2024-03-19T00:22:01.692 INFO:root:teuthology version: 0.0.1.dev262+ge691533 2024-03-19T00:22:01.693 DEBUG:teuthology.run:Teuthology command: teuthology --verbose --archive /home/teuthworker/archive/pdonnell-2024-03-18_21:05:49-fs-wip-batrick-testing-20240318.181317-distro-default-smithi/7609258 --owner scheduled_pdonnell@teuthology --name pdonnell-2024-03-18_21:05:49-fs-wip-batrick-testing-20240318.181317-distro-default-smithi --description fs/workload/{0-centos_9.stream begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v1} mount overrides/{distro/stock/{centos_9.stream k-stock} ms-die-on-skipped}} ms_mode/crc wsync/no} objectstore-ec/bluestore-bitmap omap_limit/10 overrides/{cephsqlite-timeout frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/multi/{balancer/random export-check n/3 replication/always} standby-replay tasks/{0-subvolume/{with-no-extra-options} 1-check-counter 2-scrub/no 3-snaps/no 4-flush/yes 5-workunit/suites/dbench}} -- /home/teuthworker/archive/pdonnell-2024-03-18_21:05:49-fs-wip-batrick-testing-20240318.181317-distro-default-smithi/7609258/orig.config.yaml 2024-03-19T00:22:01.746 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-03-19T00:22:01.833 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/pdonnell-2024-03-18_21:05:49-fs-wip-batrick-testing-20240318.181317-distro-default-smithi/7609258 branch: wip-batrick-testing-20240318.181317 description: fs/workload/{0-centos_9.stream begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v1} mount overrides/{distro/stock/{centos_9.stream k-stock} ms-die-on-skipped}} ms_mode/crc wsync/no} objectstore-ec/bluestore-bitmap omap_limit/10 overrides/{cephsqlite-timeout frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/multi/{balancer/random export-check n/3 replication/always} standby-replay tasks/{0-subvolume/{with-no-extra-options} 1-check-counter 2-scrub/no 3-snaps/no 4-flush/yes 5-workunit/suites/dbench}} email: pdonnell@redhat.com first_in_suite: false job_id: '7609258' kernel: client: sha1: distro kdb: true sha1: distro ktype: distro last_in_suite: false log-rotate: ceph-mds: 10G ceph-osd: 10G machine_type: smithi name: pdonnell-2024-03-18_21:05:49-fs-wip-batrick-testing-20240318.181317-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 20 - machine: disk: 200 os_type: centos os_version: 9.stream overrides: admin_socket: branch: wip-batrick-testing-20240318.181317 ceph: cephfs: max_mds: 3 session_timeout: 300 standby_replay: true conf: client: client mount timeout: 600 debug client: 20 debug ms: 1 rados mon op timeout: 900 rados osd op timeout: 900 global: ms die on skipped message: false mds: debug mds: 20 debug mds balancer: 20 debug ms: 1 mds bal fragment size max: 10000 mds bal merge size: 5 mds bal split bits: 3 mds bal split size: 100 mds debug frag: true mds debug scatterstat: true mds op complaint time: 180 mds verify scatter: true mds_bal_replicate_threshold: 1 mds_export_ephemeral_random_max: 0.1 osd op complaint time: 180 rados mon op timeout: 900 rados osd op timeout: 900 mgr: cephsqlite lock renewal timeout: 900000 debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 mon op complaint time: 120 osd: bdev async discard: true bdev enable discard: true bluestore allocator: bitmap bluestore block size: 96636764160 bluestore fsck on mount: true debug bluefs: 1/20 debug bluestore: 1/20 debug ms: 1 debug osd: 20 debug rocksdb: 4/10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore osd op complaint time: 180 osd shutdown pgref assert: true osd_max_omap_entries_per_request: 10 flavor: default fs: xfs log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - FS_DEGRADED - FS_INLINE_DATA_DEPRECATED - FS_WITH_FAILED_MDS - MDS_ALL_DOWN - MDS_DAMAGE - MDS_DEGRADED - MDS_FAILED - MDS_INSUFFICIENT_STANDBY - MDS_UP_LESS_THAN_MAX - POOL_APP_NOT_ENABLED - overall HEALTH_ - Replacing daemon - deprecated feature inline_data - PG_AVAILABILITY - PG_DEGRADED - overall HEALTH_ - \(OSD_DOWN\) - \(OSD_ - but it is still running - is not responding sha1: b6ae24918b03b1b8a19c9263a50655bba80bb810 subvols: create: 2 ceph-deploy: bluestore: true conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} osd: bdev async discard: true bdev enable discard: true bluestore block size: 96636764160 bluestore fsck on mount: true debug bluefs: 1/20 debug bluestore: 1/20 debug rocksdb: 4/10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore fs: xfs ceph-fuse: client.0: mount_subvol_num: 0 check-counter: counters: mds: - mds.exported - mds.imported - min: 3 name: mds_cache.dir_update - min: 3 name: mds_cache.dir_update_receipt install: ceph: flavor: default sha1: b6ae24918b03b1b8a19c9263a50655bba80bb810 kclient: client.0: mount_subvol_num: 1 mntopts: - ms_mode=crc - nowsync syntax: v1 selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 - scontext=system_u:system_r:getty_t:s0 - scontext=system_u:system_r:getty_t:s0 thrashosds: bdev_inject_crash: 2 bdev_inject_crash_probability: 0.5 workunit: branch: wip-batrick-testing-20240318.181317 sha1: e48b1f6c0608b2615e1890cc46286c3877d6e910 owner: scheduled_pdonnell@teuthology priority: 75 repo: https://git.ceph.com/ceph-ci.git roles: - - mon.a - mgr.x - mds.a - mds.d - mds.g - mds.j - osd.0 - osd.3 - osd.6 - osd.9 - client.0 - - mon.b - mgr.y - mds.b - mds.e - mds.h - mds.k - osd.1 - osd.4 - osd.7 - osd.10 - - mon.c - mgr.z - mds.c - mds.f - mds.i - mds.l - osd.2 - osd.5 - osd.8 - osd.11 seed: 3005 sha1: b6ae24918b03b1b8a19c9263a50655bba80bb810 sleep_before_teardown: 0 subset: 433/512 suite: fs suite_branch: wip-batrick-testing-20240318.181317 suite_path: /home/teuthworker/src/github.com_batrick_ceph_e48b1f6c0608b2615e1890cc46286c3877d6e910/qa suite_relpath: qa suite_repo: https://github.com/batrick/ceph suite_sha1: e48b1f6c0608b2615e1890cc46286c3877d6e910 syslog: ignorelist: - WARNING*.*check_session_state - WARNING*.*__ceph_remove_cap targets: smithi099.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBMWOkmh1fiqamXWIQTPEThz+6CiVZmPEHUZMQAJUAAEvdGqeZxrzOqraKfT4dEVey7xyQartsdDv292AAKfiP7Q= smithi112.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCd7Wmd7bQcoJeCcw/Zzufv4EK8DxlS46DEW+9X2hp+OA9SCNLUuEhu7gkAob5SpREz6z6IXduH3m1ZL1u9Al3Y= smithi115.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBO4emzpff0NrFFf+cZK9dZxkf4KE96HuWp15+KTIk5TJFRiV8fCyISa3xjNfpZKAlAofA9JjR5CO+jYYrdWAZOU= tasks: - install: extra_packages: deb: - python3-cephfs - cephfs-shell - cephfs-top - cephfs-mirror rpm: - python3-cephfs - cephfs-shell - cephfs-top - cephfs-mirror extra_system_packages: deb: - bison - flex - libelf-dev - libssl-dev - network-manager - iproute2 - util-linux - dump - indent - libaio-dev - libtool-bin - uuid-dev - xfslibs-dev - postgresql - postgresql-client - postgresql-common - postgresql-contrib rpm: - bison - flex - elfutils-libelf-devel - openssl-devel - NetworkManager - iproute - util-linux - libacl-devel - libaio-devel - libattr-devel - libtool - libuuid-devel - xfsdump - xfsprogs - xfsprogs-devel - libaio-devel - libtool - libuuid-devel - xfsprogs-devel - postgresql - postgresql-server - postgresql-contrib - cephadm: roleless: false - cephadm.shell: mon.a: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - cephadm.shell: mon.a: - ceph fs dump - ceph osd dump - fs.ready: timeout: 300 - kclient: null - exec: mon.a: - ceph fs set cephfs balance_automate false - ceph fs subvolumegroup pin cephfs qa random 0.10 - check-counter: {} - background_exec: mon.a: - while sleep 13; do ceph tell mds.cephfs:0 flush journal; done - workunit: clients: all: - suites/dbench.sh teuthology: fragments_dropped: [] meta: {} postmerge: - local function is_kupstream() return yaml.ktype == 'upstream' end - local function is_kdistro() return yaml.ktype == 'distro' end - local function is_fuse() return false end - local function syntax_version() return yaml.overrides.kclient.syntax end teuthology_branch: main teuthology_sha1: e691533f9cbb33d85b2187bba20d7102f098636d timestamp: 2024-03-18_21:05:49 tube: smithi user: pdonnell verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 2024-03-19T00:22:01.833 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_batrick_ceph_e48b1f6c0608b2615e1890cc46286c3877d6e910/qa; will attempt to use it 2024-03-19T00:22:01.834 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_batrick_ceph_e48b1f6c0608b2615e1890cc46286c3877d6e910/qa/tasks 2024-03-19T00:22:01.834 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-03-19T00:22:01.835 INFO:teuthology.task.internal:Checking packages... 2024-03-19T00:22:01.859 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'b6ae24918b03b1b8a19c9263a50655bba80bb810' 2024-03-19T00:22:01.859 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-03-19T00:22:01.859 INFO:teuthology.packaging:ref: None 2024-03-19T00:22:01.859 INFO:teuthology.packaging:tag: None 2024-03-19T00:22:01.859 INFO:teuthology.packaging:branch: wip-batrick-testing-20240318.181317 2024-03-19T00:22:01.859 INFO:teuthology.packaging:sha1: b6ae24918b03b1b8a19c9263a50655bba80bb810 2024-03-19T00:22:01.860 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=wip-batrick-testing-20240318.181317 2024-03-19T00:22:02.001 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-2206.gb6ae2491 2024-03-19T00:22:02.003 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-03-19T00:22:02.010 INFO:teuthology.task.internal:no buildpackages task found 2024-03-19T00:22:02.010 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-03-19T00:22:02.040 INFO:teuthology.task.internal:Saving configuration 2024-03-19T00:22:02.059 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-03-19T00:22:02.082 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-03-19T00:22:02.107 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi099.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/pdonnell-2024-03-18_21:05:49-fs-wip-batrick-testing-20240318.181317-distro-default-smithi/7609258', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-03-19 00:16:53.420364', 'locked_by': 'scheduled_pdonnell@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBMWOkmh1fiqamXWIQTPEThz+6CiVZmPEHUZMQAJUAAEvdGqeZxrzOqraKfT4dEVey7xyQartsdDv292AAKfiP7Q='} 2024-03-19T00:22:02.128 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi112.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/pdonnell-2024-03-18_21:05:49-fs-wip-batrick-testing-20240318.181317-distro-default-smithi/7609258', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-03-19 00:16:53.422319', 'locked_by': 'scheduled_pdonnell@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCd7Wmd7bQcoJeCcw/Zzufv4EK8DxlS46DEW+9X2hp+OA9SCNLUuEhu7gkAob5SpREz6z6IXduH3m1ZL1u9Al3Y='} 2024-03-19T00:22:02.149 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi115.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/pdonnell-2024-03-18_21:05:49-fs-wip-batrick-testing-20240318.181317-distro-default-smithi/7609258', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-03-19 00:16:53.423979', 'locked_by': 'scheduled_pdonnell@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBO4emzpff0NrFFf+cZK9dZxkf4KE96HuWp15+KTIk5TJFRiV8fCyISa3xjNfpZKAlAofA9JjR5CO+jYYrdWAZOU='} 2024-03-19T00:22:02.149 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-03-19T00:22:02.155 INFO:teuthology.task.internal:roles: ubuntu@smithi099.front.sepia.ceph.com - ['mon.a', 'mgr.x', 'mds.a', 'mds.d', 'mds.g', 'mds.j', 'osd.0', 'osd.3', 'osd.6', 'osd.9', 'client.0'] 2024-03-19T00:22:02.155 INFO:teuthology.task.internal:roles: ubuntu@smithi112.front.sepia.ceph.com - ['mon.b', 'mgr.y', 'mds.b', 'mds.e', 'mds.h', 'mds.k', 'osd.1', 'osd.4', 'osd.7', 'osd.10'] 2024-03-19T00:22:02.155 INFO:teuthology.task.internal:roles: ubuntu@smithi115.front.sepia.ceph.com - ['mon.c', 'mgr.z', 'mds.c', 'mds.f', 'mds.i', 'mds.l', 'osd.2', 'osd.5', 'osd.8', 'osd.11'] 2024-03-19T00:22:02.155 INFO:teuthology.run_tasks:Running task console_log... 2024-03-19T00:22:02.281 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f85847ef160>, signals=[15]) 2024-03-19T00:22:02.281 INFO:teuthology.run_tasks:Running task internal.connect... 2024-03-19T00:22:02.287 INFO:teuthology.task.internal:Opening connections... 2024-03-19T00:22:02.287 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi099.front.sepia.ceph.com 2024-03-19T00:22:02.289 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi099.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-19T00:22:02.381 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi112.front.sepia.ceph.com 2024-03-19T00:22:02.382 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi112.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-19T00:22:02.470 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi115.front.sepia.ceph.com 2024-03-19T00:22:02.471 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi115.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-19T00:22:02.556 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-03-19T00:22:02.563 DEBUG:teuthology.orchestra.run.smithi099:> uname -m 2024-03-19T00:22:02.583 INFO:teuthology.orchestra.run.smithi099.stdout:x86_64 2024-03-19T00:22:02.583 DEBUG:teuthology.orchestra.run.smithi099:> cat /etc/os-release 2024-03-19T00:22:02.640 INFO:teuthology.orchestra.run.smithi099.stdout:NAME="CentOS Stream" 2024-03-19T00:22:02.640 INFO:teuthology.orchestra.run.smithi099.stdout:VERSION="9" 2024-03-19T00:22:02.640 INFO:teuthology.orchestra.run.smithi099.stdout:ID="centos" 2024-03-19T00:22:02.640 INFO:teuthology.orchestra.run.smithi099.stdout:ID_LIKE="rhel fedora" 2024-03-19T00:22:02.640 INFO:teuthology.orchestra.run.smithi099.stdout:VERSION_ID="9" 2024-03-19T00:22:02.640 INFO:teuthology.orchestra.run.smithi099.stdout:PLATFORM_ID="platform:el9" 2024-03-19T00:22:02.640 INFO:teuthology.orchestra.run.smithi099.stdout:PRETTY_NAME="CentOS Stream 9" 2024-03-19T00:22:02.640 INFO:teuthology.orchestra.run.smithi099.stdout:ANSI_COLOR="0;31" 2024-03-19T00:22:02.640 INFO:teuthology.orchestra.run.smithi099.stdout:LOGO="fedora-logo-icon" 2024-03-19T00:22:02.640 INFO:teuthology.orchestra.run.smithi099.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-03-19T00:22:02.641 INFO:teuthology.orchestra.run.smithi099.stdout:HOME_URL="https://centos.org/" 2024-03-19T00:22:02.641 INFO:teuthology.orchestra.run.smithi099.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-03-19T00:22:02.641 INFO:teuthology.orchestra.run.smithi099.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-03-19T00:22:02.641 INFO:teuthology.orchestra.run.smithi099.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-03-19T00:22:02.641 INFO:teuthology.lock.ops:Updating smithi099.front.sepia.ceph.com on lock server 2024-03-19T00:22:02.661 DEBUG:teuthology.orchestra.run.smithi112:> uname -m 2024-03-19T00:22:02.679 INFO:teuthology.orchestra.run.smithi112.stdout:x86_64 2024-03-19T00:22:02.679 DEBUG:teuthology.orchestra.run.smithi112:> cat /etc/os-release 2024-03-19T00:22:02.736 INFO:teuthology.orchestra.run.smithi112.stdout:NAME="CentOS Stream" 2024-03-19T00:22:02.736 INFO:teuthology.orchestra.run.smithi112.stdout:VERSION="9" 2024-03-19T00:22:02.736 INFO:teuthology.orchestra.run.smithi112.stdout:ID="centos" 2024-03-19T00:22:02.736 INFO:teuthology.orchestra.run.smithi112.stdout:ID_LIKE="rhel fedora" 2024-03-19T00:22:02.736 INFO:teuthology.orchestra.run.smithi112.stdout:VERSION_ID="9" 2024-03-19T00:22:02.736 INFO:teuthology.orchestra.run.smithi112.stdout:PLATFORM_ID="platform:el9" 2024-03-19T00:22:02.736 INFO:teuthology.orchestra.run.smithi112.stdout:PRETTY_NAME="CentOS Stream 9" 2024-03-19T00:22:02.736 INFO:teuthology.orchestra.run.smithi112.stdout:ANSI_COLOR="0;31" 2024-03-19T00:22:02.736 INFO:teuthology.orchestra.run.smithi112.stdout:LOGO="fedora-logo-icon" 2024-03-19T00:22:02.736 INFO:teuthology.orchestra.run.smithi112.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-03-19T00:22:02.736 INFO:teuthology.orchestra.run.smithi112.stdout:HOME_URL="https://centos.org/" 2024-03-19T00:22:02.737 INFO:teuthology.orchestra.run.smithi112.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-03-19T00:22:02.737 INFO:teuthology.orchestra.run.smithi112.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-03-19T00:22:02.737 INFO:teuthology.orchestra.run.smithi112.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-03-19T00:22:02.737 INFO:teuthology.lock.ops:Updating smithi112.front.sepia.ceph.com on lock server 2024-03-19T00:22:02.761 DEBUG:teuthology.orchestra.run.smithi115:> uname -m 2024-03-19T00:22:02.778 INFO:teuthology.orchestra.run.smithi115.stdout:x86_64 2024-03-19T00:22:02.779 DEBUG:teuthology.orchestra.run.smithi115:> cat /etc/os-release 2024-03-19T00:22:02.834 INFO:teuthology.orchestra.run.smithi115.stdout:NAME="CentOS Stream" 2024-03-19T00:22:02.834 INFO:teuthology.orchestra.run.smithi115.stdout:VERSION="9" 2024-03-19T00:22:02.834 INFO:teuthology.orchestra.run.smithi115.stdout:ID="centos" 2024-03-19T00:22:02.834 INFO:teuthology.orchestra.run.smithi115.stdout:ID_LIKE="rhel fedora" 2024-03-19T00:22:02.834 INFO:teuthology.orchestra.run.smithi115.stdout:VERSION_ID="9" 2024-03-19T00:22:02.834 INFO:teuthology.orchestra.run.smithi115.stdout:PLATFORM_ID="platform:el9" 2024-03-19T00:22:02.834 INFO:teuthology.orchestra.run.smithi115.stdout:PRETTY_NAME="CentOS Stream 9" 2024-03-19T00:22:02.834 INFO:teuthology.orchestra.run.smithi115.stdout:ANSI_COLOR="0;31" 2024-03-19T00:22:02.834 INFO:teuthology.orchestra.run.smithi115.stdout:LOGO="fedora-logo-icon" 2024-03-19T00:22:02.835 INFO:teuthology.orchestra.run.smithi115.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-03-19T00:22:02.835 INFO:teuthology.orchestra.run.smithi115.stdout:HOME_URL="https://centos.org/" 2024-03-19T00:22:02.835 INFO:teuthology.orchestra.run.smithi115.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-03-19T00:22:02.835 INFO:teuthology.orchestra.run.smithi115.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-03-19T00:22:02.835 INFO:teuthology.orchestra.run.smithi115.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-03-19T00:22:02.835 INFO:teuthology.lock.ops:Updating smithi115.front.sepia.ceph.com on lock server 2024-03-19T00:22:02.860 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-03-19T00:22:02.873 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-03-19T00:22:02.913 INFO:teuthology.task.internal:Checking for old test directory... 2024-03-19T00:22:02.913 DEBUG:teuthology.orchestra.run.smithi099:> test '!' -e /home/ubuntu/cephtest 2024-03-19T00:22:02.916 DEBUG:teuthology.orchestra.run.smithi112:> test '!' -e /home/ubuntu/cephtest 2024-03-19T00:22:02.918 DEBUG:teuthology.orchestra.run.smithi115:> test '!' -e /home/ubuntu/cephtest 2024-03-19T00:22:02.934 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-03-19T00:22:02.942 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-03-19T00:22:02.942 DEBUG:teuthology.orchestra.run.smithi099:> test -z $(ls -A /var/lib/ceph) 2024-03-19T00:22:02.972 DEBUG:teuthology.orchestra.run.smithi112:> test -z $(ls -A /var/lib/ceph) 2024-03-19T00:22:02.975 DEBUG:teuthology.orchestra.run.smithi115:> test -z $(ls -A /var/lib/ceph) 2024-03-19T00:22:03.032 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-03-19T00:22:03.104 INFO:teuthology.run_tasks:Running task kernel... 2024-03-19T00:22:03.119 INFO:teuthology.task.kernel:normalize config orig: {'client': {'sha1': 'distro'}, 'kdb': True, 'sha1': 'distro'} 2024-03-19T00:22:03.120 INFO:teuthology.task.kernel:normalize config final: {'client.0': {'sha1': 'distro'}} 2024-03-19T00:22:03.120 INFO:teuthology.task.kernel:config {'client.0': {'sha1': 'distro'}}, timeout 300 2024-03-19T00:22:03.120 DEBUG:teuthology.orchestra.run.smithi099:> test -f /run/.containerenv -o -f /.dockerenv 2024-03-19T00:22:03.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-19T00:22:03.137 DEBUG:teuthology.orchestra.run.smithi099:> uname -r 2024-03-19T00:22:03.194 INFO:teuthology.orchestra.run.smithi099.stdout:5.14.0-361.el9.x86_64 2024-03-19T00:22:03.194 INFO:teuthology.task.kernel:Running kernel on smithi099: 5.14.0-361.el9.x86_64 2024-03-19T00:22:03.194 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum install -y kernel 2024-03-19T00:22:05.449 INFO:teuthology.orchestra.run.smithi099.stdout:CentOS-9 - AppStream 12 MB/s | 6.7 MB 00:00 2024-03-19T00:22:06.849 INFO:teuthology.orchestra.run.smithi099.stdout:CentOS-9 - BaseOS 6.6 MB/s | 2.0 MB 00:00 2024-03-19T00:35:17.054 INFO:teuthology.orchestra.run.smithi099.stdout:CentOS Stream 9 - BaseOS 10 kB/s | 8.0 MB 13:09 2024-03-19T00:48:26.671 INFO:teuthology.orchestra.run.smithi099.stdout:CentOS Stream 9 - AppStream 24 kB/s | 19 MB 13:08 2024-03-19T00:48:33.829 INFO:teuthology.orchestra.run.smithi099.stdout:CentOS Stream 9 - CRB 1.7 MB/s | 6.1 MB 00:03 2024-03-19T00:48:35.441 INFO:teuthology.orchestra.run.smithi099.stdout:CentOS Stream 9 - Extras packages 60 kB/s | 16 kB 00:00 2024-03-19T00:48:38.197 INFO:teuthology.orchestra.run.smithi099.stdout:Extra Packages for Enterprise Linux 7.8 MB/s | 21 MB 00:02 2024-03-19T00:48:43.047 INFO:teuthology.orchestra.run.smithi099.stdout:lab-extras 15 kB/s | 1.7 kB 00:00 2024-03-19T00:48:45.153 INFO:teuthology.orchestra.run.smithi099.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-03-19T00:48:45.153 INFO:teuthology.orchestra.run.smithi099.stdout:Package kernel-5.14.0-361.el9.x86_64 is already installed. 2024-03-19T00:48:45.263 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2024-03-19T00:48:45.270 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2024-03-19T00:48:45.270 INFO:teuthology.orchestra.run.smithi099.stdout: Package Architecture Version Repository Size 2024-03-19T00:48:45.271 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2024-03-19T00:48:45.271 INFO:teuthology.orchestra.run.smithi099.stdout:Installing: 2024-03-19T00:48:45.271 INFO:teuthology.orchestra.run.smithi099.stdout: kernel x86_64 5.14.0-430.el9 baseos 44 k 2024-03-19T00:48:45.271 INFO:teuthology.orchestra.run.smithi099.stdout:Installing dependencies: 2024-03-19T00:48:45.271 INFO:teuthology.orchestra.run.smithi099.stdout: kernel-core x86_64 5.14.0-430.el9 baseos 15 M 2024-03-19T00:48:45.271 INFO:teuthology.orchestra.run.smithi099.stdout: kernel-modules x86_64 5.14.0-430.el9 baseos 33 M 2024-03-19T00:48:45.271 INFO:teuthology.orchestra.run.smithi099.stdout: kernel-modules-core x86_64 5.14.0-430.el9 baseos 27 M 2024-03-19T00:48:45.271 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T00:48:45.271 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction Summary 2024-03-19T00:48:45.271 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2024-03-19T00:48:45.271 INFO:teuthology.orchestra.run.smithi099.stdout:Install 4 Packages 2024-03-19T00:48:45.271 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T00:48:45.272 INFO:teuthology.orchestra.run.smithi099.stdout:Total download size: 76 M 2024-03-19T00:48:45.272 INFO:teuthology.orchestra.run.smithi099.stdout:Installed size: 123 M 2024-03-19T00:48:45.272 INFO:teuthology.orchestra.run.smithi099.stdout:Downloading Packages: 2024-03-19T00:50:56.914 INFO:teuthology.orchestra.run.smithi099.stdout:[MIRROR] kernel-5.14.0-430.el9.x86_64.rpm: Curl error (28): Timeout was reached for http://ftp-nyc.osuosl.org/pub/centos-stream/9-stream/BaseOS/x86_64/os/Packages/kernel-5.14.0-430.el9.x86_64.rpm [Failed to connect to ftp-nyc.osuosl.org port 80: Connection timed out] 2024-03-19T00:50:56.915 INFO:teuthology.orchestra.run.smithi099.stdout:[MIRROR] kernel-core-5.14.0-430.el9.x86_64.rpm: Curl error (28): Timeout was reached for http://ftp-nyc.osuosl.org/pub/centos-stream/9-stream/BaseOS/x86_64/os/Packages/kernel-core-5.14.0-430.el9.x86_64.rpm [Failed to connect to ftp-nyc.osuosl.org port 80: Connection timed out] 2024-03-19T00:50:56.915 INFO:teuthology.orchestra.run.smithi099.stdout:[MIRROR] kernel-modules-5.14.0-430.el9.x86_64.rpm: Curl error (28): Timeout was reached for http://ftp-nyc.osuosl.org/pub/centos-stream/9-stream/BaseOS/x86_64/os/Packages/kernel-modules-5.14.0-430.el9.x86_64.rpm [Failed to connect to ftp-nyc.osuosl.org port 80: Connection timed out] 2024-03-19T00:50:57.083 INFO:teuthology.orchestra.run.smithi099.stdout:[MIRROR] kernel-modules-5.14.0-430.el9.x86_64.rpm: Status code: 404 for http://mirror.cpsc.ucalgary.ca/mirror/centos-stream/9-stream/BaseOS/x86_64/os/Packages/kernel-modules-5.14.0-430.el9.x86_64.rpm (IP: 136.159.208.97) 2024-03-19T00:50:57.087 INFO:teuthology.orchestra.run.smithi099.stdout:[MIRROR] kernel-core-5.14.0-430.el9.x86_64.rpm: Status code: 404 for http://mirror.cpsc.ucalgary.ca/mirror/centos-stream/9-stream/BaseOS/x86_64/os/Packages/kernel-core-5.14.0-430.el9.x86_64.rpm (IP: 136.159.208.97) 2024-03-19T00:50:57.088 INFO:teuthology.orchestra.run.smithi099.stdout:[MIRROR] kernel-5.14.0-430.el9.x86_64.rpm: Status code: 404 for http://mirror.cpsc.ucalgary.ca/mirror/centos-stream/9-stream/BaseOS/x86_64/os/Packages/kernel-5.14.0-430.el9.x86_64.rpm (IP: 136.159.208.97) 2024-03-19T00:53:07.987 INFO:teuthology.orchestra.run.smithi099.stdout:[MIRROR] kernel-modules-5.14.0-430.el9.x86_64.rpm: Curl error (28): Timeout was reached for https://ftp-nyc.osuosl.org/pub/centos-stream/9-stream/BaseOS/x86_64/os/Packages/kernel-modules-5.14.0-430.el9.x86_64.rpm [Failed to connect to ftp-nyc.osuosl.org port 443: Connection timed out] 2024-03-19T00:53:07.987 INFO:teuthology.orchestra.run.smithi099.stdout:[MIRROR] kernel-core-5.14.0-430.el9.x86_64.rpm: Curl error (28): Timeout was reached for https://ftp-nyc.osuosl.org/pub/centos-stream/9-stream/BaseOS/x86_64/os/Packages/kernel-core-5.14.0-430.el9.x86_64.rpm [Failed to connect to ftp-nyc.osuosl.org port 443: Connection timed out] 2024-03-19T00:53:07.987 INFO:teuthology.orchestra.run.smithi099.stdout:[MIRROR] kernel-5.14.0-430.el9.x86_64.rpm: Curl error (28): Timeout was reached for https://ftp-nyc.osuosl.org/pub/centos-stream/9-stream/BaseOS/x86_64/os/Packages/kernel-5.14.0-430.el9.x86_64.rpm [Failed to connect to ftp-nyc.osuosl.org port 443: Connection timed out] 2024-03-19T00:53:14.841 INFO:teuthology.orchestra.run.smithi099.stdout:(1/4): kernel-core-5.14.0-430.el9.x86_64.rpm 57 kB/s | 15 MB 04:27 2024-03-19T00:53:15.017 INFO:teuthology.orchestra.run.smithi099.stdout:[MIRROR] kernel-modules-core-5.14.0-430.el9.x86_64.rpm: Status code: 404 for http://mirror.cpsc.ucalgary.ca/mirror/centos-stream/9-stream/BaseOS/x86_64/os/Packages/kernel-modules-core-5.14.0-430.el9.x86_64.rpm (IP: 136.159.208.97) 2024-03-19T00:53:17.208 INFO:teuthology.orchestra.run.smithi099.stdout:(2/4): kernel-modules-core-5.14.0-430.el9.x86_6 12 MB/s | 27 MB 00:02 2024-03-19T00:53:33.580 INFO:teuthology.orchestra.run.smithi099.stdout:(3/4): kernel-modules-5.14.0-430.el9.x86_64.rpm 120 kB/s | 33 MB 04:46 2024-03-19T00:55:19.147 INFO:teuthology.orchestra.run.smithi099.stdout:[MIRROR] kernel-5.14.0-430.el9.x86_64.rpm: Curl error (28): Timeout was reached for http://ftp-nyc.osuosl.org/pub/centos-stream/9-stream/BaseOS/x86_64/os/Packages/kernel-5.14.0-430.el9.x86_64.rpm [Failed to connect to ftp-nyc.osuosl.org port 80: Connection timed out] 2024-03-19T00:55:19.499 INFO:teuthology.orchestra.run.smithi099.stdout:(4/4): kernel-5.14.0-430.el9.x86_64.rpm 116 B/s | 44 kB 06:31 2024-03-19T00:55:19.499 INFO:teuthology.orchestra.run.smithi099.stdout:-------------------------------------------------------------------------------- 2024-03-19T00:55:19.499 INFO:teuthology.orchestra.run.smithi099.stdout:Total 197 kB/s | 76 MB 06:34 2024-03-19T00:55:20.157 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction check 2024-03-19T00:55:20.515 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction check succeeded. 2024-03-19T00:55:20.515 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction test 2024-03-19T00:55:21.504 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction test succeeded. 2024-03-19T00:55:21.505 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction 2024-03-19T00:55:23.961 INFO:teuthology.orchestra.run.smithi099.stdout: Preparing : 1/1 2024-03-19T00:55:24.582 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : kernel-modules-core-5.14.0-430.el9.x86_64 1/4 2024-03-19T00:55:24.668 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : kernel-core-5.14.0-430.el9.x86_64 2/4 2024-03-19T00:55:27.415 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: kernel-core-5.14.0-430.el9.x86_64 2/4 2024-03-19T00:55:27.555 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : kernel-modules-5.14.0-430.el9.x86_64 3/4 2024-03-19T00:55:32.290 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: kernel-modules-5.14.0-430.el9.x86_64 3/4 2024-03-19T00:55:32.396 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : kernel-5.14.0-430.el9.x86_64 4/4 2024-03-19T00:55:37.138 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: kernel-modules-core-5.14.0-430.el9.x86_64 4/4 2024-03-19T00:56:06.317 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: kernel-core-5.14.0-430.el9.x86_64 4/4 2024-03-19T00:56:06.317 INFO:teuthology.orchestra.run.smithi099.stdout:dracut: Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_[AMD|INTEL]!=y 2024-03-19T00:56:06.317 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T00:56:06.361 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: kernel-modules-5.14.0-430.el9.x86_64 4/4 2024-03-19T00:56:06.734 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: kernel-5.14.0-430.el9.x86_64 4/4 2024-03-19T00:56:06.734 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : kernel-5.14.0-430.el9.x86_64 1/4 2024-03-19T00:56:06.734 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : kernel-core-5.14.0-430.el9.x86_64 2/4 2024-03-19T00:56:06.735 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : kernel-modules-5.14.0-430.el9.x86_64 3/4 2024-03-19T00:56:07.136 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : kernel-modules-core-5.14.0-430.el9.x86_64 4/4 2024-03-19T00:56:07.136 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T00:56:07.137 INFO:teuthology.orchestra.run.smithi099.stdout:Installed: 2024-03-19T00:56:07.137 INFO:teuthology.orchestra.run.smithi099.stdout: kernel-5.14.0-430.el9.x86_64 2024-03-19T00:56:07.137 INFO:teuthology.orchestra.run.smithi099.stdout: kernel-core-5.14.0-430.el9.x86_64 2024-03-19T00:56:07.137 INFO:teuthology.orchestra.run.smithi099.stdout: kernel-modules-5.14.0-430.el9.x86_64 2024-03-19T00:56:07.137 INFO:teuthology.orchestra.run.smithi099.stdout: kernel-modules-core-5.14.0-430.el9.x86_64 2024-03-19T00:56:07.137 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T00:56:07.137 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2024-03-19T00:56:07.499 DEBUG:teuthology.orchestra.run.smithi099:> rpm -q kernel | sort -rV | head -n 1 2024-03-19T00:56:07.598 INFO:teuthology.orchestra.run.smithi099.stdout:kernel-5.14.0-430.el9.x86_64 2024-03-19T00:56:07.598 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-430.el9.x86_64 2024-03-19T00:56:07.598 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-361.el9.x86_64 Expected: 5.14.0-430.el9.x86_64 2024-03-19T00:56:07.598 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-03-19T00:56:07.599 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-03-19T00:56:07.599 INFO:teuthology.task.kernel:Installing distro kernel on client.0... 2024-03-19T00:56:07.599 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi099.front.sepia.ceph.com, path=None, version=distro) 2024-03-19T00:56:07.599 DEBUG:teuthology.orchestra.run.smithi099:> rpm -q kernel | sort -rV | head -n 1 2024-03-19T00:56:07.678 INFO:teuthology.orchestra.run.smithi099.stdout:kernel-5.14.0-430.el9.x86_64 2024-03-19T00:56:07.679 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-430.el9.x86_64 2024-03-19T00:56:07.679 DEBUG:teuthology.orchestra.run.smithi099:> sudo rpm -qi grub2-tools 2024-03-19T00:56:07.713 INFO:teuthology.orchestra.run.smithi099.stdout:Name : grub2-tools 2024-03-19T00:56:07.713 INFO:teuthology.orchestra.run.smithi099.stdout:Epoch : 1 2024-03-19T00:56:07.713 INFO:teuthology.orchestra.run.smithi099.stdout:Version : 2.06 2024-03-19T00:56:07.713 INFO:teuthology.orchestra.run.smithi099.stdout:Release : 68.el9 2024-03-19T00:56:07.713 INFO:teuthology.orchestra.run.smithi099.stdout:Architecture: x86_64 2024-03-19T00:56:07.713 INFO:teuthology.orchestra.run.smithi099.stdout:Install Date: Fri 01 Sep 2023 09:14:30 PM UTC 2024-03-19T00:56:07.713 INFO:teuthology.orchestra.run.smithi099.stdout:Group : Unspecified 2024-03-19T00:56:07.713 INFO:teuthology.orchestra.run.smithi099.stdout:Size : 8264254 2024-03-19T00:56:07.713 INFO:teuthology.orchestra.run.smithi099.stdout:License : GPLv3+ 2024-03-19T00:56:07.714 INFO:teuthology.orchestra.run.smithi099.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-03-19T00:56:07.714 INFO:teuthology.orchestra.run.smithi099.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-03-19T00:56:07.714 INFO:teuthology.orchestra.run.smithi099.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-03-19T00:56:07.714 INFO:teuthology.orchestra.run.smithi099.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-03-19T00:56:07.714 INFO:teuthology.orchestra.run.smithi099.stdout:Packager : builder@centos.org 2024-03-19T00:56:07.714 INFO:teuthology.orchestra.run.smithi099.stdout:Vendor : CentOS 2024-03-19T00:56:07.714 INFO:teuthology.orchestra.run.smithi099.stdout:URL : http://www.gnu.org/software/grub/ 2024-03-19T00:56:07.714 INFO:teuthology.orchestra.run.smithi099.stdout:Summary : Support tools for GRUB. 2024-03-19T00:56:07.714 INFO:teuthology.orchestra.run.smithi099.stdout:Description : 2024-03-19T00:56:07.714 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T00:56:07.714 INFO:teuthology.orchestra.run.smithi099.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-03-19T00:56:07.714 INFO:teuthology.orchestra.run.smithi099.stdout:customizable bootloader with modular architecture. It supports a rich 2024-03-19T00:56:07.714 INFO:teuthology.orchestra.run.smithi099.stdout:variety of kernel formats, file systems, computer architectures and 2024-03-19T00:56:07.714 INFO:teuthology.orchestra.run.smithi099.stdout:hardware devices. 2024-03-19T00:56:07.715 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T00:56:07.715 INFO:teuthology.orchestra.run.smithi099.stdout:This subpackage provides tools for support of all platforms. 2024-03-19T00:56:07.715 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-03-19T00:56:07.715 INFO:teuthology.task.kernel:Updating grub on smithi099 to boot 5.14.0-430.el9.x86_64 2024-03-19T00:56:07.716 DEBUG:teuthology.orchestra.run.smithi099:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-03-19T00:56:08.309 INFO:teuthology.orchestra.run.smithi099.stderr:Generating grub configuration file ... 2024-03-19T00:56:09.782 INFO:teuthology.orchestra.run.smithi099.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-03-19T00:56:09.829 INFO:teuthology.orchestra.run.smithi099.stderr:done 2024-03-19T00:56:09.832 DEBUG:teuthology.orchestra.run.smithi099:> mktemp 2024-03-19T00:56:09.872 INFO:teuthology.orchestra.run.smithi099.stdout:/tmp/tmp.PgzmqDBxSi 2024-03-19T00:56:09.872 DEBUG:teuthology.orchestra.run.smithi099:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.PgzmqDBxSi 2024-03-19T00:56:09.951 DEBUG:teuthology.orchestra.run.smithi099:> sudo chmod 0666 /tmp/tmp.PgzmqDBxSi 2024-03-19T00:56:10.104 DEBUG:teuthology.orchestra.remote:smithi099:/tmp/tmp.PgzmqDBxSi is 6KB 2024-03-19T00:56:10.123 DEBUG:teuthology.orchestra.run.smithi099:> rm -fr /tmp/tmp.PgzmqDBxSi 2024-03-19T00:56:10.137 DEBUG:teuthology.orchestra.run.smithi099:> sudo /bin/ls /boot/loader/entries || true 2024-03-19T00:56:10.201 INFO:teuthology.orchestra.run.smithi099.stdout:15949e3c88704300882482eddfd7baae-0-rescue.conf 2024-03-19T00:56:10.202 INFO:teuthology.orchestra.run.smithi099.stdout:15949e3c88704300882482eddfd7baae-5.14.0-239.el9.x86_64.conf 2024-03-19T00:56:10.202 INFO:teuthology.orchestra.run.smithi099.stdout:15949e3c88704300882482eddfd7baae-5.14.0-361.el9.x86_64.conf 2024-03-19T00:56:10.202 INFO:teuthology.orchestra.run.smithi099.stdout:15949e3c88704300882482eddfd7baae-5.14.0-430.el9.x86_64.conf 2024-03-19T00:56:10.203 DEBUG:teuthology.orchestra.run.smithi099:> sudo grub2-set-default 15949e3c88704300882482eddfd7baae-5.14.0-430.el9.x86_64 2024-03-19T00:56:10.360 DEBUG:teuthology.orchestra.run.smithi099:> sudo shutdown -r now 2024-03-19T00:56:40.364 INFO:teuthology.task.kernel:Checking client client.0 for new kernel version... 2024-03-19T00:56:40.364 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi099.front.sepia.ceph.com' 2024-03-19T00:56:40.365 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi099.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-19T00:56:58.914 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.99 2024-03-19T00:57:07.916 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi099.front.sepia.ceph.com' 2024-03-19T00:57:07.917 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi099.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-19T00:57:10.978 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.99 2024-03-19T00:57:22.979 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi099.front.sepia.ceph.com' 2024-03-19T00:57:22.980 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi099.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-19T00:57:54.858 DEBUG:teuthology.orchestra.run.smithi099:> true 2024-03-19T00:57:55.277 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi099.front.sepia.ceph.com' 2024-03-19T00:57:55.278 INFO:teuthology.task.kernel:Checking kernel version of client.0, want "5.14.0-430.el9.x86_64"... 2024-03-19T00:57:55.278 DEBUG:teuthology.orchestra.run.smithi099:> uname -r 2024-03-19T00:57:55.292 INFO:teuthology.orchestra.run.smithi099.stdout:5.14.0-430.el9.x86_64 2024-03-19T00:57:55.292 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-430.el9.x86_64 vs 5.14.0-430.el9.x86_64 2024-03-19T00:57:55.293 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-03-19T00:57:55.293 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-03-19T00:57:56.294 DEBUG:teuthology.parallel:result is None 2024-03-19T00:57:56.294 INFO:teuthology.run_tasks:Running task internal.base... 2024-03-19T00:57:56.302 INFO:teuthology.task.internal:Creating test directory... 2024-03-19T00:57:56.302 DEBUG:teuthology.orchestra.run.smithi099:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-03-19T00:57:56.304 DEBUG:teuthology.orchestra.run.smithi112:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-03-19T00:57:56.307 DEBUG:teuthology.orchestra.run.smithi115:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-03-19T00:57:56.326 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-03-19T00:57:56.364 INFO:teuthology.run_tasks:Running task internal.archive... 2024-03-19T00:57:56.447 INFO:teuthology.task.internal:Creating archive directory... 2024-03-19T00:57:56.448 DEBUG:teuthology.orchestra.run.smithi099:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-03-19T00:57:56.450 DEBUG:teuthology.orchestra.run.smithi112:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-03-19T00:57:56.453 DEBUG:teuthology.orchestra.run.smithi115:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-03-19T00:57:56.596 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-03-19T00:57:56.603 INFO:teuthology.task.internal:Enabling coredump saving... 2024-03-19T00:57:56.604 DEBUG:teuthology.orchestra.run.smithi112:> test -f /run/.containerenv -o -f /.dockerenv 2024-03-19T00:57:56.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-19T00:57:56.620 DEBUG:teuthology.orchestra.run.smithi115:> test -f /run/.containerenv -o -f /.dockerenv 2024-03-19T00:57:56.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-19T00:57:56.636 DEBUG:teuthology.orchestra.run.smithi099:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-03-19T00:57:56.638 DEBUG:teuthology.orchestra.run.smithi112:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-03-19T00:57:56.662 DEBUG:teuthology.orchestra.run.smithi115:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-03-19T00:57:56.696 INFO:teuthology.orchestra.run.smithi112.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-03-19T00:57:56.715 INFO:teuthology.orchestra.run.smithi112.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-03-19T00:57:56.719 INFO:teuthology.orchestra.run.smithi115.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-03-19T00:57:56.742 INFO:teuthology.orchestra.run.smithi115.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-03-19T00:57:56.763 INFO:teuthology.orchestra.run.smithi099.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-03-19T00:57:56.944 INFO:teuthology.orchestra.run.smithi099.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-03-19T00:57:56.945 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-03-19T00:57:56.955 INFO:teuthology.task.internal:Configuring sudo... 2024-03-19T00:57:56.955 DEBUG:teuthology.orchestra.run.smithi099:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-03-19T00:57:56.958 DEBUG:teuthology.orchestra.run.smithi112:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-03-19T00:57:56.962 DEBUG:teuthology.orchestra.run.smithi115:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-03-19T00:57:56.993 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-03-19T00:57:57.007 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-03-19T00:57:57.007 DEBUG:teuthology.orchestra.run.smithi099:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-03-19T00:57:57.025 DEBUG:teuthology.orchestra.run.smithi112:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-03-19T00:57:57.032 DEBUG:teuthology.orchestra.run.smithi115:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-03-19T00:57:57.051 DEBUG:teuthology.orchestra.run.smithi099:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-03-19T00:57:57.118 DEBUG:teuthology.orchestra.run.smithi099:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-03-19T00:57:57.163 DEBUG:teuthology.orchestra.run.smithi099:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-03-19T00:57:57.239 DEBUG:teuthology.orchestra.run.smithi099:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-03-19T00:57:57.279 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2024-03-19T00:57:57.279 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-03-19T00:57:57.342 DEBUG:teuthology.orchestra.run.smithi112:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-03-19T00:57:57.387 DEBUG:teuthology.orchestra.run.smithi112:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-03-19T00:57:57.443 DEBUG:teuthology.orchestra.run.smithi112:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-03-19T00:57:57.524 DEBUG:teuthology.orchestra.run.smithi112:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-03-19T00:57:57.571 DEBUG:teuthology.orchestra.run.smithi112:> set -ex 2024-03-19T00:57:57.571 DEBUG:teuthology.orchestra.run.smithi112:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-03-19T00:57:57.639 DEBUG:teuthology.orchestra.run.smithi115:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-03-19T00:57:57.684 DEBUG:teuthology.orchestra.run.smithi115:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-03-19T00:57:57.744 DEBUG:teuthology.orchestra.run.smithi115:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-03-19T00:57:57.787 DEBUG:teuthology.orchestra.run.smithi115:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-03-19T00:57:57.832 DEBUG:teuthology.orchestra.run.smithi115:> set -ex 2024-03-19T00:57:57.832 DEBUG:teuthology.orchestra.run.smithi115:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-03-19T00:57:57.900 DEBUG:teuthology.orchestra.run.smithi099:> sudo service rsyslog restart 2024-03-19T00:57:57.903 DEBUG:teuthology.orchestra.run.smithi112:> sudo service rsyslog restart 2024-03-19T00:57:57.906 DEBUG:teuthology.orchestra.run.smithi115:> sudo service rsyslog restart 2024-03-19T00:57:57.945 INFO:teuthology.orchestra.run.smithi112.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-03-19T00:57:57.951 INFO:teuthology.orchestra.run.smithi099.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-03-19T00:57:57.976 INFO:teuthology.orchestra.run.smithi115.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-03-19T00:57:58.392 INFO:teuthology.run_tasks:Running task internal.timer... 2024-03-19T00:57:58.400 INFO:teuthology.task.internal:Starting timer... 2024-03-19T00:57:58.400 INFO:teuthology.run_tasks:Running task pcp... 2024-03-19T00:57:58.442 INFO:teuthology.run_tasks:Running task selinux... 2024-03-19T00:57:58.533 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0', 'scontext=system_u:system_r:getty_t:s0', 'scontext=system_u:system_r:getty_t:s0']} 2024-03-19T00:57:58.534 DEBUG:teuthology.orchestra.run.smithi099:> sudo service auditd rotate 2024-03-19T00:57:58.645 INFO:teuthology.orchestra.run.smithi099.stdout:Rotating logs: 2024-03-19T00:57:58.647 DEBUG:teuthology.orchestra.run.smithi112:> sudo service auditd rotate 2024-03-19T00:57:58.712 INFO:teuthology.orchestra.run.smithi112.stdout:Rotating logs: 2024-03-19T00:57:58.714 DEBUG:teuthology.orchestra.run.smithi115:> sudo service auditd rotate 2024-03-19T00:57:58.758 INFO:teuthology.orchestra.run.smithi115.stdout:Rotating logs: 2024-03-19T00:57:58.760 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-03-19T00:57:58.760 DEBUG:teuthology.orchestra.run.smithi099:> /usr/sbin/getenforce 2024-03-19T00:57:58.790 INFO:teuthology.orchestra.run.smithi099.stdout:Permissive 2024-03-19T00:57:58.790 DEBUG:teuthology.orchestra.run.smithi112:> /usr/sbin/getenforce 2024-03-19T00:57:58.823 INFO:teuthology.orchestra.run.smithi112.stdout:Permissive 2024-03-19T00:57:58.824 DEBUG:teuthology.orchestra.run.smithi115:> /usr/sbin/getenforce 2024-03-19T00:57:58.859 INFO:teuthology.orchestra.run.smithi115.stdout:Permissive 2024-03-19T00:57:58.859 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi099.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi112.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi115.front.sepia.ceph.com': 'permissive'} 2024-03-19T00:57:58.859 DEBUG:teuthology.orchestra.run.smithi099:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-03-19T00:57:58.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-19T00:57:58.912 DEBUG:teuthology.orchestra.run.smithi112:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-03-19T00:57:58.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-19T00:57:58.943 DEBUG:teuthology.orchestra.run.smithi115:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-03-19T00:57:58.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-19T00:57:58.971 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-03-19T00:57:58.971 DEBUG:teuthology.orchestra.run.smithi099:> sudo /usr/sbin/setenforce permissive 2024-03-19T00:57:59.308 DEBUG:teuthology.orchestra.run.smithi112:> sudo /usr/sbin/setenforce permissive 2024-03-19T00:57:59.342 DEBUG:teuthology.orchestra.run.smithi115:> sudo /usr/sbin/setenforce permissive 2024-03-19T00:57:59.378 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-03-19T00:57:59.387 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-03-19T00:57:59.402 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-03-19T00:57:59.403 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi099.front.sepia.ceph.com,smithi112.front.sepia.ceph.com,smithi115.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-03-19T01:15:36.240 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi099.front.sepia.ceph.com'), Remote(name='ubuntu@smithi112.front.sepia.ceph.com'), Remote(name='ubuntu@smithi115.front.sepia.ceph.com')] 2024-03-19T01:15:36.270 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi099.front.sepia.ceph.com' 2024-03-19T01:15:36.272 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi099.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-19T01:15:36.365 DEBUG:teuthology.orchestra.run.smithi099:> true 2024-03-19T01:15:36.442 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi099.front.sepia.ceph.com' 2024-03-19T01:15:36.442 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi112.front.sepia.ceph.com' 2024-03-19T01:15:36.443 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi112.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-19T01:15:36.533 DEBUG:teuthology.orchestra.run.smithi112:> true 2024-03-19T01:15:36.606 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi112.front.sepia.ceph.com' 2024-03-19T01:15:36.607 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi115.front.sepia.ceph.com' 2024-03-19T01:15:36.608 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi115.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-19T01:15:36.696 DEBUG:teuthology.orchestra.run.smithi115:> true 2024-03-19T01:15:36.769 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi115.front.sepia.ceph.com' 2024-03-19T01:15:36.770 INFO:teuthology.run_tasks:Running task clock... 2024-03-19T01:15:36.781 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-03-19T01:15:36.781 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-03-19T01:15:36.781 DEBUG:teuthology.orchestra.run.smithi099:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-03-19T01:15:36.784 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-03-19T01:15:36.784 DEBUG:teuthology.orchestra.run.smithi112:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-03-19T01:15:36.786 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-03-19T01:15:36.786 DEBUG:teuthology.orchestra.run.smithi115:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-03-19T01:15:36.814 INFO:teuthology.orchestra.run.smithi099.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-03-19T01:15:36.819 INFO:teuthology.orchestra.run.smithi112.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-03-19T01:15:36.832 INFO:teuthology.orchestra.run.smithi099.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-03-19T01:15:36.838 INFO:teuthology.orchestra.run.smithi112.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-03-19T01:15:36.841 INFO:teuthology.orchestra.run.smithi115.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-03-19T01:15:36.858 INFO:teuthology.orchestra.run.smithi115.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-03-19T01:15:36.863 INFO:teuthology.orchestra.run.smithi099.stderr:sudo: ntpd: command not found 2024-03-19T01:15:36.868 INFO:teuthology.orchestra.run.smithi112.stderr:sudo: ntpd: command not found 2024-03-19T01:15:36.876 INFO:teuthology.orchestra.run.smithi099.stdout:506 Cannot talk to daemon 2024-03-19T01:15:36.882 INFO:teuthology.orchestra.run.smithi112.stdout:506 Cannot talk to daemon 2024-03-19T01:15:36.888 INFO:teuthology.orchestra.run.smithi115.stderr:sudo: ntpd: command not found 2024-03-19T01:15:36.893 INFO:teuthology.orchestra.run.smithi099.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-03-19T01:15:36.899 INFO:teuthology.orchestra.run.smithi112.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-03-19T01:15:36.901 INFO:teuthology.orchestra.run.smithi115.stdout:506 Cannot talk to daemon 2024-03-19T01:15:36.910 INFO:teuthology.orchestra.run.smithi099.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-03-19T01:15:36.915 INFO:teuthology.orchestra.run.smithi112.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-03-19T01:15:36.917 INFO:teuthology.orchestra.run.smithi115.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-03-19T01:15:36.934 INFO:teuthology.orchestra.run.smithi115.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-03-19T01:15:36.959 INFO:teuthology.orchestra.run.smithi112.stderr:bash: line 1: ntpq: command not found 2024-03-19T01:15:36.962 INFO:teuthology.orchestra.run.smithi112.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-03-19T01:15:36.962 INFO:teuthology.orchestra.run.smithi112.stdout:=============================================================================== 2024-03-19T01:15:36.962 INFO:teuthology.orchestra.run.smithi112.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-19T01:15:36.962 INFO:teuthology.orchestra.run.smithi112.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-19T01:15:36.962 INFO:teuthology.orchestra.run.smithi112.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-19T01:15:36.962 INFO:teuthology.orchestra.run.smithi112.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-19T01:15:36.967 INFO:teuthology.orchestra.run.smithi099.stderr:bash: line 1: ntpq: command not found 2024-03-19T01:15:36.971 INFO:teuthology.orchestra.run.smithi099.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-03-19T01:15:36.971 INFO:teuthology.orchestra.run.smithi099.stdout:=============================================================================== 2024-03-19T01:15:36.971 INFO:teuthology.orchestra.run.smithi099.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-19T01:15:36.971 INFO:teuthology.orchestra.run.smithi099.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-19T01:15:36.971 INFO:teuthology.orchestra.run.smithi099.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-19T01:15:36.971 INFO:teuthology.orchestra.run.smithi099.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-19T01:15:36.978 INFO:teuthology.orchestra.run.smithi115.stderr:bash: line 1: ntpq: command not found 2024-03-19T01:15:36.981 INFO:teuthology.orchestra.run.smithi115.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-03-19T01:15:36.981 INFO:teuthology.orchestra.run.smithi115.stdout:=============================================================================== 2024-03-19T01:15:36.981 INFO:teuthology.orchestra.run.smithi115.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-19T01:15:36.981 INFO:teuthology.orchestra.run.smithi115.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-19T01:15:36.981 INFO:teuthology.orchestra.run.smithi115.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-19T01:15:36.981 INFO:teuthology.orchestra.run.smithi115.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-19T01:15:36.982 INFO:teuthology.run_tasks:Running task install... 2024-03-19T01:15:36.990 DEBUG:teuthology.task.install:project ceph 2024-03-19T01:15:36.990 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'b6ae24918b03b1b8a19c9263a50655bba80bb810'}} 2024-03-19T01:15:36.991 DEBUG:teuthology.task.install:config {'extra_packages': {'deb': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror'], 'rpm': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror']}, 'extra_system_packages': {'deb': ['bison', 'flex', 'libelf-dev', 'libssl-dev', 'network-manager', 'iproute2', 'util-linux', 'dump', 'indent', 'libaio-dev', 'libtool-bin', 'uuid-dev', 'xfslibs-dev', 'postgresql', 'postgresql-client', 'postgresql-common', 'postgresql-contrib'], 'rpm': ['bison', 'flex', 'elfutils-libelf-devel', 'openssl-devel', 'NetworkManager', 'iproute', 'util-linux', 'libacl-devel', 'libaio-devel', 'libattr-devel', 'libtool', 'libuuid-devel', 'xfsdump', 'xfsprogs', 'xfsprogs-devel', 'libaio-devel', 'libtool', 'libuuid-devel', 'xfsprogs-devel', 'postgresql', 'postgresql-server', 'postgresql-contrib']}, 'flavor': 'default', 'sha1': 'b6ae24918b03b1b8a19c9263a50655bba80bb810'} 2024-03-19T01:15:36.991 INFO:teuthology.task.install:Using flavor: default 2024-03-19T01:15:36.998 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'ceph-volume', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'ceph-volume', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-03-19T01:15:36.998 INFO:teuthology.task.install:extra packages: {'deb': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror'], 'rpm': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror']} 2024-03-19T01:15:36.999 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': {'deb': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror'], 'rpm': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror']}, 'extra_system_packages': {'deb': ['bison', 'flex', 'libelf-dev', 'libssl-dev', 'network-manager', 'iproute2', 'util-linux', 'dump', 'indent', 'libaio-dev', 'libtool-bin', 'uuid-dev', 'xfslibs-dev', 'postgresql', 'postgresql-client', 'postgresql-common', 'postgresql-contrib'], 'rpm': ['bison', 'flex', 'elfutils-libelf-devel', 'openssl-devel', 'NetworkManager', 'iproute', 'util-linux', 'libacl-devel', 'libaio-devel', 'libattr-devel', 'libtool', 'libuuid-devel', 'xfsdump', 'xfsprogs', 'xfsprogs-devel', 'libaio-devel', 'libtool', 'libuuid-devel', 'xfsprogs-devel', 'postgresql', 'postgresql-server', 'postgresql-contrib']}, 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'b6ae24918b03b1b8a19c9263a50655bba80bb810', 'tag': None, 'wait_for_package': False} 2024-03-19T01:15:36.999 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=b6ae24918b03b1b8a19c9263a50655bba80bb810 2024-03-19T01:15:37.001 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': {'deb': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror'], 'rpm': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror']}, 'extra_system_packages': {'deb': ['bison', 'flex', 'libelf-dev', 'libssl-dev', 'network-manager', 'iproute2', 'util-linux', 'dump', 'indent', 'libaio-dev', 'libtool-bin', 'uuid-dev', 'xfslibs-dev', 'postgresql', 'postgresql-client', 'postgresql-common', 'postgresql-contrib'], 'rpm': ['bison', 'flex', 'elfutils-libelf-devel', 'openssl-devel', 'NetworkManager', 'iproute', 'util-linux', 'libacl-devel', 'libaio-devel', 'libattr-devel', 'libtool', 'libuuid-devel', 'xfsdump', 'xfsprogs', 'xfsprogs-devel', 'libaio-devel', 'libtool', 'libuuid-devel', 'xfsprogs-devel', 'postgresql', 'postgresql-server', 'postgresql-contrib']}, 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'b6ae24918b03b1b8a19c9263a50655bba80bb810', 'tag': None, 'wait_for_package': False} 2024-03-19T01:15:37.001 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=b6ae24918b03b1b8a19c9263a50655bba80bb810 2024-03-19T01:15:37.002 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': {'deb': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror'], 'rpm': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror']}, 'extra_system_packages': {'deb': ['bison', 'flex', 'libelf-dev', 'libssl-dev', 'network-manager', 'iproute2', 'util-linux', 'dump', 'indent', 'libaio-dev', 'libtool-bin', 'uuid-dev', 'xfslibs-dev', 'postgresql', 'postgresql-client', 'postgresql-common', 'postgresql-contrib'], 'rpm': ['bison', 'flex', 'elfutils-libelf-devel', 'openssl-devel', 'NetworkManager', 'iproute', 'util-linux', 'libacl-devel', 'libaio-devel', 'libattr-devel', 'libtool', 'libuuid-devel', 'xfsdump', 'xfsprogs', 'xfsprogs-devel', 'libaio-devel', 'libtool', 'libuuid-devel', 'xfsprogs-devel', 'postgresql', 'postgresql-server', 'postgresql-contrib']}, 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'b6ae24918b03b1b8a19c9263a50655bba80bb810', 'tag': None, 'wait_for_package': False} 2024-03-19T01:15:37.003 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=b6ae24918b03b1b8a19c9263a50655bba80bb810 2024-03-19T01:15:37.152 INFO:teuthology.task.install.rpm:Pulling from https://4.chacra.ceph.com/r/ceph/wip-batrick-testing-20240318.181317/b6ae24918b03b1b8a19c9263a50655bba80bb810/centos/9/flavors/default/ 2024-03-19T01:15:37.152 INFO:teuthology.task.install.rpm:Package version is 19.0.0-2206.gb6ae2491 2024-03-19T01:15:37.155 INFO:teuthology.task.install.rpm:Pulling from https://4.chacra.ceph.com/r/ceph/wip-batrick-testing-20240318.181317/b6ae24918b03b1b8a19c9263a50655bba80bb810/centos/9/flavors/default/ 2024-03-19T01:15:37.155 INFO:teuthology.task.install.rpm:Package version is 19.0.0-2206.gb6ae2491 2024-03-19T01:15:37.220 INFO:teuthology.task.install.rpm:Pulling from https://4.chacra.ceph.com/r/ceph/wip-batrick-testing-20240318.181317/b6ae24918b03b1b8a19c9263a50655bba80bb810/centos/9/flavors/default/ 2024-03-19T01:15:37.220 INFO:teuthology.task.install.rpm:Package version is 19.0.0-2206.gb6ae2491 2024-03-19T01:15:37.271 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://4.chacra.ceph.com/r/ceph/wip-batrick-testing-20240318.181317/b6ae24918b03b1b8a19c9263a50655bba80bb810/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://4.chacra.ceph.com/r/ceph/wip-batrick-testing-20240318.181317/b6ae24918b03b1b8a19c9263a50655bba80bb810/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://4.chacra.ceph.com/r/ceph/wip-batrick-testing-20240318.181317/b6ae24918b03b1b8a19c9263a50655bba80bb810/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-03-19T01:15:37.271 DEBUG:teuthology.orchestra.run.smithi112:> set -ex 2024-03-19T01:15:37.272 DEBUG:teuthology.orchestra.run.smithi112:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-03-19T01:15:37.299 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, python3-cephfs, cephfs-shell, cephfs-top, cephfs-mirror, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib on remote rpm x86_64 2024-03-19T01:15:37.300 DEBUG:teuthology.orchestra.run.smithi112:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/b6ae24918b03b1b8a19c9263a50655bba80bb810/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-03-19T01:15:37.324 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://4.chacra.ceph.com/r/ceph/wip-batrick-testing-20240318.181317/b6ae24918b03b1b8a19c9263a50655bba80bb810/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://4.chacra.ceph.com/r/ceph/wip-batrick-testing-20240318.181317/b6ae24918b03b1b8a19c9263a50655bba80bb810/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://4.chacra.ceph.com/r/ceph/wip-batrick-testing-20240318.181317/b6ae24918b03b1b8a19c9263a50655bba80bb810/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-03-19T01:15:37.325 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2024-03-19T01:15:37.325 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-03-19T01:15:37.353 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, python3-cephfs, cephfs-shell, cephfs-top, cephfs-mirror, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib on remote rpm x86_64 2024-03-19T01:15:37.353 DEBUG:teuthology.orchestra.run.smithi099:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/b6ae24918b03b1b8a19c9263a50655bba80bb810/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-03-19T01:15:37.367 DEBUG:teuthology.orchestra.run.smithi112:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-03-19T01:15:37.392 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://4.chacra.ceph.com/r/ceph/wip-batrick-testing-20240318.181317/b6ae24918b03b1b8a19c9263a50655bba80bb810/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://4.chacra.ceph.com/r/ceph/wip-batrick-testing-20240318.181317/b6ae24918b03b1b8a19c9263a50655bba80bb810/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://4.chacra.ceph.com/r/ceph/wip-batrick-testing-20240318.181317/b6ae24918b03b1b8a19c9263a50655bba80bb810/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-03-19T01:15:37.392 DEBUG:teuthology.orchestra.run.smithi115:> set -ex 2024-03-19T01:15:37.392 DEBUG:teuthology.orchestra.run.smithi115:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-03-19T01:15:37.420 DEBUG:teuthology.orchestra.run.smithi099:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-03-19T01:15:37.422 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, python3-cephfs, cephfs-shell, cephfs-top, cephfs-mirror, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib on remote rpm x86_64 2024-03-19T01:15:37.422 DEBUG:teuthology.orchestra.run.smithi115:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/b6ae24918b03b1b8a19c9263a50655bba80bb810/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-03-19T01:15:37.447 DEBUG:teuthology.orchestra.run.smithi112:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-03-19T01:15:37.492 DEBUG:teuthology.orchestra.run.smithi115:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-03-19T01:15:37.499 DEBUG:teuthology.orchestra.run.smithi099:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-03-19T01:15:37.514 INFO:teuthology.orchestra.run.smithi112.stdout:check_obsoletes = 1 2024-03-19T01:15:37.515 DEBUG:teuthology.orchestra.run.smithi112:> sudo yum clean all 2024-03-19T01:15:37.565 INFO:teuthology.orchestra.run.smithi099.stdout:check_obsoletes = 1 2024-03-19T01:15:37.567 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum clean all 2024-03-19T01:15:37.573 DEBUG:teuthology.orchestra.run.smithi115:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-03-19T01:15:37.639 INFO:teuthology.orchestra.run.smithi115.stdout:check_obsoletes = 1 2024-03-19T01:15:37.641 DEBUG:teuthology.orchestra.run.smithi115:> sudo yum clean all 2024-03-19T01:15:37.751 INFO:teuthology.orchestra.run.smithi112.stdout:74 files removed 2024-03-19T01:15:37.778 DEBUG:teuthology.orchestra.run.smithi112:> sudo yum -y install ceph-radosgw ceph-test ceph ceph-base cephadm ceph-immutable-object-cache ceph-mgr ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-rook ceph-mgr-cephadm ceph-fuse ceph-volume librados-devel libcephfs2 libcephfs-devel librados2 librbd1 python3-rados python3-rgw python3-cephfs python3-rbd rbd-fuse rbd-mirror rbd-nbd python3-cephfs cephfs-shell cephfs-top cephfs-mirror bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib 2024-03-19T01:15:37.809 INFO:teuthology.orchestra.run.smithi099.stdout:74 files removed 2024-03-19T01:15:37.836 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum -y install ceph-radosgw ceph-test ceph ceph-base cephadm ceph-immutable-object-cache ceph-mgr ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-rook ceph-mgr-cephadm ceph-fuse ceph-volume librados-devel libcephfs2 libcephfs-devel librados2 librbd1 python3-rados python3-rgw python3-cephfs python3-rbd rbd-fuse rbd-mirror rbd-nbd python3-cephfs cephfs-shell cephfs-top cephfs-mirror bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib 2024-03-19T01:15:37.875 INFO:teuthology.orchestra.run.smithi115.stdout:74 files removed 2024-03-19T01:15:37.903 DEBUG:teuthology.orchestra.run.smithi115:> sudo yum -y install ceph-radosgw ceph-test ceph ceph-base cephadm ceph-immutable-object-cache ceph-mgr ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-rook ceph-mgr-cephadm ceph-fuse ceph-volume librados-devel libcephfs2 libcephfs-devel librados2 librbd1 python3-rados python3-rgw python3-cephfs python3-rbd rbd-fuse rbd-mirror rbd-nbd python3-cephfs cephfs-shell cephfs-top cephfs-mirror bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib 2024-03-19T01:15:38.459 INFO:teuthology.orchestra.run.smithi112.stdout:ceph packages for x86_64 211 kB/s | 83 kB 00:00 2024-03-19T01:15:38.527 INFO:teuthology.orchestra.run.smithi099.stdout:ceph packages for x86_64 212 kB/s | 83 kB 00:00 2024-03-19T01:15:38.558 INFO:teuthology.orchestra.run.smithi115.stdout:ceph packages for x86_64 227 kB/s | 83 kB 00:00 2024-03-19T01:15:38.809 INFO:teuthology.orchestra.run.smithi112.stdout:ceph noarch packages 33 kB/s | 10 kB 00:00 2024-03-19T01:15:38.934 INFO:teuthology.orchestra.run.smithi115.stdout:ceph noarch packages 30 kB/s | 10 kB 00:00 2024-03-19T01:15:38.945 INFO:teuthology.orchestra.run.smithi099.stdout:ceph noarch packages 27 kB/s | 10 kB 00:00 2024-03-19T01:15:39.236 INFO:teuthology.orchestra.run.smithi112.stdout:ceph source packages 4.8 kB/s | 1.9 kB 00:00 2024-03-19T01:15:39.309 INFO:teuthology.orchestra.run.smithi115.stdout:ceph source packages 5.5 kB/s | 1.9 kB 00:00 2024-03-19T01:15:39.345 INFO:teuthology.orchestra.run.smithi099.stdout:ceph source packages 5.2 kB/s | 1.9 kB 00:00 2024-03-19T01:15:39.798 INFO:teuthology.orchestra.run.smithi099.stdout:CentOS-9 - AppStream 16 MB/s | 6.7 MB 00:00 2024-03-19T01:15:39.828 INFO:teuthology.orchestra.run.smithi115.stdout:CentOS-9 - AppStream 14 MB/s | 6.7 MB 00:00 2024-03-19T01:15:39.912 INFO:teuthology.orchestra.run.smithi112.stdout:CentOS-9 - AppStream 10 MB/s | 6.7 MB 00:00 2024-03-19T01:15:41.314 INFO:teuthology.orchestra.run.smithi099.stdout:CentOS-9 - BaseOS 4.9 MB/s | 2.0 MB 00:00 2024-03-19T01:15:41.337 INFO:teuthology.orchestra.run.smithi112.stdout:CentOS-9 - BaseOS 6.5 MB/s | 2.0 MB 00:00 2024-03-19T01:15:41.528 INFO:teuthology.orchestra.run.smithi115.stdout:CentOS-9 - BaseOS 3.3 MB/s | 2.0 MB 00:00 2024-03-19T01:15:43.222 INFO:teuthology.orchestra.run.smithi099.stdout:CentOS Stream 9 - BaseOS 5.1 MB/s | 8.0 MB 00:01 2024-03-19T01:15:46.981 INFO:teuthology.orchestra.run.smithi099.stdout:CentOS Stream 9 - AppStream 6.9 MB/s | 19 MB 00:02 2024-03-19T01:15:51.006 INFO:teuthology.orchestra.run.smithi099.stdout:CentOS Stream 9 - CRB 10 MB/s | 6.1 MB 00:00 2024-03-19T01:15:52.640 INFO:teuthology.orchestra.run.smithi099.stdout:CentOS Stream 9 - Extras packages 59 kB/s | 16 kB 00:00 2024-03-19T01:20:07.082 INFO:teuthology.orchestra.run.smithi112.stdout:CentOS Stream 9 - BaseOS 31 kB/s | 8.0 MB 04:25 2024-03-19T01:20:09.155 INFO:teuthology.orchestra.run.smithi115.stdout:CentOS Stream 9 - BaseOS 31 kB/s | 8.0 MB 04:27 2024-03-19T01:20:11.105 INFO:teuthology.orchestra.run.smithi115.stdout:CentOS Stream 9 - AppStream 20 MB/s | 19 MB 00:00 2024-03-19T01:20:17.705 INFO:teuthology.orchestra.run.smithi115.stdout:CentOS Stream 9 - CRB 1.9 MB/s | 6.1 MB 00:03 2024-03-19T01:20:19.250 INFO:teuthology.orchestra.run.smithi112.stdout:CentOS Stream 9 - AppStream 1.7 MB/s | 19 MB 00:11 2024-03-19T01:20:19.264 INFO:teuthology.orchestra.run.smithi115.stdout:CentOS Stream 9 - Extras packages 38 kB/s | 16 kB 00:00 2024-03-19T01:20:20.743 INFO:teuthology.orchestra.run.smithi115.stdout:Extra Packages for Enterprise Linux 15 MB/s | 21 MB 00:01 2024-03-19T01:20:24.332 INFO:teuthology.orchestra.run.smithi112.stdout:CentOS Stream 9 - CRB 3.8 MB/s | 6.1 MB 00:01 2024-03-19T01:20:25.723 INFO:teuthology.orchestra.run.smithi115.stdout:lab-extras 19 kB/s | 1.7 kB 00:00 2024-03-19T01:20:25.750 INFO:teuthology.orchestra.run.smithi112.stdout:CentOS Stream 9 - Extras packages 55 kB/s | 16 kB 00:00 2024-03-19T01:20:27.435 INFO:teuthology.orchestra.run.smithi112.stdout:Extra Packages for Enterprise Linux 13 MB/s | 21 MB 00:01 2024-03-19T01:20:28.053 INFO:teuthology.orchestra.run.smithi115.stdout:Package librados2-2:16.2.4-5.el9.x86_64 is already installed. 2024-03-19T01:20:28.054 INFO:teuthology.orchestra.run.smithi115.stdout:Package librbd1-2:16.2.4-5.el9.x86_64 is already installed. 2024-03-19T01:20:28.067 INFO:teuthology.orchestra.run.smithi115.stdout:Package NetworkManager-1:1.44.0-2.el9.x86_64 is already installed. 2024-03-19T01:20:28.068 INFO:teuthology.orchestra.run.smithi115.stdout:Package iproute-6.2.0-5.el9.x86_64 is already installed. 2024-03-19T01:20:28.069 INFO:teuthology.orchestra.run.smithi115.stdout:Package util-linux-2.37.4-15.el9.x86_64 is already installed. 2024-03-19T01:20:28.070 INFO:teuthology.orchestra.run.smithi115.stdout:Package libacl-devel-2.3.1-3.el9.x86_64 is already installed. 2024-03-19T01:20:28.072 INFO:teuthology.orchestra.run.smithi115.stdout:Package libattr-devel-2.5.1-3.el9.x86_64 is already installed. 2024-03-19T01:20:28.075 INFO:teuthology.orchestra.run.smithi115.stdout:Package xfsdump-3.1.12-3.el9.x86_64 is already installed. 2024-03-19T01:20:28.076 INFO:teuthology.orchestra.run.smithi115.stdout:Package xfsprogs-5.19.0-4.el9.x86_64 is already installed. 2024-03-19T01:20:28.156 INFO:teuthology.orchestra.run.smithi115.stdout:Dependencies resolved. 2024-03-19T01:20:28.179 INFO:teuthology.orchestra.run.smithi115.stdout:============================================================================================ 2024-03-19T01:20:28.179 INFO:teuthology.orchestra.run.smithi115.stdout: Package Arch Version Repository Size 2024-03-19T01:20:28.179 INFO:teuthology.orchestra.run.smithi115.stdout:============================================================================================ 2024-03-19T01:20:28.179 INFO:teuthology.orchestra.run.smithi115.stdout:Installing: 2024-03-19T01:20:28.179 INFO:teuthology.orchestra.run.smithi115.stdout: bison x86_64 3.7.4-5.el9 CentOS-AppStream 920 k 2024-03-19T01:20:28.179 INFO:teuthology.orchestra.run.smithi115.stdout: ceph x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 6.6 k 2024-03-19T01:20:28.180 INFO:teuthology.orchestra.run.smithi115.stdout: ceph-base x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 5.4 M 2024-03-19T01:20:28.180 INFO:teuthology.orchestra.run.smithi115.stdout: ceph-fuse x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 891 k 2024-03-19T01:20:28.180 INFO:teuthology.orchestra.run.smithi115.stdout: ceph-immutable-object-cache x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 145 k 2024-03-19T01:20:28.180 INFO:teuthology.orchestra.run.smithi115.stdout: ceph-mgr x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 1.5 M 2024-03-19T01:20:28.180 INFO:teuthology.orchestra.run.smithi115.stdout: ceph-mgr-cephadm noarch 2:19.0.0-2206.gb6ae2491.el9 ceph-noarch 139 k 2024-03-19T01:20:28.180 INFO:teuthology.orchestra.run.smithi115.stdout: ceph-mgr-dashboard noarch 2:19.0.0-2206.gb6ae2491.el9 ceph-noarch 1.8 M 2024-03-19T01:20:28.180 INFO:teuthology.orchestra.run.smithi115.stdout: ceph-mgr-diskprediction-local noarch 2:19.0.0-2206.gb6ae2491.el9 ceph-noarch 7.4 M 2024-03-19T01:20:28.180 INFO:teuthology.orchestra.run.smithi115.stdout: ceph-mgr-rook noarch 2:19.0.0-2206.gb6ae2491.el9 ceph-noarch 49 k 2024-03-19T01:20:28.180 INFO:teuthology.orchestra.run.smithi115.stdout: ceph-radosgw x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 9.8 M 2024-03-19T01:20:28.180 INFO:teuthology.orchestra.run.smithi115.stdout: ceph-test x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 48 M 2024-03-19T01:20:28.180 INFO:teuthology.orchestra.run.smithi115.stdout: ceph-volume noarch 2:19.0.0-2206.gb6ae2491.el9 ceph-noarch 264 k 2024-03-19T01:20:28.180 INFO:teuthology.orchestra.run.smithi115.stdout: cephadm noarch 2:19.0.0-2206.gb6ae2491.el9 ceph-noarch 749 k 2024-03-19T01:20:28.180 INFO:teuthology.orchestra.run.smithi115.stdout: cephfs-mirror x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 221 k 2024-03-19T01:20:28.181 INFO:teuthology.orchestra.run.smithi115.stdout: cephfs-shell x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 27 k 2024-03-19T01:20:28.181 INFO:teuthology.orchestra.run.smithi115.stdout: cephfs-top noarch 2:19.0.0-2206.gb6ae2491.el9 ceph-noarch 18 k 2024-03-19T01:20:28.181 INFO:teuthology.orchestra.run.smithi115.stdout: elfutils-libelf-devel x86_64 0.190-2.el9 appstream 21 k 2024-03-19T01:20:28.181 INFO:teuthology.orchestra.run.smithi115.stdout: flex x86_64 2.6.4-9.el9 CentOS-AppStream 307 k 2024-03-19T01:20:28.181 INFO:teuthology.orchestra.run.smithi115.stdout: libaio-devel x86_64 0.3.111-13.el9 CentOS-AppStream 10 k 2024-03-19T01:20:28.181 INFO:teuthology.orchestra.run.smithi115.stdout: libcephfs-devel x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 32 k 2024-03-19T01:20:28.181 INFO:teuthology.orchestra.run.smithi115.stdout: libcephfs2 x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 746 k 2024-03-19T01:20:28.181 INFO:teuthology.orchestra.run.smithi115.stdout: librados-devel x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 127 k 2024-03-19T01:20:28.181 INFO:teuthology.orchestra.run.smithi115.stdout: libtool x86_64 2.4.6-45.el9 CentOS-AppStream 578 k 2024-03-19T01:20:28.181 INFO:teuthology.orchestra.run.smithi115.stdout: libuuid-devel x86_64 2.37.4-18.el9 appstream 25 k 2024-03-19T01:20:28.181 INFO:teuthology.orchestra.run.smithi115.stdout: openssl-devel x86_64 1:3.0.7-27.el9 appstream 4.1 M 2024-03-19T01:20:28.181 INFO:teuthology.orchestra.run.smithi115.stdout: postgresql x86_64 13.11-1.el9 CentOS-AppStream 1.5 M 2024-03-19T01:20:28.182 INFO:teuthology.orchestra.run.smithi115.stdout: postgresql-contrib x86_64 13.11-1.el9 CentOS-AppStream 811 k 2024-03-19T01:20:28.182 INFO:teuthology.orchestra.run.smithi115.stdout: postgresql-server x86_64 13.11-1.el9 CentOS-AppStream 5.7 M 2024-03-19T01:20:28.182 INFO:teuthology.orchestra.run.smithi115.stdout: python3-cephfs x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 162 k 2024-03-19T01:20:28.182 INFO:teuthology.orchestra.run.smithi115.stdout: python3-rados x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 321 k 2024-03-19T01:20:28.182 INFO:teuthology.orchestra.run.smithi115.stdout: python3-rbd x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 298 k 2024-03-19T01:20:28.182 INFO:teuthology.orchestra.run.smithi115.stdout: python3-rgw x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 99 k 2024-03-19T01:20:28.182 INFO:teuthology.orchestra.run.smithi115.stdout: rbd-fuse x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 85 k 2024-03-19T01:20:28.182 INFO:teuthology.orchestra.run.smithi115.stdout: rbd-mirror x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 3.1 M 2024-03-19T01:20:28.182 INFO:teuthology.orchestra.run.smithi115.stdout: rbd-nbd x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 170 k 2024-03-19T01:20:28.182 INFO:teuthology.orchestra.run.smithi115.stdout: xfsprogs-devel x86_64 6.3.0-1.el9 appstream 94 k 2024-03-19T01:20:28.182 INFO:teuthology.orchestra.run.smithi115.stdout:Upgrading: 2024-03-19T01:20:28.182 INFO:teuthology.orchestra.run.smithi115.stdout: NetworkManager x86_64 1:1.47.2-1.el9 baseos 2.3 M 2024-03-19T01:20:28.182 INFO:teuthology.orchestra.run.smithi115.stdout: NetworkManager-libnm x86_64 1:1.47.2-1.el9 baseos 1.8 M 2024-03-19T01:20:28.183 INFO:teuthology.orchestra.run.smithi115.stdout: NetworkManager-team x86_64 1:1.47.2-1.el9 baseos 42 k 2024-03-19T01:20:28.183 INFO:teuthology.orchestra.run.smithi115.stdout: NetworkManager-tui x86_64 1:1.47.2-1.el9 baseos 248 k 2024-03-19T01:20:28.183 INFO:teuthology.orchestra.run.smithi115.stdout: acl x86_64 2.3.1-4.el9 baseos 71 k 2024-03-19T01:20:28.183 INFO:teuthology.orchestra.run.smithi115.stdout: elfutils-debuginfod-client x86_64 0.190-2.el9 baseos 36 k 2024-03-19T01:20:28.183 INFO:teuthology.orchestra.run.smithi115.stdout: elfutils-libelf x86_64 0.190-2.el9 baseos 192 k 2024-03-19T01:20:28.183 INFO:teuthology.orchestra.run.smithi115.stdout: elfutils-libs x86_64 0.190-2.el9 baseos 256 k 2024-03-19T01:20:28.183 INFO:teuthology.orchestra.run.smithi115.stdout: libacl x86_64 2.3.1-4.el9 baseos 23 k 2024-03-19T01:20:28.183 INFO:teuthology.orchestra.run.smithi115.stdout: libacl-devel x86_64 2.3.1-4.el9 appstream 74 k 2024-03-19T01:20:28.183 INFO:teuthology.orchestra.run.smithi115.stdout: libblkid x86_64 2.37.4-18.el9 baseos 108 k 2024-03-19T01:20:28.183 INFO:teuthology.orchestra.run.smithi115.stdout: libfdisk x86_64 2.37.4-18.el9 baseos 155 k 2024-03-19T01:20:28.183 INFO:teuthology.orchestra.run.smithi115.stdout: libmount x86_64 2.37.4-18.el9 baseos 136 k 2024-03-19T01:20:28.183 INFO:teuthology.orchestra.run.smithi115.stdout: librados2 x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 3.4 M 2024-03-19T01:20:28.183 INFO:teuthology.orchestra.run.smithi115.stdout: librbd1 x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 3.1 M 2024-03-19T01:20:28.183 INFO:teuthology.orchestra.run.smithi115.stdout: libsmartcols x86_64 2.37.4-18.el9 baseos 64 k 2024-03-19T01:20:28.184 INFO:teuthology.orchestra.run.smithi115.stdout: libuuid x86_64 2.37.4-18.el9 baseos 29 k 2024-03-19T01:20:28.184 INFO:teuthology.orchestra.run.smithi115.stdout: openssl x86_64 1:3.0.7-27.el9 baseos 1.2 M 2024-03-19T01:20:28.184 INFO:teuthology.orchestra.run.smithi115.stdout: openssl-libs x86_64 1:3.0.7-27.el9 baseos 2.2 M 2024-03-19T01:20:28.184 INFO:teuthology.orchestra.run.smithi115.stdout: util-linux x86_64 2.37.4-18.el9 baseos 2.3 M 2024-03-19T01:20:28.184 INFO:teuthology.orchestra.run.smithi115.stdout: util-linux-core x86_64 2.37.4-18.el9 baseos 465 k 2024-03-19T01:20:28.184 INFO:teuthology.orchestra.run.smithi115.stdout: util-linux-user x86_64 2.37.4-18.el9 baseos 31 k 2024-03-19T01:20:28.184 INFO:teuthology.orchestra.run.smithi115.stdout: xfsdump x86_64 3.1.12-4.el9 CentOS-BaseOS 321 k 2024-03-19T01:20:28.184 INFO:teuthology.orchestra.run.smithi115.stdout: xfsprogs x86_64 6.3.0-1.el9 baseos 1.1 M 2024-03-19T01:20:28.184 INFO:teuthology.orchestra.run.smithi115.stdout:Installing dependencies: 2024-03-19T01:20:28.185 INFO:teuthology.orchestra.run.smithi115.stdout: abseil-cpp x86_64 20211102.0-3.el9 epel 548 k 2024-03-19T01:20:28.185 INFO:teuthology.orchestra.run.smithi115.stdout: automake noarch 1.16.2-8.el9 CentOS-AppStream 662 k 2024-03-19T01:20:28.185 INFO:teuthology.orchestra.run.smithi115.stdout: boost-program-options x86_64 1.75.0-8.el9 CentOS-AppStream 106 k 2024-03-19T01:20:28.185 INFO:teuthology.orchestra.run.smithi115.stdout: ceph-common x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 21 M 2024-03-19T01:20:28.185 INFO:teuthology.orchestra.run.smithi115.stdout: ceph-grafana-dashboards noarch 2:19.0.0-2206.gb6ae2491.el9 ceph-noarch 34 k 2024-03-19T01:20:28.185 INFO:teuthology.orchestra.run.smithi115.stdout: ceph-mds x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 2.4 M 2024-03-19T01:20:28.185 INFO:teuthology.orchestra.run.smithi115.stdout: ceph-mgr-modules-core noarch 2:19.0.0-2206.gb6ae2491.el9 ceph-noarch 248 k 2024-03-19T01:20:28.185 INFO:teuthology.orchestra.run.smithi115.stdout: ceph-mon x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 4.7 M 2024-03-19T01:20:28.185 INFO:teuthology.orchestra.run.smithi115.stdout: ceph-osd x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 17 M 2024-03-19T01:20:28.185 INFO:teuthology.orchestra.run.smithi115.stdout: ceph-prometheus-alerts noarch 2:19.0.0-2206.gb6ae2491.el9 ceph-noarch 16 k 2024-03-19T01:20:28.186 INFO:teuthology.orchestra.run.smithi115.stdout: ceph-selinux x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 25 k 2024-03-19T01:20:28.186 INFO:teuthology.orchestra.run.smithi115.stdout: flexiblas x86_64 3.0.4-8.el9 CentOS-AppStream 32 k 2024-03-19T01:20:28.186 INFO:teuthology.orchestra.run.smithi115.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 CentOS-AppStream 3.0 M 2024-03-19T01:20:28.186 INFO:teuthology.orchestra.run.smithi115.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 CentOS-AppStream 17 k 2024-03-19T01:20:28.186 INFO:teuthology.orchestra.run.smithi115.stdout: fuse x86_64 2.9.9-15.el9 CentOS-BaseOS 79 k 2024-03-19T01:20:28.186 INFO:teuthology.orchestra.run.smithi115.stdout: gperftools-libs x86_64 2.9.1-2.el9 epel 309 k 2024-03-19T01:20:28.186 INFO:teuthology.orchestra.run.smithi115.stdout: grpc-data noarch 1.46.7-10.el9 epel 19 k 2024-03-19T01:20:28.186 INFO:teuthology.orchestra.run.smithi115.stdout: jq x86_64 1.6-16.el9 baseos 187 k 2024-03-19T01:20:28.186 INFO:teuthology.orchestra.run.smithi115.stdout: libarrow x86_64 9.0.0-10.el9 epel 4.4 M 2024-03-19T01:20:28.186 INFO:teuthology.orchestra.run.smithi115.stdout: libarrow-doc noarch 9.0.0-10.el9 epel 26 k 2024-03-19T01:20:28.186 INFO:teuthology.orchestra.run.smithi115.stdout: libcephsqlite x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 163 k 2024-03-19T01:20:28.186 INFO:teuthology.orchestra.run.smithi115.stdout: libgfortran x86_64 11.4.1-3.el9 baseos 803 k 2024-03-19T01:20:28.186 INFO:teuthology.orchestra.run.smithi115.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-03-19T01:20:28.187 INFO:teuthology.orchestra.run.smithi115.stdout: libpmemobj x86_64 1.12.1-1.el9 CentOS-AppStream 159 k 2024-03-19T01:20:28.187 INFO:teuthology.orchestra.run.smithi115.stdout: libquadmath x86_64 11.4.1-3.el9 baseos 195 k 2024-03-19T01:20:28.187 INFO:teuthology.orchestra.run.smithi115.stdout: librabbitmq x86_64 0.11.0-7.el9 CentOS-AppStream 44 k 2024-03-19T01:20:28.187 INFO:teuthology.orchestra.run.smithi115.stdout: libradosstriper1 x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 501 k 2024-03-19T01:20:28.187 INFO:teuthology.orchestra.run.smithi115.stdout: librdkafka x86_64 1.6.1-102.el9 CentOS-AppStream 661 k 2024-03-19T01:20:28.187 INFO:teuthology.orchestra.run.smithi115.stdout: librgw2 x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 5.1 M 2024-03-19T01:20:28.187 INFO:teuthology.orchestra.run.smithi115.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-03-19T01:20:28.187 INFO:teuthology.orchestra.run.smithi115.stdout: libzstd-devel x86_64 1.5.1-2.el9 CentOS-AppStream 46 k 2024-03-19T01:20:28.187 INFO:teuthology.orchestra.run.smithi115.stdout: lttng-ust x86_64 2.12.0-6.el9 CentOS-AppStream 282 k 2024-03-19T01:20:28.187 INFO:teuthology.orchestra.run.smithi115.stdout: lua x86_64 5.4.4-4.el9 CentOS-AppStream 187 k 2024-03-19T01:20:28.187 INFO:teuthology.orchestra.run.smithi115.stdout: lua-devel x86_64 5.4.4-4.el9 crb 22 k 2024-03-19T01:20:28.187 INFO:teuthology.orchestra.run.smithi115.stdout: luarocks noarch 3.9.2-1.el9 epel 151 k 2024-03-19T01:20:28.187 INFO:teuthology.orchestra.run.smithi115.stdout: oniguruma x86_64 6.9.6-1.el9.5 CentOS-AppStream 217 k 2024-03-19T01:20:28.188 INFO:teuthology.orchestra.run.smithi115.stdout: openblas x86_64 0.3.21-2.el9 CentOS-AppStream 34 k 2024-03-19T01:20:28.188 INFO:teuthology.orchestra.run.smithi115.stdout: openblas-openmp x86_64 0.3.21-2.el9 CentOS-AppStream 4.7 M 2024-03-19T01:20:28.188 INFO:teuthology.orchestra.run.smithi115.stdout: parquet-libs x86_64 9.0.0-10.el9 epel 838 k 2024-03-19T01:20:28.188 INFO:teuthology.orchestra.run.smithi115.stdout: postgresql-private-libs x86_64 13.11-1.el9 CentOS-AppStream 134 k 2024-03-19T01:20:28.188 INFO:teuthology.orchestra.run.smithi115.stdout: protobuf x86_64 3.14.0-13.el9 CentOS-AppStream 1.0 M 2024-03-19T01:20:28.188 INFO:teuthology.orchestra.run.smithi115.stdout: protobuf-compiler x86_64 3.14.0-13.el9 crb 863 k 2024-03-19T01:20:28.188 INFO:teuthology.orchestra.run.smithi115.stdout: python3-asyncssh noarch 2.13.2-4.el9 epel 548 k 2024-03-19T01:20:28.188 INFO:teuthology.orchestra.run.smithi115.stdout: python3-attrs noarch 20.3.0-7.el9 CentOS-AppStream 78 k 2024-03-19T01:20:28.188 INFO:teuthology.orchestra.run.smithi115.stdout: python3-babel noarch 2.9.1-2.el9 CentOS-AppStream 5.8 M 2024-03-19T01:20:28.188 INFO:teuthology.orchestra.run.smithi115.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-03-19T01:20:28.188 INFO:teuthology.orchestra.run.smithi115.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-03-19T01:20:28.188 INFO:teuthology.orchestra.run.smithi115.stdout: python3-ceph-argparse x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 45 k 2024-03-19T01:20:28.188 INFO:teuthology.orchestra.run.smithi115.stdout: python3-ceph-common x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 131 k 2024-03-19T01:20:28.189 INFO:teuthology.orchestra.run.smithi115.stdout: python3-certifi noarch 2021.10.8-2.el9 epel 15 k 2024-03-19T01:20:28.189 INFO:teuthology.orchestra.run.smithi115.stdout: python3-cffi x86_64 1.14.5-5.el9 CentOS-BaseOS 241 k 2024-03-19T01:20:28.189 INFO:teuthology.orchestra.run.smithi115.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-03-19T01:20:28.189 INFO:teuthology.orchestra.run.smithi115.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-03-19T01:20:28.189 INFO:teuthology.orchestra.run.smithi115.stdout: python3-cmd2 noarch 2.4.2-2.el9 epel 303 k 2024-03-19T01:20:28.189 INFO:teuthology.orchestra.run.smithi115.stdout: python3-colorama noarch 0.4.4-12.el9 epel 35 k 2024-03-19T01:20:28.189 INFO:teuthology.orchestra.run.smithi115.stdout: python3-cryptography x86_64 36.0.1-4.el9 CentOS-BaseOS 1.1 M 2024-03-19T01:20:28.189 INFO:teuthology.orchestra.run.smithi115.stdout: python3-devel x86_64 3.9.17-2.el9 appstream 245 k 2024-03-19T01:20:28.189 INFO:teuthology.orchestra.run.smithi115.stdout: python3-google-auth noarch 1:2.28.0-1.el9 epel 203 k 2024-03-19T01:20:28.189 INFO:teuthology.orchestra.run.smithi115.stdout: python3-grpcio x86_64 1.46.7-10.el9 epel 2.0 M 2024-03-19T01:20:28.189 INFO:teuthology.orchestra.run.smithi115.stdout: python3-grpcio-tools x86_64 1.46.7-10.el9 epel 144 k 2024-03-19T01:20:28.189 INFO:teuthology.orchestra.run.smithi115.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-03-19T01:20:28.190 INFO:teuthology.orchestra.run.smithi115.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-03-19T01:20:28.190 INFO:teuthology.orchestra.run.smithi115.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-03-19T01:20:28.190 INFO:teuthology.orchestra.run.smithi115.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-03-19T01:20:28.190 INFO:teuthology.orchestra.run.smithi115.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-03-19T01:20:28.190 INFO:teuthology.orchestra.run.smithi115.stdout: python3-jinja2 noarch 2.11.3-5.el9 appstream 248 k 2024-03-19T01:20:28.190 INFO:teuthology.orchestra.run.smithi115.stdout: python3-jsonpatch noarch 1.21-16.el9 CentOS-AppStream 24 k 2024-03-19T01:20:28.190 INFO:teuthology.orchestra.run.smithi115.stdout: python3-jsonpointer noarch 2.0-4.el9 CentOS-AppStream 18 k 2024-03-19T01:20:28.190 INFO:teuthology.orchestra.run.smithi115.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-03-19T01:20:28.190 INFO:teuthology.orchestra.run.smithi115.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-03-19T01:20:28.190 INFO:teuthology.orchestra.run.smithi115.stdout: python3-mako noarch 1.1.4-6.el9 CentOS-AppStream 144 k 2024-03-19T01:20:28.190 INFO:teuthology.orchestra.run.smithi115.stdout: python3-markupsafe x86_64 1.1.1-12.el9 CentOS-AppStream 32 k 2024-03-19T01:20:28.190 INFO:teuthology.orchestra.run.smithi115.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-03-19T01:20:28.190 INFO:teuthology.orchestra.run.smithi115.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-03-19T01:20:28.190 INFO:teuthology.orchestra.run.smithi115.stdout: python3-numpy x86_64 1:1.20.1-5.el9 CentOS-AppStream 4.8 M 2024-03-19T01:20:28.191 INFO:teuthology.orchestra.run.smithi115.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 CentOS-AppStream 217 k 2024-03-19T01:20:28.191 INFO:teuthology.orchestra.run.smithi115.stdout: python3-oauthlib noarch 3.1.1-5.el9 CentOS-AppStream 174 k 2024-03-19T01:20:28.191 INFO:teuthology.orchestra.run.smithi115.stdout: python3-packaging noarch 20.9-5.el9 CentOS-AppStream 69 k 2024-03-19T01:20:28.191 INFO:teuthology.orchestra.run.smithi115.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-03-19T01:20:28.191 INFO:teuthology.orchestra.run.smithi115.stdout: python3-ply noarch 3.11-14.el9 CentOS-BaseOS 103 k 2024-03-19T01:20:28.191 INFO:teuthology.orchestra.run.smithi115.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-03-19T01:20:28.191 INFO:teuthology.orchestra.run.smithi115.stdout: python3-prettytable noarch 0.7.2-27.el9 CentOS-AppStream 41 k 2024-03-19T01:20:28.191 INFO:teuthology.orchestra.run.smithi115.stdout: python3-protobuf noarch 3.14.0-13.el9 CentOS-AppStream 238 k 2024-03-19T01:20:28.191 INFO:teuthology.orchestra.run.smithi115.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-03-19T01:20:28.191 INFO:teuthology.orchestra.run.smithi115.stdout: python3-pyasn1 noarch 0.4.8-6.el9 CentOS-AppStream 132 k 2024-03-19T01:20:28.191 INFO:teuthology.orchestra.run.smithi115.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 CentOS-AppStream 211 k 2024-03-19T01:20:28.191 INFO:teuthology.orchestra.run.smithi115.stdout: python3-pycparser noarch 2.20-6.el9 CentOS-BaseOS 124 k 2024-03-19T01:20:28.191 INFO:teuthology.orchestra.run.smithi115.stdout: python3-pyperclip noarch 1.8.2-2.el9 epel 28 k 2024-03-19T01:20:28.191 INFO:teuthology.orchestra.run.smithi115.stdout: python3-pytz noarch 2021.1-5.el9 CentOS-AppStream 47 k 2024-03-19T01:20:28.192 INFO:teuthology.orchestra.run.smithi115.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-03-19T01:20:28.192 INFO:teuthology.orchestra.run.smithi115.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-03-19T01:20:28.192 INFO:teuthology.orchestra.run.smithi115.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 CentOS-AppStream 43 k 2024-03-19T01:20:28.192 INFO:teuthology.orchestra.run.smithi115.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-03-19T01:20:28.192 INFO:teuthology.orchestra.run.smithi115.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-03-19T01:20:28.192 INFO:teuthology.orchestra.run.smithi115.stdout: python3-scipy x86_64 1.6.2-8.el9 CentOS-AppStream 16 M 2024-03-19T01:20:28.192 INFO:teuthology.orchestra.run.smithi115.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-03-19T01:20:28.192 INFO:teuthology.orchestra.run.smithi115.stdout: python3-toml noarch 0.10.2-6.el9 CentOS-AppStream 39 k 2024-03-19T01:20:28.192 INFO:teuthology.orchestra.run.smithi115.stdout: python3-typing-extensions noarch 4.4.0-2.el9 epel 51 k 2024-03-19T01:20:28.192 INFO:teuthology.orchestra.run.smithi115.stdout: python3-urllib3 noarch 1.26.5-5.el9 CentOS-BaseOS 187 k 2024-03-19T01:20:28.192 INFO:teuthology.orchestra.run.smithi115.stdout: python3-wcwidth noarch 0.2.5-8.el9 CentOS-AppStream 41 k 2024-03-19T01:20:28.192 INFO:teuthology.orchestra.run.smithi115.stdout: python3-webob noarch 1.8.7-6.el9 epel 230 k 2024-03-19T01:20:28.192 INFO:teuthology.orchestra.run.smithi115.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-03-19T01:20:28.192 INFO:teuthology.orchestra.run.smithi115.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-03-19T01:20:28.192 INFO:teuthology.orchestra.run.smithi115.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-03-19T01:20:28.193 INFO:teuthology.orchestra.run.smithi115.stdout: qatlib x86_64 23.11.0-1.el9 appstream 216 k 2024-03-19T01:20:28.193 INFO:teuthology.orchestra.run.smithi115.stdout: qatzip-libs x86_64 1.1.2-2.el9 appstream 44 k 2024-03-19T01:20:28.193 INFO:teuthology.orchestra.run.smithi115.stdout: re2 x86_64 1:20211101-3.el9 epel 192 k 2024-03-19T01:20:28.193 INFO:teuthology.orchestra.run.smithi115.stdout: socat x86_64 1.7.4.1-5.el9 CentOS-AppStream 300 k 2024-03-19T01:20:28.193 INFO:teuthology.orchestra.run.smithi115.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-03-19T01:20:28.193 INFO:teuthology.orchestra.run.smithi115.stdout: uuid x86_64 1.6.2-55.el9 CentOS-AppStream 56 k 2024-03-19T01:20:28.193 INFO:teuthology.orchestra.run.smithi115.stdout: xmlstarlet x86_64 1.6.1-20.el9 CentOS-AppStream 63 k 2024-03-19T01:20:28.193 INFO:teuthology.orchestra.run.smithi115.stdout: zlib-devel x86_64 1.2.11-40.el9 CentOS-AppStream 44 k 2024-03-19T01:20:28.193 INFO:teuthology.orchestra.run.smithi115.stdout:Installing weak dependencies: 2024-03-19T01:20:28.193 INFO:teuthology.orchestra.run.smithi115.stdout: qatlib-service x86_64 23.11.0-1.el9 appstream 35 k 2024-03-19T01:20:28.193 INFO:teuthology.orchestra.run.smithi115.stdout: 2024-03-19T01:20:28.193 INFO:teuthology.orchestra.run.smithi115.stdout:Transaction Summary 2024-03-19T01:20:28.193 INFO:teuthology.orchestra.run.smithi115.stdout:============================================================================================ 2024-03-19T01:20:28.193 INFO:teuthology.orchestra.run.smithi115.stdout:Install 150 Packages 2024-03-19T01:20:28.194 INFO:teuthology.orchestra.run.smithi115.stdout:Upgrade 24 Packages 2024-03-19T01:20:28.194 INFO:teuthology.orchestra.run.smithi115.stdout: 2024-03-19T01:20:28.194 INFO:teuthology.orchestra.run.smithi115.stdout:Total download size: 224 M 2024-03-19T01:20:28.194 INFO:teuthology.orchestra.run.smithi115.stdout:Downloading Packages: 2024-03-19T01:20:28.991 INFO:teuthology.orchestra.run.smithi115.stdout:(1/174): ceph-19.0.0-2206.gb6ae2491.el9.x86_64. 38 kB/s | 6.6 kB 00:00 2024-03-19T01:20:29.292 INFO:teuthology.orchestra.run.smithi115.stdout:(2/174): ceph-fuse-19.0.0-2206.gb6ae2491.el9.x8 2.9 MB/s | 891 kB 00:00 2024-03-19T01:20:29.375 INFO:teuthology.orchestra.run.smithi115.stdout:(3/174): ceph-base-19.0.0-2206.gb6ae2491.el9.x8 9.6 MB/s | 5.4 MB 00:00 2024-03-19T01:20:29.400 INFO:teuthology.orchestra.run.smithi115.stdout:(4/174): ceph-immutable-object-cache-19.0.0-220 1.3 MB/s | 145 kB 00:00 2024-03-19T01:20:29.526 INFO:teuthology.orchestra.run.smithi115.stdout:(5/174): ceph-mds-19.0.0-2206.gb6ae2491.el9.x86 16 MB/s | 2.4 MB 00:00 2024-03-19T01:20:29.701 INFO:teuthology.orchestra.run.smithi115.stdout:(6/174): ceph-mgr-19.0.0-2206.gb6ae2491.el9.x86 5.0 MB/s | 1.5 MB 00:00 2024-03-19T01:20:29.818 INFO:teuthology.orchestra.run.smithi115.stdout:(7/174): ceph-mon-19.0.0-2206.gb6ae2491.el9.x86 16 MB/s | 4.7 MB 00:00 2024-03-19T01:20:30.451 INFO:teuthology.orchestra.run.smithi115.stdout:(8/174): ceph-radosgw-19.0.0-2206.gb6ae2491.el9 16 MB/s | 9.8 MB 00:00 2024-03-19T01:20:30.509 INFO:teuthology.orchestra.run.smithi115.stdout:(9/174): ceph-selinux-19.0.0-2206.gb6ae2491.el9 432 kB/s | 25 kB 00:00 2024-03-19T01:20:32.385 INFO:teuthology.orchestra.run.smithi112.stdout:lab-extras 14 kB/s | 1.7 kB 00:00 2024-03-19T01:20:32.734 INFO:teuthology.orchestra.run.smithi115.stdout:(10/174): ceph-osd-19.0.0-2206.gb6ae2491.el9.x8 5.6 MB/s | 17 MB 00:03 2024-03-19T01:20:32.926 INFO:teuthology.orchestra.run.smithi115.stdout:(11/174): ceph-common-19.0.0-2206.gb6ae2491.el9 5.0 MB/s | 21 MB 00:04 2024-03-19T01:20:32.953 INFO:teuthology.orchestra.run.smithi115.stdout:(12/174): cephfs-mirror-19.0.0-2206.gb6ae2491.e 1.0 MB/s | 221 kB 00:00 2024-03-19T01:20:32.985 INFO:teuthology.orchestra.run.smithi115.stdout:(13/174): cephfs-shell-19.0.0-2206.gb6ae2491.el 463 kB/s | 27 kB 00:00 2024-03-19T01:20:33.010 INFO:teuthology.orchestra.run.smithi115.stdout:(14/174): libcephfs-devel-19.0.0-2206.gb6ae2491 540 kB/s | 32 kB 00:00 2024-03-19T01:20:33.085 INFO:teuthology.orchestra.run.smithi115.stdout:(15/174): libcephsqlite-19.0.0-2206.gb6ae2491.e 2.1 MB/s | 163 kB 00:00 2024-03-19T01:20:33.127 INFO:teuthology.orchestra.run.smithi115.stdout:(16/174): libcephfs2-19.0.0-2206.gb6ae2491.el9. 5.1 MB/s | 746 kB 00:00 2024-03-19T01:20:33.153 INFO:teuthology.orchestra.run.smithi115.stdout:(17/174): librados-devel-19.0.0-2206.gb6ae2491. 1.8 MB/s | 127 kB 00:00 2024-03-19T01:20:33.220 INFO:teuthology.orchestra.run.smithi115.stdout:(18/174): libradosstriper1-19.0.0-2206.gb6ae249 5.3 MB/s | 501 kB 00:00 2024-03-19T01:20:33.303 INFO:teuthology.orchestra.run.smithi115.stdout:(19/174): python3-ceph-argparse-19.0.0-2206.gb6 542 kB/s | 45 kB 00:00 2024-03-19T01:20:33.395 INFO:teuthology.orchestra.run.smithi115.stdout:(20/174): python3-ceph-common-19.0.0-2206.gb6ae 1.4 MB/s | 131 kB 00:00 2024-03-19T01:20:33.487 INFO:teuthology.orchestra.run.smithi115.stdout:(21/174): python3-cephfs-19.0.0-2206.gb6ae2491. 1.7 MB/s | 162 kB 00:00 2024-03-19T01:20:33.913 INFO:teuthology.orchestra.run.smithi115.stdout:(22/174): ceph-test-19.0.0-2206.gb6ae2491.el9.x 14 MB/s | 48 MB 00:03 2024-03-19T01:20:33.938 INFO:teuthology.orchestra.run.smithi115.stdout:(23/174): python3-rados-19.0.0-2206.gb6ae2491.e 714 kB/s | 321 kB 00:00 2024-03-19T01:20:33.971 INFO:teuthology.orchestra.run.smithi115.stdout:(24/174): python3-rbd-19.0.0-2206.gb6ae2491.el9 5.0 MB/s | 298 kB 00:00 2024-03-19T01:20:34.005 INFO:teuthology.orchestra.run.smithi115.stdout:(25/174): python3-rgw-19.0.0-2206.gb6ae2491.el9 1.5 MB/s | 99 kB 00:00 2024-03-19T01:20:34.022 INFO:teuthology.orchestra.run.smithi115.stdout:(26/174): rbd-fuse-19.0.0-2206.gb6ae2491.el9.x8 1.7 MB/s | 85 kB 00:00 2024-03-19T01:20:34.089 INFO:teuthology.orchestra.run.smithi115.stdout:(27/174): librgw2-19.0.0-2206.gb6ae2491.el9.x86 5.5 MB/s | 5.1 MB 00:00 2024-03-19T01:20:34.114 INFO:teuthology.orchestra.run.smithi115.stdout:(28/174): rbd-nbd-19.0.0-2206.gb6ae2491.el9.x86 1.8 MB/s | 170 kB 00:00 2024-03-19T01:20:34.131 INFO:teuthology.orchestra.run.smithi115.stdout:(29/174): ceph-grafana-dashboards-19.0.0-2206.g 813 kB/s | 34 kB 00:00 2024-03-19T01:20:34.165 INFO:teuthology.orchestra.run.smithi115.stdout:(30/174): ceph-mgr-cephadm-19.0.0-2206.gb6ae249 2.7 MB/s | 139 kB 00:00 2024-03-19T01:20:34.415 INFO:teuthology.orchestra.run.smithi115.stdout:(31/174): rbd-mirror-19.0.0-2206.gb6ae2491.el9. 7.5 MB/s | 3.1 MB 00:00 2024-03-19T01:20:34.474 INFO:teuthology.orchestra.run.smithi115.stdout:(32/174): ceph-mgr-dashboard-19.0.0-2206.gb6ae2 5.2 MB/s | 1.8 MB 00:00 2024-03-19T01:20:34.515 INFO:teuthology.orchestra.run.smithi115.stdout:(33/174): ceph-mgr-modules-core-19.0.0-2206.gb6 2.4 MB/s | 248 kB 00:00 2024-03-19T01:20:34.540 INFO:teuthology.orchestra.run.smithi115.stdout:(34/174): ceph-mgr-rook-19.0.0-2206.gb6ae2491.e 740 kB/s | 49 kB 00:00 2024-03-19T01:20:34.632 INFO:teuthology.orchestra.run.smithi115.stdout:(35/174): ceph-mgr-diskprediction-local-19.0.0- 16 MB/s | 7.4 MB 00:00 2024-03-19T01:20:34.658 INFO:teuthology.orchestra.run.smithi115.stdout:(36/174): ceph-prometheus-alerts-19.0.0-2206.gb 116 kB/s | 16 kB 00:00 2024-03-19T01:20:34.683 INFO:teuthology.orchestra.run.smithi115.stdout:(37/174): ceph-volume-19.0.0-2206.gb6ae2491.el9 1.8 MB/s | 264 kB 00:00 2024-03-19T01:20:34.717 INFO:teuthology.orchestra.run.smithi115.stdout:(38/174): cephadm-19.0.0-2206.gb6ae2491.el9.noa 8.7 MB/s | 749 kB 00:00 2024-03-19T01:20:34.738 INFO:teuthology.orchestra.run.smithi112.stdout:Package librados2-2:16.2.4-5.el9.x86_64 is already installed. 2024-03-19T01:20:34.738 INFO:teuthology.orchestra.run.smithi112.stdout:Package librbd1-2:16.2.4-5.el9.x86_64 is already installed. 2024-03-19T01:20:34.750 INFO:teuthology.orchestra.run.smithi112.stdout:Package NetworkManager-1:1.44.0-2.el9.x86_64 is already installed. 2024-03-19T01:20:34.751 INFO:teuthology.orchestra.run.smithi112.stdout:Package iproute-6.2.0-5.el9.x86_64 is already installed. 2024-03-19T01:20:34.752 INFO:teuthology.orchestra.run.smithi112.stdout:Package util-linux-2.37.4-15.el9.x86_64 is already installed. 2024-03-19T01:20:34.753 INFO:teuthology.orchestra.run.smithi112.stdout:Package libacl-devel-2.3.1-3.el9.x86_64 is already installed. 2024-03-19T01:20:34.754 INFO:teuthology.orchestra.run.smithi112.stdout:Package libattr-devel-2.5.1-3.el9.x86_64 is already installed. 2024-03-19T01:20:34.757 INFO:teuthology.orchestra.run.smithi112.stdout:Package xfsdump-3.1.12-3.el9.x86_64 is already installed. 2024-03-19T01:20:34.758 INFO:teuthology.orchestra.run.smithi112.stdout:Package xfsprogs-5.19.0-4.el9.x86_64 is already installed. 2024-03-19T01:20:34.767 INFO:teuthology.orchestra.run.smithi115.stdout:(39/174): cephfs-top-19.0.0-2206.gb6ae2491.el9. 168 kB/s | 18 kB 00:00 2024-03-19T01:20:34.838 INFO:teuthology.orchestra.run.smithi112.stdout:Dependencies resolved. 2024-03-19T01:20:34.863 INFO:teuthology.orchestra.run.smithi112.stdout:============================================================================================ 2024-03-19T01:20:34.863 INFO:teuthology.orchestra.run.smithi112.stdout: Package Arch Version Repository Size 2024-03-19T01:20:34.863 INFO:teuthology.orchestra.run.smithi112.stdout:============================================================================================ 2024-03-19T01:20:34.864 INFO:teuthology.orchestra.run.smithi112.stdout:Installing: 2024-03-19T01:20:34.864 INFO:teuthology.orchestra.run.smithi112.stdout: bison x86_64 3.7.4-5.el9 CentOS-AppStream 920 k 2024-03-19T01:20:34.864 INFO:teuthology.orchestra.run.smithi112.stdout: ceph x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 6.6 k 2024-03-19T01:20:34.864 INFO:teuthology.orchestra.run.smithi112.stdout: ceph-base x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 5.4 M 2024-03-19T01:20:34.864 INFO:teuthology.orchestra.run.smithi112.stdout: ceph-fuse x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 891 k 2024-03-19T01:20:34.864 INFO:teuthology.orchestra.run.smithi112.stdout: ceph-immutable-object-cache x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 145 k 2024-03-19T01:20:34.865 INFO:teuthology.orchestra.run.smithi112.stdout: ceph-mgr x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 1.5 M 2024-03-19T01:20:34.865 INFO:teuthology.orchestra.run.smithi112.stdout: ceph-mgr-cephadm noarch 2:19.0.0-2206.gb6ae2491.el9 ceph-noarch 139 k 2024-03-19T01:20:34.865 INFO:teuthology.orchestra.run.smithi112.stdout: ceph-mgr-dashboard noarch 2:19.0.0-2206.gb6ae2491.el9 ceph-noarch 1.8 M 2024-03-19T01:20:34.865 INFO:teuthology.orchestra.run.smithi112.stdout: ceph-mgr-diskprediction-local noarch 2:19.0.0-2206.gb6ae2491.el9 ceph-noarch 7.4 M 2024-03-19T01:20:34.865 INFO:teuthology.orchestra.run.smithi112.stdout: ceph-mgr-rook noarch 2:19.0.0-2206.gb6ae2491.el9 ceph-noarch 49 k 2024-03-19T01:20:34.865 INFO:teuthology.orchestra.run.smithi112.stdout: ceph-radosgw x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 9.8 M 2024-03-19T01:20:34.866 INFO:teuthology.orchestra.run.smithi112.stdout: ceph-test x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 48 M 2024-03-19T01:20:34.866 INFO:teuthology.orchestra.run.smithi112.stdout: ceph-volume noarch 2:19.0.0-2206.gb6ae2491.el9 ceph-noarch 264 k 2024-03-19T01:20:34.866 INFO:teuthology.orchestra.run.smithi112.stdout: cephadm noarch 2:19.0.0-2206.gb6ae2491.el9 ceph-noarch 749 k 2024-03-19T01:20:34.866 INFO:teuthology.orchestra.run.smithi112.stdout: cephfs-mirror x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 221 k 2024-03-19T01:20:34.866 INFO:teuthology.orchestra.run.smithi112.stdout: cephfs-shell x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 27 k 2024-03-19T01:20:34.866 INFO:teuthology.orchestra.run.smithi112.stdout: cephfs-top noarch 2:19.0.0-2206.gb6ae2491.el9 ceph-noarch 18 k 2024-03-19T01:20:34.867 INFO:teuthology.orchestra.run.smithi112.stdout: elfutils-libelf-devel x86_64 0.190-2.el9 appstream 21 k 2024-03-19T01:20:34.867 INFO:teuthology.orchestra.run.smithi112.stdout: flex x86_64 2.6.4-9.el9 CentOS-AppStream 307 k 2024-03-19T01:20:34.867 INFO:teuthology.orchestra.run.smithi112.stdout: libaio-devel x86_64 0.3.111-13.el9 CentOS-AppStream 10 k 2024-03-19T01:20:34.867 INFO:teuthology.orchestra.run.smithi112.stdout: libcephfs-devel x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 32 k 2024-03-19T01:20:34.867 INFO:teuthology.orchestra.run.smithi112.stdout: libcephfs2 x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 746 k 2024-03-19T01:20:34.867 INFO:teuthology.orchestra.run.smithi112.stdout: librados-devel x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 127 k 2024-03-19T01:20:34.868 INFO:teuthology.orchestra.run.smithi112.stdout: libtool x86_64 2.4.6-45.el9 CentOS-AppStream 578 k 2024-03-19T01:20:34.868 INFO:teuthology.orchestra.run.smithi112.stdout: libuuid-devel x86_64 2.37.4-18.el9 appstream 25 k 2024-03-19T01:20:34.868 INFO:teuthology.orchestra.run.smithi112.stdout: openssl-devel x86_64 1:3.0.7-27.el9 appstream 4.1 M 2024-03-19T01:20:34.868 INFO:teuthology.orchestra.run.smithi112.stdout: postgresql x86_64 13.11-1.el9 CentOS-AppStream 1.5 M 2024-03-19T01:20:34.868 INFO:teuthology.orchestra.run.smithi112.stdout: postgresql-contrib x86_64 13.11-1.el9 CentOS-AppStream 811 k 2024-03-19T01:20:34.868 INFO:teuthology.orchestra.run.smithi112.stdout: postgresql-server x86_64 13.11-1.el9 CentOS-AppStream 5.7 M 2024-03-19T01:20:34.869 INFO:teuthology.orchestra.run.smithi112.stdout: python3-cephfs x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 162 k 2024-03-19T01:20:34.869 INFO:teuthology.orchestra.run.smithi112.stdout: python3-rados x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 321 k 2024-03-19T01:20:34.869 INFO:teuthology.orchestra.run.smithi112.stdout: python3-rbd x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 298 k 2024-03-19T01:20:34.869 INFO:teuthology.orchestra.run.smithi112.stdout: python3-rgw x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 99 k 2024-03-19T01:20:34.869 INFO:teuthology.orchestra.run.smithi112.stdout: rbd-fuse x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 85 k 2024-03-19T01:20:34.870 INFO:teuthology.orchestra.run.smithi112.stdout: rbd-mirror x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 3.1 M 2024-03-19T01:20:34.870 INFO:teuthology.orchestra.run.smithi112.stdout: rbd-nbd x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 170 k 2024-03-19T01:20:34.870 INFO:teuthology.orchestra.run.smithi112.stdout: xfsprogs-devel x86_64 6.3.0-1.el9 appstream 94 k 2024-03-19T01:20:34.870 INFO:teuthology.orchestra.run.smithi112.stdout:Upgrading: 2024-03-19T01:20:34.871 INFO:teuthology.orchestra.run.smithi112.stdout: NetworkManager x86_64 1:1.47.2-1.el9 baseos 2.3 M 2024-03-19T01:20:34.871 INFO:teuthology.orchestra.run.smithi112.stdout: NetworkManager-libnm x86_64 1:1.47.2-1.el9 baseos 1.8 M 2024-03-19T01:20:34.871 INFO:teuthology.orchestra.run.smithi112.stdout: NetworkManager-team x86_64 1:1.47.2-1.el9 baseos 42 k 2024-03-19T01:20:34.871 INFO:teuthology.orchestra.run.smithi112.stdout: NetworkManager-tui x86_64 1:1.47.2-1.el9 baseos 248 k 2024-03-19T01:20:34.871 INFO:teuthology.orchestra.run.smithi112.stdout: acl x86_64 2.3.1-4.el9 baseos 71 k 2024-03-19T01:20:34.872 INFO:teuthology.orchestra.run.smithi112.stdout: elfutils-debuginfod-client x86_64 0.190-2.el9 baseos 36 k 2024-03-19T01:20:34.872 INFO:teuthology.orchestra.run.smithi112.stdout: elfutils-libelf x86_64 0.190-2.el9 baseos 192 k 2024-03-19T01:20:34.872 INFO:teuthology.orchestra.run.smithi112.stdout: elfutils-libs x86_64 0.190-2.el9 baseos 256 k 2024-03-19T01:20:34.872 INFO:teuthology.orchestra.run.smithi112.stdout: libacl x86_64 2.3.1-4.el9 baseos 23 k 2024-03-19T01:20:34.873 INFO:teuthology.orchestra.run.smithi112.stdout: libacl-devel x86_64 2.3.1-4.el9 appstream 74 k 2024-03-19T01:20:34.873 INFO:teuthology.orchestra.run.smithi112.stdout: libblkid x86_64 2.37.4-18.el9 baseos 108 k 2024-03-19T01:20:34.873 INFO:teuthology.orchestra.run.smithi112.stdout: libfdisk x86_64 2.37.4-18.el9 baseos 155 k 2024-03-19T01:20:34.873 INFO:teuthology.orchestra.run.smithi112.stdout: libmount x86_64 2.37.4-18.el9 baseos 136 k 2024-03-19T01:20:34.873 INFO:teuthology.orchestra.run.smithi112.stdout: librados2 x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 3.4 M 2024-03-19T01:20:34.874 INFO:teuthology.orchestra.run.smithi112.stdout: librbd1 x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 3.1 M 2024-03-19T01:20:34.874 INFO:teuthology.orchestra.run.smithi112.stdout: libsmartcols x86_64 2.37.4-18.el9 baseos 64 k 2024-03-19T01:20:34.874 INFO:teuthology.orchestra.run.smithi112.stdout: libuuid x86_64 2.37.4-18.el9 baseos 29 k 2024-03-19T01:20:34.874 INFO:teuthology.orchestra.run.smithi112.stdout: openssl x86_64 1:3.0.7-27.el9 baseos 1.2 M 2024-03-19T01:20:34.874 INFO:teuthology.orchestra.run.smithi112.stdout: openssl-libs x86_64 1:3.0.7-27.el9 baseos 2.2 M 2024-03-19T01:20:34.875 INFO:teuthology.orchestra.run.smithi112.stdout: util-linux x86_64 2.37.4-18.el9 baseos 2.3 M 2024-03-19T01:20:34.875 INFO:teuthology.orchestra.run.smithi112.stdout: util-linux-core x86_64 2.37.4-18.el9 baseos 465 k 2024-03-19T01:20:34.875 INFO:teuthology.orchestra.run.smithi112.stdout: util-linux-user x86_64 2.37.4-18.el9 baseos 31 k 2024-03-19T01:20:34.875 INFO:teuthology.orchestra.run.smithi112.stdout: xfsdump x86_64 3.1.12-4.el9 CentOS-BaseOS 321 k 2024-03-19T01:20:34.876 INFO:teuthology.orchestra.run.smithi112.stdout: xfsprogs x86_64 6.3.0-1.el9 baseos 1.1 M 2024-03-19T01:20:34.876 INFO:teuthology.orchestra.run.smithi112.stdout:Installing dependencies: 2024-03-19T01:20:34.876 INFO:teuthology.orchestra.run.smithi112.stdout: abseil-cpp x86_64 20211102.0-3.el9 epel 548 k 2024-03-19T01:20:34.876 INFO:teuthology.orchestra.run.smithi112.stdout: automake noarch 1.16.2-8.el9 CentOS-AppStream 662 k 2024-03-19T01:20:34.876 INFO:teuthology.orchestra.run.smithi112.stdout: boost-program-options x86_64 1.75.0-8.el9 CentOS-AppStream 106 k 2024-03-19T01:20:34.877 INFO:teuthology.orchestra.run.smithi112.stdout: ceph-common x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 21 M 2024-03-19T01:20:34.877 INFO:teuthology.orchestra.run.smithi112.stdout: ceph-grafana-dashboards noarch 2:19.0.0-2206.gb6ae2491.el9 ceph-noarch 34 k 2024-03-19T01:20:34.877 INFO:teuthology.orchestra.run.smithi112.stdout: ceph-mds x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 2.4 M 2024-03-19T01:20:34.877 INFO:teuthology.orchestra.run.smithi112.stdout: ceph-mgr-modules-core noarch 2:19.0.0-2206.gb6ae2491.el9 ceph-noarch 248 k 2024-03-19T01:20:34.878 INFO:teuthology.orchestra.run.smithi112.stdout: ceph-mon x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 4.7 M 2024-03-19T01:20:34.878 INFO:teuthology.orchestra.run.smithi112.stdout: ceph-osd x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 17 M 2024-03-19T01:20:34.878 INFO:teuthology.orchestra.run.smithi112.stdout: ceph-prometheus-alerts noarch 2:19.0.0-2206.gb6ae2491.el9 ceph-noarch 16 k 2024-03-19T01:20:34.878 INFO:teuthology.orchestra.run.smithi112.stdout: ceph-selinux x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 25 k 2024-03-19T01:20:34.878 INFO:teuthology.orchestra.run.smithi112.stdout: flexiblas x86_64 3.0.4-8.el9 CentOS-AppStream 32 k 2024-03-19T01:20:34.879 INFO:teuthology.orchestra.run.smithi112.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 CentOS-AppStream 3.0 M 2024-03-19T01:20:34.879 INFO:teuthology.orchestra.run.smithi112.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 CentOS-AppStream 17 k 2024-03-19T01:20:34.879 INFO:teuthology.orchestra.run.smithi112.stdout: fuse x86_64 2.9.9-15.el9 CentOS-BaseOS 79 k 2024-03-19T01:20:34.879 INFO:teuthology.orchestra.run.smithi112.stdout: gperftools-libs x86_64 2.9.1-2.el9 epel 309 k 2024-03-19T01:20:34.879 INFO:teuthology.orchestra.run.smithi112.stdout: grpc-data noarch 1.46.7-10.el9 epel 19 k 2024-03-19T01:20:34.880 INFO:teuthology.orchestra.run.smithi112.stdout: jq x86_64 1.6-16.el9 baseos 187 k 2024-03-19T01:20:34.880 INFO:teuthology.orchestra.run.smithi112.stdout: libarrow x86_64 9.0.0-10.el9 epel 4.4 M 2024-03-19T01:20:34.880 INFO:teuthology.orchestra.run.smithi112.stdout: libarrow-doc noarch 9.0.0-10.el9 epel 26 k 2024-03-19T01:20:34.880 INFO:teuthology.orchestra.run.smithi112.stdout: libcephsqlite x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 163 k 2024-03-19T01:20:34.880 INFO:teuthology.orchestra.run.smithi112.stdout: libgfortran x86_64 11.4.1-3.el9 baseos 803 k 2024-03-19T01:20:34.881 INFO:teuthology.orchestra.run.smithi112.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-03-19T01:20:34.881 INFO:teuthology.orchestra.run.smithi112.stdout: libpmemobj x86_64 1.12.1-1.el9 CentOS-AppStream 159 k 2024-03-19T01:20:34.881 INFO:teuthology.orchestra.run.smithi112.stdout: libquadmath x86_64 11.4.1-3.el9 baseos 195 k 2024-03-19T01:20:34.881 INFO:teuthology.orchestra.run.smithi112.stdout: librabbitmq x86_64 0.11.0-7.el9 CentOS-AppStream 44 k 2024-03-19T01:20:34.882 INFO:teuthology.orchestra.run.smithi112.stdout: libradosstriper1 x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 501 k 2024-03-19T01:20:34.882 INFO:teuthology.orchestra.run.smithi112.stdout: librdkafka x86_64 1.6.1-102.el9 CentOS-AppStream 661 k 2024-03-19T01:20:34.882 INFO:teuthology.orchestra.run.smithi112.stdout: librgw2 x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 5.1 M 2024-03-19T01:20:34.882 INFO:teuthology.orchestra.run.smithi112.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-03-19T01:20:34.882 INFO:teuthology.orchestra.run.smithi112.stdout: libzstd-devel x86_64 1.5.1-2.el9 CentOS-AppStream 46 k 2024-03-19T01:20:34.883 INFO:teuthology.orchestra.run.smithi112.stdout: lttng-ust x86_64 2.12.0-6.el9 CentOS-AppStream 282 k 2024-03-19T01:20:34.883 INFO:teuthology.orchestra.run.smithi112.stdout: lua x86_64 5.4.4-4.el9 CentOS-AppStream 187 k 2024-03-19T01:20:34.883 INFO:teuthology.orchestra.run.smithi112.stdout: lua-devel x86_64 5.4.4-4.el9 crb 22 k 2024-03-19T01:20:34.883 INFO:teuthology.orchestra.run.smithi112.stdout: luarocks noarch 3.9.2-1.el9 epel 151 k 2024-03-19T01:20:34.883 INFO:teuthology.orchestra.run.smithi112.stdout: oniguruma x86_64 6.9.6-1.el9.5 CentOS-AppStream 217 k 2024-03-19T01:20:34.884 INFO:teuthology.orchestra.run.smithi112.stdout: openblas x86_64 0.3.21-2.el9 CentOS-AppStream 34 k 2024-03-19T01:20:34.884 INFO:teuthology.orchestra.run.smithi112.stdout: openblas-openmp x86_64 0.3.21-2.el9 CentOS-AppStream 4.7 M 2024-03-19T01:20:34.884 INFO:teuthology.orchestra.run.smithi112.stdout: parquet-libs x86_64 9.0.0-10.el9 epel 838 k 2024-03-19T01:20:34.884 INFO:teuthology.orchestra.run.smithi112.stdout: postgresql-private-libs x86_64 13.11-1.el9 CentOS-AppStream 134 k 2024-03-19T01:20:34.884 INFO:teuthology.orchestra.run.smithi112.stdout: protobuf x86_64 3.14.0-13.el9 CentOS-AppStream 1.0 M 2024-03-19T01:20:34.885 INFO:teuthology.orchestra.run.smithi112.stdout: protobuf-compiler x86_64 3.14.0-13.el9 crb 863 k 2024-03-19T01:20:34.885 INFO:teuthology.orchestra.run.smithi112.stdout: python3-asyncssh noarch 2.13.2-4.el9 epel 548 k 2024-03-19T01:20:34.885 INFO:teuthology.orchestra.run.smithi112.stdout: python3-attrs noarch 20.3.0-7.el9 CentOS-AppStream 78 k 2024-03-19T01:20:34.885 INFO:teuthology.orchestra.run.smithi112.stdout: python3-babel noarch 2.9.1-2.el9 CentOS-AppStream 5.8 M 2024-03-19T01:20:34.885 INFO:teuthology.orchestra.run.smithi112.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-03-19T01:20:34.886 INFO:teuthology.orchestra.run.smithi112.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-03-19T01:20:34.886 INFO:teuthology.orchestra.run.smithi112.stdout: python3-ceph-argparse x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 45 k 2024-03-19T01:20:34.886 INFO:teuthology.orchestra.run.smithi112.stdout: python3-ceph-common x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 131 k 2024-03-19T01:20:34.886 INFO:teuthology.orchestra.run.smithi112.stdout: python3-certifi noarch 2021.10.8-2.el9 epel 15 k 2024-03-19T01:20:34.886 INFO:teuthology.orchestra.run.smithi112.stdout: python3-cffi x86_64 1.14.5-5.el9 CentOS-BaseOS 241 k 2024-03-19T01:20:34.887 INFO:teuthology.orchestra.run.smithi112.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-03-19T01:20:34.887 INFO:teuthology.orchestra.run.smithi112.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-03-19T01:20:34.887 INFO:teuthology.orchestra.run.smithi112.stdout: python3-cmd2 noarch 2.4.2-2.el9 epel 303 k 2024-03-19T01:20:34.887 INFO:teuthology.orchestra.run.smithi112.stdout: python3-colorama noarch 0.4.4-12.el9 epel 35 k 2024-03-19T01:20:34.887 INFO:teuthology.orchestra.run.smithi112.stdout: python3-cryptography x86_64 36.0.1-4.el9 CentOS-BaseOS 1.1 M 2024-03-19T01:20:34.888 INFO:teuthology.orchestra.run.smithi112.stdout: python3-devel x86_64 3.9.17-2.el9 appstream 245 k 2024-03-19T01:20:34.888 INFO:teuthology.orchestra.run.smithi112.stdout: python3-google-auth noarch 1:2.28.0-1.el9 epel 203 k 2024-03-19T01:20:34.888 INFO:teuthology.orchestra.run.smithi112.stdout: python3-grpcio x86_64 1.46.7-10.el9 epel 2.0 M 2024-03-19T01:20:34.888 INFO:teuthology.orchestra.run.smithi112.stdout: python3-grpcio-tools x86_64 1.46.7-10.el9 epel 144 k 2024-03-19T01:20:34.889 INFO:teuthology.orchestra.run.smithi112.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-03-19T01:20:34.889 INFO:teuthology.orchestra.run.smithi112.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-03-19T01:20:34.889 INFO:teuthology.orchestra.run.smithi112.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-03-19T01:20:34.889 INFO:teuthology.orchestra.run.smithi112.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-03-19T01:20:34.889 INFO:teuthology.orchestra.run.smithi112.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-03-19T01:20:34.890 INFO:teuthology.orchestra.run.smithi112.stdout: python3-jinja2 noarch 2.11.3-5.el9 appstream 248 k 2024-03-19T01:20:34.890 INFO:teuthology.orchestra.run.smithi112.stdout: python3-jsonpatch noarch 1.21-16.el9 CentOS-AppStream 24 k 2024-03-19T01:20:34.890 INFO:teuthology.orchestra.run.smithi112.stdout: python3-jsonpointer noarch 2.0-4.el9 CentOS-AppStream 18 k 2024-03-19T01:20:34.890 INFO:teuthology.orchestra.run.smithi112.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-03-19T01:20:34.890 INFO:teuthology.orchestra.run.smithi112.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-03-19T01:20:34.890 INFO:teuthology.orchestra.run.smithi112.stdout: python3-mako noarch 1.1.4-6.el9 CentOS-AppStream 144 k 2024-03-19T01:20:34.891 INFO:teuthology.orchestra.run.smithi112.stdout: python3-markupsafe x86_64 1.1.1-12.el9 CentOS-AppStream 32 k 2024-03-19T01:20:34.891 INFO:teuthology.orchestra.run.smithi112.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-03-19T01:20:34.891 INFO:teuthology.orchestra.run.smithi112.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-03-19T01:20:34.891 INFO:teuthology.orchestra.run.smithi112.stdout: python3-numpy x86_64 1:1.20.1-5.el9 CentOS-AppStream 4.8 M 2024-03-19T01:20:34.891 INFO:teuthology.orchestra.run.smithi112.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 CentOS-AppStream 217 k 2024-03-19T01:20:34.891 INFO:teuthology.orchestra.run.smithi112.stdout: python3-oauthlib noarch 3.1.1-5.el9 CentOS-AppStream 174 k 2024-03-19T01:20:34.891 INFO:teuthology.orchestra.run.smithi112.stdout: python3-packaging noarch 20.9-5.el9 CentOS-AppStream 69 k 2024-03-19T01:20:34.892 INFO:teuthology.orchestra.run.smithi112.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-03-19T01:20:34.892 INFO:teuthology.orchestra.run.smithi112.stdout: python3-ply noarch 3.11-14.el9 CentOS-BaseOS 103 k 2024-03-19T01:20:34.892 INFO:teuthology.orchestra.run.smithi112.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-03-19T01:20:34.892 INFO:teuthology.orchestra.run.smithi112.stdout: python3-prettytable noarch 0.7.2-27.el9 CentOS-AppStream 41 k 2024-03-19T01:20:34.892 INFO:teuthology.orchestra.run.smithi112.stdout: python3-protobuf noarch 3.14.0-13.el9 CentOS-AppStream 238 k 2024-03-19T01:20:34.892 INFO:teuthology.orchestra.run.smithi112.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-03-19T01:20:34.893 INFO:teuthology.orchestra.run.smithi112.stdout: python3-pyasn1 noarch 0.4.8-6.el9 CentOS-AppStream 132 k 2024-03-19T01:20:34.893 INFO:teuthology.orchestra.run.smithi112.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 CentOS-AppStream 211 k 2024-03-19T01:20:34.893 INFO:teuthology.orchestra.run.smithi112.stdout: python3-pycparser noarch 2.20-6.el9 CentOS-BaseOS 124 k 2024-03-19T01:20:34.893 INFO:teuthology.orchestra.run.smithi112.stdout: python3-pyperclip noarch 1.8.2-2.el9 epel 28 k 2024-03-19T01:20:34.893 INFO:teuthology.orchestra.run.smithi112.stdout: python3-pytz noarch 2021.1-5.el9 CentOS-AppStream 47 k 2024-03-19T01:20:34.893 INFO:teuthology.orchestra.run.smithi112.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-03-19T01:20:34.894 INFO:teuthology.orchestra.run.smithi112.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-03-19T01:20:34.894 INFO:teuthology.orchestra.run.smithi112.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 CentOS-AppStream 43 k 2024-03-19T01:20:34.894 INFO:teuthology.orchestra.run.smithi112.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-03-19T01:20:34.894 INFO:teuthology.orchestra.run.smithi112.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-03-19T01:20:34.894 INFO:teuthology.orchestra.run.smithi112.stdout: python3-scipy x86_64 1.6.2-8.el9 CentOS-AppStream 16 M 2024-03-19T01:20:34.894 INFO:teuthology.orchestra.run.smithi112.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-03-19T01:20:34.895 INFO:teuthology.orchestra.run.smithi112.stdout: python3-toml noarch 0.10.2-6.el9 CentOS-AppStream 39 k 2024-03-19T01:20:34.895 INFO:teuthology.orchestra.run.smithi112.stdout: python3-typing-extensions noarch 4.4.0-2.el9 epel 51 k 2024-03-19T01:20:34.895 INFO:teuthology.orchestra.run.smithi112.stdout: python3-urllib3 noarch 1.26.5-5.el9 CentOS-BaseOS 187 k 2024-03-19T01:20:34.895 INFO:teuthology.orchestra.run.smithi112.stdout: python3-wcwidth noarch 0.2.5-8.el9 CentOS-AppStream 41 k 2024-03-19T01:20:34.896 INFO:teuthology.orchestra.run.smithi112.stdout: python3-webob noarch 1.8.7-6.el9 epel 230 k 2024-03-19T01:20:34.896 INFO:teuthology.orchestra.run.smithi112.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-03-19T01:20:34.896 INFO:teuthology.orchestra.run.smithi112.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-03-19T01:20:34.896 INFO:teuthology.orchestra.run.smithi112.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-03-19T01:20:34.896 INFO:teuthology.orchestra.run.smithi112.stdout: qatlib x86_64 23.11.0-1.el9 appstream 216 k 2024-03-19T01:20:34.896 INFO:teuthology.orchestra.run.smithi112.stdout: qatzip-libs x86_64 1.1.2-2.el9 appstream 44 k 2024-03-19T01:20:34.896 INFO:teuthology.orchestra.run.smithi112.stdout: re2 x86_64 1:20211101-3.el9 epel 192 k 2024-03-19T01:20:34.897 INFO:teuthology.orchestra.run.smithi112.stdout: socat x86_64 1.7.4.1-5.el9 CentOS-AppStream 300 k 2024-03-19T01:20:34.897 INFO:teuthology.orchestra.run.smithi112.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-03-19T01:20:34.897 INFO:teuthology.orchestra.run.smithi112.stdout: uuid x86_64 1.6.2-55.el9 CentOS-AppStream 56 k 2024-03-19T01:20:34.897 INFO:teuthology.orchestra.run.smithi112.stdout: xmlstarlet x86_64 1.6.1-20.el9 CentOS-AppStream 63 k 2024-03-19T01:20:34.897 INFO:teuthology.orchestra.run.smithi112.stdout: zlib-devel x86_64 1.2.11-40.el9 CentOS-AppStream 44 k 2024-03-19T01:20:34.897 INFO:teuthology.orchestra.run.smithi112.stdout:Installing weak dependencies: 2024-03-19T01:20:34.898 INFO:teuthology.orchestra.run.smithi112.stdout: qatlib-service x86_64 23.11.0-1.el9 appstream 35 k 2024-03-19T01:20:34.898 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-03-19T01:20:34.898 INFO:teuthology.orchestra.run.smithi112.stdout:Transaction Summary 2024-03-19T01:20:34.898 INFO:teuthology.orchestra.run.smithi112.stdout:============================================================================================ 2024-03-19T01:20:34.898 INFO:teuthology.orchestra.run.smithi112.stdout:Install 150 Packages 2024-03-19T01:20:34.898 INFO:teuthology.orchestra.run.smithi112.stdout:Upgrade 24 Packages 2024-03-19T01:20:34.899 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-03-19T01:20:34.900 INFO:teuthology.orchestra.run.smithi112.stdout:Total download size: 224 M 2024-03-19T01:20:34.900 INFO:teuthology.orchestra.run.smithi112.stdout:Downloading Packages: 2024-03-19T01:20:34.992 INFO:teuthology.orchestra.run.smithi115.stdout:(40/174): automake-1.16.2-8.el9.noarch.rpm 2.1 MB/s | 662 kB 00:00 2024-03-19T01:20:35.017 INFO:teuthology.orchestra.run.smithi115.stdout:(41/174): boost-program-options-1.75.0-8.el9.x8 476 kB/s | 106 kB 00:00 2024-03-19T01:20:35.067 INFO:teuthology.orchestra.run.smithi115.stdout:(42/174): flex-2.6.4-9.el9.x86_64.rpm 4.0 MB/s | 307 kB 00:00 2024-03-19T01:20:35.093 INFO:teuthology.orchestra.run.smithi115.stdout:(43/174): flexiblas-3.0.4-8.el9.x86_64.rpm 421 kB/s | 32 kB 00:00 2024-03-19T01:20:35.118 INFO:teuthology.orchestra.run.smithi115.stdout:(44/174): bison-3.7.4-5.el9.x86_64.rpm 2.4 MB/s | 920 kB 00:00 2024-03-19T01:20:35.243 INFO:teuthology.orchestra.run.smithi115.stdout:(45/174): flexiblas-openblas-openmp-3.0.4-8.el9 114 kB/s | 17 kB 00:00 2024-03-19T01:20:35.268 INFO:teuthology.orchestra.run.smithi115.stdout:(46/174): libaio-devel-0.3.111-13.el9.x86_64.rp 67 kB/s | 10 kB 00:00 2024-03-19T01:20:35.327 INFO:teuthology.orchestra.run.smithi115.stdout:(47/174): flexiblas-netlib-3.0.4-8.el9.x86_64.r 12 MB/s | 3.0 MB 00:00 2024-03-19T01:20:35.352 INFO:teuthology.orchestra.run.smithi115.stdout:(48/174): libpmemobj-1.12.1-1.el9.x86_64.rpm 1.4 MB/s | 159 kB 00:00 2024-03-19T01:20:35.377 INFO:teuthology.orchestra.run.smithi115.stdout:(49/174): librabbitmq-0.11.0-7.el9.x86_64.rpm 408 kB/s | 44 kB 00:00 2024-03-19T01:20:35.412 INFO:teuthology.orchestra.run.smithi115.stdout:(50/174): librdkafka-1.6.1-102.el9.x86_64.rpm 7.8 MB/s | 661 kB 00:00 2024-03-19T01:20:35.436 INFO:teuthology.orchestra.run.smithi115.stdout:(51/174): libzstd-devel-1.5.1-2.el9.x86_64.rpm 811 kB/s | 46 kB 00:00 2024-03-19T01:20:35.462 INFO:teuthology.orchestra.run.smithi115.stdout:(52/174): libtool-2.4.6-45.el9.x86_64.rpm 6.8 MB/s | 578 kB 00:00 2024-03-19T01:20:35.497 INFO:teuthology.orchestra.run.smithi115.stdout:(53/174): lttng-ust-2.12.0-6.el9.x86_64.rpm 3.2 MB/s | 282 kB 00:00 2024-03-19T01:20:35.523 INFO:teuthology.orchestra.run.smithi115.stdout:(54/174): lua-5.4.4-4.el9.x86_64.rpm 2.2 MB/s | 187 kB 00:00 2024-03-19T01:20:35.548 INFO:teuthology.orchestra.run.smithi115.stdout:(55/174): oniguruma-6.9.6-1.el9.5.x86_64.rpm 2.5 MB/s | 217 kB 00:00 2024-03-19T01:20:35.573 INFO:teuthology.orchestra.run.smithi115.stdout:(56/174): openblas-0.3.21-2.el9.x86_64.rpm 458 kB/s | 34 kB 00:00 2024-03-19T01:20:35.682 INFO:teuthology.orchestra.run.smithi115.stdout:(57/174): postgresql-13.11-1.el9.x86_64.rpm 11 MB/s | 1.5 MB 00:00 2024-03-19T01:20:35.778 INFO:teuthology.orchestra.run.smithi112.stdout:(1/174): ceph-19.0.0-2206.gb6ae2491.el9.x86_64. 20 kB/s | 6.6 kB 00:00 2024-03-19T01:20:35.834 INFO:teuthology.orchestra.run.smithi115.stdout:(58/174): postgresql-contrib-13.11-1.el9.x86_64 3.0 MB/s | 811 kB 00:00 2024-03-19T01:20:35.899 INFO:teuthology.orchestra.run.smithi115.stdout:(59/174): openblas-openmp-0.3.21-2.el9.x86_64.r 14 MB/s | 4.7 MB 00:00 2024-03-19T01:20:35.924 INFO:teuthology.orchestra.run.smithi115.stdout:(60/174): postgresql-private-libs-13.11-1.el9.x 560 kB/s | 134 kB 00:00 2024-03-19T01:20:36.100 INFO:teuthology.orchestra.run.smithi115.stdout:(61/174): postgresql-server-13.11-1.el9.x86_64. 22 MB/s | 5.7 MB 00:00 2024-03-19T01:20:36.125 INFO:teuthology.orchestra.run.smithi115.stdout:(62/174): python3-attrs-20.3.0-7.el9.noarch.rpm 394 kB/s | 78 kB 00:00 2024-03-19T01:20:36.167 INFO:teuthology.orchestra.run.smithi115.stdout:(63/174): protobuf-3.14.0-13.el9.x86_64.rpm 4.2 MB/s | 1.0 MB 00:00 2024-03-19T01:20:36.378 INFO:teuthology.orchestra.run.smithi112.stdout:(2/174): ceph-fuse-19.0.0-2206.gb6ae2491.el9.x8 1.4 MB/s | 891 kB 00:00 2024-03-19T01:20:36.383 INFO:teuthology.orchestra.run.smithi115.stdout:(64/174): python3-babel-2.9.1-2.el9.noarch.rpm 20 MB/s | 5.8 MB 00:00 2024-03-19T01:20:36.409 INFO:teuthology.orchestra.run.smithi115.stdout:(65/174): python3-jsonpatch-1.21-16.el9.noarch. 86 kB/s | 24 kB 00:00 2024-03-19T01:20:36.434 INFO:teuthology.orchestra.run.smithi115.stdout:(66/174): python3-jsonpointer-2.0-4.el9.noarch. 68 kB/s | 18 kB 00:00 2024-03-19T01:20:36.462 INFO:teuthology.orchestra.run.smithi112.stdout:(3/174): ceph-immutable-object-cache-19.0.0-220 1.7 MB/s | 145 kB 00:00 2024-03-19T01:20:36.476 INFO:teuthology.orchestra.run.smithi115.stdout:(67/174): python3-mako-1.1.4-6.el9.noarch.rpm 2.1 MB/s | 144 kB 00:00 2024-03-19T01:20:36.501 INFO:teuthology.orchestra.run.smithi115.stdout:(68/174): python3-markupsafe-1.1.1-12.el9.x86_6 344 kB/s | 32 kB 00:00 2024-03-19T01:20:36.726 INFO:teuthology.orchestra.run.smithi115.stdout:(69/174): python3-numpy-1.20.1-5.el9.x86_64.rpm 16 MB/s | 4.8 MB 00:00 2024-03-19T01:20:36.751 INFO:teuthology.orchestra.run.smithi115.stdout:(70/174): python3-numpy-f2py-1.20.1-5.el9.x86_6 791 kB/s | 217 kB 00:00 2024-03-19T01:20:36.777 INFO:teuthology.orchestra.run.smithi115.stdout:(71/174): python3-oauthlib-3.1.1-5.el9.noarch.r 637 kB/s | 174 kB 00:00 2024-03-19T01:20:36.802 INFO:teuthology.orchestra.run.smithi115.stdout:(72/174): python3-packaging-20.9-5.el9.noarch.r 920 kB/s | 69 kB 00:00 2024-03-19T01:20:36.836 INFO:teuthology.orchestra.run.smithi115.stdout:(73/174): python3-prettytable-0.7.2-27.el9.noar 709 kB/s | 41 kB 00:00 2024-03-19T01:20:36.861 INFO:teuthology.orchestra.run.smithi115.stdout:(74/174): python3-protobuf-3.14.0-13.el9.noarch 2.8 MB/s | 238 kB 00:00 2024-03-19T01:20:36.886 INFO:teuthology.orchestra.run.smithi115.stdout:(75/174): python3-pyasn1-0.4.8-6.el9.noarch.rpm 1.5 MB/s | 132 kB 00:00 2024-03-19T01:20:36.911 INFO:teuthology.orchestra.run.smithi115.stdout:(76/174): python3-pyasn1-modules-0.4.8-6.el9.no 2.7 MB/s | 211 kB 00:00 2024-03-19T01:20:36.936 INFO:teuthology.orchestra.run.smithi115.stdout:(77/174): python3-pytz-2021.1-5.el9.noarch.rpm 629 kB/s | 47 kB 00:00 2024-03-19T01:20:36.962 INFO:teuthology.orchestra.run.smithi115.stdout:(78/174): python3-requests-oauthlib-1.3.0-12.el 571 kB/s | 43 kB 00:00 2024-03-19T01:20:36.987 INFO:teuthology.orchestra.run.smithi115.stdout:(79/174): python3-toml-0.10.2-6.el9.noarch.rpm 775 kB/s | 39 kB 00:00 2024-03-19T01:20:37.354 INFO:teuthology.orchestra.run.smithi115.stdout:(80/174): python3-wcwidth-0.2.5-8.el9.noarch.rp 104 kB/s | 41 kB 00:00 2024-03-19T01:20:37.379 INFO:teuthology.orchestra.run.smithi115.stdout:(81/174): socat-1.7.4.1-5.el9.x86_64.rpm 770 kB/s | 300 kB 00:00 2024-03-19T01:20:37.412 INFO:teuthology.orchestra.run.smithi112.stdout:(4/174): ceph-base-19.0.0-2206.gb6ae2491.el9.x8 2.7 MB/s | 5.4 MB 00:01 2024-03-19T01:20:37.613 INFO:teuthology.orchestra.run.smithi115.stdout:(82/174): python3-scipy-1.6.2-8.el9.x86_64.rpm 23 MB/s | 16 MB 00:00 2024-03-19T01:20:37.638 INFO:teuthology.orchestra.run.smithi115.stdout:(83/174): uuid-1.6.2-55.el9.x86_64.rpm 199 kB/s | 56 kB 00:00 2024-03-19T01:20:37.663 INFO:teuthology.orchestra.run.smithi115.stdout:(84/174): xmlstarlet-1.6.1-20.el9.x86_64.rpm 222 kB/s | 63 kB 00:00 2024-03-19T01:20:37.688 INFO:teuthology.orchestra.run.smithi115.stdout:(85/174): zlib-devel-1.2.11-40.el9.x86_64.rpm 589 kB/s | 44 kB 00:00 2024-03-19T01:20:37.695 INFO:teuthology.orchestra.run.smithi112.stdout:(5/174): ceph-mds-19.0.0-2206.gb6ae2491.el9.x86 1.9 MB/s | 2.4 MB 00:01 2024-03-19T01:20:37.722 INFO:teuthology.orchestra.run.smithi115.stdout:(86/174): fuse-2.9.9-15.el9.x86_64.rpm 1.3 MB/s | 79 kB 00:00 2024-03-19T01:20:37.737 INFO:teuthology.orchestra.run.smithi112.stdout:(6/174): ceph-mgr-19.0.0-2206.gb6ae2491.el9.x86 4.6 MB/s | 1.5 MB 00:00 2024-03-19T01:20:37.747 INFO:teuthology.orchestra.run.smithi115.stdout:(87/174): python3-cffi-1.14.5-5.el9.x86_64.rpm 2.8 MB/s | 241 kB 00:00 2024-03-19T01:20:37.790 INFO:teuthology.orchestra.run.smithi115.stdout:(88/174): python3-cryptography-36.0.1-4.el9.x86 11 MB/s | 1.1 MB 00:00 2024-03-19T01:20:37.815 INFO:teuthology.orchestra.run.smithi115.stdout:(89/174): python3-ply-3.11-14.el9.noarch.rpm 1.1 MB/s | 103 kB 00:00 2024-03-19T01:20:37.840 INFO:teuthology.orchestra.run.smithi115.stdout:(90/174): python3-pycparser-2.20-6.el9.noarch.r 1.3 MB/s | 124 kB 00:00 2024-03-19T01:20:37.866 INFO:teuthology.orchestra.run.smithi115.stdout:(91/174): python3-urllib3-1.26.5-5.el9.noarch.r 2.4 MB/s | 187 kB 00:00 2024-03-19T01:20:39.529 INFO:teuthology.orchestra.run.smithi112.stdout:(7/174): ceph-common-19.0.0-2206.gb6ae2491.el9. 5.1 MB/s | 21 MB 00:04 2024-03-19T01:20:39.680 INFO:teuthology.orchestra.run.smithi112.stdout:(8/174): ceph-mon-19.0.0-2206.gb6ae2491.el9.x86 2.4 MB/s | 4.7 MB 00:01 2024-03-19T01:20:39.846 INFO:teuthology.orchestra.run.smithi112.stdout:(9/174): ceph-osd-19.0.0-2206.gb6ae2491.el9.x86 8.0 MB/s | 17 MB 00:02 2024-03-19T01:20:39.871 INFO:teuthology.orchestra.run.smithi112.stdout:(10/174): ceph-selinux-19.0.0-2206.gb6ae2491.el 131 kB/s | 25 kB 00:00 2024-03-19T01:20:39.997 INFO:teuthology.orchestra.run.smithi112.stdout:(11/174): cephfs-mirror-19.0.0-2206.gb6ae2491.e 1.7 MB/s | 221 kB 00:00 2024-03-19T01:20:40.080 INFO:teuthology.orchestra.run.smithi112.stdout:(12/174): cephfs-shell-19.0.0-2206.gb6ae2491.el 326 kB/s | 27 kB 00:00 2024-03-19T01:20:40.164 INFO:teuthology.orchestra.run.smithi112.stdout:(13/174): libcephfs-devel-19.0.0-2206.gb6ae2491 381 kB/s | 32 kB 00:00 2024-03-19T01:20:40.281 INFO:teuthology.orchestra.run.smithi112.stdout:(14/174): ceph-radosgw-19.0.0-2206.gb6ae2491.el 13 MB/s | 9.8 MB 00:00 2024-03-19T01:20:40.365 INFO:teuthology.orchestra.run.smithi112.stdout:(15/174): libcephsqlite-19.0.0-2206.gb6ae2491.e 1.9 MB/s | 163 kB 00:00 2024-03-19T01:20:40.390 INFO:teuthology.orchestra.run.smithi112.stdout:(16/174): libcephfs2-19.0.0-2206.gb6ae2491.el9. 3.2 MB/s | 746 kB 00:00 2024-03-19T01:20:40.440 INFO:teuthology.orchestra.run.smithi112.stdout:(17/174): librados-devel-19.0.0-2206.gb6ae2491. 1.6 MB/s | 127 kB 00:00 2024-03-19T01:20:40.641 INFO:teuthology.orchestra.run.smithi112.stdout:(18/174): libradosstriper1-19.0.0-2206.gb6ae249 2.0 MB/s | 501 kB 00:00 2024-03-19T01:20:40.741 INFO:teuthology.orchestra.run.smithi112.stdout:(19/174): python3-ceph-argparse-19.0.0-2206.gb6 451 kB/s | 45 kB 00:00 2024-03-19T01:20:40.816 INFO:teuthology.orchestra.run.smithi112.stdout:(20/174): librgw2-19.0.0-2206.gb6ae2491.el9.x86 14 MB/s | 5.1 MB 00:00 2024-03-19T01:20:40.842 INFO:teuthology.orchestra.run.smithi112.stdout:(21/174): python3-ceph-common-19.0.0-2206.gb6ae 1.3 MB/s | 131 kB 00:00 2024-03-19T01:20:40.900 INFO:teuthology.orchestra.run.smithi112.stdout:(22/174): python3-cephfs-19.0.0-2206.gb6ae2491. 1.9 MB/s | 162 kB 00:00 2024-03-19T01:20:40.992 INFO:teuthology.orchestra.run.smithi112.stdout:(23/174): python3-rados-19.0.0-2206.gb6ae2491.e 2.1 MB/s | 321 kB 00:00 2024-03-19T01:20:41.017 INFO:teuthology.orchestra.run.smithi112.stdout:(24/174): python3-rbd-19.0.0-2206.gb6ae2491.el9 2.5 MB/s | 298 kB 00:00 2024-03-19T01:20:41.076 INFO:teuthology.orchestra.run.smithi112.stdout:(25/174): python3-rgw-19.0.0-2206.gb6ae2491.el9 1.2 MB/s | 99 kB 00:00 2024-03-19T01:20:41.101 INFO:teuthology.orchestra.run.smithi112.stdout:(26/174): rbd-fuse-19.0.0-2206.gb6ae2491.el9.x8 1.0 MB/s | 85 kB 00:00 2024-03-19T01:20:41.210 INFO:teuthology.orchestra.run.smithi112.stdout:(27/174): rbd-nbd-19.0.0-2206.gb6ae2491.el9.x86 1.5 MB/s | 170 kB 00:00 2024-03-19T01:20:41.302 INFO:teuthology.orchestra.run.smithi112.stdout:(28/174): ceph-grafana-dashboards-19.0.0-2206.g 370 kB/s | 34 kB 00:00 2024-03-19T01:20:41.394 INFO:teuthology.orchestra.run.smithi112.stdout:(29/174): ceph-mgr-cephadm-19.0.0-2206.gb6ae249 1.5 MB/s | 139 kB 00:00 2024-03-19T01:20:41.569 INFO:teuthology.orchestra.run.smithi112.stdout:(30/174): ceph-mgr-dashboard-19.0.0-2206.gb6ae2 10 MB/s | 1.8 MB 00:00 2024-03-19T01:20:42.061 INFO:teuthology.orchestra.run.smithi112.stdout:(31/174): ceph-mgr-diskprediction-local-19.0.0- 15 MB/s | 7.4 MB 00:00 2024-03-19T01:20:42.153 INFO:teuthology.orchestra.run.smithi112.stdout:(32/174): ceph-mgr-modules-core-19.0.0-2206.gb6 2.7 MB/s | 248 kB 00:00 2024-03-19T01:20:42.220 INFO:teuthology.orchestra.run.smithi112.stdout:(33/174): ceph-mgr-rook-19.0.0-2206.gb6ae2491.e 740 kB/s | 49 kB 00:00 2024-03-19T01:20:42.279 INFO:teuthology.orchestra.run.smithi112.stdout:(34/174): ceph-prometheus-alerts-19.0.0-2206.gb 281 kB/s | 16 kB 00:00 2024-03-19T01:20:42.362 INFO:teuthology.orchestra.run.smithi112.stdout:(35/174): ceph-volume-19.0.0-2206.gb6ae2491.el9 3.1 MB/s | 264 kB 00:00 2024-03-19T01:20:42.421 INFO:teuthology.orchestra.run.smithi112.stdout:(36/174): rbd-mirror-19.0.0-2206.gb6ae2491.el9. 2.3 MB/s | 3.1 MB 00:01 2024-03-19T01:20:42.454 INFO:teuthology.orchestra.run.smithi112.stdout:(37/174): cephadm-19.0.0-2206.gb6ae2491.el9.noa 7.9 MB/s | 749 kB 00:00 2024-03-19T01:20:42.513 INFO:teuthology.orchestra.run.smithi112.stdout:(38/174): cephfs-top-19.0.0-2206.gb6ae2491.el9. 198 kB/s | 18 kB 00:00 2024-03-19T01:20:42.663 INFO:teuthology.orchestra.run.smithi112.stdout:(39/174): automake-1.16.2-8.el9.noarch.rpm 3.1 MB/s | 662 kB 00:00 2024-03-19T01:20:42.714 INFO:teuthology.orchestra.run.smithi112.stdout:(40/174): bison-3.7.4-5.el9.x86_64.rpm 5.1 MB/s | 920 kB 00:00 2024-03-19T01:20:42.739 INFO:teuthology.orchestra.run.smithi112.stdout:(41/174): boost-program-options-1.75.0-8.el9.x8 1.4 MB/s | 106 kB 00:00 2024-03-19T01:20:42.767 INFO:teuthology.orchestra.run.smithi112.stdout:(42/174): flex-2.6.4-9.el9.x86_64.rpm 5.7 MB/s | 307 kB 00:00 2024-03-19T01:20:42.791 INFO:teuthology.orchestra.run.smithi112.stdout:(43/174): flexiblas-3.0.4-8.el9.x86_64.rpm 608 kB/s | 32 kB 00:00 2024-03-19T01:20:42.875 INFO:teuthology.orchestra.run.smithi112.stdout:(44/174): flexiblas-netlib-3.0.4-8.el9.x86_64.r 28 MB/s | 3.0 MB 00:00 2024-03-19T01:20:42.901 INFO:teuthology.orchestra.run.smithi112.stdout:(45/174): flexiblas-openblas-openmp-3.0.4-8.el9 160 kB/s | 17 kB 00:00 2024-03-19T01:20:42.926 INFO:teuthology.orchestra.run.smithi112.stdout:(46/174): libaio-devel-0.3.111-13.el9.x86_64.rp 200 kB/s | 10 kB 00:00 2024-03-19T01:20:42.951 INFO:teuthology.orchestra.run.smithi112.stdout:(47/174): libpmemobj-1.12.1-1.el9.x86_64.rpm 3.1 MB/s | 159 kB 00:00 2024-03-19T01:20:42.968 INFO:teuthology.orchestra.run.smithi112.stdout:(48/174): librabbitmq-0.11.0-7.el9.x86_64.rpm 1.0 MB/s | 44 kB 00:00 2024-03-19T01:20:43.002 INFO:teuthology.orchestra.run.smithi112.stdout:(49/174): librdkafka-1.6.1-102.el9.x86_64.rpm 13 MB/s | 661 kB 00:00 2024-03-19T01:20:43.027 INFO:teuthology.orchestra.run.smithi112.stdout:(50/174): libtool-2.4.6-45.el9.x86_64.rpm 9.8 MB/s | 578 kB 00:00 2024-03-19T01:20:43.044 INFO:teuthology.orchestra.run.smithi112.stdout:(51/174): libzstd-devel-1.5.1-2.el9.x86_64.rpm 1.1 MB/s | 46 kB 00:00 2024-03-19T01:20:43.069 INFO:teuthology.orchestra.run.smithi112.stdout:(52/174): lttng-ust-2.12.0-6.el9.x86_64.rpm 6.6 MB/s | 282 kB 00:00 2024-03-19T01:20:43.095 INFO:teuthology.orchestra.run.smithi112.stdout:(53/174): lua-5.4.4-4.el9.x86_64.rpm 3.6 MB/s | 187 kB 00:00 2024-03-19T01:20:43.120 INFO:teuthology.orchestra.run.smithi112.stdout:(54/174): oniguruma-6.9.6-1.el9.5.x86_64.rpm 4.2 MB/s | 217 kB 00:00 2024-03-19T01:20:43.136 INFO:teuthology.orchestra.run.smithi112.stdout:(55/174): openblas-0.3.21-2.el9.x86_64.rpm 827 kB/s | 34 kB 00:00 2024-03-19T01:20:43.254 INFO:teuthology.orchestra.run.smithi112.stdout:(56/174): openblas-openmp-0.3.21-2.el9.x86_64.r 35 MB/s | 4.7 MB 00:00 2024-03-19T01:20:43.296 INFO:teuthology.orchestra.run.smithi112.stdout:(57/174): postgresql-13.11-1.el9.x86_64.rpm 9.6 MB/s | 1.5 MB 00:00 2024-03-19T01:20:43.329 INFO:teuthology.orchestra.run.smithi112.stdout:(58/174): postgresql-contrib-13.11-1.el9.x86_64 11 MB/s | 811 kB 00:00 2024-03-19T01:20:43.354 INFO:teuthology.orchestra.run.smithi112.stdout:(59/174): postgresql-private-libs-13.11-1.el9.x 2.3 MB/s | 134 kB 00:00 2024-03-19T01:20:43.488 INFO:teuthology.orchestra.run.smithi112.stdout:(60/174): postgresql-server-13.11-1.el9.x86_64. 36 MB/s | 5.7 MB 00:00 2024-03-19T01:20:43.522 INFO:teuthology.orchestra.run.smithi112.stdout:(61/174): protobuf-3.14.0-13.el9.x86_64.rpm 6.1 MB/s | 1.0 MB 00:00 2024-03-19T01:20:43.547 INFO:teuthology.orchestra.run.smithi112.stdout:(62/174): python3-attrs-20.3.0-7.el9.noarch.rpm 1.3 MB/s | 78 kB 00:00 2024-03-19T01:20:43.681 INFO:teuthology.orchestra.run.smithi112.stdout:(63/174): python3-babel-2.9.1-2.el9.noarch.rpm 36 MB/s | 5.8 MB 00:00 2024-03-19T01:20:43.697 INFO:teuthology.orchestra.run.smithi112.stdout:(64/174): python3-jsonpatch-1.21-16.el9.noarch. 164 kB/s | 24 kB 00:00 2024-03-19T01:20:43.714 INFO:teuthology.orchestra.run.smithi112.stdout:(65/174): python3-jsonpointer-2.0-4.el9.noarch. 541 kB/s | 18 kB 00:00 2024-03-19T01:20:43.739 INFO:teuthology.orchestra.run.smithi112.stdout:(66/174): python3-mako-1.1.4-6.el9.noarch.rpm 3.4 MB/s | 144 kB 00:00 2024-03-19T01:20:43.756 INFO:teuthology.orchestra.run.smithi112.stdout:(67/174): python3-markupsafe-1.1.1-12.el9.x86_6 763 kB/s | 32 kB 00:00 2024-03-19T01:20:43.873 INFO:teuthology.orchestra.run.smithi112.stdout:(68/174): python3-numpy-1.20.1-5.el9.x86_64.rpm 36 MB/s | 4.8 MB 00:00 2024-03-19T01:20:43.899 INFO:teuthology.orchestra.run.smithi112.stdout:(69/174): python3-numpy-f2py-1.20.1-5.el9.x86_6 1.5 MB/s | 217 kB 00:00 2024-03-19T01:20:43.924 INFO:teuthology.orchestra.run.smithi112.stdout:(70/174): python3-oauthlib-3.1.1-5.el9.noarch.r 3.4 MB/s | 174 kB 00:00 2024-03-19T01:20:43.949 INFO:teuthology.orchestra.run.smithi112.stdout:(71/174): python3-packaging-20.9-5.el9.noarch.r 1.3 MB/s | 69 kB 00:00 2024-03-19T01:20:43.975 INFO:teuthology.orchestra.run.smithi112.stdout:(72/174): python3-prettytable-0.7.2-27.el9.noar 818 kB/s | 41 kB 00:00 2024-03-19T01:20:44.000 INFO:teuthology.orchestra.run.smithi112.stdout:(73/174): python3-protobuf-3.14.0-13.el9.noarch 4.6 MB/s | 238 kB 00:00 2024-03-19T01:20:44.025 INFO:teuthology.orchestra.run.smithi112.stdout:(74/174): python3-pyasn1-0.4.8-6.el9.noarch.rpm 2.6 MB/s | 132 kB 00:00 2024-03-19T01:20:44.050 INFO:teuthology.orchestra.run.smithi112.stdout:(75/174): python3-pyasn1-modules-0.4.8-6.el9.no 4.1 MB/s | 211 kB 00:00 2024-03-19T01:20:44.076 INFO:teuthology.orchestra.run.smithi112.stdout:(76/174): python3-pytz-2021.1-5.el9.noarch.rpm 947 kB/s | 47 kB 00:00 2024-03-19T01:20:44.101 INFO:teuthology.orchestra.run.smithi112.stdout:(77/174): python3-requests-oauthlib-1.3.0-12.el 854 kB/s | 43 kB 00:00 2024-03-19T01:20:44.552 INFO:teuthology.orchestra.run.smithi112.stdout:(78/174): ceph-test-19.0.0-2206.gb6ae2491.el9.x 10 MB/s | 48 MB 00:04 2024-03-19T01:20:44.618 INFO:teuthology.orchestra.run.smithi112.stdout:(79/174): python3-toml-0.10.2-6.el9.noarch.rpm 75 kB/s | 39 kB 00:00 2024-03-19T01:20:44.785 INFO:teuthology.orchestra.run.smithi112.stdout:(80/174): python3-scipy-1.6.2-8.el9.x86_64.rpm 22 MB/s | 16 MB 00:00 2024-03-19T01:20:44.813 INFO:teuthology.orchestra.run.smithi112.stdout:(81/174): python3-wcwidth-0.2.5-8.el9.noarch.rp 166 kB/s | 41 kB 00:00 2024-03-19T01:20:44.827 INFO:teuthology.orchestra.run.smithi112.stdout:(82/174): socat-1.7.4.1-5.el9.x86_64.rpm 1.4 MB/s | 300 kB 00:00 2024-03-19T01:20:44.844 INFO:teuthology.orchestra.run.smithi112.stdout:(83/174): uuid-1.6.2-55.el9.x86_64.rpm 977 kB/s | 56 kB 00:00 2024-03-19T01:20:44.861 INFO:teuthology.orchestra.run.smithi112.stdout:(84/174): zlib-devel-1.2.11-40.el9.x86_64.rpm 1.3 MB/s | 44 kB 00:00 2024-03-19T01:20:44.878 INFO:teuthology.orchestra.run.smithi112.stdout:(85/174): xmlstarlet-1.6.1-20.el9.x86_64.rpm 1.2 MB/s | 63 kB 00:00 2024-03-19T01:20:44.903 INFO:teuthology.orchestra.run.smithi112.stdout:(86/174): fuse-2.9.9-15.el9.x86_64.rpm 1.3 MB/s | 79 kB 00:00 2024-03-19T01:20:44.929 INFO:teuthology.orchestra.run.smithi112.stdout:(87/174): python3-cffi-1.14.5-5.el9.x86_64.rpm 3.5 MB/s | 241 kB 00:00 2024-03-19T01:20:44.954 INFO:teuthology.orchestra.run.smithi112.stdout:(88/174): python3-ply-3.11-14.el9.noarch.rpm 2.0 MB/s | 103 kB 00:00 2024-03-19T01:20:44.987 INFO:teuthology.orchestra.run.smithi112.stdout:(89/174): python3-cryptography-36.0.1-4.el9.x86 14 MB/s | 1.1 MB 00:00 2024-03-19T01:20:45.012 INFO:teuthology.orchestra.run.smithi112.stdout:(90/174): python3-pycparser-2.20-6.el9.noarch.r 1.5 MB/s | 124 kB 00:00 2024-03-19T01:20:45.038 INFO:teuthology.orchestra.run.smithi112.stdout:(91/174): python3-urllib3-1.26.5-5.el9.noarch.r 2.2 MB/s | 187 kB 00:00 2024-03-19T01:22:47.654 INFO:teuthology.orchestra.run.smithi115.stdout:[MIRROR] jq-1.6-16.el9.x86_64.rpm: Curl error (28): Timeout was reached for http://ftp-nyc.osuosl.org/pub/centos-stream/9-stream/BaseOS/x86_64/os/Packages/jq-1.6-16.el9.x86_64.rpm [Failed to connect to ftp-nyc.osuosl.org port 80: Connection timed out] 2024-03-19T01:22:47.655 INFO:teuthology.orchestra.run.smithi115.stdout:[MIRROR] libgfortran-11.4.1-3.el9.x86_64.rpm: Curl error (28): Timeout was reached for http://ftp-nyc.osuosl.org/pub/centos-stream/9-stream/BaseOS/x86_64/os/Packages/libgfortran-11.4.1-3.el9.x86_64.rpm [Failed to connect to ftp-nyc.osuosl.org port 80: Connection timed out] 2024-03-19T01:22:47.655 INFO:teuthology.orchestra.run.smithi115.stdout:[MIRROR] libquadmath-11.4.1-3.el9.x86_64.rpm: Curl error (28): Timeout was reached for http://ftp-nyc.osuosl.org/pub/centos-stream/9-stream/BaseOS/x86_64/os/Packages/libquadmath-11.4.1-3.el9.x86_64.rpm [Failed to connect to ftp-nyc.osuosl.org port 80: Connection timed out] 2024-03-19T01:22:48.472 INFO:teuthology.orchestra.run.smithi115.stdout:(92/174): libquadmath-11.4.1-3.el9.x86_64.rpm 1.5 kB/s | 195 kB 02:10 2024-03-19T01:22:48.497 INFO:teuthology.orchestra.run.smithi115.stdout:(93/174): jq-1.6-16.el9.x86_64.rpm 1.4 kB/s | 187 kB 02:10 2024-03-19T01:22:48.822 INFO:teuthology.orchestra.run.smithi115.stdout:(94/174): python3-requests-2.25.1-8.el9.noarch. 357 kB/s | 125 kB 00:00 2024-03-19T01:22:49.305 INFO:teuthology.orchestra.run.smithi115.stdout:(95/174): libgfortran-11.4.1-3.el9.x86_64.rpm 6.1 kB/s | 803 kB 02:11 2024-03-19T01:22:53.879 INFO:teuthology.orchestra.run.smithi112.stdout:[MIRROR] jq-1.6-16.el9.x86_64.rpm: Curl error (28): Timeout was reached for http://ftp-nyc.osuosl.org/pub/centos-stream/9-stream/BaseOS/x86_64/os/Packages/jq-1.6-16.el9.x86_64.rpm [Failed to connect to ftp-nyc.osuosl.org port 80: Connection timed out] 2024-03-19T01:22:53.879 INFO:teuthology.orchestra.run.smithi112.stdout:[MIRROR] libgfortran-11.4.1-3.el9.x86_64.rpm: Curl error (28): Timeout was reached for http://ftp-nyc.osuosl.org/pub/centos-stream/9-stream/BaseOS/x86_64/os/Packages/libgfortran-11.4.1-3.el9.x86_64.rpm [Failed to connect to ftp-nyc.osuosl.org port 80: Connection timed out] 2024-03-19T01:22:53.879 INFO:teuthology.orchestra.run.smithi112.stdout:[MIRROR] libquadmath-11.4.1-3.el9.x86_64.rpm: Curl error (28): Timeout was reached for http://ftp-nyc.osuosl.org/pub/centos-stream/9-stream/BaseOS/x86_64/os/Packages/libquadmath-11.4.1-3.el9.x86_64.rpm [Failed to connect to ftp-nyc.osuosl.org port 80: Connection timed out] 2024-03-19T01:22:54.581 INFO:teuthology.orchestra.run.smithi112.stdout:(92/174): jq-1.6-16.el9.x86_64.rpm 1.4 kB/s | 187 kB 02:09 2024-03-19T01:22:54.606 INFO:teuthology.orchestra.run.smithi112.stdout:(93/174): libquadmath-11.4.1-3.el9.x86_64.rpm 1.5 kB/s | 195 kB 02:09 2024-03-19T01:22:54.781 INFO:teuthology.orchestra.run.smithi112.stdout:(94/174): python3-requests-2.25.1-8.el9.noarch. 624 kB/s | 125 kB 00:00 2024-03-19T01:22:54.873 INFO:teuthology.orchestra.run.smithi112.stdout:(95/174): elfutils-libelf-devel-0.190-2.el9.x86 80 kB/s | 21 kB 00:00 2024-03-19T01:22:55.048 INFO:teuthology.orchestra.run.smithi112.stdout:(96/174): libgfortran-11.4.1-3.el9.x86_64.rpm 6.2 kB/s | 803 kB 02:10 2024-03-19T01:22:55.074 INFO:teuthology.orchestra.run.smithi112.stdout:(97/174): libuuid-devel-2.37.4-18.el9.x86_64.rp 87 kB/s | 25 kB 00:00 2024-03-19T01:22:55.399 INFO:teuthology.orchestra.run.smithi112.stdout:(98/174): python3-devel-3.9.17-2.el9.x86_64.rpm 700 kB/s | 245 kB 00:00 2024-03-19T01:22:55.583 INFO:teuthology.orchestra.run.smithi112.stdout:(99/174): python3-jinja2-2.11.3-5.el9.noarch.rp 487 kB/s | 248 kB 00:00 2024-03-19T01:22:55.608 INFO:teuthology.orchestra.run.smithi112.stdout:(100/174): qatlib-23.11.0-1.el9.x86_64.rpm 1.0 MB/s | 216 kB 00:00 2024-03-19T01:22:55.683 INFO:teuthology.orchestra.run.smithi112.stdout:(101/174): qatlib-service-23.11.0-1.el9.x86_64. 348 kB/s | 35 kB 00:00 2024-03-19T01:22:55.708 INFO:teuthology.orchestra.run.smithi112.stdout:(102/174): qatzip-libs-1.1.2-2.el9.x86_64.rpm 444 kB/s | 44 kB 00:00 2024-03-19T01:22:55.792 INFO:teuthology.orchestra.run.smithi112.stdout:(103/174): xfsprogs-devel-6.3.0-1.el9.x86_64.rp 864 kB/s | 94 kB 00:00 2024-03-19T01:22:55.809 INFO:teuthology.orchestra.run.smithi112.stdout:(104/174): lua-devel-5.4.4-4.el9.x86_64.rpm 222 kB/s | 22 kB 00:00 2024-03-19T01:22:55.884 INFO:teuthology.orchestra.run.smithi112.stdout:(105/174): openssl-devel-3.0.7-27.el9.x86_64.rp 4.1 MB/s | 4.1 MB 00:01 2024-03-19T01:22:56.176 INFO:teuthology.orchestra.run.smithi112.stdout:(106/174): protobuf-compiler-3.14.0-13.el9.x86_ 2.2 MB/s | 863 kB 00:00 2024-03-19T01:22:56.201 INFO:teuthology.orchestra.run.smithi112.stdout:(107/174): gperftools-libs-2.9.1-2.el9.x86_64.r 976 kB/s | 309 kB 00:00 2024-03-19T01:22:56.268 INFO:teuthology.orchestra.run.smithi112.stdout:(108/174): abseil-cpp-20211102.0-3.el9.x86_64.r 1.2 MB/s | 548 kB 00:00 2024-03-19T01:22:56.335 INFO:teuthology.orchestra.run.smithi112.stdout:(109/174): libarrow-doc-9.0.0-10.el9.noarch.rpm 389 kB/s | 26 kB 00:00 2024-03-19T01:22:56.385 INFO:teuthology.orchestra.run.smithi112.stdout:(110/174): grpc-data-1.46.7-10.el9.noarch.rpm 93 kB/s | 19 kB 00:00 2024-03-19T01:22:56.402 INFO:teuthology.orchestra.run.smithi112.stdout:(111/174): liboath-2.6.7-2.el9.x86_64.rpm 728 kB/s | 49 kB 00:00 2024-03-19T01:22:56.461 INFO:teuthology.orchestra.run.smithi112.stdout:(112/174): luarocks-3.9.2-1.el9.noarch.rpm 2.5 MB/s | 151 kB 00:00 2024-03-19T01:22:56.486 INFO:teuthology.orchestra.run.smithi112.stdout:(113/174): libunwind-1.6.2-1.el9.x86_64.rpm 670 kB/s | 67 kB 00:00 2024-03-19T01:22:56.578 INFO:teuthology.orchestra.run.smithi112.stdout:(114/174): parquet-libs-9.0.0-10.el9.x86_64.rpm 7.0 MB/s | 838 kB 00:00 2024-03-19T01:22:56.645 INFO:teuthology.orchestra.run.smithi112.stdout:(115/174): libarrow-9.0.0-10.el9.x86_64.rpm 10 MB/s | 4.4 MB 00:00 2024-03-19T01:22:56.662 INFO:teuthology.orchestra.run.smithi112.stdout:(116/174): python3-bcrypt-3.2.2-1.el9.x86_64.rp 519 kB/s | 43 kB 00:00 2024-03-19T01:22:56.695 INFO:teuthology.orchestra.run.smithi112.stdout:(117/174): python3-asyncssh-2.13.2-4.el9.noarch 2.6 MB/s | 548 kB 00:00 2024-03-19T01:22:56.712 INFO:teuthology.orchestra.run.smithi112.stdout:(118/174): python3-certifi-2021.10.8-2.el9.noar 298 kB/s | 15 kB 00:00 2024-03-19T01:22:56.729 INFO:teuthology.orchestra.run.smithi112.stdout:(119/174): python3-cachetools-4.2.4-1.el9.noarc 384 kB/s | 32 kB 00:00 2024-03-19T01:22:56.763 INFO:teuthology.orchestra.run.smithi112.stdout:(120/174): python3-cheroot-8.6.0-4.el9.noarch.r 2.5 MB/s | 172 kB 00:00 2024-03-19T01:22:56.788 INFO:teuthology.orchestra.run.smithi112.stdout:(121/174): python3-cherrypy-18.6.1-2.el9.noarch 4.6 MB/s | 358 kB 00:00 2024-03-19T01:22:56.813 INFO:teuthology.orchestra.run.smithi112.stdout:(122/174): python3-cmd2-2.4.2-2.el9.noarch.rpm 3.5 MB/s | 303 kB 00:00 2024-03-19T01:22:56.830 INFO:teuthology.orchestra.run.smithi112.stdout:(123/174): python3-colorama-0.4.4-12.el9.noarch 526 kB/s | 35 kB 00:00 2024-03-19T01:22:56.856 INFO:teuthology.orchestra.run.smithi112.stdout:(124/174): python3-google-auth-2.28.0-1.el9.noa 2.9 MB/s | 203 kB 00:00 2024-03-19T01:22:56.889 INFO:teuthology.orchestra.run.smithi112.stdout:(125/174): python3-grpcio-tools-1.46.7-10.el9.x 2.4 MB/s | 144 kB 00:00 2024-03-19T01:22:56.906 INFO:teuthology.orchestra.run.smithi112.stdout:(126/174): python3-jaraco-8.2.1-3.el9.noarch.rp 214 kB/s | 11 kB 00:00 2024-03-19T01:22:56.948 INFO:teuthology.orchestra.run.smithi112.stdout:(127/174): python3-jaraco-classes-3.2.1-5.el9.n 302 kB/s | 18 kB 00:00 2024-03-19T01:22:56.973 INFO:teuthology.orchestra.run.smithi112.stdout:(128/174): python3-jaraco-collections-3.0.0-8.e 346 kB/s | 23 kB 00:00 2024-03-19T01:22:57.024 INFO:teuthology.orchestra.run.smithi112.stdout:(129/174): python3-grpcio-1.46.7-10.el9.x86_64. 9.7 MB/s | 2.0 MB 00:00 2024-03-19T01:22:57.049 INFO:teuthology.orchestra.run.smithi112.stdout:(130/174): python3-jaraco-functools-3.5.0-2.el9 194 kB/s | 19 kB 00:00 2024-03-19T01:22:57.083 INFO:teuthology.orchestra.run.smithi112.stdout:(131/174): python3-jaraco-text-3.2.0-6.el9.noar 180 kB/s | 20 kB 00:00 2024-03-19T01:22:57.108 INFO:teuthology.orchestra.run.smithi112.stdout:(132/174): python3-logutils-0.3.5-21.el9.noarch 789 kB/s | 46 kB 00:00 2024-03-19T01:22:57.142 INFO:teuthology.orchestra.run.smithi112.stdout:(133/174): python3-more-itertools-8.12.0-2.el9. 1.3 MB/s | 79 kB 00:00 2024-03-19T01:22:57.167 INFO:teuthology.orchestra.run.smithi112.stdout:(134/174): python3-natsort-7.1.1-5.el9.noarch.r 986 kB/s | 58 kB 00:00 2024-03-19T01:22:57.200 INFO:teuthology.orchestra.run.smithi112.stdout:(135/174): python3-kubernetes-26.1.0-2.el9.noar 5.8 MB/s | 1.0 MB 00:00 2024-03-19T01:22:57.225 INFO:teuthology.orchestra.run.smithi112.stdout:(136/174): python3-portend-3.1.0-2.el9.noarch.r 281 kB/s | 16 kB 00:00 2024-03-19T01:22:57.251 INFO:teuthology.orchestra.run.smithi112.stdout:(137/174): python3-pecan-1.4.2-3.el9.noarch.rpm 2.4 MB/s | 272 kB 00:00 2024-03-19T01:22:57.276 INFO:teuthology.orchestra.run.smithi112.stdout:(138/174): python3-pyOpenSSL-21.0.0-1.el9.noarc 1.2 MB/s | 90 kB 00:00 2024-03-19T01:22:57.301 INFO:teuthology.orchestra.run.smithi112.stdout:(139/174): python3-pyperclip-1.8.2-2.el9.noarch 374 kB/s | 28 kB 00:00 2024-03-19T01:22:57.327 INFO:teuthology.orchestra.run.smithi112.stdout:(140/174): python3-repoze-lru-0.7-16.el9.noarch 407 kB/s | 31 kB 00:00 2024-03-19T01:22:57.352 INFO:teuthology.orchestra.run.smithi112.stdout:(141/174): python3-routes-2.5.1-5.el9.noarch.rp 2.4 MB/s | 188 kB 00:00 2024-03-19T01:22:57.377 INFO:teuthology.orchestra.run.smithi112.stdout:(142/174): python3-rsa-4.9-2.el9.noarch.rpm 785 kB/s | 59 kB 00:00 2024-03-19T01:22:57.402 INFO:teuthology.orchestra.run.smithi112.stdout:(143/174): python3-tempora-5.0.0-2.el9.noarch.r 475 kB/s | 36 kB 00:00 2024-03-19T01:22:57.427 INFO:teuthology.orchestra.run.smithi112.stdout:(144/174): python3-typing-extensions-4.4.0-2.el 676 kB/s | 51 kB 00:00 2024-03-19T01:22:57.453 INFO:teuthology.orchestra.run.smithi112.stdout:(145/174): python3-webob-1.8.7-6.el9.noarch.rpm 3.0 MB/s | 230 kB 00:00 2024-03-19T01:22:57.478 INFO:teuthology.orchestra.run.smithi112.stdout:(146/174): python3-websocket-client-1.2.3-2.el9 1.2 MB/s | 90 kB 00:00 2024-03-19T01:22:57.506 INFO:teuthology.orchestra.run.smithi112.stdout:(147/174): python3-werkzeug-2.0.3-3.el9.1.noarc 5.5 MB/s | 427 kB 00:00 2024-03-19T01:22:57.521 INFO:teuthology.orchestra.run.smithi112.stdout:(148/174): python3-zc-lockfile-2.0-10.el9.noarc 297 kB/s | 20 kB 00:00 2024-03-19T01:22:57.545 INFO:teuthology.orchestra.run.smithi112.stdout:(149/174): re2-20211101-3.el9.x86_64.rpm 2.8 MB/s | 192 kB 00:00 2024-03-19T01:22:57.712 INFO:teuthology.orchestra.run.smithi112.stdout:(150/174): thrift-0.15.0-2.el9.x86_64.rpm 7.6 MB/s | 1.6 MB 00:00 2024-03-19T01:22:57.838 INFO:teuthology.orchestra.run.smithi112.stdout:(151/174): xfsdump-3.1.12-4.el9.x86_64.rpm 2.5 MB/s | 321 kB 00:00 2024-03-19T01:22:57.938 INFO:teuthology.orchestra.run.smithi112.stdout:(152/174): librbd1-19.0.0-2206.gb6ae2491.el9.x8 8.0 MB/s | 3.1 MB 00:00 2024-03-19T01:22:58.163 INFO:teuthology.orchestra.run.smithi112.stdout:(153/174): librados2-19.0.0-2206.gb6ae2491.el9. 5.3 MB/s | 3.4 MB 00:00 2024-03-19T01:22:58.547 INFO:teuthology.orchestra.run.smithi112.stdout:(154/174): NetworkManager-team-1.47.2-1.el9.x86 110 kB/s | 42 kB 00:00 2024-03-19T01:22:59.014 INFO:teuthology.orchestra.run.smithi112.stdout:(155/174): NetworkManager-tui-1.47.2-1.el9.x86_ 532 kB/s | 248 kB 00:00 2024-03-19T01:22:59.139 INFO:teuthology.orchestra.run.smithi112.stdout:(156/174): acl-2.3.1-4.el9.x86_64.rpm 571 kB/s | 71 kB 00:00 2024-03-19T01:22:59.239 INFO:teuthology.orchestra.run.smithi112.stdout:(157/174): elfutils-debuginfod-client-0.190-2.e 365 kB/s | 36 kB 00:00 2024-03-19T01:22:59.523 INFO:teuthology.orchestra.run.smithi112.stdout:(158/174): elfutils-libelf-0.190-2.el9.x86_64.r 679 kB/s | 192 kB 00:00 2024-03-19T01:22:59.831 INFO:teuthology.orchestra.run.smithi112.stdout:(159/174): NetworkManager-libnm-1.47.2-1.el9.x8 991 kB/s | 1.8 MB 00:01 2024-03-19T01:22:59.856 INFO:teuthology.orchestra.run.smithi112.stdout:(160/174): elfutils-libs-0.190-2.el9.x86_64.rpm 766 kB/s | 256 kB 00:00 2024-03-19T01:22:59.940 INFO:teuthology.orchestra.run.smithi112.stdout:(161/174): libacl-2.3.1-4.el9.x86_64.rpm 212 kB/s | 23 kB 00:00 2024-03-19T01:23:00.040 INFO:teuthology.orchestra.run.smithi112.stdout:(162/174): NetworkManager-1.47.2-1.el9.x86_64.r 1.0 MB/s | 2.3 MB 00:02 2024-03-19T01:23:00.066 INFO:teuthology.orchestra.run.smithi112.stdout:(163/174): libblkid-2.37.4-18.el9.x86_64.rpm 518 kB/s | 108 kB 00:00 2024-03-19T01:23:00.149 INFO:teuthology.orchestra.run.smithi112.stdout:(164/174): libfdisk-2.37.4-18.el9.x86_64.rpm 740 kB/s | 155 kB 00:00 2024-03-19T01:23:00.183 INFO:teuthology.orchestra.run.smithi112.stdout:(165/174): libsmartcols-2.37.4-18.el9.x86_64.rp 547 kB/s | 64 kB 00:00 2024-03-19T01:23:00.258 INFO:teuthology.orchestra.run.smithi112.stdout:(166/174): libmount-2.37.4-18.el9.x86_64.rpm 624 kB/s | 136 kB 00:00 2024-03-19T01:23:00.275 INFO:teuthology.orchestra.run.smithi112.stdout:(167/174): libuuid-2.37.4-18.el9.x86_64.rpm 229 kB/s | 29 kB 00:00 2024-03-19T01:23:01.242 INFO:teuthology.orchestra.run.smithi112.stdout:(168/174): openssl-3.0.7-27.el9.x86_64.rpm 1.1 MB/s | 1.2 MB 00:01 2024-03-19T01:23:01.634 INFO:teuthology.orchestra.run.smithi112.stdout:(169/174): util-linux-core-2.37.4-18.el9.x86_64 1.2 MB/s | 465 kB 00:00 2024-03-19T01:23:01.742 INFO:teuthology.orchestra.run.smithi112.stdout:(170/174): util-linux-user-2.37.4-18.el9.x86_64 287 kB/s | 31 kB 00:00 2024-03-19T01:23:02.093 INFO:teuthology.orchestra.run.smithi112.stdout:(171/174): openssl-libs-3.0.7-27.el9.x86_64.rpm 1.2 MB/s | 2.2 MB 00:01 2024-03-19T01:23:02.176 INFO:teuthology.orchestra.run.smithi112.stdout:(172/174): util-linux-2.37.4-18.el9.x86_64.rpm 1.2 MB/s | 2.3 MB 00:01 2024-03-19T01:23:02.493 INFO:teuthology.orchestra.run.smithi112.stdout:(173/174): libacl-devel-2.3.1-4.el9.x86_64.rpm 186 kB/s | 74 kB 00:00 2024-03-19T01:23:02.635 INFO:teuthology.orchestra.run.smithi112.stdout:(174/174): xfsprogs-6.3.0-1.el9.x86_64.rpm 1.2 MB/s | 1.1 MB 00:00 2024-03-19T01:23:02.642 INFO:teuthology.orchestra.run.smithi112.stdout:-------------------------------------------------------------------------------- 2024-03-19T01:23:02.642 INFO:teuthology.orchestra.run.smithi112.stdout:Total 1.5 MB/s | 224 MB 02:27 2024-03-19T01:23:03.090 INFO:teuthology.orchestra.run.smithi112.stdout:Running transaction check 2024-03-19T01:23:03.361 INFO:teuthology.orchestra.run.smithi112.stdout:Transaction check succeeded. 2024-03-19T01:23:03.361 INFO:teuthology.orchestra.run.smithi112.stdout:Running transaction test 2024-03-19T01:23:05.137 INFO:teuthology.orchestra.run.smithi112.stdout:Transaction test succeeded. 2024-03-19T01:23:05.138 INFO:teuthology.orchestra.run.smithi112.stdout:Running transaction 2024-03-19T01:23:11.979 INFO:teuthology.orchestra.run.smithi112.stdout: Preparing : 1/1 2024-03-19T01:23:12.088 INFO:teuthology.orchestra.run.smithi112.stdout: Upgrading : openssl-libs-1:3.0.7-27.el9.x86_64 1/198 2024-03-19T01:23:12.122 INFO:teuthology.orchestra.run.smithi112.stdout: Upgrading : libuuid-2.37.4-18.el9.x86_64 2/198 2024-03-19T01:23:12.162 INFO:teuthology.orchestra.run.smithi112.stdout: Upgrading : libblkid-2.37.4-18.el9.x86_64 3/198 2024-03-19T01:23:12.170 INFO:teuthology.orchestra.run.smithi112.stdout: Running scriptlet: libblkid-2.37.4-18.el9.x86_64 3/198 2024-03-19T01:23:12.211 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 4/198 2024-03-19T01:23:12.264 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 5/198 2024-03-19T01:23:12.298 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : thrift-0.15.0-2.el9.x86_64 6/198 2024-03-19T01:23:12.332 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 7/198 2024-03-19T01:23:12.390 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 8/198 2024-03-19T01:23:12.768 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 9/198 2024-03-19T01:23:12.800 INFO:teuthology.orchestra.run.smithi112.stdout: Upgrading : librados2-2:19.0.0-2206.gb6ae2491.el9.x86_64 10/198 2024-03-19T01:23:12.904 INFO:teuthology.orchestra.run.smithi112.stdout: Running scriptlet: librados2-2:19.0.0-2206.gb6ae2491.el9.x86_64 10/198 2024-03-19T01:23:12.926 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : libcephfs2-2:19.0.0-2206.gb6ae2491.el9.x86_64 11/198 2024-03-19T01:23:13.017 INFO:teuthology.orchestra.run.smithi112.stdout: Running scriptlet: libcephfs2-2:19.0.0-2206.gb6ae2491.el9.x86_64 11/198 2024-03-19T01:23:13.046 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-rados-2:19.0.0-2206.gb6ae2491.el9.x86_64 12/198 2024-03-19T01:23:13.081 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : postgresql-private-libs-13.11-1.el9.x86_64 13/198 2024-03-19T01:23:13.261 INFO:teuthology.orchestra.run.smithi112.stdout: Upgrading : elfutils-libelf-0.190-2.el9.x86_64 14/198 2024-03-19T01:23:13.319 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : postgresql-13.11-1.el9.x86_64 15/198 2024-03-19T01:23:13.347 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : libcephsqlite-2:19.0.0-2206.gb6ae2491.el9.x86_64 16/198 2024-03-19T01:23:13.450 INFO:teuthology.orchestra.run.smithi112.stdout: Running scriptlet: libcephsqlite-2:19.0.0-2206.gb6ae2491.el9.x86_64 16/198 2024-03-19T01:23:13.472 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : libradosstriper1-2:19.0.0-2206.gb6ae2491.el9.x86 17/198 2024-03-19T01:23:13.518 INFO:teuthology.orchestra.run.smithi112.stdout: Running scriptlet: libradosstriper1-2:19.0.0-2206.gb6ae2491.el9.x86 17/198 2024-03-19T01:23:13.575 INFO:teuthology.orchestra.run.smithi112.stdout: Upgrading : libmount-2.37.4-18.el9.x86_64 18/198 2024-03-19T01:23:13.598 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : qatlib-service-23.11.0-1.el9.x86_64 19/198 2024-03-19T01:23:13.613 INFO:teuthology.orchestra.run.smithi112.stdout: Running scriptlet: qatlib-service-23.11.0-1.el9.x86_64 19/198 2024-03-19T01:23:13.805 INFO:teuthology.orchestra.run.smithi112.stdout: Running scriptlet: qatlib-23.11.0-1.el9.x86_64 20/198 2024-03-19T01:23:13.845 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : qatlib-23.11.0-1.el9.x86_64 20/198 2024-03-19T01:23:13.869 INFO:teuthology.orchestra.run.smithi112.stdout: Upgrading : libsmartcols-2.37.4-18.el9.x86_64 21/198 2024-03-19T01:23:13.965 INFO:teuthology.orchestra.run.smithi112.stdout: Upgrading : libacl-2.3.1-4.el9.x86_64 22/198 2024-03-19T01:23:14.063 INFO:teuthology.orchestra.run.smithi112.stdout: Upgrading : NetworkManager-libnm-1:1.47.2-1.el9.x86_64 23/198 2024-03-19T01:23:14.106 INFO:teuthology.orchestra.run.smithi112.stdout: Running scriptlet: NetworkManager-libnm-1:1.47.2-1.el9.x86_64 23/198 2024-03-19T01:23:14.177 INFO:teuthology.orchestra.run.smithi112.stdout: Running scriptlet: NetworkManager-1:1.47.2-1.el9.x86_64 24/198 2024-03-19T01:23:14.240 INFO:teuthology.orchestra.run.smithi112.stdout: Upgrading : NetworkManager-1:1.47.2-1.el9.x86_64 24/198 2024-03-19T01:23:14.436 INFO:teuthology.orchestra.run.smithi112.stdout: Running scriptlet: NetworkManager-1:1.47.2-1.el9.x86_64 24/198 2024-03-19T01:23:14.545 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : re2-1:20211101-3.el9.x86_64 25/198 2024-03-19T01:23:14.642 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : libarrow-9.0.0-10.el9.x86_64 26/198 2024-03-19T01:23:14.678 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 27/198 2024-03-19T01:23:14.705 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : liboath-2.6.7-2.el9.x86_64 28/198 2024-03-19T01:23:14.740 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : libquadmath-11.4.1-3.el9.x86_64 29/198 2024-03-19T01:23:14.816 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : libgfortran-11.4.1-3.el9.x86_64 30/198 2024-03-19T01:23:14.868 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-urllib3-1.26.5-5.el9.noarch 31/198 2024-03-19T01:23:14.907 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 32/198 2024-03-19T01:23:14.972 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 33/198 2024-03-19T01:23:15.006 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 34/198 2024-03-19T01:23:15.049 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 35/198 2024-03-19T01:23:15.082 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : protobuf-3.14.0-13.el9.x86_64 36/198 2024-03-19T01:23:15.110 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : lua-5.4.4-4.el9.x86_64 37/198 2024-03-19T01:23:15.179 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 38/198 2024-03-19T01:23:15.215 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-ceph-common-2:19.0.0-2206.gb6ae2491.el9. 39/198 2024-03-19T01:23:15.258 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-ceph-argparse-2:19.0.0-2206.gb6ae2491.el 40/198 2024-03-19T01:23:15.287 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-cephfs-2:19.0.0-2206.gb6ae2491.el9.x86_6 41/198 2024-03-19T01:23:15.357 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : lua-devel-5.4.4-4.el9.x86_64 42/198 2024-03-19T01:23:15.410 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : luarocks-3.9.2-1.el9.noarch 43/198 2024-03-19T01:23:15.484 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : protobuf-compiler-3.14.0-13.el9.x86_64 44/198 2024-03-19T01:23:15.616 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 45/198 2024-03-19T01:23:15.670 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 46/198 2024-03-19T01:23:16.186 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-rsa-4.9-2.el9.noarch 47/198 2024-03-19T01:23:16.267 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 48/198 2024-03-19T01:23:16.351 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-jinja2-2.11.3-5.el9.noarch 49/198 2024-03-19T01:23:16.402 INFO:teuthology.orchestra.run.smithi112.stdout: Upgrading : util-linux-core-2.37.4-18.el9.x86_64 50/198 2024-03-19T01:23:16.409 INFO:teuthology.orchestra.run.smithi112.stdout: Running scriptlet: util-linux-core-2.37.4-18.el9.x86_64 50/198 2024-03-19T01:23:16.442 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : qatzip-libs-1.1.2-2.el9.x86_64 51/198 2024-03-19T01:23:16.491 INFO:teuthology.orchestra.run.smithi112.stdout: Upgrading : elfutils-libs-0.190-2.el9.x86_64 52/198 2024-03-19T01:23:16.529 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : librados-devel-2:19.0.0-2206.gb6ae2491.el9.x86_6 53/198 2024-03-19T01:23:16.559 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 54/198 2024-03-19T01:23:16.780 INFO:teuthology.orchestra.run.smithi112.stdout: Upgrading : libfdisk-2.37.4-18.el9.x86_64 55/198 2024-03-19T01:23:16.975 INFO:teuthology.orchestra.run.smithi112.stdout: Upgrading : util-linux-2.37.4-18.el9.x86_64 56/198 2024-03-19T01:23:17.044 INFO:teuthology.orchestra.run.smithi112.stdout: Upgrading : xfsprogs-6.3.0-1.el9.x86_64 57/198 2024-03-19T01:23:17.088 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : libuuid-devel-2.37.4-18.el9.x86_64 58/198 2024-03-19T01:23:17.222 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : socat-1.7.4.1-5.el9.x86_64 59/198 2024-03-19T01:23:17.291 INFO:teuthology.orchestra.run.smithi112.stdout: Upgrading : openssl-1:3.0.7-27.el9.x86_64 60/198 2024-03-19T01:23:17.346 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 61/198 2024-03-19T01:23:17.444 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 62/198 2024-03-19T01:23:17.475 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-webob-1.8.7-6.el9.noarch 63/198 2024-03-19T01:23:17.512 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-typing-extensions-4.4.0-2.el9.noarch 64/198 2024-03-19T01:23:17.558 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 65/198 2024-03-19T01:23:17.594 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 66/198 2024-03-19T01:23:17.637 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-pyperclip-1.8.2-2.el9.noarch 67/198 2024-03-19T01:23:17.680 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 68/198 2024-03-19T01:23:17.812 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 69/198 2024-03-19T01:23:17.851 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 70/198 2024-03-19T01:23:17.881 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-colorama-0.4.4-12.el9.noarch 71/198 2024-03-19T01:23:17.922 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-certifi-2021.10.8-2.el9.noarch 72/198 2024-03-19T01:23:18.017 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 73/198 2024-03-19T01:23:18.056 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-google-auth-1:2.28.0-1.el9.noarch 74/198 2024-03-19T01:23:18.096 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 75/198 2024-03-19T01:23:18.122 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : gperftools-libs-2.9.1-2.el9.x86_64 76/198 2024-03-19T01:23:18.146 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : libarrow-doc-9.0.0-10.el9.noarch 77/198 2024-03-19T01:23:18.218 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : grpc-data-1.46.7-10.el9.noarch 78/198 2024-03-19T01:23:18.348 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : abseil-cpp-20211102.0-3.el9.x86_64 79/198 2024-03-19T01:23:18.456 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-grpcio-1.46.7-10.el9.x86_64 80/198 2024-03-19T01:23:18.496 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-devel-3.9.17-2.el9.x86_64 81/198 2024-03-19T01:23:18.545 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-ply-3.11-14.el9.noarch 82/198 2024-03-19T01:23:18.600 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 83/198 2024-03-19T01:23:18.793 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 84/198 2024-03-19T01:23:18.842 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 85/198 2024-03-19T01:23:18.980 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 86/198 2024-03-19T01:23:19.015 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-asyncssh-2.13.2-4.el9.noarch 87/198 2024-03-19T01:23:19.046 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 88/198 2024-03-19T01:23:19.077 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : fuse-2.9.9-15.el9.x86_64 89/198 2024-03-19T01:23:19.106 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : zlib-devel-1.2.11-40.el9.x86_64 90/198 2024-03-19T01:23:19.140 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 91/198 2024-03-19T01:23:19.187 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : uuid-1.6.2-55.el9.x86_64 92/198 2024-03-19T01:23:19.237 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-wcwidth-0.2.5-8.el9.noarch 93/198 2024-03-19T01:23:19.265 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 94/198 2024-03-19T01:23:19.342 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 95/198 2024-03-19T01:23:19.376 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 96/198 2024-03-19T01:23:19.409 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 97/198 2024-03-19T01:23:19.442 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 98/198 2024-03-19T01:23:19.476 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 99/198 2024-03-19T01:23:19.566 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 100/198 2024-03-19T01:23:19.648 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 101/198 2024-03-19T01:23:19.691 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-protobuf-3.14.0-13.el9.noarch 102/198 2024-03-19T01:23:19.722 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-grpcio-tools-1.46.7-10.el9.x86_64 103/198 2024-03-19T01:23:19.770 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 104/198 2024-03-19T01:23:19.867 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-packaging-20.9-5.el9.noarch 105/198 2024-03-19T01:23:19.925 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 106/198 2024-03-19T01:23:20.671 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 107/198 2024-03-19T01:23:20.729 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 108/198 2024-03-19T01:23:20.762 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 109/198 2024-03-19T01:23:20.811 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 110/198 2024-03-19T01:23:20.899 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-attrs-20.3.0-7.el9.noarch 111/198 2024-03-19T01:23:20.938 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-cmd2-2.4.2-2.el9.noarch 112/198 2024-03-19T01:23:21.117 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : openblas-0.3.21-2.el9.x86_64 113/198 2024-03-19T01:23:21.144 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : openblas-openmp-0.3.21-2.el9.x86_64 114/198 2024-03-19T01:23:21.222 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 115/198 2024-03-19T01:23:21.870 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 116/198 2024-03-19T01:23:21.975 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 117/198 2024-03-19T01:23:23.211 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 118/198 2024-03-19T01:23:23.294 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 119/198 2024-03-19T01:23:23.325 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : oniguruma-6.9.6-1.el9.5.x86_64 120/198 2024-03-19T01:23:23.356 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : jq-1.6-16.el9.x86_64 121/198 2024-03-19T01:23:23.384 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : libzstd-devel-1.5.1-2.el9.x86_64 122/198 2024-03-19T01:23:23.695 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 123/198 2024-03-19T01:23:23.730 INFO:teuthology.orchestra.run.smithi112.stdout: Upgrading : librbd1-2:19.0.0-2206.gb6ae2491.el9.x86_64 124/198 2024-03-19T01:23:23.821 INFO:teuthology.orchestra.run.smithi112.stdout: Running scriptlet: librbd1-2:19.0.0-2206.gb6ae2491.el9.x86_64 124/198 2024-03-19T01:23:23.851 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-rbd-2:19.0.0-2206.gb6ae2491.el9.x86_64 125/198 2024-03-19T01:23:23.887 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : boost-program-options-1.75.0-8.el9.x86_64 126/198 2024-03-19T01:23:24.346 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : parquet-libs-9.0.0-10.el9.x86_64 127/198 2024-03-19T01:23:24.375 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : librgw2-2:19.0.0-2206.gb6ae2491.el9.x86_64 128/198 2024-03-19T01:23:24.445 INFO:teuthology.orchestra.run.smithi112.stdout: Running scriptlet: librgw2-2:19.0.0-2206.gb6ae2491.el9.x86_64 128/198 2024-03-19T01:23:24.468 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : python3-rgw-2:19.0.0-2206.gb6ae2491.el9.x86_64 129/198 2024-03-19T01:23:26.719 INFO:teuthology.orchestra.run.smithi112.stdout: Running scriptlet: ceph-common-2:19.0.0-2206.gb6ae2491.el9.x86_64 130/198 2024-03-19T01:23:26.764 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : ceph-common-2:19.0.0-2206.gb6ae2491.el9.x86_64 130/198 2024-03-19T01:23:27.301 INFO:teuthology.orchestra.run.smithi112.stdout: Running scriptlet: ceph-common-2:19.0.0-2206.gb6ae2491.el9.x86_64 130/198 2024-03-19T01:23:27.342 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : ceph-base-2:19.0.0-2206.gb6ae2491.el9.x86_64 131/198 2024-03-19T01:23:27.424 INFO:teuthology.orchestra.run.smithi112.stdout: Running scriptlet: ceph-base-2:19.0.0-2206.gb6ae2491.el9.x86_64 131/198 2024-03-19T01:23:27.424 INFO:teuthology.orchestra.run.smithi112.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-03-19T01:23:27.424 INFO:teuthology.orchestra.run.smithi112.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-03-19T01:23:27.425 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-03-19T01:23:27.452 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : ceph-selinux-2:19.0.0-2206.gb6ae2491.el9.x86_64 132/198 2024-03-19T01:23:39.024 INFO:teuthology.orchestra.run.smithi112.stdout: Running scriptlet: ceph-selinux-2:19.0.0-2206.gb6ae2491.el9.x86_64 132/198 2024-03-19T01:23:39.025 INFO:teuthology.orchestra.run.smithi112.stdout:skipping the directory /sys 2024-03-19T01:23:39.025 INFO:teuthology.orchestra.run.smithi112.stdout:skipping the directory /proc 2024-03-19T01:23:39.025 INFO:teuthology.orchestra.run.smithi112.stdout:skipping the directory /mnt 2024-03-19T01:23:39.025 INFO:teuthology.orchestra.run.smithi112.stdout:skipping the directory /var/tmp 2024-03-19T01:23:39.025 INFO:teuthology.orchestra.run.smithi112.stdout:skipping the directory /home 2024-03-19T01:23:39.025 INFO:teuthology.orchestra.run.smithi112.stdout:skipping the directory /root 2024-03-19T01:23:39.025 INFO:teuthology.orchestra.run.smithi112.stdout:skipping the directory /tmp 2024-03-19T01:23:39.025 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-03-19T01:23:39.211 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : ceph-volume-2:19.0.0-2206.gb6ae2491.el9.noarch 133/198 2024-03-19T01:23:39.222 INFO:teuthology.orchestra.run.smithi112.stdout: Running scriptlet: ceph-volume-2:19.0.0-2206.gb6ae2491.el9.noarch 133/198 2024-03-19T01:23:39.222 INFO:teuthology.orchestra.run.smithi112.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-19T01:23:39.222 INFO:teuthology.orchestra.run.smithi112.stdout:Invalid unit name "ceph-volume@*.service" escaped as "ceph-volume@\x2a.service". 2024-03-19T01:23:39.222 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-03-19T01:23:40.685 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : ceph-osd-2:19.0.0-2206.gb6ae2491.el9.x86_64 134/198 2024-03-19T01:23:40.710 INFO:teuthology.orchestra.run.smithi112.stdout: Running scriptlet: ceph-osd-2:19.0.0-2206.gb6ae2491.el9.x86_64 134/198 2024-03-19T01:23:40.710 INFO:teuthology.orchestra.run.smithi112.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-19T01:23:40.710 INFO:teuthology.orchestra.run.smithi112.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-03-19T01:23:40.710 INFO:teuthology.orchestra.run.smithi112.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-03-19T01:23:40.710 INFO:teuthology.orchestra.run.smithi112.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-03-19T01:23:40.711 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-03-19T01:23:40.936 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : ceph-mds-2:19.0.0-2206.gb6ae2491.el9.x86_64 135/198 2024-03-19T01:23:40.958 INFO:teuthology.orchestra.run.smithi112.stdout: Running scriptlet: ceph-mds-2:19.0.0-2206.gb6ae2491.el9.x86_64 135/198 2024-03-19T01:23:40.958 INFO:teuthology.orchestra.run.smithi112.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-19T01:23:40.958 INFO:teuthology.orchestra.run.smithi112.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-03-19T01:23:40.958 INFO:teuthology.orchestra.run.smithi112.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-03-19T01:23:40.959 INFO:teuthology.orchestra.run.smithi112.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-03-19T01:23:40.959 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-03-19T01:23:41.403 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : ceph-mon-2:19.0.0-2206.gb6ae2491.el9.x86_64 136/198 2024-03-19T01:23:41.426 INFO:teuthology.orchestra.run.smithi112.stdout: Running scriptlet: ceph-mon-2:19.0.0-2206.gb6ae2491.el9.x86_64 136/198 2024-03-19T01:23:41.426 INFO:teuthology.orchestra.run.smithi112.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-19T01:23:41.426 INFO:teuthology.orchestra.run.smithi112.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-03-19T01:23:41.426 INFO:teuthology.orchestra.run.smithi112.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-03-19T01:23:41.426 INFO:teuthology.orchestra.run.smithi112.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-03-19T01:23:41.426 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-03-19T01:23:41.513 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : automake-1.16.2-8.el9.noarch 137/198 2024-03-19T01:23:41.981 INFO:teuthology.orchestra.run.smithi112.stdout: Running scriptlet: cephadm-2:19.0.0-2206.gb6ae2491.el9.noarch 138/198 2024-03-19T01:23:42.011 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : cephadm-2:19.0.0-2206.gb6ae2491.el9.noarch 138/198 2024-03-19T01:23:42.043 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : ceph-prometheus-alerts-2:19.0.0-2206.gb6ae2491.e 139/198 2024-03-19T01:23:42.103 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : ceph-grafana-dashboards-2:19.0.0-2206.gb6ae2491. 140/198 2024-03-19T01:23:42.134 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : ceph-mgr-cephadm-2:19.0.0-2206.gb6ae2491.el9.noa 141/198 2024-03-19T01:23:42.367 INFO:teuthology.orchestra.run.smithi112.stdout: Running scriptlet: ceph-mgr-cephadm-2:19.0.0-2206.gb6ae2491.el9.noa 141/198 2024-03-19T01:23:42.402 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : ceph-mgr-dashboard-2:19.0.0-2206.gb6ae2491.el9.n 142/198 2024-03-19T01:23:43.363 INFO:teuthology.orchestra.run.smithi112.stdout: Running scriptlet: ceph-mgr-dashboard-2:19.0.0-2206.gb6ae2491.el9.n 142/198 2024-03-19T01:23:43.402 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : ceph-mgr-diskprediction-local-2:19.0.0-2206.gb6a 143/198 2024-03-19T01:23:43.502 INFO:teuthology.orchestra.run.smithi112.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:19.0.0-2206.gb6a 143/198 2024-03-19T01:23:43.674 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : ceph-mgr-modules-core-2:19.0.0-2206.gb6ae2491.el 144/198 2024-03-19T01:23:43.704 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : ceph-mgr-2:19.0.0-2206.gb6ae2491.el9.x86_64 145/198 2024-03-19T01:23:43.727 INFO:teuthology.orchestra.run.smithi112.stdout: Running scriptlet: ceph-mgr-2:19.0.0-2206.gb6ae2491.el9.x86_64 145/198 2024-03-19T01:23:43.727 INFO:teuthology.orchestra.run.smithi112.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-19T01:23:43.727 INFO:teuthology.orchestra.run.smithi112.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-03-19T01:23:43.727 INFO:teuthology.orchestra.run.smithi112.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-03-19T01:23:43.728 INFO:teuthology.orchestra.run.smithi112.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-03-19T01:23:43.728 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-03-19T01:23:43.771 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : ceph-mgr-rook-2:19.0.0-2206.gb6ae2491.el9.noarch 146/198 2024-03-19T01:23:43.782 INFO:teuthology.orchestra.run.smithi112.stdout: Running scriptlet: ceph-mgr-rook-2:19.0.0-2206.gb6ae2491.el9.noarch 146/198 2024-03-19T01:23:43.845 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : ceph-2:19.0.0-2206.gb6ae2491.el9.x86_64 147/198 2024-03-19T01:23:44.713 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : libtool-2.4.6-45.el9.x86_64 148/198 2024-03-19T01:23:44.740 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : ceph-radosgw-2:19.0.0-2206.gb6ae2491.el9.x86_64 149/198 2024-03-19T01:23:44.763 INFO:teuthology.orchestra.run.smithi112.stdout: Running scriptlet: ceph-radosgw-2:19.0.0-2206.gb6ae2491.el9.x86_64 149/198 2024-03-19T01:23:44.763 INFO:teuthology.orchestra.run.smithi112.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-19T01:23:44.763 INFO:teuthology.orchestra.run.smithi112.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-03-19T01:23:44.763 INFO:teuthology.orchestra.run.smithi112.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-03-19T01:23:44.763 INFO:teuthology.orchestra.run.smithi112.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-03-19T01:23:44.763 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-03-19T01:23:44.816 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : ceph-immutable-object-cache-2:19.0.0-2206.gb6ae2 150/198 2024-03-19T01:23:44.837 INFO:teuthology.orchestra.run.smithi112.stdout: Running scriptlet: ceph-immutable-object-cache-2:19.0.0-2206.gb6ae2 150/198 2024-03-19T01:23:44.838 INFO:teuthology.orchestra.run.smithi112.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-19T01:23:44.838 INFO:teuthology.orchestra.run.smithi112.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-03-19T01:23:44.838 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-03-19T01:23:44.883 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : cephfs-mirror-2:19.0.0-2206.gb6ae2491.el9.x86_64 151/198 2024-03-19T01:23:44.905 INFO:teuthology.orchestra.run.smithi112.stdout: Running scriptlet: cephfs-mirror-2:19.0.0-2206.gb6ae2491.el9.x86_64 151/198 2024-03-19T01:23:44.905 INFO:teuthology.orchestra.run.smithi112.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-19T01:23:44.905 INFO:teuthology.orchestra.run.smithi112.stdout:Invalid unit name "cephfs-mirror@*.service" escaped as "cephfs-mirror@\x2a.service". 2024-03-19T01:23:44.905 INFO:teuthology.orchestra.run.smithi112.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/cephfs-mirror.target → /usr/lib/systemd/system/cephfs-mirror.target. 2024-03-19T01:23:44.905 INFO:teuthology.orchestra.run.smithi112.stdout:Created symlink /etc/systemd/system/ceph.target.wants/cephfs-mirror.target → /usr/lib/systemd/system/cephfs-mirror.target. 2024-03-19T01:23:44.906 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-03-19T01:23:45.209 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : rbd-mirror-2:19.0.0-2206.gb6ae2491.el9.x86_64 152/198 2024-03-19T01:23:45.231 INFO:teuthology.orchestra.run.smithi112.stdout: Running scriptlet: rbd-mirror-2:19.0.0-2206.gb6ae2491.el9.x86_64 152/198 2024-03-19T01:23:45.231 INFO:teuthology.orchestra.run.smithi112.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-19T01:23:45.231 INFO:teuthology.orchestra.run.smithi112.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-03-19T01:23:45.231 INFO:teuthology.orchestra.run.smithi112.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-03-19T01:23:45.231 INFO:teuthology.orchestra.run.smithi112.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-03-19T01:23:45.231 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-03-19T01:23:49.671 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : ceph-test-2:19.0.0-2206.gb6ae2491.el9.x86_64 153/198 2024-03-19T01:23:49.712 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : rbd-fuse-2:19.0.0-2206.gb6ae2491.el9.x86_64 154/198 2024-03-19T01:23:49.743 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : rbd-nbd-2:19.0.0-2206.gb6ae2491.el9.x86_64 155/198 2024-03-19T01:23:49.762 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : elfutils-libelf-devel-0.190-2.el9.x86_64 156/198 2024-03-19T01:23:49.943 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : cephfs-shell-2:19.0.0-2206.gb6ae2491.el9.x86_64 157/198 2024-03-19T01:23:50.053 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : postgresql-contrib-13.11-1.el9.x86_64 158/198 2024-03-19T01:23:50.093 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : ceph-fuse-2:19.0.0-2206.gb6ae2491.el9.x86_64 159/198 2024-03-19T01:23:50.133 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : xfsprogs-devel-6.3.0-1.el9.x86_64 160/198 2024-03-19T01:23:50.176 INFO:teuthology.orchestra.run.smithi112.stdout: Upgrading : xfsdump-3.1.12-4.el9.x86_64 161/198 2024-03-19T01:23:50.220 INFO:teuthology.orchestra.run.smithi112.stdout: Upgrading : util-linux-user-2.37.4-18.el9.x86_64 162/198 2024-03-19T01:23:50.252 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : libcephfs-devel-2:19.0.0-2206.gb6ae2491.el9.x86_ 163/198 2024-03-19T01:23:50.293 INFO:teuthology.orchestra.run.smithi112.stdout: Upgrading : elfutils-debuginfod-client-0.190-2.el9.x86_64 164/198 2024-03-19T01:23:50.322 INFO:teuthology.orchestra.run.smithi112.stdout: Upgrading : NetworkManager-team-1:1.47.2-1.el9.x86_64 165/198 2024-03-19T01:23:50.377 INFO:teuthology.orchestra.run.smithi112.stdout: Upgrading : NetworkManager-tui-1:1.47.2-1.el9.x86_64 166/198 2024-03-19T01:23:50.446 INFO:teuthology.orchestra.run.smithi112.stdout: Upgrading : acl-2.3.1-4.el9.x86_64 167/198 2024-03-19T01:23:50.487 INFO:teuthology.orchestra.run.smithi112.stdout: Upgrading : libacl-devel-2.3.1-4.el9.x86_64 168/198 2024-03-19T01:23:51.090 INFO:teuthology.orchestra.run.smithi112.stdout: Running scriptlet: postgresql-server-13.11-1.el9.x86_64 169/198 2024-03-19T01:23:51.135 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : postgresql-server-13.11-1.el9.x86_64 169/198 2024-03-19T01:23:51.154 INFO:teuthology.orchestra.run.smithi112.stdout: Running scriptlet: postgresql-server-13.11-1.el9.x86_64 169/198 2024-03-19T01:23:51.978 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : cephfs-top-2:19.0.0-2206.gb6ae2491.el9.noarch 170/198 2024-03-19T01:23:52.071 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : openssl-devel-1:3.0.7-27.el9.x86_64 171/198 2024-03-19T01:23:52.112 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : libaio-devel-0.3.111-13.el9.x86_64 172/198 2024-03-19T01:23:52.193 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : flex-2.6.4-9.el9.x86_64 173/198 2024-03-19T01:23:52.225 INFO:teuthology.orchestra.run.smithi112.stdout: Installing : bison-3.7.4-5.el9.x86_64 174/198 2024-03-19T01:23:52.250 INFO:teuthology.orchestra.run.smithi112.stdout: Cleanup : NetworkManager-tui-1:1.44.0-2.el9.x86_64 175/198 2024-03-19T01:23:52.275 INFO:teuthology.orchestra.run.smithi112.stdout: Cleanup : elfutils-debuginfod-client-0.189-3.el9.x86_64 176/198 2024-03-19T01:23:52.302 INFO:teuthology.orchestra.run.smithi112.stdout: Cleanup : elfutils-libs-0.189-3.el9.x86_64 177/198 2024-03-19T01:23:52.302 INFO:teuthology.orchestra.run.smithi112.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 178/198 2024-03-19T01:23:52.373 INFO:teuthology.orchestra.run.smithi112.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 178/198 2024-03-19T01:23:52.402 INFO:teuthology.orchestra.run.smithi112.stdout: Cleanup : openssl-1:3.0.7-24.el9.x86_64 179/198 2024-03-19T01:23:52.402 INFO:teuthology.orchestra.run.smithi112.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 180/198 2024-03-19T01:23:52.479 INFO:teuthology.orchestra.run.smithi112.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 180/198 2024-03-19T01:23:52.505 INFO:teuthology.orchestra.run.smithi112.stdout: Cleanup : xfsdump-3.1.12-3.el9.x86_64 181/198 2024-03-19T01:23:52.538 INFO:teuthology.orchestra.run.smithi112.stdout: Cleanup : xfsprogs-5.19.0-4.el9.x86_64 182/198 2024-03-19T01:23:52.563 INFO:teuthology.orchestra.run.smithi112.stdout: Cleanup : acl-2.3.1-3.el9.x86_64 183/198 2024-03-19T01:23:52.588 INFO:teuthology.orchestra.run.smithi112.stdout: Cleanup : libacl-devel-2.3.1-3.el9.x86_64 184/198 2024-03-19T01:23:52.621 INFO:teuthology.orchestra.run.smithi112.stdout: Cleanup : util-linux-user-2.37.4-15.el9.x86_64 185/198 2024-03-19T01:23:52.660 INFO:teuthology.orchestra.run.smithi112.stdout: Cleanup : util-linux-2.37.4-15.el9.x86_64 186/198 2024-03-19T01:23:52.693 INFO:teuthology.orchestra.run.smithi112.stdout: Cleanup : util-linux-core-2.37.4-15.el9.x86_64 187/198 2024-03-19T01:23:52.718 INFO:teuthology.orchestra.run.smithi112.stdout: Cleanup : libfdisk-2.37.4-15.el9.x86_64 188/198 2024-03-19T01:23:52.743 INFO:teuthology.orchestra.run.smithi112.stdout: Cleanup : libmount-2.37.4-15.el9.x86_64 189/198 2024-03-19T01:23:52.768 INFO:teuthology.orchestra.run.smithi112.stdout: Cleanup : libblkid-2.37.4-15.el9.x86_64 190/198 2024-03-19T01:23:52.794 INFO:teuthology.orchestra.run.smithi112.stdout: Cleanup : NetworkManager-team-1:1.44.0-2.el9.x86_64 191/198 2024-03-19T01:23:52.800 INFO:teuthology.orchestra.run.smithi112.stdout: Running scriptlet: NetworkManager-1:1.44.0-2.el9.x86_64 192/198 2024-03-19T01:23:52.800 INFO:teuthology.orchestra.run.smithi112.stdout: Cleanup : NetworkManager-1:1.44.0-2.el9.x86_64 192/198 2024-03-19T01:23:52.980 INFO:teuthology.orchestra.run.smithi112.stdout: Running scriptlet: NetworkManager-1:1.44.0-2.el9.x86_64 192/198 2024-03-19T01:23:52.980 INFO:teuthology.orchestra.run.smithi112.stdout: Cleanup : NetworkManager-libnm-1:1.44.0-2.el9.x86_64 193/198 2024-03-19T01:23:53.047 INFO:teuthology.orchestra.run.smithi112.stdout: Running scriptlet: NetworkManager-libnm-1:1.44.0-2.el9.x86_64 193/198 2024-03-19T01:23:53.071 INFO:teuthology.orchestra.run.smithi112.stdout: Cleanup : libuuid-2.37.4-15.el9.x86_64 194/198 2024-03-19T01:23:53.096 INFO:teuthology.orchestra.run.smithi112.stdout: Cleanup : libsmartcols-2.37.4-15.el9.x86_64 195/198 2024-03-19T01:23:53.130 INFO:teuthology.orchestra.run.smithi112.stdout: Cleanup : libacl-2.3.1-3.el9.x86_64 196/198 2024-03-19T01:23:53.156 INFO:teuthology.orchestra.run.smithi112.stdout: Cleanup : openssl-libs-1:3.0.7-24.el9.x86_64 197/198 2024-03-19T01:23:53.219 INFO:teuthology.orchestra.run.smithi112.stdout: Cleanup : elfutils-libelf-0.189-3.el9.x86_64 198/198 2024-03-19T01:23:58.187 INFO:teuthology.orchestra.run.smithi112.stdout: Running scriptlet: elfutils-libelf-0.189-3.el9.x86_64 198/198 2024-03-19T01:23:58.187 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : ceph-2:19.0.0-2206.gb6ae2491.el9.x86_64 1/198 2024-03-19T01:23:58.187 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : ceph-base-2:19.0.0-2206.gb6ae2491.el9.x86_64 2/198 2024-03-19T01:23:58.188 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : ceph-common-2:19.0.0-2206.gb6ae2491.el9.x86_64 3/198 2024-03-19T01:23:58.188 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : ceph-fuse-2:19.0.0-2206.gb6ae2491.el9.x86_64 4/198 2024-03-19T01:23:58.188 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : ceph-immutable-object-cache-2:19.0.0-2206.gb6ae2 5/198 2024-03-19T01:23:58.188 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : ceph-mds-2:19.0.0-2206.gb6ae2491.el9.x86_64 6/198 2024-03-19T01:23:58.188 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : ceph-mgr-2:19.0.0-2206.gb6ae2491.el9.x86_64 7/198 2024-03-19T01:23:58.188 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : ceph-mon-2:19.0.0-2206.gb6ae2491.el9.x86_64 8/198 2024-03-19T01:23:58.188 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : ceph-osd-2:19.0.0-2206.gb6ae2491.el9.x86_64 9/198 2024-03-19T01:23:58.188 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : ceph-radosgw-2:19.0.0-2206.gb6ae2491.el9.x86_64 10/198 2024-03-19T01:23:58.188 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : ceph-selinux-2:19.0.0-2206.gb6ae2491.el9.x86_64 11/198 2024-03-19T01:23:58.188 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : ceph-test-2:19.0.0-2206.gb6ae2491.el9.x86_64 12/198 2024-03-19T01:23:58.188 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : cephfs-mirror-2:19.0.0-2206.gb6ae2491.el9.x86_64 13/198 2024-03-19T01:23:58.188 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : cephfs-shell-2:19.0.0-2206.gb6ae2491.el9.x86_64 14/198 2024-03-19T01:23:58.188 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : libcephfs-devel-2:19.0.0-2206.gb6ae2491.el9.x86_ 15/198 2024-03-19T01:23:58.189 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : libcephfs2-2:19.0.0-2206.gb6ae2491.el9.x86_64 16/198 2024-03-19T01:23:58.189 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : libcephsqlite-2:19.0.0-2206.gb6ae2491.el9.x86_64 17/198 2024-03-19T01:23:58.189 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : librados-devel-2:19.0.0-2206.gb6ae2491.el9.x86_6 18/198 2024-03-19T01:23:58.189 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : libradosstriper1-2:19.0.0-2206.gb6ae2491.el9.x86 19/198 2024-03-19T01:23:58.189 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : librgw2-2:19.0.0-2206.gb6ae2491.el9.x86_64 20/198 2024-03-19T01:23:58.189 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-ceph-argparse-2:19.0.0-2206.gb6ae2491.el 21/198 2024-03-19T01:23:58.189 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-ceph-common-2:19.0.0-2206.gb6ae2491.el9. 22/198 2024-03-19T01:23:58.189 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-cephfs-2:19.0.0-2206.gb6ae2491.el9.x86_6 23/198 2024-03-19T01:23:58.189 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-rados-2:19.0.0-2206.gb6ae2491.el9.x86_64 24/198 2024-03-19T01:23:58.189 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-rbd-2:19.0.0-2206.gb6ae2491.el9.x86_64 25/198 2024-03-19T01:23:58.189 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-rgw-2:19.0.0-2206.gb6ae2491.el9.x86_64 26/198 2024-03-19T01:23:58.189 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : rbd-fuse-2:19.0.0-2206.gb6ae2491.el9.x86_64 27/198 2024-03-19T01:23:58.205 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : rbd-mirror-2:19.0.0-2206.gb6ae2491.el9.x86_64 28/198 2024-03-19T01:23:58.205 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : rbd-nbd-2:19.0.0-2206.gb6ae2491.el9.x86_64 29/198 2024-03-19T01:23:58.205 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : ceph-grafana-dashboards-2:19.0.0-2206.gb6ae2491. 30/198 2024-03-19T01:23:58.205 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : ceph-mgr-cephadm-2:19.0.0-2206.gb6ae2491.el9.noa 31/198 2024-03-19T01:23:58.206 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : ceph-mgr-dashboard-2:19.0.0-2206.gb6ae2491.el9.n 32/198 2024-03-19T01:23:58.206 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : ceph-mgr-diskprediction-local-2:19.0.0-2206.gb6a 33/198 2024-03-19T01:23:58.206 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : ceph-mgr-modules-core-2:19.0.0-2206.gb6ae2491.el 34/198 2024-03-19T01:23:58.206 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : ceph-mgr-rook-2:19.0.0-2206.gb6ae2491.el9.noarch 35/198 2024-03-19T01:23:58.206 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : ceph-prometheus-alerts-2:19.0.0-2206.gb6ae2491.e 36/198 2024-03-19T01:23:58.206 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : ceph-volume-2:19.0.0-2206.gb6ae2491.el9.noarch 37/198 2024-03-19T01:23:58.206 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : cephadm-2:19.0.0-2206.gb6ae2491.el9.noarch 38/198 2024-03-19T01:23:58.206 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : cephfs-top-2:19.0.0-2206.gb6ae2491.el9.noarch 39/198 2024-03-19T01:23:58.206 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : automake-1.16.2-8.el9.noarch 40/198 2024-03-19T01:23:58.206 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : bison-3.7.4-5.el9.x86_64 41/198 2024-03-19T01:23:58.206 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : boost-program-options-1.75.0-8.el9.x86_64 42/198 2024-03-19T01:23:58.206 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : flex-2.6.4-9.el9.x86_64 43/198 2024-03-19T01:23:58.206 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 44/198 2024-03-19T01:23:58.207 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 45/198 2024-03-19T01:23:58.207 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 46/198 2024-03-19T01:23:58.207 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : libaio-devel-0.3.111-13.el9.x86_64 47/198 2024-03-19T01:23:58.207 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 48/198 2024-03-19T01:23:58.207 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 49/198 2024-03-19T01:23:58.207 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 50/198 2024-03-19T01:23:58.207 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : libtool-2.4.6-45.el9.x86_64 51/198 2024-03-19T01:23:58.207 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : libzstd-devel-1.5.1-2.el9.x86_64 52/198 2024-03-19T01:23:58.207 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 53/198 2024-03-19T01:23:58.207 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : lua-5.4.4-4.el9.x86_64 54/198 2024-03-19T01:23:58.207 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : oniguruma-6.9.6-1.el9.5.x86_64 55/198 2024-03-19T01:23:58.207 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : openblas-0.3.21-2.el9.x86_64 56/198 2024-03-19T01:23:58.207 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : openblas-openmp-0.3.21-2.el9.x86_64 57/198 2024-03-19T01:23:58.207 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : postgresql-13.11-1.el9.x86_64 58/198 2024-03-19T01:23:58.208 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : postgresql-contrib-13.11-1.el9.x86_64 59/198 2024-03-19T01:23:58.208 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : postgresql-private-libs-13.11-1.el9.x86_64 60/198 2024-03-19T01:23:58.208 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : postgresql-server-13.11-1.el9.x86_64 61/198 2024-03-19T01:23:58.208 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : protobuf-3.14.0-13.el9.x86_64 62/198 2024-03-19T01:23:58.208 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-attrs-20.3.0-7.el9.noarch 63/198 2024-03-19T01:23:58.208 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 64/198 2024-03-19T01:23:58.208 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 65/198 2024-03-19T01:23:58.208 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 66/198 2024-03-19T01:23:58.208 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 67/198 2024-03-19T01:23:58.208 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 68/198 2024-03-19T01:23:58.208 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 69/198 2024-03-19T01:23:58.208 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 70/198 2024-03-19T01:23:58.208 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 71/198 2024-03-19T01:23:58.208 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 72/198 2024-03-19T01:23:58.209 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 73/198 2024-03-19T01:23:58.209 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-protobuf-3.14.0-13.el9.noarch 74/198 2024-03-19T01:23:58.209 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 75/198 2024-03-19T01:23:58.209 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 76/198 2024-03-19T01:23:58.209 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 77/198 2024-03-19T01:23:58.209 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 78/198 2024-03-19T01:23:58.209 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 79/198 2024-03-19T01:23:58.209 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 80/198 2024-03-19T01:23:58.209 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-wcwidth-0.2.5-8.el9.noarch 81/198 2024-03-19T01:23:58.209 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : socat-1.7.4.1-5.el9.x86_64 82/198 2024-03-19T01:23:58.209 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : uuid-1.6.2-55.el9.x86_64 83/198 2024-03-19T01:23:58.209 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 84/198 2024-03-19T01:23:58.209 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : zlib-devel-1.2.11-40.el9.x86_64 85/198 2024-03-19T01:23:58.210 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : fuse-2.9.9-15.el9.x86_64 86/198 2024-03-19T01:23:58.210 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 87/198 2024-03-19T01:23:58.210 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 88/198 2024-03-19T01:23:58.210 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-ply-3.11-14.el9.noarch 89/198 2024-03-19T01:23:58.210 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 90/198 2024-03-19T01:23:58.210 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-urllib3-1.26.5-5.el9.noarch 91/198 2024-03-19T01:23:58.210 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : jq-1.6-16.el9.x86_64 92/198 2024-03-19T01:23:58.210 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : libgfortran-11.4.1-3.el9.x86_64 93/198 2024-03-19T01:23:58.210 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : libquadmath-11.4.1-3.el9.x86_64 94/198 2024-03-19T01:23:58.210 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 95/198 2024-03-19T01:23:58.210 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : elfutils-libelf-devel-0.190-2.el9.x86_64 96/198 2024-03-19T01:23:58.210 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : libuuid-devel-2.37.4-18.el9.x86_64 97/198 2024-03-19T01:23:58.210 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : openssl-devel-1:3.0.7-27.el9.x86_64 98/198 2024-03-19T01:23:58.211 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-devel-3.9.17-2.el9.x86_64 99/198 2024-03-19T01:23:58.211 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-jinja2-2.11.3-5.el9.noarch 100/198 2024-03-19T01:23:58.211 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : qatlib-23.11.0-1.el9.x86_64 101/198 2024-03-19T01:23:58.211 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : qatlib-service-23.11.0-1.el9.x86_64 102/198 2024-03-19T01:23:58.211 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : qatzip-libs-1.1.2-2.el9.x86_64 103/198 2024-03-19T01:23:58.211 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : xfsprogs-devel-6.3.0-1.el9.x86_64 104/198 2024-03-19T01:23:58.211 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : lua-devel-5.4.4-4.el9.x86_64 105/198 2024-03-19T01:23:58.211 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : protobuf-compiler-3.14.0-13.el9.x86_64 106/198 2024-03-19T01:23:58.211 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : abseil-cpp-20211102.0-3.el9.x86_64 107/198 2024-03-19T01:23:58.211 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : gperftools-libs-2.9.1-2.el9.x86_64 108/198 2024-03-19T01:23:58.211 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : grpc-data-1.46.7-10.el9.noarch 109/198 2024-03-19T01:23:58.211 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : libarrow-9.0.0-10.el9.x86_64 110/198 2024-03-19T01:23:58.211 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : libarrow-doc-9.0.0-10.el9.noarch 111/198 2024-03-19T01:23:58.211 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 112/198 2024-03-19T01:23:58.212 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 113/198 2024-03-19T01:23:58.212 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : luarocks-3.9.2-1.el9.noarch 114/198 2024-03-19T01:23:58.212 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : parquet-libs-9.0.0-10.el9.x86_64 115/198 2024-03-19T01:23:58.212 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-asyncssh-2.13.2-4.el9.noarch 116/198 2024-03-19T01:23:58.212 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 117/198 2024-03-19T01:23:58.212 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 118/198 2024-03-19T01:23:58.212 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-certifi-2021.10.8-2.el9.noarch 119/198 2024-03-19T01:23:58.212 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 120/198 2024-03-19T01:23:58.212 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 121/198 2024-03-19T01:23:58.212 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-cmd2-2.4.2-2.el9.noarch 122/198 2024-03-19T01:23:58.212 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-colorama-0.4.4-12.el9.noarch 123/198 2024-03-19T01:23:58.212 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-google-auth-1:2.28.0-1.el9.noarch 124/198 2024-03-19T01:23:58.212 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-grpcio-1.46.7-10.el9.x86_64 125/198 2024-03-19T01:23:58.212 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-grpcio-tools-1.46.7-10.el9.x86_64 126/198 2024-03-19T01:23:58.212 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 127/198 2024-03-19T01:23:58.213 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 128/198 2024-03-19T01:23:58.213 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 129/198 2024-03-19T01:23:58.213 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 130/198 2024-03-19T01:23:58.213 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 131/198 2024-03-19T01:23:58.213 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 132/198 2024-03-19T01:23:58.213 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 133/198 2024-03-19T01:23:58.213 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 134/198 2024-03-19T01:23:58.213 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 135/198 2024-03-19T01:23:58.213 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 136/198 2024-03-19T01:23:58.213 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 137/198 2024-03-19T01:23:58.213 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 138/198 2024-03-19T01:23:58.213 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-pyperclip-1.8.2-2.el9.noarch 139/198 2024-03-19T01:23:58.213 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 140/198 2024-03-19T01:23:58.213 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 141/198 2024-03-19T01:23:58.214 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 142/198 2024-03-19T01:23:58.214 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 143/198 2024-03-19T01:23:58.214 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-typing-extensions-4.4.0-2.el9.noarch 144/198 2024-03-19T01:23:58.214 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-webob-1.8.7-6.el9.noarch 145/198 2024-03-19T01:23:58.214 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 146/198 2024-03-19T01:23:58.214 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 147/198 2024-03-19T01:23:58.214 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 148/198 2024-03-19T01:23:58.214 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : re2-1:20211101-3.el9.x86_64 149/198 2024-03-19T01:23:58.214 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 150/198 2024-03-19T01:23:58.214 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : librados2-2:19.0.0-2206.gb6ae2491.el9.x86_64 151/198 2024-03-19T01:23:58.214 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 152/198 2024-03-19T01:23:58.214 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : librbd1-2:19.0.0-2206.gb6ae2491.el9.x86_64 153/198 2024-03-19T01:23:58.214 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 154/198 2024-03-19T01:23:58.215 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : xfsdump-3.1.12-4.el9.x86_64 155/198 2024-03-19T01:23:58.215 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : xfsdump-3.1.12-3.el9.x86_64 156/198 2024-03-19T01:23:58.215 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : NetworkManager-1:1.47.2-1.el9.x86_64 157/198 2024-03-19T01:23:58.215 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : NetworkManager-1:1.44.0-2.el9.x86_64 158/198 2024-03-19T01:23:58.215 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : NetworkManager-libnm-1:1.47.2-1.el9.x86_64 159/198 2024-03-19T01:23:58.215 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : NetworkManager-libnm-1:1.44.0-2.el9.x86_64 160/198 2024-03-19T01:23:58.215 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : NetworkManager-team-1:1.47.2-1.el9.x86_64 161/198 2024-03-19T01:23:58.215 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : NetworkManager-team-1:1.44.0-2.el9.x86_64 162/198 2024-03-19T01:23:58.215 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : NetworkManager-tui-1:1.47.2-1.el9.x86_64 163/198 2024-03-19T01:23:58.215 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : NetworkManager-tui-1:1.44.0-2.el9.x86_64 164/198 2024-03-19T01:23:58.215 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : acl-2.3.1-4.el9.x86_64 165/198 2024-03-19T01:23:58.215 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : acl-2.3.1-3.el9.x86_64 166/198 2024-03-19T01:23:58.215 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : elfutils-debuginfod-client-0.190-2.el9.x86_64 167/198 2024-03-19T01:23:58.215 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : elfutils-debuginfod-client-0.189-3.el9.x86_64 168/198 2024-03-19T01:23:58.216 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : elfutils-libelf-0.190-2.el9.x86_64 169/198 2024-03-19T01:23:58.216 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : elfutils-libelf-0.189-3.el9.x86_64 170/198 2024-03-19T01:23:58.216 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : elfutils-libs-0.190-2.el9.x86_64 171/198 2024-03-19T01:23:58.216 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : elfutils-libs-0.189-3.el9.x86_64 172/198 2024-03-19T01:23:58.216 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : libacl-2.3.1-4.el9.x86_64 173/198 2024-03-19T01:23:58.216 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : libacl-2.3.1-3.el9.x86_64 174/198 2024-03-19T01:23:58.216 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : libblkid-2.37.4-18.el9.x86_64 175/198 2024-03-19T01:23:58.216 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : libblkid-2.37.4-15.el9.x86_64 176/198 2024-03-19T01:23:58.216 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : libfdisk-2.37.4-18.el9.x86_64 177/198 2024-03-19T01:23:58.216 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : libfdisk-2.37.4-15.el9.x86_64 178/198 2024-03-19T01:23:58.216 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : libmount-2.37.4-18.el9.x86_64 179/198 2024-03-19T01:23:58.216 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : libmount-2.37.4-15.el9.x86_64 180/198 2024-03-19T01:23:58.216 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : libsmartcols-2.37.4-18.el9.x86_64 181/198 2024-03-19T01:23:58.216 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : libsmartcols-2.37.4-15.el9.x86_64 182/198 2024-03-19T01:23:58.217 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : libuuid-2.37.4-18.el9.x86_64 183/198 2024-03-19T01:23:58.217 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : libuuid-2.37.4-15.el9.x86_64 184/198 2024-03-19T01:23:58.217 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : openssl-1:3.0.7-27.el9.x86_64 185/198 2024-03-19T01:23:58.217 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : openssl-1:3.0.7-24.el9.x86_64 186/198 2024-03-19T01:23:58.217 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : openssl-libs-1:3.0.7-27.el9.x86_64 187/198 2024-03-19T01:23:58.217 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : openssl-libs-1:3.0.7-24.el9.x86_64 188/198 2024-03-19T01:23:58.217 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : util-linux-2.37.4-18.el9.x86_64 189/198 2024-03-19T01:23:58.217 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : util-linux-2.37.4-15.el9.x86_64 190/198 2024-03-19T01:23:58.217 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : util-linux-core-2.37.4-18.el9.x86_64 191/198 2024-03-19T01:23:58.217 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : util-linux-core-2.37.4-15.el9.x86_64 192/198 2024-03-19T01:23:58.217 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : util-linux-user-2.37.4-18.el9.x86_64 193/198 2024-03-19T01:23:58.217 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : util-linux-user-2.37.4-15.el9.x86_64 194/198 2024-03-19T01:23:58.217 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : xfsprogs-6.3.0-1.el9.x86_64 195/198 2024-03-19T01:23:58.217 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : xfsprogs-5.19.0-4.el9.x86_64 196/198 2024-03-19T01:23:58.218 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : libacl-devel-2.3.1-4.el9.x86_64 197/198 2024-03-19T01:24:01.065 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : libacl-devel-2.3.1-3.el9.x86_64 198/198 2024-03-19T01:24:01.065 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-03-19T01:24:01.065 INFO:teuthology.orchestra.run.smithi112.stdout:Upgraded: 2024-03-19T01:24:01.065 INFO:teuthology.orchestra.run.smithi112.stdout: NetworkManager-1:1.47.2-1.el9.x86_64 2024-03-19T01:24:01.065 INFO:teuthology.orchestra.run.smithi112.stdout: NetworkManager-libnm-1:1.47.2-1.el9.x86_64 2024-03-19T01:24:01.065 INFO:teuthology.orchestra.run.smithi112.stdout: NetworkManager-team-1:1.47.2-1.el9.x86_64 2024-03-19T01:24:01.066 INFO:teuthology.orchestra.run.smithi112.stdout: NetworkManager-tui-1:1.47.2-1.el9.x86_64 2024-03-19T01:24:01.066 INFO:teuthology.orchestra.run.smithi112.stdout: acl-2.3.1-4.el9.x86_64 2024-03-19T01:24:01.066 INFO:teuthology.orchestra.run.smithi112.stdout: elfutils-debuginfod-client-0.190-2.el9.x86_64 2024-03-19T01:24:01.066 INFO:teuthology.orchestra.run.smithi112.stdout: elfutils-libelf-0.190-2.el9.x86_64 2024-03-19T01:24:01.066 INFO:teuthology.orchestra.run.smithi112.stdout: elfutils-libs-0.190-2.el9.x86_64 2024-03-19T01:24:01.066 INFO:teuthology.orchestra.run.smithi112.stdout: libacl-2.3.1-4.el9.x86_64 2024-03-19T01:24:01.066 INFO:teuthology.orchestra.run.smithi112.stdout: libacl-devel-2.3.1-4.el9.x86_64 2024-03-19T01:24:01.066 INFO:teuthology.orchestra.run.smithi112.stdout: libblkid-2.37.4-18.el9.x86_64 2024-03-19T01:24:01.066 INFO:teuthology.orchestra.run.smithi112.stdout: libfdisk-2.37.4-18.el9.x86_64 2024-03-19T01:24:01.066 INFO:teuthology.orchestra.run.smithi112.stdout: libmount-2.37.4-18.el9.x86_64 2024-03-19T01:24:01.066 INFO:teuthology.orchestra.run.smithi112.stdout: librados2-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:24:01.066 INFO:teuthology.orchestra.run.smithi112.stdout: librbd1-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:24:01.066 INFO:teuthology.orchestra.run.smithi112.stdout: libsmartcols-2.37.4-18.el9.x86_64 2024-03-19T01:24:01.067 INFO:teuthology.orchestra.run.smithi112.stdout: libuuid-2.37.4-18.el9.x86_64 2024-03-19T01:24:01.067 INFO:teuthology.orchestra.run.smithi112.stdout: openssl-1:3.0.7-27.el9.x86_64 2024-03-19T01:24:01.067 INFO:teuthology.orchestra.run.smithi112.stdout: openssl-libs-1:3.0.7-27.el9.x86_64 2024-03-19T01:24:01.067 INFO:teuthology.orchestra.run.smithi112.stdout: util-linux-2.37.4-18.el9.x86_64 2024-03-19T01:24:01.067 INFO:teuthology.orchestra.run.smithi112.stdout: util-linux-core-2.37.4-18.el9.x86_64 2024-03-19T01:24:01.067 INFO:teuthology.orchestra.run.smithi112.stdout: util-linux-user-2.37.4-18.el9.x86_64 2024-03-19T01:24:01.067 INFO:teuthology.orchestra.run.smithi112.stdout: xfsdump-3.1.12-4.el9.x86_64 2024-03-19T01:24:01.067 INFO:teuthology.orchestra.run.smithi112.stdout: xfsprogs-6.3.0-1.el9.x86_64 2024-03-19T01:24:01.067 INFO:teuthology.orchestra.run.smithi112.stdout:Installed: 2024-03-19T01:24:01.067 INFO:teuthology.orchestra.run.smithi112.stdout: abseil-cpp-20211102.0-3.el9.x86_64 2024-03-19T01:24:01.067 INFO:teuthology.orchestra.run.smithi112.stdout: automake-1.16.2-8.el9.noarch 2024-03-19T01:24:01.067 INFO:teuthology.orchestra.run.smithi112.stdout: bison-3.7.4-5.el9.x86_64 2024-03-19T01:24:01.068 INFO:teuthology.orchestra.run.smithi112.stdout: boost-program-options-1.75.0-8.el9.x86_64 2024-03-19T01:24:01.068 INFO:teuthology.orchestra.run.smithi112.stdout: ceph-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:24:01.068 INFO:teuthology.orchestra.run.smithi112.stdout: ceph-base-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:24:01.068 INFO:teuthology.orchestra.run.smithi112.stdout: ceph-common-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:24:01.068 INFO:teuthology.orchestra.run.smithi112.stdout: ceph-fuse-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:24:01.068 INFO:teuthology.orchestra.run.smithi112.stdout: ceph-grafana-dashboards-2:19.0.0-2206.gb6ae2491.el9.noarch 2024-03-19T01:24:01.068 INFO:teuthology.orchestra.run.smithi112.stdout: ceph-immutable-object-cache-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:24:01.068 INFO:teuthology.orchestra.run.smithi112.stdout: ceph-mds-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:24:01.068 INFO:teuthology.orchestra.run.smithi112.stdout: ceph-mgr-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:24:01.068 INFO:teuthology.orchestra.run.smithi112.stdout: ceph-mgr-cephadm-2:19.0.0-2206.gb6ae2491.el9.noarch 2024-03-19T01:24:01.068 INFO:teuthology.orchestra.run.smithi112.stdout: ceph-mgr-dashboard-2:19.0.0-2206.gb6ae2491.el9.noarch 2024-03-19T01:24:01.068 INFO:teuthology.orchestra.run.smithi112.stdout: ceph-mgr-diskprediction-local-2:19.0.0-2206.gb6ae2491.el9.noarch 2024-03-19T01:24:01.068 INFO:teuthology.orchestra.run.smithi112.stdout: ceph-mgr-modules-core-2:19.0.0-2206.gb6ae2491.el9.noarch 2024-03-19T01:24:01.068 INFO:teuthology.orchestra.run.smithi112.stdout: ceph-mgr-rook-2:19.0.0-2206.gb6ae2491.el9.noarch 2024-03-19T01:24:01.069 INFO:teuthology.orchestra.run.smithi112.stdout: ceph-mon-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:24:01.069 INFO:teuthology.orchestra.run.smithi112.stdout: ceph-osd-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:24:01.069 INFO:teuthology.orchestra.run.smithi112.stdout: ceph-prometheus-alerts-2:19.0.0-2206.gb6ae2491.el9.noarch 2024-03-19T01:24:01.069 INFO:teuthology.orchestra.run.smithi112.stdout: ceph-radosgw-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:24:01.069 INFO:teuthology.orchestra.run.smithi112.stdout: ceph-selinux-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:24:01.069 INFO:teuthology.orchestra.run.smithi112.stdout: ceph-test-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:24:01.069 INFO:teuthology.orchestra.run.smithi112.stdout: ceph-volume-2:19.0.0-2206.gb6ae2491.el9.noarch 2024-03-19T01:24:01.069 INFO:teuthology.orchestra.run.smithi112.stdout: cephadm-2:19.0.0-2206.gb6ae2491.el9.noarch 2024-03-19T01:24:01.069 INFO:teuthology.orchestra.run.smithi112.stdout: cephfs-mirror-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:24:01.069 INFO:teuthology.orchestra.run.smithi112.stdout: cephfs-shell-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:24:01.069 INFO:teuthology.orchestra.run.smithi112.stdout: cephfs-top-2:19.0.0-2206.gb6ae2491.el9.noarch 2024-03-19T01:24:01.069 INFO:teuthology.orchestra.run.smithi112.stdout: elfutils-libelf-devel-0.190-2.el9.x86_64 2024-03-19T01:24:01.069 INFO:teuthology.orchestra.run.smithi112.stdout: flex-2.6.4-9.el9.x86_64 2024-03-19T01:24:01.069 INFO:teuthology.orchestra.run.smithi112.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-03-19T01:24:01.070 INFO:teuthology.orchestra.run.smithi112.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-03-19T01:24:01.070 INFO:teuthology.orchestra.run.smithi112.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-03-19T01:24:01.070 INFO:teuthology.orchestra.run.smithi112.stdout: fuse-2.9.9-15.el9.x86_64 2024-03-19T01:24:01.070 INFO:teuthology.orchestra.run.smithi112.stdout: gperftools-libs-2.9.1-2.el9.x86_64 2024-03-19T01:24:01.070 INFO:teuthology.orchestra.run.smithi112.stdout: grpc-data-1.46.7-10.el9.noarch 2024-03-19T01:24:01.070 INFO:teuthology.orchestra.run.smithi112.stdout: jq-1.6-16.el9.x86_64 2024-03-19T01:24:01.070 INFO:teuthology.orchestra.run.smithi112.stdout: libaio-devel-0.3.111-13.el9.x86_64 2024-03-19T01:24:01.070 INFO:teuthology.orchestra.run.smithi112.stdout: libarrow-9.0.0-10.el9.x86_64 2024-03-19T01:24:01.070 INFO:teuthology.orchestra.run.smithi112.stdout: libarrow-doc-9.0.0-10.el9.noarch 2024-03-19T01:24:01.070 INFO:teuthology.orchestra.run.smithi112.stdout: libcephfs-devel-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:24:01.070 INFO:teuthology.orchestra.run.smithi112.stdout: libcephfs2-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:24:01.070 INFO:teuthology.orchestra.run.smithi112.stdout: libcephsqlite-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:24:01.070 INFO:teuthology.orchestra.run.smithi112.stdout: libgfortran-11.4.1-3.el9.x86_64 2024-03-19T01:24:01.071 INFO:teuthology.orchestra.run.smithi112.stdout: liboath-2.6.7-2.el9.x86_64 2024-03-19T01:24:01.071 INFO:teuthology.orchestra.run.smithi112.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-03-19T01:24:01.071 INFO:teuthology.orchestra.run.smithi112.stdout: libquadmath-11.4.1-3.el9.x86_64 2024-03-19T01:24:01.071 INFO:teuthology.orchestra.run.smithi112.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-03-19T01:24:01.071 INFO:teuthology.orchestra.run.smithi112.stdout: librados-devel-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:24:01.071 INFO:teuthology.orchestra.run.smithi112.stdout: libradosstriper1-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:24:01.071 INFO:teuthology.orchestra.run.smithi112.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-03-19T01:24:01.071 INFO:teuthology.orchestra.run.smithi112.stdout: librgw2-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:24:01.071 INFO:teuthology.orchestra.run.smithi112.stdout: libtool-2.4.6-45.el9.x86_64 2024-03-19T01:24:01.071 INFO:teuthology.orchestra.run.smithi112.stdout: libunwind-1.6.2-1.el9.x86_64 2024-03-19T01:24:01.071 INFO:teuthology.orchestra.run.smithi112.stdout: libuuid-devel-2.37.4-18.el9.x86_64 2024-03-19T01:24:01.071 INFO:teuthology.orchestra.run.smithi112.stdout: libzstd-devel-1.5.1-2.el9.x86_64 2024-03-19T01:24:01.071 INFO:teuthology.orchestra.run.smithi112.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-03-19T01:24:01.072 INFO:teuthology.orchestra.run.smithi112.stdout: lua-5.4.4-4.el9.x86_64 2024-03-19T01:24:01.072 INFO:teuthology.orchestra.run.smithi112.stdout: lua-devel-5.4.4-4.el9.x86_64 2024-03-19T01:24:01.072 INFO:teuthology.orchestra.run.smithi112.stdout: luarocks-3.9.2-1.el9.noarch 2024-03-19T01:24:01.072 INFO:teuthology.orchestra.run.smithi112.stdout: oniguruma-6.9.6-1.el9.5.x86_64 2024-03-19T01:24:01.072 INFO:teuthology.orchestra.run.smithi112.stdout: openblas-0.3.21-2.el9.x86_64 2024-03-19T01:24:01.072 INFO:teuthology.orchestra.run.smithi112.stdout: openblas-openmp-0.3.21-2.el9.x86_64 2024-03-19T01:24:01.072 INFO:teuthology.orchestra.run.smithi112.stdout: openssl-devel-1:3.0.7-27.el9.x86_64 2024-03-19T01:24:01.072 INFO:teuthology.orchestra.run.smithi112.stdout: parquet-libs-9.0.0-10.el9.x86_64 2024-03-19T01:24:01.072 INFO:teuthology.orchestra.run.smithi112.stdout: postgresql-13.11-1.el9.x86_64 2024-03-19T01:24:01.072 INFO:teuthology.orchestra.run.smithi112.stdout: postgresql-contrib-13.11-1.el9.x86_64 2024-03-19T01:24:01.072 INFO:teuthology.orchestra.run.smithi112.stdout: postgresql-private-libs-13.11-1.el9.x86_64 2024-03-19T01:24:01.072 INFO:teuthology.orchestra.run.smithi112.stdout: postgresql-server-13.11-1.el9.x86_64 2024-03-19T01:24:01.072 INFO:teuthology.orchestra.run.smithi112.stdout: protobuf-3.14.0-13.el9.x86_64 2024-03-19T01:24:01.072 INFO:teuthology.orchestra.run.smithi112.stdout: protobuf-compiler-3.14.0-13.el9.x86_64 2024-03-19T01:24:01.072 INFO:teuthology.orchestra.run.smithi112.stdout: python3-asyncssh-2.13.2-4.el9.noarch 2024-03-19T01:24:01.073 INFO:teuthology.orchestra.run.smithi112.stdout: python3-attrs-20.3.0-7.el9.noarch 2024-03-19T01:24:01.073 INFO:teuthology.orchestra.run.smithi112.stdout: python3-babel-2.9.1-2.el9.noarch 2024-03-19T01:24:01.073 INFO:teuthology.orchestra.run.smithi112.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-03-19T01:24:01.073 INFO:teuthology.orchestra.run.smithi112.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-03-19T01:24:01.073 INFO:teuthology.orchestra.run.smithi112.stdout: python3-ceph-argparse-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:24:01.073 INFO:teuthology.orchestra.run.smithi112.stdout: python3-ceph-common-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:24:01.073 INFO:teuthology.orchestra.run.smithi112.stdout: python3-cephfs-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:24:01.073 INFO:teuthology.orchestra.run.smithi112.stdout: python3-certifi-2021.10.8-2.el9.noarch 2024-03-19T01:24:01.073 INFO:teuthology.orchestra.run.smithi112.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-03-19T01:24:01.074 INFO:teuthology.orchestra.run.smithi112.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-03-19T01:24:01.074 INFO:teuthology.orchestra.run.smithi112.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-03-19T01:24:01.074 INFO:teuthology.orchestra.run.smithi112.stdout: python3-cmd2-2.4.2-2.el9.noarch 2024-03-19T01:24:01.074 INFO:teuthology.orchestra.run.smithi112.stdout: python3-colorama-0.4.4-12.el9.noarch 2024-03-19T01:24:01.074 INFO:teuthology.orchestra.run.smithi112.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-03-19T01:24:01.074 INFO:teuthology.orchestra.run.smithi112.stdout: python3-devel-3.9.17-2.el9.x86_64 2024-03-19T01:24:01.074 INFO:teuthology.orchestra.run.smithi112.stdout: python3-google-auth-1:2.28.0-1.el9.noarch 2024-03-19T01:24:01.074 INFO:teuthology.orchestra.run.smithi112.stdout: python3-grpcio-1.46.7-10.el9.x86_64 2024-03-19T01:24:01.074 INFO:teuthology.orchestra.run.smithi112.stdout: python3-grpcio-tools-1.46.7-10.el9.x86_64 2024-03-19T01:24:01.074 INFO:teuthology.orchestra.run.smithi112.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-03-19T01:24:01.074 INFO:teuthology.orchestra.run.smithi112.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-03-19T01:24:01.074 INFO:teuthology.orchestra.run.smithi112.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-03-19T01:24:01.074 INFO:teuthology.orchestra.run.smithi112.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-03-19T01:24:01.075 INFO:teuthology.orchestra.run.smithi112.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-03-19T01:24:01.075 INFO:teuthology.orchestra.run.smithi112.stdout: python3-jinja2-2.11.3-5.el9.noarch 2024-03-19T01:24:01.075 INFO:teuthology.orchestra.run.smithi112.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-03-19T01:24:01.075 INFO:teuthology.orchestra.run.smithi112.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-03-19T01:24:01.075 INFO:teuthology.orchestra.run.smithi112.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-03-19T01:24:01.075 INFO:teuthology.orchestra.run.smithi112.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-03-19T01:24:01.075 INFO:teuthology.orchestra.run.smithi112.stdout: python3-mako-1.1.4-6.el9.noarch 2024-03-19T01:24:01.075 INFO:teuthology.orchestra.run.smithi112.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-03-19T01:24:01.075 INFO:teuthology.orchestra.run.smithi112.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-03-19T01:24:01.075 INFO:teuthology.orchestra.run.smithi112.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-03-19T01:24:01.075 INFO:teuthology.orchestra.run.smithi112.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-03-19T01:24:01.075 INFO:teuthology.orchestra.run.smithi112.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-03-19T01:24:01.075 INFO:teuthology.orchestra.run.smithi112.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-03-19T01:24:01.075 INFO:teuthology.orchestra.run.smithi112.stdout: python3-packaging-20.9-5.el9.noarch 2024-03-19T01:24:01.076 INFO:teuthology.orchestra.run.smithi112.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-03-19T01:24:01.076 INFO:teuthology.orchestra.run.smithi112.stdout: python3-ply-3.11-14.el9.noarch 2024-03-19T01:24:01.076 INFO:teuthology.orchestra.run.smithi112.stdout: python3-portend-3.1.0-2.el9.noarch 2024-03-19T01:24:01.076 INFO:teuthology.orchestra.run.smithi112.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-03-19T01:24:01.076 INFO:teuthology.orchestra.run.smithi112.stdout: python3-protobuf-3.14.0-13.el9.noarch 2024-03-19T01:24:01.076 INFO:teuthology.orchestra.run.smithi112.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-03-19T01:24:01.076 INFO:teuthology.orchestra.run.smithi112.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-03-19T01:24:01.076 INFO:teuthology.orchestra.run.smithi112.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-03-19T01:24:01.076 INFO:teuthology.orchestra.run.smithi112.stdout: python3-pycparser-2.20-6.el9.noarch 2024-03-19T01:24:01.076 INFO:teuthology.orchestra.run.smithi112.stdout: python3-pyperclip-1.8.2-2.el9.noarch 2024-03-19T01:24:01.076 INFO:teuthology.orchestra.run.smithi112.stdout: python3-pytz-2021.1-5.el9.noarch 2024-03-19T01:24:01.076 INFO:teuthology.orchestra.run.smithi112.stdout: python3-rados-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:24:01.076 INFO:teuthology.orchestra.run.smithi112.stdout: python3-rbd-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:24:01.077 INFO:teuthology.orchestra.run.smithi112.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-03-19T01:24:01.077 INFO:teuthology.orchestra.run.smithi112.stdout: python3-requests-2.25.1-8.el9.noarch 2024-03-19T01:24:01.077 INFO:teuthology.orchestra.run.smithi112.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-03-19T01:24:01.077 INFO:teuthology.orchestra.run.smithi112.stdout: python3-rgw-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:24:01.077 INFO:teuthology.orchestra.run.smithi112.stdout: python3-routes-2.5.1-5.el9.noarch 2024-03-19T01:24:01.077 INFO:teuthology.orchestra.run.smithi112.stdout: python3-rsa-4.9-2.el9.noarch 2024-03-19T01:24:01.077 INFO:teuthology.orchestra.run.smithi112.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-03-19T01:24:01.077 INFO:teuthology.orchestra.run.smithi112.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-03-19T01:24:01.077 INFO:teuthology.orchestra.run.smithi112.stdout: python3-toml-0.10.2-6.el9.noarch 2024-03-19T01:24:01.077 INFO:teuthology.orchestra.run.smithi112.stdout: python3-typing-extensions-4.4.0-2.el9.noarch 2024-03-19T01:24:01.077 INFO:teuthology.orchestra.run.smithi112.stdout: python3-urllib3-1.26.5-5.el9.noarch 2024-03-19T01:24:01.077 INFO:teuthology.orchestra.run.smithi112.stdout: python3-wcwidth-0.2.5-8.el9.noarch 2024-03-19T01:24:01.077 INFO:teuthology.orchestra.run.smithi112.stdout: python3-webob-1.8.7-6.el9.noarch 2024-03-19T01:24:01.077 INFO:teuthology.orchestra.run.smithi112.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-03-19T01:24:01.078 INFO:teuthology.orchestra.run.smithi112.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-03-19T01:24:01.078 INFO:teuthology.orchestra.run.smithi112.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-03-19T01:24:01.078 INFO:teuthology.orchestra.run.smithi112.stdout: qatlib-23.11.0-1.el9.x86_64 2024-03-19T01:24:01.078 INFO:teuthology.orchestra.run.smithi112.stdout: qatlib-service-23.11.0-1.el9.x86_64 2024-03-19T01:24:01.078 INFO:teuthology.orchestra.run.smithi112.stdout: qatzip-libs-1.1.2-2.el9.x86_64 2024-03-19T01:24:01.078 INFO:teuthology.orchestra.run.smithi112.stdout: rbd-fuse-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:24:01.078 INFO:teuthology.orchestra.run.smithi112.stdout: rbd-mirror-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:24:01.078 INFO:teuthology.orchestra.run.smithi112.stdout: rbd-nbd-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:24:01.078 INFO:teuthology.orchestra.run.smithi112.stdout: re2-1:20211101-3.el9.x86_64 2024-03-19T01:24:01.078 INFO:teuthology.orchestra.run.smithi112.stdout: socat-1.7.4.1-5.el9.x86_64 2024-03-19T01:24:01.078 INFO:teuthology.orchestra.run.smithi112.stdout: thrift-0.15.0-2.el9.x86_64 2024-03-19T01:24:01.078 INFO:teuthology.orchestra.run.smithi112.stdout: uuid-1.6.2-55.el9.x86_64 2024-03-19T01:24:01.078 INFO:teuthology.orchestra.run.smithi112.stdout: xfsprogs-devel-6.3.0-1.el9.x86_64 2024-03-19T01:24:01.078 INFO:teuthology.orchestra.run.smithi112.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-03-19T01:24:01.079 INFO:teuthology.orchestra.run.smithi112.stdout: zlib-devel-1.2.11-40.el9.x86_64 2024-03-19T01:24:01.079 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-03-19T01:24:01.079 INFO:teuthology.orchestra.run.smithi112.stdout:Complete! 2024-03-19T01:24:01.512 DEBUG:teuthology.parallel:result is None 2024-03-19T01:24:58.514 INFO:teuthology.orchestra.run.smithi115.stdout:[MIRROR] elfutils-libelf-devel-0.190-2.el9.x86_64.rpm: Curl error (28): Timeout was reached for http://ftp-nyc.osuosl.org/pub/centos-stream/9-stream/AppStream/x86_64/os/Packages/elfutils-libelf-devel-0.190-2.el9.x86_64.rpm [Failed to connect to ftp-nyc.osuosl.org port 80: Connection timed out] 2024-03-19T01:24:58.515 INFO:teuthology.orchestra.run.smithi115.stdout:[MIRROR] libuuid-devel-2.37.4-18.el9.x86_64.rpm: Curl error (28): Timeout was reached for http://ftp-nyc.osuosl.org/pub/centos-stream/9-stream/AppStream/x86_64/os/Packages/libuuid-devel-2.37.4-18.el9.x86_64.rpm [Failed to connect to ftp-nyc.osuosl.org port 80: Connection timed out] 2024-03-19T01:24:58.515 INFO:teuthology.orchestra.run.smithi115.stdout:[MIRROR] openssl-devel-3.0.7-27.el9.x86_64.rpm: Curl error (28): Timeout was reached for http://ftp-nyc.osuosl.org/pub/centos-stream/9-stream/AppStream/x86_64/os/Packages/openssl-devel-3.0.7-27.el9.x86_64.rpm [Failed to connect to ftp-nyc.osuosl.org port 80: Connection timed out] 2024-03-19T01:24:58.795 INFO:teuthology.orchestra.run.smithi115.stdout:(96/174): elfutils-libelf-devel-0.190-2.el9.x86 169 B/s | 21 kB 02:10 2024-03-19T01:24:58.820 INFO:teuthology.orchestra.run.smithi115.stdout:(97/174): libuuid-devel-2.37.4-18.el9.x86_64.rp 200 B/s | 25 kB 02:09 2024-03-19T01:24:59.212 INFO:teuthology.orchestra.run.smithi115.stdout:(98/174): python3-jinja2-2.11.3-5.el9.noarch.rp 633 kB/s | 248 kB 00:00 2024-03-19T01:24:59.237 INFO:teuthology.orchestra.run.smithi115.stdout:(99/174): python3-devel-3.9.17-2.el9.x86_64.rpm 555 kB/s | 245 kB 00:00 2024-03-19T01:24:59.362 INFO:teuthology.orchestra.run.smithi115.stdout:(100/174): qatlib-service-23.11.0-1.el9.x86_64. 279 kB/s | 35 kB 00:00 2024-03-19T01:24:59.487 INFO:teuthology.orchestra.run.smithi115.stdout:(101/174): qatzip-libs-1.1.2-2.el9.x86_64.rpm 356 kB/s | 44 kB 00:00 2024-03-19T01:24:59.513 INFO:teuthology.orchestra.run.smithi115.stdout:(102/174): qatlib-23.11.0-1.el9.x86_64.rpm 720 kB/s | 216 kB 00:00 2024-03-19T01:24:59.596 INFO:teuthology.orchestra.run.smithi115.stdout:(103/174): lua-devel-5.4.4-4.el9.x86_64.rpm 267 kB/s | 22 kB 00:00 2024-03-19T01:24:59.705 INFO:teuthology.orchestra.run.smithi115.stdout:(104/174): xfsprogs-devel-6.3.0-1.el9.x86_64.rp 432 kB/s | 94 kB 00:00 2024-03-19T01:25:00.005 INFO:teuthology.orchestra.run.smithi115.stdout:(105/174): protobuf-compiler-3.14.0-13.el9.x86_ 2.1 MB/s | 863 kB 00:00 2024-03-19T01:25:00.847 INFO:teuthology.orchestra.run.smithi115.stdout:(106/174): abseil-cpp-20211102.0-3.el9.x86_64.r 480 kB/s | 548 kB 00:01 2024-03-19T01:25:00.872 INFO:teuthology.orchestra.run.smithi115.stdout:(107/174): gperftools-libs-2.9.1-2.el9.x86_64.r 356 kB/s | 309 kB 00:00 2024-03-19T01:25:00.956 INFO:teuthology.orchestra.run.smithi115.stdout:(108/174): grpc-data-1.46.7-10.el9.noarch.rpm 179 kB/s | 19 kB 00:00 2024-03-19T01:25:01.065 INFO:teuthology.orchestra.run.smithi115.stdout:(109/174): libarrow-doc-9.0.0-10.el9.noarch.rpm 239 kB/s | 26 kB 00:00 2024-03-19T01:25:01.173 INFO:teuthology.orchestra.run.smithi115.stdout:(110/174): liboath-2.6.7-2.el9.x86_64.rpm 449 kB/s | 49 kB 00:00 2024-03-19T01:25:01.282 INFO:teuthology.orchestra.run.smithi115.stdout:(111/174): libunwind-1.6.2-1.el9.x86_64.rpm 622 kB/s | 67 kB 00:00 2024-03-19T01:25:01.549 INFO:teuthology.orchestra.run.smithi115.stdout:(112/174): luarocks-3.9.2-1.el9.noarch.rpm 566 kB/s | 151 kB 00:00 2024-03-19T01:25:02.349 INFO:teuthology.orchestra.run.smithi115.stdout:(113/174): openssl-devel-3.0.7-27.el9.x86_64.rp 32 kB/s | 4.1 MB 02:13 2024-03-19T01:25:02.632 INFO:teuthology.orchestra.run.smithi115.stdout:(114/174): parquet-libs-9.0.0-10.el9.x86_64.rpm 774 kB/s | 838 kB 00:01 2024-03-19T01:25:02.749 INFO:teuthology.orchestra.run.smithi115.stdout:(115/174): python3-bcrypt-3.2.2-1.el9.x86_64.rp 372 kB/s | 43 kB 00:00 2024-03-19T01:25:02.858 INFO:teuthology.orchestra.run.smithi115.stdout:(116/174): python3-cachetools-4.2.4-1.el9.noarc 297 kB/s | 32 kB 00:00 2024-03-19T01:25:02.975 INFO:teuthology.orchestra.run.smithi115.stdout:(117/174): python3-certifi-2021.10.8-2.el9.noar 129 kB/s | 15 kB 00:00 2024-03-19T01:25:03.292 INFO:teuthology.orchestra.run.smithi115.stdout:(118/174): python3-cheroot-8.6.0-4.el9.noarch.r 542 kB/s | 172 kB 00:00 2024-03-19T01:25:03.642 INFO:teuthology.orchestra.run.smithi115.stdout:(119/174): python3-asyncssh-2.13.2-4.el9.noarch 424 kB/s | 548 kB 00:01 2024-03-19T01:25:03.859 INFO:teuthology.orchestra.run.smithi115.stdout:(120/174): python3-cherrypy-18.6.1-2.el9.noarch 632 kB/s | 358 kB 00:00 2024-03-19T01:25:03.967 INFO:teuthology.orchestra.run.smithi115.stdout:(121/174): python3-colorama-0.4.4-12.el9.noarch 325 kB/s | 35 kB 00:00 2024-03-19T01:25:04.076 INFO:teuthology.orchestra.run.smithi115.stdout:(122/174): python3-cmd2-2.4.2-2.el9.noarch.rpm 699 kB/s | 303 kB 00:00 2024-03-19T01:25:04.235 INFO:teuthology.orchestra.run.smithi115.stdout:(123/174): python3-google-auth-2.28.0-1.el9.noa 759 kB/s | 203 kB 00:00 2024-03-19T01:25:04.502 INFO:teuthology.orchestra.run.smithi115.stdout:(124/174): python3-grpcio-tools-1.46.7-10.el9.x 542 kB/s | 144 kB 00:00 2024-03-19T01:25:04.610 INFO:teuthology.orchestra.run.smithi115.stdout:(125/174): python3-jaraco-8.2.1-3.el9.noarch.rp 99 kB/s | 11 kB 00:00 2024-03-19T01:25:04.719 INFO:teuthology.orchestra.run.smithi115.stdout:(126/174): python3-jaraco-classes-3.2.1-5.el9.n 163 kB/s | 18 kB 00:00 2024-03-19T01:25:04.827 INFO:teuthology.orchestra.run.smithi115.stdout:(127/174): python3-jaraco-collections-3.0.0-8.e 214 kB/s | 23 kB 00:00 2024-03-19T01:25:04.936 INFO:teuthology.orchestra.run.smithi115.stdout:(128/174): python3-jaraco-functools-3.5.0-2.el9 179 kB/s | 19 kB 00:00 2024-03-19T01:25:05.045 INFO:teuthology.orchestra.run.smithi115.stdout:(129/174): python3-jaraco-text-3.2.0-6.el9.noar 182 kB/s | 20 kB 00:00 2024-03-19T01:25:06.445 INFO:teuthology.orchestra.run.smithi115.stdout:(130/174): libarrow-9.0.0-10.el9.x86_64.rpm 812 kB/s | 4.4 MB 00:05 2024-03-19T01:25:06.478 INFO:teuthology.orchestra.run.smithi115.stdout:(131/174): python3-kubernetes-26.1.0-2.el9.noar 729 kB/s | 1.0 MB 00:01 2024-03-19T01:25:06.545 INFO:teuthology.orchestra.run.smithi115.stdout:(132/174): python3-logutils-0.3.5-21.el9.noarch 462 kB/s | 46 kB 00:00 2024-03-19T01:25:06.587 INFO:teuthology.orchestra.run.smithi115.stdout:(133/174): python3-more-itertools-8.12.0-2.el9. 726 kB/s | 79 kB 00:00 2024-03-19T01:25:06.654 INFO:teuthology.orchestra.run.smithi115.stdout:(134/174): python3-natsort-7.1.1-5.el9.noarch.r 530 kB/s | 58 kB 00:00 2024-03-19T01:25:06.771 INFO:teuthology.orchestra.run.smithi115.stdout:(135/174): python3-grpcio-1.46.7-10.el9.x86_64. 774 kB/s | 2.0 MB 00:02 2024-03-19T01:25:06.796 INFO:teuthology.orchestra.run.smithi115.stdout:(136/174): python3-portend-3.1.0-2.el9.noarch.r 116 kB/s | 16 kB 00:00 2024-03-19T01:25:06.913 INFO:teuthology.orchestra.run.smithi115.stdout:(137/174): python3-pyperclip-1.8.2-2.el9.noarch 242 kB/s | 28 kB 00:00 2024-03-19T01:25:06.963 INFO:teuthology.orchestra.run.smithi115.stdout:(138/174): python3-pyOpenSSL-21.0.0-1.el9.noarc 468 kB/s | 90 kB 00:00 2024-03-19T01:25:07.022 INFO:teuthology.orchestra.run.smithi115.stdout:(139/174): python3-repoze-lru-0.7-16.el9.noarch 283 kB/s | 31 kB 00:00 2024-03-19T01:25:07.047 INFO:teuthology.orchestra.run.smithi115.stdout:(140/174): python3-pecan-1.4.2-3.el9.noarch.rpm 591 kB/s | 272 kB 00:00 2024-03-19T01:25:07.139 INFO:teuthology.orchestra.run.smithi115.stdout:(141/174): python3-rsa-4.9-2.el9.noarch.rpm 506 kB/s | 59 kB 00:00 2024-03-19T01:25:07.164 INFO:teuthology.orchestra.run.smithi115.stdout:(142/174): python3-tempora-5.0.0-2.el9.noarch.r 306 kB/s | 36 kB 00:00 2024-03-19T01:25:07.249 INFO:teuthology.orchestra.run.smithi115.stdout:(143/174): python3-typing-extensions-4.4.0-2.el 470 kB/s | 51 kB 00:00 2024-03-19T01:25:07.275 INFO:teuthology.orchestra.run.smithi115.stdout:(144/174): python3-routes-2.5.1-5.el9.noarch.rp 609 kB/s | 188 kB 00:00 2024-03-19T01:25:07.475 INFO:teuthology.orchestra.run.smithi115.stdout:(145/174): python3-websocket-client-1.2.3-2.el9 398 kB/s | 90 kB 00:00 2024-03-19T01:25:07.574 INFO:teuthology.orchestra.run.smithi115.stdout:(146/174): python3-webob-1.8.7-6.el9.noarch.rpm 561 kB/s | 230 kB 00:00 2024-03-19T01:25:07.608 INFO:teuthology.orchestra.run.smithi115.stdout:(147/174): python3-zc-lockfile-2.0-10.el9.noarc 149 kB/s | 20 kB 00:00 2024-03-19T01:25:07.908 INFO:teuthology.orchestra.run.smithi115.stdout:(148/174): re2-20211101-3.el9.x86_64.rpm 574 kB/s | 192 kB 00:00 2024-03-19T01:25:08.291 INFO:teuthology.orchestra.run.smithi115.stdout:(149/174): librados2-19.0.0-2206.gb6ae2491.el9. 8.9 MB/s | 3.4 MB 00:00 2024-03-19T01:25:08.475 INFO:teuthology.orchestra.run.smithi115.stdout:(150/174): librbd1-19.0.0-2206.gb6ae2491.el9.x8 17 MB/s | 3.1 MB 00:00 2024-03-19T01:25:08.917 INFO:teuthology.orchestra.run.smithi115.stdout:(151/174): xfsdump-3.1.12-4.el9.x86_64.rpm 728 kB/s | 321 kB 00:00 2024-03-19T01:25:08.984 INFO:teuthology.orchestra.run.smithi115.stdout:(152/174): python3-werkzeug-2.0.3-3.el9.1.noarc 250 kB/s | 427 kB 00:01 2024-03-19T01:25:10.042 INFO:teuthology.orchestra.run.smithi115.stdout:(153/174): thrift-0.15.0-2.el9.x86_64.rpm 665 kB/s | 1.6 MB 00:02 2024-03-19T01:25:10.342 INFO:teuthology.orchestra.run.smithi115.stdout:(154/174): NetworkManager-team-1.47.2-1.el9.x86 140 kB/s | 42 kB 00:00 2024-03-19T01:25:10.717 INFO:teuthology.orchestra.run.smithi115.stdout:(155/174): NetworkManager-tui-1.47.2-1.el9.x86_ 662 kB/s | 248 kB 00:00 2024-03-19T01:25:10.859 INFO:teuthology.orchestra.run.smithi115.stdout:(156/174): acl-2.3.1-4.el9.x86_64.rpm 504 kB/s | 71 kB 00:00 2024-03-19T01:25:10.968 INFO:teuthology.orchestra.run.smithi115.stdout:(157/174): elfutils-debuginfod-client-0.190-2.e 336 kB/s | 36 kB 00:00 2024-03-19T01:25:11.110 INFO:teuthology.orchestra.run.smithi115.stdout:(158/174): NetworkManager-libnm-1.47.2-1.el9.x8 883 kB/s | 1.8 MB 00:02 2024-03-19T01:25:11.260 INFO:teuthology.orchestra.run.smithi115.stdout:(159/174): elfutils-libelf-0.190-2.el9.x86_64.r 659 kB/s | 192 kB 00:00 2024-03-19T01:25:11.360 INFO:teuthology.orchestra.run.smithi115.stdout:(160/174): NetworkManager-1.47.2-1.el9.x86_64.r 966 kB/s | 2.3 MB 00:02 2024-03-19T01:25:11.385 INFO:teuthology.orchestra.run.smithi115.stdout:(161/174): libacl-2.3.1-4.el9.x86_64.rpm 184 kB/s | 23 kB 00:00 2024-03-19T01:25:11.419 INFO:teuthology.orchestra.run.smithi115.stdout:(162/174): elfutils-libs-0.190-2.el9.x86_64.rpm 827 kB/s | 256 kB 00:00 2024-03-19T01:25:11.469 INFO:teuthology.orchestra.run.smithi115.stdout:(163/174): libblkid-2.37.4-18.el9.x86_64.rpm 995 kB/s | 108 kB 00:00 2024-03-19T01:25:11.578 INFO:teuthology.orchestra.run.smithi115.stdout:(164/174): libsmartcols-2.37.4-18.el9.x86_64.rp 589 kB/s | 64 kB 00:00 2024-03-19T01:25:11.604 INFO:teuthology.orchestra.run.smithi115.stdout:(165/174): libfdisk-2.37.4-18.el9.x86_64.rpm 711 kB/s | 155 kB 00:00 2024-03-19T01:25:11.629 INFO:teuthology.orchestra.run.smithi115.stdout:(166/174): libmount-2.37.4-18.el9.x86_64.rpm 648 kB/s | 136 kB 00:00 2024-03-19T01:25:11.687 INFO:teuthology.orchestra.run.smithi115.stdout:(167/174): libuuid-2.37.4-18.el9.x86_64.rpm 264 kB/s | 29 kB 00:00 2024-03-19T01:25:12.596 INFO:teuthology.orchestra.run.smithi115.stdout:(168/174): openssl-3.0.7-27.el9.x86_64.rpm 1.2 MB/s | 1.2 MB 00:00 2024-03-19T01:25:13.046 INFO:teuthology.orchestra.run.smithi115.stdout:(169/174): util-linux-core-2.37.4-18.el9.x86_64 1.0 MB/s | 465 kB 00:00 2024-03-19T01:25:13.155 INFO:teuthology.orchestra.run.smithi115.stdout:(170/174): util-linux-user-2.37.4-18.el9.x86_64 287 kB/s | 31 kB 00:00 2024-03-19T01:25:13.372 INFO:teuthology.orchestra.run.smithi115.stdout:(171/174): openssl-libs-3.0.7-27.el9.x86_64.rpm 1.2 MB/s | 2.2 MB 00:01 2024-03-19T01:25:13.497 INFO:teuthology.orchestra.run.smithi115.stdout:(172/174): util-linux-2.37.4-18.el9.x86_64.rpm 1.3 MB/s | 2.3 MB 00:01 2024-03-19T01:25:13.522 INFO:teuthology.orchestra.run.smithi115.stdout:(173/174): libacl-devel-2.3.1-4.el9.x86_64.rpm 496 kB/s | 74 kB 00:00 2024-03-19T01:25:14.055 INFO:teuthology.orchestra.run.smithi115.stdout:(174/174): xfsprogs-6.3.0-1.el9.x86_64.rpm 1.2 MB/s | 1.1 MB 00:00 2024-03-19T01:25:14.060 INFO:teuthology.orchestra.run.smithi115.stdout:-------------------------------------------------------------------------------- 2024-03-19T01:25:14.060 INFO:teuthology.orchestra.run.smithi115.stdout:Total 804 kB/s | 224 MB 04:45 2024-03-19T01:25:14.511 INFO:teuthology.orchestra.run.smithi115.stdout:Running transaction check 2024-03-19T01:25:14.782 INFO:teuthology.orchestra.run.smithi115.stdout:Transaction check succeeded. 2024-03-19T01:25:14.782 INFO:teuthology.orchestra.run.smithi115.stdout:Running transaction test 2024-03-19T01:25:16.585 INFO:teuthology.orchestra.run.smithi115.stdout:Transaction test succeeded. 2024-03-19T01:25:16.586 INFO:teuthology.orchestra.run.smithi115.stdout:Running transaction 2024-03-19T01:25:23.516 INFO:teuthology.orchestra.run.smithi115.stdout: Preparing : 1/1 2024-03-19T01:25:23.625 INFO:teuthology.orchestra.run.smithi115.stdout: Upgrading : openssl-libs-1:3.0.7-27.el9.x86_64 1/198 2024-03-19T01:25:23.650 INFO:teuthology.orchestra.run.smithi115.stdout: Upgrading : libuuid-2.37.4-18.el9.x86_64 2/198 2024-03-19T01:25:23.682 INFO:teuthology.orchestra.run.smithi115.stdout: Upgrading : libblkid-2.37.4-18.el9.x86_64 3/198 2024-03-19T01:25:23.690 INFO:teuthology.orchestra.run.smithi115.stdout: Running scriptlet: libblkid-2.37.4-18.el9.x86_64 3/198 2024-03-19T01:25:23.738 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 4/198 2024-03-19T01:25:23.792 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 5/198 2024-03-19T01:25:23.826 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : thrift-0.15.0-2.el9.x86_64 6/198 2024-03-19T01:25:23.860 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 7/198 2024-03-19T01:25:23.910 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 8/198 2024-03-19T01:25:24.229 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 9/198 2024-03-19T01:25:24.295 INFO:teuthology.orchestra.run.smithi115.stdout: Upgrading : librados2-2:19.0.0-2206.gb6ae2491.el9.x86_64 10/198 2024-03-19T01:25:24.416 INFO:teuthology.orchestra.run.smithi115.stdout: Running scriptlet: librados2-2:19.0.0-2206.gb6ae2491.el9.x86_64 10/198 2024-03-19T01:25:24.446 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : libcephfs2-2:19.0.0-2206.gb6ae2491.el9.x86_64 11/198 2024-03-19T01:25:24.537 INFO:teuthology.orchestra.run.smithi115.stdout: Running scriptlet: libcephfs2-2:19.0.0-2206.gb6ae2491.el9.x86_64 11/198 2024-03-19T01:25:24.567 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-rados-2:19.0.0-2206.gb6ae2491.el9.x86_64 12/198 2024-03-19T01:25:24.604 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : postgresql-private-libs-13.11-1.el9.x86_64 13/198 2024-03-19T01:25:24.788 INFO:teuthology.orchestra.run.smithi115.stdout: Upgrading : elfutils-libelf-0.190-2.el9.x86_64 14/198 2024-03-19T01:25:24.833 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : postgresql-13.11-1.el9.x86_64 15/198 2024-03-19T01:25:24.862 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : libcephsqlite-2:19.0.0-2206.gb6ae2491.el9.x86_64 16/198 2024-03-19T01:25:24.965 INFO:teuthology.orchestra.run.smithi115.stdout: Running scriptlet: libcephsqlite-2:19.0.0-2206.gb6ae2491.el9.x86_64 16/198 2024-03-19T01:25:24.988 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : libradosstriper1-2:19.0.0-2206.gb6ae2491.el9.x86 17/198 2024-03-19T01:25:25.051 INFO:teuthology.orchestra.run.smithi115.stdout: Running scriptlet: libradosstriper1-2:19.0.0-2206.gb6ae2491.el9.x86 17/198 2024-03-19T01:25:25.100 INFO:teuthology.orchestra.run.smithi115.stdout: Upgrading : libmount-2.37.4-18.el9.x86_64 18/198 2024-03-19T01:25:25.123 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : qatlib-service-23.11.0-1.el9.x86_64 19/198 2024-03-19T01:25:25.137 INFO:teuthology.orchestra.run.smithi115.stdout: Running scriptlet: qatlib-service-23.11.0-1.el9.x86_64 19/198 2024-03-19T01:25:25.272 INFO:teuthology.orchestra.run.smithi115.stdout: Running scriptlet: qatlib-23.11.0-1.el9.x86_64 20/198 2024-03-19T01:25:25.303 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : qatlib-23.11.0-1.el9.x86_64 20/198 2024-03-19T01:25:25.335 INFO:teuthology.orchestra.run.smithi115.stdout: Upgrading : libsmartcols-2.37.4-18.el9.x86_64 21/198 2024-03-19T01:25:25.431 INFO:teuthology.orchestra.run.smithi115.stdout: Upgrading : libacl-2.3.1-4.el9.x86_64 22/198 2024-03-19T01:25:25.546 INFO:teuthology.orchestra.run.smithi115.stdout: Upgrading : NetworkManager-libnm-1:1.47.2-1.el9.x86_64 23/198 2024-03-19T01:25:25.606 INFO:teuthology.orchestra.run.smithi115.stdout: Running scriptlet: NetworkManager-libnm-1:1.47.2-1.el9.x86_64 23/198 2024-03-19T01:25:25.670 INFO:teuthology.orchestra.run.smithi115.stdout: Running scriptlet: NetworkManager-1:1.47.2-1.el9.x86_64 24/198 2024-03-19T01:25:25.724 INFO:teuthology.orchestra.run.smithi115.stdout: Upgrading : NetworkManager-1:1.47.2-1.el9.x86_64 24/198 2024-03-19T01:25:25.947 INFO:teuthology.orchestra.run.smithi115.stdout: Running scriptlet: NetworkManager-1:1.47.2-1.el9.x86_64 24/198 2024-03-19T01:25:26.053 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : re2-1:20211101-3.el9.x86_64 25/198 2024-03-19T01:25:26.139 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : libarrow-9.0.0-10.el9.x86_64 26/198 2024-03-19T01:25:26.170 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 27/198 2024-03-19T01:25:26.197 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : liboath-2.6.7-2.el9.x86_64 28/198 2024-03-19T01:25:26.232 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : libquadmath-11.4.1-3.el9.x86_64 29/198 2024-03-19T01:25:26.314 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : libgfortran-11.4.1-3.el9.x86_64 30/198 2024-03-19T01:25:26.369 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-urllib3-1.26.5-5.el9.noarch 31/198 2024-03-19T01:25:26.407 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 32/198 2024-03-19T01:25:26.471 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 33/198 2024-03-19T01:25:26.507 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 34/198 2024-03-19T01:25:26.550 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 35/198 2024-03-19T01:25:26.583 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : protobuf-3.14.0-13.el9.x86_64 36/198 2024-03-19T01:25:26.611 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : lua-5.4.4-4.el9.x86_64 37/198 2024-03-19T01:25:26.680 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 38/198 2024-03-19T01:25:26.716 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-ceph-common-2:19.0.0-2206.gb6ae2491.el9. 39/198 2024-03-19T01:25:26.759 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-ceph-argparse-2:19.0.0-2206.gb6ae2491.el 40/198 2024-03-19T01:25:26.788 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-cephfs-2:19.0.0-2206.gb6ae2491.el9.x86_6 41/198 2024-03-19T01:25:26.856 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : lua-devel-5.4.4-4.el9.x86_64 42/198 2024-03-19T01:25:26.902 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : luarocks-3.9.2-1.el9.noarch 43/198 2024-03-19T01:25:26.984 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : protobuf-compiler-3.14.0-13.el9.x86_64 44/198 2024-03-19T01:25:27.125 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 45/198 2024-03-19T01:25:27.179 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 46/198 2024-03-19T01:25:27.694 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-rsa-4.9-2.el9.noarch 47/198 2024-03-19T01:25:27.774 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 48/198 2024-03-19T01:25:27.845 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-jinja2-2.11.3-5.el9.noarch 49/198 2024-03-19T01:25:27.929 INFO:teuthology.orchestra.run.smithi115.stdout: Upgrading : util-linux-core-2.37.4-18.el9.x86_64 50/198 2024-03-19T01:25:27.936 INFO:teuthology.orchestra.run.smithi115.stdout: Running scriptlet: util-linux-core-2.37.4-18.el9.x86_64 50/198 2024-03-19T01:25:27.969 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : qatzip-libs-1.1.2-2.el9.x86_64 51/198 2024-03-19T01:25:28.009 INFO:teuthology.orchestra.run.smithi115.stdout: Upgrading : elfutils-libs-0.190-2.el9.x86_64 52/198 2024-03-19T01:25:28.047 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : librados-devel-2:19.0.0-2206.gb6ae2491.el9.x86_6 53/198 2024-03-19T01:25:28.076 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 54/198 2024-03-19T01:25:28.291 INFO:teuthology.orchestra.run.smithi115.stdout: Upgrading : libfdisk-2.37.4-18.el9.x86_64 55/198 2024-03-19T01:25:28.456 INFO:teuthology.orchestra.run.smithi115.stdout: Upgrading : util-linux-2.37.4-18.el9.x86_64 56/198 2024-03-19T01:25:28.546 INFO:teuthology.orchestra.run.smithi115.stdout: Upgrading : xfsprogs-6.3.0-1.el9.x86_64 57/198 2024-03-19T01:25:28.581 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : libuuid-devel-2.37.4-18.el9.x86_64 58/198 2024-03-19T01:25:28.720 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : socat-1.7.4.1-5.el9.x86_64 59/198 2024-03-19T01:25:28.776 INFO:teuthology.orchestra.run.smithi115.stdout: Upgrading : openssl-1:3.0.7-27.el9.x86_64 60/198 2024-03-19T01:25:28.831 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 61/198 2024-03-19T01:25:28.895 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 62/198 2024-03-19T01:25:28.935 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-webob-1.8.7-6.el9.noarch 63/198 2024-03-19T01:25:28.972 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-typing-extensions-4.4.0-2.el9.noarch 64/198 2024-03-19T01:25:29.018 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 65/198 2024-03-19T01:25:29.063 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 66/198 2024-03-19T01:25:29.106 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-pyperclip-1.8.2-2.el9.noarch 67/198 2024-03-19T01:25:29.148 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 68/198 2024-03-19T01:25:29.280 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 69/198 2024-03-19T01:25:29.320 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 70/198 2024-03-19T01:25:29.350 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-colorama-0.4.4-12.el9.noarch 71/198 2024-03-19T01:25:29.392 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-certifi-2021.10.8-2.el9.noarch 72/198 2024-03-19T01:25:29.485 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 73/198 2024-03-19T01:25:29.525 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-google-auth-1:2.28.0-1.el9.noarch 74/198 2024-03-19T01:25:29.565 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 75/198 2024-03-19T01:25:29.591 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : gperftools-libs-2.9.1-2.el9.x86_64 76/198 2024-03-19T01:25:29.616 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : libarrow-doc-9.0.0-10.el9.noarch 77/198 2024-03-19T01:25:29.688 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : grpc-data-1.46.7-10.el9.noarch 78/198 2024-03-19T01:25:29.807 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : abseil-cpp-20211102.0-3.el9.x86_64 79/198 2024-03-19T01:25:29.906 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-grpcio-1.46.7-10.el9.x86_64 80/198 2024-03-19T01:25:29.958 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-devel-3.9.17-2.el9.x86_64 81/198 2024-03-19T01:25:30.006 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-ply-3.11-14.el9.noarch 82/198 2024-03-19T01:25:30.061 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 83/198 2024-03-19T01:25:30.244 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 84/198 2024-03-19T01:25:30.293 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 85/198 2024-03-19T01:25:30.422 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 86/198 2024-03-19T01:25:30.460 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-asyncssh-2.13.2-4.el9.noarch 87/198 2024-03-19T01:25:30.492 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 88/198 2024-03-19T01:25:30.523 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : fuse-2.9.9-15.el9.x86_64 89/198 2024-03-19T01:25:30.550 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : zlib-devel-1.2.11-40.el9.x86_64 90/198 2024-03-19T01:25:30.585 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 91/198 2024-03-19T01:25:30.624 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : uuid-1.6.2-55.el9.x86_64 92/198 2024-03-19T01:25:30.656 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-wcwidth-0.2.5-8.el9.noarch 93/198 2024-03-19T01:25:30.686 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 94/198 2024-03-19T01:25:30.761 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 95/198 2024-03-19T01:25:30.796 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 96/198 2024-03-19T01:25:30.829 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 97/198 2024-03-19T01:25:30.870 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 98/198 2024-03-19T01:25:30.905 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 99/198 2024-03-19T01:25:30.994 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 100/198 2024-03-19T01:25:31.075 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 101/198 2024-03-19T01:25:31.128 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-protobuf-3.14.0-13.el9.noarch 102/198 2024-03-19T01:25:31.159 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-grpcio-tools-1.46.7-10.el9.x86_64 103/198 2024-03-19T01:25:31.215 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 104/198 2024-03-19T01:25:31.311 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-packaging-20.9-5.el9.noarch 105/198 2024-03-19T01:25:31.361 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 106/198 2024-03-19T01:25:32.082 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 107/198 2024-03-19T01:25:32.125 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 108/198 2024-03-19T01:25:32.159 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 109/198 2024-03-19T01:25:32.206 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 110/198 2024-03-19T01:25:32.284 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-attrs-20.3.0-7.el9.noarch 111/198 2024-03-19T01:25:32.335 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-cmd2-2.4.2-2.el9.noarch 112/198 2024-03-19T01:25:32.510 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : openblas-0.3.21-2.el9.x86_64 113/198 2024-03-19T01:25:32.549 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : openblas-openmp-0.3.21-2.el9.x86_64 114/198 2024-03-19T01:25:32.625 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 115/198 2024-03-19T01:25:33.243 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 116/198 2024-03-19T01:25:33.354 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 117/198 2024-03-19T01:25:34.549 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 118/198 2024-03-19T01:25:34.641 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 119/198 2024-03-19T01:25:34.672 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : oniguruma-6.9.6-1.el9.5.x86_64 120/198 2024-03-19T01:25:34.703 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : jq-1.6-16.el9.x86_64 121/198 2024-03-19T01:25:34.739 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : libzstd-devel-1.5.1-2.el9.x86_64 122/198 2024-03-19T01:25:35.049 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 123/198 2024-03-19T01:25:35.078 INFO:teuthology.orchestra.run.smithi115.stdout: Upgrading : librbd1-2:19.0.0-2206.gb6ae2491.el9.x86_64 124/198 2024-03-19T01:25:35.168 INFO:teuthology.orchestra.run.smithi115.stdout: Running scriptlet: librbd1-2:19.0.0-2206.gb6ae2491.el9.x86_64 124/198 2024-03-19T01:25:35.199 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-rbd-2:19.0.0-2206.gb6ae2491.el9.x86_64 125/198 2024-03-19T01:25:35.235 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : boost-program-options-1.75.0-8.el9.x86_64 126/198 2024-03-19T01:25:35.692 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : parquet-libs-9.0.0-10.el9.x86_64 127/198 2024-03-19T01:25:35.722 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : librgw2-2:19.0.0-2206.gb6ae2491.el9.x86_64 128/198 2024-03-19T01:25:35.776 INFO:teuthology.orchestra.run.smithi115.stdout: Running scriptlet: librgw2-2:19.0.0-2206.gb6ae2491.el9.x86_64 128/198 2024-03-19T01:25:35.799 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : python3-rgw-2:19.0.0-2206.gb6ae2491.el9.x86_64 129/198 2024-03-19T01:25:38.017 INFO:teuthology.orchestra.run.smithi115.stdout: Running scriptlet: ceph-common-2:19.0.0-2206.gb6ae2491.el9.x86_64 130/198 2024-03-19T01:25:38.054 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : ceph-common-2:19.0.0-2206.gb6ae2491.el9.x86_64 130/198 2024-03-19T01:25:38.598 INFO:teuthology.orchestra.run.smithi115.stdout: Running scriptlet: ceph-common-2:19.0.0-2206.gb6ae2491.el9.x86_64 130/198 2024-03-19T01:25:38.641 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : ceph-base-2:19.0.0-2206.gb6ae2491.el9.x86_64 131/198 2024-03-19T01:25:38.704 INFO:teuthology.orchestra.run.smithi115.stdout: Running scriptlet: ceph-base-2:19.0.0-2206.gb6ae2491.el9.x86_64 131/198 2024-03-19T01:25:38.704 INFO:teuthology.orchestra.run.smithi115.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-03-19T01:25:38.705 INFO:teuthology.orchestra.run.smithi115.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-03-19T01:25:38.705 INFO:teuthology.orchestra.run.smithi115.stdout: 2024-03-19T01:25:38.734 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : ceph-selinux-2:19.0.0-2206.gb6ae2491.el9.x86_64 132/198 2024-03-19T01:25:50.620 INFO:teuthology.orchestra.run.smithi115.stdout: Running scriptlet: ceph-selinux-2:19.0.0-2206.gb6ae2491.el9.x86_64 132/198 2024-03-19T01:25:50.621 INFO:teuthology.orchestra.run.smithi115.stdout:skipping the directory /sys 2024-03-19T01:25:50.621 INFO:teuthology.orchestra.run.smithi115.stdout:skipping the directory /proc 2024-03-19T01:25:50.621 INFO:teuthology.orchestra.run.smithi115.stdout:skipping the directory /mnt 2024-03-19T01:25:50.621 INFO:teuthology.orchestra.run.smithi115.stdout:skipping the directory /var/tmp 2024-03-19T01:25:50.621 INFO:teuthology.orchestra.run.smithi115.stdout:skipping the directory /home 2024-03-19T01:25:50.621 INFO:teuthology.orchestra.run.smithi115.stdout:skipping the directory /root 2024-03-19T01:25:50.621 INFO:teuthology.orchestra.run.smithi115.stdout:skipping the directory /tmp 2024-03-19T01:25:50.621 INFO:teuthology.orchestra.run.smithi115.stdout: 2024-03-19T01:25:50.781 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : ceph-volume-2:19.0.0-2206.gb6ae2491.el9.noarch 133/198 2024-03-19T01:25:50.796 INFO:teuthology.orchestra.run.smithi115.stdout: Running scriptlet: ceph-volume-2:19.0.0-2206.gb6ae2491.el9.noarch 133/198 2024-03-19T01:25:50.796 INFO:teuthology.orchestra.run.smithi115.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-19T01:25:50.796 INFO:teuthology.orchestra.run.smithi115.stdout:Invalid unit name "ceph-volume@*.service" escaped as "ceph-volume@\x2a.service". 2024-03-19T01:25:50.796 INFO:teuthology.orchestra.run.smithi115.stdout: 2024-03-19T01:25:52.266 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : ceph-osd-2:19.0.0-2206.gb6ae2491.el9.x86_64 134/198 2024-03-19T01:25:52.292 INFO:teuthology.orchestra.run.smithi115.stdout: Running scriptlet: ceph-osd-2:19.0.0-2206.gb6ae2491.el9.x86_64 134/198 2024-03-19T01:25:52.292 INFO:teuthology.orchestra.run.smithi115.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-19T01:25:52.292 INFO:teuthology.orchestra.run.smithi115.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-03-19T01:25:52.292 INFO:teuthology.orchestra.run.smithi115.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-03-19T01:25:52.292 INFO:teuthology.orchestra.run.smithi115.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-03-19T01:25:52.292 INFO:teuthology.orchestra.run.smithi115.stdout: 2024-03-19T01:25:52.518 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : ceph-mds-2:19.0.0-2206.gb6ae2491.el9.x86_64 135/198 2024-03-19T01:25:52.541 INFO:teuthology.orchestra.run.smithi115.stdout: Running scriptlet: ceph-mds-2:19.0.0-2206.gb6ae2491.el9.x86_64 135/198 2024-03-19T01:25:52.541 INFO:teuthology.orchestra.run.smithi115.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-19T01:25:52.541 INFO:teuthology.orchestra.run.smithi115.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-03-19T01:25:52.541 INFO:teuthology.orchestra.run.smithi115.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-03-19T01:25:52.541 INFO:teuthology.orchestra.run.smithi115.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-03-19T01:25:52.541 INFO:teuthology.orchestra.run.smithi115.stdout: 2024-03-19T01:25:52.995 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : ceph-mon-2:19.0.0-2206.gb6ae2491.el9.x86_64 136/198 2024-03-19T01:25:53.017 INFO:teuthology.orchestra.run.smithi115.stdout: Running scriptlet: ceph-mon-2:19.0.0-2206.gb6ae2491.el9.x86_64 136/198 2024-03-19T01:25:53.017 INFO:teuthology.orchestra.run.smithi115.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-19T01:25:53.018 INFO:teuthology.orchestra.run.smithi115.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-03-19T01:25:53.018 INFO:teuthology.orchestra.run.smithi115.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-03-19T01:25:53.018 INFO:teuthology.orchestra.run.smithi115.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-03-19T01:25:53.018 INFO:teuthology.orchestra.run.smithi115.stdout: 2024-03-19T01:25:53.105 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : automake-1.16.2-8.el9.noarch 137/198 2024-03-19T01:25:53.532 INFO:teuthology.orchestra.run.smithi115.stdout: Running scriptlet: cephadm-2:19.0.0-2206.gb6ae2491.el9.noarch 138/198 2024-03-19T01:25:53.562 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : cephadm-2:19.0.0-2206.gb6ae2491.el9.noarch 138/198 2024-03-19T01:25:53.594 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : ceph-prometheus-alerts-2:19.0.0-2206.gb6ae2491.e 139/198 2024-03-19T01:25:53.655 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : ceph-grafana-dashboards-2:19.0.0-2206.gb6ae2491. 140/198 2024-03-19T01:25:53.685 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : ceph-mgr-cephadm-2:19.0.0-2206.gb6ae2491.el9.noa 141/198 2024-03-19T01:25:53.918 INFO:teuthology.orchestra.run.smithi115.stdout: Running scriptlet: ceph-mgr-cephadm-2:19.0.0-2206.gb6ae2491.el9.noa 141/198 2024-03-19T01:25:53.944 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : ceph-mgr-dashboard-2:19.0.0-2206.gb6ae2491.el9.n 142/198 2024-03-19T01:25:54.905 INFO:teuthology.orchestra.run.smithi115.stdout: Running scriptlet: ceph-mgr-dashboard-2:19.0.0-2206.gb6ae2491.el9.n 142/198 2024-03-19T01:25:54.936 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : ceph-mgr-diskprediction-local-2:19.0.0-2206.gb6a 143/198 2024-03-19T01:25:55.037 INFO:teuthology.orchestra.run.smithi115.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:19.0.0-2206.gb6a 143/198 2024-03-19T01:25:55.198 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : ceph-mgr-modules-core-2:19.0.0-2206.gb6ae2491.el 144/198 2024-03-19T01:25:55.230 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : ceph-mgr-2:19.0.0-2206.gb6ae2491.el9.x86_64 145/198 2024-03-19T01:25:55.253 INFO:teuthology.orchestra.run.smithi115.stdout: Running scriptlet: ceph-mgr-2:19.0.0-2206.gb6ae2491.el9.x86_64 145/198 2024-03-19T01:25:55.253 INFO:teuthology.orchestra.run.smithi115.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-19T01:25:55.253 INFO:teuthology.orchestra.run.smithi115.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-03-19T01:25:55.253 INFO:teuthology.orchestra.run.smithi115.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-03-19T01:25:55.253 INFO:teuthology.orchestra.run.smithi115.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-03-19T01:25:55.253 INFO:teuthology.orchestra.run.smithi115.stdout: 2024-03-19T01:25:55.297 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : ceph-mgr-rook-2:19.0.0-2206.gb6ae2491.el9.noarch 146/198 2024-03-19T01:25:55.308 INFO:teuthology.orchestra.run.smithi115.stdout: Running scriptlet: ceph-mgr-rook-2:19.0.0-2206.gb6ae2491.el9.noarch 146/198 2024-03-19T01:25:55.353 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : ceph-2:19.0.0-2206.gb6ae2491.el9.x86_64 147/198 2024-03-19T01:25:56.229 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : libtool-2.4.6-45.el9.x86_64 148/198 2024-03-19T01:25:56.266 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : ceph-radosgw-2:19.0.0-2206.gb6ae2491.el9.x86_64 149/198 2024-03-19T01:25:56.289 INFO:teuthology.orchestra.run.smithi115.stdout: Running scriptlet: ceph-radosgw-2:19.0.0-2206.gb6ae2491.el9.x86_64 149/198 2024-03-19T01:25:56.289 INFO:teuthology.orchestra.run.smithi115.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-19T01:25:56.289 INFO:teuthology.orchestra.run.smithi115.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-03-19T01:25:56.289 INFO:teuthology.orchestra.run.smithi115.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-03-19T01:25:56.289 INFO:teuthology.orchestra.run.smithi115.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-03-19T01:25:56.289 INFO:teuthology.orchestra.run.smithi115.stdout: 2024-03-19T01:25:56.333 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : ceph-immutable-object-cache-2:19.0.0-2206.gb6ae2 150/198 2024-03-19T01:25:56.355 INFO:teuthology.orchestra.run.smithi115.stdout: Running scriptlet: ceph-immutable-object-cache-2:19.0.0-2206.gb6ae2 150/198 2024-03-19T01:25:56.355 INFO:teuthology.orchestra.run.smithi115.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-19T01:25:56.355 INFO:teuthology.orchestra.run.smithi115.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-03-19T01:25:56.355 INFO:teuthology.orchestra.run.smithi115.stdout: 2024-03-19T01:25:56.400 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : cephfs-mirror-2:19.0.0-2206.gb6ae2491.el9.x86_64 151/198 2024-03-19T01:25:56.423 INFO:teuthology.orchestra.run.smithi115.stdout: Running scriptlet: cephfs-mirror-2:19.0.0-2206.gb6ae2491.el9.x86_64 151/198 2024-03-19T01:25:56.423 INFO:teuthology.orchestra.run.smithi115.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-19T01:25:56.423 INFO:teuthology.orchestra.run.smithi115.stdout:Invalid unit name "cephfs-mirror@*.service" escaped as "cephfs-mirror@\x2a.service". 2024-03-19T01:25:56.423 INFO:teuthology.orchestra.run.smithi115.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/cephfs-mirror.target → /usr/lib/systemd/system/cephfs-mirror.target. 2024-03-19T01:25:56.423 INFO:teuthology.orchestra.run.smithi115.stdout:Created symlink /etc/systemd/system/ceph.target.wants/cephfs-mirror.target → /usr/lib/systemd/system/cephfs-mirror.target. 2024-03-19T01:25:56.423 INFO:teuthology.orchestra.run.smithi115.stdout: 2024-03-19T01:25:56.726 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : rbd-mirror-2:19.0.0-2206.gb6ae2491.el9.x86_64 152/198 2024-03-19T01:25:56.748 INFO:teuthology.orchestra.run.smithi115.stdout: Running scriptlet: rbd-mirror-2:19.0.0-2206.gb6ae2491.el9.x86_64 152/198 2024-03-19T01:25:56.748 INFO:teuthology.orchestra.run.smithi115.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-19T01:25:56.748 INFO:teuthology.orchestra.run.smithi115.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-03-19T01:25:56.748 INFO:teuthology.orchestra.run.smithi115.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-03-19T01:25:56.748 INFO:teuthology.orchestra.run.smithi115.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-03-19T01:25:56.748 INFO:teuthology.orchestra.run.smithi115.stdout: 2024-03-19T01:26:01.180 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : ceph-test-2:19.0.0-2206.gb6ae2491.el9.x86_64 153/198 2024-03-19T01:26:01.221 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : rbd-fuse-2:19.0.0-2206.gb6ae2491.el9.x86_64 154/198 2024-03-19T01:26:01.260 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : rbd-nbd-2:19.0.0-2206.gb6ae2491.el9.x86_64 155/198 2024-03-19T01:26:01.287 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : elfutils-libelf-devel-0.190-2.el9.x86_64 156/198 2024-03-19T01:26:01.469 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : cephfs-shell-2:19.0.0-2206.gb6ae2491.el9.x86_64 157/198 2024-03-19T01:26:01.595 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : postgresql-contrib-13.11-1.el9.x86_64 158/198 2024-03-19T01:26:01.636 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : ceph-fuse-2:19.0.0-2206.gb6ae2491.el9.x86_64 159/198 2024-03-19T01:26:01.675 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : xfsprogs-devel-6.3.0-1.el9.x86_64 160/198 2024-03-19T01:26:01.728 INFO:teuthology.orchestra.run.smithi115.stdout: Upgrading : xfsdump-3.1.12-4.el9.x86_64 161/198 2024-03-19T01:26:01.762 INFO:teuthology.orchestra.run.smithi115.stdout: Upgrading : util-linux-user-2.37.4-18.el9.x86_64 162/198 2024-03-19T01:26:01.795 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : libcephfs-devel-2:19.0.0-2206.gb6ae2491.el9.x86_ 163/198 2024-03-19T01:26:01.827 INFO:teuthology.orchestra.run.smithi115.stdout: Upgrading : elfutils-debuginfod-client-0.190-2.el9.x86_64 164/198 2024-03-19T01:26:01.865 INFO:teuthology.orchestra.run.smithi115.stdout: Upgrading : NetworkManager-team-1:1.47.2-1.el9.x86_64 165/198 2024-03-19T01:26:01.911 INFO:teuthology.orchestra.run.smithi115.stdout: Upgrading : NetworkManager-tui-1:1.47.2-1.el9.x86_64 166/198 2024-03-19T01:26:01.977 INFO:teuthology.orchestra.run.smithi115.stdout: Upgrading : acl-2.3.1-4.el9.x86_64 167/198 2024-03-19T01:26:02.022 INFO:teuthology.orchestra.run.smithi115.stdout: Upgrading : libacl-devel-2.3.1-4.el9.x86_64 168/198 2024-03-19T01:26:02.634 INFO:teuthology.orchestra.run.smithi115.stdout: Running scriptlet: postgresql-server-13.11-1.el9.x86_64 169/198 2024-03-19T01:26:02.679 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : postgresql-server-13.11-1.el9.x86_64 169/198 2024-03-19T01:26:02.698 INFO:teuthology.orchestra.run.smithi115.stdout: Running scriptlet: postgresql-server-13.11-1.el9.x86_64 169/198 2024-03-19T01:26:03.520 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : cephfs-top-2:19.0.0-2206.gb6ae2491.el9.noarch 170/198 2024-03-19T01:26:03.599 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : openssl-devel-1:3.0.7-27.el9.x86_64 171/198 2024-03-19T01:26:03.639 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : libaio-devel-0.3.111-13.el9.x86_64 172/198 2024-03-19T01:26:03.721 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : flex-2.6.4-9.el9.x86_64 173/198 2024-03-19T01:26:03.769 INFO:teuthology.orchestra.run.smithi115.stdout: Installing : bison-3.7.4-5.el9.x86_64 174/198 2024-03-19T01:26:03.794 INFO:teuthology.orchestra.run.smithi115.stdout: Cleanup : NetworkManager-tui-1:1.44.0-2.el9.x86_64 175/198 2024-03-19T01:26:03.819 INFO:teuthology.orchestra.run.smithi115.stdout: Cleanup : elfutils-debuginfod-client-0.189-3.el9.x86_64 176/198 2024-03-19T01:26:03.846 INFO:teuthology.orchestra.run.smithi115.stdout: Cleanup : elfutils-libs-0.189-3.el9.x86_64 177/198 2024-03-19T01:26:03.846 INFO:teuthology.orchestra.run.smithi115.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 178/198 2024-03-19T01:26:03.934 INFO:teuthology.orchestra.run.smithi115.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 178/198 2024-03-19T01:26:03.964 INFO:teuthology.orchestra.run.smithi115.stdout: Cleanup : openssl-1:3.0.7-24.el9.x86_64 179/198 2024-03-19T01:26:03.964 INFO:teuthology.orchestra.run.smithi115.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 180/198 2024-03-19T01:26:04.048 INFO:teuthology.orchestra.run.smithi115.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 180/198 2024-03-19T01:26:04.074 INFO:teuthology.orchestra.run.smithi115.stdout: Cleanup : xfsdump-3.1.12-3.el9.x86_64 181/198 2024-03-19T01:26:04.107 INFO:teuthology.orchestra.run.smithi115.stdout: Cleanup : xfsprogs-5.19.0-4.el9.x86_64 182/198 2024-03-19T01:26:04.133 INFO:teuthology.orchestra.run.smithi115.stdout: Cleanup : acl-2.3.1-3.el9.x86_64 183/198 2024-03-19T01:26:04.158 INFO:teuthology.orchestra.run.smithi115.stdout: Cleanup : libacl-devel-2.3.1-3.el9.x86_64 184/198 2024-03-19T01:26:04.182 INFO:teuthology.orchestra.run.smithi115.stdout: Cleanup : util-linux-user-2.37.4-15.el9.x86_64 185/198 2024-03-19T01:26:04.222 INFO:teuthology.orchestra.run.smithi115.stdout: Cleanup : util-linux-2.37.4-15.el9.x86_64 186/198 2024-03-19T01:26:04.254 INFO:teuthology.orchestra.run.smithi115.stdout: Cleanup : util-linux-core-2.37.4-15.el9.x86_64 187/198 2024-03-19T01:26:04.280 INFO:teuthology.orchestra.run.smithi115.stdout: Cleanup : libfdisk-2.37.4-15.el9.x86_64 188/198 2024-03-19T01:26:04.305 INFO:teuthology.orchestra.run.smithi115.stdout: Cleanup : libmount-2.37.4-15.el9.x86_64 189/198 2024-03-19T01:26:04.330 INFO:teuthology.orchestra.run.smithi115.stdout: Cleanup : libblkid-2.37.4-15.el9.x86_64 190/198 2024-03-19T01:26:04.356 INFO:teuthology.orchestra.run.smithi115.stdout: Cleanup : NetworkManager-team-1:1.44.0-2.el9.x86_64 191/198 2024-03-19T01:26:04.544 INFO:teuthology.orchestra.run.smithi115.stdout: Running scriptlet: NetworkManager-1:1.44.0-2.el9.x86_64 192/198 2024-03-19T01:26:04.545 INFO:teuthology.orchestra.run.smithi115.stdout: Cleanup : NetworkManager-1:1.44.0-2.el9.x86_64 192/198 2024-03-19T01:26:04.545 INFO:teuthology.orchestra.run.smithi115.stdout: Running scriptlet: NetworkManager-1:1.44.0-2.el9.x86_64 192/198 2024-03-19T01:26:04.545 INFO:teuthology.orchestra.run.smithi115.stdout: Cleanup : NetworkManager-libnm-1:1.44.0-2.el9.x86_64 193/198 2024-03-19T01:26:04.608 INFO:teuthology.orchestra.run.smithi115.stdout: Running scriptlet: NetworkManager-libnm-1:1.44.0-2.el9.x86_64 193/198 2024-03-19T01:26:04.633 INFO:teuthology.orchestra.run.smithi115.stdout: Cleanup : libuuid-2.37.4-15.el9.x86_64 194/198 2024-03-19T01:26:04.658 INFO:teuthology.orchestra.run.smithi115.stdout: Cleanup : libsmartcols-2.37.4-15.el9.x86_64 195/198 2024-03-19T01:26:04.691 INFO:teuthology.orchestra.run.smithi115.stdout: Cleanup : libacl-2.3.1-3.el9.x86_64 196/198 2024-03-19T01:26:04.717 INFO:teuthology.orchestra.run.smithi115.stdout: Cleanup : openssl-libs-1:3.0.7-24.el9.x86_64 197/198 2024-03-19T01:26:04.782 INFO:teuthology.orchestra.run.smithi115.stdout: Cleanup : elfutils-libelf-0.189-3.el9.x86_64 198/198 2024-03-19T01:26:09.566 INFO:teuthology.orchestra.run.smithi115.stdout: Running scriptlet: elfutils-libelf-0.189-3.el9.x86_64 198/198 2024-03-19T01:26:09.567 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : ceph-2:19.0.0-2206.gb6ae2491.el9.x86_64 1/198 2024-03-19T01:26:09.567 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : ceph-base-2:19.0.0-2206.gb6ae2491.el9.x86_64 2/198 2024-03-19T01:26:09.567 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : ceph-common-2:19.0.0-2206.gb6ae2491.el9.x86_64 3/198 2024-03-19T01:26:09.567 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : ceph-fuse-2:19.0.0-2206.gb6ae2491.el9.x86_64 4/198 2024-03-19T01:26:09.568 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : ceph-immutable-object-cache-2:19.0.0-2206.gb6ae2 5/198 2024-03-19T01:26:09.568 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : ceph-mds-2:19.0.0-2206.gb6ae2491.el9.x86_64 6/198 2024-03-19T01:26:09.568 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : ceph-mgr-2:19.0.0-2206.gb6ae2491.el9.x86_64 7/198 2024-03-19T01:26:09.568 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : ceph-mon-2:19.0.0-2206.gb6ae2491.el9.x86_64 8/198 2024-03-19T01:26:09.568 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : ceph-osd-2:19.0.0-2206.gb6ae2491.el9.x86_64 9/198 2024-03-19T01:26:09.569 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : ceph-radosgw-2:19.0.0-2206.gb6ae2491.el9.x86_64 10/198 2024-03-19T01:26:09.569 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : ceph-selinux-2:19.0.0-2206.gb6ae2491.el9.x86_64 11/198 2024-03-19T01:26:09.569 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : ceph-test-2:19.0.0-2206.gb6ae2491.el9.x86_64 12/198 2024-03-19T01:26:09.569 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : cephfs-mirror-2:19.0.0-2206.gb6ae2491.el9.x86_64 13/198 2024-03-19T01:26:09.569 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : cephfs-shell-2:19.0.0-2206.gb6ae2491.el9.x86_64 14/198 2024-03-19T01:26:09.569 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : libcephfs-devel-2:19.0.0-2206.gb6ae2491.el9.x86_ 15/198 2024-03-19T01:26:09.569 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : libcephfs2-2:19.0.0-2206.gb6ae2491.el9.x86_64 16/198 2024-03-19T01:26:09.569 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : libcephsqlite-2:19.0.0-2206.gb6ae2491.el9.x86_64 17/198 2024-03-19T01:26:09.570 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : librados-devel-2:19.0.0-2206.gb6ae2491.el9.x86_6 18/198 2024-03-19T01:26:09.570 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : libradosstriper1-2:19.0.0-2206.gb6ae2491.el9.x86 19/198 2024-03-19T01:26:09.570 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : librgw2-2:19.0.0-2206.gb6ae2491.el9.x86_64 20/198 2024-03-19T01:26:09.570 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-ceph-argparse-2:19.0.0-2206.gb6ae2491.el 21/198 2024-03-19T01:26:09.570 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-ceph-common-2:19.0.0-2206.gb6ae2491.el9. 22/198 2024-03-19T01:26:09.571 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-cephfs-2:19.0.0-2206.gb6ae2491.el9.x86_6 23/198 2024-03-19T01:26:09.571 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-rados-2:19.0.0-2206.gb6ae2491.el9.x86_64 24/198 2024-03-19T01:26:09.571 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-rbd-2:19.0.0-2206.gb6ae2491.el9.x86_64 25/198 2024-03-19T01:26:09.571 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-rgw-2:19.0.0-2206.gb6ae2491.el9.x86_64 26/198 2024-03-19T01:26:09.571 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : rbd-fuse-2:19.0.0-2206.gb6ae2491.el9.x86_64 27/198 2024-03-19T01:26:09.571 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : rbd-mirror-2:19.0.0-2206.gb6ae2491.el9.x86_64 28/198 2024-03-19T01:26:09.572 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : rbd-nbd-2:19.0.0-2206.gb6ae2491.el9.x86_64 29/198 2024-03-19T01:26:09.572 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : ceph-grafana-dashboards-2:19.0.0-2206.gb6ae2491. 30/198 2024-03-19T01:26:09.572 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : ceph-mgr-cephadm-2:19.0.0-2206.gb6ae2491.el9.noa 31/198 2024-03-19T01:26:09.572 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : ceph-mgr-dashboard-2:19.0.0-2206.gb6ae2491.el9.n 32/198 2024-03-19T01:26:09.573 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : ceph-mgr-diskprediction-local-2:19.0.0-2206.gb6a 33/198 2024-03-19T01:26:09.573 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : ceph-mgr-modules-core-2:19.0.0-2206.gb6ae2491.el 34/198 2024-03-19T01:26:09.573 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : ceph-mgr-rook-2:19.0.0-2206.gb6ae2491.el9.noarch 35/198 2024-03-19T01:26:09.573 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : ceph-prometheus-alerts-2:19.0.0-2206.gb6ae2491.e 36/198 2024-03-19T01:26:09.573 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : ceph-volume-2:19.0.0-2206.gb6ae2491.el9.noarch 37/198 2024-03-19T01:26:09.574 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : cephadm-2:19.0.0-2206.gb6ae2491.el9.noarch 38/198 2024-03-19T01:26:09.574 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : cephfs-top-2:19.0.0-2206.gb6ae2491.el9.noarch 39/198 2024-03-19T01:26:09.574 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : automake-1.16.2-8.el9.noarch 40/198 2024-03-19T01:26:09.574 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : bison-3.7.4-5.el9.x86_64 41/198 2024-03-19T01:26:09.574 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : boost-program-options-1.75.0-8.el9.x86_64 42/198 2024-03-19T01:26:09.574 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : flex-2.6.4-9.el9.x86_64 43/198 2024-03-19T01:26:09.575 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 44/198 2024-03-19T01:26:09.575 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 45/198 2024-03-19T01:26:09.575 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 46/198 2024-03-19T01:26:09.575 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : libaio-devel-0.3.111-13.el9.x86_64 47/198 2024-03-19T01:26:09.575 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 48/198 2024-03-19T01:26:09.575 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 49/198 2024-03-19T01:26:09.575 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 50/198 2024-03-19T01:26:09.575 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : libtool-2.4.6-45.el9.x86_64 51/198 2024-03-19T01:26:09.575 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : libzstd-devel-1.5.1-2.el9.x86_64 52/198 2024-03-19T01:26:09.575 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 53/198 2024-03-19T01:26:09.575 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : lua-5.4.4-4.el9.x86_64 54/198 2024-03-19T01:26:09.575 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : oniguruma-6.9.6-1.el9.5.x86_64 55/198 2024-03-19T01:26:09.575 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : openblas-0.3.21-2.el9.x86_64 56/198 2024-03-19T01:26:09.576 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : openblas-openmp-0.3.21-2.el9.x86_64 57/198 2024-03-19T01:26:09.576 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : postgresql-13.11-1.el9.x86_64 58/198 2024-03-19T01:26:09.576 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : postgresql-contrib-13.11-1.el9.x86_64 59/198 2024-03-19T01:26:09.576 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : postgresql-private-libs-13.11-1.el9.x86_64 60/198 2024-03-19T01:26:09.576 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : postgresql-server-13.11-1.el9.x86_64 61/198 2024-03-19T01:26:09.576 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : protobuf-3.14.0-13.el9.x86_64 62/198 2024-03-19T01:26:09.576 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-attrs-20.3.0-7.el9.noarch 63/198 2024-03-19T01:26:09.576 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 64/198 2024-03-19T01:26:09.576 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 65/198 2024-03-19T01:26:09.576 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 66/198 2024-03-19T01:26:09.576 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 67/198 2024-03-19T01:26:09.576 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 68/198 2024-03-19T01:26:09.576 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 69/198 2024-03-19T01:26:09.577 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 70/198 2024-03-19T01:26:09.577 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 71/198 2024-03-19T01:26:09.577 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 72/198 2024-03-19T01:26:09.577 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 73/198 2024-03-19T01:26:09.577 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-protobuf-3.14.0-13.el9.noarch 74/198 2024-03-19T01:26:09.577 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 75/198 2024-03-19T01:26:09.577 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 76/198 2024-03-19T01:26:09.577 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 77/198 2024-03-19T01:26:09.577 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 78/198 2024-03-19T01:26:09.577 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 79/198 2024-03-19T01:26:09.577 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 80/198 2024-03-19T01:26:09.577 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-wcwidth-0.2.5-8.el9.noarch 81/198 2024-03-19T01:26:09.577 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : socat-1.7.4.1-5.el9.x86_64 82/198 2024-03-19T01:26:09.578 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : uuid-1.6.2-55.el9.x86_64 83/198 2024-03-19T01:26:09.578 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 84/198 2024-03-19T01:26:09.578 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : zlib-devel-1.2.11-40.el9.x86_64 85/198 2024-03-19T01:26:09.578 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : fuse-2.9.9-15.el9.x86_64 86/198 2024-03-19T01:26:09.578 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 87/198 2024-03-19T01:26:09.578 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 88/198 2024-03-19T01:26:09.578 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-ply-3.11-14.el9.noarch 89/198 2024-03-19T01:26:09.578 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 90/198 2024-03-19T01:26:09.578 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-urllib3-1.26.5-5.el9.noarch 91/198 2024-03-19T01:26:09.578 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : jq-1.6-16.el9.x86_64 92/198 2024-03-19T01:26:09.578 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : libgfortran-11.4.1-3.el9.x86_64 93/198 2024-03-19T01:26:09.578 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : libquadmath-11.4.1-3.el9.x86_64 94/198 2024-03-19T01:26:09.578 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 95/198 2024-03-19T01:26:09.578 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : elfutils-libelf-devel-0.190-2.el9.x86_64 96/198 2024-03-19T01:26:09.579 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : libuuid-devel-2.37.4-18.el9.x86_64 97/198 2024-03-19T01:26:09.579 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : openssl-devel-1:3.0.7-27.el9.x86_64 98/198 2024-03-19T01:26:09.579 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-devel-3.9.17-2.el9.x86_64 99/198 2024-03-19T01:26:09.579 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-jinja2-2.11.3-5.el9.noarch 100/198 2024-03-19T01:26:09.579 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : qatlib-23.11.0-1.el9.x86_64 101/198 2024-03-19T01:26:09.579 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : qatlib-service-23.11.0-1.el9.x86_64 102/198 2024-03-19T01:26:09.579 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : qatzip-libs-1.1.2-2.el9.x86_64 103/198 2024-03-19T01:26:09.579 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : xfsprogs-devel-6.3.0-1.el9.x86_64 104/198 2024-03-19T01:26:09.579 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : lua-devel-5.4.4-4.el9.x86_64 105/198 2024-03-19T01:26:09.579 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : protobuf-compiler-3.14.0-13.el9.x86_64 106/198 2024-03-19T01:26:09.579 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : abseil-cpp-20211102.0-3.el9.x86_64 107/198 2024-03-19T01:26:09.579 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : gperftools-libs-2.9.1-2.el9.x86_64 108/198 2024-03-19T01:26:09.579 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : grpc-data-1.46.7-10.el9.noarch 109/198 2024-03-19T01:26:09.580 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : libarrow-9.0.0-10.el9.x86_64 110/198 2024-03-19T01:26:09.580 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : libarrow-doc-9.0.0-10.el9.noarch 111/198 2024-03-19T01:26:09.580 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 112/198 2024-03-19T01:26:09.580 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 113/198 2024-03-19T01:26:09.580 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : luarocks-3.9.2-1.el9.noarch 114/198 2024-03-19T01:26:09.580 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : parquet-libs-9.0.0-10.el9.x86_64 115/198 2024-03-19T01:26:09.580 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-asyncssh-2.13.2-4.el9.noarch 116/198 2024-03-19T01:26:09.580 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 117/198 2024-03-19T01:26:09.580 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 118/198 2024-03-19T01:26:09.580 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-certifi-2021.10.8-2.el9.noarch 119/198 2024-03-19T01:26:09.580 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 120/198 2024-03-19T01:26:09.580 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 121/198 2024-03-19T01:26:09.580 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-cmd2-2.4.2-2.el9.noarch 122/198 2024-03-19T01:26:09.581 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-colorama-0.4.4-12.el9.noarch 123/198 2024-03-19T01:26:09.581 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-google-auth-1:2.28.0-1.el9.noarch 124/198 2024-03-19T01:26:09.581 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-grpcio-1.46.7-10.el9.x86_64 125/198 2024-03-19T01:26:09.581 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-grpcio-tools-1.46.7-10.el9.x86_64 126/198 2024-03-19T01:26:09.581 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 127/198 2024-03-19T01:26:09.581 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 128/198 2024-03-19T01:26:09.581 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 129/198 2024-03-19T01:26:09.581 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 130/198 2024-03-19T01:26:09.581 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 131/198 2024-03-19T01:26:09.581 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 132/198 2024-03-19T01:26:09.581 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 133/198 2024-03-19T01:26:09.581 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 134/198 2024-03-19T01:26:09.581 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 135/198 2024-03-19T01:26:09.582 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 136/198 2024-03-19T01:26:09.582 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 137/198 2024-03-19T01:26:09.582 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 138/198 2024-03-19T01:26:09.582 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-pyperclip-1.8.2-2.el9.noarch 139/198 2024-03-19T01:26:09.582 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 140/198 2024-03-19T01:26:09.582 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 141/198 2024-03-19T01:26:09.582 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 142/198 2024-03-19T01:26:09.582 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 143/198 2024-03-19T01:26:09.582 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-typing-extensions-4.4.0-2.el9.noarch 144/198 2024-03-19T01:26:09.582 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-webob-1.8.7-6.el9.noarch 145/198 2024-03-19T01:26:09.582 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 146/198 2024-03-19T01:26:09.582 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 147/198 2024-03-19T01:26:09.582 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 148/198 2024-03-19T01:26:09.582 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : re2-1:20211101-3.el9.x86_64 149/198 2024-03-19T01:26:09.583 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 150/198 2024-03-19T01:26:09.583 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : librados2-2:19.0.0-2206.gb6ae2491.el9.x86_64 151/198 2024-03-19T01:26:09.583 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 152/198 2024-03-19T01:26:09.583 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : librbd1-2:19.0.0-2206.gb6ae2491.el9.x86_64 153/198 2024-03-19T01:26:09.583 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 154/198 2024-03-19T01:26:09.583 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : xfsdump-3.1.12-4.el9.x86_64 155/198 2024-03-19T01:26:09.583 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : xfsdump-3.1.12-3.el9.x86_64 156/198 2024-03-19T01:26:09.583 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : NetworkManager-1:1.47.2-1.el9.x86_64 157/198 2024-03-19T01:26:09.583 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : NetworkManager-1:1.44.0-2.el9.x86_64 158/198 2024-03-19T01:26:09.583 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : NetworkManager-libnm-1:1.47.2-1.el9.x86_64 159/198 2024-03-19T01:26:09.583 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : NetworkManager-libnm-1:1.44.0-2.el9.x86_64 160/198 2024-03-19T01:26:09.583 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : NetworkManager-team-1:1.47.2-1.el9.x86_64 161/198 2024-03-19T01:26:09.583 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : NetworkManager-team-1:1.44.0-2.el9.x86_64 162/198 2024-03-19T01:26:09.583 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : NetworkManager-tui-1:1.47.2-1.el9.x86_64 163/198 2024-03-19T01:26:09.584 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : NetworkManager-tui-1:1.44.0-2.el9.x86_64 164/198 2024-03-19T01:26:09.584 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : acl-2.3.1-4.el9.x86_64 165/198 2024-03-19T01:26:09.584 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : acl-2.3.1-3.el9.x86_64 166/198 2024-03-19T01:26:09.584 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : elfutils-debuginfod-client-0.190-2.el9.x86_64 167/198 2024-03-19T01:26:09.584 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : elfutils-debuginfod-client-0.189-3.el9.x86_64 168/198 2024-03-19T01:26:09.584 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : elfutils-libelf-0.190-2.el9.x86_64 169/198 2024-03-19T01:26:09.584 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : elfutils-libelf-0.189-3.el9.x86_64 170/198 2024-03-19T01:26:09.584 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : elfutils-libs-0.190-2.el9.x86_64 171/198 2024-03-19T01:26:09.584 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : elfutils-libs-0.189-3.el9.x86_64 172/198 2024-03-19T01:26:09.584 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : libacl-2.3.1-4.el9.x86_64 173/198 2024-03-19T01:26:09.584 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : libacl-2.3.1-3.el9.x86_64 174/198 2024-03-19T01:26:09.584 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : libblkid-2.37.4-18.el9.x86_64 175/198 2024-03-19T01:26:09.584 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : libblkid-2.37.4-15.el9.x86_64 176/198 2024-03-19T01:26:09.584 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : libfdisk-2.37.4-18.el9.x86_64 177/198 2024-03-19T01:26:09.585 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : libfdisk-2.37.4-15.el9.x86_64 178/198 2024-03-19T01:26:09.585 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : libmount-2.37.4-18.el9.x86_64 179/198 2024-03-19T01:26:09.585 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : libmount-2.37.4-15.el9.x86_64 180/198 2024-03-19T01:26:09.585 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : libsmartcols-2.37.4-18.el9.x86_64 181/198 2024-03-19T01:26:09.585 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : libsmartcols-2.37.4-15.el9.x86_64 182/198 2024-03-19T01:26:09.585 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : libuuid-2.37.4-18.el9.x86_64 183/198 2024-03-19T01:26:09.585 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : libuuid-2.37.4-15.el9.x86_64 184/198 2024-03-19T01:26:09.585 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : openssl-1:3.0.7-27.el9.x86_64 185/198 2024-03-19T01:26:09.585 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : openssl-1:3.0.7-24.el9.x86_64 186/198 2024-03-19T01:26:09.585 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : openssl-libs-1:3.0.7-27.el9.x86_64 187/198 2024-03-19T01:26:09.585 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : openssl-libs-1:3.0.7-24.el9.x86_64 188/198 2024-03-19T01:26:09.585 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : util-linux-2.37.4-18.el9.x86_64 189/198 2024-03-19T01:26:09.585 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : util-linux-2.37.4-15.el9.x86_64 190/198 2024-03-19T01:26:09.585 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : util-linux-core-2.37.4-18.el9.x86_64 191/198 2024-03-19T01:26:09.586 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : util-linux-core-2.37.4-15.el9.x86_64 192/198 2024-03-19T01:26:09.586 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : util-linux-user-2.37.4-18.el9.x86_64 193/198 2024-03-19T01:26:09.586 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : util-linux-user-2.37.4-15.el9.x86_64 194/198 2024-03-19T01:26:09.586 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : xfsprogs-6.3.0-1.el9.x86_64 195/198 2024-03-19T01:26:09.586 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : xfsprogs-5.19.0-4.el9.x86_64 196/198 2024-03-19T01:26:09.586 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : libacl-devel-2.3.1-4.el9.x86_64 197/198 2024-03-19T01:26:12.572 INFO:teuthology.orchestra.run.smithi115.stdout: Verifying : libacl-devel-2.3.1-3.el9.x86_64 198/198 2024-03-19T01:26:12.572 INFO:teuthology.orchestra.run.smithi115.stdout: 2024-03-19T01:26:12.572 INFO:teuthology.orchestra.run.smithi115.stdout:Upgraded: 2024-03-19T01:26:12.573 INFO:teuthology.orchestra.run.smithi115.stdout: NetworkManager-1:1.47.2-1.el9.x86_64 2024-03-19T01:26:12.573 INFO:teuthology.orchestra.run.smithi115.stdout: NetworkManager-libnm-1:1.47.2-1.el9.x86_64 2024-03-19T01:26:12.573 INFO:teuthology.orchestra.run.smithi115.stdout: NetworkManager-team-1:1.47.2-1.el9.x86_64 2024-03-19T01:26:12.573 INFO:teuthology.orchestra.run.smithi115.stdout: NetworkManager-tui-1:1.47.2-1.el9.x86_64 2024-03-19T01:26:12.573 INFO:teuthology.orchestra.run.smithi115.stdout: acl-2.3.1-4.el9.x86_64 2024-03-19T01:26:12.573 INFO:teuthology.orchestra.run.smithi115.stdout: elfutils-debuginfod-client-0.190-2.el9.x86_64 2024-03-19T01:26:12.573 INFO:teuthology.orchestra.run.smithi115.stdout: elfutils-libelf-0.190-2.el9.x86_64 2024-03-19T01:26:12.573 INFO:teuthology.orchestra.run.smithi115.stdout: elfutils-libs-0.190-2.el9.x86_64 2024-03-19T01:26:12.573 INFO:teuthology.orchestra.run.smithi115.stdout: libacl-2.3.1-4.el9.x86_64 2024-03-19T01:26:12.573 INFO:teuthology.orchestra.run.smithi115.stdout: libacl-devel-2.3.1-4.el9.x86_64 2024-03-19T01:26:12.573 INFO:teuthology.orchestra.run.smithi115.stdout: libblkid-2.37.4-18.el9.x86_64 2024-03-19T01:26:12.573 INFO:teuthology.orchestra.run.smithi115.stdout: libfdisk-2.37.4-18.el9.x86_64 2024-03-19T01:26:12.573 INFO:teuthology.orchestra.run.smithi115.stdout: libmount-2.37.4-18.el9.x86_64 2024-03-19T01:26:12.574 INFO:teuthology.orchestra.run.smithi115.stdout: librados2-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:26:12.574 INFO:teuthology.orchestra.run.smithi115.stdout: librbd1-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:26:12.574 INFO:teuthology.orchestra.run.smithi115.stdout: libsmartcols-2.37.4-18.el9.x86_64 2024-03-19T01:26:12.574 INFO:teuthology.orchestra.run.smithi115.stdout: libuuid-2.37.4-18.el9.x86_64 2024-03-19T01:26:12.574 INFO:teuthology.orchestra.run.smithi115.stdout: openssl-1:3.0.7-27.el9.x86_64 2024-03-19T01:26:12.574 INFO:teuthology.orchestra.run.smithi115.stdout: openssl-libs-1:3.0.7-27.el9.x86_64 2024-03-19T01:26:12.574 INFO:teuthology.orchestra.run.smithi115.stdout: util-linux-2.37.4-18.el9.x86_64 2024-03-19T01:26:12.574 INFO:teuthology.orchestra.run.smithi115.stdout: util-linux-core-2.37.4-18.el9.x86_64 2024-03-19T01:26:12.574 INFO:teuthology.orchestra.run.smithi115.stdout: util-linux-user-2.37.4-18.el9.x86_64 2024-03-19T01:26:12.574 INFO:teuthology.orchestra.run.smithi115.stdout: xfsdump-3.1.12-4.el9.x86_64 2024-03-19T01:26:12.574 INFO:teuthology.orchestra.run.smithi115.stdout: xfsprogs-6.3.0-1.el9.x86_64 2024-03-19T01:26:12.574 INFO:teuthology.orchestra.run.smithi115.stdout:Installed: 2024-03-19T01:26:12.574 INFO:teuthology.orchestra.run.smithi115.stdout: abseil-cpp-20211102.0-3.el9.x86_64 2024-03-19T01:26:12.575 INFO:teuthology.orchestra.run.smithi115.stdout: automake-1.16.2-8.el9.noarch 2024-03-19T01:26:12.575 INFO:teuthology.orchestra.run.smithi115.stdout: bison-3.7.4-5.el9.x86_64 2024-03-19T01:26:12.575 INFO:teuthology.orchestra.run.smithi115.stdout: boost-program-options-1.75.0-8.el9.x86_64 2024-03-19T01:26:12.575 INFO:teuthology.orchestra.run.smithi115.stdout: ceph-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:26:12.575 INFO:teuthology.orchestra.run.smithi115.stdout: ceph-base-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:26:12.575 INFO:teuthology.orchestra.run.smithi115.stdout: ceph-common-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:26:12.575 INFO:teuthology.orchestra.run.smithi115.stdout: ceph-fuse-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:26:12.575 INFO:teuthology.orchestra.run.smithi115.stdout: ceph-grafana-dashboards-2:19.0.0-2206.gb6ae2491.el9.noarch 2024-03-19T01:26:12.575 INFO:teuthology.orchestra.run.smithi115.stdout: ceph-immutable-object-cache-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:26:12.575 INFO:teuthology.orchestra.run.smithi115.stdout: ceph-mds-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:26:12.575 INFO:teuthology.orchestra.run.smithi115.stdout: ceph-mgr-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:26:12.575 INFO:teuthology.orchestra.run.smithi115.stdout: ceph-mgr-cephadm-2:19.0.0-2206.gb6ae2491.el9.noarch 2024-03-19T01:26:12.575 INFO:teuthology.orchestra.run.smithi115.stdout: ceph-mgr-dashboard-2:19.0.0-2206.gb6ae2491.el9.noarch 2024-03-19T01:26:12.576 INFO:teuthology.orchestra.run.smithi115.stdout: ceph-mgr-diskprediction-local-2:19.0.0-2206.gb6ae2491.el9.noarch 2024-03-19T01:26:12.576 INFO:teuthology.orchestra.run.smithi115.stdout: ceph-mgr-modules-core-2:19.0.0-2206.gb6ae2491.el9.noarch 2024-03-19T01:26:12.576 INFO:teuthology.orchestra.run.smithi115.stdout: ceph-mgr-rook-2:19.0.0-2206.gb6ae2491.el9.noarch 2024-03-19T01:26:12.576 INFO:teuthology.orchestra.run.smithi115.stdout: ceph-mon-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:26:12.576 INFO:teuthology.orchestra.run.smithi115.stdout: ceph-osd-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:26:12.576 INFO:teuthology.orchestra.run.smithi115.stdout: ceph-prometheus-alerts-2:19.0.0-2206.gb6ae2491.el9.noarch 2024-03-19T01:26:12.576 INFO:teuthology.orchestra.run.smithi115.stdout: ceph-radosgw-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:26:12.576 INFO:teuthology.orchestra.run.smithi115.stdout: ceph-selinux-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:26:12.576 INFO:teuthology.orchestra.run.smithi115.stdout: ceph-test-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:26:12.576 INFO:teuthology.orchestra.run.smithi115.stdout: ceph-volume-2:19.0.0-2206.gb6ae2491.el9.noarch 2024-03-19T01:26:12.576 INFO:teuthology.orchestra.run.smithi115.stdout: cephadm-2:19.0.0-2206.gb6ae2491.el9.noarch 2024-03-19T01:26:12.576 INFO:teuthology.orchestra.run.smithi115.stdout: cephfs-mirror-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:26:12.576 INFO:teuthology.orchestra.run.smithi115.stdout: cephfs-shell-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:26:12.576 INFO:teuthology.orchestra.run.smithi115.stdout: cephfs-top-2:19.0.0-2206.gb6ae2491.el9.noarch 2024-03-19T01:26:12.577 INFO:teuthology.orchestra.run.smithi115.stdout: elfutils-libelf-devel-0.190-2.el9.x86_64 2024-03-19T01:26:12.577 INFO:teuthology.orchestra.run.smithi115.stdout: flex-2.6.4-9.el9.x86_64 2024-03-19T01:26:12.577 INFO:teuthology.orchestra.run.smithi115.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-03-19T01:26:12.577 INFO:teuthology.orchestra.run.smithi115.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-03-19T01:26:12.577 INFO:teuthology.orchestra.run.smithi115.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-03-19T01:26:12.577 INFO:teuthology.orchestra.run.smithi115.stdout: fuse-2.9.9-15.el9.x86_64 2024-03-19T01:26:12.577 INFO:teuthology.orchestra.run.smithi115.stdout: gperftools-libs-2.9.1-2.el9.x86_64 2024-03-19T01:26:12.577 INFO:teuthology.orchestra.run.smithi115.stdout: grpc-data-1.46.7-10.el9.noarch 2024-03-19T01:26:12.577 INFO:teuthology.orchestra.run.smithi115.stdout: jq-1.6-16.el9.x86_64 2024-03-19T01:26:12.577 INFO:teuthology.orchestra.run.smithi115.stdout: libaio-devel-0.3.111-13.el9.x86_64 2024-03-19T01:26:12.577 INFO:teuthology.orchestra.run.smithi115.stdout: libarrow-9.0.0-10.el9.x86_64 2024-03-19T01:26:12.577 INFO:teuthology.orchestra.run.smithi115.stdout: libarrow-doc-9.0.0-10.el9.noarch 2024-03-19T01:26:12.577 INFO:teuthology.orchestra.run.smithi115.stdout: libcephfs-devel-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:26:12.578 INFO:teuthology.orchestra.run.smithi115.stdout: libcephfs2-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:26:12.578 INFO:teuthology.orchestra.run.smithi115.stdout: libcephsqlite-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:26:12.578 INFO:teuthology.orchestra.run.smithi115.stdout: libgfortran-11.4.1-3.el9.x86_64 2024-03-19T01:26:12.578 INFO:teuthology.orchestra.run.smithi115.stdout: liboath-2.6.7-2.el9.x86_64 2024-03-19T01:26:12.578 INFO:teuthology.orchestra.run.smithi115.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-03-19T01:26:12.578 INFO:teuthology.orchestra.run.smithi115.stdout: libquadmath-11.4.1-3.el9.x86_64 2024-03-19T01:26:12.578 INFO:teuthology.orchestra.run.smithi115.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-03-19T01:26:12.578 INFO:teuthology.orchestra.run.smithi115.stdout: librados-devel-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:26:12.578 INFO:teuthology.orchestra.run.smithi115.stdout: libradosstriper1-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:26:12.578 INFO:teuthology.orchestra.run.smithi115.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-03-19T01:26:12.578 INFO:teuthology.orchestra.run.smithi115.stdout: librgw2-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:26:12.578 INFO:teuthology.orchestra.run.smithi115.stdout: libtool-2.4.6-45.el9.x86_64 2024-03-19T01:26:12.578 INFO:teuthology.orchestra.run.smithi115.stdout: libunwind-1.6.2-1.el9.x86_64 2024-03-19T01:26:12.578 INFO:teuthology.orchestra.run.smithi115.stdout: libuuid-devel-2.37.4-18.el9.x86_64 2024-03-19T01:26:12.579 INFO:teuthology.orchestra.run.smithi115.stdout: libzstd-devel-1.5.1-2.el9.x86_64 2024-03-19T01:26:12.579 INFO:teuthology.orchestra.run.smithi115.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-03-19T01:26:12.579 INFO:teuthology.orchestra.run.smithi115.stdout: lua-5.4.4-4.el9.x86_64 2024-03-19T01:26:12.579 INFO:teuthology.orchestra.run.smithi115.stdout: lua-devel-5.4.4-4.el9.x86_64 2024-03-19T01:26:12.579 INFO:teuthology.orchestra.run.smithi115.stdout: luarocks-3.9.2-1.el9.noarch 2024-03-19T01:26:12.579 INFO:teuthology.orchestra.run.smithi115.stdout: oniguruma-6.9.6-1.el9.5.x86_64 2024-03-19T01:26:12.579 INFO:teuthology.orchestra.run.smithi115.stdout: openblas-0.3.21-2.el9.x86_64 2024-03-19T01:26:12.579 INFO:teuthology.orchestra.run.smithi115.stdout: openblas-openmp-0.3.21-2.el9.x86_64 2024-03-19T01:26:12.579 INFO:teuthology.orchestra.run.smithi115.stdout: openssl-devel-1:3.0.7-27.el9.x86_64 2024-03-19T01:26:12.579 INFO:teuthology.orchestra.run.smithi115.stdout: parquet-libs-9.0.0-10.el9.x86_64 2024-03-19T01:26:12.579 INFO:teuthology.orchestra.run.smithi115.stdout: postgresql-13.11-1.el9.x86_64 2024-03-19T01:26:12.579 INFO:teuthology.orchestra.run.smithi115.stdout: postgresql-contrib-13.11-1.el9.x86_64 2024-03-19T01:26:12.579 INFO:teuthology.orchestra.run.smithi115.stdout: postgresql-private-libs-13.11-1.el9.x86_64 2024-03-19T01:26:12.579 INFO:teuthology.orchestra.run.smithi115.stdout: postgresql-server-13.11-1.el9.x86_64 2024-03-19T01:26:12.580 INFO:teuthology.orchestra.run.smithi115.stdout: protobuf-3.14.0-13.el9.x86_64 2024-03-19T01:26:12.580 INFO:teuthology.orchestra.run.smithi115.stdout: protobuf-compiler-3.14.0-13.el9.x86_64 2024-03-19T01:26:12.580 INFO:teuthology.orchestra.run.smithi115.stdout: python3-asyncssh-2.13.2-4.el9.noarch 2024-03-19T01:26:12.580 INFO:teuthology.orchestra.run.smithi115.stdout: python3-attrs-20.3.0-7.el9.noarch 2024-03-19T01:26:12.580 INFO:teuthology.orchestra.run.smithi115.stdout: python3-babel-2.9.1-2.el9.noarch 2024-03-19T01:26:12.580 INFO:teuthology.orchestra.run.smithi115.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-03-19T01:26:12.580 INFO:teuthology.orchestra.run.smithi115.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-03-19T01:26:12.580 INFO:teuthology.orchestra.run.smithi115.stdout: python3-ceph-argparse-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:26:12.580 INFO:teuthology.orchestra.run.smithi115.stdout: python3-ceph-common-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:26:12.580 INFO:teuthology.orchestra.run.smithi115.stdout: python3-cephfs-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:26:12.580 INFO:teuthology.orchestra.run.smithi115.stdout: python3-certifi-2021.10.8-2.el9.noarch 2024-03-19T01:26:12.580 INFO:teuthology.orchestra.run.smithi115.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-03-19T01:26:12.580 INFO:teuthology.orchestra.run.smithi115.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-03-19T01:26:12.580 INFO:teuthology.orchestra.run.smithi115.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-03-19T01:26:12.581 INFO:teuthology.orchestra.run.smithi115.stdout: python3-cmd2-2.4.2-2.el9.noarch 2024-03-19T01:26:12.581 INFO:teuthology.orchestra.run.smithi115.stdout: python3-colorama-0.4.4-12.el9.noarch 2024-03-19T01:26:12.581 INFO:teuthology.orchestra.run.smithi115.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-03-19T01:26:12.581 INFO:teuthology.orchestra.run.smithi115.stdout: python3-devel-3.9.17-2.el9.x86_64 2024-03-19T01:26:12.581 INFO:teuthology.orchestra.run.smithi115.stdout: python3-google-auth-1:2.28.0-1.el9.noarch 2024-03-19T01:26:12.581 INFO:teuthology.orchestra.run.smithi115.stdout: python3-grpcio-1.46.7-10.el9.x86_64 2024-03-19T01:26:12.581 INFO:teuthology.orchestra.run.smithi115.stdout: python3-grpcio-tools-1.46.7-10.el9.x86_64 2024-03-19T01:26:12.581 INFO:teuthology.orchestra.run.smithi115.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-03-19T01:26:12.581 INFO:teuthology.orchestra.run.smithi115.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-03-19T01:26:12.581 INFO:teuthology.orchestra.run.smithi115.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-03-19T01:26:12.581 INFO:teuthology.orchestra.run.smithi115.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-03-19T01:26:12.581 INFO:teuthology.orchestra.run.smithi115.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-03-19T01:26:12.581 INFO:teuthology.orchestra.run.smithi115.stdout: python3-jinja2-2.11.3-5.el9.noarch 2024-03-19T01:26:12.581 INFO:teuthology.orchestra.run.smithi115.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-03-19T01:26:12.582 INFO:teuthology.orchestra.run.smithi115.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-03-19T01:26:12.582 INFO:teuthology.orchestra.run.smithi115.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-03-19T01:26:12.582 INFO:teuthology.orchestra.run.smithi115.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-03-19T01:26:12.582 INFO:teuthology.orchestra.run.smithi115.stdout: python3-mako-1.1.4-6.el9.noarch 2024-03-19T01:26:12.582 INFO:teuthology.orchestra.run.smithi115.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-03-19T01:26:12.582 INFO:teuthology.orchestra.run.smithi115.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-03-19T01:26:12.582 INFO:teuthology.orchestra.run.smithi115.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-03-19T01:26:12.582 INFO:teuthology.orchestra.run.smithi115.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-03-19T01:26:12.582 INFO:teuthology.orchestra.run.smithi115.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-03-19T01:26:12.582 INFO:teuthology.orchestra.run.smithi115.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-03-19T01:26:12.582 INFO:teuthology.orchestra.run.smithi115.stdout: python3-packaging-20.9-5.el9.noarch 2024-03-19T01:26:12.582 INFO:teuthology.orchestra.run.smithi115.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-03-19T01:26:12.582 INFO:teuthology.orchestra.run.smithi115.stdout: python3-ply-3.11-14.el9.noarch 2024-03-19T01:26:12.582 INFO:teuthology.orchestra.run.smithi115.stdout: python3-portend-3.1.0-2.el9.noarch 2024-03-19T01:26:12.583 INFO:teuthology.orchestra.run.smithi115.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-03-19T01:26:12.583 INFO:teuthology.orchestra.run.smithi115.stdout: python3-protobuf-3.14.0-13.el9.noarch 2024-03-19T01:26:12.583 INFO:teuthology.orchestra.run.smithi115.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-03-19T01:26:12.583 INFO:teuthology.orchestra.run.smithi115.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-03-19T01:26:12.583 INFO:teuthology.orchestra.run.smithi115.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-03-19T01:26:12.583 INFO:teuthology.orchestra.run.smithi115.stdout: python3-pycparser-2.20-6.el9.noarch 2024-03-19T01:26:12.583 INFO:teuthology.orchestra.run.smithi115.stdout: python3-pyperclip-1.8.2-2.el9.noarch 2024-03-19T01:26:12.583 INFO:teuthology.orchestra.run.smithi115.stdout: python3-pytz-2021.1-5.el9.noarch 2024-03-19T01:26:12.583 INFO:teuthology.orchestra.run.smithi115.stdout: python3-rados-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:26:12.583 INFO:teuthology.orchestra.run.smithi115.stdout: python3-rbd-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:26:12.583 INFO:teuthology.orchestra.run.smithi115.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-03-19T01:26:12.583 INFO:teuthology.orchestra.run.smithi115.stdout: python3-requests-2.25.1-8.el9.noarch 2024-03-19T01:26:12.583 INFO:teuthology.orchestra.run.smithi115.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-03-19T01:26:12.584 INFO:teuthology.orchestra.run.smithi115.stdout: python3-rgw-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:26:12.584 INFO:teuthology.orchestra.run.smithi115.stdout: python3-routes-2.5.1-5.el9.noarch 2024-03-19T01:26:12.584 INFO:teuthology.orchestra.run.smithi115.stdout: python3-rsa-4.9-2.el9.noarch 2024-03-19T01:26:12.584 INFO:teuthology.orchestra.run.smithi115.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-03-19T01:26:12.584 INFO:teuthology.orchestra.run.smithi115.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-03-19T01:26:12.584 INFO:teuthology.orchestra.run.smithi115.stdout: python3-toml-0.10.2-6.el9.noarch 2024-03-19T01:26:12.584 INFO:teuthology.orchestra.run.smithi115.stdout: python3-typing-extensions-4.4.0-2.el9.noarch 2024-03-19T01:26:12.584 INFO:teuthology.orchestra.run.smithi115.stdout: python3-urllib3-1.26.5-5.el9.noarch 2024-03-19T01:26:12.584 INFO:teuthology.orchestra.run.smithi115.stdout: python3-wcwidth-0.2.5-8.el9.noarch 2024-03-19T01:26:12.584 INFO:teuthology.orchestra.run.smithi115.stdout: python3-webob-1.8.7-6.el9.noarch 2024-03-19T01:26:12.584 INFO:teuthology.orchestra.run.smithi115.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-03-19T01:26:12.584 INFO:teuthology.orchestra.run.smithi115.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-03-19T01:26:12.584 INFO:teuthology.orchestra.run.smithi115.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-03-19T01:26:12.584 INFO:teuthology.orchestra.run.smithi115.stdout: qatlib-23.11.0-1.el9.x86_64 2024-03-19T01:26:12.585 INFO:teuthology.orchestra.run.smithi115.stdout: qatlib-service-23.11.0-1.el9.x86_64 2024-03-19T01:26:12.585 INFO:teuthology.orchestra.run.smithi115.stdout: qatzip-libs-1.1.2-2.el9.x86_64 2024-03-19T01:26:12.585 INFO:teuthology.orchestra.run.smithi115.stdout: rbd-fuse-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:26:12.585 INFO:teuthology.orchestra.run.smithi115.stdout: rbd-mirror-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:26:12.585 INFO:teuthology.orchestra.run.smithi115.stdout: rbd-nbd-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:26:12.585 INFO:teuthology.orchestra.run.smithi115.stdout: re2-1:20211101-3.el9.x86_64 2024-03-19T01:26:12.585 INFO:teuthology.orchestra.run.smithi115.stdout: socat-1.7.4.1-5.el9.x86_64 2024-03-19T01:26:12.585 INFO:teuthology.orchestra.run.smithi115.stdout: thrift-0.15.0-2.el9.x86_64 2024-03-19T01:26:12.585 INFO:teuthology.orchestra.run.smithi115.stdout: uuid-1.6.2-55.el9.x86_64 2024-03-19T01:26:12.585 INFO:teuthology.orchestra.run.smithi115.stdout: xfsprogs-devel-6.3.0-1.el9.x86_64 2024-03-19T01:26:12.585 INFO:teuthology.orchestra.run.smithi115.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-03-19T01:26:12.586 INFO:teuthology.orchestra.run.smithi115.stdout: zlib-devel-1.2.11-40.el9.x86_64 2024-03-19T01:26:12.586 INFO:teuthology.orchestra.run.smithi115.stdout: 2024-03-19T01:26:12.586 INFO:teuthology.orchestra.run.smithi115.stdout:Complete! 2024-03-19T01:26:13.007 DEBUG:teuthology.parallel:result is None 2024-03-19T01:26:49.165 INFO:teuthology.orchestra.run.smithi099.stdout:Extra Packages for Enterprise Linux 32 kB/s | 21 MB 10:56 2024-03-19T01:26:54.038 INFO:teuthology.orchestra.run.smithi099.stdout:lab-extras 24 kB/s | 1.7 kB 00:00 2024-03-19T01:26:56.410 INFO:teuthology.orchestra.run.smithi099.stdout:Package librados2-2:16.2.4-5.el9.x86_64 is already installed. 2024-03-19T01:26:56.411 INFO:teuthology.orchestra.run.smithi099.stdout:Package librbd1-2:16.2.4-5.el9.x86_64 is already installed. 2024-03-19T01:26:56.426 INFO:teuthology.orchestra.run.smithi099.stdout:Package NetworkManager-1:1.44.0-2.el9.x86_64 is already installed. 2024-03-19T01:26:56.427 INFO:teuthology.orchestra.run.smithi099.stdout:Package iproute-6.2.0-5.el9.x86_64 is already installed. 2024-03-19T01:26:56.428 INFO:teuthology.orchestra.run.smithi099.stdout:Package util-linux-2.37.4-15.el9.x86_64 is already installed. 2024-03-19T01:26:56.429 INFO:teuthology.orchestra.run.smithi099.stdout:Package libacl-devel-2.3.1-3.el9.x86_64 is already installed. 2024-03-19T01:26:56.431 INFO:teuthology.orchestra.run.smithi099.stdout:Package libattr-devel-2.5.1-3.el9.x86_64 is already installed. 2024-03-19T01:26:56.434 INFO:teuthology.orchestra.run.smithi099.stdout:Package xfsdump-3.1.12-3.el9.x86_64 is already installed. 2024-03-19T01:26:56.435 INFO:teuthology.orchestra.run.smithi099.stdout:Package xfsprogs-5.19.0-4.el9.x86_64 is already installed. 2024-03-19T01:26:56.515 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2024-03-19T01:26:56.535 INFO:teuthology.orchestra.run.smithi099.stdout:============================================================================================ 2024-03-19T01:26:56.535 INFO:teuthology.orchestra.run.smithi099.stdout: Package Arch Version Repository Size 2024-03-19T01:26:56.535 INFO:teuthology.orchestra.run.smithi099.stdout:============================================================================================ 2024-03-19T01:26:56.535 INFO:teuthology.orchestra.run.smithi099.stdout:Installing: 2024-03-19T01:26:56.535 INFO:teuthology.orchestra.run.smithi099.stdout: bison x86_64 3.7.4-5.el9 CentOS-AppStream 920 k 2024-03-19T01:26:56.535 INFO:teuthology.orchestra.run.smithi099.stdout: ceph x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 6.6 k 2024-03-19T01:26:56.535 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-base x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 5.4 M 2024-03-19T01:26:56.535 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-fuse x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 891 k 2024-03-19T01:26:56.536 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-immutable-object-cache x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 145 k 2024-03-19T01:26:56.536 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-mgr x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 1.5 M 2024-03-19T01:26:56.536 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-mgr-cephadm noarch 2:19.0.0-2206.gb6ae2491.el9 ceph-noarch 139 k 2024-03-19T01:26:56.536 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-mgr-dashboard noarch 2:19.0.0-2206.gb6ae2491.el9 ceph-noarch 1.8 M 2024-03-19T01:26:56.536 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-mgr-diskprediction-local noarch 2:19.0.0-2206.gb6ae2491.el9 ceph-noarch 7.4 M 2024-03-19T01:26:56.536 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-mgr-rook noarch 2:19.0.0-2206.gb6ae2491.el9 ceph-noarch 49 k 2024-03-19T01:26:56.536 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-radosgw x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 9.8 M 2024-03-19T01:26:56.536 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-test x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 48 M 2024-03-19T01:26:56.536 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-volume noarch 2:19.0.0-2206.gb6ae2491.el9 ceph-noarch 264 k 2024-03-19T01:26:56.536 INFO:teuthology.orchestra.run.smithi099.stdout: cephadm noarch 2:19.0.0-2206.gb6ae2491.el9 ceph-noarch 749 k 2024-03-19T01:26:56.536 INFO:teuthology.orchestra.run.smithi099.stdout: cephfs-mirror x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 221 k 2024-03-19T01:26:56.536 INFO:teuthology.orchestra.run.smithi099.stdout: cephfs-shell x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 27 k 2024-03-19T01:26:56.536 INFO:teuthology.orchestra.run.smithi099.stdout: cephfs-top noarch 2:19.0.0-2206.gb6ae2491.el9 ceph-noarch 18 k 2024-03-19T01:26:56.537 INFO:teuthology.orchestra.run.smithi099.stdout: elfutils-libelf-devel x86_64 0.190-2.el9 appstream 21 k 2024-03-19T01:26:56.537 INFO:teuthology.orchestra.run.smithi099.stdout: flex x86_64 2.6.4-9.el9 CentOS-AppStream 307 k 2024-03-19T01:26:56.537 INFO:teuthology.orchestra.run.smithi099.stdout: libaio-devel x86_64 0.3.111-13.el9 CentOS-AppStream 10 k 2024-03-19T01:26:56.537 INFO:teuthology.orchestra.run.smithi099.stdout: libcephfs-devel x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 32 k 2024-03-19T01:26:56.537 INFO:teuthology.orchestra.run.smithi099.stdout: libcephfs2 x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 746 k 2024-03-19T01:26:56.537 INFO:teuthology.orchestra.run.smithi099.stdout: librados-devel x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 127 k 2024-03-19T01:26:56.537 INFO:teuthology.orchestra.run.smithi099.stdout: libtool x86_64 2.4.6-45.el9 CentOS-AppStream 578 k 2024-03-19T01:26:56.537 INFO:teuthology.orchestra.run.smithi099.stdout: libuuid-devel x86_64 2.37.4-18.el9 appstream 25 k 2024-03-19T01:26:56.537 INFO:teuthology.orchestra.run.smithi099.stdout: openssl-devel x86_64 1:3.0.7-27.el9 appstream 4.1 M 2024-03-19T01:26:56.537 INFO:teuthology.orchestra.run.smithi099.stdout: postgresql x86_64 13.11-1.el9 CentOS-AppStream 1.5 M 2024-03-19T01:26:56.537 INFO:teuthology.orchestra.run.smithi099.stdout: postgresql-contrib x86_64 13.11-1.el9 CentOS-AppStream 811 k 2024-03-19T01:26:56.537 INFO:teuthology.orchestra.run.smithi099.stdout: postgresql-server x86_64 13.11-1.el9 CentOS-AppStream 5.7 M 2024-03-19T01:26:56.537 INFO:teuthology.orchestra.run.smithi099.stdout: python3-cephfs x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 162 k 2024-03-19T01:26:56.538 INFO:teuthology.orchestra.run.smithi099.stdout: python3-rados x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 321 k 2024-03-19T01:26:56.538 INFO:teuthology.orchestra.run.smithi099.stdout: python3-rbd x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 298 k 2024-03-19T01:26:56.538 INFO:teuthology.orchestra.run.smithi099.stdout: python3-rgw x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 99 k 2024-03-19T01:26:56.538 INFO:teuthology.orchestra.run.smithi099.stdout: rbd-fuse x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 85 k 2024-03-19T01:26:56.538 INFO:teuthology.orchestra.run.smithi099.stdout: rbd-mirror x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 3.1 M 2024-03-19T01:26:56.538 INFO:teuthology.orchestra.run.smithi099.stdout: rbd-nbd x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 170 k 2024-03-19T01:26:56.538 INFO:teuthology.orchestra.run.smithi099.stdout: xfsprogs-devel x86_64 6.3.0-1.el9 appstream 94 k 2024-03-19T01:26:56.538 INFO:teuthology.orchestra.run.smithi099.stdout:Upgrading: 2024-03-19T01:26:56.538 INFO:teuthology.orchestra.run.smithi099.stdout: NetworkManager x86_64 1:1.47.2-1.el9 baseos 2.3 M 2024-03-19T01:26:56.538 INFO:teuthology.orchestra.run.smithi099.stdout: NetworkManager-libnm x86_64 1:1.47.2-1.el9 baseos 1.8 M 2024-03-19T01:26:56.538 INFO:teuthology.orchestra.run.smithi099.stdout: NetworkManager-team x86_64 1:1.47.2-1.el9 baseos 42 k 2024-03-19T01:26:56.538 INFO:teuthology.orchestra.run.smithi099.stdout: NetworkManager-tui x86_64 1:1.47.2-1.el9 baseos 248 k 2024-03-19T01:26:56.539 INFO:teuthology.orchestra.run.smithi099.stdout: acl x86_64 2.3.1-4.el9 baseos 71 k 2024-03-19T01:26:56.539 INFO:teuthology.orchestra.run.smithi099.stdout: elfutils-debuginfod-client x86_64 0.190-2.el9 baseos 36 k 2024-03-19T01:26:56.539 INFO:teuthology.orchestra.run.smithi099.stdout: elfutils-libelf x86_64 0.190-2.el9 baseos 192 k 2024-03-19T01:26:56.539 INFO:teuthology.orchestra.run.smithi099.stdout: elfutils-libs x86_64 0.190-2.el9 baseos 256 k 2024-03-19T01:26:56.539 INFO:teuthology.orchestra.run.smithi099.stdout: libacl x86_64 2.3.1-4.el9 baseos 23 k 2024-03-19T01:26:56.539 INFO:teuthology.orchestra.run.smithi099.stdout: libacl-devel x86_64 2.3.1-4.el9 appstream 74 k 2024-03-19T01:26:56.539 INFO:teuthology.orchestra.run.smithi099.stdout: libblkid x86_64 2.37.4-18.el9 baseos 108 k 2024-03-19T01:26:56.539 INFO:teuthology.orchestra.run.smithi099.stdout: libfdisk x86_64 2.37.4-18.el9 baseos 155 k 2024-03-19T01:26:56.539 INFO:teuthology.orchestra.run.smithi099.stdout: libmount x86_64 2.37.4-18.el9 baseos 136 k 2024-03-19T01:26:56.539 INFO:teuthology.orchestra.run.smithi099.stdout: librados2 x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 3.4 M 2024-03-19T01:26:56.539 INFO:teuthology.orchestra.run.smithi099.stdout: librbd1 x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 3.1 M 2024-03-19T01:26:56.539 INFO:teuthology.orchestra.run.smithi099.stdout: libsmartcols x86_64 2.37.4-18.el9 baseos 64 k 2024-03-19T01:26:56.539 INFO:teuthology.orchestra.run.smithi099.stdout: libuuid x86_64 2.37.4-18.el9 baseos 29 k 2024-03-19T01:26:56.539 INFO:teuthology.orchestra.run.smithi099.stdout: openssl x86_64 1:3.0.7-27.el9 baseos 1.2 M 2024-03-19T01:26:56.540 INFO:teuthology.orchestra.run.smithi099.stdout: openssl-libs x86_64 1:3.0.7-27.el9 baseos 2.2 M 2024-03-19T01:26:56.540 INFO:teuthology.orchestra.run.smithi099.stdout: util-linux x86_64 2.37.4-18.el9 baseos 2.3 M 2024-03-19T01:26:56.540 INFO:teuthology.orchestra.run.smithi099.stdout: util-linux-core x86_64 2.37.4-18.el9 baseos 465 k 2024-03-19T01:26:56.540 INFO:teuthology.orchestra.run.smithi099.stdout: util-linux-user x86_64 2.37.4-18.el9 baseos 31 k 2024-03-19T01:26:56.540 INFO:teuthology.orchestra.run.smithi099.stdout: xfsdump x86_64 3.1.12-4.el9 CentOS-BaseOS 321 k 2024-03-19T01:26:56.540 INFO:teuthology.orchestra.run.smithi099.stdout: xfsprogs x86_64 6.3.0-1.el9 baseos 1.1 M 2024-03-19T01:26:56.540 INFO:teuthology.orchestra.run.smithi099.stdout:Installing dependencies: 2024-03-19T01:26:56.540 INFO:teuthology.orchestra.run.smithi099.stdout: abseil-cpp x86_64 20211102.0-3.el9 epel 548 k 2024-03-19T01:26:56.540 INFO:teuthology.orchestra.run.smithi099.stdout: automake noarch 1.16.2-8.el9 CentOS-AppStream 662 k 2024-03-19T01:26:56.540 INFO:teuthology.orchestra.run.smithi099.stdout: boost-program-options x86_64 1.75.0-8.el9 CentOS-AppStream 106 k 2024-03-19T01:26:56.540 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-common x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 21 M 2024-03-19T01:26:56.540 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-grafana-dashboards noarch 2:19.0.0-2206.gb6ae2491.el9 ceph-noarch 34 k 2024-03-19T01:26:56.540 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-mds x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 2.4 M 2024-03-19T01:26:56.541 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-mgr-modules-core noarch 2:19.0.0-2206.gb6ae2491.el9 ceph-noarch 248 k 2024-03-19T01:26:56.541 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-mon x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 4.7 M 2024-03-19T01:26:56.541 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-osd x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 17 M 2024-03-19T01:26:56.541 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-prometheus-alerts noarch 2:19.0.0-2206.gb6ae2491.el9 ceph-noarch 16 k 2024-03-19T01:26:56.541 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-selinux x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 25 k 2024-03-19T01:26:56.541 INFO:teuthology.orchestra.run.smithi099.stdout: flexiblas x86_64 3.0.4-8.el9 CentOS-AppStream 32 k 2024-03-19T01:26:56.541 INFO:teuthology.orchestra.run.smithi099.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 CentOS-AppStream 3.0 M 2024-03-19T01:26:56.541 INFO:teuthology.orchestra.run.smithi099.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 CentOS-AppStream 17 k 2024-03-19T01:26:56.541 INFO:teuthology.orchestra.run.smithi099.stdout: fuse x86_64 2.9.9-15.el9 CentOS-BaseOS 79 k 2024-03-19T01:26:56.541 INFO:teuthology.orchestra.run.smithi099.stdout: gperftools-libs x86_64 2.9.1-2.el9 epel 309 k 2024-03-19T01:26:56.541 INFO:teuthology.orchestra.run.smithi099.stdout: grpc-data noarch 1.46.7-10.el9 epel 19 k 2024-03-19T01:26:56.541 INFO:teuthology.orchestra.run.smithi099.stdout: jq x86_64 1.6-16.el9 baseos 187 k 2024-03-19T01:26:56.542 INFO:teuthology.orchestra.run.smithi099.stdout: libarrow x86_64 9.0.0-10.el9 epel 4.4 M 2024-03-19T01:26:56.542 INFO:teuthology.orchestra.run.smithi099.stdout: libarrow-doc noarch 9.0.0-10.el9 epel 26 k 2024-03-19T01:26:56.542 INFO:teuthology.orchestra.run.smithi099.stdout: libcephsqlite x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 163 k 2024-03-19T01:26:56.542 INFO:teuthology.orchestra.run.smithi099.stdout: libgfortran x86_64 11.4.1-3.el9 baseos 803 k 2024-03-19T01:26:56.542 INFO:teuthology.orchestra.run.smithi099.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-03-19T01:26:56.542 INFO:teuthology.orchestra.run.smithi099.stdout: libpmemobj x86_64 1.12.1-1.el9 CentOS-AppStream 159 k 2024-03-19T01:26:56.542 INFO:teuthology.orchestra.run.smithi099.stdout: libquadmath x86_64 11.4.1-3.el9 baseos 195 k 2024-03-19T01:26:56.542 INFO:teuthology.orchestra.run.smithi099.stdout: librabbitmq x86_64 0.11.0-7.el9 CentOS-AppStream 44 k 2024-03-19T01:26:56.542 INFO:teuthology.orchestra.run.smithi099.stdout: libradosstriper1 x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 501 k 2024-03-19T01:26:56.542 INFO:teuthology.orchestra.run.smithi099.stdout: librdkafka x86_64 1.6.1-102.el9 CentOS-AppStream 661 k 2024-03-19T01:26:56.542 INFO:teuthology.orchestra.run.smithi099.stdout: librgw2 x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 5.1 M 2024-03-19T01:26:56.542 INFO:teuthology.orchestra.run.smithi099.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-03-19T01:26:56.542 INFO:teuthology.orchestra.run.smithi099.stdout: libzstd-devel x86_64 1.5.1-2.el9 CentOS-AppStream 46 k 2024-03-19T01:26:56.543 INFO:teuthology.orchestra.run.smithi099.stdout: lttng-ust x86_64 2.12.0-6.el9 CentOS-AppStream 282 k 2024-03-19T01:26:56.543 INFO:teuthology.orchestra.run.smithi099.stdout: lua x86_64 5.4.4-4.el9 CentOS-AppStream 187 k 2024-03-19T01:26:56.543 INFO:teuthology.orchestra.run.smithi099.stdout: lua-devel x86_64 5.4.4-4.el9 crb 22 k 2024-03-19T01:26:56.543 INFO:teuthology.orchestra.run.smithi099.stdout: luarocks noarch 3.9.2-1.el9 epel 151 k 2024-03-19T01:26:56.543 INFO:teuthology.orchestra.run.smithi099.stdout: oniguruma x86_64 6.9.6-1.el9.5 CentOS-AppStream 217 k 2024-03-19T01:26:56.543 INFO:teuthology.orchestra.run.smithi099.stdout: openblas x86_64 0.3.21-2.el9 CentOS-AppStream 34 k 2024-03-19T01:26:56.543 INFO:teuthology.orchestra.run.smithi099.stdout: openblas-openmp x86_64 0.3.21-2.el9 CentOS-AppStream 4.7 M 2024-03-19T01:26:56.543 INFO:teuthology.orchestra.run.smithi099.stdout: parquet-libs x86_64 9.0.0-10.el9 epel 838 k 2024-03-19T01:26:56.543 INFO:teuthology.orchestra.run.smithi099.stdout: postgresql-private-libs x86_64 13.11-1.el9 CentOS-AppStream 134 k 2024-03-19T01:26:56.543 INFO:teuthology.orchestra.run.smithi099.stdout: protobuf x86_64 3.14.0-13.el9 CentOS-AppStream 1.0 M 2024-03-19T01:26:56.543 INFO:teuthology.orchestra.run.smithi099.stdout: protobuf-compiler x86_64 3.14.0-13.el9 crb 863 k 2024-03-19T01:26:56.543 INFO:teuthology.orchestra.run.smithi099.stdout: python3-asyncssh noarch 2.13.2-4.el9 epel 548 k 2024-03-19T01:26:56.543 INFO:teuthology.orchestra.run.smithi099.stdout: python3-attrs noarch 20.3.0-7.el9 CentOS-AppStream 78 k 2024-03-19T01:26:56.543 INFO:teuthology.orchestra.run.smithi099.stdout: python3-babel noarch 2.9.1-2.el9 CentOS-AppStream 5.8 M 2024-03-19T01:26:56.544 INFO:teuthology.orchestra.run.smithi099.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-03-19T01:26:56.544 INFO:teuthology.orchestra.run.smithi099.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-03-19T01:26:56.544 INFO:teuthology.orchestra.run.smithi099.stdout: python3-ceph-argparse x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 45 k 2024-03-19T01:26:56.544 INFO:teuthology.orchestra.run.smithi099.stdout: python3-ceph-common x86_64 2:19.0.0-2206.gb6ae2491.el9 ceph 131 k 2024-03-19T01:26:56.544 INFO:teuthology.orchestra.run.smithi099.stdout: python3-certifi noarch 2021.10.8-2.el9 epel 15 k 2024-03-19T01:26:56.544 INFO:teuthology.orchestra.run.smithi099.stdout: python3-cffi x86_64 1.14.5-5.el9 CentOS-BaseOS 241 k 2024-03-19T01:26:56.544 INFO:teuthology.orchestra.run.smithi099.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-03-19T01:26:56.544 INFO:teuthology.orchestra.run.smithi099.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-03-19T01:26:56.544 INFO:teuthology.orchestra.run.smithi099.stdout: python3-cmd2 noarch 2.4.2-2.el9 epel 303 k 2024-03-19T01:26:56.544 INFO:teuthology.orchestra.run.smithi099.stdout: python3-colorama noarch 0.4.4-12.el9 epel 35 k 2024-03-19T01:26:56.544 INFO:teuthology.orchestra.run.smithi099.stdout: python3-cryptography x86_64 36.0.1-4.el9 CentOS-BaseOS 1.1 M 2024-03-19T01:26:56.544 INFO:teuthology.orchestra.run.smithi099.stdout: python3-devel x86_64 3.9.17-2.el9 appstream 245 k 2024-03-19T01:26:56.544 INFO:teuthology.orchestra.run.smithi099.stdout: python3-google-auth noarch 1:2.28.0-1.el9 epel 203 k 2024-03-19T01:26:56.544 INFO:teuthology.orchestra.run.smithi099.stdout: python3-grpcio x86_64 1.46.7-10.el9 epel 2.0 M 2024-03-19T01:26:56.545 INFO:teuthology.orchestra.run.smithi099.stdout: python3-grpcio-tools x86_64 1.46.7-10.el9 epel 144 k 2024-03-19T01:26:56.545 INFO:teuthology.orchestra.run.smithi099.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-03-19T01:26:56.545 INFO:teuthology.orchestra.run.smithi099.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-03-19T01:26:56.545 INFO:teuthology.orchestra.run.smithi099.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-03-19T01:26:56.545 INFO:teuthology.orchestra.run.smithi099.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-03-19T01:26:56.545 INFO:teuthology.orchestra.run.smithi099.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-03-19T01:26:56.545 INFO:teuthology.orchestra.run.smithi099.stdout: python3-jinja2 noarch 2.11.3-5.el9 appstream 248 k 2024-03-19T01:26:56.545 INFO:teuthology.orchestra.run.smithi099.stdout: python3-jsonpatch noarch 1.21-16.el9 CentOS-AppStream 24 k 2024-03-19T01:26:56.545 INFO:teuthology.orchestra.run.smithi099.stdout: python3-jsonpointer noarch 2.0-4.el9 CentOS-AppStream 18 k 2024-03-19T01:26:56.545 INFO:teuthology.orchestra.run.smithi099.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-03-19T01:26:56.545 INFO:teuthology.orchestra.run.smithi099.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-03-19T01:26:56.545 INFO:teuthology.orchestra.run.smithi099.stdout: python3-mako noarch 1.1.4-6.el9 CentOS-AppStream 144 k 2024-03-19T01:26:56.546 INFO:teuthology.orchestra.run.smithi099.stdout: python3-markupsafe x86_64 1.1.1-12.el9 CentOS-AppStream 32 k 2024-03-19T01:26:56.546 INFO:teuthology.orchestra.run.smithi099.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-03-19T01:26:56.546 INFO:teuthology.orchestra.run.smithi099.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-03-19T01:26:56.546 INFO:teuthology.orchestra.run.smithi099.stdout: python3-numpy x86_64 1:1.20.1-5.el9 CentOS-AppStream 4.8 M 2024-03-19T01:26:56.546 INFO:teuthology.orchestra.run.smithi099.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 CentOS-AppStream 217 k 2024-03-19T01:26:56.546 INFO:teuthology.orchestra.run.smithi099.stdout: python3-oauthlib noarch 3.1.1-5.el9 CentOS-AppStream 174 k 2024-03-19T01:26:56.546 INFO:teuthology.orchestra.run.smithi099.stdout: python3-packaging noarch 20.9-5.el9 CentOS-AppStream 69 k 2024-03-19T01:26:56.546 INFO:teuthology.orchestra.run.smithi099.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-03-19T01:26:56.546 INFO:teuthology.orchestra.run.smithi099.stdout: python3-ply noarch 3.11-14.el9 CentOS-BaseOS 103 k 2024-03-19T01:26:56.546 INFO:teuthology.orchestra.run.smithi099.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-03-19T01:26:56.546 INFO:teuthology.orchestra.run.smithi099.stdout: python3-prettytable noarch 0.7.2-27.el9 CentOS-AppStream 41 k 2024-03-19T01:26:56.547 INFO:teuthology.orchestra.run.smithi099.stdout: python3-protobuf noarch 3.14.0-13.el9 CentOS-AppStream 238 k 2024-03-19T01:26:56.547 INFO:teuthology.orchestra.run.smithi099.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-03-19T01:26:56.547 INFO:teuthology.orchestra.run.smithi099.stdout: python3-pyasn1 noarch 0.4.8-6.el9 CentOS-AppStream 132 k 2024-03-19T01:26:56.547 INFO:teuthology.orchestra.run.smithi099.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 CentOS-AppStream 211 k 2024-03-19T01:26:56.547 INFO:teuthology.orchestra.run.smithi099.stdout: python3-pycparser noarch 2.20-6.el9 CentOS-BaseOS 124 k 2024-03-19T01:26:56.547 INFO:teuthology.orchestra.run.smithi099.stdout: python3-pyperclip noarch 1.8.2-2.el9 epel 28 k 2024-03-19T01:26:56.547 INFO:teuthology.orchestra.run.smithi099.stdout: python3-pytz noarch 2021.1-5.el9 CentOS-AppStream 47 k 2024-03-19T01:26:56.547 INFO:teuthology.orchestra.run.smithi099.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-03-19T01:26:56.547 INFO:teuthology.orchestra.run.smithi099.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-03-19T01:26:56.547 INFO:teuthology.orchestra.run.smithi099.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 CentOS-AppStream 43 k 2024-03-19T01:26:56.547 INFO:teuthology.orchestra.run.smithi099.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-03-19T01:26:56.547 INFO:teuthology.orchestra.run.smithi099.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-03-19T01:26:56.547 INFO:teuthology.orchestra.run.smithi099.stdout: python3-scipy x86_64 1.6.2-8.el9 CentOS-AppStream 16 M 2024-03-19T01:26:56.547 INFO:teuthology.orchestra.run.smithi099.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-03-19T01:26:56.547 INFO:teuthology.orchestra.run.smithi099.stdout: python3-toml noarch 0.10.2-6.el9 CentOS-AppStream 39 k 2024-03-19T01:26:56.547 INFO:teuthology.orchestra.run.smithi099.stdout: python3-typing-extensions noarch 4.4.0-2.el9 epel 51 k 2024-03-19T01:26:56.548 INFO:teuthology.orchestra.run.smithi099.stdout: python3-urllib3 noarch 1.26.5-5.el9 CentOS-BaseOS 187 k 2024-03-19T01:26:56.548 INFO:teuthology.orchestra.run.smithi099.stdout: python3-wcwidth noarch 0.2.5-8.el9 CentOS-AppStream 41 k 2024-03-19T01:26:56.548 INFO:teuthology.orchestra.run.smithi099.stdout: python3-webob noarch 1.8.7-6.el9 epel 230 k 2024-03-19T01:26:56.548 INFO:teuthology.orchestra.run.smithi099.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-03-19T01:26:56.548 INFO:teuthology.orchestra.run.smithi099.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-03-19T01:26:56.548 INFO:teuthology.orchestra.run.smithi099.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-03-19T01:26:56.548 INFO:teuthology.orchestra.run.smithi099.stdout: qatlib x86_64 23.11.0-1.el9 appstream 216 k 2024-03-19T01:26:56.548 INFO:teuthology.orchestra.run.smithi099.stdout: qatzip-libs x86_64 1.1.2-2.el9 appstream 44 k 2024-03-19T01:26:56.548 INFO:teuthology.orchestra.run.smithi099.stdout: re2 x86_64 1:20211101-3.el9 epel 192 k 2024-03-19T01:26:56.548 INFO:teuthology.orchestra.run.smithi099.stdout: socat x86_64 1.7.4.1-5.el9 CentOS-AppStream 300 k 2024-03-19T01:26:56.548 INFO:teuthology.orchestra.run.smithi099.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-03-19T01:26:56.548 INFO:teuthology.orchestra.run.smithi099.stdout: uuid x86_64 1.6.2-55.el9 CentOS-AppStream 56 k 2024-03-19T01:26:56.548 INFO:teuthology.orchestra.run.smithi099.stdout: xmlstarlet x86_64 1.6.1-20.el9 CentOS-AppStream 63 k 2024-03-19T01:26:56.548 INFO:teuthology.orchestra.run.smithi099.stdout: zlib-devel x86_64 1.2.11-40.el9 CentOS-AppStream 44 k 2024-03-19T01:26:56.548 INFO:teuthology.orchestra.run.smithi099.stdout:Installing weak dependencies: 2024-03-19T01:26:56.548 INFO:teuthology.orchestra.run.smithi099.stdout: qatlib-service x86_64 23.11.0-1.el9 appstream 35 k 2024-03-19T01:26:56.549 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:26:56.549 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction Summary 2024-03-19T01:26:56.549 INFO:teuthology.orchestra.run.smithi099.stdout:============================================================================================ 2024-03-19T01:26:56.549 INFO:teuthology.orchestra.run.smithi099.stdout:Install 150 Packages 2024-03-19T01:26:56.549 INFO:teuthology.orchestra.run.smithi099.stdout:Upgrade 24 Packages 2024-03-19T01:26:56.549 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:26:56.549 INFO:teuthology.orchestra.run.smithi099.stdout:Total download size: 224 M 2024-03-19T01:26:56.549 INFO:teuthology.orchestra.run.smithi099.stdout:Downloading Packages: 2024-03-19T01:26:57.264 INFO:teuthology.orchestra.run.smithi099.stdout:(1/174): ceph-19.0.0-2206.gb6ae2491.el9.x86_64. 37 kB/s | 6.6 kB 00:00 2024-03-19T01:26:57.456 INFO:teuthology.orchestra.run.smithi099.stdout:(2/174): ceph-fuse-19.0.0-2206.gb6ae2491.el9.x8 4.6 MB/s | 891 kB 00:00 2024-03-19T01:26:57.506 INFO:teuthology.orchestra.run.smithi099.stdout:(3/174): ceph-immutable-object-cache-19.0.0-220 2.8 MB/s | 145 kB 00:00 2024-03-19T01:26:57.648 INFO:teuthology.orchestra.run.smithi099.stdout:(4/174): ceph-mds-19.0.0-2206.gb6ae2491.el9.x86 17 MB/s | 2.4 MB 00:00 2024-03-19T01:26:57.748 INFO:teuthology.orchestra.run.smithi099.stdout:(5/174): ceph-mgr-19.0.0-2206.gb6ae2491.el9.x86 15 MB/s | 1.5 MB 00:00 2024-03-19T01:26:57.999 INFO:teuthology.orchestra.run.smithi099.stdout:(6/174): ceph-mon-19.0.0-2206.gb6ae2491.el9.x86 19 MB/s | 4.7 MB 00:00 2024-03-19T01:26:58.824 INFO:teuthology.orchestra.run.smithi099.stdout:(7/174): ceph-osd-19.0.0-2206.gb6ae2491.el9.x86 20 MB/s | 17 MB 00:00 2024-03-19T01:26:59.341 INFO:teuthology.orchestra.run.smithi099.stdout:(8/174): ceph-radosgw-19.0.0-2206.gb6ae2491.el9 19 MB/s | 9.8 MB 00:00 2024-03-19T01:26:59.458 INFO:teuthology.orchestra.run.smithi099.stdout:(9/174): ceph-selinux-19.0.0-2206.gb6ae2491.el9 216 kB/s | 25 kB 00:00 2024-03-19T01:26:59.958 INFO:teuthology.orchestra.run.smithi099.stdout:(10/174): ceph-base-19.0.0-2206.gb6ae2491.el9.x 1.9 MB/s | 5.4 MB 00:02 2024-03-19T01:27:00.142 INFO:teuthology.orchestra.run.smithi099.stdout:(11/174): cephfs-mirror-19.0.0-2206.gb6ae2491.e 1.2 MB/s | 221 kB 00:00 2024-03-19T01:27:00.242 INFO:teuthology.orchestra.run.smithi099.stdout:(12/174): cephfs-shell-19.0.0-2206.gb6ae2491.el 271 kB/s | 27 kB 00:00 2024-03-19T01:27:00.342 INFO:teuthology.orchestra.run.smithi099.stdout:(13/174): libcephfs-devel-19.0.0-2206.gb6ae2491 317 kB/s | 32 kB 00:00 2024-03-19T01:27:00.801 INFO:teuthology.orchestra.run.smithi099.stdout:(14/174): libcephfs2-19.0.0-2206.gb6ae2491.el9. 1.6 MB/s | 746 kB 00:00 2024-03-19T01:27:00.926 INFO:teuthology.orchestra.run.smithi099.stdout:(15/174): libcephsqlite-19.0.0-2206.gb6ae2491.e 1.3 MB/s | 163 kB 00:00 2024-03-19T01:27:01.043 INFO:teuthology.orchestra.run.smithi099.stdout:(16/174): librados-devel-19.0.0-2206.gb6ae2491. 1.1 MB/s | 127 kB 00:00 2024-03-19T01:27:01.327 INFO:teuthology.orchestra.run.smithi099.stdout:(17/174): libradosstriper1-19.0.0-2206.gb6ae249 1.7 MB/s | 501 kB 00:00 2024-03-19T01:27:02.102 INFO:teuthology.orchestra.run.smithi099.stdout:(18/174): ceph-test-19.0.0-2206.gb6ae2491.el9.x 18 MB/s | 48 MB 00:02 2024-03-19T01:27:02.311 INFO:teuthology.orchestra.run.smithi099.stdout:(19/174): ceph-common-19.0.0-2206.gb6ae2491.el9 4.0 MB/s | 21 MB 00:05 2024-03-19T01:27:02.327 INFO:teuthology.orchestra.run.smithi099.stdout:(20/174): python3-ceph-argparse-19.0.0-2206.gb6 201 kB/s | 45 kB 00:00 2024-03-19T01:27:02.378 INFO:teuthology.orchestra.run.smithi099.stdout:(21/174): python3-cephfs-19.0.0-2206.gb6ae2491. 3.2 MB/s | 162 kB 00:00 2024-03-19T01:27:02.403 INFO:teuthology.orchestra.run.smithi099.stdout:(22/174): python3-ceph-common-19.0.0-2206.gb6ae 1.4 MB/s | 131 kB 00:00 2024-03-19T01:27:02.428 INFO:teuthology.orchestra.run.smithi099.stdout:(23/174): python3-rados-19.0.0-2206.gb6ae2491.e 6.3 MB/s | 321 kB 00:00 2024-03-19T01:27:02.478 INFO:teuthology.orchestra.run.smithi099.stdout:(24/174): python3-rbd-19.0.0-2206.gb6ae2491.el9 3.9 MB/s | 298 kB 00:00 2024-03-19T01:27:02.504 INFO:teuthology.orchestra.run.smithi099.stdout:(25/174): python3-rgw-19.0.0-2206.gb6ae2491.el9 1.3 MB/s | 99 kB 00:00 2024-03-19T01:27:02.537 INFO:teuthology.orchestra.run.smithi099.stdout:(26/174): rbd-fuse-19.0.0-2206.gb6ae2491.el9.x8 1.4 MB/s | 85 kB 00:00 2024-03-19T01:27:02.629 INFO:teuthology.orchestra.run.smithi099.stdout:(27/174): rbd-nbd-19.0.0-2206.gb6ae2491.el9.x86 1.8 MB/s | 170 kB 00:00 2024-03-19T01:27:02.696 INFO:teuthology.orchestra.run.smithi099.stdout:(28/174): rbd-mirror-19.0.0-2206.gb6ae2491.el9. 16 MB/s | 3.1 MB 00:00 2024-03-19T01:27:02.714 INFO:teuthology.orchestra.run.smithi099.stdout:(29/174): ceph-grafana-dashboards-19.0.0-2206.g 406 kB/s | 34 kB 00:00 2024-03-19T01:27:02.747 INFO:teuthology.orchestra.run.smithi099.stdout:(30/174): ceph-mgr-cephadm-19.0.0-2206.gb6ae249 2.7 MB/s | 139 kB 00:00 2024-03-19T01:27:02.947 INFO:teuthology.orchestra.run.smithi099.stdout:(31/174): ceph-mgr-dashboard-19.0.0-2206.gb6ae2 7.7 MB/s | 1.8 MB 00:00 2024-03-19T01:27:03.047 INFO:teuthology.orchestra.run.smithi099.stdout:(32/174): ceph-mgr-modules-core-19.0.0-2206.gb6 2.4 MB/s | 248 kB 00:00 2024-03-19T01:27:03.176 INFO:teuthology.orchestra.run.smithi099.stdout:(33/174): ceph-mgr-diskprediction-local-19.0.0- 17 MB/s | 7.4 MB 00:00 2024-03-19T01:27:03.201 INFO:teuthology.orchestra.run.smithi099.stdout:(34/174): ceph-mgr-rook-19.0.0-2206.gb6ae2491.e 321 kB/s | 49 kB 00:00 2024-03-19T01:27:03.226 INFO:teuthology.orchestra.run.smithi099.stdout:(35/174): ceph-prometheus-alerts-19.0.0-2206.gb 329 kB/s | 16 kB 00:00 2024-03-19T01:27:03.268 INFO:teuthology.orchestra.run.smithi099.stdout:(36/174): ceph-volume-19.0.0-2206.gb6ae2491.el9 3.8 MB/s | 264 kB 00:00 2024-03-19T01:27:03.336 INFO:teuthology.orchestra.run.smithi099.stdout:(37/174): librgw2-19.0.0-2206.gb6ae2491.el9.x86 2.6 MB/s | 5.1 MB 00:02 2024-03-19T01:27:03.353 INFO:teuthology.orchestra.run.smithi099.stdout:(38/174): cephfs-top-19.0.0-2206.gb6ae2491.el9. 217 kB/s | 18 kB 00:00 2024-03-19T01:27:03.395 INFO:teuthology.orchestra.run.smithi099.stdout:(39/174): cephadm-19.0.0-2206.gb6ae2491.el9.noa 4.4 MB/s | 749 kB 00:00 2024-03-19T01:27:03.553 INFO:teuthology.orchestra.run.smithi099.stdout:(40/174): boost-program-options-1.75.0-8.el9.x8 1.0 MB/s | 106 kB 00:00 2024-03-19T01:27:03.595 INFO:teuthology.orchestra.run.smithi099.stdout:(41/174): bison-3.7.4-5.el9.x86_64.rpm 3.7 MB/s | 920 kB 00:00 2024-03-19T01:27:03.662 INFO:teuthology.orchestra.run.smithi099.stdout:(42/174): automake-1.16.2-8.el9.noarch.rpm 2.0 MB/s | 662 kB 00:00 2024-03-19T01:27:03.704 INFO:teuthology.orchestra.run.smithi099.stdout:(43/174): flex-2.6.4-9.el9.x86_64.rpm 2.0 MB/s | 307 kB 00:00 2024-03-19T01:27:03.721 INFO:teuthology.orchestra.run.smithi099.stdout:(44/174): flexiblas-3.0.4-8.el9.x86_64.rpm 252 kB/s | 32 kB 00:00 2024-03-19T01:27:03.788 INFO:teuthology.orchestra.run.smithi099.stdout:(45/174): flexiblas-openblas-openmp-3.0.4-8.el9 206 kB/s | 17 kB 00:00 2024-03-19T01:27:03.805 INFO:teuthology.orchestra.run.smithi099.stdout:(46/174): libaio-devel-0.3.111-13.el9.x86_64.rp 121 kB/s | 10 kB 00:00 2024-03-19T01:27:03.872 INFO:teuthology.orchestra.run.smithi099.stdout:(47/174): flexiblas-netlib-3.0.4-8.el9.x86_64.r 14 MB/s | 3.0 MB 00:00 2024-03-19T01:27:03.897 INFO:teuthology.orchestra.run.smithi099.stdout:(48/174): libpmemobj-1.12.1-1.el9.x86_64.rpm 1.4 MB/s | 159 kB 00:00 2024-03-19T01:27:03.914 INFO:teuthology.orchestra.run.smithi099.stdout:(49/174): librabbitmq-0.11.0-7.el9.x86_64.rpm 409 kB/s | 44 kB 00:00 2024-03-19T01:27:03.947 INFO:teuthology.orchestra.run.smithi099.stdout:(50/174): librdkafka-1.6.1-102.el9.x86_64.rpm 8.6 MB/s | 661 kB 00:00 2024-03-19T01:27:03.965 INFO:teuthology.orchestra.run.smithi099.stdout:(51/174): libzstd-devel-1.5.1-2.el9.x86_64.rpm 948 kB/s | 46 kB 00:00 2024-03-19T01:27:03.998 INFO:teuthology.orchestra.run.smithi099.stdout:(52/174): libtool-2.4.6-45.el9.x86_64.rpm 6.8 MB/s | 578 kB 00:00 2024-03-19T01:27:04.023 INFO:teuthology.orchestra.run.smithi099.stdout:(53/174): lttng-ust-2.12.0-6.el9.x86_64.rpm 3.7 MB/s | 282 kB 00:00 2024-03-19T01:27:04.049 INFO:teuthology.orchestra.run.smithi099.stdout:(54/174): lua-5.4.4-4.el9.x86_64.rpm 2.2 MB/s | 187 kB 00:00 2024-03-19T01:27:04.074 INFO:teuthology.orchestra.run.smithi099.stdout:(55/174): oniguruma-6.9.6-1.el9.5.x86_64.rpm 2.8 MB/s | 217 kB 00:00 2024-03-19T01:27:04.099 INFO:teuthology.orchestra.run.smithi099.stdout:(56/174): openblas-0.3.21-2.el9.x86_64.rpm 687 kB/s | 34 kB 00:00 2024-03-19T01:27:04.291 INFO:teuthology.orchestra.run.smithi099.stdout:(57/174): openblas-openmp-0.3.21-2.el9.x86_64.r 20 MB/s | 4.7 MB 00:00 2024-03-19T01:27:04.325 INFO:teuthology.orchestra.run.smithi099.stdout:(58/174): postgresql-13.11-1.el9.x86_64.rpm 6.1 MB/s | 1.5 MB 00:00 2024-03-19T01:27:04.350 INFO:teuthology.orchestra.run.smithi099.stdout:(59/174): postgresql-contrib-13.11-1.el9.x86_64 3.2 MB/s | 811 kB 00:00 2024-03-19T01:27:04.375 INFO:teuthology.orchestra.run.smithi099.stdout:(60/174): postgresql-private-libs-13.11-1.el9.x 1.6 MB/s | 134 kB 00:00 2024-03-19T01:27:04.442 INFO:teuthology.orchestra.run.smithi099.stdout:(61/174): protobuf-3.14.0-13.el9.x86_64.rpm 11 MB/s | 1.0 MB 00:00 2024-03-19T01:27:04.526 INFO:teuthology.orchestra.run.smithi099.stdout:(62/174): python3-attrs-20.3.0-7.el9.noarch.rpm 524 kB/s | 78 kB 00:00 2024-03-19T01:27:04.601 INFO:teuthology.orchestra.run.smithi099.stdout:(63/174): postgresql-server-13.11-1.el9.x86_64. 23 MB/s | 5.7 MB 00:00 2024-03-19T01:27:04.685 INFO:teuthology.orchestra.run.smithi099.stdout:(64/174): python3-babel-2.9.1-2.el9.noarch.rpm 24 MB/s | 5.8 MB 00:00 2024-03-19T01:27:04.710 INFO:teuthology.orchestra.run.smithi099.stdout:(65/174): python3-jsonpatch-1.21-16.el9.noarch. 136 kB/s | 24 kB 00:00 2024-03-19T01:27:04.753 INFO:teuthology.orchestra.run.smithi099.stdout:(66/174): python3-jsonpointer-2.0-4.el9.noarch. 121 kB/s | 18 kB 00:00 2024-03-19T01:27:04.786 INFO:teuthology.orchestra.run.smithi099.stdout:(67/174): python3-mako-1.1.4-6.el9.noarch.rpm 1.4 MB/s | 144 kB 00:00 2024-03-19T01:27:04.811 INFO:teuthology.orchestra.run.smithi099.stdout:(68/174): python3-markupsafe-1.1.1-12.el9.x86_6 552 kB/s | 32 kB 00:00 2024-03-19T01:27:04.919 INFO:teuthology.orchestra.run.smithi099.stdout:(69/174): python3-numpy-1.20.1-5.el9.x86_64.rpm 29 MB/s | 4.8 MB 00:00 2024-03-19T01:27:04.945 INFO:teuthology.orchestra.run.smithi099.stdout:(70/174): python3-numpy-f2py-1.20.1-5.el9.x86_6 1.3 MB/s | 217 kB 00:00 2024-03-19T01:27:04.970 INFO:teuthology.orchestra.run.smithi099.stdout:(71/174): python3-oauthlib-3.1.1-5.el9.noarch.r 1.1 MB/s | 174 kB 00:00 2024-03-19T01:27:04.995 INFO:teuthology.orchestra.run.smithi099.stdout:(72/174): python3-packaging-20.9-5.el9.noarch.r 920 kB/s | 69 kB 00:00 2024-03-19T01:27:05.012 INFO:teuthology.orchestra.run.smithi099.stdout:(73/174): python3-prettytable-0.7.2-27.el9.noar 989 kB/s | 41 kB 00:00 2024-03-19T01:27:05.037 INFO:teuthology.orchestra.run.smithi099.stdout:(74/174): python3-protobuf-3.14.0-13.el9.noarch 3.5 MB/s | 238 kB 00:00 2024-03-19T01:27:05.063 INFO:teuthology.orchestra.run.smithi099.stdout:(75/174): python3-pyasn1-0.4.8-6.el9.noarch.rpm 1.9 MB/s | 132 kB 00:00 2024-03-19T01:27:05.079 INFO:teuthology.orchestra.run.smithi099.stdout:(76/174): python3-pytz-2021.1-5.el9.noarch.rpm 1.1 MB/s | 47 kB 00:00 2024-03-19T01:27:05.105 INFO:teuthology.orchestra.run.smithi099.stdout:(77/174): python3-pyasn1-modules-0.4.8-6.el9.no 3.1 MB/s | 211 kB 00:00 2024-03-19T01:27:05.121 INFO:teuthology.orchestra.run.smithi099.stdout:(78/174): python3-requests-oauthlib-1.3.0-12.el 734 kB/s | 43 kB 00:00 2024-03-19T01:27:05.305 INFO:teuthology.orchestra.run.smithi099.stdout:(79/174): python3-toml-0.10.2-6.el9.noarch.rpm 214 kB/s | 39 kB 00:00 2024-03-19T01:27:05.322 INFO:teuthology.orchestra.run.smithi099.stdout:(80/174): python3-wcwidth-0.2.5-8.el9.noarch.rp 207 kB/s | 41 kB 00:00 2024-03-19T01:27:05.481 INFO:teuthology.orchestra.run.smithi099.stdout:(81/174): python3-scipy-1.6.2-8.el9.x86_64.rpm 39 MB/s | 16 MB 00:00 2024-03-19T01:27:05.506 INFO:teuthology.orchestra.run.smithi099.stdout:(82/174): socat-1.7.4.1-5.el9.x86_64.rpm 1.5 MB/s | 300 kB 00:00 2024-03-19T01:27:05.523 INFO:teuthology.orchestra.run.smithi099.stdout:(83/174): uuid-1.6.2-55.el9.x86_64.rpm 284 kB/s | 56 kB 00:00 2024-03-19T01:27:05.540 INFO:teuthology.orchestra.run.smithi099.stdout:(84/174): xmlstarlet-1.6.1-20.el9.x86_64.rpm 1.1 MB/s | 63 kB 00:00 2024-03-19T01:27:05.556 INFO:teuthology.orchestra.run.smithi099.stdout:(85/174): zlib-devel-1.2.11-40.el9.x86_64.rpm 1.3 MB/s | 44 kB 00:00 2024-03-19T01:27:05.582 INFO:teuthology.orchestra.run.smithi099.stdout:(86/174): fuse-2.9.9-15.el9.x86_64.rpm 1.3 MB/s | 79 kB 00:00 2024-03-19T01:27:05.607 INFO:teuthology.orchestra.run.smithi099.stdout:(87/174): python3-cffi-1.14.5-5.el9.x86_64.rpm 3.5 MB/s | 241 kB 00:00 2024-03-19T01:27:05.632 INFO:teuthology.orchestra.run.smithi099.stdout:(88/174): python3-ply-3.11-14.el9.noarch.rpm 2.0 MB/s | 103 kB 00:00 2024-03-19T01:27:05.666 INFO:teuthology.orchestra.run.smithi099.stdout:(89/174): python3-cryptography-36.0.1-4.el9.x86 14 MB/s | 1.1 MB 00:00 2024-03-19T01:27:05.683 INFO:teuthology.orchestra.run.smithi099.stdout:(90/174): python3-pycparser-2.20-6.el9.noarch.r 1.6 MB/s | 124 kB 00:00 2024-03-19T01:27:05.706 INFO:teuthology.orchestra.run.smithi099.stdout:(91/174): python3-urllib3-1.26.5-5.el9.noarch.r 2.6 MB/s | 187 kB 00:00 2024-03-19T01:29:15.529 INFO:teuthology.orchestra.run.smithi099.stdout:[MIRROR] jq-1.6-16.el9.x86_64.rpm: Curl error (28): Timeout was reached for http://ftp-nyc.osuosl.org/pub/centos-stream/9-stream/BaseOS/x86_64/os/Packages/jq-1.6-16.el9.x86_64.rpm [Failed to connect to ftp-nyc.osuosl.org port 80: Connection timed out] 2024-03-19T01:29:15.530 INFO:teuthology.orchestra.run.smithi099.stdout:[MIRROR] libgfortran-11.4.1-3.el9.x86_64.rpm: Curl error (28): Timeout was reached for http://ftp-nyc.osuosl.org/pub/centos-stream/9-stream/BaseOS/x86_64/os/Packages/libgfortran-11.4.1-3.el9.x86_64.rpm [Failed to connect to ftp-nyc.osuosl.org port 80: Connection timed out] 2024-03-19T01:29:15.530 INFO:teuthology.orchestra.run.smithi099.stdout:[MIRROR] libquadmath-11.4.1-3.el9.x86_64.rpm: Curl error (28): Timeout was reached for http://ftp-nyc.osuosl.org/pub/centos-stream/9-stream/BaseOS/x86_64/os/Packages/libquadmath-11.4.1-3.el9.x86_64.rpm [Failed to connect to ftp-nyc.osuosl.org port 80: Connection timed out] 2024-03-19T01:29:16.078 INFO:teuthology.orchestra.run.smithi099.stdout:(92/174): jq-1.6-16.el9.x86_64.rpm 1.4 kB/s | 187 kB 02:10 2024-03-19T01:29:16.104 INFO:teuthology.orchestra.run.smithi099.stdout:(93/174): libquadmath-11.4.1-3.el9.x86_64.rpm 1.5 kB/s | 195 kB 02:10 2024-03-19T01:29:16.271 INFO:teuthology.orchestra.run.smithi099.stdout:(94/174): python3-requests-2.25.1-8.el9.noarch. 651 kB/s | 125 kB 00:00 2024-03-19T01:29:16.313 INFO:teuthology.orchestra.run.smithi099.stdout:(95/174): elfutils-libelf-devel-0.190-2.el9.x86 103 kB/s | 21 kB 00:00 2024-03-19T01:29:16.488 INFO:teuthology.orchestra.run.smithi099.stdout:(96/174): libuuid-devel-2.37.4-18.el9.x86_64.rp 117 kB/s | 25 kB 00:00 2024-03-19T01:29:16.621 INFO:teuthology.orchestra.run.smithi099.stdout:(97/174): libgfortran-11.4.1-3.el9.x86_64.rpm 6.1 kB/s | 803 kB 02:10 2024-03-19T01:29:16.838 INFO:teuthology.orchestra.run.smithi099.stdout:(98/174): python3-devel-3.9.17-2.el9.x86_64.rpm 701 kB/s | 245 kB 00:00 2024-03-19T01:29:16.980 INFO:teuthology.orchestra.run.smithi099.stdout:(99/174): python3-jinja2-2.11.3-5.el9.noarch.rp 693 kB/s | 248 kB 00:00 2024-03-19T01:29:17.164 INFO:teuthology.orchestra.run.smithi099.stdout:(100/174): qatlib-23.11.0-1.el9.x86_64.rpm 666 kB/s | 216 kB 00:00 2024-03-19T01:29:17.230 INFO:teuthology.orchestra.run.smithi099.stdout:(101/174): openssl-devel-3.0.7-27.el9.x86_64.rp 4.6 MB/s | 4.1 MB 00:00 2024-03-19T01:29:17.281 INFO:teuthology.orchestra.run.smithi099.stdout:(102/174): qatlib-service-23.11.0-1.el9.x86_64. 116 kB/s | 35 kB 00:00 2024-03-19T01:29:17.356 INFO:teuthology.orchestra.run.smithi099.stdout:(103/174): lua-devel-5.4.4-4.el9.x86_64.rpm 297 kB/s | 22 kB 00:00 2024-03-19T01:29:17.432 INFO:teuthology.orchestra.run.smithi099.stdout:(104/174): qatzip-libs-1.1.2-2.el9.x86_64.rpm 166 kB/s | 44 kB 00:00 2024-03-19T01:29:17.515 INFO:teuthology.orchestra.run.smithi099.stdout:(105/174): protobuf-compiler-3.14.0-13.el9.x86_ 5.3 MB/s | 863 kB 00:00 2024-03-19T01:29:17.565 INFO:teuthology.orchestra.run.smithi099.stdout:(106/174): xfsprogs-devel-6.3.0-1.el9.x86_64.rp 304 kB/s | 94 kB 00:00 2024-03-19T01:29:17.640 INFO:teuthology.orchestra.run.smithi099.stdout:(107/174): grpc-data-1.46.7-10.el9.noarch.rpm 260 kB/s | 19 kB 00:00 2024-03-19T01:29:17.682 INFO:teuthology.orchestra.run.smithi099.stdout:(108/174): gperftools-libs-2.9.1-2.el9.x86_64.r 1.8 MB/s | 309 kB 00:00 2024-03-19T01:29:17.707 INFO:teuthology.orchestra.run.smithi099.stdout:(109/174): abseil-cpp-20211102.0-3.el9.x86_64.r 1.9 MB/s | 548 kB 00:00 2024-03-19T01:29:17.733 INFO:teuthology.orchestra.run.smithi099.stdout:(110/174): libarrow-doc-9.0.0-10.el9.noarch.rpm 518 kB/s | 26 kB 00:00 2024-03-19T01:29:17.758 INFO:teuthology.orchestra.run.smithi099.stdout:(111/174): liboath-2.6.7-2.el9.x86_64.rpm 967 kB/s | 49 kB 00:00 2024-03-19T01:29:17.786 INFO:teuthology.orchestra.run.smithi099.stdout:(112/174): libunwind-1.6.2-1.el9.x86_64.rpm 1.3 MB/s | 67 kB 00:00 2024-03-19T01:29:17.817 INFO:teuthology.orchestra.run.smithi099.stdout:(113/174): luarocks-3.9.2-1.el9.noarch.rpm 2.5 MB/s | 151 kB 00:00 2024-03-19T01:29:18.009 INFO:teuthology.orchestra.run.smithi099.stdout:(114/174): python3-asyncssh-2.13.2-4.el9.noarch 2.8 MB/s | 548 kB 00:00 2024-03-19T01:29:18.051 INFO:teuthology.orchestra.run.smithi099.stdout:(115/174): python3-bcrypt-3.2.2-1.el9.x86_64.rp 1.0 MB/s | 43 kB 00:00 2024-03-19T01:29:18.109 INFO:teuthology.orchestra.run.smithi099.stdout:(116/174): python3-cachetools-4.2.4-1.el9.noarc 550 kB/s | 32 kB 00:00 2024-03-19T01:29:18.152 INFO:teuthology.orchestra.run.smithi099.stdout:(117/174): python3-certifi-2021.10.8-2.el9.noar 360 kB/s | 15 kB 00:00 2024-03-19T01:29:18.210 INFO:teuthology.orchestra.run.smithi099.stdout:(118/174): python3-cheroot-8.6.0-4.el9.noarch.r 2.9 MB/s | 172 kB 00:00 2024-03-19T01:29:18.235 INFO:teuthology.orchestra.run.smithi099.stdout:(119/174): parquet-libs-9.0.0-10.el9.x86_64.rpm 1.8 MB/s | 838 kB 00:00 2024-03-19T01:29:18.285 INFO:teuthology.orchestra.run.smithi099.stdout:(120/174): python3-cherrypy-18.6.1-2.el9.noarch 4.7 MB/s | 358 kB 00:00 2024-03-19T01:29:18.327 INFO:teuthology.orchestra.run.smithi099.stdout:(121/174): python3-cmd2-2.4.2-2.el9.noarch.rpm 3.2 MB/s | 303 kB 00:00 2024-03-19T01:29:18.344 INFO:teuthology.orchestra.run.smithi099.stdout:(122/174): python3-colorama-0.4.4-12.el9.noarch 602 kB/s | 35 kB 00:00 2024-03-19T01:29:18.403 INFO:teuthology.orchestra.run.smithi099.stdout:(123/174): python3-google-auth-2.28.0-1.el9.noa 2.6 MB/s | 203 kB 00:00 2024-03-19T01:29:18.478 INFO:teuthology.orchestra.run.smithi099.stdout:(124/174): python3-grpcio-1.46.7-10.el9.x86_64. 15 MB/s | 2.0 MB 00:00 2024-03-19T01:29:18.503 INFO:teuthology.orchestra.run.smithi099.stdout:(125/174): python3-grpcio-tools-1.46.7-10.el9.x 1.4 MB/s | 144 kB 00:00 2024-03-19T01:29:18.520 INFO:teuthology.orchestra.run.smithi099.stdout:(126/174): python3-jaraco-8.2.1-3.el9.noarch.rp 255 kB/s | 11 kB 00:00 2024-03-19T01:29:18.537 INFO:teuthology.orchestra.run.smithi099.stdout:(127/174): python3-jaraco-classes-3.2.1-5.el9.n 527 kB/s | 18 kB 00:00 2024-03-19T01:29:18.554 INFO:teuthology.orchestra.run.smithi099.stdout:(128/174): python3-jaraco-collections-3.0.0-8.e 689 kB/s | 23 kB 00:00 2024-03-19T01:29:18.572 INFO:teuthology.orchestra.run.smithi099.stdout:(129/174): python3-jaraco-functools-3.5.0-2.el9 579 kB/s | 19 kB 00:00 2024-03-19T01:29:18.588 INFO:teuthology.orchestra.run.smithi099.stdout:(130/174): python3-jaraco-text-3.2.0-6.el9.noar 589 kB/s | 20 kB 00:00 2024-03-19T01:29:18.622 INFO:teuthology.orchestra.run.smithi099.stdout:(131/174): python3-logutils-0.3.5-21.el9.noarch 1.4 MB/s | 46 kB 00:00 2024-03-19T01:29:18.664 INFO:teuthology.orchestra.run.smithi099.stdout:(132/174): python3-more-itertools-8.12.0-2.el9. 1.9 MB/s | 79 kB 00:00 2024-03-19T01:29:18.697 INFO:teuthology.orchestra.run.smithi099.stdout:(133/174): python3-natsort-7.1.1-5.el9.noarch.r 1.7 MB/s | 58 kB 00:00 2024-03-19T01:29:18.748 INFO:teuthology.orchestra.run.smithi099.stdout:(134/174): python3-pecan-1.4.2-3.el9.noarch.rpm 5.3 MB/s | 272 kB 00:00 2024-03-19T01:29:18.790 INFO:teuthology.orchestra.run.smithi099.stdout:(135/174): python3-kubernetes-26.1.0-2.el9.noar 4.7 MB/s | 1.0 MB 00:00 2024-03-19T01:29:18.807 INFO:teuthology.orchestra.run.smithi099.stdout:(136/174): python3-portend-3.1.0-2.el9.noarch.r 281 kB/s | 16 kB 00:00 2024-03-19T01:29:18.890 INFO:teuthology.orchestra.run.smithi099.stdout:(137/174): python3-pyperclip-1.8.2-2.el9.noarch 339 kB/s | 28 kB 00:00 2024-03-19T01:29:18.916 INFO:teuthology.orchestra.run.smithi099.stdout:(138/174): python3-pyOpenSSL-21.0.0-1.el9.noarc 717 kB/s | 90 kB 00:00 2024-03-19T01:29:18.932 INFO:teuthology.orchestra.run.smithi099.stdout:(139/174): python3-repoze-lru-0.7-16.el9.noarch 733 kB/s | 31 kB 00:00 2024-03-19T01:29:19.008 INFO:teuthology.orchestra.run.smithi099.stdout:(140/174): libarrow-9.0.0-10.el9.x86_64.rpm 3.2 MB/s | 4.4 MB 00:01 2024-03-19T01:29:19.025 INFO:teuthology.orchestra.run.smithi099.stdout:(141/174): python3-rsa-4.9-2.el9.noarch.rpm 644 kB/s | 59 kB 00:00 2024-03-19T01:29:19.050 INFO:teuthology.orchestra.run.smithi099.stdout:(142/174): python3-routes-2.5.1-5.el9.noarch.rp 1.4 MB/s | 188 kB 00:00 2024-03-19T01:29:19.066 INFO:teuthology.orchestra.run.smithi099.stdout:(143/174): python3-tempora-5.0.0-2.el9.noarch.r 609 kB/s | 36 kB 00:00 2024-03-19T01:29:19.084 INFO:teuthology.orchestra.run.smithi099.stdout:(144/174): python3-typing-extensions-4.4.0-2.el 870 kB/s | 51 kB 00:00 2024-03-19T01:29:19.117 INFO:teuthology.orchestra.run.smithi099.stdout:(145/174): python3-webob-1.8.7-6.el9.noarch.rpm 3.3 MB/s | 230 kB 00:00 2024-03-19T01:29:19.143 INFO:teuthology.orchestra.run.smithi099.stdout:(146/174): python3-websocket-client-1.2.3-2.el9 1.2 MB/s | 90 kB 00:00 2024-03-19T01:29:19.168 INFO:teuthology.orchestra.run.smithi099.stdout:(147/174): python3-werkzeug-2.0.3-3.el9.1.noarc 5.0 MB/s | 427 kB 00:00 2024-03-19T01:29:19.210 INFO:teuthology.orchestra.run.smithi099.stdout:(148/174): python3-zc-lockfile-2.0-10.el9.noarc 216 kB/s | 20 kB 00:00 2024-03-19T01:29:19.235 INFO:teuthology.orchestra.run.smithi099.stdout:(149/174): re2-20211101-3.el9.x86_64.rpm 2.0 MB/s | 192 kB 00:00 2024-03-19T01:29:19.286 INFO:teuthology.orchestra.run.smithi099.stdout:(150/174): thrift-0.15.0-2.el9.x86_64.rpm 13 MB/s | 1.6 MB 00:00 2024-03-19T01:29:19.469 INFO:teuthology.orchestra.run.smithi099.stdout:(151/174): xfsdump-3.1.12-4.el9.x86_64.rpm 2.4 MB/s | 321 kB 00:00 2024-03-19T01:29:19.686 INFO:teuthology.orchestra.run.smithi099.stdout:(152/174): librados2-19.0.0-2206.gb6ae2491.el9. 7.1 MB/s | 3.4 MB 00:00 2024-03-19T01:29:20.353 INFO:teuthology.orchestra.run.smithi099.stdout:(153/174): librbd1-19.0.0-2206.gb6ae2491.el9.x8 2.8 MB/s | 3.1 MB 00:01 2024-03-19T01:29:20.628 INFO:teuthology.orchestra.run.smithi099.stdout:(154/174): NetworkManager-team-1.47.2-1.el9.x86 153 kB/s | 42 kB 00:00 2024-03-19T01:29:20.995 INFO:teuthology.orchestra.run.smithi099.stdout:(155/174): NetworkManager-tui-1.47.2-1.el9.x86_ 677 kB/s | 248 kB 00:00 2024-03-19T01:29:21.103 INFO:teuthology.orchestra.run.smithi099.stdout:(156/174): acl-2.3.1-4.el9.x86_64.rpm 659 kB/s | 71 kB 00:00 2024-03-19T01:29:21.270 INFO:teuthology.orchestra.run.smithi099.stdout:(157/174): elfutils-debuginfod-client-0.190-2.e 219 kB/s | 36 kB 00:00 2024-03-19T01:29:21.437 INFO:teuthology.orchestra.run.smithi099.stdout:(158/174): NetworkManager-libnm-1.47.2-1.el9.x8 1.0 MB/s | 1.8 MB 00:01 2024-03-19T01:29:21.463 INFO:teuthology.orchestra.run.smithi099.stdout:(159/174): elfutils-libelf-0.190-2.el9.x86_64.r 1.0 MB/s | 192 kB 00:00 2024-03-19T01:29:21.571 INFO:teuthology.orchestra.run.smithi099.stdout:(160/174): NetworkManager-1.47.2-1.el9.x86_64.r 1.1 MB/s | 2.3 MB 00:02 2024-03-19T01:29:21.596 INFO:teuthology.orchestra.run.smithi099.stdout:(161/174): libacl-2.3.1-4.el9.x86_64.rpm 173 kB/s | 23 kB 00:00 2024-03-19T01:29:21.696 INFO:teuthology.orchestra.run.smithi099.stdout:(162/174): libblkid-2.37.4-18.el9.x86_64.rpm 865 kB/s | 108 kB 00:00 2024-03-19T01:29:21.730 INFO:teuthology.orchestra.run.smithi099.stdout:(163/174): elfutils-libs-0.190-2.el9.x86_64.rpm 873 kB/s | 256 kB 00:00 2024-03-19T01:29:21.847 INFO:teuthology.orchestra.run.smithi099.stdout:(164/174): libfdisk-2.37.4-18.el9.x86_64.rpm 618 kB/s | 155 kB 00:00 2024-03-19T01:29:21.872 INFO:teuthology.orchestra.run.smithi099.stdout:(165/174): libsmartcols-2.37.4-18.el9.x86_64.rp 450 kB/s | 64 kB 00:00 2024-03-19T01:29:21.931 INFO:teuthology.orchestra.run.smithi099.stdout:(166/174): libmount-2.37.4-18.el9.x86_64.rpm 580 kB/s | 136 kB 00:00 2024-03-19T01:29:21.956 INFO:teuthology.orchestra.run.smithi099.stdout:(167/174): libuuid-2.37.4-18.el9.x86_64.rpm 263 kB/s | 29 kB 00:00 2024-03-19T01:29:23.323 INFO:teuthology.orchestra.run.smithi099.stdout:(168/174): openssl-3.0.7-27.el9.x86_64.rpm 854 kB/s | 1.2 MB 00:01 2024-03-19T01:29:23.806 INFO:teuthology.orchestra.run.smithi099.stdout:(169/174): util-linux-core-2.37.4-18.el9.x86_64 963 kB/s | 465 kB 00:00 2024-03-19T01:29:23.923 INFO:teuthology.orchestra.run.smithi099.stdout:(170/174): openssl-libs-3.0.7-27.el9.x86_64.rpm 1.1 MB/s | 2.2 MB 00:01 2024-03-19T01:29:23.948 INFO:teuthology.orchestra.run.smithi099.stdout:(171/174): util-linux-user-2.37.4-18.el9.x86_64 219 kB/s | 31 kB 00:00 2024-03-19T01:29:24.057 INFO:teuthology.orchestra.run.smithi099.stdout:(172/174): util-linux-2.37.4-18.el9.x86_64.rpm 1.1 MB/s | 2.3 MB 00:02 2024-03-19T01:29:24.199 INFO:teuthology.orchestra.run.smithi099.stdout:(173/174): libacl-devel-2.3.1-4.el9.x86_64.rpm 297 kB/s | 74 kB 00:00 2024-03-19T01:29:24.840 INFO:teuthology.orchestra.run.smithi099.stdout:(174/174): xfsprogs-6.3.0-1.el9.x86_64.rpm 1.2 MB/s | 1.1 MB 00:00 2024-03-19T01:29:24.844 INFO:teuthology.orchestra.run.smithi099.stdout:-------------------------------------------------------------------------------- 2024-03-19T01:29:24.844 INFO:teuthology.orchestra.run.smithi099.stdout:Total 1.5 MB/s | 224 MB 02:28 2024-03-19T01:29:25.289 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction check 2024-03-19T01:29:25.558 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction check succeeded. 2024-03-19T01:29:25.559 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction test 2024-03-19T01:29:27.572 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction test succeeded. 2024-03-19T01:29:27.572 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction 2024-03-19T01:29:34.465 INFO:teuthology.orchestra.run.smithi099.stdout: Preparing : 1/1 2024-03-19T01:29:34.561 INFO:teuthology.orchestra.run.smithi099.stdout: Upgrading : openssl-libs-1:3.0.7-27.el9.x86_64 1/198 2024-03-19T01:29:34.595 INFO:teuthology.orchestra.run.smithi099.stdout: Upgrading : libuuid-2.37.4-18.el9.x86_64 2/198 2024-03-19T01:29:34.642 INFO:teuthology.orchestra.run.smithi099.stdout: Upgrading : libblkid-2.37.4-18.el9.x86_64 3/198 2024-03-19T01:29:34.651 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: libblkid-2.37.4-18.el9.x86_64 3/198 2024-03-19T01:29:34.698 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 4/198 2024-03-19T01:29:34.754 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 5/198 2024-03-19T01:29:34.787 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : thrift-0.15.0-2.el9.x86_64 6/198 2024-03-19T01:29:34.812 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 7/198 2024-03-19T01:29:34.865 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 8/198 2024-03-19T01:29:35.193 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 9/198 2024-03-19T01:29:35.255 INFO:teuthology.orchestra.run.smithi099.stdout: Upgrading : librados2-2:19.0.0-2206.gb6ae2491.el9.x86_64 10/198 2024-03-19T01:29:35.376 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: librados2-2:19.0.0-2206.gb6ae2491.el9.x86_64 10/198 2024-03-19T01:29:35.406 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : libcephfs2-2:19.0.0-2206.gb6ae2491.el9.x86_64 11/198 2024-03-19T01:29:35.496 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: libcephfs2-2:19.0.0-2206.gb6ae2491.el9.x86_64 11/198 2024-03-19T01:29:35.536 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-rados-2:19.0.0-2206.gb6ae2491.el9.x86_64 12/198 2024-03-19T01:29:35.586 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : postgresql-private-libs-13.11-1.el9.x86_64 13/198 2024-03-19T01:29:35.771 INFO:teuthology.orchestra.run.smithi099.stdout: Upgrading : elfutils-libelf-0.190-2.el9.x86_64 14/198 2024-03-19T01:29:35.816 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : postgresql-13.11-1.el9.x86_64 15/198 2024-03-19T01:29:35.844 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : libcephsqlite-2:19.0.0-2206.gb6ae2491.el9.x86_64 16/198 2024-03-19T01:29:35.946 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: libcephsqlite-2:19.0.0-2206.gb6ae2491.el9.x86_64 16/198 2024-03-19T01:29:35.970 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : libradosstriper1-2:19.0.0-2206.gb6ae2491.el9.x86 17/198 2024-03-19T01:29:36.027 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: libradosstriper1-2:19.0.0-2206.gb6ae2491.el9.x86 17/198 2024-03-19T01:29:36.065 INFO:teuthology.orchestra.run.smithi099.stdout: Upgrading : libmount-2.37.4-18.el9.x86_64 18/198 2024-03-19T01:29:36.088 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : qatlib-service-23.11.0-1.el9.x86_64 19/198 2024-03-19T01:29:36.103 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: qatlib-service-23.11.0-1.el9.x86_64 19/198 2024-03-19T01:29:36.293 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: qatlib-23.11.0-1.el9.x86_64 20/198 2024-03-19T01:29:36.320 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : qatlib-23.11.0-1.el9.x86_64 20/198 2024-03-19T01:29:36.351 INFO:teuthology.orchestra.run.smithi099.stdout: Upgrading : libsmartcols-2.37.4-18.el9.x86_64 21/198 2024-03-19T01:29:36.528 INFO:teuthology.orchestra.run.smithi099.stdout: Upgrading : libacl-2.3.1-4.el9.x86_64 22/198 2024-03-19T01:29:36.678 INFO:teuthology.orchestra.run.smithi099.stdout: Upgrading : NetworkManager-libnm-1:1.47.2-1.el9.x86_64 23/198 2024-03-19T01:29:36.729 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: NetworkManager-libnm-1:1.47.2-1.el9.x86_64 23/198 2024-03-19T01:29:36.823 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: NetworkManager-1:1.47.2-1.el9.x86_64 24/198 2024-03-19T01:29:36.872 INFO:teuthology.orchestra.run.smithi099.stdout: Upgrading : NetworkManager-1:1.47.2-1.el9.x86_64 24/198 2024-03-19T01:29:37.065 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: NetworkManager-1:1.47.2-1.el9.x86_64 24/198 2024-03-19T01:29:37.177 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : re2-1:20211101-3.el9.x86_64 25/198 2024-03-19T01:29:37.272 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : libarrow-9.0.0-10.el9.x86_64 26/198 2024-03-19T01:29:37.311 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 27/198 2024-03-19T01:29:37.338 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : liboath-2.6.7-2.el9.x86_64 28/198 2024-03-19T01:29:37.373 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : libquadmath-11.4.1-3.el9.x86_64 29/198 2024-03-19T01:29:37.447 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : libgfortran-11.4.1-3.el9.x86_64 30/198 2024-03-19T01:29:37.500 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-urllib3-1.26.5-5.el9.noarch 31/198 2024-03-19T01:29:37.539 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 32/198 2024-03-19T01:29:37.604 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 33/198 2024-03-19T01:29:37.646 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 34/198 2024-03-19T01:29:37.689 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 35/198 2024-03-19T01:29:37.723 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : protobuf-3.14.0-13.el9.x86_64 36/198 2024-03-19T01:29:37.750 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : lua-5.4.4-4.el9.x86_64 37/198 2024-03-19T01:29:37.818 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 38/198 2024-03-19T01:29:37.855 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-ceph-common-2:19.0.0-2206.gb6ae2491.el9. 39/198 2024-03-19T01:29:37.898 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-ceph-argparse-2:19.0.0-2206.gb6ae2491.el 40/198 2024-03-19T01:29:37.927 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-cephfs-2:19.0.0-2206.gb6ae2491.el9.x86_6 41/198 2024-03-19T01:29:37.997 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : lua-devel-5.4.4-4.el9.x86_64 42/198 2024-03-19T01:29:38.050 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : luarocks-3.9.2-1.el9.noarch 43/198 2024-03-19T01:29:38.123 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : protobuf-compiler-3.14.0-13.el9.x86_64 44/198 2024-03-19T01:29:38.256 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 45/198 2024-03-19T01:29:38.310 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 46/198 2024-03-19T01:29:38.825 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-rsa-4.9-2.el9.noarch 47/198 2024-03-19T01:29:38.906 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 48/198 2024-03-19T01:29:38.993 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-jinja2-2.11.3-5.el9.noarch 49/198 2024-03-19T01:29:39.050 INFO:teuthology.orchestra.run.smithi099.stdout: Upgrading : util-linux-core-2.37.4-18.el9.x86_64 50/198 2024-03-19T01:29:39.073 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: util-linux-core-2.37.4-18.el9.x86_64 50/198 2024-03-19T01:29:39.128 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : qatzip-libs-1.1.2-2.el9.x86_64 51/198 2024-03-19T01:29:39.174 INFO:teuthology.orchestra.run.smithi099.stdout: Upgrading : elfutils-libs-0.190-2.el9.x86_64 52/198 2024-03-19T01:29:39.203 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : librados-devel-2:19.0.0-2206.gb6ae2491.el9.x86_6 53/198 2024-03-19T01:29:39.233 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 54/198 2024-03-19T01:29:39.521 INFO:teuthology.orchestra.run.smithi099.stdout: Upgrading : libfdisk-2.37.4-18.el9.x86_64 55/198 2024-03-19T01:29:39.918 INFO:teuthology.orchestra.run.smithi099.stdout: Upgrading : util-linux-2.37.4-18.el9.x86_64 56/198 2024-03-19T01:29:40.117 INFO:teuthology.orchestra.run.smithi099.stdout: Upgrading : xfsprogs-6.3.0-1.el9.x86_64 57/198 2024-03-19T01:29:40.161 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : libuuid-devel-2.37.4-18.el9.x86_64 58/198 2024-03-19T01:29:40.302 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : socat-1.7.4.1-5.el9.x86_64 59/198 2024-03-19T01:29:40.371 INFO:teuthology.orchestra.run.smithi099.stdout: Upgrading : openssl-1:3.0.7-27.el9.x86_64 60/198 2024-03-19T01:29:40.425 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 61/198 2024-03-19T01:29:40.498 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 62/198 2024-03-19T01:29:40.530 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-webob-1.8.7-6.el9.noarch 63/198 2024-03-19T01:29:40.567 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-typing-extensions-4.4.0-2.el9.noarch 64/198 2024-03-19T01:29:40.614 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 65/198 2024-03-19T01:29:40.650 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 66/198 2024-03-19T01:29:40.692 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-pyperclip-1.8.2-2.el9.noarch 67/198 2024-03-19T01:29:40.735 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 68/198 2024-03-19T01:29:40.867 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 69/198 2024-03-19T01:29:40.915 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 70/198 2024-03-19T01:29:40.945 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-colorama-0.4.4-12.el9.noarch 71/198 2024-03-19T01:29:40.987 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-certifi-2021.10.8-2.el9.noarch 72/198 2024-03-19T01:29:41.079 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 73/198 2024-03-19T01:29:41.120 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-google-auth-1:2.28.0-1.el9.noarch 74/198 2024-03-19T01:29:41.161 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 75/198 2024-03-19T01:29:41.195 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : gperftools-libs-2.9.1-2.el9.x86_64 76/198 2024-03-19T01:29:41.219 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : libarrow-doc-9.0.0-10.el9.noarch 77/198 2024-03-19T01:29:41.305 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : grpc-data-1.46.7-10.el9.noarch 78/198 2024-03-19T01:29:41.427 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : abseil-cpp-20211102.0-3.el9.x86_64 79/198 2024-03-19T01:29:41.535 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-grpcio-1.46.7-10.el9.x86_64 80/198 2024-03-19T01:29:41.577 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-devel-3.9.17-2.el9.x86_64 81/198 2024-03-19T01:29:41.625 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-ply-3.11-14.el9.noarch 82/198 2024-03-19T01:29:41.697 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 83/198 2024-03-19T01:29:41.886 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 84/198 2024-03-19T01:29:41.941 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 85/198 2024-03-19T01:29:42.065 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 86/198 2024-03-19T01:29:42.105 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-asyncssh-2.13.2-4.el9.noarch 87/198 2024-03-19T01:29:42.136 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 88/198 2024-03-19T01:29:42.167 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : fuse-2.9.9-15.el9.x86_64 89/198 2024-03-19T01:29:42.195 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : zlib-devel-1.2.11-40.el9.x86_64 90/198 2024-03-19T01:29:42.229 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 91/198 2024-03-19T01:29:42.268 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : uuid-1.6.2-55.el9.x86_64 92/198 2024-03-19T01:29:42.300 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-wcwidth-0.2.5-8.el9.noarch 93/198 2024-03-19T01:29:42.330 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 94/198 2024-03-19T01:29:42.406 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 95/198 2024-03-19T01:29:42.441 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 96/198 2024-03-19T01:29:42.474 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 97/198 2024-03-19T01:29:42.515 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 98/198 2024-03-19T01:29:42.550 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 99/198 2024-03-19T01:29:42.640 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 100/198 2024-03-19T01:29:42.729 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 101/198 2024-03-19T01:29:42.772 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-protobuf-3.14.0-13.el9.noarch 102/198 2024-03-19T01:29:42.803 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-grpcio-tools-1.46.7-10.el9.x86_64 103/198 2024-03-19T01:29:42.850 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 104/198 2024-03-19T01:29:42.946 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-packaging-20.9-5.el9.noarch 105/198 2024-03-19T01:29:42.996 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 106/198 2024-03-19T01:29:43.713 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 107/198 2024-03-19T01:29:43.760 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 108/198 2024-03-19T01:29:43.794 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 109/198 2024-03-19T01:29:43.841 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 110/198 2024-03-19T01:29:43.920 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-attrs-20.3.0-7.el9.noarch 111/198 2024-03-19T01:29:43.969 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-cmd2-2.4.2-2.el9.noarch 112/198 2024-03-19T01:29:44.145 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : openblas-0.3.21-2.el9.x86_64 113/198 2024-03-19T01:29:44.184 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : openblas-openmp-0.3.21-2.el9.x86_64 114/198 2024-03-19T01:29:44.260 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 115/198 2024-03-19T01:29:44.891 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 116/198 2024-03-19T01:29:44.998 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 117/198 2024-03-19T01:29:46.221 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 118/198 2024-03-19T01:29:46.308 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 119/198 2024-03-19T01:29:46.340 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : oniguruma-6.9.6-1.el9.5.x86_64 120/198 2024-03-19T01:29:46.379 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : jq-1.6-16.el9.x86_64 121/198 2024-03-19T01:29:46.406 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : libzstd-devel-1.5.1-2.el9.x86_64 122/198 2024-03-19T01:29:46.717 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 123/198 2024-03-19T01:29:46.745 INFO:teuthology.orchestra.run.smithi099.stdout: Upgrading : librbd1-2:19.0.0-2206.gb6ae2491.el9.x86_64 124/198 2024-03-19T01:29:46.835 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: librbd1-2:19.0.0-2206.gb6ae2491.el9.x86_64 124/198 2024-03-19T01:29:46.866 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-rbd-2:19.0.0-2206.gb6ae2491.el9.x86_64 125/198 2024-03-19T01:29:46.903 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : boost-program-options-1.75.0-8.el9.x86_64 126/198 2024-03-19T01:29:47.359 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : parquet-libs-9.0.0-10.el9.x86_64 127/198 2024-03-19T01:29:47.389 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : librgw2-2:19.0.0-2206.gb6ae2491.el9.x86_64 128/198 2024-03-19T01:29:47.443 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: librgw2-2:19.0.0-2206.gb6ae2491.el9.x86_64 128/198 2024-03-19T01:29:47.466 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-rgw-2:19.0.0-2206.gb6ae2491.el9.x86_64 129/198 2024-03-19T01:29:49.624 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: ceph-common-2:19.0.0-2206.gb6ae2491.el9.x86_64 130/198 2024-03-19T01:29:49.670 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-common-2:19.0.0-2206.gb6ae2491.el9.x86_64 130/198 2024-03-19T01:29:50.224 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: ceph-common-2:19.0.0-2206.gb6ae2491.el9.x86_64 130/198 2024-03-19T01:29:50.265 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-base-2:19.0.0-2206.gb6ae2491.el9.x86_64 131/198 2024-03-19T01:29:50.359 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: ceph-base-2:19.0.0-2206.gb6ae2491.el9.x86_64 131/198 2024-03-19T01:29:50.360 INFO:teuthology.orchestra.run.smithi099.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-03-19T01:29:50.360 INFO:teuthology.orchestra.run.smithi099.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-03-19T01:29:50.360 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:29:50.400 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-selinux-2:19.0.0-2206.gb6ae2491.el9.x86_64 132/198 2024-03-19T01:30:01.988 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: ceph-selinux-2:19.0.0-2206.gb6ae2491.el9.x86_64 132/198 2024-03-19T01:30:01.988 INFO:teuthology.orchestra.run.smithi099.stdout:skipping the directory /sys 2024-03-19T01:30:01.988 INFO:teuthology.orchestra.run.smithi099.stdout:skipping the directory /proc 2024-03-19T01:30:01.988 INFO:teuthology.orchestra.run.smithi099.stdout:skipping the directory /mnt 2024-03-19T01:30:01.988 INFO:teuthology.orchestra.run.smithi099.stdout:skipping the directory /var/tmp 2024-03-19T01:30:01.988 INFO:teuthology.orchestra.run.smithi099.stdout:skipping the directory /home 2024-03-19T01:30:01.988 INFO:teuthology.orchestra.run.smithi099.stdout:skipping the directory /root 2024-03-19T01:30:01.988 INFO:teuthology.orchestra.run.smithi099.stdout:skipping the directory /tmp 2024-03-19T01:30:01.989 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:30:02.153 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-volume-2:19.0.0-2206.gb6ae2491.el9.noarch 133/198 2024-03-19T01:30:02.167 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: ceph-volume-2:19.0.0-2206.gb6ae2491.el9.noarch 133/198 2024-03-19T01:30:02.167 INFO:teuthology.orchestra.run.smithi099.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-19T01:30:02.168 INFO:teuthology.orchestra.run.smithi099.stdout:Invalid unit name "ceph-volume@*.service" escaped as "ceph-volume@\x2a.service". 2024-03-19T01:30:02.168 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:30:03.639 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-osd-2:19.0.0-2206.gb6ae2491.el9.x86_64 134/198 2024-03-19T01:30:03.664 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: ceph-osd-2:19.0.0-2206.gb6ae2491.el9.x86_64 134/198 2024-03-19T01:30:03.665 INFO:teuthology.orchestra.run.smithi099.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-19T01:30:03.665 INFO:teuthology.orchestra.run.smithi099.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-03-19T01:30:03.665 INFO:teuthology.orchestra.run.smithi099.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-03-19T01:30:03.665 INFO:teuthology.orchestra.run.smithi099.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-03-19T01:30:03.665 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:30:03.881 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-mds-2:19.0.0-2206.gb6ae2491.el9.x86_64 135/198 2024-03-19T01:30:03.903 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: ceph-mds-2:19.0.0-2206.gb6ae2491.el9.x86_64 135/198 2024-03-19T01:30:03.903 INFO:teuthology.orchestra.run.smithi099.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-19T01:30:03.903 INFO:teuthology.orchestra.run.smithi099.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-03-19T01:30:03.903 INFO:teuthology.orchestra.run.smithi099.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-03-19T01:30:03.903 INFO:teuthology.orchestra.run.smithi099.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-03-19T01:30:03.903 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:30:04.340 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-mon-2:19.0.0-2206.gb6ae2491.el9.x86_64 136/198 2024-03-19T01:30:04.362 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: ceph-mon-2:19.0.0-2206.gb6ae2491.el9.x86_64 136/198 2024-03-19T01:30:04.362 INFO:teuthology.orchestra.run.smithi099.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-19T01:30:04.362 INFO:teuthology.orchestra.run.smithi099.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-03-19T01:30:04.362 INFO:teuthology.orchestra.run.smithi099.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-03-19T01:30:04.362 INFO:teuthology.orchestra.run.smithi099.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-03-19T01:30:04.362 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:30:04.475 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : automake-1.16.2-8.el9.noarch 137/198 2024-03-19T01:30:04.959 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: cephadm-2:19.0.0-2206.gb6ae2491.el9.noarch 138/198 2024-03-19T01:30:04.982 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : cephadm-2:19.0.0-2206.gb6ae2491.el9.noarch 138/198 2024-03-19T01:30:05.014 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-prometheus-alerts-2:19.0.0-2206.gb6ae2491.e 139/198 2024-03-19T01:30:05.073 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-grafana-dashboards-2:19.0.0-2206.gb6ae2491. 140/198 2024-03-19T01:30:05.096 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-mgr-cephadm-2:19.0.0-2206.gb6ae2491.el9.noa 141/198 2024-03-19T01:30:05.330 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: ceph-mgr-cephadm-2:19.0.0-2206.gb6ae2491.el9.noa 141/198 2024-03-19T01:30:05.364 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-mgr-dashboard-2:19.0.0-2206.gb6ae2491.el9.n 142/198 2024-03-19T01:30:06.325 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: ceph-mgr-dashboard-2:19.0.0-2206.gb6ae2491.el9.n 142/198 2024-03-19T01:30:06.355 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-mgr-diskprediction-local-2:19.0.0-2206.gb6a 143/198 2024-03-19T01:30:06.456 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:19.0.0-2206.gb6a 143/198 2024-03-19T01:30:06.620 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-mgr-modules-core-2:19.0.0-2206.gb6ae2491.el 144/198 2024-03-19T01:30:06.660 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-mgr-2:19.0.0-2206.gb6ae2491.el9.x86_64 145/198 2024-03-19T01:30:06.682 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: ceph-mgr-2:19.0.0-2206.gb6ae2491.el9.x86_64 145/198 2024-03-19T01:30:06.682 INFO:teuthology.orchestra.run.smithi099.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-19T01:30:06.682 INFO:teuthology.orchestra.run.smithi099.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-03-19T01:30:06.682 INFO:teuthology.orchestra.run.smithi099.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-03-19T01:30:06.682 INFO:teuthology.orchestra.run.smithi099.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-03-19T01:30:06.682 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:30:06.752 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-mgr-rook-2:19.0.0-2206.gb6ae2491.el9.noarch 146/198 2024-03-19T01:30:06.762 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: ceph-mgr-rook-2:19.0.0-2206.gb6ae2491.el9.noarch 146/198 2024-03-19T01:30:06.810 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-2:19.0.0-2206.gb6ae2491.el9.x86_64 147/198 2024-03-19T01:30:07.677 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : libtool-2.4.6-45.el9.x86_64 148/198 2024-03-19T01:30:07.704 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-radosgw-2:19.0.0-2206.gb6ae2491.el9.x86_64 149/198 2024-03-19T01:30:07.727 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: ceph-radosgw-2:19.0.0-2206.gb6ae2491.el9.x86_64 149/198 2024-03-19T01:30:07.727 INFO:teuthology.orchestra.run.smithi099.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-19T01:30:07.727 INFO:teuthology.orchestra.run.smithi099.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-03-19T01:30:07.727 INFO:teuthology.orchestra.run.smithi099.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-03-19T01:30:07.727 INFO:teuthology.orchestra.run.smithi099.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-03-19T01:30:07.727 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:30:07.771 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-immutable-object-cache-2:19.0.0-2206.gb6ae2 150/198 2024-03-19T01:30:07.792 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: ceph-immutable-object-cache-2:19.0.0-2206.gb6ae2 150/198 2024-03-19T01:30:07.793 INFO:teuthology.orchestra.run.smithi099.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-19T01:30:07.793 INFO:teuthology.orchestra.run.smithi099.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-03-19T01:30:07.793 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:30:07.872 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : cephfs-mirror-2:19.0.0-2206.gb6ae2491.el9.x86_64 151/198 2024-03-19T01:30:07.893 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: cephfs-mirror-2:19.0.0-2206.gb6ae2491.el9.x86_64 151/198 2024-03-19T01:30:07.893 INFO:teuthology.orchestra.run.smithi099.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-19T01:30:07.893 INFO:teuthology.orchestra.run.smithi099.stdout:Invalid unit name "cephfs-mirror@*.service" escaped as "cephfs-mirror@\x2a.service". 2024-03-19T01:30:07.894 INFO:teuthology.orchestra.run.smithi099.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/cephfs-mirror.target → /usr/lib/systemd/system/cephfs-mirror.target. 2024-03-19T01:30:07.894 INFO:teuthology.orchestra.run.smithi099.stdout:Created symlink /etc/systemd/system/ceph.target.wants/cephfs-mirror.target → /usr/lib/systemd/system/cephfs-mirror.target. 2024-03-19T01:30:07.894 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:30:08.198 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : rbd-mirror-2:19.0.0-2206.gb6ae2491.el9.x86_64 152/198 2024-03-19T01:30:08.220 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: rbd-mirror-2:19.0.0-2206.gb6ae2491.el9.x86_64 152/198 2024-03-19T01:30:08.220 INFO:teuthology.orchestra.run.smithi099.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-19T01:30:08.220 INFO:teuthology.orchestra.run.smithi099.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-03-19T01:30:08.220 INFO:teuthology.orchestra.run.smithi099.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-03-19T01:30:08.220 INFO:teuthology.orchestra.run.smithi099.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-03-19T01:30:08.220 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:30:12.643 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-test-2:19.0.0-2206.gb6ae2491.el9.x86_64 153/198 2024-03-19T01:30:12.684 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : rbd-fuse-2:19.0.0-2206.gb6ae2491.el9.x86_64 154/198 2024-03-19T01:30:12.714 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : rbd-nbd-2:19.0.0-2206.gb6ae2491.el9.x86_64 155/198 2024-03-19T01:30:12.742 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : elfutils-libelf-devel-0.190-2.el9.x86_64 156/198 2024-03-19T01:30:12.929 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : cephfs-shell-2:19.0.0-2206.gb6ae2491.el9.x86_64 157/198 2024-03-19T01:30:13.049 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : postgresql-contrib-13.11-1.el9.x86_64 158/198 2024-03-19T01:30:13.097 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-fuse-2:19.0.0-2206.gb6ae2491.el9.x86_64 159/198 2024-03-19T01:30:13.151 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : xfsprogs-devel-6.3.0-1.el9.x86_64 160/198 2024-03-19T01:30:13.207 INFO:teuthology.orchestra.run.smithi099.stdout: Upgrading : xfsdump-3.1.12-4.el9.x86_64 161/198 2024-03-19T01:30:13.242 INFO:teuthology.orchestra.run.smithi099.stdout: Upgrading : util-linux-user-2.37.4-18.el9.x86_64 162/198 2024-03-19T01:30:13.274 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : libcephfs-devel-2:19.0.0-2206.gb6ae2491.el9.x86_ 163/198 2024-03-19T01:30:13.306 INFO:teuthology.orchestra.run.smithi099.stdout: Upgrading : elfutils-debuginfod-client-0.190-2.el9.x86_64 164/198 2024-03-19T01:30:13.336 INFO:teuthology.orchestra.run.smithi099.stdout: Upgrading : NetworkManager-team-1:1.47.2-1.el9.x86_64 165/198 2024-03-19T01:30:13.391 INFO:teuthology.orchestra.run.smithi099.stdout: Upgrading : NetworkManager-tui-1:1.47.2-1.el9.x86_64 166/198 2024-03-19T01:30:13.464 INFO:teuthology.orchestra.run.smithi099.stdout: Upgrading : acl-2.3.1-4.el9.x86_64 167/198 2024-03-19T01:30:13.501 INFO:teuthology.orchestra.run.smithi099.stdout: Upgrading : libacl-devel-2.3.1-4.el9.x86_64 168/198 2024-03-19T01:30:14.082 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: postgresql-server-13.11-1.el9.x86_64 169/198 2024-03-19T01:30:14.148 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : postgresql-server-13.11-1.el9.x86_64 169/198 2024-03-19T01:30:14.168 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: postgresql-server-13.11-1.el9.x86_64 169/198 2024-03-19T01:30:14.971 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : cephfs-top-2:19.0.0-2206.gb6ae2491.el9.noarch 170/198 2024-03-19T01:30:15.060 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : openssl-devel-1:3.0.7-27.el9.x86_64 171/198 2024-03-19T01:30:15.101 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : libaio-devel-0.3.111-13.el9.x86_64 172/198 2024-03-19T01:30:15.185 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : flex-2.6.4-9.el9.x86_64 173/198 2024-03-19T01:30:15.213 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : bison-3.7.4-5.el9.x86_64 174/198 2024-03-19T01:30:15.238 INFO:teuthology.orchestra.run.smithi099.stdout: Cleanup : NetworkManager-tui-1:1.44.0-2.el9.x86_64 175/198 2024-03-19T01:30:15.267 INFO:teuthology.orchestra.run.smithi099.stdout: Cleanup : elfutils-debuginfod-client-0.189-3.el9.x86_64 176/198 2024-03-19T01:30:15.290 INFO:teuthology.orchestra.run.smithi099.stdout: Cleanup : elfutils-libs-0.189-3.el9.x86_64 177/198 2024-03-19T01:30:15.290 INFO:teuthology.orchestra.run.smithi099.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 178/198 2024-03-19T01:30:15.370 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 178/198 2024-03-19T01:30:15.399 INFO:teuthology.orchestra.run.smithi099.stdout: Cleanup : openssl-1:3.0.7-24.el9.x86_64 179/198 2024-03-19T01:30:15.399 INFO:teuthology.orchestra.run.smithi099.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 180/198 2024-03-19T01:30:15.484 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 180/198 2024-03-19T01:30:15.510 INFO:teuthology.orchestra.run.smithi099.stdout: Cleanup : xfsdump-3.1.12-3.el9.x86_64 181/198 2024-03-19T01:30:15.543 INFO:teuthology.orchestra.run.smithi099.stdout: Cleanup : xfsprogs-5.19.0-4.el9.x86_64 182/198 2024-03-19T01:30:15.569 INFO:teuthology.orchestra.run.smithi099.stdout: Cleanup : acl-2.3.1-3.el9.x86_64 183/198 2024-03-19T01:30:15.593 INFO:teuthology.orchestra.run.smithi099.stdout: Cleanup : libacl-devel-2.3.1-3.el9.x86_64 184/198 2024-03-19T01:30:15.627 INFO:teuthology.orchestra.run.smithi099.stdout: Cleanup : util-linux-user-2.37.4-15.el9.x86_64 185/198 2024-03-19T01:30:15.666 INFO:teuthology.orchestra.run.smithi099.stdout: Cleanup : util-linux-2.37.4-15.el9.x86_64 186/198 2024-03-19T01:30:15.698 INFO:teuthology.orchestra.run.smithi099.stdout: Cleanup : util-linux-core-2.37.4-15.el9.x86_64 187/198 2024-03-19T01:30:15.723 INFO:teuthology.orchestra.run.smithi099.stdout: Cleanup : libfdisk-2.37.4-15.el9.x86_64 188/198 2024-03-19T01:30:15.748 INFO:teuthology.orchestra.run.smithi099.stdout: Cleanup : libmount-2.37.4-15.el9.x86_64 189/198 2024-03-19T01:30:15.774 INFO:teuthology.orchestra.run.smithi099.stdout: Cleanup : libblkid-2.37.4-15.el9.x86_64 190/198 2024-03-19T01:30:15.791 INFO:teuthology.orchestra.run.smithi099.stdout: Cleanup : NetworkManager-team-1:1.44.0-2.el9.x86_64 191/198 2024-03-19T01:30:15.797 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: NetworkManager-1:1.44.0-2.el9.x86_64 192/198 2024-03-19T01:30:15.797 INFO:teuthology.orchestra.run.smithi099.stdout: Cleanup : NetworkManager-1:1.44.0-2.el9.x86_64 192/198 2024-03-19T01:30:15.977 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: NetworkManager-1:1.44.0-2.el9.x86_64 192/198 2024-03-19T01:30:15.977 INFO:teuthology.orchestra.run.smithi099.stdout: Cleanup : NetworkManager-libnm-1:1.44.0-2.el9.x86_64 193/198 2024-03-19T01:30:16.060 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: NetworkManager-libnm-1:1.44.0-2.el9.x86_64 193/198 2024-03-19T01:30:16.085 INFO:teuthology.orchestra.run.smithi099.stdout: Cleanup : libuuid-2.37.4-15.el9.x86_64 194/198 2024-03-19T01:30:16.110 INFO:teuthology.orchestra.run.smithi099.stdout: Cleanup : libsmartcols-2.37.4-15.el9.x86_64 195/198 2024-03-19T01:30:16.142 INFO:teuthology.orchestra.run.smithi099.stdout: Cleanup : libacl-2.3.1-3.el9.x86_64 196/198 2024-03-19T01:30:16.169 INFO:teuthology.orchestra.run.smithi099.stdout: Cleanup : openssl-libs-1:3.0.7-24.el9.x86_64 197/198 2024-03-19T01:30:16.234 INFO:teuthology.orchestra.run.smithi099.stdout: Cleanup : elfutils-libelf-0.189-3.el9.x86_64 198/198 2024-03-19T01:30:21.499 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: elfutils-libelf-0.189-3.el9.x86_64 198/198 2024-03-19T01:30:21.499 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-2:19.0.0-2206.gb6ae2491.el9.x86_64 1/198 2024-03-19T01:30:21.500 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-base-2:19.0.0-2206.gb6ae2491.el9.x86_64 2/198 2024-03-19T01:30:21.500 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-common-2:19.0.0-2206.gb6ae2491.el9.x86_64 3/198 2024-03-19T01:30:21.500 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-fuse-2:19.0.0-2206.gb6ae2491.el9.x86_64 4/198 2024-03-19T01:30:21.500 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-immutable-object-cache-2:19.0.0-2206.gb6ae2 5/198 2024-03-19T01:30:21.500 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-mds-2:19.0.0-2206.gb6ae2491.el9.x86_64 6/198 2024-03-19T01:30:21.500 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-mgr-2:19.0.0-2206.gb6ae2491.el9.x86_64 7/198 2024-03-19T01:30:21.500 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-mon-2:19.0.0-2206.gb6ae2491.el9.x86_64 8/198 2024-03-19T01:30:21.500 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-osd-2:19.0.0-2206.gb6ae2491.el9.x86_64 9/198 2024-03-19T01:30:21.500 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-radosgw-2:19.0.0-2206.gb6ae2491.el9.x86_64 10/198 2024-03-19T01:30:21.500 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-selinux-2:19.0.0-2206.gb6ae2491.el9.x86_64 11/198 2024-03-19T01:30:21.500 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-test-2:19.0.0-2206.gb6ae2491.el9.x86_64 12/198 2024-03-19T01:30:21.500 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : cephfs-mirror-2:19.0.0-2206.gb6ae2491.el9.x86_64 13/198 2024-03-19T01:30:21.500 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : cephfs-shell-2:19.0.0-2206.gb6ae2491.el9.x86_64 14/198 2024-03-19T01:30:21.501 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : libcephfs-devel-2:19.0.0-2206.gb6ae2491.el9.x86_ 15/198 2024-03-19T01:30:21.501 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : libcephfs2-2:19.0.0-2206.gb6ae2491.el9.x86_64 16/198 2024-03-19T01:30:21.501 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : libcephsqlite-2:19.0.0-2206.gb6ae2491.el9.x86_64 17/198 2024-03-19T01:30:21.501 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : librados-devel-2:19.0.0-2206.gb6ae2491.el9.x86_6 18/198 2024-03-19T01:30:21.501 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : libradosstriper1-2:19.0.0-2206.gb6ae2491.el9.x86 19/198 2024-03-19T01:30:21.501 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : librgw2-2:19.0.0-2206.gb6ae2491.el9.x86_64 20/198 2024-03-19T01:30:21.501 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-ceph-argparse-2:19.0.0-2206.gb6ae2491.el 21/198 2024-03-19T01:30:21.501 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-ceph-common-2:19.0.0-2206.gb6ae2491.el9. 22/198 2024-03-19T01:30:21.501 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-cephfs-2:19.0.0-2206.gb6ae2491.el9.x86_6 23/198 2024-03-19T01:30:21.501 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-rados-2:19.0.0-2206.gb6ae2491.el9.x86_64 24/198 2024-03-19T01:30:21.501 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-rbd-2:19.0.0-2206.gb6ae2491.el9.x86_64 25/198 2024-03-19T01:30:21.501 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-rgw-2:19.0.0-2206.gb6ae2491.el9.x86_64 26/198 2024-03-19T01:30:21.501 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : rbd-fuse-2:19.0.0-2206.gb6ae2491.el9.x86_64 27/198 2024-03-19T01:30:21.501 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : rbd-mirror-2:19.0.0-2206.gb6ae2491.el9.x86_64 28/198 2024-03-19T01:30:21.502 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : rbd-nbd-2:19.0.0-2206.gb6ae2491.el9.x86_64 29/198 2024-03-19T01:30:21.502 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-grafana-dashboards-2:19.0.0-2206.gb6ae2491. 30/198 2024-03-19T01:30:21.502 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-mgr-cephadm-2:19.0.0-2206.gb6ae2491.el9.noa 31/198 2024-03-19T01:30:21.502 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-mgr-dashboard-2:19.0.0-2206.gb6ae2491.el9.n 32/198 2024-03-19T01:30:21.502 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-mgr-diskprediction-local-2:19.0.0-2206.gb6a 33/198 2024-03-19T01:30:21.502 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-mgr-modules-core-2:19.0.0-2206.gb6ae2491.el 34/198 2024-03-19T01:30:21.502 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-mgr-rook-2:19.0.0-2206.gb6ae2491.el9.noarch 35/198 2024-03-19T01:30:21.502 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-prometheus-alerts-2:19.0.0-2206.gb6ae2491.e 36/198 2024-03-19T01:30:21.502 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-volume-2:19.0.0-2206.gb6ae2491.el9.noarch 37/198 2024-03-19T01:30:21.502 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : cephadm-2:19.0.0-2206.gb6ae2491.el9.noarch 38/198 2024-03-19T01:30:21.502 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : cephfs-top-2:19.0.0-2206.gb6ae2491.el9.noarch 39/198 2024-03-19T01:30:21.502 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : automake-1.16.2-8.el9.noarch 40/198 2024-03-19T01:30:21.503 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : bison-3.7.4-5.el9.x86_64 41/198 2024-03-19T01:30:21.503 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : boost-program-options-1.75.0-8.el9.x86_64 42/198 2024-03-19T01:30:21.503 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : flex-2.6.4-9.el9.x86_64 43/198 2024-03-19T01:30:21.503 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 44/198 2024-03-19T01:30:21.503 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 45/198 2024-03-19T01:30:21.503 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 46/198 2024-03-19T01:30:21.503 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : libaio-devel-0.3.111-13.el9.x86_64 47/198 2024-03-19T01:30:21.503 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 48/198 2024-03-19T01:30:21.503 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 49/198 2024-03-19T01:30:21.503 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 50/198 2024-03-19T01:30:21.503 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : libtool-2.4.6-45.el9.x86_64 51/198 2024-03-19T01:30:21.503 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : libzstd-devel-1.5.1-2.el9.x86_64 52/198 2024-03-19T01:30:21.503 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 53/198 2024-03-19T01:30:21.504 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : lua-5.4.4-4.el9.x86_64 54/198 2024-03-19T01:30:21.504 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : oniguruma-6.9.6-1.el9.5.x86_64 55/198 2024-03-19T01:30:21.504 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : openblas-0.3.21-2.el9.x86_64 56/198 2024-03-19T01:30:21.504 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : openblas-openmp-0.3.21-2.el9.x86_64 57/198 2024-03-19T01:30:21.504 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : postgresql-13.11-1.el9.x86_64 58/198 2024-03-19T01:30:21.504 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : postgresql-contrib-13.11-1.el9.x86_64 59/198 2024-03-19T01:30:21.504 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : postgresql-private-libs-13.11-1.el9.x86_64 60/198 2024-03-19T01:30:21.504 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : postgresql-server-13.11-1.el9.x86_64 61/198 2024-03-19T01:30:21.504 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : protobuf-3.14.0-13.el9.x86_64 62/198 2024-03-19T01:30:21.504 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-attrs-20.3.0-7.el9.noarch 63/198 2024-03-19T01:30:21.504 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 64/198 2024-03-19T01:30:21.504 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 65/198 2024-03-19T01:30:21.504 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 66/198 2024-03-19T01:30:21.504 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 67/198 2024-03-19T01:30:21.505 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 68/198 2024-03-19T01:30:21.505 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 69/198 2024-03-19T01:30:21.505 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 70/198 2024-03-19T01:30:21.505 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 71/198 2024-03-19T01:30:21.505 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 72/198 2024-03-19T01:30:21.505 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 73/198 2024-03-19T01:30:21.505 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-protobuf-3.14.0-13.el9.noarch 74/198 2024-03-19T01:30:21.505 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 75/198 2024-03-19T01:30:21.505 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 76/198 2024-03-19T01:30:21.505 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 77/198 2024-03-19T01:30:21.505 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 78/198 2024-03-19T01:30:21.505 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 79/198 2024-03-19T01:30:21.506 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 80/198 2024-03-19T01:30:21.506 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-wcwidth-0.2.5-8.el9.noarch 81/198 2024-03-19T01:30:21.506 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : socat-1.7.4.1-5.el9.x86_64 82/198 2024-03-19T01:30:21.506 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : uuid-1.6.2-55.el9.x86_64 83/198 2024-03-19T01:30:21.506 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 84/198 2024-03-19T01:30:21.506 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : zlib-devel-1.2.11-40.el9.x86_64 85/198 2024-03-19T01:30:21.506 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : fuse-2.9.9-15.el9.x86_64 86/198 2024-03-19T01:30:21.506 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 87/198 2024-03-19T01:30:21.506 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 88/198 2024-03-19T01:30:21.506 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-ply-3.11-14.el9.noarch 89/198 2024-03-19T01:30:21.506 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 90/198 2024-03-19T01:30:21.506 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-urllib3-1.26.5-5.el9.noarch 91/198 2024-03-19T01:30:21.507 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : jq-1.6-16.el9.x86_64 92/198 2024-03-19T01:30:21.507 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : libgfortran-11.4.1-3.el9.x86_64 93/198 2024-03-19T01:30:21.507 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : libquadmath-11.4.1-3.el9.x86_64 94/198 2024-03-19T01:30:21.507 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 95/198 2024-03-19T01:30:21.507 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : elfutils-libelf-devel-0.190-2.el9.x86_64 96/198 2024-03-19T01:30:21.507 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : libuuid-devel-2.37.4-18.el9.x86_64 97/198 2024-03-19T01:30:21.507 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : openssl-devel-1:3.0.7-27.el9.x86_64 98/198 2024-03-19T01:30:21.507 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-devel-3.9.17-2.el9.x86_64 99/198 2024-03-19T01:30:21.507 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-jinja2-2.11.3-5.el9.noarch 100/198 2024-03-19T01:30:21.507 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : qatlib-23.11.0-1.el9.x86_64 101/198 2024-03-19T01:30:21.507 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : qatlib-service-23.11.0-1.el9.x86_64 102/198 2024-03-19T01:30:21.508 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : qatzip-libs-1.1.2-2.el9.x86_64 103/198 2024-03-19T01:30:21.508 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : xfsprogs-devel-6.3.0-1.el9.x86_64 104/198 2024-03-19T01:30:21.508 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : lua-devel-5.4.4-4.el9.x86_64 105/198 2024-03-19T01:30:21.508 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : protobuf-compiler-3.14.0-13.el9.x86_64 106/198 2024-03-19T01:30:21.508 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : abseil-cpp-20211102.0-3.el9.x86_64 107/198 2024-03-19T01:30:21.508 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : gperftools-libs-2.9.1-2.el9.x86_64 108/198 2024-03-19T01:30:21.508 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : grpc-data-1.46.7-10.el9.noarch 109/198 2024-03-19T01:30:21.508 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : libarrow-9.0.0-10.el9.x86_64 110/198 2024-03-19T01:30:21.508 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : libarrow-doc-9.0.0-10.el9.noarch 111/198 2024-03-19T01:30:21.508 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 112/198 2024-03-19T01:30:21.508 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 113/198 2024-03-19T01:30:21.508 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : luarocks-3.9.2-1.el9.noarch 114/198 2024-03-19T01:30:21.508 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : parquet-libs-9.0.0-10.el9.x86_64 115/198 2024-03-19T01:30:21.509 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-asyncssh-2.13.2-4.el9.noarch 116/198 2024-03-19T01:30:21.509 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 117/198 2024-03-19T01:30:21.509 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 118/198 2024-03-19T01:30:21.509 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-certifi-2021.10.8-2.el9.noarch 119/198 2024-03-19T01:30:21.509 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 120/198 2024-03-19T01:30:21.509 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 121/198 2024-03-19T01:30:21.509 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-cmd2-2.4.2-2.el9.noarch 122/198 2024-03-19T01:30:21.509 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-colorama-0.4.4-12.el9.noarch 123/198 2024-03-19T01:30:21.509 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-google-auth-1:2.28.0-1.el9.noarch 124/198 2024-03-19T01:30:21.509 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-grpcio-1.46.7-10.el9.x86_64 125/198 2024-03-19T01:30:21.510 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-grpcio-tools-1.46.7-10.el9.x86_64 126/198 2024-03-19T01:30:21.510 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 127/198 2024-03-19T01:30:21.510 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 128/198 2024-03-19T01:30:21.510 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 129/198 2024-03-19T01:30:21.510 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 130/198 2024-03-19T01:30:21.510 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 131/198 2024-03-19T01:30:21.510 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 132/198 2024-03-19T01:30:21.510 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 133/198 2024-03-19T01:30:21.510 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 134/198 2024-03-19T01:30:21.510 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 135/198 2024-03-19T01:30:21.510 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 136/198 2024-03-19T01:30:21.510 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 137/198 2024-03-19T01:30:21.510 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 138/198 2024-03-19T01:30:21.511 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-pyperclip-1.8.2-2.el9.noarch 139/198 2024-03-19T01:30:21.511 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 140/198 2024-03-19T01:30:21.511 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 141/198 2024-03-19T01:30:21.511 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 142/198 2024-03-19T01:30:21.511 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 143/198 2024-03-19T01:30:21.511 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-typing-extensions-4.4.0-2.el9.noarch 144/198 2024-03-19T01:30:21.511 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-webob-1.8.7-6.el9.noarch 145/198 2024-03-19T01:30:21.511 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 146/198 2024-03-19T01:30:21.511 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 147/198 2024-03-19T01:30:21.511 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 148/198 2024-03-19T01:30:21.511 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : re2-1:20211101-3.el9.x86_64 149/198 2024-03-19T01:30:21.511 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 150/198 2024-03-19T01:30:21.512 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : librados2-2:19.0.0-2206.gb6ae2491.el9.x86_64 151/198 2024-03-19T01:30:21.512 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 152/198 2024-03-19T01:30:21.512 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : librbd1-2:19.0.0-2206.gb6ae2491.el9.x86_64 153/198 2024-03-19T01:30:21.512 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 154/198 2024-03-19T01:30:21.512 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : xfsdump-3.1.12-4.el9.x86_64 155/198 2024-03-19T01:30:21.512 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : xfsdump-3.1.12-3.el9.x86_64 156/198 2024-03-19T01:30:21.512 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : NetworkManager-1:1.47.2-1.el9.x86_64 157/198 2024-03-19T01:30:21.512 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : NetworkManager-1:1.44.0-2.el9.x86_64 158/198 2024-03-19T01:30:21.512 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : NetworkManager-libnm-1:1.47.2-1.el9.x86_64 159/198 2024-03-19T01:30:21.512 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : NetworkManager-libnm-1:1.44.0-2.el9.x86_64 160/198 2024-03-19T01:30:21.512 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : NetworkManager-team-1:1.47.2-1.el9.x86_64 161/198 2024-03-19T01:30:21.512 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : NetworkManager-team-1:1.44.0-2.el9.x86_64 162/198 2024-03-19T01:30:21.512 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : NetworkManager-tui-1:1.47.2-1.el9.x86_64 163/198 2024-03-19T01:30:21.512 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : NetworkManager-tui-1:1.44.0-2.el9.x86_64 164/198 2024-03-19T01:30:21.513 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : acl-2.3.1-4.el9.x86_64 165/198 2024-03-19T01:30:21.513 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : acl-2.3.1-3.el9.x86_64 166/198 2024-03-19T01:30:21.513 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : elfutils-debuginfod-client-0.190-2.el9.x86_64 167/198 2024-03-19T01:30:21.513 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : elfutils-debuginfod-client-0.189-3.el9.x86_64 168/198 2024-03-19T01:30:21.513 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : elfutils-libelf-0.190-2.el9.x86_64 169/198 2024-03-19T01:30:21.513 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : elfutils-libelf-0.189-3.el9.x86_64 170/198 2024-03-19T01:30:21.513 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : elfutils-libs-0.190-2.el9.x86_64 171/198 2024-03-19T01:30:21.513 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : elfutils-libs-0.189-3.el9.x86_64 172/198 2024-03-19T01:30:21.513 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : libacl-2.3.1-4.el9.x86_64 173/198 2024-03-19T01:30:21.513 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : libacl-2.3.1-3.el9.x86_64 174/198 2024-03-19T01:30:21.513 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : libblkid-2.37.4-18.el9.x86_64 175/198 2024-03-19T01:30:21.513 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : libblkid-2.37.4-15.el9.x86_64 176/198 2024-03-19T01:30:21.513 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : libfdisk-2.37.4-18.el9.x86_64 177/198 2024-03-19T01:30:21.513 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : libfdisk-2.37.4-15.el9.x86_64 178/198 2024-03-19T01:30:21.513 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : libmount-2.37.4-18.el9.x86_64 179/198 2024-03-19T01:30:21.514 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : libmount-2.37.4-15.el9.x86_64 180/198 2024-03-19T01:30:21.514 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : libsmartcols-2.37.4-18.el9.x86_64 181/198 2024-03-19T01:30:21.514 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : libsmartcols-2.37.4-15.el9.x86_64 182/198 2024-03-19T01:30:21.514 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : libuuid-2.37.4-18.el9.x86_64 183/198 2024-03-19T01:30:21.514 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : libuuid-2.37.4-15.el9.x86_64 184/198 2024-03-19T01:30:21.514 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : openssl-1:3.0.7-27.el9.x86_64 185/198 2024-03-19T01:30:21.514 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : openssl-1:3.0.7-24.el9.x86_64 186/198 2024-03-19T01:30:21.514 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : openssl-libs-1:3.0.7-27.el9.x86_64 187/198 2024-03-19T01:30:21.514 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : openssl-libs-1:3.0.7-24.el9.x86_64 188/198 2024-03-19T01:30:21.514 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : util-linux-2.37.4-18.el9.x86_64 189/198 2024-03-19T01:30:21.514 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : util-linux-2.37.4-15.el9.x86_64 190/198 2024-03-19T01:30:21.514 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : util-linux-core-2.37.4-18.el9.x86_64 191/198 2024-03-19T01:30:21.514 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : util-linux-core-2.37.4-15.el9.x86_64 192/198 2024-03-19T01:30:21.515 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : util-linux-user-2.37.4-18.el9.x86_64 193/198 2024-03-19T01:30:21.515 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : util-linux-user-2.37.4-15.el9.x86_64 194/198 2024-03-19T01:30:21.515 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : xfsprogs-6.3.0-1.el9.x86_64 195/198 2024-03-19T01:30:21.515 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : xfsprogs-5.19.0-4.el9.x86_64 196/198 2024-03-19T01:30:21.515 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : libacl-devel-2.3.1-4.el9.x86_64 197/198 2024-03-19T01:30:24.539 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : libacl-devel-2.3.1-3.el9.x86_64 198/198 2024-03-19T01:30:24.539 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:30:24.540 INFO:teuthology.orchestra.run.smithi099.stdout:Upgraded: 2024-03-19T01:30:24.540 INFO:teuthology.orchestra.run.smithi099.stdout: NetworkManager-1:1.47.2-1.el9.x86_64 2024-03-19T01:30:24.540 INFO:teuthology.orchestra.run.smithi099.stdout: NetworkManager-libnm-1:1.47.2-1.el9.x86_64 2024-03-19T01:30:24.540 INFO:teuthology.orchestra.run.smithi099.stdout: NetworkManager-team-1:1.47.2-1.el9.x86_64 2024-03-19T01:30:24.540 INFO:teuthology.orchestra.run.smithi099.stdout: NetworkManager-tui-1:1.47.2-1.el9.x86_64 2024-03-19T01:30:24.540 INFO:teuthology.orchestra.run.smithi099.stdout: acl-2.3.1-4.el9.x86_64 2024-03-19T01:30:24.540 INFO:teuthology.orchestra.run.smithi099.stdout: elfutils-debuginfod-client-0.190-2.el9.x86_64 2024-03-19T01:30:24.540 INFO:teuthology.orchestra.run.smithi099.stdout: elfutils-libelf-0.190-2.el9.x86_64 2024-03-19T01:30:24.540 INFO:teuthology.orchestra.run.smithi099.stdout: elfutils-libs-0.190-2.el9.x86_64 2024-03-19T01:30:24.540 INFO:teuthology.orchestra.run.smithi099.stdout: libacl-2.3.1-4.el9.x86_64 2024-03-19T01:30:24.540 INFO:teuthology.orchestra.run.smithi099.stdout: libacl-devel-2.3.1-4.el9.x86_64 2024-03-19T01:30:24.540 INFO:teuthology.orchestra.run.smithi099.stdout: libblkid-2.37.4-18.el9.x86_64 2024-03-19T01:30:24.541 INFO:teuthology.orchestra.run.smithi099.stdout: libfdisk-2.37.4-18.el9.x86_64 2024-03-19T01:30:24.541 INFO:teuthology.orchestra.run.smithi099.stdout: libmount-2.37.4-18.el9.x86_64 2024-03-19T01:30:24.541 INFO:teuthology.orchestra.run.smithi099.stdout: librados2-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:30:24.541 INFO:teuthology.orchestra.run.smithi099.stdout: librbd1-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:30:24.541 INFO:teuthology.orchestra.run.smithi099.stdout: libsmartcols-2.37.4-18.el9.x86_64 2024-03-19T01:30:24.541 INFO:teuthology.orchestra.run.smithi099.stdout: libuuid-2.37.4-18.el9.x86_64 2024-03-19T01:30:24.541 INFO:teuthology.orchestra.run.smithi099.stdout: openssl-1:3.0.7-27.el9.x86_64 2024-03-19T01:30:24.541 INFO:teuthology.orchestra.run.smithi099.stdout: openssl-libs-1:3.0.7-27.el9.x86_64 2024-03-19T01:30:24.541 INFO:teuthology.orchestra.run.smithi099.stdout: util-linux-2.37.4-18.el9.x86_64 2024-03-19T01:30:24.541 INFO:teuthology.orchestra.run.smithi099.stdout: util-linux-core-2.37.4-18.el9.x86_64 2024-03-19T01:30:24.542 INFO:teuthology.orchestra.run.smithi099.stdout: util-linux-user-2.37.4-18.el9.x86_64 2024-03-19T01:30:24.542 INFO:teuthology.orchestra.run.smithi099.stdout: xfsdump-3.1.12-4.el9.x86_64 2024-03-19T01:30:24.542 INFO:teuthology.orchestra.run.smithi099.stdout: xfsprogs-6.3.0-1.el9.x86_64 2024-03-19T01:30:24.542 INFO:teuthology.orchestra.run.smithi099.stdout:Installed: 2024-03-19T01:30:24.542 INFO:teuthology.orchestra.run.smithi099.stdout: abseil-cpp-20211102.0-3.el9.x86_64 2024-03-19T01:30:24.542 INFO:teuthology.orchestra.run.smithi099.stdout: automake-1.16.2-8.el9.noarch 2024-03-19T01:30:24.542 INFO:teuthology.orchestra.run.smithi099.stdout: bison-3.7.4-5.el9.x86_64 2024-03-19T01:30:24.542 INFO:teuthology.orchestra.run.smithi099.stdout: boost-program-options-1.75.0-8.el9.x86_64 2024-03-19T01:30:24.542 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:30:24.542 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-base-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:30:24.542 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-common-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:30:24.542 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-fuse-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:30:24.543 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-grafana-dashboards-2:19.0.0-2206.gb6ae2491.el9.noarch 2024-03-19T01:30:24.543 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-immutable-object-cache-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:30:24.543 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-mds-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:30:24.543 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-mgr-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:30:24.543 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-mgr-cephadm-2:19.0.0-2206.gb6ae2491.el9.noarch 2024-03-19T01:30:24.543 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-mgr-dashboard-2:19.0.0-2206.gb6ae2491.el9.noarch 2024-03-19T01:30:24.543 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-mgr-diskprediction-local-2:19.0.0-2206.gb6ae2491.el9.noarch 2024-03-19T01:30:24.543 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-mgr-modules-core-2:19.0.0-2206.gb6ae2491.el9.noarch 2024-03-19T01:30:24.543 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-mgr-rook-2:19.0.0-2206.gb6ae2491.el9.noarch 2024-03-19T01:30:24.543 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-mon-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:30:24.543 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-osd-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:30:24.543 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-prometheus-alerts-2:19.0.0-2206.gb6ae2491.el9.noarch 2024-03-19T01:30:24.543 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-radosgw-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:30:24.543 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-selinux-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:30:24.544 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-test-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:30:24.544 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-volume-2:19.0.0-2206.gb6ae2491.el9.noarch 2024-03-19T01:30:24.544 INFO:teuthology.orchestra.run.smithi099.stdout: cephadm-2:19.0.0-2206.gb6ae2491.el9.noarch 2024-03-19T01:30:24.544 INFO:teuthology.orchestra.run.smithi099.stdout: cephfs-mirror-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:30:24.544 INFO:teuthology.orchestra.run.smithi099.stdout: cephfs-shell-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:30:24.544 INFO:teuthology.orchestra.run.smithi099.stdout: cephfs-top-2:19.0.0-2206.gb6ae2491.el9.noarch 2024-03-19T01:30:24.544 INFO:teuthology.orchestra.run.smithi099.stdout: elfutils-libelf-devel-0.190-2.el9.x86_64 2024-03-19T01:30:24.544 INFO:teuthology.orchestra.run.smithi099.stdout: flex-2.6.4-9.el9.x86_64 2024-03-19T01:30:24.544 INFO:teuthology.orchestra.run.smithi099.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-03-19T01:30:24.544 INFO:teuthology.orchestra.run.smithi099.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-03-19T01:30:24.544 INFO:teuthology.orchestra.run.smithi099.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-03-19T01:30:24.544 INFO:teuthology.orchestra.run.smithi099.stdout: fuse-2.9.9-15.el9.x86_64 2024-03-19T01:30:24.544 INFO:teuthology.orchestra.run.smithi099.stdout: gperftools-libs-2.9.1-2.el9.x86_64 2024-03-19T01:30:24.544 INFO:teuthology.orchestra.run.smithi099.stdout: grpc-data-1.46.7-10.el9.noarch 2024-03-19T01:30:24.545 INFO:teuthology.orchestra.run.smithi099.stdout: jq-1.6-16.el9.x86_64 2024-03-19T01:30:24.545 INFO:teuthology.orchestra.run.smithi099.stdout: libaio-devel-0.3.111-13.el9.x86_64 2024-03-19T01:30:24.545 INFO:teuthology.orchestra.run.smithi099.stdout: libarrow-9.0.0-10.el9.x86_64 2024-03-19T01:30:24.545 INFO:teuthology.orchestra.run.smithi099.stdout: libarrow-doc-9.0.0-10.el9.noarch 2024-03-19T01:30:24.545 INFO:teuthology.orchestra.run.smithi099.stdout: libcephfs-devel-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:30:24.545 INFO:teuthology.orchestra.run.smithi099.stdout: libcephfs2-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:30:24.545 INFO:teuthology.orchestra.run.smithi099.stdout: libcephsqlite-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:30:24.545 INFO:teuthology.orchestra.run.smithi099.stdout: libgfortran-11.4.1-3.el9.x86_64 2024-03-19T01:30:24.545 INFO:teuthology.orchestra.run.smithi099.stdout: liboath-2.6.7-2.el9.x86_64 2024-03-19T01:30:24.545 INFO:teuthology.orchestra.run.smithi099.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-03-19T01:30:24.545 INFO:teuthology.orchestra.run.smithi099.stdout: libquadmath-11.4.1-3.el9.x86_64 2024-03-19T01:30:24.545 INFO:teuthology.orchestra.run.smithi099.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-03-19T01:30:24.545 INFO:teuthology.orchestra.run.smithi099.stdout: librados-devel-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:30:24.546 INFO:teuthology.orchestra.run.smithi099.stdout: libradosstriper1-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:30:24.546 INFO:teuthology.orchestra.run.smithi099.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-03-19T01:30:24.546 INFO:teuthology.orchestra.run.smithi099.stdout: librgw2-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:30:24.546 INFO:teuthology.orchestra.run.smithi099.stdout: libtool-2.4.6-45.el9.x86_64 2024-03-19T01:30:24.546 INFO:teuthology.orchestra.run.smithi099.stdout: libunwind-1.6.2-1.el9.x86_64 2024-03-19T01:30:24.546 INFO:teuthology.orchestra.run.smithi099.stdout: libuuid-devel-2.37.4-18.el9.x86_64 2024-03-19T01:30:24.546 INFO:teuthology.orchestra.run.smithi099.stdout: libzstd-devel-1.5.1-2.el9.x86_64 2024-03-19T01:30:24.546 INFO:teuthology.orchestra.run.smithi099.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-03-19T01:30:24.546 INFO:teuthology.orchestra.run.smithi099.stdout: lua-5.4.4-4.el9.x86_64 2024-03-19T01:30:24.546 INFO:teuthology.orchestra.run.smithi099.stdout: lua-devel-5.4.4-4.el9.x86_64 2024-03-19T01:30:24.546 INFO:teuthology.orchestra.run.smithi099.stdout: luarocks-3.9.2-1.el9.noarch 2024-03-19T01:30:24.546 INFO:teuthology.orchestra.run.smithi099.stdout: oniguruma-6.9.6-1.el9.5.x86_64 2024-03-19T01:30:24.546 INFO:teuthology.orchestra.run.smithi099.stdout: openblas-0.3.21-2.el9.x86_64 2024-03-19T01:30:24.546 INFO:teuthology.orchestra.run.smithi099.stdout: openblas-openmp-0.3.21-2.el9.x86_64 2024-03-19T01:30:24.547 INFO:teuthology.orchestra.run.smithi099.stdout: openssl-devel-1:3.0.7-27.el9.x86_64 2024-03-19T01:30:24.547 INFO:teuthology.orchestra.run.smithi099.stdout: parquet-libs-9.0.0-10.el9.x86_64 2024-03-19T01:30:24.547 INFO:teuthology.orchestra.run.smithi099.stdout: postgresql-13.11-1.el9.x86_64 2024-03-19T01:30:24.547 INFO:teuthology.orchestra.run.smithi099.stdout: postgresql-contrib-13.11-1.el9.x86_64 2024-03-19T01:30:24.547 INFO:teuthology.orchestra.run.smithi099.stdout: postgresql-private-libs-13.11-1.el9.x86_64 2024-03-19T01:30:24.547 INFO:teuthology.orchestra.run.smithi099.stdout: postgresql-server-13.11-1.el9.x86_64 2024-03-19T01:30:24.547 INFO:teuthology.orchestra.run.smithi099.stdout: protobuf-3.14.0-13.el9.x86_64 2024-03-19T01:30:24.547 INFO:teuthology.orchestra.run.smithi099.stdout: protobuf-compiler-3.14.0-13.el9.x86_64 2024-03-19T01:30:24.547 INFO:teuthology.orchestra.run.smithi099.stdout: python3-asyncssh-2.13.2-4.el9.noarch 2024-03-19T01:30:24.547 INFO:teuthology.orchestra.run.smithi099.stdout: python3-attrs-20.3.0-7.el9.noarch 2024-03-19T01:30:24.547 INFO:teuthology.orchestra.run.smithi099.stdout: python3-babel-2.9.1-2.el9.noarch 2024-03-19T01:30:24.547 INFO:teuthology.orchestra.run.smithi099.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-03-19T01:30:24.547 INFO:teuthology.orchestra.run.smithi099.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-03-19T01:30:24.548 INFO:teuthology.orchestra.run.smithi099.stdout: python3-ceph-argparse-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:30:24.548 INFO:teuthology.orchestra.run.smithi099.stdout: python3-ceph-common-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:30:24.548 INFO:teuthology.orchestra.run.smithi099.stdout: python3-cephfs-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:30:24.548 INFO:teuthology.orchestra.run.smithi099.stdout: python3-certifi-2021.10.8-2.el9.noarch 2024-03-19T01:30:24.548 INFO:teuthology.orchestra.run.smithi099.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-03-19T01:30:24.548 INFO:teuthology.orchestra.run.smithi099.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-03-19T01:30:24.548 INFO:teuthology.orchestra.run.smithi099.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-03-19T01:30:24.548 INFO:teuthology.orchestra.run.smithi099.stdout: python3-cmd2-2.4.2-2.el9.noarch 2024-03-19T01:30:24.548 INFO:teuthology.orchestra.run.smithi099.stdout: python3-colorama-0.4.4-12.el9.noarch 2024-03-19T01:30:24.548 INFO:teuthology.orchestra.run.smithi099.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-03-19T01:30:24.548 INFO:teuthology.orchestra.run.smithi099.stdout: python3-devel-3.9.17-2.el9.x86_64 2024-03-19T01:30:24.548 INFO:teuthology.orchestra.run.smithi099.stdout: python3-google-auth-1:2.28.0-1.el9.noarch 2024-03-19T01:30:24.548 INFO:teuthology.orchestra.run.smithi099.stdout: python3-grpcio-1.46.7-10.el9.x86_64 2024-03-19T01:30:24.548 INFO:teuthology.orchestra.run.smithi099.stdout: python3-grpcio-tools-1.46.7-10.el9.x86_64 2024-03-19T01:30:24.549 INFO:teuthology.orchestra.run.smithi099.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-03-19T01:30:24.549 INFO:teuthology.orchestra.run.smithi099.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-03-19T01:30:24.549 INFO:teuthology.orchestra.run.smithi099.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-03-19T01:30:24.549 INFO:teuthology.orchestra.run.smithi099.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-03-19T01:30:24.549 INFO:teuthology.orchestra.run.smithi099.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-03-19T01:30:24.549 INFO:teuthology.orchestra.run.smithi099.stdout: python3-jinja2-2.11.3-5.el9.noarch 2024-03-19T01:30:24.549 INFO:teuthology.orchestra.run.smithi099.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-03-19T01:30:24.549 INFO:teuthology.orchestra.run.smithi099.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-03-19T01:30:24.549 INFO:teuthology.orchestra.run.smithi099.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-03-19T01:30:24.549 INFO:teuthology.orchestra.run.smithi099.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-03-19T01:30:24.549 INFO:teuthology.orchestra.run.smithi099.stdout: python3-mako-1.1.4-6.el9.noarch 2024-03-19T01:30:24.549 INFO:teuthology.orchestra.run.smithi099.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-03-19T01:30:24.549 INFO:teuthology.orchestra.run.smithi099.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-03-19T01:30:24.550 INFO:teuthology.orchestra.run.smithi099.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-03-19T01:30:24.550 INFO:teuthology.orchestra.run.smithi099.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-03-19T01:30:24.550 INFO:teuthology.orchestra.run.smithi099.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-03-19T01:30:24.550 INFO:teuthology.orchestra.run.smithi099.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-03-19T01:30:24.550 INFO:teuthology.orchestra.run.smithi099.stdout: python3-packaging-20.9-5.el9.noarch 2024-03-19T01:30:24.550 INFO:teuthology.orchestra.run.smithi099.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-03-19T01:30:24.550 INFO:teuthology.orchestra.run.smithi099.stdout: python3-ply-3.11-14.el9.noarch 2024-03-19T01:30:24.550 INFO:teuthology.orchestra.run.smithi099.stdout: python3-portend-3.1.0-2.el9.noarch 2024-03-19T01:30:24.550 INFO:teuthology.orchestra.run.smithi099.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-03-19T01:30:24.550 INFO:teuthology.orchestra.run.smithi099.stdout: python3-protobuf-3.14.0-13.el9.noarch 2024-03-19T01:30:24.550 INFO:teuthology.orchestra.run.smithi099.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-03-19T01:30:24.550 INFO:teuthology.orchestra.run.smithi099.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-03-19T01:30:24.550 INFO:teuthology.orchestra.run.smithi099.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-03-19T01:30:24.550 INFO:teuthology.orchestra.run.smithi099.stdout: python3-pycparser-2.20-6.el9.noarch 2024-03-19T01:30:24.551 INFO:teuthology.orchestra.run.smithi099.stdout: python3-pyperclip-1.8.2-2.el9.noarch 2024-03-19T01:30:24.551 INFO:teuthology.orchestra.run.smithi099.stdout: python3-pytz-2021.1-5.el9.noarch 2024-03-19T01:30:24.551 INFO:teuthology.orchestra.run.smithi099.stdout: python3-rados-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:30:24.551 INFO:teuthology.orchestra.run.smithi099.stdout: python3-rbd-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:30:24.551 INFO:teuthology.orchestra.run.smithi099.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-03-19T01:30:24.551 INFO:teuthology.orchestra.run.smithi099.stdout: python3-requests-2.25.1-8.el9.noarch 2024-03-19T01:30:24.551 INFO:teuthology.orchestra.run.smithi099.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-03-19T01:30:24.551 INFO:teuthology.orchestra.run.smithi099.stdout: python3-rgw-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:30:24.551 INFO:teuthology.orchestra.run.smithi099.stdout: python3-routes-2.5.1-5.el9.noarch 2024-03-19T01:30:24.552 INFO:teuthology.orchestra.run.smithi099.stdout: python3-rsa-4.9-2.el9.noarch 2024-03-19T01:30:24.552 INFO:teuthology.orchestra.run.smithi099.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-03-19T01:30:24.552 INFO:teuthology.orchestra.run.smithi099.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-03-19T01:30:24.552 INFO:teuthology.orchestra.run.smithi099.stdout: python3-toml-0.10.2-6.el9.noarch 2024-03-19T01:30:24.552 INFO:teuthology.orchestra.run.smithi099.stdout: python3-typing-extensions-4.4.0-2.el9.noarch 2024-03-19T01:30:24.552 INFO:teuthology.orchestra.run.smithi099.stdout: python3-urllib3-1.26.5-5.el9.noarch 2024-03-19T01:30:24.552 INFO:teuthology.orchestra.run.smithi099.stdout: python3-wcwidth-0.2.5-8.el9.noarch 2024-03-19T01:30:24.552 INFO:teuthology.orchestra.run.smithi099.stdout: python3-webob-1.8.7-6.el9.noarch 2024-03-19T01:30:24.552 INFO:teuthology.orchestra.run.smithi099.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-03-19T01:30:24.552 INFO:teuthology.orchestra.run.smithi099.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-03-19T01:30:24.553 INFO:teuthology.orchestra.run.smithi099.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-03-19T01:30:24.553 INFO:teuthology.orchestra.run.smithi099.stdout: qatlib-23.11.0-1.el9.x86_64 2024-03-19T01:30:24.553 INFO:teuthology.orchestra.run.smithi099.stdout: qatlib-service-23.11.0-1.el9.x86_64 2024-03-19T01:30:24.553 INFO:teuthology.orchestra.run.smithi099.stdout: qatzip-libs-1.1.2-2.el9.x86_64 2024-03-19T01:30:24.553 INFO:teuthology.orchestra.run.smithi099.stdout: rbd-fuse-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:30:24.553 INFO:teuthology.orchestra.run.smithi099.stdout: rbd-mirror-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:30:24.553 INFO:teuthology.orchestra.run.smithi099.stdout: rbd-nbd-2:19.0.0-2206.gb6ae2491.el9.x86_64 2024-03-19T01:30:24.553 INFO:teuthology.orchestra.run.smithi099.stdout: re2-1:20211101-3.el9.x86_64 2024-03-19T01:30:24.553 INFO:teuthology.orchestra.run.smithi099.stdout: socat-1.7.4.1-5.el9.x86_64 2024-03-19T01:30:24.553 INFO:teuthology.orchestra.run.smithi099.stdout: thrift-0.15.0-2.el9.x86_64 2024-03-19T01:30:24.553 INFO:teuthology.orchestra.run.smithi099.stdout: uuid-1.6.2-55.el9.x86_64 2024-03-19T01:30:24.553 INFO:teuthology.orchestra.run.smithi099.stdout: xfsprogs-devel-6.3.0-1.el9.x86_64 2024-03-19T01:30:24.553 INFO:teuthology.orchestra.run.smithi099.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-03-19T01:30:24.554 INFO:teuthology.orchestra.run.smithi099.stdout: zlib-devel-1.2.11-40.el9.x86_64 2024-03-19T01:30:24.554 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:30:24.555 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2024-03-19T01:30:25.048 DEBUG:teuthology.parallel:result is None 2024-03-19T01:30:25.049 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=b6ae24918b03b1b8a19c9263a50655bba80bb810 2024-03-19T01:30:25.190 DEBUG:teuthology.orchestra.run.smithi099:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-03-19T01:30:25.211 INFO:teuthology.orchestra.run.smithi099.stdout:19.0.0-2206.gb6ae2491.el9 2024-03-19T01:30:25.212 INFO:teuthology.packaging:The installed version of ceph is 19.0.0-2206.gb6ae2491.el9 2024-03-19T01:30:25.212 INFO:teuthology.task.install:The correct ceph version 19.0.0-2206.gb6ae2491 is installed. 2024-03-19T01:30:25.214 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=b6ae24918b03b1b8a19c9263a50655bba80bb810 2024-03-19T01:30:25.350 DEBUG:teuthology.orchestra.run.smithi112:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-03-19T01:30:25.376 INFO:teuthology.orchestra.run.smithi112.stdout:19.0.0-2206.gb6ae2491.el9 2024-03-19T01:30:25.376 INFO:teuthology.packaging:The installed version of ceph is 19.0.0-2206.gb6ae2491.el9 2024-03-19T01:30:25.377 INFO:teuthology.task.install:The correct ceph version 19.0.0-2206.gb6ae2491 is installed. 2024-03-19T01:30:25.379 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=b6ae24918b03b1b8a19c9263a50655bba80bb810 2024-03-19T01:30:25.516 DEBUG:teuthology.orchestra.run.smithi115:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-03-19T01:30:25.544 INFO:teuthology.orchestra.run.smithi115.stdout:19.0.0-2206.gb6ae2491.el9 2024-03-19T01:30:25.544 INFO:teuthology.packaging:The installed version of ceph is 19.0.0-2206.gb6ae2491.el9 2024-03-19T01:30:25.544 INFO:teuthology.task.install:The correct ceph version 19.0.0-2206.gb6ae2491 is installed. 2024-03-19T01:30:25.546 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2024-03-19T01:30:25.546 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2024-03-19T01:30:25.546 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-03-19T01:30:25.578 DEBUG:teuthology.orchestra.run.smithi112:> set -ex 2024-03-19T01:30:25.578 DEBUG:teuthology.orchestra.run.smithi112:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-03-19T01:30:25.613 DEBUG:teuthology.orchestra.run.smithi115:> set -ex 2024-03-19T01:30:25.613 DEBUG:teuthology.orchestra.run.smithi115:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-03-19T01:30:25.648 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2024-03-19T01:30:25.648 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2024-03-19T01:30:25.648 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd of=/usr/bin/daemon-helper 2024-03-19T01:30:25.676 DEBUG:teuthology.orchestra.run.smithi099:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-03-19T01:30:25.744 DEBUG:teuthology.orchestra.run.smithi112:> set -ex 2024-03-19T01:30:25.744 DEBUG:teuthology.orchestra.run.smithi112:> sudo dd of=/usr/bin/daemon-helper 2024-03-19T01:30:25.775 DEBUG:teuthology.orchestra.run.smithi112:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-03-19T01:30:25.845 DEBUG:teuthology.orchestra.run.smithi115:> set -ex 2024-03-19T01:30:25.845 DEBUG:teuthology.orchestra.run.smithi115:> sudo dd of=/usr/bin/daemon-helper 2024-03-19T01:30:25.876 DEBUG:teuthology.orchestra.run.smithi115:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-03-19T01:30:25.948 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2024-03-19T01:30:25.948 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2024-03-19T01:30:25.948 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd of=/usr/bin/adjust-ulimits 2024-03-19T01:30:25.975 DEBUG:teuthology.orchestra.run.smithi099:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-03-19T01:30:26.042 DEBUG:teuthology.orchestra.run.smithi112:> set -ex 2024-03-19T01:30:26.043 DEBUG:teuthology.orchestra.run.smithi112:> sudo dd of=/usr/bin/adjust-ulimits 2024-03-19T01:30:26.071 DEBUG:teuthology.orchestra.run.smithi112:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-03-19T01:30:26.140 DEBUG:teuthology.orchestra.run.smithi115:> set -ex 2024-03-19T01:30:26.141 DEBUG:teuthology.orchestra.run.smithi115:> sudo dd of=/usr/bin/adjust-ulimits 2024-03-19T01:30:26.172 DEBUG:teuthology.orchestra.run.smithi115:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-03-19T01:30:26.242 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2024-03-19T01:30:26.242 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2024-03-19T01:30:26.242 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd of=/usr/bin/stdin-killer 2024-03-19T01:30:26.270 DEBUG:teuthology.orchestra.run.smithi099:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-03-19T01:30:26.336 DEBUG:teuthology.orchestra.run.smithi112:> set -ex 2024-03-19T01:30:26.337 DEBUG:teuthology.orchestra.run.smithi112:> sudo dd of=/usr/bin/stdin-killer 2024-03-19T01:30:26.371 DEBUG:teuthology.orchestra.run.smithi112:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-03-19T01:30:26.440 DEBUG:teuthology.orchestra.run.smithi115:> set -ex 2024-03-19T01:30:26.441 DEBUG:teuthology.orchestra.run.smithi115:> sudo dd of=/usr/bin/stdin-killer 2024-03-19T01:30:26.472 DEBUG:teuthology.orchestra.run.smithi115:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-03-19T01:30:26.543 INFO:teuthology.run_tasks:Running task cephadm... 2024-03-19T01:30:26.659 INFO:tasks.cephadm:Config: {'roleless': False, 'cephfs': {'max_mds': 3, 'session_timeout': 300, 'standby_replay': True}, 'conf': {'client': {'client mount timeout': 600, 'debug client': 20, 'debug ms': 1, 'rados mon op timeout': 900, 'rados osd op timeout': 900}, 'global': {'ms die on skipped message': False}, 'mds': {'debug mds': 20, 'debug mds balancer': 20, 'debug ms': 1, 'mds bal fragment size max': 10000, 'mds bal merge size': 5, 'mds bal split bits': 3, 'mds bal split size': 100, 'mds debug frag': True, 'mds debug scatterstat': True, 'mds op complaint time': 180, 'mds verify scatter': True, 'mds_bal_replicate_threshold': 1, 'mds_export_ephemeral_random_max': 0.1, 'osd op complaint time': 180, 'rados mon op timeout': 900, 'rados osd op timeout': 900}, 'mgr': {'cephsqlite lock renewal timeout': 900000, 'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20, 'mon op complaint time': 120}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'bitmap', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore', 'osd op complaint time': 180, 'osd shutdown pgref assert': True, 'osd_max_omap_entries_per_request': 10}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'FS_DEGRADED', 'FS_INLINE_DATA_DEPRECATED', 'FS_WITH_FAILED_MDS', 'MDS_ALL_DOWN', 'MDS_DAMAGE', 'MDS_DEGRADED', 'MDS_FAILED', 'MDS_INSUFFICIENT_STANDBY', 'MDS_UP_LESS_THAN_MAX', 'POOL_APP_NOT_ENABLED', 'overall HEALTH_', 'Replacing daemon', 'deprecated feature inline_data', 'PG_AVAILABILITY', 'PG_DEGRADED', 'overall HEALTH_', '\\(OSD_DOWN\\)', '\\(OSD_', 'but it is still running', 'is not responding'], 'sha1': 'b6ae24918b03b1b8a19c9263a50655bba80bb810', 'subvols': {'create': 2}} 2024-03-19T01:30:26.660 INFO:tasks.cephadm:Cluster image is quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 2024-03-19T01:30:26.660 INFO:tasks.cephadm:Cluster fsid is 438fb69c-e590-11ee-95c9-87774f69a715 2024-03-19T01:30:26.661 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-03-19T01:30:26.661 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.99', 'mon.b': '172.21.15.112', 'mon.c': '172.21.15.115'} 2024-03-19T01:30:26.661 INFO:tasks.cephadm:First mon is mon.a on smithi099 2024-03-19T01:30:26.661 INFO:tasks.cephadm:First mgr is x 2024-03-19T01:30:26.662 INFO:tasks.cephadm:Normalizing hostnames... 2024-03-19T01:30:26.662 DEBUG:teuthology.orchestra.run.smithi099:> sudo hostname $(hostname -s) 2024-03-19T01:30:26.690 DEBUG:teuthology.orchestra.run.smithi112:> sudo hostname $(hostname -s) 2024-03-19T01:30:26.721 DEBUG:teuthology.orchestra.run.smithi115:> sudo hostname $(hostname -s) 2024-03-19T01:30:26.752 INFO:tasks.cephadm:Downloading "compiled" cephadm from cachra 2024-03-19T01:30:26.753 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=b6ae24918b03b1b8a19c9263a50655bba80bb810 2024-03-19T01:30:26.984 INFO:tasks.cephadm:builder_project result: [{'status': 'ready', 'sha1': 'b6ae24918b03b1b8a19c9263a50655bba80bb810', 'extra': {'build_url': 'https://jenkins.ceph.com/job/ceph-dev-new-build/ARCH=x86_64,AVAILABLE_ARCH=x86_64,AVAILABLE_DIST=centos9,DIST=centos9,MACHINE_SIZE=gigantic/77736/', 'root_build_cause': 'SCMTRIGGER', 'version': '19.0.0-2206-gb6ae2491', 'node_name': '172.21.2.14+braggi14', 'job_name': 'ceph-dev-new-build/ARCH=x86_64,AVAILABLE_ARCH=x86_64,AVAILABLE_DIST=centos9,DIST=centos9,MACHINE_SIZE=gigantic', 'package_manager_version': '19.0.0-2206.gb6ae2491'}, 'url': 'https://4.chacra.ceph.com/r/ceph/wip-batrick-testing-20240318.181317/b6ae24918b03b1b8a19c9263a50655bba80bb810/centos/9/flavors/default/', 'distro_codename': None, 'modified': '2024-03-18 20:42:27.872142', 'distro_version': '9', 'project': 'ceph', 'flavor': 'default', 'ref': 'wip-batrick-testing-20240318.181317', 'chacra_url': 'https://4.chacra.ceph.com/repos/ceph/wip-batrick-testing-20240318.181317/b6ae24918b03b1b8a19c9263a50655bba80bb810/centos/9/flavors/default/', 'archs': ['source', 'x86_64'], 'distro': 'centos'}] 2024-03-19T01:30:27.126 INFO:tasks.util.chacra:got chacra host 4.chacra.ceph.com, ref wip-batrick-testing-20240318.181317, sha1 b6ae24918b03b1b8a19c9263a50655bba80bb810 from https://shaman.ceph.com/api/search/?project=ceph&distros=centos%2F9%2Fx86_64&flavor=default&sha1=b6ae24918b03b1b8a19c9263a50655bba80bb810 2024-03-19T01:30:27.128 INFO:tasks.cephadm:Discovered cachra url: https://4.chacra.ceph.com/binaries/ceph/wip-batrick-testing-20240318.181317/b6ae24918b03b1b8a19c9263a50655bba80bb810/centos/9/x86_64/flavors/default/cephadm 2024-03-19T01:30:27.128 DEBUG:teuthology.orchestra.run.smithi099:> curl --silent -L https://4.chacra.ceph.com/binaries/ceph/wip-batrick-testing-20240318.181317/b6ae24918b03b1b8a19c9263a50655bba80bb810/centos/9/x86_64/flavors/default/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-03-19T01:30:27.720 INFO:teuthology.orchestra.run.smithi099.stdout:-rw-r--r--. 1 ubuntu ubuntu 767404 Mar 19 01:30 /home/ubuntu/cephtest/cephadm 2024-03-19T01:30:27.721 DEBUG:teuthology.orchestra.run.smithi112:> curl --silent -L https://4.chacra.ceph.com/binaries/ceph/wip-batrick-testing-20240318.181317/b6ae24918b03b1b8a19c9263a50655bba80bb810/centos/9/x86_64/flavors/default/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-03-19T01:30:28.171 INFO:teuthology.orchestra.run.smithi112.stdout:-rw-r--r--. 1 ubuntu ubuntu 767404 Mar 19 01:30 /home/ubuntu/cephtest/cephadm 2024-03-19T01:30:28.171 DEBUG:teuthology.orchestra.run.smithi115:> curl --silent -L https://4.chacra.ceph.com/binaries/ceph/wip-batrick-testing-20240318.181317/b6ae24918b03b1b8a19c9263a50655bba80bb810/centos/9/x86_64/flavors/default/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-03-19T01:30:29.341 INFO:teuthology.orchestra.run.smithi115.stdout:-rw-r--r--. 1 ubuntu ubuntu 767404 Mar 19 01:30 /home/ubuntu/cephtest/cephadm 2024-03-19T01:30:29.342 DEBUG:teuthology.orchestra.run.smithi099:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-03-19T01:30:29.360 DEBUG:teuthology.orchestra.run.smithi112:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-03-19T01:30:29.382 DEBUG:teuthology.orchestra.run.smithi115:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-03-19T01:30:29.445 INFO:tasks.cephadm:Pulling image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 on all hosts... 2024-03-19T01:30:29.445 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 pull 2024-03-19T01:30:29.448 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 pull 2024-03-19T01:30:29.450 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 pull 2024-03-19T01:30:30.218 INFO:teuthology.orchestra.run.smithi115.stderr:Pulling container image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810... 2024-03-19T01:30:30.236 INFO:teuthology.orchestra.run.smithi112.stderr:Pulling container image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810... 2024-03-19T01:30:30.394 INFO:teuthology.orchestra.run.smithi099.stderr:Pulling container image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810... 2024-03-19T01:31:00.837 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-03-19T01:31:00.837 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph_version": "ceph version 19.0.0-2206-gb6ae2491 (b6ae24918b03b1b8a19c9263a50655bba80bb810) squid (dev)", 2024-03-19T01:31:00.837 INFO:teuthology.orchestra.run.smithi112.stdout: "image_id": "2312c8e42d0069a2dbb2364bddb00264db4b800e7f18a7e7e4eb8f3ecab2ff1c", 2024-03-19T01:31:00.837 INFO:teuthology.orchestra.run.smithi112.stdout: "repo_digests": [ 2024-03-19T01:31:00.837 INFO:teuthology.orchestra.run.smithi112.stdout: "quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639" 2024-03-19T01:31:00.837 INFO:teuthology.orchestra.run.smithi112.stdout: ] 2024-03-19T01:31:00.838 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-03-19T01:31:00.870 INFO:teuthology.orchestra.run.smithi115.stdout:{ 2024-03-19T01:31:00.870 INFO:teuthology.orchestra.run.smithi115.stdout: "ceph_version": "ceph version 19.0.0-2206-gb6ae2491 (b6ae24918b03b1b8a19c9263a50655bba80bb810) squid (dev)", 2024-03-19T01:31:00.871 INFO:teuthology.orchestra.run.smithi115.stdout: "image_id": "2312c8e42d0069a2dbb2364bddb00264db4b800e7f18a7e7e4eb8f3ecab2ff1c", 2024-03-19T01:31:00.871 INFO:teuthology.orchestra.run.smithi115.stdout: "repo_digests": [ 2024-03-19T01:31:00.871 INFO:teuthology.orchestra.run.smithi115.stdout: "quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639" 2024-03-19T01:31:00.871 INFO:teuthology.orchestra.run.smithi115.stdout: ] 2024-03-19T01:31:00.871 INFO:teuthology.orchestra.run.smithi115.stdout:} 2024-03-19T01:31:01.183 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2024-03-19T01:31:01.183 INFO:teuthology.orchestra.run.smithi099.stdout: "ceph_version": "ceph version 19.0.0-2206-gb6ae2491 (b6ae24918b03b1b8a19c9263a50655bba80bb810) squid (dev)", 2024-03-19T01:31:01.184 INFO:teuthology.orchestra.run.smithi099.stdout: "image_id": "2312c8e42d0069a2dbb2364bddb00264db4b800e7f18a7e7e4eb8f3ecab2ff1c", 2024-03-19T01:31:01.184 INFO:teuthology.orchestra.run.smithi099.stdout: "repo_digests": [ 2024-03-19T01:31:01.184 INFO:teuthology.orchestra.run.smithi099.stdout: "quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639" 2024-03-19T01:31:01.184 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-03-19T01:31:01.184 INFO:teuthology.orchestra.run.smithi099.stdout:} 2024-03-19T01:31:01.208 DEBUG:teuthology.orchestra.run.smithi099:> sudo mkdir -p /etc/ceph 2024-03-19T01:31:01.270 DEBUG:teuthology.orchestra.run.smithi112:> sudo mkdir -p /etc/ceph 2024-03-19T01:31:01.301 DEBUG:teuthology.orchestra.run.smithi115:> sudo mkdir -p /etc/ceph 2024-03-19T01:31:01.333 DEBUG:teuthology.orchestra.run.smithi099:> sudo chmod 777 /etc/ceph 2024-03-19T01:31:01.359 DEBUG:teuthology.orchestra.run.smithi112:> sudo chmod 777 /etc/ceph 2024-03-19T01:31:01.386 DEBUG:teuthology.orchestra.run.smithi115:> sudo chmod 777 /etc/ceph 2024-03-19T01:31:01.415 INFO:tasks.cephadm:Writing seed config... 2024-03-19T01:31:01.416 INFO:tasks.cephadm: override: [client] client mount timeout = 600 2024-03-19T01:31:01.416 INFO:tasks.cephadm: override: [client] debug client = 20 2024-03-19T01:31:01.416 INFO:tasks.cephadm: override: [client] debug ms = 1 2024-03-19T01:31:01.416 INFO:tasks.cephadm: override: [client] rados mon op timeout = 900 2024-03-19T01:31:01.416 INFO:tasks.cephadm: override: [client] rados osd op timeout = 900 2024-03-19T01:31:01.416 INFO:tasks.cephadm: override: [global] ms die on skipped message = False 2024-03-19T01:31:01.416 INFO:tasks.cephadm: override: [mds] debug mds = 20 2024-03-19T01:31:01.416 INFO:tasks.cephadm: override: [mds] debug mds balancer = 20 2024-03-19T01:31:01.417 INFO:tasks.cephadm: override: [mds] debug ms = 1 2024-03-19T01:31:01.417 INFO:tasks.cephadm: override: [mds] mds bal fragment size max = 10000 2024-03-19T01:31:01.417 INFO:tasks.cephadm: override: [mds] mds bal merge size = 5 2024-03-19T01:31:01.417 INFO:tasks.cephadm: override: [mds] mds bal split bits = 3 2024-03-19T01:31:01.417 INFO:tasks.cephadm: override: [mds] mds bal split size = 100 2024-03-19T01:31:01.417 INFO:tasks.cephadm: override: [mds] mds debug frag = True 2024-03-19T01:31:01.417 INFO:tasks.cephadm: override: [mds] mds debug scatterstat = True 2024-03-19T01:31:01.417 INFO:tasks.cephadm: override: [mds] mds op complaint time = 180 2024-03-19T01:31:01.417 INFO:tasks.cephadm: override: [mds] mds verify scatter = True 2024-03-19T01:31:01.417 INFO:tasks.cephadm: override: [mds] mds_bal_replicate_threshold = 1 2024-03-19T01:31:01.417 INFO:tasks.cephadm: override: [mds] mds_export_ephemeral_random_max = 0.1 2024-03-19T01:31:01.417 INFO:tasks.cephadm: override: [mds] osd op complaint time = 180 2024-03-19T01:31:01.417 INFO:tasks.cephadm: override: [mds] rados mon op timeout = 900 2024-03-19T01:31:01.417 INFO:tasks.cephadm: override: [mds] rados osd op timeout = 900 2024-03-19T01:31:01.418 INFO:tasks.cephadm: override: [mgr] cephsqlite lock renewal timeout = 900000 2024-03-19T01:31:01.418 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-03-19T01:31:01.418 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-03-19T01:31:01.418 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-03-19T01:31:01.418 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-03-19T01:31:01.418 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-03-19T01:31:01.418 INFO:tasks.cephadm: override: [mon] mon op complaint time = 120 2024-03-19T01:31:01.418 INFO:tasks.cephadm: override: [osd] bdev async discard = True 2024-03-19T01:31:01.418 INFO:tasks.cephadm: override: [osd] bdev enable discard = True 2024-03-19T01:31:01.418 INFO:tasks.cephadm: override: [osd] bluestore allocator = bitmap 2024-03-19T01:31:01.418 INFO:tasks.cephadm: override: [osd] bluestore block size = 96636764160 2024-03-19T01:31:01.418 INFO:tasks.cephadm: override: [osd] bluestore fsck on mount = True 2024-03-19T01:31:01.418 INFO:tasks.cephadm: override: [osd] debug bluefs = 1/20 2024-03-19T01:31:01.418 INFO:tasks.cephadm: override: [osd] debug bluestore = 1/20 2024-03-19T01:31:01.418 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-03-19T01:31:01.419 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-03-19T01:31:01.419 INFO:tasks.cephadm: override: [osd] debug rocksdb = 4/10 2024-03-19T01:31:01.419 INFO:tasks.cephadm: override: [osd] mon osd backfillfull_ratio = 0.85 2024-03-19T01:31:01.419 INFO:tasks.cephadm: override: [osd] mon osd full ratio = 0.9 2024-03-19T01:31:01.419 INFO:tasks.cephadm: override: [osd] mon osd nearfull ratio = 0.8 2024-03-19T01:31:01.419 INFO:tasks.cephadm: override: [osd] osd failsafe full ratio = 0.95 2024-03-19T01:31:01.419 INFO:tasks.cephadm: override: [osd] osd objectstore = bluestore 2024-03-19T01:31:01.419 INFO:tasks.cephadm: override: [osd] osd op complaint time = 180 2024-03-19T01:31:01.419 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-03-19T01:31:01.419 INFO:tasks.cephadm: override: [osd] osd_max_omap_entries_per_request = 10 2024-03-19T01:31:01.420 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2024-03-19T01:31:01.420 DEBUG:teuthology.orchestra.run.smithi099:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-03-19T01:31:01.434 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 438fb69c-e590-11ee-95c9-87774f69a715 ms die on skipped message = False [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true bdev async discard = True bdev enable discard = True bluestore allocator = bitmap bluestore block size = 96636764160 bluestore fsck on mount = True debug bluefs = 1/20 debug bluestore = 1/20 debug ms = 1 debug osd = 20 debug rocksdb = 4/10 mon osd backfillfull_ratio = 0.85 mon osd full ratio = 0.9 mon osd nearfull ratio = 0.8 osd failsafe full ratio = 0.95 osd objectstore = bluestore osd op complaint time = 180 osd_max_omap_entries_per_request = 10 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false cephsqlite lock renewal timeout = 900000 debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 mon op complaint time = 120 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true [client] client mount timeout = 600 debug client = 20 debug ms = 1 rados mon op timeout = 900 rados osd op timeout = 900 [mds] debug mds = 20 debug mds balancer = 20 debug ms = 1 mds bal fragment size max = 10000 mds bal merge size = 5 mds bal split bits = 3 mds bal split size = 100 mds debug frag = True mds debug scatterstat = True mds op complaint time = 180 mds verify scatter = True mds_bal_replicate_threshold = 1 mds_export_ephemeral_random_max = 0.1 osd op complaint time = 180 rados mon op timeout = 900 rados osd op timeout = 900 2024-03-19T01:31:01.435 DEBUG:teuthology.orchestra.run.smithi099:mon.a> sudo journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@mon.a.service 2024-03-19T01:31:01.477 DEBUG:teuthology.orchestra.run.smithi099:mgr.x> sudo journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@mgr.x.service 2024-03-19T01:31:01.519 INFO:tasks.cephadm:Bootstrapping... 2024-03-19T01:31:01.519 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 -v bootstrap --fsid 438fb69c-e590-11ee-95c9-87774f69a715 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id x --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.99 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-03-19T01:31:01.708 INFO:teuthology.orchestra.run.smithi099.stdout:-------------------------------------------------------------------------------- 2024-03-19T01:31:01.709 INFO:teuthology.orchestra.run.smithi099.stdout:cephadm ['--image', 'quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810', '-v', 'bootstrap', '--fsid', '438fb69c-e590-11ee-95c9-87774f69a715', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'x', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.99', '--skip-admin-label'] 2024-03-19T01:31:01.734 INFO:teuthology.orchestra.run.smithi099.stdout:/bin/podman: stdout 4.9.3-dev 2024-03-19T01:31:01.734 INFO:teuthology.orchestra.run.smithi099.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2024-03-19T01:31:01.734 INFO:teuthology.orchestra.run.smithi099.stdout:Verifying podman|docker is present... 2024-03-19T01:31:01.757 INFO:teuthology.orchestra.run.smithi099.stdout:/bin/podman: stdout 4.9.3-dev 2024-03-19T01:31:01.757 INFO:teuthology.orchestra.run.smithi099.stdout:Verifying lvm2 is present... 2024-03-19T01:31:01.758 INFO:teuthology.orchestra.run.smithi099.stdout:Verifying time synchronization is in place... 2024-03-19T01:31:01.765 INFO:teuthology.orchestra.run.smithi099.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-03-19T01:31:01.765 INFO:teuthology.orchestra.run.smithi099.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-03-19T01:31:01.770 INFO:teuthology.orchestra.run.smithi099.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-03-19T01:31:01.771 INFO:teuthology.orchestra.run.smithi099.stdout:systemctl: stdout inactive 2024-03-19T01:31:01.777 INFO:teuthology.orchestra.run.smithi099.stdout:systemctl: stdout enabled 2024-03-19T01:31:01.783 INFO:teuthology.orchestra.run.smithi099.stdout:systemctl: stdout active 2024-03-19T01:31:01.783 INFO:teuthology.orchestra.run.smithi099.stdout:Unit chronyd.service is enabled and running 2024-03-19T01:31:01.783 INFO:teuthology.orchestra.run.smithi099.stdout:Repeating the final host check... 2024-03-19T01:31:01.806 INFO:teuthology.orchestra.run.smithi099.stdout:/bin/podman: stdout 4.9.3-dev 2024-03-19T01:31:01.806 INFO:teuthology.orchestra.run.smithi099.stdout:podman (/bin/podman) version 4.9.3 is present 2024-03-19T01:31:01.806 INFO:teuthology.orchestra.run.smithi099.stdout:systemctl is present 2024-03-19T01:31:01.806 INFO:teuthology.orchestra.run.smithi099.stdout:lvcreate is present 2024-03-19T01:31:01.812 INFO:teuthology.orchestra.run.smithi099.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-03-19T01:31:01.812 INFO:teuthology.orchestra.run.smithi099.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-03-19T01:31:01.818 INFO:teuthology.orchestra.run.smithi099.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-03-19T01:31:01.818 INFO:teuthology.orchestra.run.smithi099.stdout:systemctl: stdout inactive 2024-03-19T01:31:01.825 INFO:teuthology.orchestra.run.smithi099.stdout:systemctl: stdout enabled 2024-03-19T01:31:01.830 INFO:teuthology.orchestra.run.smithi099.stdout:systemctl: stdout active 2024-03-19T01:31:01.830 INFO:teuthology.orchestra.run.smithi099.stdout:Unit chronyd.service is enabled and running 2024-03-19T01:31:01.831 INFO:teuthology.orchestra.run.smithi099.stdout:Host looks OK 2024-03-19T01:31:01.831 INFO:teuthology.orchestra.run.smithi099.stdout:Cluster fsid: 438fb69c-e590-11ee-95c9-87774f69a715 2024-03-19T01:31:01.831 INFO:teuthology.orchestra.run.smithi099.stdout:Acquiring lock 140414941409184 on /run/cephadm/438fb69c-e590-11ee-95c9-87774f69a715.lock 2024-03-19T01:31:01.831 INFO:teuthology.orchestra.run.smithi099.stdout:Lock 140414941409184 acquired on /run/cephadm/438fb69c-e590-11ee-95c9-87774f69a715.lock 2024-03-19T01:31:01.831 INFO:teuthology.orchestra.run.smithi099.stdout:Verifying IP 172.21.15.99 port 3300 ... 2024-03-19T01:31:01.831 INFO:teuthology.orchestra.run.smithi099.stdout:Verifying IP 172.21.15.99 port 6789 ... 2024-03-19T01:31:01.831 INFO:teuthology.orchestra.run.smithi099.stdout:Base mon IP(s) is [172.21.15.99:3300, 172.21.15.99:6789], mon addrv is [v2:172.21.15.99:3300,v1:172.21.15.99:6789] 2024-03-19T01:31:01.835 INFO:teuthology.orchestra.run.smithi099.stdout:/sbin/ip: stdout default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.99 metric 100 2024-03-19T01:31:01.835 INFO:teuthology.orchestra.run.smithi099.stdout:/sbin/ip: stdout 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.99 metric 100 2024-03-19T01:31:01.837 INFO:teuthology.orchestra.run.smithi099.stdout:/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2024-03-19T01:31:01.837 INFO:teuthology.orchestra.run.smithi099.stdout:/sbin/ip: stdout fe80::/64 dev enp3s0f1 proto kernel metric 256 pref medium 2024-03-19T01:31:01.838 INFO:teuthology.orchestra.run.smithi099.stdout:/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 1024 expires 1799sec hoplimit 64 pref medium 2024-03-19T01:31:01.840 INFO:teuthology.orchestra.run.smithi099.stdout:/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-03-19T01:31:01.840 INFO:teuthology.orchestra.run.smithi099.stdout:/sbin/ip: stdout inet6 ::1/128 scope host 2024-03-19T01:31:01.840 INFO:teuthology.orchestra.run.smithi099.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-03-19T01:31:01.840 INFO:teuthology.orchestra.run.smithi099.stdout:/sbin/ip: stdout 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-03-19T01:31:01.840 INFO:teuthology.orchestra.run.smithi099.stdout:/sbin/ip: stdout inet6 fe80::ec4:7aff:fe8f:d803/64 scope link 2024-03-19T01:31:01.840 INFO:teuthology.orchestra.run.smithi099.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-03-19T01:31:01.841 INFO:teuthology.orchestra.run.smithi099.stdout:Mon IP `172.21.15.99` is in CIDR network `172.21.0.0/20` 2024-03-19T01:31:01.841 INFO:teuthology.orchestra.run.smithi099.stdout:Mon IP `172.21.15.99` is in CIDR network `172.21.0.0/20` 2024-03-19T01:31:01.841 INFO:teuthology.orchestra.run.smithi099.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2024-03-19T01:31:01.841 INFO:teuthology.orchestra.run.smithi099.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-03-19T01:31:01.842 INFO:teuthology.orchestra.run.smithi099.stdout:Pulling container image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810... 2024-03-19T01:31:02.311 INFO:teuthology.orchestra.run.smithi099.stdout:/bin/podman: stdout 2312c8e42d0069a2dbb2364bddb00264db4b800e7f18a7e7e4eb8f3ecab2ff1c 2024-03-19T01:31:02.312 INFO:teuthology.orchestra.run.smithi099.stdout:/bin/podman: stderr Trying to pull quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810... 2024-03-19T01:31:02.312 INFO:teuthology.orchestra.run.smithi099.stdout:/bin/podman: stderr Getting image source signatures 2024-03-19T01:31:02.312 INFO:teuthology.orchestra.run.smithi099.stdout:/bin/podman: stderr Copying blob sha256:93b5a8d362aeee7e8b1f0bbf18238e1bb256c50573ad5b72aff9e382b1bb3ccb 2024-03-19T01:31:02.312 INFO:teuthology.orchestra.run.smithi099.stdout:/bin/podman: stderr Copying blob sha256:f58b4670ef4403d11d1c912aa26a4dbcf73447df5302e9dc66f6a7fc0d9d1e47 2024-03-19T01:31:02.312 INFO:teuthology.orchestra.run.smithi099.stdout:/bin/podman: stderr Copying config sha256:2312c8e42d0069a2dbb2364bddb00264db4b800e7f18a7e7e4eb8f3ecab2ff1c 2024-03-19T01:31:02.312 INFO:teuthology.orchestra.run.smithi099.stdout:/bin/podman: stderr Writing manifest to image destination 2024-03-19T01:31:03.232 INFO:teuthology.orchestra.run.smithi099.stdout:ceph: stdout ceph version 19.0.0-2206-gb6ae2491 (b6ae24918b03b1b8a19c9263a50655bba80bb810) squid (dev) 2024-03-19T01:31:03.232 INFO:teuthology.orchestra.run.smithi099.stdout:Ceph version: ceph version 19.0.0-2206-gb6ae2491 (b6ae24918b03b1b8a19c9263a50655bba80bb810) squid (dev) 2024-03-19T01:31:03.233 INFO:teuthology.orchestra.run.smithi099.stdout:Extracting ceph user uid/gid from container image... 2024-03-19T01:31:04.132 INFO:teuthology.orchestra.run.smithi099.stdout:stat: stdout 167 167 2024-03-19T01:31:04.132 INFO:teuthology.orchestra.run.smithi099.stdout:Creating initial keys... 2024-03-19T01:31:05.096 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-authtool: stdout AQDY6vhlaiJTGhAAfsEsMMAeWSYy88Ps/FuNRg== 2024-03-19T01:31:05.986 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-authtool: stdout AQDZ6vhlwAe1GRAAUH0r1iKEXAE9j3sE4s7IKA== 2024-03-19T01:31:06.891 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-authtool: stdout AQDa6vhlAWYoEhAAli7WhpEYjiLkjWhmRFDWPw== 2024-03-19T01:31:06.892 INFO:teuthology.orchestra.run.smithi099.stdout:Creating initial monmap... 2024-03-19T01:31:07.821 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2024-03-19T01:31:07.822 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = quincy 2024-03-19T01:31:07.822 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to 438fb69c-e590-11ee-95c9-87774f69a715 2024-03-19T01:31:07.822 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-03-19T01:31:07.822 INFO:teuthology.orchestra.run.smithi099.stdout:monmaptool for a [v2:172.21.15.99:3300,v1:172.21.15.99:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-03-19T01:31:07.822 INFO:teuthology.orchestra.run.smithi099.stdout:setting min_mon_release = quincy 2024-03-19T01:31:07.822 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/monmaptool: set fsid to 438fb69c-e590-11ee-95c9-87774f69a715 2024-03-19T01:31:07.822 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-03-19T01:31:07.822 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:31:07.822 INFO:teuthology.orchestra.run.smithi099.stdout:Creating mon... 2024-03-19T01:31:08.764 INFO:teuthology.orchestra.run.smithi099.stdout:create mon.a on 2024-03-19T01:31:08.977 INFO:teuthology.orchestra.run.smithi099.stdout:systemctl: stderr Removed "/etc/systemd/system/multi-user.target.wants/ceph.target". 2024-03-19T01:31:09.117 INFO:teuthology.orchestra.run.smithi099.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-03-19T01:31:09.306 INFO:teuthology.orchestra.run.smithi099.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-438fb69c-e590-11ee-95c9-87774f69a715.target → /etc/systemd/system/ceph-438fb69c-e590-11ee-95c9-87774f69a715.target. 2024-03-19T01:31:09.306 INFO:teuthology.orchestra.run.smithi099.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-438fb69c-e590-11ee-95c9-87774f69a715.target → /etc/systemd/system/ceph-438fb69c-e590-11ee-95c9-87774f69a715.target. 2024-03-19T01:31:09.541 INFO:teuthology.orchestra.run.smithi099.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-438fb69c-e590-11ee-95c9-87774f69a715@mon.a 2024-03-19T01:31:09.541 INFO:teuthology.orchestra.run.smithi099.stdout:systemctl: stderr Failed to reset failed state of unit ceph-438fb69c-e590-11ee-95c9-87774f69a715@mon.a.service: Unit ceph-438fb69c-e590-11ee-95c9-87774f69a715@mon.a.service not loaded. 2024-03-19T01:31:09.698 INFO:teuthology.orchestra.run.smithi099.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-438fb69c-e590-11ee-95c9-87774f69a715.target.wants/ceph-438fb69c-e590-11ee-95c9-87774f69a715@mon.a.service → /etc/systemd/system/ceph-438fb69c-e590-11ee-95c9-87774f69a715@.service. 2024-03-19T01:31:10.196 INFO:teuthology.orchestra.run.smithi099.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-03-19T01:31:10.196 INFO:teuthology.orchestra.run.smithi099.stdout:systemctl: stdout disabled 2024-03-19T01:31:10.203 INFO:teuthology.orchestra.run.smithi099.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-03-19T01:31:10.203 INFO:teuthology.orchestra.run.smithi099.stdout:systemctl: stdout inactive 2024-03-19T01:31:10.203 INFO:teuthology.orchestra.run.smithi099.stdout:firewalld.service is not enabled 2024-03-19T01:31:10.203 INFO:teuthology.orchestra.run.smithi099.stdout:Not possible to enable service . firewalld.service is not available 2024-03-19T01:31:10.203 INFO:teuthology.orchestra.run.smithi099.stdout:Waiting for mon to start... 2024-03-19T01:31:10.204 INFO:teuthology.orchestra.run.smithi099.stdout:Waiting for mon... 2024-03-19T01:31:10.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:10 smithi099 systemd[1]: Started Ceph mon.a for 438fb69c-e590-11ee-95c9-87774f69a715. 2024-03-19T01:31:11.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:11 smithi099 ceph-mon[62577]: mon.a is new leader, mons a in quorum (ranks 0) 2024-03-19T01:31:11.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:11 smithi099 ceph-mon[62577]: monmap epoch 1 2024-03-19T01:31:11.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:11 smithi099 ceph-mon[62577]: fsid 438fb69c-e590-11ee-95c9-87774f69a715 2024-03-19T01:31:11.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:11 smithi099 ceph-mon[62577]: last_changed 2024-03-19T01:31:07.215663+0000 2024-03-19T01:31:11.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:11 smithi099 ceph-mon[62577]: created 2024-03-19T01:31:07.215663+0000 2024-03-19T01:31:11.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:11 smithi099 ceph-mon[62577]: min_mon_release 19 (squid) 2024-03-19T01:31:11.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:11 smithi099 ceph-mon[62577]: election_strategy: 1 2024-03-19T01:31:11.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:11 smithi099 ceph-mon[62577]: 0: [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] mon.a 2024-03-19T01:31:11.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:11 smithi099 ceph-mon[62577]: fsmap 2024-03-19T01:31:11.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:11 smithi099 ceph-mon[62577]: osdmap e1: 0 total, 0 up, 0 in 2024-03-19T01:31:11.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:11 smithi099 ceph-mon[62577]: mgrmap e1: no daemons active 2024-03-19T01:31:11.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:11 smithi099 ceph-mon[62577]: from='client.? 172.21.15.99:0/4145405100' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-03-19T01:31:11.343 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout cluster: 2024-03-19T01:31:11.349 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout id: 438fb69c-e590-11ee-95c9-87774f69a715 2024-03-19T01:31:11.349 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2024-03-19T01:31:11.350 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout 2024-03-19T01:31:11.350 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout services: 2024-03-19T01:31:11.350 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.562893s) 2024-03-19T01:31:11.350 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2024-03-19T01:31:11.350 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2024-03-19T01:31:11.350 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout 2024-03-19T01:31:11.350 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout data: 2024-03-19T01:31:11.350 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2024-03-19T01:31:11.350 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2024-03-19T01:31:11.350 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2024-03-19T01:31:11.350 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout pgs: 2024-03-19T01:31:11.350 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout 2024-03-19T01:31:11.350 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:10.650+0000 7f7fe953e700 1 Processor -- start 2024-03-19T01:31:11.351 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:10.650+0000 7f7fe953e700 1 -- start start 2024-03-19T01:31:11.351 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:10.650+0000 7f7fe953e700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7fe4106cd0 0x7f7fe41070f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:31:11.351 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:10.650+0000 7f7fe953e700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f7fe41076c0 con 0x7f7fe4106cd0 2024-03-19T01:31:11.351 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:10.650+0000 7f7fe2ffd700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7fe4106cd0 0x7f7fe41070f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:31:11.351 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:10.650+0000 7f7fe2ffd700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7fe4106cd0 0x7f7fe41070f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:53868/0 (socket says 172.21.15.99:53868) 2024-03-19T01:31:11.351 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:10.650+0000 7f7fe2ffd700 1 -- 172.21.15.99:0/52114775 learned_addr learned my addr 172.21.15.99:0/52114775 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:31:11.351 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:10.651+0000 7f7fe2ffd700 1 -- 172.21.15.99:0/52114775 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7fe4107ed0 con 0x7f7fe4106cd0 2024-03-19T01:31:11.351 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:10.651+0000 7f7fe2ffd700 1 --2- 172.21.15.99:0/52114775 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7fe4106cd0 0x7f7fe41070f0 secure :-1 s=READY pgs=1 cs=0 l=1 rev1=1 crypto rx=0x7f7fd40098a0 tx=0x7f7fd4009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3f9db1e0159f667 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:31:11.351 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:10.651+0000 7f7fe1ffb700 1 -- 172.21.15.99:0/52114775 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f7fd4004160 con 0x7f7fe4106cd0 2024-03-19T01:31:11.351 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:10.651+0000 7f7fe1ffb700 1 -- 172.21.15.99:0/52114775 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(0 keys) ==== 4+0+0 (secure 0 0 0) 0x7f7fd4004300 con 0x7f7fe4106cd0 2024-03-19T01:31:11.351 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:10.651+0000 7f7fe1ffb700 1 -- 172.21.15.99:0/52114775 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f7fd40036a0 con 0x7f7fe4106cd0 2024-03-19T01:31:11.351 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:10.651+0000 7f7fe953e700 1 -- 172.21.15.99:0/52114775 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7fe4106cd0 msgr2=0x7f7fe41070f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:31:11.351 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:10.651+0000 7f7fe953e700 1 --2- 172.21.15.99:0/52114775 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7fe4106cd0 0x7f7fe41070f0 secure :-1 s=READY pgs=1 cs=0 l=1 rev1=1 crypto rx=0x7f7fd40098a0 tx=0x7f7fd4009bb0 comp rx=0 tx=0).stop 2024-03-19T01:31:11.352 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:10.652+0000 7f7fe953e700 1 -- 172.21.15.99:0/52114775 shutdown_connections 2024-03-19T01:31:11.352 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:10.652+0000 7f7fe953e700 1 --2- 172.21.15.99:0/52114775 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7fe4106cd0 0x7f7fe41070f0 unknown :-1 s=CLOSED pgs=1 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:11.352 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:10.652+0000 7f7fe953e700 1 -- 172.21.15.99:0/52114775 >> 172.21.15.99:0/52114775 conn(0x7f7fe4102270 msgr2=0x7f7fe41046b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:31:11.352 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:10.652+0000 7f7fe953e700 1 -- 172.21.15.99:0/52114775 shutdown_connections 2024-03-19T01:31:11.352 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:10.652+0000 7f7fe953e700 1 -- 172.21.15.99:0/52114775 wait complete. 2024-03-19T01:31:11.352 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:10.652+0000 7f7fe953e700 1 Processor -- start 2024-03-19T01:31:11.352 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:10.652+0000 7f7fe953e700 1 -- start start 2024-03-19T01:31:11.352 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:10.653+0000 7f7fe953e700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7fe4106cd0 0x7f7fe419a410 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:31:11.352 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:10.653+0000 7f7fe953e700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f7fe4108400 con 0x7f7fe4106cd0 2024-03-19T01:31:11.353 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:10.653+0000 7f7fe2ffd700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7fe4106cd0 0x7f7fe419a410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:31:11.353 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:10.653+0000 7f7fe2ffd700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7fe4106cd0 0x7f7fe419a410 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:53876/0 (socket says 172.21.15.99:53876) 2024-03-19T01:31:11.353 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:10.653+0000 7f7fe2ffd700 1 -- 172.21.15.99:0/4145405100 learned_addr learned my addr 172.21.15.99:0/4145405100 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:31:11.353 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:10.653+0000 7f7fe2ffd700 1 -- 172.21.15.99:0/4145405100 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7fe419a950 con 0x7f7fe4106cd0 2024-03-19T01:31:11.353 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:10.653+0000 7f7fe2ffd700 1 --2- 172.21.15.99:0/4145405100 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7fe4106cd0 0x7f7fe419a410 secure :-1 s=READY pgs=2 cs=0 l=1 rev1=1 crypto rx=0x7f7fd4000c00 tx=0x7f7fd4004750 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:31:11.353 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:10.653+0000 7f7fcbfff700 1 -- 172.21.15.99:0/4145405100 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f7fd4003db0 con 0x7f7fe4106cd0 2024-03-19T01:31:11.353 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:10.653+0000 7f7fcbfff700 1 -- 172.21.15.99:0/4145405100 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(0 keys) ==== 4+0+0 (secure 0 0 0) 0x7f7fd4011630 con 0x7f7fe4106cd0 2024-03-19T01:31:11.354 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:10.653+0000 7f7fe953e700 1 -- 172.21.15.99:0/4145405100 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f7fe419abe0 con 0x7f7fe4106cd0 2024-03-19T01:31:11.354 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:10.653+0000 7f7fcbfff700 1 -- 172.21.15.99:0/4145405100 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f7fd4011950 con 0x7f7fe4106cd0 2024-03-19T01:31:11.354 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:10.653+0000 7f7fe953e700 1 -- 172.21.15.99:0/4145405100 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f7fe419d8d0 con 0x7f7fe4106cd0 2024-03-19T01:31:11.354 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:10.653+0000 7f7fcbfff700 1 -- 172.21.15.99:0/4145405100 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7f7fd401a460 con 0x7f7fe4106cd0 2024-03-19T01:31:11.354 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:10.654+0000 7f7fcbfff700 1 -- 172.21.15.99:0/4145405100 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7f7fd401b780 con 0x7f7fe4106cd0 2024-03-19T01:31:11.354 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:10.654+0000 7f7fe953e700 1 -- 172.21.15.99:0/4145405100 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f7fe4069420 con 0x7f7fe4106cd0 2024-03-19T01:31:11.354 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:10.657+0000 7f7fcbfff700 1 -- 172.21.15.99:0/4145405100 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+73141 (secure 0 0 0) 0x7f7fd4031020 con 0x7f7fe4106cd0 2024-03-19T01:31:11.354 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:10.726+0000 7f7fe953e700 1 -- 172.21.15.99:0/4145405100 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "status"} v 0) -- 0x7f7fe41938c0 con 0x7f7fe4106cd0 2024-03-19T01:31:11.354 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:10.726+0000 7f7fcbfff700 1 -- 172.21.15.99:0/4145405100 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "status"}]=0 v0) ==== 54+0+317 (secure 0 0 0) 0x7f7fd401b9d0 con 0x7f7fe4106cd0 2024-03-19T01:31:11.354 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:10.726+0000 7f7fe953e700 1 -- 172.21.15.99:0/4145405100 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7fe4106cd0 msgr2=0x7f7fe419a410 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:31:11.354 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:10.727+0000 7f7fe953e700 1 --2- 172.21.15.99:0/4145405100 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7fe4106cd0 0x7f7fe419a410 secure :-1 s=READY pgs=2 cs=0 l=1 rev1=1 crypto rx=0x7f7fd4000c00 tx=0x7f7fd4004750 comp rx=0 tx=0).stop 2024-03-19T01:31:11.354 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:10.727+0000 7f7fe953e700 1 -- 172.21.15.99:0/4145405100 shutdown_connections 2024-03-19T01:31:11.355 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:10.727+0000 7f7fe953e700 1 --2- 172.21.15.99:0/4145405100 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7fe4106cd0 0x7f7fe419a410 unknown :-1 s=CLOSED pgs=2 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:11.355 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:10.727+0000 7f7fe953e700 1 -- 172.21.15.99:0/4145405100 >> 172.21.15.99:0/4145405100 conn(0x7f7fe4102270 msgr2=0x7f7fe4191020 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:31:11.355 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:10.727+0000 7f7fe953e700 1 -- 172.21.15.99:0/4145405100 shutdown_connections 2024-03-19T01:31:11.355 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:10.727+0000 7f7fe953e700 1 -- 172.21.15.99:0/4145405100 wait complete. 2024-03-19T01:31:11.355 INFO:teuthology.orchestra.run.smithi099.stdout:mon is available 2024-03-19T01:31:11.355 INFO:teuthology.orchestra.run.smithi099.stdout:Assimilating anything we can from ceph.conf... 2024-03-19T01:31:12.447 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:12 smithi099 ceph-mon[62577]: from='client.? 172.21.15.99:0/4265824385' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-03-19T01:31:12.447 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:12 smithi099 ceph-mon[62577]: from='client.? 172.21.15.99:0/4265824385' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-03-19T01:31:12.455 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout 2024-03-19T01:31:12.455 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout [global] 2024-03-19T01:31:12.455 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout fsid = 438fb69c-e590-11ee-95c9-87774f69a715 2024-03-19T01:31:12.455 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.99:3300,v1:172.21.15.99:6789] 2024-03-19T01:31:12.456 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-03-19T01:31:12.456 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-03-19T01:31:12.456 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-03-19T01:31:12.456 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-03-19T01:31:12.456 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout 2024-03-19T01:31:12.456 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout [mgr] 2024-03-19T01:31:12.456 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-03-19T01:31:12.456 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout 2024-03-19T01:31:12.456 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout [osd] 2024-03-19T01:31:12.456 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout bdev_async_discard = True 2024-03-19T01:31:12.456 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout mon_osd_backfillfull_ratio = 0.85 2024-03-19T01:31:12.456 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout mon_osd_full_ratio = 0.9 2024-03-19T01:31:12.456 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout mon_osd_nearfull_ratio = 0.8 2024-03-19T01:31:12.456 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-03-19T01:31:12.456 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-03-19T01:31:12.457 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:11.781+0000 7f5a4a012700 1 Processor -- start 2024-03-19T01:31:12.457 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:11.781+0000 7f5a4a012700 1 -- start start 2024-03-19T01:31:12.457 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:11.781+0000 7f5a4a012700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5a44106cc0 0x7f5a441070e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:31:12.457 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:11.781+0000 7f5a4a012700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f5a441076b0 con 0x7f5a44106cc0 2024-03-19T01:31:12.457 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:11.782+0000 7f5a437fe700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5a44106cc0 0x7f5a441070e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:31:12.457 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:11.782+0000 7f5a437fe700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5a44106cc0 0x7f5a441070e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:53888/0 (socket says 172.21.15.99:53888) 2024-03-19T01:31:12.457 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:11.782+0000 7f5a437fe700 1 -- 172.21.15.99:0/3383469270 learned_addr learned my addr 172.21.15.99:0/3383469270 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:31:12.457 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:11.782+0000 7f5a437fe700 1 -- 172.21.15.99:0/3383469270 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f5a44107ec0 con 0x7f5a44106cc0 2024-03-19T01:31:12.457 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:11.782+0000 7f5a437fe700 1 --2- 172.21.15.99:0/3383469270 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5a44106cc0 0x7f5a441070e0 secure :-1 s=READY pgs=3 cs=0 l=1 rev1=1 crypto rx=0x7f5a340098a0 tx=0x7f5a34009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9673cb33be76bee7 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:31:12.457 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:11.782+0000 7f5a427fc700 1 -- 172.21.15.99:0/3383469270 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f5a34004160 con 0x7f5a44106cc0 2024-03-19T01:31:12.457 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:11.782+0000 7f5a427fc700 1 -- 172.21.15.99:0/3383469270 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(0 keys) ==== 4+0+0 (secure 0 0 0) 0x7f5a34004300 con 0x7f5a44106cc0 2024-03-19T01:31:12.457 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:11.782+0000 7f5a427fc700 1 -- 172.21.15.99:0/3383469270 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f5a340036a0 con 0x7f5a44106cc0 2024-03-19T01:31:12.457 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:11.783+0000 7f5a4a012700 1 -- 172.21.15.99:0/3383469270 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5a44106cc0 msgr2=0x7f5a441070e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:31:12.457 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:11.783+0000 7f5a4a012700 1 --2- 172.21.15.99:0/3383469270 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5a44106cc0 0x7f5a441070e0 secure :-1 s=READY pgs=3 cs=0 l=1 rev1=1 crypto rx=0x7f5a340098a0 tx=0x7f5a34009bb0 comp rx=0 tx=0).stop 2024-03-19T01:31:12.458 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:11.783+0000 7f5a4a012700 1 -- 172.21.15.99:0/3383469270 shutdown_connections 2024-03-19T01:31:12.458 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:11.783+0000 7f5a4a012700 1 --2- 172.21.15.99:0/3383469270 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5a44106cc0 0x7f5a441070e0 unknown :-1 s=CLOSED pgs=3 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:12.458 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:11.783+0000 7f5a4a012700 1 -- 172.21.15.99:0/3383469270 >> 172.21.15.99:0/3383469270 conn(0x7f5a44102220 msgr2=0x7f5a441046a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:31:12.458 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:11.783+0000 7f5a4a012700 1 -- 172.21.15.99:0/3383469270 shutdown_connections 2024-03-19T01:31:12.458 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:11.783+0000 7f5a4a012700 1 -- 172.21.15.99:0/3383469270 wait complete. 2024-03-19T01:31:12.458 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:11.783+0000 7f5a4a012700 1 Processor -- start 2024-03-19T01:31:12.458 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:11.784+0000 7f5a4a012700 1 -- start start 2024-03-19T01:31:12.458 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:11.784+0000 7f5a4a012700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5a44106cc0 0x7f5a4419b550 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:31:12.458 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:11.784+0000 7f5a4a012700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f5a441083f0 con 0x7f5a44106cc0 2024-03-19T01:31:12.458 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:11.784+0000 7f5a437fe700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5a44106cc0 0x7f5a4419b550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:31:12.458 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:11.784+0000 7f5a437fe700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5a44106cc0 0x7f5a4419b550 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:53904/0 (socket says 172.21.15.99:53904) 2024-03-19T01:31:12.458 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:11.784+0000 7f5a437fe700 1 -- 172.21.15.99:0/4265824385 learned_addr learned my addr 172.21.15.99:0/4265824385 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:31:12.458 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:11.784+0000 7f5a437fe700 1 -- 172.21.15.99:0/4265824385 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f5a4419ba90 con 0x7f5a44106cc0 2024-03-19T01:31:12.459 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:11.784+0000 7f5a437fe700 1 --2- 172.21.15.99:0/4265824385 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5a44106cc0 0x7f5a4419b550 secure :-1 s=READY pgs=4 cs=0 l=1 rev1=1 crypto rx=0x7f5a34000c00 tx=0x7f5a34004750 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:31:12.459 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:11.784+0000 7f5a40ff9700 1 -- 172.21.15.99:0/4265824385 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f5a34003db0 con 0x7f5a44106cc0 2024-03-19T01:31:12.459 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:11.784+0000 7f5a40ff9700 1 -- 172.21.15.99:0/4265824385 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(0 keys) ==== 4+0+0 (secure 0 0 0) 0x7f5a34011630 con 0x7f5a44106cc0 2024-03-19T01:31:12.459 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:11.785+0000 7f5a4a012700 1 -- 172.21.15.99:0/4265824385 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f5a4419bd20 con 0x7f5a44106cc0 2024-03-19T01:31:12.459 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:11.785+0000 7f5a40ff9700 1 -- 172.21.15.99:0/4265824385 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f5a34011950 con 0x7f5a44106cc0 2024-03-19T01:31:12.459 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:11.785+0000 7f5a4a012700 1 -- 172.21.15.99:0/4265824385 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f5a4419ea10 con 0x7f5a44106cc0 2024-03-19T01:31:12.459 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:11.785+0000 7f5a40ff9700 1 -- 172.21.15.99:0/4265824385 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7f5a3401a460 con 0x7f5a44106cc0 2024-03-19T01:31:12.459 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:11.785+0000 7f5a40ff9700 1 -- 172.21.15.99:0/4265824385 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7f5a3401b780 con 0x7f5a44106cc0 2024-03-19T01:31:12.459 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:11.786+0000 7f5a4a012700 1 -- 172.21.15.99:0/4265824385 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f5a44069420 con 0x7f5a44106cc0 2024-03-19T01:31:12.459 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:11.788+0000 7f5a40ff9700 1 -- 172.21.15.99:0/4265824385 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+73141 (secure 0 0 0) 0x7f5a3401b980 con 0x7f5a44106cc0 2024-03-19T01:31:12.459 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:11.854+0000 7f5a4a012700 1 -- 172.21.15.99:0/4265824385 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "config assimilate-conf"} v 0) -- 0x7f5a4419c200 con 0x7f5a44106cc0 2024-03-19T01:31:12.459 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:11.857+0000 7f5a40ff9700 1 -- 172.21.15.99:0/4265824385 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "config assimilate-conf"}]=0 v2) ==== 70+0+456 (secure 0 0 0) 0x7f5a4419c200 con 0x7f5a44106cc0 2024-03-19T01:31:12.459 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:11.858+0000 7f5a4a012700 1 -- 172.21.15.99:0/4265824385 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5a44106cc0 msgr2=0x7f5a4419b550 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:31:12.460 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:11.858+0000 7f5a4a012700 1 --2- 172.21.15.99:0/4265824385 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5a44106cc0 0x7f5a4419b550 secure :-1 s=READY pgs=4 cs=0 l=1 rev1=1 crypto rx=0x7f5a34000c00 tx=0x7f5a34004750 comp rx=0 tx=0).stop 2024-03-19T01:31:12.460 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:11.858+0000 7f5a40ff9700 1 -- 172.21.15.99:0/4265824385 <== mon.0 v2:172.21.15.99:3300/0 8 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7f5a3401ada0 con 0x7f5a44106cc0 2024-03-19T01:31:12.460 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:11.858+0000 7f5a4a012700 1 -- 172.21.15.99:0/4265824385 shutdown_connections 2024-03-19T01:31:12.460 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:11.858+0000 7f5a4a012700 1 --2- 172.21.15.99:0/4265824385 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5a44106cc0 0x7f5a4419b550 unknown :-1 s=CLOSED pgs=4 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:12.460 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:11.858+0000 7f5a4a012700 1 -- 172.21.15.99:0/4265824385 >> 172.21.15.99:0/4265824385 conn(0x7f5a44102220 msgr2=0x7f5a44192130 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:31:12.460 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:11.858+0000 7f5a4a012700 1 -- 172.21.15.99:0/4265824385 shutdown_connections 2024-03-19T01:31:12.460 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:11.858+0000 7f5a4a012700 1 -- 172.21.15.99:0/4265824385 wait complete. 2024-03-19T01:31:12.460 INFO:teuthology.orchestra.run.smithi099.stdout:Generating new minimal ceph.conf... 2024-03-19T01:31:13.503 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:13 smithi099 ceph-mon[62577]: from='client.? 172.21.15.99:0/1076107561' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:31:13.511 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:12.846+0000 7fb9822a0700 1 Processor -- start 2024-03-19T01:31:13.511 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:12.846+0000 7fb9822a0700 1 -- start start 2024-03-19T01:31:13.511 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:12.846+0000 7fb9822a0700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb97c106cc0 0x7fb97c1070e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:31:13.511 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:12.846+0000 7fb9822a0700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fb97c1076b0 con 0x7fb97c106cc0 2024-03-19T01:31:13.511 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:12.847+0000 7fb97bfff700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb97c106cc0 0x7fb97c1070e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:31:13.511 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:12.847+0000 7fb97bfff700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb97c106cc0 0x7fb97c1070e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:40088/0 (socket says 172.21.15.99:40088) 2024-03-19T01:31:13.511 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:12.847+0000 7fb97bfff700 1 -- 172.21.15.99:0/2600983598 learned_addr learned my addr 172.21.15.99:0/2600983598 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:31:13.511 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:12.847+0000 7fb97bfff700 1 -- 172.21.15.99:0/2600983598 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb97c107ec0 con 0x7fb97c106cc0 2024-03-19T01:31:13.511 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:12.847+0000 7fb97bfff700 1 --2- 172.21.15.99:0/2600983598 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb97c106cc0 0x7fb97c1070e0 secure :-1 s=READY pgs=5 cs=0 l=1 rev1=1 crypto rx=0x7fb96c0098a0 tx=0x7fb96c009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=891b51681c90183e server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:31:13.511 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:12.847+0000 7fb97affd700 1 -- 172.21.15.99:0/2600983598 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fb96c004160 con 0x7fb97c106cc0 2024-03-19T01:31:13.512 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:12.847+0000 7fb97affd700 1 -- 172.21.15.99:0/2600983598 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7fb96c004300 con 0x7fb97c106cc0 2024-03-19T01:31:13.512 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:12.847+0000 7fb97affd700 1 -- 172.21.15.99:0/2600983598 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fb96c011b50 con 0x7fb97c106cc0 2024-03-19T01:31:13.512 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:12.848+0000 7fb9822a0700 1 -- 172.21.15.99:0/2600983598 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb97c106cc0 msgr2=0x7fb97c1070e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:31:13.512 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:12.848+0000 7fb9822a0700 1 --2- 172.21.15.99:0/2600983598 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb97c106cc0 0x7fb97c1070e0 secure :-1 s=READY pgs=5 cs=0 l=1 rev1=1 crypto rx=0x7fb96c0098a0 tx=0x7fb96c009bb0 comp rx=0 tx=0).stop 2024-03-19T01:31:13.512 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:12.848+0000 7fb9822a0700 1 -- 172.21.15.99:0/2600983598 shutdown_connections 2024-03-19T01:31:13.512 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:12.848+0000 7fb9822a0700 1 --2- 172.21.15.99:0/2600983598 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb97c106cc0 0x7fb97c1070e0 unknown :-1 s=CLOSED pgs=5 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:13.512 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:12.848+0000 7fb9822a0700 1 -- 172.21.15.99:0/2600983598 >> 172.21.15.99:0/2600983598 conn(0x7fb97c102220 msgr2=0x7fb97c1046a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:31:13.512 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:12.848+0000 7fb9822a0700 1 -- 172.21.15.99:0/2600983598 shutdown_connections 2024-03-19T01:31:13.512 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:12.848+0000 7fb9822a0700 1 -- 172.21.15.99:0/2600983598 wait complete. 2024-03-19T01:31:13.512 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:12.848+0000 7fb9822a0700 1 Processor -- start 2024-03-19T01:31:13.512 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:12.848+0000 7fb9822a0700 1 -- start start 2024-03-19T01:31:13.512 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:12.849+0000 7fb9822a0700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb97c106cc0 0x7fb97c19ad10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:31:13.512 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:12.849+0000 7fb9822a0700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fb97c10b270 con 0x7fb97c106cc0 2024-03-19T01:31:13.512 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:12.849+0000 7fb97bfff700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb97c106cc0 0x7fb97c19ad10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:31:13.513 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:12.849+0000 7fb97bfff700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb97c106cc0 0x7fb97c19ad10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:40104/0 (socket says 172.21.15.99:40104) 2024-03-19T01:31:13.513 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:12.849+0000 7fb97bfff700 1 -- 172.21.15.99:0/1076107561 learned_addr learned my addr 172.21.15.99:0/1076107561 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:31:13.513 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:12.849+0000 7fb97bfff700 1 -- 172.21.15.99:0/1076107561 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb97c19b250 con 0x7fb97c106cc0 2024-03-19T01:31:13.513 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:12.849+0000 7fb97bfff700 1 --2- 172.21.15.99:0/1076107561 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb97c106cc0 0x7fb97c19ad10 secure :-1 s=READY pgs=6 cs=0 l=1 rev1=1 crypto rx=0x7fb96c011850 tx=0x7fb96c011c10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:31:13.513 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:12.849+0000 7fb9797fa700 1 -- 172.21.15.99:0/1076107561 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fb96c003af0 con 0x7fb97c106cc0 2024-03-19T01:31:13.513 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:12.849+0000 7fb9797fa700 1 -- 172.21.15.99:0/1076107561 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7fb96c020070 con 0x7fb97c106cc0 2024-03-19T01:31:13.513 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:12.849+0000 7fb9822a0700 1 -- 172.21.15.99:0/1076107561 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb97c19b4e0 con 0x7fb97c106cc0 2024-03-19T01:31:13.513 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:12.849+0000 7fb9797fa700 1 -- 172.21.15.99:0/1076107561 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fb96c0195a0 con 0x7fb97c106cc0 2024-03-19T01:31:13.513 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:12.849+0000 7fb9822a0700 1 -- 172.21.15.99:0/1076107561 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fb97c19e1c0 con 0x7fb97c106cc0 2024-03-19T01:31:13.513 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:12.850+0000 7fb9797fa700 1 -- 172.21.15.99:0/1076107561 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7fb96c019be0 con 0x7fb97c106cc0 2024-03-19T01:31:13.513 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:12.850+0000 7fb9797fa700 1 -- 172.21.15.99:0/1076107561 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7fb96c011e00 con 0x7fb97c106cc0 2024-03-19T01:31:13.513 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:12.850+0000 7fb9822a0700 1 -- 172.21.15.99:0/1076107561 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fb97c069420 con 0x7fb97c106cc0 2024-03-19T01:31:13.514 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:12.852+0000 7fb9797fa700 1 -- 172.21.15.99:0/1076107561 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+73141 (secure 0 0 0) 0x7fb96c003c90 con 0x7fb97c106cc0 2024-03-19T01:31:13.514 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:12.919+0000 7fb9822a0700 1 -- 172.21.15.99:0/1076107561 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "config generate-minimal-conf"} v 0) -- 0x7fb97c19e390 con 0x7fb97c106cc0 2024-03-19T01:31:13.514 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:12.920+0000 7fb9797fa700 1 -- 172.21.15.99:0/1076107561 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "config generate-minimal-conf"}]=0 v2) ==== 76+0+175 (secure 0 0 0) 0x7fb96c025460 con 0x7fb97c106cc0 2024-03-19T01:31:13.514 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:12.920+0000 7fb9822a0700 1 -- 172.21.15.99:0/1076107561 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb97c106cc0 msgr2=0x7fb97c19ad10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:31:13.514 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:12.920+0000 7fb9822a0700 1 --2- 172.21.15.99:0/1076107561 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb97c106cc0 0x7fb97c19ad10 secure :-1 s=READY pgs=6 cs=0 l=1 rev1=1 crypto rx=0x7fb96c011850 tx=0x7fb96c011c10 comp rx=0 tx=0).stop 2024-03-19T01:31:13.514 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:12.921+0000 7fb9822a0700 1 -- 172.21.15.99:0/1076107561 shutdown_connections 2024-03-19T01:31:13.514 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:12.921+0000 7fb9822a0700 1 --2- 172.21.15.99:0/1076107561 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb97c106cc0 0x7fb97c19ad10 unknown :-1 s=CLOSED pgs=6 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:13.514 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:12.921+0000 7fb9822a0700 1 -- 172.21.15.99:0/1076107561 >> 172.21.15.99:0/1076107561 conn(0x7fb97c102220 msgr2=0x7fb97c102e10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:31:13.515 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:12.921+0000 7fb9822a0700 1 -- 172.21.15.99:0/1076107561 shutdown_connections 2024-03-19T01:31:13.515 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:12.921+0000 7fb9822a0700 1 -- 172.21.15.99:0/1076107561 wait complete. 2024-03-19T01:31:13.515 INFO:teuthology.orchestra.run.smithi099.stdout:Restarting the monitor... 2024-03-19T01:31:13.757 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:13 smithi099 systemd[1]: Stopping Ceph mon.a for 438fb69c-e590-11ee-95c9-87774f69a715... 2024-03-19T01:31:14.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:13 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mon-a[62573]: 2024-03-19T01:31:13.756+0000 7f0c5e12f700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-03-19T01:31:14.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:13 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mon-a[62573]: 2024-03-19T01:31:13.756+0000 7f0c5e12f700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-03-19T01:31:14.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:13 smithi099 podman[62821]: 2024-03-19 01:31:13.900362314 +0000 UTC m=+0.230334521 container died ef0105337c4e3f2542d5a3f780ad250d3e4f5a0de1506a76c818d1c94da70415 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mon-a, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240307, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=wip-batrick-testing-20240318.181317-b6ae249) 2024-03-19T01:31:14.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:14 smithi099 podman[62821]: 2024-03-19 01:31:14.240411269 +0000 UTC m=+0.570383484 container cleanup ef0105337c4e3f2542d5a3f780ad250d3e4f5a0de1506a76c818d1c94da70415 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mon-a, CEPH_POINT_RELEASE=, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240307, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_BRANCH=HEAD) 2024-03-19T01:31:14.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:14 smithi099 bash[62821]: ceph-438fb69c-e590-11ee-95c9-87774f69a715-mon-a 2024-03-19T01:31:14.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:14 smithi099 podman[62835]: 2024-03-19 01:31:14.416363647 +0000 UTC m=+0.515121958 container remove ef0105337c4e3f2542d5a3f780ad250d3e4f5a0de1506a76c818d1c94da70415 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mon-a, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240307, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-19T01:31:14.964 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:14 smithi099 systemd[1]: ceph-438fb69c-e590-11ee-95c9-87774f69a715@mon.a.service: Deactivated successfully. 2024-03-19T01:31:14.964 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:14 smithi099 systemd[1]: Stopped Ceph mon.a for 438fb69c-e590-11ee-95c9-87774f69a715. 2024-03-19T01:31:14.964 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:14 smithi099 systemd[1]: Starting Ceph mon.a for 438fb69c-e590-11ee-95c9-87774f69a715... 2024-03-19T01:31:14.965 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:14 smithi099 podman[62932]: 2024-03-19 01:31:14.764138373 +0000 UTC m=+0.018545476 image pull 2312c8e42d0069a2dbb2364bddb00264db4b800e7f18a7e7e4eb8f3ecab2ff1c quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 2024-03-19T01:31:14.965 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:14 smithi099 podman[62932]: 2024-03-19 01:31:14.867412962 +0000 UTC m=+0.121820058 container create e320e9f5ecea50796d28569f5e924ed9515a79ba91911c669944c318a5d0037c (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mon-a, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240307, GIT_BRANCH=HEAD, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30) 2024-03-19T01:31:15.124 INFO:teuthology.orchestra.run.smithi099.stdout:Setting public_network to 172.21.0.0/20 in mon config section 2024-03-19T01:31:15.238 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 podman[62932]: 2024-03-19 01:31:15.044727927 +0000 UTC m=+0.299135031 container init e320e9f5ecea50796d28569f5e924ed9515a79ba91911c669944c318a5d0037c (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mon-a, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.build-date=20240307, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-19T01:31:15.238 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 podman[62932]: 2024-03-19 01:31:15.04761438 +0000 UTC m=+0.302021467 container start e320e9f5ecea50796d28569f5e924ed9515a79ba91911c669944c318a5d0037c (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mon-a, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240307, ceph=True, GIT_CLEAN=True) 2024-03-19T01:31:15.238 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: set uid:gid to 167:167 (ceph:ceph) 2024-03-19T01:31:15.238 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: ceph version 19.0.0-2206-gb6ae2491 (b6ae24918b03b1b8a19c9263a50655bba80bb810) squid (dev), process ceph-mon, pid 2 2024-03-19T01:31:15.238 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: pidfile_write: ignore empty --pid-file 2024-03-19T01:31:15.238 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: load: jerasure load: lrc 2024-03-19T01:31:15.238 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: RocksDB version: 7.9.2 2024-03-19T01:31:15.238 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Git sha 0 2024-03-19T01:31:15.238 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Compile date 2024-03-18 18:57:12 2024-03-19T01:31:15.238 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: DB SUMMARY 2024-03-19T01:31:15.239 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: DB Session ID: 21BQIHF56YJLPKTBAAPE 2024-03-19T01:31:15.239 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: CURRENT file: CURRENT 2024-03-19T01:31:15.239 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: IDENTITY file: IDENTITY 2024-03-19T01:31:15.239 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: MANIFEST file: MANIFEST-000010 size: 179 Bytes 2024-03-19T01:31:15.239 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-03-19T01:31:15.239 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000009.log size: 105246 ; 2024-03-19T01:31:15.239 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.error_if_exists: 0 2024-03-19T01:31:15.239 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.create_if_missing: 0 2024-03-19T01:31:15.239 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.paranoid_checks: 1 2024-03-19T01:31:15.239 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.flush_verify_memtable_count: 1 2024-03-19T01:31:15.239 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-03-19T01:31:15.239 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-03-19T01:31:15.239 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.env: 0x55cee1a41c60 2024-03-19T01:31:15.239 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.fs: PosixFileSystem 2024-03-19T01:31:15.240 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.info_log: 0x55cee2ab7ae0 2024-03-19T01:31:15.240 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.max_file_opening_threads: 16 2024-03-19T01:31:15.240 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.statistics: (nil) 2024-03-19T01:31:15.240 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.use_fsync: 0 2024-03-19T01:31:15.240 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.max_log_file_size: 0 2024-03-19T01:31:15.240 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-03-19T01:31:15.240 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.log_file_time_to_roll: 0 2024-03-19T01:31:15.240 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.keep_log_file_num: 1000 2024-03-19T01:31:15.240 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.recycle_log_file_num: 0 2024-03-19T01:31:15.240 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.allow_fallocate: 1 2024-03-19T01:31:15.240 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.allow_mmap_reads: 0 2024-03-19T01:31:15.240 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.allow_mmap_writes: 0 2024-03-19T01:31:15.240 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.use_direct_reads: 0 2024-03-19T01:31:15.241 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-03-19T01:31:15.241 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.create_missing_column_families: 0 2024-03-19T01:31:15.241 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.db_log_dir: 2024-03-19T01:31:15.241 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.wal_dir: 2024-03-19T01:31:15.241 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.table_cache_numshardbits: 6 2024-03-19T01:31:15.241 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.WAL_ttl_seconds: 0 2024-03-19T01:31:15.241 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.WAL_size_limit_MB: 0 2024-03-19T01:31:15.241 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-03-19T01:31:15.241 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-03-19T01:31:15.241 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.is_fd_close_on_exec: 1 2024-03-19T01:31:15.241 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.advise_random_on_open: 1 2024-03-19T01:31:15.241 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.db_write_buffer_size: 0 2024-03-19T01:31:15.241 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.write_buffer_manager: 0x55cee2a8a1e0 2024-03-19T01:31:15.241 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-03-19T01:31:15.241 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-03-19T01:31:15.242 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.use_adaptive_mutex: 0 2024-03-19T01:31:15.242 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.rate_limiter: (nil) 2024-03-19T01:31:15.242 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-03-19T01:31:15.242 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.wal_recovery_mode: 2 2024-03-19T01:31:15.242 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.enable_thread_tracking: 0 2024-03-19T01:31:15.242 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.enable_pipelined_write: 0 2024-03-19T01:31:15.242 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.unordered_write: 0 2024-03-19T01:31:15.242 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-03-19T01:31:15.242 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-03-19T01:31:15.242 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-03-19T01:31:15.242 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-03-19T01:31:15.242 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.row_cache: None 2024-03-19T01:31:15.242 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.wal_filter: None 2024-03-19T01:31:15.244 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-03-19T01:31:15.244 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.allow_ingest_behind: 0 2024-03-19T01:31:15.244 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.two_write_queues: 0 2024-03-19T01:31:15.244 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.manual_wal_flush: 0 2024-03-19T01:31:15.244 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.wal_compression: 0 2024-03-19T01:31:15.244 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.atomic_flush: 0 2024-03-19T01:31:15.244 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-03-19T01:31:15.244 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.persist_stats_to_disk: 0 2024-03-19T01:31:15.244 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.write_dbid_to_manifest: 0 2024-03-19T01:31:15.245 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.log_readahead_size: 0 2024-03-19T01:31:15.245 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-03-19T01:31:15.245 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.best_efforts_recovery: 0 2024-03-19T01:31:15.245 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-03-19T01:31:15.245 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-03-19T01:31:15.245 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.allow_data_in_errors: 0 2024-03-19T01:31:15.245 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.db_host_id: __hostname__ 2024-03-19T01:31:15.245 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.enforce_single_del_contracts: true 2024-03-19T01:31:15.245 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.max_background_jobs: 2 2024-03-19T01:31:15.245 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.max_background_compactions: -1 2024-03-19T01:31:15.245 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.max_subcompactions: 1 2024-03-19T01:31:15.245 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-03-19T01:31:15.245 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-03-19T01:31:15.245 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.delayed_write_rate : 16777216 2024-03-19T01:31:15.246 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.max_total_wal_size: 0 2024-03-19T01:31:15.246 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-03-19T01:31:15.246 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.stats_dump_period_sec: 600 2024-03-19T01:31:15.246 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.stats_persist_period_sec: 600 2024-03-19T01:31:15.246 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-03-19T01:31:15.246 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.max_open_files: -1 2024-03-19T01:31:15.246 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.bytes_per_sync: 0 2024-03-19T01:31:15.246 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.wal_bytes_per_sync: 0 2024-03-19T01:31:15.246 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.strict_bytes_per_sync: 0 2024-03-19T01:31:15.246 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.compaction_readahead_size: 0 2024-03-19T01:31:15.246 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.max_background_flushes: -1 2024-03-19T01:31:15.246 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Compression algorithms supported: 2024-03-19T01:31:15.246 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-03-19T01:31:15.246 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: kZSTD supported: 0 2024-03-19T01:31:15.247 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: kXpressCompression supported: 0 2024-03-19T01:31:15.247 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: kLZ4HCCompression supported: 1 2024-03-19T01:31:15.247 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: kZlibCompression supported: 1 2024-03-19T01:31:15.247 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: kSnappyCompression supported: 1 2024-03-19T01:31:15.247 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: kLZ4Compression supported: 1 2024-03-19T01:31:15.247 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: kBZip2Compression supported: 0 2024-03-19T01:31:15.247 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Fast CRC32 supported: Supported on x86 2024-03-19T01:31:15.247 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: DMutex implementation: pthread_mutex_t 2024-03-19T01:31:15.247 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000010 2024-03-19T01:31:15.247 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-03-19T01:31:15.247 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-03-19T01:31:15.247 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.merge_operator: 2024-03-19T01:31:15.247 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.compaction_filter: None 2024-03-19T01:31:15.247 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.compaction_filter_factory: None 2024-03-19T01:31:15.248 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.sst_partitioner_factory: None 2024-03-19T01:31:15.248 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.memtable_factory: SkipListFactory 2024-03-19T01:31:15.248 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.table_factory: BlockBasedTable 2024-03-19T01:31:15.248 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55cee2ab7be0) 2024-03-19T01:31:15.248 INFO:journalctl@ceph.mon.a.smithi099.stdout: cache_index_and_filter_blocks: 1 2024-03-19T01:31:15.248 INFO:journalctl@ceph.mon.a.smithi099.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-03-19T01:31:15.248 INFO:journalctl@ceph.mon.a.smithi099.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-03-19T01:31:15.248 INFO:journalctl@ceph.mon.a.smithi099.stdout: pin_top_level_index_and_filter: 1 2024-03-19T01:31:15.248 INFO:journalctl@ceph.mon.a.smithi099.stdout: index_type: 0 2024-03-19T01:31:15.248 INFO:journalctl@ceph.mon.a.smithi099.stdout: data_block_index_type: 0 2024-03-19T01:31:15.248 INFO:journalctl@ceph.mon.a.smithi099.stdout: index_shortening: 1 2024-03-19T01:31:15.248 INFO:journalctl@ceph.mon.a.smithi099.stdout: data_block_hash_table_util_ratio: 0.750000 2024-03-19T01:31:15.248 INFO:journalctl@ceph.mon.a.smithi099.stdout: checksum: 4 2024-03-19T01:31:15.248 INFO:journalctl@ceph.mon.a.smithi099.stdout: no_block_cache: 0 2024-03-19T01:31:15.248 INFO:journalctl@ceph.mon.a.smithi099.stdout: block_cache: 0x55cee2abd090 2024-03-19T01:31:15.249 INFO:journalctl@ceph.mon.a.smithi099.stdout: block_cache_name: BinnedLRUCache 2024-03-19T01:31:15.249 INFO:journalctl@ceph.mon.a.smithi099.stdout: block_cache_options: 2024-03-19T01:31:15.249 INFO:journalctl@ceph.mon.a.smithi099.stdout: capacity : 536870912 2024-03-19T01:31:15.249 INFO:journalctl@ceph.mon.a.smithi099.stdout: num_shard_bits : 4 2024-03-19T01:31:15.249 INFO:journalctl@ceph.mon.a.smithi099.stdout: strict_capacity_limit : 0 2024-03-19T01:31:15.249 INFO:journalctl@ceph.mon.a.smithi099.stdout: high_pri_pool_ratio: 0.000 2024-03-19T01:31:15.249 INFO:journalctl@ceph.mon.a.smithi099.stdout: block_cache_compressed: (nil) 2024-03-19T01:31:15.249 INFO:journalctl@ceph.mon.a.smithi099.stdout: persistent_cache: (nil) 2024-03-19T01:31:15.249 INFO:journalctl@ceph.mon.a.smithi099.stdout: block_size: 4096 2024-03-19T01:31:15.249 INFO:journalctl@ceph.mon.a.smithi099.stdout: block_size_deviation: 10 2024-03-19T01:31:15.249 INFO:journalctl@ceph.mon.a.smithi099.stdout: block_restart_interval: 16 2024-03-19T01:31:15.249 INFO:journalctl@ceph.mon.a.smithi099.stdout: index_block_restart_interval: 1 2024-03-19T01:31:15.249 INFO:journalctl@ceph.mon.a.smithi099.stdout: metadata_block_size: 4096 2024-03-19T01:31:15.249 INFO:journalctl@ceph.mon.a.smithi099.stdout: partition_filters: 0 2024-03-19T01:31:15.250 INFO:journalctl@ceph.mon.a.smithi099.stdout: use_delta_encoding: 1 2024-03-19T01:31:15.250 INFO:journalctl@ceph.mon.a.smithi099.stdout: filter_policy: bloomfilter 2024-03-19T01:31:15.250 INFO:journalctl@ceph.mon.a.smithi099.stdout: whole_key_filtering: 1 2024-03-19T01:31:15.250 INFO:journalctl@ceph.mon.a.smithi099.stdout: verify_compression: 0 2024-03-19T01:31:15.250 INFO:journalctl@ceph.mon.a.smithi099.stdout: read_amp_bytes_per_bit: 0 2024-03-19T01:31:15.250 INFO:journalctl@ceph.mon.a.smithi099.stdout: format_version: 5 2024-03-19T01:31:15.250 INFO:journalctl@ceph.mon.a.smithi099.stdout: enable_index_compression: 1 2024-03-19T01:31:15.250 INFO:journalctl@ceph.mon.a.smithi099.stdout: block_align: 0 2024-03-19T01:31:15.250 INFO:journalctl@ceph.mon.a.smithi099.stdout: max_auto_readahead_size: 262144 2024-03-19T01:31:15.250 INFO:journalctl@ceph.mon.a.smithi099.stdout: prepopulate_block_cache: 0 2024-03-19T01:31:15.250 INFO:journalctl@ceph.mon.a.smithi099.stdout: initial_auto_readahead_size: 8192 2024-03-19T01:31:15.250 INFO:journalctl@ceph.mon.a.smithi099.stdout: num_file_reads_for_auto_readahead: 2 2024-03-19T01:31:15.250 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.write_buffer_size: 33554432 2024-03-19T01:31:15.250 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.max_write_buffer_number: 2 2024-03-19T01:31:15.251 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.compression: NoCompression 2024-03-19T01:31:15.251 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.bottommost_compression: Disabled 2024-03-19T01:31:15.251 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.prefix_extractor: nullptr 2024-03-19T01:31:15.251 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-03-19T01:31:15.251 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.num_levels: 7 2024-03-19T01:31:15.251 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-03-19T01:31:15.251 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-03-19T01:31:15.251 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-03-19T01:31:15.251 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-03-19T01:31:15.251 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-03-19T01:31:15.251 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-03-19T01:31:15.251 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-03-19T01:31:15.251 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-03-19T01:31:15.251 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-03-19T01:31:15.251 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-03-19T01:31:15.252 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-03-19T01:31:15.252 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-03-19T01:31:15.252 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.compression_opts.window_bits: -14 2024-03-19T01:31:15.252 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.compression_opts.level: 32767 2024-03-19T01:31:15.252 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.compression_opts.strategy: 0 2024-03-19T01:31:15.252 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-03-19T01:31:15.252 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-03-19T01:31:15.252 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-03-19T01:31:15.252 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-03-19T01:31:15.252 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.compression_opts.enabled: false 2024-03-19T01:31:15.252 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-03-19T01:31:15.252 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-03-19T01:31:15.252 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-03-19T01:31:15.252 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-03-19T01:31:15.253 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.target_file_size_base: 67108864 2024-03-19T01:31:15.253 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.target_file_size_multiplier: 1 2024-03-19T01:31:15.253 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-03-19T01:31:15.253 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-03-19T01:31:15.253 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-03-19T01:31:15.253 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-03-19T01:31:15.253 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-03-19T01:31:15.253 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-03-19T01:31:15.253 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-03-19T01:31:15.253 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-03-19T01:31:15.253 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-03-19T01:31:15.253 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-03-19T01:31:15.253 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-03-19T01:31:15.253 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-03-19T01:31:15.253 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-03-19T01:31:15.254 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.arena_block_size: 1048576 2024-03-19T01:31:15.254 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-03-19T01:31:15.254 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-03-19T01:31:15.254 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.disable_auto_compactions: 0 2024-03-19T01:31:15.254 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-03-19T01:31:15.254 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-03-19T01:31:15.254 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-03-19T01:31:15.254 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-03-19T01:31:15.254 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-03-19T01:31:15.254 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-03-19T01:31:15.254 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-03-19T01:31:15.254 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-03-19T01:31:15.254 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-03-19T01:31:15.254 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-03-19T01:31:15.255 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.table_properties_collectors: 2024-03-19T01:31:15.255 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.inplace_update_support: 0 2024-03-19T01:31:15.255 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.inplace_update_num_locks: 10000 2024-03-19T01:31:15.255 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-03-19T01:31:15.255 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-03-19T01:31:15.255 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.memtable_huge_page_size: 0 2024-03-19T01:31:15.255 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.bloom_locality: 0 2024-03-19T01:31:15.255 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.max_successive_merges: 0 2024-03-19T01:31:15.255 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.optimize_filters_for_hits: 0 2024-03-19T01:31:15.255 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.paranoid_file_checks: 0 2024-03-19T01:31:15.255 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.force_consistency_checks: 1 2024-03-19T01:31:15.255 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.report_bg_io_stats: 0 2024-03-19T01:31:15.255 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.ttl: 2592000 2024-03-19T01:31:15.255 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.periodic_compaction_seconds: 0 2024-03-19T01:31:15.256 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-03-19T01:31:15.256 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-03-19T01:31:15.256 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.enable_blob_files: false 2024-03-19T01:31:15.256 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.min_blob_size: 0 2024-03-19T01:31:15.256 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.blob_file_size: 268435456 2024-03-19T01:31:15.256 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.blob_compression_type: NoCompression 2024-03-19T01:31:15.256 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.enable_blob_garbage_collection: false 2024-03-19T01:31:15.256 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-03-19T01:31:15.256 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-03-19T01:31:15.256 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-03-19T01:31:15.256 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.blob_file_starting_level: 0 2024-03-19T01:31:15.256 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-03-19T01:31:15.256 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000010 succeeded,manifest_file_number is 10, next_file_number is 12, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 5 2024-03-19T01:31:15.256 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 5 2024-03-19T01:31:15.257 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: c9c17e57-96f3-4398-8c01-74686ecb6793 2024-03-19T01:31:15.257 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: EVENT_LOG_v1 {"time_micros": 1710811875076430, "job": 1, "event": "recovery_started", "wal_files": [9]} 2024-03-19T01:31:15.257 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #9 mode 2 2024-03-19T01:31:15.257 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: EVENT_LOG_v1 {"time_micros": 1710811875077445, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 13, "file_size": 100360, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 8, "largest_seqno": 326, "table_properties": {"data_size": 98271, "index_size": 307, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 773, "raw_key_size": 14211, "raw_average_key_size": 49, "raw_value_size": 90540, "raw_average_value_size": 317, "num_data_blocks": 12, "num_entries": 285, "num_filter_entries": 285, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1710811875, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "c9c17e57-96f3-4398-8c01-74686ecb6793", "db_session_id": "21BQIHF56YJLPKTBAAPE", "orig_file_number": 13, "seqno_to_time_mapping": "N/A"}} 2024-03-19T01:31:15.257 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: EVENT_LOG_v1 {"time_micros": 1710811875077526, "job": 1, "event": "recovery_finished"} 2024-03-19T01:31:15.257 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: [db/version_set.cc:5047] Creating manifest 15 2024-03-19T01:31:15.257 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000009.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-03-19T01:31:15.257 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55cee2bb0000 2024-03-19T01:31:15.257 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: DB pointer 0x55cee2b9a000 2024-03-19T01:31:15.257 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-03-19T01:31:15.257 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-03-19T01:31:15.257 INFO:journalctl@ceph.mon.a.smithi099.stdout: ** DB Stats ** 2024-03-19T01:31:15.257 INFO:journalctl@ceph.mon.a.smithi099.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-19T01:31:15.257 INFO:journalctl@ceph.mon.a.smithi099.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-03-19T01:31:15.258 INFO:journalctl@ceph.mon.a.smithi099.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-19T01:31:15.258 INFO:journalctl@ceph.mon.a.smithi099.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-19T01:31:15.258 INFO:journalctl@ceph.mon.a.smithi099.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-03-19T01:31:15.258 INFO:journalctl@ceph.mon.a.smithi099.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-19T01:31:15.258 INFO:journalctl@ceph.mon.a.smithi099.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-19T01:31:15.258 INFO:journalctl@ceph.mon.a.smithi099.stdout: 2024-03-19T01:31:15.258 INFO:journalctl@ceph.mon.a.smithi099.stdout: ** Compaction Stats [default] ** 2024-03-19T01:31:15.258 INFO:journalctl@ceph.mon.a.smithi099.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-19T01:31:15.258 INFO:journalctl@ceph.mon.a.smithi099.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-03-19T01:31:15.258 INFO:journalctl@ceph.mon.a.smithi099.stdout: L0 2/0 99.84 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 143.3 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-03-19T01:31:15.258 INFO:journalctl@ceph.mon.a.smithi099.stdout: Sum 2/0 99.84 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 143.3 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-03-19T01:31:15.258 INFO:journalctl@ceph.mon.a.smithi099.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 143.3 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-03-19T01:31:15.258 INFO:journalctl@ceph.mon.a.smithi099.stdout: 2024-03-19T01:31:15.258 INFO:journalctl@ceph.mon.a.smithi099.stdout: ** Compaction Stats [default] ** 2024-03-19T01:31:15.259 INFO:journalctl@ceph.mon.a.smithi099.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-19T01:31:15.259 INFO:journalctl@ceph.mon.a.smithi099.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-19T01:31:15.259 INFO:journalctl@ceph.mon.a.smithi099.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 143.3 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-03-19T01:31:15.259 INFO:journalctl@ceph.mon.a.smithi099.stdout: 2024-03-19T01:31:15.259 INFO:journalctl@ceph.mon.a.smithi099.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-03-19T01:31:15.259 INFO:journalctl@ceph.mon.a.smithi099.stdout: 2024-03-19T01:31:15.259 INFO:journalctl@ceph.mon.a.smithi099.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-19T01:31:15.259 INFO:journalctl@ceph.mon.a.smithi099.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-03-19T01:31:15.259 INFO:journalctl@ceph.mon.a.smithi099.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-03-19T01:31:15.259 INFO:journalctl@ceph.mon.a.smithi099.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-03-19T01:31:15.259 INFO:journalctl@ceph.mon.a.smithi099.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-03-19T01:31:15.259 INFO:journalctl@ceph.mon.a.smithi099.stdout: AddFile(Keys): cumulative 0, interval 0 2024-03-19T01:31:15.259 INFO:journalctl@ceph.mon.a.smithi099.stdout: Cumulative compaction: 0.00 GB write, 32.89 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-19T01:31:15.259 INFO:journalctl@ceph.mon.a.smithi099.stdout: Interval compaction: 0.00 GB write, 32.89 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-19T01:31:15.260 INFO:journalctl@ceph.mon.a.smithi099.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-03-19T01:31:15.260 INFO:journalctl@ceph.mon.a.smithi099.stdout: Block cache BinnedLRUCache@0x55cee2abd090#2 capacity: 512.00 MB usage: 1.52 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 8e-06 secs_since: 0 2024-03-19T01:31:15.260 INFO:journalctl@ceph.mon.a.smithi099.stdout: Block cache entry stats(count,size,portion): FilterBlock(2,1.02 KB,0.000193715%) IndexBlock(2,0.50 KB,9.53674e-05%) Misc(1,0.00 KB,0%) 2024-03-19T01:31:15.260 INFO:journalctl@ceph.mon.a.smithi099.stdout: 2024-03-19T01:31:15.260 INFO:journalctl@ceph.mon.a.smithi099.stdout: ** File Read Latency Histogram By Level [default] ** 2024-03-19T01:31:15.260 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: starting mon.a rank 0 at public addrs [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] at bind addrs [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 438fb69c-e590-11ee-95c9-87774f69a715 2024-03-19T01:31:15.260 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: mon.a@-1(???) e1 preinit fsid 438fb69c-e590-11ee-95c9-87774f69a715 2024-03-19T01:31:15.260 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: mon.a@-1(???).mds e1 new map 2024-03-19T01:31:15.260 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: mon.a@-1(???).mds e1 print_map 2024-03-19T01:31:15.260 INFO:journalctl@ceph.mon.a.smithi099.stdout: e1 2024-03-19T01:31:15.260 INFO:journalctl@ceph.mon.a.smithi099.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-03-19T01:31:15.260 INFO:journalctl@ceph.mon.a.smithi099.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2,11=minor log segments,12=quiesce subvolumes} 2024-03-19T01:31:15.260 INFO:journalctl@ceph.mon.a.smithi099.stdout: legacy client fscid: -1 2024-03-19T01:31:15.260 INFO:journalctl@ceph.mon.a.smithi099.stdout: 2024-03-19T01:31:15.260 INFO:journalctl@ceph.mon.a.smithi099.stdout: No filesystems configured 2024-03-19T01:31:15.261 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-03-19T01:31:15.261 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-03-19T01:31:15.261 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-03-19T01:31:15.261 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-03-19T01:31:15.261 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-03-19T01:31:15.261 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 bash[62932]: e320e9f5ecea50796d28569f5e924ed9515a79ba91911c669944c318a5d0037c 2024-03-19T01:31:15.261 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 systemd[1]: Started Ceph mon.a for 438fb69c-e590-11ee-95c9-87774f69a715. 2024-03-19T01:31:15.261 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: mon.a is new leader, mons a in quorum (ranks 0) 2024-03-19T01:31:15.261 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: monmap epoch 1 2024-03-19T01:31:15.261 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: fsid 438fb69c-e590-11ee-95c9-87774f69a715 2024-03-19T01:31:15.261 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: last_changed 2024-03-19T01:31:07.215663+0000 2024-03-19T01:31:15.261 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: created 2024-03-19T01:31:07.215663+0000 2024-03-19T01:31:15.261 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: min_mon_release 19 (squid) 2024-03-19T01:31:15.261 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: election_strategy: 1 2024-03-19T01:31:15.262 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: 0: [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] mon.a 2024-03-19T01:31:15.262 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: fsmap 2024-03-19T01:31:15.262 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: osdmap e1: 0 total, 0 up, 0 in 2024-03-19T01:31:15.262 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:15 smithi099 ceph-mon[62951]: mgrmap e1: no daemons active 2024-03-19T01:31:16.183 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:15.565+0000 7fbf0c0e4700 1 Processor -- start 2024-03-19T01:31:16.184 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:15.565+0000 7fbf0c0e4700 1 -- start start 2024-03-19T01:31:16.184 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:15.566+0000 7fbf0c0e4700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbf0407dec0 0x7fbf0407c320 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:31:16.184 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:15.566+0000 7fbf0c0e4700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fbf0407c8f0 con 0x7fbf0407dec0 2024-03-19T01:31:16.184 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:15.566+0000 7fbf09e80700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbf0407dec0 0x7fbf0407c320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:31:16.184 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:15.566+0000 7fbf09e80700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbf0407dec0 0x7fbf0407c320 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:40118/0 (socket says 172.21.15.99:40118) 2024-03-19T01:31:16.184 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:15.566+0000 7fbf09e80700 1 -- 172.21.15.99:0/1200670592 learned_addr learned my addr 172.21.15.99:0/1200670592 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:31:16.184 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:15.566+0000 7fbf09e80700 1 -- 172.21.15.99:0/1200670592 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fbf0407ca70 con 0x7fbf0407dec0 2024-03-19T01:31:16.184 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:15.566+0000 7fbf09e80700 1 --2- 172.21.15.99:0/1200670592 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbf0407dec0 0x7fbf0407c320 secure :-1 s=READY pgs=1 cs=0 l=1 rev1=1 crypto rx=0x7fbef40098a0 tx=0x7fbef4009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9269284722045b3f server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:31:16.184 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:15.566+0000 7fbf08e7e700 1 -- 172.21.15.99:0/1200670592 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fbef4004160 con 0x7fbf0407dec0 2024-03-19T01:31:16.184 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:15.566+0000 7fbf08e7e700 1 -- 172.21.15.99:0/1200670592 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7fbef4004300 con 0x7fbf0407dec0 2024-03-19T01:31:16.184 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:15.566+0000 7fbf08e7e700 1 -- 172.21.15.99:0/1200670592 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fbef4011b50 con 0x7fbf0407dec0 2024-03-19T01:31:16.184 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:15.567+0000 7fbf0c0e4700 1 -- 172.21.15.99:0/1200670592 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbf0407dec0 msgr2=0x7fbf0407c320 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:31:16.184 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:15.567+0000 7fbf0c0e4700 1 --2- 172.21.15.99:0/1200670592 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbf0407dec0 0x7fbf0407c320 secure :-1 s=READY pgs=1 cs=0 l=1 rev1=1 crypto rx=0x7fbef40098a0 tx=0x7fbef4009bb0 comp rx=0 tx=0).stop 2024-03-19T01:31:16.185 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:15.567+0000 7fbf0c0e4700 1 -- 172.21.15.99:0/1200670592 shutdown_connections 2024-03-19T01:31:16.185 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:15.567+0000 7fbf0c0e4700 1 --2- 172.21.15.99:0/1200670592 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbf0407dec0 0x7fbf0407c320 unknown :-1 s=CLOSED pgs=1 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:16.185 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:15.567+0000 7fbf0c0e4700 1 -- 172.21.15.99:0/1200670592 >> 172.21.15.99:0/1200670592 conn(0x7fbf041033a0 msgr2=0x7fbf041057c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:31:16.185 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:15.567+0000 7fbf0c0e4700 1 -- 172.21.15.99:0/1200670592 shutdown_connections 2024-03-19T01:31:16.185 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:15.567+0000 7fbf0c0e4700 1 -- 172.21.15.99:0/1200670592 wait complete. 2024-03-19T01:31:16.185 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:15.568+0000 7fbf0c0e4700 1 Processor -- start 2024-03-19T01:31:16.185 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:15.568+0000 7fbf0c0e4700 1 -- start start 2024-03-19T01:31:16.185 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:15.568+0000 7fbf0c0e4700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbf0407dec0 0x7fbf0419dfa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:31:16.185 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:15.568+0000 7fbf0c0e4700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fbf0410e390 con 0x7fbf0407dec0 2024-03-19T01:31:16.185 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:15.568+0000 7fbf09e80700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbf0407dec0 0x7fbf0419dfa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:31:16.185 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:15.568+0000 7fbf09e80700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbf0407dec0 0x7fbf0419dfa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:40134/0 (socket says 172.21.15.99:40134) 2024-03-19T01:31:16.185 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:15.568+0000 7fbf09e80700 1 -- 172.21.15.99:0/2018176743 learned_addr learned my addr 172.21.15.99:0/2018176743 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:31:16.185 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:15.568+0000 7fbf09e80700 1 -- 172.21.15.99:0/2018176743 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fbf0419e4e0 con 0x7fbf0407dec0 2024-03-19T01:31:16.185 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:15.568+0000 7fbf09e80700 1 --2- 172.21.15.99:0/2018176743 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbf0407dec0 0x7fbf0419dfa0 secure :-1 s=READY pgs=2 cs=0 l=1 rev1=1 crypto rx=0x7fbef4011670 tx=0x7fbef4011b60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:31:16.186 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:15.568+0000 7fbefaffd700 1 -- 172.21.15.99:0/2018176743 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fbef40039c0 con 0x7fbf0407dec0 2024-03-19T01:31:16.186 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:15.568+0000 7fbefaffd700 1 -- 172.21.15.99:0/2018176743 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7fbef4011e50 con 0x7fbf0407dec0 2024-03-19T01:31:16.186 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:15.568+0000 7fbf0c0e4700 1 -- 172.21.15.99:0/2018176743 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fbf0419e770 con 0x7fbf0407dec0 2024-03-19T01:31:16.186 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:15.569+0000 7fbefaffd700 1 -- 172.21.15.99:0/2018176743 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fbef401a5c0 con 0x7fbf0407dec0 2024-03-19T01:31:16.186 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:15.569+0000 7fbf0c0e4700 1 -- 172.21.15.99:0/2018176743 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fbf041a1460 con 0x7fbf0407dec0 2024-03-19T01:31:16.186 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:15.569+0000 7fbefaffd700 1 -- 172.21.15.99:0/2018176743 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7fbef401ac50 con 0x7fbf0407dec0 2024-03-19T01:31:16.186 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:15.569+0000 7fbefaffd700 1 -- 172.21.15.99:0/2018176743 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7fbef4019700 con 0x7fbf0407dec0 2024-03-19T01:31:16.186 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:15.569+0000 7fbf0c0e4700 1 -- 172.21.15.99:0/2018176743 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fbf04063940 con 0x7fbf0407dec0 2024-03-19T01:31:16.186 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:15.572+0000 7fbefaffd700 1 -- 172.21.15.99:0/2018176743 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+73141 (secure 0 0 0) 0x7fbef4003b60 con 0x7fbf0407dec0 2024-03-19T01:31:16.186 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:15.638+0000 7fbf0c0e4700 1 -- 172.21.15.99:0/2018176743 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command([{prefix=config set, name=public_network}] v 0) -- 0x7fbf041a1630 con 0x7fbf0407dec0 2024-03-19T01:31:16.186 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:15.640+0000 7fbefaffd700 1 -- 172.21.15.99:0/2018176743 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{prefix=config set, name=public_network}]=0 v3)=0 v3) ==== 124+0+0 (secure 0 0 0) 0x7fbef4019900 con 0x7fbf0407dec0 2024-03-19T01:31:16.186 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:15.640+0000 7fbf0c0e4700 1 -- 172.21.15.99:0/2018176743 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbf0407dec0 msgr2=0x7fbf0419dfa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:31:16.186 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:15.640+0000 7fbf0c0e4700 1 --2- 172.21.15.99:0/2018176743 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbf0407dec0 0x7fbf0419dfa0 secure :-1 s=READY pgs=2 cs=0 l=1 rev1=1 crypto rx=0x7fbef4011670 tx=0x7fbef4011b60 comp rx=0 tx=0).stop 2024-03-19T01:31:16.187 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:15.641+0000 7fbf0c0e4700 1 -- 172.21.15.99:0/2018176743 shutdown_connections 2024-03-19T01:31:16.187 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:15.641+0000 7fbf0c0e4700 1 --2- 172.21.15.99:0/2018176743 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbf0407dec0 0x7fbf0419dfa0 unknown :-1 s=CLOSED pgs=2 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:16.187 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:15.641+0000 7fbf0c0e4700 1 -- 172.21.15.99:0/2018176743 >> 172.21.15.99:0/2018176743 conn(0x7fbf041033a0 msgr2=0x7fbf04103f50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:31:16.187 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:15.641+0000 7fbf0c0e4700 1 -- 172.21.15.99:0/2018176743 shutdown_connections 2024-03-19T01:31:16.187 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:15.641+0000 7fbf0c0e4700 1 -- 172.21.15.99:0/2018176743 wait complete. 2024-03-19T01:31:16.187 INFO:teuthology.orchestra.run.smithi099.stdout:Wrote config to /etc/ceph/ceph.conf 2024-03-19T01:31:16.201 INFO:teuthology.orchestra.run.smithi099.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-03-19T01:31:16.201 INFO:teuthology.orchestra.run.smithi099.stdout:Creating mgr... 2024-03-19T01:31:16.202 INFO:teuthology.orchestra.run.smithi099.stdout:Verifying port 0.0.0.0:9283 ... 2024-03-19T01:31:16.202 INFO:teuthology.orchestra.run.smithi099.stdout:Verifying port 0.0.0.0:8765 ... 2024-03-19T01:31:16.451 INFO:teuthology.orchestra.run.smithi099.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-438fb69c-e590-11ee-95c9-87774f69a715@mgr.x 2024-03-19T01:31:16.451 INFO:teuthology.orchestra.run.smithi099.stdout:systemctl: stderr Failed to reset failed state of unit ceph-438fb69c-e590-11ee-95c9-87774f69a715@mgr.x.service: Unit ceph-438fb69c-e590-11ee-95c9-87774f69a715@mgr.x.service not loaded. 2024-03-19T01:31:16.601 INFO:teuthology.orchestra.run.smithi099.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-438fb69c-e590-11ee-95c9-87774f69a715.target.wants/ceph-438fb69c-e590-11ee-95c9-87774f69a715@mgr.x.service → /etc/systemd/system/ceph-438fb69c-e590-11ee-95c9-87774f69a715@.service. 2024-03-19T01:31:16.943 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:16 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/2018176743' entity='client.admin' 2024-03-19T01:31:17.120 INFO:teuthology.orchestra.run.smithi099.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-03-19T01:31:17.120 INFO:teuthology.orchestra.run.smithi099.stdout:systemctl: stdout disabled 2024-03-19T01:31:17.127 INFO:teuthology.orchestra.run.smithi099.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-03-19T01:31:17.127 INFO:teuthology.orchestra.run.smithi099.stdout:systemctl: stdout inactive 2024-03-19T01:31:17.127 INFO:teuthology.orchestra.run.smithi099.stdout:firewalld.service is not enabled 2024-03-19T01:31:17.127 INFO:teuthology.orchestra.run.smithi099.stdout:Not possible to enable service . firewalld.service is not available 2024-03-19T01:31:17.135 INFO:teuthology.orchestra.run.smithi099.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-03-19T01:31:17.135 INFO:teuthology.orchestra.run.smithi099.stdout:systemctl: stdout disabled 2024-03-19T01:31:17.142 INFO:teuthology.orchestra.run.smithi099.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-03-19T01:31:17.142 INFO:teuthology.orchestra.run.smithi099.stdout:systemctl: stdout inactive 2024-03-19T01:31:17.142 INFO:teuthology.orchestra.run.smithi099.stdout:firewalld.service is not enabled 2024-03-19T01:31:17.143 INFO:teuthology.orchestra.run.smithi099.stdout:Not possible to open ports <[9283, 8765]>. firewalld.service is not available 2024-03-19T01:31:17.143 INFO:teuthology.orchestra.run.smithi099.stdout:Waiting for mgr to start... 2024-03-19T01:31:17.143 INFO:teuthology.orchestra.run.smithi099.stdout:Waiting for mgr... 2024-03-19T01:31:17.988 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:17 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/4026830645' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-03-19T01:31:18.278 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout 2024-03-19T01:31:18.278 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout { 2024-03-19T01:31:18.278 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "fsid": "438fb69c-e590-11ee-95c9-87774f69a715", 2024-03-19T01:31:18.278 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "health": { 2024-03-19T01:31:18.278 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-03-19T01:31:18.278 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-03-19T01:31:18.278 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-03-19T01:31:18.279 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout }, 2024-03-19T01:31:18.279 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-03-19T01:31:18.279 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-03-19T01:31:18.279 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout 0 2024-03-19T01:31:18.279 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout ], 2024-03-19T01:31:18.279 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-03-19T01:31:18.279 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "a" 2024-03-19T01:31:18.279 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout ], 2024-03-19T01:31:18.279 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "quorum_age": 2, 2024-03-19T01:31:18.279 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "monmap": { 2024-03-19T01:31:18.279 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-19T01:31:18.279 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "squid", 2024-03-19T01:31:18.279 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-03-19T01:31:18.279 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout }, 2024-03-19T01:31:18.280 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-03-19T01:31:18.280 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-19T01:31:18.280 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-03-19T01:31:18.280 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-03-19T01:31:18.280 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-03-19T01:31:18.289 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-03-19T01:31:18.289 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-03-19T01:31:18.289 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-03-19T01:31:18.289 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout }, 2024-03-19T01:31:18.289 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-03-19T01:31:18.289 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-03-19T01:31:18.289 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-03-19T01:31:18.290 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-03-19T01:31:18.290 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-03-19T01:31:18.290 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-03-19T01:31:18.290 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-03-19T01:31:18.290 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-03-19T01:31:18.290 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-03-19T01:31:18.290 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout }, 2024-03-19T01:31:18.290 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-03-19T01:31:18.290 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-19T01:31:18.290 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-03-19T01:31:18.290 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-03-19T01:31:18.290 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout }, 2024-03-19T01:31:18.290 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-03-19T01:31:18.290 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "available": false, 2024-03-19T01:31:18.291 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-03-19T01:31:18.291 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "modules": [ 2024-03-19T01:31:18.291 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "iostat", 2024-03-19T01:31:18.291 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "nfs", 2024-03-19T01:31:18.291 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "restful" 2024-03-19T01:31:18.291 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout ], 2024-03-19T01:31:18.291 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "services": {} 2024-03-19T01:31:18.291 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout }, 2024-03-19T01:31:18.291 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-03-19T01:31:18.291 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-19T01:31:18.291 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "modified": "2024-03-19T01:31:10.165529+0000", 2024-03-19T01:31:18.291 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "services": {} 2024-03-19T01:31:18.291 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout }, 2024-03-19T01:31:18.291 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-03-19T01:31:18.292 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout } 2024-03-19T01:31:18.292 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:17.627+0000 7fe1f3fc3700 1 Processor -- start 2024-03-19T01:31:18.292 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:17.627+0000 7fe1f3fc3700 1 -- start start 2024-03-19T01:31:18.292 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:17.627+0000 7fe1f3fc3700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe1ec108860 0x7fe1ec10ac50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:31:18.292 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:17.627+0000 7fe1f3fc3700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fe1ec07be70 con 0x7fe1ec108860 2024-03-19T01:31:18.292 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:17.627+0000 7fe1f1d5f700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe1ec108860 0x7fe1ec10ac50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:31:18.292 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:17.628+0000 7fe1f1d5f700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe1ec108860 0x7fe1ec10ac50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:40168/0 (socket says 172.21.15.99:40168) 2024-03-19T01:31:18.292 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:17.628+0000 7fe1f1d5f700 1 -- 172.21.15.99:0/3991811243 learned_addr learned my addr 172.21.15.99:0/3991811243 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:31:18.292 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:17.628+0000 7fe1f1d5f700 1 -- 172.21.15.99:0/3991811243 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fe1ec10b190 con 0x7fe1ec108860 2024-03-19T01:31:18.292 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:17.628+0000 7fe1f1d5f700 1 --2- 172.21.15.99:0/3991811243 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe1ec108860 0x7fe1ec10ac50 secure :-1 s=READY pgs=5 cs=0 l=1 rev1=1 crypto rx=0x7fe1e80098a0 tx=0x7fe1e8009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=813e0fda9e0fb4c6 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:31:18.292 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:17.628+0000 7fe1f0d5d700 1 -- 172.21.15.99:0/3991811243 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fe1e8004160 con 0x7fe1ec108860 2024-03-19T01:31:18.292 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:17.628+0000 7fe1f0d5d700 1 -- 172.21.15.99:0/3991811243 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7fe1e8004300 con 0x7fe1ec108860 2024-03-19T01:31:18.292 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:17.628+0000 7fe1f0d5d700 1 -- 172.21.15.99:0/3991811243 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fe1e8011b50 con 0x7fe1ec108860 2024-03-19T01:31:18.292 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:17.629+0000 7fe1f3fc3700 1 -- 172.21.15.99:0/3991811243 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe1ec108860 msgr2=0x7fe1ec10ac50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:31:18.293 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:17.629+0000 7fe1f3fc3700 1 --2- 172.21.15.99:0/3991811243 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe1ec108860 0x7fe1ec10ac50 secure :-1 s=READY pgs=5 cs=0 l=1 rev1=1 crypto rx=0x7fe1e80098a0 tx=0x7fe1e8009bb0 comp rx=0 tx=0).stop 2024-03-19T01:31:18.293 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:17.629+0000 7fe1f3fc3700 1 -- 172.21.15.99:0/3991811243 shutdown_connections 2024-03-19T01:31:18.293 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:17.629+0000 7fe1f3fc3700 1 --2- 172.21.15.99:0/3991811243 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe1ec108860 0x7fe1ec10ac50 unknown :-1 s=CLOSED pgs=5 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:18.293 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:17.629+0000 7fe1f3fc3700 1 -- 172.21.15.99:0/3991811243 >> 172.21.15.99:0/3991811243 conn(0x7fe1ec102210 msgr2=0x7fe1ec104670 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:31:18.293 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:17.629+0000 7fe1f3fc3700 1 -- 172.21.15.99:0/3991811243 shutdown_connections 2024-03-19T01:31:18.293 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:17.629+0000 7fe1f3fc3700 1 -- 172.21.15.99:0/3991811243 wait complete. 2024-03-19T01:31:18.293 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:17.630+0000 7fe1f3fc3700 1 Processor -- start 2024-03-19T01:31:18.293 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:17.630+0000 7fe1f3fc3700 1 -- start start 2024-03-19T01:31:18.293 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:17.630+0000 7fe1f3fc3700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe1ec108860 0x7fe1ec199b90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:31:18.293 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:17.630+0000 7fe1f3fc3700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fe1ec107140 con 0x7fe1ec108860 2024-03-19T01:31:18.293 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:17.630+0000 7fe1f1d5f700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe1ec108860 0x7fe1ec199b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:31:18.293 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:17.630+0000 7fe1f1d5f700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe1ec108860 0x7fe1ec199b90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:40178/0 (socket says 172.21.15.99:40178) 2024-03-19T01:31:18.293 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:17.630+0000 7fe1f1d5f700 1 -- 172.21.15.99:0/4026830645 learned_addr learned my addr 172.21.15.99:0/4026830645 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:31:18.293 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:17.630+0000 7fe1f1d5f700 1 -- 172.21.15.99:0/4026830645 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fe1ec19a0d0 con 0x7fe1ec108860 2024-03-19T01:31:18.293 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:17.630+0000 7fe1f1d5f700 1 --2- 172.21.15.99:0/4026830645 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe1ec108860 0x7fe1ec199b90 secure :-1 s=READY pgs=6 cs=0 l=1 rev1=1 crypto rx=0x7fe1e8011b60 tx=0x7fe1e8011c40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:31:18.294 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:17.630+0000 7fe1e2ffd700 1 -- 172.21.15.99:0/4026830645 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fe1e8003970 con 0x7fe1ec108860 2024-03-19T01:31:18.294 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:17.631+0000 7fe1f3fc3700 1 -- 172.21.15.99:0/4026830645 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fe1ec19a360 con 0x7fe1ec108860 2024-03-19T01:31:18.294 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:17.631+0000 7fe1e2ffd700 1 -- 172.21.15.99:0/4026830645 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7fe1e8003b10 con 0x7fe1ec108860 2024-03-19T01:31:18.294 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:17.631+0000 7fe1e2ffd700 1 -- 172.21.15.99:0/4026830645 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fe1e8024610 con 0x7fe1ec108860 2024-03-19T01:31:18.294 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:17.631+0000 7fe1f3fc3700 1 -- 172.21.15.99:0/4026830645 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fe1ec19d050 con 0x7fe1ec108860 2024-03-19T01:31:18.294 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:17.631+0000 7fe1e2ffd700 1 -- 172.21.15.99:0/4026830645 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7fe1e8024c60 con 0x7fe1ec108860 2024-03-19T01:31:18.294 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:17.631+0000 7fe1e2ffd700 1 -- 172.21.15.99:0/4026830645 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7fe1e8011e00 con 0x7fe1ec108860 2024-03-19T01:31:18.294 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:17.632+0000 7fe1f3fc3700 1 -- 172.21.15.99:0/4026830645 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fe1ec069420 con 0x7fe1ec108860 2024-03-19T01:31:18.294 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:17.634+0000 7fe1e2ffd700 1 -- 172.21.15.99:0/4026830645 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+73141 (secure 0 0 0) 0x7fe1e801a9f0 con 0x7fe1ec108860 2024-03-19T01:31:18.294 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:17.706+0000 7fe1f3fc3700 1 -- 172.21.15.99:0/4026830645 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "status", "format": "json-pretty"} v 0) -- 0x7fe1ec02ddb0 con 0x7fe1ec108860 2024-03-19T01:31:18.294 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:17.707+0000 7fe1e2ffd700 1 -- 172.21.15.99:0/4026830645 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "status", "format": "json-pretty"}]=0 v0) ==== 79+0+1239 (secure 0 0 0) 0x7fe1e801a430 con 0x7fe1ec108860 2024-03-19T01:31:18.294 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:17.708+0000 7fe1f3fc3700 1 -- 172.21.15.99:0/4026830645 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe1ec108860 msgr2=0x7fe1ec199b90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:31:18.294 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:17.708+0000 7fe1f3fc3700 1 --2- 172.21.15.99:0/4026830645 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe1ec108860 0x7fe1ec199b90 secure :-1 s=READY pgs=6 cs=0 l=1 rev1=1 crypto rx=0x7fe1e8011b60 tx=0x7fe1e8011c40 comp rx=0 tx=0).stop 2024-03-19T01:31:18.295 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:17.708+0000 7fe1f3fc3700 1 -- 172.21.15.99:0/4026830645 shutdown_connections 2024-03-19T01:31:18.295 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:17.708+0000 7fe1f3fc3700 1 --2- 172.21.15.99:0/4026830645 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe1ec108860 0x7fe1ec199b90 unknown :-1 s=CLOSED pgs=6 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:18.295 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:17.708+0000 7fe1f3fc3700 1 -- 172.21.15.99:0/4026830645 >> 172.21.15.99:0/4026830645 conn(0x7fe1ec102210 msgr2=0x7fe1ec102b00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:31:18.295 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:17.708+0000 7fe1f3fc3700 1 -- 172.21.15.99:0/4026830645 shutdown_connections 2024-03-19T01:31:18.295 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:17.708+0000 7fe1f3fc3700 1 -- 172.21.15.99:0/4026830645 wait complete. 2024-03-19T01:31:18.295 INFO:teuthology.orchestra.run.smithi099.stdout:mgr not available, waiting (1/15)... 2024-03-19T01:31:18.592 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:18 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:31:18.447+0000 7ff1a46e1200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-19T01:31:19.052 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:18 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:31:18.596+0000 7ff1a46e1200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-19T01:31:19.052 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:18 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:31:18.692+0000 7ff1a46e1200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-19T01:31:19.052 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:18 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:31:18.789+0000 7ff1a46e1200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-19T01:31:19.052 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:19 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:31:19.051+0000 7ff1a46e1200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-19T01:31:19.341 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:19 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:31:19.161+0000 7ff1a46e1200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-19T01:31:19.341 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:19 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:31:19.245+0000 7ff1a46e1200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-19T01:31:19.841 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:19 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:31:19.346+0000 7ff1a46e1200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-19T01:31:19.841 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:19 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:31:19.518+0000 7ff1a46e1200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-19T01:31:20.692 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:20 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:31:20.474+0000 7ff1a46e1200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-19T01:31:20.692 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:20 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:31:20.607+0000 7ff1a46e1200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-19T01:31:20.951 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:20 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/2891141473' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-03-19T01:31:21.235 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:20 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:31:20.950+0000 7ff1a46e1200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-19T01:31:21.235 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:21 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:31:21.040+0000 7ff1a46e1200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-19T01:31:21.235 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:21 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:31:21.234+0000 7ff1a46e1200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-19T01:31:21.342 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout 2024-03-19T01:31:21.342 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout { 2024-03-19T01:31:21.342 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "fsid": "438fb69c-e590-11ee-95c9-87774f69a715", 2024-03-19T01:31:21.342 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "health": { 2024-03-19T01:31:21.342 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-03-19T01:31:21.342 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-03-19T01:31:21.342 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-03-19T01:31:21.342 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout }, 2024-03-19T01:31:21.342 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-03-19T01:31:21.342 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-03-19T01:31:21.342 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout 0 2024-03-19T01:31:21.342 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout ], 2024-03-19T01:31:21.343 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-03-19T01:31:21.343 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "a" 2024-03-19T01:31:21.343 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout ], 2024-03-19T01:31:21.343 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "quorum_age": 5, 2024-03-19T01:31:21.343 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "monmap": { 2024-03-19T01:31:21.343 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-19T01:31:21.343 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "squid", 2024-03-19T01:31:21.343 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-03-19T01:31:21.343 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout }, 2024-03-19T01:31:21.343 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-03-19T01:31:21.343 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-19T01:31:21.343 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-03-19T01:31:21.343 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-03-19T01:31:21.344 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-03-19T01:31:21.352 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-03-19T01:31:21.352 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-03-19T01:31:21.352 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-03-19T01:31:21.352 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout }, 2024-03-19T01:31:21.352 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-03-19T01:31:21.353 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-03-19T01:31:21.353 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-03-19T01:31:21.353 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-03-19T01:31:21.353 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-03-19T01:31:21.353 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-03-19T01:31:21.353 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-03-19T01:31:21.353 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-03-19T01:31:21.353 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-03-19T01:31:21.353 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout }, 2024-03-19T01:31:21.353 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-03-19T01:31:21.353 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-19T01:31:21.353 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-03-19T01:31:21.353 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-03-19T01:31:21.353 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout }, 2024-03-19T01:31:21.354 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-03-19T01:31:21.354 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "available": false, 2024-03-19T01:31:21.354 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-03-19T01:31:21.354 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "modules": [ 2024-03-19T01:31:21.354 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "iostat", 2024-03-19T01:31:21.354 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "nfs", 2024-03-19T01:31:21.354 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "restful" 2024-03-19T01:31:21.354 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout ], 2024-03-19T01:31:21.354 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "services": {} 2024-03-19T01:31:21.354 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout }, 2024-03-19T01:31:21.354 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-03-19T01:31:21.354 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-19T01:31:21.354 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "modified": "2024-03-19T01:31:10.165529+0000", 2024-03-19T01:31:21.355 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "services": {} 2024-03-19T01:31:21.355 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout }, 2024-03-19T01:31:21.355 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-03-19T01:31:21.355 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout } 2024-03-19T01:31:21.355 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:20.706+0000 7f78086dd700 1 Processor -- start 2024-03-19T01:31:21.355 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:20.706+0000 7f78086dd700 1 -- start start 2024-03-19T01:31:21.355 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:20.706+0000 7f78086dd700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7800106cc0 0x7f78001070e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:31:21.355 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:20.706+0000 7f78086dd700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f78001076b0 con 0x7f7800106cc0 2024-03-19T01:31:21.355 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:20.706+0000 7f7806479700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7800106cc0 0x7f78001070e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:31:21.355 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:20.706+0000 7f7806479700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7800106cc0 0x7f78001070e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:40190/0 (socket says 172.21.15.99:40190) 2024-03-19T01:31:21.355 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:20.706+0000 7f7806479700 1 -- 172.21.15.99:0/4147883758 learned_addr learned my addr 172.21.15.99:0/4147883758 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:31:21.355 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:20.707+0000 7f7806479700 1 -- 172.21.15.99:0/4147883758 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7800107ec0 con 0x7f7800106cc0 2024-03-19T01:31:21.355 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:20.707+0000 7f7806479700 1 --2- 172.21.15.99:0/4147883758 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7800106cc0 0x7f78001070e0 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7f77fc0098a0 tx=0x7f77fc009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a54e2506458b7c7c server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:31:21.355 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:20.707+0000 7f7805477700 1 -- 172.21.15.99:0/4147883758 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f77fc004160 con 0x7f7800106cc0 2024-03-19T01:31:21.356 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:20.707+0000 7f7805477700 1 -- 172.21.15.99:0/4147883758 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7f77fc004300 con 0x7f7800106cc0 2024-03-19T01:31:21.356 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:20.707+0000 7f7805477700 1 -- 172.21.15.99:0/4147883758 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f77fc011b50 con 0x7f7800106cc0 2024-03-19T01:31:21.356 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:20.708+0000 7f78086dd700 1 -- 172.21.15.99:0/4147883758 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7800106cc0 msgr2=0x7f78001070e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:31:21.356 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:20.708+0000 7f78086dd700 1 --2- 172.21.15.99:0/4147883758 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7800106cc0 0x7f78001070e0 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7f77fc0098a0 tx=0x7f77fc009bb0 comp rx=0 tx=0).stop 2024-03-19T01:31:21.356 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:20.708+0000 7f78086dd700 1 -- 172.21.15.99:0/4147883758 shutdown_connections 2024-03-19T01:31:21.356 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:20.708+0000 7f78086dd700 1 --2- 172.21.15.99:0/4147883758 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7800106cc0 0x7f78001070e0 unknown :-1 s=CLOSED pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:21.356 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:20.708+0000 7f78086dd700 1 -- 172.21.15.99:0/4147883758 >> 172.21.15.99:0/4147883758 conn(0x7f7800102220 msgr2=0x7f78001046a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:31:21.356 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:20.708+0000 7f78086dd700 1 -- 172.21.15.99:0/4147883758 shutdown_connections 2024-03-19T01:31:21.356 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:20.708+0000 7f78086dd700 1 -- 172.21.15.99:0/4147883758 wait complete. 2024-03-19T01:31:21.356 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:20.708+0000 7f78086dd700 1 Processor -- start 2024-03-19T01:31:21.356 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:20.708+0000 7f78086dd700 1 -- start start 2024-03-19T01:31:21.356 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:20.709+0000 7f78086dd700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7800106cc0 0x7f780007ba70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:31:21.356 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:20.709+0000 7f78086dd700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f780010b270 con 0x7f7800106cc0 2024-03-19T01:31:21.356 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:20.709+0000 7f7806479700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7800106cc0 0x7f780007ba70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:31:21.357 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:20.709+0000 7f7806479700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7800106cc0 0x7f780007ba70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:40192/0 (socket says 172.21.15.99:40192) 2024-03-19T01:31:21.357 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:20.709+0000 7f7806479700 1 -- 172.21.15.99:0/2891141473 learned_addr learned my addr 172.21.15.99:0/2891141473 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:31:21.357 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:20.709+0000 7f7806479700 1 -- 172.21.15.99:0/2891141473 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f780019eb60 con 0x7f7800106cc0 2024-03-19T01:31:21.357 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:20.709+0000 7f7806479700 1 --2- 172.21.15.99:0/2891141473 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7800106cc0 0x7f780007ba70 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7f77fc009fb0 tx=0x7f77fc011bf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:31:21.357 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:20.709+0000 7f77f77fe700 1 -- 172.21.15.99:0/2891141473 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f77fc0038b0 con 0x7f7800106cc0 2024-03-19T01:31:21.357 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:20.709+0000 7f78086dd700 1 -- 172.21.15.99:0/2891141473 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f780019ed30 con 0x7f7800106cc0 2024-03-19T01:31:21.357 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:20.709+0000 7f77f77fe700 1 -- 172.21.15.99:0/2891141473 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7f77fc011e50 con 0x7f7800106cc0 2024-03-19T01:31:21.357 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:20.709+0000 7f77f77fe700 1 -- 172.21.15.99:0/2891141473 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f77fc019550 con 0x7f7800106cc0 2024-03-19T01:31:21.357 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:20.709+0000 7f78086dd700 1 -- 172.21.15.99:0/2891141473 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f780019f0d0 con 0x7f7800106cc0 2024-03-19T01:31:21.357 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:20.710+0000 7f77f77fe700 1 -- 172.21.15.99:0/2891141473 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7f77fc019be0 con 0x7f7800106cc0 2024-03-19T01:31:21.357 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:20.710+0000 7f77f77fe700 1 -- 172.21.15.99:0/2891141473 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7f77fc024730 con 0x7f7800106cc0 2024-03-19T01:31:21.357 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:20.710+0000 7f78086dd700 1 -- 172.21.15.99:0/2891141473 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f7800069420 con 0x7f7800106cc0 2024-03-19T01:31:21.357 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:20.713+0000 7f77f77fe700 1 -- 172.21.15.99:0/2891141473 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+73141 (secure 0 0 0) 0x7f77fc003a50 con 0x7f7800106cc0 2024-03-19T01:31:21.357 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:20.785+0000 7f78086dd700 1 -- 172.21.15.99:0/2891141473 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "status", "format": "json-pretty"} v 0) -- 0x7f780019f510 con 0x7f7800106cc0 2024-03-19T01:31:21.358 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:20.785+0000 7f77f77fe700 1 -- 172.21.15.99:0/2891141473 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "status", "format": "json-pretty"}]=0 v0) ==== 79+0+1239 (secure 0 0 0) 0x7f77fc0249d0 con 0x7f7800106cc0 2024-03-19T01:31:21.358 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:20.787+0000 7f78086dd700 1 -- 172.21.15.99:0/2891141473 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7800106cc0 msgr2=0x7f780007ba70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:31:21.358 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:20.787+0000 7f78086dd700 1 --2- 172.21.15.99:0/2891141473 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7800106cc0 0x7f780007ba70 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7f77fc009fb0 tx=0x7f77fc011bf0 comp rx=0 tx=0).stop 2024-03-19T01:31:21.358 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:20.787+0000 7f78086dd700 1 -- 172.21.15.99:0/2891141473 shutdown_connections 2024-03-19T01:31:21.358 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:20.787+0000 7f78086dd700 1 --2- 172.21.15.99:0/2891141473 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7800106cc0 0x7f780007ba70 unknown :-1 s=CLOSED pgs=8 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:21.358 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:20.787+0000 7f78086dd700 1 -- 172.21.15.99:0/2891141473 >> 172.21.15.99:0/2891141473 conn(0x7f7800102220 msgr2=0x7f7800102e10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:31:21.358 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:20.787+0000 7f78086dd700 1 -- 172.21.15.99:0/2891141473 shutdown_connections 2024-03-19T01:31:21.358 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:20.787+0000 7f78086dd700 1 -- 172.21.15.99:0/2891141473 wait complete. 2024-03-19T01:31:21.358 INFO:teuthology.orchestra.run.smithi099.stdout:mgr not available, waiting (2/15)... 2024-03-19T01:31:21.591 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:21 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:31:21.419+0000 7ff1a46e1200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-19T01:31:21.958 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:21 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:31:21.701+0000 7ff1a46e1200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-19T01:31:21.958 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:21 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:31:21.957+0000 7ff1a46e1200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-19T01:31:22.341 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:22 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:31:22.193+0000 7ff1a46e1200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-19T01:31:23.342 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:22 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:31:22.998+0000 7ff1a46e1200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-19T01:31:23.841 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:23 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:31:23.570+0000 7ff1a46e1200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-19T01:31:23.841 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:23 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:31:23.667+0000 7ff1a46e1200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-19T01:31:24.341 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:23 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:31:23.879+0000 7ff1a46e1200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-19T01:31:24.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:23 smithi099 ceph-mon[62951]: Activating manager daemon x 2024-03-19T01:31:24.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:23 smithi099 ceph-mon[62951]: mgrmap e2: x(active, starting, since 0.00310072s) 2024-03-19T01:31:24.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:23 smithi099 ceph-mon[62951]: from='mgr.14100 172.21.15.99:0/2086207454' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-19T01:31:24.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:23 smithi099 ceph-mon[62951]: from='mgr.14100 172.21.15.99:0/2086207454' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-19T01:31:24.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:23 smithi099 ceph-mon[62951]: from='mgr.14100 172.21.15.99:0/2086207454' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-19T01:31:24.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:23 smithi099 ceph-mon[62951]: from='mgr.14100 172.21.15.99:0/2086207454' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-19T01:31:24.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:23 smithi099 ceph-mon[62951]: from='mgr.14100 172.21.15.99:0/2086207454' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-19T01:31:24.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:23 smithi099 ceph-mon[62951]: Manager daemon x is now available 2024-03-19T01:31:24.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:23 smithi099 ceph-mon[62951]: from='mgr.14100 172.21.15.99:0/2086207454' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-03-19T01:31:24.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:23 smithi099 ceph-mon[62951]: from='mgr.14100 172.21.15.99:0/2086207454' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-03-19T01:31:24.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:23 smithi099 ceph-mon[62951]: from='mgr.14100 172.21.15.99:0/2086207454' entity='mgr.x' 2024-03-19T01:31:24.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:23 smithi099 ceph-mon[62951]: from='mgr.14100 172.21.15.99:0/2086207454' entity='mgr.x' 2024-03-19T01:31:24.343 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:23 smithi099 ceph-mon[62951]: from='mgr.14100 172.21.15.99:0/2086207454' entity='mgr.x' 2024-03-19T01:31:26.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:25 smithi099 ceph-mon[62951]: mgrmap e3: x(active, since 1.00699s) 2024-03-19T01:31:28.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:27 smithi099 ceph-mon[62951]: mgrmap e4: x(active, since 3s) 2024-03-19T01:31:28.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:27 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/616162539' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-03-19T01:31:31.421 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout 2024-03-19T01:31:31.421 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout { 2024-03-19T01:31:31.421 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "fsid": "438fb69c-e590-11ee-95c9-87774f69a715", 2024-03-19T01:31:31.421 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "health": { 2024-03-19T01:31:31.421 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-03-19T01:31:31.421 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-03-19T01:31:31.421 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-03-19T01:31:31.421 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout }, 2024-03-19T01:31:31.421 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-03-19T01:31:31.422 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-03-19T01:31:31.422 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout 0 2024-03-19T01:31:31.422 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout ], 2024-03-19T01:31:31.422 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-03-19T01:31:31.422 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "a" 2024-03-19T01:31:31.422 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout ], 2024-03-19T01:31:31.422 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "quorum_age": 12, 2024-03-19T01:31:31.422 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "monmap": { 2024-03-19T01:31:31.422 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-19T01:31:31.422 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "squid", 2024-03-19T01:31:31.422 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-03-19T01:31:31.422 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout }, 2024-03-19T01:31:31.422 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-03-19T01:31:31.423 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-19T01:31:31.423 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-03-19T01:31:31.423 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-03-19T01:31:31.423 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-03-19T01:31:31.423 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-03-19T01:31:31.423 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-03-19T01:31:31.423 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-03-19T01:31:31.423 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout }, 2024-03-19T01:31:31.423 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-03-19T01:31:31.423 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-03-19T01:31:31.423 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-03-19T01:31:31.423 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-03-19T01:31:31.423 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-03-19T01:31:31.423 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-03-19T01:31:31.424 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-03-19T01:31:31.424 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-03-19T01:31:31.424 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-03-19T01:31:31.424 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout }, 2024-03-19T01:31:31.424 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-03-19T01:31:31.424 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-19T01:31:31.424 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-03-19T01:31:31.424 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-03-19T01:31:31.424 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout }, 2024-03-19T01:31:31.424 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-03-19T01:31:31.424 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "available": true, 2024-03-19T01:31:31.424 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-03-19T01:31:31.424 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "modules": [ 2024-03-19T01:31:31.424 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "iostat", 2024-03-19T01:31:31.425 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "nfs", 2024-03-19T01:31:31.425 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "restful" 2024-03-19T01:31:31.425 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout ], 2024-03-19T01:31:31.425 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "services": {} 2024-03-19T01:31:31.425 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout }, 2024-03-19T01:31:31.425 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-03-19T01:31:31.425 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-19T01:31:31.425 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "modified": "2024-03-19T01:31:10.165529+0000", 2024-03-19T01:31:31.425 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "services": {} 2024-03-19T01:31:31.425 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout }, 2024-03-19T01:31:31.425 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-03-19T01:31:31.425 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout } 2024-03-19T01:31:31.425 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:26.965+0000 7f27ffc79700 1 Processor -- start 2024-03-19T01:31:31.425 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:26.966+0000 7f27ffc79700 1 -- start start 2024-03-19T01:31:31.426 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:26.966+0000 7f27ffc79700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f27f807dec0 0x7f27f807c320 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:31:31.426 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:26.966+0000 7f27ffc79700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f27f807c8f0 con 0x7f27f807dec0 2024-03-19T01:31:31.426 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:26.966+0000 7f27fda15700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f27f807dec0 0x7f27f807c320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:31:31.426 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:26.966+0000 7f27fda15700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f27f807dec0 0x7f27f807c320 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:35920/0 (socket says 172.21.15.99:35920) 2024-03-19T01:31:31.426 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:26.966+0000 7f27fda15700 1 -- 172.21.15.99:0/905859808 learned_addr learned my addr 172.21.15.99:0/905859808 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:31:31.426 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:26.966+0000 7f27fda15700 1 -- 172.21.15.99:0/905859808 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f27f807ca70 con 0x7f27f807dec0 2024-03-19T01:31:31.426 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:26.966+0000 7f27fda15700 1 --2- 172.21.15.99:0/905859808 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f27f807dec0 0x7f27f807c320 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7f27f40098a0 tx=0x7f27f4009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c65300c84d67edbb server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:31:31.426 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:26.966+0000 7f27fca13700 1 -- 172.21.15.99:0/905859808 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f27f4004160 con 0x7f27f807dec0 2024-03-19T01:31:31.426 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:26.966+0000 7f27fca13700 1 -- 172.21.15.99:0/905859808 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7f27f4004300 con 0x7f27f807dec0 2024-03-19T01:31:31.426 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:26.966+0000 7f27fca13700 1 -- 172.21.15.99:0/905859808 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f27f4011b50 con 0x7f27f807dec0 2024-03-19T01:31:31.426 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:26.967+0000 7f27ffc79700 1 -- 172.21.15.99:0/905859808 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f27f807dec0 msgr2=0x7f27f807c320 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:31:31.426 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:26.967+0000 7f27ffc79700 1 --2- 172.21.15.99:0/905859808 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f27f807dec0 0x7f27f807c320 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7f27f40098a0 tx=0x7f27f4009bb0 comp rx=0 tx=0).stop 2024-03-19T01:31:31.427 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:26.967+0000 7f27ffc79700 1 -- 172.21.15.99:0/905859808 shutdown_connections 2024-03-19T01:31:31.427 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:26.967+0000 7f27ffc79700 1 --2- 172.21.15.99:0/905859808 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f27f807dec0 0x7f27f807c320 unknown :-1 s=CLOSED pgs=16 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:31.427 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:26.967+0000 7f27ffc79700 1 -- 172.21.15.99:0/905859808 >> 172.21.15.99:0/905859808 conn(0x7f27f8103390 msgr2=0x7f27f81057b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:31:31.427 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:26.967+0000 7f27ffc79700 1 -- 172.21.15.99:0/905859808 shutdown_connections 2024-03-19T01:31:31.427 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:26.967+0000 7f27ffc79700 1 -- 172.21.15.99:0/905859808 wait complete. 2024-03-19T01:31:31.427 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:26.968+0000 7f27ffc79700 1 Processor -- start 2024-03-19T01:31:31.427 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:26.968+0000 7f27ffc79700 1 -- start start 2024-03-19T01:31:31.427 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:26.968+0000 7f27ffc79700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f27f807dec0 0x7f27f819dff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:31:31.427 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:26.968+0000 7f27ffc79700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f27f810e3e0 con 0x7f27f807dec0 2024-03-19T01:31:31.427 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:26.968+0000 7f27fda15700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f27f807dec0 0x7f27f819dff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:31:31.427 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:26.968+0000 7f27fda15700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f27f807dec0 0x7f27f819dff0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:35928/0 (socket says 172.21.15.99:35928) 2024-03-19T01:31:31.427 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:26.968+0000 7f27fda15700 1 -- 172.21.15.99:0/616162539 learned_addr learned my addr 172.21.15.99:0/616162539 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:31:31.427 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:26.968+0000 7f27fda15700 1 -- 172.21.15.99:0/616162539 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f27f819e530 con 0x7f27f807dec0 2024-03-19T01:31:31.428 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:26.968+0000 7f27fda15700 1 --2- 172.21.15.99:0/616162539 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f27f807dec0 0x7f27f819dff0 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7f27f4009fb0 tx=0x7f27f4011bf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:31:31.428 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:26.968+0000 7f27eeffd700 1 -- 172.21.15.99:0/616162539 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f27f40038b0 con 0x7f27f807dec0 2024-03-19T01:31:31.428 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:26.969+0000 7f27eeffd700 1 -- 172.21.15.99:0/616162539 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7f27f4011e50 con 0x7f27f807dec0 2024-03-19T01:31:31.428 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:26.969+0000 7f27eeffd700 1 -- 172.21.15.99:0/616162539 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f27f4019550 con 0x7f27f807dec0 2024-03-19T01:31:31.428 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:26.969+0000 7f27ffc79700 1 -- 172.21.15.99:0/616162539 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f27f819e7c0 con 0x7f27f807dec0 2024-03-19T01:31:31.428 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:26.969+0000 7f27ffc79700 1 -- 172.21.15.99:0/616162539 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f27f81a14b0 con 0x7f27f807dec0 2024-03-19T01:31:31.428 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:26.969+0000 7f27eeffd700 1 -- 172.21.15.99:0/616162539 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 4) ==== 46095+0+0 (secure 0 0 0) 0x7f27f4003a50 con 0x7f27f807dec0 2024-03-19T01:31:31.428 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:26.969+0000 7f27ffc79700 1 -- 172.21.15.99:0/616162539 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f27f8069420 con 0x7f27f807dec0 2024-03-19T01:31:31.428 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:26.970+0000 7f27eeffd700 1 --2- 172.21.15.99:0/616162539 >> [v2:172.21.15.99:6800/4223048293,v1:172.21.15.99:6801/4223048293] conn(0x7f27e4038ed0 0x7f27e403b390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:31:31.428 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:26.970+0000 7f27eeffd700 1 -- 172.21.15.99:0/616162539 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7f27f404cfa0 con 0x7f27f807dec0 2024-03-19T01:31:31.428 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:26.970+0000 7f27fd214700 1 --2- 172.21.15.99:0/616162539 >> [v2:172.21.15.99:6800/4223048293,v1:172.21.15.99:6801/4223048293] conn(0x7f27e4038ed0 0x7f27e403b390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:31:31.428 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:26.970+0000 7f27fd214700 1 --2- 172.21.15.99:0/616162539 >> [v2:172.21.15.99:6800/4223048293,v1:172.21.15.99:6801/4223048293] conn(0x7f27e4038ed0 0x7f27e403b390 secure :-1 s=READY pgs=6 cs=0 l=1 rev1=1 crypto rx=0x7f27e8006fd0 tx=0x7f27e8006e40 comp rx=0 tx=0).ready entity=mgr.14100 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:31:31.428 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:26.975+0000 7f27eeffd700 1 -- 172.21.15.99:0/616162539 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f27f4023390 con 0x7f27f807dec0 2024-03-19T01:31:31.428 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:27.253+0000 7f27ffc79700 1 -- 172.21.15.99:0/616162539 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "status", "format": "json-pretty"} v 0) -- 0x7f27f81a1680 con 0x7f27f807dec0 2024-03-19T01:31:31.429 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:27.254+0000 7f27eeffd700 1 -- 172.21.15.99:0/616162539 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "status", "format": "json-pretty"}]=0 v0) ==== 79+0+1239 (secure 0 0 0) 0x7f27f81a1680 con 0x7f27f807dec0 2024-03-19T01:31:31.429 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:27.256+0000 7f27ffc79700 1 -- 172.21.15.99:0/616162539 >> [v2:172.21.15.99:6800/4223048293,v1:172.21.15.99:6801/4223048293] conn(0x7f27e4038ed0 msgr2=0x7f27e403b390 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:31:31.429 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:27.256+0000 7f27ffc79700 1 --2- 172.21.15.99:0/616162539 >> [v2:172.21.15.99:6800/4223048293,v1:172.21.15.99:6801/4223048293] conn(0x7f27e4038ed0 0x7f27e403b390 secure :-1 s=READY pgs=6 cs=0 l=1 rev1=1 crypto rx=0x7f27e8006fd0 tx=0x7f27e8006e40 comp rx=0 tx=0).stop 2024-03-19T01:31:31.429 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:27.256+0000 7f27ffc79700 1 -- 172.21.15.99:0/616162539 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f27f807dec0 msgr2=0x7f27f819dff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:31:31.429 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:27.256+0000 7f27ffc79700 1 --2- 172.21.15.99:0/616162539 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f27f807dec0 0x7f27f819dff0 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7f27f4009fb0 tx=0x7f27f4011bf0 comp rx=0 tx=0).stop 2024-03-19T01:31:31.429 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:27.256+0000 7f27ffc79700 1 -- 172.21.15.99:0/616162539 shutdown_connections 2024-03-19T01:31:31.429 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:27.256+0000 7f27ffc79700 1 --2- 172.21.15.99:0/616162539 >> [v2:172.21.15.99:6800/4223048293,v1:172.21.15.99:6801/4223048293] conn(0x7f27e4038ed0 0x7f27e403b390 unknown :-1 s=CLOSED pgs=6 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:31.429 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:27.256+0000 7f27ffc79700 1 --2- 172.21.15.99:0/616162539 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f27f807dec0 0x7f27f819dff0 unknown :-1 s=CLOSED pgs=17 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:31.429 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:27.256+0000 7f27ffc79700 1 -- 172.21.15.99:0/616162539 >> 172.21.15.99:0/616162539 conn(0x7f27f8103390 msgr2=0x7f27f8103f10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:31:31.429 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:27.256+0000 7f27ffc79700 1 -- 172.21.15.99:0/616162539 shutdown_connections 2024-03-19T01:31:31.429 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:27.256+0000 7f27ffc79700 1 -- 172.21.15.99:0/616162539 wait complete. 2024-03-19T01:31:31.429 INFO:teuthology.orchestra.run.smithi099.stdout:mgr is available 2024-03-19T01:31:32.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:32 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/496673876' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-03-19T01:31:32.541 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout 2024-03-19T01:31:32.541 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout [global] 2024-03-19T01:31:32.541 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout fsid = 438fb69c-e590-11ee-95c9-87774f69a715 2024-03-19T01:31:32.541 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-03-19T01:31:32.541 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-03-19T01:31:32.541 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-03-19T01:31:32.542 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-03-19T01:31:32.542 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout 2024-03-19T01:31:32.542 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout [mgr] 2024-03-19T01:31:32.542 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-03-19T01:31:32.542 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout 2024-03-19T01:31:32.542 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout [osd] 2024-03-19T01:31:32.542 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout bdev_async_discard = True 2024-03-19T01:31:32.542 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout mon_osd_backfillfull_ratio = 0.85 2024-03-19T01:31:32.542 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout mon_osd_full_ratio = 0.9 2024-03-19T01:31:32.542 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout mon_osd_nearfull_ratio = 0.8 2024-03-19T01:31:32.542 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-03-19T01:31:32.542 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-03-19T01:31:32.542 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:31.814+0000 7fe39d631700 1 Processor -- start 2024-03-19T01:31:32.543 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:31.814+0000 7fe39d631700 1 -- start start 2024-03-19T01:31:32.543 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:31.814+0000 7fe39d631700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe398108830 0x7fe39810ac20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:31:32.543 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:31.814+0000 7fe39d631700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fe39807bed0 con 0x7fe398108830 2024-03-19T01:31:32.543 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:31.815+0000 7fe396ffd700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe398108830 0x7fe39810ac20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:31:32.543 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:31.815+0000 7fe396ffd700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe398108830 0x7fe39810ac20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:35942/0 (socket says 172.21.15.99:35942) 2024-03-19T01:31:32.543 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:31.815+0000 7fe396ffd700 1 -- 172.21.15.99:0/2633250627 learned_addr learned my addr 172.21.15.99:0/2633250627 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:31:32.543 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:31.815+0000 7fe396ffd700 1 -- 172.21.15.99:0/2633250627 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fe39810b160 con 0x7fe398108830 2024-03-19T01:31:32.543 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:31.815+0000 7fe396ffd700 1 --2- 172.21.15.99:0/2633250627 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe398108830 0x7fe39810ac20 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7fe3880098a0 tx=0x7fe388009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8c9613c8ef83eeda server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:31:32.543 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:31.815+0000 7fe3967fc700 1 -- 172.21.15.99:0/2633250627 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fe388004160 con 0x7fe398108830 2024-03-19T01:31:32.543 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:31.815+0000 7fe3967fc700 1 -- 172.21.15.99:0/2633250627 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7fe388004300 con 0x7fe398108830 2024-03-19T01:31:32.543 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:31.815+0000 7fe3967fc700 1 -- 172.21.15.99:0/2633250627 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fe388011b50 con 0x7fe398108830 2024-03-19T01:31:32.543 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:31.816+0000 7fe39d631700 1 -- 172.21.15.99:0/2633250627 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe398108830 msgr2=0x7fe39810ac20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:31:32.543 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:31.816+0000 7fe39d631700 1 --2- 172.21.15.99:0/2633250627 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe398108830 0x7fe39810ac20 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7fe3880098a0 tx=0x7fe388009bb0 comp rx=0 tx=0).stop 2024-03-19T01:31:32.543 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:31.816+0000 7fe39d631700 1 -- 172.21.15.99:0/2633250627 shutdown_connections 2024-03-19T01:31:32.544 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:31.816+0000 7fe39d631700 1 --2- 172.21.15.99:0/2633250627 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe398108830 0x7fe39810ac20 unknown :-1 s=CLOSED pgs=18 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:32.544 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:31.816+0000 7fe39d631700 1 -- 172.21.15.99:0/2633250627 >> 172.21.15.99:0/2633250627 conn(0x7fe398102220 msgr2=0x7fe3981046a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:31:32.544 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:31.816+0000 7fe39d631700 1 -- 172.21.15.99:0/2633250627 shutdown_connections 2024-03-19T01:31:32.544 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:31.816+0000 7fe39d631700 1 -- 172.21.15.99:0/2633250627 wait complete. 2024-03-19T01:31:32.544 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:31.817+0000 7fe39d631700 1 Processor -- start 2024-03-19T01:31:32.544 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:31.817+0000 7fe39d631700 1 -- start start 2024-03-19T01:31:32.544 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:31.817+0000 7fe39d631700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe398108830 0x7fe39819bdc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:31:32.544 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:31.817+0000 7fe39d631700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fe39810c190 con 0x7fe398108830 2024-03-19T01:31:32.544 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:31.817+0000 7fe396ffd700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe398108830 0x7fe39819bdc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:31:32.544 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:31.817+0000 7fe396ffd700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe398108830 0x7fe39819bdc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:35954/0 (socket says 172.21.15.99:35954) 2024-03-19T01:31:32.544 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:31.817+0000 7fe396ffd700 1 -- 172.21.15.99:0/496673876 learned_addr learned my addr 172.21.15.99:0/496673876 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:31:32.544 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:31.817+0000 7fe396ffd700 1 -- 172.21.15.99:0/496673876 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fe39819c300 con 0x7fe398108830 2024-03-19T01:31:32.544 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:31.817+0000 7fe396ffd700 1 --2- 172.21.15.99:0/496673876 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe398108830 0x7fe39819bdc0 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7fe388011b60 tx=0x7fe388011c40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:31:32.544 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:31.817+0000 7fe394ff9700 1 -- 172.21.15.99:0/496673876 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fe388003970 con 0x7fe398108830 2024-03-19T01:31:32.545 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:31.817+0000 7fe394ff9700 1 -- 172.21.15.99:0/496673876 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7fe388003b10 con 0x7fe398108830 2024-03-19T01:31:32.545 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:31.817+0000 7fe39d631700 1 -- 172.21.15.99:0/496673876 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fe39819c590 con 0x7fe398108830 2024-03-19T01:31:32.545 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:31.817+0000 7fe394ff9700 1 -- 172.21.15.99:0/496673876 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fe388024610 con 0x7fe398108830 2024-03-19T01:31:32.545 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:31.818+0000 7fe39d631700 1 -- 172.21.15.99:0/496673876 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fe39819f280 con 0x7fe398108830 2024-03-19T01:31:32.545 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:31.818+0000 7fe394ff9700 1 -- 172.21.15.99:0/496673876 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 4) ==== 46095+0+0 (secure 0 0 0) 0x7fe388024910 con 0x7fe398108830 2024-03-19T01:31:32.545 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:31.818+0000 7fe394ff9700 1 --2- 172.21.15.99:0/496673876 >> [v2:172.21.15.99:6800/4223048293,v1:172.21.15.99:6801/4223048293] conn(0x7fe384038ed0 0x7fe38403b390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:31:32.545 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:31.818+0000 7fe394ff9700 1 -- 172.21.15.99:0/496673876 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7fe388011e00 con 0x7fe398108830 2024-03-19T01:31:32.545 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:31.818+0000 7fe39d631700 1 -- 172.21.15.99:0/496673876 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fe398069420 con 0x7fe398108830 2024-03-19T01:31:32.545 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:31.819+0000 7fe38ffff700 1 --2- 172.21.15.99:0/496673876 >> [v2:172.21.15.99:6800/4223048293,v1:172.21.15.99:6801/4223048293] conn(0x7fe384038ed0 0x7fe38403b390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:31:32.545 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:31.819+0000 7fe38ffff700 1 --2- 172.21.15.99:0/496673876 >> [v2:172.21.15.99:6800/4223048293,v1:172.21.15.99:6801/4223048293] conn(0x7fe384038ed0 0x7fe38403b390 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7fe380006fd0 tx=0x7fe380006e40 comp rx=0 tx=0).ready entity=mgr.14100 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:31:32.545 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:31.824+0000 7fe394ff9700 1 -- 172.21.15.99:0/496673876 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7fe38801a7b0 con 0x7fe398108830 2024-03-19T01:31:32.545 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:32.017+0000 7fe39d631700 1 -- 172.21.15.99:0/496673876 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "config assimilate-conf"} v 0) -- 0x7fe3981a09d0 con 0x7fe398108830 2024-03-19T01:31:32.545 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:32.018+0000 7fe394ff9700 1 -- 172.21.15.99:0/496673876 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "config assimilate-conf"}]=0 v3) ==== 70+0+400 (secure 0 0 0) 0x7fe38802d3f0 con 0x7fe398108830 2024-03-19T01:31:32.545 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:32.020+0000 7fe39d631700 1 -- 172.21.15.99:0/496673876 >> [v2:172.21.15.99:6800/4223048293,v1:172.21.15.99:6801/4223048293] conn(0x7fe384038ed0 msgr2=0x7fe38403b390 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:31:32.545 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:32.020+0000 7fe39d631700 1 --2- 172.21.15.99:0/496673876 >> [v2:172.21.15.99:6800/4223048293,v1:172.21.15.99:6801/4223048293] conn(0x7fe384038ed0 0x7fe38403b390 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7fe380006fd0 tx=0x7fe380006e40 comp rx=0 tx=0).stop 2024-03-19T01:31:32.546 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:32.020+0000 7fe39d631700 1 -- 172.21.15.99:0/496673876 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe398108830 msgr2=0x7fe39819bdc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:31:32.546 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:32.020+0000 7fe39d631700 1 --2- 172.21.15.99:0/496673876 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe398108830 0x7fe39819bdc0 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7fe388011b60 tx=0x7fe388011c40 comp rx=0 tx=0).stop 2024-03-19T01:31:32.546 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:32.020+0000 7fe39d631700 1 -- 172.21.15.99:0/496673876 shutdown_connections 2024-03-19T01:31:32.546 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:32.020+0000 7fe39d631700 1 --2- 172.21.15.99:0/496673876 >> [v2:172.21.15.99:6800/4223048293,v1:172.21.15.99:6801/4223048293] conn(0x7fe384038ed0 0x7fe38403b390 unknown :-1 s=CLOSED pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:32.546 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:32.020+0000 7fe39d631700 1 --2- 172.21.15.99:0/496673876 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe398108830 0x7fe39819bdc0 unknown :-1 s=CLOSED pgs=19 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:32.546 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:32.020+0000 7fe39d631700 1 -- 172.21.15.99:0/496673876 >> 172.21.15.99:0/496673876 conn(0x7fe398102220 msgr2=0x7fe398102e10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:31:32.546 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:32.021+0000 7fe39d631700 1 -- 172.21.15.99:0/496673876 shutdown_connections 2024-03-19T01:31:32.546 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:32.021+0000 7fe39d631700 1 -- 172.21.15.99:0/496673876 wait complete. 2024-03-19T01:31:32.546 INFO:teuthology.orchestra.run.smithi099.stdout:Enabling cephadm module... 2024-03-19T01:31:33.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:33 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/2937661384' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-03-19T01:31:33.591 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:33 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: ignoring --setuser ceph since I am not root 2024-03-19T01:31:33.592 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:33 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: ignoring --setgroup ceph since I am not root 2024-03-19T01:31:34.091 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:33 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:31:33.803+0000 7f8e195ae200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-19T01:31:34.091 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:33 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:31:33.898+0000 7f8e195ae200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-19T01:31:34.122 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:32.940+0000 7feb70740700 1 Processor -- start 2024-03-19T01:31:34.122 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:32.940+0000 7feb70740700 1 -- start start 2024-03-19T01:31:34.122 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:32.940+0000 7feb70740700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7feb68108870 0x7feb6810ac60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:31:34.122 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:32.940+0000 7feb70740700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7feb6807be90 con 0x7feb68108870 2024-03-19T01:31:34.122 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:32.940+0000 7feb6e4dc700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7feb68108870 0x7feb6810ac60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:31:34.122 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:32.940+0000 7feb6e4dc700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7feb68108870 0x7feb6810ac60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:50898/0 (socket says 172.21.15.99:50898) 2024-03-19T01:31:34.122 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:32.940+0000 7feb6e4dc700 1 -- 172.21.15.99:0/1832016443 learned_addr learned my addr 172.21.15.99:0/1832016443 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:31:34.122 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:32.940+0000 7feb6e4dc700 1 -- 172.21.15.99:0/1832016443 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7feb6810b1a0 con 0x7feb68108870 2024-03-19T01:31:34.122 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:32.941+0000 7feb6e4dc700 1 --2- 172.21.15.99:0/1832016443 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7feb68108870 0x7feb6810ac60 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7feb640098a0 tx=0x7feb64009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4f7e8057363ef649 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:31:34.123 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:32.941+0000 7feb6d4da700 1 -- 172.21.15.99:0/1832016443 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7feb64004160 con 0x7feb68108870 2024-03-19T01:31:34.123 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:32.941+0000 7feb6d4da700 1 -- 172.21.15.99:0/1832016443 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7feb64004300 con 0x7feb68108870 2024-03-19T01:31:34.123 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:32.941+0000 7feb6d4da700 1 -- 172.21.15.99:0/1832016443 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7feb64011b50 con 0x7feb68108870 2024-03-19T01:31:34.123 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:32.941+0000 7feb70740700 1 -- 172.21.15.99:0/1832016443 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7feb68108870 msgr2=0x7feb6810ac60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:31:34.123 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:32.941+0000 7feb70740700 1 --2- 172.21.15.99:0/1832016443 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7feb68108870 0x7feb6810ac60 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7feb640098a0 tx=0x7feb64009bb0 comp rx=0 tx=0).stop 2024-03-19T01:31:34.123 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:32.941+0000 7feb70740700 1 -- 172.21.15.99:0/1832016443 shutdown_connections 2024-03-19T01:31:34.123 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:32.941+0000 7feb70740700 1 --2- 172.21.15.99:0/1832016443 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7feb68108870 0x7feb6810ac60 unknown :-1 s=CLOSED pgs=20 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:34.123 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:32.941+0000 7feb70740700 1 -- 172.21.15.99:0/1832016443 >> 172.21.15.99:0/1832016443 conn(0x7feb68102260 msgr2=0x7feb681046a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:31:34.123 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:32.941+0000 7feb70740700 1 -- 172.21.15.99:0/1832016443 shutdown_connections 2024-03-19T01:31:34.123 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:32.942+0000 7feb70740700 1 -- 172.21.15.99:0/1832016443 wait complete. 2024-03-19T01:31:34.123 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:32.942+0000 7feb70740700 1 Processor -- start 2024-03-19T01:31:34.123 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:32.942+0000 7feb70740700 1 -- start start 2024-03-19T01:31:34.123 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:32.942+0000 7feb70740700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7feb68108870 0x7feb681080f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:31:34.124 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:32.942+0000 7feb70740700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7feb6810c1c0 con 0x7feb68108870 2024-03-19T01:31:34.124 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:32.942+0000 7feb6e4dc700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7feb68108870 0x7feb681080f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:31:34.124 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:32.942+0000 7feb6e4dc700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7feb68108870 0x7feb681080f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:50906/0 (socket says 172.21.15.99:50906) 2024-03-19T01:31:34.124 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:32.942+0000 7feb6e4dc700 1 -- 172.21.15.99:0/2937661384 learned_addr learned my addr 172.21.15.99:0/2937661384 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:31:34.124 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:32.943+0000 7feb6e4dc700 1 -- 172.21.15.99:0/2937661384 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7feb68108630 con 0x7feb68108870 2024-03-19T01:31:34.124 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:32.943+0000 7feb6e4dc700 1 --2- 172.21.15.99:0/2937661384 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7feb68108870 0x7feb681080f0 secure :-1 s=READY pgs=21 cs=0 l=1 rev1=1 crypto rx=0x7feb64011b60 tx=0x7feb64011c40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:31:34.124 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:32.943+0000 7feb5f7fe700 1 -- 172.21.15.99:0/2937661384 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7feb64003b20 con 0x7feb68108870 2024-03-19T01:31:34.124 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:32.943+0000 7feb70740700 1 -- 172.21.15.99:0/2937661384 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7feb68106820 con 0x7feb68108870 2024-03-19T01:31:34.124 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:32.943+0000 7feb70740700 1 -- 172.21.15.99:0/2937661384 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7feb68106d60 con 0x7feb68108870 2024-03-19T01:31:34.124 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:32.943+0000 7feb5f7fe700 1 -- 172.21.15.99:0/2937661384 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7feb64003cc0 con 0x7feb68108870 2024-03-19T01:31:34.124 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:32.943+0000 7feb5f7fe700 1 -- 172.21.15.99:0/2937661384 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7feb64004300 con 0x7feb68108870 2024-03-19T01:31:34.124 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:32.944+0000 7feb5f7fe700 1 -- 172.21.15.99:0/2937661384 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 4) ==== 46095+0+0 (secure 0 0 0) 0x7feb64022020 con 0x7feb68108870 2024-03-19T01:31:34.124 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:32.944+0000 7feb70740700 1 -- 172.21.15.99:0/2937661384 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7feb4c005360 con 0x7feb68108870 2024-03-19T01:31:34.124 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:32.944+0000 7feb5f7fe700 1 --2- 172.21.15.99:0/2937661384 >> [v2:172.21.15.99:6800/4223048293,v1:172.21.15.99:6801/4223048293] conn(0x7feb54039020 0x7feb5403b4e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:31:34.125 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:32.944+0000 7feb5f7fe700 1 -- 172.21.15.99:0/2937661384 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7feb64011e00 con 0x7feb68108870 2024-03-19T01:31:34.125 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:32.944+0000 7feb6dcdb700 1 --2- 172.21.15.99:0/2937661384 >> [v2:172.21.15.99:6800/4223048293,v1:172.21.15.99:6801/4223048293] conn(0x7feb54039020 0x7feb5403b4e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:31:34.125 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:32.944+0000 7feb6dcdb700 1 --2- 172.21.15.99:0/2937661384 >> [v2:172.21.15.99:6800/4223048293,v1:172.21.15.99:6801/4223048293] conn(0x7feb54039020 0x7feb5403b4e0 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7feb58006fd0 tx=0x7feb58006e40 comp rx=0 tx=0).ready entity=mgr.14100 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:31:34.125 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:32.949+0000 7feb5f7fe700 1 -- 172.21.15.99:0/2937661384 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7feb6404ca30 con 0x7feb68108870 2024-03-19T01:31:34.125 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:33.194+0000 7feb70740700 1 -- 172.21.15.99:0/2937661384 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "mgr module enable", "module": "cephadm"} v 0) -- 0x7feb4c005650 con 0x7feb68108870 2024-03-19T01:31:34.125 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:33.246+0000 7feb5f7fe700 1 -- 172.21.15.99:0/2937661384 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "mgr module enable", "module": "cephadm"}]=0 v5) ==== 86+0+0 (secure 0 0 0) 0x7feb6400ba20 con 0x7feb68108870 2024-03-19T01:31:34.125 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:33.247+0000 7feb5f7fe700 1 -- 172.21.15.99:0/2937661384 <== mon.0 v2:172.21.15.99:3300/0 8 ==== mgrmap(e 5) ==== 46106+0+0 (secure 0 0 0) 0x7feb6402d530 con 0x7feb68108870 2024-03-19T01:31:34.125 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:33.249+0000 7feb70740700 1 -- 172.21.15.99:0/2937661384 >> [v2:172.21.15.99:6800/4223048293,v1:172.21.15.99:6801/4223048293] conn(0x7feb54039020 msgr2=0x7feb5403b4e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:31:34.125 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:33.249+0000 7feb70740700 1 --2- 172.21.15.99:0/2937661384 >> [v2:172.21.15.99:6800/4223048293,v1:172.21.15.99:6801/4223048293] conn(0x7feb54039020 0x7feb5403b4e0 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7feb58006fd0 tx=0x7feb58006e40 comp rx=0 tx=0).stop 2024-03-19T01:31:34.125 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:33.249+0000 7feb70740700 1 -- 172.21.15.99:0/2937661384 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7feb68108870 msgr2=0x7feb681080f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:31:34.125 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:33.249+0000 7feb70740700 1 --2- 172.21.15.99:0/2937661384 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7feb68108870 0x7feb681080f0 secure :-1 s=READY pgs=21 cs=0 l=1 rev1=1 crypto rx=0x7feb64011b60 tx=0x7feb64011c40 comp rx=0 tx=0).stop 2024-03-19T01:31:34.125 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:33.249+0000 7feb70740700 1 -- 172.21.15.99:0/2937661384 shutdown_connections 2024-03-19T01:31:34.125 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:33.249+0000 7feb70740700 1 --2- 172.21.15.99:0/2937661384 >> [v2:172.21.15.99:6800/4223048293,v1:172.21.15.99:6801/4223048293] conn(0x7feb54039020 0x7feb5403b4e0 unknown :-1 s=CLOSED pgs=8 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:34.126 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:33.249+0000 7feb70740700 1 --2- 172.21.15.99:0/2937661384 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7feb68108870 0x7feb681080f0 unknown :-1 s=CLOSED pgs=21 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:34.126 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:33.249+0000 7feb70740700 1 -- 172.21.15.99:0/2937661384 >> 172.21.15.99:0/2937661384 conn(0x7feb68102260 msgr2=0x7feb68102d80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:31:34.126 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:33.249+0000 7feb70740700 1 -- 172.21.15.99:0/2937661384 shutdown_connections 2024-03-19T01:31:34.126 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:33.249+0000 7feb70740700 1 -- 172.21.15.99:0/2937661384 wait complete. 2024-03-19T01:31:34.399 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:34 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/2937661384' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-03-19T01:31:34.399 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:34 smithi099 ceph-mon[62951]: mgrmap e5: x(active, since 9s) 2024-03-19T01:31:34.655 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:34 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:31:34.557+0000 7f8e195ae200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-19T01:31:34.655 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:34 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:31:34.653+0000 7f8e195ae200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-19T01:31:34.987 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:34 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:31:34.804+0000 7f8e195ae200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-19T01:31:34.988 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:34 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:31:34.902+0000 7f8e195ae200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-19T01:31:34.988 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:34 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:31:34.986+0000 7f8e195ae200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-19T01:31:35.369 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout { 2024-03-19T01:31:35.370 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "epoch": 5, 2024-03-19T01:31:35.370 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "available": true, 2024-03-19T01:31:35.370 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "active_name": "x", 2024-03-19T01:31:35.370 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-03-19T01:31:35.370 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout } 2024-03-19T01:31:35.370 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:34.496+0000 7fee56909700 1 Processor -- start 2024-03-19T01:31:35.370 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:34.496+0000 7fee56909700 1 -- start start 2024-03-19T01:31:35.370 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:34.497+0000 7fee56909700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fee50106630 0x7fee50106a50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:31:35.370 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:34.497+0000 7fee56909700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fee5007be90 con 0x7fee50106630 2024-03-19T01:31:35.370 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:34.497+0000 7fee4ffff700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fee50106630 0x7fee50106a50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:31:35.370 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:34.497+0000 7fee4ffff700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fee50106630 0x7fee50106a50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:50934/0 (socket says 172.21.15.99:50934) 2024-03-19T01:31:35.370 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:34.497+0000 7fee4ffff700 1 -- 172.21.15.99:0/3573697181 learned_addr learned my addr 172.21.15.99:0/3573697181 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:31:35.371 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:34.497+0000 7fee4ffff700 1 -- 172.21.15.99:0/3573697181 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fee50106f90 con 0x7fee50106630 2024-03-19T01:31:35.371 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:34.497+0000 7fee4ffff700 1 --2- 172.21.15.99:0/3573697181 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fee50106630 0x7fee50106a50 secure :-1 s=READY pgs=24 cs=0 l=1 rev1=1 crypto rx=0x7fee400098a0 tx=0x7fee40009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d4001fd35855d796 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:31:35.371 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:34.497+0000 7fee4effd700 1 -- 172.21.15.99:0/3573697181 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fee40004160 con 0x7fee50106630 2024-03-19T01:31:35.371 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:34.497+0000 7fee4effd700 1 -- 172.21.15.99:0/3573697181 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7fee40004300 con 0x7fee50106630 2024-03-19T01:31:35.371 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:34.497+0000 7fee4effd700 1 -- 172.21.15.99:0/3573697181 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fee40011b50 con 0x7fee50106630 2024-03-19T01:31:35.371 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:34.498+0000 7fee56909700 1 -- 172.21.15.99:0/3573697181 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fee50106630 msgr2=0x7fee50106a50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:31:35.371 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:34.498+0000 7fee56909700 1 --2- 172.21.15.99:0/3573697181 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fee50106630 0x7fee50106a50 secure :-1 s=READY pgs=24 cs=0 l=1 rev1=1 crypto rx=0x7fee400098a0 tx=0x7fee40009bb0 comp rx=0 tx=0).stop 2024-03-19T01:31:35.371 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:34.498+0000 7fee56909700 1 -- 172.21.15.99:0/3573697181 shutdown_connections 2024-03-19T01:31:35.371 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:34.498+0000 7fee56909700 1 --2- 172.21.15.99:0/3573697181 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fee50106630 0x7fee50106a50 unknown :-1 s=CLOSED pgs=24 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:35.371 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:34.498+0000 7fee56909700 1 -- 172.21.15.99:0/3573697181 >> 172.21.15.99:0/3573697181 conn(0x7fee50102270 msgr2=0x7fee50104690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:31:35.371 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:34.498+0000 7fee56909700 1 -- 172.21.15.99:0/3573697181 shutdown_connections 2024-03-19T01:31:35.371 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:34.498+0000 7fee56909700 1 -- 172.21.15.99:0/3573697181 wait complete. 2024-03-19T01:31:35.371 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:34.499+0000 7fee56909700 1 Processor -- start 2024-03-19T01:31:35.371 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:34.499+0000 7fee56909700 1 -- start start 2024-03-19T01:31:35.372 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:34.499+0000 7fee56909700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fee50106630 0x7fee5019df50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:31:35.372 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:34.499+0000 7fee56909700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fee50108020 con 0x7fee50106630 2024-03-19T01:31:35.372 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:34.500+0000 7fee4ffff700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fee50106630 0x7fee5019df50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:31:35.372 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:34.500+0000 7fee4ffff700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fee50106630 0x7fee5019df50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:50946/0 (socket says 172.21.15.99:50946) 2024-03-19T01:31:35.372 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:34.500+0000 7fee4ffff700 1 -- 172.21.15.99:0/1262182924 learned_addr learned my addr 172.21.15.99:0/1262182924 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:31:35.372 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:34.500+0000 7fee4ffff700 1 -- 172.21.15.99:0/1262182924 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fee5019e490 con 0x7fee50106630 2024-03-19T01:31:35.372 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:34.500+0000 7fee4ffff700 1 --2- 172.21.15.99:0/1262182924 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fee50106630 0x7fee5019df50 secure :-1 s=READY pgs=25 cs=0 l=1 rev1=1 crypto rx=0x7fee40011670 tx=0x7fee40011b60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:31:35.372 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:34.500+0000 7fee4d7fa700 1 -- 172.21.15.99:0/1262182924 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fee400039c0 con 0x7fee50106630 2024-03-19T01:31:35.372 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:34.500+0000 7fee56909700 1 -- 172.21.15.99:0/1262182924 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fee5019e720 con 0x7fee50106630 2024-03-19T01:31:35.372 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:34.500+0000 7fee4d7fa700 1 -- 172.21.15.99:0/1262182924 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7fee40011e50 con 0x7fee50106630 2024-03-19T01:31:35.372 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:34.500+0000 7fee4d7fa700 1 -- 172.21.15.99:0/1262182924 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fee4001a5c0 con 0x7fee50106630 2024-03-19T01:31:35.372 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:34.500+0000 7fee56909700 1 -- 172.21.15.99:0/1262182924 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fee501a1410 con 0x7fee50106630 2024-03-19T01:31:35.372 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:34.501+0000 7fee56909700 1 -- 172.21.15.99:0/1262182924 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fee50069420 con 0x7fee50106630 2024-03-19T01:31:35.373 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:34.501+0000 7fee4d7fa700 1 -- 172.21.15.99:0/1262182924 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 5) ==== 46106+0+0 (secure 0 0 0) 0x7fee4001a8c0 con 0x7fee50106630 2024-03-19T01:31:35.373 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:34.501+0000 7fee4d7fa700 1 --2- 172.21.15.99:0/1262182924 >> [v2:172.21.15.99:6800/4223048293,v1:172.21.15.99:6801/4223048293] conn(0x7fee38038ed0 0x7fee3803b390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:31:35.373 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:34.501+0000 7fee4d7fa700 1 -- 172.21.15.99:0/1262182924 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7fee4004d480 con 0x7fee50106630 2024-03-19T01:31:35.373 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:34.501+0000 7fee4f7fe700 1 -- 172.21.15.99:0/1262182924 >> [v2:172.21.15.99:6800/4223048293,v1:172.21.15.99:6801/4223048293] conn(0x7fee38038ed0 msgr2=0x7fee3803b390 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.99:6800/4223048293 2024-03-19T01:31:35.373 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:34.501+0000 7fee4f7fe700 1 --2- 172.21.15.99:0/1262182924 >> [v2:172.21.15.99:6800/4223048293,v1:172.21.15.99:6801/4223048293] conn(0x7fee38038ed0 0x7fee3803b390 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2024-03-19T01:31:35.373 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:34.510+0000 7fee4d7fa700 1 -- 172.21.15.99:0/1262182924 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7fee40019c70 con 0x7fee50106630 2024-03-19T01:31:35.373 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:34.702+0000 7fee4f7fe700 1 -- 172.21.15.99:0/1262182924 >> [v2:172.21.15.99:6800/4223048293,v1:172.21.15.99:6801/4223048293] conn(0x7fee38038ed0 msgr2=0x7fee3803b390 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.99:6800/4223048293 2024-03-19T01:31:35.373 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:34.702+0000 7fee4f7fe700 1 --2- 172.21.15.99:0/1262182924 >> [v2:172.21.15.99:6800/4223048293,v1:172.21.15.99:6801/4223048293] conn(0x7fee38038ed0 0x7fee3803b390 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.400000 2024-03-19T01:31:35.373 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:34.769+0000 7fee56909700 1 -- 172.21.15.99:0/1262182924 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "mgr stat"} v 0) -- 0x7fee500008d0 con 0x7fee50106630 2024-03-19T01:31:35.373 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:34.770+0000 7fee4d7fa700 1 -- 172.21.15.99:0/1262182924 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "mgr stat"}]=0 v5) ==== 56+0+88 (secure 0 0 0) 0x7fee4002cbc0 con 0x7fee50106630 2024-03-19T01:31:35.373 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:34.772+0000 7fee56909700 1 -- 172.21.15.99:0/1262182924 >> [v2:172.21.15.99:6800/4223048293,v1:172.21.15.99:6801/4223048293] conn(0x7fee38038ed0 msgr2=0x7fee3803b390 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-03-19T01:31:35.373 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:34.772+0000 7fee56909700 1 --2- 172.21.15.99:0/1262182924 >> [v2:172.21.15.99:6800/4223048293,v1:172.21.15.99:6801/4223048293] conn(0x7fee38038ed0 0x7fee3803b390 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:35.373 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:34.772+0000 7fee56909700 1 -- 172.21.15.99:0/1262182924 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fee50106630 msgr2=0x7fee5019df50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:31:35.374 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:34.772+0000 7fee56909700 1 --2- 172.21.15.99:0/1262182924 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fee50106630 0x7fee5019df50 secure :-1 s=READY pgs=25 cs=0 l=1 rev1=1 crypto rx=0x7fee40011670 tx=0x7fee40011b60 comp rx=0 tx=0).stop 2024-03-19T01:31:35.374 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:34.772+0000 7fee56909700 1 -- 172.21.15.99:0/1262182924 shutdown_connections 2024-03-19T01:31:35.374 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:34.772+0000 7fee56909700 1 --2- 172.21.15.99:0/1262182924 >> [v2:172.21.15.99:6800/4223048293,v1:172.21.15.99:6801/4223048293] conn(0x7fee38038ed0 0x7fee3803b390 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:35.374 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:34.772+0000 7fee56909700 1 --2- 172.21.15.99:0/1262182924 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fee50106630 0x7fee5019df50 unknown :-1 s=CLOSED pgs=25 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:35.374 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:34.772+0000 7fee56909700 1 -- 172.21.15.99:0/1262182924 >> 172.21.15.99:0/1262182924 conn(0x7fee50102270 msgr2=0x7fee50194b30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:31:35.374 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:34.772+0000 7fee56909700 1 -- 172.21.15.99:0/1262182924 shutdown_connections 2024-03-19T01:31:35.374 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:34.772+0000 7fee56909700 1 -- 172.21.15.99:0/1262182924 wait complete. 2024-03-19T01:31:35.374 INFO:teuthology.orchestra.run.smithi099.stdout:Waiting for the mgr to restart... 2024-03-19T01:31:35.374 INFO:teuthology.orchestra.run.smithi099.stdout:Waiting for mgr epoch 5... 2024-03-19T01:31:35.563 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:35 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/1262182924' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-03-19T01:31:35.564 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:35 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:31:35.252+0000 7f8e195ae200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-19T01:31:35.564 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:35 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:31:35.364+0000 7f8e195ae200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-19T01:31:35.564 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:35 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:31:35.462+0000 7f8e195ae200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-19T01:31:35.841 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:35 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:31:35.562+0000 7f8e195ae200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-19T01:31:35.841 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:35 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:31:35.742+0000 7f8e195ae200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-19T01:31:37.094 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:36 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:31:36.717+0000 7f8e195ae200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-19T01:31:37.094 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:36 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:31:36.850+0000 7f8e195ae200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-19T01:31:37.358 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:37 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:31:37.184+0000 7f8e195ae200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-19T01:31:37.358 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:37 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:31:37.277+0000 7f8e195ae200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-19T01:31:37.678 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:37 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:31:37.483+0000 7f8e195ae200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-19T01:31:37.678 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:37 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:31:37.676+0000 7f8e195ae200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-19T01:31:38.234 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:37 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:31:37.968+0000 7f8e195ae200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-19T01:31:38.234 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:38 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:31:38.233+0000 7f8e195ae200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-19T01:31:38.591 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:38 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:31:38.472+0000 7f8e195ae200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-19T01:31:39.591 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:39 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:31:39.269+0000 7f8e195ae200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-19T01:31:40.091 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:39 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:31:39.835+0000 7f8e195ae200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-19T01:31:40.091 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:39 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:31:39.932+0000 7f8e195ae200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-19T01:31:40.591 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:40 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:31:40.147+0000 7f8e195ae200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-19T01:31:40.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:40 smithi099 ceph-mon[62951]: Active manager daemon x restarted 2024-03-19T01:31:40.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:40 smithi099 ceph-mon[62951]: Activating manager daemon x 2024-03-19T01:31:40.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:40 smithi099 ceph-mon[62951]: osdmap e2: 0 total, 0 up, 0 in 2024-03-19T01:31:40.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:40 smithi099 ceph-mon[62951]: mgrmap e6: x(active, starting, since 0.00343647s) 2024-03-19T01:31:40.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:40 smithi099 ceph-mon[62951]: from='mgr.14118 172.21.15.99:0/2932773335' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-19T01:31:40.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:40 smithi099 ceph-mon[62951]: from='mgr.14118 172.21.15.99:0/2932773335' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-19T01:31:40.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:40 smithi099 ceph-mon[62951]: from='mgr.14118 172.21.15.99:0/2932773335' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-19T01:31:40.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:40 smithi099 ceph-mon[62951]: from='mgr.14118 172.21.15.99:0/2932773335' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-19T01:31:40.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:40 smithi099 ceph-mon[62951]: from='mgr.14118 172.21.15.99:0/2932773335' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-19T01:31:40.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:40 smithi099 ceph-mon[62951]: Manager daemon x is now available 2024-03-19T01:31:40.593 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:40 smithi099 ceph-mon[62951]: from='mgr.14118 172.21.15.99:0/2932773335' entity='mgr.x' 2024-03-19T01:31:40.593 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:40 smithi099 ceph-mon[62951]: from='mgr.14118 172.21.15.99:0/2932773335' entity='mgr.x' 2024-03-19T01:31:40.593 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:40 smithi099 ceph-mon[62951]: from='mgr.14118 172.21.15.99:0/2932773335' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:31:40.593 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:40 smithi099 ceph-mon[62951]: from='mgr.14118 172.21.15.99:0/2932773335' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:31:41.456 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:41 smithi099 ceph-mon[62951]: Found migration_current of "None". Setting to last migration. 2024-03-19T01:31:41.456 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:41 smithi099 ceph-mon[62951]: from='mgr.14118 172.21.15.99:0/2932773335' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:31:41.456 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:41 smithi099 ceph-mon[62951]: from='mgr.14118 172.21.15.99:0/2932773335' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-03-19T01:31:41.456 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:41 smithi099 ceph-mon[62951]: from='mgr.14118 172.21.15.99:0/2932773335' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-03-19T01:31:41.456 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:41 smithi099 ceph-mon[62951]: from='mgr.14118 172.21.15.99:0/2932773335' entity='mgr.x' 2024-03-19T01:31:41.456 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:41 smithi099 ceph-mon[62951]: from='mgr.14118 172.21.15.99:0/2932773335' entity='mgr.x' 2024-03-19T01:31:41.457 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:41 smithi099 ceph-mon[62951]: mgrmap e7: x(active, since 1.00539s) 2024-03-19T01:31:41.725 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout { 2024-03-19T01:31:41.726 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2024-03-19T01:31:41.726 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "initialized": true 2024-03-19T01:31:41.726 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout } 2024-03-19T01:31:41.726 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:35.779+0000 7f1638c6c700 1 Processor -- start 2024-03-19T01:31:41.726 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:35.779+0000 7f1638c6c700 1 -- start start 2024-03-19T01:31:41.726 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:35.779+0000 7f1638c6c700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1634108860 0x7f163410ac50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:31:41.726 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:35.779+0000 7f1638c6c700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f163407be70 con 0x7f1634108860 2024-03-19T01:31:41.726 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:35.779+0000 7f163259c700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1634108860 0x7f163410ac50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:31:41.726 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:35.780+0000 7f163259c700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1634108860 0x7f163410ac50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:50948/0 (socket says 172.21.15.99:50948) 2024-03-19T01:31:41.726 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:35.780+0000 7f163259c700 1 -- 172.21.15.99:0/1291859213 learned_addr learned my addr 172.21.15.99:0/1291859213 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:31:41.726 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:35.780+0000 7f163259c700 1 -- 172.21.15.99:0/1291859213 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f163410b190 con 0x7f1634108860 2024-03-19T01:31:41.727 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:35.780+0000 7f163259c700 1 --2- 172.21.15.99:0/1291859213 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1634108860 0x7f163410ac50 secure :-1 s=READY pgs=26 cs=0 l=1 rev1=1 crypto rx=0x7f16280098a0 tx=0x7f1628009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e2ba33ec526880eb server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:31:41.727 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:35.780+0000 7f163159a700 1 -- 172.21.15.99:0/1291859213 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f1628004160 con 0x7f1634108860 2024-03-19T01:31:41.727 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:35.780+0000 7f163159a700 1 -- 172.21.15.99:0/1291859213 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7f1628004300 con 0x7f1634108860 2024-03-19T01:31:41.727 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:35.780+0000 7f163159a700 1 -- 172.21.15.99:0/1291859213 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f1628011b50 con 0x7f1634108860 2024-03-19T01:31:41.727 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:35.781+0000 7f1638c6c700 1 -- 172.21.15.99:0/1291859213 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1634108860 msgr2=0x7f163410ac50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:31:41.727 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:35.781+0000 7f1638c6c700 1 --2- 172.21.15.99:0/1291859213 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1634108860 0x7f163410ac50 secure :-1 s=READY pgs=26 cs=0 l=1 rev1=1 crypto rx=0x7f16280098a0 tx=0x7f1628009bb0 comp rx=0 tx=0).stop 2024-03-19T01:31:41.727 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:35.781+0000 7f1638c6c700 1 -- 172.21.15.99:0/1291859213 shutdown_connections 2024-03-19T01:31:41.727 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:35.781+0000 7f1638c6c700 1 --2- 172.21.15.99:0/1291859213 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1634108860 0x7f163410ac50 unknown :-1 s=CLOSED pgs=26 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:41.727 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:35.781+0000 7f1638c6c700 1 -- 172.21.15.99:0/1291859213 >> 172.21.15.99:0/1291859213 conn(0x7f1634102270 msgr2=0x7f1634104690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:31:41.727 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:35.781+0000 7f1638c6c700 1 -- 172.21.15.99:0/1291859213 shutdown_connections 2024-03-19T01:31:41.727 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:35.781+0000 7f1638c6c700 1 -- 172.21.15.99:0/1291859213 wait complete. 2024-03-19T01:31:41.728 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:35.782+0000 7f1638c6c700 1 Processor -- start 2024-03-19T01:31:41.728 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:35.782+0000 7f1638c6c700 1 -- start start 2024-03-19T01:31:41.728 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:35.782+0000 7f1638c6c700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1634108860 0x7f163419bd00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:31:41.728 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:35.782+0000 7f1638c6c700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f163410c160 con 0x7f1634108860 2024-03-19T01:31:41.728 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:35.782+0000 7f163259c700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1634108860 0x7f163419bd00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:31:41.728 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:35.782+0000 7f163259c700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1634108860 0x7f163419bd00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:50952/0 (socket says 172.21.15.99:50952) 2024-03-19T01:31:41.728 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:35.782+0000 7f163259c700 1 -- 172.21.15.99:0/1197927612 learned_addr learned my addr 172.21.15.99:0/1197927612 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:31:41.728 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:35.782+0000 7f163259c700 1 -- 172.21.15.99:0/1197927612 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f163419c240 con 0x7f1634108860 2024-03-19T01:31:41.728 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:35.782+0000 7f163259c700 1 --2- 172.21.15.99:0/1197927612 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1634108860 0x7f163419bd00 secure :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0x7f1628011670 tx=0x7f1628011b60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:31:41.728 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:35.783+0000 7f16237fe700 1 -- 172.21.15.99:0/1197927612 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f16280039c0 con 0x7f1634108860 2024-03-19T01:31:41.728 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:35.783+0000 7f1638c6c700 1 -- 172.21.15.99:0/1197927612 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f163419c4d0 con 0x7f1634108860 2024-03-19T01:31:41.728 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:35.783+0000 7f16237fe700 1 -- 172.21.15.99:0/1197927612 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7f1628011e50 con 0x7f1634108860 2024-03-19T01:31:41.728 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:35.783+0000 7f16237fe700 1 -- 172.21.15.99:0/1197927612 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f162801a5c0 con 0x7f1634108860 2024-03-19T01:31:41.729 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:35.783+0000 7f1638c6c700 1 -- 172.21.15.99:0/1197927612 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f163419f1b0 con 0x7f1634108860 2024-03-19T01:31:41.729 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:35.784+0000 7f16237fe700 1 -- 172.21.15.99:0/1197927612 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 5) ==== 46106+0+0 (secure 0 0 0) 0x7f162801a870 con 0x7f1634108860 2024-03-19T01:31:41.729 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:35.784+0000 7f16237fe700 1 --2- 172.21.15.99:0/1197927612 >> [v2:172.21.15.99:6800/4223048293,v1:172.21.15.99:6801/4223048293] conn(0x7f161c038f20 0x7f161c03b3e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:31:41.729 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:35.784+0000 7f16237fe700 1 -- 172.21.15.99:0/1197927612 --> [v2:172.21.15.99:6800/4223048293,v1:172.21.15.99:6801/4223048293] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x7f161c03baf0 con 0x7f161c038f20 2024-03-19T01:31:41.729 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:35.784+0000 7f16237fe700 1 -- 172.21.15.99:0/1197927612 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7f162804d080 con 0x7f1634108860 2024-03-19T01:31:41.729 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:35.784+0000 7f1631d9b700 1 -- 172.21.15.99:0/1197927612 >> [v2:172.21.15.99:6800/4223048293,v1:172.21.15.99:6801/4223048293] conn(0x7f161c038f20 msgr2=0x7f161c03b3e0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.99:6800/4223048293 2024-03-19T01:31:41.729 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:35.784+0000 7f1631d9b700 1 --2- 172.21.15.99:0/1197927612 >> [v2:172.21.15.99:6800/4223048293,v1:172.21.15.99:6801/4223048293] conn(0x7f161c038f20 0x7f161c03b3e0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2024-03-19T01:31:41.729 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:35.985+0000 7f1631d9b700 1 -- 172.21.15.99:0/1197927612 >> [v2:172.21.15.99:6800/4223048293,v1:172.21.15.99:6801/4223048293] conn(0x7f161c038f20 msgr2=0x7f161c03b3e0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.99:6800/4223048293 2024-03-19T01:31:41.729 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:35.985+0000 7f1631d9b700 1 --2- 172.21.15.99:0/1197927612 >> [v2:172.21.15.99:6800/4223048293,v1:172.21.15.99:6801/4223048293] conn(0x7f161c038f20 0x7f161c03b3e0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.400000 2024-03-19T01:31:41.729 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:36.386+0000 7f1631d9b700 1 -- 172.21.15.99:0/1197927612 >> [v2:172.21.15.99:6800/4223048293,v1:172.21.15.99:6801/4223048293] conn(0x7f161c038f20 msgr2=0x7f161c03b3e0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.99:6800/4223048293 2024-03-19T01:31:41.729 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:36.386+0000 7f1631d9b700 1 --2- 172.21.15.99:0/1197927612 >> [v2:172.21.15.99:6800/4223048293,v1:172.21.15.99:6801/4223048293] conn(0x7f161c038f20 0x7f161c03b3e0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.800000 2024-03-19T01:31:41.729 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:37.187+0000 7f1631d9b700 1 -- 172.21.15.99:0/1197927612 >> [v2:172.21.15.99:6800/4223048293,v1:172.21.15.99:6801/4223048293] conn(0x7f161c038f20 msgr2=0x7f161c03b3e0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.99:6800/4223048293 2024-03-19T01:31:41.729 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:37.187+0000 7f1631d9b700 1 --2- 172.21.15.99:0/1197927612 >> [v2:172.21.15.99:6800/4223048293,v1:172.21.15.99:6801/4223048293] conn(0x7f161c038f20 0x7f161c03b3e0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 1.600000 2024-03-19T01:31:41.730 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:38.789+0000 7f1631d9b700 1 -- 172.21.15.99:0/1197927612 >> [v2:172.21.15.99:6800/4223048293,v1:172.21.15.99:6801/4223048293] conn(0x7f161c038f20 msgr2=0x7f161c03b3e0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.99:6800/4223048293 2024-03-19T01:31:41.730 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:38.789+0000 7f1631d9b700 1 --2- 172.21.15.99:0/1197927612 >> [v2:172.21.15.99:6800/4223048293,v1:172.21.15.99:6801/4223048293] conn(0x7f161c038f20 0x7f161c03b3e0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 3.200000 2024-03-19T01:31:41.730 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:40.163+0000 7f16237fe700 1 -- 172.21.15.99:0/1197927612 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mgrmap(e 6) ==== 45873+0+0 (secure 0 0 0) 0x7f162801ab60 con 0x7f1634108860 2024-03-19T01:31:41.730 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:40.163+0000 7f16237fe700 1 -- 172.21.15.99:0/1197927612 >> [v2:172.21.15.99:6800/4223048293,v1:172.21.15.99:6801/4223048293] conn(0x7f161c038f20 msgr2=0x7f161c03b3e0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-03-19T01:31:41.730 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:40.163+0000 7f16237fe700 1 --2- 172.21.15.99:0/1197927612 >> [v2:172.21.15.99:6800/4223048293,v1:172.21.15.99:6801/4223048293] conn(0x7f161c038f20 0x7f161c03b3e0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:41.730 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:41.164+0000 7f16237fe700 1 -- 172.21.15.99:0/1197927612 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mgrmap(e 7) ==== 46000+0+0 (secure 0 0 0) 0x7f162804ef90 con 0x7f1634108860 2024-03-19T01:31:41.730 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:41.164+0000 7f16237fe700 1 --2- 172.21.15.99:0/1197927612 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7f161c03c810 0x7f161c03ec00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:31:41.730 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:41.164+0000 7f16237fe700 1 -- 172.21.15.99:0/1197927612 --> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x7f161c03baf0 con 0x7f161c03c810 2024-03-19T01:31:41.730 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:41.164+0000 7f1631d9b700 1 --2- 172.21.15.99:0/1197927612 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7f161c03c810 0x7f161c03ec00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:31:41.730 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:41.165+0000 7f1631d9b700 1 --2- 172.21.15.99:0/1197927612 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7f161c03c810 0x7f161c03ec00 secure :-1 s=READY pgs=1 cs=0 l=1 rev1=1 crypto rx=0x7f1624003a10 tx=0x7f16240092b0 comp rx=0 tx=0).ready entity=mgr.14118 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:31:41.730 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:41.166+0000 7f16237fe700 1 -- 172.21.15.99:0/1197927612 <== mgr.14118 v2:172.21.15.99:6800/1328245934 1 ==== command_reply(tid 0: 0 ) ==== 8+0+7774 (secure 0 0 0) 0x7f161c03baf0 con 0x7f161c03c810 2024-03-19T01:31:41.730 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:41.172+0000 7f1638c6c700 1 -- 172.21.15.99:0/1197927612 --> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] -- command(tid 1: {"prefix": "mgr_status"}) -- 0x7f1634069420 con 0x7f161c03c810 2024-03-19T01:31:41.730 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:41.172+0000 7f16237fe700 1 -- 172.21.15.99:0/1197927612 <== mgr.14118 v2:172.21.15.99:6800/1328245934 2 ==== command_reply(tid 1: 0 ) ==== 8+0+51 (secure 0 0 0) 0x7f1634069420 con 0x7f161c03c810 2024-03-19T01:31:41.731 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:41.172+0000 7f1638c6c700 1 -- 172.21.15.99:0/1197927612 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7f161c03c810 msgr2=0x7f161c03ec00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:31:41.731 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:41.172+0000 7f1638c6c700 1 --2- 172.21.15.99:0/1197927612 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7f161c03c810 0x7f161c03ec00 secure :-1 s=READY pgs=1 cs=0 l=1 rev1=1 crypto rx=0x7f1624003a10 tx=0x7f16240092b0 comp rx=0 tx=0).stop 2024-03-19T01:31:41.731 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:41.172+0000 7f1638c6c700 1 -- 172.21.15.99:0/1197927612 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1634108860 msgr2=0x7f163419bd00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:31:41.731 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:41.172+0000 7f1638c6c700 1 --2- 172.21.15.99:0/1197927612 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1634108860 0x7f163419bd00 secure :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0x7f1628011670 tx=0x7f1628011b60 comp rx=0 tx=0).stop 2024-03-19T01:31:41.731 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:41.173+0000 7f1638c6c700 1 -- 172.21.15.99:0/1197927612 shutdown_connections 2024-03-19T01:31:41.731 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:41.173+0000 7f1638c6c700 1 --2- 172.21.15.99:0/1197927612 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7f161c03c810 0x7f161c03ec00 unknown :-1 s=CLOSED pgs=1 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:41.731 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:41.173+0000 7f1638c6c700 1 --2- 172.21.15.99:0/1197927612 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1634108860 0x7f163419bd00 unknown :-1 s=CLOSED pgs=27 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:41.731 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:41.173+0000 7f1638c6c700 1 --2- 172.21.15.99:0/1197927612 >> [v2:172.21.15.99:6800/4223048293,v1:172.21.15.99:6801/4223048293] conn(0x7f161c038f20 0x7f161c03b3e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:41.731 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:41.173+0000 7f1638c6c700 1 -- 172.21.15.99:0/1197927612 >> 172.21.15.99:0/1197927612 conn(0x7f1634102270 msgr2=0x7f1634102d60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:31:41.731 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:41.173+0000 7f1638c6c700 1 -- 172.21.15.99:0/1197927612 shutdown_connections 2024-03-19T01:31:41.731 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:41.173+0000 7f1638c6c700 1 -- 172.21.15.99:0/1197927612 wait complete. 2024-03-19T01:31:41.731 INFO:teuthology.orchestra.run.smithi099.stdout:mgr epoch 5 is available 2024-03-19T01:31:41.731 INFO:teuthology.orchestra.run.smithi099.stdout:Setting orchestrator backend to cephadm... 2024-03-19T01:31:42.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:42 smithi099 ceph-mon[62951]: [19/Mar/2024:01:31:40] ENGINE Bus STARTING 2024-03-19T01:31:42.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:42 smithi099 ceph-mon[62951]: [19/Mar/2024:01:31:41] ENGINE Serving on http://172.21.15.99:8765 2024-03-19T01:31:42.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:42 smithi099 ceph-mon[62951]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-03-19T01:31:42.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:42 smithi099 ceph-mon[62951]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-03-19T01:31:42.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:42 smithi099 ceph-mon[62951]: [19/Mar/2024:01:31:41] ENGINE Serving on https://172.21.15.99:7150 2024-03-19T01:31:42.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:42 smithi099 ceph-mon[62951]: [19/Mar/2024:01:31:41] ENGINE Bus STARTED 2024-03-19T01:31:42.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:42 smithi099 ceph-mon[62951]: from='mgr.14118 172.21.15.99:0/2932773335' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:31:43.005 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:42.148+0000 7f87ee075700 1 Processor -- start 2024-03-19T01:31:43.005 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:42.148+0000 7f87ee075700 1 -- start start 2024-03-19T01:31:43.005 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:42.148+0000 7f87ee075700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f87e81088f0 0x7f87e810ace0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:31:43.005 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:42.148+0000 7f87ee075700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f87e807be10 con 0x7f87e81088f0 2024-03-19T01:31:43.005 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:42.149+0000 7f87e77fe700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f87e81088f0 0x7f87e810ace0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:31:43.005 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:42.149+0000 7f87e77fe700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f87e81088f0 0x7f87e810ace0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:51008/0 (socket says 172.21.15.99:51008) 2024-03-19T01:31:43.005 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:42.149+0000 7f87e77fe700 1 -- 172.21.15.99:0/1534245520 learned_addr learned my addr 172.21.15.99:0/1534245520 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:31:43.005 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:42.149+0000 7f87e77fe700 1 -- 172.21.15.99:0/1534245520 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f87e810b220 con 0x7f87e81088f0 2024-03-19T01:31:43.005 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:42.149+0000 7f87e77fe700 1 --2- 172.21.15.99:0/1534245520 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f87e81088f0 0x7f87e810ace0 secure :-1 s=READY pgs=35 cs=0 l=1 rev1=1 crypto rx=0x7f87d80098a0 tx=0x7f87d8009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9864e6b22f3a6ef6 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:31:43.005 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:42.149+0000 7f87e67fc700 1 -- 172.21.15.99:0/1534245520 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f87d8004160 con 0x7f87e81088f0 2024-03-19T01:31:43.005 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:42.149+0000 7f87e67fc700 1 -- 172.21.15.99:0/1534245520 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7f87d8004300 con 0x7f87e81088f0 2024-03-19T01:31:43.006 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:42.149+0000 7f87e67fc700 1 -- 172.21.15.99:0/1534245520 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f87d8011b50 con 0x7f87e81088f0 2024-03-19T01:31:43.006 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:42.150+0000 7f87ee075700 1 -- 172.21.15.99:0/1534245520 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f87e81088f0 msgr2=0x7f87e810ace0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:31:43.006 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:42.150+0000 7f87ee075700 1 --2- 172.21.15.99:0/1534245520 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f87e81088f0 0x7f87e810ace0 secure :-1 s=READY pgs=35 cs=0 l=1 rev1=1 crypto rx=0x7f87d80098a0 tx=0x7f87d8009bb0 comp rx=0 tx=0).stop 2024-03-19T01:31:43.006 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:42.150+0000 7f87ee075700 1 -- 172.21.15.99:0/1534245520 shutdown_connections 2024-03-19T01:31:43.006 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:42.150+0000 7f87ee075700 1 --2- 172.21.15.99:0/1534245520 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f87e81088f0 0x7f87e810ace0 unknown :-1 s=CLOSED pgs=35 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:43.006 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:42.150+0000 7f87ee075700 1 -- 172.21.15.99:0/1534245520 >> 172.21.15.99:0/1534245520 conn(0x7f87e8102260 msgr2=0x7f87e81046a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:31:43.006 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:42.150+0000 7f87ee075700 1 -- 172.21.15.99:0/1534245520 shutdown_connections 2024-03-19T01:31:43.006 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:42.150+0000 7f87ee075700 1 -- 172.21.15.99:0/1534245520 wait complete. 2024-03-19T01:31:43.006 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:42.151+0000 7f87ee075700 1 Processor -- start 2024-03-19T01:31:43.006 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:42.151+0000 7f87ee075700 1 -- start start 2024-03-19T01:31:43.006 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:42.151+0000 7f87ee075700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f87e81088f0 0x7f87e819bd50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:31:43.006 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:42.151+0000 7f87ee075700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f87e810c190 con 0x7f87e81088f0 2024-03-19T01:31:43.007 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:42.151+0000 7f87e77fe700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f87e81088f0 0x7f87e819bd50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:31:43.007 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:42.151+0000 7f87e77fe700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f87e81088f0 0x7f87e819bd50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:51010/0 (socket says 172.21.15.99:51010) 2024-03-19T01:31:43.007 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:42.151+0000 7f87e77fe700 1 -- 172.21.15.99:0/1953851660 learned_addr learned my addr 172.21.15.99:0/1953851660 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:31:43.007 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:42.151+0000 7f87e77fe700 1 -- 172.21.15.99:0/1953851660 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f87e819c290 con 0x7f87e81088f0 2024-03-19T01:31:43.007 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:42.151+0000 7f87e77fe700 1 --2- 172.21.15.99:0/1953851660 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f87e81088f0 0x7f87e819bd50 secure :-1 s=READY pgs=36 cs=0 l=1 rev1=1 crypto rx=0x7f87d8011670 tx=0x7f87d8011b60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:31:43.007 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:42.152+0000 7f87e4ff9700 1 -- 172.21.15.99:0/1953851660 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f87d80039c0 con 0x7f87e81088f0 2024-03-19T01:31:43.007 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:42.152+0000 7f87e4ff9700 1 -- 172.21.15.99:0/1953851660 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7f87d8011e50 con 0x7f87e81088f0 2024-03-19T01:31:43.007 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:42.152+0000 7f87ee075700 1 -- 172.21.15.99:0/1953851660 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f87e819c520 con 0x7f87e81088f0 2024-03-19T01:31:43.007 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:42.152+0000 7f87e4ff9700 1 -- 172.21.15.99:0/1953851660 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f87d801a5c0 con 0x7f87e81088f0 2024-03-19T01:31:43.007 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:42.152+0000 7f87ee075700 1 -- 172.21.15.99:0/1953851660 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f87e819f210 con 0x7f87e81088f0 2024-03-19T01:31:43.007 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:42.152+0000 7f87e4ff9700 1 -- 172.21.15.99:0/1953851660 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 7) ==== 46000+0+0 (secure 0 0 0) 0x7f87d801a8c0 con 0x7f87e81088f0 2024-03-19T01:31:43.007 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:42.153+0000 7f87ee075700 1 -- 172.21.15.99:0/1953851660 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f87e8063940 con 0x7f87e81088f0 2024-03-19T01:31:43.007 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:42.153+0000 7f87e4ff9700 1 --2- 172.21.15.99:0/1953851660 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7f87d0038dc0 0x7f87d003b280 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:31:43.007 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:42.153+0000 7f87e4ff9700 1 -- 172.21.15.99:0/1953851660 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(2..2 src has 1..2) ==== 940+0+0 (secure 0 0 0) 0x7f87d804d050 con 0x7f87e81088f0 2024-03-19T01:31:43.008 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:42.153+0000 7f87e6ffd700 1 --2- 172.21.15.99:0/1953851660 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7f87d0038dc0 0x7f87d003b280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:31:43.008 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:42.153+0000 7f87e6ffd700 1 --2- 172.21.15.99:0/1953851660 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7f87d0038dc0 0x7f87d003b280 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7f87dc006fd0 tx=0x7f87dc006e40 comp rx=0 tx=0).ready entity=mgr.14118 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:31:43.008 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:42.158+0000 7f87e4ff9700 1 -- 172.21.15.99:0/1953851660 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f87d8019c70 con 0x7f87e81088f0 2024-03-19T01:31:43.008 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:42.218+0000 7f87e4ff9700 1 -- 172.21.15.99:0/1953851660 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mgrmap(e 8) ==== 46106+0+0 (secure 0 0 0) 0x7f87d802d910 con 0x7f87e81088f0 2024-03-19T01:31:43.008 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:42.370+0000 7f87ee075700 1 -- 172.21.15.99:0/1953851660 --> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] -- mgr_command(tid 0: {"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}) -- 0x7f87e8103490 con 0x7f87d0038dc0 2024-03-19T01:31:43.008 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:42.379+0000 7f87e4ff9700 1 -- 172.21.15.99:0/1953851660 <== mgr.14118 v2:172.21.15.99:6800/1328245934 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+0 (secure 0 0 0) 0x7f87e8103490 con 0x7f87d0038dc0 2024-03-19T01:31:43.008 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:42.381+0000 7f87ee075700 1 -- 172.21.15.99:0/1953851660 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7f87d0038dc0 msgr2=0x7f87d003b280 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:31:43.008 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:42.381+0000 7f87ee075700 1 --2- 172.21.15.99:0/1953851660 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7f87d0038dc0 0x7f87d003b280 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7f87dc006fd0 tx=0x7f87dc006e40 comp rx=0 tx=0).stop 2024-03-19T01:31:43.008 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:42.381+0000 7f87ee075700 1 -- 172.21.15.99:0/1953851660 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f87e81088f0 msgr2=0x7f87e819bd50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:31:43.008 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:42.381+0000 7f87ee075700 1 --2- 172.21.15.99:0/1953851660 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f87e81088f0 0x7f87e819bd50 secure :-1 s=READY pgs=36 cs=0 l=1 rev1=1 crypto rx=0x7f87d8011670 tx=0x7f87d8011b60 comp rx=0 tx=0).stop 2024-03-19T01:31:43.008 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:42.381+0000 7f87ee075700 1 -- 172.21.15.99:0/1953851660 shutdown_connections 2024-03-19T01:31:43.008 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:42.381+0000 7f87ee075700 1 --2- 172.21.15.99:0/1953851660 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7f87d0038dc0 0x7f87d003b280 unknown :-1 s=CLOSED pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:43.008 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:42.381+0000 7f87ee075700 1 --2- 172.21.15.99:0/1953851660 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f87e81088f0 0x7f87e819bd50 unknown :-1 s=CLOSED pgs=36 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:43.008 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:42.381+0000 7f87ee075700 1 -- 172.21.15.99:0/1953851660 >> 172.21.15.99:0/1953851660 conn(0x7f87e8102260 msgr2=0x7f87e8102d80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:31:43.009 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:42.381+0000 7f87ee075700 1 -- 172.21.15.99:0/1953851660 shutdown_connections 2024-03-19T01:31:43.009 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:42.381+0000 7f87ee075700 1 -- 172.21.15.99:0/1953851660 wait complete. 2024-03-19T01:31:43.299 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:43 smithi099 ceph-mon[62951]: mgrmap e8: x(active, since 2s) 2024-03-19T01:31:43.300 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:43 smithi099 ceph-mon[62951]: from='mgr.14118 172.21.15.99:0/2932773335' entity='mgr.x' 2024-03-19T01:31:43.300 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:43 smithi099 ceph-mon[62951]: from='mgr.14118 172.21.15.99:0/2932773335' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:31:44.203 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout value unchanged 2024-03-19T01:31:44.203 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:43.405+0000 7fc1f620a700 1 Processor -- start 2024-03-19T01:31:44.204 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:43.405+0000 7fc1f620a700 1 -- start start 2024-03-19T01:31:44.204 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:43.405+0000 7fc1f620a700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc1f0107dd0 0x7fc1f01081f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:31:44.204 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:43.405+0000 7fc1f620a700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fc1f01087c0 con 0x7fc1f0107dd0 2024-03-19T01:31:44.204 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:43.406+0000 7fc1ef7fe700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc1f0107dd0 0x7fc1f01081f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:31:44.204 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:43.406+0000 7fc1ef7fe700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc1f0107dd0 0x7fc1f01081f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:45512/0 (socket says 172.21.15.99:45512) 2024-03-19T01:31:44.204 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:43.406+0000 7fc1ef7fe700 1 -- 172.21.15.99:0/170114212 learned_addr learned my addr 172.21.15.99:0/170114212 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:31:44.204 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:43.406+0000 7fc1ef7fe700 1 -- 172.21.15.99:0/170114212 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fc1f0108fd0 con 0x7fc1f0107dd0 2024-03-19T01:31:44.204 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:43.406+0000 7fc1ef7fe700 1 --2- 172.21.15.99:0/170114212 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc1f0107dd0 0x7fc1f01081f0 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 crypto rx=0x7fc1e0009e80 tx=0x7fc1e000b330 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b96d041862a04f9 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:31:44.204 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:43.406+0000 7fc1ee7fc700 1 -- 172.21.15.99:0/170114212 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fc1e0017070 con 0x7fc1f0107dd0 2024-03-19T01:31:44.204 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:43.406+0000 7fc1ee7fc700 1 -- 172.21.15.99:0/170114212 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7fc1e000b8c0 con 0x7fc1f0107dd0 2024-03-19T01:31:44.204 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:43.406+0000 7fc1ee7fc700 1 -- 172.21.15.99:0/170114212 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fc1e00045c0 con 0x7fc1f0107dd0 2024-03-19T01:31:44.204 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:43.407+0000 7fc1f620a700 1 -- 172.21.15.99:0/170114212 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc1f0107dd0 msgr2=0x7fc1f01081f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:31:44.204 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:43.407+0000 7fc1f620a700 1 --2- 172.21.15.99:0/170114212 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc1f0107dd0 0x7fc1f01081f0 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 crypto rx=0x7fc1e0009e80 tx=0x7fc1e000b330 comp rx=0 tx=0).stop 2024-03-19T01:31:44.205 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:43.407+0000 7fc1f620a700 1 -- 172.21.15.99:0/170114212 shutdown_connections 2024-03-19T01:31:44.205 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:43.407+0000 7fc1f620a700 1 --2- 172.21.15.99:0/170114212 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc1f0107dd0 0x7fc1f01081f0 unknown :-1 s=CLOSED pgs=37 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:44.205 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:43.407+0000 7fc1f620a700 1 -- 172.21.15.99:0/170114212 >> 172.21.15.99:0/170114212 conn(0x7fc1f0103390 msgr2=0x7fc1f01057b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:31:44.205 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:43.407+0000 7fc1f620a700 1 -- 172.21.15.99:0/170114212 shutdown_connections 2024-03-19T01:31:44.205 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:43.407+0000 7fc1f620a700 1 -- 172.21.15.99:0/170114212 wait complete. 2024-03-19T01:31:44.205 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:43.407+0000 7fc1f620a700 1 Processor -- start 2024-03-19T01:31:44.205 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:43.408+0000 7fc1f620a700 1 -- start start 2024-03-19T01:31:44.205 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:43.408+0000 7fc1f620a700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc1f0107dd0 0x7fc1f007db90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:31:44.205 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:43.408+0000 7fc1f620a700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fc1f010c380 con 0x7fc1f0107dd0 2024-03-19T01:31:44.205 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:43.408+0000 7fc1ef7fe700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc1f0107dd0 0x7fc1f007db90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:31:44.205 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:43.408+0000 7fc1ef7fe700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc1f0107dd0 0x7fc1f007db90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:45516/0 (socket says 172.21.15.99:45516) 2024-03-19T01:31:44.205 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:43.408+0000 7fc1ef7fe700 1 -- 172.21.15.99:0/836959593 learned_addr learned my addr 172.21.15.99:0/836959593 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:31:44.205 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:43.408+0000 7fc1ef7fe700 1 -- 172.21.15.99:0/836959593 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fc1f007e0d0 con 0x7fc1f0107dd0 2024-03-19T01:31:44.205 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:43.408+0000 7fc1ef7fe700 1 --2- 172.21.15.99:0/836959593 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc1f0107dd0 0x7fc1f007db90 secure :-1 s=READY pgs=38 cs=0 l=1 rev1=1 crypto rx=0x7fc1e0009e50 tx=0x7fc1e000bd30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:31:44.205 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:43.409+0000 7fc1ecff9700 1 -- 172.21.15.99:0/836959593 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fc1e0020020 con 0x7fc1f0107dd0 2024-03-19T01:31:44.206 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:43.409+0000 7fc1ecff9700 1 -- 172.21.15.99:0/836959593 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7fc1e0003b50 con 0x7fc1f0107dd0 2024-03-19T01:31:44.206 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:43.409+0000 7fc1f620a700 1 -- 172.21.15.99:0/836959593 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fc1f007c2c0 con 0x7fc1f0107dd0 2024-03-19T01:31:44.206 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:43.409+0000 7fc1ecff9700 1 -- 172.21.15.99:0/836959593 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fc1e0017040 con 0x7fc1f0107dd0 2024-03-19T01:31:44.206 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:43.409+0000 7fc1f620a700 1 -- 172.21.15.99:0/836959593 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fc1f007c800 con 0x7fc1f0107dd0 2024-03-19T01:31:44.206 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:43.409+0000 7fc1ecff9700 1 -- 172.21.15.99:0/836959593 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 8) ==== 46106+0+0 (secure 0 0 0) 0x7fc1e0023020 con 0x7fc1f0107dd0 2024-03-19T01:31:44.206 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:43.410+0000 7fc1ecff9700 1 --2- 172.21.15.99:0/836959593 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7fc1d8038ed0 0x7fc1d803b390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:31:44.206 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:43.410+0000 7fc1f620a700 1 -- 172.21.15.99:0/836959593 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fc1f0069420 con 0x7fc1f0107dd0 2024-03-19T01:31:44.206 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:43.410+0000 7fc1ecff9700 1 -- 172.21.15.99:0/836959593 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(2..2 src has 1..2) ==== 940+0+0 (secure 0 0 0) 0x7fc1e000f050 con 0x7fc1f0107dd0 2024-03-19T01:31:44.206 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:43.410+0000 7fc1eeffd700 1 --2- 172.21.15.99:0/836959593 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7fc1d8038ed0 0x7fc1d803b390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:31:44.206 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:43.410+0000 7fc1eeffd700 1 --2- 172.21.15.99:0/836959593 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7fc1d8038ed0 0x7fc1d803b390 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7fc1e4006fd0 tx=0x7fc1e4006e40 comp rx=0 tx=0).ready entity=mgr.14118 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:31:44.206 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:43.415+0000 7fc1ecff9700 1 -- 172.21.15.99:0/836959593 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7fc1e002a020 con 0x7fc1f0107dd0 2024-03-19T01:31:44.206 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:43.612+0000 7fc1f620a700 1 -- 172.21.15.99:0/836959593 --> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] -- mgr_command(tid 0: {"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}) -- 0x7fc1f01045f0 con 0x7fc1d8038ed0 2024-03-19T01:31:44.207 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:43.613+0000 7fc1ecff9700 1 -- 172.21.15.99:0/836959593 <== mgr.14118 v2:172.21.15.99:6800/1328245934 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+16 (secure 0 0 0) 0x7fc1f01045f0 con 0x7fc1d8038ed0 2024-03-19T01:31:44.207 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:43.615+0000 7fc1f620a700 1 -- 172.21.15.99:0/836959593 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7fc1d8038ed0 msgr2=0x7fc1d803b390 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:31:44.207 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:43.615+0000 7fc1f620a700 1 --2- 172.21.15.99:0/836959593 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7fc1d8038ed0 0x7fc1d803b390 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7fc1e4006fd0 tx=0x7fc1e4006e40 comp rx=0 tx=0).stop 2024-03-19T01:31:44.207 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:43.615+0000 7fc1f620a700 1 -- 172.21.15.99:0/836959593 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc1f0107dd0 msgr2=0x7fc1f007db90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:31:44.207 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:43.615+0000 7fc1f620a700 1 --2- 172.21.15.99:0/836959593 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc1f0107dd0 0x7fc1f007db90 secure :-1 s=READY pgs=38 cs=0 l=1 rev1=1 crypto rx=0x7fc1e0009e50 tx=0x7fc1e000bd30 comp rx=0 tx=0).stop 2024-03-19T01:31:44.207 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:43.615+0000 7fc1f620a700 1 -- 172.21.15.99:0/836959593 shutdown_connections 2024-03-19T01:31:44.207 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:43.615+0000 7fc1f620a700 1 --2- 172.21.15.99:0/836959593 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7fc1d8038ed0 0x7fc1d803b390 unknown :-1 s=CLOSED pgs=8 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:44.207 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:43.615+0000 7fc1f620a700 1 --2- 172.21.15.99:0/836959593 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc1f0107dd0 0x7fc1f007db90 unknown :-1 s=CLOSED pgs=38 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:44.207 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:43.615+0000 7fc1f620a700 1 -- 172.21.15.99:0/836959593 >> 172.21.15.99:0/836959593 conn(0x7fc1f0103390 msgr2=0x7fc1f0103ee0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:31:44.207 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:43.615+0000 7fc1f620a700 1 -- 172.21.15.99:0/836959593 shutdown_connections 2024-03-19T01:31:44.207 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:43.615+0000 7fc1f620a700 1 -- 172.21.15.99:0/836959593 wait complete. 2024-03-19T01:31:44.207 INFO:teuthology.orchestra.run.smithi099.stdout:Generating ssh key... 2024-03-19T01:31:44.531 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:44 smithi099 ceph-mon[62951]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:31:45.092 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:45 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: Generating public/private rsa key pair. 2024-03-19T01:31:45.092 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:45 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: Your identification has been saved in /tmp/tmp4lbua9cy/key. 2024-03-19T01:31:45.092 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:45 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: Your public key has been saved in /tmp/tmp4lbua9cy/key.pub. 2024-03-19T01:31:45.092 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:45 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: The key fingerprint is: 2024-03-19T01:31:45.092 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:45 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: SHA256:D8kLNWiHhAWKjHmDlxte40oBgYguZbTO9l0zfbsCPIU ceph-438fb69c-e590-11ee-95c9-87774f69a715 2024-03-19T01:31:45.092 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:45 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: The key's randomart image is: 2024-03-19T01:31:45.092 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:45 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: +---[RSA 3072]----+ 2024-03-19T01:31:45.092 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:45 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: |*o. .+o | 2024-03-19T01:31:45.092 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:45 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: |*=o+.. o | 2024-03-19T01:31:45.092 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:45 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: |*+X o + +. | 2024-03-19T01:31:45.092 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:45 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: |.B B o +E+. | 2024-03-19T01:31:45.093 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:45 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: |. B . ..S.. . | 2024-03-19T01:31:45.093 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:45 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: | o o . o+* . . | 2024-03-19T01:31:45.093 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:45 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: | . . . .o. . | 2024-03-19T01:31:45.093 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:45 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: | . . | 2024-03-19T01:31:45.093 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:45 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: | .. | 2024-03-19T01:31:45.093 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:45 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: +----[SHA256]-----+ 2024-03-19T01:31:45.536 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:45 smithi099 ceph-mon[62951]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:31:45.536 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:45 smithi099 ceph-mon[62951]: from='mgr.14118 172.21.15.99:0/2932773335' entity='mgr.x' 2024-03-19T01:31:45.536 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:45 smithi099 ceph-mon[62951]: from='mgr.14118 172.21.15.99:0/2932773335' entity='mgr.x' 2024-03-19T01:31:45.543 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:44.635+0000 7f546eecb700 1 Processor -- start 2024-03-19T01:31:45.543 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:44.635+0000 7f546eecb700 1 -- start start 2024-03-19T01:31:45.543 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:44.635+0000 7f546eecb700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f54681066b0 0x7f5468106ad0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:31:45.544 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:44.635+0000 7f546eecb700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f546807be50 con 0x7f54681066b0 2024-03-19T01:31:45.544 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:44.635+0000 7f546cc67700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f54681066b0 0x7f5468106ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:31:45.544 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:44.635+0000 7f546cc67700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f54681066b0 0x7f5468106ad0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:45530/0 (socket says 172.21.15.99:45530) 2024-03-19T01:31:45.544 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:44.635+0000 7f546cc67700 1 -- 172.21.15.99:0/3391342531 learned_addr learned my addr 172.21.15.99:0/3391342531 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:31:45.544 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:44.635+0000 7f546cc67700 1 -- 172.21.15.99:0/3391342531 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f5468107010 con 0x7f54681066b0 2024-03-19T01:31:45.544 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:44.636+0000 7f546cc67700 1 --2- 172.21.15.99:0/3391342531 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f54681066b0 0x7f5468106ad0 secure :-1 s=READY pgs=39 cs=0 l=1 rev1=1 crypto rx=0x7f5460009e80 tx=0x7f546000b330 comp rx=0 tx=0).ready entity=mon.0 client_cookie=cf659c78f2748b34 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:31:45.544 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:44.636+0000 7f54677fe700 1 -- 172.21.15.99:0/3391342531 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f5460017070 con 0x7f54681066b0 2024-03-19T01:31:45.544 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:44.636+0000 7f54677fe700 1 -- 172.21.15.99:0/3391342531 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7f546000b8c0 con 0x7f54681066b0 2024-03-19T01:31:45.544 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:44.636+0000 7f54677fe700 1 -- 172.21.15.99:0/3391342531 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f54600045c0 con 0x7f54681066b0 2024-03-19T01:31:45.544 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:44.636+0000 7f546eecb700 1 -- 172.21.15.99:0/3391342531 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f54681066b0 msgr2=0x7f5468106ad0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:31:45.544 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:44.636+0000 7f546eecb700 1 --2- 172.21.15.99:0/3391342531 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f54681066b0 0x7f5468106ad0 secure :-1 s=READY pgs=39 cs=0 l=1 rev1=1 crypto rx=0x7f5460009e80 tx=0x7f546000b330 comp rx=0 tx=0).stop 2024-03-19T01:31:45.544 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:44.636+0000 7f546eecb700 1 -- 172.21.15.99:0/3391342531 shutdown_connections 2024-03-19T01:31:45.544 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:44.636+0000 7f546eecb700 1 --2- 172.21.15.99:0/3391342531 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f54681066b0 0x7f5468106ad0 unknown :-1 s=CLOSED pgs=39 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:45.544 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:44.636+0000 7f546eecb700 1 -- 172.21.15.99:0/3391342531 >> 172.21.15.99:0/3391342531 conn(0x7f5468102270 msgr2=0x7f54681046b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:31:45.545 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:44.636+0000 7f546eecb700 1 -- 172.21.15.99:0/3391342531 shutdown_connections 2024-03-19T01:31:45.545 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:44.636+0000 7f546eecb700 1 -- 172.21.15.99:0/3391342531 wait complete. 2024-03-19T01:31:45.545 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:44.637+0000 7f546eecb700 1 Processor -- start 2024-03-19T01:31:45.545 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:44.637+0000 7f546eecb700 1 -- start start 2024-03-19T01:31:45.545 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:44.637+0000 7f546eecb700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f54681066b0 0x7f546819df60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:31:45.545 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:44.637+0000 7f546eecb700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f54681080a0 con 0x7f54681066b0 2024-03-19T01:31:45.545 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:44.637+0000 7f546cc67700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f54681066b0 0x7f546819df60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:31:45.545 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:44.637+0000 7f546cc67700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f54681066b0 0x7f546819df60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:45540/0 (socket says 172.21.15.99:45540) 2024-03-19T01:31:45.545 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:44.637+0000 7f546cc67700 1 -- 172.21.15.99:0/3217608940 learned_addr learned my addr 172.21.15.99:0/3217608940 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:31:45.545 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:44.637+0000 7f546cc67700 1 -- 172.21.15.99:0/3217608940 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f546819e4a0 con 0x7f54681066b0 2024-03-19T01:31:45.546 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:44.638+0000 7f546cc67700 1 --2- 172.21.15.99:0/3217608940 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f54681066b0 0x7f546819df60 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7f5460009530 tx=0x7f546000bd50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:31:45.546 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:44.638+0000 7f5465ffb700 1 -- 172.21.15.99:0/3217608940 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f5460020020 con 0x7f54681066b0 2024-03-19T01:31:45.546 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:44.638+0000 7f5465ffb700 1 -- 172.21.15.99:0/3217608940 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7f5460003b50 con 0x7f54681066b0 2024-03-19T01:31:45.546 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:44.638+0000 7f546eecb700 1 -- 172.21.15.99:0/3217608940 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f546819e730 con 0x7f54681066b0 2024-03-19T01:31:45.546 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:44.638+0000 7f5465ffb700 1 -- 172.21.15.99:0/3217608940 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f5460017040 con 0x7f54681066b0 2024-03-19T01:31:45.546 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:44.638+0000 7f546eecb700 1 -- 172.21.15.99:0/3217608940 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f54681a1420 con 0x7f54681066b0 2024-03-19T01:31:45.546 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:44.639+0000 7f5465ffb700 1 -- 172.21.15.99:0/3217608940 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 8) ==== 46106+0+0 (secure 0 0 0) 0x7f5460023020 con 0x7f54681066b0 2024-03-19T01:31:45.546 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:44.639+0000 7f546eecb700 1 -- 172.21.15.99:0/3217608940 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f5468069420 con 0x7f54681066b0 2024-03-19T01:31:45.546 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:44.639+0000 7f5465ffb700 1 --2- 172.21.15.99:0/3217608940 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7f5450038f70 0x7f545003b430 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:31:45.546 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:44.639+0000 7f5465ffb700 1 -- 172.21.15.99:0/3217608940 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(2..2 src has 1..2) ==== 940+0+0 (secure 0 0 0) 0x7f546000f050 con 0x7f54681066b0 2024-03-19T01:31:45.546 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:44.639+0000 7f5467fff700 1 --2- 172.21.15.99:0/3217608940 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7f5450038f70 0x7f545003b430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:31:45.546 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:44.639+0000 7f5467fff700 1 --2- 172.21.15.99:0/3217608940 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7f5450038f70 0x7f545003b430 secure :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0x7f545c006fd0 tx=0x7f545c006e40 comp rx=0 tx=0).ready entity=mgr.14118 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:31:45.547 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:44.644+0000 7f5465ffb700 1 -- 172.21.15.99:0/3217608940 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f546002a020 con 0x7f54681066b0 2024-03-19T01:31:45.547 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:44.837+0000 7f546eecb700 1 -- 172.21.15.99:0/3217608940 --> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] -- mgr_command(tid 0: {"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}) -- 0x7f546802df90 con 0x7f5450038f70 2024-03-19T01:31:45.547 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:45.037+0000 7f5465ffb700 1 -- 172.21.15.99:0/3217608940 <== mgr.14118 v2:172.21.15.99:6800/1328245934 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+0 (secure 0 0 0) 0x7f546802df90 con 0x7f5450038f70 2024-03-19T01:31:45.547 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:45.039+0000 7f546eecb700 1 -- 172.21.15.99:0/3217608940 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7f5450038f70 msgr2=0x7f545003b430 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:31:45.547 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:45.039+0000 7f546eecb700 1 --2- 172.21.15.99:0/3217608940 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7f5450038f70 0x7f545003b430 secure :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0x7f545c006fd0 tx=0x7f545c006e40 comp rx=0 tx=0).stop 2024-03-19T01:31:45.547 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:45.039+0000 7f546eecb700 1 -- 172.21.15.99:0/3217608940 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f54681066b0 msgr2=0x7f546819df60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:31:45.547 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:45.039+0000 7f546eecb700 1 --2- 172.21.15.99:0/3217608940 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f54681066b0 0x7f546819df60 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7f5460009530 tx=0x7f546000bd50 comp rx=0 tx=0).stop 2024-03-19T01:31:45.547 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:45.039+0000 7f546eecb700 1 -- 172.21.15.99:0/3217608940 shutdown_connections 2024-03-19T01:31:45.547 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:45.039+0000 7f546eecb700 1 --2- 172.21.15.99:0/3217608940 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7f5450038f70 0x7f545003b430 unknown :-1 s=CLOSED pgs=9 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:45.547 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:45.039+0000 7f546eecb700 1 --2- 172.21.15.99:0/3217608940 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f54681066b0 0x7f546819df60 unknown :-1 s=CLOSED pgs=40 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:45.547 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:45.039+0000 7f546eecb700 1 -- 172.21.15.99:0/3217608940 >> 172.21.15.99:0/3217608940 conn(0x7f5468102270 msgr2=0x7f5468194bc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:31:45.547 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:45.040+0000 7f546eecb700 1 -- 172.21.15.99:0/3217608940 shutdown_connections 2024-03-19T01:31:45.547 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:45.040+0000 7f546eecb700 1 -- 172.21.15.99:0/3217608940 wait complete. 2024-03-19T01:31:46.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:46 smithi099 ceph-mon[62951]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:31:46.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:46 smithi099 ceph-mon[62951]: Generating ssh key... 2024-03-19T01:31:46.759 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCqyGEa2vlwe+9WcbWbCC8Jy3h1Z37hzemIAnr34gVudgT0k4Qy1DYzyVd7ZuGzAqx/B2RXQ0iLaPODqTEgreeURYtg6GBe50j+IZfs1EDwEOak/kjyni5/GCf9Uk2tOV84qyqGruaQbyZMtnd/+ycmBfi/gGOK1DJSNTE14lG6Y0inaNAj2Vy9p4S1hIxR4AtC9uPFdTGx1w04RJkZg6OIM/af0PVlNRRU3QTrMDYozsvYhXWorO/J+ZYXkU0RCMBYFtTdo55LkevfCVKuuDfgKG0gYEWRYFdtGTKUuW8TTpF0hMd9KFjAOs93y8REG2vOxb3NwOcvVaURenuHpl5V/xUj/yf19zafiVi8T+0SsRPHUJzGH0ITzpiDOwXxJji90aBzs19UGMicPEeIYUdmSFhQg2wR1kAcvBYvEDWwvui/SO+ALHX9SZKm5eYxN/TEvNqDaAX12pq49dcfu0BuMDkrk3x+7ZgFC8MGW+opCFjMMBnkdaMNW6M/v1Pc3W0= ceph-438fb69c-e590-11ee-95c9-87774f69a715 2024-03-19T01:31:46.759 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:45.962+0000 7f8e8e2cd700 1 Processor -- start 2024-03-19T01:31:46.759 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:45.962+0000 7f8e8e2cd700 1 -- start start 2024-03-19T01:31:46.759 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:45.962+0000 7f8e8e2cd700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8e88106410 0x7f8e88106830 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:31:46.760 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:45.962+0000 7f8e8e2cd700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f8e88106e00 con 0x7f8e88106410 2024-03-19T01:31:46.760 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:45.962+0000 7f8e87fff700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8e88106410 0x7f8e88106830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:31:46.760 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:45.962+0000 7f8e87fff700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8e88106410 0x7f8e88106830 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:45556/0 (socket says 172.21.15.99:45556) 2024-03-19T01:31:46.760 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:45.962+0000 7f8e87fff700 1 -- 172.21.15.99:0/3463328955 learned_addr learned my addr 172.21.15.99:0/3463328955 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:31:46.760 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:45.962+0000 7f8e87fff700 1 -- 172.21.15.99:0/3463328955 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f8e88107610 con 0x7f8e88106410 2024-03-19T01:31:46.760 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:45.962+0000 7f8e87fff700 1 --2- 172.21.15.99:0/3463328955 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8e88106410 0x7f8e88106830 secure :-1 s=READY pgs=41 cs=0 l=1 rev1=1 crypto rx=0x7f8e780098a0 tx=0x7f8e78009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f85bc926c27cb6c server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:31:46.760 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:45.963+0000 7f8e86ffd700 1 -- 172.21.15.99:0/3463328955 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f8e78004160 con 0x7f8e88106410 2024-03-19T01:31:46.760 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:45.963+0000 7f8e86ffd700 1 -- 172.21.15.99:0/3463328955 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7f8e78004300 con 0x7f8e88106410 2024-03-19T01:31:46.760 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:45.963+0000 7f8e86ffd700 1 -- 172.21.15.99:0/3463328955 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f8e78011b50 con 0x7f8e88106410 2024-03-19T01:31:46.760 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:45.963+0000 7f8e8e2cd700 1 -- 172.21.15.99:0/3463328955 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8e88106410 msgr2=0x7f8e88106830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:31:46.760 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:45.963+0000 7f8e8e2cd700 1 --2- 172.21.15.99:0/3463328955 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8e88106410 0x7f8e88106830 secure :-1 s=READY pgs=41 cs=0 l=1 rev1=1 crypto rx=0x7f8e780098a0 tx=0x7f8e78009bb0 comp rx=0 tx=0).stop 2024-03-19T01:31:46.760 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:45.963+0000 7f8e8e2cd700 1 -- 172.21.15.99:0/3463328955 shutdown_connections 2024-03-19T01:31:46.760 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:45.963+0000 7f8e8e2cd700 1 --2- 172.21.15.99:0/3463328955 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8e88106410 0x7f8e88106830 unknown :-1 s=CLOSED pgs=41 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:46.760 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:45.963+0000 7f8e8e2cd700 1 -- 172.21.15.99:0/3463328955 >> 172.21.15.99:0/3463328955 conn(0x7f8e881019f0 msgr2=0x7f8e88103e10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:31:46.761 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:45.963+0000 7f8e8e2cd700 1 -- 172.21.15.99:0/3463328955 shutdown_connections 2024-03-19T01:31:46.761 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:45.963+0000 7f8e8e2cd700 1 -- 172.21.15.99:0/3463328955 wait complete. 2024-03-19T01:31:46.761 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:45.964+0000 7f8e8e2cd700 1 Processor -- start 2024-03-19T01:31:46.761 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:45.964+0000 7f8e8e2cd700 1 -- start start 2024-03-19T01:31:46.761 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:45.964+0000 7f8e8e2cd700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8e88106410 0x7f8e8819be20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:31:46.761 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:45.964+0000 7f8e8e2cd700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f8e8810a9c0 con 0x7f8e88106410 2024-03-19T01:31:46.761 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:45.964+0000 7f8e87fff700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8e88106410 0x7f8e8819be20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:31:46.761 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:45.964+0000 7f8e87fff700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8e88106410 0x7f8e8819be20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:45566/0 (socket says 172.21.15.99:45566) 2024-03-19T01:31:46.761 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:45.964+0000 7f8e87fff700 1 -- 172.21.15.99:0/1598770060 learned_addr learned my addr 172.21.15.99:0/1598770060 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:31:46.761 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:45.965+0000 7f8e87fff700 1 -- 172.21.15.99:0/1598770060 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f8e8819c360 con 0x7f8e88106410 2024-03-19T01:31:46.761 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:45.965+0000 7f8e87fff700 1 --2- 172.21.15.99:0/1598770060 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8e88106410 0x7f8e8819be20 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7f8e78009fb0 tx=0x7f8e78011bf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:31:46.761 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:45.965+0000 7f8e857fa700 1 -- 172.21.15.99:0/1598770060 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f8e780038b0 con 0x7f8e88106410 2024-03-19T01:31:46.761 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:45.965+0000 7f8e857fa700 1 -- 172.21.15.99:0/1598770060 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7f8e78011e50 con 0x7f8e88106410 2024-03-19T01:31:46.761 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:45.965+0000 7f8e8e2cd700 1 -- 172.21.15.99:0/1598770060 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f8e8819c5f0 con 0x7f8e88106410 2024-03-19T01:31:46.762 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:45.965+0000 7f8e857fa700 1 -- 172.21.15.99:0/1598770060 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f8e78019550 con 0x7f8e88106410 2024-03-19T01:31:46.762 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:45.965+0000 7f8e8e2cd700 1 -- 172.21.15.99:0/1598770060 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f8e8819f2e0 con 0x7f8e88106410 2024-03-19T01:31:46.762 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:45.966+0000 7f8e857fa700 1 -- 172.21.15.99:0/1598770060 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 8) ==== 46106+0+0 (secure 0 0 0) 0x7f8e78019800 con 0x7f8e88106410 2024-03-19T01:31:46.762 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:45.966+0000 7f8e857fa700 1 --2- 172.21.15.99:0/1598770060 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7f8e70038f20 0x7f8e7003b3e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:31:46.762 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:45.966+0000 7f8e8e2cd700 1 -- 172.21.15.99:0/1598770060 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f8e88069420 con 0x7f8e88106410 2024-03-19T01:31:46.762 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:45.966+0000 7f8e857fa700 1 -- 172.21.15.99:0/1598770060 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(2..2 src has 1..2) ==== 940+0+0 (secure 0 0 0) 0x7f8e7804d200 con 0x7f8e88106410 2024-03-19T01:31:46.762 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:45.966+0000 7f8e877fe700 1 --2- 172.21.15.99:0/1598770060 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7f8e70038f20 0x7f8e7003b3e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:31:46.762 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:45.967+0000 7f8e877fe700 1 --2- 172.21.15.99:0/1598770060 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7f8e70038f20 0x7f8e7003b3e0 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0x7f8e7c006fd0 tx=0x7f8e7c006e40 comp rx=0 tx=0).ready entity=mgr.14118 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:31:46.762 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:45.971+0000 7f8e857fa700 1 -- 172.21.15.99:0/1598770060 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f8e7802d960 con 0x7f8e88106410 2024-03-19T01:31:46.762 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:46.168+0000 7f8e8e2cd700 1 -- 172.21.15.99:0/1598770060 --> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] -- mgr_command(tid 0: {"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}) -- 0x7f8e880008d0 con 0x7f8e70038f20 2024-03-19T01:31:46.762 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:46.169+0000 7f8e857fa700 1 -- 172.21.15.99:0/1598770060 <== mgr.14118 v2:172.21.15.99:6800/1328245934 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+595 (secure 0 0 0) 0x7f8e880008d0 con 0x7f8e70038f20 2024-03-19T01:31:46.762 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:46.171+0000 7f8e8e2cd700 1 -- 172.21.15.99:0/1598770060 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7f8e70038f20 msgr2=0x7f8e7003b3e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:31:46.763 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:46.171+0000 7f8e8e2cd700 1 --2- 172.21.15.99:0/1598770060 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7f8e70038f20 0x7f8e7003b3e0 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0x7f8e7c006fd0 tx=0x7f8e7c006e40 comp rx=0 tx=0).stop 2024-03-19T01:31:46.763 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:46.171+0000 7f8e8e2cd700 1 -- 172.21.15.99:0/1598770060 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8e88106410 msgr2=0x7f8e8819be20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:31:46.763 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:46.171+0000 7f8e8e2cd700 1 --2- 172.21.15.99:0/1598770060 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8e88106410 0x7f8e8819be20 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7f8e78009fb0 tx=0x7f8e78011bf0 comp rx=0 tx=0).stop 2024-03-19T01:31:46.763 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:46.171+0000 7f8e8e2cd700 1 -- 172.21.15.99:0/1598770060 shutdown_connections 2024-03-19T01:31:46.763 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:46.171+0000 7f8e8e2cd700 1 --2- 172.21.15.99:0/1598770060 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7f8e70038f20 0x7f8e7003b3e0 unknown :-1 s=CLOSED pgs=10 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:46.763 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:46.171+0000 7f8e8e2cd700 1 --2- 172.21.15.99:0/1598770060 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8e88106410 0x7f8e8819be20 unknown :-1 s=CLOSED pgs=42 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:46.763 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:46.171+0000 7f8e8e2cd700 1 -- 172.21.15.99:0/1598770060 >> 172.21.15.99:0/1598770060 conn(0x7f8e881019f0 msgr2=0x7f8e88102d20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:31:46.763 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:46.171+0000 7f8e8e2cd700 1 -- 172.21.15.99:0/1598770060 shutdown_connections 2024-03-19T01:31:46.763 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:46.171+0000 7f8e8e2cd700 1 -- 172.21.15.99:0/1598770060 wait complete. 2024-03-19T01:31:46.763 INFO:teuthology.orchestra.run.smithi099.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-03-19T01:31:46.763 INFO:teuthology.orchestra.run.smithi099.stdout:Adding key to root@localhost authorized_keys... 2024-03-19T01:31:46.763 INFO:teuthology.orchestra.run.smithi099.stdout:Adding host smithi099... 2024-03-19T01:31:47.480 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:47 smithi099 ceph-mon[62951]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:31:48.609 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:48 smithi099 ceph-mon[62951]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi099", "addr": "172.21.15.99", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:31:48.609 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:48 smithi099 ceph-mon[62951]: Deploying cephadm binary to smithi099 2024-03-19T01:31:52.283 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout Added host 'smithi099' with addr '172.21.15.99' 2024-03-19T01:31:52.283 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:47.152+0000 7f6e9f003700 1 Processor -- start 2024-03-19T01:31:52.283 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:47.152+0000 7f6e9f003700 1 -- start start 2024-03-19T01:31:52.283 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:47.152+0000 7f6e9f003700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6e98107190 0x7f6e981075b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:31:52.283 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:47.152+0000 7f6e9f003700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f6e98107b80 con 0x7f6e98107190 2024-03-19T01:31:52.283 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:47.152+0000 7f6e9cd9f700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6e98107190 0x7f6e981075b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:31:52.283 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:47.152+0000 7f6e9cd9f700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6e98107190 0x7f6e981075b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:45574/0 (socket says 172.21.15.99:45574) 2024-03-19T01:31:52.284 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:47.152+0000 7f6e9cd9f700 1 -- 172.21.15.99:0/2116891524 learned_addr learned my addr 172.21.15.99:0/2116891524 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:31:52.284 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:47.153+0000 7f6e9cd9f700 1 -- 172.21.15.99:0/2116891524 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f6e98107eb0 con 0x7f6e98107190 2024-03-19T01:31:52.284 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:47.153+0000 7f6e9cd9f700 1 --2- 172.21.15.99:0/2116891524 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6e98107190 0x7f6e981075b0 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 crypto rx=0x7f6e8c0098a0 tx=0x7f6e8c009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=270901b4e47bcf5 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:31:52.284 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:47.153+0000 7f6e977fe700 1 -- 172.21.15.99:0/2116891524 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f6e8c004160 con 0x7f6e98107190 2024-03-19T01:31:52.284 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:47.153+0000 7f6e977fe700 1 -- 172.21.15.99:0/2116891524 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7f6e8c004300 con 0x7f6e98107190 2024-03-19T01:31:52.284 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:47.153+0000 7f6e977fe700 1 -- 172.21.15.99:0/2116891524 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f6e8c011b50 con 0x7f6e98107190 2024-03-19T01:31:52.284 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:47.154+0000 7f6e9f003700 1 -- 172.21.15.99:0/2116891524 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6e98107190 msgr2=0x7f6e981075b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:31:52.284 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:47.154+0000 7f6e9f003700 1 --2- 172.21.15.99:0/2116891524 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6e98107190 0x7f6e981075b0 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 crypto rx=0x7f6e8c0098a0 tx=0x7f6e8c009bb0 comp rx=0 tx=0).stop 2024-03-19T01:31:52.284 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:47.154+0000 7f6e9f003700 1 -- 172.21.15.99:0/2116891524 shutdown_connections 2024-03-19T01:31:52.284 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:47.154+0000 7f6e9f003700 1 --2- 172.21.15.99:0/2116891524 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6e98107190 0x7f6e981075b0 unknown :-1 s=CLOSED pgs=43 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:52.285 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:47.154+0000 7f6e9f003700 1 -- 172.21.15.99:0/2116891524 >> 172.21.15.99:0/2116891524 conn(0x7f6e981019f0 msgr2=0x7f6e98103e10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:31:52.285 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:47.154+0000 7f6e9f003700 1 -- 172.21.15.99:0/2116891524 shutdown_connections 2024-03-19T01:31:52.285 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:47.154+0000 7f6e9f003700 1 -- 172.21.15.99:0/2116891524 wait complete. 2024-03-19T01:31:52.285 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:47.154+0000 7f6e9f003700 1 Processor -- start 2024-03-19T01:31:52.285 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:47.154+0000 7f6e9f003700 1 -- start start 2024-03-19T01:31:52.285 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:47.154+0000 7f6e9f003700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6e98107190 0x7f6e98106790 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:31:52.285 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:47.154+0000 7f6e9f003700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f6e9810b260 con 0x7f6e98107190 2024-03-19T01:31:52.285 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:47.155+0000 7f6e9cd9f700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6e98107190 0x7f6e98106790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:31:52.285 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:47.155+0000 7f6e9cd9f700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6e98107190 0x7f6e98106790 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:45578/0 (socket says 172.21.15.99:45578) 2024-03-19T01:31:52.285 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:47.155+0000 7f6e9cd9f700 1 -- 172.21.15.99:0/3147095210 learned_addr learned my addr 172.21.15.99:0/3147095210 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:31:52.286 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:47.155+0000 7f6e9cd9f700 1 -- 172.21.15.99:0/3147095210 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f6e9819eae0 con 0x7f6e98107190 2024-03-19T01:31:52.286 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:47.155+0000 7f6e9cd9f700 1 --2- 172.21.15.99:0/3147095210 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6e98107190 0x7f6e98106790 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7f6e8c009fb0 tx=0x7f6e8c011bf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:31:52.286 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:47.155+0000 7f6e95ffb700 1 -- 172.21.15.99:0/3147095210 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f6e8c0038b0 con 0x7f6e98107190 2024-03-19T01:31:52.286 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:47.155+0000 7f6e9f003700 1 -- 172.21.15.99:0/3147095210 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f6e9819ecb0 con 0x7f6e98107190 2024-03-19T01:31:52.286 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:47.155+0000 7f6e95ffb700 1 -- 172.21.15.99:0/3147095210 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7f6e8c011e50 con 0x7f6e98107190 2024-03-19T01:31:52.286 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:47.155+0000 7f6e95ffb700 1 -- 172.21.15.99:0/3147095210 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f6e8c019420 con 0x7f6e98107190 2024-03-19T01:31:52.286 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:47.155+0000 7f6e9f003700 1 -- 172.21.15.99:0/3147095210 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f6e9819f0d0 con 0x7f6e98107190 2024-03-19T01:31:52.286 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:47.156+0000 7f6e95ffb700 1 -- 172.21.15.99:0/3147095210 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 8) ==== 46106+0+0 (secure 0 0 0) 0x7f6e8c019720 con 0x7f6e98107190 2024-03-19T01:31:52.286 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:47.156+0000 7f6e9f003700 1 -- 172.21.15.99:0/3147095210 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f6e98063940 con 0x7f6e98107190 2024-03-19T01:31:52.286 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:47.156+0000 7f6e95ffb700 1 --2- 172.21.15.99:0/3147095210 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7f6e88038ed0 0x7f6e8803b390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:31:52.286 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:47.156+0000 7f6e95ffb700 1 -- 172.21.15.99:0/3147095210 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(2..2 src has 1..2) ==== 940+0+0 (secure 0 0 0) 0x7f6e8c04d350 con 0x7f6e98107190 2024-03-19T01:31:52.286 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:47.157+0000 7f6e97fff700 1 --2- 172.21.15.99:0/3147095210 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7f6e88038ed0 0x7f6e8803b390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:31:52.287 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:47.157+0000 7f6e97fff700 1 --2- 172.21.15.99:0/3147095210 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7f6e88038ed0 0x7f6e8803b390 secure :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0x7f6e84006fd0 tx=0x7f6e84006e40 comp rx=0 tx=0).ready entity=mgr.14118 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:31:52.287 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:47.161+0000 7f6e95ffb700 1 -- 172.21.15.99:0/3147095210 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f6e8c00ba30 con 0x7f6e98107190 2024-03-19T01:31:52.287 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:47.364+0000 7f6e9f003700 1 -- 172.21.15.99:0/3147095210 --> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] -- mgr_command(tid 0: {"prefix": "orch host add", "hostname": "smithi099", "addr": "172.21.15.99", "target": ["mon-mgr", ""]}) -- 0x7f6e98102cb0 con 0x7f6e88038ed0 2024-03-19T01:31:52.287 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:51.663+0000 7f6e95ffb700 1 -- 172.21.15.99:0/3147095210 <== mgr.14118 v2:172.21.15.99:6800/1328245934 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+48 (secure 0 0 0) 0x7f6e98102cb0 con 0x7f6e88038ed0 2024-03-19T01:31:52.287 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:51.666+0000 7f6e9f003700 1 -- 172.21.15.99:0/3147095210 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7f6e88038ed0 msgr2=0x7f6e8803b390 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:31:52.287 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:51.666+0000 7f6e9f003700 1 --2- 172.21.15.99:0/3147095210 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7f6e88038ed0 0x7f6e8803b390 secure :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0x7f6e84006fd0 tx=0x7f6e84006e40 comp rx=0 tx=0).stop 2024-03-19T01:31:52.287 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:51.666+0000 7f6e9f003700 1 -- 172.21.15.99:0/3147095210 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6e98107190 msgr2=0x7f6e98106790 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:31:52.287 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:51.666+0000 7f6e9f003700 1 --2- 172.21.15.99:0/3147095210 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6e98107190 0x7f6e98106790 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7f6e8c009fb0 tx=0x7f6e8c011bf0 comp rx=0 tx=0).stop 2024-03-19T01:31:52.287 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:51.666+0000 7f6e9f003700 1 -- 172.21.15.99:0/3147095210 shutdown_connections 2024-03-19T01:31:52.287 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:51.666+0000 7f6e9f003700 1 --2- 172.21.15.99:0/3147095210 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7f6e88038ed0 0x7f6e8803b390 unknown :-1 s=CLOSED pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:52.287 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:51.666+0000 7f6e9f003700 1 --2- 172.21.15.99:0/3147095210 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6e98107190 0x7f6e98106790 unknown :-1 s=CLOSED pgs=44 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:52.287 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:51.666+0000 7f6e9f003700 1 -- 172.21.15.99:0/3147095210 >> 172.21.15.99:0/3147095210 conn(0x7f6e981019f0 msgr2=0x7f6e981025a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:31:52.288 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:51.666+0000 7f6e9f003700 1 -- 172.21.15.99:0/3147095210 shutdown_connections 2024-03-19T01:31:52.288 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:51.666+0000 7f6e9f003700 1 -- 172.21.15.99:0/3147095210 wait complete. 2024-03-19T01:31:52.288 INFO:teuthology.orchestra.run.smithi099.stdout:Deploying unmanaged mon service... 2024-03-19T01:31:52.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:52 smithi099 ceph-mon[62951]: from='mgr.14118 172.21.15.99:0/2932773335' entity='mgr.x' 2024-03-19T01:31:52.842 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:52 smithi099 ceph-mon[62951]: Added host smithi099 2024-03-19T01:31:52.842 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:52 smithi099 ceph-mon[62951]: from='mgr.14118 172.21.15.99:0/2932773335' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:31:54.208 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:53 smithi099 ceph-mon[62951]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:31:54.208 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:53 smithi099 ceph-mon[62951]: Saving service mon spec with placement count:5 2024-03-19T01:31:54.208 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:53 smithi099 ceph-mon[62951]: from='mgr.14118 172.21.15.99:0/2932773335' entity='mgr.x' 2024-03-19T01:31:54.426 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2024-03-19T01:31:54.426 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:52.691+0000 7f3be7cf7700 1 Processor -- start 2024-03-19T01:31:54.426 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:52.691+0000 7f3be7cf7700 1 -- start start 2024-03-19T01:31:54.426 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:52.691+0000 7f3be7cf7700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3be007dee0 0x7f3be007c340 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:31:54.426 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:52.691+0000 7f3be7cf7700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f3be007c910 con 0x7f3be007dee0 2024-03-19T01:31:54.426 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:52.691+0000 7f3be5a93700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3be007dee0 0x7f3be007c340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:31:54.426 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:52.691+0000 7f3be5a93700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3be007dee0 0x7f3be007c340 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:59442/0 (socket says 172.21.15.99:59442) 2024-03-19T01:31:54.427 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:52.691+0000 7f3be5a93700 1 -- 172.21.15.99:0/3109693648 learned_addr learned my addr 172.21.15.99:0/3109693648 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:31:54.427 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:52.691+0000 7f3be5a93700 1 -- 172.21.15.99:0/3109693648 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f3be007ca90 con 0x7f3be007dee0 2024-03-19T01:31:54.427 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:52.692+0000 7f3be5a93700 1 --2- 172.21.15.99:0/3109693648 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3be007dee0 0x7f3be007c340 secure :-1 s=READY pgs=45 cs=0 l=1 rev1=1 crypto rx=0x7f3bdc0098a0 tx=0x7f3bdc009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8933049bc41d3fb9 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:31:54.427 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:52.692+0000 7f3be4a91700 1 -- 172.21.15.99:0/3109693648 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f3bdc004160 con 0x7f3be007dee0 2024-03-19T01:31:54.427 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:52.692+0000 7f3be4a91700 1 -- 172.21.15.99:0/3109693648 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7f3bdc004300 con 0x7f3be007dee0 2024-03-19T01:31:54.427 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:52.692+0000 7f3be4a91700 1 -- 172.21.15.99:0/3109693648 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f3bdc011b50 con 0x7f3be007dee0 2024-03-19T01:31:54.427 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:52.692+0000 7f3be7cf7700 1 -- 172.21.15.99:0/3109693648 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3be007dee0 msgr2=0x7f3be007c340 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:31:54.427 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:52.692+0000 7f3be7cf7700 1 --2- 172.21.15.99:0/3109693648 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3be007dee0 0x7f3be007c340 secure :-1 s=READY pgs=45 cs=0 l=1 rev1=1 crypto rx=0x7f3bdc0098a0 tx=0x7f3bdc009bb0 comp rx=0 tx=0).stop 2024-03-19T01:31:54.427 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:52.692+0000 7f3be7cf7700 1 -- 172.21.15.99:0/3109693648 shutdown_connections 2024-03-19T01:31:54.427 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:52.692+0000 7f3be7cf7700 1 --2- 172.21.15.99:0/3109693648 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3be007dee0 0x7f3be007c340 unknown :-1 s=CLOSED pgs=45 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:54.427 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:52.692+0000 7f3be7cf7700 1 -- 172.21.15.99:0/3109693648 >> 172.21.15.99:0/3109693648 conn(0x7f3be01033a0 msgr2=0x7f3be01057c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:31:54.427 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:52.692+0000 7f3be7cf7700 1 -- 172.21.15.99:0/3109693648 shutdown_connections 2024-03-19T01:31:54.427 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:52.693+0000 7f3be7cf7700 1 -- 172.21.15.99:0/3109693648 wait complete. 2024-03-19T01:31:54.428 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:52.693+0000 7f3be7cf7700 1 Processor -- start 2024-03-19T01:31:54.428 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:52.693+0000 7f3be7cf7700 1 -- start start 2024-03-19T01:31:54.428 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:52.693+0000 7f3be7cf7700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3be007dee0 0x7f3be019df30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:31:54.428 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:52.693+0000 7f3be7cf7700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f3be010e370 con 0x7f3be007dee0 2024-03-19T01:31:54.428 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:52.693+0000 7f3be5a93700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3be007dee0 0x7f3be019df30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:31:54.428 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:52.693+0000 7f3be5a93700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3be007dee0 0x7f3be019df30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:59446/0 (socket says 172.21.15.99:59446) 2024-03-19T01:31:54.428 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:52.693+0000 7f3be5a93700 1 -- 172.21.15.99:0/1578836612 learned_addr learned my addr 172.21.15.99:0/1578836612 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:31:54.428 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:52.693+0000 7f3be5a93700 1 -- 172.21.15.99:0/1578836612 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f3be019e470 con 0x7f3be007dee0 2024-03-19T01:31:54.428 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:52.694+0000 7f3be5a93700 1 --2- 172.21.15.99:0/1578836612 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3be007dee0 0x7f3be019df30 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7f3bdc000c00 tx=0x7f3bdc011c10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:31:54.428 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:52.694+0000 7f3bd6ffd700 1 -- 172.21.15.99:0/1578836612 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f3bdc003b20 con 0x7f3be007dee0 2024-03-19T01:31:54.428 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:52.694+0000 7f3bd6ffd700 1 -- 172.21.15.99:0/1578836612 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1109+0+0 (secure 0 0 0) 0x7f3bdc003cc0 con 0x7f3be007dee0 2024-03-19T01:31:54.428 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:52.694+0000 7f3bd6ffd700 1 -- 172.21.15.99:0/1578836612 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f3bdc024910 con 0x7f3be007dee0 2024-03-19T01:31:54.428 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:52.694+0000 7f3be7cf7700 1 -- 172.21.15.99:0/1578836612 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f3be019e700 con 0x7f3be007dee0 2024-03-19T01:31:54.429 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:52.694+0000 7f3be7cf7700 1 -- 172.21.15.99:0/1578836612 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f3be01a13f0 con 0x7f3be007dee0 2024-03-19T01:31:54.429 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:52.695+0000 7f3bd6ffd700 1 -- 172.21.15.99:0/1578836612 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 8) ==== 46106+0+0 (secure 0 0 0) 0x7f3bdc022020 con 0x7f3be007dee0 2024-03-19T01:31:54.429 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:52.695+0000 7f3be7cf7700 1 -- 172.21.15.99:0/1578836612 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f3be0063940 con 0x7f3be007dee0 2024-03-19T01:31:54.429 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:52.695+0000 7f3bd6ffd700 1 --2- 172.21.15.99:0/1578836612 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7f3bcc038f70 0x7f3bcc03b430 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:31:54.429 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:52.695+0000 7f3bd6ffd700 1 -- 172.21.15.99:0/1578836612 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(2..2 src has 1..2) ==== 940+0+0 (secure 0 0 0) 0x7f3bdc011e00 con 0x7f3be007dee0 2024-03-19T01:31:54.429 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:52.695+0000 7f3be5292700 1 --2- 172.21.15.99:0/1578836612 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7f3bcc038f70 0x7f3bcc03b430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:31:54.429 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:52.695+0000 7f3be5292700 1 --2- 172.21.15.99:0/1578836612 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7f3bcc038f70 0x7f3bcc03b430 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0x7f3bd0006fd0 tx=0x7f3bd0006e40 comp rx=0 tx=0).ready entity=mgr.14118 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:31:54.429 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:52.700+0000 7f3bd6ffd700 1 -- 172.21.15.99:0/1578836612 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f3bdc02cdf0 con 0x7f3be007dee0 2024-03-19T01:31:54.429 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:52.911+0000 7f3be7cf7700 1 -- 172.21.15.99:0/1578836612 --> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] -- mgr_command(tid 0: {"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}) -- 0x7f3be01045c0 con 0x7f3bcc038f70 2024-03-19T01:31:54.429 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:52.916+0000 7f3bd6ffd700 1 -- 172.21.15.99:0/1578836612 <== mgr.14118 v2:172.21.15.99:6800/1328245934 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+24 (secure 0 0 0) 0x7f3be01045c0 con 0x7f3bcc038f70 2024-03-19T01:31:54.429 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:52.918+0000 7f3be7cf7700 1 -- 172.21.15.99:0/1578836612 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7f3bcc038f70 msgr2=0x7f3bcc03b430 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:31:54.429 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:52.918+0000 7f3be7cf7700 1 --2- 172.21.15.99:0/1578836612 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7f3bcc038f70 0x7f3bcc03b430 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0x7f3bd0006fd0 tx=0x7f3bd0006e40 comp rx=0 tx=0).stop 2024-03-19T01:31:54.429 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:52.918+0000 7f3be7cf7700 1 -- 172.21.15.99:0/1578836612 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3be007dee0 msgr2=0x7f3be019df30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:31:54.429 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:52.918+0000 7f3be7cf7700 1 --2- 172.21.15.99:0/1578836612 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3be007dee0 0x7f3be019df30 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7f3bdc000c00 tx=0x7f3bdc011c10 comp rx=0 tx=0).stop 2024-03-19T01:31:54.430 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:52.918+0000 7f3be7cf7700 1 -- 172.21.15.99:0/1578836612 shutdown_connections 2024-03-19T01:31:54.430 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:52.918+0000 7f3be7cf7700 1 --2- 172.21.15.99:0/1578836612 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7f3bcc038f70 0x7f3bcc03b430 unknown :-1 s=CLOSED pgs=12 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:54.430 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:52.918+0000 7f3be7cf7700 1 --2- 172.21.15.99:0/1578836612 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3be007dee0 0x7f3be019df30 unknown :-1 s=CLOSED pgs=46 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:54.430 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:52.918+0000 7f3be7cf7700 1 -- 172.21.15.99:0/1578836612 >> 172.21.15.99:0/1578836612 conn(0x7f3be01033a0 msgr2=0x7f3be0103eb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:31:54.430 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:52.918+0000 7f3be7cf7700 1 -- 172.21.15.99:0/1578836612 shutdown_connections 2024-03-19T01:31:54.430 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:52.918+0000 7f3be7cf7700 1 -- 172.21.15.99:0/1578836612 wait complete. 2024-03-19T01:31:54.430 INFO:teuthology.orchestra.run.smithi099.stdout:Deploying unmanaged mgr service... 2024-03-19T01:31:55.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:55 smithi099 ceph-mon[62951]: from='mgr.14118 172.21.15.99:0/2932773335' entity='mgr.x' 2024-03-19T01:31:55.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:55 smithi099 ceph-mon[62951]: from='mgr.14118 172.21.15.99:0/2932773335' entity='mgr.x' 2024-03-19T01:31:55.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:55 smithi099 ceph-mon[62951]: from='mgr.14118 172.21.15.99:0/2932773335' entity='mgr.x' 2024-03-19T01:31:55.690 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2024-03-19T01:31:55.690 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:54.852+0000 7f8ec5a3b700 1 Processor -- start 2024-03-19T01:31:55.690 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:54.852+0000 7f8ec5a3b700 1 -- start start 2024-03-19T01:31:55.690 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:54.852+0000 7f8ec5a3b700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8ec007df00 0x7f8ec007c360 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:31:55.690 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:54.852+0000 7f8ec5a3b700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f8ec007c930 con 0x7f8ec007df00 2024-03-19T01:31:55.691 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:54.853+0000 7f8ebeffd700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8ec007df00 0x7f8ec007c360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:31:55.691 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:54.853+0000 7f8ebeffd700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8ec007df00 0x7f8ec007c360 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:59462/0 (socket says 172.21.15.99:59462) 2024-03-19T01:31:55.691 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:54.853+0000 7f8ebeffd700 1 -- 172.21.15.99:0/3087648298 learned_addr learned my addr 172.21.15.99:0/3087648298 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:31:55.691 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:54.853+0000 7f8ebeffd700 1 -- 172.21.15.99:0/3087648298 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f8ec007cab0 con 0x7f8ec007df00 2024-03-19T01:31:55.691 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:54.853+0000 7f8ebeffd700 1 --2- 172.21.15.99:0/3087648298 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8ec007df00 0x7f8ec007c360 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7f8eb8009e80 tx=0x7f8eb800b330 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d7ea49d94d871356 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:31:55.691 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:54.853+0000 7f8ebdffb700 1 -- 172.21.15.99:0/3087648298 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f8eb8017070 con 0x7f8ec007df00 2024-03-19T01:31:55.691 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:54.853+0000 7f8ebdffb700 1 -- 172.21.15.99:0/3087648298 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f8eb800b8c0 con 0x7f8ec007df00 2024-03-19T01:31:55.691 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:54.853+0000 7f8ebdffb700 1 -- 172.21.15.99:0/3087648298 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f8eb80045c0 con 0x7f8ec007df00 2024-03-19T01:31:55.691 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:54.854+0000 7f8ec5a3b700 1 -- 172.21.15.99:0/3087648298 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8ec007df00 msgr2=0x7f8ec007c360 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:31:55.691 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:54.854+0000 7f8ec5a3b700 1 --2- 172.21.15.99:0/3087648298 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8ec007df00 0x7f8ec007c360 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7f8eb8009e80 tx=0x7f8eb800b330 comp rx=0 tx=0).stop 2024-03-19T01:31:55.691 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:54.854+0000 7f8ec5a3b700 1 -- 172.21.15.99:0/3087648298 shutdown_connections 2024-03-19T01:31:55.691 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:54.854+0000 7f8ec5a3b700 1 --2- 172.21.15.99:0/3087648298 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8ec007df00 0x7f8ec007c360 unknown :-1 s=CLOSED pgs=47 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:55.692 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:54.854+0000 7f8ec5a3b700 1 -- 172.21.15.99:0/3087648298 >> 172.21.15.99:0/3087648298 conn(0x7f8ec01033a0 msgr2=0x7f8ec01057c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:31:55.692 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:54.854+0000 7f8ec5a3b700 1 -- 172.21.15.99:0/3087648298 shutdown_connections 2024-03-19T01:31:55.692 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:54.854+0000 7f8ec5a3b700 1 -- 172.21.15.99:0/3087648298 wait complete. 2024-03-19T01:31:55.692 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:54.854+0000 7f8ec5a3b700 1 Processor -- start 2024-03-19T01:31:55.692 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:54.855+0000 7f8ec5a3b700 1 -- start start 2024-03-19T01:31:55.692 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:54.855+0000 7f8ec5a3b700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8ec007df00 0x7f8ec019df40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:31:55.692 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:54.855+0000 7f8ec5a3b700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f8ec010e3f0 con 0x7f8ec007df00 2024-03-19T01:31:55.692 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:54.855+0000 7f8ebeffd700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8ec007df00 0x7f8ec019df40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:31:55.692 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:54.855+0000 7f8ebeffd700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8ec007df00 0x7f8ec019df40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:59464/0 (socket says 172.21.15.99:59464) 2024-03-19T01:31:55.692 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:54.855+0000 7f8ebeffd700 1 -- 172.21.15.99:0/3176240089 learned_addr learned my addr 172.21.15.99:0/3176240089 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:31:55.692 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:54.855+0000 7f8ebeffd700 1 -- 172.21.15.99:0/3176240089 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f8ec019e480 con 0x7f8ec007df00 2024-03-19T01:31:55.692 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:54.855+0000 7f8ebeffd700 1 --2- 172.21.15.99:0/3176240089 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8ec007df00 0x7f8ec019df40 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7f8eb80048f0 tx=0x7f8eb80049d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:31:55.692 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:54.855+0000 7f8ec4a39700 1 -- 172.21.15.99:0/3176240089 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f8eb8020020 con 0x7f8ec007df00 2024-03-19T01:31:55.693 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:54.856+0000 7f8ec4a39700 1 -- 172.21.15.99:0/3176240089 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f8eb8012430 con 0x7f8ec007df00 2024-03-19T01:31:55.693 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:54.856+0000 7f8ec5a3b700 1 -- 172.21.15.99:0/3176240089 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f8ec019e710 con 0x7f8ec007df00 2024-03-19T01:31:55.693 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:54.856+0000 7f8ec4a39700 1 -- 172.21.15.99:0/3176240089 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f8eb8017040 con 0x7f8ec007df00 2024-03-19T01:31:55.693 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:54.856+0000 7f8ec5a3b700 1 -- 172.21.15.99:0/3176240089 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f8ec01a1400 con 0x7f8ec007df00 2024-03-19T01:31:55.693 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:54.856+0000 7f8ec4a39700 1 -- 172.21.15.99:0/3176240089 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 8) ==== 46106+0+0 (secure 0 0 0) 0x7f8eb8023020 con 0x7f8ec007df00 2024-03-19T01:31:55.693 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:54.857+0000 7f8ec5a3b700 1 -- 172.21.15.99:0/3176240089 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f8ec0063940 con 0x7f8ec007df00 2024-03-19T01:31:55.693 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:54.857+0000 7f8ec4a39700 1 --2- 172.21.15.99:0/3176240089 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7f8ea8038ed0 0x7f8ea803b390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:31:55.693 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:54.857+0000 7f8ec4a39700 1 -- 172.21.15.99:0/3176240089 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(2..2 src has 1..2) ==== 940+0+0 (secure 0 0 0) 0x7f8eb800f050 con 0x7f8ec007df00 2024-03-19T01:31:55.693 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:54.857+0000 7f8ebe7fc700 1 --2- 172.21.15.99:0/3176240089 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7f8ea8038ed0 0x7f8ea803b390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:31:55.693 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:54.857+0000 7f8ebe7fc700 1 --2- 172.21.15.99:0/3176240089 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7f8ea8038ed0 0x7f8ea803b390 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7f8eb4006fd0 tx=0x7f8eb4006e40 comp rx=0 tx=0).ready entity=mgr.14118 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:31:55.693 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:54.862+0000 7f8ec4a39700 1 -- 172.21.15.99:0/3176240089 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f8eb802a020 con 0x7f8ec007df00 2024-03-19T01:31:55.693 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:55.085+0000 7f8ec5a3b700 1 -- 172.21.15.99:0/3176240089 --> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] -- mgr_command(tid 0: {"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}) -- 0x7f8ec01045c0 con 0x7f8ea8038ed0 2024-03-19T01:31:55.694 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:55.089+0000 7f8ec4a39700 1 -- 172.21.15.99:0/3176240089 <== mgr.14118 v2:172.21.15.99:6800/1328245934 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+24 (secure 0 0 0) 0x7f8ec01045c0 con 0x7f8ea8038ed0 2024-03-19T01:31:55.694 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:55.092+0000 7f8ec5a3b700 1 -- 172.21.15.99:0/3176240089 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7f8ea8038ed0 msgr2=0x7f8ea803b390 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:31:55.694 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:55.092+0000 7f8ec5a3b700 1 --2- 172.21.15.99:0/3176240089 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7f8ea8038ed0 0x7f8ea803b390 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7f8eb4006fd0 tx=0x7f8eb4006e40 comp rx=0 tx=0).stop 2024-03-19T01:31:55.694 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:55.092+0000 7f8ec5a3b700 1 -- 172.21.15.99:0/3176240089 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8ec007df00 msgr2=0x7f8ec019df40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:31:55.694 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:55.092+0000 7f8ec5a3b700 1 --2- 172.21.15.99:0/3176240089 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8ec007df00 0x7f8ec019df40 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7f8eb80048f0 tx=0x7f8eb80049d0 comp rx=0 tx=0).stop 2024-03-19T01:31:55.694 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:55.092+0000 7f8ec5a3b700 1 -- 172.21.15.99:0/3176240089 shutdown_connections 2024-03-19T01:31:55.694 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:55.092+0000 7f8ec5a3b700 1 --2- 172.21.15.99:0/3176240089 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7f8ea8038ed0 0x7f8ea803b390 unknown :-1 s=CLOSED pgs=13 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:55.694 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:55.092+0000 7f8ec5a3b700 1 --2- 172.21.15.99:0/3176240089 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8ec007df00 0x7f8ec019df40 unknown :-1 s=CLOSED pgs=48 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:55.694 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:55.092+0000 7f8ec5a3b700 1 -- 172.21.15.99:0/3176240089 >> 172.21.15.99:0/3176240089 conn(0x7f8ec01033a0 msgr2=0x7f8ec0103eb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:31:55.694 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:55.092+0000 7f8ec5a3b700 1 -- 172.21.15.99:0/3176240089 shutdown_connections 2024-03-19T01:31:55.694 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:55.092+0000 7f8ec5a3b700 1 -- 172.21.15.99:0/3176240089 wait complete. 2024-03-19T01:31:56.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:56 smithi099 ceph-mon[62951]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:31:56.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:56 smithi099 ceph-mon[62951]: Saving service mgr spec with placement count:2 2024-03-19T01:31:57.040 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:56.254+0000 7fe976798700 1 Processor -- start 2024-03-19T01:31:57.040 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:56.254+0000 7fe976798700 1 -- start start 2024-03-19T01:31:57.040 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:56.254+0000 7fe976798700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe97007dec0 0x7fe97007c320 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:31:57.041 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:56.254+0000 7fe976798700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fe97007c8f0 con 0x7fe97007dec0 2024-03-19T01:31:57.041 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:56.254+0000 7fe96ffff700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe97007dec0 0x7fe97007c320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:31:57.041 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:56.254+0000 7fe96ffff700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe97007dec0 0x7fe97007c320 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:59470/0 (socket says 172.21.15.99:59470) 2024-03-19T01:31:57.041 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:56.254+0000 7fe96ffff700 1 -- 172.21.15.99:0/360819398 learned_addr learned my addr 172.21.15.99:0/360819398 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:31:57.041 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:56.255+0000 7fe96ffff700 1 -- 172.21.15.99:0/360819398 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fe97007ca70 con 0x7fe97007dec0 2024-03-19T01:31:57.041 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:56.255+0000 7fe96ffff700 1 --2- 172.21.15.99:0/360819398 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe97007dec0 0x7fe97007c320 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7fe9600098a0 tx=0x7fe960009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1c4fe43c90e38f54 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:31:57.041 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:56.255+0000 7fe96effd700 1 -- 172.21.15.99:0/360819398 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fe960004160 con 0x7fe97007dec0 2024-03-19T01:31:57.041 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:56.255+0000 7fe96effd700 1 -- 172.21.15.99:0/360819398 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe960004300 con 0x7fe97007dec0 2024-03-19T01:31:57.041 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:56.255+0000 7fe96effd700 1 -- 172.21.15.99:0/360819398 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fe960011b20 con 0x7fe97007dec0 2024-03-19T01:31:57.041 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:56.256+0000 7fe976798700 1 -- 172.21.15.99:0/360819398 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe97007dec0 msgr2=0x7fe97007c320 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:31:57.041 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:56.256+0000 7fe976798700 1 --2- 172.21.15.99:0/360819398 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe97007dec0 0x7fe97007c320 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7fe9600098a0 tx=0x7fe960009bb0 comp rx=0 tx=0).stop 2024-03-19T01:31:57.042 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:56.256+0000 7fe976798700 1 -- 172.21.15.99:0/360819398 shutdown_connections 2024-03-19T01:31:57.042 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:56.256+0000 7fe976798700 1 --2- 172.21.15.99:0/360819398 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe97007dec0 0x7fe97007c320 unknown :-1 s=CLOSED pgs=49 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:57.042 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:56.256+0000 7fe976798700 1 -- 172.21.15.99:0/360819398 >> 172.21.15.99:0/360819398 conn(0x7fe970103390 msgr2=0x7fe9701057b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:31:57.042 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:56.256+0000 7fe976798700 1 -- 172.21.15.99:0/360819398 shutdown_connections 2024-03-19T01:31:57.042 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:56.256+0000 7fe976798700 1 -- 172.21.15.99:0/360819398 wait complete. 2024-03-19T01:31:57.042 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:56.256+0000 7fe976798700 1 Processor -- start 2024-03-19T01:31:57.042 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:56.256+0000 7fe976798700 1 -- start start 2024-03-19T01:31:57.042 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:56.257+0000 7fe976798700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe97007dec0 0x7fe97019dfa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:31:57.042 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:56.257+0000 7fe976798700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fe97010e430 con 0x7fe97007dec0 2024-03-19T01:31:57.042 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:56.257+0000 7fe96ffff700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe97007dec0 0x7fe97019dfa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:31:57.043 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:56.257+0000 7fe96ffff700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe97007dec0 0x7fe97019dfa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:59482/0 (socket says 172.21.15.99:59482) 2024-03-19T01:31:57.043 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:56.257+0000 7fe96ffff700 1 -- 172.21.15.99:0/4049452828 learned_addr learned my addr 172.21.15.99:0/4049452828 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:31:57.043 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:56.257+0000 7fe96ffff700 1 -- 172.21.15.99:0/4049452828 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fe97019e4e0 con 0x7fe97007dec0 2024-03-19T01:31:57.043 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:56.257+0000 7fe96ffff700 1 --2- 172.21.15.99:0/4049452828 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe97007dec0 0x7fe97019dfa0 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7fe960000c00 tx=0x7fe960011c10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:31:57.043 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:56.257+0000 7fe96d7fa700 1 -- 172.21.15.99:0/4049452828 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fe960003b60 con 0x7fe97007dec0 2024-03-19T01:31:57.043 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:56.257+0000 7fe976798700 1 -- 172.21.15.99:0/4049452828 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fe97019e770 con 0x7fe97007dec0 2024-03-19T01:31:57.043 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:56.257+0000 7fe96d7fa700 1 -- 172.21.15.99:0/4049452828 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe96001a910 con 0x7fe97007dec0 2024-03-19T01:31:57.043 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:56.257+0000 7fe96d7fa700 1 -- 172.21.15.99:0/4049452828 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fe960024930 con 0x7fe97007dec0 2024-03-19T01:31:57.043 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:56.257+0000 7fe976798700 1 -- 172.21.15.99:0/4049452828 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fe97019ebd0 con 0x7fe97007dec0 2024-03-19T01:31:57.043 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:56.258+0000 7fe96d7fa700 1 -- 172.21.15.99:0/4049452828 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 8) ==== 46106+0+0 (secure 0 0 0) 0x7fe960022020 con 0x7fe97007dec0 2024-03-19T01:31:57.043 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:56.258+0000 7fe976798700 1 -- 172.21.15.99:0/4049452828 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fe970069420 con 0x7fe97007dec0 2024-03-19T01:31:57.044 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:56.259+0000 7fe96d7fa700 1 --2- 172.21.15.99:0/4049452828 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7fe958038f20 0x7fe95803b3e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:31:57.044 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:56.259+0000 7fe96d7fa700 1 -- 172.21.15.99:0/4049452828 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(2..2 src has 1..2) ==== 940+0+0 (secure 0 0 0) 0x7fe96004d090 con 0x7fe97007dec0 2024-03-19T01:31:57.044 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:56.259+0000 7fe96f7fe700 1 --2- 172.21.15.99:0/4049452828 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7fe958038f20 0x7fe95803b3e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:31:57.044 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:56.259+0000 7fe96f7fe700 1 --2- 172.21.15.99:0/4049452828 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7fe958038f20 0x7fe95803b3e0 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7fe964006fd0 tx=0x7fe964006e40 comp rx=0 tx=0).ready entity=mgr.14118 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:31:57.044 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:56.264+0000 7fe96d7fa700 1 -- 172.21.15.99:0/4049452828 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7fe96000b3e0 con 0x7fe97007dec0 2024-03-19T01:31:57.044 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:56.468+0000 7fe976798700 1 -- 172.21.15.99:0/4049452828 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command([{prefix=config set, name=mgr/cephadm/container_init}] v 0) -- 0x7fe9701a1460 con 0x7fe97007dec0 2024-03-19T01:31:57.044 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:56.470+0000 7fe96d7fa700 1 -- 172.21.15.99:0/4049452828 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mgr/cephadm/container_init}]=0 v7)=0 v7) ==== 142+0+0 (secure 0 0 0) 0x7fe9600172b0 con 0x7fe97007dec0 2024-03-19T01:31:57.044 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:56.473+0000 7fe976798700 1 -- 172.21.15.99:0/4049452828 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7fe958038f20 msgr2=0x7fe95803b3e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:31:57.044 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:56.473+0000 7fe976798700 1 --2- 172.21.15.99:0/4049452828 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7fe958038f20 0x7fe95803b3e0 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7fe964006fd0 tx=0x7fe964006e40 comp rx=0 tx=0).stop 2024-03-19T01:31:57.044 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:56.473+0000 7fe976798700 1 -- 172.21.15.99:0/4049452828 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe97007dec0 msgr2=0x7fe97019dfa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:31:57.044 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:56.473+0000 7fe976798700 1 --2- 172.21.15.99:0/4049452828 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe97007dec0 0x7fe97019dfa0 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7fe960000c00 tx=0x7fe960011c10 comp rx=0 tx=0).stop 2024-03-19T01:31:57.045 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:56.473+0000 7fe976798700 1 -- 172.21.15.99:0/4049452828 shutdown_connections 2024-03-19T01:31:57.045 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:56.473+0000 7fe976798700 1 --2- 172.21.15.99:0/4049452828 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7fe958038f20 0x7fe95803b3e0 unknown :-1 s=CLOSED pgs=14 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:57.045 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:56.473+0000 7fe976798700 1 --2- 172.21.15.99:0/4049452828 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe97007dec0 0x7fe97019dfa0 unknown :-1 s=CLOSED pgs=50 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:57.045 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:56.473+0000 7fe976798700 1 -- 172.21.15.99:0/4049452828 >> 172.21.15.99:0/4049452828 conn(0x7fe970103390 msgr2=0x7fe970103ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:31:57.045 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:56.473+0000 7fe976798700 1 -- 172.21.15.99:0/4049452828 shutdown_connections 2024-03-19T01:31:57.045 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:56.473+0000 7fe976798700 1 -- 172.21.15.99:0/4049452828 wait complete. 2024-03-19T01:31:57.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:57 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/4049452828' entity='client.admin' 2024-03-19T01:31:58.690 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:57.909+0000 7f4ea957e700 1 Processor -- start 2024-03-19T01:31:58.691 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:57.909+0000 7f4ea957e700 1 -- start start 2024-03-19T01:31:58.691 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:57.909+0000 7f4ea957e700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4ea4108840 0x7f4ea410ac30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:31:58.691 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:57.909+0000 7f4ea957e700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f4ea407be90 con 0x7f4ea4108840 2024-03-19T01:31:58.691 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:57.909+0000 7f4ea2ffd700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4ea4108840 0x7f4ea410ac30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:31:58.691 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:57.909+0000 7f4ea2ffd700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4ea4108840 0x7f4ea410ac30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:59486/0 (socket says 172.21.15.99:59486) 2024-03-19T01:31:58.691 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:57.909+0000 7f4ea2ffd700 1 -- 172.21.15.99:0/2692408063 learned_addr learned my addr 172.21.15.99:0/2692408063 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:31:58.691 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:57.910+0000 7f4ea2ffd700 1 -- 172.21.15.99:0/2692408063 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f4ea410b170 con 0x7f4ea4108840 2024-03-19T01:31:58.691 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:57.910+0000 7f4ea2ffd700 1 --2- 172.21.15.99:0/2692408063 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4ea4108840 0x7f4ea410ac30 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7f4e980098a0 tx=0x7f4e98009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=768eba32ce8bad99 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:31:58.691 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:57.910+0000 7f4ea1ffb700 1 -- 172.21.15.99:0/2692408063 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f4e98004160 con 0x7f4ea4108840 2024-03-19T01:31:58.691 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:57.910+0000 7f4ea1ffb700 1 -- 172.21.15.99:0/2692408063 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f4e98004300 con 0x7f4ea4108840 2024-03-19T01:31:58.691 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:57.910+0000 7f4ea1ffb700 1 -- 172.21.15.99:0/2692408063 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f4e98011b20 con 0x7f4ea4108840 2024-03-19T01:31:58.691 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:57.911+0000 7f4ea957e700 1 -- 172.21.15.99:0/2692408063 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4ea4108840 msgr2=0x7f4ea410ac30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:31:58.692 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:57.911+0000 7f4ea957e700 1 --2- 172.21.15.99:0/2692408063 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4ea4108840 0x7f4ea410ac30 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7f4e980098a0 tx=0x7f4e98009bb0 comp rx=0 tx=0).stop 2024-03-19T01:31:58.692 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:57.911+0000 7f4ea957e700 1 -- 172.21.15.99:0/2692408063 shutdown_connections 2024-03-19T01:31:58.692 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:57.911+0000 7f4ea957e700 1 --2- 172.21.15.99:0/2692408063 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4ea4108840 0x7f4ea410ac30 unknown :-1 s=CLOSED pgs=51 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:58.692 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:57.911+0000 7f4ea957e700 1 -- 172.21.15.99:0/2692408063 >> 172.21.15.99:0/2692408063 conn(0x7f4ea4102270 msgr2=0x7f4ea4104690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:31:58.692 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:57.911+0000 7f4ea957e700 1 -- 172.21.15.99:0/2692408063 shutdown_connections 2024-03-19T01:31:58.692 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:57.911+0000 7f4ea957e700 1 -- 172.21.15.99:0/2692408063 wait complete. 2024-03-19T01:31:58.692 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:57.911+0000 7f4ea957e700 1 Processor -- start 2024-03-19T01:31:58.692 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:57.911+0000 7f4ea957e700 1 -- start start 2024-03-19T01:31:58.692 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:57.912+0000 7f4ea957e700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4ea4108840 0x7f4ea41080c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:31:58.692 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:57.912+0000 7f4ea957e700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f4ea410c1d0 con 0x7f4ea4108840 2024-03-19T01:31:58.692 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:57.912+0000 7f4ea2ffd700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4ea4108840 0x7f4ea41080c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:31:58.692 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:57.912+0000 7f4ea2ffd700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4ea4108840 0x7f4ea41080c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:59500/0 (socket says 172.21.15.99:59500) 2024-03-19T01:31:58.692 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:57.912+0000 7f4ea2ffd700 1 -- 172.21.15.99:0/3033126691 learned_addr learned my addr 172.21.15.99:0/3033126691 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:31:58.693 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:57.912+0000 7f4ea2ffd700 1 -- 172.21.15.99:0/3033126691 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f4ea4108600 con 0x7f4ea4108840 2024-03-19T01:31:58.693 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:57.912+0000 7f4ea2ffd700 1 --2- 172.21.15.99:0/3033126691 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4ea4108840 0x7f4ea41080c0 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7f4e980117e0 tx=0x7f4e98011c10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:31:58.693 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:57.912+0000 7f4e8bfff700 1 -- 172.21.15.99:0/3033126691 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f4e98003bf0 con 0x7f4ea4108840 2024-03-19T01:31:58.693 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:57.912+0000 7f4e8bfff700 1 -- 172.21.15.99:0/3033126691 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f4e9801a910 con 0x7f4ea4108840 2024-03-19T01:31:58.693 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:57.912+0000 7f4ea957e700 1 -- 172.21.15.99:0/3033126691 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f4ea41067f0 con 0x7f4ea4108840 2024-03-19T01:31:58.693 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:57.912+0000 7f4e8bfff700 1 -- 172.21.15.99:0/3033126691 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f4e98024970 con 0x7f4ea4108840 2024-03-19T01:31:58.693 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:57.912+0000 7f4ea957e700 1 -- 172.21.15.99:0/3033126691 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f4ea4106cb0 con 0x7f4ea4108840 2024-03-19T01:31:58.693 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:57.913+0000 7f4e8bfff700 1 -- 172.21.15.99:0/3033126691 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 8) ==== 46106+0+0 (secure 0 0 0) 0x7f4e98022020 con 0x7f4ea4108840 2024-03-19T01:31:58.693 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:57.913+0000 7f4ea957e700 1 -- 172.21.15.99:0/3033126691 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f4ea4063940 con 0x7f4ea4108840 2024-03-19T01:31:58.693 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:57.914+0000 7f4e8bfff700 1 --2- 172.21.15.99:0/3033126691 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7f4e8c038ed0 0x7f4e8c03b390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:31:58.693 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:57.914+0000 7f4e8bfff700 1 -- 172.21.15.99:0/3033126691 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(2..2 src has 1..2) ==== 940+0+0 (secure 0 0 0) 0x7f4e9804d270 con 0x7f4ea4108840 2024-03-19T01:31:58.693 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:57.914+0000 7f4ea27fc700 1 --2- 172.21.15.99:0/3033126691 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7f4e8c038ed0 0x7f4e8c03b390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:31:58.693 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:57.914+0000 7f4ea27fc700 1 --2- 172.21.15.99:0/3033126691 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7f4e8c038ed0 0x7f4e8c03b390 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7f4e94006fd0 tx=0x7f4e94006e40 comp rx=0 tx=0).ready entity=mgr.14118 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:31:58.693 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:57.919+0000 7f4e8bfff700 1 -- 172.21.15.99:0/3033126691 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f4e98011e20 con 0x7f4ea4108840 2024-03-19T01:31:58.694 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:58.125+0000 7f4ea957e700 1 -- 172.21.15.99:0/3033126691 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command([{prefix=config set, name=mgr/dashboard/ssl_server_port}] v 0) -- 0x7f4ea4106f30 con 0x7f4ea4108840 2024-03-19T01:31:58.694 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:58.128+0000 7f4e8bfff700 1 -- 172.21.15.99:0/3033126691 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mgr/dashboard/ssl_server_port}]=0 v8)=0 v8) ==== 130+0+0 (secure 0 0 0) 0x7f4e9801ab70 con 0x7f4ea4108840 2024-03-19T01:31:58.694 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:58.131+0000 7f4ea957e700 1 -- 172.21.15.99:0/3033126691 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7f4e8c038ed0 msgr2=0x7f4e8c03b390 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:31:58.694 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:58.131+0000 7f4ea957e700 1 --2- 172.21.15.99:0/3033126691 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7f4e8c038ed0 0x7f4e8c03b390 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7f4e94006fd0 tx=0x7f4e94006e40 comp rx=0 tx=0).stop 2024-03-19T01:31:58.694 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:58.131+0000 7f4ea957e700 1 -- 172.21.15.99:0/3033126691 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4ea4108840 msgr2=0x7f4ea41080c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:31:58.694 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:58.131+0000 7f4ea957e700 1 --2- 172.21.15.99:0/3033126691 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4ea4108840 0x7f4ea41080c0 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7f4e980117e0 tx=0x7f4e98011c10 comp rx=0 tx=0).stop 2024-03-19T01:31:58.694 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:58.131+0000 7f4ea957e700 1 -- 172.21.15.99:0/3033126691 shutdown_connections 2024-03-19T01:31:58.694 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:58.131+0000 7f4ea957e700 1 --2- 172.21.15.99:0/3033126691 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7f4e8c038ed0 0x7f4e8c03b390 unknown :-1 s=CLOSED pgs=15 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:58.694 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:58.131+0000 7f4ea957e700 1 --2- 172.21.15.99:0/3033126691 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4ea4108840 0x7f4ea41080c0 unknown :-1 s=CLOSED pgs=52 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:31:58.694 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:58.131+0000 7f4ea957e700 1 -- 172.21.15.99:0/3033126691 >> 172.21.15.99:0/3033126691 conn(0x7f4ea4102270 msgr2=0x7f4ea4102cc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:31:58.694 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:58.131+0000 7f4ea957e700 1 -- 172.21.15.99:0/3033126691 shutdown_connections 2024-03-19T01:31:58.694 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:58.131+0000 7f4ea957e700 1 -- 172.21.15.99:0/3033126691 wait complete. 2024-03-19T01:31:58.694 INFO:teuthology.orchestra.run.smithi099.stdout:Enabling the dashboard module... 2024-03-19T01:31:58.783 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:58 smithi099 ceph-mon[62951]: from='mgr.14118 172.21.15.99:0/2932773335' entity='mgr.x' 2024-03-19T01:31:58.784 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:58 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/3033126691' entity='client.admin' 2024-03-19T01:32:00.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:59 smithi099 ceph-mon[62951]: from='mgr.14118 172.21.15.99:0/2932773335' entity='mgr.x' 2024-03-19T01:32:00.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:31:59 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/2973158712' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-03-19T01:32:00.092 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:59 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: ignoring --setuser ceph since I am not root 2024-03-19T01:32:00.092 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:31:59 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: ignoring --setgroup ceph since I am not root 2024-03-19T01:32:00.438 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:32:00 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:32:00.343+0000 7f6861571200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-19T01:32:00.461 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:59.115+0000 7fd6f3ea2700 1 Processor -- start 2024-03-19T01:32:00.461 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:59.115+0000 7fd6f3ea2700 1 -- start start 2024-03-19T01:32:00.462 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:59.115+0000 7fd6f3ea2700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd6ec108850 0x7fd6ec10ac40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:32:00.462 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:59.115+0000 7fd6f3ea2700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fd6ec07be70 con 0x7fd6ec108850 2024-03-19T01:32:00.462 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:59.115+0000 7fd6f1c3e700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd6ec108850 0x7fd6ec10ac40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:32:00.462 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:59.115+0000 7fd6f1c3e700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd6ec108850 0x7fd6ec10ac40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:59512/0 (socket says 172.21.15.99:59512) 2024-03-19T01:32:00.463 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:59.115+0000 7fd6f1c3e700 1 -- 172.21.15.99:0/2162618214 learned_addr learned my addr 172.21.15.99:0/2162618214 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:32:00.463 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:59.115+0000 7fd6f1c3e700 1 -- 172.21.15.99:0/2162618214 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd6ec10b180 con 0x7fd6ec108850 2024-03-19T01:32:00.463 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:59.116+0000 7fd6f1c3e700 1 --2- 172.21.15.99:0/2162618214 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd6ec108850 0x7fd6ec10ac40 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7fd6e80098a0 tx=0x7fd6e8009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=aa648f10b649883d server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:32:00.463 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:59.116+0000 7fd6f0c3c700 1 -- 172.21.15.99:0/2162618214 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fd6e8004160 con 0x7fd6ec108850 2024-03-19T01:32:00.464 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:59.116+0000 7fd6f0c3c700 1 -- 172.21.15.99:0/2162618214 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd6e8004300 con 0x7fd6ec108850 2024-03-19T01:32:00.464 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:59.116+0000 7fd6f0c3c700 1 -- 172.21.15.99:0/2162618214 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fd6e8011b20 con 0x7fd6ec108850 2024-03-19T01:32:00.464 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:59.116+0000 7fd6f3ea2700 1 -- 172.21.15.99:0/2162618214 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd6ec108850 msgr2=0x7fd6ec10ac40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:32:00.464 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:59.116+0000 7fd6f3ea2700 1 --2- 172.21.15.99:0/2162618214 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd6ec108850 0x7fd6ec10ac40 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7fd6e80098a0 tx=0x7fd6e8009bb0 comp rx=0 tx=0).stop 2024-03-19T01:32:00.464 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:59.116+0000 7fd6f3ea2700 1 -- 172.21.15.99:0/2162618214 shutdown_connections 2024-03-19T01:32:00.464 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:59.116+0000 7fd6f3ea2700 1 --2- 172.21.15.99:0/2162618214 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd6ec108850 0x7fd6ec10ac40 unknown :-1 s=CLOSED pgs=53 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:32:00.464 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:59.116+0000 7fd6f3ea2700 1 -- 172.21.15.99:0/2162618214 >> 172.21.15.99:0/2162618214 conn(0x7fd6ec102240 msgr2=0x7fd6ec104660 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:32:00.464 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:59.117+0000 7fd6f3ea2700 1 -- 172.21.15.99:0/2162618214 shutdown_connections 2024-03-19T01:32:00.464 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:59.117+0000 7fd6f3ea2700 1 -- 172.21.15.99:0/2162618214 wait complete. 2024-03-19T01:32:00.464 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:59.117+0000 7fd6f3ea2700 1 Processor -- start 2024-03-19T01:32:00.464 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:59.117+0000 7fd6f3ea2700 1 -- start start 2024-03-19T01:32:00.464 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:59.117+0000 7fd6f3ea2700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd6ec108850 0x7fd6ec1080d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:32:00.465 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:59.117+0000 7fd6f3ea2700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fd6ec10c190 con 0x7fd6ec108850 2024-03-19T01:32:00.465 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:59.117+0000 7fd6f1c3e700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd6ec108850 0x7fd6ec1080d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:32:00.465 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:59.117+0000 7fd6f1c3e700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd6ec108850 0x7fd6ec1080d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:59518/0 (socket says 172.21.15.99:59518) 2024-03-19T01:32:00.465 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:59.117+0000 7fd6f1c3e700 1 -- 172.21.15.99:0/2973158712 learned_addr learned my addr 172.21.15.99:0/2973158712 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:32:00.465 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:59.118+0000 7fd6f1c3e700 1 -- 172.21.15.99:0/2973158712 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd6ec108610 con 0x7fd6ec108850 2024-03-19T01:32:00.465 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:59.118+0000 7fd6f1c3e700 1 --2- 172.21.15.99:0/2973158712 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd6ec108850 0x7fd6ec1080d0 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 crypto rx=0x7fd6e80117e0 tx=0x7fd6e8011c10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:32:00.465 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:59.118+0000 7fd6e2ffd700 1 -- 172.21.15.99:0/2973158712 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fd6e8003bf0 con 0x7fd6ec108850 2024-03-19T01:32:00.465 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:59.118+0000 7fd6e2ffd700 1 -- 172.21.15.99:0/2973158712 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd6e801a910 con 0x7fd6ec108850 2024-03-19T01:32:00.465 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:59.118+0000 7fd6f3ea2700 1 -- 172.21.15.99:0/2973158712 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd6ec106800 con 0x7fd6ec108850 2024-03-19T01:32:00.465 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:59.118+0000 7fd6e2ffd700 1 -- 172.21.15.99:0/2973158712 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fd6e8024970 con 0x7fd6ec108850 2024-03-19T01:32:00.465 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:59.118+0000 7fd6f3ea2700 1 -- 172.21.15.99:0/2973158712 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fd6ec106cc0 con 0x7fd6ec108850 2024-03-19T01:32:00.465 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:59.119+0000 7fd6e2ffd700 1 -- 172.21.15.99:0/2973158712 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 8) ==== 46106+0+0 (secure 0 0 0) 0x7fd6e8022020 con 0x7fd6ec108850 2024-03-19T01:32:00.465 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:59.119+0000 7fd6e2ffd700 1 --2- 172.21.15.99:0/2973158712 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7fd6d8038ed0 0x7fd6d803b390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:32:00.466 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:59.119+0000 7fd6e2ffd700 1 -- 172.21.15.99:0/2973158712 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(2..2 src has 1..2) ==== 940+0+0 (secure 0 0 0) 0x7fd6e804d4e0 con 0x7fd6ec108850 2024-03-19T01:32:00.466 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:59.119+0000 7fd6f3ea2700 1 -- 172.21.15.99:0/2973158712 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fd6ec063940 con 0x7fd6ec108850 2024-03-19T01:32:00.466 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:59.119+0000 7fd6f143d700 1 --2- 172.21.15.99:0/2973158712 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7fd6d8038ed0 0x7fd6d803b390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:32:00.466 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:59.120+0000 7fd6f143d700 1 --2- 172.21.15.99:0/2973158712 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7fd6d8038ed0 0x7fd6d803b390 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7fd6dc006fd0 tx=0x7fd6dc006e40 comp rx=0 tx=0).ready entity=mgr.14118 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:32:00.466 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:59.124+0000 7fd6e2ffd700 1 -- 172.21.15.99:0/2973158712 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7fd6e8011e20 con 0x7fd6ec108850 2024-03-19T01:32:00.466 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:59.406+0000 7fd6f3ea2700 1 -- 172.21.15.99:0/2973158712 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "mgr module enable", "module": "dashboard"} v 0) -- 0x7fd6ec107340 con 0x7fd6ec108850 2024-03-19T01:32:00.466 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:59.768+0000 7fd6e2ffd700 1 -- 172.21.15.99:0/2973158712 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "mgr module enable", "module": "dashboard"}]=0 v9) ==== 88+0+0 (secure 0 0 0) 0x7fd6e8019c70 con 0x7fd6ec108850 2024-03-19T01:32:00.466 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:59.769+0000 7fd6e2ffd700 1 -- 172.21.15.99:0/2973158712 <== mon.0 v2:172.21.15.99:3300/0 8 ==== mgrmap(e 9) ==== 46119+0+0 (secure 0 0 0) 0x7fd6e802d910 con 0x7fd6ec108850 2024-03-19T01:32:00.466 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:59.771+0000 7fd6f3ea2700 1 -- 172.21.15.99:0/2973158712 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7fd6d8038ed0 msgr2=0x7fd6d803b390 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:32:00.466 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:59.771+0000 7fd6f3ea2700 1 --2- 172.21.15.99:0/2973158712 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7fd6d8038ed0 0x7fd6d803b390 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7fd6dc006fd0 tx=0x7fd6dc006e40 comp rx=0 tx=0).stop 2024-03-19T01:32:00.466 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:59.771+0000 7fd6f3ea2700 1 -- 172.21.15.99:0/2973158712 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd6ec108850 msgr2=0x7fd6ec1080d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:32:00.466 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:59.771+0000 7fd6f3ea2700 1 --2- 172.21.15.99:0/2973158712 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd6ec108850 0x7fd6ec1080d0 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 crypto rx=0x7fd6e80117e0 tx=0x7fd6e8011c10 comp rx=0 tx=0).stop 2024-03-19T01:32:00.467 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:59.771+0000 7fd6f3ea2700 1 -- 172.21.15.99:0/2973158712 shutdown_connections 2024-03-19T01:32:00.467 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:59.771+0000 7fd6f3ea2700 1 --2- 172.21.15.99:0/2973158712 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7fd6d8038ed0 0x7fd6d803b390 unknown :-1 s=CLOSED pgs=16 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:32:00.467 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:59.771+0000 7fd6f3ea2700 1 --2- 172.21.15.99:0/2973158712 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd6ec108850 0x7fd6ec1080d0 unknown :-1 s=CLOSED pgs=54 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:32:00.467 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:59.771+0000 7fd6f3ea2700 1 -- 172.21.15.99:0/2973158712 >> 172.21.15.99:0/2973158712 conn(0x7fd6ec102240 msgr2=0x7fd6ec102cc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:32:00.467 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:59.771+0000 7fd6f3ea2700 1 -- 172.21.15.99:0/2973158712 shutdown_connections 2024-03-19T01:32:00.467 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:31:59.771+0000 7fd6f3ea2700 1 -- 172.21.15.99:0/2973158712 wait complete. 2024-03-19T01:32:00.769 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:32:00 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:32:00.437+0000 7f6861571200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-19T01:32:01.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:00 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/2973158712' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-03-19T01:32:01.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:00 smithi099 ceph-mon[62951]: mgrmap e9: x(active, since 19s) 2024-03-19T01:32:01.407 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:32:01 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:32:01.096+0000 7f6861571200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-19T01:32:01.408 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:32:01 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:32:01.193+0000 7f6861571200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-19T01:32:01.408 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:32:01 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:32:01.339+0000 7f6861571200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-19T01:32:01.769 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:32:01 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:32:01.436+0000 7f6861571200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-19T01:32:01.769 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:32:01 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:32:01.525+0000 7f6861571200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-19T01:32:02.088 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:01 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/1412968326' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-03-19T01:32:02.089 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:32:01 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:32:01.793+0000 7f6861571200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-19T01:32:02.089 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:32:01 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:32:01.905+0000 7f6861571200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-19T01:32:02.089 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:32:01 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:32:01.991+0000 7f6861571200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-19T01:32:02.089 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:32:02 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:32:02.087+0000 7f6861571200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-19T01:32:02.114 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout { 2024-03-19T01:32:02.115 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "epoch": 9, 2024-03-19T01:32:02.115 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "available": true, 2024-03-19T01:32:02.115 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "active_name": "x", 2024-03-19T01:32:02.115 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-03-19T01:32:02.115 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout } 2024-03-19T01:32:02.115 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:01.201+0000 7f0715c4d700 1 Processor -- start 2024-03-19T01:32:02.115 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:01.201+0000 7f0715c4d700 1 -- start start 2024-03-19T01:32:02.115 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:01.201+0000 7f0715c4d700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f07101088e0 0x7f071010acd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:32:02.115 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:01.201+0000 7f0715c4d700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f071007bdf0 con 0x7f07101088e0 2024-03-19T01:32:02.115 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:01.201+0000 7f070f7fe700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f07101088e0 0x7f071010acd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:32:02.115 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:01.201+0000 7f070f7fe700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f07101088e0 0x7f071010acd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:59540/0 (socket says 172.21.15.99:59540) 2024-03-19T01:32:02.115 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:01.201+0000 7f070f7fe700 1 -- 172.21.15.99:0/275029233 learned_addr learned my addr 172.21.15.99:0/275029233 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:32:02.116 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:01.202+0000 7f070f7fe700 1 -- 172.21.15.99:0/275029233 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f071010b210 con 0x7f07101088e0 2024-03-19T01:32:02.116 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:01.202+0000 7f070f7fe700 1 --2- 172.21.15.99:0/275029233 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f07101088e0 0x7f071010acd0 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7f07040098a0 tx=0x7f0704009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=285cfeb641396f4f server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:32:02.116 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:01.202+0000 7f070e7fc700 1 -- 172.21.15.99:0/275029233 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f0704004160 con 0x7f07101088e0 2024-03-19T01:32:02.116 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:01.202+0000 7f070e7fc700 1 -- 172.21.15.99:0/275029233 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f0704004300 con 0x7f07101088e0 2024-03-19T01:32:02.116 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:01.202+0000 7f070e7fc700 1 -- 172.21.15.99:0/275029233 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f0704011b20 con 0x7f07101088e0 2024-03-19T01:32:02.116 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:01.203+0000 7f0715c4d700 1 -- 172.21.15.99:0/275029233 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f07101088e0 msgr2=0x7f071010acd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:32:02.116 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:01.203+0000 7f0715c4d700 1 --2- 172.21.15.99:0/275029233 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f07101088e0 0x7f071010acd0 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7f07040098a0 tx=0x7f0704009bb0 comp rx=0 tx=0).stop 2024-03-19T01:32:02.116 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:01.203+0000 7f0715c4d700 1 -- 172.21.15.99:0/275029233 shutdown_connections 2024-03-19T01:32:02.116 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:01.203+0000 7f0715c4d700 1 --2- 172.21.15.99:0/275029233 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f07101088e0 0x7f071010acd0 unknown :-1 s=CLOSED pgs=57 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:32:02.116 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:01.203+0000 7f0715c4d700 1 -- 172.21.15.99:0/275029233 >> 172.21.15.99:0/275029233 conn(0x7f0710102270 msgr2=0x7f0710104690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:32:02.116 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:01.203+0000 7f0715c4d700 1 -- 172.21.15.99:0/275029233 shutdown_connections 2024-03-19T01:32:02.116 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:01.203+0000 7f0715c4d700 1 -- 172.21.15.99:0/275029233 wait complete. 2024-03-19T01:32:02.116 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:01.203+0000 7f0715c4d700 1 Processor -- start 2024-03-19T01:32:02.116 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:01.204+0000 7f0715c4d700 1 -- start start 2024-03-19T01:32:02.117 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:01.204+0000 7f0715c4d700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f07101088e0 0x7f0710108160 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:32:02.117 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:01.204+0000 7f0715c4d700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f071010c1b0 con 0x7f07101088e0 2024-03-19T01:32:02.117 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:01.204+0000 7f070f7fe700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f07101088e0 0x7f0710108160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:32:02.117 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:01.204+0000 7f070f7fe700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f07101088e0 0x7f0710108160 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:59554/0 (socket says 172.21.15.99:59554) 2024-03-19T01:32:02.117 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:01.204+0000 7f070f7fe700 1 -- 172.21.15.99:0/1412968326 learned_addr learned my addr 172.21.15.99:0/1412968326 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:32:02.117 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:01.204+0000 7f070f7fe700 1 -- 172.21.15.99:0/1412968326 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f07101086a0 con 0x7f07101088e0 2024-03-19T01:32:02.117 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:01.204+0000 7f070f7fe700 1 --2- 172.21.15.99:0/1412968326 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f07101088e0 0x7f0710108160 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 crypto rx=0x7f07040117e0 tx=0x7f0704011c10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:32:02.117 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:01.204+0000 7f070cff9700 1 -- 172.21.15.99:0/1412968326 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f0704003bf0 con 0x7f07101088e0 2024-03-19T01:32:02.117 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:01.205+0000 7f070cff9700 1 -- 172.21.15.99:0/1412968326 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f070401a910 con 0x7f07101088e0 2024-03-19T01:32:02.117 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:01.205+0000 7f070cff9700 1 -- 172.21.15.99:0/1412968326 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f0704024970 con 0x7f07101088e0 2024-03-19T01:32:02.117 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:01.205+0000 7f0715c4d700 1 -- 172.21.15.99:0/1412968326 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f0710106890 con 0x7f07101088e0 2024-03-19T01:32:02.117 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:01.205+0000 7f0715c4d700 1 -- 172.21.15.99:0/1412968326 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f0710106d50 con 0x7f07101088e0 2024-03-19T01:32:02.117 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:01.206+0000 7f070cff9700 1 -- 172.21.15.99:0/1412968326 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 9) ==== 46119+0+0 (secure 0 0 0) 0x7f0704022020 con 0x7f07101088e0 2024-03-19T01:32:02.118 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:01.206+0000 7f0715c4d700 1 -- 172.21.15.99:0/1412968326 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f0710069420 con 0x7f07101088e0 2024-03-19T01:32:02.118 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:01.206+0000 7f070cff9700 1 --2- 172.21.15.99:0/1412968326 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7f06f8038f20 0x7f06f803b3e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:32:02.118 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:01.206+0000 7f070cff9700 1 -- 172.21.15.99:0/1412968326 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(2..2 src has 1..2) ==== 940+0+0 (secure 0 0 0) 0x7f070404d240 con 0x7f07101088e0 2024-03-19T01:32:02.118 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:01.206+0000 7f070effd700 1 -- 172.21.15.99:0/1412968326 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7f06f8038f20 msgr2=0x7f06f803b3e0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.99:6800/1328245934 2024-03-19T01:32:02.118 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:01.206+0000 7f070effd700 1 --2- 172.21.15.99:0/1412968326 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7f06f8038f20 0x7f06f803b3e0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2024-03-19T01:32:02.118 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:01.211+0000 7f070cff9700 1 -- 172.21.15.99:0/1412968326 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f0704011e20 con 0x7f07101088e0 2024-03-19T01:32:02.118 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:01.407+0000 7f070effd700 1 -- 172.21.15.99:0/1412968326 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7f06f8038f20 msgr2=0x7f06f803b3e0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.99:6800/1328245934 2024-03-19T01:32:02.118 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:01.407+0000 7f070effd700 1 --2- 172.21.15.99:0/1412968326 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7f06f8038f20 0x7f06f803b3e0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.400000 2024-03-19T01:32:02.118 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:01.475+0000 7f0715c4d700 1 -- 172.21.15.99:0/1412968326 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "mgr stat"} v 0) -- 0x7f07101075f0 con 0x7f07101088e0 2024-03-19T01:32:02.118 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:01.475+0000 7f070cff9700 1 -- 172.21.15.99:0/1412968326 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "mgr stat"}]=0 v9) ==== 56+0+88 (secure 0 0 0) 0x7f070400ba00 con 0x7f07101088e0 2024-03-19T01:32:02.118 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:01.477+0000 7f0715c4d700 1 -- 172.21.15.99:0/1412968326 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7f06f8038f20 msgr2=0x7f06f803b3e0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-03-19T01:32:02.118 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:01.477+0000 7f0715c4d700 1 --2- 172.21.15.99:0/1412968326 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7f06f8038f20 0x7f06f803b3e0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:32:02.118 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:01.477+0000 7f0715c4d700 1 -- 172.21.15.99:0/1412968326 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f07101088e0 msgr2=0x7f0710108160 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:32:02.119 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:01.477+0000 7f0715c4d700 1 --2- 172.21.15.99:0/1412968326 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f07101088e0 0x7f0710108160 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 crypto rx=0x7f07040117e0 tx=0x7f0704011c10 comp rx=0 tx=0).stop 2024-03-19T01:32:02.119 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:01.477+0000 7f0715c4d700 1 -- 172.21.15.99:0/1412968326 shutdown_connections 2024-03-19T01:32:02.119 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:01.477+0000 7f0715c4d700 1 --2- 172.21.15.99:0/1412968326 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7f06f8038f20 0x7f06f803b3e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:32:02.119 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:01.477+0000 7f0715c4d700 1 --2- 172.21.15.99:0/1412968326 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f07101088e0 0x7f0710108160 unknown :-1 s=CLOSED pgs=58 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:32:02.119 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:01.478+0000 7f0715c4d700 1 -- 172.21.15.99:0/1412968326 >> 172.21.15.99:0/1412968326 conn(0x7f0710102270 msgr2=0x7f0710102cd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:32:02.119 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:01.478+0000 7f0715c4d700 1 -- 172.21.15.99:0/1412968326 shutdown_connections 2024-03-19T01:32:02.119 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:01.478+0000 7f0715c4d700 1 -- 172.21.15.99:0/1412968326 wait complete. 2024-03-19T01:32:02.119 INFO:teuthology.orchestra.run.smithi099.stdout:Waiting for the mgr to restart... 2024-03-19T01:32:02.119 INFO:teuthology.orchestra.run.smithi099.stdout:Waiting for mgr epoch 9... 2024-03-19T01:32:02.341 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:32:02 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:32:02.273+0000 7f6861571200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-19T01:32:03.591 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:32:03 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:32:03.241+0000 7f6861571200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-19T01:32:03.591 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:32:03 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:32:03.373+0000 7f6861571200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-19T01:32:03.994 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:32:03 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:32:03.706+0000 7f6861571200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-19T01:32:03.994 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:32:03 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:32:03.797+0000 7f6861571200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-19T01:32:03.994 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:32:03 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:32:03.993+0000 7f6861571200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-19T01:32:04.341 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:32:04 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:32:04.180+0000 7f6861571200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-19T01:32:04.722 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:32:04 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:32:04.465+0000 7f6861571200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-19T01:32:04.722 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:32:04 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:32:04.721+0000 7f6861571200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-19T01:32:05.091 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:32:04 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:32:04.957+0000 7f6861571200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-19T01:32:05.841 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:32:05 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:32:05.762+0000 7f6861571200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-19T01:32:06.591 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:32:06 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:32:06.331+0000 7f6861571200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-19T01:32:06.591 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:32:06 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:32:06.429+0000 7f6861571200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-19T01:32:07.091 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:32:06 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:32:06.647+0000 7f6861571200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-19T01:32:07.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:06 smithi099 ceph-mon[62951]: Active manager daemon x restarted 2024-03-19T01:32:07.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:06 smithi099 ceph-mon[62951]: Activating manager daemon x 2024-03-19T01:32:07.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:06 smithi099 ceph-mon[62951]: osdmap e3: 0 total, 0 up, 0 in 2024-03-19T01:32:07.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:06 smithi099 ceph-mon[62951]: mgrmap e10: x(active, starting, since 0.00340946s) 2024-03-19T01:32:07.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:06 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-19T01:32:07.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:06 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-19T01:32:07.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:06 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-19T01:32:07.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:06 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-19T01:32:07.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:06 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-19T01:32:07.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:06 smithi099 ceph-mon[62951]: Manager daemon x is now available 2024-03-19T01:32:07.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:06 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:32:07.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:06 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:32:07.967 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:07 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-03-19T01:32:07.967 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:07 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-03-19T01:32:07.967 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:07 smithi099 ceph-mon[62951]: mgrmap e11: x(active, since 1.00674s) 2024-03-19T01:32:08.309 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout { 2024-03-19T01:32:08.317 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2024-03-19T01:32:08.317 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "initialized": true 2024-03-19T01:32:08.317 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout } 2024-03-19T01:32:08.317 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:02.517+0000 7fcfd8769700 1 Processor -- start 2024-03-19T01:32:08.317 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:02.518+0000 7fcfd8769700 1 -- start start 2024-03-19T01:32:08.317 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:02.518+0000 7fcfd8769700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fcfd0106c90 0x7fcfd01070b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:32:08.317 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:02.518+0000 7fcfd8769700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fcfd0107680 con 0x7fcfd0106c90 2024-03-19T01:32:08.317 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:02.518+0000 7fcfd6505700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fcfd0106c90 0x7fcfd01070b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:32:08.317 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:02.518+0000 7fcfd6505700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fcfd0106c90 0x7fcfd01070b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:59564/0 (socket says 172.21.15.99:59564) 2024-03-19T01:32:08.317 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:02.518+0000 7fcfd6505700 1 -- 172.21.15.99:0/4191503017 learned_addr learned my addr 172.21.15.99:0/4191503017 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:32:08.318 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:02.518+0000 7fcfd6505700 1 -- 172.21.15.99:0/4191503017 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fcfd0107e90 con 0x7fcfd0106c90 2024-03-19T01:32:08.318 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:02.519+0000 7fcfd6505700 1 --2- 172.21.15.99:0/4191503017 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fcfd0106c90 0x7fcfd01070b0 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7fcfcc0098a0 tx=0x7fcfcc009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6030c61143759ed3 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:32:08.318 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:02.519+0000 7fcfd5503700 1 -- 172.21.15.99:0/4191503017 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fcfcc004160 con 0x7fcfd0106c90 2024-03-19T01:32:08.318 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:02.519+0000 7fcfd5503700 1 -- 172.21.15.99:0/4191503017 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fcfcc004300 con 0x7fcfd0106c90 2024-03-19T01:32:08.318 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:02.519+0000 7fcfd5503700 1 -- 172.21.15.99:0/4191503017 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fcfcc011b20 con 0x7fcfd0106c90 2024-03-19T01:32:08.318 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:02.519+0000 7fcfd8769700 1 -- 172.21.15.99:0/4191503017 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fcfd0106c90 msgr2=0x7fcfd01070b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:32:08.318 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:02.519+0000 7fcfd8769700 1 --2- 172.21.15.99:0/4191503017 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fcfd0106c90 0x7fcfd01070b0 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7fcfcc0098a0 tx=0x7fcfcc009bb0 comp rx=0 tx=0).stop 2024-03-19T01:32:08.318 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:02.520+0000 7fcfd8769700 1 -- 172.21.15.99:0/4191503017 shutdown_connections 2024-03-19T01:32:08.318 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:02.520+0000 7fcfd8769700 1 --2- 172.21.15.99:0/4191503017 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fcfd0106c90 0x7fcfd01070b0 unknown :-1 s=CLOSED pgs=59 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:32:08.318 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:02.520+0000 7fcfd8769700 1 -- 172.21.15.99:0/4191503017 >> 172.21.15.99:0/4191503017 conn(0x7fcfd0102270 msgr2=0x7fcfd0104690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:32:08.318 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:02.520+0000 7fcfd8769700 1 -- 172.21.15.99:0/4191503017 shutdown_connections 2024-03-19T01:32:08.318 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:02.520+0000 7fcfd8769700 1 -- 172.21.15.99:0/4191503017 wait complete. 2024-03-19T01:32:08.319 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:02.520+0000 7fcfd8769700 1 Processor -- start 2024-03-19T01:32:08.319 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:02.520+0000 7fcfd8769700 1 -- start start 2024-03-19T01:32:08.319 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:02.520+0000 7fcfd8769700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fcfd0106c90 0x7fcfd019bda0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:32:08.319 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:02.520+0000 7fcfd8769700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fcfd010b240 con 0x7fcfd0106c90 2024-03-19T01:32:08.319 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:02.521+0000 7fcfd6505700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fcfd0106c90 0x7fcfd019bda0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:32:08.319 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:02.521+0000 7fcfd6505700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fcfd0106c90 0x7fcfd019bda0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:59568/0 (socket says 172.21.15.99:59568) 2024-03-19T01:32:08.319 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:02.521+0000 7fcfd6505700 1 -- 172.21.15.99:0/3671605753 learned_addr learned my addr 172.21.15.99:0/3671605753 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:32:08.319 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:02.521+0000 7fcfd6505700 1 -- 172.21.15.99:0/3671605753 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fcfd019c2e0 con 0x7fcfd0106c90 2024-03-19T01:32:08.319 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:02.521+0000 7fcfd6505700 1 --2- 172.21.15.99:0/3671605753 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fcfd0106c90 0x7fcfd019bda0 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7fcfcc0117e0 tx=0x7fcfcc011c10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:32:08.319 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:02.521+0000 7fcfc77fe700 1 -- 172.21.15.99:0/3671605753 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fcfcc003bf0 con 0x7fcfd0106c90 2024-03-19T01:32:08.319 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:02.521+0000 7fcfc77fe700 1 -- 172.21.15.99:0/3671605753 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fcfcc01a910 con 0x7fcfd0106c90 2024-03-19T01:32:08.319 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:02.521+0000 7fcfd8769700 1 -- 172.21.15.99:0/3671605753 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fcfd019c570 con 0x7fcfd0106c90 2024-03-19T01:32:08.319 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:02.521+0000 7fcfc77fe700 1 -- 172.21.15.99:0/3671605753 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fcfcc0249a0 con 0x7fcfd0106c90 2024-03-19T01:32:08.319 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:02.521+0000 7fcfd8769700 1 -- 172.21.15.99:0/3671605753 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fcfd019f220 con 0x7fcfd0106c90 2024-03-19T01:32:08.320 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:02.522+0000 7fcfc77fe700 1 -- 172.21.15.99:0/3671605753 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 9) ==== 46119+0+0 (secure 0 0 0) 0x7fcfcc022020 con 0x7fcfd0106c90 2024-03-19T01:32:08.320 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:02.522+0000 7fcfc77fe700 1 --2- 172.21.15.99:0/3671605753 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7fcfbc038f80 0x7fcfbc03b440 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:32:08.320 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:02.522+0000 7fcfc77fe700 1 -- 172.21.15.99:0/3671605753 --> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x7fcfbc03bb50 con 0x7fcfbc038f80 2024-03-19T01:32:08.320 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:02.522+0000 7fcfc77fe700 1 -- 172.21.15.99:0/3671605753 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(2..2 src has 1..2) ==== 940+0+0 (secure 0 0 0) 0x7fcfcc04d1a0 con 0x7fcfd0106c90 2024-03-19T01:32:08.320 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:02.523+0000 7fcfd5d04700 1 -- 172.21.15.99:0/3671605753 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7fcfbc038f80 msgr2=0x7fcfbc03b440 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.99:6800/1328245934 2024-03-19T01:32:08.320 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:02.523+0000 7fcfd5d04700 1 --2- 172.21.15.99:0/3671605753 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7fcfbc038f80 0x7fcfbc03b440 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2024-03-19T01:32:08.320 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:02.724+0000 7fcfd5d04700 1 -- 172.21.15.99:0/3671605753 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7fcfbc038f80 msgr2=0x7fcfbc03b440 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.99:6800/1328245934 2024-03-19T01:32:08.320 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:02.724+0000 7fcfd5d04700 1 --2- 172.21.15.99:0/3671605753 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7fcfbc038f80 0x7fcfbc03b440 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.400000 2024-03-19T01:32:08.320 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:03.125+0000 7fcfd5d04700 1 -- 172.21.15.99:0/3671605753 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7fcfbc038f80 msgr2=0x7fcfbc03b440 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.99:6800/1328245934 2024-03-19T01:32:08.320 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:03.125+0000 7fcfd5d04700 1 --2- 172.21.15.99:0/3671605753 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7fcfbc038f80 0x7fcfbc03b440 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.800000 2024-03-19T01:32:08.320 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:03.926+0000 7fcfd5d04700 1 -- 172.21.15.99:0/3671605753 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7fcfbc038f80 msgr2=0x7fcfbc03b440 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.99:6800/1328245934 2024-03-19T01:32:08.320 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:03.926+0000 7fcfd5d04700 1 --2- 172.21.15.99:0/3671605753 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7fcfbc038f80 0x7fcfbc03b440 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 1.600000 2024-03-19T01:32:08.320 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:05.527+0000 7fcfd5d04700 1 -- 172.21.15.99:0/3671605753 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7fcfbc038f80 msgr2=0x7fcfbc03b440 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.99:6800/1328245934 2024-03-19T01:32:08.320 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:05.527+0000 7fcfd5d04700 1 --2- 172.21.15.99:0/3671605753 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7fcfbc038f80 0x7fcfbc03b440 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 3.200000 2024-03-19T01:32:08.321 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:06.662+0000 7fcfc77fe700 1 -- 172.21.15.99:0/3671605753 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mgrmap(e 10) ==== 45886+0+0 (secure 0 0 0) 0x7fcfcc024b40 con 0x7fcfd0106c90 2024-03-19T01:32:08.321 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:06.662+0000 7fcfc77fe700 1 -- 172.21.15.99:0/3671605753 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7fcfbc038f80 msgr2=0x7fcfbc03b440 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-03-19T01:32:08.321 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:06.662+0000 7fcfc77fe700 1 --2- 172.21.15.99:0/3671605753 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7fcfbc038f80 0x7fcfbc03b440 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:32:08.321 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:07.664+0000 7fcfc77fe700 1 -- 172.21.15.99:0/3671605753 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mgrmap(e 11) ==== 46013+0+0 (secure 0 0 0) 0x7fcfcc04f100 con 0x7fcfd0106c90 2024-03-19T01:32:08.321 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:07.664+0000 7fcfc77fe700 1 --2- 172.21.15.99:0/3671605753 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fcfbc03ca30 0x7fcfbc03ee20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:32:08.321 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:07.664+0000 7fcfc77fe700 1 -- 172.21.15.99:0/3671605753 --> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x7fcfcc01ac60 con 0x7fcfbc03ca30 2024-03-19T01:32:08.321 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:07.664+0000 7fcfd5d04700 1 --2- 172.21.15.99:0/3671605753 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fcfbc03ca30 0x7fcfbc03ee20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:32:08.321 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:07.665+0000 7fcfd5d04700 1 --2- 172.21.15.99:0/3671605753 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fcfbc03ca30 0x7fcfbc03ee20 secure :-1 s=READY pgs=4 cs=0 l=1 rev1=1 crypto rx=0x7fcfc0003a10 tx=0x7fcfc0009270 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:32:08.321 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:07.666+0000 7fcfc77fe700 1 -- 172.21.15.99:0/3671605753 <== mgr.14150 v2:172.21.15.99:6800/4272410295 1 ==== command_reply(tid 0: 0 ) ==== 8+0+7774 (secure 0 0 0) 0x7fcfcc01ac60 con 0x7fcfbc03ca30 2024-03-19T01:32:08.321 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:07.673+0000 7fcfd8769700 1 -- 172.21.15.99:0/3671605753 --> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] -- command(tid 1: {"prefix": "mgr_status"}) -- 0x7fcfd0069420 con 0x7fcfbc03ca30 2024-03-19T01:32:08.321 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:07.673+0000 7fcfc77fe700 1 -- 172.21.15.99:0/3671605753 <== mgr.14150 v2:172.21.15.99:6800/4272410295 2 ==== command_reply(tid 1: 0 ) ==== 8+0+52 (secure 0 0 0) 0x7fcfd0069420 con 0x7fcfbc03ca30 2024-03-19T01:32:08.321 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:07.673+0000 7fcfd8769700 1 -- 172.21.15.99:0/3671605753 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fcfbc03ca30 msgr2=0x7fcfbc03ee20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:32:08.321 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:07.673+0000 7fcfd8769700 1 --2- 172.21.15.99:0/3671605753 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fcfbc03ca30 0x7fcfbc03ee20 secure :-1 s=READY pgs=4 cs=0 l=1 rev1=1 crypto rx=0x7fcfc0003a10 tx=0x7fcfc0009270 comp rx=0 tx=0).stop 2024-03-19T01:32:08.322 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:07.673+0000 7fcfd8769700 1 -- 172.21.15.99:0/3671605753 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fcfd0106c90 msgr2=0x7fcfd019bda0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:32:08.322 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:07.673+0000 7fcfd8769700 1 --2- 172.21.15.99:0/3671605753 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fcfd0106c90 0x7fcfd019bda0 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7fcfcc0117e0 tx=0x7fcfcc011c10 comp rx=0 tx=0).stop 2024-03-19T01:32:08.322 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:07.673+0000 7fcfd8769700 1 -- 172.21.15.99:0/3671605753 shutdown_connections 2024-03-19T01:32:08.322 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:07.673+0000 7fcfd8769700 1 --2- 172.21.15.99:0/3671605753 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fcfbc03ca30 0x7fcfbc03ee20 unknown :-1 s=CLOSED pgs=4 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:32:08.322 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:07.673+0000 7fcfd8769700 1 --2- 172.21.15.99:0/3671605753 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fcfd0106c90 0x7fcfd019bda0 unknown :-1 s=CLOSED pgs=60 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:32:08.322 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:07.673+0000 7fcfd8769700 1 --2- 172.21.15.99:0/3671605753 >> [v2:172.21.15.99:6800/1328245934,v1:172.21.15.99:6801/1328245934] conn(0x7fcfbc038f80 0x7fcfbc03b440 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:32:08.322 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:07.673+0000 7fcfd8769700 1 -- 172.21.15.99:0/3671605753 >> 172.21.15.99:0/3671605753 conn(0x7fcfd0102270 msgr2=0x7fcfd0103100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:32:08.322 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:07.674+0000 7fcfd8769700 1 -- 172.21.15.99:0/3671605753 shutdown_connections 2024-03-19T01:32:08.322 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:07.674+0000 7fcfd8769700 1 -- 172.21.15.99:0/3671605753 wait complete. 2024-03-19T01:32:08.322 INFO:teuthology.orchestra.run.smithi099.stdout:mgr epoch 9 is available 2024-03-19T01:32:08.322 INFO:teuthology.orchestra.run.smithi099.stdout:Generating a dashboard self-signed certificate... 2024-03-19T01:32:08.932 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:08 smithi099 ceph-mon[62951]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-03-19T01:32:08.932 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:08 smithi099 ceph-mon[62951]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-03-19T01:32:09.572 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2024-03-19T01:32:09.572 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:08.718+0000 7efdf1f70700 1 Processor -- start 2024-03-19T01:32:09.573 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:08.719+0000 7efdf1f70700 1 -- start start 2024-03-19T01:32:09.573 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:08.719+0000 7efdf1f70700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7efdec108240 0x7efdec108660 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:32:09.573 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:08.719+0000 7efdf1f70700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7efdec108c30 con 0x7efdec108240 2024-03-19T01:32:09.573 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:08.719+0000 7efdeb7fe700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7efdec108240 0x7efdec108660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:32:09.573 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:08.719+0000 7efdeb7fe700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7efdec108240 0x7efdec108660 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:51236/0 (socket says 172.21.15.99:51236) 2024-03-19T01:32:09.573 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:08.719+0000 7efdeb7fe700 1 -- 172.21.15.99:0/2268790525 learned_addr learned my addr 172.21.15.99:0/2268790525 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:32:09.573 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:08.719+0000 7efdeb7fe700 1 -- 172.21.15.99:0/2268790525 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7efdec106e50 con 0x7efdec108240 2024-03-19T01:32:09.573 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:08.719+0000 7efdeb7fe700 1 --2- 172.21.15.99:0/2268790525 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7efdec108240 0x7efdec108660 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7efddc00bb40 tx=0x7efddc00be50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e6d9dcedb70b2415 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:32:09.573 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:08.720+0000 7efdea7fc700 1 -- 172.21.15.99:0/2268790525 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7efddc00dbe0 con 0x7efdec108240 2024-03-19T01:32:09.573 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:08.720+0000 7efdea7fc700 1 -- 172.21.15.99:0/2268790525 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7efddc004510 con 0x7efdec108240 2024-03-19T01:32:09.573 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:08.720+0000 7efdea7fc700 1 -- 172.21.15.99:0/2268790525 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7efddc010470 con 0x7efdec108240 2024-03-19T01:32:09.573 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:08.720+0000 7efdf1f70700 1 -- 172.21.15.99:0/2268790525 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7efdec108240 msgr2=0x7efdec108660 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:32:09.573 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:08.720+0000 7efdf1f70700 1 --2- 172.21.15.99:0/2268790525 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7efdec108240 0x7efdec108660 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7efddc00bb40 tx=0x7efddc00be50 comp rx=0 tx=0).stop 2024-03-19T01:32:09.573 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:08.720+0000 7efdf1f70700 1 -- 172.21.15.99:0/2268790525 shutdown_connections 2024-03-19T01:32:09.574 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:08.720+0000 7efdf1f70700 1 --2- 172.21.15.99:0/2268790525 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7efdec108240 0x7efdec108660 unknown :-1 s=CLOSED pgs=68 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:32:09.574 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:08.720+0000 7efdf1f70700 1 -- 172.21.15.99:0/2268790525 >> 172.21.15.99:0/2268790525 conn(0x7efdec1019c0 msgr2=0x7efdec103de0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:32:09.574 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:08.720+0000 7efdf1f70700 1 -- 172.21.15.99:0/2268790525 shutdown_connections 2024-03-19T01:32:09.574 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:08.720+0000 7efdf1f70700 1 -- 172.21.15.99:0/2268790525 wait complete. 2024-03-19T01:32:09.574 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:08.721+0000 7efdf1f70700 1 Processor -- start 2024-03-19T01:32:09.574 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:08.721+0000 7efdf1f70700 1 -- start start 2024-03-19T01:32:09.574 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:08.722+0000 7efdf1f70700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7efdec108240 0x7efdec199ba0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:32:09.574 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:08.722+0000 7efdf1f70700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7efdec105d40 con 0x7efdec108240 2024-03-19T01:32:09.574 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:08.722+0000 7efdeb7fe700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7efdec108240 0x7efdec199ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:32:09.574 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:08.722+0000 7efdeb7fe700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7efdec108240 0x7efdec199ba0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:51240/0 (socket says 172.21.15.99:51240) 2024-03-19T01:32:09.574 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:08.722+0000 7efdeb7fe700 1 -- 172.21.15.99:0/219111320 learned_addr learned my addr 172.21.15.99:0/219111320 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:32:09.574 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:08.722+0000 7efdeb7fe700 1 -- 172.21.15.99:0/219111320 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7efdec19a0e0 con 0x7efdec108240 2024-03-19T01:32:09.574 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:08.722+0000 7efdeb7fe700 1 --2- 172.21.15.99:0/219111320 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7efdec108240 0x7efdec199ba0 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x7efddc003bb0 tx=0x7efddc0040b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:32:09.575 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:08.723+0000 7efde8ff9700 1 -- 172.21.15.99:0/219111320 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7efddc027070 con 0x7efdec108240 2024-03-19T01:32:09.575 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:08.723+0000 7efde8ff9700 1 -- 172.21.15.99:0/219111320 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7efddc00de20 con 0x7efdec108240 2024-03-19T01:32:09.575 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:08.723+0000 7efdf1f70700 1 -- 172.21.15.99:0/219111320 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7efdec19a370 con 0x7efdec108240 2024-03-19T01:32:09.575 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:08.723+0000 7efde8ff9700 1 -- 172.21.15.99:0/219111320 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7efddc022b80 con 0x7efdec108240 2024-03-19T01:32:09.575 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:08.723+0000 7efdf1f70700 1 -- 172.21.15.99:0/219111320 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7efdec19a790 con 0x7efdec108240 2024-03-19T01:32:09.575 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:08.724+0000 7efde8ff9700 1 -- 172.21.15.99:0/219111320 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 12) ==== 46119+0+0 (secure 0 0 0) 0x7efddc02a070 con 0x7efdec108240 2024-03-19T01:32:09.575 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:08.724+0000 7efdf1f70700 1 -- 172.21.15.99:0/219111320 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7efdec069420 con 0x7efdec108240 2024-03-19T01:32:09.575 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:08.724+0000 7efde8ff9700 1 --2- 172.21.15.99:0/219111320 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7efdd4038f20 0x7efdd403b3e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:32:09.575 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:08.724+0000 7efde8ff9700 1 -- 172.21.15.99:0/219111320 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(3..3 src has 1..3) ==== 1155+0+0 (secure 0 0 0) 0x7efddc022e00 con 0x7efdec108240 2024-03-19T01:32:09.575 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:08.725+0000 7efdeaffd700 1 --2- 172.21.15.99:0/219111320 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7efdd4038f20 0x7efdd403b3e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:32:09.575 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:08.725+0000 7efdeaffd700 1 --2- 172.21.15.99:0/219111320 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7efdd4038f20 0x7efdd403b3e0 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7efde0006fd0 tx=0x7efde0006e40 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:32:09.575 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:08.729+0000 7efde8ff9700 1 -- 172.21.15.99:0/219111320 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7efddc015070 con 0x7efdec108240 2024-03-19T01:32:09.576 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:08.936+0000 7efdf1f70700 1 -- 172.21.15.99:0/219111320 --> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] -- mgr_command(tid 0: {"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}) -- 0x7efdec062750 con 0x7efdd4038f20 2024-03-19T01:32:09.576 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:09.017+0000 7efde8ff9700 1 -- 172.21.15.99:0/219111320 <== mgr.14150 v2:172.21.15.99:6800/4272410295 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+32 (secure 0 0 0) 0x7efdec062750 con 0x7efdd4038f20 2024-03-19T01:32:09.576 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:09.019+0000 7efdf1f70700 1 -- 172.21.15.99:0/219111320 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7efdd4038f20 msgr2=0x7efdd403b3e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:32:09.576 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:09.019+0000 7efdf1f70700 1 --2- 172.21.15.99:0/219111320 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7efdd4038f20 0x7efdd403b3e0 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7efde0006fd0 tx=0x7efde0006e40 comp rx=0 tx=0).stop 2024-03-19T01:32:09.576 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:09.019+0000 7efdf1f70700 1 -- 172.21.15.99:0/219111320 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7efdec108240 msgr2=0x7efdec199ba0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:32:09.576 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:09.019+0000 7efdf1f70700 1 --2- 172.21.15.99:0/219111320 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7efdec108240 0x7efdec199ba0 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x7efddc003bb0 tx=0x7efddc0040b0 comp rx=0 tx=0).stop 2024-03-19T01:32:09.576 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:09.019+0000 7efdf1f70700 1 -- 172.21.15.99:0/219111320 shutdown_connections 2024-03-19T01:32:09.576 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:09.019+0000 7efdf1f70700 1 --2- 172.21.15.99:0/219111320 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7efdd4038f20 0x7efdd403b3e0 unknown :-1 s=CLOSED pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:32:09.576 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:09.019+0000 7efdf1f70700 1 --2- 172.21.15.99:0/219111320 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7efdec108240 0x7efdec199ba0 unknown :-1 s=CLOSED pgs=69 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:32:09.576 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:09.019+0000 7efdf1f70700 1 -- 172.21.15.99:0/219111320 >> 172.21.15.99:0/219111320 conn(0x7efdec1019c0 msgr2=0x7efdec1901e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:32:09.576 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:09.019+0000 7efdf1f70700 1 -- 172.21.15.99:0/219111320 shutdown_connections 2024-03-19T01:32:09.576 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:09.019+0000 7efdf1f70700 1 -- 172.21.15.99:0/219111320 wait complete. 2024-03-19T01:32:09.576 INFO:teuthology.orchestra.run.smithi099.stdout:Creating initial admin user... 2024-03-19T01:32:09.836 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:09 smithi099 ceph-mon[62951]: [19/Mar/2024:01:32:07] ENGINE Bus STARTING 2024-03-19T01:32:09.837 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:09 smithi099 ceph-mon[62951]: [19/Mar/2024:01:32:08] ENGINE Serving on https://172.21.15.99:7150 2024-03-19T01:32:09.837 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:09 smithi099 ceph-mon[62951]: [19/Mar/2024:01:32:08] ENGINE Serving on http://172.21.15.99:8765 2024-03-19T01:32:09.837 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:09 smithi099 ceph-mon[62951]: [19/Mar/2024:01:32:08] ENGINE Bus STARTED 2024-03-19T01:32:09.837 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:09 smithi099 ceph-mon[62951]: mgrmap e12: x(active, since 2s) 2024-03-19T01:32:09.837 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:09 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:09.837 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:09 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:11.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:10 smithi099 ceph-mon[62951]: from='client.14162 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:32:11.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:10 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:11.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:10 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:11.223 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$JkVAoBjhCSNTVA9K3qCkSOmdnpazttAL7IciA5YWfHTlpifLgqUqe", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1710811930, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-03-19T01:32:11.223 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:10.065+0000 7fa2c2721700 1 Processor -- start 2024-03-19T01:32:11.223 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:10.065+0000 7fa2c2721700 1 -- start start 2024-03-19T01:32:11.223 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:10.065+0000 7fa2c2721700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa2bc106050 0x7fa2bc106470 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:32:11.223 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:10.065+0000 7fa2c2721700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fa2bc106a40 con 0x7fa2bc106050 2024-03-19T01:32:11.223 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:10.065+0000 7fa2bbfff700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa2bc106050 0x7fa2bc106470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:32:11.224 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:10.065+0000 7fa2bbfff700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa2bc106050 0x7fa2bc106470 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:51242/0 (socket says 172.21.15.99:51242) 2024-03-19T01:32:11.224 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:10.065+0000 7fa2bbfff700 1 -- 172.21.15.99:0/2183928735 learned_addr learned my addr 172.21.15.99:0/2183928735 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:32:11.224 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:10.066+0000 7fa2bbfff700 1 -- 172.21.15.99:0/2183928735 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa2bc108fe0 con 0x7fa2bc106050 2024-03-19T01:32:11.224 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:10.066+0000 7fa2bbfff700 1 --2- 172.21.15.99:0/2183928735 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa2bc106050 0x7fa2bc106470 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7fa2b00098a0 tx=0x7fa2b0009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8561cb039929cddf server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:32:11.224 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:10.066+0000 7fa2baffd700 1 -- 172.21.15.99:0/2183928735 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fa2b0004160 con 0x7fa2bc106050 2024-03-19T01:32:11.224 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:10.066+0000 7fa2baffd700 1 -- 172.21.15.99:0/2183928735 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa2b0004300 con 0x7fa2bc106050 2024-03-19T01:32:11.224 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:10.066+0000 7fa2baffd700 1 -- 172.21.15.99:0/2183928735 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fa2b0011b20 con 0x7fa2bc106050 2024-03-19T01:32:11.224 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:10.067+0000 7fa2c2721700 1 -- 172.21.15.99:0/2183928735 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa2bc106050 msgr2=0x7fa2bc106470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:32:11.224 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:10.067+0000 7fa2c2721700 1 --2- 172.21.15.99:0/2183928735 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa2bc106050 0x7fa2bc106470 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7fa2b00098a0 tx=0x7fa2b0009bb0 comp rx=0 tx=0).stop 2024-03-19T01:32:11.224 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:10.067+0000 7fa2c2721700 1 -- 172.21.15.99:0/2183928735 shutdown_connections 2024-03-19T01:32:11.224 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:10.067+0000 7fa2c2721700 1 --2- 172.21.15.99:0/2183928735 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa2bc106050 0x7fa2bc106470 unknown :-1 s=CLOSED pgs=70 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:32:11.224 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:10.067+0000 7fa2c2721700 1 -- 172.21.15.99:0/2183928735 >> 172.21.15.99:0/2183928735 conn(0x7fa2bc1019c0 msgr2=0x7fa2bc103de0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:32:11.225 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:10.067+0000 7fa2c2721700 1 -- 172.21.15.99:0/2183928735 shutdown_connections 2024-03-19T01:32:11.225 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:10.067+0000 7fa2c2721700 1 -- 172.21.15.99:0/2183928735 wait complete. 2024-03-19T01:32:11.225 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:10.067+0000 7fa2c2721700 1 Processor -- start 2024-03-19T01:32:11.225 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:10.067+0000 7fa2c2721700 1 -- start start 2024-03-19T01:32:11.225 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:10.068+0000 7fa2c2721700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa2bc106050 0x7fa2bc108810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:32:11.225 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:10.068+0000 7fa2c2721700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fa2bc10c390 con 0x7fa2bc106050 2024-03-19T01:32:11.225 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:10.068+0000 7fa2bbfff700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa2bc106050 0x7fa2bc108810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:32:11.225 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:10.068+0000 7fa2bbfff700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa2bc106050 0x7fa2bc108810 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:51256/0 (socket says 172.21.15.99:51256) 2024-03-19T01:32:11.225 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:10.068+0000 7fa2bbfff700 1 -- 172.21.15.99:0/2360842512 learned_addr learned my addr 172.21.15.99:0/2360842512 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:32:11.225 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:10.068+0000 7fa2bbfff700 1 -- 172.21.15.99:0/2360842512 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa2bc108d50 con 0x7fa2bc106050 2024-03-19T01:32:11.225 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:10.068+0000 7fa2bbfff700 1 --2- 172.21.15.99:0/2360842512 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa2bc106050 0x7fa2bc108810 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7fa2b00116a0 tx=0x7fa2b0011ba0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:32:11.225 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:10.068+0000 7fa2b97fa700 1 -- 172.21.15.99:0/2360842512 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fa2b0003a40 con 0x7fa2bc106050 2024-03-19T01:32:11.225 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:10.068+0000 7fa2b97fa700 1 -- 172.21.15.99:0/2360842512 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa2b0003be0 con 0x7fa2bc106050 2024-03-19T01:32:11.226 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:10.068+0000 7fa2c2721700 1 -- 172.21.15.99:0/2360842512 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa2bc106f90 con 0x7fa2bc106050 2024-03-19T01:32:11.226 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:10.068+0000 7fa2b97fa700 1 -- 172.21.15.99:0/2360842512 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fa2b001a720 con 0x7fa2bc106050 2024-03-19T01:32:11.226 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:10.068+0000 7fa2c2721700 1 -- 172.21.15.99:0/2360842512 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fa2bc1073f0 con 0x7fa2bc106050 2024-03-19T01:32:11.226 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:10.069+0000 7fa2b97fa700 1 -- 172.21.15.99:0/2360842512 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 12) ==== 46119+0+0 (secure 0 0 0) 0x7fa2b001aa60 con 0x7fa2bc106050 2024-03-19T01:32:11.226 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:10.069+0000 7fa2b97fa700 1 --2- 172.21.15.99:0/2360842512 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa2a4038f20 0x7fa2a403b3e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:32:11.226 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:10.069+0000 7fa2b97fa700 1 -- 172.21.15.99:0/2360842512 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(3..3 src has 1..3) ==== 1155+0+0 (secure 0 0 0) 0x7fa2b004d5e0 con 0x7fa2bc106050 2024-03-19T01:32:11.226 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:10.070+0000 7fa2bb7fe700 1 --2- 172.21.15.99:0/2360842512 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa2a4038f20 0x7fa2a403b3e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:32:11.226 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:10.070+0000 7fa2c2721700 1 -- 172.21.15.99:0/2360842512 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fa2bc069420 con 0x7fa2bc106050 2024-03-19T01:32:11.226 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:10.070+0000 7fa2bb7fe700 1 --2- 172.21.15.99:0/2360842512 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa2a4038f20 0x7fa2a403b3e0 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7fa2ac006fd0 tx=0x7fa2ac006e40 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:32:11.226 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:10.079+0000 7fa2b97fa700 1 -- 172.21.15.99:0/2360842512 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7fa2b0011e20 con 0x7fa2bc106050 2024-03-19T01:32:11.226 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:10.313+0000 7fa2c2721700 1 -- 172.21.15.99:0/2360842512 --> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] -- mgr_command(tid 0: {"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}) -- 0x7fa2bc107ac0 con 0x7fa2a4038f20 2024-03-19T01:32:11.227 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:10.540+0000 7fa2b97fa700 1 -- 172.21.15.99:0/2360842512 <== mgr.14150 v2:172.21.15.99:6800/4272410295 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+252 (secure 0 0 0) 0x7fa2bc107ac0 con 0x7fa2a4038f20 2024-03-19T01:32:11.227 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:10.543+0000 7fa2c2721700 1 -- 172.21.15.99:0/2360842512 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa2a4038f20 msgr2=0x7fa2a403b3e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:32:11.227 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:10.543+0000 7fa2c2721700 1 --2- 172.21.15.99:0/2360842512 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa2a4038f20 0x7fa2a403b3e0 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7fa2ac006fd0 tx=0x7fa2ac006e40 comp rx=0 tx=0).stop 2024-03-19T01:32:11.227 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:10.543+0000 7fa2c2721700 1 -- 172.21.15.99:0/2360842512 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa2bc106050 msgr2=0x7fa2bc108810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:32:11.227 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:10.543+0000 7fa2c2721700 1 --2- 172.21.15.99:0/2360842512 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa2bc106050 0x7fa2bc108810 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7fa2b00116a0 tx=0x7fa2b0011ba0 comp rx=0 tx=0).stop 2024-03-19T01:32:11.227 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:10.543+0000 7fa2c2721700 1 -- 172.21.15.99:0/2360842512 shutdown_connections 2024-03-19T01:32:11.227 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:10.543+0000 7fa2c2721700 1 --2- 172.21.15.99:0/2360842512 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa2a4038f20 0x7fa2a403b3e0 unknown :-1 s=CLOSED pgs=8 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:32:11.227 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:10.543+0000 7fa2c2721700 1 --2- 172.21.15.99:0/2360842512 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa2bc106050 0x7fa2bc108810 unknown :-1 s=CLOSED pgs=71 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:32:11.227 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:10.543+0000 7fa2c2721700 1 -- 172.21.15.99:0/2360842512 >> 172.21.15.99:0/2360842512 conn(0x7fa2bc1019c0 msgr2=0x7fa2bc1024c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:32:11.227 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:10.543+0000 7fa2c2721700 1 -- 172.21.15.99:0/2360842512 shutdown_connections 2024-03-19T01:32:11.227 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:10.543+0000 7fa2c2721700 1 -- 172.21.15.99:0/2360842512 wait complete. 2024-03-19T01:32:11.228 INFO:teuthology.orchestra.run.smithi099.stdout:Fetching dashboard port number... 2024-03-19T01:32:12.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:11 smithi099 ceph-mon[62951]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:32:12.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:11 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:12.456 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout 8443 2024-03-19T01:32:12.456 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:11.565+0000 7f7cbc833700 1 Processor -- start 2024-03-19T01:32:12.456 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:11.565+0000 7f7cbc833700 1 -- start start 2024-03-19T01:32:12.456 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:11.566+0000 7f7cbc833700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7cb8108890 0x7f7cb810ac80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:32:12.456 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:11.566+0000 7f7cbc833700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f7cb807be70 con 0x7f7cb8108890 2024-03-19T01:32:12.456 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:11.566+0000 7f7cb659c700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7cb8108890 0x7f7cb810ac80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:32:12.456 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:11.566+0000 7f7cb659c700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7cb8108890 0x7f7cb810ac80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:51258/0 (socket says 172.21.15.99:51258) 2024-03-19T01:32:12.456 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:11.566+0000 7f7cb659c700 1 -- 172.21.15.99:0/3977686212 learned_addr learned my addr 172.21.15.99:0/3977686212 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:32:12.457 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:11.566+0000 7f7cb659c700 1 -- 172.21.15.99:0/3977686212 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7cb810b1c0 con 0x7f7cb8108890 2024-03-19T01:32:12.457 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:11.567+0000 7f7cb659c700 1 --2- 172.21.15.99:0/3977686212 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7cb8108890 0x7f7cb810ac80 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7f7cac0098a0 tx=0x7f7cac009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=536368a47949c92f server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:32:12.457 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:11.567+0000 7f7cb559a700 1 -- 172.21.15.99:0/3977686212 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f7cac004160 con 0x7f7cb8108890 2024-03-19T01:32:12.457 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:11.567+0000 7f7cb559a700 1 -- 172.21.15.99:0/3977686212 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7cac004300 con 0x7f7cb8108890 2024-03-19T01:32:12.457 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:11.567+0000 7f7cb559a700 1 -- 172.21.15.99:0/3977686212 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f7cac011b20 con 0x7f7cb8108890 2024-03-19T01:32:12.457 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:11.567+0000 7f7cbc833700 1 -- 172.21.15.99:0/3977686212 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7cb8108890 msgr2=0x7f7cb810ac80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:32:12.457 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:11.567+0000 7f7cbc833700 1 --2- 172.21.15.99:0/3977686212 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7cb8108890 0x7f7cb810ac80 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7f7cac0098a0 tx=0x7f7cac009bb0 comp rx=0 tx=0).stop 2024-03-19T01:32:12.457 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:11.567+0000 7f7cbc833700 1 -- 172.21.15.99:0/3977686212 shutdown_connections 2024-03-19T01:32:12.457 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:11.567+0000 7f7cbc833700 1 --2- 172.21.15.99:0/3977686212 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7cb8108890 0x7f7cb810ac80 unknown :-1 s=CLOSED pgs=72 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:32:12.457 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:11.567+0000 7f7cbc833700 1 -- 172.21.15.99:0/3977686212 >> 172.21.15.99:0/3977686212 conn(0x7f7cb8102260 msgr2=0x7f7cb81046a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:32:12.457 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:11.567+0000 7f7cbc833700 1 -- 172.21.15.99:0/3977686212 shutdown_connections 2024-03-19T01:32:12.457 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:11.567+0000 7f7cbc833700 1 -- 172.21.15.99:0/3977686212 wait complete. 2024-03-19T01:32:12.457 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:11.568+0000 7f7cbc833700 1 Processor -- start 2024-03-19T01:32:12.457 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:11.568+0000 7f7cbc833700 1 -- start start 2024-03-19T01:32:12.457 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:11.568+0000 7f7cbc833700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7cb8108890 0x7f7cb8108110 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:32:12.458 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:11.568+0000 7f7cbc833700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f7cb810c1d0 con 0x7f7cb8108890 2024-03-19T01:32:12.458 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:11.568+0000 7f7cb659c700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7cb8108890 0x7f7cb8108110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:32:12.458 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:11.568+0000 7f7cb659c700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7cb8108890 0x7f7cb8108110 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:51272/0 (socket says 172.21.15.99:51272) 2024-03-19T01:32:12.458 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:11.569+0000 7f7cb659c700 1 -- 172.21.15.99:0/1286936405 learned_addr learned my addr 172.21.15.99:0/1286936405 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:32:12.458 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:11.569+0000 7f7cb659c700 1 -- 172.21.15.99:0/1286936405 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7cb8108650 con 0x7f7cb8108890 2024-03-19T01:32:12.458 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:11.569+0000 7f7cb659c700 1 --2- 172.21.15.99:0/1286936405 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7cb8108890 0x7f7cb8108110 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7f7cac0117e0 tx=0x7f7cac011c10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:32:12.458 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:11.569+0000 7f7ca77fe700 1 -- 172.21.15.99:0/1286936405 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f7cac003bf0 con 0x7f7cb8108890 2024-03-19T01:32:12.458 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:11.569+0000 7f7ca77fe700 1 -- 172.21.15.99:0/1286936405 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7cac01a910 con 0x7f7cb8108890 2024-03-19T01:32:12.458 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:11.569+0000 7f7cbc833700 1 -- 172.21.15.99:0/1286936405 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f7cb8106840 con 0x7f7cb8108890 2024-03-19T01:32:12.458 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:11.569+0000 7f7ca77fe700 1 -- 172.21.15.99:0/1286936405 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f7cac024970 con 0x7f7cb8108890 2024-03-19T01:32:12.458 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:11.569+0000 7f7cbc833700 1 -- 172.21.15.99:0/1286936405 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f7cb8106d00 con 0x7f7cb8108890 2024-03-19T01:32:12.458 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:11.570+0000 7f7ca77fe700 1 -- 172.21.15.99:0/1286936405 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 12) ==== 46119+0+0 (secure 0 0 0) 0x7f7cac022020 con 0x7f7cb8108890 2024-03-19T01:32:12.459 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:11.570+0000 7f7cbc833700 1 -- 172.21.15.99:0/1286936405 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f7cb8063940 con 0x7f7cb8108890 2024-03-19T01:32:12.459 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:11.570+0000 7f7ca77fe700 1 --2- 172.21.15.99:0/1286936405 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f7ca0038f20 0x7f7ca003b3e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:32:12.459 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:11.570+0000 7f7ca77fe700 1 -- 172.21.15.99:0/1286936405 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(3..3 src has 1..3) ==== 1155+0+0 (secure 0 0 0) 0x7f7cac04d2c0 con 0x7f7cb8108890 2024-03-19T01:32:12.459 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:11.570+0000 7f7cb5d9b700 1 --2- 172.21.15.99:0/1286936405 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f7ca0038f20 0x7f7ca003b3e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:32:12.459 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:11.571+0000 7f7cb5d9b700 1 --2- 172.21.15.99:0/1286936405 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f7ca0038f20 0x7f7ca003b3e0 secure :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0x7f7ca8006fd0 tx=0x7f7ca8006e40 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:32:12.459 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:11.575+0000 7f7ca77fe700 1 -- 172.21.15.99:0/1286936405 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f7cac011e20 con 0x7f7cb8108890 2024-03-19T01:32:12.459 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:11.785+0000 7f7cbc833700 1 -- 172.21.15.99:0/1286936405 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"} v 0) -- 0x7f7cb8107040 con 0x7f7cb8108890 2024-03-19T01:32:12.459 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:11.786+0000 7f7ca77fe700 1 -- 172.21.15.99:0/1286936405 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]=0 v8) ==== 112+0+5 (secure 0 0 0) 0x7f7cb8107040 con 0x7f7cb8108890 2024-03-19T01:32:12.459 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:11.788+0000 7f7cbc833700 1 -- 172.21.15.99:0/1286936405 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f7ca0038f20 msgr2=0x7f7ca003b3e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:32:12.459 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:11.788+0000 7f7cbc833700 1 --2- 172.21.15.99:0/1286936405 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f7ca0038f20 0x7f7ca003b3e0 secure :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0x7f7ca8006fd0 tx=0x7f7ca8006e40 comp rx=0 tx=0).stop 2024-03-19T01:32:12.459 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:11.788+0000 7f7cbc833700 1 -- 172.21.15.99:0/1286936405 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7cb8108890 msgr2=0x7f7cb8108110 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:32:12.459 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:11.788+0000 7f7cbc833700 1 --2- 172.21.15.99:0/1286936405 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7cb8108890 0x7f7cb8108110 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7f7cac0117e0 tx=0x7f7cac011c10 comp rx=0 tx=0).stop 2024-03-19T01:32:12.459 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:11.788+0000 7f7cbc833700 1 -- 172.21.15.99:0/1286936405 shutdown_connections 2024-03-19T01:32:12.459 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:11.788+0000 7f7cbc833700 1 --2- 172.21.15.99:0/1286936405 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f7ca0038f20 0x7f7ca003b3e0 unknown :-1 s=CLOSED pgs=9 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:32:12.460 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:11.788+0000 7f7cbc833700 1 --2- 172.21.15.99:0/1286936405 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7cb8108890 0x7f7cb8108110 unknown :-1 s=CLOSED pgs=73 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:32:12.460 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:11.788+0000 7f7cbc833700 1 -- 172.21.15.99:0/1286936405 >> 172.21.15.99:0/1286936405 conn(0x7f7cb8102260 msgr2=0x7f7cb8102d20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:32:12.460 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:11.788+0000 7f7cbc833700 1 -- 172.21.15.99:0/1286936405 shutdown_connections 2024-03-19T01:32:12.460 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:11.788+0000 7f7cbc833700 1 -- 172.21.15.99:0/1286936405 wait complete. 2024-03-19T01:32:12.463 INFO:teuthology.orchestra.run.smithi099.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-03-19T01:32:12.463 INFO:teuthology.orchestra.run.smithi099.stdout:systemctl: stdout disabled 2024-03-19T01:32:12.470 INFO:teuthology.orchestra.run.smithi099.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-03-19T01:32:12.470 INFO:teuthology.orchestra.run.smithi099.stdout:systemctl: stdout inactive 2024-03-19T01:32:12.471 INFO:teuthology.orchestra.run.smithi099.stdout:firewalld.service is not enabled 2024-03-19T01:32:12.471 INFO:teuthology.orchestra.run.smithi099.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2024-03-19T01:32:12.472 INFO:teuthology.orchestra.run.smithi099.stdout:Ceph Dashboard is now available at: 2024-03-19T01:32:12.472 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:32:12.472 INFO:teuthology.orchestra.run.smithi099.stdout: URL: https://smithi099.front.sepia.ceph.com:8443/ 2024-03-19T01:32:12.472 INFO:teuthology.orchestra.run.smithi099.stdout: User: admin 2024-03-19T01:32:12.472 INFO:teuthology.orchestra.run.smithi099.stdout: Password: phod09a7k7 2024-03-19T01:32:12.472 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:32:12.473 INFO:teuthology.orchestra.run.smithi099.stdout:Saving cluster configuration to /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/config directory 2024-03-19T01:32:12.473 INFO:teuthology.orchestra.run.smithi099.stdout:Enabling autotune for osd_memory_target 2024-03-19T01:32:13.011 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:12 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/1286936405' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-03-19T01:32:13.832 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:13 smithi099 ceph-mon[62951]: mgrmap e13: x(active, since 6s) 2024-03-19T01:32:14.142 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:12.870+0000 7fbc5da5c700 1 Processor -- start 2024-03-19T01:32:14.143 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:12.870+0000 7fbc5da5c700 1 -- start start 2024-03-19T01:32:14.143 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:12.870+0000 7fbc5da5c700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbc58107dd0 0x7fbc581081f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:32:14.143 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:12.870+0000 7fbc5da5c700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fbc581087c0 con 0x7fbc58107dd0 2024-03-19T01:32:14.143 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:12.870+0000 7fbc56ffd700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbc58107dd0 0x7fbc581081f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:32:14.143 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:12.870+0000 7fbc56ffd700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbc58107dd0 0x7fbc581081f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:49000/0 (socket says 172.21.15.99:49000) 2024-03-19T01:32:14.143 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:12.870+0000 7fbc56ffd700 1 -- 172.21.15.99:0/1338130032 learned_addr learned my addr 172.21.15.99:0/1338130032 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:32:14.143 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:12.871+0000 7fbc56ffd700 1 -- 172.21.15.99:0/1338130032 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fbc58108fd0 con 0x7fbc58107dd0 2024-03-19T01:32:14.143 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:12.871+0000 7fbc56ffd700 1 --2- 172.21.15.99:0/1338130032 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbc58107dd0 0x7fbc581081f0 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x7fbc480098a0 tx=0x7fbc48009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a12f9ef5761e0458 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:32:14.143 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:12.871+0000 7fbc55ffb700 1 -- 172.21.15.99:0/1338130032 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fbc48004160 con 0x7fbc58107dd0 2024-03-19T01:32:14.143 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:12.871+0000 7fbc55ffb700 1 -- 172.21.15.99:0/1338130032 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fbc48004300 con 0x7fbc58107dd0 2024-03-19T01:32:14.143 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:12.871+0000 7fbc55ffb700 1 -- 172.21.15.99:0/1338130032 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fbc48011b20 con 0x7fbc58107dd0 2024-03-19T01:32:14.143 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:12.871+0000 7fbc5da5c700 1 -- 172.21.15.99:0/1338130032 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbc58107dd0 msgr2=0x7fbc581081f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:32:14.144 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:12.871+0000 7fbc5da5c700 1 --2- 172.21.15.99:0/1338130032 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbc58107dd0 0x7fbc581081f0 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x7fbc480098a0 tx=0x7fbc48009bb0 comp rx=0 tx=0).stop 2024-03-19T01:32:14.144 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:12.872+0000 7fbc5da5c700 1 -- 172.21.15.99:0/1338130032 shutdown_connections 2024-03-19T01:32:14.144 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:12.872+0000 7fbc5da5c700 1 --2- 172.21.15.99:0/1338130032 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbc58107dd0 0x7fbc581081f0 unknown :-1 s=CLOSED pgs=74 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:32:14.144 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:12.872+0000 7fbc5da5c700 1 -- 172.21.15.99:0/1338130032 >> 172.21.15.99:0/1338130032 conn(0x7fbc58103390 msgr2=0x7fbc581057b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:32:14.144 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:12.872+0000 7fbc5da5c700 1 -- 172.21.15.99:0/1338130032 shutdown_connections 2024-03-19T01:32:14.144 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:12.872+0000 7fbc5da5c700 1 -- 172.21.15.99:0/1338130032 wait complete. 2024-03-19T01:32:14.144 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:12.872+0000 7fbc5da5c700 1 Processor -- start 2024-03-19T01:32:14.144 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:12.872+0000 7fbc5da5c700 1 -- start start 2024-03-19T01:32:14.144 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:12.873+0000 7fbc5da5c700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbc58107dd0 0x7fbc5819bda0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:32:14.144 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:12.873+0000 7fbc5da5c700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fbc5810c380 con 0x7fbc58107dd0 2024-03-19T01:32:14.144 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:12.873+0000 7fbc56ffd700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbc58107dd0 0x7fbc5819bda0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:32:14.144 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:12.873+0000 7fbc56ffd700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbc58107dd0 0x7fbc5819bda0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:49012/0 (socket says 172.21.15.99:49012) 2024-03-19T01:32:14.144 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:12.873+0000 7fbc56ffd700 1 -- 172.21.15.99:0/1011906599 learned_addr learned my addr 172.21.15.99:0/1011906599 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:32:14.144 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:12.873+0000 7fbc56ffd700 1 -- 172.21.15.99:0/1011906599 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fbc5819c2e0 con 0x7fbc58107dd0 2024-03-19T01:32:14.144 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:12.873+0000 7fbc56ffd700 1 --2- 172.21.15.99:0/1011906599 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbc58107dd0 0x7fbc5819bda0 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x7fbc480116a0 tx=0x7fbc48011ba0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:32:14.145 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:12.873+0000 7fbc5ca5a700 1 -- 172.21.15.99:0/1011906599 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fbc48003a40 con 0x7fbc58107dd0 2024-03-19T01:32:14.145 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:12.873+0000 7fbc5ca5a700 1 -- 172.21.15.99:0/1011906599 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fbc48003be0 con 0x7fbc58107dd0 2024-03-19T01:32:14.145 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:12.873+0000 7fbc5ca5a700 1 -- 172.21.15.99:0/1011906599 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fbc4801a720 con 0x7fbc58107dd0 2024-03-19T01:32:14.145 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:12.873+0000 7fbc5da5c700 1 -- 172.21.15.99:0/1011906599 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fbc5819c570 con 0x7fbc58107dd0 2024-03-19T01:32:14.145 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:12.873+0000 7fbc5da5c700 1 -- 172.21.15.99:0/1011906599 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fbc5819c9d0 con 0x7fbc58107dd0 2024-03-19T01:32:14.145 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:12.874+0000 7fbc5ca5a700 1 -- 172.21.15.99:0/1011906599 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 13) ==== 46162+0+0 (secure 0 0 0) 0x7fbc4801aa20 con 0x7fbc58107dd0 2024-03-19T01:32:14.145 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:12.874+0000 7fbc5da5c700 1 -- 172.21.15.99:0/1011906599 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fbc58063940 con 0x7fbc58107dd0 2024-03-19T01:32:14.145 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:12.874+0000 7fbc5ca5a700 1 --2- 172.21.15.99:0/1011906599 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fbc40038fc0 0x7fbc4003b480 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:32:14.145 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:12.874+0000 7fbc5ca5a700 1 -- 172.21.15.99:0/1011906599 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(3..3 src has 1..3) ==== 1155+0+0 (secure 0 0 0) 0x7fbc4804d640 con 0x7fbc58107dd0 2024-03-19T01:32:14.145 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:12.874+0000 7fbc567fc700 1 --2- 172.21.15.99:0/1011906599 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fbc40038fc0 0x7fbc4003b480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:32:14.145 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:12.875+0000 7fbc567fc700 1 --2- 172.21.15.99:0/1011906599 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fbc40038fc0 0x7fbc4003b480 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0x7fbc4c006fd0 tx=0x7fbc4c006e40 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:32:14.145 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:12.879+0000 7fbc5ca5a700 1 -- 172.21.15.99:0/1011906599 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7fbc48011e20 con 0x7fbc58107dd0 2024-03-19T01:32:14.145 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:13.078+0000 7fbc5da5c700 1 -- 172.21.15.99:0/1011906599 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command([{prefix=config set, name=osd_memory_target_autotune}] v 0) -- 0x7fbc5819f3a0 con 0x7fbc58107dd0 2024-03-19T01:32:14.145 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:13.078+0000 7fbc5ca5a700 1 -- 172.21.15.99:0/1011906599 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{prefix=config set, name=osd_memory_target_autotune}]=0 v8)=0 v8) ==== 127+0+0 (secure 0 0 0) 0x7fbc48024b50 con 0x7fbc58107dd0 2024-03-19T01:32:14.145 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:13.081+0000 7fbc5da5c700 1 -- 172.21.15.99:0/1011906599 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fbc40038fc0 msgr2=0x7fbc4003b480 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:32:14.146 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:13.081+0000 7fbc5da5c700 1 --2- 172.21.15.99:0/1011906599 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fbc40038fc0 0x7fbc4003b480 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0x7fbc4c006fd0 tx=0x7fbc4c006e40 comp rx=0 tx=0).stop 2024-03-19T01:32:14.146 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:13.081+0000 7fbc5da5c700 1 -- 172.21.15.99:0/1011906599 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbc58107dd0 msgr2=0x7fbc5819bda0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:32:14.146 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:13.081+0000 7fbc5da5c700 1 --2- 172.21.15.99:0/1011906599 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbc58107dd0 0x7fbc5819bda0 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x7fbc480116a0 tx=0x7fbc48011ba0 comp rx=0 tx=0).stop 2024-03-19T01:32:14.146 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:13.081+0000 7fbc5da5c700 1 -- 172.21.15.99:0/1011906599 shutdown_connections 2024-03-19T01:32:14.146 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:13.081+0000 7fbc5da5c700 1 --2- 172.21.15.99:0/1011906599 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fbc40038fc0 0x7fbc4003b480 unknown :-1 s=CLOSED pgs=10 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:32:14.146 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:13.081+0000 7fbc5da5c700 1 --2- 172.21.15.99:0/1011906599 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbc58107dd0 0x7fbc5819bda0 unknown :-1 s=CLOSED pgs=75 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:32:14.146 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:13.081+0000 7fbc5da5c700 1 -- 172.21.15.99:0/1011906599 >> 172.21.15.99:0/1011906599 conn(0x7fbc58103390 msgr2=0x7fbc58103ee0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:32:14.146 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:13.081+0000 7fbc5da5c700 1 -- 172.21.15.99:0/1011906599 shutdown_connections 2024-03-19T01:32:14.146 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:13.081+0000 7fbc5da5c700 1 -- 172.21.15.99:0/1011906599 wait complete. 2024-03-19T01:32:15.375 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:14.547+0000 7fdcb21e7700 1 Processor -- start 2024-03-19T01:32:15.375 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:14.547+0000 7fdcb21e7700 1 -- start start 2024-03-19T01:32:15.375 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:14.547+0000 7fdcb21e7700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fdcac108910 0x7fdcac10ad00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:32:15.375 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:14.547+0000 7fdcb21e7700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fdcac07bdf0 con 0x7fdcac108910 2024-03-19T01:32:15.375 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:14.547+0000 7fdcab7fe700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fdcac108910 0x7fdcac10ad00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:32:15.376 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:14.547+0000 7fdcab7fe700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fdcac108910 0x7fdcac10ad00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:49018/0 (socket says 172.21.15.99:49018) 2024-03-19T01:32:15.376 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:14.547+0000 7fdcab7fe700 1 -- 172.21.15.99:0/4211279191 learned_addr learned my addr 172.21.15.99:0/4211279191 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:32:15.376 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:14.547+0000 7fdcab7fe700 1 -- 172.21.15.99:0/4211279191 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fdcac10b240 con 0x7fdcac108910 2024-03-19T01:32:15.376 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:14.547+0000 7fdcab7fe700 1 --2- 172.21.15.99:0/4211279191 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fdcac108910 0x7fdcac10ad00 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7fdc9c0098a0 tx=0x7fdc9c009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=64cbfbb084c3c228 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:32:15.376 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:14.547+0000 7fdcaaffd700 1 -- 172.21.15.99:0/4211279191 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fdc9c004160 con 0x7fdcac108910 2024-03-19T01:32:15.376 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:14.548+0000 7fdcaaffd700 1 -- 172.21.15.99:0/4211279191 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fdc9c004300 con 0x7fdcac108910 2024-03-19T01:32:15.376 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:14.548+0000 7fdcaaffd700 1 -- 172.21.15.99:0/4211279191 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fdc9c011b20 con 0x7fdcac108910 2024-03-19T01:32:15.376 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:14.548+0000 7fdcb21e7700 1 -- 172.21.15.99:0/4211279191 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fdcac108910 msgr2=0x7fdcac10ad00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:32:15.376 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:14.548+0000 7fdcb21e7700 1 --2- 172.21.15.99:0/4211279191 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fdcac108910 0x7fdcac10ad00 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7fdc9c0098a0 tx=0x7fdc9c009bb0 comp rx=0 tx=0).stop 2024-03-19T01:32:15.376 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:14.548+0000 7fdcb21e7700 1 -- 172.21.15.99:0/4211279191 shutdown_connections 2024-03-19T01:32:15.376 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:14.548+0000 7fdcb21e7700 1 --2- 172.21.15.99:0/4211279191 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fdcac108910 0x7fdcac10ad00 unknown :-1 s=CLOSED pgs=76 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:32:15.376 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:14.548+0000 7fdcb21e7700 1 -- 172.21.15.99:0/4211279191 >> 172.21.15.99:0/4211279191 conn(0x7fdcac102260 msgr2=0x7fdcac1046a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:32:15.376 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:14.548+0000 7fdcb21e7700 1 -- 172.21.15.99:0/4211279191 shutdown_connections 2024-03-19T01:32:15.377 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:14.548+0000 7fdcb21e7700 1 -- 172.21.15.99:0/4211279191 wait complete. 2024-03-19T01:32:15.377 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:14.549+0000 7fdcb21e7700 1 Processor -- start 2024-03-19T01:32:15.377 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:14.549+0000 7fdcb21e7700 1 -- start start 2024-03-19T01:32:15.377 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:14.549+0000 7fdcb21e7700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fdcac108910 0x7fdcac19bd40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:32:15.377 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:14.549+0000 7fdcb21e7700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fdcac10c1e0 con 0x7fdcac108910 2024-03-19T01:32:15.377 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:14.549+0000 7fdcab7fe700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fdcac108910 0x7fdcac19bd40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:32:15.377 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:14.549+0000 7fdcab7fe700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fdcac108910 0x7fdcac19bd40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:49020/0 (socket says 172.21.15.99:49020) 2024-03-19T01:32:15.377 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:14.549+0000 7fdcab7fe700 1 -- 172.21.15.99:0/3654969734 learned_addr learned my addr 172.21.15.99:0/3654969734 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:32:15.377 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:14.549+0000 7fdcab7fe700 1 -- 172.21.15.99:0/3654969734 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fdcac19c280 con 0x7fdcac108910 2024-03-19T01:32:15.377 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:14.549+0000 7fdcab7fe700 1 --2- 172.21.15.99:0/3654969734 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fdcac108910 0x7fdcac19bd40 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 crypto rx=0x7fdc9c0117e0 tx=0x7fdc9c011ba0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:32:15.377 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:14.550+0000 7fdca97fa700 1 -- 172.21.15.99:0/3654969734 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fdc9c003ba0 con 0x7fdcac108910 2024-03-19T01:32:15.377 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:14.550+0000 7fdca97fa700 1 -- 172.21.15.99:0/3654969734 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fdc9c01a910 con 0x7fdcac108910 2024-03-19T01:32:15.377 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:14.550+0000 7fdcb21e7700 1 -- 172.21.15.99:0/3654969734 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fdcac19c510 con 0x7fdcac108910 2024-03-19T01:32:15.377 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:14.550+0000 7fdca97fa700 1 -- 172.21.15.99:0/3654969734 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fdc9c0248f0 con 0x7fdcac108910 2024-03-19T01:32:15.378 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:14.550+0000 7fdcb21e7700 1 -- 172.21.15.99:0/3654969734 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fdcac19c970 con 0x7fdcac108910 2024-03-19T01:32:15.378 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:14.551+0000 7fdca97fa700 1 -- 172.21.15.99:0/3654969734 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 13) ==== 46162+0+0 (secure 0 0 0) 0x7fdc9c022020 con 0x7fdcac108910 2024-03-19T01:32:15.378 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:14.551+0000 7fdcb21e7700 1 -- 172.21.15.99:0/3654969734 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fdcac063940 con 0x7fdcac108910 2024-03-19T01:32:15.378 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:14.551+0000 7fdca97fa700 1 --2- 172.21.15.99:0/3654969734 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fdc98039070 0x7fdc9803b530 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:32:15.378 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:14.551+0000 7fdca97fa700 1 -- 172.21.15.99:0/3654969734 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(3..3 src has 1..3) ==== 1155+0+0 (secure 0 0 0) 0x7fdc9c04d5c0 con 0x7fdcac108910 2024-03-19T01:32:15.378 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:14.551+0000 7fdca3fff700 1 --2- 172.21.15.99:0/3654969734 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fdc98039070 0x7fdc9803b530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:32:15.378 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:14.551+0000 7fdca3fff700 1 --2- 172.21.15.99:0/3654969734 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fdc98039070 0x7fdc9803b530 secure :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0x7fdc94006fd0 tx=0x7fdc94006e40 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:32:15.378 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:14.556+0000 7fdca97fa700 1 -- 172.21.15.99:0/3654969734 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7fdc9c01a430 con 0x7fdcac108910 2024-03-19T01:32:15.378 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:14.862+0000 7fdcb21e7700 1 -- 172.21.15.99:0/3654969734 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command([{prefix=config-key set, key=mgr/dashboard/cluster/status}] v 0) -- 0x7fdcac19f3e0 con 0x7fdcac108910 2024-03-19T01:32:15.378 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:14.865+0000 7fdca97fa700 1 -- 172.21.15.99:0/3654969734 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{prefix=config-key set, key=mgr/dashboard/cluster/status}]=0 set mgr/dashboard/cluster/status v28)=0 set mgr/dashboard/cluster/status v28) ==== 153+0+0 (secure 0 0 0) 0x7fdc9c01a650 con 0x7fdcac108910 2024-03-19T01:32:15.378 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:14.867+0000 7fdcb21e7700 1 -- 172.21.15.99:0/3654969734 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fdc98039070 msgr2=0x7fdc9803b530 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:32:15.378 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:14.867+0000 7fdcb21e7700 1 --2- 172.21.15.99:0/3654969734 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fdc98039070 0x7fdc9803b530 secure :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0x7fdc94006fd0 tx=0x7fdc94006e40 comp rx=0 tx=0).stop 2024-03-19T01:32:15.378 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:14.867+0000 7fdcb21e7700 1 -- 172.21.15.99:0/3654969734 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fdcac108910 msgr2=0x7fdcac19bd40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:32:15.379 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:14.867+0000 7fdcb21e7700 1 --2- 172.21.15.99:0/3654969734 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fdcac108910 0x7fdcac19bd40 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 crypto rx=0x7fdc9c0117e0 tx=0x7fdc9c011ba0 comp rx=0 tx=0).stop 2024-03-19T01:32:15.379 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:14.867+0000 7fdcb21e7700 1 -- 172.21.15.99:0/3654969734 shutdown_connections 2024-03-19T01:32:15.379 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:14.867+0000 7fdcb21e7700 1 --2- 172.21.15.99:0/3654969734 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fdc98039070 0x7fdc9803b530 unknown :-1 s=CLOSED pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:32:15.379 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:14.867+0000 7fdcb21e7700 1 --2- 172.21.15.99:0/3654969734 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fdcac108910 0x7fdcac19bd40 unknown :-1 s=CLOSED pgs=77 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:32:15.379 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:14.867+0000 7fdcb21e7700 1 -- 172.21.15.99:0/3654969734 >> 172.21.15.99:0/3654969734 conn(0x7fdcac102260 msgr2=0x7fdcac102d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:32:15.379 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:14.867+0000 7fdcb21e7700 1 -- 172.21.15.99:0/3654969734 shutdown_connections 2024-03-19T01:32:15.379 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr 2024-03-19T01:32:14.867+0000 7fdcb21e7700 1 -- 172.21.15.99:0/3654969734 wait complete. 2024-03-19T01:32:15.379 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2024-03-19T01:32:15.379 INFO:teuthology.orchestra.run.smithi099.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2024-03-19T01:32:15.379 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:32:15.379 INFO:teuthology.orchestra.run.smithi099.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-03-19T01:32:15.379 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:32:15.379 INFO:teuthology.orchestra.run.smithi099.stdout:Or, if you are only running a single cluster on this host: 2024-03-19T01:32:15.379 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:32:15.379 INFO:teuthology.orchestra.run.smithi099.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2024-03-19T01:32:15.380 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:32:15.380 INFO:teuthology.orchestra.run.smithi099.stdout:Please consider enabling telemetry to help improve Ceph: 2024-03-19T01:32:15.380 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:32:15.380 INFO:teuthology.orchestra.run.smithi099.stdout: ceph telemetry on 2024-03-19T01:32:15.380 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:32:15.380 INFO:teuthology.orchestra.run.smithi099.stdout:For more information see: 2024-03-19T01:32:15.380 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:32:15.380 INFO:teuthology.orchestra.run.smithi099.stdout: https://docs.ceph.com/en/latest/mgr/telemetry/ 2024-03-19T01:32:15.380 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:32:15.380 INFO:teuthology.orchestra.run.smithi099.stdout:Bootstrap complete. 2024-03-19T01:32:15.412 INFO:tasks.cephadm:Fetching config... 2024-03-19T01:32:15.413 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2024-03-19T01:32:15.413 DEBUG:teuthology.orchestra.run.smithi099:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-03-19T01:32:15.427 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-03-19T01:32:15.428 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2024-03-19T01:32:15.428 DEBUG:teuthology.orchestra.run.smithi099:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-03-19T01:32:15.482 INFO:tasks.cephadm:Fetching mon keyring... 2024-03-19T01:32:15.482 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2024-03-19T01:32:15.482 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd if=/var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/keyring of=/dev/stdout 2024-03-19T01:32:15.549 INFO:tasks.cephadm:Fetching pub ssh key... 2024-03-19T01:32:15.549 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2024-03-19T01:32:15.549 DEBUG:teuthology.orchestra.run.smithi099:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-03-19T01:32:15.604 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-03-19T01:32:15.604 DEBUG:teuthology.orchestra.run.smithi099:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCqyGEa2vlwe+9WcbWbCC8Jy3h1Z37hzemIAnr34gVudgT0k4Qy1DYzyVd7ZuGzAqx/B2RXQ0iLaPODqTEgreeURYtg6GBe50j+IZfs1EDwEOak/kjyni5/GCf9Uk2tOV84qyqGruaQbyZMtnd/+ycmBfi/gGOK1DJSNTE14lG6Y0inaNAj2Vy9p4S1hIxR4AtC9uPFdTGx1w04RJkZg6OIM/af0PVlNRRU3QTrMDYozsvYhXWorO/J+ZYXkU0RCMBYFtTdo55LkevfCVKuuDfgKG0gYEWRYFdtGTKUuW8TTpF0hMd9KFjAOs93y8REG2vOxb3NwOcvVaURenuHpl5V/xUj/yf19zafiVi8T+0SsRPHUJzGH0ITzpiDOwXxJji90aBzs19UGMicPEeIYUdmSFhQg2wR1kAcvBYvEDWwvui/SO+ALHX9SZKm5eYxN/TEvNqDaAX12pq49dcfu0BuMDkrk3x+7ZgFC8MGW+opCFjMMBnkdaMNW6M/v1Pc3W0= ceph-438fb69c-e590-11ee-95c9-87774f69a715' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-03-19T01:32:15.682 INFO:teuthology.orchestra.run.smithi099.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCqyGEa2vlwe+9WcbWbCC8Jy3h1Z37hzemIAnr34gVudgT0k4Qy1DYzyVd7ZuGzAqx/B2RXQ0iLaPODqTEgreeURYtg6GBe50j+IZfs1EDwEOak/kjyni5/GCf9Uk2tOV84qyqGruaQbyZMtnd/+ycmBfi/gGOK1DJSNTE14lG6Y0inaNAj2Vy9p4S1hIxR4AtC9uPFdTGx1w04RJkZg6OIM/af0PVlNRRU3QTrMDYozsvYhXWorO/J+ZYXkU0RCMBYFtTdo55LkevfCVKuuDfgKG0gYEWRYFdtGTKUuW8TTpF0hMd9KFjAOs93y8REG2vOxb3NwOcvVaURenuHpl5V/xUj/yf19zafiVi8T+0SsRPHUJzGH0ITzpiDOwXxJji90aBzs19UGMicPEeIYUdmSFhQg2wR1kAcvBYvEDWwvui/SO+ALHX9SZKm5eYxN/TEvNqDaAX12pq49dcfu0BuMDkrk3x+7ZgFC8MGW+opCFjMMBnkdaMNW6M/v1Pc3W0= ceph-438fb69c-e590-11ee-95c9-87774f69a715 2024-03-19T01:32:15.697 DEBUG:teuthology.orchestra.run.smithi112:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCqyGEa2vlwe+9WcbWbCC8Jy3h1Z37hzemIAnr34gVudgT0k4Qy1DYzyVd7ZuGzAqx/B2RXQ0iLaPODqTEgreeURYtg6GBe50j+IZfs1EDwEOak/kjyni5/GCf9Uk2tOV84qyqGruaQbyZMtnd/+ycmBfi/gGOK1DJSNTE14lG6Y0inaNAj2Vy9p4S1hIxR4AtC9uPFdTGx1w04RJkZg6OIM/af0PVlNRRU3QTrMDYozsvYhXWorO/J+ZYXkU0RCMBYFtTdo55LkevfCVKuuDfgKG0gYEWRYFdtGTKUuW8TTpF0hMd9KFjAOs93y8REG2vOxb3NwOcvVaURenuHpl5V/xUj/yf19zafiVi8T+0SsRPHUJzGH0ITzpiDOwXxJji90aBzs19UGMicPEeIYUdmSFhQg2wR1kAcvBYvEDWwvui/SO+ALHX9SZKm5eYxN/TEvNqDaAX12pq49dcfu0BuMDkrk3x+7ZgFC8MGW+opCFjMMBnkdaMNW6M/v1Pc3W0= ceph-438fb69c-e590-11ee-95c9-87774f69a715' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-03-19T01:32:15.743 INFO:teuthology.orchestra.run.smithi112.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCqyGEa2vlwe+9WcbWbCC8Jy3h1Z37hzemIAnr34gVudgT0k4Qy1DYzyVd7ZuGzAqx/B2RXQ0iLaPODqTEgreeURYtg6GBe50j+IZfs1EDwEOak/kjyni5/GCf9Uk2tOV84qyqGruaQbyZMtnd/+ycmBfi/gGOK1DJSNTE14lG6Y0inaNAj2Vy9p4S1hIxR4AtC9uPFdTGx1w04RJkZg6OIM/af0PVlNRRU3QTrMDYozsvYhXWorO/J+ZYXkU0RCMBYFtTdo55LkevfCVKuuDfgKG0gYEWRYFdtGTKUuW8TTpF0hMd9KFjAOs93y8REG2vOxb3NwOcvVaURenuHpl5V/xUj/yf19zafiVi8T+0SsRPHUJzGH0ITzpiDOwXxJji90aBzs19UGMicPEeIYUdmSFhQg2wR1kAcvBYvEDWwvui/SO+ALHX9SZKm5eYxN/TEvNqDaAX12pq49dcfu0BuMDkrk3x+7ZgFC8MGW+opCFjMMBnkdaMNW6M/v1Pc3W0= ceph-438fb69c-e590-11ee-95c9-87774f69a715 2024-03-19T01:32:15.758 DEBUG:teuthology.orchestra.run.smithi115:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCqyGEa2vlwe+9WcbWbCC8Jy3h1Z37hzemIAnr34gVudgT0k4Qy1DYzyVd7ZuGzAqx/B2RXQ0iLaPODqTEgreeURYtg6GBe50j+IZfs1EDwEOak/kjyni5/GCf9Uk2tOV84qyqGruaQbyZMtnd/+ycmBfi/gGOK1DJSNTE14lG6Y0inaNAj2Vy9p4S1hIxR4AtC9uPFdTGx1w04RJkZg6OIM/af0PVlNRRU3QTrMDYozsvYhXWorO/J+ZYXkU0RCMBYFtTdo55LkevfCVKuuDfgKG0gYEWRYFdtGTKUuW8TTpF0hMd9KFjAOs93y8REG2vOxb3NwOcvVaURenuHpl5V/xUj/yf19zafiVi8T+0SsRPHUJzGH0ITzpiDOwXxJji90aBzs19UGMicPEeIYUdmSFhQg2wR1kAcvBYvEDWwvui/SO+ALHX9SZKm5eYxN/TEvNqDaAX12pq49dcfu0BuMDkrk3x+7ZgFC8MGW+opCFjMMBnkdaMNW6M/v1Pc3W0= ceph-438fb69c-e590-11ee-95c9-87774f69a715' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-03-19T01:32:15.803 INFO:teuthology.orchestra.run.smithi115.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCqyGEa2vlwe+9WcbWbCC8Jy3h1Z37hzemIAnr34gVudgT0k4Qy1DYzyVd7ZuGzAqx/B2RXQ0iLaPODqTEgreeURYtg6GBe50j+IZfs1EDwEOak/kjyni5/GCf9Uk2tOV84qyqGruaQbyZMtnd/+ycmBfi/gGOK1DJSNTE14lG6Y0inaNAj2Vy9p4S1hIxR4AtC9uPFdTGx1w04RJkZg6OIM/af0PVlNRRU3QTrMDYozsvYhXWorO/J+ZYXkU0RCMBYFtTdo55LkevfCVKuuDfgKG0gYEWRYFdtGTKUuW8TTpF0hMd9KFjAOs93y8REG2vOxb3NwOcvVaURenuHpl5V/xUj/yf19zafiVi8T+0SsRPHUJzGH0ITzpiDOwXxJji90aBzs19UGMicPEeIYUdmSFhQg2wR1kAcvBYvEDWwvui/SO+ALHX9SZKm5eYxN/TEvNqDaAX12pq49dcfu0BuMDkrk3x+7ZgFC8MGW+opCFjMMBnkdaMNW6M/v1Pc3W0= ceph-438fb69c-e590-11ee-95c9-87774f69a715 2024-03-19T01:32:15.818 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-03-19T01:32:16.066 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:32:16.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:15 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/3654969734' entity='client.admin' 2024-03-19T01:32:18.629 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:18.626+0000 7f2a54165700 1 -- 172.21.15.99:0/138209436 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2a4c101d60 msgr2=0x7f2a4c102180 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:32:18.630 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:18.626+0000 7f2a54165700 1 --2- 172.21.15.99:0/138209436 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2a4c101d60 0x7f2a4c102180 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7f2a48009910 tx=0x7f2a48009c20 comp rx=0 tx=0).stop 2024-03-19T01:32:18.630 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:18.626+0000 7f2a54165700 1 -- 172.21.15.99:0/138209436 shutdown_connections 2024-03-19T01:32:18.630 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:18.626+0000 7f2a54165700 1 --2- 172.21.15.99:0/138209436 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2a4c101d60 0x7f2a4c102180 unknown :-1 s=CLOSED pgs=78 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:32:18.630 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:18.626+0000 7f2a54165700 1 -- 172.21.15.99:0/138209436 >> 172.21.15.99:0/138209436 conn(0x7f2a4c0fd300 msgr2=0x7f2a4c0ff740 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:32:18.630 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:18.627+0000 7f2a54165700 1 -- 172.21.15.99:0/138209436 shutdown_connections 2024-03-19T01:32:18.630 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:18.627+0000 7f2a54165700 1 -- 172.21.15.99:0/138209436 wait complete. 2024-03-19T01:32:18.630 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:18.627+0000 7f2a54165700 1 Processor -- start 2024-03-19T01:32:18.630 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:18.627+0000 7f2a54165700 1 -- start start 2024-03-19T01:32:18.630 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:18.627+0000 7f2a54165700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2a4c101d60 0x7f2a4c197440 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:32:18.630 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:18.627+0000 7f2a54165700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f2a4c107df0 con 0x7f2a4c101d60 2024-03-19T01:32:18.630 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:18.627+0000 7f2a51f01700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2a4c101d60 0x7f2a4c197440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:32:18.631 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:18.627+0000 7f2a51f01700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2a4c101d60 0x7f2a4c197440 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:49044/0 (socket says 172.21.15.99:49044) 2024-03-19T01:32:18.631 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:18.627+0000 7f2a51f01700 1 -- 172.21.15.99:0/3821986930 learned_addr learned my addr 172.21.15.99:0/3821986930 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:32:18.631 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:18.628+0000 7f2a51f01700 1 -- 172.21.15.99:0/3821986930 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f2a4c197980 con 0x7f2a4c101d60 2024-03-19T01:32:18.631 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:18.628+0000 7f2a51f01700 1 --2- 172.21.15.99:0/3821986930 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2a4c101d60 0x7f2a4c197440 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7f2a48006010 tx=0x7f2a48005050 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:32:18.631 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:18.628+0000 7f2a42ffd700 1 -- 172.21.15.99:0/3821986930 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f2a48027070 con 0x7f2a4c101d60 2024-03-19T01:32:18.631 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:18.628+0000 7f2a42ffd700 1 -- 172.21.15.99:0/3821986930 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f2a4800be40 con 0x7f2a4c101d60 2024-03-19T01:32:18.631 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:18.628+0000 7f2a54165700 1 -- 172.21.15.99:0/3821986930 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f2a4c197c10 con 0x7f2a4c101d60 2024-03-19T01:32:18.631 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:18.628+0000 7f2a42ffd700 1 -- 172.21.15.99:0/3821986930 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f2a4800f460 con 0x7f2a4c101d60 2024-03-19T01:32:18.631 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:18.628+0000 7f2a54165700 1 -- 172.21.15.99:0/3821986930 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f2a4c19a900 con 0x7f2a4c101d60 2024-03-19T01:32:18.632 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:18.629+0000 7f2a42ffd700 1 -- 172.21.15.99:0/3821986930 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 13) ==== 46162+0+0 (secure 0 0 0) 0x7f2a4802a070 con 0x7f2a4c101d60 2024-03-19T01:32:18.632 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:18.629+0000 7f2a54165700 1 -- 172.21.15.99:0/3821986930 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f2a4c108570 con 0x7f2a4c101d60 2024-03-19T01:32:18.632 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:18.629+0000 7f2a42ffd700 1 --2- 172.21.15.99:0/3821986930 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f2a38038fc0 0x7f2a3803b480 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:32:18.632 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:18.629+0000 7f2a42ffd700 1 -- 172.21.15.99:0/3821986930 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(3..3 src has 1..3) ==== 1155+0+0 (secure 0 0 0) 0x7f2a4801e030 con 0x7f2a4c101d60 2024-03-19T01:32:18.632 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:18.629+0000 7f2a51700700 1 --2- 172.21.15.99:0/3821986930 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f2a38038fc0 0x7f2a3803b480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:32:18.632 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:18.630+0000 7f2a51700700 1 --2- 172.21.15.99:0/3821986930 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f2a38038fc0 0x7f2a3803b480 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0x7f2a3c006fd0 tx=0x7f2a3c006e40 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:32:18.636 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:18.634+0000 7f2a42ffd700 1 -- 172.21.15.99:0/3821986930 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f2a48013070 con 0x7f2a4c101d60 2024-03-19T01:32:18.840 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:18.838+0000 7f2a54165700 1 -- 172.21.15.99:0/3821986930 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command([{prefix=config set, name=mgr/cephadm/allow_ptrace}] v 0) -- 0x7f2a4c02dcc0 con 0x7f2a4c101d60 2024-03-19T01:32:18.840 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:18 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:18.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:18 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:18.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:18 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi099", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:32:18.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:18 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:18.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:18 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:32:18.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:18 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:18.843 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:18.842+0000 7f2a42ffd700 1 -- 172.21.15.99:0/3821986930 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mgr/cephadm/allow_ptrace}]=0 v9)=0 v9) ==== 125+0+0 (secure 0 0 0) 0x7f2a48022470 con 0x7f2a4c101d60 2024-03-19T01:32:18.846 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:18.844+0000 7f2a54165700 1 -- 172.21.15.99:0/3821986930 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f2a38038fc0 msgr2=0x7f2a3803b480 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:32:18.846 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:18.844+0000 7f2a54165700 1 --2- 172.21.15.99:0/3821986930 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f2a38038fc0 0x7f2a3803b480 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0x7f2a3c006fd0 tx=0x7f2a3c006e40 comp rx=0 tx=0).stop 2024-03-19T01:32:18.846 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:18.844+0000 7f2a54165700 1 -- 172.21.15.99:0/3821986930 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2a4c101d60 msgr2=0x7f2a4c197440 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:32:18.846 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:18.844+0000 7f2a54165700 1 --2- 172.21.15.99:0/3821986930 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2a4c101d60 0x7f2a4c197440 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7f2a48006010 tx=0x7f2a48005050 comp rx=0 tx=0).stop 2024-03-19T01:32:18.846 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:18.844+0000 7f2a54165700 1 -- 172.21.15.99:0/3821986930 shutdown_connections 2024-03-19T01:32:18.846 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:18.844+0000 7f2a54165700 1 --2- 172.21.15.99:0/3821986930 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f2a38038fc0 0x7f2a3803b480 unknown :-1 s=CLOSED pgs=12 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:32:18.846 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:18.844+0000 7f2a54165700 1 --2- 172.21.15.99:0/3821986930 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2a4c101d60 0x7f2a4c197440 unknown :-1 s=CLOSED pgs=79 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:32:18.846 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:18.844+0000 7f2a54165700 1 -- 172.21.15.99:0/3821986930 >> 172.21.15.99:0/3821986930 conn(0x7f2a4c0fd300 msgr2=0x7f2a4c0fdd30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:32:18.846 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:18.844+0000 7f2a54165700 1 -- 172.21.15.99:0/3821986930 shutdown_connections 2024-03-19T01:32:18.847 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:18.844+0000 7f2a54165700 1 -- 172.21.15.99:0/3821986930 wait complete. 2024-03-19T01:32:19.955 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:19 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/3821986930' entity='client.admin' 2024-03-19T01:32:19.955 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:19 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:32:19.955 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:19 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:19.982 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-03-19T01:32:19.983 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-03-19T01:32:20.219 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:32:21.596 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:21.593+0000 7f3d9b8f6700 1 -- 172.21.15.99:0/1198104940 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3d94101d60 msgr2=0x7f3d94102180 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:32:21.596 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:21.593+0000 7f3d9b8f6700 1 --2- 172.21.15.99:0/1198104940 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3d94101d60 0x7f3d94102180 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7f3d90009910 tx=0x7f3d90009c20 comp rx=0 tx=0).stop 2024-03-19T01:32:21.597 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:21.593+0000 7f3d9b8f6700 1 -- 172.21.15.99:0/1198104940 shutdown_connections 2024-03-19T01:32:21.597 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:21.593+0000 7f3d9b8f6700 1 --2- 172.21.15.99:0/1198104940 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3d94101d60 0x7f3d94102180 unknown :-1 s=CLOSED pgs=80 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:32:21.597 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:21.593+0000 7f3d9b8f6700 1 -- 172.21.15.99:0/1198104940 >> 172.21.15.99:0/1198104940 conn(0x7f3d940fd300 msgr2=0x7f3d940ff740 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:32:21.597 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:21.593+0000 7f3d9b8f6700 1 -- 172.21.15.99:0/1198104940 shutdown_connections 2024-03-19T01:32:21.597 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:21.593+0000 7f3d9b8f6700 1 -- 172.21.15.99:0/1198104940 wait complete. 2024-03-19T01:32:21.597 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:21.594+0000 7f3d9b8f6700 1 Processor -- start 2024-03-19T01:32:21.597 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:21.594+0000 7f3d9b8f6700 1 -- start start 2024-03-19T01:32:21.597 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:21.594+0000 7f3d9b8f6700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3d94101d60 0x7f3d94075f80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:32:21.597 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:21.594+0000 7f3d9b8f6700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f3d94107df0 con 0x7f3d94101d60 2024-03-19T01:32:21.597 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:21.594+0000 7f3d99692700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3d94101d60 0x7f3d94075f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:32:21.597 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:21.594+0000 7f3d99692700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3d94101d60 0x7f3d94075f80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:49062/0 (socket says 172.21.15.99:49062) 2024-03-19T01:32:21.597 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:21.594+0000 7f3d99692700 1 -- 172.21.15.99:0/3508314921 learned_addr learned my addr 172.21.15.99:0/3508314921 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:32:21.597 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:21.594+0000 7f3d99692700 1 -- 172.21.15.99:0/3508314921 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f3d940764c0 con 0x7f3d94101d60 2024-03-19T01:32:21.598 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:21.594+0000 7f3d99692700 1 --2- 172.21.15.99:0/3508314921 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3d94101d60 0x7f3d94075f80 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 crypto rx=0x7f3d90006010 tx=0x7f3d90005050 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:32:21.598 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:21.594+0000 7f3d8a7fc700 1 -- 172.21.15.99:0/3508314921 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f3d90027070 con 0x7f3d94101d60 2024-03-19T01:32:21.598 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:21.594+0000 7f3d8a7fc700 1 -- 172.21.15.99:0/3508314921 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f3d9000be40 con 0x7f3d94101d60 2024-03-19T01:32:21.598 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:21.595+0000 7f3d9b8f6700 1 -- 172.21.15.99:0/3508314921 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f3d94074700 con 0x7f3d94101d60 2024-03-19T01:32:21.598 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:21.595+0000 7f3d8a7fc700 1 -- 172.21.15.99:0/3508314921 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f3d9000f460 con 0x7f3d94101d60 2024-03-19T01:32:21.598 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:21.595+0000 7f3d9b8f6700 1 -- 172.21.15.99:0/3508314921 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f3d94074be0 con 0x7f3d94101d60 2024-03-19T01:32:21.599 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:21.595+0000 7f3d8a7fc700 1 -- 172.21.15.99:0/3508314921 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 13) ==== 46162+0+0 (secure 0 0 0) 0x7f3d9002a070 con 0x7f3d94101d60 2024-03-19T01:32:21.599 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:21.595+0000 7f3d8a7fc700 1 --2- 172.21.15.99:0/3508314921 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f3d80039020 0x7f3d8003b4e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:32:21.599 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:21.596+0000 7f3d8a7fc700 1 -- 172.21.15.99:0/3508314921 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(3..3 src has 1..3) ==== 1155+0+0 (secure 0 0 0) 0x7f3d9001e030 con 0x7f3d94101d60 2024-03-19T01:32:21.599 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:21.596+0000 7f3d9b8f6700 1 -- 172.21.15.99:0/3508314921 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f3d94108570 con 0x7f3d94101d60 2024-03-19T01:32:21.599 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:21.596+0000 7f3d98e91700 1 --2- 172.21.15.99:0/3508314921 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f3d80039020 0x7f3d8003b4e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:32:21.599 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:21.596+0000 7f3d98e91700 1 --2- 172.21.15.99:0/3508314921 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f3d80039020 0x7f3d8003b4e0 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7f3d84006fd0 tx=0x7f3d84006e40 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:32:21.602 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:21.601+0000 7f3d8a7fc700 1 -- 172.21.15.99:0/3508314921 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f3d90031070 con 0x7f3d94101d60 2024-03-19T01:32:21.801 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:21.800+0000 7f3d9b8f6700 1 -- 172.21.15.99:0/3508314921 --> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] -- mgr_command(tid 0: {"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}) -- 0x7f3d94190700 con 0x7f3d80039020 2024-03-19T01:32:21.805 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:21.803+0000 7f3d8a7fc700 1 -- 172.21.15.99:0/3508314921 <== mgr.14150 v2:172.21.15.99:6800/4272410295 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+0 (secure 0 0 0) 0x7f3d94190700 con 0x7f3d80039020 2024-03-19T01:32:21.807 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:21.806+0000 7f3d9b8f6700 1 -- 172.21.15.99:0/3508314921 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f3d80039020 msgr2=0x7f3d8003b4e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:32:21.807 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:21.806+0000 7f3d9b8f6700 1 --2- 172.21.15.99:0/3508314921 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f3d80039020 0x7f3d8003b4e0 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7f3d84006fd0 tx=0x7f3d84006e40 comp rx=0 tx=0).stop 2024-03-19T01:32:21.807 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:21.806+0000 7f3d9b8f6700 1 -- 172.21.15.99:0/3508314921 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3d94101d60 msgr2=0x7f3d94075f80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:32:21.807 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:21.806+0000 7f3d9b8f6700 1 --2- 172.21.15.99:0/3508314921 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3d94101d60 0x7f3d94075f80 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 crypto rx=0x7f3d90006010 tx=0x7f3d90005050 comp rx=0 tx=0).stop 2024-03-19T01:32:21.807 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:21.806+0000 7f3d9b8f6700 1 -- 172.21.15.99:0/3508314921 shutdown_connections 2024-03-19T01:32:21.808 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:21.806+0000 7f3d9b8f6700 1 --2- 172.21.15.99:0/3508314921 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f3d80039020 0x7f3d8003b4e0 unknown :-1 s=CLOSED pgs=13 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:32:21.808 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:21.806+0000 7f3d9b8f6700 1 --2- 172.21.15.99:0/3508314921 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3d94101d60 0x7f3d94075f80 unknown :-1 s=CLOSED pgs=81 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:32:21.808 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:21.806+0000 7f3d9b8f6700 1 -- 172.21.15.99:0/3508314921 >> 172.21.15.99:0/3508314921 conn(0x7f3d940fd300 msgr2=0x7f3d940fde30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:32:21.808 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:21.806+0000 7f3d9b8f6700 1 -- 172.21.15.99:0/3508314921 shutdown_connections 2024-03-19T01:32:21.808 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:21.806+0000 7f3d9b8f6700 1 -- 172.21.15.99:0/3508314921 wait complete. 2024-03-19T01:32:22.551 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi112 2024-03-19T01:32:22.552 DEBUG:teuthology.orchestra.run.smithi112:> set -ex 2024-03-19T01:32:22.552 DEBUG:teuthology.orchestra.run.smithi112:> dd of=/etc/ceph/ceph.conf 2024-03-19T01:32:22.570 DEBUG:teuthology.orchestra.run.smithi112:> set -ex 2024-03-19T01:32:22.570 DEBUG:teuthology.orchestra.run.smithi112:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-03-19T01:32:22.628 INFO:tasks.cephadm:Adding host smithi112 to orchestrator... 2024-03-19T01:32:22.629 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph orch host add smithi112 2024-03-19T01:32:22.869 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:32:22.991 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:22 smithi099 ceph-mon[62951]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:32:22.991 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:22 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:22.991 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:22 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:32:22.991 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:22 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:32:22.991 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:22 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:32:22.991 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:22 smithi099 ceph-mon[62951]: Updating smithi099:/etc/ceph/ceph.conf 2024-03-19T01:32:22.992 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:22 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:22.992 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:22 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:22.992 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:22 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:24.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:23 smithi099 ceph-mon[62951]: Updating smithi099:/var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/config/ceph.conf 2024-03-19T01:32:24.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:23 smithi099 ceph-mon[62951]: Updating smithi099:/etc/ceph/ceph.client.admin.keyring 2024-03-19T01:32:24.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:23 smithi099 ceph-mon[62951]: Updating smithi099:/var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/config/ceph.client.admin.keyring 2024-03-19T01:32:24.284 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:24.283+0000 7f7fbf7c3700 1 -- 172.21.15.99:0/2324885679 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7fb8102e90 msgr2=0x7f7fb8103270 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:32:24.284 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:24.283+0000 7f7fbf7c3700 1 --2- 172.21.15.99:0/2324885679 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7fb8102e90 0x7f7fb8103270 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7f7fa800b0a0 tx=0x7f7fa800b3b0 comp rx=0 tx=0).stop 2024-03-19T01:32:24.284 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:24.283+0000 7f7fbf7c3700 1 -- 172.21.15.99:0/2324885679 shutdown_connections 2024-03-19T01:32:24.284 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:24.283+0000 7f7fbf7c3700 1 --2- 172.21.15.99:0/2324885679 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7fb8102e90 0x7f7fb8103270 unknown :-1 s=CLOSED pgs=82 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:32:24.284 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:24.283+0000 7f7fbf7c3700 1 -- 172.21.15.99:0/2324885679 >> 172.21.15.99:0/2324885679 conn(0x7f7fb80fc6c0 msgr2=0x7f7fb80feae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:32:24.286 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:24.284+0000 7f7fbf7c3700 1 -- 172.21.15.99:0/2324885679 shutdown_connections 2024-03-19T01:32:24.286 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:24.284+0000 7f7fbf7c3700 1 -- 172.21.15.99:0/2324885679 wait complete. 2024-03-19T01:32:24.286 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:24.284+0000 7f7fbf7c3700 1 Processor -- start 2024-03-19T01:32:24.286 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:24.285+0000 7f7fbf7c3700 1 -- start start 2024-03-19T01:32:24.289 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:24.285+0000 7f7fbf7c3700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7fb8102e90 0x7f7fb8100af0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:32:24.289 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:24.285+0000 7f7fbf7c3700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f7fb810c380 con 0x7f7fb8102e90 2024-03-19T01:32:24.290 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:24.285+0000 7f7fbd55f700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7fb8102e90 0x7f7fb8100af0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:32:24.290 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:24.285+0000 7f7fbd55f700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7fb8102e90 0x7f7fb8100af0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:39708/0 (socket says 172.21.15.99:39708) 2024-03-19T01:32:24.290 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:24.285+0000 7f7fbd55f700 1 -- 172.21.15.99:0/2942001368 learned_addr learned my addr 172.21.15.99:0/2942001368 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:32:24.290 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:24.285+0000 7f7fbd55f700 1 -- 172.21.15.99:0/2942001368 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7fb81029e0 con 0x7f7fb8102e90 2024-03-19T01:32:24.290 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:24.285+0000 7f7fbd55f700 1 --2- 172.21.15.99:0/2942001368 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7fb8102e90 0x7f7fb8100af0 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 crypto rx=0x7f7fa80062a0 tx=0x7f7fa800bdd0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:32:24.290 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:24.285+0000 7f7fb67fc700 1 -- 172.21.15.99:0/2942001368 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f7fa8017050 con 0x7f7fb8102e90 2024-03-19T01:32:24.290 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:24.285+0000 7f7fb67fc700 1 -- 172.21.15.99:0/2942001368 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7fa8023070 con 0x7f7fb8102e90 2024-03-19T01:32:24.290 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:24.285+0000 7f7fbf7c3700 1 -- 172.21.15.99:0/2942001368 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f7fb8101030 con 0x7f7fb8102e90 2024-03-19T01:32:24.290 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:24.285+0000 7f7fbf7c3700 1 -- 172.21.15.99:0/2942001368 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f7fb81014f0 con 0x7f7fb8102e90 2024-03-19T01:32:24.290 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:24.285+0000 7f7fb67fc700 1 -- 172.21.15.99:0/2942001368 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f7fa8004000 con 0x7f7fb8102e90 2024-03-19T01:32:24.291 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:24.286+0000 7f7fb67fc700 1 -- 172.21.15.99:0/2942001368 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 13) ==== 46162+0+0 (secure 0 0 0) 0x7f7fa80042d0 con 0x7f7fb8102e90 2024-03-19T01:32:24.291 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:24.286+0000 7f7fbf7c3700 1 -- 172.21.15.99:0/2942001368 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f7fb806a870 con 0x7f7fb8102e90 2024-03-19T01:32:24.291 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:24.286+0000 7f7fb67fc700 1 --2- 172.21.15.99:0/2942001368 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f7fa0039020 0x7f7fa003b4e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:32:24.291 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:24.286+0000 7f7fb67fc700 1 -- 172.21.15.99:0/2942001368 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(3..3 src has 1..3) ==== 1155+0+0 (secure 0 0 0) 0x7f7fa800f050 con 0x7f7fb8102e90 2024-03-19T01:32:24.291 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:24.286+0000 7f7fbcd5e700 1 --2- 172.21.15.99:0/2942001368 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f7fa0039020 0x7f7fa003b4e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:32:24.292 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:24.287+0000 7f7fbcd5e700 1 --2- 172.21.15.99:0/2942001368 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f7fa0039020 0x7f7fa003b4e0 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7f7fac006fd0 tx=0x7f7fac006e40 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:32:24.292 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:24.291+0000 7f7fb67fc700 1 -- 172.21.15.99:0/2942001368 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f7fa80287e0 con 0x7f7fb8102e90 2024-03-19T01:32:24.492 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:24.492+0000 7f7fbf7c3700 1 -- 172.21.15.99:0/2942001368 --> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] -- mgr_command(tid 0: {"prefix": "orch host add", "hostname": "smithi112", "target": ["mon-mgr", ""]}) -- 0x7f7fb8101720 con 0x7f7fa0039020 2024-03-19T01:32:25.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:24 smithi099 ceph-mon[62951]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi112", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:32:26.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:25 smithi099 ceph-mon[62951]: Deploying cephadm binary to smithi112 2024-03-19T01:32:28.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:27 smithi099 ceph-mon[62951]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:32:28.593 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:28.592+0000 7f7fb67fc700 1 -- 172.21.15.99:0/2942001368 <== mgr.14150 v2:172.21.15.99:6800/4272410295 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+49 (secure 0 0 0) 0x7f7fb8101720 con 0x7f7fa0039020 2024-03-19T01:32:28.594 INFO:teuthology.orchestra.run.smithi099.stdout:Added host 'smithi112' with addr '172.21.15.112' 2024-03-19T01:32:28.596 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:28.595+0000 7f7fbf7c3700 1 -- 172.21.15.99:0/2942001368 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f7fa0039020 msgr2=0x7f7fa003b4e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:32:28.596 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:28.595+0000 7f7fbf7c3700 1 --2- 172.21.15.99:0/2942001368 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f7fa0039020 0x7f7fa003b4e0 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7f7fac006fd0 tx=0x7f7fac006e40 comp rx=0 tx=0).stop 2024-03-19T01:32:28.596 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:28.595+0000 7f7fbf7c3700 1 -- 172.21.15.99:0/2942001368 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7fb8102e90 msgr2=0x7f7fb8100af0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:32:28.596 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:28.595+0000 7f7fbf7c3700 1 --2- 172.21.15.99:0/2942001368 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7fb8102e90 0x7f7fb8100af0 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 crypto rx=0x7f7fa80062a0 tx=0x7f7fa800bdd0 comp rx=0 tx=0).stop 2024-03-19T01:32:28.596 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:28.595+0000 7f7fbf7c3700 1 -- 172.21.15.99:0/2942001368 shutdown_connections 2024-03-19T01:32:28.596 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:28.595+0000 7f7fbf7c3700 1 --2- 172.21.15.99:0/2942001368 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f7fa0039020 0x7f7fa003b4e0 unknown :-1 s=CLOSED pgs=14 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:32:28.596 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:28.595+0000 7f7fbf7c3700 1 --2- 172.21.15.99:0/2942001368 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7fb8102e90 0x7f7fb8100af0 unknown :-1 s=CLOSED pgs=83 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:32:28.596 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:28.595+0000 7f7fbf7c3700 1 -- 172.21.15.99:0/2942001368 >> 172.21.15.99:0/2942001368 conn(0x7f7fb80fc6c0 msgr2=0x7f7fb8103c40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:32:28.596 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:28.595+0000 7f7fbf7c3700 1 -- 172.21.15.99:0/2942001368 shutdown_connections 2024-03-19T01:32:28.596 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:28.595+0000 7f7fbf7c3700 1 -- 172.21.15.99:0/2942001368 wait complete. 2024-03-19T01:32:29.229 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph orch host ls --format=json 2024-03-19T01:32:29.464 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:32:29.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:29 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:29.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:29 smithi099 ceph-mon[62951]: Added host smithi112 2024-03-19T01:32:29.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:29 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:32:29.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:29 smithi099 ceph-mon[62951]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:32:29.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:29 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:30.849 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:30.846+0000 7f04dc512700 1 -- 172.21.15.99:0/1724403085 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f04d4101f50 msgr2=0x7f04d4102330 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:32:30.849 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:30.846+0000 7f04dc512700 1 --2- 172.21.15.99:0/1724403085 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f04d4101f50 0x7f04d4102330 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 crypto rx=0x7f04d0009910 tx=0x7f04d0009c20 comp rx=0 tx=0).stop 2024-03-19T01:32:30.849 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:30.846+0000 7f04dc512700 1 -- 172.21.15.99:0/1724403085 shutdown_connections 2024-03-19T01:32:30.849 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:30.846+0000 7f04dc512700 1 --2- 172.21.15.99:0/1724403085 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f04d4101f50 0x7f04d4102330 unknown :-1 s=CLOSED pgs=84 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:32:30.849 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:30.846+0000 7f04dc512700 1 -- 172.21.15.99:0/1724403085 >> 172.21.15.99:0/1724403085 conn(0x7f04d40fd800 msgr2=0x7f04d40ffc20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:32:30.849 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:30.846+0000 7f04dc512700 1 -- 172.21.15.99:0/1724403085 shutdown_connections 2024-03-19T01:32:30.849 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:30.846+0000 7f04dc512700 1 -- 172.21.15.99:0/1724403085 wait complete. 2024-03-19T01:32:30.849 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:30.847+0000 7f04dc512700 1 Processor -- start 2024-03-19T01:32:30.849 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:30.847+0000 7f04dc512700 1 -- start start 2024-03-19T01:32:30.849 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:30.847+0000 7f04dc512700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f04d4101f50 0x7f04d419d580 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:32:30.850 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:30.847+0000 7f04dc512700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f04d410c320 con 0x7f04d4101f50 2024-03-19T01:32:30.850 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:30.847+0000 7f04da2ae700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f04d4101f50 0x7f04d419d580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:32:30.850 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:30.847+0000 7f04da2ae700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f04d4101f50 0x7f04d419d580 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:39736/0 (socket says 172.21.15.99:39736) 2024-03-19T01:32:30.850 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:30.847+0000 7f04da2ae700 1 -- 172.21.15.99:0/2794852124 learned_addr learned my addr 172.21.15.99:0/2794852124 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:32:30.850 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:30.847+0000 7f04da2ae700 1 -- 172.21.15.99:0/2794852124 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f04d419dac0 con 0x7f04d4101f50 2024-03-19T01:32:30.850 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:30.848+0000 7f04da2ae700 1 --2- 172.21.15.99:0/2794852124 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f04d4101f50 0x7f04d419d580 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7f04d0005850 tx=0x7f04d0005030 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:32:30.850 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:30.848+0000 7f04cb7fe700 1 -- 172.21.15.99:0/2794852124 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f04d0026070 con 0x7f04d4101f50 2024-03-19T01:32:30.850 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:30.848+0000 7f04cb7fe700 1 -- 172.21.15.99:0/2794852124 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f04d000bbd0 con 0x7f04d4101f50 2024-03-19T01:32:30.850 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:30.848+0000 7f04dc512700 1 -- 172.21.15.99:0/2794852124 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f04d419ddb0 con 0x7f04d4101f50 2024-03-19T01:32:30.850 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:30.848+0000 7f04cb7fe700 1 -- 172.21.15.99:0/2794852124 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f04d0003970 con 0x7f04d4101f50 2024-03-19T01:32:30.850 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:30.848+0000 7f04dc512700 1 -- 172.21.15.99:0/2794852124 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f04d4197820 con 0x7f04d4101f50 2024-03-19T01:32:30.851 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:30.848+0000 7f04cb7fe700 1 -- 172.21.15.99:0/2794852124 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 13) ==== 46162+0+0 (secure 0 0 0) 0x7f04d0029070 con 0x7f04d4101f50 2024-03-19T01:32:30.851 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:30.849+0000 7f04dc512700 1 -- 172.21.15.99:0/2794852124 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f04d4103430 con 0x7f04d4101f50 2024-03-19T01:32:30.851 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:30.849+0000 7f04cb7fe700 1 --2- 172.21.15.99:0/2794852124 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f04c0038fc0 0x7f04c003b480 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:32:30.851 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:30.849+0000 7f04cb7fe700 1 -- 172.21.15.99:0/2794852124 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(3..3 src has 1..3) ==== 1155+0+0 (secure 0 0 0) 0x7f04d001e070 con 0x7f04d4101f50 2024-03-19T01:32:30.852 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:30.849+0000 7f04d9aad700 1 --2- 172.21.15.99:0/2794852124 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f04c0038fc0 0x7f04c003b480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:32:30.852 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:30.849+0000 7f04d9aad700 1 --2- 172.21.15.99:0/2794852124 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f04c0038fc0 0x7f04c003b480 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7f04c4006fd0 tx=0x7f04c4006e40 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:32:30.854 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:30.854+0000 7f04cb7fe700 1 -- 172.21.15.99:0/2794852124 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f04d0013070 con 0x7f04d4101f50 2024-03-19T01:32:31.052 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:31.052+0000 7f04dc512700 1 -- 172.21.15.99:0/2794852124 --> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] -- mgr_command(tid 0: {"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7f04d4198bb0 con 0x7f04c0038fc0 2024-03-19T01:32:31.053 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:30 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:31.053 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:31.053+0000 7f04cb7fe700 1 -- 172.21.15.99:0/2794852124 <== mgr.14150 v2:172.21.15.99:6800/4272410295 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+160 (secure 0 0 0) 0x7f04d4198bb0 con 0x7f04c0038fc0 2024-03-19T01:32:31.054 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:32:31.054 INFO:teuthology.orchestra.run.smithi099.stdout:[{"addr": "172.21.15.99", "hostname": "smithi099", "labels": [], "status": ""}, {"addr": "172.21.15.112", "hostname": "smithi112", "labels": [], "status": ""}] 2024-03-19T01:32:31.056 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:31.055+0000 7f04dc512700 1 -- 172.21.15.99:0/2794852124 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f04c0038fc0 msgr2=0x7f04c003b480 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:32:31.056 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:31.055+0000 7f04dc512700 1 --2- 172.21.15.99:0/2794852124 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f04c0038fc0 0x7f04c003b480 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7f04c4006fd0 tx=0x7f04c4006e40 comp rx=0 tx=0).stop 2024-03-19T01:32:31.056 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:31.055+0000 7f04dc512700 1 -- 172.21.15.99:0/2794852124 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f04d4101f50 msgr2=0x7f04d419d580 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:32:31.056 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:31.055+0000 7f04dc512700 1 --2- 172.21.15.99:0/2794852124 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f04d4101f50 0x7f04d419d580 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7f04d0005850 tx=0x7f04d0005030 comp rx=0 tx=0).stop 2024-03-19T01:32:31.057 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:31.056+0000 7f04dc512700 1 -- 172.21.15.99:0/2794852124 shutdown_connections 2024-03-19T01:32:31.057 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:31.056+0000 7f04dc512700 1 --2- 172.21.15.99:0/2794852124 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f04c0038fc0 0x7f04c003b480 unknown :-1 s=CLOSED pgs=15 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:32:31.057 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:31.056+0000 7f04dc512700 1 --2- 172.21.15.99:0/2794852124 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f04d4101f50 0x7f04d419d580 unknown :-1 s=CLOSED pgs=85 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:32:31.057 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:31.056+0000 7f04dc512700 1 -- 172.21.15.99:0/2794852124 >> 172.21.15.99:0/2794852124 conn(0x7f04d40fd800 msgr2=0x7f04d4104cd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:32:31.057 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:31.056+0000 7f04dc512700 1 -- 172.21.15.99:0/2794852124 shutdown_connections 2024-03-19T01:32:31.057 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:31.056+0000 7f04dc512700 1 -- 172.21.15.99:0/2794852124 wait complete. 2024-03-19T01:32:31.627 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi115 2024-03-19T01:32:31.627 DEBUG:teuthology.orchestra.run.smithi115:> set -ex 2024-03-19T01:32:31.627 DEBUG:teuthology.orchestra.run.smithi115:> dd of=/etc/ceph/ceph.conf 2024-03-19T01:32:31.646 DEBUG:teuthology.orchestra.run.smithi115:> set -ex 2024-03-19T01:32:31.647 DEBUG:teuthology.orchestra.run.smithi115:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-03-19T01:32:31.704 INFO:tasks.cephadm:Adding host smithi115 to orchestrator... 2024-03-19T01:32:31.704 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph orch host add smithi115 2024-03-19T01:32:31.938 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:32:32.049 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:31 smithi099 ceph-mon[62951]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:32:32.050 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:31 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:32.999 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:32 smithi099 ceph-mon[62951]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-19T01:32:33.336 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:33.335+0000 7f0621cb0700 1 -- 172.21.15.99:0/888032531 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f061c101f50 msgr2=0x7f061c102330 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:32:33.336 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:33.335+0000 7f0621cb0700 1 --2- 172.21.15.99:0/888032531 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f061c101f50 0x7f061c102330 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 crypto rx=0x7f060c009910 tx=0x7f060c009c20 comp rx=0 tx=0).stop 2024-03-19T01:32:33.336 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:33.335+0000 7f0621cb0700 1 -- 172.21.15.99:0/888032531 shutdown_connections 2024-03-19T01:32:33.336 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:33.335+0000 7f0621cb0700 1 --2- 172.21.15.99:0/888032531 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f061c101f50 0x7f061c102330 unknown :-1 s=CLOSED pgs=86 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:32:33.336 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:33.335+0000 7f0621cb0700 1 -- 172.21.15.99:0/888032531 >> 172.21.15.99:0/888032531 conn(0x7f061c0fd820 msgr2=0x7f061c0ffc40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:32:33.336 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:33.335+0000 7f0621cb0700 1 -- 172.21.15.99:0/888032531 shutdown_connections 2024-03-19T01:32:33.336 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:33.335+0000 7f0621cb0700 1 -- 172.21.15.99:0/888032531 wait complete. 2024-03-19T01:32:33.336 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:33.335+0000 7f0621cb0700 1 Processor -- start 2024-03-19T01:32:33.336 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:33.335+0000 7f0621cb0700 1 -- start start 2024-03-19T01:32:33.338 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:33.336+0000 7f0621cb0700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f061c101f50 0x7f061c074610 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:32:33.338 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:33.336+0000 7f0621cb0700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f061c10c320 con 0x7f061c101f50 2024-03-19T01:32:33.338 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:33.336+0000 7f061b7fe700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f061c101f50 0x7f061c074610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:32:33.338 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:33.336+0000 7f061b7fe700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f061c101f50 0x7f061c074610 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:36972/0 (socket says 172.21.15.99:36972) 2024-03-19T01:32:33.339 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:33.336+0000 7f061b7fe700 1 -- 172.21.15.99:0/4188281409 learned_addr learned my addr 172.21.15.99:0/4188281409 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:32:33.339 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:33.336+0000 7f061b7fe700 1 -- 172.21.15.99:0/4188281409 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f061c076500 con 0x7f061c101f50 2024-03-19T01:32:33.339 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:33.336+0000 7f061b7fe700 1 --2- 172.21.15.99:0/4188281409 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f061c101f50 0x7f061c074610 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7f060c005850 tx=0x7f060c005030 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:32:33.339 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:33.336+0000 7f0618ff9700 1 -- 172.21.15.99:0/4188281409 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f060c026070 con 0x7f061c101f50 2024-03-19T01:32:33.339 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:33.336+0000 7f0618ff9700 1 -- 172.21.15.99:0/4188281409 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f060c00bc30 con 0x7f061c101f50 2024-03-19T01:32:33.339 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:33.336+0000 7f0618ff9700 1 -- 172.21.15.99:0/4188281409 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f060c003a20 con 0x7f061c101f50 2024-03-19T01:32:33.339 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:33.336+0000 7f0621cb0700 1 -- 172.21.15.99:0/4188281409 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f061c074bb0 con 0x7f061c101f50 2024-03-19T01:32:33.339 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:33.336+0000 7f0621cb0700 1 -- 172.21.15.99:0/4188281409 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f061c075070 con 0x7f061c101f50 2024-03-19T01:32:33.339 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:33.337+0000 7f0618ff9700 1 -- 172.21.15.99:0/4188281409 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 13) ==== 46162+0+0 (secure 0 0 0) 0x7f060c029070 con 0x7f061c101f50 2024-03-19T01:32:33.339 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:33.337+0000 7f0621cb0700 1 -- 172.21.15.99:0/4188281409 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f061c103430 con 0x7f061c101f50 2024-03-19T01:32:33.339 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:33.337+0000 7f0618ff9700 1 --2- 172.21.15.99:0/4188281409 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f0604039020 0x7f060403b4e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:32:33.339 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:33.337+0000 7f0618ff9700 1 -- 172.21.15.99:0/4188281409 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(3..3 src has 1..3) ==== 1155+0+0 (secure 0 0 0) 0x7f060c01e070 con 0x7f061c101f50 2024-03-19T01:32:33.339 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:33.337+0000 7f061affd700 1 --2- 172.21.15.99:0/4188281409 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f0604039020 0x7f060403b4e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:32:33.340 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:33.338+0000 7f061affd700 1 --2- 172.21.15.99:0/4188281409 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f0604039020 0x7f060403b4e0 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7f0610006fd0 tx=0x7f0610006e40 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:32:33.343 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:33.342+0000 7f0618ff9700 1 -- 172.21.15.99:0/4188281409 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f060c013070 con 0x7f061c101f50 2024-03-19T01:32:33.541 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:33.540+0000 7f0621cb0700 1 -- 172.21.15.99:0/4188281409 --> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] -- mgr_command(tid 0: {"prefix": "orch host add", "hostname": "smithi115", "target": ["mon-mgr", ""]}) -- 0x7f061c075b40 con 0x7f0604039020 2024-03-19T01:32:34.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:33 smithi099 ceph-mon[62951]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:32:35.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:34 smithi099 ceph-mon[62951]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi115", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:32:35.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:34 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:35.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:34 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:35.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:34 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:35.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:34 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:35.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:34 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi112", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:32:35.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:34 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:32:35.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:34 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:32:36.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:35 smithi099 ceph-mon[62951]: Deploying cephadm binary to smithi115 2024-03-19T01:32:36.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:35 smithi099 ceph-mon[62951]: Updating smithi112:/etc/ceph/ceph.conf 2024-03-19T01:32:36.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:35 smithi099 ceph-mon[62951]: Updating smithi112:/var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/config/ceph.conf 2024-03-19T01:32:36.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:35 smithi099 ceph-mon[62951]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:32:37.708 INFO:teuthology.orchestra.run.smithi099.stdout:Added host 'smithi115' with addr '172.21.15.115' 2024-03-19T01:32:37.709 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:37.707+0000 7f0618ff9700 1 -- 172.21.15.99:0/4188281409 <== mgr.14150 v2:172.21.15.99:6800/4272410295 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+49 (secure 0 0 0) 0x7f061c075b40 con 0x7f0604039020 2024-03-19T01:32:37.710 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:37.709+0000 7f0621cb0700 1 -- 172.21.15.99:0/4188281409 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f0604039020 msgr2=0x7f060403b4e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:32:37.711 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:37.709+0000 7f0621cb0700 1 --2- 172.21.15.99:0/4188281409 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f0604039020 0x7f060403b4e0 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7f0610006fd0 tx=0x7f0610006e40 comp rx=0 tx=0).stop 2024-03-19T01:32:37.711 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:37.709+0000 7f0621cb0700 1 -- 172.21.15.99:0/4188281409 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f061c101f50 msgr2=0x7f061c074610 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:32:37.711 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:37.709+0000 7f0621cb0700 1 --2- 172.21.15.99:0/4188281409 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f061c101f50 0x7f061c074610 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7f060c005850 tx=0x7f060c005030 comp rx=0 tx=0).stop 2024-03-19T01:32:37.711 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:37.710+0000 7f0621cb0700 1 -- 172.21.15.99:0/4188281409 shutdown_connections 2024-03-19T01:32:37.711 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:37.710+0000 7f0621cb0700 1 --2- 172.21.15.99:0/4188281409 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f0604039020 0x7f060403b4e0 unknown :-1 s=CLOSED pgs=16 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:32:37.711 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:37.710+0000 7f0621cb0700 1 --2- 172.21.15.99:0/4188281409 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f061c101f50 0x7f061c074610 unknown :-1 s=CLOSED pgs=87 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:32:37.711 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:37.710+0000 7f0621cb0700 1 -- 172.21.15.99:0/4188281409 >> 172.21.15.99:0/4188281409 conn(0x7f061c0fd820 msgr2=0x7f061c104d80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:32:37.711 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:37.710+0000 7f0621cb0700 1 -- 172.21.15.99:0/4188281409 shutdown_connections 2024-03-19T01:32:37.711 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:37.710+0000 7f0621cb0700 1 -- 172.21.15.99:0/4188281409 wait complete. 2024-03-19T01:32:38.061 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:37 smithi099 ceph-mon[62951]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:32:38.061 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:37 smithi099 ceph-mon[62951]: Updating smithi112:/etc/ceph/ceph.client.admin.keyring 2024-03-19T01:32:38.061 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:37 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:38.061 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:37 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:38.061 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:37 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:38.061 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:37 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:38.363 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph orch host ls --format=json 2024-03-19T01:32:38.596 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:32:39.031 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:38 smithi099 ceph-mon[62951]: Updating smithi112:/var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/config/ceph.client.admin.keyring 2024-03-19T01:32:39.031 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:38 smithi099 ceph-mon[62951]: Added host smithi115 2024-03-19T01:32:39.031 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:38 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:32:39.031 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:38 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:40.038 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:39 smithi099 ceph-mon[62951]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:32:40.038 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:39 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:40.142 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:40.141+0000 7f1359a50700 1 -- 172.21.15.99:0/1625332866 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1354101f40 msgr2=0x7f1354102320 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:32:40.142 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:40.141+0000 7f1359a50700 1 --2- 172.21.15.99:0/1625332866 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1354101f40 0x7f1354102320 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 crypto rx=0x7f133c009910 tx=0x7f133c009c20 comp rx=0 tx=0).stop 2024-03-19T01:32:40.142 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:40.141+0000 7f1359a50700 1 -- 172.21.15.99:0/1625332866 shutdown_connections 2024-03-19T01:32:40.142 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:40.141+0000 7f1359a50700 1 --2- 172.21.15.99:0/1625332866 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1354101f40 0x7f1354102320 unknown :-1 s=CLOSED pgs=88 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:32:40.142 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:40.141+0000 7f1359a50700 1 -- 172.21.15.99:0/1625332866 >> 172.21.15.99:0/1625332866 conn(0x7f13540fd7d0 msgr2=0x7f13540ffbf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:32:40.142 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:40.141+0000 7f1359a50700 1 -- 172.21.15.99:0/1625332866 shutdown_connections 2024-03-19T01:32:40.142 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:40.141+0000 7f1359a50700 1 -- 172.21.15.99:0/1625332866 wait complete. 2024-03-19T01:32:40.144 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:40.142+0000 7f1359a50700 1 Processor -- start 2024-03-19T01:32:40.145 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:40.142+0000 7f1359a50700 1 -- start start 2024-03-19T01:32:40.145 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:40.142+0000 7f1359a50700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1354101f40 0x7f1354074610 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:32:40.145 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:40.142+0000 7f1359a50700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f135410c310 con 0x7f1354101f40 2024-03-19T01:32:40.145 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:40.142+0000 7f1352ffd700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1354101f40 0x7f1354074610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:32:40.145 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:40.142+0000 7f1352ffd700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1354101f40 0x7f1354074610 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:36990/0 (socket says 172.21.15.99:36990) 2024-03-19T01:32:40.145 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:40.142+0000 7f1352ffd700 1 -- 172.21.15.99:0/713079557 learned_addr learned my addr 172.21.15.99:0/713079557 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:32:40.145 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:40.142+0000 7f1352ffd700 1 -- 172.21.15.99:0/713079557 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f1354076500 con 0x7f1354101f40 2024-03-19T01:32:40.145 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:40.142+0000 7f1352ffd700 1 --2- 172.21.15.99:0/713079557 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1354101f40 0x7f1354074610 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x7f133c005850 tx=0x7f133c005030 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:32:40.145 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:40.143+0000 7f1358a4e700 1 -- 172.21.15.99:0/713079557 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f133c026070 con 0x7f1354101f40 2024-03-19T01:32:40.145 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:40.143+0000 7f1359a50700 1 -- 172.21.15.99:0/713079557 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f1354074bb0 con 0x7f1354101f40 2024-03-19T01:32:40.145 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:40.143+0000 7f1358a4e700 1 -- 172.21.15.99:0/713079557 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f133c00bc30 con 0x7f1354101f40 2024-03-19T01:32:40.145 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:40.143+0000 7f1358a4e700 1 -- 172.21.15.99:0/713079557 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f133c003a20 con 0x7f1354101f40 2024-03-19T01:32:40.145 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:40.143+0000 7f1359a50700 1 -- 172.21.15.99:0/713079557 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f1354075070 con 0x7f1354101f40 2024-03-19T01:32:40.146 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:40.144+0000 7f1358a4e700 1 -- 172.21.15.99:0/713079557 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 13) ==== 46162+0+0 (secure 0 0 0) 0x7f133c029070 con 0x7f1354101f40 2024-03-19T01:32:40.146 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:40.144+0000 7f1359a50700 1 -- 172.21.15.99:0/713079557 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f1354103420 con 0x7f1354101f40 2024-03-19T01:32:40.146 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:40.144+0000 7f1358a4e700 1 --2- 172.21.15.99:0/713079557 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f1340038fc0 0x7f134003b480 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:32:40.146 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:40.144+0000 7f1358a4e700 1 -- 172.21.15.99:0/713079557 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(3..3 src has 1..3) ==== 1155+0+0 (secure 0 0 0) 0x7f133c01e070 con 0x7f1354101f40 2024-03-19T01:32:40.146 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:40.144+0000 7f13527fc700 1 --2- 172.21.15.99:0/713079557 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f1340038fc0 0x7f134003b480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:32:40.146 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:40.144+0000 7f13527fc700 1 --2- 172.21.15.99:0/713079557 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f1340038fc0 0x7f134003b480 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7f1344006fd0 tx=0x7f1344006e40 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:32:40.149 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:40.149+0000 7f1358a4e700 1 -- 172.21.15.99:0/713079557 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f133c013070 con 0x7f1354101f40 2024-03-19T01:32:40.372 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:40.371+0000 7f1359a50700 1 -- 172.21.15.99:0/713079557 --> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] -- mgr_command(tid 0: {"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7f1354075b40 con 0x7f1340038fc0 2024-03-19T01:32:40.373 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:40.373+0000 7f1358a4e700 1 -- 172.21.15.99:0/713079557 <== mgr.14150 v2:172.21.15.99:6800/4272410295 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+240 (secure 0 0 0) 0x7f1354075b40 con 0x7f1340038fc0 2024-03-19T01:32:40.373 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:32:40.373 INFO:teuthology.orchestra.run.smithi099.stdout:[{"addr": "172.21.15.99", "hostname": "smithi099", "labels": [], "status": ""}, {"addr": "172.21.15.112", "hostname": "smithi112", "labels": [], "status": ""}, {"addr": "172.21.15.115", "hostname": "smithi115", "labels": [], "status": ""}] 2024-03-19T01:32:40.377 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:40.376+0000 7f1359a50700 1 -- 172.21.15.99:0/713079557 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f1340038fc0 msgr2=0x7f134003b480 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:32:40.377 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:40.376+0000 7f1359a50700 1 --2- 172.21.15.99:0/713079557 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f1340038fc0 0x7f134003b480 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7f1344006fd0 tx=0x7f1344006e40 comp rx=0 tx=0).stop 2024-03-19T01:32:40.377 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:40.376+0000 7f1359a50700 1 -- 172.21.15.99:0/713079557 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1354101f40 msgr2=0x7f1354074610 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:32:40.377 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:40.376+0000 7f1359a50700 1 --2- 172.21.15.99:0/713079557 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1354101f40 0x7f1354074610 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x7f133c005850 tx=0x7f133c005030 comp rx=0 tx=0).stop 2024-03-19T01:32:40.377 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:40.376+0000 7f1359a50700 1 -- 172.21.15.99:0/713079557 shutdown_connections 2024-03-19T01:32:40.377 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:40.376+0000 7f1359a50700 1 --2- 172.21.15.99:0/713079557 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f1340038fc0 0x7f134003b480 unknown :-1 s=CLOSED pgs=17 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:32:40.377 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:40.376+0000 7f1359a50700 1 --2- 172.21.15.99:0/713079557 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1354101f40 0x7f1354074610 unknown :-1 s=CLOSED pgs=89 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:32:40.377 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:40.376+0000 7f1359a50700 1 -- 172.21.15.99:0/713079557 >> 172.21.15.99:0/713079557 conn(0x7f13540fd7d0 msgr2=0x7f1354103910 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:32:40.377 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:40.376+0000 7f1359a50700 1 -- 172.21.15.99:0/713079557 shutdown_connections 2024-03-19T01:32:40.377 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:40.376+0000 7f1359a50700 1 -- 172.21.15.99:0/713079557 wait complete. 2024-03-19T01:32:41.002 INFO:tasks.cephadm:Setting crush tunables to default 2024-03-19T01:32:41.002 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph osd crush tunables default 2024-03-19T01:32:41.249 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:32:41.553 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:41 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:41.553 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:41 smithi099 ceph-mon[62951]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-19T01:32:41.553 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:41 smithi099 ceph-mon[62951]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:32:42.781 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:42.780+0000 7fecf4b1a700 1 -- 172.21.15.99:0/1917710093 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fecf0102ea0 msgr2=0x7fecf0103280 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:32:42.781 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:42.780+0000 7fecf4b1a700 1 --2- 172.21.15.99:0/1917710093 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fecf0102ea0 0x7fecf0103280 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 crypto rx=0x7fece000b0a0 tx=0x7fece000b3b0 comp rx=0 tx=0).stop 2024-03-19T01:32:42.781 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:42.780+0000 7fecf4b1a700 1 -- 172.21.15.99:0/1917710093 shutdown_connections 2024-03-19T01:32:42.781 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:42.780+0000 7fecf4b1a700 1 --2- 172.21.15.99:0/1917710093 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fecf0102ea0 0x7fecf0103280 unknown :-1 s=CLOSED pgs=90 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:32:42.781 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:42.780+0000 7fecf4b1a700 1 -- 172.21.15.99:0/1917710093 >> 172.21.15.99:0/1917710093 conn(0x7fecf0077850 msgr2=0x7fecf00feb20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:32:42.781 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:42.780+0000 7fecf4b1a700 1 -- 172.21.15.99:0/1917710093 shutdown_connections 2024-03-19T01:32:42.781 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:42.780+0000 7fecf4b1a700 1 -- 172.21.15.99:0/1917710093 wait complete. 2024-03-19T01:32:42.783 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:42.781+0000 7fecf4b1a700 1 Processor -- start 2024-03-19T01:32:42.784 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:42.781+0000 7fecf4b1a700 1 -- start start 2024-03-19T01:32:42.784 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:42.781+0000 7fecf4b1a700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fecf0102ea0 0x7fecf019d430 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:32:42.784 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:42.781+0000 7fecf4b1a700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fecf010c390 con 0x7fecf0102ea0 2024-03-19T01:32:42.784 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:42.781+0000 7fecee59c700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fecf0102ea0 0x7fecf019d430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:32:42.784 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:42.781+0000 7fecee59c700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fecf0102ea0 0x7fecf019d430 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:50258/0 (socket says 172.21.15.99:50258) 2024-03-19T01:32:42.784 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:42.781+0000 7fecee59c700 1 -- 172.21.15.99:0/4026667028 learned_addr learned my addr 172.21.15.99:0/4026667028 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:32:42.784 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:42.781+0000 7fecee59c700 1 -- 172.21.15.99:0/4026667028 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fecf019d970 con 0x7fecf0102ea0 2024-03-19T01:32:42.784 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:42.782+0000 7fecee59c700 1 --2- 172.21.15.99:0/4026667028 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fecf0102ea0 0x7fecf019d430 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7fece00062a0 tx=0x7fece000bdd0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:32:42.784 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:42.782+0000 7fecdf7fe700 1 -- 172.21.15.99:0/4026667028 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fece0017050 con 0x7fecf0102ea0 2024-03-19T01:32:42.784 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:42.782+0000 7fecdf7fe700 1 -- 172.21.15.99:0/4026667028 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fece0023070 con 0x7fecf0102ea0 2024-03-19T01:32:42.784 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:42.782+0000 7fecf4b1a700 1 -- 172.21.15.99:0/4026667028 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fecf019dc00 con 0x7fecf0102ea0 2024-03-19T01:32:42.784 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:42.782+0000 7fecdf7fe700 1 -- 172.21.15.99:0/4026667028 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fece0004000 con 0x7fecf0102ea0 2024-03-19T01:32:42.784 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:42.782+0000 7fecf4b1a700 1 -- 172.21.15.99:0/4026667028 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fecf0197640 con 0x7fecf0102ea0 2024-03-19T01:32:42.785 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:42.783+0000 7fecdf7fe700 1 -- 172.21.15.99:0/4026667028 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 13) ==== 46162+0+0 (secure 0 0 0) 0x7fece0004270 con 0x7fecf0102ea0 2024-03-19T01:32:42.785 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:42.783+0000 7fecdf7fe700 1 --2- 172.21.15.99:0/4026667028 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fecd8039020 0x7fecd803b4e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:32:42.785 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:42.783+0000 7fecdf7fe700 1 -- 172.21.15.99:0/4026667028 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(3..3 src has 1..3) ==== 1155+0+0 (secure 0 0 0) 0x7fece000f050 con 0x7fecf0102ea0 2024-03-19T01:32:42.785 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:42.783+0000 7fecedd9b700 1 --2- 172.21.15.99:0/4026667028 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fecd8039020 0x7fecd803b4e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:32:42.785 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:42.783+0000 7fecf4b1a700 1 -- 172.21.15.99:0/4026667028 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fecf0069fe0 con 0x7fecf0102ea0 2024-03-19T01:32:42.785 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:42.783+0000 7fecedd9b700 1 --2- 172.21.15.99:0/4026667028 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fecd8039020 0x7fecd803b4e0 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7fece4006fd0 tx=0x7fece4006e40 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:32:42.788 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:42.788+0000 7fecdf7fe700 1 -- 172.21.15.99:0/4026667028 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7fece0026be0 con 0x7fecf0102ea0 2024-03-19T01:32:42.990 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:42.989+0000 7fecf4b1a700 1 -- 172.21.15.99:0/4026667028 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "osd crush tunables", "profile": "default"} v 0) -- 0x7fecf0063940 con 0x7fecf0102ea0 2024-03-19T01:32:43.722 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:43.721+0000 7fecdf7fe700 1 -- 172.21.15.99:0/4026667028 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "osd crush tunables", "profile": "default"}]=0 adjusted tunables profile to default v4) ==== 124+0+0 (secure 0 0 0) 0x7fece002d200 con 0x7fecf0102ea0 2024-03-19T01:32:43.725 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:43.724+0000 7fecf4b1a700 1 -- 172.21.15.99:0/4026667028 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fecd8039020 msgr2=0x7fecd803b4e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:32:43.725 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:43.724+0000 7fecf4b1a700 1 --2- 172.21.15.99:0/4026667028 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fecd8039020 0x7fecd803b4e0 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7fece4006fd0 tx=0x7fece4006e40 comp rx=0 tx=0).stop 2024-03-19T01:32:43.725 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:43.724+0000 7fecf4b1a700 1 -- 172.21.15.99:0/4026667028 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fecf0102ea0 msgr2=0x7fecf019d430 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:32:43.725 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:43.724+0000 7fecf4b1a700 1 --2- 172.21.15.99:0/4026667028 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fecf0102ea0 0x7fecf019d430 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7fece00062a0 tx=0x7fece000bdd0 comp rx=0 tx=0).stop 2024-03-19T01:32:43.725 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:43.724+0000 7fecf4b1a700 1 -- 172.21.15.99:0/4026667028 shutdown_connections 2024-03-19T01:32:43.725 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:43.724+0000 7fecf4b1a700 1 --2- 172.21.15.99:0/4026667028 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fecd8039020 0x7fecd803b4e0 unknown :-1 s=CLOSED pgs=18 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:32:43.725 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:43.724+0000 7fecf4b1a700 1 --2- 172.21.15.99:0/4026667028 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fecf0102ea0 0x7fecf019d430 unknown :-1 s=CLOSED pgs=91 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:32:43.726 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:43.724+0000 7fecf4b1a700 1 -- 172.21.15.99:0/4026667028 >> 172.21.15.99:0/4026667028 conn(0x7fecf0077850 msgr2=0x7fecf0103c50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:32:43.726 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:43.724+0000 7fecf4b1a700 1 -- 172.21.15.99:0/4026667028 shutdown_connections 2024-03-19T01:32:43.726 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:32:43.725+0000 7fecf4b1a700 1 -- 172.21.15.99:0/4026667028 wait complete. 2024-03-19T01:32:43.726 INFO:teuthology.orchestra.run.smithi099.stderr:adjusted tunables profile to default 2024-03-19T01:32:44.089 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:43 smithi099 ceph-mon[62951]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:32:44.089 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:43 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/4026667028' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-03-19T01:32:44.089 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:43 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:44.089 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:43 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:44.089 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:43 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:44.089 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:43 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:44.089 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:43 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi115", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:32:44.089 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:43 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:32:44.089 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:43 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:32:44.357 INFO:tasks.cephadm:Adding mon.a on smithi099 2024-03-19T01:32:44.357 INFO:tasks.cephadm:Adding mon.b on smithi112 2024-03-19T01:32:44.357 INFO:tasks.cephadm:Adding mon.c on smithi115 2024-03-19T01:32:44.357 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph orch apply mon '3;smithi099:172.21.15.99=a;smithi112:172.21.15.112=b;smithi115:172.21.15.115=c' 2024-03-19T01:32:44.583 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/config/ceph.conf 2024-03-19T01:32:45.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:44 smithi099 ceph-mon[62951]: Updating smithi115:/etc/ceph/ceph.conf 2024-03-19T01:32:45.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:44 smithi099 ceph-mon[62951]: Updating smithi115:/var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/config/ceph.conf 2024-03-19T01:32:45.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:44 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/4026667028' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-03-19T01:32:45.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:44 smithi099 ceph-mon[62951]: osdmap e4: 0 total, 0 up, 0 in 2024-03-19T01:32:45.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:44 smithi099 ceph-mon[62951]: Updating smithi115:/etc/ceph/ceph.client.admin.keyring 2024-03-19T01:32:45.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:44 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:45.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:44 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:45.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:44 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:46.001 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:45.999+0000 7f926736e700 1 -- 172.21.15.115:0/938450142 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9260100e40 msgr2=0x7f9260101220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:32:46.001 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:45.999+0000 7f926736e700 1 --2- 172.21.15.115:0/938450142 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9260100e40 0x7f9260101220 secure :-1 s=READY pgs=92 cs=0 l=1 rev1=1 crypto rx=0x7f9250009910 tx=0x7f9250009c20 comp rx=0 tx=0).stop 2024-03-19T01:32:46.002 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:45.999+0000 7f926736e700 1 -- 172.21.15.115:0/938450142 shutdown_connections 2024-03-19T01:32:46.002 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:45.999+0000 7f926736e700 1 --2- 172.21.15.115:0/938450142 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9260100e40 0x7f9260101220 unknown :-1 s=CLOSED pgs=92 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:32:46.002 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:45.999+0000 7f926736e700 1 -- 172.21.15.115:0/938450142 >> 172.21.15.115:0/938450142 conn(0x7f9260077850 msgr2=0x7f92600feb20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:32:46.002 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:45.999+0000 7f926736e700 1 -- 172.21.15.115:0/938450142 shutdown_connections 2024-03-19T01:32:46.002 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:45.999+0000 7f926736e700 1 -- 172.21.15.115:0/938450142 wait complete. 2024-03-19T01:32:46.002 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:46.000+0000 7f926736e700 1 Processor -- start 2024-03-19T01:32:46.002 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:46.000+0000 7f926736e700 1 -- start start 2024-03-19T01:32:46.002 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:46.000+0000 7f926736e700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9260100e40 0x7f92601028e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:32:46.002 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:46.000+0000 7f926736e700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f926010b210 con 0x7f9260100e40 2024-03-19T01:32:46.002 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:46.000+0000 7f926510a700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9260100e40 0x7f92601028e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:32:46.002 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:46.000+0000 7f926510a700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9260100e40 0x7f92601028e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.115:55486/0 (socket says 172.21.15.115:55486) 2024-03-19T01:32:46.002 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:46.000+0000 7f926510a700 1 -- 172.21.15.115:0/711697936 learned_addr learned my addr 172.21.15.115:0/711697936 (peer_addr_for_me v2:172.21.15.115:0/0) 2024-03-19T01:32:46.004 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:46.001+0000 7f926510a700 1 -- 172.21.15.115:0/711697936 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f92601047d0 con 0x7f9260100e40 2024-03-19T01:32:46.004 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:46.001+0000 7f926510a700 1 --2- 172.21.15.115:0/711697936 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9260100e40 0x7f92601028e0 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7f9250005850 tx=0x7f9250005030 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:32:46.004 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:46.001+0000 7f92567fc700 1 -- 172.21.15.115:0/711697936 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f9250026070 con 0x7f9260100e40 2024-03-19T01:32:46.004 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:46.001+0000 7f926736e700 1 -- 172.21.15.115:0/711697936 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f9260102e80 con 0x7f9260100e40 2024-03-19T01:32:46.004 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:46.001+0000 7f92567fc700 1 -- 172.21.15.115:0/711697936 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f925000bc30 con 0x7f9260100e40 2024-03-19T01:32:46.004 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:46.001+0000 7f92567fc700 1 -- 172.21.15.115:0/711697936 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f9250003a20 con 0x7f9260100e40 2024-03-19T01:32:46.004 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:46.001+0000 7f926736e700 1 -- 172.21.15.115:0/711697936 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f9260103340 con 0x7f9260100e40 2024-03-19T01:32:46.004 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:46.002+0000 7f926736e700 1 -- 172.21.15.115:0/711697936 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f9260102320 con 0x7f9260100e40 2024-03-19T01:32:46.004 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:46.002+0000 7f92567fc700 1 -- 172.21.15.115:0/711697936 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 13) ==== 46162+0+0 (secure 0 0 0) 0x7f9250029070 con 0x7f9260100e40 2024-03-19T01:32:46.005 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:46.002+0000 7f92567fc700 1 --2- 172.21.15.115:0/711697936 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f924c038fc0 0x7f924c03b480 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:32:46.005 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:46.002+0000 7f92567fc700 1 -- 172.21.15.115:0/711697936 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(4..4 src has 1..4) ==== 1155+0+0 (secure 0 0 0) 0x7f925001e070 con 0x7f9260100e40 2024-03-19T01:32:46.005 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:46.002+0000 7f9264909700 1 --2- 172.21.15.115:0/711697936 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f924c038fc0 0x7f924c03b480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:32:46.005 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:46.003+0000 7f9264909700 1 --2- 172.21.15.115:0/711697936 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f924c038fc0 0x7f924c03b480 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7f925c006fd0 tx=0x7f925c006e40 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:32:46.009 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:46.008+0000 7f92567fc700 1 -- 172.21.15.115:0/711697936 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f9250013070 con 0x7f9260100e40 2024-03-19T01:32:46.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:45 smithi099 ceph-mon[62951]: Updating smithi115:/var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/config/ceph.client.admin.keyring 2024-03-19T01:32:46.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:45 smithi099 ceph-mon[62951]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:32:46.223 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:46.222+0000 7f926736e700 1 -- 172.21.15.115:0/711697936 --> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] -- mgr_command(tid 0: {"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi099:172.21.15.99=a;smithi112:172.21.15.112=b;smithi115:172.21.15.115=c", "target": ["mon-mgr", ""]}) -- 0x7f9260103e10 con 0x7f924c038fc0 2024-03-19T01:32:46.232 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:46.231+0000 7f92567fc700 1 -- 172.21.15.115:0/711697936 <== mgr.14150 v2:172.21.15.99:6800/4272410295 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+24 (secure 0 0 0) 0x7f9260103e10 con 0x7f924c038fc0 2024-03-19T01:32:46.232 INFO:teuthology.orchestra.run.smithi115.stdout:Scheduled mon update... 2024-03-19T01:32:46.234 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:46.233+0000 7f926736e700 1 -- 172.21.15.115:0/711697936 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f924c038fc0 msgr2=0x7f924c03b480 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:32:46.234 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:46.233+0000 7f926736e700 1 --2- 172.21.15.115:0/711697936 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f924c038fc0 0x7f924c03b480 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7f925c006fd0 tx=0x7f925c006e40 comp rx=0 tx=0).stop 2024-03-19T01:32:46.234 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:46.233+0000 7f926736e700 1 -- 172.21.15.115:0/711697936 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9260100e40 msgr2=0x7f92601028e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:32:46.235 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:46.233+0000 7f926736e700 1 --2- 172.21.15.115:0/711697936 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9260100e40 0x7f92601028e0 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7f9250005850 tx=0x7f9250005030 comp rx=0 tx=0).stop 2024-03-19T01:32:46.235 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:46.233+0000 7f926736e700 1 -- 172.21.15.115:0/711697936 shutdown_connections 2024-03-19T01:32:46.235 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:46.233+0000 7f926736e700 1 --2- 172.21.15.115:0/711697936 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f924c038fc0 0x7f924c03b480 unknown :-1 s=CLOSED pgs=19 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:32:46.235 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:46.233+0000 7f926736e700 1 --2- 172.21.15.115:0/711697936 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9260100e40 0x7f92601028e0 unknown :-1 s=CLOSED pgs=93 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:32:46.235 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:46.233+0000 7f926736e700 1 -- 172.21.15.115:0/711697936 >> 172.21.15.115:0/711697936 conn(0x7f9260077850 msgr2=0x7f9260069700 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:32:46.235 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:46.233+0000 7f926736e700 1 -- 172.21.15.115:0/711697936 shutdown_connections 2024-03-19T01:32:46.235 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:46.233+0000 7f926736e700 1 -- 172.21.15.115:0/711697936 wait complete. 2024-03-19T01:32:46.807 DEBUG:teuthology.orchestra.run.smithi112:mon.b> sudo journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@mon.b.service 2024-03-19T01:32:46.810 DEBUG:teuthology.orchestra.run.smithi115:mon.c> sudo journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@mon.c.service 2024-03-19T01:32:46.812 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-03-19T01:32:46.812 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph mon dump -f json 2024-03-19T01:32:47.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:47 smithi099 ceph-mon[62951]: from='client.14186 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi099:172.21.15.99=a;smithi112:172.21.15.112=b;smithi115:172.21.15.115=c", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:32:47.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:47 smithi099 ceph-mon[62951]: Saving service mon spec with placement smithi099:172.21.15.99=a;smithi112:172.21.15.112=b;smithi115:172.21.15.115=c;count:3 2024-03-19T01:32:47.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:47 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:47.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:47 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:32:47.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:47 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:32:47.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:47 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:32:47.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:47 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:47.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:47 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-19T01:32:47.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:47 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:32:47.593 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:47 smithi099 ceph-mon[62951]: Deploying daemon mon.c on smithi115 2024-03-19T01:32:47.593 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:47 smithi099 ceph-mon[62951]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:32:47.932 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/config/ceph.conf 2024-03-19T01:32:49.988 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:49 smithi115 systemd[1]: Starting Ceph mon.c for 438fb69c-e590-11ee-95c9-87774f69a715... 2024-03-19T01:32:50.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:49 smithi099 ceph-mon[62951]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:32:50.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 podman[67286]: 2024-03-19 01:32:50.272204617 +0000 UTC m=+0.198138016 image pull 2312c8e42d0069a2dbb2364bddb00264db4b800e7f18a7e7e4eb8f3ecab2ff1c quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639 2024-03-19T01:32:50.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 podman[67286]: 2024-03-19 01:32:50.382497551 +0000 UTC m=+0.308430951 container create 30f5aa3b44b18e0930bfcf3673fa632f7856352342cbabc3e17d7dddfc17505c (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mon-c, ceph=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240307, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.vendor=CentOS) 2024-03-19T01:32:50.679 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:50.678+0000 7f481a3da700 1 -- 172.21.15.115:0/1830034733 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4814100e50 msgr2=0x7f4814101230 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:32:50.679 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:50.678+0000 7f481a3da700 1 --2- 172.21.15.115:0/1830034733 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4814100e50 0x7f4814101230 secure :-1 s=READY pgs=94 cs=0 l=1 rev1=1 crypto rx=0x7f4804009910 tx=0x7f4804009c20 comp rx=0 tx=0).stop 2024-03-19T01:32:50.679 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:50.678+0000 7f481a3da700 1 -- 172.21.15.115:0/1830034733 shutdown_connections 2024-03-19T01:32:50.680 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:50.678+0000 7f481a3da700 1 --2- 172.21.15.115:0/1830034733 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4814100e50 0x7f4814101230 unknown :-1 s=CLOSED pgs=94 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:32:50.680 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:50.678+0000 7f481a3da700 1 -- 172.21.15.115:0/1830034733 >> 172.21.15.115:0/1830034733 conn(0x7f48140fc700 msgr2=0x7f48140feb20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:32:50.680 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:50.678+0000 7f481a3da700 1 -- 172.21.15.115:0/1830034733 shutdown_connections 2024-03-19T01:32:50.680 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:50.678+0000 7f481a3da700 1 -- 172.21.15.115:0/1830034733 wait complete. 2024-03-19T01:32:50.680 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:50.678+0000 7f481a3da700 1 Processor -- start 2024-03-19T01:32:50.680 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:50.678+0000 7f481a3da700 1 -- start start 2024-03-19T01:32:50.680 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:50.679+0000 7f481a3da700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4814100e50 0x7f481419c430 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:32:50.680 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:50.679+0000 7f481a3da700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f481410b220 con 0x7f4814100e50 2024-03-19T01:32:50.680 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:50.679+0000 7f4813fff700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4814100e50 0x7f481419c430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:32:50.680 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:50.679+0000 7f4813fff700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4814100e50 0x7f481419c430 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.115:49530/0 (socket says 172.21.15.115:49530) 2024-03-19T01:32:50.680 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:50.679+0000 7f4813fff700 1 -- 172.21.15.115:0/3795310894 learned_addr learned my addr 172.21.15.115:0/3795310894 (peer_addr_for_me v2:172.21.15.115:0/0) 2024-03-19T01:32:50.680 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:50.679+0000 7f4813fff700 1 -- 172.21.15.115:0/3795310894 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f481419c970 con 0x7f4814100e50 2024-03-19T01:32:50.680 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:50.679+0000 7f4813fff700 1 --2- 172.21.15.115:0/3795310894 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4814100e50 0x7f481419c430 secure :-1 s=READY pgs=95 cs=0 l=1 rev1=1 crypto rx=0x7f4804005850 tx=0x7f4804005030 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:32:50.682 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:50.680+0000 7f48117fa700 1 -- 172.21.15.115:0/3795310894 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f4804028070 con 0x7f4814100e50 2024-03-19T01:32:50.682 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:50.680+0000 7f481a3da700 1 -- 172.21.15.115:0/3795310894 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f481419cc00 con 0x7f4814100e50 2024-03-19T01:32:50.682 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:50.680+0000 7f48117fa700 1 -- 172.21.15.115:0/3795310894 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f480400bbd0 con 0x7f4814100e50 2024-03-19T01:32:50.682 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:50.680+0000 7f48117fa700 1 -- 172.21.15.115:0/3795310894 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f4804003970 con 0x7f4814100e50 2024-03-19T01:32:50.682 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:50.680+0000 7f481a3da700 1 -- 172.21.15.115:0/3795310894 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f48141965f0 con 0x7f4814100e50 2024-03-19T01:32:50.682 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:50.681+0000 7f48117fa700 1 -- 172.21.15.115:0/3795310894 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 13) ==== 46162+0+0 (secure 0 0 0) 0x7f480402b070 con 0x7f4814100e50 2024-03-19T01:32:50.682 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:50.681+0000 7f481a3da700 1 -- 172.21.15.115:0/3795310894 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f4814102330 con 0x7f4814100e50 2024-03-19T01:32:50.682 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:50.681+0000 7f48117fa700 1 --2- 172.21.15.115:0/3795310894 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f47fc039020 0x7f47fc03b4e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:32:50.682 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:50.681+0000 7f48117fa700 1 -- 172.21.15.115:0/3795310894 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(4..4 src has 1..4) ==== 1155+0+0 (secure 0 0 0) 0x7f4804020070 con 0x7f4814100e50 2024-03-19T01:32:50.683 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:50.681+0000 7f48137fe700 1 --2- 172.21.15.115:0/3795310894 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f47fc039020 0x7f47fc03b4e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:32:50.683 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:50.682+0000 7f48137fe700 1 --2- 172.21.15.115:0/3795310894 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f47fc039020 0x7f47fc03b4e0 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7f4808006fd0 tx=0x7f4808006e40 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:32:50.686 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:50.686+0000 7f48117fa700 1 -- 172.21.15.115:0/3795310894 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f4804013070 con 0x7f4814100e50 2024-03-19T01:32:50.827 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:50.827+0000 7f48117fa700 1 -- 172.21.15.115:0/3795310894 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_map magic: 0 ==== 309+0+0 (secure 0 0 0) 0x7f480402b6e0 con 0x7f4814100e50 2024-03-19T01:32:50.827 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 podman[67286]: 2024-03-19 01:32:50.768971517 +0000 UTC m=+0.694904925 container init 30f5aa3b44b18e0930bfcf3673fa632f7856352342cbabc3e17d7dddfc17505c (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mon-c, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240307, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-03-19T01:32:50.828 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 podman[67286]: 2024-03-19 01:32:50.772283155 +0000 UTC m=+0.698216555 container start 30f5aa3b44b18e0930bfcf3673fa632f7856352342cbabc3e17d7dddfc17505c (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mon-c, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.build-date=20240307, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-19T01:32:50.828 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: set uid:gid to 167:167 (ceph:ceph) 2024-03-19T01:32:50.828 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: ceph version 19.0.0-2206-gb6ae2491 (b6ae24918b03b1b8a19c9263a50655bba80bb810) squid (dev), process ceph-mon, pid 2 2024-03-19T01:32:50.828 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: pidfile_write: ignore empty --pid-file 2024-03-19T01:32:50.828 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: load: jerasure load: lrc 2024-03-19T01:32:50.828 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: RocksDB version: 7.9.2 2024-03-19T01:32:50.828 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Git sha 0 2024-03-19T01:32:50.828 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Compile date 2024-03-18 18:57:12 2024-03-19T01:32:50.828 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: DB SUMMARY 2024-03-19T01:32:50.828 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: DB Session ID: ELKICXDD1Y309IUY6F8Z 2024-03-19T01:32:50.828 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: CURRENT file: CURRENT 2024-03-19T01:32:50.829 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: IDENTITY file: IDENTITY 2024-03-19T01:32:50.829 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: MANIFEST file: MANIFEST-000005 size: 59 Bytes 2024-03-19T01:32:50.829 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2024-03-19T01:32:50.829 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 511 ; 2024-03-19T01:32:50.829 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.error_if_exists: 0 2024-03-19T01:32:50.829 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.create_if_missing: 0 2024-03-19T01:32:50.829 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.paranoid_checks: 1 2024-03-19T01:32:50.829 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.flush_verify_memtable_count: 1 2024-03-19T01:32:50.829 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-03-19T01:32:50.829 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-03-19T01:32:50.829 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.env: 0x5644ab73bc60 2024-03-19T01:32:50.829 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.fs: PosixFileSystem 2024-03-19T01:32:50.829 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.info_log: 0x5644ad305ae0 2024-03-19T01:32:50.829 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.max_file_opening_threads: 16 2024-03-19T01:32:50.830 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.statistics: (nil) 2024-03-19T01:32:50.830 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.use_fsync: 0 2024-03-19T01:32:50.830 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.max_log_file_size: 0 2024-03-19T01:32:50.830 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-03-19T01:32:50.830 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.log_file_time_to_roll: 0 2024-03-19T01:32:50.830 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.keep_log_file_num: 1000 2024-03-19T01:32:50.830 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.recycle_log_file_num: 0 2024-03-19T01:32:50.830 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.allow_fallocate: 1 2024-03-19T01:32:50.830 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.allow_mmap_reads: 0 2024-03-19T01:32:50.830 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.allow_mmap_writes: 0 2024-03-19T01:32:50.830 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.use_direct_reads: 0 2024-03-19T01:32:50.830 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-03-19T01:32:50.830 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.create_missing_column_families: 0 2024-03-19T01:32:50.830 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.db_log_dir: 2024-03-19T01:32:50.831 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.wal_dir: 2024-03-19T01:32:50.831 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.table_cache_numshardbits: 6 2024-03-19T01:32:50.831 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.WAL_ttl_seconds: 0 2024-03-19T01:32:50.831 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.WAL_size_limit_MB: 0 2024-03-19T01:32:50.831 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-03-19T01:32:50.831 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-03-19T01:32:50.831 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.is_fd_close_on_exec: 1 2024-03-19T01:32:50.831 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.advise_random_on_open: 1 2024-03-19T01:32:50.831 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.db_write_buffer_size: 0 2024-03-19T01:32:50.831 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.write_buffer_manager: 0x5644ad2d81e0 2024-03-19T01:32:50.831 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-03-19T01:32:50.831 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-03-19T01:32:50.831 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.use_adaptive_mutex: 0 2024-03-19T01:32:50.831 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.rate_limiter: (nil) 2024-03-19T01:32:50.832 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-03-19T01:32:50.832 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.wal_recovery_mode: 2 2024-03-19T01:32:50.832 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.enable_thread_tracking: 0 2024-03-19T01:32:50.832 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.enable_pipelined_write: 0 2024-03-19T01:32:50.832 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.unordered_write: 0 2024-03-19T01:32:50.832 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-03-19T01:32:50.832 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-03-19T01:32:50.832 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-03-19T01:32:50.832 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-03-19T01:32:50.832 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.row_cache: None 2024-03-19T01:32:50.832 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.wal_filter: None 2024-03-19T01:32:50.834 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-03-19T01:32:50.834 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.allow_ingest_behind: 0 2024-03-19T01:32:50.834 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.two_write_queues: 0 2024-03-19T01:32:50.834 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.manual_wal_flush: 0 2024-03-19T01:32:50.834 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.wal_compression: 0 2024-03-19T01:32:50.834 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.atomic_flush: 0 2024-03-19T01:32:50.834 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-03-19T01:32:50.834 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.persist_stats_to_disk: 0 2024-03-19T01:32:50.834 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.write_dbid_to_manifest: 0 2024-03-19T01:32:50.834 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.log_readahead_size: 0 2024-03-19T01:32:50.835 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-03-19T01:32:50.835 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.best_efforts_recovery: 0 2024-03-19T01:32:50.835 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-03-19T01:32:50.835 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-03-19T01:32:50.835 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.allow_data_in_errors: 0 2024-03-19T01:32:50.835 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.db_host_id: __hostname__ 2024-03-19T01:32:50.835 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.enforce_single_del_contracts: true 2024-03-19T01:32:50.835 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.max_background_jobs: 2 2024-03-19T01:32:50.835 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.max_background_compactions: -1 2024-03-19T01:32:50.835 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.max_subcompactions: 1 2024-03-19T01:32:50.835 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-03-19T01:32:50.835 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-03-19T01:32:50.835 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.delayed_write_rate : 16777216 2024-03-19T01:32:50.835 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.max_total_wal_size: 0 2024-03-19T01:32:50.835 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-03-19T01:32:50.835 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.stats_dump_period_sec: 600 2024-03-19T01:32:50.836 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.stats_persist_period_sec: 600 2024-03-19T01:32:50.836 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-03-19T01:32:50.836 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.max_open_files: -1 2024-03-19T01:32:50.836 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.bytes_per_sync: 0 2024-03-19T01:32:50.836 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.wal_bytes_per_sync: 0 2024-03-19T01:32:50.836 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.strict_bytes_per_sync: 0 2024-03-19T01:32:50.836 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.compaction_readahead_size: 0 2024-03-19T01:32:50.836 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.max_background_flushes: -1 2024-03-19T01:32:50.836 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Compression algorithms supported: 2024-03-19T01:32:50.836 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-03-19T01:32:50.836 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: kZSTD supported: 0 2024-03-19T01:32:50.836 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: kXpressCompression supported: 0 2024-03-19T01:32:50.836 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: kLZ4HCCompression supported: 1 2024-03-19T01:32:50.836 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: kZlibCompression supported: 1 2024-03-19T01:32:50.836 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: kSnappyCompression supported: 1 2024-03-19T01:32:50.837 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: kLZ4Compression supported: 1 2024-03-19T01:32:50.837 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: kBZip2Compression supported: 0 2024-03-19T01:32:50.837 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Fast CRC32 supported: Supported on x86 2024-03-19T01:32:50.837 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: DMutex implementation: pthread_mutex_t 2024-03-19T01:32:50.837 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 2024-03-19T01:32:50.837 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-03-19T01:32:50.837 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-03-19T01:32:50.837 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.merge_operator: 2024-03-19T01:32:50.837 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.compaction_filter: None 2024-03-19T01:32:50.837 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.compaction_filter_factory: None 2024-03-19T01:32:50.837 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.sst_partitioner_factory: None 2024-03-19T01:32:50.837 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.memtable_factory: SkipListFactory 2024-03-19T01:32:50.837 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.table_factory: BlockBasedTable 2024-03-19T01:32:50.837 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5644ad305be0) 2024-03-19T01:32:50.837 INFO:journalctl@ceph.mon.c.smithi115.stdout: cache_index_and_filter_blocks: 1 2024-03-19T01:32:50.838 INFO:journalctl@ceph.mon.c.smithi115.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-03-19T01:32:50.838 INFO:journalctl@ceph.mon.c.smithi115.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-03-19T01:32:50.838 INFO:journalctl@ceph.mon.c.smithi115.stdout: pin_top_level_index_and_filter: 1 2024-03-19T01:32:50.838 INFO:journalctl@ceph.mon.c.smithi115.stdout: index_type: 0 2024-03-19T01:32:50.838 INFO:journalctl@ceph.mon.c.smithi115.stdout: data_block_index_type: 0 2024-03-19T01:32:50.838 INFO:journalctl@ceph.mon.c.smithi115.stdout: index_shortening: 1 2024-03-19T01:32:50.838 INFO:journalctl@ceph.mon.c.smithi115.stdout: data_block_hash_table_util_ratio: 0.750000 2024-03-19T01:32:50.838 INFO:journalctl@ceph.mon.c.smithi115.stdout: checksum: 4 2024-03-19T01:32:50.838 INFO:journalctl@ceph.mon.c.smithi115.stdout: no_block_cache: 0 2024-03-19T01:32:50.838 INFO:journalctl@ceph.mon.c.smithi115.stdout: block_cache: 0x5644ad30b090 2024-03-19T01:32:50.838 INFO:journalctl@ceph.mon.c.smithi115.stdout: block_cache_name: BinnedLRUCache 2024-03-19T01:32:50.838 INFO:journalctl@ceph.mon.c.smithi115.stdout: block_cache_options: 2024-03-19T01:32:50.838 INFO:journalctl@ceph.mon.c.smithi115.stdout: capacity : 536870912 2024-03-19T01:32:50.838 INFO:journalctl@ceph.mon.c.smithi115.stdout: num_shard_bits : 4 2024-03-19T01:32:50.839 INFO:journalctl@ceph.mon.c.smithi115.stdout: strict_capacity_limit : 0 2024-03-19T01:32:50.839 INFO:journalctl@ceph.mon.c.smithi115.stdout: high_pri_pool_ratio: 0.000 2024-03-19T01:32:50.839 INFO:journalctl@ceph.mon.c.smithi115.stdout: block_cache_compressed: (nil) 2024-03-19T01:32:50.839 INFO:journalctl@ceph.mon.c.smithi115.stdout: persistent_cache: (nil) 2024-03-19T01:32:50.839 INFO:journalctl@ceph.mon.c.smithi115.stdout: block_size: 4096 2024-03-19T01:32:50.839 INFO:journalctl@ceph.mon.c.smithi115.stdout: block_size_deviation: 10 2024-03-19T01:32:50.839 INFO:journalctl@ceph.mon.c.smithi115.stdout: block_restart_interval: 16 2024-03-19T01:32:50.839 INFO:journalctl@ceph.mon.c.smithi115.stdout: index_block_restart_interval: 1 2024-03-19T01:32:50.839 INFO:journalctl@ceph.mon.c.smithi115.stdout: metadata_block_size: 4096 2024-03-19T01:32:50.839 INFO:journalctl@ceph.mon.c.smithi115.stdout: partition_filters: 0 2024-03-19T01:32:50.839 INFO:journalctl@ceph.mon.c.smithi115.stdout: use_delta_encoding: 1 2024-03-19T01:32:50.839 INFO:journalctl@ceph.mon.c.smithi115.stdout: filter_policy: bloomfilter 2024-03-19T01:32:50.839 INFO:journalctl@ceph.mon.c.smithi115.stdout: whole_key_filtering: 1 2024-03-19T01:32:50.839 INFO:journalctl@ceph.mon.c.smithi115.stdout: verify_compression: 0 2024-03-19T01:32:50.839 INFO:journalctl@ceph.mon.c.smithi115.stdout: read_amp_bytes_per_bit: 0 2024-03-19T01:32:50.840 INFO:journalctl@ceph.mon.c.smithi115.stdout: format_version: 5 2024-03-19T01:32:50.840 INFO:journalctl@ceph.mon.c.smithi115.stdout: enable_index_compression: 1 2024-03-19T01:32:50.840 INFO:journalctl@ceph.mon.c.smithi115.stdout: block_align: 0 2024-03-19T01:32:50.840 INFO:journalctl@ceph.mon.c.smithi115.stdout: max_auto_readahead_size: 262144 2024-03-19T01:32:50.840 INFO:journalctl@ceph.mon.c.smithi115.stdout: prepopulate_block_cache: 0 2024-03-19T01:32:50.840 INFO:journalctl@ceph.mon.c.smithi115.stdout: initial_auto_readahead_size: 8192 2024-03-19T01:32:50.840 INFO:journalctl@ceph.mon.c.smithi115.stdout: num_file_reads_for_auto_readahead: 2 2024-03-19T01:32:50.840 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.write_buffer_size: 33554432 2024-03-19T01:32:50.840 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.max_write_buffer_number: 2 2024-03-19T01:32:50.840 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.compression: NoCompression 2024-03-19T01:32:50.840 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.bottommost_compression: Disabled 2024-03-19T01:32:50.840 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.prefix_extractor: nullptr 2024-03-19T01:32:50.840 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-03-19T01:32:50.840 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.num_levels: 7 2024-03-19T01:32:50.841 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-03-19T01:32:50.841 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-03-19T01:32:50.841 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-03-19T01:32:50.841 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-03-19T01:32:50.841 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-03-19T01:32:50.841 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-03-19T01:32:50.841 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-03-19T01:32:50.841 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-03-19T01:32:50.841 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-03-19T01:32:50.841 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-03-19T01:32:50.841 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-03-19T01:32:50.841 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-03-19T01:32:50.841 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.compression_opts.window_bits: -14 2024-03-19T01:32:50.841 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.compression_opts.level: 32767 2024-03-19T01:32:50.842 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.compression_opts.strategy: 0 2024-03-19T01:32:50.842 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-03-19T01:32:50.842 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-03-19T01:32:50.842 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-03-19T01:32:50.842 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-03-19T01:32:50.842 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.compression_opts.enabled: false 2024-03-19T01:32:50.842 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-03-19T01:32:50.842 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-03-19T01:32:50.842 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-03-19T01:32:50.842 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-03-19T01:32:50.842 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.target_file_size_base: 67108864 2024-03-19T01:32:50.842 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.target_file_size_multiplier: 1 2024-03-19T01:32:50.842 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-03-19T01:32:50.842 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-03-19T01:32:50.843 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-03-19T01:32:50.843 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-03-19T01:32:50.843 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-03-19T01:32:50.843 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-03-19T01:32:50.843 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-03-19T01:32:50.843 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-03-19T01:32:50.843 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-03-19T01:32:50.843 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-03-19T01:32:50.843 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-03-19T01:32:50.843 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-03-19T01:32:50.843 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-03-19T01:32:50.843 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.arena_block_size: 1048576 2024-03-19T01:32:50.843 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-03-19T01:32:50.843 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-03-19T01:32:50.844 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.disable_auto_compactions: 0 2024-03-19T01:32:50.844 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-03-19T01:32:50.844 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-03-19T01:32:50.844 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-03-19T01:32:50.844 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-03-19T01:32:50.844 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-03-19T01:32:50.844 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-03-19T01:32:50.844 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-03-19T01:32:50.844 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-03-19T01:32:50.844 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-03-19T01:32:50.844 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-03-19T01:32:50.844 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.table_properties_collectors: 2024-03-19T01:32:50.844 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.inplace_update_support: 0 2024-03-19T01:32:50.844 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.inplace_update_num_locks: 10000 2024-03-19T01:32:50.844 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-03-19T01:32:50.845 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-03-19T01:32:50.845 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.memtable_huge_page_size: 0 2024-03-19T01:32:50.845 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.bloom_locality: 0 2024-03-19T01:32:50.845 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.max_successive_merges: 0 2024-03-19T01:32:50.845 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.optimize_filters_for_hits: 0 2024-03-19T01:32:50.845 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.paranoid_file_checks: 0 2024-03-19T01:32:50.845 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.force_consistency_checks: 1 2024-03-19T01:32:50.845 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.report_bg_io_stats: 0 2024-03-19T01:32:50.845 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.ttl: 2592000 2024-03-19T01:32:50.845 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.periodic_compaction_seconds: 0 2024-03-19T01:32:50.845 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-03-19T01:32:50.845 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-03-19T01:32:50.845 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.enable_blob_files: false 2024-03-19T01:32:50.845 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.min_blob_size: 0 2024-03-19T01:32:50.845 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.blob_file_size: 268435456 2024-03-19T01:32:50.846 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.blob_compression_type: NoCompression 2024-03-19T01:32:50.846 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.enable_blob_garbage_collection: false 2024-03-19T01:32:50.846 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-03-19T01:32:50.846 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-03-19T01:32:50.846 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-03-19T01:32:50.846 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.blob_file_starting_level: 0 2024-03-19T01:32:50.846 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-03-19T01:32:50.846 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-03-19T01:32:50.846 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 0 2024-03-19T01:32:50.846 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 5baf83b0-3dc3-4626-943f-5884d3228951 2024-03-19T01:32:50.846 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: EVENT_LOG_v1 {"time_micros": 1710811970801803, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-03-19T01:32:50.846 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #4 mode 2 2024-03-19T01:32:50.846 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: EVENT_LOG_v1 {"time_micros": 1710811970802212, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1622, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 1, "largest_seqno": 5, "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_filter_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1710811970, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "5baf83b0-3dc3-4626-943f-5884d3228951", "db_session_id": "ELKICXDD1Y309IUY6F8Z", "orig_file_number": 8, "seqno_to_time_mapping": "N/A"}} 2024-03-19T01:32:50.846 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: EVENT_LOG_v1 {"time_micros": 1710811970802289, "job": 1, "event": "recovery_finished"} 2024-03-19T01:32:50.847 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: [db/version_set.cc:5047] Creating manifest 10 2024-03-19T01:32:50.847 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-03-19T01:32:50.847 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5644ad3fe000 2024-03-19T01:32:50.847 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: DB pointer 0x5644ad3e8000 2024-03-19T01:32:50.847 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-03-19T01:32:50.847 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-03-19T01:32:50.847 INFO:journalctl@ceph.mon.c.smithi115.stdout: ** DB Stats ** 2024-03-19T01:32:50.847 INFO:journalctl@ceph.mon.c.smithi115.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-19T01:32:50.847 INFO:journalctl@ceph.mon.c.smithi115.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-03-19T01:32:50.847 INFO:journalctl@ceph.mon.c.smithi115.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-19T01:32:50.847 INFO:journalctl@ceph.mon.c.smithi115.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-19T01:32:50.847 INFO:journalctl@ceph.mon.c.smithi115.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-03-19T01:32:50.847 INFO:journalctl@ceph.mon.c.smithi115.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-19T01:32:50.847 INFO:journalctl@ceph.mon.c.smithi115.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-19T01:32:50.847 INFO:journalctl@ceph.mon.c.smithi115.stdout: 2024-03-19T01:32:50.848 INFO:journalctl@ceph.mon.c.smithi115.stdout: ** Compaction Stats [default] ** 2024-03-19T01:32:50.848 INFO:journalctl@ceph.mon.c.smithi115.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-19T01:32:50.848 INFO:journalctl@ceph.mon.c.smithi115.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-03-19T01:32:50.848 INFO:journalctl@ceph.mon.c.smithi115.stdout: L0 1/0 1.58 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.0 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-03-19T01:32:50.848 INFO:journalctl@ceph.mon.c.smithi115.stdout: Sum 1/0 1.58 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.0 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-03-19T01:32:50.848 INFO:journalctl@ceph.mon.c.smithi115.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.0 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-03-19T01:32:50.848 INFO:journalctl@ceph.mon.c.smithi115.stdout: 2024-03-19T01:32:50.848 INFO:journalctl@ceph.mon.c.smithi115.stdout: ** Compaction Stats [default] ** 2024-03-19T01:32:50.848 INFO:journalctl@ceph.mon.c.smithi115.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-19T01:32:50.848 INFO:journalctl@ceph.mon.c.smithi115.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-19T01:32:50.848 INFO:journalctl@ceph.mon.c.smithi115.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.0 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-03-19T01:32:50.848 INFO:journalctl@ceph.mon.c.smithi115.stdout: 2024-03-19T01:32:50.848 INFO:journalctl@ceph.mon.c.smithi115.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-03-19T01:32:50.848 INFO:journalctl@ceph.mon.c.smithi115.stdout: 2024-03-19T01:32:50.849 INFO:journalctl@ceph.mon.c.smithi115.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-19T01:32:50.849 INFO:journalctl@ceph.mon.c.smithi115.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-03-19T01:32:50.849 INFO:journalctl@ceph.mon.c.smithi115.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-03-19T01:32:50.849 INFO:journalctl@ceph.mon.c.smithi115.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-03-19T01:32:50.849 INFO:journalctl@ceph.mon.c.smithi115.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-03-19T01:32:50.849 INFO:journalctl@ceph.mon.c.smithi115.stdout: AddFile(Keys): cumulative 0, interval 0 2024-03-19T01:32:50.849 INFO:journalctl@ceph.mon.c.smithi115.stdout: Cumulative compaction: 0.00 GB write, 0.66 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-19T01:32:50.849 INFO:journalctl@ceph.mon.c.smithi115.stdout: Interval compaction: 0.00 GB write, 0.66 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-19T01:32:50.849 INFO:journalctl@ceph.mon.c.smithi115.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-03-19T01:32:50.849 INFO:journalctl@ceph.mon.c.smithi115.stdout: Block cache BinnedLRUCache@0x5644ad30b090#2 capacity: 512.00 MB usage: 0.22 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 9e-06 secs_since: 0 2024-03-19T01:32:50.849 INFO:journalctl@ceph.mon.c.smithi115.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.11 KB,2.08616e-05%) IndexBlock(1,0.11 KB,2.08616e-05%) Misc(1,0.00 KB,0%) 2024-03-19T01:32:50.849 INFO:journalctl@ceph.mon.c.smithi115.stdout: 2024-03-19T01:32:50.849 INFO:journalctl@ceph.mon.c.smithi115.stdout: ** File Read Latency Histogram By Level [default] ** 2024-03-19T01:32:50.849 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: mon.c does not exist in monmap, will attempt to join an existing cluster 2024-03-19T01:32:50.849 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: using public_addr v2:172.21.15.115:0/0 -> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] 2024-03-19T01:32:50.850 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: starting mon.c rank -1 at public addrs [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] at bind addrs [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] mon_data /var/lib/ceph/mon/ceph-c fsid 438fb69c-e590-11ee-95c9-87774f69a715 2024-03-19T01:32:50.850 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: mon.c@-1(???) e0 preinit fsid 438fb69c-e590-11ee-95c9-87774f69a715 2024-03-19T01:32:50.850 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: mon.c@-1(synchronizing).mds e1 new map 2024-03-19T01:32:50.850 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: mon.c@-1(synchronizing).mds e1 print_map 2024-03-19T01:32:50.850 INFO:journalctl@ceph.mon.c.smithi115.stdout: e1 2024-03-19T01:32:50.850 INFO:journalctl@ceph.mon.c.smithi115.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-03-19T01:32:50.850 INFO:journalctl@ceph.mon.c.smithi115.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2,11=minor log segments,12=quiesce subvolumes} 2024-03-19T01:32:50.850 INFO:journalctl@ceph.mon.c.smithi115.stdout: legacy client fscid: -1 2024-03-19T01:32:50.850 INFO:journalctl@ceph.mon.c.smithi115.stdout: 2024-03-19T01:32:50.850 INFO:journalctl@ceph.mon.c.smithi115.stdout: No filesystems configured 2024-03-19T01:32:50.850 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-03-19T01:32:50.850 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-03-19T01:32:50.850 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-03-19T01:32:50.850 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-03-19T01:32:50.851 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-03-19T01:32:50.851 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-03-19T01:32:50.851 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-03-19T01:32:50.851 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-03-19T01:32:50.851 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-03-19T01:32:50.851 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-03-19T01:32:50.851 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:50.851 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:32:50.851 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:50.851 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-19T01:32:50.851 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:32:50.851 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi115", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:32:50.851 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:50.851 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:50.851 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:50.852 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:50.852 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi112", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:32:50.852 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:32:50.852 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:32:50.852 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: Deploying cephadm binary to smithi115 2024-03-19T01:32:50.852 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: Updating smithi112:/etc/ceph/ceph.conf 2024-03-19T01:32:50.852 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: Updating smithi112:/var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/config/ceph.conf 2024-03-19T01:32:50.852 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:32:50.852 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:32:50.852 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: Updating smithi112:/etc/ceph/ceph.client.admin.keyring 2024-03-19T01:32:50.852 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:50.852 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:50.852 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:50.852 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:50.853 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: Updating smithi112:/var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/config/ceph.client.admin.keyring 2024-03-19T01:32:50.853 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: Added host smithi115 2024-03-19T01:32:50.853 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:32:50.853 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:50.853 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:32:50.853 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:50.853 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:50.853 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-19T01:32:50.853 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:32:50.853 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:32:50.853 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/4026667028' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-03-19T01:32:50.853 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:50.853 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:50.853 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:50.853 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:50.854 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi115", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:32:50.854 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:32:50.854 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:32:50.854 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: Updating smithi115:/etc/ceph/ceph.conf 2024-03-19T01:32:50.854 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: Updating smithi115:/var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/config/ceph.conf 2024-03-19T01:32:50.854 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/4026667028' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-03-19T01:32:50.854 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: osdmap e4: 0 total, 0 up, 0 in 2024-03-19T01:32:50.854 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: Updating smithi115:/etc/ceph/ceph.client.admin.keyring 2024-03-19T01:32:50.854 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:50.854 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:50.854 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:50.854 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: Updating smithi115:/var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/config/ceph.client.admin.keyring 2024-03-19T01:32:50.854 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:32:50.855 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: from='client.14186 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi099:172.21.15.99=a;smithi112:172.21.15.112=b;smithi115:172.21.15.115=c", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:32:50.855 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: Saving service mon spec with placement smithi099:172.21.15.99=a;smithi112:172.21.15.112=b;smithi115:172.21.15.115=c;count:3 2024-03-19T01:32:50.855 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:50.855 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:32:50.855 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:32:50.855 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:32:50.855 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:50.855 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-19T01:32:50.855 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:32:50.855 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: Deploying daemon mon.c on smithi115 2024-03-19T01:32:50.855 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:32:50.855 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:32:50.855 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 ceph-mon[67332]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-03-19T01:32:50.975 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:50.975+0000 7f481a3da700 1 -- 172.21.15.115:0/3795310894 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) -- 0x7f4814063940 con 0x7f4814100e50 2024-03-19T01:32:51.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 bash[67286]: 30f5aa3b44b18e0930bfcf3673fa632f7856352342cbabc3e17d7dddfc17505c 2024-03-19T01:32:51.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:50 smithi115 systemd[1]: Started Ceph mon.c for 438fb69c-e590-11ee-95c9-87774f69a715. 2024-03-19T01:32:55.873 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:55.873+0000 7f48117fa700 1 -- 172.21.15.115:0/3795310894 <== mon.0 v2:172.21.15.99:3300/0 8 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 2 v2) ==== 95+0+1015 (secure 0 0 0) 0x7f4804032020 con 0x7f4814100e50 2024-03-19T01:32:55.873 INFO:teuthology.orchestra.run.smithi115.stdout: 2024-03-19T01:32:55.873 INFO:teuthology.orchestra.run.smithi115.stdout:{"epoch":2,"fsid":"438fb69c-e590-11ee-95c9-87774f69a715","modified":"2024-03-19T01:32:50.824049Z","created":"2024-03-19T01:31:07.215663Z","min_mon_release":19,"min_mon_release_name":"squid","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy","reef","squid"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:3300","nonce":0},{"type":"v1","addr":"172.21.15.99:6789","nonce":0}]},"addr":"172.21.15.99:6789/0","public_addr":"172.21.15.99:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:3300","nonce":0},{"type":"v1","addr":"172.21.15.115:6789","nonce":0}]},"addr":"172.21.15.115:6789/0","public_addr":"172.21.15.115:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-03-19T01:32:55.876 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:55.876+0000 7f481a3da700 1 -- 172.21.15.115:0/3795310894 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f47fc039020 msgr2=0x7f47fc03b4e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:32:55.876 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:55.876+0000 7f481a3da700 1 --2- 172.21.15.115:0/3795310894 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f47fc039020 0x7f47fc03b4e0 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7f4808006fd0 tx=0x7f4808006e40 comp rx=0 tx=0).stop 2024-03-19T01:32:55.876 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:55.876+0000 7f481a3da700 1 -- 172.21.15.115:0/3795310894 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4814100e50 msgr2=0x7f481419c430 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:32:55.876 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:55.876+0000 7f481a3da700 1 --2- 172.21.15.115:0/3795310894 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4814100e50 0x7f481419c430 secure :-1 s=READY pgs=95 cs=0 l=1 rev1=1 crypto rx=0x7f4804005850 tx=0x7f4804005030 comp rx=0 tx=0).stop 2024-03-19T01:32:55.876 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:55.876+0000 7f481a3da700 1 -- 172.21.15.115:0/3795310894 shutdown_connections 2024-03-19T01:32:55.876 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:55.876+0000 7f481a3da700 1 --2- 172.21.15.115:0/3795310894 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f47fc039020 0x7f47fc03b4e0 unknown :-1 s=CLOSED pgs=20 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:32:55.876 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:55.876+0000 7f481a3da700 1 --2- 172.21.15.115:0/3795310894 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4814100e50 0x7f481419c430 unknown :-1 s=CLOSED pgs=95 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:32:55.876 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:55.876+0000 7f481a3da700 1 -- 172.21.15.115:0/3795310894 >> 172.21.15.115:0/3795310894 conn(0x7f48140fc700 msgr2=0x7f4814103c80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:32:55.877 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:55.876+0000 7f481a3da700 1 -- 172.21.15.115:0/3795310894 shutdown_connections 2024-03-19T01:32:55.877 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:55.876+0000 7f481a3da700 1 -- 172.21.15.115:0/3795310894 wait complete. 2024-03-19T01:32:55.877 INFO:teuthology.orchestra.run.smithi115.stderr:dumped monmap epoch 2 2024-03-19T01:32:56.179 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:55 smithi115 ceph-mon[67332]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:32:56.180 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:55 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-19T01:32:56.180 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:55 smithi115 ceph-mon[67332]: mon.a calling monitor election 2024-03-19T01:32:56.180 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:55 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-19T01:32:56.180 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:55 smithi115 ceph-mon[67332]: from='client.? 172.21.15.115:0/3795310894' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-19T01:32:56.180 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:55 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-19T01:32:56.180 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:55 smithi115 ceph-mon[67332]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:32:56.180 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:55 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-19T01:32:56.180 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:55 smithi115 ceph-mon[67332]: mon.c calling monitor election 2024-03-19T01:32:56.180 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:55 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-19T01:32:56.180 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:55 smithi115 ceph-mon[67332]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:32:56.180 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:55 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-19T01:32:56.180 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:55 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-19T01:32:56.181 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:55 smithi115 ceph-mon[67332]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-03-19T01:32:56.181 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:55 smithi115 ceph-mon[67332]: monmap epoch 2 2024-03-19T01:32:56.181 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:55 smithi115 ceph-mon[67332]: fsid 438fb69c-e590-11ee-95c9-87774f69a715 2024-03-19T01:32:56.181 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:55 smithi115 ceph-mon[67332]: last_changed 2024-03-19T01:32:50.824049+0000 2024-03-19T01:32:56.181 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:55 smithi115 ceph-mon[67332]: created 2024-03-19T01:31:07.215663+0000 2024-03-19T01:32:56.181 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:55 smithi115 ceph-mon[67332]: min_mon_release 19 (squid) 2024-03-19T01:32:56.181 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:55 smithi115 ceph-mon[67332]: election_strategy: 1 2024-03-19T01:32:56.181 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:55 smithi115 ceph-mon[67332]: 0: [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] mon.a 2024-03-19T01:32:56.181 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:55 smithi115 ceph-mon[67332]: 1: [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] mon.c 2024-03-19T01:32:56.181 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:55 smithi115 ceph-mon[67332]: fsmap 2024-03-19T01:32:56.181 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:55 smithi115 ceph-mon[67332]: osdmap e4: 0 total, 0 up, 0 in 2024-03-19T01:32:56.181 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:55 smithi115 ceph-mon[67332]: mgrmap e13: x(active, since 49s) 2024-03-19T01:32:56.181 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:55 smithi115 ceph-mon[67332]: overall HEALTH_OK 2024-03-19T01:32:56.181 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:55 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:56.182 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:55 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:56.182 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:55 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:56.182 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:55 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-19T01:32:56.182 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:55 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:32:56.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:55 smithi099 ceph-mon[62951]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:32:56.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:55 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-19T01:32:56.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:55 smithi099 ceph-mon[62951]: mon.a calling monitor election 2024-03-19T01:32:56.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:55 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-19T01:32:56.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:55 smithi099 ceph-mon[62951]: from='client.? 172.21.15.115:0/3795310894' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-19T01:32:56.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:55 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-19T01:32:56.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:55 smithi099 ceph-mon[62951]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:32:56.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:55 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-19T01:32:56.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:55 smithi099 ceph-mon[62951]: mon.c calling monitor election 2024-03-19T01:32:56.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:55 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-19T01:32:56.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:55 smithi099 ceph-mon[62951]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:32:56.343 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:55 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-19T01:32:56.343 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:55 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-19T01:32:56.343 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:55 smithi099 ceph-mon[62951]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-03-19T01:32:56.343 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:55 smithi099 ceph-mon[62951]: monmap epoch 2 2024-03-19T01:32:56.343 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:55 smithi099 ceph-mon[62951]: fsid 438fb69c-e590-11ee-95c9-87774f69a715 2024-03-19T01:32:56.343 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:55 smithi099 ceph-mon[62951]: last_changed 2024-03-19T01:32:50.824049+0000 2024-03-19T01:32:56.343 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:55 smithi099 ceph-mon[62951]: created 2024-03-19T01:31:07.215663+0000 2024-03-19T01:32:56.343 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:55 smithi099 ceph-mon[62951]: min_mon_release 19 (squid) 2024-03-19T01:32:56.343 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:55 smithi099 ceph-mon[62951]: election_strategy: 1 2024-03-19T01:32:56.343 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:55 smithi099 ceph-mon[62951]: 0: [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] mon.a 2024-03-19T01:32:56.343 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:55 smithi099 ceph-mon[62951]: 1: [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] mon.c 2024-03-19T01:32:56.343 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:55 smithi099 ceph-mon[62951]: fsmap 2024-03-19T01:32:56.343 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:55 smithi099 ceph-mon[62951]: osdmap e4: 0 total, 0 up, 0 in 2024-03-19T01:32:56.343 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:55 smithi099 ceph-mon[62951]: mgrmap e13: x(active, since 49s) 2024-03-19T01:32:56.344 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:55 smithi099 ceph-mon[62951]: overall HEALTH_OK 2024-03-19T01:32:56.344 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:55 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:56.344 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:55 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:56.344 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:55 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:56.344 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:55 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-19T01:32:56.344 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:55 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:32:57.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:56 smithi115 ceph-mon[67332]: Deploying daemon mon.b on smithi112 2024-03-19T01:32:57.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:56 smithi115 ceph-mon[67332]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:32:57.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:32:56 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-19T01:32:57.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:56 smithi099 ceph-mon[62951]: Deploying daemon mon.b on smithi112 2024-03-19T01:32:57.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:56 smithi099 ceph-mon[62951]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:32:57.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:32:56 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-19T01:32:57.498 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-03-19T01:32:57.498 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph mon dump -f json 2024-03-19T01:32:57.738 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.c/config 2024-03-19T01:32:59.306 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:59.304+0000 7f971d7b1700 1 -- 172.21.15.115:0/2666223962 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9718106e10 msgr2=0x7f97181071f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:32:59.306 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:59.304+0000 7f971d7b1700 1 --2- 172.21.15.115:0/2666223962 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9718106e10 0x7f97181071f0 secure :-1 s=READY pgs=98 cs=0 l=1 rev1=1 crypto rx=0x7f9708009910 tx=0x7f9708009c20 comp rx=0 tx=0).stop 2024-03-19T01:32:59.307 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:59.304+0000 7f971d7b1700 1 -- 172.21.15.115:0/2666223962 shutdown_connections 2024-03-19T01:32:59.307 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:59.304+0000 7f971d7b1700 1 --2- 172.21.15.115:0/2666223962 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9718106e10 0x7f97181071f0 unknown :-1 s=CLOSED pgs=98 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:32:59.307 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:59.304+0000 7f971d7b1700 1 -- 172.21.15.115:0/2666223962 >> 172.21.15.115:0/2666223962 conn(0x7f9718077400 msgr2=0x7f9718077810 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:32:59.307 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:59.304+0000 7f971d7b1700 1 -- 172.21.15.115:0/2666223962 shutdown_connections 2024-03-19T01:32:59.307 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:59.304+0000 7f971d7b1700 1 -- 172.21.15.115:0/2666223962 wait complete. 2024-03-19T01:32:59.307 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:59.304+0000 7f971d7b1700 1 Processor -- start 2024-03-19T01:32:59.307 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:59.305+0000 7f971d7b1700 1 -- start start 2024-03-19T01:32:59.307 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:59.305+0000 7f971d7b1700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9718106e10 0x7f9718198980 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:32:59.307 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:59.305+0000 7f971d7b1700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9718198ec0 0x7f9718199340 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:32:59.307 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:59.305+0000 7f971d7b1700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f971810b340 con 0x7f9718198ec0 2024-03-19T01:32:59.307 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:59.305+0000 7f971d7b1700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f971810b1c0 con 0x7f9718106e10 2024-03-19T01:32:59.307 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:59.305+0000 7f9716ffd700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9718106e10 0x7f9718198980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:32:59.307 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:59.305+0000 7f9716ffd700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9718106e10 0x7f9718198980 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.115:34952/0 (socket says 172.21.15.115:34952) 2024-03-19T01:32:59.307 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:59.305+0000 7f9716ffd700 1 -- 172.21.15.115:0/1731484822 learned_addr learned my addr 172.21.15.115:0/1731484822 (peer_addr_for_me v2:172.21.15.115:0/0) 2024-03-19T01:32:59.308 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:59.305+0000 7f97167fc700 1 --2- 172.21.15.115:0/1731484822 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9718198ec0 0x7f9718199340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:32:59.308 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:59.305+0000 7f9716ffd700 1 -- 172.21.15.115:0/1731484822 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9718106e10 msgr2=0x7f9718198980 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 12 2024-03-19T01:32:59.308 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:59.305+0000 7f9716ffd700 1 -- 172.21.15.115:0/1731484822 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9718106e10 msgr2=0x7f9718198980 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2024-03-19T01:32:59.308 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:59.305+0000 7f9716ffd700 1 --2- 172.21.15.115:0/1731484822 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9718106e10 0x7f9718198980 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 2024-03-19T01:32:59.308 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:59.305+0000 7f9716ffd700 1 --2- 172.21.15.115:0/1731484822 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9718106e10 0x7f9718198980 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2024-03-19T01:32:59.308 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:59.305+0000 7f97167fc700 1 -- 172.21.15.115:0/1731484822 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9718106e10 msgr2=0x7f9718198980 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-03-19T01:32:59.308 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:59.305+0000 7f97167fc700 1 --2- 172.21.15.115:0/1731484822 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9718106e10 0x7f9718198980 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:32:59.308 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:59.305+0000 7f97167fc700 1 -- 172.21.15.115:0/1731484822 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f97181a19f0 con 0x7f9718198ec0 2024-03-19T01:32:59.308 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:59.305+0000 7f97167fc700 1 --2- 172.21.15.115:0/1731484822 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9718198ec0 0x7f9718199340 secure :-1 s=READY pgs=99 cs=0 l=1 rev1=1 crypto rx=0x7f970c00b4d0 tx=0x7f970c00b890 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:32:59.308 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:59.306+0000 7f96fffff700 1 -- 172.21.15.115:0/1731484822 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 309+0+0 (secure 0 0 0) 0x7f970c011630 con 0x7f9718198ec0 2024-03-19T01:32:59.308 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:59.306+0000 7f971d7b1700 1 -- 172.21.15.115:0/1731484822 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f97181a1bc0 con 0x7f9718198ec0 2024-03-19T01:32:59.308 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:59.306+0000 7f96fffff700 1 -- 172.21.15.115:0/1731484822 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f970c011cb0 con 0x7f9718198ec0 2024-03-19T01:32:59.308 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:59.306+0000 7f96fffff700 1 -- 172.21.15.115:0/1731484822 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 309+0+0 (secure 0 0 0) 0x7f970c010e60 con 0x7f9718198ec0 2024-03-19T01:32:59.308 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:59.306+0000 7f971d7b1700 1 -- 172.21.15.115:0/1731484822 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f97181a2150 con 0x7f9718198ec0 2024-03-19T01:32:59.309 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:59.307+0000 7f971d7b1700 1 -- 172.21.15.115:0/1731484822 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f97181024f0 con 0x7f9718198ec0 2024-03-19T01:32:59.309 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:59.307+0000 7f96fffff700 1 -- 172.21.15.115:0/1731484822 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 13) ==== 46162+0+0 (secure 0 0 0) 0x7f970c018020 con 0x7f9718198ec0 2024-03-19T01:32:59.309 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:59.307+0000 7f96fffff700 1 --2- 172.21.15.115:0/1731484822 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f9700038d00 0x7f970003b1c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:32:59.309 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:59.307+0000 7f96fffff700 1 -- 172.21.15.115:0/1731484822 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(4..4 src has 1..4) ==== 1155+0+0 (secure 0 0 0) 0x7f970c04b980 con 0x7f9718198ec0 2024-03-19T01:32:59.310 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:59.307+0000 7f9716ffd700 1 --2- 172.21.15.115:0/1731484822 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f9700038d00 0x7f970003b1c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:32:59.310 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:59.308+0000 7f9716ffd700 1 --2- 172.21.15.115:0/1731484822 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f9700038d00 0x7f970003b1c0 secure :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0x7f970800b5c0 tx=0x7f9708009650 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:32:59.312 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:59.312+0000 7f96fffff700 1 -- 172.21.15.115:0/1731484822 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f970c01f020 con 0x7f9718198ec0 2024-03-19T01:32:59.558 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: mon.b@-1(synchronizing).mds e1 new map 2024-03-19T01:32:59.558 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: mon.b@-1(synchronizing).mds e1 print_map 2024-03-19T01:32:59.558 INFO:journalctl@ceph.mon.b.smithi112.stdout: e1 2024-03-19T01:32:59.558 INFO:journalctl@ceph.mon.b.smithi112.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-03-19T01:32:59.558 INFO:journalctl@ceph.mon.b.smithi112.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2,11=minor log segments,12=quiesce subvolumes} 2024-03-19T01:32:59.558 INFO:journalctl@ceph.mon.b.smithi112.stdout: legacy client fscid: -1 2024-03-19T01:32:59.558 INFO:journalctl@ceph.mon.b.smithi112.stdout: 2024-03-19T01:32:59.559 INFO:journalctl@ceph.mon.b.smithi112.stdout: No filesystems configured 2024-03-19T01:32:59.559 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-03-19T01:32:59.559 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-03-19T01:32:59.563 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:59.563+0000 7f96fffff700 1 -- 172.21.15.115:0/1731484822 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f970c00fa20 con 0x7f9718198ec0 2024-03-19T01:32:59.589 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:32:59.589+0000 7f971d7b1700 1 -- 172.21.15.115:0/1731484822 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) -- 0x7f97181a2430 con 0x7f9718198ec0 2024-03-19T01:32:59.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-03-19T01:32:59.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-03-19T01:32:59.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-03-19T01:32:59.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-03-19T01:32:59.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-03-19T01:32:59.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-03-19T01:32:59.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-03-19T01:32:59.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-03-19T01:32:59.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:59.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:32:59.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:59.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-19T01:32:59.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:32:59.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi115", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:32:59.864 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:59.864 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:59.864 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:59.864 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:59.864 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi112", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:32:59.864 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:32:59.864 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:32:59.864 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: Deploying cephadm binary to smithi115 2024-03-19T01:32:59.864 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: Updating smithi112:/etc/ceph/ceph.conf 2024-03-19T01:32:59.864 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: Updating smithi112:/var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/config/ceph.conf 2024-03-19T01:32:59.864 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:32:59.864 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:32:59.864 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: Updating smithi112:/etc/ceph/ceph.client.admin.keyring 2024-03-19T01:32:59.865 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:59.865 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:59.865 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:59.865 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:59.865 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: Updating smithi112:/var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/config/ceph.client.admin.keyring 2024-03-19T01:32:59.865 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: Added host smithi115 2024-03-19T01:32:59.865 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:32:59.865 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:59.865 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:32:59.865 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:59.865 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:59.865 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-19T01:32:59.865 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:32:59.865 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:32:59.865 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/4026667028' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-03-19T01:32:59.866 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:59.866 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:59.866 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:59.866 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:59.866 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi115", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:32:59.866 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:32:59.866 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:32:59.866 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: Updating smithi115:/etc/ceph/ceph.conf 2024-03-19T01:32:59.866 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: Updating smithi115:/var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/config/ceph.conf 2024-03-19T01:32:59.866 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/4026667028' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-03-19T01:32:59.866 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: osdmap e4: 0 total, 0 up, 0 in 2024-03-19T01:32:59.866 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: Updating smithi115:/etc/ceph/ceph.client.admin.keyring 2024-03-19T01:32:59.866 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:59.867 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:59.867 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:59.867 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: Updating smithi115:/var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/config/ceph.client.admin.keyring 2024-03-19T01:32:59.867 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:32:59.867 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: from='client.14186 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi099:172.21.15.99=a;smithi112:172.21.15.112=b;smithi115:172.21.15.115=c", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:32:59.867 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: Saving service mon spec with placement smithi099:172.21.15.99=a;smithi112:172.21.15.112=b;smithi115:172.21.15.115=c;count:3 2024-03-19T01:32:59.867 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:59.867 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:32:59.867 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:32:59.867 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:32:59.867 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:59.868 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-19T01:32:59.868 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:32:59.868 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: Deploying daemon mon.c on smithi115 2024-03-19T01:32:59.868 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:32:59.868 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:32:59.868 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:32:59.868 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-19T01:32:59.868 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: mon.a calling monitor election 2024-03-19T01:32:59.868 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-19T01:32:59.869 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: from='client.? 172.21.15.115:0/3795310894' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-19T01:32:59.869 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-19T01:32:59.869 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:32:59.869 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-19T01:32:59.869 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: mon.c calling monitor election 2024-03-19T01:32:59.869 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-19T01:32:59.869 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:32:59.869 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-19T01:32:59.869 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-19T01:32:59.869 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-03-19T01:32:59.869 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: monmap epoch 2 2024-03-19T01:32:59.869 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: fsid 438fb69c-e590-11ee-95c9-87774f69a715 2024-03-19T01:32:59.870 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: last_changed 2024-03-19T01:32:50.824049+0000 2024-03-19T01:32:59.870 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: created 2024-03-19T01:31:07.215663+0000 2024-03-19T01:32:59.870 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: min_mon_release 19 (squid) 2024-03-19T01:32:59.870 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: election_strategy: 1 2024-03-19T01:32:59.870 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: 0: [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] mon.a 2024-03-19T01:32:59.870 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: 1: [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] mon.c 2024-03-19T01:32:59.870 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: fsmap 2024-03-19T01:32:59.870 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: osdmap e4: 0 total, 0 up, 0 in 2024-03-19T01:32:59.871 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: mgrmap e13: x(active, since 49s) 2024-03-19T01:32:59.871 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: overall HEALTH_OK 2024-03-19T01:32:59.871 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:59.871 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:59.871 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:32:59.871 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-19T01:32:59.871 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:32:59.871 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: Deploying daemon mon.b on smithi112 2024-03-19T01:32:59.871 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:32:59.871 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-19T01:32:59.872 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 ceph-mon[66819]: mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-03-19T01:32:59.872 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 bash[66800]: d293a73943f87f7340eabc303ef0ff99db8bc08a92f0b8105844ae628f2ac93e 2024-03-19T01:32:59.872 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:32:59 smithi112 systemd[1]: Started Ceph mon.b for 438fb69c-e590-11ee-95c9-87774f69a715. 2024-03-19T01:33:04.596 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:04.596+0000 7f96fffff700 1 -- 172.21.15.115:0/1731484822 <== mon.0 v2:172.21.15.99:3300/0 8 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 3 v3) ==== 95+0+1283 (secure 0 0 0) 0x7f970c00f2e0 con 0x7f9718198ec0 2024-03-19T01:33:04.597 INFO:teuthology.orchestra.run.smithi115.stdout: 2024-03-19T01:33:04.597 INFO:teuthology.orchestra.run.smithi115.stdout:{"epoch":3,"fsid":"438fb69c-e590-11ee-95c9-87774f69a715","modified":"2024-03-19T01:32:59.563140Z","created":"2024-03-19T01:31:07.215663Z","min_mon_release":19,"min_mon_release_name":"squid","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy","reef","squid"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:3300","nonce":0},{"type":"v1","addr":"172.21.15.99:6789","nonce":0}]},"addr":"172.21.15.99:6789/0","public_addr":"172.21.15.99:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:3300","nonce":0},{"type":"v1","addr":"172.21.15.115:6789","nonce":0}]},"addr":"172.21.15.115:6789/0","public_addr":"172.21.15.115:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:3300","nonce":0},{"type":"v1","addr":"172.21.15.112:6789","nonce":0}]},"addr":"172.21.15.112:6789/0","public_addr":"172.21.15.112:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-03-19T01:33:04.599 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:04.599+0000 7f971d7b1700 1 -- 172.21.15.115:0/1731484822 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f9700038d00 msgr2=0x7f970003b1c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:33:04.599 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:04.599+0000 7f971d7b1700 1 --2- 172.21.15.115:0/1731484822 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f9700038d00 0x7f970003b1c0 secure :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0x7f970800b5c0 tx=0x7f9708009650 comp rx=0 tx=0).stop 2024-03-19T01:33:04.599 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:04.599+0000 7f971d7b1700 1 -- 172.21.15.115:0/1731484822 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9718198ec0 msgr2=0x7f9718199340 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:33:04.599 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:04.599+0000 7f971d7b1700 1 --2- 172.21.15.115:0/1731484822 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9718198ec0 0x7f9718199340 secure :-1 s=READY pgs=99 cs=0 l=1 rev1=1 crypto rx=0x7f970c00b4d0 tx=0x7f970c00b890 comp rx=0 tx=0).stop 2024-03-19T01:33:04.600 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:04.599+0000 7f971d7b1700 1 -- 172.21.15.115:0/1731484822 shutdown_connections 2024-03-19T01:33:04.600 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:04.599+0000 7f971d7b1700 1 --2- 172.21.15.115:0/1731484822 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f9700038d00 0x7f970003b1c0 unknown :-1 s=CLOSED pgs=29 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:33:04.600 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:04.599+0000 7f971d7b1700 1 --2- 172.21.15.115:0/1731484822 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9718106e10 0x7f9718198980 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:33:04.600 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:04.599+0000 7f971d7b1700 1 --2- 172.21.15.115:0/1731484822 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9718198ec0 0x7f9718199340 unknown :-1 s=CLOSED pgs=99 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:33:04.600 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:04.599+0000 7f971d7b1700 1 -- 172.21.15.115:0/1731484822 >> 172.21.15.115:0/1731484822 conn(0x7f9718077400 msgr2=0x7f97180fd910 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:33:04.600 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:04.599+0000 7f971d7b1700 1 -- 172.21.15.115:0/1731484822 shutdown_connections 2024-03-19T01:33:04.600 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:04.599+0000 7f971d7b1700 1 -- 172.21.15.115:0/1731484822 wait complete. 2024-03-19T01:33:04.600 INFO:teuthology.orchestra.run.smithi115.stderr:dumped monmap epoch 3 2024-03-19T01:33:04.999 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:04 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-19T01:33:04.999 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:04 smithi115 ceph-mon[67332]: mon.a calling monitor election 2024-03-19T01:33:04.999 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:04 smithi115 ceph-mon[67332]: mon.c calling monitor election 2024-03-19T01:33:05.000 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:04 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-19T01:33:05.000 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:04 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-19T01:33:05.000 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:04 smithi115 ceph-mon[67332]: from='client.? 172.21.15.115:0/1731484822' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-19T01:33:05.000 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:04 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-19T01:33:05.000 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:04 smithi115 ceph-mon[67332]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:05.000 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:04 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-19T01:33:05.000 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:04 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-19T01:33:05.000 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:04 smithi115 ceph-mon[67332]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:05.000 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:04 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-19T01:33:05.000 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:04 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-19T01:33:05.000 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:04 smithi115 ceph-mon[67332]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-03-19T01:33:05.000 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:04 smithi115 ceph-mon[67332]: monmap epoch 3 2024-03-19T01:33:05.001 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:04 smithi115 ceph-mon[67332]: fsid 438fb69c-e590-11ee-95c9-87774f69a715 2024-03-19T01:33:05.001 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:04 smithi115 ceph-mon[67332]: last_changed 2024-03-19T01:32:59.563140+0000 2024-03-19T01:33:05.001 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:04 smithi115 ceph-mon[67332]: created 2024-03-19T01:31:07.215663+0000 2024-03-19T01:33:05.001 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:04 smithi115 ceph-mon[67332]: min_mon_release 19 (squid) 2024-03-19T01:33:05.001 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:04 smithi115 ceph-mon[67332]: election_strategy: 1 2024-03-19T01:33:05.001 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:04 smithi115 ceph-mon[67332]: 0: [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] mon.a 2024-03-19T01:33:05.001 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:04 smithi115 ceph-mon[67332]: 1: [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] mon.c 2024-03-19T01:33:05.001 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:04 smithi115 ceph-mon[67332]: 2: [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] mon.b 2024-03-19T01:33:05.001 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:04 smithi115 ceph-mon[67332]: fsmap 2024-03-19T01:33:05.001 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:04 smithi115 ceph-mon[67332]: osdmap e4: 0 total, 0 up, 0 in 2024-03-19T01:33:05.001 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:04 smithi115 ceph-mon[67332]: mgrmap e13: x(active, since 57s) 2024-03-19T01:33:05.001 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:04 smithi115 ceph-mon[67332]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-03-19T01:33:05.002 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:04 smithi115 ceph-mon[67332]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,c 2024-03-19T01:33:05.002 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:04 smithi115 ceph-mon[67332]: [WRN] MON_DOWN: 1/3 mons down, quorum a,c 2024-03-19T01:33:05.002 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:04 smithi115 ceph-mon[67332]: mon.b (rank 2) addr [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] is down (out of quorum) 2024-03-19T01:33:05.002 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:04 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:05.002 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:04 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:05.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:04 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-19T01:33:05.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:04 smithi099 ceph-mon[62951]: mon.a calling monitor election 2024-03-19T01:33:05.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:04 smithi099 ceph-mon[62951]: mon.c calling monitor election 2024-03-19T01:33:05.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:04 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-19T01:33:05.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:04 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-19T01:33:05.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:04 smithi099 ceph-mon[62951]: from='client.? 172.21.15.115:0/1731484822' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-19T01:33:05.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:04 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-19T01:33:05.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:04 smithi099 ceph-mon[62951]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:05.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:04 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-19T01:33:05.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:04 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-19T01:33:05.093 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:04 smithi099 ceph-mon[62951]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:05.093 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:04 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-19T01:33:05.093 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:04 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-19T01:33:05.093 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:04 smithi099 ceph-mon[62951]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-03-19T01:33:05.093 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:04 smithi099 ceph-mon[62951]: monmap epoch 3 2024-03-19T01:33:05.093 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:04 smithi099 ceph-mon[62951]: fsid 438fb69c-e590-11ee-95c9-87774f69a715 2024-03-19T01:33:05.093 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:04 smithi099 ceph-mon[62951]: last_changed 2024-03-19T01:32:59.563140+0000 2024-03-19T01:33:05.093 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:04 smithi099 ceph-mon[62951]: created 2024-03-19T01:31:07.215663+0000 2024-03-19T01:33:05.093 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:04 smithi099 ceph-mon[62951]: min_mon_release 19 (squid) 2024-03-19T01:33:05.093 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:04 smithi099 ceph-mon[62951]: election_strategy: 1 2024-03-19T01:33:05.093 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:04 smithi099 ceph-mon[62951]: 0: [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] mon.a 2024-03-19T01:33:05.093 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:04 smithi099 ceph-mon[62951]: 1: [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] mon.c 2024-03-19T01:33:05.093 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:04 smithi099 ceph-mon[62951]: 2: [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] mon.b 2024-03-19T01:33:05.093 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:04 smithi099 ceph-mon[62951]: fsmap 2024-03-19T01:33:05.094 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:04 smithi099 ceph-mon[62951]: osdmap e4: 0 total, 0 up, 0 in 2024-03-19T01:33:05.094 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:04 smithi099 ceph-mon[62951]: mgrmap e13: x(active, since 57s) 2024-03-19T01:33:05.094 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:04 smithi099 ceph-mon[62951]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-03-19T01:33:05.094 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:04 smithi099 ceph-mon[62951]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,c 2024-03-19T01:33:05.094 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:04 smithi099 ceph-mon[62951]: [WRN] MON_DOWN: 1/3 mons down, quorum a,c 2024-03-19T01:33:05.094 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:04 smithi099 ceph-mon[62951]: mon.b (rank 2) addr [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] is down (out of quorum) 2024-03-19T01:33:05.094 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:04 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:05.094 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:04 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:05.296 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-03-19T01:33:05.297 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph config generate-minimal-conf 2024-03-19T01:33:05.547 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:33:05.998 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:05 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:05.998 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:05 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:05.998 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:05 smithi099 ceph-mon[62951]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:05.998 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:05 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:33:05.999 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:05 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-19T01:33:06.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:05 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:06.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:05 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:06.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:05 smithi115 ceph-mon[67332]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:06.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:05 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:33:06.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:05 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-19T01:33:06.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:06 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-19T01:33:06.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:06 smithi112 ceph-mon[66819]: mon.a calling monitor election 2024-03-19T01:33:06.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:06 smithi112 ceph-mon[66819]: mon.c calling monitor election 2024-03-19T01:33:06.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:06 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-19T01:33:06.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:06 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-19T01:33:06.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:06 smithi112 ceph-mon[66819]: from='client.? 172.21.15.115:0/1731484822' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-19T01:33:06.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:06 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-19T01:33:06.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:06 smithi112 ceph-mon[66819]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:06.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:06 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-19T01:33:06.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:06 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-19T01:33:06.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:06 smithi112 ceph-mon[66819]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:06.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:06 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-19T01:33:06.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:06 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-19T01:33:06.864 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:06 smithi112 ceph-mon[66819]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-03-19T01:33:06.864 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:06 smithi112 ceph-mon[66819]: monmap epoch 3 2024-03-19T01:33:06.864 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:06 smithi112 ceph-mon[66819]: fsid 438fb69c-e590-11ee-95c9-87774f69a715 2024-03-19T01:33:06.864 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:06 smithi112 ceph-mon[66819]: last_changed 2024-03-19T01:32:59.563140+0000 2024-03-19T01:33:06.864 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:06 smithi112 ceph-mon[66819]: created 2024-03-19T01:31:07.215663+0000 2024-03-19T01:33:06.864 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:06 smithi112 ceph-mon[66819]: min_mon_release 19 (squid) 2024-03-19T01:33:06.864 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:06 smithi112 ceph-mon[66819]: election_strategy: 1 2024-03-19T01:33:06.864 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:06 smithi112 ceph-mon[66819]: 0: [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] mon.a 2024-03-19T01:33:06.864 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:06 smithi112 ceph-mon[66819]: 1: [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] mon.c 2024-03-19T01:33:06.864 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:06 smithi112 ceph-mon[66819]: 2: [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] mon.b 2024-03-19T01:33:06.864 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:06 smithi112 ceph-mon[66819]: fsmap 2024-03-19T01:33:06.864 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:06 smithi112 ceph-mon[66819]: osdmap e4: 0 total, 0 up, 0 in 2024-03-19T01:33:06.865 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:06 smithi112 ceph-mon[66819]: mgrmap e13: x(active, since 57s) 2024-03-19T01:33:06.865 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:06 smithi112 ceph-mon[66819]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-03-19T01:33:06.865 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:06 smithi112 ceph-mon[66819]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,c 2024-03-19T01:33:06.865 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:06 smithi112 ceph-mon[66819]: [WRN] MON_DOWN: 1/3 mons down, quorum a,c 2024-03-19T01:33:06.865 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:06 smithi112 ceph-mon[66819]: mon.b (rank 2) addr [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] is down (out of quorum) 2024-03-19T01:33:06.865 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:06 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:06.865 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:06 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:06.865 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:06 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:06.865 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:06 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:06.865 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:06 smithi112 ceph-mon[66819]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:06.865 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:06 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:33:06.865 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:06 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-19T01:33:07.203 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:07.201+0000 7f36db2d1700 1 -- 172.21.15.99:0/3540943629 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f36d4100e30 msgr2=0x7f36d4101210 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:33:07.203 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:07.201+0000 7f36db2d1700 1 --2- 172.21.15.99:0/3540943629 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f36d4100e30 0x7f36d4101210 secure :-1 s=READY pgs=102 cs=0 l=1 rev1=1 crypto rx=0x7f36c4009910 tx=0x7f36c4009c20 comp rx=0 tx=0).stop 2024-03-19T01:33:07.203 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:07.201+0000 7f36db2d1700 1 -- 172.21.15.99:0/3540943629 shutdown_connections 2024-03-19T01:33:07.203 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:07.201+0000 7f36db2d1700 1 --2- 172.21.15.99:0/3540943629 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f36d4100e30 0x7f36d4101210 unknown :-1 s=CLOSED pgs=102 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:33:07.203 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:07.201+0000 7f36db2d1700 1 -- 172.21.15.99:0/3540943629 >> 172.21.15.99:0/3540943629 conn(0x7f36d40fc6c0 msgr2=0x7f36d40feae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:33:07.203 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:07.201+0000 7f36db2d1700 1 -- 172.21.15.99:0/3540943629 shutdown_connections 2024-03-19T01:33:07.203 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:07.201+0000 7f36db2d1700 1 -- 172.21.15.99:0/3540943629 wait complete. 2024-03-19T01:33:07.203 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:07.202+0000 7f36db2d1700 1 Processor -- start 2024-03-19T01:33:07.203 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:07.202+0000 7f36db2d1700 1 -- start start 2024-03-19T01:33:07.205 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:07.202+0000 7f36db2d1700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f36d4100e30 0x7f36d4198900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:33:07.205 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:07.202+0000 7f36db2d1700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f36d4198e40 0x7f36d41992c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:33:07.205 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:07.202+0000 7f36db2d1700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f36d41a1970 0x7f36d41a3d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:33:07.205 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:07.203+0000 7f36db2d1700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f36d410b1c0 con 0x7f36d4198e40 2024-03-19T01:33:07.205 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:07.203+0000 7f36db2d1700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f36d410b040 con 0x7f36d41a1970 2024-03-19T01:33:07.205 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:07.203+0000 7f36db2d1700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f36d410b340 con 0x7f36d4100e30 2024-03-19T01:33:07.206 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:07.203+0000 7f36d886c700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f36d4198e40 0x7f36d41992c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:33:07.206 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:07.203+0000 7f36d886c700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f36d4198e40 0x7f36d41992c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58790/0 (socket says 172.21.15.99:58790) 2024-03-19T01:33:07.206 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:07.203+0000 7f36d886c700 1 -- 172.21.15.99:0/710635343 learned_addr learned my addr 172.21.15.99:0/710635343 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:33:07.206 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:07.203+0000 7f36d906d700 1 --2- 172.21.15.99:0/710635343 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f36d4100e30 0x7f36d4198900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:33:07.206 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:07.203+0000 7f36d986e700 1 --2- 172.21.15.99:0/710635343 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f36d41a1970 0x7f36d41a3d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:33:07.206 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:07.203+0000 7f36d886c700 1 -- 172.21.15.99:0/710635343 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f36d4100e30 msgr2=0x7f36d4198900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:33:07.206 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:07.203+0000 7f36d886c700 1 --2- 172.21.15.99:0/710635343 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f36d4100e30 0x7f36d4198900 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:33:07.206 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:07.203+0000 7f36d886c700 1 -- 172.21.15.99:0/710635343 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f36d41a1970 msgr2=0x7f36d41a3d60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:33:07.206 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:07.203+0000 7f36d886c700 1 --2- 172.21.15.99:0/710635343 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f36d41a1970 0x7f36d41a3d60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:33:07.206 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:07.203+0000 7f36d886c700 1 -- 172.21.15.99:0/710635343 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f36d41a4570 con 0x7f36d4198e40 2024-03-19T01:33:07.206 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:07.203+0000 7f36d906d700 1 --2- 172.21.15.99:0/710635343 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f36d4100e30 0x7f36d4198900 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-19T01:33:07.206 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:07.203+0000 7f36d986e700 1 --2- 172.21.15.99:0/710635343 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f36d41a1970 0x7f36d41a3d60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-19T01:33:07.207 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:07.203+0000 7f36d886c700 1 --2- 172.21.15.99:0/710635343 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f36d4198e40 0x7f36d41992c0 secure :-1 s=READY pgs=103 cs=0 l=1 rev1=1 crypto rx=0x7f36d000b780 tx=0x7f36d000bb40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:33:07.207 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:07.203+0000 7f36ca7fc700 1 -- 172.21.15.99:0/710635343 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f36d0004230 con 0x7f36d4198e40 2024-03-19T01:33:07.207 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:07.203+0000 7f36ca7fc700 1 -- 172.21.15.99:0/710635343 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f36d00043d0 con 0x7f36d4198e40 2024-03-19T01:33:07.207 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:07.203+0000 7f36db2d1700 1 -- 172.21.15.99:0/710635343 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f36d41a4860 con 0x7f36d4198e40 2024-03-19T01:33:07.207 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:07.203+0000 7f36ca7fc700 1 -- 172.21.15.99:0/710635343 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f36d0011c60 con 0x7f36d4198e40 2024-03-19T01:33:07.207 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:07.204+0000 7f36db2d1700 1 -- 172.21.15.99:0/710635343 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f36d41a4df0 con 0x7f36d4198e40 2024-03-19T01:33:07.207 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:07.204+0000 7f36ca7fc700 1 -- 172.21.15.99:0/710635343 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 13) ==== 46162+0+0 (secure 0 0 0) 0x7f36d001a400 con 0x7f36d4198e40 2024-03-19T01:33:07.207 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:07.204+0000 7f36ca7fc700 1 --2- 172.21.15.99:0/710635343 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f36c0039450 0x7f36c003b910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:33:07.207 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:07.204+0000 7f36db2d1700 1 -- 172.21.15.99:0/710635343 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f36d4063940 con 0x7f36d4198e40 2024-03-19T01:33:07.207 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:07.205+0000 7f36ca7fc700 1 -- 172.21.15.99:0/710635343 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(4..4 src has 1..4) ==== 1155+0+0 (secure 0 0 0) 0x7f36d004bda0 con 0x7f36d4198e40 2024-03-19T01:33:07.208 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:07.205+0000 7f36d906d700 1 --2- 172.21.15.99:0/710635343 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f36c0039450 0x7f36c003b910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:33:07.208 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:07.205+0000 7f36d906d700 1 --2- 172.21.15.99:0/710635343 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f36c0039450 0x7f36c003b910 secure :-1 s=READY pgs=38 cs=0 l=1 rev1=1 crypto rx=0x7f36c4009750 tx=0x7f36c400b540 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:33:07.210 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:07.210+0000 7f36ca7fc700 1 -- 172.21.15.99:0/710635343 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f36d0003c80 con 0x7f36d4198e40 2024-03-19T01:33:07.414 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:07.413+0000 7f36db2d1700 1 -- 172.21.15.99:0/710635343 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "config generate-minimal-conf"} v 0) -- 0x7f36d41a5190 con 0x7f36d4198e40 2024-03-19T01:33:07.415 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:07.414+0000 7f36ca7fc700 1 -- 172.21.15.99:0/710635343 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "config generate-minimal-conf"}]=0 v9) ==== 76+0+275 (secure 0 0 0) 0x7f36d00193a0 con 0x7f36d4198e40 2024-03-19T01:33:07.415 INFO:teuthology.orchestra.run.smithi099.stdout:# minimal ceph.conf for 438fb69c-e590-11ee-95c9-87774f69a715 2024-03-19T01:33:07.415 INFO:teuthology.orchestra.run.smithi099.stdout:[global] 2024-03-19T01:33:07.415 INFO:teuthology.orchestra.run.smithi099.stdout: fsid = 438fb69c-e590-11ee-95c9-87774f69a715 2024-03-19T01:33:07.415 INFO:teuthology.orchestra.run.smithi099.stdout: mon_host = [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] 2024-03-19T01:33:07.416 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:07.416+0000 7f36db2d1700 1 -- 172.21.15.99:0/710635343 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f36c0039450 msgr2=0x7f36c003b910 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:33:07.416 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:07.416+0000 7f36db2d1700 1 --2- 172.21.15.99:0/710635343 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f36c0039450 0x7f36c003b910 secure :-1 s=READY pgs=38 cs=0 l=1 rev1=1 crypto rx=0x7f36c4009750 tx=0x7f36c400b540 comp rx=0 tx=0).stop 2024-03-19T01:33:07.416 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:07.416+0000 7f36db2d1700 1 -- 172.21.15.99:0/710635343 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f36d4198e40 msgr2=0x7f36d41992c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:33:07.416 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:07.416+0000 7f36db2d1700 1 --2- 172.21.15.99:0/710635343 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f36d4198e40 0x7f36d41992c0 secure :-1 s=READY pgs=103 cs=0 l=1 rev1=1 crypto rx=0x7f36d000b780 tx=0x7f36d000bb40 comp rx=0 tx=0).stop 2024-03-19T01:33:07.417 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:07.416+0000 7f36db2d1700 1 -- 172.21.15.99:0/710635343 shutdown_connections 2024-03-19T01:33:07.417 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:07.416+0000 7f36db2d1700 1 --2- 172.21.15.99:0/710635343 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f36c0039450 0x7f36c003b910 unknown :-1 s=CLOSED pgs=38 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:33:07.417 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:07.416+0000 7f36db2d1700 1 --2- 172.21.15.99:0/710635343 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f36d4100e30 0x7f36d4198900 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:33:07.417 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:07.416+0000 7f36db2d1700 1 --2- 172.21.15.99:0/710635343 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f36d41a1970 0x7f36d41a3d60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:33:07.417 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:07.416+0000 7f36db2d1700 1 --2- 172.21.15.99:0/710635343 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f36d4198e40 0x7f36d41992c0 unknown :-1 s=CLOSED pgs=103 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:33:07.417 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:07.416+0000 7f36db2d1700 1 -- 172.21.15.99:0/710635343 >> 172.21.15.99:0/710635343 conn(0x7f36d40fc6c0 msgr2=0x7f36d4102800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:33:07.417 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:07.416+0000 7f36db2d1700 1 -- 172.21.15.99:0/710635343 shutdown_connections 2024-03-19T01:33:07.417 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:07.416+0000 7f36db2d1700 1 -- 172.21.15.99:0/710635343 wait complete. 2024-03-19T01:33:07.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:07 smithi112 ceph-mon[66819]: mon.b calling monitor election 2024-03-19T01:33:07.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:07 smithi112 ceph-mon[66819]: Updating smithi099:/etc/ceph/ceph.conf 2024-03-19T01:33:07.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:07 smithi112 ceph-mon[66819]: Updating smithi112:/etc/ceph/ceph.conf 2024-03-19T01:33:07.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:07 smithi112 ceph-mon[66819]: Updating smithi115:/etc/ceph/ceph.conf 2024-03-19T01:33:07.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:07 smithi112 ceph-mon[66819]: Updating smithi112:/var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/config/ceph.conf 2024-03-19T01:33:07.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:07 smithi112 ceph-mon[66819]: Updating smithi099:/var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/config/ceph.conf 2024-03-19T01:33:07.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:07 smithi112 ceph-mon[66819]: Updating smithi115:/var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/config/ceph.conf 2024-03-19T01:33:07.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:07 smithi112 ceph-mon[66819]: mon.b calling monitor election 2024-03-19T01:33:07.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:07 smithi112 ceph-mon[66819]: mon.c calling monitor election 2024-03-19T01:33:07.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:07 smithi112 ceph-mon[66819]: mon.a calling monitor election 2024-03-19T01:33:07.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:07 smithi112 ceph-mon[66819]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-03-19T01:33:07.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:07 smithi112 ceph-mon[66819]: monmap epoch 3 2024-03-19T01:33:07.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:07 smithi112 ceph-mon[66819]: fsid 438fb69c-e590-11ee-95c9-87774f69a715 2024-03-19T01:33:07.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:07 smithi112 ceph-mon[66819]: last_changed 2024-03-19T01:32:59.563140+0000 2024-03-19T01:33:07.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:07 smithi112 ceph-mon[66819]: created 2024-03-19T01:31:07.215663+0000 2024-03-19T01:33:07.864 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:07 smithi112 ceph-mon[66819]: min_mon_release 19 (squid) 2024-03-19T01:33:07.864 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:07 smithi112 ceph-mon[66819]: election_strategy: 1 2024-03-19T01:33:07.864 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:07 smithi112 ceph-mon[66819]: 0: [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] mon.a 2024-03-19T01:33:07.864 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:07 smithi112 ceph-mon[66819]: 1: [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] mon.c 2024-03-19T01:33:07.864 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:07 smithi112 ceph-mon[66819]: 2: [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] mon.b 2024-03-19T01:33:07.864 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:07 smithi112 ceph-mon[66819]: fsmap 2024-03-19T01:33:07.864 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:07 smithi112 ceph-mon[66819]: osdmap e4: 0 total, 0 up, 0 in 2024-03-19T01:33:07.864 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:07 smithi112 ceph-mon[66819]: mgrmap e13: x(active, since 59s) 2024-03-19T01:33:07.864 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:07 smithi112 ceph-mon[66819]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-03-19T01:33:07.864 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:07 smithi112 ceph-mon[66819]: Cluster is now healthy 2024-03-19T01:33:07.864 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:07 smithi112 ceph-mon[66819]: overall HEALTH_OK 2024-03-19T01:33:07.864 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:07 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:07.864 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:07 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:07.864 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:07 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:07.865 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:07 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:07.865 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:07 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:07.865 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:07 smithi112 ceph-mon[66819]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:07.865 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:07 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:07.865 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:07 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:07.865 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:07 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:07.865 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:07 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:07.865 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:07 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:07.865 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:07 smithi112 ceph-mon[66819]: Reconfiguring mon.a (unknown last config time)... 2024-03-19T01:33:07.865 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:07 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-19T01:33:07.865 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:07 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-19T01:33:07.865 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:07 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:33:07.865 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:07 smithi112 ceph-mon[66819]: Reconfiguring daemon mon.a on smithi099 2024-03-19T01:33:07.865 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:07 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/710635343' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:33:07.865 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:07 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-19T01:33:07.868 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:07 smithi099 ceph-mon[62951]: mon.b calling monitor election 2024-03-19T01:33:07.868 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:07 smithi099 ceph-mon[62951]: Updating smithi099:/etc/ceph/ceph.conf 2024-03-19T01:33:07.868 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:07 smithi099 ceph-mon[62951]: Updating smithi112:/etc/ceph/ceph.conf 2024-03-19T01:33:07.868 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:07 smithi099 ceph-mon[62951]: Updating smithi115:/etc/ceph/ceph.conf 2024-03-19T01:33:07.868 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:07 smithi099 ceph-mon[62951]: Updating smithi112:/var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/config/ceph.conf 2024-03-19T01:33:07.868 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:07 smithi099 ceph-mon[62951]: Updating smithi099:/var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/config/ceph.conf 2024-03-19T01:33:07.868 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:07 smithi099 ceph-mon[62951]: Updating smithi115:/var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/config/ceph.conf 2024-03-19T01:33:07.868 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:07 smithi099 ceph-mon[62951]: mon.b calling monitor election 2024-03-19T01:33:07.868 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:07 smithi099 ceph-mon[62951]: mon.c calling monitor election 2024-03-19T01:33:07.868 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:07 smithi099 ceph-mon[62951]: mon.a calling monitor election 2024-03-19T01:33:07.868 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:07 smithi099 ceph-mon[62951]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-03-19T01:33:07.868 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:07 smithi099 ceph-mon[62951]: monmap epoch 3 2024-03-19T01:33:07.869 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:07 smithi099 ceph-mon[62951]: fsid 438fb69c-e590-11ee-95c9-87774f69a715 2024-03-19T01:33:07.869 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:07 smithi099 ceph-mon[62951]: last_changed 2024-03-19T01:32:59.563140+0000 2024-03-19T01:33:07.869 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:07 smithi099 ceph-mon[62951]: created 2024-03-19T01:31:07.215663+0000 2024-03-19T01:33:07.869 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:07 smithi099 ceph-mon[62951]: min_mon_release 19 (squid) 2024-03-19T01:33:07.869 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:07 smithi099 ceph-mon[62951]: election_strategy: 1 2024-03-19T01:33:07.869 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:07 smithi099 ceph-mon[62951]: 0: [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] mon.a 2024-03-19T01:33:07.869 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:07 smithi099 ceph-mon[62951]: 1: [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] mon.c 2024-03-19T01:33:07.869 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:07 smithi099 ceph-mon[62951]: 2: [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] mon.b 2024-03-19T01:33:07.869 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:07 smithi099 ceph-mon[62951]: fsmap 2024-03-19T01:33:07.869 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:07 smithi099 ceph-mon[62951]: osdmap e4: 0 total, 0 up, 0 in 2024-03-19T01:33:07.869 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:07 smithi099 ceph-mon[62951]: mgrmap e13: x(active, since 59s) 2024-03-19T01:33:07.869 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:07 smithi099 ceph-mon[62951]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-03-19T01:33:07.869 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:07 smithi099 ceph-mon[62951]: Cluster is now healthy 2024-03-19T01:33:07.870 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:07 smithi099 ceph-mon[62951]: overall HEALTH_OK 2024-03-19T01:33:07.870 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:07 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:07.870 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:07 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:07.870 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:07 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:07.870 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:07 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:07.870 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:07 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:07.870 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:07 smithi099 ceph-mon[62951]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:07.870 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:07 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:07.870 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:07 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:07.870 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:07 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:07.870 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:07 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:07.870 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:07 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:07.870 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:07 smithi099 ceph-mon[62951]: Reconfiguring mon.a (unknown last config time)... 2024-03-19T01:33:07.870 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:07 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-19T01:33:07.871 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:07 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-19T01:33:07.871 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:07 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:33:07.871 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:07 smithi099 ceph-mon[62951]: Reconfiguring daemon mon.a on smithi099 2024-03-19T01:33:07.871 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:07 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/710635343' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:33:07.871 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:07 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-19T01:33:08.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:07 smithi115 ceph-mon[67332]: mon.b calling monitor election 2024-03-19T01:33:08.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:07 smithi115 ceph-mon[67332]: Updating smithi099:/etc/ceph/ceph.conf 2024-03-19T01:33:08.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:07 smithi115 ceph-mon[67332]: Updating smithi112:/etc/ceph/ceph.conf 2024-03-19T01:33:08.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:07 smithi115 ceph-mon[67332]: Updating smithi115:/etc/ceph/ceph.conf 2024-03-19T01:33:08.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:07 smithi115 ceph-mon[67332]: Updating smithi112:/var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/config/ceph.conf 2024-03-19T01:33:08.058 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:07 smithi115 ceph-mon[67332]: Updating smithi099:/var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/config/ceph.conf 2024-03-19T01:33:08.058 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:07 smithi115 ceph-mon[67332]: Updating smithi115:/var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/config/ceph.conf 2024-03-19T01:33:08.058 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:07 smithi115 ceph-mon[67332]: mon.b calling monitor election 2024-03-19T01:33:08.058 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:07 smithi115 ceph-mon[67332]: mon.c calling monitor election 2024-03-19T01:33:08.058 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:07 smithi115 ceph-mon[67332]: mon.a calling monitor election 2024-03-19T01:33:08.058 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:07 smithi115 ceph-mon[67332]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-03-19T01:33:08.058 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:07 smithi115 ceph-mon[67332]: monmap epoch 3 2024-03-19T01:33:08.058 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:07 smithi115 ceph-mon[67332]: fsid 438fb69c-e590-11ee-95c9-87774f69a715 2024-03-19T01:33:08.058 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:07 smithi115 ceph-mon[67332]: last_changed 2024-03-19T01:32:59.563140+0000 2024-03-19T01:33:08.058 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:07 smithi115 ceph-mon[67332]: created 2024-03-19T01:31:07.215663+0000 2024-03-19T01:33:08.058 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:07 smithi115 ceph-mon[67332]: min_mon_release 19 (squid) 2024-03-19T01:33:08.058 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:07 smithi115 ceph-mon[67332]: election_strategy: 1 2024-03-19T01:33:08.058 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:07 smithi115 ceph-mon[67332]: 0: [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] mon.a 2024-03-19T01:33:08.058 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:07 smithi115 ceph-mon[67332]: 1: [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] mon.c 2024-03-19T01:33:08.059 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:07 smithi115 ceph-mon[67332]: 2: [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] mon.b 2024-03-19T01:33:08.059 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:07 smithi115 ceph-mon[67332]: fsmap 2024-03-19T01:33:08.059 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:07 smithi115 ceph-mon[67332]: osdmap e4: 0 total, 0 up, 0 in 2024-03-19T01:33:08.059 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:07 smithi115 ceph-mon[67332]: mgrmap e13: x(active, since 59s) 2024-03-19T01:33:08.059 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:07 smithi115 ceph-mon[67332]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-03-19T01:33:08.059 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:07 smithi115 ceph-mon[67332]: Cluster is now healthy 2024-03-19T01:33:08.059 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:07 smithi115 ceph-mon[67332]: overall HEALTH_OK 2024-03-19T01:33:08.059 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:07 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:08.059 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:07 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:08.059 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:07 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:08.059 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:07 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:08.059 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:07 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:08.059 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:07 smithi115 ceph-mon[67332]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:08.059 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:07 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:08.059 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:07 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:08.060 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:07 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:08.060 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:07 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:08.060 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:07 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:08.060 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:07 smithi115 ceph-mon[67332]: Reconfiguring mon.a (unknown last config time)... 2024-03-19T01:33:08.060 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:07 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-19T01:33:08.060 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:07 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-19T01:33:08.060 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:07 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:33:08.060 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:07 smithi115 ceph-mon[67332]: Reconfiguring daemon mon.a on smithi099 2024-03-19T01:33:08.060 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:07 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/710635343' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:33:08.060 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:07 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-19T01:33:08.478 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-03-19T01:33:08.478 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2024-03-19T01:33:08.478 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd of=/etc/ceph/ceph.conf 2024-03-19T01:33:08.507 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2024-03-19T01:33:08.507 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-03-19T01:33:08.574 DEBUG:teuthology.orchestra.run.smithi112:> set -ex 2024-03-19T01:33:08.574 DEBUG:teuthology.orchestra.run.smithi112:> sudo dd of=/etc/ceph/ceph.conf 2024-03-19T01:33:08.609 DEBUG:teuthology.orchestra.run.smithi112:> set -ex 2024-03-19T01:33:08.609 DEBUG:teuthology.orchestra.run.smithi112:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-03-19T01:33:08.678 DEBUG:teuthology.orchestra.run.smithi115:> set -ex 2024-03-19T01:33:08.678 DEBUG:teuthology.orchestra.run.smithi115:> sudo dd of=/etc/ceph/ceph.conf 2024-03-19T01:33:08.713 DEBUG:teuthology.orchestra.run.smithi115:> set -ex 2024-03-19T01:33:08.714 DEBUG:teuthology.orchestra.run.smithi115:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-03-19T01:33:08.784 INFO:tasks.cephadm:Adding mgr.x on smithi099 2024-03-19T01:33:08.785 INFO:tasks.cephadm:Adding mgr.y on smithi112 2024-03-19T01:33:08.785 INFO:tasks.cephadm:Adding mgr.z on smithi115 2024-03-19T01:33:08.785 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph orch apply mgr '3;smithi099=x;smithi112=y;smithi115=z' 2024-03-19T01:33:09.057 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.c/config 2024-03-19T01:33:10.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:09 smithi115 ceph-mon[67332]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:10.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:09 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:10.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:09 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:10.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:09 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-19T01:33:10.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:09 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-19T01:33:10.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:09 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:33:10.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:09 smithi099 ceph-mon[62951]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:10.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:09 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:10.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:09 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:10.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:09 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-19T01:33:10.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:09 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-19T01:33:10.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:09 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:33:10.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:09 smithi112 ceph-mon[66819]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:10.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:09 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:10.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:09 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:10.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:09 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-19T01:33:10.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:09 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-19T01:33:10.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:09 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:33:10.653 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:10.649+0000 7fbd81217700 1 -- 172.21.15.115:0/1969087379 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbd7c100d30 msgr2=0x7fbd7c101150 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:33:10.653 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:10.649+0000 7fbd81217700 1 --2- 172.21.15.115:0/1969087379 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbd7c100d30 0x7fbd7c101150 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 crypto rx=0x7fbd6c009910 tx=0x7fbd6c009c20 comp rx=0 tx=0).stop 2024-03-19T01:33:10.653 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:10.649+0000 7fbd81217700 1 -- 172.21.15.115:0/1969087379 shutdown_connections 2024-03-19T01:33:10.653 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:10.649+0000 7fbd81217700 1 --2- 172.21.15.115:0/1969087379 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbd7c100d30 0x7fbd7c101150 unknown :-1 s=CLOSED pgs=104 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:33:10.653 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:10.649+0000 7fbd81217700 1 -- 172.21.15.115:0/1969087379 >> 172.21.15.115:0/1969087379 conn(0x7fbd7c0fc2f0 msgr2=0x7fbd7c0fe710 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:33:10.654 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:10.649+0000 7fbd81217700 1 -- 172.21.15.115:0/1969087379 shutdown_connections 2024-03-19T01:33:10.654 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:10.649+0000 7fbd81217700 1 -- 172.21.15.115:0/1969087379 wait complete. 2024-03-19T01:33:10.654 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:10.649+0000 7fbd81217700 1 Processor -- start 2024-03-19T01:33:10.654 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:10.650+0000 7fbd81217700 1 -- start start 2024-03-19T01:33:10.654 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:10.650+0000 7fbd81217700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbd7c100d30 0x7fbd7c197690 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:33:10.654 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:10.650+0000 7fbd81217700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fbd7c197bd0 0x7fbd7c19cc40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:33:10.654 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:10.650+0000 7fbd81217700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fbd7c198050 0x7fbd7c19f1d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:33:10.654 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:10.650+0000 7fbd81217700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fbd7c106d70 con 0x7fbd7c100d30 2024-03-19T01:33:10.654 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:10.650+0000 7fbd81217700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fbd7c106bf0 con 0x7fbd7c197bd0 2024-03-19T01:33:10.655 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:10.650+0000 7fbd81217700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fbd7c106ef0 con 0x7fbd7c198050 2024-03-19T01:33:10.655 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:10.650+0000 7fbd7b59e700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fbd7c198050 0x7fbd7c19f1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:33:10.655 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:10.650+0000 7fbd7b59e700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fbd7c198050 0x7fbd7c19f1d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.115:54472/0 (socket says 172.21.15.115:54472) 2024-03-19T01:33:10.655 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:10.650+0000 7fbd7b59e700 1 -- 172.21.15.115:0/274469810 learned_addr learned my addr 172.21.15.115:0/274469810 (peer_addr_for_me v2:172.21.15.115:0/0) 2024-03-19T01:33:10.655 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:10.650+0000 7fbd7a59c700 1 --2- 172.21.15.115:0/274469810 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fbd7c197bd0 0x7fbd7c19cc40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:33:10.655 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:10.650+0000 7fbd7ad9d700 1 --2- 172.21.15.115:0/274469810 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbd7c100d30 0x7fbd7c197690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:33:10.655 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:10.650+0000 7fbd7b59e700 1 -- 172.21.15.115:0/274469810 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fbd7c197bd0 msgr2=0x7fbd7c19cc40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:33:10.655 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:10.650+0000 7fbd7b59e700 1 --2- 172.21.15.115:0/274469810 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fbd7c197bd0 0x7fbd7c19cc40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:33:10.655 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:10.650+0000 7fbd7b59e700 1 -- 172.21.15.115:0/274469810 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbd7c100d30 msgr2=0x7fbd7c197690 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:33:10.655 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:10.650+0000 7fbd7b59e700 1 --2- 172.21.15.115:0/274469810 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbd7c100d30 0x7fbd7c197690 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:33:10.655 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:10.650+0000 7fbd7b59e700 1 -- 172.21.15.115:0/274469810 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fbd7c0fc6d0 con 0x7fbd7c198050 2024-03-19T01:33:10.655 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:10.651+0000 7fbd7b59e700 1 --2- 172.21.15.115:0/274469810 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fbd7c198050 0x7fbd7c19f1d0 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7fbd7400ba50 tx=0x7fbd7400bd60 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:33:10.655 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:10.651+0000 7fbd63fff700 1 -- 172.21.15.115:0/274469810 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbd74019070 con 0x7fbd7c198050 2024-03-19T01:33:10.656 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:10.651+0000 7fbd81217700 1 -- 172.21.15.115:0/274469810 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fbd7c0fc9c0 con 0x7fbd7c198050 2024-03-19T01:33:10.656 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:10.651+0000 7fbd63fff700 1 -- 172.21.15.115:0/274469810 <== mon.1 v2:172.21.15.115:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fbd7400ce10 con 0x7fbd7c198050 2024-03-19T01:33:10.656 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:10.651+0000 7fbd63fff700 1 -- 172.21.15.115:0/274469810 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbd74007350 con 0x7fbd7c198050 2024-03-19T01:33:10.656 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:10.651+0000 7fbd81217700 1 -- 172.21.15.115:0/274469810 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fbd7c0fcf80 con 0x7fbd7c198050 2024-03-19T01:33:10.656 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:10.652+0000 7fbd63fff700 1 -- 172.21.15.115:0/274469810 <== mon.1 v2:172.21.15.115:3300/0 4 ==== mgrmap(e 13) ==== 46162+0+0 (secure 0 0 0) 0x7fbd7401c070 con 0x7fbd7c198050 2024-03-19T01:33:10.656 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:10.652+0000 7fbd81217700 1 -- 172.21.15.115:0/274469810 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fbd7c063940 con 0x7fbd7c198050 2024-03-19T01:33:10.656 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:10.652+0000 7fbd63fff700 1 --2- 172.21.15.115:0/274469810 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fbd64039450 0x7fbd6403b910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:33:10.656 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:10.652+0000 7fbd63fff700 1 -- 172.21.15.115:0/274469810 <== mon.1 v2:172.21.15.115:3300/0 5 ==== osd_map(4..4 src has 1..4) ==== 1155+0+0 (secure 0 0 0) 0x7fbd7404b650 con 0x7fbd7c198050 2024-03-19T01:33:10.656 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:10.652+0000 7fbd7ad9d700 1 --2- 172.21.15.115:0/274469810 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fbd64039450 0x7fbd6403b910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:33:10.657 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:10.653+0000 7fbd7ad9d700 1 --2- 172.21.15.115:0/274469810 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fbd64039450 0x7fbd6403b910 secure :-1 s=READY pgs=41 cs=0 l=1 rev1=1 crypto rx=0x7fbd6c005200 tx=0x7fbd6c01a040 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:33:10.662 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:10.662+0000 7fbd63fff700 1 -- 172.21.15.115:0/274469810 <== mon.1 v2:172.21.15.115:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7fbd7400f160 con 0x7fbd7c198050 2024-03-19T01:33:10.873 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:10.872+0000 7fbd81217700 1 -- 172.21.15.115:0/274469810 --> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] -- mgr_command(tid 0: {"prefix": "orch apply", "service_type": "mgr", "placement": "3;smithi099=x;smithi112=y;smithi115=z", "target": ["mon-mgr", ""]}) -- 0x7fbd7c190c10 con 0x7fbd64039450 2024-03-19T01:33:10.878 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:10.878+0000 7fbd63fff700 1 -- 172.21.15.115:0/274469810 <== mgr.14150 v2:172.21.15.99:6800/4272410295 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+24 (secure 0 0 0) 0x7fbd7c190c10 con 0x7fbd64039450 2024-03-19T01:33:10.879 INFO:teuthology.orchestra.run.smithi115.stdout:Scheduled mgr update... 2024-03-19T01:33:10.883 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:10.880+0000 7fbd81217700 1 -- 172.21.15.115:0/274469810 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fbd64039450 msgr2=0x7fbd6403b910 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:33:10.883 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:10.880+0000 7fbd81217700 1 --2- 172.21.15.115:0/274469810 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fbd64039450 0x7fbd6403b910 secure :-1 s=READY pgs=41 cs=0 l=1 rev1=1 crypto rx=0x7fbd6c005200 tx=0x7fbd6c01a040 comp rx=0 tx=0).stop 2024-03-19T01:33:10.883 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:10.880+0000 7fbd81217700 1 -- 172.21.15.115:0/274469810 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fbd7c198050 msgr2=0x7fbd7c19f1d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:33:10.884 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:10.880+0000 7fbd81217700 1 --2- 172.21.15.115:0/274469810 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fbd7c198050 0x7fbd7c19f1d0 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7fbd7400ba50 tx=0x7fbd7400bd60 comp rx=0 tx=0).stop 2024-03-19T01:33:10.884 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:10.880+0000 7fbd81217700 1 -- 172.21.15.115:0/274469810 shutdown_connections 2024-03-19T01:33:10.884 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:10.880+0000 7fbd81217700 1 --2- 172.21.15.115:0/274469810 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fbd64039450 0x7fbd6403b910 unknown :-1 s=CLOSED pgs=41 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:33:10.884 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:10.880+0000 7fbd81217700 1 --2- 172.21.15.115:0/274469810 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fbd7c198050 0x7fbd7c19f1d0 unknown :-1 s=CLOSED pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:33:10.884 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:10.880+0000 7fbd81217700 1 --2- 172.21.15.115:0/274469810 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbd7c100d30 0x7fbd7c197690 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:33:10.884 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:10.880+0000 7fbd81217700 1 --2- 172.21.15.115:0/274469810 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fbd7c197bd0 0x7fbd7c19cc40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:33:10.884 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:10.880+0000 7fbd81217700 1 -- 172.21.15.115:0/274469810 >> 172.21.15.115:0/274469810 conn(0x7fbd7c0fc2f0 msgr2=0x7fbd7c06a1a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:33:10.884 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:10.880+0000 7fbd81217700 1 -- 172.21.15.115:0/274469810 shutdown_connections 2024-03-19T01:33:10.884 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:33:10.880+0000 7fbd81217700 1 -- 172.21.15.115:0/274469810 wait complete. 2024-03-19T01:33:11.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:10 smithi115 ceph-mon[67332]: Reconfiguring mon.b (monmap changed)... 2024-03-19T01:33:11.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:10 smithi115 ceph-mon[67332]: Reconfiguring daemon mon.b on smithi112 2024-03-19T01:33:11.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:10 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:11.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:10 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:11.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:10 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-19T01:33:11.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:10 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-19T01:33:11.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:10 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:33:11.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:10 smithi099 ceph-mon[62951]: Reconfiguring mon.b (monmap changed)... 2024-03-19T01:33:11.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:10 smithi099 ceph-mon[62951]: Reconfiguring daemon mon.b on smithi112 2024-03-19T01:33:11.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:10 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:11.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:10 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:11.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:10 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-19T01:33:11.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:10 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-19T01:33:11.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:10 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:33:11.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:10 smithi112 ceph-mon[66819]: Reconfiguring mon.b (monmap changed)... 2024-03-19T01:33:11.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:10 smithi112 ceph-mon[66819]: Reconfiguring daemon mon.b on smithi112 2024-03-19T01:33:11.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:10 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:11.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:10 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:11.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:10 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-19T01:33:11.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:10 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-19T01:33:11.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:10 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:33:11.384 DEBUG:teuthology.orchestra.run.smithi112:mgr.y> sudo journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@mgr.y.service 2024-03-19T01:33:11.387 DEBUG:teuthology.orchestra.run.smithi115:mgr.z> sudo journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@mgr.z.service 2024-03-19T01:33:11.389 INFO:tasks.cephadm:Deploying OSDs... 2024-03-19T01:33:11.389 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2024-03-19T01:33:11.390 DEBUG:teuthology.orchestra.run.smithi099:> dd if=/scratch_devs of=/dev/stdout 2024-03-19T01:33:11.409 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-03-19T01:33:11.409 DEBUG:teuthology.orchestra.run.smithi099:> stat /dev/vg_nvme/lv_1 2024-03-19T01:33:11.467 INFO:teuthology.orchestra.run.smithi099.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-03-19T01:33:11.467 INFO:teuthology.orchestra.run.smithi099.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-19T01:33:11.467 INFO:teuthology.orchestra.run.smithi099.stdout:Device: 5h/5d Inode: 666 Links: 1 2024-03-19T01:33:11.467 INFO:teuthology.orchestra.run.smithi099.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-19T01:33:11.468 INFO:teuthology.orchestra.run.smithi099.stdout:Context: system_u:object_r:device_t:s0 2024-03-19T01:33:11.468 INFO:teuthology.orchestra.run.smithi099.stdout:Access: 2024-03-19 01:32:18.743962967 +0000 2024-03-19T01:33:11.468 INFO:teuthology.orchestra.run.smithi099.stdout:Modify: 2024-03-19 01:04:03.701902828 +0000 2024-03-19T01:33:11.468 INFO:teuthology.orchestra.run.smithi099.stdout:Change: 2024-03-19 01:04:03.701902828 +0000 2024-03-19T01:33:11.468 INFO:teuthology.orchestra.run.smithi099.stdout: Birth: 2024-03-19 01:04:03.701902828 +0000 2024-03-19T01:33:11.468 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-03-19T01:33:11.539 INFO:teuthology.orchestra.run.smithi099.stderr:1+0 records in 2024-03-19T01:33:11.539 INFO:teuthology.orchestra.run.smithi099.stderr:1+0 records out 2024-03-19T01:33:11.539 INFO:teuthology.orchestra.run.smithi099.stderr:512 bytes copied, 0.000234538 s, 2.2 MB/s 2024-03-19T01:33:11.540 DEBUG:teuthology.orchestra.run.smithi099:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-03-19T01:33:11.599 DEBUG:teuthology.orchestra.run.smithi099:> stat /dev/vg_nvme/lv_2 2024-03-19T01:33:11.656 INFO:teuthology.orchestra.run.smithi099.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-03-19T01:33:11.656 INFO:teuthology.orchestra.run.smithi099.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-19T01:33:11.656 INFO:teuthology.orchestra.run.smithi099.stdout:Device: 5h/5d Inode: 686 Links: 1 2024-03-19T01:33:11.656 INFO:teuthology.orchestra.run.smithi099.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-19T01:33:11.656 INFO:teuthology.orchestra.run.smithi099.stdout:Context: system_u:object_r:device_t:s0 2024-03-19T01:33:11.656 INFO:teuthology.orchestra.run.smithi099.stdout:Access: 2024-03-19 01:32:18.777962201 +0000 2024-03-19T01:33:11.656 INFO:teuthology.orchestra.run.smithi099.stdout:Modify: 2024-03-19 01:04:04.032895700 +0000 2024-03-19T01:33:11.656 INFO:teuthology.orchestra.run.smithi099.stdout:Change: 2024-03-19 01:04:04.032895700 +0000 2024-03-19T01:33:11.657 INFO:teuthology.orchestra.run.smithi099.stdout: Birth: 2024-03-19 01:04:04.032895700 +0000 2024-03-19T01:33:11.657 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-03-19T01:33:11.724 INFO:teuthology.orchestra.run.smithi099.stderr:1+0 records in 2024-03-19T01:33:11.724 INFO:teuthology.orchestra.run.smithi099.stderr:1+0 records out 2024-03-19T01:33:11.724 INFO:teuthology.orchestra.run.smithi099.stderr:512 bytes copied, 0.000227816 s, 2.2 MB/s 2024-03-19T01:33:11.725 DEBUG:teuthology.orchestra.run.smithi099:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-03-19T01:33:11.782 DEBUG:teuthology.orchestra.run.smithi099:> stat /dev/vg_nvme/lv_3 2024-03-19T01:33:11.838 INFO:teuthology.orchestra.run.smithi099.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-03-19T01:33:11.838 INFO:teuthology.orchestra.run.smithi099.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-19T01:33:11.839 INFO:teuthology.orchestra.run.smithi099.stdout:Device: 5h/5d Inode: 705 Links: 1 2024-03-19T01:33:11.839 INFO:teuthology.orchestra.run.smithi099.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-19T01:33:11.839 INFO:teuthology.orchestra.run.smithi099.stdout:Context: system_u:object_r:device_t:s0 2024-03-19T01:33:11.839 INFO:teuthology.orchestra.run.smithi099.stdout:Access: 2024-03-19 01:32:18.810961456 +0000 2024-03-19T01:33:11.839 INFO:teuthology.orchestra.run.smithi099.stdout:Modify: 2024-03-19 01:04:04.372888379 +0000 2024-03-19T01:33:11.839 INFO:teuthology.orchestra.run.smithi099.stdout:Change: 2024-03-19 01:04:04.372888379 +0000 2024-03-19T01:33:11.839 INFO:teuthology.orchestra.run.smithi099.stdout: Birth: 2024-03-19 01:04:04.372888379 +0000 2024-03-19T01:33:11.839 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-03-19T01:33:11.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:11 smithi099 ceph-mon[62951]: Reconfiguring mon.c (monmap changed)... 2024-03-19T01:33:11.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:11 smithi099 ceph-mon[62951]: Reconfiguring daemon mon.c on smithi115 2024-03-19T01:33:11.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:11 smithi099 ceph-mon[62951]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:11.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:11 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:11.867 INFO:teuthology.orchestra.run.smithi099.stderr:1+0 records in 2024-03-19T01:33:11.867 INFO:teuthology.orchestra.run.smithi099.stderr:1+0 records out 2024-03-19T01:33:11.867 INFO:teuthology.orchestra.run.smithi099.stderr:512 bytes copied, 0.000214667 s, 2.4 MB/s 2024-03-19T01:33:11.868 DEBUG:teuthology.orchestra.run.smithi099:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-03-19T01:33:11.910 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:11 smithi115 ceph-mon[67332]: Reconfiguring mon.c (monmap changed)... 2024-03-19T01:33:11.910 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:11 smithi115 ceph-mon[67332]: Reconfiguring daemon mon.c on smithi115 2024-03-19T01:33:11.910 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:11 smithi115 ceph-mon[67332]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:11.910 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:11 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:11.924 DEBUG:teuthology.orchestra.run.smithi099:> stat /dev/vg_nvme/lv_4 2024-03-19T01:33:11.979 INFO:teuthology.orchestra.run.smithi099.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-03-19T01:33:11.979 INFO:teuthology.orchestra.run.smithi099.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-19T01:33:11.979 INFO:teuthology.orchestra.run.smithi099.stdout:Device: 5h/5d Inode: 725 Links: 1 2024-03-19T01:33:11.979 INFO:teuthology.orchestra.run.smithi099.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-19T01:33:11.979 INFO:teuthology.orchestra.run.smithi099.stdout:Context: system_u:object_r:device_t:s0 2024-03-19T01:33:11.980 INFO:teuthology.orchestra.run.smithi099.stdout:Access: 2024-03-19 01:32:18.844960690 +0000 2024-03-19T01:33:11.980 INFO:teuthology.orchestra.run.smithi099.stdout:Modify: 2024-03-19 01:04:04.727880734 +0000 2024-03-19T01:33:11.980 INFO:teuthology.orchestra.run.smithi099.stdout:Change: 2024-03-19 01:04:04.727880734 +0000 2024-03-19T01:33:11.980 INFO:teuthology.orchestra.run.smithi099.stdout: Birth: 2024-03-19 01:04:04.727880734 +0000 2024-03-19T01:33:11.980 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-03-19T01:33:12.045 INFO:teuthology.orchestra.run.smithi099.stderr:1+0 records in 2024-03-19T01:33:12.045 INFO:teuthology.orchestra.run.smithi099.stderr:1+0 records out 2024-03-19T01:33:12.045 INFO:teuthology.orchestra.run.smithi099.stderr:512 bytes copied, 0.000196152 s, 2.6 MB/s 2024-03-19T01:33:12.047 DEBUG:teuthology.orchestra.run.smithi099:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-03-19T01:33:12.102 DEBUG:teuthology.orchestra.run.smithi112:> set -ex 2024-03-19T01:33:12.102 DEBUG:teuthology.orchestra.run.smithi112:> dd if=/scratch_devs of=/dev/stdout 2024-03-19T01:33:12.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:11 smithi112 ceph-mon[66819]: Reconfiguring mon.c (monmap changed)... 2024-03-19T01:33:12.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:11 smithi112 ceph-mon[66819]: Reconfiguring daemon mon.c on smithi115 2024-03-19T01:33:12.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:11 smithi112 ceph-mon[66819]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:12.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:11 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:12.123 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-03-19T01:33:12.123 DEBUG:teuthology.orchestra.run.smithi112:> stat /dev/vg_nvme/lv_1 2024-03-19T01:33:12.181 INFO:teuthology.orchestra.run.smithi112.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-03-19T01:33:12.181 INFO:teuthology.orchestra.run.smithi112.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-19T01:33:12.181 INFO:teuthology.orchestra.run.smithi112.stdout:Device: 5h/5d Inode: 663 Links: 1 2024-03-19T01:33:12.182 INFO:teuthology.orchestra.run.smithi112.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-19T01:33:12.182 INFO:teuthology.orchestra.run.smithi112.stdout:Context: system_u:object_r:device_t:s0 2024-03-19T01:33:12.182 INFO:teuthology.orchestra.run.smithi112.stdout:Access: 2024-03-19 01:32:59.013499204 +0000 2024-03-19T01:33:12.182 INFO:teuthology.orchestra.run.smithi112.stdout:Modify: 2024-03-19 01:12:59.705627512 +0000 2024-03-19T01:33:12.182 INFO:teuthology.orchestra.run.smithi112.stdout:Change: 2024-03-19 01:12:59.705627512 +0000 2024-03-19T01:33:12.182 INFO:teuthology.orchestra.run.smithi112.stdout: Birth: 2024-03-19 01:12:59.705627512 +0000 2024-03-19T01:33:12.182 DEBUG:teuthology.orchestra.run.smithi112:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-03-19T01:33:12.252 INFO:teuthology.orchestra.run.smithi112.stderr:1+0 records in 2024-03-19T01:33:12.252 INFO:teuthology.orchestra.run.smithi112.stderr:1+0 records out 2024-03-19T01:33:12.252 INFO:teuthology.orchestra.run.smithi112.stderr:512 bytes copied, 0.000349079 s, 1.5 MB/s 2024-03-19T01:33:12.253 DEBUG:teuthology.orchestra.run.smithi112:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-03-19T01:33:12.312 DEBUG:teuthology.orchestra.run.smithi112:> stat /dev/vg_nvme/lv_2 2024-03-19T01:33:12.370 INFO:teuthology.orchestra.run.smithi112.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-03-19T01:33:12.370 INFO:teuthology.orchestra.run.smithi112.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-19T01:33:12.370 INFO:teuthology.orchestra.run.smithi112.stdout:Device: 5h/5d Inode: 681 Links: 1 2024-03-19T01:33:12.370 INFO:teuthology.orchestra.run.smithi112.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-19T01:33:12.370 INFO:teuthology.orchestra.run.smithi112.stdout:Context: system_u:object_r:device_t:s0 2024-03-19T01:33:12.371 INFO:teuthology.orchestra.run.smithi112.stdout:Access: 2024-03-19 01:32:59.014499178 +0000 2024-03-19T01:33:12.371 INFO:teuthology.orchestra.run.smithi112.stdout:Modify: 2024-03-19 01:13:00.052618917 +0000 2024-03-19T01:33:12.371 INFO:teuthology.orchestra.run.smithi112.stdout:Change: 2024-03-19 01:13:00.052618917 +0000 2024-03-19T01:33:12.371 INFO:teuthology.orchestra.run.smithi112.stdout: Birth: 2024-03-19 01:13:00.052618917 +0000 2024-03-19T01:33:12.371 DEBUG:teuthology.orchestra.run.smithi112:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-03-19T01:33:12.438 INFO:teuthology.orchestra.run.smithi112.stderr:1+0 records in 2024-03-19T01:33:12.439 INFO:teuthology.orchestra.run.smithi112.stderr:1+0 records out 2024-03-19T01:33:12.439 INFO:teuthology.orchestra.run.smithi112.stderr:512 bytes copied, 0.000216584 s, 2.4 MB/s 2024-03-19T01:33:12.440 DEBUG:teuthology.orchestra.run.smithi112:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-03-19T01:33:12.497 DEBUG:teuthology.orchestra.run.smithi112:> stat /dev/vg_nvme/lv_3 2024-03-19T01:33:12.553 INFO:teuthology.orchestra.run.smithi112.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-03-19T01:33:12.553 INFO:teuthology.orchestra.run.smithi112.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-19T01:33:12.553 INFO:teuthology.orchestra.run.smithi112.stdout:Device: 5h/5d Inode: 700 Links: 1 2024-03-19T01:33:12.553 INFO:teuthology.orchestra.run.smithi112.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-19T01:33:12.553 INFO:teuthology.orchestra.run.smithi112.stdout:Context: system_u:object_r:device_t:s0 2024-03-19T01:33:12.553 INFO:teuthology.orchestra.run.smithi112.stdout:Access: 2024-03-19 01:32:59.014499178 +0000 2024-03-19T01:33:12.553 INFO:teuthology.orchestra.run.smithi112.stdout:Modify: 2024-03-19 01:13:00.397610371 +0000 2024-03-19T01:33:12.553 INFO:teuthology.orchestra.run.smithi112.stdout:Change: 2024-03-19 01:13:00.397610371 +0000 2024-03-19T01:33:12.554 INFO:teuthology.orchestra.run.smithi112.stdout: Birth: 2024-03-19 01:13:00.397610371 +0000 2024-03-19T01:33:12.554 DEBUG:teuthology.orchestra.run.smithi112:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-03-19T01:33:12.619 INFO:teuthology.orchestra.run.smithi112.stderr:1+0 records in 2024-03-19T01:33:12.620 INFO:teuthology.orchestra.run.smithi112.stderr:1+0 records out 2024-03-19T01:33:12.620 INFO:teuthology.orchestra.run.smithi112.stderr:512 bytes copied, 0.000175601 s, 2.9 MB/s 2024-03-19T01:33:12.621 DEBUG:teuthology.orchestra.run.smithi112:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-03-19T01:33:12.678 DEBUG:teuthology.orchestra.run.smithi112:> stat /dev/vg_nvme/lv_4 2024-03-19T01:33:12.735 INFO:teuthology.orchestra.run.smithi112.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-03-19T01:33:12.735 INFO:teuthology.orchestra.run.smithi112.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-19T01:33:12.735 INFO:teuthology.orchestra.run.smithi112.stdout:Device: 5h/5d Inode: 718 Links: 1 2024-03-19T01:33:12.735 INFO:teuthology.orchestra.run.smithi112.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-19T01:33:12.735 INFO:teuthology.orchestra.run.smithi112.stdout:Context: system_u:object_r:device_t:s0 2024-03-19T01:33:12.735 INFO:teuthology.orchestra.run.smithi112.stdout:Access: 2024-03-19 01:32:59.014499178 +0000 2024-03-19T01:33:12.735 INFO:teuthology.orchestra.run.smithi112.stdout:Modify: 2024-03-19 01:13:00.727602198 +0000 2024-03-19T01:33:12.735 INFO:teuthology.orchestra.run.smithi112.stdout:Change: 2024-03-19 01:13:00.727602198 +0000 2024-03-19T01:33:12.735 INFO:teuthology.orchestra.run.smithi112.stdout: Birth: 2024-03-19 01:13:00.727602198 +0000 2024-03-19T01:33:12.736 DEBUG:teuthology.orchestra.run.smithi112:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-03-19T01:33:12.802 INFO:teuthology.orchestra.run.smithi112.stderr:1+0 records in 2024-03-19T01:33:12.802 INFO:teuthology.orchestra.run.smithi112.stderr:1+0 records out 2024-03-19T01:33:12.802 INFO:teuthology.orchestra.run.smithi112.stderr:512 bytes copied, 0.000212918 s, 2.4 MB/s 2024-03-19T01:33:12.803 DEBUG:teuthology.orchestra.run.smithi112:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-03-19T01:33:12.860 DEBUG:teuthology.orchestra.run.smithi115:> set -ex 2024-03-19T01:33:12.860 DEBUG:teuthology.orchestra.run.smithi115:> dd if=/scratch_devs of=/dev/stdout 2024-03-19T01:33:12.877 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-03-19T01:33:12.877 DEBUG:teuthology.orchestra.run.smithi115:> stat /dev/vg_nvme/lv_1 2024-03-19T01:33:12.934 INFO:teuthology.orchestra.run.smithi115.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-03-19T01:33:12.935 INFO:teuthology.orchestra.run.smithi115.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-19T01:33:12.935 INFO:teuthology.orchestra.run.smithi115.stdout:Device: 5h/5d Inode: 664 Links: 1 2024-03-19T01:33:12.935 INFO:teuthology.orchestra.run.smithi115.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-19T01:33:12.935 INFO:teuthology.orchestra.run.smithi115.stdout:Context: system_u:object_r:device_t:s0 2024-03-19T01:33:12.935 INFO:teuthology.orchestra.run.smithi115.stdout:Access: 2024-03-19 01:32:49.790375955 +0000 2024-03-19T01:33:12.935 INFO:teuthology.orchestra.run.smithi115.stdout:Modify: 2024-03-19 01:15:13.723789967 +0000 2024-03-19T01:33:12.935 INFO:teuthology.orchestra.run.smithi115.stdout:Change: 2024-03-19 01:15:13.723789967 +0000 2024-03-19T01:33:12.935 INFO:teuthology.orchestra.run.smithi115.stdout: Birth: 2024-03-19 01:15:13.723789967 +0000 2024-03-19T01:33:12.935 DEBUG:teuthology.orchestra.run.smithi115:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-03-19T01:33:13.001 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:12 smithi115 ceph-mon[67332]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "3;smithi099=x;smithi112=y;smithi115=z", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:33:13.001 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:12 smithi115 ceph-mon[67332]: Saving service mgr spec with placement smithi099=x;smithi112=y;smithi115=z;count:3 2024-03-19T01:33:13.001 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:12 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:13.001 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:12 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:13.001 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:12 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:33:13.003 INFO:teuthology.orchestra.run.smithi115.stderr:1+0 records in 2024-03-19T01:33:13.003 INFO:teuthology.orchestra.run.smithi115.stderr:1+0 records out 2024-03-19T01:33:13.003 INFO:teuthology.orchestra.run.smithi115.stderr:512 bytes copied, 0.000272278 s, 1.9 MB/s 2024-03-19T01:33:13.004 DEBUG:teuthology.orchestra.run.smithi115:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-03-19T01:33:13.062 DEBUG:teuthology.orchestra.run.smithi115:> stat /dev/vg_nvme/lv_2 2024-03-19T01:33:13.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:12 smithi099 ceph-mon[62951]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "3;smithi099=x;smithi112=y;smithi115=z", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:33:13.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:12 smithi099 ceph-mon[62951]: Saving service mgr spec with placement smithi099=x;smithi112=y;smithi115=z;count:3 2024-03-19T01:33:13.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:12 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:13.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:12 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:13.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:12 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:33:13.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:12 smithi112 ceph-mon[66819]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "3;smithi099=x;smithi112=y;smithi115=z", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:33:13.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:12 smithi112 ceph-mon[66819]: Saving service mgr spec with placement smithi099=x;smithi112=y;smithi115=z;count:3 2024-03-19T01:33:13.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:12 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:13.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:12 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:13.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:12 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:33:13.116 INFO:teuthology.orchestra.run.smithi115.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-03-19T01:33:13.116 INFO:teuthology.orchestra.run.smithi115.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-19T01:33:13.117 INFO:teuthology.orchestra.run.smithi115.stdout:Device: 5h/5d Inode: 680 Links: 1 2024-03-19T01:33:13.117 INFO:teuthology.orchestra.run.smithi115.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-19T01:33:13.117 INFO:teuthology.orchestra.run.smithi115.stdout:Context: system_u:object_r:device_t:s0 2024-03-19T01:33:13.117 INFO:teuthology.orchestra.run.smithi115.stdout:Access: 2024-03-19 01:32:49.790375955 +0000 2024-03-19T01:33:13.117 INFO:teuthology.orchestra.run.smithi115.stdout:Modify: 2024-03-19 01:15:14.070782943 +0000 2024-03-19T01:33:13.117 INFO:teuthology.orchestra.run.smithi115.stdout:Change: 2024-03-19 01:15:14.070782943 +0000 2024-03-19T01:33:13.117 INFO:teuthology.orchestra.run.smithi115.stdout: Birth: 2024-03-19 01:15:14.070782943 +0000 2024-03-19T01:33:13.117 DEBUG:teuthology.orchestra.run.smithi115:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-03-19T01:33:13.182 INFO:teuthology.orchestra.run.smithi115.stderr:1+0 records in 2024-03-19T01:33:13.182 INFO:teuthology.orchestra.run.smithi115.stderr:1+0 records out 2024-03-19T01:33:13.182 INFO:teuthology.orchestra.run.smithi115.stderr:512 bytes copied, 0.00021255 s, 2.4 MB/s 2024-03-19T01:33:13.183 DEBUG:teuthology.orchestra.run.smithi115:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-03-19T01:33:13.241 DEBUG:teuthology.orchestra.run.smithi115:> stat /dev/vg_nvme/lv_3 2024-03-19T01:33:13.295 INFO:teuthology.orchestra.run.smithi115.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-03-19T01:33:13.296 INFO:teuthology.orchestra.run.smithi115.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-19T01:33:13.296 INFO:teuthology.orchestra.run.smithi115.stdout:Device: 5h/5d Inode: 701 Links: 1 2024-03-19T01:33:13.296 INFO:teuthology.orchestra.run.smithi115.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-19T01:33:13.296 INFO:teuthology.orchestra.run.smithi115.stdout:Context: system_u:object_r:device_t:s0 2024-03-19T01:33:13.296 INFO:teuthology.orchestra.run.smithi115.stdout:Access: 2024-03-19 01:32:49.790375955 +0000 2024-03-19T01:33:13.296 INFO:teuthology.orchestra.run.smithi115.stdout:Modify: 2024-03-19 01:15:14.402776222 +0000 2024-03-19T01:33:13.296 INFO:teuthology.orchestra.run.smithi115.stdout:Change: 2024-03-19 01:15:14.402776222 +0000 2024-03-19T01:33:13.296 INFO:teuthology.orchestra.run.smithi115.stdout: Birth: 2024-03-19 01:15:14.402776222 +0000 2024-03-19T01:33:13.297 DEBUG:teuthology.orchestra.run.smithi115:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-03-19T01:33:13.362 INFO:teuthology.orchestra.run.smithi115.stderr:1+0 records in 2024-03-19T01:33:13.362 INFO:teuthology.orchestra.run.smithi115.stderr:1+0 records out 2024-03-19T01:33:13.362 INFO:teuthology.orchestra.run.smithi115.stderr:512 bytes copied, 0.000214462 s, 2.4 MB/s 2024-03-19T01:33:13.363 DEBUG:teuthology.orchestra.run.smithi115:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-03-19T01:33:13.419 DEBUG:teuthology.orchestra.run.smithi115:> stat /dev/vg_nvme/lv_4 2024-03-19T01:33:13.475 INFO:teuthology.orchestra.run.smithi115.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-03-19T01:33:13.475 INFO:teuthology.orchestra.run.smithi115.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-19T01:33:13.476 INFO:teuthology.orchestra.run.smithi115.stdout:Device: 5h/5d Inode: 718 Links: 1 2024-03-19T01:33:13.476 INFO:teuthology.orchestra.run.smithi115.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-19T01:33:13.476 INFO:teuthology.orchestra.run.smithi115.stdout:Context: system_u:object_r:device_t:s0 2024-03-19T01:33:13.476 INFO:teuthology.orchestra.run.smithi115.stdout:Access: 2024-03-19 01:32:49.791375935 +0000 2024-03-19T01:33:13.476 INFO:teuthology.orchestra.run.smithi115.stdout:Modify: 2024-03-19 01:15:14.734769501 +0000 2024-03-19T01:33:13.476 INFO:teuthology.orchestra.run.smithi115.stdout:Change: 2024-03-19 01:15:14.734769501 +0000 2024-03-19T01:33:13.476 INFO:teuthology.orchestra.run.smithi115.stdout: Birth: 2024-03-19 01:15:14.734769501 +0000 2024-03-19T01:33:13.476 DEBUG:teuthology.orchestra.run.smithi115:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-03-19T01:33:13.542 INFO:teuthology.orchestra.run.smithi115.stderr:1+0 records in 2024-03-19T01:33:13.543 INFO:teuthology.orchestra.run.smithi115.stderr:1+0 records out 2024-03-19T01:33:13.543 INFO:teuthology.orchestra.run.smithi115.stderr:512 bytes copied, 0.000192108 s, 2.7 MB/s 2024-03-19T01:33:13.547 DEBUG:teuthology.orchestra.run.smithi115:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-03-19T01:33:13.602 INFO:tasks.cephadm:Deploying osd.0 on smithi099 with /dev/vg_nvme/lv_4... 2024-03-19T01:33:13.603 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-03-19T01:33:13.838 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:33:13.960 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:13 smithi099 ceph-mon[62951]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:13.960 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:13 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:33:13.961 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:13 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:33:13.961 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:13 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:13.961 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:13 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.z", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-19T01:33:13.961 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:13 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.z", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-03-19T01:33:13.961 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:13 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-19T01:33:13.961 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:13 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:33:14.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:13 smithi115 ceph-mon[67332]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:14.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:13 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:33:14.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:13 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:33:14.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:13 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:14.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:13 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.z", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-19T01:33:14.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:13 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.z", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-03-19T01:33:14.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:13 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-19T01:33:14.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:13 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:33:14.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:13 smithi112 ceph-mon[66819]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:14.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:13 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:33:14.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:13 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:33:14.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:13 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:14.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:13 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.z", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-19T01:33:14.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:13 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.z", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-03-19T01:33:14.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:13 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-19T01:33:14.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:13 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:33:14.800 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:14 smithi115 ceph-mon[67332]: Deploying daemon mgr.z on smithi115 2024-03-19T01:33:14.998 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:14 smithi099 ceph-mon[62951]: Deploying daemon mgr.z on smithi115 2024-03-19T01:33:15.056 INFO:journalctl@ceph.mgr.z.smithi115.stdout:Mar 19 01:33:14 smithi115 systemd[1]: Starting Ceph mgr.z for 438fb69c-e590-11ee-95c9-87774f69a715... 2024-03-19T01:33:15.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:14 smithi112 ceph-mon[66819]: Deploying daemon mgr.z on smithi115 2024-03-19T01:33:15.448 INFO:journalctl@ceph.mgr.z.smithi115.stdout:Mar 19 01:33:15 smithi115 podman[69026]: 2024-03-19 01:33:15.154495385 +0000 UTC m=+0.085096521 container create b2c0b74c91363968eeca6c8bb8d7e012f574e5797037b5a338196015ca290688 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-z, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240307, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, ceph=True, org.label-schema.vendor=CentOS, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, io.buildah.version=1.33.5) 2024-03-19T01:33:15.448 INFO:journalctl@ceph.mgr.z.smithi115.stdout:Mar 19 01:33:15 smithi115 podman[69026]: 2024-03-19 01:33:15.086829036 +0000 UTC m=+0.017430162 image pull 2312c8e42d0069a2dbb2364bddb00264db4b800e7f18a7e7e4eb8f3ecab2ff1c quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639 2024-03-19T01:33:15.448 INFO:journalctl@ceph.mgr.z.smithi115.stdout:Mar 19 01:33:15 smithi115 podman[69026]: 2024-03-19 01:33:15.364878991 +0000 UTC m=+0.295480098 container init b2c0b74c91363968eeca6c8bb8d7e012f574e5797037b5a338196015ca290688 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-z, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240307, GIT_BRANCH=HEAD) 2024-03-19T01:33:15.448 INFO:journalctl@ceph.mgr.z.smithi115.stdout:Mar 19 01:33:15 smithi115 podman[69026]: 2024-03-19 01:33:15.367533194 +0000 UTC m=+0.298134309 container start b2c0b74c91363968eeca6c8bb8d7e012f574e5797037b5a338196015ca290688 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-z, org.label-schema.build-date=20240307, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30) 2024-03-19T01:33:15.448 INFO:journalctl@ceph.mgr.z.smithi115.stdout:Mar 19 01:33:15 smithi115 ceph-mgr[69045]: -- 172.21.15.115:0/1731494324 <== mon.1 v2:172.21.15.115:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5575333a85a0 con 0x55753337cc00 2024-03-19T01:33:15.448 INFO:journalctl@ceph.mgr.z.smithi115.stdout:Mar 19 01:33:15 smithi115 bash[69026]: b2c0b74c91363968eeca6c8bb8d7e012f574e5797037b5a338196015ca290688 2024-03-19T01:33:15.743 INFO:journalctl@ceph.mgr.z.smithi115.stdout:Mar 19 01:33:15 smithi115 systemd[1]: Started Ceph mgr.z for 438fb69c-e590-11ee-95c9-87774f69a715. 2024-03-19T01:33:15.831 INFO:teuthology.orchestra.run.smithi099.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-03-19T01:33:15.831 INFO:teuthology.orchestra.run.smithi099.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-03-19T01:33:15.831 INFO:teuthology.orchestra.run.smithi099.stderr: stderr: 10+0 records in 2024-03-19T01:33:15.831 INFO:teuthology.orchestra.run.smithi099.stderr:10+0 records out 2024-03-19T01:33:15.831 INFO:teuthology.orchestra.run.smithi099.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0117069 s, 896 MB/s 2024-03-19T01:33:15.831 INFO:teuthology.orchestra.run.smithi099.stderr:--> Zapping successful for: 2024-03-19T01:33:16.007 INFO:journalctl@ceph.mgr.z.smithi115.stdout:Mar 19 01:33:15 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-z[69041]: 2024-03-19T01:33:15.913+0000 7fe0c33b0200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-19T01:33:16.008 INFO:journalctl@ceph.mgr.z.smithi115.stdout:Mar 19 01:33:16 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-z[69041]: 2024-03-19T01:33:16.006+0000 7fe0c33b0200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-19T01:33:16.008 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:15 smithi115 ceph-mon[67332]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:16.008 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:15 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:16.008 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:15 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:16.008 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:15 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:16.008 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:15 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-19T01:33:16.008 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:15 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-03-19T01:33:16.008 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:15 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-19T01:33:16.008 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:15 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:33:16.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:15 smithi099 ceph-mon[62951]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:16.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:15 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:16.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:15 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:16.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:15 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:16.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:15 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-19T01:33:16.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:15 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-03-19T01:33:16.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:15 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-19T01:33:16.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:15 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:33:16.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:15 smithi112 ceph-mon[66819]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:16.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:15 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:16.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:15 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:16.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:15 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:16.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:15 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-19T01:33:16.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:15 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-03-19T01:33:16.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:15 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-19T01:33:16.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:15 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:33:16.460 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph orch daemon add osd smithi099:vg_nvme/lv_4 2024-03-19T01:33:16.694 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:33:16.972 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:16 smithi115 ceph-mon[67332]: Deploying daemon mgr.y on smithi112 2024-03-19T01:33:16.973 INFO:journalctl@ceph.mgr.z.smithi115.stdout:Mar 19 01:33:16 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-z[69041]: 2024-03-19T01:33:16.643+0000 7fe0c33b0200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-19T01:33:16.973 INFO:journalctl@ceph.mgr.z.smithi115.stdout:Mar 19 01:33:16 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-z[69041]: 2024-03-19T01:33:16.735+0000 7fe0c33b0200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-19T01:33:16.973 INFO:journalctl@ceph.mgr.z.smithi115.stdout:Mar 19 01:33:16 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-z[69041]: 2024-03-19T01:33:16.878+0000 7fe0c33b0200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-19T01:33:17.019 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:16 smithi099 ceph-mon[62951]: Deploying daemon mgr.y on smithi112 2024-03-19T01:33:17.032 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:16 smithi112 ceph-mon[66819]: Deploying daemon mgr.y on smithi112 2024-03-19T01:33:17.284 INFO:journalctl@ceph.mgr.z.smithi115.stdout:Mar 19 01:33:16 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-z[69041]: 2024-03-19T01:33:16.971+0000 7fe0c33b0200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-19T01:33:17.284 INFO:journalctl@ceph.mgr.z.smithi115.stdout:Mar 19 01:33:17 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-z[69041]: 2024-03-19T01:33:17.056+0000 7fe0c33b0200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-19T01:33:17.536 INFO:journalctl@ceph.mgr.z.smithi115.stdout:Mar 19 01:33:17 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-z[69041]: 2024-03-19T01:33:17.330+0000 7fe0c33b0200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-19T01:33:17.536 INFO:journalctl@ceph.mgr.z.smithi115.stdout:Mar 19 01:33:17 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-z[69041]: 2024-03-19T01:33:17.447+0000 7fe0c33b0200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-19T01:33:17.536 INFO:journalctl@ceph.mgr.z.smithi115.stdout:Mar 19 01:33:17 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-z[69041]: 2024-03-19T01:33:17.535+0000 7fe0c33b0200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-19T01:33:17.612 INFO:journalctl@ceph.mgr.y.smithi112.stdout:Mar 19 01:33:17 smithi112 systemd[1]: Starting Ceph mgr.y for 438fb69c-e590-11ee-95c9-87774f69a715... 2024-03-19T01:33:17.806 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:17 smithi115 ceph-mon[67332]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:17.807 INFO:journalctl@ceph.mgr.z.smithi115.stdout:Mar 19 01:33:17 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-z[69041]: 2024-03-19T01:33:17.631+0000 7fe0c33b0200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-19T01:33:17.986 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:17 smithi112 ceph-mon[66819]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:17.986 INFO:journalctl@ceph.mgr.y.smithi112.stdout:Mar 19 01:33:17 smithi112 podman[68181]: 2024-03-19 01:33:17.587254917 +0000 UTC m=+0.017639583 image pull 2312c8e42d0069a2dbb2364bddb00264db4b800e7f18a7e7e4eb8f3ecab2ff1c quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639 2024-03-19T01:33:17.986 INFO:journalctl@ceph.mgr.y.smithi112.stdout:Mar 19 01:33:17 smithi112 podman[68181]: 2024-03-19 01:33:17.69241561 +0000 UTC m=+0.122800260 container create c0916e7a53e9babca1da4ae0a3ad92ecad1e923624d89a3c7847949878e39490 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-y, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, io.buildah.version=1.33.5, org.label-schema.build-date=20240307, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.schema-version=1.0) 2024-03-19T01:33:17.986 INFO:journalctl@ceph.mgr.y.smithi112.stdout:Mar 19 01:33:17 smithi112 podman[68181]: 2024-03-19 01:33:17.90301874 +0000 UTC m=+0.333403389 container init c0916e7a53e9babca1da4ae0a3ad92ecad1e923624d89a3c7847949878e39490 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-y, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.build-date=20240307, org.label-schema.license=GPLv2) 2024-03-19T01:33:17.986 INFO:journalctl@ceph.mgr.y.smithi112.stdout:Mar 19 01:33:17 smithi112 podman[68181]: 2024-03-19 01:33:17.905722269 +0000 UTC m=+0.336106914 container start c0916e7a53e9babca1da4ae0a3ad92ecad1e923624d89a3c7847949878e39490 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-y, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240307, io.buildah.version=1.33.5, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=wip-batrick-testing-20240318.181317-b6ae249) 2024-03-19T01:33:17.986 INFO:journalctl@ceph.mgr.y.smithi112.stdout:Mar 19 01:33:17 smithi112 ceph-mgr[68200]: -- 172.21.15.112:0/2280496115 <== mon.2 v2:172.21.15.112:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x560cc9d385a0 con 0x560cc9d0c800 2024-03-19T01:33:17.986 INFO:journalctl@ceph.mgr.y.smithi112.stdout:Mar 19 01:33:17 smithi112 bash[68181]: c0916e7a53e9babca1da4ae0a3ad92ecad1e923624d89a3c7847949878e39490 2024-03-19T01:33:18.022 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:17 smithi099 ceph-mon[62951]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:18.242 INFO:journalctl@ceph.mgr.z.smithi115.stdout:Mar 19 01:33:17 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-z[69041]: 2024-03-19T01:33:17.809+0000 7fe0c33b0200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-19T01:33:18.258 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:18.255+0000 7fd5a45bc700 1 -- 172.21.15.99:0/3600453672 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd59c102430 msgr2=0x7fd59c102890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:33:18.258 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:18.255+0000 7fd5a45bc700 1 --2- 172.21.15.99:0/3600453672 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd59c102430 0x7fd59c102890 secure :-1 s=READY pgs=105 cs=0 l=1 rev1=1 crypto rx=0x7fd58c0099c0 tx=0x7fd58c009cd0 comp rx=0 tx=0).stop 2024-03-19T01:33:18.258 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:18.255+0000 7fd5a45bc700 1 -- 172.21.15.99:0/3600453672 shutdown_connections 2024-03-19T01:33:18.258 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:18.255+0000 7fd5a45bc700 1 --2- 172.21.15.99:0/3600453672 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd59c101230 0x7fd59c101650 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:33:18.258 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:18.255+0000 7fd5a45bc700 1 --2- 172.21.15.99:0/3600453672 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd59c102dd0 0x7fd59c109680 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:33:18.258 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:18.255+0000 7fd5a45bc700 1 --2- 172.21.15.99:0/3600453672 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd59c102430 0x7fd59c102890 unknown :-1 s=CLOSED pgs=105 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:33:18.258 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:18.255+0000 7fd5a45bc700 1 -- 172.21.15.99:0/3600453672 >> 172.21.15.99:0/3600453672 conn(0x7fd59c0fc7d0 msgr2=0x7fd59c0fec10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:33:18.258 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:18.255+0000 7fd5a45bc700 1 -- 172.21.15.99:0/3600453672 shutdown_connections 2024-03-19T01:33:18.258 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:18.255+0000 7fd5a45bc700 1 -- 172.21.15.99:0/3600453672 wait complete. 2024-03-19T01:33:18.258 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:18.256+0000 7fd5a45bc700 1 Processor -- start 2024-03-19T01:33:18.259 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:18.256+0000 7fd5a45bc700 1 -- start start 2024-03-19T01:33:18.259 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:18.256+0000 7fd5a45bc700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd59c101230 0x7fd59c198540 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:33:18.259 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:18.256+0000 7fd5a45bc700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd59c102430 0x7fd59c198a80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:33:18.259 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:18.256+0000 7fd5a45bc700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd59c102dd0 0x7fd59c19fb00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:33:18.259 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:18.256+0000 7fd5a45bc700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fd59c10ce40 con 0x7fd59c102dd0 2024-03-19T01:33:18.259 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:18.256+0000 7fd5a45bc700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fd59c10ccc0 con 0x7fd59c101230 2024-03-19T01:33:18.259 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:18.256+0000 7fd5a45bc700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fd59c10cfc0 con 0x7fd59c102430 2024-03-19T01:33:18.259 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:18.256+0000 7fd5a2b59700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd59c102dd0 0x7fd59c19fb00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:33:18.259 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:18.256+0000 7fd5a2b59700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd59c102dd0 0x7fd59c19fb00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:43024/0 (socket says 172.21.15.99:43024) 2024-03-19T01:33:18.259 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:18.256+0000 7fd5a1b57700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd59c102430 0x7fd59c198a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:33:18.259 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:18.256+0000 7fd5a2b59700 1 -- 172.21.15.99:0/1220182910 learned_addr learned my addr 172.21.15.99:0/1220182910 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:33:18.259 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:18.256+0000 7fd5a2358700 1 --2- 172.21.15.99:0/1220182910 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd59c101230 0x7fd59c198540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:33:18.259 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:18.256+0000 7fd5a2b59700 1 -- 172.21.15.99:0/1220182910 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd59c102430 msgr2=0x7fd59c198a80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:33:18.260 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:18.256+0000 7fd5a2b59700 1 --2- 172.21.15.99:0/1220182910 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd59c102430 0x7fd59c198a80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:33:18.260 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:18.256+0000 7fd5a2b59700 1 -- 172.21.15.99:0/1220182910 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd59c101230 msgr2=0x7fd59c198540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:33:18.260 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:18.256+0000 7fd5a2b59700 1 --2- 172.21.15.99:0/1220182910 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd59c101230 0x7fd59c198540 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:33:18.260 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:18.256+0000 7fd5a2b59700 1 -- 172.21.15.99:0/1220182910 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd59c1a01a0 con 0x7fd59c102dd0 2024-03-19T01:33:18.260 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:18.256+0000 7fd5a2b59700 1 --2- 172.21.15.99:0/1220182910 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd59c102dd0 0x7fd59c19fb00 secure :-1 s=READY pgs=106 cs=0 l=1 rev1=1 crypto rx=0x7fd594007f10 tx=0x7fd59400d3b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:33:18.260 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:18.257+0000 7fd5937fe700 1 -- 172.21.15.99:0/1220182910 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd594004bb0 con 0x7fd59c102dd0 2024-03-19T01:33:18.260 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:18.257+0000 7fd5937fe700 1 -- 172.21.15.99:0/1220182910 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd594004d50 con 0x7fd59c102dd0 2024-03-19T01:33:18.260 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:18.257+0000 7fd5a45bc700 1 -- 172.21.15.99:0/1220182910 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd59c1a0490 con 0x7fd59c102dd0 2024-03-19T01:33:18.260 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:18.257+0000 7fd5937fe700 1 -- 172.21.15.99:0/1220182910 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd5940059f0 con 0x7fd59c102dd0 2024-03-19T01:33:18.260 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:18.257+0000 7fd5a45bc700 1 -- 172.21.15.99:0/1220182910 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fd59c1a09d0 con 0x7fd59c102dd0 2024-03-19T01:33:18.261 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:18.258+0000 7fd5937fe700 1 -- 172.21.15.99:0/1220182910 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 13) ==== 46162+0+0 (secure 0 0 0) 0x7fd59401a020 con 0x7fd59c102dd0 2024-03-19T01:33:18.261 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:18.258+0000 7fd5a45bc700 1 -- 172.21.15.99:0/1220182910 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fd59c068fd0 con 0x7fd59c102dd0 2024-03-19T01:33:18.261 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:18.258+0000 7fd5937fe700 1 --2- 172.21.15.99:0/1220182910 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd588039200 0x7fd58803b6c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:33:18.261 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:18.258+0000 7fd5937fe700 1 -- 172.21.15.99:0/1220182910 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(4..4 src has 1..4) ==== 1155+0+0 (secure 0 0 0) 0x7fd59404ba40 con 0x7fd59c102dd0 2024-03-19T01:33:18.261 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:18.258+0000 7fd5a2358700 1 --2- 172.21.15.99:0/1220182910 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd588039200 0x7fd58803b6c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:33:18.262 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:18.258+0000 7fd5a2358700 1 --2- 172.21.15.99:0/1220182910 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd588039200 0x7fd58803b6c0 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7fd59c102290 tx=0x7fd598007400 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:33:18.263 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:18.263+0000 7fd5937fe700 1 -- 172.21.15.99:0/1220182910 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7fd594010070 con 0x7fd59c102dd0 2024-03-19T01:33:18.330 INFO:journalctl@ceph.mgr.y.smithi112.stdout:Mar 19 01:33:17 smithi112 systemd[1]: Started Ceph mgr.y for 438fb69c-e590-11ee-95c9-87774f69a715. 2024-03-19T01:33:18.472 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:18.471+0000 7fd5a45bc700 1 -- 172.21.15.99:0/1220182910 --> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_4", "target": ["mon-mgr", ""]}) -- 0x7fd59c06af60 con 0x7fd588039200 2024-03-19T01:33:18.612 INFO:journalctl@ceph.mgr.y.smithi112.stdout:Mar 19 01:33:18 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-y[68196]: 2024-03-19T01:33:18.471+0000 7f4f36ec0200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-19T01:33:18.612 INFO:journalctl@ceph.mgr.y.smithi112.stdout:Mar 19 01:33:18 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-y[68196]: 2024-03-19T01:33:18.570+0000 7f4f36ec0200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-19T01:33:19.178 INFO:journalctl@ceph.mgr.z.smithi115.stdout:Mar 19 01:33:18 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-z[69041]: 2024-03-19T01:33:18.834+0000 7fe0c33b0200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-19T01:33:19.179 INFO:journalctl@ceph.mgr.z.smithi115.stdout:Mar 19 01:33:18 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-z[69041]: 2024-03-19T01:33:18.977+0000 7fe0c33b0200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-19T01:33:19.179 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:19 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:19.179 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:19 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:19.179 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:19 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:19.179 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:19 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:19.179 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:19 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:33:19.179 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:19 smithi115 ceph-mon[67332]: from='client.14214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:33:19.179 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:19 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-19T01:33:19.179 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:19 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-19T01:33:19.179 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:19 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:33:19.180 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:19 smithi115 ceph-mon[67332]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:19.323 INFO:journalctl@ceph.mgr.y.smithi112.stdout:Mar 19 01:33:19 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-y[68196]: 2024-03-19T01:33:19.262+0000 7f4f36ec0200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-19T01:33:19.323 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:19 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:19.323 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:19 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:19.324 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:19 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:19.324 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:19 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:19.324 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:19 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:33:19.324 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:19 smithi112 ceph-mon[66819]: from='client.14214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:33:19.324 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:19 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-19T01:33:19.324 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:19 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-19T01:33:19.324 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:19 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:33:19.324 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:19 smithi112 ceph-mon[66819]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:19.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:19 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:19.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:19 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:19.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:19 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:19.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:19 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:19.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:19 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:33:19.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:19 smithi099 ceph-mon[62951]: from='client.14214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:33:19.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:19 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-19T01:33:19.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:19 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-19T01:33:19.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:19 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:33:19.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:19 smithi099 ceph-mon[62951]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:19.556 INFO:journalctl@ceph.mgr.z.smithi115.stdout:Mar 19 01:33:19 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-z[69041]: 2024-03-19T01:33:19.318+0000 7fe0c33b0200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-19T01:33:19.557 INFO:journalctl@ceph.mgr.z.smithi115.stdout:Mar 19 01:33:19 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-z[69041]: 2024-03-19T01:33:19.407+0000 7fe0c33b0200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-19T01:33:19.612 INFO:journalctl@ceph.mgr.y.smithi112.stdout:Mar 19 01:33:19 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-y[68196]: 2024-03-19T01:33:19.358+0000 7f4f36ec0200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-19T01:33:19.612 INFO:journalctl@ceph.mgr.y.smithi112.stdout:Mar 19 01:33:19 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-y[68196]: 2024-03-19T01:33:19.508+0000 7f4f36ec0200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-19T01:33:19.969 INFO:journalctl@ceph.mgr.y.smithi112.stdout:Mar 19 01:33:19 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-y[68196]: 2024-03-19T01:33:19.620+0000 7f4f36ec0200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-19T01:33:19.970 INFO:journalctl@ceph.mgr.y.smithi112.stdout:Mar 19 01:33:19 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-y[68196]: 2024-03-19T01:33:19.702+0000 7f4f36ec0200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-19T01:33:19.970 INFO:journalctl@ceph.mgr.y.smithi112.stdout:Mar 19 01:33:19 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-y[68196]: 2024-03-19T01:33:19.968+0000 7f4f36ec0200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-19T01:33:20.056 INFO:journalctl@ceph.mgr.z.smithi115.stdout:Mar 19 01:33:19 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-z[69041]: 2024-03-19T01:33:19.606+0000 7fe0c33b0200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-19T01:33:20.057 INFO:journalctl@ceph.mgr.z.smithi115.stdout:Mar 19 01:33:19 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-z[69041]: 2024-03-19T01:33:19.800+0000 7fe0c33b0200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-19T01:33:20.255 INFO:journalctl@ceph.mgr.y.smithi112.stdout:Mar 19 01:33:20 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-y[68196]: 2024-03-19T01:33:20.076+0000 7f4f36ec0200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-19T01:33:20.255 INFO:journalctl@ceph.mgr.y.smithi112.stdout:Mar 19 01:33:20 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-y[68196]: 2024-03-19T01:33:20.161+0000 7f4f36ec0200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-19T01:33:20.255 INFO:journalctl@ceph.mgr.y.smithi112.stdout:Mar 19 01:33:20 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-y[68196]: 2024-03-19T01:33:20.254+0000 7f4f36ec0200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-19T01:33:20.365 INFO:journalctl@ceph.mgr.z.smithi115.stdout:Mar 19 01:33:20 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-z[69041]: 2024-03-19T01:33:20.086+0000 7fe0c33b0200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-19T01:33:20.366 INFO:journalctl@ceph.mgr.z.smithi115.stdout:Mar 19 01:33:20 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-z[69041]: 2024-03-19T01:33:20.364+0000 7fe0c33b0200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-19T01:33:20.557 INFO:journalctl@ceph.mgr.y.smithi112.stdout:Mar 19 01:33:20 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-y[68196]: 2024-03-19T01:33:20.426+0000 7f4f36ec0200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-19T01:33:20.806 INFO:journalctl@ceph.mgr.z.smithi115.stdout:Mar 19 01:33:20 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-z[69041]: 2024-03-19T01:33:20.602+0000 7fe0c33b0200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-19T01:33:20.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:20 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:20.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:20 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:20.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:20 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:20.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:20 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:20.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:20 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:33:20.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:20 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:33:20.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:20 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:20.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:20 smithi115 ceph-mon[67332]: Reconfiguring mgr.x (unknown last config time)... 2024-03-19T01:33:20.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:20 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-19T01:33:20.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:20 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-19T01:33:20.808 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:20 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:33:20.808 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:20 smithi115 ceph-mon[67332]: Reconfiguring daemon mgr.x on smithi099 2024-03-19T01:33:20.842 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:20 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:20.842 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:20 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:20.843 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:20 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:20.843 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:20 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:20.843 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:20 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:33:20.843 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:20 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:33:20.843 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:20 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:20.843 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:20 smithi099 ceph-mon[62951]: Reconfiguring mgr.x (unknown last config time)... 2024-03-19T01:33:20.843 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:20 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-19T01:33:20.843 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:20 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-19T01:33:20.843 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:20 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:33:20.843 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:20 smithi099 ceph-mon[62951]: Reconfiguring daemon mgr.x on smithi099 2024-03-19T01:33:20.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:20 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:20.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:20 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:20.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:20 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:20.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:20 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:20.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:20 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:33:20.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:20 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:33:20.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:20 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:20.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:20 smithi112 ceph-mon[66819]: Reconfiguring mgr.x (unknown last config time)... 2024-03-19T01:33:20.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:20 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-19T01:33:20.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:20 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-19T01:33:20.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:20 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:33:20.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:20 smithi112 ceph-mon[66819]: Reconfiguring daemon mgr.x on smithi099 2024-03-19T01:33:21.806 INFO:journalctl@ceph.mgr.z.smithi115.stdout:Mar 19 01:33:21 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-z[69041]: 2024-03-19T01:33:21.397+0000 7fe0c33b0200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-19T01:33:21.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:21 smithi115 ceph-mon[67332]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:21.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:21 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/3719141649' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0c407a0e-a40e-43d7-ad9a-bb8f3cd4a9ba"}]: dispatch 2024-03-19T01:33:21.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:21 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/3719141649' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0c407a0e-a40e-43d7-ad9a-bb8f3cd4a9ba"}]': finished 2024-03-19T01:33:21.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:21 smithi115 ceph-mon[67332]: osdmap e5: 1 total, 0 up, 1 in 2024-03-19T01:33:21.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:21 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-19T01:33:21.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:21 smithi099 ceph-mon[62951]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:21.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:21 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/3719141649' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0c407a0e-a40e-43d7-ad9a-bb8f3cd4a9ba"}]: dispatch 2024-03-19T01:33:21.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:21 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/3719141649' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0c407a0e-a40e-43d7-ad9a-bb8f3cd4a9ba"}]': finished 2024-03-19T01:33:21.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:21 smithi099 ceph-mon[62951]: osdmap e5: 1 total, 0 up, 1 in 2024-03-19T01:33:21.842 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:21 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-19T01:33:21.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:21 smithi112 ceph-mon[66819]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:21.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:21 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/3719141649' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0c407a0e-a40e-43d7-ad9a-bb8f3cd4a9ba"}]: dispatch 2024-03-19T01:33:21.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:21 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/3719141649' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0c407a0e-a40e-43d7-ad9a-bb8f3cd4a9ba"}]': finished 2024-03-19T01:33:21.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:21 smithi112 ceph-mon[66819]: osdmap e5: 1 total, 0 up, 1 in 2024-03-19T01:33:21.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:21 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-19T01:33:21.862 INFO:journalctl@ceph.mgr.y.smithi112.stdout:Mar 19 01:33:21 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-y[68196]: 2024-03-19T01:33:21.387+0000 7f4f36ec0200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-19T01:33:21.863 INFO:journalctl@ceph.mgr.y.smithi112.stdout:Mar 19 01:33:21 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-y[68196]: 2024-03-19T01:33:21.515+0000 7f4f36ec0200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-19T01:33:22.142 INFO:journalctl@ceph.mgr.y.smithi112.stdout:Mar 19 01:33:21 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-y[68196]: 2024-03-19T01:33:21.861+0000 7f4f36ec0200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-19T01:33:22.143 INFO:journalctl@ceph.mgr.y.smithi112.stdout:Mar 19 01:33:21 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-y[68196]: 2024-03-19T01:33:21.949+0000 7f4f36ec0200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-19T01:33:22.143 INFO:journalctl@ceph.mgr.y.smithi112.stdout:Mar 19 01:33:22 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-y[68196]: 2024-03-19T01:33:22.141+0000 7f4f36ec0200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-19T01:33:22.270 INFO:journalctl@ceph.mgr.z.smithi115.stdout:Mar 19 01:33:21 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-z[69041]: 2024-03-19T01:33:21.960+0000 7fe0c33b0200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-19T01:33:22.271 INFO:journalctl@ceph.mgr.z.smithi115.stdout:Mar 19 01:33:22 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-z[69041]: 2024-03-19T01:33:22.056+0000 7fe0c33b0200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-19T01:33:22.271 INFO:journalctl@ceph.mgr.z.smithi115.stdout:Mar 19 01:33:22 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-z[69041]: 2024-03-19T01:33:22.269+0000 7fe0c33b0200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-19T01:33:22.566 INFO:journalctl@ceph.mgr.y.smithi112.stdout:Mar 19 01:33:22 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-y[68196]: 2024-03-19T01:33:22.324+0000 7f4f36ec0200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-19T01:33:22.740 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:22.739+0000 7fd5937fe700 1 -- 172.21.15.99:0/1220182910 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mgrmap(e 14) ==== 91698+0+0 (secure 0 0 0) 0x7fd5940133f0 con 0x7fd59c102dd0 2024-03-19T01:33:22.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:22 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/3988640623' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-19T01:33:22.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:22 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:22.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:22 smithi099 ceph-mon[62951]: Standby manager daemon z started 2024-03-19T01:33:22.842 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:22 smithi099 ceph-mon[62951]: from='mgr.? 172.21.15.115:0/3323089196' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/crt"}]: dispatch 2024-03-19T01:33:22.842 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:22 smithi099 ceph-mon[62951]: from='mgr.? 172.21.15.115:0/3323089196' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-19T01:33:22.842 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:22 smithi099 ceph-mon[62951]: from='mgr.? 172.21.15.115:0/3323089196' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/key"}]: dispatch 2024-03-19T01:33:22.842 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:22 smithi099 ceph-mon[62951]: from='mgr.? 172.21.15.115:0/3323089196' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-19T01:33:22.862 INFO:journalctl@ceph.mgr.y.smithi112.stdout:Mar 19 01:33:22 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-y[68196]: 2024-03-19T01:33:22.607+0000 7f4f36ec0200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-19T01:33:22.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:22 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/3988640623' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-19T01:33:22.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:22 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:22.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:22 smithi112 ceph-mon[66819]: Standby manager daemon z started 2024-03-19T01:33:22.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:22 smithi112 ceph-mon[66819]: from='mgr.? 172.21.15.115:0/3323089196' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/crt"}]: dispatch 2024-03-19T01:33:22.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:22 smithi112 ceph-mon[66819]: from='mgr.? 172.21.15.115:0/3323089196' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-19T01:33:22.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:22 smithi112 ceph-mon[66819]: from='mgr.? 172.21.15.115:0/3323089196' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/key"}]: dispatch 2024-03-19T01:33:22.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:22 smithi112 ceph-mon[66819]: from='mgr.? 172.21.15.115:0/3323089196' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-19T01:33:23.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:22 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/3988640623' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-19T01:33:23.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:22 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:23.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:22 smithi115 ceph-mon[67332]: Standby manager daemon z started 2024-03-19T01:33:23.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:22 smithi115 ceph-mon[67332]: from='mgr.? 172.21.15.115:0/3323089196' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/crt"}]: dispatch 2024-03-19T01:33:23.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:22 smithi115 ceph-mon[67332]: from='mgr.? 172.21.15.115:0/3323089196' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-19T01:33:23.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:22 smithi115 ceph-mon[67332]: from='mgr.? 172.21.15.115:0/3323089196' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/key"}]: dispatch 2024-03-19T01:33:23.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:22 smithi115 ceph-mon[67332]: from='mgr.? 172.21.15.115:0/3323089196' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-19T01:33:23.362 INFO:journalctl@ceph.mgr.y.smithi112.stdout:Mar 19 01:33:22 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-y[68196]: 2024-03-19T01:33:22.863+0000 7f4f36ec0200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-19T01:33:23.362 INFO:journalctl@ceph.mgr.y.smithi112.stdout:Mar 19 01:33:23 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-y[68196]: 2024-03-19T01:33:23.095+0000 7f4f36ec0200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-19T01:33:24.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:23 smithi115 ceph-mon[67332]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:24.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:23 smithi115 ceph-mon[67332]: mgrmap e14: x(active, since 76s), standbys: z 2024-03-19T01:33:24.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:23 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "z", "id": "z"}]: dispatch 2024-03-19T01:33:24.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:23 smithi099 ceph-mon[62951]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:24.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:23 smithi099 ceph-mon[62951]: mgrmap e14: x(active, since 76s), standbys: z 2024-03-19T01:33:24.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:23 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "z", "id": "z"}]: dispatch 2024-03-19T01:33:24.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:23 smithi112 ceph-mon[66819]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:24.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:23 smithi112 ceph-mon[66819]: mgrmap e14: x(active, since 76s), standbys: z 2024-03-19T01:33:24.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:23 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "z", "id": "z"}]: dispatch 2024-03-19T01:33:24.112 INFO:journalctl@ceph.mgr.y.smithi112.stdout:Mar 19 01:33:23 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-y[68196]: 2024-03-19T01:33:23.891+0000 7f4f36ec0200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-19T01:33:24.766 INFO:journalctl@ceph.mgr.y.smithi112.stdout:Mar 19 01:33:24 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-y[68196]: 2024-03-19T01:33:24.454+0000 7f4f36ec0200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-19T01:33:24.767 INFO:journalctl@ceph.mgr.y.smithi112.stdout:Mar 19 01:33:24 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-y[68196]: 2024-03-19T01:33:24.550+0000 7f4f36ec0200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-19T01:33:24.767 INFO:journalctl@ceph.mgr.y.smithi112.stdout:Mar 19 01:33:24 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-y[68196]: 2024-03-19T01:33:24.765+0000 7f4f36ec0200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-19T01:33:25.752 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:25.751+0000 7fd5937fe700 1 -- 172.21.15.99:0/1220182910 <== mon.0 v2:172.21.15.99:3300/0 8 ==== mgrmap(e 15) ==== 137234+0+0 (secure 0 0 0) 0x7fd594005020 con 0x7fd59c102dd0 2024-03-19T01:33:25.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:25 smithi099 ceph-mon[62951]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:25.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:25 smithi099 ceph-mon[62951]: Standby manager daemon y started 2024-03-19T01:33:25.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:25 smithi099 ceph-mon[62951]: from='mgr.? 172.21.15.112:0/2492416855' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-03-19T01:33:25.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:25 smithi099 ceph-mon[62951]: from='mgr.? 172.21.15.112:0/2492416855' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-19T01:33:25.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:25 smithi099 ceph-mon[62951]: from='mgr.? 172.21.15.112:0/2492416855' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-03-19T01:33:25.842 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:25 smithi099 ceph-mon[62951]: from='mgr.? 172.21.15.112:0/2492416855' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-19T01:33:26.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:25 smithi115 ceph-mon[67332]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:26.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:25 smithi115 ceph-mon[67332]: Standby manager daemon y started 2024-03-19T01:33:26.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:25 smithi115 ceph-mon[67332]: from='mgr.? 172.21.15.112:0/2492416855' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-03-19T01:33:26.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:25 smithi115 ceph-mon[67332]: from='mgr.? 172.21.15.112:0/2492416855' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-19T01:33:26.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:25 smithi115 ceph-mon[67332]: from='mgr.? 172.21.15.112:0/2492416855' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-03-19T01:33:26.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:25 smithi115 ceph-mon[67332]: from='mgr.? 172.21.15.112:0/2492416855' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-19T01:33:26.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:25 smithi112 ceph-mon[66819]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:26.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:25 smithi112 ceph-mon[66819]: Standby manager daemon y started 2024-03-19T01:33:26.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:25 smithi112 ceph-mon[66819]: from='mgr.? 172.21.15.112:0/2492416855' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-03-19T01:33:26.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:25 smithi112 ceph-mon[66819]: from='mgr.? 172.21.15.112:0/2492416855' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-19T01:33:26.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:25 smithi112 ceph-mon[66819]: from='mgr.? 172.21.15.112:0/2492416855' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-03-19T01:33:26.113 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:25 smithi112 ceph-mon[66819]: from='mgr.? 172.21.15.112:0/2492416855' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-19T01:33:26.943 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:26 smithi099 ceph-mon[62951]: mgrmap e15: x(active, since 79s), standbys: z, y 2024-03-19T01:33:26.943 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:26 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-19T01:33:26.943 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:26 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:26.943 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:26 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:26.943 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:26 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:33:26.943 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:26 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:33:26.943 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:26 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:33:26.943 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:26 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:27.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:26 smithi115 ceph-mon[67332]: mgrmap e15: x(active, since 79s), standbys: z, y 2024-03-19T01:33:27.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:26 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-19T01:33:27.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:26 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:27.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:26 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:27.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:26 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:33:27.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:26 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:33:27.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:26 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:33:27.058 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:26 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:27.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:26 smithi112 ceph-mon[66819]: mgrmap e15: x(active, since 79s), standbys: z, y 2024-03-19T01:33:27.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:26 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-19T01:33:27.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:26 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:27.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:26 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:27.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:26 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:33:27.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:26 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:33:27.113 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:26 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:33:27.113 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:26 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:28.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:27 smithi115 ceph-mon[67332]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:28.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:27 smithi099 ceph-mon[62951]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:28.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:27 smithi112 ceph-mon[66819]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:29.928 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:29 smithi099 ceph-mon[62951]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:29.928 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:29 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-19T01:33:29.928 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:29 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:33:30.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:29 smithi115 ceph-mon[67332]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:30.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:29 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-19T01:33:30.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:29 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:33:30.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:29 smithi112 ceph-mon[66819]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:30.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:29 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-19T01:33:30.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:29 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:33:31.015 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:30 smithi099 ceph-mon[62951]: Deploying daemon osd.0 on smithi099 2024-03-19T01:33:31.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:30 smithi115 ceph-mon[67332]: Deploying daemon osd.0 on smithi099 2024-03-19T01:33:31.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:30 smithi112 ceph-mon[66819]: Deploying daemon osd.0 on smithi099 2024-03-19T01:33:32.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:31 smithi115 ceph-mon[67332]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:32.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:31 smithi099 ceph-mon[62951]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:32.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:31 smithi112 ceph-mon[66819]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:33.981 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:33 smithi099 ceph-mon[62951]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:34.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:33 smithi115 ceph-mon[67332]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:34.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:33 smithi112 ceph-mon[66819]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:35.531 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:35 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:35.531 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:35 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:35.531 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:35 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:33:35.531 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:35 smithi099 ceph-mon[62951]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:35.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:35 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:35.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:35 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:35.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:35 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:33:35.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:35 smithi115 ceph-mon[67332]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:35.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:35 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:35.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:35 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:35.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:35 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:33:35.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:35 smithi112 ceph-mon[66819]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:37.781 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:37 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:37.782 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:37 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:37.782 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:37 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:33:37.782 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:37 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:33:37.782 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:37 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:37.782 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:37 smithi099 ceph-mon[62951]: pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:37.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:37 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:37.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:37 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:37.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:37 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:33:37.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:37 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:33:37.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:37 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:37.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:37 smithi115 ceph-mon[67332]: pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:37.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:37 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:37.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:37 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:37.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:37 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:33:37.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:37 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:33:37.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:37 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:37.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:37 smithi112 ceph-mon[66819]: pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:37.877 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:37.876+0000 7fd5937fe700 1 -- 172.21.15.99:0/1220182910 <== mgr.14150 v2:172.21.15.99:6800/4272410295 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+37 (secure 0 0 0) 0x7fd59c06af60 con 0x7fd588039200 2024-03-19T01:33:37.877 INFO:teuthology.orchestra.run.smithi099.stdout:Created osd(s) 0 on host 'smithi099' 2024-03-19T01:33:37.879 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:37.878+0000 7fd5a45bc700 1 -- 172.21.15.99:0/1220182910 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd588039200 msgr2=0x7fd58803b6c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:33:37.879 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:37.878+0000 7fd5a45bc700 1 --2- 172.21.15.99:0/1220182910 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd588039200 0x7fd58803b6c0 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7fd59c102290 tx=0x7fd598007400 comp rx=0 tx=0).stop 2024-03-19T01:33:37.879 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:37.878+0000 7fd5a45bc700 1 -- 172.21.15.99:0/1220182910 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd59c102dd0 msgr2=0x7fd59c19fb00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:33:37.879 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:37.878+0000 7fd5a45bc700 1 --2- 172.21.15.99:0/1220182910 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd59c102dd0 0x7fd59c19fb00 secure :-1 s=READY pgs=106 cs=0 l=1 rev1=1 crypto rx=0x7fd594007f10 tx=0x7fd59400d3b0 comp rx=0 tx=0).stop 2024-03-19T01:33:37.879 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:37.879+0000 7fd5a45bc700 1 -- 172.21.15.99:0/1220182910 shutdown_connections 2024-03-19T01:33:37.880 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:37.879+0000 7fd5a45bc700 1 --2- 172.21.15.99:0/1220182910 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd588039200 0x7fd58803b6c0 unknown :-1 s=CLOSED pgs=42 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:33:37.880 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:37.879+0000 7fd5a45bc700 1 --2- 172.21.15.99:0/1220182910 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd59c102dd0 0x7fd59c19fb00 unknown :-1 s=CLOSED pgs=106 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:33:37.880 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:37.879+0000 7fd5a45bc700 1 --2- 172.21.15.99:0/1220182910 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd59c101230 0x7fd59c198540 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:33:37.880 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:37.879+0000 7fd5a45bc700 1 --2- 172.21.15.99:0/1220182910 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd59c102430 0x7fd59c198a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:33:37.880 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:37.879+0000 7fd5a45bc700 1 -- 172.21.15.99:0/1220182910 >> 172.21.15.99:0/1220182910 conn(0x7fd59c0fc7d0 msgr2=0x7fd59c06a850 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:33:37.880 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:37.879+0000 7fd5a45bc700 1 -- 172.21.15.99:0/1220182910 shutdown_connections 2024-03-19T01:33:37.880 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:33:37.879+0000 7fd5a45bc700 1 -- 172.21.15.99:0/1220182910 wait complete. 2024-03-19T01:33:38.515 DEBUG:teuthology.orchestra.run.smithi099:osd.0> sudo journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@osd.0.service 2024-03-19T01:33:38.517 INFO:tasks.cephadm:Deploying osd.1 on smithi112 with /dev/vg_nvme/lv_4... 2024-03-19T01:33:38.517 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-03-19T01:33:38.682 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:38 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:33:38.683 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:38 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:38.683 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:38 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:38.683 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:38 smithi099 ceph-mon[62951]: from='osd.0 [v2:172.21.15.99:6802/2935260112,v1:172.21.15.99:6803/2935260112]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-19T01:33:38.758 INFO:teuthology.orchestra.run.smithi112.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.b/config 2024-03-19T01:33:38.806 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:38 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:33:38.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:38 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:38.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:38 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:38.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:38 smithi115 ceph-mon[67332]: from='osd.0 [v2:172.21.15.99:6802/2935260112,v1:172.21.15.99:6803/2935260112]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-19T01:33:38.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:38 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:33:38.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:38 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:38.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:38 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:38.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:38 smithi112 ceph-mon[66819]: from='osd.0 [v2:172.21.15.99:6802/2935260112,v1:172.21.15.99:6803/2935260112]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-19T01:33:40.055 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:39 smithi112 ceph-mon[66819]: pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:40.055 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:39 smithi112 ceph-mon[66819]: from='osd.0 [v2:172.21.15.99:6802/2935260112,v1:172.21.15.99:6803/2935260112]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-03-19T01:33:40.055 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:39 smithi112 ceph-mon[66819]: osdmap e6: 1 total, 0 up, 1 in 2024-03-19T01:33:40.055 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:39 smithi112 ceph-mon[66819]: from='osd.0 [v2:172.21.15.99:6802/2935260112,v1:172.21.15.99:6803/2935260112]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2024-03-19T01:33:40.055 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:39 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-19T01:33:40.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:39 smithi099 ceph-mon[62951]: pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:40.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:39 smithi099 ceph-mon[62951]: from='osd.0 [v2:172.21.15.99:6802/2935260112,v1:172.21.15.99:6803/2935260112]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-03-19T01:33:40.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:39 smithi099 ceph-mon[62951]: osdmap e6: 1 total, 0 up, 1 in 2024-03-19T01:33:40.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:39 smithi099 ceph-mon[62951]: from='osd.0 [v2:172.21.15.99:6802/2935260112,v1:172.21.15.99:6803/2935260112]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2024-03-19T01:33:40.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:39 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-19T01:33:40.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:39 smithi115 ceph-mon[67332]: pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:40.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:39 smithi115 ceph-mon[67332]: from='osd.0 [v2:172.21.15.99:6802/2935260112,v1:172.21.15.99:6803/2935260112]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-03-19T01:33:40.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:39 smithi115 ceph-mon[67332]: osdmap e6: 1 total, 0 up, 1 in 2024-03-19T01:33:40.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:39 smithi115 ceph-mon[67332]: from='osd.0 [v2:172.21.15.99:6802/2935260112,v1:172.21.15.99:6803/2935260112]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2024-03-19T01:33:40.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:39 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-19T01:33:40.589 INFO:teuthology.orchestra.run.smithi112.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-03-19T01:33:40.590 INFO:teuthology.orchestra.run.smithi112.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-03-19T01:33:40.590 INFO:teuthology.orchestra.run.smithi112.stderr: stderr: 10+0 records in 2024-03-19T01:33:40.590 INFO:teuthology.orchestra.run.smithi112.stderr:10+0 records out 2024-03-19T01:33:40.590 INFO:teuthology.orchestra.run.smithi112.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0127987 s, 819 MB/s 2024-03-19T01:33:40.590 INFO:teuthology.orchestra.run.smithi112.stderr: stderr: 2024-03-19T01:33:40.590 INFO:teuthology.orchestra.run.smithi112.stderr:--> Zapping successful for: 2024-03-19T01:33:40.909 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:40 smithi112 ceph-mon[66819]: from='osd.0 [v2:172.21.15.99:6802/2935260112,v1:172.21.15.99:6803/2935260112]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2024-03-19T01:33:40.909 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:40 smithi112 ceph-mon[66819]: osdmap e7: 1 total, 0 up, 1 in 2024-03-19T01:33:40.910 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:40 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-19T01:33:40.910 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:40 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-19T01:33:40.910 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:40 smithi112 ceph-mon[66819]: from='osd.0 [v2:172.21.15.99:6802/2935260112,v1:172.21.15.99:6803/2935260112]' entity='osd.0' 2024-03-19T01:33:41.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:40 smithi099 ceph-mon[62951]: from='osd.0 [v2:172.21.15.99:6802/2935260112,v1:172.21.15.99:6803/2935260112]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2024-03-19T01:33:41.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:40 smithi099 ceph-mon[62951]: osdmap e7: 1 total, 0 up, 1 in 2024-03-19T01:33:41.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:40 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-19T01:33:41.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:40 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-19T01:33:41.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:40 smithi099 ceph-mon[62951]: from='osd.0 [v2:172.21.15.99:6802/2935260112,v1:172.21.15.99:6803/2935260112]' entity='osd.0' 2024-03-19T01:33:41.092 INFO:journalctl@ceph.osd.0.smithi099.stdout:Mar 19 01:33:40 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-0[77885]: 2024-03-19T01:33:40.779+0000 7f97111fc700 -1 osd.0 0 waiting for initial osdmap 2024-03-19T01:33:41.092 INFO:journalctl@ceph.osd.0.smithi099.stdout:Mar 19 01:33:40 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-0[77885]: 2024-03-19T01:33:40.980+0000 7f9708fe7700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-19T01:33:41.161 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph orch daemon add osd smithi112:vg_nvme/lv_4 2024-03-19T01:33:41.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:40 smithi115 ceph-mon[67332]: from='osd.0 [v2:172.21.15.99:6802/2935260112,v1:172.21.15.99:6803/2935260112]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2024-03-19T01:33:41.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:40 smithi115 ceph-mon[67332]: osdmap e7: 1 total, 0 up, 1 in 2024-03-19T01:33:41.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:40 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-19T01:33:41.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:40 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-19T01:33:41.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:40 smithi115 ceph-mon[67332]: from='osd.0 [v2:172.21.15.99:6802/2935260112,v1:172.21.15.99:6803/2935260112]' entity='osd.0' 2024-03-19T01:33:41.401 INFO:teuthology.orchestra.run.smithi112.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.b/config 2024-03-19T01:33:42.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:41 smithi112 ceph-mon[66819]: purged_snaps scrub starts 2024-03-19T01:33:42.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:41 smithi112 ceph-mon[66819]: purged_snaps scrub ok 2024-03-19T01:33:42.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:41 smithi112 ceph-mon[66819]: pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:42.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:41 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-19T01:33:42.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:41 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:42.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:41 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:42.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:41 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:33:42.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:41 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:42.113 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:41 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:33:42.113 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:41 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:33:42.113 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:41 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:42.113 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:41 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-19T01:33:42.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:41 smithi115 ceph-mon[67332]: purged_snaps scrub starts 2024-03-19T01:33:42.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:41 smithi115 ceph-mon[67332]: purged_snaps scrub ok 2024-03-19T01:33:42.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:41 smithi115 ceph-mon[67332]: pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:42.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:41 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-19T01:33:42.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:41 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:42.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:41 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:42.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:41 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:33:42.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:41 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:42.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:41 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:33:42.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:41 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:33:42.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:41 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:42.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:41 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-19T01:33:42.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:41 smithi099 ceph-mon[62951]: purged_snaps scrub starts 2024-03-19T01:33:42.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:41 smithi099 ceph-mon[62951]: purged_snaps scrub ok 2024-03-19T01:33:42.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:41 smithi099 ceph-mon[62951]: pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:42.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:41 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-19T01:33:42.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:41 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:42.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:41 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:42.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:41 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:33:42.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:41 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:42.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:41 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:33:42.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:41 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:33:42.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:41 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:42.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:41 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-19T01:33:42.924 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:33:42.922+0000 7f77d43f3700 1 -- 172.21.15.112:0/1922648558 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f77cc101230 msgr2=0x7f77cc101650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:33:42.924 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:33:42.922+0000 7f77d43f3700 1 --2- 172.21.15.112:0/1922648558 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f77cc101230 0x7f77cc101650 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7f77c80099c0 tx=0x7f77c8009cd0 comp rx=0 tx=0).stop 2024-03-19T01:33:42.924 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:33:42.922+0000 7f77d43f3700 1 -- 172.21.15.112:0/1922648558 shutdown_connections 2024-03-19T01:33:42.925 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:33:42.922+0000 7f77d43f3700 1 --2- 172.21.15.112:0/1922648558 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f77cc102dd0 0x7f77cc109680 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:33:42.925 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:33:42.922+0000 7f77d43f3700 1 --2- 172.21.15.112:0/1922648558 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f77cc101230 0x7f77cc101650 unknown :-1 s=CLOSED pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:33:42.925 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:33:42.922+0000 7f77d43f3700 1 --2- 172.21.15.112:0/1922648558 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f77cc102430 0x7f77cc102890 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:33:42.925 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:33:42.922+0000 7f77d43f3700 1 -- 172.21.15.112:0/1922648558 >> 172.21.15.112:0/1922648558 conn(0x7f77cc0fc7f0 msgr2=0x7f77cc0fec10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:33:42.925 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:33:42.922+0000 7f77d43f3700 1 -- 172.21.15.112:0/1922648558 shutdown_connections 2024-03-19T01:33:42.925 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:33:42.922+0000 7f77d43f3700 1 -- 172.21.15.112:0/1922648558 wait complete. 2024-03-19T01:33:42.925 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:33:42.922+0000 7f77d43f3700 1 Processor -- start 2024-03-19T01:33:42.925 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:33:42.922+0000 7f77d43f3700 1 -- start start 2024-03-19T01:33:42.925 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:33:42.923+0000 7f77d43f3700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f77cc101230 0x7f77cc198630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:33:42.925 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:33:42.923+0000 7f77d43f3700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f77cc102430 0x7f77cc198b70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:33:42.925 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:33:42.923+0000 7f77d43f3700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f77cc102dd0 0x7f77cc19fb80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:33:42.925 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:33:42.923+0000 7f77d43f3700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f77cc10ce70 con 0x7f77cc102dd0 2024-03-19T01:33:42.925 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:33:42.923+0000 7f77d43f3700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f77cc10ccf0 con 0x7f77cc101230 2024-03-19T01:33:42.925 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:33:42.923+0000 7f77d43f3700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f77cc10cff0 con 0x7f77cc102430 2024-03-19T01:33:42.926 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:33:42.923+0000 7f77d218f700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f77cc101230 0x7f77cc198630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:33:42.926 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:33:42.923+0000 7f77d218f700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f77cc101230 0x7f77cc198630 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.112:47364/0 (socket says 172.21.15.112:47364) 2024-03-19T01:33:42.926 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:33:42.923+0000 7f77d218f700 1 -- 172.21.15.112:0/4012766605 learned_addr learned my addr 172.21.15.112:0/4012766605 (peer_addr_for_me v2:172.21.15.112:0/0) 2024-03-19T01:33:42.926 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:33:42.923+0000 7f77d2990700 1 --2- 172.21.15.112:0/4012766605 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f77cc102dd0 0x7f77cc19fb80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:33:42.926 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:33:42.923+0000 7f77d198e700 1 --2- 172.21.15.112:0/4012766605 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f77cc102430 0x7f77cc198b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:33:42.926 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:33:42.923+0000 7f77d218f700 1 -- 172.21.15.112:0/4012766605 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f77cc102430 msgr2=0x7f77cc198b70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:33:42.926 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:33:42.923+0000 7f77d218f700 1 --2- 172.21.15.112:0/4012766605 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f77cc102430 0x7f77cc198b70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:33:42.926 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:33:42.923+0000 7f77d218f700 1 -- 172.21.15.112:0/4012766605 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f77cc102dd0 msgr2=0x7f77cc19fb80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:33:42.926 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:33:42.923+0000 7f77d218f700 1 --2- 172.21.15.112:0/4012766605 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f77cc102dd0 0x7f77cc19fb80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:33:42.926 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:33:42.923+0000 7f77d218f700 1 -- 172.21.15.112:0/4012766605 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f77cc1a0220 con 0x7f77cc101230 2024-03-19T01:33:42.926 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:33:42.923+0000 7f77d2990700 1 --2- 172.21.15.112:0/4012766605 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f77cc102dd0 0x7f77cc19fb80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-19T01:33:42.926 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:33:42.923+0000 7f77d218f700 1 --2- 172.21.15.112:0/4012766605 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f77cc101230 0x7f77cc198630 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7f77c800c010 tx=0x7f77c8009580 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:33:42.926 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:33:42.923+0000 7f77c37fe700 1 -- 172.21.15.112:0/4012766605 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f77c8026070 con 0x7f77cc101230 2024-03-19T01:33:42.926 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:33:42.923+0000 7f77d43f3700 1 -- 172.21.15.112:0/4012766605 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f77cc1a04b0 con 0x7f77cc101230 2024-03-19T01:33:42.927 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:33:42.923+0000 7f77c37fe700 1 -- 172.21.15.112:0/4012766605 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f77c8004150 con 0x7f77cc101230 2024-03-19T01:33:42.927 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:33:42.923+0000 7f77c37fe700 1 -- 172.21.15.112:0/4012766605 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f77c800fe50 con 0x7f77cc101230 2024-03-19T01:33:42.927 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:33:42.923+0000 7f77d43f3700 1 -- 172.21.15.112:0/4012766605 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f77cc1a0a60 con 0x7f77cc101230 2024-03-19T01:33:42.928 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:33:42.924+0000 7f77d43f3700 1 -- 172.21.15.112:0/4012766605 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f77cc068fd0 con 0x7f77cc101230 2024-03-19T01:33:42.928 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:33:42.925+0000 7f77c37fe700 1 -- 172.21.15.112:0/4012766605 <== mon.2 v2:172.21.15.112:3300/0 4 ==== mgrmap(e 15) ==== 137234+0+0 (secure 0 0 0) 0x7f77c8029070 con 0x7f77cc101230 2024-03-19T01:33:42.928 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:33:42.926+0000 7f77c37fe700 1 --2- 172.21.15.112:0/4012766605 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f77b80a29a0 0x7f77b80a4e60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:33:42.928 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:33:42.926+0000 7f77c37fe700 1 -- 172.21.15.112:0/4012766605 <== mon.2 v2:172.21.15.112:3300/0 5 ==== osd_map(9..9 src has 1..9) ==== 1767+0+0 (secure 0 0 0) 0x7f77c80ce120 con 0x7f77cc101230 2024-03-19T01:33:42.928 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:33:42.926+0000 7f77d198e700 1 --2- 172.21.15.112:0/4012766605 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f77b80a29a0 0x7f77b80a4e60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:33:42.928 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:33:42.926+0000 7f77d198e700 1 --2- 172.21.15.112:0/4012766605 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f77b80a29a0 0x7f77b80a4e60 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7f77bc006fd0 tx=0x7f77bc008040 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:33:42.930 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:33:42.930+0000 7f77c37fe700 1 -- 172.21.15.112:0/4012766605 <== mon.2 v2:172.21.15.112:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f77c8030080 con 0x7f77cc101230 2024-03-19T01:33:43.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:42 smithi112 ceph-mon[66819]: Detected new or changed devices on smithi099 2024-03-19T01:33:43.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:42 smithi112 ceph-mon[66819]: Adjusting osd_memory_target on smithi099 to 17088M 2024-03-19T01:33:43.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:42 smithi112 ceph-mon[66819]: osd.0 [v2:172.21.15.99:6802/2935260112,v1:172.21.15.99:6803/2935260112] boot 2024-03-19T01:33:43.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:42 smithi112 ceph-mon[66819]: osdmap e8: 1 total, 1 up, 1 in 2024-03-19T01:33:43.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:42 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-19T01:33:43.134 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:33:43.133+0000 7f77d43f3700 1 -- 172.21.15.112:0/4012766605 --> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi112:vg_nvme/lv_4", "target": ["mon-mgr", ""]}) -- 0x7f77cc0fe9b0 con 0x7f77b80a29a0 2024-03-19T01:33:43.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:42 smithi115 ceph-mon[67332]: Detected new or changed devices on smithi099 2024-03-19T01:33:43.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:42 smithi115 ceph-mon[67332]: Adjusting osd_memory_target on smithi099 to 17088M 2024-03-19T01:33:43.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:42 smithi115 ceph-mon[67332]: osd.0 [v2:172.21.15.99:6802/2935260112,v1:172.21.15.99:6803/2935260112] boot 2024-03-19T01:33:43.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:42 smithi115 ceph-mon[67332]: osdmap e8: 1 total, 1 up, 1 in 2024-03-19T01:33:43.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:42 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-19T01:33:43.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:42 smithi099 ceph-mon[62951]: Detected new or changed devices on smithi099 2024-03-19T01:33:43.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:42 smithi099 ceph-mon[62951]: Adjusting osd_memory_target on smithi099 to 17088M 2024-03-19T01:33:43.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:42 smithi099 ceph-mon[62951]: osd.0 [v2:172.21.15.99:6802/2935260112,v1:172.21.15.99:6803/2935260112] boot 2024-03-19T01:33:43.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:42 smithi099 ceph-mon[62951]: osdmap e8: 1 total, 1 up, 1 in 2024-03-19T01:33:43.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:42 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-19T01:33:44.039 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:43 smithi112 ceph-mon[66819]: pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:44.039 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:43 smithi112 ceph-mon[66819]: osdmap e9: 1 total, 1 up, 1 in 2024-03-19T01:33:44.039 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:43 smithi112 ceph-mon[66819]: from='client.24143 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi112:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:33:44.040 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:43 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-19T01:33:44.040 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:43 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-19T01:33:44.040 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:43 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:33:44.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:43 smithi115 ceph-mon[67332]: pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:44.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:43 smithi115 ceph-mon[67332]: osdmap e9: 1 total, 1 up, 1 in 2024-03-19T01:33:44.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:43 smithi115 ceph-mon[67332]: from='client.24143 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi112:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:33:44.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:43 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-19T01:33:44.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:43 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-19T01:33:44.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:43 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:33:44.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:43 smithi099 ceph-mon[62951]: pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-19T01:33:44.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:43 smithi099 ceph-mon[62951]: osdmap e9: 1 total, 1 up, 1 in 2024-03-19T01:33:44.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:43 smithi099 ceph-mon[62951]: from='client.24143 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi112:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:33:44.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:43 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-19T01:33:44.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:43 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-19T01:33:44.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:43 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:33:44.939 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:44 smithi112 ceph-mon[66819]: pgmap v48: 0 pgs: ; 0 B data, 31 MiB used, 89 GiB / 89 GiB avail 2024-03-19T01:33:45.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:44 smithi115 ceph-mon[67332]: pgmap v48: 0 pgs: ; 0 B data, 31 MiB used, 89 GiB / 89 GiB avail 2024-03-19T01:33:45.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:44 smithi099 ceph-mon[62951]: pgmap v48: 0 pgs: ; 0 B data, 31 MiB used, 89 GiB / 89 GiB avail 2024-03-19T01:33:46.207 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:45 smithi115 ceph-mon[67332]: from='client.? 172.21.15.112:0/3379200136' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a48f9173-4a06-49f5-bca2-2c9047b8f666"}]: dispatch 2024-03-19T01:33:46.208 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:45 smithi115 ceph-mon[67332]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a48f9173-4a06-49f5-bca2-2c9047b8f666"}]: dispatch 2024-03-19T01:33:46.208 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:45 smithi115 ceph-mon[67332]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a48f9173-4a06-49f5-bca2-2c9047b8f666"}]': finished 2024-03-19T01:33:46.208 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:45 smithi115 ceph-mon[67332]: osdmap e10: 2 total, 1 up, 2 in 2024-03-19T01:33:46.208 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:45 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-19T01:33:46.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:45 smithi099 ceph-mon[62951]: from='client.? 172.21.15.112:0/3379200136' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a48f9173-4a06-49f5-bca2-2c9047b8f666"}]: dispatch 2024-03-19T01:33:46.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:45 smithi099 ceph-mon[62951]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a48f9173-4a06-49f5-bca2-2c9047b8f666"}]: dispatch 2024-03-19T01:33:46.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:45 smithi099 ceph-mon[62951]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a48f9173-4a06-49f5-bca2-2c9047b8f666"}]': finished 2024-03-19T01:33:46.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:45 smithi099 ceph-mon[62951]: osdmap e10: 2 total, 1 up, 2 in 2024-03-19T01:33:46.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:45 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-19T01:33:46.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:45 smithi112 ceph-mon[66819]: from='client.? 172.21.15.112:0/3379200136' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a48f9173-4a06-49f5-bca2-2c9047b8f666"}]: dispatch 2024-03-19T01:33:46.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:45 smithi112 ceph-mon[66819]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a48f9173-4a06-49f5-bca2-2c9047b8f666"}]: dispatch 2024-03-19T01:33:46.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:45 smithi112 ceph-mon[66819]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a48f9173-4a06-49f5-bca2-2c9047b8f666"}]': finished 2024-03-19T01:33:46.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:45 smithi112 ceph-mon[66819]: osdmap e10: 2 total, 1 up, 2 in 2024-03-19T01:33:46.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:45 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-19T01:33:47.263 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:46 smithi112 ceph-mon[66819]: from='client.? 172.21.15.112:0/3864825960' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-19T01:33:47.263 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:46 smithi112 ceph-mon[66819]: pgmap v50: 0 pgs: ; 0 B data, 31 MiB used, 89 GiB / 89 GiB avail 2024-03-19T01:33:47.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:46 smithi115 ceph-mon[67332]: from='client.? 172.21.15.112:0/3864825960' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-19T01:33:47.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:46 smithi115 ceph-mon[67332]: pgmap v50: 0 pgs: ; 0 B data, 31 MiB used, 89 GiB / 89 GiB avail 2024-03-19T01:33:47.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:46 smithi099 ceph-mon[62951]: from='client.? 172.21.15.112:0/3864825960' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-19T01:33:47.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:46 smithi099 ceph-mon[62951]: pgmap v50: 0 pgs: ; 0 B data, 31 MiB used, 89 GiB / 89 GiB avail 2024-03-19T01:33:50.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:49 smithi115 ceph-mon[67332]: pgmap v51: 0 pgs: ; 0 B data, 31 MiB used, 89 GiB / 89 GiB avail 2024-03-19T01:33:50.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:49 smithi099 ceph-mon[62951]: pgmap v51: 0 pgs: ; 0 B data, 31 MiB used, 89 GiB / 89 GiB avail 2024-03-19T01:33:50.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:49 smithi112 ceph-mon[66819]: pgmap v51: 0 pgs: ; 0 B data, 31 MiB used, 89 GiB / 89 GiB avail 2024-03-19T01:33:52.032 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:51 smithi112 ceph-mon[66819]: pgmap v52: 0 pgs: ; 0 B data, 31 MiB used, 89 GiB / 89 GiB avail 2024-03-19T01:33:52.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:51 smithi115 ceph-mon[67332]: pgmap v52: 0 pgs: ; 0 B data, 31 MiB used, 89 GiB / 89 GiB avail 2024-03-19T01:33:52.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:51 smithi099 ceph-mon[62951]: pgmap v52: 0 pgs: ; 0 B data, 31 MiB used, 89 GiB / 89 GiB avail 2024-03-19T01:33:54.032 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:53 smithi112 ceph-mon[66819]: pgmap v53: 0 pgs: ; 0 B data, 31 MiB used, 89 GiB / 89 GiB avail 2024-03-19T01:33:54.032 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:53 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-19T01:33:54.033 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:53 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:33:54.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:53 smithi115 ceph-mon[67332]: pgmap v53: 0 pgs: ; 0 B data, 31 MiB used, 89 GiB / 89 GiB avail 2024-03-19T01:33:54.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:53 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-19T01:33:54.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:53 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:33:54.093 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:53 smithi099 ceph-mon[62951]: pgmap v53: 0 pgs: ; 0 B data, 31 MiB used, 89 GiB / 89 GiB avail 2024-03-19T01:33:54.094 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:53 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-19T01:33:54.094 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:53 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:33:55.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:54 smithi115 ceph-mon[67332]: Deploying daemon osd.1 on smithi112 2024-03-19T01:33:55.058 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:54 smithi112 ceph-mon[66819]: Deploying daemon osd.1 on smithi112 2024-03-19T01:33:55.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:54 smithi099 ceph-mon[62951]: Deploying daemon osd.1 on smithi112 2024-03-19T01:33:55.958 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:55 smithi112 ceph-mon[66819]: pgmap v54: 0 pgs: ; 0 B data, 31 MiB used, 89 GiB / 89 GiB avail 2024-03-19T01:33:56.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:55 smithi115 ceph-mon[67332]: pgmap v54: 0 pgs: ; 0 B data, 31 MiB used, 89 GiB / 89 GiB avail 2024-03-19T01:33:56.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:55 smithi099 ceph-mon[62951]: pgmap v54: 0 pgs: ; 0 B data, 31 MiB used, 89 GiB / 89 GiB avail 2024-03-19T01:33:57.916 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:57 smithi112 ceph-mon[66819]: pgmap v55: 0 pgs: ; 0 B data, 31 MiB used, 89 GiB / 89 GiB avail 2024-03-19T01:33:58.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:57 smithi115 ceph-mon[67332]: pgmap v55: 0 pgs: ; 0 B data, 31 MiB used, 89 GiB / 89 GiB avail 2024-03-19T01:33:58.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:57 smithi099 ceph-mon[62951]: pgmap v55: 0 pgs: ; 0 B data, 31 MiB used, 89 GiB / 89 GiB avail 2024-03-19T01:33:59.806 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:59 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:59.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:59 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:59.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:59 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:33:59.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:33:59 smithi115 ceph-mon[67332]: pgmap v56: 0 pgs: ; 0 B data, 31 MiB used, 89 GiB / 89 GiB avail 2024-03-19T01:33:59.842 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:59 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:59.842 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:59 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:59.842 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:59 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:33:59.842 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:33:59 smithi099 ceph-mon[62951]: pgmap v56: 0 pgs: ; 0 B data, 31 MiB used, 89 GiB / 89 GiB avail 2024-03-19T01:33:59.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:59 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:59.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:59 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:33:59.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:59 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:33:59.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:33:59 smithi112 ceph-mon[66819]: pgmap v56: 0 pgs: ; 0 B data, 31 MiB used, 89 GiB / 89 GiB avail 2024-03-19T01:34:01.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:01 smithi112 ceph-mon[66819]: pgmap v57: 0 pgs: ; 0 B data, 31 MiB used, 89 GiB / 89 GiB avail 2024-03-19T01:34:01.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:01 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:01.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:01 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:01.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:01 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:34:01.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:01 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:34:01.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:01 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:02.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:01 smithi115 ceph-mon[67332]: pgmap v57: 0 pgs: ; 0 B data, 31 MiB used, 89 GiB / 89 GiB avail 2024-03-19T01:34:02.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:01 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:02.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:01 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:02.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:01 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:34:02.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:01 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:34:02.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:01 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:02.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:01 smithi099 ceph-mon[62951]: pgmap v57: 0 pgs: ; 0 B data, 31 MiB used, 89 GiB / 89 GiB avail 2024-03-19T01:34:02.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:01 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:02.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:01 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:02.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:01 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:34:02.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:01 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:34:02.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:01 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:02.257 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:34:02.255+0000 7f77c37fe700 1 -- 172.21.15.112:0/4012766605 <== mgr.14150 v2:172.21.15.99:6800/4272410295 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+37 (secure 0 0 0) 0x7f77cc0fe9b0 con 0x7f77b80a29a0 2024-03-19T01:34:02.257 INFO:teuthology.orchestra.run.smithi112.stdout:Created osd(s) 1 on host 'smithi112' 2024-03-19T01:34:02.258 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:34:02.257+0000 7f77d43f3700 1 -- 172.21.15.112:0/4012766605 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f77b80a29a0 msgr2=0x7f77b80a4e60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:34:02.258 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:34:02.257+0000 7f77d43f3700 1 --2- 172.21.15.112:0/4012766605 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f77b80a29a0 0x7f77b80a4e60 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7f77bc006fd0 tx=0x7f77bc008040 comp rx=0 tx=0).stop 2024-03-19T01:34:02.259 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:34:02.257+0000 7f77d43f3700 1 -- 172.21.15.112:0/4012766605 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f77cc101230 msgr2=0x7f77cc198630 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:34:02.259 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:34:02.257+0000 7f77d43f3700 1 --2- 172.21.15.112:0/4012766605 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f77cc101230 0x7f77cc198630 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7f77c800c010 tx=0x7f77c8009580 comp rx=0 tx=0).stop 2024-03-19T01:34:02.259 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:34:02.257+0000 7f77d43f3700 1 -- 172.21.15.112:0/4012766605 shutdown_connections 2024-03-19T01:34:02.259 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:34:02.257+0000 7f77d43f3700 1 --2- 172.21.15.112:0/4012766605 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f77b80a29a0 0x7f77b80a4e60 unknown :-1 s=CLOSED pgs=53 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:34:02.259 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:34:02.257+0000 7f77d43f3700 1 --2- 172.21.15.112:0/4012766605 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f77cc102dd0 0x7f77cc19fb80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:34:02.259 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:34:02.257+0000 7f77d43f3700 1 --2- 172.21.15.112:0/4012766605 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f77cc101230 0x7f77cc198630 unknown :-1 s=CLOSED pgs=8 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:34:02.259 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:34:02.257+0000 7f77d43f3700 1 --2- 172.21.15.112:0/4012766605 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f77cc102430 0x7f77cc198b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:34:02.259 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:34:02.257+0000 7f77d43f3700 1 -- 172.21.15.112:0/4012766605 >> 172.21.15.112:0/4012766605 conn(0x7f77cc0fc7f0 msgr2=0x7f77cc0fe2a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:34:02.259 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:34:02.258+0000 7f77d43f3700 1 -- 172.21.15.112:0/4012766605 shutdown_connections 2024-03-19T01:34:02.259 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:34:02.258+0000 7f77d43f3700 1 -- 172.21.15.112:0/4012766605 wait complete. 2024-03-19T01:34:02.923 DEBUG:teuthology.orchestra.run.smithi112:osd.1> sudo journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@osd.1.service 2024-03-19T01:34:02.925 INFO:tasks.cephadm:Deploying osd.2 on smithi115 with /dev/vg_nvme/lv_4... 2024-03-19T01:34:02.925 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-03-19T01:34:03.166 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.c/config 2024-03-19T01:34:03.509 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:03 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:03.509 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:03 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:34:03.509 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:03 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:03.509 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:03 smithi112 ceph-mon[66819]: from='osd.1 [v2:172.21.15.112:6800/1468854289,v1:172.21.15.112:6801/1468854289]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-19T01:34:03.509 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:03 smithi112 ceph-mon[66819]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-19T01:34:03.509 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:03 smithi112 ceph-mon[66819]: pgmap v58: 0 pgs: ; 0 B data, 31 MiB used, 89 GiB / 89 GiB avail 2024-03-19T01:34:03.555 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:03 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:03.555 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:03 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:34:03.555 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:03 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:03.555 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:03 smithi115 ceph-mon[67332]: from='osd.1 [v2:172.21.15.112:6800/1468854289,v1:172.21.15.112:6801/1468854289]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-19T01:34:03.555 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:03 smithi115 ceph-mon[67332]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-19T01:34:03.555 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:03 smithi115 ceph-mon[67332]: pgmap v58: 0 pgs: ; 0 B data, 31 MiB used, 89 GiB / 89 GiB avail 2024-03-19T01:34:03.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:03 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:03.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:03 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:34:03.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:03 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:03.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:03 smithi099 ceph-mon[62951]: from='osd.1 [v2:172.21.15.112:6800/1468854289,v1:172.21.15.112:6801/1468854289]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-19T01:34:03.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:03 smithi099 ceph-mon[62951]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-19T01:34:03.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:03 smithi099 ceph-mon[62951]: pgmap v58: 0 pgs: ; 0 B data, 31 MiB used, 89 GiB / 89 GiB avail 2024-03-19T01:34:04.293 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:04 smithi115 ceph-mon[67332]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-03-19T01:34:04.293 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:04 smithi115 ceph-mon[67332]: from='osd.1 [v2:172.21.15.112:6800/1468854289,v1:172.21.15.112:6801/1468854289]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-03-19T01:34:04.293 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:04 smithi115 ceph-mon[67332]: osdmap e11: 2 total, 1 up, 2 in 2024-03-19T01:34:04.293 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:04 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-19T01:34:04.293 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:04 smithi115 ceph-mon[67332]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-03-19T01:34:04.499 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:04 smithi112 ceph-mon[66819]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-03-19T01:34:04.499 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:04 smithi112 ceph-mon[66819]: from='osd.1 [v2:172.21.15.112:6800/1468854289,v1:172.21.15.112:6801/1468854289]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-03-19T01:34:04.499 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:04 smithi112 ceph-mon[66819]: osdmap e11: 2 total, 1 up, 2 in 2024-03-19T01:34:04.499 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:04 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-19T01:34:04.499 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:04 smithi112 ceph-mon[66819]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-03-19T01:34:04.499 INFO:journalctl@ceph.osd.1.smithi112.stdout:Mar 19 01:34:04 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-1[71392]: 2024-03-19T01:34:04.240+0000 7fe304530700 -1 osd.1 0 waiting for initial osdmap 2024-03-19T01:34:04.499 INFO:journalctl@ceph.osd.1.smithi112.stdout:Mar 19 01:34:04 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-1[71392]: 2024-03-19T01:34:04.246+0000 7fe300323700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-19T01:34:04.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:04 smithi099 ceph-mon[62951]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-03-19T01:34:04.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:04 smithi099 ceph-mon[62951]: from='osd.1 [v2:172.21.15.112:6800/1468854289,v1:172.21.15.112:6801/1468854289]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-03-19T01:34:04.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:04 smithi099 ceph-mon[62951]: osdmap e11: 2 total, 1 up, 2 in 2024-03-19T01:34:04.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:04 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-19T01:34:04.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:04 smithi099 ceph-mon[62951]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-03-19T01:34:04.798 INFO:teuthology.orchestra.run.smithi115.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-03-19T01:34:04.798 INFO:teuthology.orchestra.run.smithi115.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-03-19T01:34:04.798 INFO:teuthology.orchestra.run.smithi115.stderr: stderr: 10+0 records in 2024-03-19T01:34:04.798 INFO:teuthology.orchestra.run.smithi115.stderr:10+0 records out 2024-03-19T01:34:04.798 INFO:teuthology.orchestra.run.smithi115.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0112265 s, 934 MB/s 2024-03-19T01:34:04.798 INFO:teuthology.orchestra.run.smithi115.stderr: stderr: 2024-03-19T01:34:04.798 INFO:teuthology.orchestra.run.smithi115.stderr:--> Zapping successful for: 2024-03-19T01:34:05.357 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph orch daemon add osd smithi115:vg_nvme/lv_4 2024-03-19T01:34:05.371 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:05 smithi115 ceph-mon[67332]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]': finished 2024-03-19T01:34:05.371 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:05 smithi115 ceph-mon[67332]: osdmap e12: 2 total, 1 up, 2 in 2024-03-19T01:34:05.371 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:05 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-19T01:34:05.371 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:05 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-19T01:34:05.371 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:05 smithi115 ceph-mon[67332]: pgmap v61: 0 pgs: ; 0 B data, 31 MiB used, 89 GiB / 89 GiB avail 2024-03-19T01:34:05.434 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:05 smithi112 ceph-mon[66819]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]': finished 2024-03-19T01:34:05.434 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:05 smithi112 ceph-mon[66819]: osdmap e12: 2 total, 1 up, 2 in 2024-03-19T01:34:05.435 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:05 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-19T01:34:05.435 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:05 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-19T01:34:05.435 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:05 smithi112 ceph-mon[66819]: pgmap v61: 0 pgs: ; 0 B data, 31 MiB used, 89 GiB / 89 GiB avail 2024-03-19T01:34:05.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:05 smithi099 ceph-mon[62951]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]': finished 2024-03-19T01:34:05.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:05 smithi099 ceph-mon[62951]: osdmap e12: 2 total, 1 up, 2 in 2024-03-19T01:34:05.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:05 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-19T01:34:05.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:05 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-19T01:34:05.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:05 smithi099 ceph-mon[62951]: pgmap v61: 0 pgs: ; 0 B data, 31 MiB used, 89 GiB / 89 GiB avail 2024-03-19T01:34:05.592 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.c/config 2024-03-19T01:34:06.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:06 smithi115 ceph-mon[67332]: purged_snaps scrub starts 2024-03-19T01:34:06.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:06 smithi115 ceph-mon[67332]: purged_snaps scrub ok 2024-03-19T01:34:06.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:06 smithi115 ceph-mon[67332]: osd.1 [v2:172.21.15.112:6800/1468854289,v1:172.21.15.112:6801/1468854289] boot 2024-03-19T01:34:06.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:06 smithi115 ceph-mon[67332]: osdmap e13: 2 total, 2 up, 2 in 2024-03-19T01:34:06.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:06 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-19T01:34:06.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:06 smithi115 ceph-mon[67332]: Detected new or changed devices on smithi112 2024-03-19T01:34:06.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:06 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:06.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:06 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:06.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:06 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:34:06.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:06 smithi115 ceph-mon[67332]: Adjusting osd_memory_target on smithi112 to 17080M 2024-03-19T01:34:06.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:06 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:06.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:06 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:34:06.558 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:06 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:34:06.558 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:06 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:06.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:06 smithi099 ceph-mon[62951]: purged_snaps scrub starts 2024-03-19T01:34:06.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:06 smithi099 ceph-mon[62951]: purged_snaps scrub ok 2024-03-19T01:34:06.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:06 smithi099 ceph-mon[62951]: osd.1 [v2:172.21.15.112:6800/1468854289,v1:172.21.15.112:6801/1468854289] boot 2024-03-19T01:34:06.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:06 smithi099 ceph-mon[62951]: osdmap e13: 2 total, 2 up, 2 in 2024-03-19T01:34:06.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:06 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-19T01:34:06.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:06 smithi099 ceph-mon[62951]: Detected new or changed devices on smithi112 2024-03-19T01:34:06.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:06 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:06.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:06 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:06.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:06 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:34:06.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:06 smithi099 ceph-mon[62951]: Adjusting osd_memory_target on smithi112 to 17080M 2024-03-19T01:34:06.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:06 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:06.593 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:06 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:34:06.593 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:06 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:34:06.593 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:06 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:06.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:06 smithi112 ceph-mon[66819]: purged_snaps scrub starts 2024-03-19T01:34:06.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:06 smithi112 ceph-mon[66819]: purged_snaps scrub ok 2024-03-19T01:34:06.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:06 smithi112 ceph-mon[66819]: osd.1 [v2:172.21.15.112:6800/1468854289,v1:172.21.15.112:6801/1468854289] boot 2024-03-19T01:34:06.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:06 smithi112 ceph-mon[66819]: osdmap e13: 2 total, 2 up, 2 in 2024-03-19T01:34:06.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:06 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-19T01:34:06.613 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:06 smithi112 ceph-mon[66819]: Detected new or changed devices on smithi112 2024-03-19T01:34:06.613 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:06 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:06.613 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:06 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:06.613 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:06 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:34:06.613 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:06 smithi112 ceph-mon[66819]: Adjusting osd_memory_target on smithi112 to 17080M 2024-03-19T01:34:06.613 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:06 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:06.613 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:06 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:34:06.613 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:06 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:34:06.613 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:06 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:07.223 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:34:07.219+0000 7f78933f3700 1 -- 172.21.15.115:0/2830383260 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f788c103630 msgr2=0x7f788c103a90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:34:07.223 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:34:07.219+0000 7f78933f3700 1 --2- 172.21.15.115:0/2830383260 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f788c103630 0x7f788c103a90 secure :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0x7f7888009990 tx=0x7f7888009ca0 comp rx=0 tx=0).stop 2024-03-19T01:34:07.223 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:34:07.219+0000 7f78933f3700 1 -- 172.21.15.115:0/2830383260 shutdown_connections 2024-03-19T01:34:07.223 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:34:07.219+0000 7f78933f3700 1 --2- 172.21.15.115:0/2830383260 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f788c103fd0 0x7f788c10a880 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:34:07.223 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:34:07.219+0000 7f78933f3700 1 --2- 172.21.15.115:0/2830383260 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f788c100650 0x7f788c100a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:34:07.223 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:34:07.219+0000 7f78933f3700 1 --2- 172.21.15.115:0/2830383260 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f788c103630 0x7f788c103a90 unknown :-1 s=CLOSED pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:34:07.223 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:34:07.219+0000 7f78933f3700 1 -- 172.21.15.115:0/2830383260 >> 172.21.15.115:0/2830383260 conn(0x7f788c0fbfb0 msgr2=0x7f788c0fe3f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:34:07.223 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:34:07.220+0000 7f78933f3700 1 -- 172.21.15.115:0/2830383260 shutdown_connections 2024-03-19T01:34:07.224 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:34:07.220+0000 7f78933f3700 1 -- 172.21.15.115:0/2830383260 wait complete. 2024-03-19T01:34:07.224 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:34:07.220+0000 7f78933f3700 1 Processor -- start 2024-03-19T01:34:07.224 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:34:07.220+0000 7f78933f3700 1 -- start start 2024-03-19T01:34:07.224 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:34:07.220+0000 7f78933f3700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f788c100650 0x7f788c198490 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:34:07.224 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:34:07.220+0000 7f78933f3700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f788c103630 0x7f788c1989d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:34:07.224 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:34:07.220+0000 7f78933f3700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f788c103fd0 0x7f788c19fa50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:34:07.224 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:34:07.220+0000 7f78933f3700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f788c10df40 con 0x7f788c103fd0 2024-03-19T01:34:07.224 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:34:07.220+0000 7f78933f3700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f788c10ddc0 con 0x7f788c100650 2024-03-19T01:34:07.224 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:34:07.220+0000 7f78933f3700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f788c10e0c0 con 0x7f788c103630 2024-03-19T01:34:07.224 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:34:07.220+0000 7f789098e700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f788c103630 0x7f788c1989d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:34:07.224 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:34:07.220+0000 7f789098e700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f788c103630 0x7f788c1989d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.115:38878/0 (socket says 172.21.15.115:38878) 2024-03-19T01:34:07.224 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:34:07.220+0000 7f789098e700 1 -- 172.21.15.115:0/1232863003 learned_addr learned my addr 172.21.15.115:0/1232863003 (peer_addr_for_me v2:172.21.15.115:0/0) 2024-03-19T01:34:07.225 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:34:07.221+0000 7f7891990700 1 --2- 172.21.15.115:0/1232863003 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f788c103fd0 0x7f788c19fa50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:34:07.225 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:34:07.221+0000 7f789118f700 1 --2- 172.21.15.115:0/1232863003 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f788c100650 0x7f788c198490 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:34:07.225 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:34:07.221+0000 7f789098e700 1 -- 172.21.15.115:0/1232863003 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f788c100650 msgr2=0x7f788c198490 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:34:07.225 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:34:07.221+0000 7f789098e700 1 --2- 172.21.15.115:0/1232863003 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f788c100650 0x7f788c198490 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:34:07.225 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:34:07.221+0000 7f789098e700 1 -- 172.21.15.115:0/1232863003 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f788c103fd0 msgr2=0x7f788c19fa50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:34:07.225 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:34:07.221+0000 7f789098e700 1 --2- 172.21.15.115:0/1232863003 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f788c103fd0 0x7f788c19fa50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:34:07.225 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:34:07.221+0000 7f789098e700 1 -- 172.21.15.115:0/1232863003 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f788c1a00f0 con 0x7f788c103630 2024-03-19T01:34:07.225 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:34:07.221+0000 7f7891990700 1 --2- 172.21.15.115:0/1232863003 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f788c103fd0 0x7f788c19fa50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-19T01:34:07.225 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:34:07.221+0000 7f789098e700 1 --2- 172.21.15.115:0/1232863003 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f788c103630 0x7f788c1989d0 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0x7f7888004790 tx=0x7f7888009580 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:34:07.225 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:34:07.221+0000 7f78827fc700 1 -- 172.21.15.115:0/1232863003 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7888027070 con 0x7f788c103630 2024-03-19T01:34:07.225 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:34:07.221+0000 7f78827fc700 1 -- 172.21.15.115:0/1232863003 <== mon.1 v2:172.21.15.115:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7888004210 con 0x7f788c103630 2024-03-19T01:34:07.225 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:34:07.221+0000 7f78827fc700 1 -- 172.21.15.115:0/1232863003 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f788800f460 con 0x7f788c103630 2024-03-19T01:34:07.226 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:34:07.221+0000 7f78933f3700 1 -- 172.21.15.115:0/1232863003 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f788c1a0380 con 0x7f788c103630 2024-03-19T01:34:07.226 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:34:07.221+0000 7f78933f3700 1 -- 172.21.15.115:0/1232863003 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f788c1a07d0 con 0x7f788c103630 2024-03-19T01:34:07.226 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:34:07.222+0000 7f78933f3700 1 -- 172.21.15.115:0/1232863003 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f788c068fd0 con 0x7f788c103630 2024-03-19T01:34:07.227 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:34:07.224+0000 7f78827fc700 1 -- 172.21.15.115:0/1232863003 <== mon.1 v2:172.21.15.115:3300/0 4 ==== mgrmap(e 15) ==== 137234+0+0 (secure 0 0 0) 0x7f788802a070 con 0x7f788c103630 2024-03-19T01:34:07.227 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:34:07.224+0000 7f78827fc700 1 --2- 172.21.15.115:0/1232863003 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f78780a2800 0x7f78780a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:34:07.227 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:34:07.225+0000 7f78827fc700 1 -- 172.21.15.115:0/1232863003 <== mon.1 v2:172.21.15.115:3300/0 5 ==== osd_map(14..14 src has 1..14) ==== 2309+0+0 (secure 0 0 0) 0x7f78880ce4f0 con 0x7f788c103630 2024-03-19T01:34:07.227 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:34:07.225+0000 7f789118f700 1 --2- 172.21.15.115:0/1232863003 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f78780a2800 0x7f78780a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:34:07.227 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:34:07.225+0000 7f789118f700 1 --2- 172.21.15.115:0/1232863003 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f78780a2800 0x7f78780a4cc0 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7f788c06a7f0 tx=0x7f787c00b5b0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:34:07.229 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:34:07.228+0000 7f78827fc700 1 -- 172.21.15.115:0/1232863003 <== mon.1 v2:172.21.15.115:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f7888013070 con 0x7f788c103630 2024-03-19T01:34:07.435 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:34:07.434+0000 7f78933f3700 1 -- 172.21.15.115:0/1232863003 --> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi115:vg_nvme/lv_4", "target": ["mon-mgr", ""]}) -- 0x7f788c062750 con 0x7f78780a2800 2024-03-19T01:34:07.435 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:07 smithi115 ceph-mon[67332]: osdmap e14: 2 total, 2 up, 2 in 2024-03-19T01:34:07.435 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:07 smithi115 ceph-mon[67332]: pgmap v64: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-03-19T01:34:07.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:07 smithi099 ceph-mon[62951]: osdmap e14: 2 total, 2 up, 2 in 2024-03-19T01:34:07.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:07 smithi099 ceph-mon[62951]: pgmap v64: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-03-19T01:34:07.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:07 smithi112 ceph-mon[66819]: osdmap e14: 2 total, 2 up, 2 in 2024-03-19T01:34:07.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:07 smithi112 ceph-mon[66819]: pgmap v64: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-03-19T01:34:08.292 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:08 smithi115 ceph-mon[67332]: from='client.24157 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi115:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:34:08.292 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:08 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-19T01:34:08.292 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:08 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-19T01:34:08.292 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:08 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:34:08.513 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:08 smithi099 ceph-mon[62951]: from='client.24157 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi115:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:34:08.513 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:08 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-19T01:34:08.513 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:08 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-19T01:34:08.513 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:08 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:34:08.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:08 smithi112 ceph-mon[66819]: from='client.24157 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi115:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:34:08.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:08 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-19T01:34:08.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:08 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-19T01:34:08.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:08 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:34:09.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:09 smithi115 ceph-mon[67332]: pgmap v65: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-03-19T01:34:09.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:09 smithi099 ceph-mon[62951]: pgmap v65: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-03-19T01:34:09.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:09 smithi112 ceph-mon[66819]: pgmap v65: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-03-19T01:34:10.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:10 smithi115 ceph-mon[67332]: from='client.? 172.21.15.115:0/3829762027' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b7715fdd-b6c9-4348-bd16-5eb61b427eef"}]: dispatch 2024-03-19T01:34:10.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:10 smithi115 ceph-mon[67332]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b7715fdd-b6c9-4348-bd16-5eb61b427eef"}]: dispatch 2024-03-19T01:34:10.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:10 smithi115 ceph-mon[67332]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b7715fdd-b6c9-4348-bd16-5eb61b427eef"}]': finished 2024-03-19T01:34:10.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:10 smithi115 ceph-mon[67332]: osdmap e15: 3 total, 2 up, 3 in 2024-03-19T01:34:10.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:10 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-19T01:34:10.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:10 smithi099 ceph-mon[62951]: from='client.? 172.21.15.115:0/3829762027' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b7715fdd-b6c9-4348-bd16-5eb61b427eef"}]: dispatch 2024-03-19T01:34:10.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:10 smithi099 ceph-mon[62951]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b7715fdd-b6c9-4348-bd16-5eb61b427eef"}]: dispatch 2024-03-19T01:34:10.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:10 smithi099 ceph-mon[62951]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b7715fdd-b6c9-4348-bd16-5eb61b427eef"}]': finished 2024-03-19T01:34:10.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:10 smithi099 ceph-mon[62951]: osdmap e15: 3 total, 2 up, 3 in 2024-03-19T01:34:10.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:10 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-19T01:34:10.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:10 smithi112 ceph-mon[66819]: from='client.? 172.21.15.115:0/3829762027' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b7715fdd-b6c9-4348-bd16-5eb61b427eef"}]: dispatch 2024-03-19T01:34:10.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:10 smithi112 ceph-mon[66819]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b7715fdd-b6c9-4348-bd16-5eb61b427eef"}]: dispatch 2024-03-19T01:34:10.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:10 smithi112 ceph-mon[66819]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b7715fdd-b6c9-4348-bd16-5eb61b427eef"}]': finished 2024-03-19T01:34:10.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:10 smithi112 ceph-mon[66819]: osdmap e15: 3 total, 2 up, 3 in 2024-03-19T01:34:10.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:10 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-19T01:34:11.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:11 smithi115 ceph-mon[67332]: from='client.? 172.21.15.115:0/3705769168' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-19T01:34:11.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:11 smithi115 ceph-mon[67332]: pgmap v67: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-03-19T01:34:11.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:11 smithi099 ceph-mon[62951]: from='client.? 172.21.15.115:0/3705769168' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-19T01:34:11.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:11 smithi099 ceph-mon[62951]: pgmap v67: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-03-19T01:34:11.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:11 smithi112 ceph-mon[66819]: from='client.? 172.21.15.115:0/3705769168' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-19T01:34:11.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:11 smithi112 ceph-mon[66819]: pgmap v67: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-03-19T01:34:14.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:13 smithi115 ceph-mon[67332]: pgmap v68: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-03-19T01:34:14.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:13 smithi099 ceph-mon[62951]: pgmap v68: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-03-19T01:34:14.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:13 smithi112 ceph-mon[66819]: pgmap v68: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-03-19T01:34:16.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:15 smithi115 ceph-mon[67332]: pgmap v69: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-03-19T01:34:16.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:15 smithi099 ceph-mon[62951]: pgmap v69: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-03-19T01:34:16.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:15 smithi112 ceph-mon[66819]: pgmap v69: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-03-19T01:34:18.016 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:17 smithi115 ceph-mon[67332]: pgmap v70: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-03-19T01:34:18.017 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:17 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-19T01:34:18.017 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:17 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:34:18.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:17 smithi099 ceph-mon[62951]: pgmap v70: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-03-19T01:34:18.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:17 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-19T01:34:18.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:17 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:34:18.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:17 smithi112 ceph-mon[66819]: pgmap v70: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-03-19T01:34:18.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:17 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-19T01:34:18.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:17 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:34:19.014 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:18 smithi115 ceph-mon[67332]: Deploying daemon osd.2 on smithi115 2024-03-19T01:34:19.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:18 smithi099 ceph-mon[62951]: Deploying daemon osd.2 on smithi115 2024-03-19T01:34:19.101 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:18 smithi112 ceph-mon[66819]: Deploying daemon osd.2 on smithi115 2024-03-19T01:34:20.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:19 smithi115 ceph-mon[67332]: pgmap v71: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-03-19T01:34:20.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:19 smithi099 ceph-mon[62951]: pgmap v71: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-03-19T01:34:20.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:19 smithi112 ceph-mon[66819]: pgmap v71: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-03-19T01:34:21.989 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:21 smithi115 ceph-mon[67332]: pgmap v72: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-03-19T01:34:22.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:21 smithi099 ceph-mon[62951]: pgmap v72: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-03-19T01:34:22.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:21 smithi112 ceph-mon[66819]: pgmap v72: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-03-19T01:34:23.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:23 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:23.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:23 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:23.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:23 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:34:23.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:23 smithi115 ceph-mon[67332]: pgmap v73: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-03-19T01:34:23.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:23 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:23.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:23 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:23.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:23 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:34:23.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:23 smithi099 ceph-mon[62951]: pgmap v73: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-03-19T01:34:23.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:23 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:23.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:23 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:23.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:23 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:34:23.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:23 smithi112 ceph-mon[66819]: pgmap v73: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-03-19T01:34:25.098 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:34:25.096+0000 7f78827fc700 1 -- 172.21.15.115:0/1232863003 <== mgr.14150 v2:172.21.15.99:6800/4272410295 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+37 (secure 0 0 0) 0x7f788c062750 con 0x7f78780a2800 2024-03-19T01:34:25.098 INFO:teuthology.orchestra.run.smithi115.stdout:Created osd(s) 2 on host 'smithi115' 2024-03-19T01:34:25.099 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:34:25.098+0000 7f78933f3700 1 -- 172.21.15.115:0/1232863003 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f78780a2800 msgr2=0x7f78780a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:34:25.099 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:34:25.098+0000 7f78933f3700 1 --2- 172.21.15.115:0/1232863003 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f78780a2800 0x7f78780a4cc0 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7f788c06a7f0 tx=0x7f787c00b5b0 comp rx=0 tx=0).stop 2024-03-19T01:34:25.099 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:34:25.098+0000 7f78933f3700 1 -- 172.21.15.115:0/1232863003 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f788c103630 msgr2=0x7f788c1989d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:34:25.100 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:34:25.098+0000 7f78933f3700 1 --2- 172.21.15.115:0/1232863003 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f788c103630 0x7f788c1989d0 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0x7f7888004790 tx=0x7f7888009580 comp rx=0 tx=0).stop 2024-03-19T01:34:25.100 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:34:25.099+0000 7f78933f3700 1 -- 172.21.15.115:0/1232863003 shutdown_connections 2024-03-19T01:34:25.100 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:34:25.099+0000 7f78933f3700 1 --2- 172.21.15.115:0/1232863003 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f78780a2800 0x7f78780a4cc0 unknown :-1 s=CLOSED pgs=59 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:34:25.100 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:34:25.099+0000 7f78933f3700 1 --2- 172.21.15.115:0/1232863003 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f788c103fd0 0x7f788c19fa50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:34:25.100 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:34:25.099+0000 7f78933f3700 1 --2- 172.21.15.115:0/1232863003 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f788c100650 0x7f788c198490 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:34:25.100 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:34:25.099+0000 7f78933f3700 1 --2- 172.21.15.115:0/1232863003 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f788c103630 0x7f788c1989d0 unknown :-1 s=CLOSED pgs=12 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:34:25.100 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:34:25.099+0000 7f78933f3700 1 -- 172.21.15.115:0/1232863003 >> 172.21.15.115:0/1232863003 conn(0x7f788c0fbfb0 msgr2=0x7f788c106860 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:34:25.101 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:34:25.099+0000 7f78933f3700 1 -- 172.21.15.115:0/1232863003 shutdown_connections 2024-03-19T01:34:25.101 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:34:25.099+0000 7f78933f3700 1 -- 172.21.15.115:0/1232863003 wait complete. 2024-03-19T01:34:25.980 DEBUG:teuthology.orchestra.run.smithi115:osd.2> sudo journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@osd.2.service 2024-03-19T01:34:26.022 INFO:tasks.cephadm:Deploying osd.3 on smithi099 with /dev/vg_nvme/lv_3... 2024-03-19T01:34:26.022 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-03-19T01:34:26.045 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:25 smithi115 ceph-mon[67332]: pgmap v74: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-03-19T01:34:26.045 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:25 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:26.045 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:25 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:26.051 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:25 smithi099 ceph-mon[62951]: pgmap v74: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-03-19T01:34:26.052 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:25 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:26.052 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:25 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:26.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:25 smithi112 ceph-mon[66819]: pgmap v74: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-03-19T01:34:26.113 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:25 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:26.113 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:25 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:26.270 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:34:26.306 INFO:journalctl@ceph.osd.2.smithi115.stdout:Mar 19 01:34:26 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-2[72330]: 2024-03-19T01:34:26.143+0000 7fe9a3c9d7c0 -1 osd.2 0 log_to_monitors true 2024-03-19T01:34:27.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:27 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:27.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:27 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:27.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:27 smithi115 ceph-mon[67332]: from='osd.2 [v2:172.21.15.115:6800/3766690472,v1:172.21.15.115:6801/3766690472]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-19T01:34:27.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:27 smithi115 ceph-mon[67332]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-19T01:34:27.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:27 smithi115 ceph-mon[67332]: pgmap v75: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-03-19T01:34:27.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:27 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:27.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:27 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:27.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:27 smithi099 ceph-mon[62951]: from='osd.2 [v2:172.21.15.115:6800/3766690472,v1:172.21.15.115:6801/3766690472]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-19T01:34:27.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:27 smithi099 ceph-mon[62951]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-19T01:34:27.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:27 smithi099 ceph-mon[62951]: pgmap v75: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-03-19T01:34:27.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:27 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:27.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:27 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:27.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:27 smithi112 ceph-mon[66819]: from='osd.2 [v2:172.21.15.115:6800/3766690472,v1:172.21.15.115:6801/3766690472]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-19T01:34:27.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:27 smithi112 ceph-mon[66819]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-19T01:34:27.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:27 smithi112 ceph-mon[66819]: pgmap v75: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-03-19T01:34:28.120 INFO:teuthology.orchestra.run.smithi099.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-03-19T01:34:28.120 INFO:teuthology.orchestra.run.smithi099.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-03-19T01:34:28.120 INFO:teuthology.orchestra.run.smithi099.stderr: stderr: 10+0 records in 2024-03-19T01:34:28.120 INFO:teuthology.orchestra.run.smithi099.stderr:10+0 records out 2024-03-19T01:34:28.121 INFO:teuthology.orchestra.run.smithi099.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0112083 s, 936 MB/s 2024-03-19T01:34:28.121 INFO:teuthology.orchestra.run.smithi099.stderr:--> Zapping successful for: 2024-03-19T01:34:28.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:28 smithi099 ceph-mon[62951]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-03-19T01:34:28.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:28 smithi099 ceph-mon[62951]: from='osd.2 [v2:172.21.15.115:6800/3766690472,v1:172.21.15.115:6801/3766690472]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi115", "root=default"]}]: dispatch 2024-03-19T01:34:28.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:28 smithi099 ceph-mon[62951]: osdmap e16: 3 total, 2 up, 3 in 2024-03-19T01:34:28.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:28 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-19T01:34:28.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:28 smithi099 ceph-mon[62951]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi115", "root=default"]}]: dispatch 2024-03-19T01:34:28.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:28 smithi112 ceph-mon[66819]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-03-19T01:34:28.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:28 smithi112 ceph-mon[66819]: from='osd.2 [v2:172.21.15.115:6800/3766690472,v1:172.21.15.115:6801/3766690472]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi115", "root=default"]}]: dispatch 2024-03-19T01:34:28.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:28 smithi112 ceph-mon[66819]: osdmap e16: 3 total, 2 up, 3 in 2024-03-19T01:34:28.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:28 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-19T01:34:28.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:28 smithi112 ceph-mon[66819]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi115", "root=default"]}]: dispatch 2024-03-19T01:34:28.430 INFO:journalctl@ceph.osd.2.smithi115.stdout:Mar 19 01:34:28 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-2[72330]: 2024-03-19T01:34:28.073+0000 7fe999302700 -1 osd.2 0 waiting for initial osdmap 2024-03-19T01:34:28.431 INFO:journalctl@ceph.osd.2.smithi115.stdout:Mar 19 01:34:28 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-2[72330]: 2024-03-19T01:34:28.077+0000 7fe9950f5700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-19T01:34:28.431 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:28 smithi115 ceph-mon[67332]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-03-19T01:34:28.431 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:28 smithi115 ceph-mon[67332]: from='osd.2 [v2:172.21.15.115:6800/3766690472,v1:172.21.15.115:6801/3766690472]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi115", "root=default"]}]: dispatch 2024-03-19T01:34:28.431 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:28 smithi115 ceph-mon[67332]: osdmap e16: 3 total, 2 up, 3 in 2024-03-19T01:34:28.431 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:28 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-19T01:34:28.431 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:28 smithi115 ceph-mon[67332]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi115", "root=default"]}]: dispatch 2024-03-19T01:34:28.717 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph orch daemon add osd smithi099:vg_nvme/lv_3 2024-03-19T01:34:28.996 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:34:29.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:29 smithi115 ceph-mon[67332]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi115", "root=default"]}]': finished 2024-03-19T01:34:29.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:29 smithi115 ceph-mon[67332]: osdmap e17: 3 total, 2 up, 3 in 2024-03-19T01:34:29.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:29 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-19T01:34:29.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:29 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-19T01:34:29.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:29 smithi115 ceph-mon[67332]: pgmap v78: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-03-19T01:34:29.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:29 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-19T01:34:29.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:29 smithi099 ceph-mon[62951]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi115", "root=default"]}]': finished 2024-03-19T01:34:29.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:29 smithi099 ceph-mon[62951]: osdmap e17: 3 total, 2 up, 3 in 2024-03-19T01:34:29.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:29 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-19T01:34:29.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:29 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-19T01:34:29.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:29 smithi099 ceph-mon[62951]: pgmap v78: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-03-19T01:34:29.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:29 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-19T01:34:29.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:29 smithi112 ceph-mon[66819]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi115", "root=default"]}]': finished 2024-03-19T01:34:29.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:29 smithi112 ceph-mon[66819]: osdmap e17: 3 total, 2 up, 3 in 2024-03-19T01:34:29.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:29 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-19T01:34:29.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:29 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-19T01:34:29.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:29 smithi112 ceph-mon[66819]: pgmap v78: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-03-19T01:34:29.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:29 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-19T01:34:30.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:30 smithi099 ceph-mon[62951]: purged_snaps scrub starts 2024-03-19T01:34:30.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:30 smithi099 ceph-mon[62951]: purged_snaps scrub ok 2024-03-19T01:34:30.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:30 smithi099 ceph-mon[62951]: osd.2 [v2:172.21.15.115:6800/3766690472,v1:172.21.15.115:6801/3766690472] boot 2024-03-19T01:34:30.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:30 smithi099 ceph-mon[62951]: osdmap e18: 3 total, 3 up, 3 in 2024-03-19T01:34:30.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:30 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-19T01:34:30.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:30 smithi099 ceph-mon[62951]: Detected new or changed devices on smithi115 2024-03-19T01:34:30.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:30 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:30.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:30 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:30.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:30 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:34:30.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:30 smithi099 ceph-mon[62951]: Adjusting osd_memory_target on smithi115 to 17080M 2024-03-19T01:34:30.343 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:30 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:30.343 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:30 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:34:30.343 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:30 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:34:30.343 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:30 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:30.343 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:30 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:34:30.343 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:30 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:34:30.343 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:30 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:34:30.343 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:30 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:30.364 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:30 smithi112 ceph-mon[66819]: purged_snaps scrub starts 2024-03-19T01:34:30.364 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:30 smithi112 ceph-mon[66819]: purged_snaps scrub ok 2024-03-19T01:34:30.364 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:30 smithi112 ceph-mon[66819]: osd.2 [v2:172.21.15.115:6800/3766690472,v1:172.21.15.115:6801/3766690472] boot 2024-03-19T01:34:30.364 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:30 smithi112 ceph-mon[66819]: osdmap e18: 3 total, 3 up, 3 in 2024-03-19T01:34:30.364 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:30 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-19T01:34:30.364 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:30 smithi112 ceph-mon[66819]: Detected new or changed devices on smithi115 2024-03-19T01:34:30.364 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:30 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:30.364 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:30 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:30.364 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:30 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:34:30.364 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:30 smithi112 ceph-mon[66819]: Adjusting osd_memory_target on smithi115 to 17080M 2024-03-19T01:34:30.364 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:30 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:30.364 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:30 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:34:30.364 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:30 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:34:30.365 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:30 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:30.365 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:30 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:34:30.365 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:30 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:34:30.365 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:30 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:34:30.365 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:30 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:30.408 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:34:30.403+0000 7f34afda6700 1 -- 172.21.15.99:0/2369168699 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f34a8103630 msgr2=0x7f34a8103a90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:34:30.408 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:34:30.403+0000 7f34afda6700 1 --2- 172.21.15.99:0/2369168699 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f34a8103630 0x7f34a8103a90 secure :-1 s=READY pgs=113 cs=0 l=1 rev1=1 crypto rx=0x7f3498009990 tx=0x7f3498009ca0 comp rx=0 tx=0).stop 2024-03-19T01:34:30.408 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:34:30.403+0000 7f34afda6700 1 -- 172.21.15.99:0/2369168699 shutdown_connections 2024-03-19T01:34:30.408 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:34:30.403+0000 7f34afda6700 1 --2- 172.21.15.99:0/2369168699 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f34a8100650 0x7f34a8100a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:34:30.408 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:34:30.403+0000 7f34afda6700 1 --2- 172.21.15.99:0/2369168699 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f34a8103fd0 0x7f34a810a880 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:34:30.408 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:34:30.403+0000 7f34afda6700 1 --2- 172.21.15.99:0/2369168699 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f34a8103630 0x7f34a8103a90 unknown :-1 s=CLOSED pgs=113 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:34:30.408 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:34:30.403+0000 7f34afda6700 1 -- 172.21.15.99:0/2369168699 >> 172.21.15.99:0/2369168699 conn(0x7f34a80fbfb0 msgr2=0x7f34a80fe3f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:34:30.408 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:34:30.403+0000 7f34afda6700 1 -- 172.21.15.99:0/2369168699 shutdown_connections 2024-03-19T01:34:30.409 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:34:30.403+0000 7f34afda6700 1 -- 172.21.15.99:0/2369168699 wait complete. 2024-03-19T01:34:30.409 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:34:30.403+0000 7f34afda6700 1 Processor -- start 2024-03-19T01:34:30.409 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:34:30.404+0000 7f34afda6700 1 -- start start 2024-03-19T01:34:30.409 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:34:30.404+0000 7f34afda6700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f34a8100650 0x7f34a81984a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:34:30.409 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:34:30.404+0000 7f34afda6700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f34a8103630 0x7f34a81989e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:34:30.409 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:34:30.404+0000 7f34afda6700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f34a8103fd0 0x7f34a819fa60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:34:30.409 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:34:30.404+0000 7f34afda6700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f34a810df50 con 0x7f34a8103630 2024-03-19T01:34:30.409 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:34:30.404+0000 7f34afda6700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f34a810ddd0 con 0x7f34a8100650 2024-03-19T01:34:30.409 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:34:30.404+0000 7f34afda6700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f34a810e0d0 con 0x7f34a8103fd0 2024-03-19T01:34:30.409 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:34:30.404+0000 7f34ad341700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f34a8103630 0x7f34a81989e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:34:30.409 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:34:30.404+0000 7f34ad341700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f34a8103630 0x7f34a81989e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:33396/0 (socket says 172.21.15.99:33396) 2024-03-19T01:34:30.409 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:34:30.404+0000 7f34ad341700 1 -- 172.21.15.99:0/4027534142 learned_addr learned my addr 172.21.15.99:0/4027534142 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:34:30.412 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:34:30.404+0000 7f34adb42700 1 --2- 172.21.15.99:0/4027534142 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f34a8100650 0x7f34a81984a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:34:30.412 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:34:30.404+0000 7f34ae343700 1 --2- 172.21.15.99:0/4027534142 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f34a8103fd0 0x7f34a819fa60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:34:30.413 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:34:30.404+0000 7f34ad341700 1 -- 172.21.15.99:0/4027534142 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f34a8103fd0 msgr2=0x7f34a819fa60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:34:30.413 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:34:30.404+0000 7f34ad341700 1 --2- 172.21.15.99:0/4027534142 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f34a8103fd0 0x7f34a819fa60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:34:30.413 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:34:30.404+0000 7f34ad341700 1 -- 172.21.15.99:0/4027534142 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f34a8100650 msgr2=0x7f34a81984a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:34:30.413 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:34:30.404+0000 7f34ad341700 1 --2- 172.21.15.99:0/4027534142 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f34a8100650 0x7f34a81984a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:34:30.413 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:34:30.404+0000 7f34ad341700 1 -- 172.21.15.99:0/4027534142 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f34a81a0100 con 0x7f34a8103630 2024-03-19T01:34:30.413 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:34:30.404+0000 7f34ae343700 1 --2- 172.21.15.99:0/4027534142 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f34a8103fd0 0x7f34a819fa60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-19T01:34:30.413 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:34:30.404+0000 7f34ad341700 1 --2- 172.21.15.99:0/4027534142 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f34a8103630 0x7f34a81989e0 secure :-1 s=READY pgs=114 cs=0 l=1 rev1=1 crypto rx=0x7f3498000c00 tx=0x7f34980057c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:34:30.413 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:34:30.404+0000 7f349effd700 1 -- 172.21.15.99:0/4027534142 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3498027070 con 0x7f34a8103630 2024-03-19T01:34:30.414 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:34:30.404+0000 7f349effd700 1 -- 172.21.15.99:0/4027534142 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f3498004150 con 0x7f34a8103630 2024-03-19T01:34:30.414 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:34:30.404+0000 7f349effd700 1 -- 172.21.15.99:0/4027534142 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f349800f460 con 0x7f34a8103630 2024-03-19T01:34:30.414 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:34:30.404+0000 7f34afda6700 1 -- 172.21.15.99:0/4027534142 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f34a81a0390 con 0x7f34a8103630 2024-03-19T01:34:30.414 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:34:30.404+0000 7f34afda6700 1 -- 172.21.15.99:0/4027534142 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f34a81a07e0 con 0x7f34a8103630 2024-03-19T01:34:30.414 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:34:30.405+0000 7f34afda6700 1 -- 172.21.15.99:0/4027534142 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f34a8068fd0 con 0x7f34a8103630 2024-03-19T01:34:30.414 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:34:30.406+0000 7f349effd700 1 -- 172.21.15.99:0/4027534142 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 15) ==== 137234+0+0 (secure 0 0 0) 0x7f349802a070 con 0x7f34a8103630 2024-03-19T01:34:30.414 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:34:30.407+0000 7f349effd700 1 --2- 172.21.15.99:0/4027534142 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f34940a2800 0x7f34940a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:34:30.414 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:34:30.407+0000 7f34adb42700 1 --2- 172.21.15.99:0/4027534142 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f34940a2800 0x7f34940a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:34:30.415 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:34:30.407+0000 7f349effd700 1 -- 172.21.15.99:0/4027534142 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(19..19 src has 1..19) ==== 2851+0+0 (secure 0 0 0) 0x7f34980ce200 con 0x7f34a8103630 2024-03-19T01:34:30.415 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:34:30.407+0000 7f34adb42700 1 --2- 172.21.15.99:0/4027534142 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f34940a2800 0x7f34940a4cc0 secure :-1 s=READY pgs=65 cs=0 l=1 rev1=1 crypto rx=0x7f34a806a830 tx=0x7f34a400a3b0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:34:30.415 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:34:30.411+0000 7f349effd700 1 -- 172.21.15.99:0/4027534142 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f349805e740 con 0x7f34a8103630 2024-03-19T01:34:30.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:30 smithi115 ceph-mon[67332]: purged_snaps scrub starts 2024-03-19T01:34:30.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:30 smithi115 ceph-mon[67332]: purged_snaps scrub ok 2024-03-19T01:34:30.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:30 smithi115 ceph-mon[67332]: osd.2 [v2:172.21.15.115:6800/3766690472,v1:172.21.15.115:6801/3766690472] boot 2024-03-19T01:34:30.558 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:30 smithi115 ceph-mon[67332]: osdmap e18: 3 total, 3 up, 3 in 2024-03-19T01:34:30.558 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:30 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-19T01:34:30.558 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:30 smithi115 ceph-mon[67332]: Detected new or changed devices on smithi115 2024-03-19T01:34:30.558 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:30 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:30.558 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:30 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:30.558 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:30 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:34:30.558 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:30 smithi115 ceph-mon[67332]: Adjusting osd_memory_target on smithi115 to 17080M 2024-03-19T01:34:30.558 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:30 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:30.558 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:30 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:34:30.558 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:30 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:34:30.559 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:30 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:30.559 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:30 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:34:30.559 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:30 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:34:30.559 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:30 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:34:30.559 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:30 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:30.619 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:34:30.618+0000 7f34afda6700 1 -- 172.21.15.99:0/4027534142 --> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_3", "target": ["mon-mgr", ""]}) -- 0x7f34a8062750 con 0x7f34940a2800 2024-03-19T01:34:31.351 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:31 smithi099 ceph-mon[62951]: osdmap e19: 3 total, 3 up, 3 in 2024-03-19T01:34:31.351 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:31 smithi099 ceph-mon[62951]: from='client.14289 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:34:31.351 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:31 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-19T01:34:31.351 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:31 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-19T01:34:31.351 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:31 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:34:31.351 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:31 smithi099 ceph-mon[62951]: pgmap v81: 0 pgs: ; 0 B data, 84 MiB used, 268 GiB / 268 GiB avail 2024-03-19T01:34:31.351 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:31 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2024-03-19T01:34:31.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:31 smithi112 ceph-mon[66819]: osdmap e19: 3 total, 3 up, 3 in 2024-03-19T01:34:31.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:31 smithi112 ceph-mon[66819]: from='client.14289 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:34:31.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:31 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-19T01:34:31.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:31 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-19T01:34:31.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:31 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:34:31.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:31 smithi112 ceph-mon[66819]: pgmap v81: 0 pgs: ; 0 B data, 84 MiB used, 268 GiB / 268 GiB avail 2024-03-19T01:34:31.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:31 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2024-03-19T01:34:31.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:31 smithi115 ceph-mon[67332]: osdmap e19: 3 total, 3 up, 3 in 2024-03-19T01:34:31.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:31 smithi115 ceph-mon[67332]: from='client.14289 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:34:31.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:31 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-19T01:34:31.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:31 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-19T01:34:31.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:31 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:34:31.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:31 smithi115 ceph-mon[67332]: pgmap v81: 0 pgs: ; 0 B data, 84 MiB used, 268 GiB / 268 GiB avail 2024-03-19T01:34:31.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:31 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2024-03-19T01:34:32.191 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:32 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2024-03-19T01:34:32.191 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:32 smithi099 ceph-mon[62951]: osdmap e20: 3 total, 3 up, 3 in 2024-03-19T01:34:32.191 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:32 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-03-19T01:34:32.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:32 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2024-03-19T01:34:32.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:32 smithi112 ceph-mon[66819]: osdmap e20: 3 total, 3 up, 3 in 2024-03-19T01:34:32.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:32 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-03-19T01:34:32.448 INFO:journalctl@ceph.osd.0.smithi099.stdout:Mar 19 01:34:32 smithi099 sudo[82717]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-03-19T01:34:32.449 INFO:journalctl@ceph.osd.0.smithi099.stdout:Mar 19 01:34:32 smithi099 sudo[82717]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-03-19T01:34:32.449 INFO:journalctl@ceph.osd.0.smithi099.stdout:Mar 19 01:34:32 smithi099 sudo[82717]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-03-19T01:34:32.449 INFO:journalctl@ceph.osd.0.smithi099.stdout:Mar 19 01:34:32 smithi099 sudo[82717]: pam_unix(sudo:session): session closed for user root 2024-03-19T01:34:32.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:32 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2024-03-19T01:34:32.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:32 smithi115 ceph-mon[67332]: osdmap e20: 3 total, 3 up, 3 in 2024-03-19T01:34:32.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:32 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-03-19T01:34:33.091 INFO:journalctl@ceph.osd.0.smithi099.stdout:Mar 19 01:34:32 smithi099 sudo[82726]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-03-19T01:34:33.091 INFO:journalctl@ceph.osd.0.smithi099.stdout:Mar 19 01:34:32 smithi099 sudo[82726]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-03-19T01:34:33.091 INFO:journalctl@ceph.osd.0.smithi099.stdout:Mar 19 01:34:32 smithi099 sudo[82726]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-03-19T01:34:33.092 INFO:journalctl@ceph.osd.0.smithi099.stdout:Mar 19 01:34:32 smithi099 sudo[82726]: pam_unix(sudo:session): session closed for user root 2024-03-19T01:34:33.221 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:33 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-03-19T01:34:33.221 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:33 smithi112 ceph-mon[66819]: osdmap e21: 3 total, 3 up, 3 in 2024-03-19T01:34:33.221 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:33 smithi112 ceph-mon[66819]: pgmap v84: 1 pgs: 1 creating+peering; 0 B data, 84 MiB used, 268 GiB / 268 GiB avail 2024-03-19T01:34:33.221 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:33 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:33.222 INFO:journalctl@ceph.osd.1.smithi112.stdout:Mar 19 01:34:32 smithi112 sudo[75548]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-03-19T01:34:33.222 INFO:journalctl@ceph.osd.1.smithi112.stdout:Mar 19 01:34:32 smithi112 sudo[75548]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-03-19T01:34:33.222 INFO:journalctl@ceph.osd.1.smithi112.stdout:Mar 19 01:34:32 smithi112 sudo[75548]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-03-19T01:34:33.222 INFO:journalctl@ceph.osd.1.smithi112.stdout:Mar 19 01:34:32 smithi112 sudo[75548]: pam_unix(sudo:session): session closed for user root 2024-03-19T01:34:33.487 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:33 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-03-19T01:34:33.487 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:33 smithi115 ceph-mon[67332]: osdmap e21: 3 total, 3 up, 3 in 2024-03-19T01:34:33.487 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:33 smithi115 ceph-mon[67332]: pgmap v84: 1 pgs: 1 creating+peering; 0 B data, 84 MiB used, 268 GiB / 268 GiB avail 2024-03-19T01:34:33.487 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:33 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:33.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:33 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-03-19T01:34:33.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:33 smithi099 ceph-mon[62951]: osdmap e21: 3 total, 3 up, 3 in 2024-03-19T01:34:33.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:33 smithi099 ceph-mon[62951]: pgmap v84: 1 pgs: 1 creating+peering; 0 B data, 84 MiB used, 268 GiB / 268 GiB avail 2024-03-19T01:34:33.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:33 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:33.612 INFO:journalctl@ceph.osd.1.smithi112.stdout:Mar 19 01:34:33 smithi112 sudo[75551]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-03-19T01:34:33.612 INFO:journalctl@ceph.osd.1.smithi112.stdout:Mar 19 01:34:33 smithi112 sudo[75551]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-03-19T01:34:33.612 INFO:journalctl@ceph.osd.1.smithi112.stdout:Mar 19 01:34:33 smithi112 sudo[75551]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-03-19T01:34:33.612 INFO:journalctl@ceph.osd.1.smithi112.stdout:Mar 19 01:34:33 smithi112 sudo[75551]: pam_unix(sudo:session): session closed for user root 2024-03-19T01:34:33.739 INFO:journalctl@ceph.osd.2.smithi115.stdout:Mar 19 01:34:33 smithi115 sudo[75962]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-03-19T01:34:33.739 INFO:journalctl@ceph.osd.2.smithi115.stdout:Mar 19 01:34:33 smithi115 sudo[75962]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-03-19T01:34:33.740 INFO:journalctl@ceph.osd.2.smithi115.stdout:Mar 19 01:34:33 smithi115 sudo[75962]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-03-19T01:34:33.740 INFO:journalctl@ceph.osd.2.smithi115.stdout:Mar 19 01:34:33 smithi115 sudo[75962]: pam_unix(sudo:session): session closed for user root 2024-03-19T01:34:34.057 INFO:journalctl@ceph.osd.2.smithi115.stdout:Mar 19 01:34:33 smithi115 sudo[75965]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-03-19T01:34:34.057 INFO:journalctl@ceph.osd.2.smithi115.stdout:Mar 19 01:34:33 smithi115 sudo[75965]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-03-19T01:34:34.057 INFO:journalctl@ceph.osd.2.smithi115.stdout:Mar 19 01:34:33 smithi115 sudo[75965]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-03-19T01:34:34.057 INFO:journalctl@ceph.osd.2.smithi115.stdout:Mar 19 01:34:33 smithi115 sudo[75965]: pam_unix(sudo:session): session closed for user root 2024-03-19T01:34:34.271 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:34 smithi099 sudo[82808]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-03-19T01:34:34.271 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:34 smithi099 sudo[82808]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-03-19T01:34:34.271 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:34 smithi099 sudo[82808]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-03-19T01:34:34.271 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:34 smithi099 sudo[82808]: pam_unix(sudo:session): session closed for user root 2024-03-19T01:34:34.528 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:34 smithi115 ceph-mon[67332]: osdmap e22: 3 total, 3 up, 3 in 2024-03-19T01:34:34.528 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:34 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:34.528 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:34 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/2122926406' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6697ed1b-96bf-42de-8440-d17e9785b359"}]: dispatch 2024-03-19T01:34:34.528 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:34 smithi115 ceph-mon[67332]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6697ed1b-96bf-42de-8440-d17e9785b359"}]: dispatch 2024-03-19T01:34:34.528 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:34 smithi115 ceph-mon[67332]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6697ed1b-96bf-42de-8440-d17e9785b359"}]': finished 2024-03-19T01:34:34.528 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:34 smithi115 ceph-mon[67332]: osdmap e23: 4 total, 3 up, 4 in 2024-03-19T01:34:34.529 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:34 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-19T01:34:34.529 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:34 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:34.529 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:34 smithi115 ceph-mon[67332]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-19T01:34:34.529 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:34 smithi115 sudo[75968]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-03-19T01:34:34.529 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:34 smithi115 sudo[75968]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-03-19T01:34:34.529 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:34 smithi115 sudo[75968]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-03-19T01:34:34.529 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:34 smithi115 sudo[75968]: pam_unix(sudo:session): session closed for user root 2024-03-19T01:34:34.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:34 smithi099 sudo[82811]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-03-19T01:34:34.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:34 smithi099 sudo[82811]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-03-19T01:34:34.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:34 smithi099 sudo[82811]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-03-19T01:34:34.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:34 smithi099 sudo[82811]: pam_unix(sudo:session): session closed for user root 2024-03-19T01:34:34.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:34 smithi099 ceph-mon[62951]: osdmap e22: 3 total, 3 up, 3 in 2024-03-19T01:34:34.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:34 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:34.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:34 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/2122926406' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6697ed1b-96bf-42de-8440-d17e9785b359"}]: dispatch 2024-03-19T01:34:34.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:34 smithi099 ceph-mon[62951]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6697ed1b-96bf-42de-8440-d17e9785b359"}]: dispatch 2024-03-19T01:34:34.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:34 smithi099 ceph-mon[62951]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6697ed1b-96bf-42de-8440-d17e9785b359"}]': finished 2024-03-19T01:34:34.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:34 smithi099 ceph-mon[62951]: osdmap e23: 4 total, 3 up, 4 in 2024-03-19T01:34:34.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:34 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-19T01:34:34.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:34 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:34.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:34 smithi099 ceph-mon[62951]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-19T01:34:34.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:34 smithi112 ceph-mon[66819]: osdmap e22: 3 total, 3 up, 3 in 2024-03-19T01:34:34.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:34 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:34.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:34 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/2122926406' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6697ed1b-96bf-42de-8440-d17e9785b359"}]: dispatch 2024-03-19T01:34:34.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:34 smithi112 ceph-mon[66819]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6697ed1b-96bf-42de-8440-d17e9785b359"}]: dispatch 2024-03-19T01:34:34.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:34 smithi112 ceph-mon[66819]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6697ed1b-96bf-42de-8440-d17e9785b359"}]': finished 2024-03-19T01:34:34.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:34 smithi112 ceph-mon[66819]: osdmap e23: 4 total, 3 up, 4 in 2024-03-19T01:34:34.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:34 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-19T01:34:34.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:34 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:34.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:34 smithi112 ceph-mon[66819]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-19T01:34:34.775 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:34:34.773+0000 7f349effd700 1 -- 172.21.15.99:0/4027534142 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mgrmap(e 16) ==== 137290+0+0 (secure 0 0 0) 0x7f3498098960 con 0x7f34a8103630 2024-03-19T01:34:34.806 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:34 smithi115 sudo[75971]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-03-19T01:34:34.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:34 smithi115 sudo[75971]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-03-19T01:34:34.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:34 smithi115 sudo[75971]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-03-19T01:34:34.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:34 smithi115 sudo[75971]: pam_unix(sudo:session): session closed for user root 2024-03-19T01:34:35.256 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:35 smithi112 sudo[75554]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-03-19T01:34:35.256 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:35 smithi112 sudo[75554]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-03-19T01:34:35.256 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:35 smithi112 sudo[75554]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-03-19T01:34:35.257 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:35 smithi112 sudo[75554]: pam_unix(sudo:session): session closed for user root 2024-03-19T01:34:35.257 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:35 smithi112 sudo[75557]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-03-19T01:34:35.257 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:35 smithi112 sudo[75557]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-03-19T01:34:35.257 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:35 smithi112 sudo[75557]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-03-19T01:34:35.257 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:35 smithi112 sudo[75557]: pam_unix(sudo:session): session closed for user root 2024-03-19T01:34:35.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:35 smithi115 ceph-mon[67332]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-19T01:34:35.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:35 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-19T01:34:35.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:35 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/2802218493' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-19T01:34:35.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:35 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-19T01:34:35.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:35 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-19T01:34:35.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:35 smithi115 ceph-mon[67332]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-19T01:34:35.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:35 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-19T01:34:35.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:35 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-19T01:34:35.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:35 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-19T01:34:35.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:35 smithi115 ceph-mon[67332]: pgmap v87: 1 pgs: 1 creating+peering; 0 B data, 84 MiB used, 268 GiB / 268 GiB avail 2024-03-19T01:34:35.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:35 smithi115 ceph-mon[67332]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-19T01:34:35.558 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:35 smithi115 ceph-mon[67332]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-19T01:34:35.558 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:35 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-19T01:34:35.558 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:35 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-19T01:34:35.558 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:35 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-19T01:34:35.558 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:35 smithi115 ceph-mon[67332]: mgrmap e16: x(active, since 2m), standbys: z, y 2024-03-19T01:34:35.558 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:35 smithi115 ceph-mon[67332]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-19T01:34:35.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:35 smithi099 ceph-mon[62951]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-19T01:34:35.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:35 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-19T01:34:35.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:35 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/2802218493' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-19T01:34:35.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:35 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-19T01:34:35.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:35 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-19T01:34:35.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:35 smithi099 ceph-mon[62951]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-19T01:34:35.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:35 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-19T01:34:35.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:35 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-19T01:34:35.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:35 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-19T01:34:35.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:35 smithi099 ceph-mon[62951]: pgmap v87: 1 pgs: 1 creating+peering; 0 B data, 84 MiB used, 268 GiB / 268 GiB avail 2024-03-19T01:34:35.593 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:35 smithi099 ceph-mon[62951]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-19T01:34:35.593 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:35 smithi099 ceph-mon[62951]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-19T01:34:35.593 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:35 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-19T01:34:35.593 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:35 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-19T01:34:35.593 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:35 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-19T01:34:35.593 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:35 smithi099 ceph-mon[62951]: mgrmap e16: x(active, since 2m), standbys: z, y 2024-03-19T01:34:35.593 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:35 smithi099 ceph-mon[62951]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-19T01:34:35.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:35 smithi112 ceph-mon[66819]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-19T01:34:35.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:35 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-19T01:34:35.613 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:35 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/2802218493' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-19T01:34:35.613 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:35 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-19T01:34:35.613 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:35 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-19T01:34:35.613 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:35 smithi112 ceph-mon[66819]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-19T01:34:35.613 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:35 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-19T01:34:35.613 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:35 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-19T01:34:35.613 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:35 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-19T01:34:35.613 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:35 smithi112 ceph-mon[66819]: pgmap v87: 1 pgs: 1 creating+peering; 0 B data, 84 MiB used, 268 GiB / 268 GiB avail 2024-03-19T01:34:35.613 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:35 smithi112 ceph-mon[66819]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-19T01:34:35.613 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:35 smithi112 ceph-mon[66819]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-19T01:34:35.613 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:35 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-19T01:34:35.613 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:35 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-19T01:34:35.613 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:35 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-19T01:34:35.614 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:35 smithi112 ceph-mon[66819]: mgrmap e16: x(active, since 2m), standbys: z, y 2024-03-19T01:34:35.614 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:35 smithi112 ceph-mon[66819]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-19T01:34:37.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:37 smithi099 ceph-mon[62951]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-03-19T01:34:38.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:37 smithi115 ceph-mon[67332]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-03-19T01:34:38.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:37 smithi112 ceph-mon[66819]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-03-19T01:34:39.907 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:39 smithi099 ceph-mon[62951]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 87 MiB used, 268 GiB / 268 GiB avail 2024-03-19T01:34:40.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:39 smithi115 ceph-mon[67332]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 87 MiB used, 268 GiB / 268 GiB avail 2024-03-19T01:34:40.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:39 smithi112 ceph-mon[66819]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 87 MiB used, 268 GiB / 268 GiB avail 2024-03-19T01:34:42.042 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:41 smithi099 ceph-mon[62951]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-03-19T01:34:42.042 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:41 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-19T01:34:42.042 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:41 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:34:42.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:41 smithi115 ceph-mon[67332]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-03-19T01:34:42.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:41 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-19T01:34:42.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:41 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:34:42.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:41 smithi112 ceph-mon[66819]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-03-19T01:34:42.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:41 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-19T01:34:42.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:41 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:34:43.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:42 smithi115 ceph-mon[67332]: Deploying daemon osd.3 on smithi099 2024-03-19T01:34:43.066 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:42 smithi099 ceph-mon[62951]: Deploying daemon osd.3 on smithi099 2024-03-19T01:34:43.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:42 smithi112 ceph-mon[66819]: Deploying daemon osd.3 on smithi099 2024-03-19T01:34:43.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:43 smithi099 ceph-mon[62951]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-03-19T01:34:44.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:43 smithi115 ceph-mon[67332]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-03-19T01:34:44.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:43 smithi112 ceph-mon[66819]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-03-19T01:34:45.842 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:45 smithi099 ceph-mon[62951]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-03-19T01:34:46.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:45 smithi115 ceph-mon[67332]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-03-19T01:34:46.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:45 smithi112 ceph-mon[66819]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-03-19T01:34:47.806 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:47 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:47.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:47 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:47.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:47 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:34:47.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:47 smithi115 ceph-mon[67332]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-03-19T01:34:47.812 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:47 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:47.812 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:47 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:47.812 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:47 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:34:47.812 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:47 smithi099 ceph-mon[62951]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-03-19T01:34:47.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:47 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:47.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:47 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:47.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:47 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:34:47.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:47 smithi112 ceph-mon[66819]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-03-19T01:34:49.790 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:49 smithi099 ceph-mon[62951]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-03-19T01:34:49.790 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:49 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:49.790 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:49 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:49.790 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:49 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:34:49.790 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:49 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:34:49.790 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:49 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:49.900 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:34:49.899+0000 7f349effd700 1 -- 172.21.15.99:0/4027534142 <== mgr.14150 v2:172.21.15.99:6800/4272410295 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+37 (secure 0 0 0) 0x7f34a8062750 con 0x7f34940a2800 2024-03-19T01:34:49.900 INFO:teuthology.orchestra.run.smithi099.stdout:Created osd(s) 3 on host 'smithi099' 2024-03-19T01:34:49.902 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:34:49.901+0000 7f34afda6700 1 -- 172.21.15.99:0/4027534142 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f34940a2800 msgr2=0x7f34940a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:34:49.902 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:34:49.901+0000 7f34afda6700 1 --2- 172.21.15.99:0/4027534142 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f34940a2800 0x7f34940a4cc0 secure :-1 s=READY pgs=65 cs=0 l=1 rev1=1 crypto rx=0x7f34a806a830 tx=0x7f34a400a3b0 comp rx=0 tx=0).stop 2024-03-19T01:34:49.902 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:34:49.901+0000 7f34afda6700 1 -- 172.21.15.99:0/4027534142 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f34a8103630 msgr2=0x7f34a81989e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:34:49.902 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:34:49.901+0000 7f34afda6700 1 --2- 172.21.15.99:0/4027534142 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f34a8103630 0x7f34a81989e0 secure :-1 s=READY pgs=114 cs=0 l=1 rev1=1 crypto rx=0x7f3498000c00 tx=0x7f34980057c0 comp rx=0 tx=0).stop 2024-03-19T01:34:49.902 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:34:49.902+0000 7f34afda6700 1 -- 172.21.15.99:0/4027534142 shutdown_connections 2024-03-19T01:34:49.903 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:34:49.902+0000 7f34afda6700 1 --2- 172.21.15.99:0/4027534142 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f34940a2800 0x7f34940a4cc0 unknown :-1 s=CLOSED pgs=65 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:34:49.903 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:34:49.902+0000 7f34afda6700 1 --2- 172.21.15.99:0/4027534142 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f34a8103fd0 0x7f34a819fa60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:34:49.903 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:34:49.902+0000 7f34afda6700 1 --2- 172.21.15.99:0/4027534142 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f34a8100650 0x7f34a81984a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:34:49.903 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:34:49.902+0000 7f34afda6700 1 --2- 172.21.15.99:0/4027534142 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f34a8103630 0x7f34a81989e0 unknown :-1 s=CLOSED pgs=114 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:34:49.903 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:34:49.902+0000 7f34afda6700 1 -- 172.21.15.99:0/4027534142 >> 172.21.15.99:0/4027534142 conn(0x7f34a80fbfb0 msgr2=0x7f34a8106860 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:34:49.903 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:34:49.902+0000 7f34afda6700 1 -- 172.21.15.99:0/4027534142 shutdown_connections 2024-03-19T01:34:49.903 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:34:49.902+0000 7f34afda6700 1 -- 172.21.15.99:0/4027534142 wait complete. 2024-03-19T01:34:50.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:49 smithi115 ceph-mon[67332]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-03-19T01:34:50.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:49 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:50.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:49 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:50.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:49 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:34:50.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:49 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:34:50.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:49 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:50.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:49 smithi112 ceph-mon[66819]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-03-19T01:34:50.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:49 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:50.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:49 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:50.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:49 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:34:50.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:49 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:34:50.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:49 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:50.494 DEBUG:teuthology.orchestra.run.smithi099:osd.3> sudo journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@osd.3.service 2024-03-19T01:34:50.496 INFO:tasks.cephadm:Deploying osd.4 on smithi112 with /dev/vg_nvme/lv_3... 2024-03-19T01:34:50.497 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-03-19T01:34:50.740 INFO:teuthology.orchestra.run.smithi112.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.b/config 2024-03-19T01:34:51.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:50 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:51.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:50 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:51.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:50 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:34:51.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:50 smithi099 ceph-mon[62951]: from='osd.3 [v2:172.21.15.99:6810/3756732895,v1:172.21.15.99:6811/3756732895]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-19T01:34:51.169 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:50 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:51.170 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:50 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:51.170 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:50 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:34:51.170 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:50 smithi112 ceph-mon[66819]: from='osd.3 [v2:172.21.15.99:6810/3756732895,v1:172.21.15.99:6811/3756732895]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-19T01:34:51.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:50 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:51.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:50 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:51.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:50 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:34:51.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:50 smithi115 ceph-mon[67332]: from='osd.3 [v2:172.21.15.99:6810/3756732895,v1:172.21.15.99:6811/3756732895]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-19T01:34:52.047 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:51 smithi099 ceph-mon[62951]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-03-19T01:34:52.047 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:51 smithi099 ceph-mon[62951]: from='osd.3 [v2:172.21.15.99:6810/3756732895,v1:172.21.15.99:6811/3756732895]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-03-19T01:34:52.047 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:51 smithi099 ceph-mon[62951]: osdmap e24: 4 total, 3 up, 4 in 2024-03-19T01:34:52.047 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:51 smithi099 ceph-mon[62951]: from='osd.3 [v2:172.21.15.99:6810/3756732895,v1:172.21.15.99:6811/3756732895]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2024-03-19T01:34:52.047 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:51 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-19T01:34:52.073 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:51 smithi112 ceph-mon[66819]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-03-19T01:34:52.073 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:51 smithi112 ceph-mon[66819]: from='osd.3 [v2:172.21.15.99:6810/3756732895,v1:172.21.15.99:6811/3756732895]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-03-19T01:34:52.073 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:51 smithi112 ceph-mon[66819]: osdmap e24: 4 total, 3 up, 4 in 2024-03-19T01:34:52.073 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:51 smithi112 ceph-mon[66819]: from='osd.3 [v2:172.21.15.99:6810/3756732895,v1:172.21.15.99:6811/3756732895]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2024-03-19T01:34:52.073 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:51 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-19T01:34:52.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:51 smithi115 ceph-mon[67332]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-03-19T01:34:52.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:51 smithi115 ceph-mon[67332]: from='osd.3 [v2:172.21.15.99:6810/3756732895,v1:172.21.15.99:6811/3756732895]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-03-19T01:34:52.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:51 smithi115 ceph-mon[67332]: osdmap e24: 4 total, 3 up, 4 in 2024-03-19T01:34:52.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:51 smithi115 ceph-mon[67332]: from='osd.3 [v2:172.21.15.99:6810/3756732895,v1:172.21.15.99:6811/3756732895]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2024-03-19T01:34:52.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:51 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-19T01:34:52.604 INFO:teuthology.orchestra.run.smithi112.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-03-19T01:34:52.604 INFO:teuthology.orchestra.run.smithi112.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-03-19T01:34:52.604 INFO:teuthology.orchestra.run.smithi112.stderr: stderr: 10+0 records in 2024-03-19T01:34:52.604 INFO:teuthology.orchestra.run.smithi112.stderr:10+0 records out 2024-03-19T01:34:52.604 INFO:teuthology.orchestra.run.smithi112.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0115305 s, 909 MB/s 2024-03-19T01:34:52.604 INFO:teuthology.orchestra.run.smithi112.stderr:--> Zapping successful for: 2024-03-19T01:34:53.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:52 smithi099 ceph-mon[62951]: from='osd.3 [v2:172.21.15.99:6810/3756732895,v1:172.21.15.99:6811/3756732895]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2024-03-19T01:34:53.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:52 smithi099 ceph-mon[62951]: osdmap e25: 4 total, 3 up, 4 in 2024-03-19T01:34:53.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:52 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-19T01:34:53.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:52 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-19T01:34:53.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:52 smithi099 ceph-mon[62951]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-03-19T01:34:53.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:52 smithi099 ceph-mon[62951]: from='osd.3 [v2:172.21.15.99:6810/3756732895,v1:172.21.15.99:6811/3756732895]' entity='osd.3' 2024-03-19T01:34:53.092 INFO:journalctl@ceph.osd.3.smithi099.stdout:Mar 19 01:34:52 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-3[84808]: 2024-03-19T01:34:52.798+0000 7f5093691700 -1 osd.3 0 waiting for initial osdmap 2024-03-19T01:34:53.092 INFO:journalctl@ceph.osd.3.smithi099.stdout:Mar 19 01:34:52 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-3[84808]: 2024-03-19T01:34:52.968+0000 7f508b47c700 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-19T01:34:53.254 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:52 smithi112 ceph-mon[66819]: from='osd.3 [v2:172.21.15.99:6810/3756732895,v1:172.21.15.99:6811/3756732895]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2024-03-19T01:34:53.254 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:52 smithi112 ceph-mon[66819]: osdmap e25: 4 total, 3 up, 4 in 2024-03-19T01:34:53.255 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:52 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-19T01:34:53.255 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:52 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-19T01:34:53.255 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:52 smithi112 ceph-mon[66819]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-03-19T01:34:53.255 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:52 smithi112 ceph-mon[66819]: from='osd.3 [v2:172.21.15.99:6810/3756732895,v1:172.21.15.99:6811/3756732895]' entity='osd.3' 2024-03-19T01:34:53.279 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph orch daemon add osd smithi112:vg_nvme/lv_3 2024-03-19T01:34:53.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:52 smithi115 ceph-mon[67332]: from='osd.3 [v2:172.21.15.99:6810/3756732895,v1:172.21.15.99:6811/3756732895]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2024-03-19T01:34:53.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:52 smithi115 ceph-mon[67332]: osdmap e25: 4 total, 3 up, 4 in 2024-03-19T01:34:53.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:52 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-19T01:34:53.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:52 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-19T01:34:53.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:52 smithi115 ceph-mon[67332]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-03-19T01:34:53.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:52 smithi115 ceph-mon[67332]: from='osd.3 [v2:172.21.15.99:6810/3756732895,v1:172.21.15.99:6811/3756732895]' entity='osd.3' 2024-03-19T01:34:53.514 INFO:teuthology.orchestra.run.smithi112.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.b/config 2024-03-19T01:34:54.203 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:53 smithi112 ceph-mon[66819]: purged_snaps scrub starts 2024-03-19T01:34:54.203 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:53 smithi112 ceph-mon[66819]: purged_snaps scrub ok 2024-03-19T01:34:54.203 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:53 smithi112 ceph-mon[66819]: osdmap e26: 4 total, 3 up, 4 in 2024-03-19T01:34:54.203 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:53 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-19T01:34:54.203 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:53 smithi112 ceph-mon[66819]: Detected new or changed devices on smithi099 2024-03-19T01:34:54.203 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:53 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:54.203 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:53 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:54.203 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:53 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:34:54.203 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:53 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:34:54.203 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:53 smithi112 ceph-mon[66819]: Adjusting osd_memory_target on smithi099 to 8544M 2024-03-19T01:34:54.203 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:53 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:54.203 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:53 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:34:54.204 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:53 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:34:54.204 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:53 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:54.204 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:53 smithi112 ceph-mon[66819]: osd.3 [v2:172.21.15.99:6810/3756732895,v1:172.21.15.99:6811/3756732895] boot 2024-03-19T01:34:54.204 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:53 smithi112 ceph-mon[66819]: osdmap e27: 4 total, 4 up, 4 in 2024-03-19T01:34:54.204 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:53 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-19T01:34:54.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:53 smithi115 ceph-mon[67332]: purged_snaps scrub starts 2024-03-19T01:34:54.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:53 smithi115 ceph-mon[67332]: purged_snaps scrub ok 2024-03-19T01:34:54.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:53 smithi115 ceph-mon[67332]: osdmap e26: 4 total, 3 up, 4 in 2024-03-19T01:34:54.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:53 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-19T01:34:54.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:53 smithi115 ceph-mon[67332]: Detected new or changed devices on smithi099 2024-03-19T01:34:54.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:53 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:54.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:53 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:54.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:53 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:34:54.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:53 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:34:54.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:53 smithi115 ceph-mon[67332]: Adjusting osd_memory_target on smithi099 to 8544M 2024-03-19T01:34:54.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:53 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:54.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:53 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:34:54.308 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:53 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:34:54.308 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:53 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:54.308 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:53 smithi115 ceph-mon[67332]: osd.3 [v2:172.21.15.99:6810/3756732895,v1:172.21.15.99:6811/3756732895] boot 2024-03-19T01:34:54.308 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:53 smithi115 ceph-mon[67332]: osdmap e27: 4 total, 4 up, 4 in 2024-03-19T01:34:54.308 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:53 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-19T01:34:54.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:53 smithi099 ceph-mon[62951]: purged_snaps scrub starts 2024-03-19T01:34:54.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:53 smithi099 ceph-mon[62951]: purged_snaps scrub ok 2024-03-19T01:34:54.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:53 smithi099 ceph-mon[62951]: osdmap e26: 4 total, 3 up, 4 in 2024-03-19T01:34:54.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:53 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-19T01:34:54.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:53 smithi099 ceph-mon[62951]: Detected new or changed devices on smithi099 2024-03-19T01:34:54.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:53 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:54.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:53 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:54.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:53 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:34:54.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:53 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:34:54.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:53 smithi099 ceph-mon[62951]: Adjusting osd_memory_target on smithi099 to 8544M 2024-03-19T01:34:54.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:53 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:54.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:53 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:34:54.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:53 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:34:54.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:53 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:34:54.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:53 smithi099 ceph-mon[62951]: osd.3 [v2:172.21.15.99:6810/3756732895,v1:172.21.15.99:6811/3756732895] boot 2024-03-19T01:34:54.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:53 smithi099 ceph-mon[62951]: osdmap e27: 4 total, 4 up, 4 in 2024-03-19T01:34:54.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:53 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-19T01:34:55.036 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:34:55.032+0000 7f4c9dc22700 1 -- 172.21.15.112:0/1948876491 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4c98103600 msgr2=0x7f4c98103a60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:34:55.036 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:34:55.032+0000 7f4c9dc22700 1 --2- 172.21.15.112:0/1948876491 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4c98103600 0x7f4c98103a60 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7f4c8c009990 tx=0x7f4c8c009ca0 comp rx=0 tx=0).stop 2024-03-19T01:34:55.036 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:34:55.032+0000 7f4c9dc22700 1 -- 172.21.15.112:0/1948876491 shutdown_connections 2024-03-19T01:34:55.036 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:34:55.032+0000 7f4c9dc22700 1 --2- 172.21.15.112:0/1948876491 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4c98100620 0x7f4c98100a40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:34:55.036 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:34:55.032+0000 7f4c9dc22700 1 --2- 172.21.15.112:0/1948876491 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4c98103fa0 0x7f4c9810a850 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:34:55.036 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:34:55.032+0000 7f4c9dc22700 1 --2- 172.21.15.112:0/1948876491 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4c98103600 0x7f4c98103a60 unknown :-1 s=CLOSED pgs=18 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:34:55.037 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:34:55.032+0000 7f4c9dc22700 1 -- 172.21.15.112:0/1948876491 >> 172.21.15.112:0/1948876491 conn(0x7f4c980fbfa0 msgr2=0x7f4c980fe3c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:34:55.037 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:34:55.033+0000 7f4c9dc22700 1 -- 172.21.15.112:0/1948876491 shutdown_connections 2024-03-19T01:34:55.037 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:34:55.033+0000 7f4c9dc22700 1 -- 172.21.15.112:0/1948876491 wait complete. 2024-03-19T01:34:55.037 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:34:55.033+0000 7f4c9dc22700 1 Processor -- start 2024-03-19T01:34:55.037 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:34:55.033+0000 7f4c9dc22700 1 -- start start 2024-03-19T01:34:55.037 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:34:55.034+0000 7f4c9dc22700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4c98100620 0x7f4c98198530 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:34:55.037 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:34:55.034+0000 7f4c9dc22700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4c98103600 0x7f4c98198a70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:34:55.037 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:34:55.034+0000 7f4c9dc22700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4c98103fa0 0x7f4c9819faf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:34:55.037 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:34:55.034+0000 7f4c9dc22700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f4c9810df40 con 0x7f4c98103fa0 2024-03-19T01:34:55.037 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:34:55.034+0000 7f4c9dc22700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f4c9810ddc0 con 0x7f4c98103600 2024-03-19T01:34:55.037 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:34:55.034+0000 7f4c9dc22700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f4c9810e0c0 con 0x7f4c98100620 2024-03-19T01:34:55.037 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:34:55.034+0000 7f4c96ffd700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4c98103600 0x7f4c98198a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:34:55.037 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:34:55.034+0000 7f4c96ffd700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4c98103600 0x7f4c98198a70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.112:34152/0 (socket says 172.21.15.112:34152) 2024-03-19T01:34:55.038 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:34:55.034+0000 7f4c96ffd700 1 -- 172.21.15.112:0/2935070791 learned_addr learned my addr 172.21.15.112:0/2935070791 (peer_addr_for_me v2:172.21.15.112:0/0) 2024-03-19T01:34:55.038 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:34:55.034+0000 7f4c977fe700 1 --2- 172.21.15.112:0/2935070791 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4c98100620 0x7f4c98198530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:34:55.038 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:34:55.034+0000 7f4c96ffd700 1 -- 172.21.15.112:0/2935070791 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4c98100620 msgr2=0x7f4c98198530 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:34:55.038 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:34:55.034+0000 7f4c96ffd700 1 --2- 172.21.15.112:0/2935070791 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4c98100620 0x7f4c98198530 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:34:55.038 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:34:55.034+0000 7f4c96ffd700 1 -- 172.21.15.112:0/2935070791 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4c98103fa0 msgr2=0x7f4c9819faf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:34:55.038 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:34:55.034+0000 7f4c97fff700 1 --2- 172.21.15.112:0/2935070791 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4c98103fa0 0x7f4c9819faf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:34:55.038 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:34:55.034+0000 7f4c96ffd700 1 --2- 172.21.15.112:0/2935070791 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4c98103fa0 0x7f4c9819faf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:34:55.038 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:34:55.034+0000 7f4c96ffd700 1 -- 172.21.15.112:0/2935070791 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f4c981a0190 con 0x7f4c98103600 2024-03-19T01:34:55.038 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:34:55.034+0000 7f4c977fe700 1 --2- 172.21.15.112:0/2935070791 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4c98100620 0x7f4c98198530 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-19T01:34:55.038 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:34:55.034+0000 7f4c96ffd700 1 --2- 172.21.15.112:0/2935070791 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4c98103600 0x7f4c98198a70 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7f4c8c005910 tx=0x7f4c8c00bc60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:34:55.038 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:34:55.034+0000 7f4c94ff9700 1 -- 172.21.15.112:0/2935070791 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4c8c026070 con 0x7f4c98103600 2024-03-19T01:34:55.038 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:34:55.034+0000 7f4c9dc22700 1 -- 172.21.15.112:0/2935070791 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f4c981a0420 con 0x7f4c98103600 2024-03-19T01:34:55.038 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:34:55.034+0000 7f4c94ff9700 1 -- 172.21.15.112:0/2935070791 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f4c8c00be40 con 0x7f4c98103600 2024-03-19T01:34:55.038 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:34:55.034+0000 7f4c9dc22700 1 -- 172.21.15.112:0/2935070791 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f4c981a08d0 con 0x7f4c98103600 2024-03-19T01:34:55.039 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:34:55.034+0000 7f4c94ff9700 1 -- 172.21.15.112:0/2935070791 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4c8c021700 con 0x7f4c98103600 2024-03-19T01:34:55.039 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:34:55.035+0000 7f4c9dc22700 1 -- 172.21.15.112:0/2935070791 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f4c98068fd0 con 0x7f4c98103600 2024-03-19T01:34:55.039 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:34:55.037+0000 7f4c94ff9700 1 -- 172.21.15.112:0/2935070791 <== mon.2 v2:172.21.15.112:3300/0 4 ==== mgrmap(e 16) ==== 137290+0+0 (secure 0 0 0) 0x7f4c8c029070 con 0x7f4c98103600 2024-03-19T01:34:55.040 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:34:55.037+0000 7f4c94ff9700 1 --2- 172.21.15.112:0/2935070791 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f4c800a2890 0x7f4c800a4d50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:34:55.040 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:34:55.038+0000 7f4c94ff9700 1 -- 172.21.15.112:0/2935070791 <== mon.2 v2:172.21.15.112:3300/0 5 ==== osd_map(28..28 src has 1..28) ==== 3698+0+0 (secure 0 0 0) 0x7f4c8c0cebc0 con 0x7f4c98103600 2024-03-19T01:34:55.040 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:34:55.038+0000 7f4c977fe700 1 --2- 172.21.15.112:0/2935070791 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f4c800a2890 0x7f4c800a4d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:34:55.040 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:34:55.038+0000 7f4c977fe700 1 --2- 172.21.15.112:0/2935070791 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f4c800a2890 0x7f4c800a4d50 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7f4c88005950 tx=0x7f4c880058e0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:34:55.042 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:34:55.041+0000 7f4c94ff9700 1 -- 172.21.15.112:0/2935070791 <== mon.2 v2:172.21.15.112:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f4c8c030020 con 0x7f4c98103600 2024-03-19T01:34:55.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:54 smithi112 ceph-mon[66819]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-03-19T01:34:55.249 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:34:55.248+0000 7f4c9dc22700 1 -- 172.21.15.112:0/2935070791 --> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi112:vg_nvme/lv_3", "target": ["mon-mgr", ""]}) -- 0x7f4c98062750 con 0x7f4c800a2890 2024-03-19T01:34:55.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:54 smithi115 ceph-mon[67332]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-03-19T01:34:55.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:54 smithi099 ceph-mon[62951]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-03-19T01:34:56.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:55 smithi112 ceph-mon[66819]: osdmap e28: 4 total, 4 up, 4 in 2024-03-19T01:34:56.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:55 smithi112 ceph-mon[66819]: from='client.24209 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi112:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:34:56.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:55 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-19T01:34:56.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:55 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-19T01:34:56.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:55 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:34:56.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:55 smithi115 ceph-mon[67332]: osdmap e28: 4 total, 4 up, 4 in 2024-03-19T01:34:56.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:55 smithi115 ceph-mon[67332]: from='client.24209 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi112:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:34:56.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:55 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-19T01:34:56.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:55 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-19T01:34:56.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:55 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:34:56.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:55 smithi099 ceph-mon[62951]: osdmap e28: 4 total, 4 up, 4 in 2024-03-19T01:34:56.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:55 smithi099 ceph-mon[62951]: from='client.24209 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi112:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:34:56.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:55 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-19T01:34:56.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:55 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-19T01:34:56.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:55 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:34:57.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:56 smithi115 ceph-mon[67332]: osdmap e29: 4 total, 4 up, 4 in 2024-03-19T01:34:57.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:56 smithi115 ceph-mon[67332]: pgmap v104: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail; 2/6 objects degraded (33.333%) 2024-03-19T01:34:57.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:56 smithi099 ceph-mon[62951]: osdmap e29: 4 total, 4 up, 4 in 2024-03-19T01:34:57.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:56 smithi099 ceph-mon[62951]: pgmap v104: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail; 2/6 objects degraded (33.333%) 2024-03-19T01:34:57.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:56 smithi112 ceph-mon[66819]: osdmap e29: 4 total, 4 up, 4 in 2024-03-19T01:34:57.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:56 smithi112 ceph-mon[66819]: pgmap v104: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail; 2/6 objects degraded (33.333%) 2024-03-19T01:34:58.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:57 smithi115 ceph-mon[67332]: Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2024-03-19T01:34:58.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:57 smithi115 ceph-mon[67332]: osdmap e30: 4 total, 4 up, 4 in 2024-03-19T01:34:58.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:57 smithi115 ceph-mon[67332]: from='client.? 172.21.15.112:0/1161088139' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d958d6b9-67f6-43d0-b704-aa55dc9df83a"}]: dispatch 2024-03-19T01:34:58.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:57 smithi115 ceph-mon[67332]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d958d6b9-67f6-43d0-b704-aa55dc9df83a"}]: dispatch 2024-03-19T01:34:58.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:57 smithi115 ceph-mon[67332]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d958d6b9-67f6-43d0-b704-aa55dc9df83a"}]': finished 2024-03-19T01:34:58.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:57 smithi115 ceph-mon[67332]: osdmap e31: 5 total, 4 up, 5 in 2024-03-19T01:34:58.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:57 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-19T01:34:58.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:57 smithi099 ceph-mon[62951]: Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2024-03-19T01:34:58.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:57 smithi099 ceph-mon[62951]: osdmap e30: 4 total, 4 up, 4 in 2024-03-19T01:34:58.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:57 smithi099 ceph-mon[62951]: from='client.? 172.21.15.112:0/1161088139' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d958d6b9-67f6-43d0-b704-aa55dc9df83a"}]: dispatch 2024-03-19T01:34:58.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:57 smithi099 ceph-mon[62951]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d958d6b9-67f6-43d0-b704-aa55dc9df83a"}]: dispatch 2024-03-19T01:34:58.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:57 smithi099 ceph-mon[62951]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d958d6b9-67f6-43d0-b704-aa55dc9df83a"}]': finished 2024-03-19T01:34:58.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:57 smithi099 ceph-mon[62951]: osdmap e31: 5 total, 4 up, 5 in 2024-03-19T01:34:58.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:57 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-19T01:34:58.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:57 smithi112 ceph-mon[66819]: Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2024-03-19T01:34:58.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:57 smithi112 ceph-mon[66819]: osdmap e30: 4 total, 4 up, 4 in 2024-03-19T01:34:58.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:57 smithi112 ceph-mon[66819]: from='client.? 172.21.15.112:0/1161088139' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d958d6b9-67f6-43d0-b704-aa55dc9df83a"}]: dispatch 2024-03-19T01:34:58.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:57 smithi112 ceph-mon[66819]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d958d6b9-67f6-43d0-b704-aa55dc9df83a"}]: dispatch 2024-03-19T01:34:58.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:57 smithi112 ceph-mon[66819]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d958d6b9-67f6-43d0-b704-aa55dc9df83a"}]': finished 2024-03-19T01:34:58.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:57 smithi112 ceph-mon[66819]: osdmap e31: 5 total, 4 up, 5 in 2024-03-19T01:34:58.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:57 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-19T01:34:59.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:58 smithi115 ceph-mon[67332]: from='client.? 172.21.15.112:0/434663070' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-19T01:34:59.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:34:58 smithi115 ceph-mon[67332]: pgmap v107: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail; 2/6 objects degraded (33.333%) 2024-03-19T01:34:59.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:58 smithi099 ceph-mon[62951]: from='client.? 172.21.15.112:0/434663070' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-19T01:34:59.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:34:58 smithi099 ceph-mon[62951]: pgmap v107: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail; 2/6 objects degraded (33.333%) 2024-03-19T01:34:59.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:58 smithi112 ceph-mon[66819]: from='client.? 172.21.15.112:0/434663070' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-19T01:34:59.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:34:58 smithi112 ceph-mon[66819]: pgmap v107: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail; 2/6 objects degraded (33.333%) 2024-03-19T01:35:01.971 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:01 smithi112 ceph-mon[66819]: pgmap v108: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 117 MiB used, 357 GiB / 358 GiB avail; 2/6 objects degraded (33.333%) 2024-03-19T01:35:02.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:01 smithi115 ceph-mon[67332]: pgmap v108: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 117 MiB used, 357 GiB / 358 GiB avail; 2/6 objects degraded (33.333%) 2024-03-19T01:35:02.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:01 smithi099 ceph-mon[62951]: pgmap v108: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 117 MiB used, 357 GiB / 358 GiB avail; 2/6 objects degraded (33.333%) 2024-03-19T01:35:03.058 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:02 smithi115 ceph-mon[67332]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded) 2024-03-19T01:35:03.058 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:02 smithi115 ceph-mon[67332]: Cluster is now healthy 2024-03-19T01:35:03.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:02 smithi099 ceph-mon[62951]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded) 2024-03-19T01:35:03.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:02 smithi099 ceph-mon[62951]: Cluster is now healthy 2024-03-19T01:35:03.108 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:02 smithi112 ceph-mon[66819]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded) 2024-03-19T01:35:03.108 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:02 smithi112 ceph-mon[66819]: Cluster is now healthy 2024-03-19T01:35:03.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:03 smithi112 ceph-mon[66819]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 117 MiB used, 357 GiB / 358 GiB avail; 85 KiB/s, 0 objects/s recovering 2024-03-19T01:35:04.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:03 smithi115 ceph-mon[67332]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 117 MiB used, 357 GiB / 358 GiB avail; 85 KiB/s, 0 objects/s recovering 2024-03-19T01:35:04.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:03 smithi099 ceph-mon[62951]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 117 MiB used, 357 GiB / 358 GiB avail; 85 KiB/s, 0 objects/s recovering 2024-03-19T01:35:06.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:05 smithi115 ceph-mon[67332]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-03-19T01:35:06.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:05 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-19T01:35:06.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:05 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:35:06.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:05 smithi099 ceph-mon[62951]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-03-19T01:35:06.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:05 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-19T01:35:06.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:05 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:35:06.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:05 smithi112 ceph-mon[66819]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-03-19T01:35:06.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:05 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-19T01:35:06.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:05 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:35:06.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:06 smithi112 ceph-mon[66819]: Deploying daemon osd.4 on smithi112 2024-03-19T01:35:07.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:06 smithi099 ceph-mon[62951]: Deploying daemon osd.4 on smithi112 2024-03-19T01:35:07.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:06 smithi115 ceph-mon[67332]: Deploying daemon osd.4 on smithi112 2024-03-19T01:35:07.897 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:07 smithi112 ceph-mon[66819]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail; 59 KiB/s, 0 objects/s recovering 2024-03-19T01:35:08.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:07 smithi099 ceph-mon[62951]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail; 59 KiB/s, 0 objects/s recovering 2024-03-19T01:35:08.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:07 smithi115 ceph-mon[67332]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail; 59 KiB/s, 0 objects/s recovering 2024-03-19T01:35:09.930 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:09 smithi112 ceph-mon[66819]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail; 53 KiB/s, 0 objects/s recovering 2024-03-19T01:35:10.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:09 smithi099 ceph-mon[62951]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail; 53 KiB/s, 0 objects/s recovering 2024-03-19T01:35:10.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:09 smithi115 ceph-mon[67332]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail; 53 KiB/s, 0 objects/s recovering 2024-03-19T01:35:11.806 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:11 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:11.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:11 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:11.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:11 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:35:11.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:11 smithi115 ceph-mon[67332]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-03-19T01:35:11.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:11 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:11.842 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:11 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:11.842 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:11 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:35:11.842 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:11 smithi099 ceph-mon[62951]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-03-19T01:35:11.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:11 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:11.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:11 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:11.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:11 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:35:11.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:11 smithi112 ceph-mon[66819]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-03-19T01:35:13.813 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:35:13.812+0000 7f4c94ff9700 1 -- 172.21.15.112:0/2935070791 <== mgr.14150 v2:172.21.15.99:6800/4272410295 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+37 (secure 0 0 0) 0x7f4c98062750 con 0x7f4c800a2890 2024-03-19T01:35:13.813 INFO:teuthology.orchestra.run.smithi112.stdout:Created osd(s) 4 on host 'smithi112' 2024-03-19T01:35:13.814 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:35:13.814+0000 7f4c9dc22700 1 -- 172.21.15.112:0/2935070791 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f4c800a2890 msgr2=0x7f4c800a4d50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:35:13.815 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:35:13.814+0000 7f4c9dc22700 1 --2- 172.21.15.112:0/2935070791 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f4c800a2890 0x7f4c800a4d50 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7f4c88005950 tx=0x7f4c880058e0 comp rx=0 tx=0).stop 2024-03-19T01:35:13.815 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:35:13.814+0000 7f4c9dc22700 1 -- 172.21.15.112:0/2935070791 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4c98103600 msgr2=0x7f4c98198a70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:35:13.815 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:35:13.814+0000 7f4c9dc22700 1 --2- 172.21.15.112:0/2935070791 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4c98103600 0x7f4c98198a70 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7f4c8c005910 tx=0x7f4c8c00bc60 comp rx=0 tx=0).stop 2024-03-19T01:35:13.815 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:35:13.815+0000 7f4c9dc22700 1 -- 172.21.15.112:0/2935070791 shutdown_connections 2024-03-19T01:35:13.815 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:35:13.815+0000 7f4c9dc22700 1 --2- 172.21.15.112:0/2935070791 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f4c800a2890 0x7f4c800a4d50 unknown :-1 s=CLOSED pgs=73 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:35:13.815 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:35:13.815+0000 7f4c9dc22700 1 --2- 172.21.15.112:0/2935070791 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4c98100620 0x7f4c98198530 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:35:13.815 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:35:13.815+0000 7f4c9dc22700 1 --2- 172.21.15.112:0/2935070791 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4c98103fa0 0x7f4c9819faf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:35:13.815 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:35:13.815+0000 7f4c9dc22700 1 --2- 172.21.15.112:0/2935070791 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4c98103600 0x7f4c98198a70 unknown :-1 s=CLOSED pgs=19 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:35:13.815 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:35:13.815+0000 7f4c9dc22700 1 -- 172.21.15.112:0/2935070791 >> 172.21.15.112:0/2935070791 conn(0x7f4c980fbfa0 msgr2=0x7f4c98106900 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:35:13.815 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:35:13.815+0000 7f4c9dc22700 1 -- 172.21.15.112:0/2935070791 shutdown_connections 2024-03-19T01:35:13.815 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:35:13.815+0000 7f4c9dc22700 1 -- 172.21.15.112:0/2935070791 wait complete. 2024-03-19T01:35:14.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:13 smithi115 ceph-mon[67332]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-03-19T01:35:14.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:13 smithi099 ceph-mon[62951]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-03-19T01:35:14.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:13 smithi112 ceph-mon[66819]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-03-19T01:35:14.465 DEBUG:teuthology.orchestra.run.smithi112:osd.4> sudo journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@osd.4.service 2024-03-19T01:35:14.468 INFO:tasks.cephadm:Deploying osd.5 on smithi115 with /dev/vg_nvme/lv_3... 2024-03-19T01:35:14.468 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-03-19T01:35:14.711 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.c/config 2024-03-19T01:35:15.049 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:14 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:15.049 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:14 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:15.049 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:14 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:15.049 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:14 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:15.049 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:14 smithi112 ceph-mon[66819]: from='osd.4 [v2:172.21.15.112:6808/3717986391,v1:172.21.15.112:6809/3717986391]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-19T01:35:15.049 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:14 smithi112 ceph-mon[66819]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-19T01:35:15.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:14 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:15.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:14 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:15.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:14 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:15.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:14 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:15.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:14 smithi115 ceph-mon[67332]: from='osd.4 [v2:172.21.15.112:6808/3717986391,v1:172.21.15.112:6809/3717986391]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-19T01:35:15.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:14 smithi115 ceph-mon[67332]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-19T01:35:15.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:14 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:15.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:14 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:15.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:14 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:15.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:14 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:15.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:14 smithi099 ceph-mon[62951]: from='osd.4 [v2:172.21.15.112:6808/3717986391,v1:172.21.15.112:6809/3717986391]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-19T01:35:15.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:14 smithi099 ceph-mon[62951]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-19T01:35:15.990 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:15 smithi112 ceph-mon[66819]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-03-19T01:35:15.990 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:15 smithi112 ceph-mon[66819]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-03-19T01:35:15.990 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:15 smithi112 ceph-mon[66819]: osdmap e32: 5 total, 4 up, 5 in 2024-03-19T01:35:15.990 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:15 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-19T01:35:15.990 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:15 smithi112 ceph-mon[66819]: from='osd.4 [v2:172.21.15.112:6808/3717986391,v1:172.21.15.112:6809/3717986391]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-03-19T01:35:15.990 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:15 smithi112 ceph-mon[66819]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-03-19T01:35:16.041 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:15 smithi115 ceph-mon[67332]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-03-19T01:35:16.042 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:15 smithi115 ceph-mon[67332]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-03-19T01:35:16.042 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:15 smithi115 ceph-mon[67332]: osdmap e32: 5 total, 4 up, 5 in 2024-03-19T01:35:16.042 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:15 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-19T01:35:16.042 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:15 smithi115 ceph-mon[67332]: from='osd.4 [v2:172.21.15.112:6808/3717986391,v1:172.21.15.112:6809/3717986391]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-03-19T01:35:16.042 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:15 smithi115 ceph-mon[67332]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-03-19T01:35:16.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:15 smithi099 ceph-mon[62951]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-03-19T01:35:16.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:15 smithi099 ceph-mon[62951]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-03-19T01:35:16.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:15 smithi099 ceph-mon[62951]: osdmap e32: 5 total, 4 up, 5 in 2024-03-19T01:35:16.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:15 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-19T01:35:16.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:15 smithi099 ceph-mon[62951]: from='osd.4 [v2:172.21.15.112:6808/3717986391,v1:172.21.15.112:6809/3717986391]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-03-19T01:35:16.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:15 smithi099 ceph-mon[62951]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-03-19T01:35:16.362 INFO:journalctl@ceph.osd.4.smithi112.stdout:Mar 19 01:35:16 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-4[78359]: 2024-03-19T01:35:16.146+0000 7f39ac75b700 -1 osd.4 0 waiting for initial osdmap 2024-03-19T01:35:16.362 INFO:journalctl@ceph.osd.4.smithi112.stdout:Mar 19 01:35:16 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-4[78359]: 2024-03-19T01:35:16.152+0000 7f39a854e700 -1 osd.4 33 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-19T01:35:16.594 INFO:teuthology.orchestra.run.smithi115.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-03-19T01:35:16.594 INFO:teuthology.orchestra.run.smithi115.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-03-19T01:35:16.594 INFO:teuthology.orchestra.run.smithi115.stderr: stderr: 10+0 records in 2024-03-19T01:35:16.594 INFO:teuthology.orchestra.run.smithi115.stderr:10+0 records out 2024-03-19T01:35:16.595 INFO:teuthology.orchestra.run.smithi115.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0137139 s, 765 MB/s 2024-03-19T01:35:16.595 INFO:teuthology.orchestra.run.smithi115.stderr:--> Zapping successful for: 2024-03-19T01:35:17.169 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph orch daemon add osd smithi115:vg_nvme/lv_3 2024-03-19T01:35:17.359 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:17 smithi112 ceph-mon[66819]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]': finished 2024-03-19T01:35:17.359 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:17 smithi112 ceph-mon[66819]: osdmap e33: 5 total, 4 up, 5 in 2024-03-19T01:35:17.359 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:17 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-19T01:35:17.359 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:17 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-19T01:35:17.359 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:17 smithi112 ceph-mon[66819]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-03-19T01:35:17.411 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.c/config 2024-03-19T01:35:17.452 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:17 smithi115 ceph-mon[67332]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]': finished 2024-03-19T01:35:17.452 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:17 smithi115 ceph-mon[67332]: osdmap e33: 5 total, 4 up, 5 in 2024-03-19T01:35:17.452 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:17 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-19T01:35:17.452 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:17 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-19T01:35:17.452 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:17 smithi115 ceph-mon[67332]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-03-19T01:35:17.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:17 smithi099 ceph-mon[62951]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]': finished 2024-03-19T01:35:17.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:17 smithi099 ceph-mon[62951]: osdmap e33: 5 total, 4 up, 5 in 2024-03-19T01:35:17.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:17 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-19T01:35:17.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:17 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-19T01:35:17.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:17 smithi099 ceph-mon[62951]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-03-19T01:35:18.400 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:18 smithi115 ceph-mon[67332]: purged_snaps scrub starts 2024-03-19T01:35:18.400 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:18 smithi115 ceph-mon[67332]: purged_snaps scrub ok 2024-03-19T01:35:18.400 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:18 smithi115 ceph-mon[67332]: osd.4 [v2:172.21.15.112:6808/3717986391,v1:172.21.15.112:6809/3717986391] boot 2024-03-19T01:35:18.400 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:18 smithi115 ceph-mon[67332]: osdmap e34: 5 total, 5 up, 5 in 2024-03-19T01:35:18.400 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:18 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-19T01:35:18.401 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:18 smithi115 ceph-mon[67332]: Detected new or changed devices on smithi112 2024-03-19T01:35:18.401 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:18 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:18.401 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:18 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:18.401 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:18 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:35:18.401 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:18 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:35:18.401 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:18 smithi115 ceph-mon[67332]: Adjusting osd_memory_target on smithi112 to 8540M 2024-03-19T01:35:18.401 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:18 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:18.401 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:18 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:35:18.401 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:18 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:35:18.401 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:18 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:18.401 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:18 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:35:18.401 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:18 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:35:18.401 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:18 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:35:18.401 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:18 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:18.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:18 smithi099 ceph-mon[62951]: purged_snaps scrub starts 2024-03-19T01:35:18.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:18 smithi099 ceph-mon[62951]: purged_snaps scrub ok 2024-03-19T01:35:18.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:18 smithi099 ceph-mon[62951]: osd.4 [v2:172.21.15.112:6808/3717986391,v1:172.21.15.112:6809/3717986391] boot 2024-03-19T01:35:18.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:18 smithi099 ceph-mon[62951]: osdmap e34: 5 total, 5 up, 5 in 2024-03-19T01:35:18.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:18 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-19T01:35:18.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:18 smithi099 ceph-mon[62951]: Detected new or changed devices on smithi112 2024-03-19T01:35:18.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:18 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:18.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:18 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:18.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:18 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:35:18.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:18 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:35:18.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:18 smithi099 ceph-mon[62951]: Adjusting osd_memory_target on smithi112 to 8540M 2024-03-19T01:35:18.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:18 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:18.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:18 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:35:18.593 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:18 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:35:18.593 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:18 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:18.593 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:18 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:35:18.593 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:18 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:35:18.593 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:18 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:35:18.593 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:18 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:18.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:18 smithi112 ceph-mon[66819]: purged_snaps scrub starts 2024-03-19T01:35:18.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:18 smithi112 ceph-mon[66819]: purged_snaps scrub ok 2024-03-19T01:35:18.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:18 smithi112 ceph-mon[66819]: osd.4 [v2:172.21.15.112:6808/3717986391,v1:172.21.15.112:6809/3717986391] boot 2024-03-19T01:35:18.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:18 smithi112 ceph-mon[66819]: osdmap e34: 5 total, 5 up, 5 in 2024-03-19T01:35:18.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:18 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-19T01:35:18.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:18 smithi112 ceph-mon[66819]: Detected new or changed devices on smithi112 2024-03-19T01:35:18.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:18 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:18.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:18 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:18.613 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:18 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:35:18.613 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:18 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:35:18.613 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:18 smithi112 ceph-mon[66819]: Adjusting osd_memory_target on smithi112 to 8540M 2024-03-19T01:35:18.613 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:18 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:18.613 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:18 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:35:18.613 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:18 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:35:18.613 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:18 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:18.613 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:18 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:35:18.613 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:18 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:35:18.613 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:18 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:35:18.613 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:18 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:18.797 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:35:18.795+0000 7f58b0ecd700 1 -- 172.21.15.115:0/1663906443 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f58ac102e60 msgr2=0x7f58ac105250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:35:18.797 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:35:18.795+0000 7f58b0ecd700 1 --2- 172.21.15.115:0/1663906443 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f58ac102e60 0x7f58ac105250 secure :-1 s=READY pgs=21 cs=0 l=1 rev1=1 crypto rx=0x7f589c00b0a0 tx=0x7f589c00b3b0 comp rx=0 tx=0).stop 2024-03-19T01:35:18.797 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:35:18.796+0000 7f58b0ecd700 1 -- 172.21.15.115:0/1663906443 shutdown_connections 2024-03-19T01:35:18.797 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:35:18.796+0000 7f58b0ecd700 1 --2- 172.21.15.115:0/1663906443 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f58ac102e60 0x7f58ac105250 unknown :-1 s=CLOSED pgs=21 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:35:18.797 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:35:18.796+0000 7f58b0ecd700 1 --2- 172.21.15.115:0/1663906443 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f58ac1080c0 0x7f58ac10a4d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:35:18.798 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:35:18.796+0000 7f58b0ecd700 1 --2- 172.21.15.115:0/1663906443 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f58ac105790 0x7f58ac107b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:35:18.798 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:35:18.796+0000 7f58b0ecd700 1 -- 172.21.15.115:0/1663906443 >> 172.21.15.115:0/1663906443 conn(0x7f58ac0fc830 msgr2=0x7f58ac0fec50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:35:18.798 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:35:18.796+0000 7f58b0ecd700 1 -- 172.21.15.115:0/1663906443 shutdown_connections 2024-03-19T01:35:18.798 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:35:18.796+0000 7f58b0ecd700 1 -- 172.21.15.115:0/1663906443 wait complete. 2024-03-19T01:35:18.798 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:35:18.796+0000 7f58b0ecd700 1 Processor -- start 2024-03-19T01:35:18.798 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:35:18.796+0000 7f58b0ecd700 1 -- start start 2024-03-19T01:35:18.798 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:35:18.796+0000 7f58b0ecd700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f58ac102e60 0x7f58ac198580 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:35:18.798 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:35:18.797+0000 7f58b0ecd700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f58ac105790 0x7f58ac198ac0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:35:18.798 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:35:18.797+0000 7f58b0ecd700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f58ac1080c0 0x7f58ac19fb40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:35:18.798 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:35:18.797+0000 7f58b0ecd700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f58ac10df80 con 0x7f58ac1080c0 2024-03-19T01:35:18.798 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:35:18.797+0000 7f58b0ecd700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f58ac10de00 con 0x7f58ac102e60 2024-03-19T01:35:18.798 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:35:18.797+0000 7f58b0ecd700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f58ac10e100 con 0x7f58ac105790 2024-03-19T01:35:18.798 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:35:18.797+0000 7f58a9d9b700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f58ac105790 0x7f58ac198ac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:35:18.799 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:35:18.797+0000 7f58a9d9b700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f58ac105790 0x7f58ac198ac0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.115:58598/0 (socket says 172.21.15.115:58598) 2024-03-19T01:35:18.799 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:35:18.797+0000 7f58aa59c700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f58ac102e60 0x7f58ac198580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:35:18.799 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:35:18.797+0000 7f58a9d9b700 1 -- 172.21.15.115:0/2338565467 learned_addr learned my addr 172.21.15.115:0/2338565467 (peer_addr_for_me v2:172.21.15.115:0/0) 2024-03-19T01:35:18.799 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:35:18.797+0000 7f58aad9d700 1 --2- 172.21.15.115:0/2338565467 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f58ac1080c0 0x7f58ac19fb40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:35:18.799 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:35:18.797+0000 7f58a9d9b700 1 -- 172.21.15.115:0/2338565467 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f58ac102e60 msgr2=0x7f58ac198580 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:35:18.799 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:35:18.797+0000 7f58a9d9b700 1 --2- 172.21.15.115:0/2338565467 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f58ac102e60 0x7f58ac198580 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:35:18.799 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:35:18.797+0000 7f58a9d9b700 1 -- 172.21.15.115:0/2338565467 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f58ac1080c0 msgr2=0x7f58ac19fb40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:35:18.799 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:35:18.797+0000 7f58a9d9b700 1 --2- 172.21.15.115:0/2338565467 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f58ac1080c0 0x7f58ac19fb40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:35:18.799 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:35:18.797+0000 7f58a9d9b700 1 -- 172.21.15.115:0/2338565467 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f58ac1a01e0 con 0x7f58ac105790 2024-03-19T01:35:18.799 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:35:18.797+0000 7f58a9d9b700 1 --2- 172.21.15.115:0/2338565467 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f58ac105790 0x7f58ac198ac0 secure :-1 s=READY pgs=22 cs=0 l=1 rev1=1 crypto rx=0x7f58a000d9b0 tx=0x7f58a000dcc0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:35:18.799 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:35:18.797+0000 7f589b7fe700 1 -- 172.21.15.115:0/2338565467 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f58a0017070 con 0x7f58ac105790 2024-03-19T01:35:18.799 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:35:18.797+0000 7f589b7fe700 1 -- 172.21.15.115:0/2338565467 <== mon.1 v2:172.21.15.115:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f58a0005020 con 0x7f58ac105790 2024-03-19T01:35:18.799 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:35:18.797+0000 7f58b0ecd700 1 -- 172.21.15.115:0/2338565467 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f58ac1a04d0 con 0x7f58ac105790 2024-03-19T01:35:18.799 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:35:18.797+0000 7f589b7fe700 1 -- 172.21.15.115:0/2338565467 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f58a0009a90 con 0x7f58ac105790 2024-03-19T01:35:18.800 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:35:18.797+0000 7f58b0ecd700 1 -- 172.21.15.115:0/2338565467 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f58ac1a09c0 con 0x7f58ac105790 2024-03-19T01:35:18.800 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:35:18.798+0000 7f58b0ecd700 1 -- 172.21.15.115:0/2338565467 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f58ac068fd0 con 0x7f58ac105790 2024-03-19T01:35:18.800 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:35:18.799+0000 7f589b7fe700 1 -- 172.21.15.115:0/2338565467 <== mon.1 v2:172.21.15.115:3300/0 4 ==== mgrmap(e 16) ==== 137290+0+0 (secure 0 0 0) 0x7f58a00051d0 con 0x7f58ac105790 2024-03-19T01:35:18.800 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:35:18.800+0000 7f589b7fe700 1 --2- 172.21.15.115:0/2338565467 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f58940a2890 0x7f58940a4d50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:35:18.801 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:35:18.800+0000 7f589b7fe700 1 -- 172.21.15.115:0/2338565467 <== mon.1 v2:172.21.15.115:3300/0 5 ==== osd_map(35..35 src has 1..35) ==== 4101+0+0 (secure 0 0 0) 0x7f58a00cd850 con 0x7f58ac105790 2024-03-19T01:35:18.801 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:35:18.800+0000 7f58aa59c700 1 --2- 172.21.15.115:0/2338565467 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f58940a2890 0x7f58940a4d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:35:18.801 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:35:18.800+0000 7f58aa59c700 1 --2- 172.21.15.115:0/2338565467 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f58940a2890 0x7f58940a4d50 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7f589c009fd0 tx=0x7f589c00b7b0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:35:18.804 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:35:18.804+0000 7f589b7fe700 1 -- 172.21.15.115:0/2338565467 <== mon.1 v2:172.21.15.115:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f58a0020020 con 0x7f58ac105790 2024-03-19T01:35:19.009 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:35:19.009+0000 7f58b0ecd700 1 -- 172.21.15.115:0/2338565467 --> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi115:vg_nvme/lv_3", "target": ["mon-mgr", ""]}) -- 0x7f58ac062750 con 0x7f58940a2890 2024-03-19T01:35:19.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:19 smithi115 ceph-mon[67332]: osdmap e35: 5 total, 5 up, 5 in 2024-03-19T01:35:19.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:19 smithi115 ceph-mon[67332]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-03-19T01:35:19.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:19 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-19T01:35:19.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:19 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-19T01:35:19.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:19 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:35:19.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:19 smithi112 ceph-mon[66819]: osdmap e35: 5 total, 5 up, 5 in 2024-03-19T01:35:19.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:19 smithi112 ceph-mon[66819]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-03-19T01:35:19.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:19 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-19T01:35:19.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:19 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-19T01:35:19.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:19 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:35:19.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:19 smithi099 ceph-mon[62951]: osdmap e35: 5 total, 5 up, 5 in 2024-03-19T01:35:19.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:19 smithi099 ceph-mon[62951]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-03-19T01:35:19.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:19 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-19T01:35:19.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:19 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-19T01:35:19.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:19 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:35:20.160 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:20 smithi115 ceph-mon[67332]: from='client.24220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi115:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:35:20.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:20 smithi112 ceph-mon[66819]: from='client.24220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi115:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:35:20.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:20 smithi099 ceph-mon[62951]: from='client.24220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi115:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:35:21.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:21 smithi115 ceph-mon[67332]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-03-19T01:35:21.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:21 smithi099 ceph-mon[62951]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-03-19T01:35:21.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:21 smithi112 ceph-mon[66819]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-03-19T01:35:22.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:22 smithi115 ceph-mon[67332]: from='client.? 172.21.15.115:0/3327834951' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a73eb49a-7925-49ba-bc4f-515e17f8b8ae"}]: dispatch 2024-03-19T01:35:22.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:22 smithi115 ceph-mon[67332]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a73eb49a-7925-49ba-bc4f-515e17f8b8ae"}]: dispatch 2024-03-19T01:35:22.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:22 smithi115 ceph-mon[67332]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a73eb49a-7925-49ba-bc4f-515e17f8b8ae"}]': finished 2024-03-19T01:35:22.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:22 smithi115 ceph-mon[67332]: osdmap e36: 6 total, 5 up, 6 in 2024-03-19T01:35:22.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:22 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-19T01:35:22.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:22 smithi099 ceph-mon[62951]: from='client.? 172.21.15.115:0/3327834951' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a73eb49a-7925-49ba-bc4f-515e17f8b8ae"}]: dispatch 2024-03-19T01:35:22.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:22 smithi099 ceph-mon[62951]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a73eb49a-7925-49ba-bc4f-515e17f8b8ae"}]: dispatch 2024-03-19T01:35:22.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:22 smithi099 ceph-mon[62951]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a73eb49a-7925-49ba-bc4f-515e17f8b8ae"}]': finished 2024-03-19T01:35:22.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:22 smithi099 ceph-mon[62951]: osdmap e36: 6 total, 5 up, 6 in 2024-03-19T01:35:22.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:22 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-19T01:35:22.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:22 smithi112 ceph-mon[66819]: from='client.? 172.21.15.115:0/3327834951' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a73eb49a-7925-49ba-bc4f-515e17f8b8ae"}]: dispatch 2024-03-19T01:35:22.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:22 smithi112 ceph-mon[66819]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a73eb49a-7925-49ba-bc4f-515e17f8b8ae"}]: dispatch 2024-03-19T01:35:22.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:22 smithi112 ceph-mon[66819]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a73eb49a-7925-49ba-bc4f-515e17f8b8ae"}]': finished 2024-03-19T01:35:22.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:22 smithi112 ceph-mon[66819]: osdmap e36: 6 total, 5 up, 6 in 2024-03-19T01:35:22.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:22 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-19T01:35:23.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:23 smithi115 ceph-mon[67332]: from='client.? 172.21.15.115:0/1065866756' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-19T01:35:23.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:23 smithi115 ceph-mon[67332]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-03-19T01:35:23.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:23 smithi099 ceph-mon[62951]: from='client.? 172.21.15.115:0/1065866756' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-19T01:35:23.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:23 smithi099 ceph-mon[62951]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-03-19T01:35:23.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:23 smithi112 ceph-mon[66819]: from='client.? 172.21.15.115:0/1065866756' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-19T01:35:23.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:23 smithi112 ceph-mon[66819]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-03-19T01:35:25.863 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:25 smithi115 ceph-mon[67332]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-03-19T01:35:26.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:25 smithi099 ceph-mon[62951]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-03-19T01:35:26.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:25 smithi112 ceph-mon[66819]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-03-19T01:35:27.956 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:27 smithi115 ceph-mon[67332]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-03-19T01:35:28.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:27 smithi099 ceph-mon[62951]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-03-19T01:35:28.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:27 smithi112 ceph-mon[66819]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-03-19T01:35:30.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:29 smithi115 ceph-mon[67332]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-03-19T01:35:30.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:29 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-19T01:35:30.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:29 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:35:30.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:29 smithi115 ceph-mon[67332]: Deploying daemon osd.5 on smithi115 2024-03-19T01:35:30.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:29 smithi099 ceph-mon[62951]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-03-19T01:35:30.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:29 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-19T01:35:30.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:29 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:35:30.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:29 smithi099 ceph-mon[62951]: Deploying daemon osd.5 on smithi115 2024-03-19T01:35:30.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:29 smithi112 ceph-mon[66819]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-03-19T01:35:30.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:29 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-19T01:35:30.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:29 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:35:30.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:29 smithi112 ceph-mon[66819]: Deploying daemon osd.5 on smithi115 2024-03-19T01:35:32.009 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:31 smithi115 ceph-mon[67332]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-03-19T01:35:32.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:31 smithi099 ceph-mon[62951]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-03-19T01:35:32.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:31 smithi112 ceph-mon[66819]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-03-19T01:35:34.049 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:33 smithi115 ceph-mon[67332]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-03-19T01:35:34.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:33 smithi099 ceph-mon[62951]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-03-19T01:35:34.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:33 smithi112 ceph-mon[66819]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-03-19T01:35:35.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:35 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:35.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:35 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:35.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:35 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:35:35.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:35 smithi112 ceph-mon[66819]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-03-19T01:35:35.371 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:35 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:35.371 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:35 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:35.371 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:35 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:35:35.371 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:35 smithi115 ceph-mon[67332]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-03-19T01:35:35.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:35 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:35.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:35 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:35.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:35 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:35:35.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:35 smithi099 ceph-mon[62951]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-03-19T01:35:37.372 INFO:teuthology.orchestra.run.smithi115.stdout:Created osd(s) 5 on host 'smithi115' 2024-03-19T01:35:37.373 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:35:37.372+0000 7f589b7fe700 1 -- 172.21.15.115:0/2338565467 <== mgr.14150 v2:172.21.15.99:6800/4272410295 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+37 (secure 0 0 0) 0x7f58ac062750 con 0x7f58940a2890 2024-03-19T01:35:37.375 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:35:37.375+0000 7f58b0ecd700 1 -- 172.21.15.115:0/2338565467 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f58940a2890 msgr2=0x7f58940a4d50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:35:37.375 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:35:37.375+0000 7f58b0ecd700 1 --2- 172.21.15.115:0/2338565467 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f58940a2890 0x7f58940a4d50 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7f589c009fd0 tx=0x7f589c00b7b0 comp rx=0 tx=0).stop 2024-03-19T01:35:37.375 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:35:37.375+0000 7f58b0ecd700 1 -- 172.21.15.115:0/2338565467 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f58ac105790 msgr2=0x7f58ac198ac0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:35:37.375 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:35:37.375+0000 7f58b0ecd700 1 --2- 172.21.15.115:0/2338565467 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f58ac105790 0x7f58ac198ac0 secure :-1 s=READY pgs=22 cs=0 l=1 rev1=1 crypto rx=0x7f58a000d9b0 tx=0x7f58a000dcc0 comp rx=0 tx=0).stop 2024-03-19T01:35:37.375 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:35:37.375+0000 7f58b0ecd700 1 -- 172.21.15.115:0/2338565467 shutdown_connections 2024-03-19T01:35:37.375 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:35:37.375+0000 7f58b0ecd700 1 --2- 172.21.15.115:0/2338565467 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f58940a2890 0x7f58940a4d50 unknown :-1 s=CLOSED pgs=78 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:35:37.375 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:35:37.375+0000 7f58b0ecd700 1 --2- 172.21.15.115:0/2338565467 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f58ac1080c0 0x7f58ac19fb40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:35:37.375 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:35:37.375+0000 7f58b0ecd700 1 --2- 172.21.15.115:0/2338565467 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f58ac102e60 0x7f58ac198580 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:35:37.375 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:35:37.375+0000 7f58b0ecd700 1 --2- 172.21.15.115:0/2338565467 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f58ac105790 0x7f58ac198ac0 unknown :-1 s=CLOSED pgs=22 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:35:37.375 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:35:37.375+0000 7f58b0ecd700 1 -- 172.21.15.115:0/2338565467 >> 172.21.15.115:0/2338565467 conn(0x7f58ac0fc830 msgr2=0x7f58ac1037c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:35:37.375 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:35:37.375+0000 7f58b0ecd700 1 -- 172.21.15.115:0/2338565467 shutdown_connections 2024-03-19T01:35:37.376 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:35:37.375+0000 7f58b0ecd700 1 -- 172.21.15.115:0/2338565467 wait complete. 2024-03-19T01:35:38.025 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:37 smithi115 ceph-mon[67332]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-03-19T01:35:38.025 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:37 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:38.025 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:37 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:38.025 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:37 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:38.025 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:37 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:38.027 DEBUG:teuthology.orchestra.run.smithi115:osd.5> sudo journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@osd.5.service 2024-03-19T01:35:38.029 INFO:tasks.cephadm:Deploying osd.6 on smithi099 with /dev/vg_nvme/lv_2... 2024-03-19T01:35:38.029 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-03-19T01:35:38.059 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:37 smithi099 ceph-mon[62951]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-03-19T01:35:38.059 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:37 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:38.059 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:37 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:38.059 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:37 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:38.059 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:37 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:38.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:37 smithi112 ceph-mon[66819]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-03-19T01:35:38.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:37 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:38.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:37 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:38.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:37 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:38.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:37 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:38.281 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:35:39.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:38 smithi115 ceph-mon[67332]: from='osd.5 [v2:172.21.15.115:6808/1971806408,v1:172.21.15.115:6809/1971806408]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-19T01:35:39.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:38 smithi115 ceph-mon[67332]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-19T01:35:39.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:38 smithi099 ceph-mon[62951]: from='osd.5 [v2:172.21.15.115:6808/1971806408,v1:172.21.15.115:6809/1971806408]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-19T01:35:39.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:38 smithi099 ceph-mon[62951]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-19T01:35:39.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:38 smithi112 ceph-mon[66819]: from='osd.5 [v2:172.21.15.115:6808/1971806408,v1:172.21.15.115:6809/1971806408]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-19T01:35:39.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:38 smithi112 ceph-mon[66819]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-19T01:35:39.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:39 smithi099 ceph-mon[62951]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-03-19T01:35:39.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:39 smithi099 ceph-mon[62951]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-03-19T01:35:39.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:39 smithi099 ceph-mon[62951]: from='osd.5 [v2:172.21.15.115:6808/1971806408,v1:172.21.15.115:6809/1971806408]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi115", "root=default"]}]: dispatch 2024-03-19T01:35:39.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:39 smithi099 ceph-mon[62951]: osdmap e37: 6 total, 5 up, 6 in 2024-03-19T01:35:39.842 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:39 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-19T01:35:39.842 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:39 smithi099 ceph-mon[62951]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi115", "root=default"]}]: dispatch 2024-03-19T01:35:40.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:39 smithi115 ceph-mon[67332]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-03-19T01:35:40.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:39 smithi115 ceph-mon[67332]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-03-19T01:35:40.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:39 smithi115 ceph-mon[67332]: from='osd.5 [v2:172.21.15.115:6808/1971806408,v1:172.21.15.115:6809/1971806408]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi115", "root=default"]}]: dispatch 2024-03-19T01:35:40.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:39 smithi115 ceph-mon[67332]: osdmap e37: 6 total, 5 up, 6 in 2024-03-19T01:35:40.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:39 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-19T01:35:40.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:39 smithi115 ceph-mon[67332]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi115", "root=default"]}]: dispatch 2024-03-19T01:35:40.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:39 smithi112 ceph-mon[66819]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-03-19T01:35:40.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:39 smithi112 ceph-mon[66819]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-03-19T01:35:40.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:39 smithi112 ceph-mon[66819]: from='osd.5 [v2:172.21.15.115:6808/1971806408,v1:172.21.15.115:6809/1971806408]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi115", "root=default"]}]: dispatch 2024-03-19T01:35:40.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:39 smithi112 ceph-mon[66819]: osdmap e37: 6 total, 5 up, 6 in 2024-03-19T01:35:40.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:39 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-19T01:35:40.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:39 smithi112 ceph-mon[66819]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi115", "root=default"]}]: dispatch 2024-03-19T01:35:40.216 INFO:teuthology.orchestra.run.smithi099.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-03-19T01:35:40.216 INFO:teuthology.orchestra.run.smithi099.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-03-19T01:35:40.216 INFO:teuthology.orchestra.run.smithi099.stderr: stderr: 10+0 records in 2024-03-19T01:35:40.216 INFO:teuthology.orchestra.run.smithi099.stderr:10+0 records out 2024-03-19T01:35:40.217 INFO:teuthology.orchestra.run.smithi099.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.011437 s, 917 MB/s 2024-03-19T01:35:40.217 INFO:teuthology.orchestra.run.smithi099.stderr:--> Zapping successful for: 2024-03-19T01:35:40.790 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph orch daemon add osd smithi099:vg_nvme/lv_2 2024-03-19T01:35:40.982 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:40 smithi115 ceph-mon[67332]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi115", "root=default"]}]': finished 2024-03-19T01:35:40.982 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:40 smithi115 ceph-mon[67332]: osdmap e38: 6 total, 5 up, 6 in 2024-03-19T01:35:40.982 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:40 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-19T01:35:40.982 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:40 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-19T01:35:40.982 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:40 smithi115 ceph-mon[67332]: osdmap e39: 6 total, 5 up, 6 in 2024-03-19T01:35:40.982 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:40 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-19T01:35:40.982 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:40 smithi115 ceph-mon[67332]: from='osd.5 ' entity='osd.5' 2024-03-19T01:35:40.983 INFO:journalctl@ceph.osd.5.smithi115.stdout:Mar 19 01:35:40 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-5[78850]: 2024-03-19T01:35:40.681+0000 7fba431a5700 -1 osd.5 0 waiting for initial osdmap 2024-03-19T01:35:40.983 INFO:journalctl@ceph.osd.5.smithi115.stdout:Mar 19 01:35:40 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-5[78850]: 2024-03-19T01:35:40.757+0000 7fba3af90700 -1 osd.5 39 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-19T01:35:41.070 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:35:41.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:40 smithi099 ceph-mon[62951]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi115", "root=default"]}]': finished 2024-03-19T01:35:41.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:40 smithi099 ceph-mon[62951]: osdmap e38: 6 total, 5 up, 6 in 2024-03-19T01:35:41.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:40 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-19T01:35:41.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:40 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-19T01:35:41.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:40 smithi099 ceph-mon[62951]: osdmap e39: 6 total, 5 up, 6 in 2024-03-19T01:35:41.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:40 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-19T01:35:41.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:40 smithi099 ceph-mon[62951]: from='osd.5 ' entity='osd.5' 2024-03-19T01:35:41.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:40 smithi112 ceph-mon[66819]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi115", "root=default"]}]': finished 2024-03-19T01:35:41.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:40 smithi112 ceph-mon[66819]: osdmap e38: 6 total, 5 up, 6 in 2024-03-19T01:35:41.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:40 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-19T01:35:41.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:40 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-19T01:35:41.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:40 smithi112 ceph-mon[66819]: osdmap e39: 6 total, 5 up, 6 in 2024-03-19T01:35:41.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:40 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-19T01:35:41.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:40 smithi112 ceph-mon[66819]: from='osd.5 ' entity='osd.5' 2024-03-19T01:35:42.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:41 smithi115 ceph-mon[67332]: purged_snaps scrub starts 2024-03-19T01:35:42.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:41 smithi115 ceph-mon[67332]: purged_snaps scrub ok 2024-03-19T01:35:42.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:41 smithi115 ceph-mon[67332]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-03-19T01:35:42.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:41 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-19T01:35:42.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:41 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:42.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:41 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:42.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:41 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:35:42.058 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:41 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:35:42.058 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:41 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:42.058 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:41 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:35:42.058 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:41 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:35:42.058 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:41 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:42.058 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:41 smithi115 ceph-mon[67332]: osd.5 [v2:172.21.15.115:6808/1971806408,v1:172.21.15.115:6809/1971806408] boot 2024-03-19T01:35:42.058 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:41 smithi115 ceph-mon[67332]: osdmap e40: 6 total, 6 up, 6 in 2024-03-19T01:35:42.058 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:41 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-19T01:35:42.058 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:41 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:35:42.058 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:41 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:35:42.058 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:41 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:35:42.058 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:41 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:42.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:41 smithi099 ceph-mon[62951]: purged_snaps scrub starts 2024-03-19T01:35:42.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:41 smithi099 ceph-mon[62951]: purged_snaps scrub ok 2024-03-19T01:35:42.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:41 smithi099 ceph-mon[62951]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-03-19T01:35:42.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:41 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-19T01:35:42.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:41 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:42.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:41 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:42.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:41 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:35:42.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:41 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:35:42.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:41 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:42.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:41 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:35:42.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:41 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:35:42.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:41 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:42.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:41 smithi099 ceph-mon[62951]: osd.5 [v2:172.21.15.115:6808/1971806408,v1:172.21.15.115:6809/1971806408] boot 2024-03-19T01:35:42.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:41 smithi099 ceph-mon[62951]: osdmap e40: 6 total, 6 up, 6 in 2024-03-19T01:35:42.093 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:41 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-19T01:35:42.093 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:41 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:35:42.093 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:41 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:35:42.093 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:41 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:35:42.093 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:41 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:42.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:41 smithi112 ceph-mon[66819]: purged_snaps scrub starts 2024-03-19T01:35:42.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:41 smithi112 ceph-mon[66819]: purged_snaps scrub ok 2024-03-19T01:35:42.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:41 smithi112 ceph-mon[66819]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-03-19T01:35:42.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:41 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-19T01:35:42.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:41 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:42.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:41 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:42.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:41 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:35:42.113 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:41 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:35:42.113 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:41 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:42.113 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:41 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:35:42.113 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:41 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:35:42.113 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:41 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:42.113 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:41 smithi112 ceph-mon[66819]: osd.5 [v2:172.21.15.115:6808/1971806408,v1:172.21.15.115:6809/1971806408] boot 2024-03-19T01:35:42.113 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:41 smithi112 ceph-mon[66819]: osdmap e40: 6 total, 6 up, 6 in 2024-03-19T01:35:42.113 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:41 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-19T01:35:42.113 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:41 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:35:42.113 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:41 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:35:42.114 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:41 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:35:42.114 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:41 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:42.608 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:35:42.608+0000 7faf3e5d5700 1 -- 172.21.15.99:0/2923810507 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7faf38103540 msgr2=0x7faf381039a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:35:42.608 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:35:42.608+0000 7faf3e5d5700 1 --2- 172.21.15.99:0/2923810507 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7faf38103540 0x7faf381039a0 secure :-1 s=READY pgs=124 cs=0 l=1 rev1=1 crypto rx=0x7faf2c0099c0 tx=0x7faf2c009cd0 comp rx=0 tx=0).stop 2024-03-19T01:35:42.608 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:35:42.608+0000 7faf3e5d5700 1 -- 172.21.15.99:0/2923810507 shutdown_connections 2024-03-19T01:35:42.608 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:35:42.608+0000 7faf3e5d5700 1 --2- 172.21.15.99:0/2923810507 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7faf38102340 0x7faf38102760 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:35:42.608 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:35:42.608+0000 7faf3e5d5700 1 --2- 172.21.15.99:0/2923810507 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7faf38103ee0 0x7faf3810a790 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:35:42.609 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:35:42.608+0000 7faf3e5d5700 1 --2- 172.21.15.99:0/2923810507 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7faf38103540 0x7faf381039a0 unknown :-1 s=CLOSED pgs=124 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:35:42.609 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:35:42.608+0000 7faf3e5d5700 1 -- 172.21.15.99:0/2923810507 >> 172.21.15.99:0/2923810507 conn(0x7faf380fd920 msgr2=0x7faf380ffd40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:35:42.609 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:35:42.608+0000 7faf3e5d5700 1 -- 172.21.15.99:0/2923810507 shutdown_connections 2024-03-19T01:35:42.609 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:35:42.608+0000 7faf3e5d5700 1 -- 172.21.15.99:0/2923810507 wait complete. 2024-03-19T01:35:42.611 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:35:42.608+0000 7faf3e5d5700 1 Processor -- start 2024-03-19T01:35:42.611 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:35:42.608+0000 7faf3e5d5700 1 -- start start 2024-03-19T01:35:42.611 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:35:42.609+0000 7faf3e5d5700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7faf38102340 0x7faf381985c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:35:42.611 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:35:42.609+0000 7faf3e5d5700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7faf38103540 0x7faf38198b00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:35:42.611 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:35:42.609+0000 7faf3e5d5700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7faf38103ee0 0x7faf3819fb80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:35:42.611 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:35:42.609+0000 7faf3e5d5700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7faf3810dfa0 con 0x7faf38103ee0 2024-03-19T01:35:42.611 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:35:42.609+0000 7faf3e5d5700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7faf3810de20 con 0x7faf38102340 2024-03-19T01:35:42.611 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:35:42.609+0000 7faf3e5d5700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7faf3810e120 con 0x7faf38103540 2024-03-19T01:35:42.611 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:35:42.609+0000 7faf3cb72700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7faf38103ee0 0x7faf3819fb80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:35:42.612 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:35:42.609+0000 7faf3cb72700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7faf38103ee0 0x7faf3819fb80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:59190/0 (socket says 172.21.15.99:59190) 2024-03-19T01:35:42.612 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:35:42.609+0000 7faf3cb72700 1 -- 172.21.15.99:0/1434962157 learned_addr learned my addr 172.21.15.99:0/1434962157 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:35:42.612 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:35:42.609+0000 7faf377fe700 1 --2- 172.21.15.99:0/1434962157 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7faf38103540 0x7faf38198b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:35:42.612 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:35:42.609+0000 7faf37fff700 1 --2- 172.21.15.99:0/1434962157 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7faf38102340 0x7faf381985c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:35:42.612 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:35:42.609+0000 7faf3cb72700 1 -- 172.21.15.99:0/1434962157 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7faf38103540 msgr2=0x7faf38198b00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:35:42.612 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:35:42.609+0000 7faf3cb72700 1 --2- 172.21.15.99:0/1434962157 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7faf38103540 0x7faf38198b00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:35:42.612 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:35:42.609+0000 7faf3cb72700 1 -- 172.21.15.99:0/1434962157 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7faf38102340 msgr2=0x7faf381985c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:35:42.612 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:35:42.609+0000 7faf3cb72700 1 --2- 172.21.15.99:0/1434962157 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7faf38102340 0x7faf381985c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:35:42.612 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:35:42.609+0000 7faf3cb72700 1 -- 172.21.15.99:0/1434962157 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7faf381a0220 con 0x7faf38103ee0 2024-03-19T01:35:42.612 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:35:42.609+0000 7faf3cb72700 1 --2- 172.21.15.99:0/1434962157 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7faf38103ee0 0x7faf3819fb80 secure :-1 s=READY pgs=125 cs=0 l=1 rev1=1 crypto rx=0x7faf3000b9a0 tx=0x7faf3000bcb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:35:42.612 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:35:42.609+0000 7faf357fa700 1 -- 172.21.15.99:0/1434962157 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7faf3000d850 con 0x7faf38103ee0 2024-03-19T01:35:42.612 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:35:42.609+0000 7faf3e5d5700 1 -- 172.21.15.99:0/1434962157 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7faf380745a0 con 0x7faf38103ee0 2024-03-19T01:35:42.612 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:35:42.609+0000 7faf357fa700 1 -- 172.21.15.99:0/1434962157 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7faf3000e420 con 0x7faf38103ee0 2024-03-19T01:35:42.612 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:35:42.609+0000 7faf357fa700 1 -- 172.21.15.99:0/1434962157 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7faf30017660 con 0x7faf38103ee0 2024-03-19T01:35:42.613 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:35:42.609+0000 7faf3e5d5700 1 -- 172.21.15.99:0/1434962157 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7faf38074b60 con 0x7faf38103ee0 2024-03-19T01:35:42.613 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:35:42.610+0000 7faf3e5d5700 1 -- 172.21.15.99:0/1434962157 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7faf38068fd0 con 0x7faf38103ee0 2024-03-19T01:35:42.613 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:35:42.611+0000 7faf357fa700 1 -- 172.21.15.99:0/1434962157 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 16) ==== 137290+0+0 (secure 0 0 0) 0x7faf3000e5d0 con 0x7faf38103ee0 2024-03-19T01:35:42.613 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:35:42.612+0000 7faf357fa700 1 --2- 172.21.15.99:0/1434962157 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7faf200a2890 0x7faf200a4d50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:35:42.614 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:35:42.612+0000 7faf37fff700 1 --2- 172.21.15.99:0/1434962157 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7faf200a2890 0x7faf200a4d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:35:42.614 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:35:42.612+0000 7faf357fa700 1 -- 172.21.15.99:0/1434962157 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(41..41 src has 1..41) ==== 4562+0+0 (secure 0 0 0) 0x7faf300cdc70 con 0x7faf38103ee0 2024-03-19T01:35:42.614 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:35:42.613+0000 7faf37fff700 1 --2- 172.21.15.99:0/1434962157 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7faf200a2890 0x7faf200a4d50 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 crypto rx=0x7faf381033a0 tx=0x7faf2800b410 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:35:42.616 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:35:42.616+0000 7faf357fa700 1 -- 172.21.15.99:0/1434962157 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7faf30013030 con 0x7faf38103ee0 2024-03-19T01:35:42.824 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:35:42.824+0000 7faf3e5d5700 1 -- 172.21.15.99:0/1434962157 --> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_2", "target": ["mon-mgr", ""]}) -- 0x7faf38062750 con 0x7faf200a2890 2024-03-19T01:35:43.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:42 smithi115 ceph-mon[67332]: Detected new or changed devices on smithi115 2024-03-19T01:35:43.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:42 smithi115 ceph-mon[67332]: Adjusting osd_memory_target on smithi115 to 8540M 2024-03-19T01:35:43.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:42 smithi115 ceph-mon[67332]: osdmap e41: 6 total, 6 up, 6 in 2024-03-19T01:35:43.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:42 smithi099 ceph-mon[62951]: Detected new or changed devices on smithi115 2024-03-19T01:35:43.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:42 smithi099 ceph-mon[62951]: Adjusting osd_memory_target on smithi115 to 8540M 2024-03-19T01:35:43.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:42 smithi099 ceph-mon[62951]: osdmap e41: 6 total, 6 up, 6 in 2024-03-19T01:35:43.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:42 smithi112 ceph-mon[66819]: Detected new or changed devices on smithi115 2024-03-19T01:35:43.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:42 smithi112 ceph-mon[66819]: Adjusting osd_memory_target on smithi115 to 8540M 2024-03-19T01:35:43.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:42 smithi112 ceph-mon[66819]: osdmap e41: 6 total, 6 up, 6 in 2024-03-19T01:35:44.001 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:43 smithi099 ceph-mon[62951]: pgmap v139: 1 pgs: 1 remapped+peering; 577 KiB data, 178 MiB used, 536 GiB / 536 GiB avail 2024-03-19T01:35:44.002 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:43 smithi099 ceph-mon[62951]: from='client.14364 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:35:44.002 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:43 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-19T01:35:44.002 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:43 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-19T01:35:44.002 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:43 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:35:44.002 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:43 smithi099 ceph-mon[62951]: osdmap e42: 6 total, 6 up, 6 in 2024-03-19T01:35:44.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:43 smithi115 ceph-mon[67332]: pgmap v139: 1 pgs: 1 remapped+peering; 577 KiB data, 178 MiB used, 536 GiB / 536 GiB avail 2024-03-19T01:35:44.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:43 smithi115 ceph-mon[67332]: from='client.14364 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:35:44.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:43 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-19T01:35:44.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:43 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-19T01:35:44.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:43 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:35:44.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:43 smithi115 ceph-mon[67332]: osdmap e42: 6 total, 6 up, 6 in 2024-03-19T01:35:44.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:43 smithi112 ceph-mon[66819]: pgmap v139: 1 pgs: 1 remapped+peering; 577 KiB data, 178 MiB used, 536 GiB / 536 GiB avail 2024-03-19T01:35:44.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:43 smithi112 ceph-mon[66819]: from='client.14364 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:35:44.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:43 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-19T01:35:44.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:43 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-19T01:35:44.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:43 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:35:44.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:43 smithi112 ceph-mon[66819]: osdmap e42: 6 total, 6 up, 6 in 2024-03-19T01:35:46.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:45 smithi115 ceph-mon[67332]: pgmap v141: 1 pgs: 1 remapped+peering; 577 KiB data, 178 MiB used, 536 GiB / 536 GiB avail 2024-03-19T01:35:46.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:45 smithi115 ceph-mon[67332]: osdmap e43: 6 total, 6 up, 6 in 2024-03-19T01:35:46.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:45 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/683518698' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "caa718cd-e08b-48de-832a-29546d71e508"}]: dispatch 2024-03-19T01:35:46.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:45 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/683518698' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "caa718cd-e08b-48de-832a-29546d71e508"}]': finished 2024-03-19T01:35:46.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:45 smithi115 ceph-mon[67332]: osdmap e44: 7 total, 6 up, 7 in 2024-03-19T01:35:46.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:45 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-19T01:35:46.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:45 smithi099 ceph-mon[62951]: pgmap v141: 1 pgs: 1 remapped+peering; 577 KiB data, 178 MiB used, 536 GiB / 536 GiB avail 2024-03-19T01:35:46.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:45 smithi099 ceph-mon[62951]: osdmap e43: 6 total, 6 up, 6 in 2024-03-19T01:35:46.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:45 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/683518698' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "caa718cd-e08b-48de-832a-29546d71e508"}]: dispatch 2024-03-19T01:35:46.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:45 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/683518698' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "caa718cd-e08b-48de-832a-29546d71e508"}]': finished 2024-03-19T01:35:46.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:45 smithi099 ceph-mon[62951]: osdmap e44: 7 total, 6 up, 7 in 2024-03-19T01:35:46.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:45 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-19T01:35:46.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:45 smithi112 ceph-mon[66819]: pgmap v141: 1 pgs: 1 remapped+peering; 577 KiB data, 178 MiB used, 536 GiB / 536 GiB avail 2024-03-19T01:35:46.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:45 smithi112 ceph-mon[66819]: osdmap e43: 6 total, 6 up, 6 in 2024-03-19T01:35:46.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:45 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/683518698' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "caa718cd-e08b-48de-832a-29546d71e508"}]: dispatch 2024-03-19T01:35:46.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:45 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/683518698' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "caa718cd-e08b-48de-832a-29546d71e508"}]': finished 2024-03-19T01:35:46.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:45 smithi112 ceph-mon[66819]: osdmap e44: 7 total, 6 up, 7 in 2024-03-19T01:35:46.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:45 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-19T01:35:47.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:46 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/45104545' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-19T01:35:47.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:46 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/45104545' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-19T01:35:47.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:46 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/45104545' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-19T01:35:48.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:47 smithi115 ceph-mon[67332]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 178 MiB used, 536 GiB / 536 GiB avail; 126 KiB/s, 0 objects/s recovering 2024-03-19T01:35:48.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:47 smithi099 ceph-mon[62951]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 178 MiB used, 536 GiB / 536 GiB avail; 126 KiB/s, 0 objects/s recovering 2024-03-19T01:35:48.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:47 smithi112 ceph-mon[66819]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 178 MiB used, 536 GiB / 536 GiB avail; 126 KiB/s, 0 objects/s recovering 2024-03-19T01:35:49.994 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:49 smithi099 ceph-mon[62951]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 179 MiB used, 536 GiB / 536 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-03-19T01:35:50.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:49 smithi115 ceph-mon[67332]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 179 MiB used, 536 GiB / 536 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-03-19T01:35:50.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:49 smithi112 ceph-mon[66819]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 179 MiB used, 536 GiB / 536 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-03-19T01:35:52.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:51 smithi115 ceph-mon[67332]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 179 MiB used, 536 GiB / 536 GiB avail; 76 KiB/s, 0 objects/s recovering 2024-03-19T01:35:52.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:51 smithi099 ceph-mon[62951]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 179 MiB used, 536 GiB / 536 GiB avail; 76 KiB/s, 0 objects/s recovering 2024-03-19T01:35:52.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:51 smithi112 ceph-mon[66819]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 179 MiB used, 536 GiB / 536 GiB avail; 76 KiB/s, 0 objects/s recovering 2024-03-19T01:35:54.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:53 smithi115 ceph-mon[67332]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 179 MiB used, 536 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-03-19T01:35:54.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:53 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-19T01:35:54.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:53 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:35:54.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:53 smithi115 ceph-mon[67332]: Deploying daemon osd.6 on smithi099 2024-03-19T01:35:54.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:53 smithi099 ceph-mon[62951]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 179 MiB used, 536 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-03-19T01:35:54.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:53 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-19T01:35:54.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:53 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:35:54.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:53 smithi099 ceph-mon[62951]: Deploying daemon osd.6 on smithi099 2024-03-19T01:35:54.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:53 smithi112 ceph-mon[66819]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 179 MiB used, 536 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-03-19T01:35:54.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:53 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-19T01:35:54.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:53 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:35:54.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:53 smithi112 ceph-mon[66819]: Deploying daemon osd.6 on smithi099 2024-03-19T01:35:55.950 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:55 smithi099 ceph-mon[62951]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 179 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-03-19T01:35:56.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:55 smithi115 ceph-mon[67332]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 179 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-03-19T01:35:56.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:55 smithi112 ceph-mon[66819]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 179 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-03-19T01:35:58.023 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:57 smithi099 ceph-mon[62951]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 179 MiB used, 536 GiB / 536 GiB avail; 51 KiB/s, 0 objects/s recovering 2024-03-19T01:35:58.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:57 smithi112 ceph-mon[66819]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 179 MiB used, 536 GiB / 536 GiB avail; 51 KiB/s, 0 objects/s recovering 2024-03-19T01:35:58.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:57 smithi115 ceph-mon[67332]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 179 MiB used, 536 GiB / 536 GiB avail; 51 KiB/s, 0 objects/s recovering 2024-03-19T01:35:59.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:59 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:59.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:59 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:59.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:59 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:35:59.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:35:59 smithi099 ceph-mon[62951]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 179 MiB used, 536 GiB / 536 GiB avail 2024-03-19T01:35:59.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:59 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:59.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:59 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:59.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:59 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:35:59.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:35:59 smithi115 ceph-mon[67332]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 179 MiB used, 536 GiB / 536 GiB avail 2024-03-19T01:35:59.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:59 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:59.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:59 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:35:59.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:59 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:35:59.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:35:59 smithi112 ceph-mon[66819]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 179 MiB used, 536 GiB / 536 GiB avail 2024-03-19T01:36:01.942 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:36:01.940+0000 7faf357fa700 1 -- 172.21.15.99:0/1434962157 <== mgr.14150 v2:172.21.15.99:6800/4272410295 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+37 (secure 0 0 0) 0x7faf38062750 con 0x7faf200a2890 2024-03-19T01:36:01.943 INFO:teuthology.orchestra.run.smithi099.stdout:Created osd(s) 6 on host 'smithi099' 2024-03-19T01:36:01.944 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:36:01.943+0000 7faf3e5d5700 1 -- 172.21.15.99:0/1434962157 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7faf200a2890 msgr2=0x7faf200a4d50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:36:01.944 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:36:01.943+0000 7faf3e5d5700 1 --2- 172.21.15.99:0/1434962157 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7faf200a2890 0x7faf200a4d50 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 crypto rx=0x7faf381033a0 tx=0x7faf2800b410 comp rx=0 tx=0).stop 2024-03-19T01:36:01.945 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:36:01.943+0000 7faf3e5d5700 1 -- 172.21.15.99:0/1434962157 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7faf38103ee0 msgr2=0x7faf3819fb80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:36:01.945 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:36:01.943+0000 7faf3e5d5700 1 --2- 172.21.15.99:0/1434962157 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7faf38103ee0 0x7faf3819fb80 secure :-1 s=READY pgs=125 cs=0 l=1 rev1=1 crypto rx=0x7faf3000b9a0 tx=0x7faf3000bcb0 comp rx=0 tx=0).stop 2024-03-19T01:36:01.945 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:36:01.943+0000 7faf3e5d5700 1 -- 172.21.15.99:0/1434962157 shutdown_connections 2024-03-19T01:36:01.945 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:36:01.943+0000 7faf3e5d5700 1 --2- 172.21.15.99:0/1434962157 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7faf200a2890 0x7faf200a4d50 unknown :-1 s=CLOSED pgs=84 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:36:01.945 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:36:01.943+0000 7faf3e5d5700 1 --2- 172.21.15.99:0/1434962157 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7faf38103ee0 0x7faf3819fb80 unknown :-1 s=CLOSED pgs=125 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:36:01.945 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:36:01.943+0000 7faf3e5d5700 1 --2- 172.21.15.99:0/1434962157 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7faf38102340 0x7faf381985c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:36:01.945 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:36:01.943+0000 7faf3e5d5700 1 --2- 172.21.15.99:0/1434962157 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7faf38103540 0x7faf38198b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:36:01.945 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:36:01.943+0000 7faf3e5d5700 1 -- 172.21.15.99:0/1434962157 >> 172.21.15.99:0/1434962157 conn(0x7faf380fd920 msgr2=0x7faf38106840 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:36:01.945 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:36:01.943+0000 7faf3e5d5700 1 -- 172.21.15.99:0/1434962157 shutdown_connections 2024-03-19T01:36:01.945 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:36:01.943+0000 7faf3e5d5700 1 -- 172.21.15.99:0/1434962157 wait complete. 2024-03-19T01:36:02.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:01 smithi115 ceph-mon[67332]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 179 MiB used, 536 GiB / 536 GiB avail 2024-03-19T01:36:02.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:01 smithi099 ceph-mon[62951]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 179 MiB used, 536 GiB / 536 GiB avail 2024-03-19T01:36:02.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:01 smithi112 ceph-mon[66819]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 179 MiB used, 536 GiB / 536 GiB avail 2024-03-19T01:36:02.661 DEBUG:teuthology.orchestra.run.smithi099:osd.6> sudo journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@osd.6.service 2024-03-19T01:36:02.663 INFO:tasks.cephadm:Deploying osd.7 on smithi112 with /dev/vg_nvme/lv_2... 2024-03-19T01:36:02.663 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-03-19T01:36:02.908 INFO:teuthology.orchestra.run.smithi112.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.b/config 2024-03-19T01:36:03.029 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:02 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:03.029 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:02 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:03.030 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:02 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:03.030 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:02 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:03.030 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:02 smithi112 ceph-mon[66819]: from='osd.6 [v2:172.21.15.99:6818/1130832772,v1:172.21.15.99:6819/1130832772]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-19T01:36:03.030 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:02 smithi112 ceph-mon[66819]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 179 MiB used, 536 GiB / 536 GiB avail 2024-03-19T01:36:03.172 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:02 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:03.172 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:02 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:03.172 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:02 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:03.172 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:02 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:03.172 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:02 smithi099 ceph-mon[62951]: from='osd.6 [v2:172.21.15.99:6818/1130832772,v1:172.21.15.99:6819/1130832772]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-19T01:36:03.172 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:02 smithi099 ceph-mon[62951]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 179 MiB used, 536 GiB / 536 GiB avail 2024-03-19T01:36:03.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:02 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:03.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:02 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:03.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:02 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:03.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:02 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:03.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:02 smithi115 ceph-mon[67332]: from='osd.6 [v2:172.21.15.99:6818/1130832772,v1:172.21.15.99:6819/1130832772]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-19T01:36:03.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:02 smithi115 ceph-mon[67332]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 179 MiB used, 536 GiB / 536 GiB avail 2024-03-19T01:36:04.246 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:03 smithi099 ceph-mon[62951]: from='osd.6 [v2:172.21.15.99:6818/1130832772,v1:172.21.15.99:6819/1130832772]' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-03-19T01:36:04.247 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:03 smithi099 ceph-mon[62951]: osdmap e45: 7 total, 6 up, 7 in 2024-03-19T01:36:04.247 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:03 smithi099 ceph-mon[62951]: from='osd.6 [v2:172.21.15.99:6818/1130832772,v1:172.21.15.99:6819/1130832772]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2024-03-19T01:36:04.247 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:03 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-19T01:36:04.260 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:03 smithi112 ceph-mon[66819]: from='osd.6 [v2:172.21.15.99:6818/1130832772,v1:172.21.15.99:6819/1130832772]' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-03-19T01:36:04.260 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:03 smithi112 ceph-mon[66819]: osdmap e45: 7 total, 6 up, 7 in 2024-03-19T01:36:04.260 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:03 smithi112 ceph-mon[66819]: from='osd.6 [v2:172.21.15.99:6818/1130832772,v1:172.21.15.99:6819/1130832772]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2024-03-19T01:36:04.260 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:03 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-19T01:36:04.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:03 smithi115 ceph-mon[67332]: from='osd.6 [v2:172.21.15.99:6818/1130832772,v1:172.21.15.99:6819/1130832772]' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-03-19T01:36:04.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:03 smithi115 ceph-mon[67332]: osdmap e45: 7 total, 6 up, 7 in 2024-03-19T01:36:04.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:03 smithi115 ceph-mon[67332]: from='osd.6 [v2:172.21.15.99:6818/1130832772,v1:172.21.15.99:6819/1130832772]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2024-03-19T01:36:04.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:03 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-19T01:36:04.814 INFO:teuthology.orchestra.run.smithi112.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-03-19T01:36:04.814 INFO:teuthology.orchestra.run.smithi112.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-03-19T01:36:04.814 INFO:teuthology.orchestra.run.smithi112.stderr: stderr: 10+0 records in 2024-03-19T01:36:04.814 INFO:teuthology.orchestra.run.smithi112.stderr:10+0 records out 2024-03-19T01:36:04.814 INFO:teuthology.orchestra.run.smithi112.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0130188 s, 805 MB/s 2024-03-19T01:36:04.814 INFO:teuthology.orchestra.run.smithi112.stderr:--> Zapping successful for: 2024-03-19T01:36:05.136 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:04 smithi099 ceph-mon[62951]: from='osd.6 [v2:172.21.15.99:6818/1130832772,v1:172.21.15.99:6819/1130832772]' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2024-03-19T01:36:05.136 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:04 smithi099 ceph-mon[62951]: osdmap e46: 7 total, 6 up, 7 in 2024-03-19T01:36:05.136 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:04 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-19T01:36:05.137 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:04 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-19T01:36:05.137 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:04 smithi099 ceph-mon[62951]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 179 MiB used, 536 GiB / 536 GiB avail 2024-03-19T01:36:05.137 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:04 smithi099 ceph-mon[62951]: from='osd.6 [v2:172.21.15.99:6818/1130832772,v1:172.21.15.99:6819/1130832772]' entity='osd.6' 2024-03-19T01:36:05.137 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 01:36:04 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T01:36:04.883+0000 7f6b8c87f700 -1 osd.6 0 waiting for initial osdmap 2024-03-19T01:36:05.137 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 01:36:05 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T01:36:05.030+0000 7f6b8466a700 -1 osd.6 46 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-19T01:36:05.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:04 smithi115 ceph-mon[67332]: from='osd.6 [v2:172.21.15.99:6818/1130832772,v1:172.21.15.99:6819/1130832772]' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2024-03-19T01:36:05.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:04 smithi115 ceph-mon[67332]: osdmap e46: 7 total, 6 up, 7 in 2024-03-19T01:36:05.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:04 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-19T01:36:05.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:04 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-19T01:36:05.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:04 smithi115 ceph-mon[67332]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 179 MiB used, 536 GiB / 536 GiB avail 2024-03-19T01:36:05.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:04 smithi115 ceph-mon[67332]: from='osd.6 [v2:172.21.15.99:6818/1130832772,v1:172.21.15.99:6819/1130832772]' entity='osd.6' 2024-03-19T01:36:05.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:04 smithi112 ceph-mon[66819]: from='osd.6 [v2:172.21.15.99:6818/1130832772,v1:172.21.15.99:6819/1130832772]' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2024-03-19T01:36:05.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:04 smithi112 ceph-mon[66819]: osdmap e46: 7 total, 6 up, 7 in 2024-03-19T01:36:05.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:04 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-19T01:36:05.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:04 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-19T01:36:05.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:04 smithi112 ceph-mon[66819]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 179 MiB used, 536 GiB / 536 GiB avail 2024-03-19T01:36:05.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:04 smithi112 ceph-mon[66819]: from='osd.6 [v2:172.21.15.99:6818/1130832772,v1:172.21.15.99:6819/1130832772]' entity='osd.6' 2024-03-19T01:36:05.439 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph orch daemon add osd smithi112:vg_nvme/lv_2 2024-03-19T01:36:05.680 INFO:teuthology.orchestra.run.smithi112.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.b/config 2024-03-19T01:36:06.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:05 smithi115 ceph-mon[67332]: purged_snaps scrub starts 2024-03-19T01:36:06.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:05 smithi115 ceph-mon[67332]: purged_snaps scrub ok 2024-03-19T01:36:06.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:05 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-19T01:36:06.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:05 smithi115 ceph-mon[67332]: Detected new or changed devices on smithi099 2024-03-19T01:36:06.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:05 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:06.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:05 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:06.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:05 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:36:06.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:05 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:36:06.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:05 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:36:06.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:05 smithi115 ceph-mon[67332]: Adjusting osd_memory_target on smithi099 to 5696M 2024-03-19T01:36:06.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:05 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:06.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:05 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:36:06.308 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:05 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:36:06.308 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:05 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:06.308 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:05 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:36:06.308 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:05 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:36:06.308 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:05 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:36:06.308 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:05 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:06.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:05 smithi099 ceph-mon[62951]: purged_snaps scrub starts 2024-03-19T01:36:06.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:05 smithi099 ceph-mon[62951]: purged_snaps scrub ok 2024-03-19T01:36:06.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:05 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-19T01:36:06.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:05 smithi099 ceph-mon[62951]: Detected new or changed devices on smithi099 2024-03-19T01:36:06.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:05 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:06.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:05 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:06.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:05 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:36:06.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:05 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:36:06.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:05 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:36:06.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:05 smithi099 ceph-mon[62951]: Adjusting osd_memory_target on smithi099 to 5696M 2024-03-19T01:36:06.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:05 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:06.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:05 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:36:06.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:05 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:36:06.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:05 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:06.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:05 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:36:06.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:05 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:36:06.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:05 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:36:06.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:05 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:06.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:05 smithi112 ceph-mon[66819]: purged_snaps scrub starts 2024-03-19T01:36:06.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:05 smithi112 ceph-mon[66819]: purged_snaps scrub ok 2024-03-19T01:36:06.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:05 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-19T01:36:06.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:05 smithi112 ceph-mon[66819]: Detected new or changed devices on smithi099 2024-03-19T01:36:06.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:05 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:06.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:05 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:06.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:05 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:36:06.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:05 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:36:06.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:05 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:36:06.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:05 smithi112 ceph-mon[66819]: Adjusting osd_memory_target on smithi099 to 5696M 2024-03-19T01:36:06.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:05 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:06.363 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:05 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:36:06.363 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:05 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:36:06.363 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:05 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:06.363 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:05 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:36:06.363 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:05 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:36:06.363 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:05 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:36:06.363 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:05 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:07.189 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:06 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-19T01:36:07.189 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:06 smithi112 ceph-mon[66819]: osd.6 [v2:172.21.15.99:6818/1130832772,v1:172.21.15.99:6819/1130832772] boot 2024-03-19T01:36:07.189 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:06 smithi112 ceph-mon[66819]: osdmap e47: 7 total, 7 up, 7 in 2024-03-19T01:36:07.189 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:06 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-19T01:36:07.189 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:06 smithi112 ceph-mon[66819]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 179 MiB used, 536 GiB / 536 GiB avail 2024-03-19T01:36:07.217 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:36:07.214+0000 7fda02317700 1 -- 172.21.15.112:0/3717563063 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd9fc103530 msgr2=0x7fd9fc103990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:36:07.217 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:36:07.214+0000 7fda02317700 1 --2- 172.21.15.112:0/3717563063 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd9fc103530 0x7fd9fc103990 secure :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0x7fd9f00099c0 tx=0x7fd9f0009cd0 comp rx=0 tx=0).stop 2024-03-19T01:36:07.217 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:36:07.214+0000 7fda02317700 1 -- 172.21.15.112:0/3717563063 shutdown_connections 2024-03-19T01:36:07.217 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:36:07.214+0000 7fda02317700 1 --2- 172.21.15.112:0/3717563063 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd9fc103ed0 0x7fd9fc10a780 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:36:07.217 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:36:07.214+0000 7fda02317700 1 --2- 172.21.15.112:0/3717563063 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd9fc102330 0x7fd9fc102750 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:36:07.217 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:36:07.214+0000 7fda02317700 1 --2- 172.21.15.112:0/3717563063 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd9fc103530 0x7fd9fc103990 unknown :-1 s=CLOSED pgs=27 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:36:07.217 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:36:07.214+0000 7fda02317700 1 -- 172.21.15.112:0/3717563063 >> 172.21.15.112:0/3717563063 conn(0x7fd9fc0fd8f0 msgr2=0x7fd9fc0ffd10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:36:07.217 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:36:07.214+0000 7fda02317700 1 -- 172.21.15.112:0/3717563063 shutdown_connections 2024-03-19T01:36:07.218 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:36:07.214+0000 7fda02317700 1 -- 172.21.15.112:0/3717563063 wait complete. 2024-03-19T01:36:07.218 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:36:07.215+0000 7fda02317700 1 Processor -- start 2024-03-19T01:36:07.218 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:36:07.215+0000 7fda02317700 1 -- start start 2024-03-19T01:36:07.218 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:36:07.215+0000 7fda02317700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd9fc102330 0x7fd9fc198520 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:36:07.218 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:36:07.215+0000 7fda02317700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd9fc103530 0x7fd9fc198a60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:36:07.218 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:36:07.215+0000 7fda02317700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd9fc103ed0 0x7fd9fc19fae0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:36:07.218 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:36:07.215+0000 7fda02317700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fd9fc10df00 con 0x7fd9fc103530 2024-03-19T01:36:07.218 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:36:07.215+0000 7fda02317700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fd9fc10dd80 con 0x7fd9fc102330 2024-03-19T01:36:07.219 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:36:07.215+0000 7fda02317700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fd9fc10e080 con 0x7fd9fc103ed0 2024-03-19T01:36:07.219 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:36:07.215+0000 7fd9fbfff700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd9fc102330 0x7fd9fc198520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:36:07.219 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:36:07.215+0000 7fd9fbfff700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd9fc102330 0x7fd9fc198520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.112:36918/0 (socket says 172.21.15.112:36918) 2024-03-19T01:36:07.219 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:36:07.215+0000 7fd9fbfff700 1 -- 172.21.15.112:0/3097565615 learned_addr learned my addr 172.21.15.112:0/3097565615 (peer_addr_for_me v2:172.21.15.112:0/0) 2024-03-19T01:36:07.219 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:36:07.215+0000 7fd9fb7fe700 1 --2- 172.21.15.112:0/3097565615 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd9fc103530 0x7fd9fc198a60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:36:07.219 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:36:07.216+0000 7fda008b4700 1 --2- 172.21.15.112:0/3097565615 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd9fc103ed0 0x7fd9fc19fae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:36:07.219 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:36:07.216+0000 7fd9fbfff700 1 -- 172.21.15.112:0/3097565615 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd9fc103ed0 msgr2=0x7fd9fc19fae0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:36:07.219 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:36:07.216+0000 7fd9fbfff700 1 --2- 172.21.15.112:0/3097565615 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd9fc103ed0 0x7fd9fc19fae0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:36:07.219 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:36:07.216+0000 7fd9fbfff700 1 -- 172.21.15.112:0/3097565615 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd9fc103530 msgr2=0x7fd9fc198a60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:36:07.219 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:36:07.216+0000 7fd9fbfff700 1 --2- 172.21.15.112:0/3097565615 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd9fc103530 0x7fd9fc198a60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:36:07.219 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:36:07.216+0000 7fd9fbfff700 1 -- 172.21.15.112:0/3097565615 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd9fc1a0180 con 0x7fd9fc102330 2024-03-19T01:36:07.219 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:36:07.216+0000 7fd9fbfff700 1 --2- 172.21.15.112:0/3097565615 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd9fc102330 0x7fd9fc198520 secure :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0x7fd9ec00d980 tx=0x7fd9ec00dc90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:36:07.219 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:36:07.216+0000 7fd9f97fa700 1 -- 172.21.15.112:0/3097565615 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd9ec017030 con 0x7fd9fc102330 2024-03-19T01:36:07.220 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:36:07.216+0000 7fda02317700 1 -- 172.21.15.112:0/3097565615 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd9fc1a0470 con 0x7fd9fc102330 2024-03-19T01:36:07.220 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:36:07.216+0000 7fd9f97fa700 1 -- 172.21.15.112:0/3097565615 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd9ec004500 con 0x7fd9fc102330 2024-03-19T01:36:07.221 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:36:07.216+0000 7fd9f97fa700 1 -- 172.21.15.112:0/3097565615 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd9ec0039e0 con 0x7fd9fc102330 2024-03-19T01:36:07.221 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:36:07.216+0000 7fda02317700 1 -- 172.21.15.112:0/3097565615 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fd9fc1a09b0 con 0x7fd9fc102330 2024-03-19T01:36:07.221 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:36:07.217+0000 7fda02317700 1 -- 172.21.15.112:0/3097565615 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fd9fc068fd0 con 0x7fd9fc102330 2024-03-19T01:36:07.221 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:36:07.218+0000 7fd9f97fa700 1 -- 172.21.15.112:0/3097565615 <== mon.2 v2:172.21.15.112:3300/0 4 ==== mgrmap(e 16) ==== 137290+0+0 (secure 0 0 0) 0x7fd9ec01a070 con 0x7fd9fc102330 2024-03-19T01:36:07.221 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:36:07.219+0000 7fd9f97fa700 1 --2- 172.21.15.112:0/3097565615 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd9e40a2890 0x7fd9e40a4d50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:36:07.221 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:36:07.219+0000 7fd9f97fa700 1 -- 172.21.15.112:0/3097565615 <== mon.2 v2:172.21.15.112:3300/0 5 ==== osd_map(48..48 src has 1..48) ==== 4965+0+0 (secure 0 0 0) 0x7fd9ec0cd8e0 con 0x7fd9fc102330 2024-03-19T01:36:07.221 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:36:07.219+0000 7fd9fb7fe700 1 --2- 172.21.15.112:0/3097565615 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd9e40a2890 0x7fd9e40a4d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:36:07.221 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:36:07.219+0000 7fd9fb7fe700 1 --2- 172.21.15.112:0/3097565615 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd9e40a2890 0x7fd9e40a4d50 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7fd9f0000c00 tx=0x7fd9f0005fb0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:36:07.223 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:36:07.222+0000 7fd9f97fa700 1 -- 172.21.15.112:0/3097565615 <== mon.2 v2:172.21.15.112:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7fd9ec09b690 con 0x7fd9fc102330 2024-03-19T01:36:07.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:06 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-19T01:36:07.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:06 smithi115 ceph-mon[67332]: osd.6 [v2:172.21.15.99:6818/1130832772,v1:172.21.15.99:6819/1130832772] boot 2024-03-19T01:36:07.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:06 smithi115 ceph-mon[67332]: osdmap e47: 7 total, 7 up, 7 in 2024-03-19T01:36:07.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:06 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-19T01:36:07.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:06 smithi115 ceph-mon[67332]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 179 MiB used, 536 GiB / 536 GiB avail 2024-03-19T01:36:07.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:06 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-19T01:36:07.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:06 smithi099 ceph-mon[62951]: osd.6 [v2:172.21.15.99:6818/1130832772,v1:172.21.15.99:6819/1130832772] boot 2024-03-19T01:36:07.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:06 smithi099 ceph-mon[62951]: osdmap e47: 7 total, 7 up, 7 in 2024-03-19T01:36:07.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:06 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-19T01:36:07.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:06 smithi099 ceph-mon[62951]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 179 MiB used, 536 GiB / 536 GiB avail 2024-03-19T01:36:07.426 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:36:07.425+0000 7fda02317700 1 -- 172.21.15.112:0/3097565615 --> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi112:vg_nvme/lv_2", "target": ["mon-mgr", ""]}) -- 0x7fd9fc062750 con 0x7fd9e40a2890 2024-03-19T01:36:08.259 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:08 smithi099 ceph-mon[62951]: osdmap e48: 7 total, 7 up, 7 in 2024-03-19T01:36:08.259 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:08 smithi099 ceph-mon[62951]: from='client.24254 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi112:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:36:08.259 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:08 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-19T01:36:08.259 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:08 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-19T01:36:08.259 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:08 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:36:08.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:08 smithi115 ceph-mon[67332]: osdmap e48: 7 total, 7 up, 7 in 2024-03-19T01:36:08.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:08 smithi115 ceph-mon[67332]: from='client.24254 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi112:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:36:08.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:08 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-19T01:36:08.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:08 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-19T01:36:08.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:08 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:36:08.308 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:08 smithi112 ceph-mon[66819]: osdmap e48: 7 total, 7 up, 7 in 2024-03-19T01:36:08.308 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:08 smithi112 ceph-mon[66819]: from='client.24254 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi112:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:36:08.308 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:08 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-19T01:36:08.308 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:08 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-19T01:36:08.308 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:08 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:36:09.233 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:09 smithi112 ceph-mon[66819]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 210 MiB used, 626 GiB / 626 GiB avail 2024-03-19T01:36:09.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:09 smithi115 ceph-mon[67332]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 210 MiB used, 626 GiB / 626 GiB avail 2024-03-19T01:36:09.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:09 smithi099 ceph-mon[62951]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 210 MiB used, 626 GiB / 626 GiB avail 2024-03-19T01:36:10.428 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:10 smithi112 ceph-mon[66819]: from='client.? 172.21.15.112:0/3516330901' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "dbee19a1-2315-40d3-88e7-876fcde28903"}]: dispatch 2024-03-19T01:36:10.428 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:10 smithi112 ceph-mon[66819]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "dbee19a1-2315-40d3-88e7-876fcde28903"}]: dispatch 2024-03-19T01:36:10.428 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:10 smithi112 ceph-mon[66819]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "dbee19a1-2315-40d3-88e7-876fcde28903"}]': finished 2024-03-19T01:36:10.428 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:10 smithi112 ceph-mon[66819]: osdmap e49: 8 total, 7 up, 8 in 2024-03-19T01:36:10.428 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:10 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-19T01:36:10.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:10 smithi115 ceph-mon[67332]: from='client.? 172.21.15.112:0/3516330901' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "dbee19a1-2315-40d3-88e7-876fcde28903"}]: dispatch 2024-03-19T01:36:10.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:10 smithi115 ceph-mon[67332]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "dbee19a1-2315-40d3-88e7-876fcde28903"}]: dispatch 2024-03-19T01:36:10.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:10 smithi115 ceph-mon[67332]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "dbee19a1-2315-40d3-88e7-876fcde28903"}]': finished 2024-03-19T01:36:10.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:10 smithi115 ceph-mon[67332]: osdmap e49: 8 total, 7 up, 8 in 2024-03-19T01:36:10.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:10 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-19T01:36:10.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:10 smithi099 ceph-mon[62951]: from='client.? 172.21.15.112:0/3516330901' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "dbee19a1-2315-40d3-88e7-876fcde28903"}]: dispatch 2024-03-19T01:36:10.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:10 smithi099 ceph-mon[62951]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "dbee19a1-2315-40d3-88e7-876fcde28903"}]: dispatch 2024-03-19T01:36:10.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:10 smithi099 ceph-mon[62951]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "dbee19a1-2315-40d3-88e7-876fcde28903"}]': finished 2024-03-19T01:36:10.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:10 smithi099 ceph-mon[62951]: osdmap e49: 8 total, 7 up, 8 in 2024-03-19T01:36:10.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:10 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-19T01:36:11.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:11 smithi115 ceph-mon[67332]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 210 MiB used, 626 GiB / 626 GiB avail 2024-03-19T01:36:11.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:11 smithi115 ceph-mon[67332]: from='client.? 172.21.15.112:0/513303756' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-19T01:36:11.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:11 smithi099 ceph-mon[62951]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 210 MiB used, 626 GiB / 626 GiB avail 2024-03-19T01:36:11.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:11 smithi099 ceph-mon[62951]: from='client.? 172.21.15.112:0/513303756' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-19T01:36:11.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:11 smithi112 ceph-mon[66819]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 210 MiB used, 626 GiB / 626 GiB avail 2024-03-19T01:36:11.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:11 smithi112 ceph-mon[66819]: from='client.? 172.21.15.112:0/513303756' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-19T01:36:14.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:13 smithi115 ceph-mon[67332]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 210 MiB used, 626 GiB / 626 GiB avail 2024-03-19T01:36:14.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:13 smithi099 ceph-mon[62951]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 210 MiB used, 626 GiB / 626 GiB avail 2024-03-19T01:36:14.113 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:13 smithi112 ceph-mon[66819]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 210 MiB used, 626 GiB / 626 GiB avail 2024-03-19T01:36:16.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:15 smithi115 ceph-mon[67332]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 210 MiB used, 626 GiB / 626 GiB avail 2024-03-19T01:36:16.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:15 smithi099 ceph-mon[62951]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 210 MiB used, 626 GiB / 626 GiB avail 2024-03-19T01:36:16.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:15 smithi112 ceph-mon[66819]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 210 MiB used, 626 GiB / 626 GiB avail 2024-03-19T01:36:18.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:17 smithi115 ceph-mon[67332]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 210 MiB used, 626 GiB / 626 GiB avail 2024-03-19T01:36:18.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:17 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-19T01:36:18.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:17 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:36:18.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:17 smithi099 ceph-mon[62951]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 210 MiB used, 626 GiB / 626 GiB avail 2024-03-19T01:36:18.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:17 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-19T01:36:18.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:17 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:36:18.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:17 smithi112 ceph-mon[66819]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 210 MiB used, 626 GiB / 626 GiB avail 2024-03-19T01:36:18.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:17 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-19T01:36:18.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:17 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:36:19.024 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:18 smithi112 ceph-mon[66819]: Deploying daemon osd.7 on smithi112 2024-03-19T01:36:19.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:18 smithi115 ceph-mon[67332]: Deploying daemon osd.7 on smithi112 2024-03-19T01:36:19.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:18 smithi099 ceph-mon[62951]: Deploying daemon osd.7 on smithi112 2024-03-19T01:36:19.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:19 smithi099 ceph-mon[62951]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 210 MiB used, 626 GiB / 626 GiB avail 2024-03-19T01:36:20.026 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:19 smithi112 ceph-mon[66819]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 210 MiB used, 626 GiB / 626 GiB avail 2024-03-19T01:36:20.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:19 smithi115 ceph-mon[67332]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 210 MiB used, 626 GiB / 626 GiB avail 2024-03-19T01:36:22.013 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:21 smithi112 ceph-mon[66819]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 210 MiB used, 626 GiB / 626 GiB avail 2024-03-19T01:36:22.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:21 smithi115 ceph-mon[67332]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 210 MiB used, 626 GiB / 626 GiB avail 2024-03-19T01:36:22.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:21 smithi099 ceph-mon[62951]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 210 MiB used, 626 GiB / 626 GiB avail 2024-03-19T01:36:24.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:23 smithi112 ceph-mon[66819]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 210 MiB used, 626 GiB / 626 GiB avail 2024-03-19T01:36:24.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:23 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:24.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:23 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:24.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:23 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:36:24.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:23 smithi115 ceph-mon[67332]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 210 MiB used, 626 GiB / 626 GiB avail 2024-03-19T01:36:24.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:23 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:24.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:23 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:24.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:23 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:36:24.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:23 smithi099 ceph-mon[62951]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 210 MiB used, 626 GiB / 626 GiB avail 2024-03-19T01:36:24.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:23 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:24.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:23 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:24.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:23 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:36:26.259 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:25 smithi112 ceph-mon[66819]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 210 MiB used, 626 GiB / 626 GiB avail 2024-03-19T01:36:26.259 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:25 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:26.259 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:25 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:26.259 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:25 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:36:26.259 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:25 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:36:26.259 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:25 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:26.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:25 smithi115 ceph-mon[67332]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 210 MiB used, 626 GiB / 626 GiB avail 2024-03-19T01:36:26.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:25 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:26.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:25 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:26.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:25 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:36:26.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:25 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:36:26.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:25 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:26.339 INFO:teuthology.orchestra.run.smithi112.stdout:Created osd(s) 7 on host 'smithi112' 2024-03-19T01:36:26.339 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:36:26.338+0000 7fd9f97fa700 1 -- 172.21.15.112:0/3097565615 <== mgr.14150 v2:172.21.15.99:6800/4272410295 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+37 (secure 0 0 0) 0x7fd9fc062750 con 0x7fd9e40a2890 2024-03-19T01:36:26.341 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:36:26.340+0000 7fda02317700 1 -- 172.21.15.112:0/3097565615 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd9e40a2890 msgr2=0x7fd9e40a4d50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:36:26.341 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:36:26.340+0000 7fda02317700 1 --2- 172.21.15.112:0/3097565615 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd9e40a2890 0x7fd9e40a4d50 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7fd9f0000c00 tx=0x7fd9f0005fb0 comp rx=0 tx=0).stop 2024-03-19T01:36:26.341 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:36:26.340+0000 7fda02317700 1 -- 172.21.15.112:0/3097565615 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd9fc102330 msgr2=0x7fd9fc198520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:36:26.341 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:36:26.340+0000 7fda02317700 1 --2- 172.21.15.112:0/3097565615 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd9fc102330 0x7fd9fc198520 secure :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0x7fd9ec00d980 tx=0x7fd9ec00dc90 comp rx=0 tx=0).stop 2024-03-19T01:36:26.341 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:36:26.341+0000 7fda02317700 1 -- 172.21.15.112:0/3097565615 shutdown_connections 2024-03-19T01:36:26.341 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:36:26.341+0000 7fda02317700 1 --2- 172.21.15.112:0/3097565615 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd9e40a2890 0x7fd9e40a4d50 unknown :-1 s=CLOSED pgs=91 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:36:26.341 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:36:26.341+0000 7fda02317700 1 --2- 172.21.15.112:0/3097565615 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd9fc103ed0 0x7fd9fc19fae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:36:26.341 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:36:26.341+0000 7fda02317700 1 --2- 172.21.15.112:0/3097565615 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd9fc102330 0x7fd9fc198520 unknown :-1 s=CLOSED pgs=28 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:36:26.341 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:36:26.341+0000 7fda02317700 1 --2- 172.21.15.112:0/3097565615 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd9fc103530 0x7fd9fc198a60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:36:26.341 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:36:26.341+0000 7fda02317700 1 -- 172.21.15.112:0/3097565615 >> 172.21.15.112:0/3097565615 conn(0x7fd9fc0fd8f0 msgr2=0x7fd9fc106900 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:36:26.341 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:36:26.341+0000 7fda02317700 1 -- 172.21.15.112:0/3097565615 shutdown_connections 2024-03-19T01:36:26.341 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:36:26.341+0000 7fda02317700 1 -- 172.21.15.112:0/3097565615 wait complete. 2024-03-19T01:36:26.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:25 smithi099 ceph-mon[62951]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 210 MiB used, 626 GiB / 626 GiB avail 2024-03-19T01:36:26.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:25 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:26.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:25 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:26.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:25 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:36:26.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:25 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:36:26.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:25 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:26.965 DEBUG:teuthology.orchestra.run.smithi112:osd.7> sudo journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@osd.7.service 2024-03-19T01:36:26.967 INFO:tasks.cephadm:Deploying osd.8 on smithi115 with /dev/vg_nvme/lv_2... 2024-03-19T01:36:26.967 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-03-19T01:36:27.213 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.c/config 2024-03-19T01:36:27.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:27 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:27.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:27 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:27.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:27 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:36:27.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:27 smithi115 ceph-mon[67332]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 210 MiB used, 626 GiB / 626 GiB avail 2024-03-19T01:36:27.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:27 smithi115 ceph-mon[67332]: from='osd.7 [v2:172.21.15.112:6816/3610356576,v1:172.21.15.112:6817/3610356576]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-19T01:36:27.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:27 smithi115 ceph-mon[67332]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-19T01:36:27.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:27 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:27.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:27 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:27.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:27 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:36:27.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:27 smithi099 ceph-mon[62951]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 210 MiB used, 626 GiB / 626 GiB avail 2024-03-19T01:36:27.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:27 smithi099 ceph-mon[62951]: from='osd.7 [v2:172.21.15.112:6816/3610356576,v1:172.21.15.112:6817/3610356576]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-19T01:36:27.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:27 smithi099 ceph-mon[62951]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-19T01:36:27.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:27 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:27.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:27 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:27.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:27 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:36:27.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:27 smithi112 ceph-mon[66819]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 210 MiB used, 626 GiB / 626 GiB avail 2024-03-19T01:36:27.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:27 smithi112 ceph-mon[66819]: from='osd.7 [v2:172.21.15.112:6816/3610356576,v1:172.21.15.112:6817/3610356576]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-19T01:36:27.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:27 smithi112 ceph-mon[66819]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-19T01:36:28.312 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:28 smithi112 ceph-mon[66819]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-03-19T01:36:28.312 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:28 smithi112 ceph-mon[66819]: osdmap e50: 8 total, 7 up, 8 in 2024-03-19T01:36:28.312 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:28 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-19T01:36:28.312 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:28 smithi112 ceph-mon[66819]: from='osd.7 [v2:172.21.15.112:6816/3610356576,v1:172.21.15.112:6817/3610356576]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-03-19T01:36:28.312 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:28 smithi112 ceph-mon[66819]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-03-19T01:36:28.534 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:28 smithi115 ceph-mon[67332]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-03-19T01:36:28.534 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:28 smithi115 ceph-mon[67332]: osdmap e50: 8 total, 7 up, 8 in 2024-03-19T01:36:28.535 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:28 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-19T01:36:28.535 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:28 smithi115 ceph-mon[67332]: from='osd.7 [v2:172.21.15.112:6816/3610356576,v1:172.21.15.112:6817/3610356576]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-03-19T01:36:28.535 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:28 smithi115 ceph-mon[67332]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-03-19T01:36:28.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:28 smithi099 ceph-mon[62951]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-03-19T01:36:28.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:28 smithi099 ceph-mon[62951]: osdmap e50: 8 total, 7 up, 8 in 2024-03-19T01:36:28.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:28 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-19T01:36:28.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:28 smithi099 ceph-mon[62951]: from='osd.7 [v2:172.21.15.112:6816/3610356576,v1:172.21.15.112:6817/3610356576]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-03-19T01:36:28.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:28 smithi099 ceph-mon[62951]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-03-19T01:36:29.088 INFO:teuthology.orchestra.run.smithi115.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-03-19T01:36:29.088 INFO:teuthology.orchestra.run.smithi115.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-03-19T01:36:29.088 INFO:teuthology.orchestra.run.smithi115.stderr: stderr: 10+0 records in 2024-03-19T01:36:29.088 INFO:teuthology.orchestra.run.smithi115.stderr:10+0 records out 2024-03-19T01:36:29.088 INFO:teuthology.orchestra.run.smithi115.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0112302 s, 934 MB/s 2024-03-19T01:36:29.088 INFO:teuthology.orchestra.run.smithi115.stderr:--> Zapping successful for: 2024-03-19T01:36:29.422 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:29 smithi115 ceph-mon[67332]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]': finished 2024-03-19T01:36:29.422 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:29 smithi115 ceph-mon[67332]: osdmap e51: 8 total, 7 up, 8 in 2024-03-19T01:36:29.422 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:29 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-19T01:36:29.422 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:29 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-19T01:36:29.422 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:29 smithi115 ceph-mon[67332]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 210 MiB used, 626 GiB / 626 GiB avail 2024-03-19T01:36:29.423 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:29 smithi115 ceph-mon[67332]: from='osd.7 ' entity='osd.7' 2024-03-19T01:36:29.504 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:29 smithi112 ceph-mon[66819]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]': finished 2024-03-19T01:36:29.505 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:29 smithi112 ceph-mon[66819]: osdmap e51: 8 total, 7 up, 8 in 2024-03-19T01:36:29.505 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:29 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-19T01:36:29.505 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:29 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-19T01:36:29.505 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:29 smithi112 ceph-mon[66819]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 210 MiB used, 626 GiB / 626 GiB avail 2024-03-19T01:36:29.505 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:29 smithi112 ceph-mon[66819]: from='osd.7 ' entity='osd.7' 2024-03-19T01:36:29.505 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 01:36:29 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T01:36:29.254+0000 7f02a129e700 -1 osd.7 0 waiting for initial osdmap 2024-03-19T01:36:29.505 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 01:36:29 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T01:36:29.420+0000 7f0298888700 -1 osd.7 52 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-19T01:36:29.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:29 smithi099 ceph-mon[62951]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]': finished 2024-03-19T01:36:29.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:29 smithi099 ceph-mon[62951]: osdmap e51: 8 total, 7 up, 8 in 2024-03-19T01:36:29.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:29 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-19T01:36:29.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:29 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-19T01:36:29.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:29 smithi099 ceph-mon[62951]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 210 MiB used, 626 GiB / 626 GiB avail 2024-03-19T01:36:29.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:29 smithi099 ceph-mon[62951]: from='osd.7 ' entity='osd.7' 2024-03-19T01:36:29.723 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph orch daemon add osd smithi115:vg_nvme/lv_2 2024-03-19T01:36:29.958 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.c/config 2024-03-19T01:36:30.367 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:30 smithi115 ceph-mon[67332]: purged_snaps scrub starts 2024-03-19T01:36:30.367 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:30 smithi115 ceph-mon[67332]: purged_snaps scrub ok 2024-03-19T01:36:30.367 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:30 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-19T01:36:30.367 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:30 smithi115 ceph-mon[67332]: osdmap e52: 8 total, 7 up, 8 in 2024-03-19T01:36:30.367 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:30 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-19T01:36:30.367 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:30 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:30.367 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:30 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:30.367 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:30 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:36:30.367 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:30 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:36:30.367 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:30 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:36:30.368 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:30 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:30.368 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:30 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:36:30.368 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:30 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:36:30.368 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:30 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:30.368 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:30 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-19T01:36:30.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:30 smithi099 ceph-mon[62951]: purged_snaps scrub starts 2024-03-19T01:36:30.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:30 smithi099 ceph-mon[62951]: purged_snaps scrub ok 2024-03-19T01:36:30.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:30 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-19T01:36:30.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:30 smithi099 ceph-mon[62951]: osdmap e52: 8 total, 7 up, 8 in 2024-03-19T01:36:30.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:30 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-19T01:36:30.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:30 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:30.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:30 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:30.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:30 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:36:30.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:30 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:36:30.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:30 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:36:30.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:30 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:30.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:30 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:36:30.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:30 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:36:30.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:30 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:30.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:30 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-19T01:36:30.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:30 smithi112 ceph-mon[66819]: purged_snaps scrub starts 2024-03-19T01:36:30.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:30 smithi112 ceph-mon[66819]: purged_snaps scrub ok 2024-03-19T01:36:30.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:30 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-19T01:36:30.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:30 smithi112 ceph-mon[66819]: osdmap e52: 8 total, 7 up, 8 in 2024-03-19T01:36:30.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:30 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-19T01:36:30.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:30 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:30.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:30 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:30.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:30 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:36:30.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:30 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:36:30.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:30 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:36:30.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:30 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:30.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:30 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:36:30.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:30 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:36:30.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:30 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:30.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:30 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-19T01:36:31.332 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:36:31.325+0000 7f29d3456700 1 -- 172.21.15.115:0/72578109 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f29cc06a630 msgr2=0x7f29cc100f90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:36:31.332 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:36:31.325+0000 7f29d3456700 1 --2- 172.21.15.115:0/72578109 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f29cc06a630 0x7f29cc100f90 secure :-1 s=READY pgs=30 cs=0 l=1 rev1=1 crypto rx=0x7f29bc009990 tx=0x7f29bc009ca0 comp rx=0 tx=0).stop 2024-03-19T01:36:31.332 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:36:31.325+0000 7f29d3456700 1 -- 172.21.15.115:0/72578109 shutdown_connections 2024-03-19T01:36:31.332 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:36:31.325+0000 7f29d3456700 1 --2- 172.21.15.115:0/72578109 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f29cc101c10 0x7f29cc1020a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:36:31.332 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:36:31.325+0000 7f29d3456700 1 --2- 172.21.15.115:0/72578109 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f29cc06a180 0x7f29cc100a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:36:31.333 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:36:31.325+0000 7f29d3456700 1 --2- 172.21.15.115:0/72578109 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f29cc06a630 0x7f29cc100f90 unknown :-1 s=CLOSED pgs=30 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:36:31.333 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:36:31.325+0000 7f29d3456700 1 -- 172.21.15.115:0/72578109 >> 172.21.15.115:0/72578109 conn(0x7f29cc0fbfb0 msgr2=0x7f29cc0fe3f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:36:31.333 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:36:31.325+0000 7f29d3456700 1 -- 172.21.15.115:0/72578109 shutdown_connections 2024-03-19T01:36:31.333 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:36:31.325+0000 7f29d3456700 1 -- 172.21.15.115:0/72578109 wait complete. 2024-03-19T01:36:31.333 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:36:31.326+0000 7f29d3456700 1 Processor -- start 2024-03-19T01:36:31.333 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:36:31.326+0000 7f29d3456700 1 -- start start 2024-03-19T01:36:31.333 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:36:31.326+0000 7f29d3456700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f29cc06a180 0x7f29cc198580 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:36:31.333 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:36:31.326+0000 7f29d3456700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f29cc06a630 0x7f29cc198ac0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:36:31.333 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:36:31.326+0000 7f29d3456700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f29cc101c10 0x7f29cc19fb40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:36:31.333 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:36:31.326+0000 7f29d3456700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f29cc10ce70 con 0x7f29cc06a180 2024-03-19T01:36:31.333 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:36:31.326+0000 7f29d3456700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f29cc10ccf0 con 0x7f29cc101c10 2024-03-19T01:36:31.333 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:36:31.326+0000 7f29d3456700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f29cc10cff0 con 0x7f29cc06a630 2024-03-19T01:36:31.333 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:36:31.326+0000 7f29d09f1700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f29cc06a630 0x7f29cc198ac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:36:31.333 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:36:31.326+0000 7f29d09f1700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f29cc06a630 0x7f29cc198ac0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.115:54894/0 (socket says 172.21.15.115:54894) 2024-03-19T01:36:31.333 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:36:31.326+0000 7f29d09f1700 1 -- 172.21.15.115:0/2071662689 learned_addr learned my addr 172.21.15.115:0/2071662689 (peer_addr_for_me v2:172.21.15.115:0/0) 2024-03-19T01:36:31.334 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:36:31.326+0000 7f29d11f2700 1 --2- 172.21.15.115:0/2071662689 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f29cc06a180 0x7f29cc198580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:36:31.334 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:36:31.327+0000 7f29d09f1700 1 -- 172.21.15.115:0/2071662689 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f29cc101c10 msgr2=0x7f29cc19fb40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:36:31.334 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:36:31.327+0000 7f29d19f3700 1 --2- 172.21.15.115:0/2071662689 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f29cc101c10 0x7f29cc19fb40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:36:31.334 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:36:31.327+0000 7f29d09f1700 1 --2- 172.21.15.115:0/2071662689 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f29cc101c10 0x7f29cc19fb40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:36:31.334 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:36:31.327+0000 7f29d09f1700 1 -- 172.21.15.115:0/2071662689 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f29cc06a180 msgr2=0x7f29cc198580 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:36:31.334 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:36:31.327+0000 7f29d09f1700 1 --2- 172.21.15.115:0/2071662689 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f29cc06a180 0x7f29cc198580 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:36:31.334 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:36:31.327+0000 7f29d09f1700 1 -- 172.21.15.115:0/2071662689 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f29cc1a01e0 con 0x7f29cc06a630 2024-03-19T01:36:31.334 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:36:31.327+0000 7f29d09f1700 1 --2- 172.21.15.115:0/2071662689 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f29cc06a630 0x7f29cc198ac0 secure :-1 s=READY pgs=31 cs=0 l=1 rev1=1 crypto rx=0x7f29bc000c00 tx=0x7f29bc0047b0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:36:31.334 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:36:31.327+0000 7f29c27fc700 1 -- 172.21.15.115:0/2071662689 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f29bc003c70 con 0x7f29cc06a630 2024-03-19T01:36:31.334 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:36:31.327+0000 7f29d3456700 1 -- 172.21.15.115:0/2071662689 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f29cc1a0470 con 0x7f29cc06a630 2024-03-19T01:36:31.334 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:36:31.327+0000 7f29c27fc700 1 -- 172.21.15.115:0/2071662689 <== mon.1 v2:172.21.15.115:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f29bc004830 con 0x7f29cc06a630 2024-03-19T01:36:31.334 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:36:31.327+0000 7f29c27fc700 1 -- 172.21.15.115:0/2071662689 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f29bc0053a0 con 0x7f29cc06a630 2024-03-19T01:36:31.335 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:36:31.327+0000 7f29d3456700 1 -- 172.21.15.115:0/2071662689 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f29cc1a0860 con 0x7f29cc06a630 2024-03-19T01:36:31.335 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:36:31.328+0000 7f29d3456700 1 -- 172.21.15.115:0/2071662689 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f29cc068fd0 con 0x7f29cc06a630 2024-03-19T01:36:31.335 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:36:31.329+0000 7f29c27fc700 1 -- 172.21.15.115:0/2071662689 <== mon.1 v2:172.21.15.115:3300/0 4 ==== mgrmap(e 16) ==== 137290+0+0 (secure 0 0 0) 0x7f29bc005540 con 0x7f29cc06a630 2024-03-19T01:36:31.335 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:36:31.330+0000 7f29c27fc700 1 --2- 172.21.15.115:0/2071662689 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f29b80a2890 0x7f29b80a4d50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:36:31.335 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:36:31.330+0000 7f29c27fc700 1 -- 172.21.15.115:0/2071662689 <== mon.1 v2:172.21.15.115:3300/0 5 ==== osd_map(53..53 src has 1..53) ==== 5430+0+0 (secure 0 0 0) 0x7f29bc0cef00 con 0x7f29cc06a630 2024-03-19T01:36:31.335 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:36:31.330+0000 7f29d11f2700 1 --2- 172.21.15.115:0/2071662689 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f29b80a2890 0x7f29b80a4d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:36:31.335 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:36:31.330+0000 7f29d11f2700 1 --2- 172.21.15.115:0/2071662689 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f29b80a2890 0x7f29b80a4d50 secure :-1 s=READY pgs=98 cs=0 l=1 rev1=1 crypto rx=0x7f29cc101a70 tx=0x7f29c8007400 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:36:31.335 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:36:31.334+0000 7f29c27fc700 1 -- 172.21.15.115:0/2071662689 <== mon.1 v2:172.21.15.115:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f29bc013070 con 0x7f29cc06a630 2024-03-19T01:36:31.542 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:36:31.541+0000 7f29d3456700 1 -- 172.21.15.115:0/2071662689 --> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi115:vg_nvme/lv_2", "target": ["mon-mgr", ""]}) -- 0x7f29cc062750 con 0x7f29b80a2890 2024-03-19T01:36:31.542 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:31 smithi115 ceph-mon[67332]: Detected new or changed devices on smithi112 2024-03-19T01:36:31.542 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:31 smithi115 ceph-mon[67332]: Adjusting osd_memory_target on smithi112 to 5693M 2024-03-19T01:36:31.542 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:31 smithi115 ceph-mon[67332]: osd.7 [v2:172.21.15.112:6816/3610356576,v1:172.21.15.112:6817/3610356576] boot 2024-03-19T01:36:31.543 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:31 smithi115 ceph-mon[67332]: osdmap e53: 8 total, 8 up, 8 in 2024-03-19T01:36:31.543 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:31 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-19T01:36:31.543 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:31 smithi115 ceph-mon[67332]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 210 MiB used, 626 GiB / 626 GiB avail 2024-03-19T01:36:31.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:31 smithi112 ceph-mon[66819]: Detected new or changed devices on smithi112 2024-03-19T01:36:31.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:31 smithi112 ceph-mon[66819]: Adjusting osd_memory_target on smithi112 to 5693M 2024-03-19T01:36:31.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:31 smithi112 ceph-mon[66819]: osd.7 [v2:172.21.15.112:6816/3610356576,v1:172.21.15.112:6817/3610356576] boot 2024-03-19T01:36:31.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:31 smithi112 ceph-mon[66819]: osdmap e53: 8 total, 8 up, 8 in 2024-03-19T01:36:31.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:31 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-19T01:36:31.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:31 smithi112 ceph-mon[66819]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 210 MiB used, 626 GiB / 626 GiB avail 2024-03-19T01:36:31.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:31 smithi099 ceph-mon[62951]: Detected new or changed devices on smithi112 2024-03-19T01:36:31.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:31 smithi099 ceph-mon[62951]: Adjusting osd_memory_target on smithi112 to 5693M 2024-03-19T01:36:31.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:31 smithi099 ceph-mon[62951]: osd.7 [v2:172.21.15.112:6816/3610356576,v1:172.21.15.112:6817/3610356576] boot 2024-03-19T01:36:31.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:31 smithi099 ceph-mon[62951]: osdmap e53: 8 total, 8 up, 8 in 2024-03-19T01:36:31.842 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:31 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-19T01:36:31.842 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:31 smithi099 ceph-mon[62951]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 210 MiB used, 626 GiB / 626 GiB avail 2024-03-19T01:36:32.393 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:32 smithi115 ceph-mon[67332]: osdmap e54: 8 total, 8 up, 8 in 2024-03-19T01:36:32.393 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:32 smithi115 ceph-mon[67332]: from='client.24280 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi115:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:36:32.393 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:32 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-19T01:36:32.393 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:32 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-19T01:36:32.393 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:32 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:36:32.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:32 smithi112 ceph-mon[66819]: osdmap e54: 8 total, 8 up, 8 in 2024-03-19T01:36:32.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:32 smithi112 ceph-mon[66819]: from='client.24280 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi115:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:36:32.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:32 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-19T01:36:32.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:32 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-19T01:36:32.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:32 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:36:32.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:32 smithi099 ceph-mon[62951]: osdmap e54: 8 total, 8 up, 8 in 2024-03-19T01:36:32.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:32 smithi099 ceph-mon[62951]: from='client.24280 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi115:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:36:32.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:32 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-19T01:36:32.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:32 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-19T01:36:32.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:32 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:36:33.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:33 smithi112 ceph-mon[66819]: osdmap e55: 8 total, 8 up, 8 in 2024-03-19T01:36:33.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:33 smithi112 ceph-mon[66819]: pgmap v178: 1 pgs: 1 remapped+peering; 577 KiB data, 241 MiB used, 715 GiB / 715 GiB avail 2024-03-19T01:36:33.806 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:33 smithi115 ceph-mon[67332]: osdmap e55: 8 total, 8 up, 8 in 2024-03-19T01:36:33.806 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:33 smithi115 ceph-mon[67332]: pgmap v178: 1 pgs: 1 remapped+peering; 577 KiB data, 241 MiB used, 715 GiB / 715 GiB avail 2024-03-19T01:36:33.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:33 smithi099 ceph-mon[62951]: osdmap e55: 8 total, 8 up, 8 in 2024-03-19T01:36:33.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:33 smithi099 ceph-mon[62951]: pgmap v178: 1 pgs: 1 remapped+peering; 577 KiB data, 241 MiB used, 715 GiB / 715 GiB avail 2024-03-19T01:36:34.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:34 smithi112 ceph-mon[66819]: osdmap e56: 8 total, 8 up, 8 in 2024-03-19T01:36:34.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:34 smithi112 ceph-mon[66819]: from='client.? 172.21.15.115:0/1286425684' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "14a866dc-5ce3-42fc-8bcf-9088b68726cb"}]: dispatch 2024-03-19T01:36:34.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:34 smithi112 ceph-mon[66819]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "14a866dc-5ce3-42fc-8bcf-9088b68726cb"}]: dispatch 2024-03-19T01:36:34.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:34 smithi112 ceph-mon[66819]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "14a866dc-5ce3-42fc-8bcf-9088b68726cb"}]': finished 2024-03-19T01:36:34.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:34 smithi112 ceph-mon[66819]: osdmap e57: 9 total, 8 up, 9 in 2024-03-19T01:36:34.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:34 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-03-19T01:36:34.806 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:34 smithi115 ceph-mon[67332]: osdmap e56: 8 total, 8 up, 8 in 2024-03-19T01:36:34.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:34 smithi115 ceph-mon[67332]: from='client.? 172.21.15.115:0/1286425684' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "14a866dc-5ce3-42fc-8bcf-9088b68726cb"}]: dispatch 2024-03-19T01:36:34.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:34 smithi115 ceph-mon[67332]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "14a866dc-5ce3-42fc-8bcf-9088b68726cb"}]: dispatch 2024-03-19T01:36:34.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:34 smithi115 ceph-mon[67332]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "14a866dc-5ce3-42fc-8bcf-9088b68726cb"}]': finished 2024-03-19T01:36:34.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:34 smithi115 ceph-mon[67332]: osdmap e57: 9 total, 8 up, 9 in 2024-03-19T01:36:34.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:34 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-03-19T01:36:34.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:34 smithi099 ceph-mon[62951]: osdmap e56: 8 total, 8 up, 8 in 2024-03-19T01:36:34.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:34 smithi099 ceph-mon[62951]: from='client.? 172.21.15.115:0/1286425684' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "14a866dc-5ce3-42fc-8bcf-9088b68726cb"}]: dispatch 2024-03-19T01:36:34.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:34 smithi099 ceph-mon[62951]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "14a866dc-5ce3-42fc-8bcf-9088b68726cb"}]: dispatch 2024-03-19T01:36:34.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:34 smithi099 ceph-mon[62951]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "14a866dc-5ce3-42fc-8bcf-9088b68726cb"}]': finished 2024-03-19T01:36:34.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:34 smithi099 ceph-mon[62951]: osdmap e57: 9 total, 8 up, 9 in 2024-03-19T01:36:34.842 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:34 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-03-19T01:36:35.806 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:35 smithi115 ceph-mon[67332]: pgmap v181: 1 pgs: 1 remapped+peering; 577 KiB data, 241 MiB used, 715 GiB / 715 GiB avail 2024-03-19T01:36:35.806 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:35 smithi115 ceph-mon[67332]: from='client.? 172.21.15.115:0/1330798296' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-19T01:36:35.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:35 smithi099 ceph-mon[62951]: pgmap v181: 1 pgs: 1 remapped+peering; 577 KiB data, 241 MiB used, 715 GiB / 715 GiB avail 2024-03-19T01:36:35.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:35 smithi099 ceph-mon[62951]: from='client.? 172.21.15.115:0/1330798296' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-19T01:36:35.861 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:35 smithi112 ceph-mon[66819]: pgmap v181: 1 pgs: 1 remapped+peering; 577 KiB data, 241 MiB used, 715 GiB / 715 GiB avail 2024-03-19T01:36:35.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:35 smithi112 ceph-mon[66819]: from='client.? 172.21.15.115:0/1330798296' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-19T01:36:38.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:37 smithi115 ceph-mon[67332]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 241 MiB used, 715 GiB / 715 GiB avail 2024-03-19T01:36:38.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:37 smithi099 ceph-mon[62951]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 241 MiB used, 715 GiB / 715 GiB avail 2024-03-19T01:36:38.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:37 smithi112 ceph-mon[66819]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 241 MiB used, 715 GiB / 715 GiB avail 2024-03-19T01:36:40.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:39 smithi115 ceph-mon[67332]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 242 MiB used, 715 GiB / 715 GiB avail 2024-03-19T01:36:40.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:39 smithi099 ceph-mon[62951]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 242 MiB used, 715 GiB / 715 GiB avail 2024-03-19T01:36:40.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:39 smithi112 ceph-mon[66819]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 242 MiB used, 715 GiB / 715 GiB avail 2024-03-19T01:36:42.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:41 smithi115 ceph-mon[67332]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 242 MiB used, 715 GiB / 715 GiB avail 2024-03-19T01:36:42.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:41 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.8"}]: dispatch 2024-03-19T01:36:42.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:41 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:36:42.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:41 smithi099 ceph-mon[62951]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 242 MiB used, 715 GiB / 715 GiB avail 2024-03-19T01:36:42.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:41 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.8"}]: dispatch 2024-03-19T01:36:42.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:41 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:36:42.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:41 smithi112 ceph-mon[66819]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 242 MiB used, 715 GiB / 715 GiB avail 2024-03-19T01:36:42.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:41 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.8"}]: dispatch 2024-03-19T01:36:42.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:41 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:36:43.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:42 smithi115 ceph-mon[67332]: Deploying daemon osd.8 on smithi115 2024-03-19T01:36:43.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:42 smithi099 ceph-mon[62951]: Deploying daemon osd.8 on smithi115 2024-03-19T01:36:43.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:42 smithi112 ceph-mon[66819]: Deploying daemon osd.8 on smithi115 2024-03-19T01:36:44.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:43 smithi115 ceph-mon[67332]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 242 MiB used, 715 GiB / 715 GiB avail 2024-03-19T01:36:44.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:43 smithi099 ceph-mon[62951]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 242 MiB used, 715 GiB / 715 GiB avail 2024-03-19T01:36:44.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:43 smithi112 ceph-mon[66819]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 242 MiB used, 715 GiB / 715 GiB avail 2024-03-19T01:36:45.892 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:45 smithi115 ceph-mon[67332]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 242 MiB used, 715 GiB / 715 GiB avail 2024-03-19T01:36:46.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:45 smithi099 ceph-mon[62951]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 242 MiB used, 715 GiB / 715 GiB avail 2024-03-19T01:36:46.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:45 smithi112 ceph-mon[66819]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 242 MiB used, 715 GiB / 715 GiB avail 2024-03-19T01:36:48.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:47 smithi115 ceph-mon[67332]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 242 MiB used, 715 GiB / 715 GiB avail 2024-03-19T01:36:48.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:47 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:48.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:47 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:48.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:47 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:36:48.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:47 smithi099 ceph-mon[62951]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 242 MiB used, 715 GiB / 715 GiB avail 2024-03-19T01:36:48.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:47 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:48.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:47 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:48.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:47 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:36:48.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:47 smithi112 ceph-mon[66819]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 242 MiB used, 715 GiB / 715 GiB avail 2024-03-19T01:36:48.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:47 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:48.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:47 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:48.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:47 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:36:50.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:49 smithi115 ceph-mon[67332]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 242 MiB used, 715 GiB / 715 GiB avail 2024-03-19T01:36:50.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:49 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:50.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:49 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:50.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:49 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:36:50.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:49 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:36:50.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:49 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:50.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:49 smithi099 ceph-mon[62951]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 242 MiB used, 715 GiB / 715 GiB avail 2024-03-19T01:36:50.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:49 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:50.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:49 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:50.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:49 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:36:50.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:49 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:36:50.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:49 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:50.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:49 smithi112 ceph-mon[66819]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 242 MiB used, 715 GiB / 715 GiB avail 2024-03-19T01:36:50.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:49 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:50.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:49 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:50.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:49 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:36:50.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:49 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:36:50.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:49 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:50.456 INFO:teuthology.orchestra.run.smithi115.stdout:Created osd(s) 8 on host 'smithi115' 2024-03-19T01:36:50.456 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:36:50.455+0000 7f29c27fc700 1 -- 172.21.15.115:0/2071662689 <== mgr.14150 v2:172.21.15.99:6800/4272410295 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+37 (secure 0 0 0) 0x7f29cc062750 con 0x7f29b80a2890 2024-03-19T01:36:50.458 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:36:50.457+0000 7f29d3456700 1 -- 172.21.15.115:0/2071662689 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f29b80a2890 msgr2=0x7f29b80a4d50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:36:50.459 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:36:50.457+0000 7f29d3456700 1 --2- 172.21.15.115:0/2071662689 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f29b80a2890 0x7f29b80a4d50 secure :-1 s=READY pgs=98 cs=0 l=1 rev1=1 crypto rx=0x7f29cc101a70 tx=0x7f29c8007400 comp rx=0 tx=0).stop 2024-03-19T01:36:50.459 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:36:50.457+0000 7f29d3456700 1 -- 172.21.15.115:0/2071662689 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f29cc06a630 msgr2=0x7f29cc198ac0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:36:50.459 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:36:50.457+0000 7f29d3456700 1 --2- 172.21.15.115:0/2071662689 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f29cc06a630 0x7f29cc198ac0 secure :-1 s=READY pgs=31 cs=0 l=1 rev1=1 crypto rx=0x7f29bc000c00 tx=0x7f29bc0047b0 comp rx=0 tx=0).stop 2024-03-19T01:36:50.459 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:36:50.457+0000 7f29d3456700 1 -- 172.21.15.115:0/2071662689 shutdown_connections 2024-03-19T01:36:50.459 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:36:50.457+0000 7f29d3456700 1 --2- 172.21.15.115:0/2071662689 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f29b80a2890 0x7f29b80a4d50 unknown :-1 s=CLOSED pgs=98 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:36:50.459 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:36:50.457+0000 7f29d3456700 1 --2- 172.21.15.115:0/2071662689 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f29cc101c10 0x7f29cc19fb40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:36:50.459 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:36:50.457+0000 7f29d3456700 1 --2- 172.21.15.115:0/2071662689 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f29cc06a180 0x7f29cc198580 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:36:50.459 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:36:50.457+0000 7f29d3456700 1 --2- 172.21.15.115:0/2071662689 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f29cc06a630 0x7f29cc198ac0 unknown :-1 s=CLOSED pgs=31 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:36:50.459 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:36:50.457+0000 7f29d3456700 1 -- 172.21.15.115:0/2071662689 >> 172.21.15.115:0/2071662689 conn(0x7f29cc0fbfb0 msgr2=0x7f29cc0fdab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:36:50.459 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:36:50.458+0000 7f29d3456700 1 -- 172.21.15.115:0/2071662689 shutdown_connections 2024-03-19T01:36:50.459 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:36:50.458+0000 7f29d3456700 1 -- 172.21.15.115:0/2071662689 wait complete. 2024-03-19T01:36:51.104 DEBUG:teuthology.orchestra.run.smithi115:osd.8> sudo journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@osd.8.service 2024-03-19T01:36:51.106 INFO:tasks.cephadm:Deploying osd.9 on smithi099 with /dev/vg_nvme/lv_1... 2024-03-19T01:36:51.106 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-03-19T01:36:51.357 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:36:51.716 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:51 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:51.716 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:51 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:51.717 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:51 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:36:51.717 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:51 smithi099 ceph-mon[62951]: from='osd.8 [v2:172.21.15.115:6816/4255803111,v1:172.21.15.115:6817/4255803111]' entity='osd.8' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["8"]}]: dispatch 2024-03-19T01:36:51.717 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:51 smithi099 ceph-mon[62951]: from='osd.8 ' entity='osd.8' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["8"]}]: dispatch 2024-03-19T01:36:51.717 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:51 smithi099 ceph-mon[62951]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 242 MiB used, 715 GiB / 715 GiB avail 2024-03-19T01:36:51.766 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:51 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:51.766 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:51 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:51.766 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:51 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:36:51.767 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:51 smithi115 ceph-mon[67332]: from='osd.8 [v2:172.21.15.115:6816/4255803111,v1:172.21.15.115:6817/4255803111]' entity='osd.8' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["8"]}]: dispatch 2024-03-19T01:36:51.767 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:51 smithi115 ceph-mon[67332]: from='osd.8 ' entity='osd.8' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["8"]}]: dispatch 2024-03-19T01:36:51.767 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:51 smithi115 ceph-mon[67332]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 242 MiB used, 715 GiB / 715 GiB avail 2024-03-19T01:36:51.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:51 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:51.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:51 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:51.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:51 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:36:51.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:51 smithi112 ceph-mon[66819]: from='osd.8 [v2:172.21.15.115:6816/4255803111,v1:172.21.15.115:6817/4255803111]' entity='osd.8' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["8"]}]: dispatch 2024-03-19T01:36:51.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:51 smithi112 ceph-mon[66819]: from='osd.8 ' entity='osd.8' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["8"]}]: dispatch 2024-03-19T01:36:51.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:51 smithi112 ceph-mon[66819]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 242 MiB used, 715 GiB / 715 GiB avail 2024-03-19T01:36:52.788 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:52 smithi099 ceph-mon[62951]: from='osd.8 ' entity='osd.8' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["8"]}]': finished 2024-03-19T01:36:52.788 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:52 smithi099 ceph-mon[62951]: osdmap e58: 9 total, 8 up, 9 in 2024-03-19T01:36:52.788 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:52 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-03-19T01:36:52.789 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:52 smithi099 ceph-mon[62951]: from='osd.8 [v2:172.21.15.115:6816/4255803111,v1:172.21.15.115:6817/4255803111]' entity='osd.8' cmd=[{"prefix": "osd crush create-or-move", "id": 8, "weight":0.0873, "args": ["host=smithi115", "root=default"]}]: dispatch 2024-03-19T01:36:52.789 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:52 smithi099 ceph-mon[62951]: from='osd.8 ' entity='osd.8' cmd=[{"prefix": "osd crush create-or-move", "id": 8, "weight":0.0873, "args": ["host=smithi115", "root=default"]}]: dispatch 2024-03-19T01:36:52.806 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:52 smithi115 ceph-mon[67332]: from='osd.8 ' entity='osd.8' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["8"]}]': finished 2024-03-19T01:36:52.806 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:52 smithi115 ceph-mon[67332]: osdmap e58: 9 total, 8 up, 9 in 2024-03-19T01:36:52.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:52 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-03-19T01:36:52.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:52 smithi115 ceph-mon[67332]: from='osd.8 [v2:172.21.15.115:6816/4255803111,v1:172.21.15.115:6817/4255803111]' entity='osd.8' cmd=[{"prefix": "osd crush create-or-move", "id": 8, "weight":0.0873, "args": ["host=smithi115", "root=default"]}]: dispatch 2024-03-19T01:36:52.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:52 smithi115 ceph-mon[67332]: from='osd.8 ' entity='osd.8' cmd=[{"prefix": "osd crush create-or-move", "id": 8, "weight":0.0873, "args": ["host=smithi115", "root=default"]}]: dispatch 2024-03-19T01:36:52.807 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 01:36:52 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T01:36:52.453+0000 7f3e971f8700 -1 osd.8 0 waiting for initial osdmap 2024-03-19T01:36:52.807 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 01:36:52 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T01:36:52.470+0000 7f3e92feb700 -1 osd.8 59 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-19T01:36:52.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:52 smithi112 ceph-mon[66819]: from='osd.8 ' entity='osd.8' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["8"]}]': finished 2024-03-19T01:36:52.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:52 smithi112 ceph-mon[66819]: osdmap e58: 9 total, 8 up, 9 in 2024-03-19T01:36:52.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:52 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-03-19T01:36:52.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:52 smithi112 ceph-mon[66819]: from='osd.8 [v2:172.21.15.115:6816/4255803111,v1:172.21.15.115:6817/4255803111]' entity='osd.8' cmd=[{"prefix": "osd crush create-or-move", "id": 8, "weight":0.0873, "args": ["host=smithi115", "root=default"]}]: dispatch 2024-03-19T01:36:52.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:52 smithi112 ceph-mon[66819]: from='osd.8 ' entity='osd.8' cmd=[{"prefix": "osd crush create-or-move", "id": 8, "weight":0.0873, "args": ["host=smithi115", "root=default"]}]: dispatch 2024-03-19T01:36:53.486 INFO:teuthology.orchestra.run.smithi099.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-03-19T01:36:53.486 INFO:teuthology.orchestra.run.smithi099.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-03-19T01:36:53.486 INFO:teuthology.orchestra.run.smithi099.stderr: stderr: 10+0 records in 2024-03-19T01:36:53.486 INFO:teuthology.orchestra.run.smithi099.stderr:10+0 records out 2024-03-19T01:36:53.486 INFO:teuthology.orchestra.run.smithi099.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0133168 s, 787 MB/s 2024-03-19T01:36:53.486 INFO:teuthology.orchestra.run.smithi099.stderr:--> Zapping successful for: 2024-03-19T01:36:53.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:53 smithi115 ceph-mon[67332]: from='osd.8 ' entity='osd.8' cmd='[{"prefix": "osd crush create-or-move", "id": 8, "weight":0.0873, "args": ["host=smithi115", "root=default"]}]': finished 2024-03-19T01:36:53.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:53 smithi115 ceph-mon[67332]: osdmap e59: 9 total, 8 up, 9 in 2024-03-19T01:36:53.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:53 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-03-19T01:36:53.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:53 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-03-19T01:36:53.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:53 smithi115 ceph-mon[67332]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 242 MiB used, 715 GiB / 715 GiB avail 2024-03-19T01:36:53.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:53 smithi099 ceph-mon[62951]: from='osd.8 ' entity='osd.8' cmd='[{"prefix": "osd crush create-or-move", "id": 8, "weight":0.0873, "args": ["host=smithi115", "root=default"]}]': finished 2024-03-19T01:36:53.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:53 smithi099 ceph-mon[62951]: osdmap e59: 9 total, 8 up, 9 in 2024-03-19T01:36:53.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:53 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-03-19T01:36:53.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:53 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-03-19T01:36:53.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:53 smithi099 ceph-mon[62951]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 242 MiB used, 715 GiB / 715 GiB avail 2024-03-19T01:36:53.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:53 smithi112 ceph-mon[66819]: from='osd.8 ' entity='osd.8' cmd='[{"prefix": "osd crush create-or-move", "id": 8, "weight":0.0873, "args": ["host=smithi115", "root=default"]}]': finished 2024-03-19T01:36:53.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:53 smithi112 ceph-mon[66819]: osdmap e59: 9 total, 8 up, 9 in 2024-03-19T01:36:53.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:53 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-03-19T01:36:53.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:53 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-03-19T01:36:53.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:53 smithi112 ceph-mon[66819]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 242 MiB used, 715 GiB / 715 GiB avail 2024-03-19T01:36:54.223 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph orch daemon add osd smithi099:vg_nvme/lv_1 2024-03-19T01:36:54.466 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:36:54.728 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:54 smithi099 ceph-mon[62951]: purged_snaps scrub starts 2024-03-19T01:36:54.728 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:54 smithi099 ceph-mon[62951]: purged_snaps scrub ok 2024-03-19T01:36:54.728 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:54 smithi099 ceph-mon[62951]: osd.8 [v2:172.21.15.115:6816/4255803111,v1:172.21.15.115:6817/4255803111] boot 2024-03-19T01:36:54.728 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:54 smithi099 ceph-mon[62951]: osdmap e60: 9 total, 9 up, 9 in 2024-03-19T01:36:54.728 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:54 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-03-19T01:36:54.728 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:54 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:54.728 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:54 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:54.728 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:54 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:36:54.728 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:54 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:36:54.729 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:54 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:36:54.729 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:54 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:54.729 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:54 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:36:54.729 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:54 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:36:54.729 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:54 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:54.729 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:54 smithi099 ceph-mon[62951]: osdmap e61: 9 total, 9 up, 9 in 2024-03-19T01:36:54.806 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:54 smithi115 ceph-mon[67332]: purged_snaps scrub starts 2024-03-19T01:36:54.806 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:54 smithi115 ceph-mon[67332]: purged_snaps scrub ok 2024-03-19T01:36:54.806 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:54 smithi115 ceph-mon[67332]: osd.8 [v2:172.21.15.115:6816/4255803111,v1:172.21.15.115:6817/4255803111] boot 2024-03-19T01:36:54.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:54 smithi115 ceph-mon[67332]: osdmap e60: 9 total, 9 up, 9 in 2024-03-19T01:36:54.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:54 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-03-19T01:36:54.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:54 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:54.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:54 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:54.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:54 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:36:54.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:54 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:36:54.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:54 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:36:54.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:54 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:54.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:54 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:36:54.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:54 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:36:54.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:54 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:54.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:54 smithi115 ceph-mon[67332]: osdmap e61: 9 total, 9 up, 9 in 2024-03-19T01:36:54.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:54 smithi112 ceph-mon[66819]: purged_snaps scrub starts 2024-03-19T01:36:54.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:54 smithi112 ceph-mon[66819]: purged_snaps scrub ok 2024-03-19T01:36:54.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:54 smithi112 ceph-mon[66819]: osd.8 [v2:172.21.15.115:6816/4255803111,v1:172.21.15.115:6817/4255803111] boot 2024-03-19T01:36:54.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:54 smithi112 ceph-mon[66819]: osdmap e60: 9 total, 9 up, 9 in 2024-03-19T01:36:54.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:54 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-03-19T01:36:54.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:54 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:54.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:54 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:54.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:54 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:36:54.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:54 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:36:54.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:54 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:36:54.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:54 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:54.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:54 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:36:54.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:54 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:36:54.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:54 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:36:54.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:54 smithi112 ceph-mon[66819]: osdmap e61: 9 total, 9 up, 9 in 2024-03-19T01:36:55.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:55 smithi099 ceph-mon[62951]: Detected new or changed devices on smithi115 2024-03-19T01:36:55.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:55 smithi099 ceph-mon[62951]: Adjusting osd_memory_target on smithi115 to 5693M 2024-03-19T01:36:55.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:55 smithi099 ceph-mon[62951]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 268 MiB used, 804 GiB / 805 GiB avail 2024-03-19T01:36:55.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:55 smithi099 ceph-mon[62951]: osdmap e62: 9 total, 9 up, 9 in 2024-03-19T01:36:55.806 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:55 smithi115 ceph-mon[67332]: Detected new or changed devices on smithi115 2024-03-19T01:36:55.806 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:55 smithi115 ceph-mon[67332]: Adjusting osd_memory_target on smithi115 to 5693M 2024-03-19T01:36:55.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:55 smithi115 ceph-mon[67332]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 268 MiB used, 804 GiB / 805 GiB avail 2024-03-19T01:36:55.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:55 smithi115 ceph-mon[67332]: osdmap e62: 9 total, 9 up, 9 in 2024-03-19T01:36:55.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:55 smithi112 ceph-mon[66819]: Detected new or changed devices on smithi115 2024-03-19T01:36:55.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:55 smithi112 ceph-mon[66819]: Adjusting osd_memory_target on smithi115 to 5693M 2024-03-19T01:36:55.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:55 smithi112 ceph-mon[66819]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 268 MiB used, 804 GiB / 805 GiB avail 2024-03-19T01:36:55.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:55 smithi112 ceph-mon[66819]: osdmap e62: 9 total, 9 up, 9 in 2024-03-19T01:36:56.112 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:36:56.109+0000 7f044a2fd700 1 -- 172.21.15.99:0/2461594660 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0444102440 msgr2=0x7f04441028a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:36:56.112 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:36:56.109+0000 7f044a2fd700 1 --2- 172.21.15.99:0/2461594660 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0444102440 0x7f04441028a0 secure :-1 s=READY pgs=133 cs=0 l=1 rev1=1 crypto rx=0x7f04380099c0 tx=0x7f0438009cd0 comp rx=0 tx=0).stop 2024-03-19T01:36:56.112 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:36:56.109+0000 7f044a2fd700 1 -- 172.21.15.99:0/2461594660 shutdown_connections 2024-03-19T01:36:56.113 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:36:56.109+0000 7f044a2fd700 1 --2- 172.21.15.99:0/2461594660 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f0444101240 0x7f0444101660 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:36:56.113 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:36:56.109+0000 7f044a2fd700 1 --2- 172.21.15.99:0/2461594660 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f0444102de0 0x7f0444109690 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:36:56.113 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:36:56.109+0000 7f044a2fd700 1 --2- 172.21.15.99:0/2461594660 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0444102440 0x7f04441028a0 unknown :-1 s=CLOSED pgs=133 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:36:56.113 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:36:56.109+0000 7f044a2fd700 1 -- 172.21.15.99:0/2461594660 >> 172.21.15.99:0/2461594660 conn(0x7f04440fc7e0 msgr2=0x7f04440fec20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:36:56.113 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:36:56.109+0000 7f044a2fd700 1 -- 172.21.15.99:0/2461594660 shutdown_connections 2024-03-19T01:36:56.113 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:36:56.109+0000 7f044a2fd700 1 -- 172.21.15.99:0/2461594660 wait complete. 2024-03-19T01:36:56.113 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:36:56.110+0000 7f044a2fd700 1 Processor -- start 2024-03-19T01:36:56.113 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:36:56.110+0000 7f044a2fd700 1 -- start start 2024-03-19T01:36:56.113 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:36:56.110+0000 7f044a2fd700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f0444101240 0x7f04441974a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:36:56.113 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:36:56.110+0000 7f044a2fd700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0444102440 0x7f04441979e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:36:56.114 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:36:56.110+0000 7f044a2fd700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f0444102de0 0x7f044419ea60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:36:56.114 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:36:56.110+0000 7f044a2fd700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f044410ce30 con 0x7f0444102440 2024-03-19T01:36:56.114 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:36:56.110+0000 7f044a2fd700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f044410ccb0 con 0x7f0444101240 2024-03-19T01:36:56.114 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:36:56.110+0000 7f044a2fd700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f044410cfb0 con 0x7f0444102de0 2024-03-19T01:36:56.114 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:36:56.110+0000 7f04437fe700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0444102440 0x7f04441979e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:36:56.114 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:36:56.110+0000 7f04437fe700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0444102440 0x7f04441979e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:44774/0 (socket says 172.21.15.99:44774) 2024-03-19T01:36:56.114 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:36:56.110+0000 7f04437fe700 1 -- 172.21.15.99:0/3970253689 learned_addr learned my addr 172.21.15.99:0/3970253689 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:36:56.114 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:36:56.110+0000 7f0443fff700 1 --2- 172.21.15.99:0/3970253689 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f0444101240 0x7f04441974a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:36:56.114 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:36:56.110+0000 7f044889a700 1 --2- 172.21.15.99:0/3970253689 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f0444102de0 0x7f044419ea60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:36:56.114 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:36:56.110+0000 7f04437fe700 1 -- 172.21.15.99:0/3970253689 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f0444102de0 msgr2=0x7f044419ea60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:36:56.114 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:36:56.111+0000 7f04437fe700 1 --2- 172.21.15.99:0/3970253689 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f0444102de0 0x7f044419ea60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:36:56.114 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:36:56.111+0000 7f04437fe700 1 -- 172.21.15.99:0/3970253689 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f0444101240 msgr2=0x7f04441974a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:36:56.115 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:36:56.111+0000 7f04437fe700 1 --2- 172.21.15.99:0/3970253689 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f0444101240 0x7f04441974a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:36:56.115 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:36:56.111+0000 7f04437fe700 1 -- 172.21.15.99:0/3970253689 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f044419f100 con 0x7f0444102440 2024-03-19T01:36:56.115 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:36:56.111+0000 7f044889a700 1 --2- 172.21.15.99:0/3970253689 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f0444102de0 0x7f044419ea60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-19T01:36:56.115 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:36:56.111+0000 7f04437fe700 1 --2- 172.21.15.99:0/3970253689 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0444102440 0x7f04441979e0 secure :-1 s=READY pgs=134 cs=0 l=1 rev1=1 crypto rx=0x7f0438000c00 tx=0x7f0438009f10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:36:56.115 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:36:56.111+0000 7f04417fa700 1 -- 172.21.15.99:0/3970253689 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0438026070 con 0x7f0444102440 2024-03-19T01:36:56.115 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:36:56.111+0000 7f04417fa700 1 -- 172.21.15.99:0/3970253689 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f0438004150 con 0x7f0444102440 2024-03-19T01:36:56.115 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:36:56.111+0000 7f044a2fd700 1 -- 172.21.15.99:0/3970253689 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f044419f390 con 0x7f0444102440 2024-03-19T01:36:56.115 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:36:56.111+0000 7f04417fa700 1 -- 172.21.15.99:0/3970253689 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0438004a00 con 0x7f0444102440 2024-03-19T01:36:56.115 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:36:56.111+0000 7f044a2fd700 1 -- 172.21.15.99:0/3970253689 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f044419f7e0 con 0x7f0444102440 2024-03-19T01:36:56.115 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:36:56.112+0000 7f044a2fd700 1 -- 172.21.15.99:0/3970253689 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f0444068fd0 con 0x7f0444102440 2024-03-19T01:36:56.116 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:36:56.113+0000 7f04417fa700 1 -- 172.21.15.99:0/3970253689 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 16) ==== 137290+0+0 (secure 0 0 0) 0x7f0438029070 con 0x7f0444102440 2024-03-19T01:36:56.116 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:36:56.114+0000 7f04417fa700 1 --2- 172.21.15.99:0/3970253689 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f042c0a2890 0x7f042c0a4d50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:36:56.116 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:36:56.114+0000 7f0443fff700 1 --2- 172.21.15.99:0/3970253689 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f042c0a2890 0x7f042c0a4d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:36:56.117 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:36:56.114+0000 7f04417fa700 1 -- 172.21.15.99:0/3970253689 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(62..62 src has 1..62) ==== 5858+0+0 (secure 0 0 0) 0x7f04380cef90 con 0x7f0444102440 2024-03-19T01:36:56.117 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:36:56.114+0000 7f0443fff700 1 --2- 172.21.15.99:0/3970253689 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f042c0a2890 0x7f042c0a4d50 secure :-1 s=READY pgs=103 cs=0 l=1 rev1=1 crypto rx=0x7f04441022a0 tx=0x7f043400b410 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:36:56.118 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:36:56.118+0000 7f04417fa700 1 -- 172.21.15.99:0/3970253689 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f0438030020 con 0x7f0444102440 2024-03-19T01:36:56.321 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:36:56.321+0000 7f044a2fd700 1 -- 172.21.15.99:0/3970253689 --> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_1", "target": ["mon-mgr", ""]}) -- 0x7f0444062750 con 0x7f042c0a2890 2024-03-19T01:36:56.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:56 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-19T01:36:56.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:56 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-19T01:36:56.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:56 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:36:56.806 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:56 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-19T01:36:56.806 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:56 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-19T01:36:56.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:56 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:36:56.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:56 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-19T01:36:56.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:56 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-19T01:36:56.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:56 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:36:57.806 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:57 smithi115 ceph-mon[67332]: from='client.14442 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:36:57.806 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:57 smithi115 ceph-mon[67332]: osdmap e63: 9 total, 9 up, 9 in 2024-03-19T01:36:57.806 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:57 smithi115 ceph-mon[67332]: pgmap v198: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-03-19T01:36:57.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:57 smithi099 ceph-mon[62951]: from='client.14442 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:36:57.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:57 smithi099 ceph-mon[62951]: osdmap e63: 9 total, 9 up, 9 in 2024-03-19T01:36:57.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:57 smithi099 ceph-mon[62951]: pgmap v198: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-03-19T01:36:57.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:57 smithi112 ceph-mon[66819]: from='client.14442 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:36:57.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:57 smithi112 ceph-mon[66819]: osdmap e63: 9 total, 9 up, 9 in 2024-03-19T01:36:57.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:57 smithi112 ceph-mon[66819]: pgmap v198: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-03-19T01:36:58.806 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:58 smithi115 ceph-mon[67332]: osdmap e64: 9 total, 9 up, 9 in 2024-03-19T01:36:58.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:58 smithi099 ceph-mon[62951]: osdmap e64: 9 total, 9 up, 9 in 2024-03-19T01:36:58.861 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:58 smithi112 ceph-mon[66819]: osdmap e64: 9 total, 9 up, 9 in 2024-03-19T01:36:59.806 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:59 smithi115 ceph-mon[67332]: pgmap v200: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 270 MiB used, 804 GiB / 805 GiB avail; 135 KiB/s, 0 objects/s recovering 2024-03-19T01:36:59.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:59 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/606402641' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1db93652-6035-4906-91ae-d64d5a909502"}]: dispatch 2024-03-19T01:36:59.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:59 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/606402641' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1db93652-6035-4906-91ae-d64d5a909502"}]': finished 2024-03-19T01:36:59.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:59 smithi115 ceph-mon[67332]: osdmap e65: 10 total, 9 up, 10 in 2024-03-19T01:36:59.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:36:59 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-03-19T01:36:59.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:59 smithi099 ceph-mon[62951]: pgmap v200: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 270 MiB used, 804 GiB / 805 GiB avail; 135 KiB/s, 0 objects/s recovering 2024-03-19T01:36:59.842 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:59 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/606402641' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1db93652-6035-4906-91ae-d64d5a909502"}]: dispatch 2024-03-19T01:36:59.842 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:59 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/606402641' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1db93652-6035-4906-91ae-d64d5a909502"}]': finished 2024-03-19T01:36:59.842 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:59 smithi099 ceph-mon[62951]: osdmap e65: 10 total, 9 up, 10 in 2024-03-19T01:36:59.842 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:36:59 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-03-19T01:36:59.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:59 smithi112 ceph-mon[66819]: pgmap v200: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 270 MiB used, 804 GiB / 805 GiB avail; 135 KiB/s, 0 objects/s recovering 2024-03-19T01:36:59.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:59 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/606402641' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1db93652-6035-4906-91ae-d64d5a909502"}]: dispatch 2024-03-19T01:36:59.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:59 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/606402641' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1db93652-6035-4906-91ae-d64d5a909502"}]': finished 2024-03-19T01:36:59.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:59 smithi112 ceph-mon[66819]: osdmap e65: 10 total, 9 up, 10 in 2024-03-19T01:36:59.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:36:59 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-03-19T01:37:00.806 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:00 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/1548642301' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-19T01:37:00.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:00 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/1548642301' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-19T01:37:00.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:00 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/1548642301' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-19T01:37:01.806 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:01 smithi115 ceph-mon[67332]: pgmap v202: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail; 110 KiB/s, 0 objects/s recovering 2024-03-19T01:37:01.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:01 smithi099 ceph-mon[62951]: pgmap v202: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail; 110 KiB/s, 0 objects/s recovering 2024-03-19T01:37:01.861 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:01 smithi112 ceph-mon[66819]: pgmap v202: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail; 110 KiB/s, 0 objects/s recovering 2024-03-19T01:37:04.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:03 smithi115 ceph-mon[67332]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail 2024-03-19T01:37:04.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:03 smithi099 ceph-mon[62951]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail 2024-03-19T01:37:04.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:03 smithi112 ceph-mon[66819]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail 2024-03-19T01:37:06.039 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:05 smithi099 ceph-mon[62951]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail 2024-03-19T01:37:06.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:05 smithi115 ceph-mon[67332]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail 2024-03-19T01:37:06.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:05 smithi112 ceph-mon[66819]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail 2024-03-19T01:37:06.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:06 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.9"}]: dispatch 2024-03-19T01:37:06.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:06 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:37:07.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:06 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.9"}]: dispatch 2024-03-19T01:37:07.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:06 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:37:07.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:06 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.9"}]: dispatch 2024-03-19T01:37:07.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:06 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:37:07.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:07 smithi099 ceph-mon[62951]: Deploying daemon osd.9 on smithi099 2024-03-19T01:37:07.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:07 smithi099 ceph-mon[62951]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail 2024-03-19T01:37:08.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:07 smithi115 ceph-mon[67332]: Deploying daemon osd.9 on smithi099 2024-03-19T01:37:08.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:07 smithi115 ceph-mon[67332]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail 2024-03-19T01:37:08.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:07 smithi112 ceph-mon[66819]: Deploying daemon osd.9 on smithi099 2024-03-19T01:37:08.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:07 smithi112 ceph-mon[66819]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail 2024-03-19T01:37:09.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:09 smithi099 ceph-mon[62951]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail 2024-03-19T01:37:10.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:09 smithi115 ceph-mon[67332]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail 2024-03-19T01:37:10.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:09 smithi112 ceph-mon[66819]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail 2024-03-19T01:37:12.039 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:11 smithi099 ceph-mon[62951]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail 2024-03-19T01:37:12.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:11 smithi115 ceph-mon[67332]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail 2024-03-19T01:37:12.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:11 smithi112 ceph-mon[66819]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail 2024-03-19T01:37:13.806 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:13 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:13.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:13 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:13.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:13 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:37:13.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:13 smithi115 ceph-mon[67332]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail 2024-03-19T01:37:13.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:13 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:13.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:13 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:13.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:13 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:37:13.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:13 smithi099 ceph-mon[62951]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail 2024-03-19T01:37:13.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:13 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:13.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:13 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:13.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:13 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:37:13.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:13 smithi112 ceph-mon[66819]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail 2024-03-19T01:37:15.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:15 smithi099 ceph-mon[62951]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail 2024-03-19T01:37:15.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:15 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:15.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:15 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:15.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:15 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:37:15.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:15 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:37:15.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:15 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:16.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:15 smithi115 ceph-mon[67332]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail 2024-03-19T01:37:16.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:15 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:16.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:15 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:16.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:15 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:37:16.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:15 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:37:16.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:15 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:16.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:15 smithi112 ceph-mon[66819]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail 2024-03-19T01:37:16.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:15 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:16.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:15 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:16.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:15 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:37:16.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:15 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:37:16.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:15 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:16.227 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:37:16.226+0000 7f04417fa700 1 -- 172.21.15.99:0/3970253689 <== mgr.14150 v2:172.21.15.99:6800/4272410295 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+37 (secure 0 0 0) 0x7f0444062750 con 0x7f042c0a2890 2024-03-19T01:37:16.227 INFO:teuthology.orchestra.run.smithi099.stdout:Created osd(s) 9 on host 'smithi099' 2024-03-19T01:37:16.229 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:37:16.228+0000 7f044a2fd700 1 -- 172.21.15.99:0/3970253689 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f042c0a2890 msgr2=0x7f042c0a4d50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:37:16.229 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:37:16.228+0000 7f044a2fd700 1 --2- 172.21.15.99:0/3970253689 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f042c0a2890 0x7f042c0a4d50 secure :-1 s=READY pgs=103 cs=0 l=1 rev1=1 crypto rx=0x7f04441022a0 tx=0x7f043400b410 comp rx=0 tx=0).stop 2024-03-19T01:37:16.229 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:37:16.228+0000 7f044a2fd700 1 -- 172.21.15.99:0/3970253689 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0444102440 msgr2=0x7f04441979e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:37:16.229 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:37:16.228+0000 7f044a2fd700 1 --2- 172.21.15.99:0/3970253689 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0444102440 0x7f04441979e0 secure :-1 s=READY pgs=134 cs=0 l=1 rev1=1 crypto rx=0x7f0438000c00 tx=0x7f0438009f10 comp rx=0 tx=0).stop 2024-03-19T01:37:16.229 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:37:16.228+0000 7f044a2fd700 1 -- 172.21.15.99:0/3970253689 shutdown_connections 2024-03-19T01:37:16.229 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:37:16.228+0000 7f044a2fd700 1 --2- 172.21.15.99:0/3970253689 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f042c0a2890 0x7f042c0a4d50 unknown :-1 s=CLOSED pgs=103 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:37:16.229 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:37:16.228+0000 7f044a2fd700 1 --2- 172.21.15.99:0/3970253689 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f0444102de0 0x7f044419ea60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:37:16.229 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:37:16.228+0000 7f044a2fd700 1 --2- 172.21.15.99:0/3970253689 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f0444101240 0x7f04441974a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:37:16.229 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:37:16.228+0000 7f044a2fd700 1 --2- 172.21.15.99:0/3970253689 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0444102440 0x7f04441979e0 unknown :-1 s=CLOSED pgs=134 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:37:16.230 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:37:16.228+0000 7f044a2fd700 1 -- 172.21.15.99:0/3970253689 >> 172.21.15.99:0/3970253689 conn(0x7f04440fc7e0 msgr2=0x7f0444105760 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:37:16.230 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:37:16.228+0000 7f044a2fd700 1 -- 172.21.15.99:0/3970253689 shutdown_connections 2024-03-19T01:37:16.230 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:37:16.228+0000 7f044a2fd700 1 -- 172.21.15.99:0/3970253689 wait complete. 2024-03-19T01:37:16.970 DEBUG:teuthology.orchestra.run.smithi099:osd.9> sudo journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@osd.9.service 2024-03-19T01:37:16.974 INFO:tasks.cephadm:Deploying osd.10 on smithi112 with /dev/vg_nvme/lv_1... 2024-03-19T01:37:16.974 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-03-19T01:37:17.237 INFO:teuthology.orchestra.run.smithi112.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.b/config 2024-03-19T01:37:17.312 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:17 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:17.312 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:17 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:17.313 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:17 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:37:17.313 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:17 smithi112 ceph-mon[66819]: from='osd.9 [v2:172.21.15.99:6826/1410921523,v1:172.21.15.99:6827/1410921523]' entity='osd.9' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["9"]}]: dispatch 2024-03-19T01:37:17.313 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:17 smithi112 ceph-mon[66819]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail 2024-03-19T01:37:17.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:17 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:17.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:17 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:17.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:17 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:37:17.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:17 smithi115 ceph-mon[67332]: from='osd.9 [v2:172.21.15.99:6826/1410921523,v1:172.21.15.99:6827/1410921523]' entity='osd.9' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["9"]}]: dispatch 2024-03-19T01:37:17.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:17 smithi115 ceph-mon[67332]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail 2024-03-19T01:37:17.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:17 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:17.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:17 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:17.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:17 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:37:17.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:17 smithi099 ceph-mon[62951]: from='osd.9 [v2:172.21.15.99:6826/1410921523,v1:172.21.15.99:6827/1410921523]' entity='osd.9' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["9"]}]: dispatch 2024-03-19T01:37:17.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:17 smithi099 ceph-mon[62951]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail 2024-03-19T01:37:18.488 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:18 smithi112 ceph-mon[66819]: from='osd.9 [v2:172.21.15.99:6826/1410921523,v1:172.21.15.99:6827/1410921523]' entity='osd.9' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["9"]}]': finished 2024-03-19T01:37:18.488 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:18 smithi112 ceph-mon[66819]: osdmap e66: 10 total, 9 up, 10 in 2024-03-19T01:37:18.488 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:18 smithi112 ceph-mon[66819]: from='osd.9 [v2:172.21.15.99:6826/1410921523,v1:172.21.15.99:6827/1410921523]' entity='osd.9' cmd=[{"prefix": "osd crush create-or-move", "id": 9, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2024-03-19T01:37:18.488 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:18 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-03-19T01:37:18.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:18 smithi115 ceph-mon[67332]: from='osd.9 [v2:172.21.15.99:6826/1410921523,v1:172.21.15.99:6827/1410921523]' entity='osd.9' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["9"]}]': finished 2024-03-19T01:37:18.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:18 smithi115 ceph-mon[67332]: osdmap e66: 10 total, 9 up, 10 in 2024-03-19T01:37:18.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:18 smithi115 ceph-mon[67332]: from='osd.9 [v2:172.21.15.99:6826/1410921523,v1:172.21.15.99:6827/1410921523]' entity='osd.9' cmd=[{"prefix": "osd crush create-or-move", "id": 9, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2024-03-19T01:37:18.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:18 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-03-19T01:37:18.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:18 smithi099 ceph-mon[62951]: from='osd.9 [v2:172.21.15.99:6826/1410921523,v1:172.21.15.99:6827/1410921523]' entity='osd.9' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["9"]}]': finished 2024-03-19T01:37:18.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:18 smithi099 ceph-mon[62951]: osdmap e66: 10 total, 9 up, 10 in 2024-03-19T01:37:18.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:18 smithi099 ceph-mon[62951]: from='osd.9 [v2:172.21.15.99:6826/1410921523,v1:172.21.15.99:6827/1410921523]' entity='osd.9' cmd=[{"prefix": "osd crush create-or-move", "id": 9, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2024-03-19T01:37:18.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:18 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-03-19T01:37:19.195 INFO:teuthology.orchestra.run.smithi112.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-03-19T01:37:19.195 INFO:teuthology.orchestra.run.smithi112.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-03-19T01:37:19.195 INFO:teuthology.orchestra.run.smithi112.stderr: stderr: 10+0 records in 2024-03-19T01:37:19.195 INFO:teuthology.orchestra.run.smithi112.stderr:10+0 records out 2024-03-19T01:37:19.195 INFO:teuthology.orchestra.run.smithi112.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.011414 s, 919 MB/s 2024-03-19T01:37:19.195 INFO:teuthology.orchestra.run.smithi112.stderr: stderr: 2024-03-19T01:37:19.195 INFO:teuthology.orchestra.run.smithi112.stderr:--> Zapping successful for: 2024-03-19T01:37:19.521 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:19 smithi112 ceph-mon[66819]: from='osd.9 [v2:172.21.15.99:6826/1410921523,v1:172.21.15.99:6827/1410921523]' entity='osd.9' cmd='[{"prefix": "osd crush create-or-move", "id": 9, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2024-03-19T01:37:19.521 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:19 smithi112 ceph-mon[66819]: osdmap e67: 10 total, 9 up, 10 in 2024-03-19T01:37:19.521 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:19 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-03-19T01:37:19.521 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:19 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-03-19T01:37:19.521 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:19 smithi112 ceph-mon[66819]: pgmap v213: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail 2024-03-19T01:37:19.521 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:19 smithi112 ceph-mon[66819]: from='osd.9 [v2:172.21.15.99:6826/1410921523,v1:172.21.15.99:6827/1410921523]' entity='osd.9' 2024-03-19T01:37:19.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:19 smithi115 ceph-mon[67332]: from='osd.9 [v2:172.21.15.99:6826/1410921523,v1:172.21.15.99:6827/1410921523]' entity='osd.9' cmd='[{"prefix": "osd crush create-or-move", "id": 9, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2024-03-19T01:37:19.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:19 smithi115 ceph-mon[67332]: osdmap e67: 10 total, 9 up, 10 in 2024-03-19T01:37:19.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:19 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-03-19T01:37:19.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:19 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-03-19T01:37:19.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:19 smithi115 ceph-mon[67332]: pgmap v213: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail 2024-03-19T01:37:19.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:19 smithi115 ceph-mon[67332]: from='osd.9 [v2:172.21.15.99:6826/1410921523,v1:172.21.15.99:6827/1410921523]' entity='osd.9' 2024-03-19T01:37:19.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:19 smithi099 ceph-mon[62951]: from='osd.9 [v2:172.21.15.99:6826/1410921523,v1:172.21.15.99:6827/1410921523]' entity='osd.9' cmd='[{"prefix": "osd crush create-or-move", "id": 9, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2024-03-19T01:37:19.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:19 smithi099 ceph-mon[62951]: osdmap e67: 10 total, 9 up, 10 in 2024-03-19T01:37:19.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:19 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-03-19T01:37:19.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:19 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-03-19T01:37:19.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:19 smithi099 ceph-mon[62951]: pgmap v213: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail 2024-03-19T01:37:19.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:19 smithi099 ceph-mon[62951]: from='osd.9 [v2:172.21.15.99:6826/1410921523,v1:172.21.15.99:6827/1410921523]' entity='osd.9' 2024-03-19T01:37:19.592 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 01:37:19 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T01:37:19.130+0000 7f2fc2dbd700 -1 osd.9 0 waiting for initial osdmap 2024-03-19T01:37:19.592 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 01:37:19 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T01:37:19.314+0000 7f2fbaba8700 -1 osd.9 67 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-19T01:37:19.825 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph orch daemon add osd smithi112:vg_nvme/lv_1 2024-03-19T01:37:20.062 INFO:teuthology.orchestra.run.smithi112.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.b/config 2024-03-19T01:37:20.525 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:20 smithi112 ceph-mon[66819]: purged_snaps scrub starts 2024-03-19T01:37:20.525 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:20 smithi112 ceph-mon[66819]: purged_snaps scrub ok 2024-03-19T01:37:20.525 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:20 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-03-19T01:37:20.525 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:20 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:20.525 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:20 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:20.525 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:20 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:37:20.525 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:20 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:37:20.525 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:20 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:37:20.525 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:20 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.9", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:37:20.526 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:20 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:20.526 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:20 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:37:20.526 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:20 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:37:20.526 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:20 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:20.526 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:20 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-03-19T01:37:20.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:20 smithi115 ceph-mon[67332]: purged_snaps scrub starts 2024-03-19T01:37:20.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:20 smithi115 ceph-mon[67332]: purged_snaps scrub ok 2024-03-19T01:37:20.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:20 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-03-19T01:37:20.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:20 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:20.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:20 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:20.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:20 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:37:20.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:20 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:37:20.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:20 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:37:20.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:20 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.9", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:37:20.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:20 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:20.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:20 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:37:20.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:20 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:37:20.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:20 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:20.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:20 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-03-19T01:37:20.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:20 smithi099 ceph-mon[62951]: purged_snaps scrub starts 2024-03-19T01:37:20.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:20 smithi099 ceph-mon[62951]: purged_snaps scrub ok 2024-03-19T01:37:20.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:20 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-03-19T01:37:20.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:20 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:20.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:20 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:20.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:20 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:37:20.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:20 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:37:20.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:20 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:37:20.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:20 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.9", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:37:20.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:20 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:20.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:20 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:37:20.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:20 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:37:20.593 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:20 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:20.593 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:20 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-03-19T01:37:21.429 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:21 smithi112 ceph-mon[66819]: Detected new or changed devices on smithi099 2024-03-19T01:37:21.429 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:21 smithi112 ceph-mon[66819]: Adjusting osd_memory_target on smithi099 to 4272M 2024-03-19T01:37:21.429 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:21 smithi112 ceph-mon[66819]: osd.9 [v2:172.21.15.99:6826/1410921523,v1:172.21.15.99:6827/1410921523] boot 2024-03-19T01:37:21.429 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:21 smithi112 ceph-mon[66819]: osdmap e68: 10 total, 10 up, 10 in 2024-03-19T01:37:21.429 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:21 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-03-19T01:37:21.429 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:21 smithi112 ceph-mon[66819]: pgmap v215: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail 2024-03-19T01:37:21.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:21 smithi115 ceph-mon[67332]: Detected new or changed devices on smithi099 2024-03-19T01:37:21.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:21 smithi115 ceph-mon[67332]: Adjusting osd_memory_target on smithi099 to 4272M 2024-03-19T01:37:21.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:21 smithi115 ceph-mon[67332]: osd.9 [v2:172.21.15.99:6826/1410921523,v1:172.21.15.99:6827/1410921523] boot 2024-03-19T01:37:21.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:21 smithi115 ceph-mon[67332]: osdmap e68: 10 total, 10 up, 10 in 2024-03-19T01:37:21.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:21 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-03-19T01:37:21.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:21 smithi115 ceph-mon[67332]: pgmap v215: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail 2024-03-19T01:37:21.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:21 smithi099 ceph-mon[62951]: Detected new or changed devices on smithi099 2024-03-19T01:37:21.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:21 smithi099 ceph-mon[62951]: Adjusting osd_memory_target on smithi099 to 4272M 2024-03-19T01:37:21.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:21 smithi099 ceph-mon[62951]: osd.9 [v2:172.21.15.99:6826/1410921523,v1:172.21.15.99:6827/1410921523] boot 2024-03-19T01:37:21.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:21 smithi099 ceph-mon[62951]: osdmap e68: 10 total, 10 up, 10 in 2024-03-19T01:37:21.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:21 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-03-19T01:37:21.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:21 smithi099 ceph-mon[62951]: pgmap v215: 1 pgs: 1 active+clean; 577 KiB data, 269 MiB used, 804 GiB / 805 GiB avail 2024-03-19T01:37:21.677 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:37:21.674+0000 7f78275e6700 1 -- 172.21.15.112:0/1040579598 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f7820100600 msgr2=0x7f7820100a20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:37:21.677 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:37:21.674+0000 7f78275e6700 1 --2- 172.21.15.112:0/1040579598 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f7820100600 0x7f7820100a20 secure :-1 s=READY pgs=36 cs=0 l=1 rev1=1 crypto rx=0x7f781c009990 tx=0x7f781c009ca0 comp rx=0 tx=0).stop 2024-03-19T01:37:21.678 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:37:21.675+0000 7f78275e6700 1 -- 172.21.15.112:0/1040579598 shutdown_connections 2024-03-19T01:37:21.678 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:37:21.675+0000 7f78275e6700 1 --2- 172.21.15.112:0/1040579598 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7820103f80 0x7f782010a830 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:37:21.678 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:37:21.675+0000 7f78275e6700 1 --2- 172.21.15.112:0/1040579598 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f7820100600 0x7f7820100a20 unknown :-1 s=CLOSED pgs=36 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:37:21.678 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:37:21.675+0000 7f78275e6700 1 --2- 172.21.15.112:0/1040579598 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f78201035e0 0x7f7820103a40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:37:21.678 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:37:21.675+0000 7f78275e6700 1 -- 172.21.15.112:0/1040579598 >> 172.21.15.112:0/1040579598 conn(0x7f78200fbf80 msgr2=0x7f78200fe3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:37:21.678 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:37:21.675+0000 7f78275e6700 1 -- 172.21.15.112:0/1040579598 shutdown_connections 2024-03-19T01:37:21.678 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:37:21.675+0000 7f78275e6700 1 -- 172.21.15.112:0/1040579598 wait complete. 2024-03-19T01:37:21.678 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:37:21.675+0000 7f78275e6700 1 Processor -- start 2024-03-19T01:37:21.678 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:37:21.675+0000 7f78275e6700 1 -- start start 2024-03-19T01:37:21.678 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:37:21.675+0000 7f78275e6700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f7820100600 0x7f7820102f40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:37:21.678 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:37:21.675+0000 7f78275e6700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f78201035e0 0x7f7820101590 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:37:21.678 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:37:21.675+0000 7f78275e6700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7820103f80 0x7f7820101ad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:37:21.679 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:37:21.675+0000 7f78275e6700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f782010df10 con 0x7f7820103f80 2024-03-19T01:37:21.679 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:37:21.675+0000 7f78275e6700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f782010dd90 con 0x7f78201035e0 2024-03-19T01:37:21.679 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:37:21.675+0000 7f78275e6700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f782010e090 con 0x7f7820100600 2024-03-19T01:37:21.679 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:37:21.676+0000 7f7824b81700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f78201035e0 0x7f7820101590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:37:21.679 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:37:21.676+0000 7f7824b81700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f78201035e0 0x7f7820101590 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.112:44566/0 (socket says 172.21.15.112:44566) 2024-03-19T01:37:21.679 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:37:21.676+0000 7f7824b81700 1 -- 172.21.15.112:0/176712835 learned_addr learned my addr 172.21.15.112:0/176712835 (peer_addr_for_me v2:172.21.15.112:0/0) 2024-03-19T01:37:21.679 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:37:21.676+0000 7f7825382700 1 --2- 172.21.15.112:0/176712835 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f7820100600 0x7f7820102f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:37:21.679 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:37:21.676+0000 7f7825b83700 1 --2- 172.21.15.112:0/176712835 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7820103f80 0x7f7820101ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:37:21.679 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:37:21.676+0000 7f7824b81700 1 -- 172.21.15.112:0/176712835 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f7820100600 msgr2=0x7f7820102f40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:37:21.679 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:37:21.676+0000 7f7824b81700 1 --2- 172.21.15.112:0/176712835 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f7820100600 0x7f7820102f40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:37:21.679 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:37:21.676+0000 7f7824b81700 1 -- 172.21.15.112:0/176712835 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7820103f80 msgr2=0x7f7820101ad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:37:21.679 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:37:21.676+0000 7f7824b81700 1 --2- 172.21.15.112:0/176712835 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7820103f80 0x7f7820101ad0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:37:21.680 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:37:21.676+0000 7f7824b81700 1 -- 172.21.15.112:0/176712835 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7820102380 con 0x7f78201035e0 2024-03-19T01:37:21.680 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:37:21.676+0000 7f7824b81700 1 --2- 172.21.15.112:0/176712835 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f78201035e0 0x7f7820101590 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 crypto rx=0x7f781000d980 tx=0x7f781000dc90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:37:21.680 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:37:21.676+0000 7f78167fc700 1 -- 172.21.15.112:0/176712835 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7810017070 con 0x7f78201035e0 2024-03-19T01:37:21.680 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:37:21.676+0000 7f78275e6700 1 -- 172.21.15.112:0/176712835 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f78201a26f0 con 0x7f78201035e0 2024-03-19T01:37:21.680 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:37:21.676+0000 7f78167fc700 1 -- 172.21.15.112:0/176712835 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7810005020 con 0x7f78201035e0 2024-03-19T01:37:21.680 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:37:21.676+0000 7f78167fc700 1 -- 172.21.15.112:0/176712835 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f781000fbf0 con 0x7f78201035e0 2024-03-19T01:37:21.680 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:37:21.676+0000 7f78275e6700 1 -- 172.21.15.112:0/176712835 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f78201a2bb0 con 0x7f78201035e0 2024-03-19T01:37:21.681 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:37:21.677+0000 7f78275e6700 1 -- 172.21.15.112:0/176712835 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f7820068fd0 con 0x7f78201035e0 2024-03-19T01:37:21.681 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:37:21.678+0000 7f78167fc700 1 -- 172.21.15.112:0/176712835 <== mon.2 v2:172.21.15.112:3300/0 4 ==== mgrmap(e 16) ==== 137290+0+0 (secure 0 0 0) 0x7f78100051d0 con 0x7f78201035e0 2024-03-19T01:37:21.681 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:37:21.679+0000 7f78167fc700 1 --2- 172.21.15.112:0/176712835 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f780c0a2890 0x7f780c0a4d50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:37:21.681 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:37:21.679+0000 7f78167fc700 1 -- 172.21.15.112:0/176712835 <== mon.2 v2:172.21.15.112:3300/0 5 ==== osd_map(69..69 src has 1..69) ==== 6261+0+0 (secure 0 0 0) 0x7f78100ce470 con 0x7f78201035e0 2024-03-19T01:37:21.681 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:37:21.679+0000 7f7825382700 1 --2- 172.21.15.112:0/176712835 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f780c0a2890 0x7f780c0a4d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:37:21.681 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:37:21.680+0000 7f7825382700 1 --2- 172.21.15.112:0/176712835 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f780c0a2890 0x7f780c0a4d50 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 crypto rx=0x7f781c00b7b0 tx=0x7f781c005fd0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:37:21.683 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:37:21.683+0000 7f78167fc700 1 -- 172.21.15.112:0/176712835 <== mon.2 v2:172.21.15.112:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f7810020020 con 0x7f78201035e0 2024-03-19T01:37:21.885 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:37:21.884+0000 7f78275e6700 1 -- 172.21.15.112:0/176712835 --> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi112:vg_nvme/lv_1", "target": ["mon-mgr", ""]}) -- 0x7f782002e000 con 0x7f780c0a2890 2024-03-19T01:37:22.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:22 smithi115 ceph-mon[67332]: osdmap e69: 10 total, 10 up, 10 in 2024-03-19T01:37:22.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:22 smithi115 ceph-mon[67332]: from='client.24314 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi112:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:37:22.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:22 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-19T01:37:22.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:22 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-19T01:37:22.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:22 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:37:22.580 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:22 smithi112 ceph-mon[66819]: osdmap e69: 10 total, 10 up, 10 in 2024-03-19T01:37:22.580 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:22 smithi112 ceph-mon[66819]: from='client.24314 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi112:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:37:22.580 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:22 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-19T01:37:22.580 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:22 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-19T01:37:22.580 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:22 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:37:22.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:22 smithi099 ceph-mon[62951]: osdmap e69: 10 total, 10 up, 10 in 2024-03-19T01:37:22.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:22 smithi099 ceph-mon[62951]: from='client.24314 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi112:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:37:22.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:22 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-19T01:37:22.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:22 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-19T01:37:22.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:22 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:37:23.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:23 smithi115 ceph-mon[67332]: pgmap v217: 1 pgs: 1 active+clean; 577 KiB data, 301 MiB used, 894 GiB / 894 GiB avail 2024-03-19T01:37:23.561 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:23 smithi112 ceph-mon[66819]: pgmap v217: 1 pgs: 1 active+clean; 577 KiB data, 301 MiB used, 894 GiB / 894 GiB avail 2024-03-19T01:37:23.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:23 smithi099 ceph-mon[62951]: pgmap v217: 1 pgs: 1 active+clean; 577 KiB data, 301 MiB used, 894 GiB / 894 GiB avail 2024-03-19T01:37:24.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:24 smithi099 ceph-mon[62951]: from='client.? 172.21.15.112:0/523724366' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6c50a8a7-3345-4f4a-af0e-5606c7aca453"}]: dispatch 2024-03-19T01:37:24.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:24 smithi099 ceph-mon[62951]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6c50a8a7-3345-4f4a-af0e-5606c7aca453"}]: dispatch 2024-03-19T01:37:24.842 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:24 smithi099 ceph-mon[62951]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6c50a8a7-3345-4f4a-af0e-5606c7aca453"}]': finished 2024-03-19T01:37:24.842 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:24 smithi099 ceph-mon[62951]: osdmap e70: 11 total, 10 up, 11 in 2024-03-19T01:37:24.842 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:24 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-03-19T01:37:24.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:24 smithi112 ceph-mon[66819]: from='client.? 172.21.15.112:0/523724366' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6c50a8a7-3345-4f4a-af0e-5606c7aca453"}]: dispatch 2024-03-19T01:37:24.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:24 smithi112 ceph-mon[66819]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6c50a8a7-3345-4f4a-af0e-5606c7aca453"}]: dispatch 2024-03-19T01:37:24.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:24 smithi112 ceph-mon[66819]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6c50a8a7-3345-4f4a-af0e-5606c7aca453"}]': finished 2024-03-19T01:37:24.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:24 smithi112 ceph-mon[66819]: osdmap e70: 11 total, 10 up, 11 in 2024-03-19T01:37:24.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:24 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-03-19T01:37:25.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:24 smithi115 ceph-mon[67332]: from='client.? 172.21.15.112:0/523724366' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6c50a8a7-3345-4f4a-af0e-5606c7aca453"}]: dispatch 2024-03-19T01:37:25.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:24 smithi115 ceph-mon[67332]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6c50a8a7-3345-4f4a-af0e-5606c7aca453"}]: dispatch 2024-03-19T01:37:25.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:24 smithi115 ceph-mon[67332]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6c50a8a7-3345-4f4a-af0e-5606c7aca453"}]': finished 2024-03-19T01:37:25.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:24 smithi115 ceph-mon[67332]: osdmap e70: 11 total, 10 up, 11 in 2024-03-19T01:37:25.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:24 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-03-19T01:37:25.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:25 smithi112 ceph-mon[66819]: pgmap v219: 1 pgs: 1 active+clean; 577 KiB data, 301 MiB used, 894 GiB / 894 GiB avail 2024-03-19T01:37:25.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:25 smithi112 ceph-mon[66819]: from='client.? 172.21.15.112:0/3766095210' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-19T01:37:26.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:25 smithi115 ceph-mon[67332]: pgmap v219: 1 pgs: 1 active+clean; 577 KiB data, 301 MiB used, 894 GiB / 894 GiB avail 2024-03-19T01:37:26.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:25 smithi115 ceph-mon[67332]: from='client.? 172.21.15.112:0/3766095210' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-19T01:37:26.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:25 smithi099 ceph-mon[62951]: pgmap v219: 1 pgs: 1 active+clean; 577 KiB data, 301 MiB used, 894 GiB / 894 GiB avail 2024-03-19T01:37:26.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:25 smithi099 ceph-mon[62951]: from='client.? 172.21.15.112:0/3766095210' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-19T01:37:28.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:27 smithi115 ceph-mon[67332]: pgmap v220: 1 pgs: 1 active+clean; 577 KiB data, 301 MiB used, 894 GiB / 894 GiB avail 2024-03-19T01:37:28.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:27 smithi099 ceph-mon[62951]: pgmap v220: 1 pgs: 1 active+clean; 577 KiB data, 301 MiB used, 894 GiB / 894 GiB avail 2024-03-19T01:37:28.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:27 smithi112 ceph-mon[66819]: pgmap v220: 1 pgs: 1 active+clean; 577 KiB data, 301 MiB used, 894 GiB / 894 GiB avail 2024-03-19T01:37:30.024 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:29 smithi112 ceph-mon[66819]: pgmap v221: 1 pgs: 1 active+clean; 577 KiB data, 301 MiB used, 894 GiB / 894 GiB avail 2024-03-19T01:37:30.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:29 smithi115 ceph-mon[67332]: pgmap v221: 1 pgs: 1 active+clean; 577 KiB data, 301 MiB used, 894 GiB / 894 GiB avail 2024-03-19T01:37:30.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:29 smithi099 ceph-mon[62951]: pgmap v221: 1 pgs: 1 active+clean; 577 KiB data, 301 MiB used, 894 GiB / 894 GiB avail 2024-03-19T01:37:31.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:31 smithi112 ceph-mon[66819]: pgmap v222: 1 pgs: 1 active+clean; 577 KiB data, 301 MiB used, 894 GiB / 894 GiB avail 2024-03-19T01:37:32.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:31 smithi115 ceph-mon[67332]: pgmap v222: 1 pgs: 1 active+clean; 577 KiB data, 301 MiB used, 894 GiB / 894 GiB avail 2024-03-19T01:37:32.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:31 smithi099 ceph-mon[62951]: pgmap v222: 1 pgs: 1 active+clean; 577 KiB data, 301 MiB used, 894 GiB / 894 GiB avail 2024-03-19T01:37:32.820 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:32 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.10"}]: dispatch 2024-03-19T01:37:32.821 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:32 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:37:33.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:32 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.10"}]: dispatch 2024-03-19T01:37:33.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:32 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:37:33.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:32 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.10"}]: dispatch 2024-03-19T01:37:33.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:32 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:37:34.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:33 smithi115 ceph-mon[67332]: Deploying daemon osd.10 on smithi112 2024-03-19T01:37:34.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:33 smithi115 ceph-mon[67332]: pgmap v223: 1 pgs: 1 active+clean; 577 KiB data, 301 MiB used, 894 GiB / 894 GiB avail 2024-03-19T01:37:34.085 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:33 smithi112 ceph-mon[66819]: Deploying daemon osd.10 on smithi112 2024-03-19T01:37:34.085 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:33 smithi112 ceph-mon[66819]: pgmap v223: 1 pgs: 1 active+clean; 577 KiB data, 301 MiB used, 894 GiB / 894 GiB avail 2024-03-19T01:37:34.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:33 smithi099 ceph-mon[62951]: Deploying daemon osd.10 on smithi112 2024-03-19T01:37:34.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:33 smithi099 ceph-mon[62951]: pgmap v223: 1 pgs: 1 active+clean; 577 KiB data, 301 MiB used, 894 GiB / 894 GiB avail 2024-03-19T01:37:35.927 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:35 smithi112 ceph-mon[66819]: pgmap v224: 1 pgs: 1 active+clean; 577 KiB data, 301 MiB used, 894 GiB / 894 GiB avail 2024-03-19T01:37:36.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:35 smithi115 ceph-mon[67332]: pgmap v224: 1 pgs: 1 active+clean; 577 KiB data, 301 MiB used, 894 GiB / 894 GiB avail 2024-03-19T01:37:36.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:35 smithi099 ceph-mon[62951]: pgmap v224: 1 pgs: 1 active+clean; 577 KiB data, 301 MiB used, 894 GiB / 894 GiB avail 2024-03-19T01:37:38.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:37 smithi115 ceph-mon[67332]: pgmap v225: 1 pgs: 1 active+clean; 577 KiB data, 301 MiB used, 894 GiB / 894 GiB avail 2024-03-19T01:37:38.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:37 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:38.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:37 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:38.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:37 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:37:38.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:37 smithi099 ceph-mon[62951]: pgmap v225: 1 pgs: 1 active+clean; 577 KiB data, 301 MiB used, 894 GiB / 894 GiB avail 2024-03-19T01:37:38.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:37 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:38.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:37 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:38.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:37 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:37:38.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:37 smithi112 ceph-mon[66819]: pgmap v225: 1 pgs: 1 active+clean; 577 KiB data, 301 MiB used, 894 GiB / 894 GiB avail 2024-03-19T01:37:38.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:37 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:38.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:37 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:38.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:37 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:37:40.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:39 smithi099 ceph-mon[62951]: pgmap v226: 1 pgs: 1 active+clean; 577 KiB data, 301 MiB used, 894 GiB / 894 GiB avail 2024-03-19T01:37:40.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:39 smithi112 ceph-mon[66819]: pgmap v226: 1 pgs: 1 active+clean; 577 KiB data, 301 MiB used, 894 GiB / 894 GiB avail 2024-03-19T01:37:40.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:39 smithi115 ceph-mon[67332]: pgmap v226: 1 pgs: 1 active+clean; 577 KiB data, 301 MiB used, 894 GiB / 894 GiB avail 2024-03-19T01:37:40.651 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:37:40.650+0000 7f78167fc700 1 -- 172.21.15.112:0/176712835 <== mgr.14150 v2:172.21.15.99:6800/4272410295 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+38 (secure 0 0 0) 0x7f782002e000 con 0x7f780c0a2890 2024-03-19T01:37:40.651 INFO:teuthology.orchestra.run.smithi112.stdout:Created osd(s) 10 on host 'smithi112' 2024-03-19T01:37:40.653 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:37:40.653+0000 7f78275e6700 1 -- 172.21.15.112:0/176712835 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f780c0a2890 msgr2=0x7f780c0a4d50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:37:40.653 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:37:40.653+0000 7f78275e6700 1 --2- 172.21.15.112:0/176712835 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f780c0a2890 0x7f780c0a4d50 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 crypto rx=0x7f781c00b7b0 tx=0x7f781c005fd0 comp rx=0 tx=0).stop 2024-03-19T01:37:40.653 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:37:40.653+0000 7f78275e6700 1 -- 172.21.15.112:0/176712835 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f78201035e0 msgr2=0x7f7820101590 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:37:40.653 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:37:40.653+0000 7f78275e6700 1 --2- 172.21.15.112:0/176712835 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f78201035e0 0x7f7820101590 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 crypto rx=0x7f781000d980 tx=0x7f781000dc90 comp rx=0 tx=0).stop 2024-03-19T01:37:40.653 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:37:40.653+0000 7f78275e6700 1 -- 172.21.15.112:0/176712835 shutdown_connections 2024-03-19T01:37:40.653 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:37:40.653+0000 7f78275e6700 1 --2- 172.21.15.112:0/176712835 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f780c0a2890 0x7f780c0a4d50 unknown :-1 s=CLOSED pgs=110 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:37:40.653 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:37:40.653+0000 7f78275e6700 1 --2- 172.21.15.112:0/176712835 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f7820100600 0x7f7820102f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:37:40.653 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:37:40.653+0000 7f78275e6700 1 --2- 172.21.15.112:0/176712835 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7820103f80 0x7f7820101ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:37:40.653 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:37:40.653+0000 7f78275e6700 1 --2- 172.21.15.112:0/176712835 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f78201035e0 0x7f7820101590 unknown :-1 s=CLOSED pgs=37 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:37:40.653 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:37:40.653+0000 7f78275e6700 1 -- 172.21.15.112:0/176712835 >> 172.21.15.112:0/176712835 conn(0x7f78200fbf80 msgr2=0x7f7820106960 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:37:40.654 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:37:40.653+0000 7f78275e6700 1 -- 172.21.15.112:0/176712835 shutdown_connections 2024-03-19T01:37:40.654 INFO:teuthology.orchestra.run.smithi112.stderr:2024-03-19T01:37:40.653+0000 7f78275e6700 1 -- 172.21.15.112:0/176712835 wait complete. 2024-03-19T01:37:41.271 DEBUG:teuthology.orchestra.run.smithi112:osd.10> sudo journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@osd.10.service 2024-03-19T01:37:41.273 INFO:tasks.cephadm:Deploying osd.11 on smithi115 with /dev/vg_nvme/lv_1... 2024-03-19T01:37:41.274 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-03-19T01:37:41.519 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.c/config 2024-03-19T01:37:41.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:41 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:41.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:41 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:41.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:41 smithi112 ceph-mon[66819]: pgmap v227: 1 pgs: 1 active+clean; 577 KiB data, 301 MiB used, 894 GiB / 894 GiB avail 2024-03-19T01:37:41.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:41 smithi112 ceph-mon[66819]: from='osd.10 [v2:172.21.15.112:6824/891805922,v1:172.21.15.112:6825/891805922]' entity='osd.10' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["10"]}]: dispatch 2024-03-19T01:37:41.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:41 smithi112 ceph-mon[66819]: from='osd.10 ' entity='osd.10' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["10"]}]: dispatch 2024-03-19T01:37:41.862 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 01:37:41 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T01:37:41.506+0000 7f1e17f507c0 -1 osd.10 0 log_to_monitors true 2024-03-19T01:37:41.866 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:41 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:41.866 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:41 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:41.866 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:41 smithi115 ceph-mon[67332]: pgmap v227: 1 pgs: 1 active+clean; 577 KiB data, 301 MiB used, 894 GiB / 894 GiB avail 2024-03-19T01:37:41.866 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:41 smithi115 ceph-mon[67332]: from='osd.10 [v2:172.21.15.112:6824/891805922,v1:172.21.15.112:6825/891805922]' entity='osd.10' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["10"]}]: dispatch 2024-03-19T01:37:41.866 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:41 smithi115 ceph-mon[67332]: from='osd.10 ' entity='osd.10' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["10"]}]: dispatch 2024-03-19T01:37:42.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:41 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:42.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:41 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:42.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:41 smithi099 ceph-mon[62951]: pgmap v227: 1 pgs: 1 active+clean; 577 KiB data, 301 MiB used, 894 GiB / 894 GiB avail 2024-03-19T01:37:42.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:41 smithi099 ceph-mon[62951]: from='osd.10 [v2:172.21.15.112:6824/891805922,v1:172.21.15.112:6825/891805922]' entity='osd.10' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["10"]}]: dispatch 2024-03-19T01:37:42.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:41 smithi099 ceph-mon[62951]: from='osd.10 ' entity='osd.10' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["10"]}]: dispatch 2024-03-19T01:37:42.718 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:42 smithi112 ceph-mon[66819]: from='osd.10 ' entity='osd.10' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["10"]}]': finished 2024-03-19T01:37:42.718 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:42 smithi112 ceph-mon[66819]: osdmap e71: 11 total, 10 up, 11 in 2024-03-19T01:37:42.718 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:42 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-03-19T01:37:42.718 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:42 smithi112 ceph-mon[66819]: from='osd.10 [v2:172.21.15.112:6824/891805922,v1:172.21.15.112:6825/891805922]' entity='osd.10' cmd=[{"prefix": "osd crush create-or-move", "id": 10, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-03-19T01:37:42.718 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:42 smithi112 ceph-mon[66819]: from='osd.10 ' entity='osd.10' cmd=[{"prefix": "osd crush create-or-move", "id": 10, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-03-19T01:37:42.718 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:42 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:42.718 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:42 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:42.970 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:42 smithi115 ceph-mon[67332]: from='osd.10 ' entity='osd.10' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["10"]}]': finished 2024-03-19T01:37:42.970 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:42 smithi115 ceph-mon[67332]: osdmap e71: 11 total, 10 up, 11 in 2024-03-19T01:37:42.970 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:42 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-03-19T01:37:42.970 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:42 smithi115 ceph-mon[67332]: from='osd.10 [v2:172.21.15.112:6824/891805922,v1:172.21.15.112:6825/891805922]' entity='osd.10' cmd=[{"prefix": "osd crush create-or-move", "id": 10, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-03-19T01:37:42.970 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:42 smithi115 ceph-mon[67332]: from='osd.10 ' entity='osd.10' cmd=[{"prefix": "osd crush create-or-move", "id": 10, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-03-19T01:37:42.970 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:42 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:42.970 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:42 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:43.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:42 smithi099 ceph-mon[62951]: from='osd.10 ' entity='osd.10' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["10"]}]': finished 2024-03-19T01:37:43.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:42 smithi099 ceph-mon[62951]: osdmap e71: 11 total, 10 up, 11 in 2024-03-19T01:37:43.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:42 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-03-19T01:37:43.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:42 smithi099 ceph-mon[62951]: from='osd.10 [v2:172.21.15.112:6824/891805922,v1:172.21.15.112:6825/891805922]' entity='osd.10' cmd=[{"prefix": "osd crush create-or-move", "id": 10, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-03-19T01:37:43.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:42 smithi099 ceph-mon[62951]: from='osd.10 ' entity='osd.10' cmd=[{"prefix": "osd crush create-or-move", "id": 10, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-03-19T01:37:43.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:42 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:43.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:42 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:43.556 INFO:teuthology.orchestra.run.smithi115.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-03-19T01:37:43.556 INFO:teuthology.orchestra.run.smithi115.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-03-19T01:37:43.556 INFO:teuthology.orchestra.run.smithi115.stderr: stderr: 10+0 records in 2024-03-19T01:37:43.556 INFO:teuthology.orchestra.run.smithi115.stderr:10+0 records out 2024-03-19T01:37:43.556 INFO:teuthology.orchestra.run.smithi115.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.011546 s, 908 MB/s 2024-03-19T01:37:43.556 INFO:teuthology.orchestra.run.smithi115.stderr:--> Zapping successful for: 2024-03-19T01:37:43.606 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 01:37:43 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T01:37:43.565+0000 7f1e0ddb6700 -1 osd.10 0 waiting for initial osdmap 2024-03-19T01:37:43.862 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 01:37:43 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T01:37:43.741+0000 7f1e053a0700 -1 osd.10 73 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-19T01:37:43.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:43 smithi112 ceph-mon[66819]: from='osd.10 ' entity='osd.10' cmd='[{"prefix": "osd crush create-or-move", "id": 10, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]': finished 2024-03-19T01:37:43.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:43 smithi112 ceph-mon[66819]: osdmap e72: 11 total, 10 up, 11 in 2024-03-19T01:37:43.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:43 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-03-19T01:37:43.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:43 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-03-19T01:37:43.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:43 smithi112 ceph-mon[66819]: pgmap v230: 1 pgs: 1 active+clean; 577 KiB data, 301 MiB used, 894 GiB / 894 GiB avail 2024-03-19T01:37:43.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:43 smithi112 ceph-mon[66819]: from='osd.10 ' entity='osd.10' 2024-03-19T01:37:43.890 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:43 smithi115 ceph-mon[67332]: from='osd.10 ' entity='osd.10' cmd='[{"prefix": "osd crush create-or-move", "id": 10, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]': finished 2024-03-19T01:37:43.890 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:43 smithi115 ceph-mon[67332]: osdmap e72: 11 total, 10 up, 11 in 2024-03-19T01:37:43.890 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:43 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-03-19T01:37:43.890 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:43 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-03-19T01:37:43.890 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:43 smithi115 ceph-mon[67332]: pgmap v230: 1 pgs: 1 active+clean; 577 KiB data, 301 MiB used, 894 GiB / 894 GiB avail 2024-03-19T01:37:43.891 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:43 smithi115 ceph-mon[67332]: from='osd.10 ' entity='osd.10' 2024-03-19T01:37:44.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:43 smithi099 ceph-mon[62951]: from='osd.10 ' entity='osd.10' cmd='[{"prefix": "osd crush create-or-move", "id": 10, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]': finished 2024-03-19T01:37:44.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:43 smithi099 ceph-mon[62951]: osdmap e72: 11 total, 10 up, 11 in 2024-03-19T01:37:44.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:43 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-03-19T01:37:44.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:43 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-03-19T01:37:44.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:43 smithi099 ceph-mon[62951]: pgmap v230: 1 pgs: 1 active+clean; 577 KiB data, 301 MiB used, 894 GiB / 894 GiB avail 2024-03-19T01:37:44.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:43 smithi099 ceph-mon[62951]: from='osd.10 ' entity='osd.10' 2024-03-19T01:37:44.209 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph orch daemon add osd smithi115:vg_nvme/lv_1 2024-03-19T01:37:44.448 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.c/config 2024-03-19T01:37:44.744 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:44 smithi112 ceph-mon[66819]: purged_snaps scrub starts 2024-03-19T01:37:44.744 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:44 smithi112 ceph-mon[66819]: purged_snaps scrub ok 2024-03-19T01:37:44.744 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:44 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-03-19T01:37:44.744 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:44 smithi112 ceph-mon[66819]: osdmap e73: 11 total, 10 up, 11 in 2024-03-19T01:37:44.745 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:44 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-03-19T01:37:44.745 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:44 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-03-19T01:37:44.806 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:44 smithi115 ceph-mon[67332]: purged_snaps scrub starts 2024-03-19T01:37:44.806 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:44 smithi115 ceph-mon[67332]: purged_snaps scrub ok 2024-03-19T01:37:44.806 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:44 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-03-19T01:37:44.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:44 smithi115 ceph-mon[67332]: osdmap e73: 11 total, 10 up, 11 in 2024-03-19T01:37:44.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:44 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-03-19T01:37:44.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:44 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-03-19T01:37:45.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:44 smithi099 ceph-mon[62951]: purged_snaps scrub starts 2024-03-19T01:37:45.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:44 smithi099 ceph-mon[62951]: purged_snaps scrub ok 2024-03-19T01:37:45.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:44 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-03-19T01:37:45.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:44 smithi099 ceph-mon[62951]: osdmap e73: 11 total, 10 up, 11 in 2024-03-19T01:37:45.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:44 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-03-19T01:37:45.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:44 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-03-19T01:37:45.950 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:45 smithi115 ceph-mon[67332]: osd.10 [v2:172.21.15.112:6824/891805922,v1:172.21.15.112:6825/891805922] boot 2024-03-19T01:37:45.950 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:45 smithi115 ceph-mon[67332]: osdmap e74: 11 total, 11 up, 11 in 2024-03-19T01:37:45.950 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:45 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-03-19T01:37:45.950 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:45 smithi115 ceph-mon[67332]: pgmap v233: 1 pgs: 1 active+clean; 577 KiB data, 301 MiB used, 894 GiB / 894 GiB avail 2024-03-19T01:37:45.951 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:45 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:45.951 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:45 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:45.951 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:45 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:37:45.951 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:45 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:37:45.951 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:45 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:37:45.951 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:45 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.10", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:37:45.951 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:45 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:45.951 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:45 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:37:45.951 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:45 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:37:45.951 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:45 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:45.951 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:45 smithi115 ceph-mon[67332]: osdmap e75: 11 total, 11 up, 11 in 2024-03-19T01:37:45.951 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:45 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:37:45.951 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:45 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:37:45.951 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:45 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:37:45.952 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:45 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:46.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:45 smithi099 ceph-mon[62951]: osd.10 [v2:172.21.15.112:6824/891805922,v1:172.21.15.112:6825/891805922] boot 2024-03-19T01:37:46.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:45 smithi099 ceph-mon[62951]: osdmap e74: 11 total, 11 up, 11 in 2024-03-19T01:37:46.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:45 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-03-19T01:37:46.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:45 smithi099 ceph-mon[62951]: pgmap v233: 1 pgs: 1 active+clean; 577 KiB data, 301 MiB used, 894 GiB / 894 GiB avail 2024-03-19T01:37:46.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:45 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:46.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:45 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:46.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:45 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:37:46.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:45 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:37:46.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:45 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:37:46.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:45 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.10", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:37:46.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:45 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:46.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:45 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:37:46.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:45 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:37:46.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:45 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:46.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:45 smithi099 ceph-mon[62951]: osdmap e75: 11 total, 11 up, 11 in 2024-03-19T01:37:46.093 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:45 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:37:46.093 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:45 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:37:46.093 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:45 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:37:46.093 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:45 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:46.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:45 smithi112 ceph-mon[66819]: osd.10 [v2:172.21.15.112:6824/891805922,v1:172.21.15.112:6825/891805922] boot 2024-03-19T01:37:46.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:45 smithi112 ceph-mon[66819]: osdmap e74: 11 total, 11 up, 11 in 2024-03-19T01:37:46.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:45 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-03-19T01:37:46.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:45 smithi112 ceph-mon[66819]: pgmap v233: 1 pgs: 1 active+clean; 577 KiB data, 301 MiB used, 894 GiB / 894 GiB avail 2024-03-19T01:37:46.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:45 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:46.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:45 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:46.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:45 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:37:46.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:45 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:37:46.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:45 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:37:46.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:45 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.10", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:37:46.113 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:45 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:46.113 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:45 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:37:46.113 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:45 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:37:46.113 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:45 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:46.113 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:45 smithi112 ceph-mon[66819]: osdmap e75: 11 total, 11 up, 11 in 2024-03-19T01:37:46.113 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:45 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:37:46.113 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:45 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:37:46.113 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:45 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:37:46.113 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:45 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:37:46.145 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:37:46.143+0000 7fa525078700 1 -- 172.21.15.115:0/2333985340 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa5201027f0 msgr2=0x7fa520102c10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:37:46.145 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:37:46.143+0000 7fa525078700 1 --2- 172.21.15.115:0/2333985340 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa5201027f0 0x7fa520102c10 secure :-1 s=READY pgs=39 cs=0 l=1 rev1=1 crypto rx=0x7fa5100099c0 tx=0x7fa510009cd0 comp rx=0 tx=0).stop 2024-03-19T01:37:46.145 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:37:46.143+0000 7fa525078700 1 -- 172.21.15.115:0/2333985340 shutdown_connections 2024-03-19T01:37:46.145 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:37:46.143+0000 7fa525078700 1 --2- 172.21.15.115:0/2333985340 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa5201027f0 0x7fa520102c10 unknown :-1 s=CLOSED pgs=39 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:37:46.145 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:37:46.143+0000 7fa525078700 1 --2- 172.21.15.115:0/2333985340 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa520101ed0 0x7fa520109790 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:37:46.145 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:37:46.143+0000 7fa525078700 1 --2- 172.21.15.115:0/2333985340 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa520101530 0x7fa520101990 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:37:46.145 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:37:46.143+0000 7fa525078700 1 -- 172.21.15.115:0/2333985340 >> 172.21.15.115:0/2333985340 conn(0x7fa5200fbfa0 msgr2=0x7fa5200fe3c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:37:46.145 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:37:46.143+0000 7fa525078700 1 -- 172.21.15.115:0/2333985340 shutdown_connections 2024-03-19T01:37:46.145 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:37:46.143+0000 7fa525078700 1 -- 172.21.15.115:0/2333985340 wait complete. 2024-03-19T01:37:46.145 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:37:46.144+0000 7fa525078700 1 Processor -- start 2024-03-19T01:37:46.147 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:37:46.144+0000 7fa525078700 1 -- start start 2024-03-19T01:37:46.147 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:37:46.145+0000 7fa525078700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa520101530 0x7fa5201974c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:37:46.147 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:37:46.145+0000 7fa525078700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa520101ed0 0x7fa520197a00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:37:46.147 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:37:46.145+0000 7fa525078700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa5201027f0 0x7fa52019ea80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:37:46.147 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:37:46.145+0000 7fa525078700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fa52010ce40 con 0x7fa520101ed0 2024-03-19T01:37:46.148 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:37:46.145+0000 7fa525078700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fa52010ccc0 con 0x7fa5201027f0 2024-03-19T01:37:46.148 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:37:46.145+0000 7fa525078700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fa52010cfc0 con 0x7fa520101530 2024-03-19T01:37:46.148 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:37:46.145+0000 7fa51ed9d700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa520101530 0x7fa5201974c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:37:46.148 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:37:46.145+0000 7fa51ed9d700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa520101530 0x7fa5201974c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.115:42324/0 (socket says 172.21.15.115:42324) 2024-03-19T01:37:46.148 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:37:46.145+0000 7fa51ed9d700 1 -- 172.21.15.115:0/2830326607 learned_addr learned my addr 172.21.15.115:0/2830326607 (peer_addr_for_me v2:172.21.15.115:0/0) 2024-03-19T01:37:46.148 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:37:46.145+0000 7fa51e59c700 1 --2- 172.21.15.115:0/2830326607 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa520101ed0 0x7fa520197a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:37:46.148 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:37:46.145+0000 7fa51f59e700 1 --2- 172.21.15.115:0/2830326607 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa5201027f0 0x7fa52019ea80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:37:46.148 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:37:46.145+0000 7fa51ed9d700 1 -- 172.21.15.115:0/2830326607 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa5201027f0 msgr2=0x7fa52019ea80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:37:46.148 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:37:46.145+0000 7fa51ed9d700 1 --2- 172.21.15.115:0/2830326607 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa5201027f0 0x7fa52019ea80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:37:46.148 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:37:46.145+0000 7fa51ed9d700 1 -- 172.21.15.115:0/2830326607 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa520101ed0 msgr2=0x7fa520197a00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:37:46.148 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:37:46.145+0000 7fa51ed9d700 1 --2- 172.21.15.115:0/2830326607 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa520101ed0 0x7fa520197a00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:37:46.148 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:37:46.145+0000 7fa51ed9d700 1 -- 172.21.15.115:0/2830326607 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa52019f120 con 0x7fa520101530 2024-03-19T01:37:46.148 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:37:46.145+0000 7fa51ed9d700 1 --2- 172.21.15.115:0/2830326607 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa520101530 0x7fa5201974c0 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7fa510005b00 tx=0x7fa510009510 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:37:46.149 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:37:46.145+0000 7fa507fff700 1 -- 172.21.15.115:0/2830326607 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa510028070 con 0x7fa520101530 2024-03-19T01:37:46.149 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:37:46.146+0000 7fa525078700 1 -- 172.21.15.115:0/2830326607 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa520100420 con 0x7fa520101530 2024-03-19T01:37:46.149 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:37:46.146+0000 7fa507fff700 1 -- 172.21.15.115:0/2830326607 <== mon.1 v2:172.21.15.115:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa510004150 con 0x7fa520101530 2024-03-19T01:37:46.149 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:37:46.146+0000 7fa507fff700 1 -- 172.21.15.115:0/2830326607 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa51000f460 con 0x7fa520101530 2024-03-19T01:37:46.149 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:37:46.146+0000 7fa525078700 1 -- 172.21.15.115:0/2830326607 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fa520100950 con 0x7fa520101530 2024-03-19T01:37:46.149 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:37:46.146+0000 7fa525078700 1 -- 172.21.15.115:0/2830326607 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fa520068fd0 con 0x7fa520101530 2024-03-19T01:37:46.150 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:37:46.148+0000 7fa507fff700 1 -- 172.21.15.115:0/2830326607 <== mon.1 v2:172.21.15.115:3300/0 4 ==== mgrmap(e 16) ==== 137290+0+0 (secure 0 0 0) 0x7fa51002b070 con 0x7fa520101530 2024-03-19T01:37:46.150 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:37:46.148+0000 7fa507fff700 1 --2- 172.21.15.115:0/2830326607 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa5080a2890 0x7fa5080a4d50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:37:46.150 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:37:46.148+0000 7fa507fff700 1 -- 172.21.15.115:0/2830326607 <== mon.1 v2:172.21.15.115:3300/0 5 ==== osd_map(75..75 src has 1..75) ==== 6723+0+0 (secure 0 0 0) 0x7fa5100d13d0 con 0x7fa520101530 2024-03-19T01:37:46.150 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:37:46.148+0000 7fa51e59c700 1 --2- 172.21.15.115:0/2830326607 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa5080a2890 0x7fa5080a4d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:37:46.150 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:37:46.149+0000 7fa51e59c700 1 --2- 172.21.15.115:0/2830326607 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa5080a2890 0x7fa5080a4d50 secure :-1 s=READY pgs=117 cs=0 l=1 rev1=1 crypto rx=0x7fa514005950 tx=0x7fa51400a300 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:37:46.152 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:37:46.152+0000 7fa507fff700 1 -- 172.21.15.115:0/2830326607 <== mon.1 v2:172.21.15.115:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7fa510032020 con 0x7fa520101530 2024-03-19T01:37:46.363 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:37:46.363+0000 7fa525078700 1 -- 172.21.15.115:0/2830326607 --> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi115:vg_nvme/lv_1", "target": ["mon-mgr", ""]}) -- 0x7fa520100bb0 con 0x7fa5080a2890 2024-03-19T01:37:46.963 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:46 smithi115 ceph-mon[67332]: Detected new or changed devices on smithi112 2024-03-19T01:37:46.964 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:46 smithi115 ceph-mon[67332]: Adjusting osd_memory_target on smithi112 to 4270M 2024-03-19T01:37:46.964 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:46 smithi115 ceph-mon[67332]: osdmap e76: 11 total, 11 up, 11 in 2024-03-19T01:37:46.964 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:46 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-19T01:37:46.964 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:46 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-19T01:37:46.964 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:46 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:37:47.014 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:46 smithi112 ceph-mon[66819]: Detected new or changed devices on smithi112 2024-03-19T01:37:47.014 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:46 smithi112 ceph-mon[66819]: Adjusting osd_memory_target on smithi112 to 4270M 2024-03-19T01:37:47.014 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:46 smithi112 ceph-mon[66819]: osdmap e76: 11 total, 11 up, 11 in 2024-03-19T01:37:47.015 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:46 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-19T01:37:47.015 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:46 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-19T01:37:47.015 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:46 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:37:47.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:46 smithi099 ceph-mon[62951]: Detected new or changed devices on smithi112 2024-03-19T01:37:47.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:46 smithi099 ceph-mon[62951]: Adjusting osd_memory_target on smithi112 to 4270M 2024-03-19T01:37:47.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:46 smithi099 ceph-mon[62951]: osdmap e76: 11 total, 11 up, 11 in 2024-03-19T01:37:47.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:46 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-19T01:37:47.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:46 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-19T01:37:47.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:46 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:37:48.034 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:47 smithi115 ceph-mon[67332]: from='client.24355 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi115:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:37:48.034 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:47 smithi115 ceph-mon[67332]: pgmap v236: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 332 MiB used, 983 GiB / 983 GiB avail; 2/6 objects degraded (33.333%) 2024-03-19T01:37:48.034 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:47 smithi115 ceph-mon[67332]: osdmap e77: 11 total, 11 up, 11 in 2024-03-19T01:37:48.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:47 smithi099 ceph-mon[62951]: from='client.24355 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi115:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:37:48.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:47 smithi099 ceph-mon[62951]: pgmap v236: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 332 MiB used, 983 GiB / 983 GiB avail; 2/6 objects degraded (33.333%) 2024-03-19T01:37:48.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:47 smithi099 ceph-mon[62951]: osdmap e77: 11 total, 11 up, 11 in 2024-03-19T01:37:48.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:47 smithi112 ceph-mon[66819]: from='client.24355 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi115:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:37:48.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:47 smithi112 ceph-mon[66819]: pgmap v236: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 332 MiB used, 983 GiB / 983 GiB avail; 2/6 objects degraded (33.333%) 2024-03-19T01:37:48.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:47 smithi112 ceph-mon[66819]: osdmap e77: 11 total, 11 up, 11 in 2024-03-19T01:37:49.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:48 smithi115 ceph-mon[67332]: Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2024-03-19T01:37:49.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:48 smithi115 ceph-mon[67332]: osdmap e78: 11 total, 11 up, 11 in 2024-03-19T01:37:49.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:48 smithi099 ceph-mon[62951]: Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2024-03-19T01:37:49.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:48 smithi099 ceph-mon[62951]: osdmap e78: 11 total, 11 up, 11 in 2024-03-19T01:37:49.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:48 smithi112 ceph-mon[66819]: Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2024-03-19T01:37:49.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:48 smithi112 ceph-mon[66819]: osdmap e78: 11 total, 11 up, 11 in 2024-03-19T01:37:50.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:49 smithi115 ceph-mon[67332]: pgmap v239: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 337 MiB used, 983 GiB / 983 GiB avail; 2/6 objects degraded (33.333%) 2024-03-19T01:37:50.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:49 smithi115 ceph-mon[67332]: from='client.? 172.21.15.115:0/3066054248' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fd63b7fd-0db5-46d5-82fa-9707bbd0b495"}]: dispatch 2024-03-19T01:37:50.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:49 smithi115 ceph-mon[67332]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fd63b7fd-0db5-46d5-82fa-9707bbd0b495"}]: dispatch 2024-03-19T01:37:50.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:49 smithi115 ceph-mon[67332]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fd63b7fd-0db5-46d5-82fa-9707bbd0b495"}]': finished 2024-03-19T01:37:50.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:49 smithi115 ceph-mon[67332]: osdmap e79: 12 total, 11 up, 12 in 2024-03-19T01:37:50.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:49 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-03-19T01:37:50.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:49 smithi099 ceph-mon[62951]: pgmap v239: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 337 MiB used, 983 GiB / 983 GiB avail; 2/6 objects degraded (33.333%) 2024-03-19T01:37:50.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:49 smithi099 ceph-mon[62951]: from='client.? 172.21.15.115:0/3066054248' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fd63b7fd-0db5-46d5-82fa-9707bbd0b495"}]: dispatch 2024-03-19T01:37:50.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:49 smithi099 ceph-mon[62951]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fd63b7fd-0db5-46d5-82fa-9707bbd0b495"}]: dispatch 2024-03-19T01:37:50.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:49 smithi099 ceph-mon[62951]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fd63b7fd-0db5-46d5-82fa-9707bbd0b495"}]': finished 2024-03-19T01:37:50.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:49 smithi099 ceph-mon[62951]: osdmap e79: 12 total, 11 up, 12 in 2024-03-19T01:37:50.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:49 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-03-19T01:37:50.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:49 smithi112 ceph-mon[66819]: pgmap v239: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 337 MiB used, 983 GiB / 983 GiB avail; 2/6 objects degraded (33.333%) 2024-03-19T01:37:50.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:49 smithi112 ceph-mon[66819]: from='client.? 172.21.15.115:0/3066054248' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fd63b7fd-0db5-46d5-82fa-9707bbd0b495"}]: dispatch 2024-03-19T01:37:50.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:49 smithi112 ceph-mon[66819]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fd63b7fd-0db5-46d5-82fa-9707bbd0b495"}]: dispatch 2024-03-19T01:37:50.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:49 smithi112 ceph-mon[66819]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fd63b7fd-0db5-46d5-82fa-9707bbd0b495"}]': finished 2024-03-19T01:37:50.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:49 smithi112 ceph-mon[66819]: osdmap e79: 12 total, 11 up, 12 in 2024-03-19T01:37:50.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:49 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-03-19T01:37:51.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:50 smithi115 ceph-mon[67332]: from='client.? 172.21.15.115:0/3506055440' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-19T01:37:51.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:50 smithi099 ceph-mon[62951]: from='client.? 172.21.15.115:0/3506055440' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-19T01:37:51.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:50 smithi112 ceph-mon[66819]: from='client.? 172.21.15.115:0/3506055440' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-19T01:37:52.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:51 smithi115 ceph-mon[67332]: pgmap v241: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 338 MiB used, 983 GiB / 983 GiB avail; 2/6 objects degraded (33.333%) 2024-03-19T01:37:52.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:51 smithi099 ceph-mon[62951]: pgmap v241: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 338 MiB used, 983 GiB / 983 GiB avail; 2/6 objects degraded (33.333%) 2024-03-19T01:37:52.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:51 smithi112 ceph-mon[66819]: pgmap v241: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 338 MiB used, 983 GiB / 983 GiB avail; 2/6 objects degraded (33.333%) 2024-03-19T01:37:53.088 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:52 smithi115 ceph-mon[67332]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded) 2024-03-19T01:37:53.088 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:52 smithi115 ceph-mon[67332]: Cluster is now healthy 2024-03-19T01:37:53.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:52 smithi099 ceph-mon[62951]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded) 2024-03-19T01:37:53.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:52 smithi099 ceph-mon[62951]: Cluster is now healthy 2024-03-19T01:37:53.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:52 smithi112 ceph-mon[66819]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded) 2024-03-19T01:37:53.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:52 smithi112 ceph-mon[66819]: Cluster is now healthy 2024-03-19T01:37:54.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:53 smithi099 ceph-mon[62951]: pgmap v242: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 983 GiB / 983 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-03-19T01:37:54.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:53 smithi112 ceph-mon[66819]: pgmap v242: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 983 GiB / 983 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-03-19T01:37:54.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:53 smithi115 ceph-mon[67332]: pgmap v242: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 983 GiB / 983 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-03-19T01:37:56.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:55 smithi115 ceph-mon[67332]: pgmap v243: 1 pgs: 1 active+clean; 577 KiB data, 337 MiB used, 983 GiB / 983 GiB avail; 76 KiB/s, 0 objects/s recovering 2024-03-19T01:37:56.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:55 smithi099 ceph-mon[62951]: pgmap v243: 1 pgs: 1 active+clean; 577 KiB data, 337 MiB used, 983 GiB / 983 GiB avail; 76 KiB/s, 0 objects/s recovering 2024-03-19T01:37:56.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:55 smithi112 ceph-mon[66819]: pgmap v243: 1 pgs: 1 active+clean; 577 KiB data, 337 MiB used, 983 GiB / 983 GiB avail; 76 KiB/s, 0 objects/s recovering 2024-03-19T01:37:58.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:57 smithi099 ceph-mon[62951]: pgmap v244: 1 pgs: 1 active+clean; 577 KiB data, 337 MiB used, 983 GiB / 983 GiB avail; 67 KiB/s, 0 objects/s recovering 2024-03-19T01:37:58.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:57 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.11"}]: dispatch 2024-03-19T01:37:58.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:57 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:37:58.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:57 smithi112 ceph-mon[66819]: pgmap v244: 1 pgs: 1 active+clean; 577 KiB data, 337 MiB used, 983 GiB / 983 GiB avail; 67 KiB/s, 0 objects/s recovering 2024-03-19T01:37:58.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:57 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.11"}]: dispatch 2024-03-19T01:37:58.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:57 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:37:58.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:57 smithi115 ceph-mon[67332]: pgmap v244: 1 pgs: 1 active+clean; 577 KiB data, 337 MiB used, 983 GiB / 983 GiB avail; 67 KiB/s, 0 objects/s recovering 2024-03-19T01:37:58.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:57 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.11"}]: dispatch 2024-03-19T01:37:58.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:57 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:37:58.891 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:58 smithi115 ceph-mon[67332]: Deploying daemon osd.11 on smithi115 2024-03-19T01:37:59.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:58 smithi112 ceph-mon[66819]: Deploying daemon osd.11 on smithi115 2024-03-19T01:37:59.343 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:58 smithi099 ceph-mon[62951]: Deploying daemon osd.11 on smithi115 2024-03-19T01:38:00.011 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:37:59 smithi115 ceph-mon[67332]: pgmap v245: 1 pgs: 1 active+clean; 577 KiB data, 337 MiB used, 983 GiB / 983 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-03-19T01:38:00.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:37:59 smithi112 ceph-mon[66819]: pgmap v245: 1 pgs: 1 active+clean; 577 KiB data, 337 MiB used, 983 GiB / 983 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-03-19T01:38:00.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:37:59 smithi099 ceph-mon[62951]: pgmap v245: 1 pgs: 1 active+clean; 577 KiB data, 337 MiB used, 983 GiB / 983 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-03-19T01:38:02.036 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:01 smithi115 ceph-mon[67332]: pgmap v246: 1 pgs: 1 active+clean; 577 KiB data, 337 MiB used, 983 GiB / 983 GiB avail; 50 KiB/s, 0 objects/s recovering 2024-03-19T01:38:02.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:01 smithi112 ceph-mon[66819]: pgmap v246: 1 pgs: 1 active+clean; 577 KiB data, 337 MiB used, 983 GiB / 983 GiB avail; 50 KiB/s, 0 objects/s recovering 2024-03-19T01:38:02.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:01 smithi099 ceph-mon[62951]: pgmap v246: 1 pgs: 1 active+clean; 577 KiB data, 337 MiB used, 983 GiB / 983 GiB avail; 50 KiB/s, 0 objects/s recovering 2024-03-19T01:38:03.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:03 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:03.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:03 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:03.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:03 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:38:03.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:03 smithi112 ceph-mon[66819]: pgmap v247: 1 pgs: 1 active+clean; 577 KiB data, 337 MiB used, 983 GiB / 983 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-03-19T01:38:04.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:03 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:04.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:03 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:04.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:03 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:38:04.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:03 smithi115 ceph-mon[67332]: pgmap v247: 1 pgs: 1 active+clean; 577 KiB data, 337 MiB used, 983 GiB / 983 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-03-19T01:38:04.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:03 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:04.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:03 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:04.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:03 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:38:04.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:03 smithi099 ceph-mon[62951]: pgmap v247: 1 pgs: 1 active+clean; 577 KiB data, 337 MiB used, 983 GiB / 983 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-03-19T01:38:06.033 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:05 smithi115 ceph-mon[67332]: pgmap v248: 1 pgs: 1 active+clean; 577 KiB data, 337 MiB used, 983 GiB / 983 GiB avail 2024-03-19T01:38:06.078 INFO:teuthology.orchestra.run.smithi115.stdout:Created osd(s) 11 on host 'smithi115' 2024-03-19T01:38:06.079 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:06.078+0000 7fa507fff700 1 -- 172.21.15.115:0/2830326607 <== mgr.14150 v2:172.21.15.99:6800/4272410295 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+38 (secure 0 0 0) 0x7fa520100bb0 con 0x7fa5080a2890 2024-03-19T01:38:06.080 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:06.080+0000 7fa525078700 1 -- 172.21.15.115:0/2830326607 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa5080a2890 msgr2=0x7fa5080a4d50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:06.080 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:06.080+0000 7fa525078700 1 --2- 172.21.15.115:0/2830326607 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa5080a2890 0x7fa5080a4d50 secure :-1 s=READY pgs=117 cs=0 l=1 rev1=1 crypto rx=0x7fa514005950 tx=0x7fa51400a300 comp rx=0 tx=0).stop 2024-03-19T01:38:06.080 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:06.080+0000 7fa525078700 1 -- 172.21.15.115:0/2830326607 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa520101530 msgr2=0x7fa5201974c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:06.081 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:06.080+0000 7fa525078700 1 --2- 172.21.15.115:0/2830326607 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa520101530 0x7fa5201974c0 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7fa510005b00 tx=0x7fa510009510 comp rx=0 tx=0).stop 2024-03-19T01:38:06.081 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:06.080+0000 7fa525078700 1 -- 172.21.15.115:0/2830326607 shutdown_connections 2024-03-19T01:38:06.081 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:06.080+0000 7fa525078700 1 --2- 172.21.15.115:0/2830326607 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa5080a2890 0x7fa5080a4d50 unknown :-1 s=CLOSED pgs=117 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:06.081 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:06.080+0000 7fa525078700 1 --2- 172.21.15.115:0/2830326607 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa520101530 0x7fa5201974c0 unknown :-1 s=CLOSED pgs=40 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:06.081 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:06.080+0000 7fa525078700 1 --2- 172.21.15.115:0/2830326607 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa5201027f0 0x7fa52019ea80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:06.081 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:06.080+0000 7fa525078700 1 --2- 172.21.15.115:0/2830326607 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa520101ed0 0x7fa520197a00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:06.081 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:06.080+0000 7fa525078700 1 -- 172.21.15.115:0/2830326607 >> 172.21.15.115:0/2830326607 conn(0x7fa5200fbfa0 msgr2=0x7fa5201057e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:06.081 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:06.081+0000 7fa525078700 1 -- 172.21.15.115:0/2830326607 shutdown_connections 2024-03-19T01:38:06.081 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:06.081+0000 7fa525078700 1 -- 172.21.15.115:0/2830326607 wait complete. 2024-03-19T01:38:06.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:05 smithi099 ceph-mon[62951]: pgmap v248: 1 pgs: 1 active+clean; 577 KiB data, 337 MiB used, 983 GiB / 983 GiB avail 2024-03-19T01:38:06.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:05 smithi112 ceph-mon[66819]: pgmap v248: 1 pgs: 1 active+clean; 577 KiB data, 337 MiB used, 983 GiB / 983 GiB avail 2024-03-19T01:38:06.720 DEBUG:teuthology.orchestra.run.smithi115:osd.11> sudo journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@osd.11.service 2024-03-19T01:38:06.722 INFO:tasks.cephadm:Waiting for 12 OSDs to come up... 2024-03-19T01:38:06.723 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph osd stat -f json 2024-03-19T01:38:06.974 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:38:07.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:07 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:07.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:07 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:07.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:07 smithi099 ceph-mon[62951]: from='osd.11 [v2:172.21.15.115:6824/938436244,v1:172.21.15.115:6825/938436244]' entity='osd.11' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["11"]}]: dispatch 2024-03-19T01:38:07.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:07 smithi099 ceph-mon[62951]: from='osd.11 ' entity='osd.11' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["11"]}]: dispatch 2024-03-19T01:38:07.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:07 smithi099 ceph-mon[62951]: pgmap v249: 1 pgs: 1 active+clean; 577 KiB data, 337 MiB used, 983 GiB / 983 GiB avail 2024-03-19T01:38:07.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:07 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:07.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:07 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:07.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:07 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:07.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:07 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:07.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:07 smithi112 ceph-mon[66819]: from='osd.11 [v2:172.21.15.115:6824/938436244,v1:172.21.15.115:6825/938436244]' entity='osd.11' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["11"]}]: dispatch 2024-03-19T01:38:07.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:07 smithi112 ceph-mon[66819]: from='osd.11 ' entity='osd.11' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["11"]}]: dispatch 2024-03-19T01:38:07.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:07 smithi112 ceph-mon[66819]: pgmap v249: 1 pgs: 1 active+clean; 577 KiB data, 337 MiB used, 983 GiB / 983 GiB avail 2024-03-19T01:38:07.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:07 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:07.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:07 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:07.537 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:07 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:07.537 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:07 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:07.537 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:07 smithi115 ceph-mon[67332]: from='osd.11 [v2:172.21.15.115:6824/938436244,v1:172.21.15.115:6825/938436244]' entity='osd.11' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["11"]}]: dispatch 2024-03-19T01:38:07.538 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:07 smithi115 ceph-mon[67332]: from='osd.11 ' entity='osd.11' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["11"]}]: dispatch 2024-03-19T01:38:07.538 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:07 smithi115 ceph-mon[67332]: pgmap v249: 1 pgs: 1 active+clean; 577 KiB data, 337 MiB used, 983 GiB / 983 GiB avail 2024-03-19T01:38:07.538 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:07 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:07.538 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:07 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:08.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:08 smithi099 ceph-mon[62951]: from='osd.11 ' entity='osd.11' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["11"]}]': finished 2024-03-19T01:38:08.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:08 smithi099 ceph-mon[62951]: osdmap e80: 12 total, 11 up, 12 in 2024-03-19T01:38:08.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:08 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-03-19T01:38:08.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:08 smithi099 ceph-mon[62951]: from='osd.11 [v2:172.21.15.115:6824/938436244,v1:172.21.15.115:6825/938436244]' entity='osd.11' cmd=[{"prefix": "osd crush create-or-move", "id": 11, "weight":0.0873, "args": ["host=smithi115", "root=default"]}]: dispatch 2024-03-19T01:38:08.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:08 smithi099 ceph-mon[62951]: from='osd.11 ' entity='osd.11' cmd=[{"prefix": "osd crush create-or-move", "id": 11, "weight":0.0873, "args": ["host=smithi115", "root=default"]}]: dispatch 2024-03-19T01:38:08.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:08 smithi112 ceph-mon[66819]: from='osd.11 ' entity='osd.11' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["11"]}]': finished 2024-03-19T01:38:08.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:08 smithi112 ceph-mon[66819]: osdmap e80: 12 total, 11 up, 12 in 2024-03-19T01:38:08.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:08 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-03-19T01:38:08.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:08 smithi112 ceph-mon[66819]: from='osd.11 [v2:172.21.15.115:6824/938436244,v1:172.21.15.115:6825/938436244]' entity='osd.11' cmd=[{"prefix": "osd crush create-or-move", "id": 11, "weight":0.0873, "args": ["host=smithi115", "root=default"]}]: dispatch 2024-03-19T01:38:08.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:08 smithi112 ceph-mon[66819]: from='osd.11 ' entity='osd.11' cmd=[{"prefix": "osd crush create-or-move", "id": 11, "weight":0.0873, "args": ["host=smithi115", "root=default"]}]: dispatch 2024-03-19T01:38:08.491 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:08 smithi115 ceph-mon[67332]: from='osd.11 ' entity='osd.11' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["11"]}]': finished 2024-03-19T01:38:08.491 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:08 smithi115 ceph-mon[67332]: osdmap e80: 12 total, 11 up, 12 in 2024-03-19T01:38:08.491 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:08 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-03-19T01:38:08.491 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:08 smithi115 ceph-mon[67332]: from='osd.11 [v2:172.21.15.115:6824/938436244,v1:172.21.15.115:6825/938436244]' entity='osd.11' cmd=[{"prefix": "osd crush create-or-move", "id": 11, "weight":0.0873, "args": ["host=smithi115", "root=default"]}]: dispatch 2024-03-19T01:38:08.491 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:08 smithi115 ceph-mon[67332]: from='osd.11 ' entity='osd.11' cmd=[{"prefix": "osd crush create-or-move", "id": 11, "weight":0.0873, "args": ["host=smithi115", "root=default"]}]: dispatch 2024-03-19T01:38:08.492 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 01:38:08 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T01:38:08.088+0000 7fd8c4bc2700 -1 osd.11 0 waiting for initial osdmap 2024-03-19T01:38:08.492 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 01:38:08 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T01:38:08.106+0000 7fd8c09b5700 -1 osd.11 81 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-19T01:38:08.924 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:08.923+0000 7f0c0cadd700 1 -- 172.21.15.99:0/2190615560 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0c08105700 msgr2=0x7f0c08107af0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:08.924 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:08.923+0000 7f0c0cadd700 1 --2- 172.21.15.99:0/2190615560 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0c08105700 0x7f0c08107af0 secure :-1 s=READY pgs=142 cs=0 l=1 rev1=1 crypto rx=0x7f0bfc009990 tx=0x7f0bfc009ca0 comp rx=0 tx=0).stop 2024-03-19T01:38:08.924 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:08.923+0000 7f0c0cadd700 1 -- 172.21.15.99:0/2190615560 shutdown_connections 2024-03-19T01:38:08.924 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:08.923+0000 7f0c0cadd700 1 --2- 172.21.15.99:0/2190615560 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f0c08108030 0x7f0c0810a4a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:08.924 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:08.923+0000 7f0c0cadd700 1 --2- 172.21.15.99:0/2190615560 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f0c08102dd0 0x7f0c081051c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:08.924 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:08.923+0000 7f0c0cadd700 1 --2- 172.21.15.99:0/2190615560 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0c08105700 0x7f0c08107af0 unknown :-1 s=CLOSED pgs=142 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:08.925 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:08.923+0000 7f0c0cadd700 1 -- 172.21.15.99:0/2190615560 >> 172.21.15.99:0/2190615560 conn(0x7f0c080fc800 msgr2=0x7f0c080fec20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:08.925 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:08.923+0000 7f0c0cadd700 1 -- 172.21.15.99:0/2190615560 shutdown_connections 2024-03-19T01:38:08.925 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:08.923+0000 7f0c0cadd700 1 -- 172.21.15.99:0/2190615560 wait complete. 2024-03-19T01:38:08.927 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:08.924+0000 7f0c0cadd700 1 Processor -- start 2024-03-19T01:38:08.927 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:08.924+0000 7f0c0cadd700 1 -- start start 2024-03-19T01:38:08.927 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:08.924+0000 7f0c0cadd700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0c08102dd0 0x7f0c08102730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:08.927 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:08.924+0000 7f0c0cadd700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f0c08105700 0x7f0c08100d80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:08.927 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:08.924+0000 7f0c0cadd700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f0c08108030 0x7f0c081012c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:08.927 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:08.924+0000 7f0c0cadd700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f0c0810df50 con 0x7f0c08102dd0 2024-03-19T01:38:08.928 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:08.924+0000 7f0c0cadd700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f0c0810ddd0 con 0x7f0c08105700 2024-03-19T01:38:08.928 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:08.924+0000 7f0c0cadd700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f0c0810e0d0 con 0x7f0c08108030 2024-03-19T01:38:08.928 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:08.924+0000 7f0c0659c700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0c08102dd0 0x7f0c08102730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:08.928 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:08.924+0000 7f0c0659c700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0c08102dd0 0x7f0c08102730 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:37466/0 (socket says 172.21.15.99:37466) 2024-03-19T01:38:08.928 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:08.924+0000 7f0c0659c700 1 -- 172.21.15.99:0/3618882135 learned_addr learned my addr 172.21.15.99:0/3618882135 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:38:08.928 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:08.924+0000 7f0c06d9d700 1 --2- 172.21.15.99:0/3618882135 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f0c08108030 0x7f0c081012c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:08.928 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:08.924+0000 7f0c05d9b700 1 --2- 172.21.15.99:0/3618882135 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f0c08105700 0x7f0c08100d80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:08.928 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:08.924+0000 7f0c0659c700 1 -- 172.21.15.99:0/3618882135 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f0c08108030 msgr2=0x7f0c081012c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:08.928 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:08.924+0000 7f0c0659c700 1 --2- 172.21.15.99:0/3618882135 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f0c08108030 0x7f0c081012c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:08.928 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:08.924+0000 7f0c0659c700 1 -- 172.21.15.99:0/3618882135 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f0c08105700 msgr2=0x7f0c08100d80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:08.929 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:08.924+0000 7f0c0659c700 1 --2- 172.21.15.99:0/3618882135 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f0c08105700 0x7f0c08100d80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:08.929 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:08.924+0000 7f0c0659c700 1 -- 172.21.15.99:0/3618882135 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f0c08101b70 con 0x7f0c08102dd0 2024-03-19T01:38:08.929 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:08.925+0000 7f0c0659c700 1 --2- 172.21.15.99:0/3618882135 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0c08102dd0 0x7f0c08102730 secure :-1 s=READY pgs=143 cs=0 l=1 rev1=1 crypto rx=0x7f0c000060b0 tx=0x7f0c00007f90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:08.929 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:08.925+0000 7f0bf77fe700 1 -- 172.21.15.99:0/3618882135 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0c0001a070 con 0x7f0c08102dd0 2024-03-19T01:38:08.929 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:08.925+0000 7f0c0cadd700 1 -- 172.21.15.99:0/3618882135 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f0c081a26f0 con 0x7f0c08102dd0 2024-03-19T01:38:08.929 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:08.925+0000 7f0bf77fe700 1 -- 172.21.15.99:0/3618882135 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f0c0000dd60 con 0x7f0c08102dd0 2024-03-19T01:38:08.929 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:08.925+0000 7f0bf77fe700 1 -- 172.21.15.99:0/3618882135 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0c000147c0 con 0x7f0c08102dd0 2024-03-19T01:38:08.929 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:08.925+0000 7f0c0cadd700 1 -- 172.21.15.99:0/3618882135 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f0c081a2c30 con 0x7f0c08102dd0 2024-03-19T01:38:08.929 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:08.926+0000 7f0c0cadd700 1 -- 172.21.15.99:0/3618882135 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f0c08068fd0 con 0x7f0c08102dd0 2024-03-19T01:38:08.929 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:08.927+0000 7f0bf77fe700 1 -- 172.21.15.99:0/3618882135 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 16) ==== 137290+0+0 (secure 0 0 0) 0x7f0c0001d070 con 0x7f0c08102dd0 2024-03-19T01:38:08.930 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:08.927+0000 7f0bf77fe700 1 --2- 172.21.15.99:0/3618882135 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f0bf00a2890 0x7f0bf00a4d50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:08.930 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:08.928+0000 7f0c05d9b700 1 --2- 172.21.15.99:0/3618882135 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f0bf00a2890 0x7f0bf00a4d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:08.930 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:08.928+0000 7f0bf77fe700 1 -- 172.21.15.99:0/3618882135 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 6847+0+0 (secure 0 0 0) 0x7f0c000cdf00 con 0x7f0c08102dd0 2024-03-19T01:38:08.930 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:08.928+0000 7f0c05d9b700 1 --2- 172.21.15.99:0/3618882135 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f0bf00a2890 0x7f0bf00a4d50 secure :-1 s=READY pgs=121 cs=0 l=1 rev1=1 crypto rx=0x7f0bfc00b5c0 tx=0x7f0bfc005fb0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:08.931 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:08.931+0000 7f0bf77fe700 1 -- 172.21.15.99:0/3618882135 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f0c00010070 con 0x7f0c08102dd0 2024-03-19T01:38:09.134 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:09.133+0000 7f0c0cadd700 1 -- 172.21.15.99:0/3618882135 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "osd stat", "format": "json"} v 0) -- 0x7f0c0802dbb0 con 0x7f0c08102dd0 2024-03-19T01:38:09.135 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:09.134+0000 7f0bf77fe700 1 -- 172.21.15.99:0/3618882135 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "osd stat", "format": "json"}]=0 v82) ==== 74+0+133 (secure 0 0 0) 0x7f0c0009b560 con 0x7f0c08102dd0 2024-03-19T01:38:09.135 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:38:09.137 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:09.136+0000 7f0c0cadd700 1 -- 172.21.15.99:0/3618882135 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f0bf00a2890 msgr2=0x7f0bf00a4d50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:09.137 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:09.136+0000 7f0c0cadd700 1 --2- 172.21.15.99:0/3618882135 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f0bf00a2890 0x7f0bf00a4d50 secure :-1 s=READY pgs=121 cs=0 l=1 rev1=1 crypto rx=0x7f0bfc00b5c0 tx=0x7f0bfc005fb0 comp rx=0 tx=0).stop 2024-03-19T01:38:09.137 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:09.136+0000 7f0c0cadd700 1 -- 172.21.15.99:0/3618882135 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0c08102dd0 msgr2=0x7f0c08102730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:09.137 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:09.136+0000 7f0c0cadd700 1 --2- 172.21.15.99:0/3618882135 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0c08102dd0 0x7f0c08102730 secure :-1 s=READY pgs=143 cs=0 l=1 rev1=1 crypto rx=0x7f0c000060b0 tx=0x7f0c00007f90 comp rx=0 tx=0).stop 2024-03-19T01:38:09.137 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:09.136+0000 7f0c0cadd700 1 -- 172.21.15.99:0/3618882135 shutdown_connections 2024-03-19T01:38:09.137 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:09.136+0000 7f0c0cadd700 1 --2- 172.21.15.99:0/3618882135 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f0bf00a2890 0x7f0bf00a4d50 unknown :-1 s=CLOSED pgs=121 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:09.137 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:09.136+0000 7f0c0cadd700 1 --2- 172.21.15.99:0/3618882135 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f0c08108030 0x7f0c081012c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:09.137 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:09.136+0000 7f0c0cadd700 1 --2- 172.21.15.99:0/3618882135 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0c08102dd0 0x7f0c08102730 unknown :-1 s=CLOSED pgs=143 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:09.137 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:09.136+0000 7f0c0cadd700 1 --2- 172.21.15.99:0/3618882135 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f0c08105700 0x7f0c08100d80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:09.138 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:09.136+0000 7f0c0cadd700 1 -- 172.21.15.99:0/3618882135 >> 172.21.15.99:0/3618882135 conn(0x7f0c080fc800 msgr2=0x7f0c08103710 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:09.138 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:09.136+0000 7f0c0cadd700 1 -- 172.21.15.99:0/3618882135 shutdown_connections 2024-03-19T01:38:09.138 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:09.136+0000 7f0c0cadd700 1 -- 172.21.15.99:0/3618882135 wait complete. 2024-03-19T01:38:09.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:09 smithi099 ceph-mon[62951]: from='osd.11 ' entity='osd.11' cmd='[{"prefix": "osd crush create-or-move", "id": 11, "weight":0.0873, "args": ["host=smithi115", "root=default"]}]': finished 2024-03-19T01:38:09.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:09 smithi099 ceph-mon[62951]: osdmap e81: 12 total, 11 up, 12 in 2024-03-19T01:38:09.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:09 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-03-19T01:38:09.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:09 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-03-19T01:38:09.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:09 smithi099 ceph-mon[62951]: pgmap v252: 1 pgs: 1 active+clean; 577 KiB data, 337 MiB used, 983 GiB / 983 GiB avail 2024-03-19T01:38:09.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:09 smithi112 ceph-mon[66819]: from='osd.11 ' entity='osd.11' cmd='[{"prefix": "osd crush create-or-move", "id": 11, "weight":0.0873, "args": ["host=smithi115", "root=default"]}]': finished 2024-03-19T01:38:09.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:09 smithi112 ceph-mon[66819]: osdmap e81: 12 total, 11 up, 12 in 2024-03-19T01:38:09.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:09 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-03-19T01:38:09.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:09 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-03-19T01:38:09.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:09 smithi112 ceph-mon[66819]: pgmap v252: 1 pgs: 1 active+clean; 577 KiB data, 337 MiB used, 983 GiB / 983 GiB avail 2024-03-19T01:38:09.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:09 smithi115 ceph-mon[67332]: from='osd.11 ' entity='osd.11' cmd='[{"prefix": "osd crush create-or-move", "id": 11, "weight":0.0873, "args": ["host=smithi115", "root=default"]}]': finished 2024-03-19T01:38:09.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:09 smithi115 ceph-mon[67332]: osdmap e81: 12 total, 11 up, 12 in 2024-03-19T01:38:09.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:09 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-03-19T01:38:09.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:09 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-03-19T01:38:09.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:09 smithi115 ceph-mon[67332]: pgmap v252: 1 pgs: 1 active+clean; 577 KiB data, 337 MiB used, 983 GiB / 983 GiB avail 2024-03-19T01:38:09.749 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":82,"num_osds":12,"num_up_osds":12,"osd_up_since":1710812289,"num_in_osds":12,"osd_in_since":1710812269,"num_remapped_pgs":0} 2024-03-19T01:38:09.750 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph osd dump --format=json 2024-03-19T01:38:09.993 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:38:10.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:10 smithi099 ceph-mon[62951]: purged_snaps scrub starts 2024-03-19T01:38:10.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:10 smithi099 ceph-mon[62951]: purged_snaps scrub ok 2024-03-19T01:38:10.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:10 smithi099 ceph-mon[62951]: osd.11 [v2:172.21.15.115:6824/938436244,v1:172.21.15.115:6825/938436244] boot 2024-03-19T01:38:10.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:10 smithi099 ceph-mon[62951]: osdmap e82: 12 total, 12 up, 12 in 2024-03-19T01:38:10.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:10 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-03-19T01:38:10.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:10 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/3618882135' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-03-19T01:38:10.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:10 smithi112 ceph-mon[66819]: purged_snaps scrub starts 2024-03-19T01:38:10.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:10 smithi112 ceph-mon[66819]: purged_snaps scrub ok 2024-03-19T01:38:10.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:10 smithi112 ceph-mon[66819]: osd.11 [v2:172.21.15.115:6824/938436244,v1:172.21.15.115:6825/938436244] boot 2024-03-19T01:38:10.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:10 smithi112 ceph-mon[66819]: osdmap e82: 12 total, 12 up, 12 in 2024-03-19T01:38:10.363 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:10 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-03-19T01:38:10.363 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:10 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/3618882135' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-03-19T01:38:10.473 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:10 smithi115 ceph-mon[67332]: purged_snaps scrub starts 2024-03-19T01:38:10.474 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:10 smithi115 ceph-mon[67332]: purged_snaps scrub ok 2024-03-19T01:38:10.474 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:10 smithi115 ceph-mon[67332]: osd.11 [v2:172.21.15.115:6824/938436244,v1:172.21.15.115:6825/938436244] boot 2024-03-19T01:38:10.474 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:10 smithi115 ceph-mon[67332]: osdmap e82: 12 total, 12 up, 12 in 2024-03-19T01:38:10.474 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:10 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-03-19T01:38:10.474 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:10 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/3618882135' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-03-19T01:38:11.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:11 smithi099 ceph-mon[62951]: osdmap e83: 12 total, 12 up, 12 in 2024-03-19T01:38:11.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:11 smithi099 ceph-mon[62951]: Detected new or changed devices on smithi115 2024-03-19T01:38:11.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:11 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:11.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:11 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:11.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:11 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:38:11.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:11 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:38:11.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:11 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:38:11.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:11 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.11", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:38:11.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:11 smithi099 ceph-mon[62951]: Adjusting osd_memory_target on smithi115 to 4270M 2024-03-19T01:38:11.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:11 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:11.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:11 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:38:11.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:11 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:38:11.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:11 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:11.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:11 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:38:11.343 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:11 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:38:11.343 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:11 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:38:11.343 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:11 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:11.343 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:11 smithi099 ceph-mon[62951]: pgmap v255: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:11.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:11 smithi112 ceph-mon[66819]: osdmap e83: 12 total, 12 up, 12 in 2024-03-19T01:38:11.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:11 smithi112 ceph-mon[66819]: Detected new or changed devices on smithi115 2024-03-19T01:38:11.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:11 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:11.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:11 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:11.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:11 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:38:11.363 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:11 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:38:11.363 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:11 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:38:11.363 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:11 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.11", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:38:11.363 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:11 smithi112 ceph-mon[66819]: Adjusting osd_memory_target on smithi115 to 4270M 2024-03-19T01:38:11.363 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:11 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:11.363 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:11 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:38:11.363 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:11 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:38:11.363 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:11 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:11.363 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:11 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:38:11.363 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:11 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:38:11.363 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:11 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:38:11.363 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:11 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:11.363 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:11 smithi112 ceph-mon[66819]: pgmap v255: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:11.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:11 smithi115 ceph-mon[67332]: osdmap e83: 12 total, 12 up, 12 in 2024-03-19T01:38:11.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:11 smithi115 ceph-mon[67332]: Detected new or changed devices on smithi115 2024-03-19T01:38:11.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:11 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:11.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:11 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:11.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:11 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:38:11.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:11 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:38:11.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:11 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:38:11.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:11 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.11", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:38:11.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:11 smithi115 ceph-mon[67332]: Adjusting osd_memory_target on smithi115 to 4270M 2024-03-19T01:38:11.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:11 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:11.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:11 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:38:11.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:11 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:38:11.558 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:11 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:11.558 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:11 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:38:11.558 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:11 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:38:11.558 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:11 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:38:11.558 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:11 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:11.558 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:11 smithi115 ceph-mon[67332]: pgmap v255: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:11.583 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:11.583+0000 7f36bead9700 1 -- 172.21.15.99:0/2716212768 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f36b81076e0 msgr2=0x7f36b810f2f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:11.583 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:11.583+0000 7f36bead9700 1 --2- 172.21.15.99:0/2716212768 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f36b81076e0 0x7f36b810f2f0 secure :-1 s=READY pgs=144 cs=0 l=1 rev1=1 crypto rx=0x7f36b000b0d0 tx=0x7f36b000b3e0 comp rx=0 tx=0).stop 2024-03-19T01:38:11.583 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:11.583+0000 7f36bead9700 1 -- 172.21.15.99:0/2716212768 shutdown_connections 2024-03-19T01:38:11.584 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:11.583+0000 7f36bead9700 1 --2- 172.21.15.99:0/2716212768 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f36b8100f10 0x7f36b81012f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:11.584 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:11.583+0000 7f36bead9700 1 --2- 172.21.15.99:0/2716212768 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f36b81076e0 0x7f36b810f2f0 unknown :-1 s=CLOSED pgs=144 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:11.584 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:11.583+0000 7f36bead9700 1 --2- 172.21.15.99:0/2716212768 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f36b81018c0 0x7f36b8107000 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:11.584 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:11.583+0000 7f36bead9700 1 -- 172.21.15.99:0/2716212768 >> 172.21.15.99:0/2716212768 conn(0x7f36b8076590 msgr2=0x7f36b80769a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:11.584 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:11.583+0000 7f36bead9700 1 -- 172.21.15.99:0/2716212768 shutdown_connections 2024-03-19T01:38:11.584 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:11.583+0000 7f36bead9700 1 -- 172.21.15.99:0/2716212768 wait complete. 2024-03-19T01:38:11.586 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:11.584+0000 7f36bead9700 1 Processor -- start 2024-03-19T01:38:11.586 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:11.584+0000 7f36bead9700 1 -- start start 2024-03-19T01:38:11.587 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:11.584+0000 7f36bead9700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f36b8100f10 0x7f36b819e800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:11.587 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:11.584+0000 7f36bead9700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f36b81018c0 0x7f36b819ed40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:11.587 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:11.584+0000 7f36bead9700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f36b81076e0 0x7f36b81989c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:11.587 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:11.584+0000 7f36bead9700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f36b8112780 con 0x7f36b81018c0 2024-03-19T01:38:11.587 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:11.584+0000 7f36bead9700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f36b8112600 con 0x7f36b81076e0 2024-03-19T01:38:11.587 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:11.584+0000 7f36bead9700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f36b8112900 con 0x7f36b8100f10 2024-03-19T01:38:11.587 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:11.584+0000 7f36affff700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f36b81018c0 0x7f36b819ed40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:11.587 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:11.584+0000 7f36affff700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f36b81018c0 0x7f36b819ed40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:37490/0 (socket says 172.21.15.99:37490) 2024-03-19T01:38:11.587 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:11.584+0000 7f36affff700 1 -- 172.21.15.99:0/2440461681 learned_addr learned my addr 172.21.15.99:0/2440461681 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:38:11.587 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:11.584+0000 7f36bc875700 1 --2- 172.21.15.99:0/2440461681 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f36b8100f10 0x7f36b819e800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:11.587 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:11.585+0000 7f36bd076700 1 --2- 172.21.15.99:0/2440461681 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f36b81076e0 0x7f36b81989c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:11.587 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:11.585+0000 7f36affff700 1 -- 172.21.15.99:0/2440461681 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f36b8100f10 msgr2=0x7f36b819e800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:11.587 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:11.585+0000 7f36affff700 1 --2- 172.21.15.99:0/2440461681 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f36b8100f10 0x7f36b819e800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:11.588 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:11.585+0000 7f36affff700 1 -- 172.21.15.99:0/2440461681 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f36b81076e0 msgr2=0x7f36b81989c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:11.588 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:11.585+0000 7f36affff700 1 --2- 172.21.15.99:0/2440461681 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f36b81076e0 0x7f36b81989c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:11.588 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:11.585+0000 7f36affff700 1 -- 172.21.15.99:0/2440461681 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f36b8199120 con 0x7f36b81018c0 2024-03-19T01:38:11.588 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:11.585+0000 7f36bc875700 1 --2- 172.21.15.99:0/2440461681 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f36b8100f10 0x7f36b819e800 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-19T01:38:11.588 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:11.585+0000 7f36bd076700 1 --2- 172.21.15.99:0/2440461681 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f36b81076e0 0x7f36b81989c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-19T01:38:11.588 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:11.585+0000 7f36affff700 1 --2- 172.21.15.99:0/2440461681 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f36b81018c0 0x7f36b819ed40 secure :-1 s=READY pgs=145 cs=0 l=1 rev1=1 crypto rx=0x7f36a800d950 tx=0x7f36a800dd10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:11.588 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:11.585+0000 7f36adffb700 1 -- 172.21.15.99:0/2440461681 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f36a8016070 con 0x7f36b81018c0 2024-03-19T01:38:11.588 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:11.585+0000 7f36adffb700 1 -- 172.21.15.99:0/2440461681 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f36a800de30 con 0x7f36b81018c0 2024-03-19T01:38:11.588 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:11.585+0000 7f36adffb700 1 -- 172.21.15.99:0/2440461681 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f36a8005420 con 0x7f36b81018c0 2024-03-19T01:38:11.588 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:11.585+0000 7f36bead9700 1 -- 172.21.15.99:0/2440461681 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f36b8199410 con 0x7f36b81018c0 2024-03-19T01:38:11.588 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:11.585+0000 7f36bead9700 1 -- 172.21.15.99:0/2440461681 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f36b81a56d0 con 0x7f36b81018c0 2024-03-19T01:38:11.588 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:11.586+0000 7f36bead9700 1 -- 172.21.15.99:0/2440461681 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f36b8068fd0 con 0x7f36b81018c0 2024-03-19T01:38:11.589 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:11.587+0000 7f36adffb700 1 -- 172.21.15.99:0/2440461681 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 16) ==== 137290+0+0 (secure 0 0 0) 0x7f36a8004750 con 0x7f36b81018c0 2024-03-19T01:38:11.589 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:11.588+0000 7f36adffb700 1 --2- 172.21.15.99:0/2440461681 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f36a00a2910 0x7f36a00a4dd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:11.589 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:11.588+0000 7f36bc875700 1 --2- 172.21.15.99:0/2440461681 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f36a00a2910 0x7f36a00a4dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:11.589 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:11.588+0000 7f36adffb700 1 -- 172.21.15.99:0/2440461681 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(83..83 src has 1..83) ==== 7127+0+0 (secure 0 0 0) 0x7f36a80cde00 con 0x7f36b81018c0 2024-03-19T01:38:11.589 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:11.588+0000 7f36bc875700 1 --2- 172.21.15.99:0/2440461681 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f36a00a2910 0x7f36a00a4dd0 secure :-1 s=READY pgs=124 cs=0 l=1 rev1=1 crypto rx=0x7f36b8199b00 tx=0x7f36b4008040 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:11.593 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:11.592+0000 7f36adffb700 1 -- 172.21.15.99:0/2440461681 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f36a801f070 con 0x7f36b81018c0 2024-03-19T01:38:11.792 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:11.791+0000 7f36bead9700 1 -- 172.21.15.99:0/2440461681 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f36b81a5900 con 0x7f36b81018c0 2024-03-19T01:38:11.793 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:11.792+0000 7f36adffb700 1 -- 172.21.15.99:0/2440461681 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v83) ==== 74+0+18470 (secure 0 0 0) 0x7f36a809b340 con 0x7f36b81018c0 2024-03-19T01:38:11.793 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:38:11.793 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":83,"fsid":"438fb69c-e590-11ee-95c9-87774f69a715","created":"2024-03-19T01:31:10.165073+0000","modified":"2024-03-19T01:38:10.088747+0000","last_up_change":"2024-03-19T01:38:09.082688+0000","last_in_change":"2024-03-19T01:37:49.115793+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-19T01:34:31.001309+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"0c407a0e-a40e-43d7-ad9a-bb8f3cd4a9ba","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6803","nonce":2935260112}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6805","nonce":2935260112}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6808","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6809","nonce":2935260112}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6807","nonce":2935260112}]},"public_addr":"172.21.15.99:6803/2935260112","cluster_addr":"172.21.15.99:6805/2935260112","heartbeat_back_addr":"172.21.15.99:6809/2935260112","heartbeat_front_addr":"172.21.15.99:6807/2935260112","state":["exists","up"]},{"osd":1,"uuid":"a48f9173-4a06-49f5-bca2-2c9047b8f666","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6800","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6801","nonce":1468854289}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6802","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6803","nonce":1468854289}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6806","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6807","nonce":1468854289}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6804","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6805","nonce":1468854289}]},"public_addr":"172.21.15.112:6801/1468854289","cluster_addr":"172.21.15.112:6803/1468854289","heartbeat_back_addr":"172.21.15.112:6807/1468854289","heartbeat_front_addr":"172.21.15.112:6805/1468854289","state":["exists","up"]},{"osd":2,"uuid":"b7715fdd-b6c9-4348-bd16-5eb61b427eef","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6800","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6801","nonce":3766690472}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6802","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6803","nonce":3766690472}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6806","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6807","nonce":3766690472}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6804","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6805","nonce":3766690472}]},"public_addr":"172.21.15.115:6801/3766690472","cluster_addr":"172.21.15.115:6803/3766690472","heartbeat_back_addr":"172.21.15.115:6807/3766690472","heartbeat_front_addr":"172.21.15.115:6805/3766690472","state":["exists","up"]},{"osd":3,"uuid":"6697ed1b-96bf-42de-8440-d17e9785b359","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6810","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6811","nonce":3756732895}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6812","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6813","nonce":3756732895}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6816","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6817","nonce":3756732895}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6814","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6815","nonce":3756732895}]},"public_addr":"172.21.15.99:6811/3756732895","cluster_addr":"172.21.15.99:6813/3756732895","heartbeat_back_addr":"172.21.15.99:6817/3756732895","heartbeat_front_addr":"172.21.15.99:6815/3756732895","state":["exists","up"]},{"osd":4,"uuid":"d958d6b9-67f6-43d0-b704-aa55dc9df83a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":34,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6808","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6809","nonce":3717986391}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6810","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6811","nonce":3717986391}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6814","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6815","nonce":3717986391}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6812","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6813","nonce":3717986391}]},"public_addr":"172.21.15.112:6809/3717986391","cluster_addr":"172.21.15.112:6811/3717986391","heartbeat_back_addr":"172.21.15.112:6815/3717986391","heartbeat_front_addr":"172.21.15.112:6813/3717986391","state":["exists","up"]},{"osd":5,"uuid":"a73eb49a-7925-49ba-bc4f-515e17f8b8ae","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6808","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6809","nonce":1971806408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6810","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6811","nonce":1971806408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6814","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6815","nonce":1971806408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6812","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6813","nonce":1971806408}]},"public_addr":"172.21.15.115:6809/1971806408","cluster_addr":"172.21.15.115:6811/1971806408","heartbeat_back_addr":"172.21.15.115:6815/1971806408","heartbeat_front_addr":"172.21.15.115:6813/1971806408","state":["exists","up"]},{"osd":6,"uuid":"caa718cd-e08b-48de-832a-29546d71e508","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6818","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6819","nonce":1130832772}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6820","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6821","nonce":1130832772}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6824","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6825","nonce":1130832772}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6822","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6823","nonce":1130832772}]},"public_addr":"172.21.15.99:6819/1130832772","cluster_addr":"172.21.15.99:6821/1130832772","heartbeat_back_addr":"172.21.15.99:6825/1130832772","heartbeat_front_addr":"172.21.15.99:6823/1130832772","state":["exists","up"]},{"osd":7,"uuid":"dbee19a1-2315-40d3-88e7-876fcde28903","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":53,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6816","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6817","nonce":3610356576}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6818","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6819","nonce":3610356576}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6822","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6823","nonce":3610356576}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6820","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6821","nonce":3610356576}]},"public_addr":"172.21.15.112:6817/3610356576","cluster_addr":"172.21.15.112:6819/3610356576","heartbeat_back_addr":"172.21.15.112:6823/3610356576","heartbeat_front_addr":"172.21.15.112:6821/3610356576","state":["exists","up"]},{"osd":8,"uuid":"14a866dc-5ce3-42fc-8bcf-9088b68726cb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":60,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6816","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6817","nonce":4255803111}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6818","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6819","nonce":4255803111}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6822","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6823","nonce":4255803111}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6820","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6821","nonce":4255803111}]},"public_addr":"172.21.15.115:6817/4255803111","cluster_addr":"172.21.15.115:6819/4255803111","heartbeat_back_addr":"172.21.15.115:6823/4255803111","heartbeat_front_addr":"172.21.15.115:6821/4255803111","state":["exists","up"]},{"osd":9,"uuid":"1db93652-6035-4906-91ae-d64d5a909502","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":68,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6826","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6827","nonce":1410921523}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6828","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6829","nonce":1410921523}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6832","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6833","nonce":1410921523}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6830","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6831","nonce":1410921523}]},"public_addr":"172.21.15.99:6827/1410921523","cluster_addr":"172.21.15.99:6829/1410921523","heartbeat_back_addr":"172.21.15.99:6833/1410921523","heartbeat_front_addr":"172.21.15.99:6831/1410921523","state":["exists","up"]},{"osd":10,"uuid":"6c50a8a7-3345-4f4a-af0e-5606c7aca453","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":74,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6824","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6825","nonce":891805922}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6826","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6827","nonce":891805922}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6830","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6831","nonce":891805922}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6828","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6829","nonce":891805922}]},"public_addr":"172.21.15.112:6825/891805922","cluster_addr":"172.21.15.112:6827/891805922","heartbeat_back_addr":"172.21.15.112:6831/891805922","heartbeat_front_addr":"172.21.15.112:6829/891805922","state":["exists","up"]},{"osd":11,"uuid":"fd63b7fd-0db5-46d5-82fa-9707bbd0b495","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":82,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6824","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6825","nonce":938436244}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6826","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6827","nonce":938436244}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6830","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6831","nonce":938436244}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6828","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6829","nonce":938436244}]},"public_addr":"172.21.15.115:6825/938436244","cluster_addr":"172.21.15.115:6827/938436244","heartbeat_back_addr":"172.21.15.115:6831/938436244","heartbeat_front_addr":"172.21.15.115:6829/938436244","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:33:39.036822+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:34:03.339566+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:34:27.149273+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:34:51.315561+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:35:15.396031+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:35:38.957168+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:36:03.213914+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:36:27.763699+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:36:51.688670+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:37:17.280165+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:37:42.554633+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:38:07.438039+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.99:6801/1328245934":"2024-03-20T01:32:06.661002+0000","172.21.15.99:0/1618965837":"2024-03-20T01:32:06.661002+0000","172.21.15.99:0/2491429851":"2024-03-20T01:32:06.661002+0000","172.21.15.99:6800/1328245934":"2024-03-20T01:32:06.661002+0000","172.21.15.99:0/2112231766":"2024-03-20T01:31:40.161751+0000","172.21.15.99:0/3341836545":"2024-03-20T01:31:40.161751+0000","172.21.15.99:6800/4223048293":"2024-03-20T01:31:40.161751+0000","172.21.15.99:6801/4223048293":"2024-03-20T01:31:40.161751+0000","172.21.15.99:0/2377621863":"2024-03-20T01:31:40.161751+0000","172.21.15.99:0/1904400935":"2024-03-20T01:32:06.661002+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-19T01:38:11.796 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:11.795+0000 7f36bead9700 1 -- 172.21.15.99:0/2440461681 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f36a00a2910 msgr2=0x7f36a00a4dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:11.796 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:11.795+0000 7f36bead9700 1 --2- 172.21.15.99:0/2440461681 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f36a00a2910 0x7f36a00a4dd0 secure :-1 s=READY pgs=124 cs=0 l=1 rev1=1 crypto rx=0x7f36b8199b00 tx=0x7f36b4008040 comp rx=0 tx=0).stop 2024-03-19T01:38:11.796 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:11.795+0000 7f36bead9700 1 -- 172.21.15.99:0/2440461681 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f36b81018c0 msgr2=0x7f36b819ed40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:11.796 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:11.795+0000 7f36bead9700 1 --2- 172.21.15.99:0/2440461681 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f36b81018c0 0x7f36b819ed40 secure :-1 s=READY pgs=145 cs=0 l=1 rev1=1 crypto rx=0x7f36a800d950 tx=0x7f36a800dd10 comp rx=0 tx=0).stop 2024-03-19T01:38:11.796 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:11.795+0000 7f36bead9700 1 -- 172.21.15.99:0/2440461681 shutdown_connections 2024-03-19T01:38:11.796 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:11.795+0000 7f36bead9700 1 --2- 172.21.15.99:0/2440461681 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f36a00a2910 0x7f36a00a4dd0 unknown :-1 s=CLOSED pgs=124 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:11.796 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:11.795+0000 7f36bead9700 1 --2- 172.21.15.99:0/2440461681 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f36b8100f10 0x7f36b819e800 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:11.796 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:11.795+0000 7f36bead9700 1 --2- 172.21.15.99:0/2440461681 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f36b81076e0 0x7f36b81989c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:11.796 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:11.795+0000 7f36bead9700 1 --2- 172.21.15.99:0/2440461681 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f36b81018c0 0x7f36b819ed40 unknown :-1 s=CLOSED pgs=145 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:11.796 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:11.795+0000 7f36bead9700 1 -- 172.21.15.99:0/2440461681 >> 172.21.15.99:0/2440461681 conn(0x7f36b8076590 msgr2=0x7f36b80fec20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:11.796 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:11.795+0000 7f36bead9700 1 -- 172.21.15.99:0/2440461681 shutdown_connections 2024-03-19T01:38:11.796 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:11.795+0000 7f36bead9700 1 -- 172.21.15.99:0/2440461681 wait complete. 2024-03-19T01:38:12.517 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-19T01:34:31.001309+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 12, 'score_stable': 12, 'optimal_score': 0.25, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-19T01:38:12.517 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph osd pool get .mgr pg_num 2024-03-19T01:38:12.758 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:38:12.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:12 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/2440461681' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-19T01:38:13.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:12 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/2440461681' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-19T01:38:13.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:12 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/2440461681' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-19T01:38:13.985 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:13 smithi099 ceph-mon[62951]: pgmap v256: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:14.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:13 smithi115 ceph-mon[67332]: pgmap v256: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:14.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:13 smithi112 ceph-mon[66819]: pgmap v256: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:14.353 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:14.351+0000 7fcfcdaba700 1 -- 172.21.15.99:0/2354132715 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fcfc806a960 msgr2=0x7fcfc810f1a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:14.354 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:14.351+0000 7fcfcdaba700 1 --2- 172.21.15.99:0/2354132715 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fcfc806a960 0x7fcfc810f1a0 secure :-1 s=READY pgs=146 cs=0 l=1 rev1=1 crypto rx=0x7fcfc000b0d0 tx=0x7fcfc000b3e0 comp rx=0 tx=0).stop 2024-03-19T01:38:14.354 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:14.351+0000 7fcfcdaba700 1 -- 172.21.15.99:0/2354132715 shutdown_connections 2024-03-19T01:38:14.354 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:14.351+0000 7fcfcdaba700 1 --2- 172.21.15.99:0/2354132715 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fcfc8108360 0x7fcfc8108740 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:14.354 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:14.351+0000 7fcfcdaba700 1 --2- 172.21.15.99:0/2354132715 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fcfc806a960 0x7fcfc810f1a0 unknown :-1 s=CLOSED pgs=146 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:14.354 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:14.351+0000 7fcfcdaba700 1 --2- 172.21.15.99:0/2354132715 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fcfc8069fc0 0x7fcfc806a420 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:14.354 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:14.351+0000 7fcfcdaba700 1 -- 172.21.15.99:0/2354132715 >> 172.21.15.99:0/2354132715 conn(0x7fcfc8078140 msgr2=0x7fcfc8078550 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:14.354 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:14.351+0000 7fcfcdaba700 1 -- 172.21.15.99:0/2354132715 shutdown_connections 2024-03-19T01:38:14.354 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:14.351+0000 7fcfcdaba700 1 -- 172.21.15.99:0/2354132715 wait complete. 2024-03-19T01:38:14.354 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:14.352+0000 7fcfcdaba700 1 Processor -- start 2024-03-19T01:38:14.354 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:14.352+0000 7fcfcdaba700 1 -- start start 2024-03-19T01:38:14.354 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:14.352+0000 7fcfcdaba700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fcfc8069fc0 0x7fcfc819e620 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:14.354 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:14.352+0000 7fcfcdaba700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fcfc806a960 0x7fcfc819eb60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:14.355 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:14.352+0000 7fcfcdaba700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fcfc8108360 0x7fcfc81987a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:14.355 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:14.352+0000 7fcfcdaba700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fcfc8112620 con 0x7fcfc8069fc0 2024-03-19T01:38:14.355 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:14.352+0000 7fcfcdaba700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fcfc81124a0 con 0x7fcfc8108360 2024-03-19T01:38:14.355 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:14.352+0000 7fcfcdaba700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fcfc81127a0 con 0x7fcfc806a960 2024-03-19T01:38:14.355 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:14.352+0000 7fcfc77fe700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fcfc8069fc0 0x7fcfc819e620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:14.355 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:14.352+0000 7fcfc77fe700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fcfc8069fc0 0x7fcfc819e620 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:32942/0 (socket says 172.21.15.99:32942) 2024-03-19T01:38:14.355 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:14.352+0000 7fcfc77fe700 1 -- 172.21.15.99:0/158508771 learned_addr learned my addr 172.21.15.99:0/158508771 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:38:14.355 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:14.352+0000 7fcfc6ffd700 1 --2- 172.21.15.99:0/158508771 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fcfc806a960 0x7fcfc819eb60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:14.355 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:14.352+0000 7fcfc7fff700 1 --2- 172.21.15.99:0/158508771 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fcfc8108360 0x7fcfc81987a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:14.355 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:14.352+0000 7fcfc77fe700 1 -- 172.21.15.99:0/158508771 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fcfc806a960 msgr2=0x7fcfc819eb60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:14.355 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:14.352+0000 7fcfc77fe700 1 --2- 172.21.15.99:0/158508771 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fcfc806a960 0x7fcfc819eb60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:14.355 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:14.352+0000 7fcfc77fe700 1 -- 172.21.15.99:0/158508771 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fcfc8108360 msgr2=0x7fcfc81987a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:14.355 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:14.352+0000 7fcfc77fe700 1 --2- 172.21.15.99:0/158508771 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fcfc8108360 0x7fcfc81987a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:14.355 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:14.352+0000 7fcfc77fe700 1 -- 172.21.15.99:0/158508771 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fcfc8198f80 con 0x7fcfc8069fc0 2024-03-19T01:38:14.356 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:14.352+0000 7fcfc7fff700 1 --2- 172.21.15.99:0/158508771 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fcfc8108360 0x7fcfc81987a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-19T01:38:14.356 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:14.353+0000 7fcfc77fe700 1 --2- 172.21.15.99:0/158508771 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fcfc8069fc0 0x7fcfc819e620 secure :-1 s=READY pgs=147 cs=0 l=1 rev1=1 crypto rx=0x7fcfb800b530 tx=0x7fcfb800b840 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:14.356 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:14.353+0000 7fcfc4ff9700 1 -- 172.21.15.99:0/158508771 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fcfb8015020 con 0x7fcfc8069fc0 2024-03-19T01:38:14.356 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:14.353+0000 7fcfcdaba700 1 -- 172.21.15.99:0/158508771 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fcfc8199270 con 0x7fcfc8069fc0 2024-03-19T01:38:14.356 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:14.353+0000 7fcfc4ff9700 1 -- 172.21.15.99:0/158508771 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fcfb8011d10 con 0x7fcfc8069fc0 2024-03-19T01:38:14.356 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:14.353+0000 7fcfc4ff9700 1 -- 172.21.15.99:0/158508771 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fcfb800f2e0 con 0x7fcfc8069fc0 2024-03-19T01:38:14.356 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:14.353+0000 7fcfcdaba700 1 -- 172.21.15.99:0/158508771 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fcfc81a5480 con 0x7fcfc8069fc0 2024-03-19T01:38:14.357 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:14.354+0000 7fcfcdaba700 1 -- 172.21.15.99:0/158508771 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fcfc8103a10 con 0x7fcfc8069fc0 2024-03-19T01:38:14.357 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:14.355+0000 7fcfc4ff9700 1 -- 172.21.15.99:0/158508771 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 16) ==== 137290+0+0 (secure 0 0 0) 0x7fcfb8018020 con 0x7fcfc8069fc0 2024-03-19T01:38:14.357 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:14.356+0000 7fcfc4ff9700 1 --2- 172.21.15.99:0/158508771 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fcfb00a2890 0x7fcfb00a4d50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:14.357 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:14.356+0000 7fcfc6ffd700 1 --2- 172.21.15.99:0/158508771 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fcfb00a2890 0x7fcfb00a4d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:14.357 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:14.356+0000 7fcfc4ff9700 1 -- 172.21.15.99:0/158508771 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(83..83 src has 1..83) ==== 7127+0+0 (secure 0 0 0) 0x7fcfb80cdec0 con 0x7fcfc8069fc0 2024-03-19T01:38:14.357 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:14.356+0000 7fcfc6ffd700 1 --2- 172.21.15.99:0/158508771 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fcfb00a2890 0x7fcfb00a4d50 secure :-1 s=READY pgs=125 cs=0 l=1 rev1=1 crypto rx=0x7fcfc80785f0 tx=0x7fcfbc00a380 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:14.360 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:14.360+0000 7fcfc4ff9700 1 -- 172.21.15.99:0/158508771 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7fcfb801f020 con 0x7fcfc8069fc0 2024-03-19T01:38:14.559 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:14.558+0000 7fcfcdaba700 1 -- 172.21.15.99:0/158508771 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) -- 0x7fcfc81a5770 con 0x7fcfc8069fc0 2024-03-19T01:38:14.559 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:14.559+0000 7fcfc4ff9700 1 -- 172.21.15.99:0/158508771 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v83) ==== 93+0+10 (secure 0 0 0) 0x7fcfb809b400 con 0x7fcfc8069fc0 2024-03-19T01:38:14.559 INFO:teuthology.orchestra.run.smithi099.stdout:pg_num: 1 2024-03-19T01:38:14.561 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:14.561+0000 7fcfcdaba700 1 -- 172.21.15.99:0/158508771 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fcfb00a2890 msgr2=0x7fcfb00a4d50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:14.561 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:14.561+0000 7fcfcdaba700 1 --2- 172.21.15.99:0/158508771 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fcfb00a2890 0x7fcfb00a4d50 secure :-1 s=READY pgs=125 cs=0 l=1 rev1=1 crypto rx=0x7fcfc80785f0 tx=0x7fcfbc00a380 comp rx=0 tx=0).stop 2024-03-19T01:38:14.561 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:14.561+0000 7fcfcdaba700 1 -- 172.21.15.99:0/158508771 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fcfc8069fc0 msgr2=0x7fcfc819e620 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:14.561 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:14.561+0000 7fcfcdaba700 1 --2- 172.21.15.99:0/158508771 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fcfc8069fc0 0x7fcfc819e620 secure :-1 s=READY pgs=147 cs=0 l=1 rev1=1 crypto rx=0x7fcfb800b530 tx=0x7fcfb800b840 comp rx=0 tx=0).stop 2024-03-19T01:38:14.561 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:14.561+0000 7fcfcdaba700 1 -- 172.21.15.99:0/158508771 shutdown_connections 2024-03-19T01:38:14.562 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:14.561+0000 7fcfcdaba700 1 --2- 172.21.15.99:0/158508771 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fcfb00a2890 0x7fcfb00a4d50 unknown :-1 s=CLOSED pgs=125 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:14.562 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:14.561+0000 7fcfcdaba700 1 --2- 172.21.15.99:0/158508771 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fcfc8108360 0x7fcfc81987a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:14.562 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:14.561+0000 7fcfcdaba700 1 --2- 172.21.15.99:0/158508771 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fcfc8069fc0 0x7fcfc819e620 unknown :-1 s=CLOSED pgs=147 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:14.562 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:14.561+0000 7fcfcdaba700 1 --2- 172.21.15.99:0/158508771 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fcfc806a960 0x7fcfc819eb60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:14.562 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:14.561+0000 7fcfcdaba700 1 -- 172.21.15.99:0/158508771 >> 172.21.15.99:0/158508771 conn(0x7fcfc8078140 msgr2=0x7fcfc80ffb90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:14.562 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:14.561+0000 7fcfcdaba700 1 -- 172.21.15.99:0/158508771 shutdown_connections 2024-03-19T01:38:14.562 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:14.561+0000 7fcfcdaba700 1 -- 172.21.15.99:0/158508771 wait complete. 2024-03-19T01:38:14.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:14 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/158508771' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-03-19T01:38:15.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:14 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/158508771' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-03-19T01:38:15.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:14 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/158508771' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-03-19T01:38:15.246 INFO:tasks.cephadm:Adding mds.a on smithi099 2024-03-19T01:38:15.246 INFO:tasks.cephadm:Adding mds.d on smithi099 2024-03-19T01:38:15.246 INFO:tasks.cephadm:Adding mds.g on smithi099 2024-03-19T01:38:15.247 INFO:tasks.cephadm:Adding mds.j on smithi099 2024-03-19T01:38:15.247 INFO:tasks.cephadm:Adding mds.b on smithi112 2024-03-19T01:38:15.247 INFO:tasks.cephadm:Adding mds.e on smithi112 2024-03-19T01:38:15.247 INFO:tasks.cephadm:Adding mds.h on smithi112 2024-03-19T01:38:15.247 INFO:tasks.cephadm:Adding mds.k on smithi112 2024-03-19T01:38:15.247 INFO:tasks.cephadm:Adding mds.c on smithi115 2024-03-19T01:38:15.247 INFO:tasks.cephadm:Adding mds.f on smithi115 2024-03-19T01:38:15.247 INFO:tasks.cephadm:Adding mds.i on smithi115 2024-03-19T01:38:15.247 INFO:tasks.cephadm:Adding mds.l on smithi115 2024-03-19T01:38:15.247 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph orch apply mds all '12;smithi099=a;smithi099=d;smithi099=g;smithi099=j;smithi112=b;smithi112=e;smithi112=h;smithi112=k;smithi115=c;smithi115=f;smithi115=i;smithi115=l' 2024-03-19T01:38:15.497 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.c/config 2024-03-19T01:38:15.959 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:15 smithi115 ceph-mon[67332]: pgmap v257: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:16.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:15 smithi099 ceph-mon[62951]: pgmap v257: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:16.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:15 smithi112 ceph-mon[66819]: pgmap v257: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:17.383 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:17.364+0000 7faa0f434700 1 -- 172.21.15.115:0/721466630 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7faa08102430 msgr2=0x7faa08102890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:17.383 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:17.364+0000 7faa0f434700 1 --2- 172.21.15.115:0/721466630 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7faa08102430 0x7faa08102890 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7fa9f8009990 tx=0x7fa9f8009ca0 comp rx=0 tx=0).stop 2024-03-19T01:38:17.384 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:17.364+0000 7faa0f434700 1 -- 172.21.15.115:0/721466630 shutdown_connections 2024-03-19T01:38:17.384 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:17.364+0000 7faa0f434700 1 --2- 172.21.15.115:0/721466630 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7faa08102dd0 0x7faa08109680 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:17.384 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:17.364+0000 7faa0f434700 1 --2- 172.21.15.115:0/721466630 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7faa08101230 0x7faa08101650 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:17.384 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:17.364+0000 7faa0f434700 1 --2- 172.21.15.115:0/721466630 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7faa08102430 0x7faa08102890 unknown :-1 s=CLOSED pgs=48 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:17.384 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:17.364+0000 7faa0f434700 1 -- 172.21.15.115:0/721466630 >> 172.21.15.115:0/721466630 conn(0x7faa080fc7d0 msgr2=0x7faa080fec10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:17.384 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:17.364+0000 7faa0f434700 1 -- 172.21.15.115:0/721466630 shutdown_connections 2024-03-19T01:38:17.384 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:17.364+0000 7faa0f434700 1 -- 172.21.15.115:0/721466630 wait complete. 2024-03-19T01:38:17.384 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:17.364+0000 7faa0f434700 1 Processor -- start 2024-03-19T01:38:17.384 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:17.364+0000 7faa0f434700 1 -- start start 2024-03-19T01:38:17.384 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:17.365+0000 7faa0f434700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7faa08101230 0x7faa0806b360 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:17.384 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:17.365+0000 7faa0f434700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7faa08102430 0x7faa0819d440 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:17.384 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:17.365+0000 7faa0f434700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7faa08102dd0 0x7faa0819fa10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:17.384 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:17.365+0000 7faa0f434700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7faa0810cf10 con 0x7faa08102dd0 2024-03-19T01:38:17.384 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:17.365+0000 7faa0f434700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7faa0810cd90 con 0x7faa08102430 2024-03-19T01:38:17.385 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:17.365+0000 7faa0f434700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7faa0810d090 con 0x7faa08101230 2024-03-19T01:38:17.385 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:17.365+0000 7faa0d1d0700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7faa08101230 0x7faa0806b360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:17.385 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:17.365+0000 7faa0d1d0700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7faa08101230 0x7faa0806b360 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.115:35676/0 (socket says 172.21.15.115:35676) 2024-03-19T01:38:17.385 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:17.365+0000 7faa0d1d0700 1 -- 172.21.15.115:0/4038306274 learned_addr learned my addr 172.21.15.115:0/4038306274 (peer_addr_for_me v2:172.21.15.115:0/0) 2024-03-19T01:38:17.385 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:17.365+0000 7faa0c9cf700 1 --2- 172.21.15.115:0/4038306274 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7faa08102430 0x7faa0819d440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:17.385 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:17.365+0000 7faa0d9d1700 1 --2- 172.21.15.115:0/4038306274 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7faa08102dd0 0x7faa0819fa10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:17.385 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:17.365+0000 7faa0d1d0700 1 -- 172.21.15.115:0/4038306274 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7faa08102430 msgr2=0x7faa0819d440 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:17.385 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:17.365+0000 7faa0d1d0700 1 --2- 172.21.15.115:0/4038306274 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7faa08102430 0x7faa0819d440 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:17.385 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:17.365+0000 7faa0d1d0700 1 -- 172.21.15.115:0/4038306274 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7faa08102dd0 msgr2=0x7faa0819fa10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:17.385 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:17.365+0000 7faa0d1d0700 1 --2- 172.21.15.115:0/4038306274 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7faa08102dd0 0x7faa0819fa10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:17.385 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:17.365+0000 7faa0d1d0700 1 -- 172.21.15.115:0/4038306274 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7faa081a02e0 con 0x7faa08101230 2024-03-19T01:38:17.385 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:17.365+0000 7faa0d1d0700 1 --2- 172.21.15.115:0/4038306274 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7faa08101230 0x7faa0806b360 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7faa0400b500 tx=0x7faa0400b8c0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:17.385 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:17.365+0000 7fa9fe7fc700 1 -- 172.21.15.115:0/4038306274 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7faa04015020 con 0x7faa08101230 2024-03-19T01:38:17.385 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:17.365+0000 7faa0f434700 1 -- 172.21.15.115:0/4038306274 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7faa081a05d0 con 0x7faa08101230 2024-03-19T01:38:17.386 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:17.365+0000 7fa9fe7fc700 1 -- 172.21.15.115:0/4038306274 <== mon.1 v2:172.21.15.115:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7faa04011d10 con 0x7faa08101230 2024-03-19T01:38:17.386 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:17.365+0000 7fa9fe7fc700 1 -- 172.21.15.115:0/4038306274 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7faa0400f2e0 con 0x7faa08101230 2024-03-19T01:38:17.386 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:17.365+0000 7faa0f434700 1 -- 172.21.15.115:0/4038306274 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7faa081a0b40 con 0x7faa08101230 2024-03-19T01:38:17.386 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:17.366+0000 7faa0f434700 1 -- 172.21.15.115:0/4038306274 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7faa08068fd0 con 0x7faa08101230 2024-03-19T01:38:17.386 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:17.367+0000 7fa9fe7fc700 1 -- 172.21.15.115:0/4038306274 <== mon.1 v2:172.21.15.115:3300/0 4 ==== mgrmap(e 16) ==== 137290+0+0 (secure 0 0 0) 0x7faa04018020 con 0x7faa08101230 2024-03-19T01:38:17.386 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:17.368+0000 7fa9fe7fc700 1 --2- 172.21.15.115:0/4038306274 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa9f40a2890 0x7fa9f40a4d50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:17.386 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:17.368+0000 7fa9fe7fc700 1 -- 172.21.15.115:0/4038306274 <== mon.1 v2:172.21.15.115:3300/0 5 ==== osd_map(83..83 src has 1..83) ==== 7127+0+0 (secure 0 0 0) 0x7faa040cdeb0 con 0x7faa08101230 2024-03-19T01:38:17.386 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:17.368+0000 7faa0c9cf700 1 --2- 172.21.15.115:0/4038306274 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa9f40a2890 0x7fa9f40a4d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:17.386 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:17.369+0000 7faa0c9cf700 1 --2- 172.21.15.115:0/4038306274 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa9f40a2890 0x7fa9f40a4d50 secure :-1 s=READY pgs=126 cs=0 l=1 rev1=1 crypto rx=0x7fa9f800b5c0 tx=0x7fa9f8005fb0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:17.386 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:17.373+0000 7fa9fe7fc700 1 -- 172.21.15.115:0/4038306274 <== mon.1 v2:172.21.15.115:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7faa0401f020 con 0x7faa08101230 2024-03-19T01:38:17.602 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:17.601+0000 7faa0f434700 1 -- 172.21.15.115:0/4038306274 --> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] -- mgr_command(tid 0: {"prefix": "orch apply mds", "fs_name": "all", "placement": "12;smithi099=a;smithi099=d;smithi099=g;smithi099=j;smithi112=b;smithi112=e;smithi112=h;smithi112=k;smithi115=c;smithi115=f;smithi115=i;smithi115=l", "target": ["mon-mgr", ""]}) -- 0x7faa08062950 con 0x7fa9f40a2890 2024-03-19T01:38:17.610 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:17.609+0000 7fa9fe7fc700 1 -- 172.21.15.115:0/4038306274 <== mgr.14150 v2:172.21.15.99:6800/4272410295 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+28 (secure 0 0 0) 0x7faa08062950 con 0x7fa9f40a2890 2024-03-19T01:38:17.610 INFO:teuthology.orchestra.run.smithi115.stdout:Scheduled mds.all update... 2024-03-19T01:38:17.612 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:17.611+0000 7faa0f434700 1 -- 172.21.15.115:0/4038306274 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa9f40a2890 msgr2=0x7fa9f40a4d50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:17.612 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:17.611+0000 7faa0f434700 1 --2- 172.21.15.115:0/4038306274 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa9f40a2890 0x7fa9f40a4d50 secure :-1 s=READY pgs=126 cs=0 l=1 rev1=1 crypto rx=0x7fa9f800b5c0 tx=0x7fa9f8005fb0 comp rx=0 tx=0).stop 2024-03-19T01:38:17.612 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:17.611+0000 7faa0f434700 1 -- 172.21.15.115:0/4038306274 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7faa08101230 msgr2=0x7faa0806b360 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:17.613 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:17.611+0000 7faa0f434700 1 --2- 172.21.15.115:0/4038306274 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7faa08101230 0x7faa0806b360 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7faa0400b500 tx=0x7faa0400b8c0 comp rx=0 tx=0).stop 2024-03-19T01:38:17.613 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:17.611+0000 7faa0f434700 1 -- 172.21.15.115:0/4038306274 shutdown_connections 2024-03-19T01:38:17.613 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:17.611+0000 7faa0f434700 1 --2- 172.21.15.115:0/4038306274 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa9f40a2890 0x7fa9f40a4d50 unknown :-1 s=CLOSED pgs=126 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:17.613 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:17.611+0000 7faa0f434700 1 --2- 172.21.15.115:0/4038306274 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7faa08101230 0x7faa0806b360 unknown :-1 s=CLOSED pgs=49 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:17.613 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:17.611+0000 7faa0f434700 1 --2- 172.21.15.115:0/4038306274 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7faa08102dd0 0x7faa0819fa10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:17.613 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:17.611+0000 7faa0f434700 1 --2- 172.21.15.115:0/4038306274 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7faa08102430 0x7faa0819d440 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:17.613 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:17.611+0000 7faa0f434700 1 -- 172.21.15.115:0/4038306274 >> 172.21.15.115:0/4038306274 conn(0x7faa080fc7d0 msgr2=0x7faa08105750 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:17.613 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:17.611+0000 7faa0f434700 1 -- 172.21.15.115:0/4038306274 shutdown_connections 2024-03-19T01:38:17.613 INFO:teuthology.orchestra.run.smithi115.stderr:2024-03-19T01:38:17.611+0000 7faa0f434700 1 -- 172.21.15.115:0/4038306274 wait complete. 2024-03-19T01:38:17.936 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:17 smithi115 ceph-mon[67332]: pgmap v258: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:17.936 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:17 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:17.936 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:17 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:38:18.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:17 smithi099 ceph-mon[62951]: pgmap v258: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:18.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:17 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:18.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:17 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:38:18.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:17 smithi112 ceph-mon[66819]: pgmap v258: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:18.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:17 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:18.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:17 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:38:18.250 DEBUG:teuthology.orchestra.run.smithi099:mds.a> sudo journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@mds.a.service 2024-03-19T01:38:18.252 DEBUG:teuthology.orchestra.run.smithi099:mds.d> sudo journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@mds.d.service 2024-03-19T01:38:18.295 DEBUG:teuthology.orchestra.run.smithi099:mds.g> sudo journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@mds.g.service 2024-03-19T01:38:18.338 DEBUG:teuthology.orchestra.run.smithi099:mds.j> sudo journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@mds.j.service 2024-03-19T01:38:18.381 DEBUG:teuthology.orchestra.run.smithi112:mds.b> sudo journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@mds.b.service 2024-03-19T01:38:18.387 DEBUG:teuthology.orchestra.run.smithi112:mds.e> sudo journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@mds.e.service 2024-03-19T01:38:18.431 DEBUG:teuthology.orchestra.run.smithi112:mds.h> sudo journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@mds.h.service 2024-03-19T01:38:18.473 DEBUG:teuthology.orchestra.run.smithi112:mds.k> sudo journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@mds.k.service 2024-03-19T01:38:18.517 DEBUG:teuthology.orchestra.run.smithi115:mds.c> sudo journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@mds.c.service 2024-03-19T01:38:18.520 DEBUG:teuthology.orchestra.run.smithi115:mds.f> sudo journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@mds.f.service 2024-03-19T01:38:18.562 DEBUG:teuthology.orchestra.run.smithi115:mds.i> sudo journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@mds.i.service 2024-03-19T01:38:18.605 DEBUG:teuthology.orchestra.run.smithi115:mds.l> sudo journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@mds.l.service 2024-03-19T01:38:18.648 INFO:tasks.cephadm:Setting up CephFS filesystem(s)... 2024-03-19T01:38:18.649 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-19T01:38:18.766 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:18.762+0000 7f77c0b53640 1 -- 172.21.15.99:0/3450448093 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f77bc100740 msgr2=0x7f77bc100b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:18.766 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:18.762+0000 7f77c0b53640 1 --2- 172.21.15.99:0/3450448093 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f77bc100740 0x7f77bc100b60 secure :-1 s=READY pgs=148 cs=0 l=1 rev1=1 crypto rx=0x7f77b0009900 tx=0x7f77b002bc30 comp rx=0 tx=0).stop 2024-03-19T01:38:18.766 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:18.762+0000 7f77c0b53640 1 -- 172.21.15.99:0/3450448093 shutdown_connections 2024-03-19T01:38:18.766 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:18.762+0000 7f77c0b53640 1 --2- 172.21.15.99:0/3450448093 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f77bc1022f0 0x7f77bc108b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:18.766 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:18.762+0000 7f77c0b53640 1 --2- 172.21.15.99:0/3450448093 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f77bc101930 0x7f77bc101db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:18.766 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:18.762+0000 7f77c0b53640 1 --2- 172.21.15.99:0/3450448093 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f77bc100740 0x7f77bc100b60 unknown :-1 s=CLOSED pgs=148 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:18.766 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:18.762+0000 7f77c0b53640 1 -- 172.21.15.99:0/3450448093 >> 172.21.15.99:0/3450448093 conn(0x7f77bc0fbcb0 msgr2=0x7f77bc0fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:18.766 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:18.762+0000 7f77c0b53640 1 -- 172.21.15.99:0/3450448093 shutdown_connections 2024-03-19T01:38:18.766 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:18.762+0000 7f77c0b53640 1 -- 172.21.15.99:0/3450448093 wait complete. 2024-03-19T01:38:18.766 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:18.763+0000 7f77c0b53640 1 Processor -- start 2024-03-19T01:38:18.766 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:18.763+0000 7f77c0b53640 1 -- start start 2024-03-19T01:38:18.767 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:18.763+0000 7f77c0b53640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f77bc100740 0x7f77bc10b0c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:18.767 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:18.763+0000 7f77c0b53640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f77bc101930 0x7f77bc109700 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:18.767 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:18.763+0000 7f77c0b53640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f77bc1022f0 0x7f77bc109c40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:18.767 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:18.763+0000 7f77c0b53640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f77bc10d4e0 con 0x7f77bc1022f0 2024-03-19T01:38:18.767 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:18.763+0000 7f77c0b53640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f77bc10d360 con 0x7f77bc101930 2024-03-19T01:38:18.767 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:18.763+0000 7f77c0b53640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f77bc10d660 con 0x7f77bc100740 2024-03-19T01:38:18.767 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:18.763+0000 7f77bad9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f77bc1022f0 0x7f77bc109c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:18.767 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:18.763+0000 7f77ba59c640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f77bc100740 0x7f77bc10b0c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:18.767 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:18.763+0000 7f77bad9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f77bc1022f0 0x7f77bc109c40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:32954/0 (socket says 172.21.15.99:32954) 2024-03-19T01:38:18.767 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:18.763+0000 7f77bad9d640 1 -- 172.21.15.99:0/3113912413 learned_addr learned my addr 172.21.15.99:0/3113912413 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:38:18.767 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:18.763+0000 7f77b9d9b640 1 --2- 172.21.15.99:0/3113912413 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f77bc101930 0x7f77bc109700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:18.767 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:18.763+0000 7f77bad9d640 1 -- 172.21.15.99:0/3113912413 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f77bc100740 msgr2=0x7f77bc10b0c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:18.767 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:18.763+0000 7f77bad9d640 1 --2- 172.21.15.99:0/3113912413 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f77bc100740 0x7f77bc10b0c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:18.768 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:18.763+0000 7f77bad9d640 1 -- 172.21.15.99:0/3113912413 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f77bc101930 msgr2=0x7f77bc109700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:18.768 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:18.763+0000 7f77bad9d640 1 --2- 172.21.15.99:0/3113912413 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f77bc101930 0x7f77bc109700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:18.768 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:18.763+0000 7f77bad9d640 1 -- 172.21.15.99:0/3113912413 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f77bc10a4d0 con 0x7f77bc1022f0 2024-03-19T01:38:18.768 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:18.763+0000 7f77bad9d640 1 --2- 172.21.15.99:0/3113912413 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f77bc1022f0 0x7f77bc109c40 secure :-1 s=READY pgs=149 cs=0 l=1 rev1=1 crypto rx=0x7f77b400cad0 tx=0x7f77b400cf90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:18.768 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:18.764+0000 7f77ab7fe640 1 -- 172.21.15.99:0/3113912413 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f77b400ae20 con 0x7f77bc1022f0 2024-03-19T01:38:18.768 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:18.764+0000 7f77ab7fe640 1 -- 172.21.15.99:0/3113912413 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f77b4005020 con 0x7f77bc1022f0 2024-03-19T01:38:18.768 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:18.764+0000 7f77ab7fe640 1 -- 172.21.15.99:0/3113912413 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f77b400f6f0 con 0x7f77bc1022f0 2024-03-19T01:38:18.768 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:18.764+0000 7f77c0b53640 1 -- 172.21.15.99:0/3113912413 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f77bc1a1ce0 con 0x7f77bc1022f0 2024-03-19T01:38:18.768 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:18.764+0000 7f77c0b53640 1 -- 172.21.15.99:0/3113912413 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f77bc1a2180 con 0x7f77bc1022f0 2024-03-19T01:38:18.768 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:18.765+0000 7f77c0b53640 1 -- 172.21.15.99:0/3113912413 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f77bc107ae0 con 0x7f77bc1022f0 2024-03-19T01:38:18.769 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:18.766+0000 7f77ab7fe640 1 -- 172.21.15.99:0/3113912413 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 16) ==== 137290+0+0 (secure 0 0 0) 0x7f77b40051d0 con 0x7f77bc1022f0 2024-03-19T01:38:18.769 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:18.766+0000 7f77ab7fe640 1 --2- 172.21.15.99:0/3113912413 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f77a40a2880 0x7f77a40a4d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:18.769 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:18.766+0000 7f77ba59c640 1 --2- 172.21.15.99:0/3113912413 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f77a40a2880 0x7f77a40a4d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:18.769 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:18.766+0000 7f77ab7fe640 1 -- 172.21.15.99:0/3113912413 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(83..83 src has 1..83) ==== 7127+0+0 (secure 0 0 0) 0x7f77b40ce650 con 0x7f77bc1022f0 2024-03-19T01:38:18.769 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:18.767+0000 7f77ba59c640 1 --2- 172.21.15.99:0/3113912413 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f77a40a2880 0x7f77a40a4d40 secure :-1 s=READY pgs=127 cs=0 l=1 rev1=1 crypto rx=0x7f77b0000c00 tx=0x7f77b0005fd0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:18.772 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:18.771+0000 7f77ab7fe640 1 -- 172.21.15.99:0/3113912413 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f77b4014030 con 0x7f77bc1022f0 2024-03-19T01:38:18.806 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:18 smithi115 ceph-mon[67332]: from='client.24394 -' entity='client.admin' cmd=[{"prefix": "orch apply mds", "fs_name": "all", "placement": "12;smithi099=a;smithi099=d;smithi099=g;smithi099=j;smithi112=b;smithi112=e;smithi112=h;smithi112=k;smithi115=c;smithi115=f;smithi115=i;smithi115=l", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:38:18.806 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:18 smithi115 ceph-mon[67332]: Saving service mds.all spec with placement smithi099=a;smithi099=d;smithi099=g;smithi099=j;smithi112=b;smithi112=e;smithi112=h;smithi112=k;smithi115=c;smithi115=f;smithi115=i;smithi115=l;count:12 2024-03-19T01:38:18.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:18 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:38:18.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:18 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:38:18.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:18 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:18.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:18 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.l", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-19T01:38:18.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:18 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.l", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-19T01:38:18.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:18 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:38:18.952 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:18.951+0000 7f77c0b53640 1 -- 172.21.15.99:0/3113912413 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f77bc02d410 con 0x7f77bc1022f0 2024-03-19T01:38:18.953 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:18.952+0000 7f77ab7fe640 1 -- 172.21.15.99:0/3113912413 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v83) ==== 74+0+18470 (secure 0 0 0) 0x7f77b409bb90 con 0x7f77bc1022f0 2024-03-19T01:38:18.953 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:38:18.953 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":83,"fsid":"438fb69c-e590-11ee-95c9-87774f69a715","created":"2024-03-19T01:31:10.165073+0000","modified":"2024-03-19T01:38:10.088747+0000","last_up_change":"2024-03-19T01:38:09.082688+0000","last_in_change":"2024-03-19T01:37:49.115793+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-19T01:34:31.001309+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"0c407a0e-a40e-43d7-ad9a-bb8f3cd4a9ba","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6803","nonce":2935260112}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6805","nonce":2935260112}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6808","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6809","nonce":2935260112}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6807","nonce":2935260112}]},"public_addr":"172.21.15.99:6803/2935260112","cluster_addr":"172.21.15.99:6805/2935260112","heartbeat_back_addr":"172.21.15.99:6809/2935260112","heartbeat_front_addr":"172.21.15.99:6807/2935260112","state":["exists","up"]},{"osd":1,"uuid":"a48f9173-4a06-49f5-bca2-2c9047b8f666","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6800","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6801","nonce":1468854289}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6802","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6803","nonce":1468854289}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6806","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6807","nonce":1468854289}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6804","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6805","nonce":1468854289}]},"public_addr":"172.21.15.112:6801/1468854289","cluster_addr":"172.21.15.112:6803/1468854289","heartbeat_back_addr":"172.21.15.112:6807/1468854289","heartbeat_front_addr":"172.21.15.112:6805/1468854289","state":["exists","up"]},{"osd":2,"uuid":"b7715fdd-b6c9-4348-bd16-5eb61b427eef","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6800","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6801","nonce":3766690472}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6802","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6803","nonce":3766690472}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6806","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6807","nonce":3766690472}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6804","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6805","nonce":3766690472}]},"public_addr":"172.21.15.115:6801/3766690472","cluster_addr":"172.21.15.115:6803/3766690472","heartbeat_back_addr":"172.21.15.115:6807/3766690472","heartbeat_front_addr":"172.21.15.115:6805/3766690472","state":["exists","up"]},{"osd":3,"uuid":"6697ed1b-96bf-42de-8440-d17e9785b359","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6810","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6811","nonce":3756732895}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6812","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6813","nonce":3756732895}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6816","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6817","nonce":3756732895}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6814","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6815","nonce":3756732895}]},"public_addr":"172.21.15.99:6811/3756732895","cluster_addr":"172.21.15.99:6813/3756732895","heartbeat_back_addr":"172.21.15.99:6817/3756732895","heartbeat_front_addr":"172.21.15.99:6815/3756732895","state":["exists","up"]},{"osd":4,"uuid":"d958d6b9-67f6-43d0-b704-aa55dc9df83a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":34,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6808","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6809","nonce":3717986391}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6810","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6811","nonce":3717986391}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6814","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6815","nonce":3717986391}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6812","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6813","nonce":3717986391}]},"public_addr":"172.21.15.112:6809/3717986391","cluster_addr":"172.21.15.112:6811/3717986391","heartbeat_back_addr":"172.21.15.112:6815/3717986391","heartbeat_front_addr":"172.21.15.112:6813/3717986391","state":["exists","up"]},{"osd":5,"uuid":"a73eb49a-7925-49ba-bc4f-515e17f8b8ae","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6808","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6809","nonce":1971806408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6810","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6811","nonce":1971806408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6814","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6815","nonce":1971806408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6812","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6813","nonce":1971806408}]},"public_addr":"172.21.15.115:6809/1971806408","cluster_addr":"172.21.15.115:6811/1971806408","heartbeat_back_addr":"172.21.15.115:6815/1971806408","heartbeat_front_addr":"172.21.15.115:6813/1971806408","state":["exists","up"]},{"osd":6,"uuid":"caa718cd-e08b-48de-832a-29546d71e508","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6818","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6819","nonce":1130832772}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6820","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6821","nonce":1130832772}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6824","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6825","nonce":1130832772}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6822","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6823","nonce":1130832772}]},"public_addr":"172.21.15.99:6819/1130832772","cluster_addr":"172.21.15.99:6821/1130832772","heartbeat_back_addr":"172.21.15.99:6825/1130832772","heartbeat_front_addr":"172.21.15.99:6823/1130832772","state":["exists","up"]},{"osd":7,"uuid":"dbee19a1-2315-40d3-88e7-876fcde28903","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":53,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6816","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6817","nonce":3610356576}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6818","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6819","nonce":3610356576}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6822","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6823","nonce":3610356576}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6820","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6821","nonce":3610356576}]},"public_addr":"172.21.15.112:6817/3610356576","cluster_addr":"172.21.15.112:6819/3610356576","heartbeat_back_addr":"172.21.15.112:6823/3610356576","heartbeat_front_addr":"172.21.15.112:6821/3610356576","state":["exists","up"]},{"osd":8,"uuid":"14a866dc-5ce3-42fc-8bcf-9088b68726cb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":60,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6816","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6817","nonce":4255803111}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6818","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6819","nonce":4255803111}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6822","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6823","nonce":4255803111}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6820","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6821","nonce":4255803111}]},"public_addr":"172.21.15.115:6817/4255803111","cluster_addr":"172.21.15.115:6819/4255803111","heartbeat_back_addr":"172.21.15.115:6823/4255803111","heartbeat_front_addr":"172.21.15.115:6821/4255803111","state":["exists","up"]},{"osd":9,"uuid":"1db93652-6035-4906-91ae-d64d5a909502","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":68,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6826","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6827","nonce":1410921523}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6828","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6829","nonce":1410921523}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6832","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6833","nonce":1410921523}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6830","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6831","nonce":1410921523}]},"public_addr":"172.21.15.99:6827/1410921523","cluster_addr":"172.21.15.99:6829/1410921523","heartbeat_back_addr":"172.21.15.99:6833/1410921523","heartbeat_front_addr":"172.21.15.99:6831/1410921523","state":["exists","up"]},{"osd":10,"uuid":"6c50a8a7-3345-4f4a-af0e-5606c7aca453","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":74,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6824","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6825","nonce":891805922}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6826","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6827","nonce":891805922}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6830","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6831","nonce":891805922}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6828","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6829","nonce":891805922}]},"public_addr":"172.21.15.112:6825/891805922","cluster_addr":"172.21.15.112:6827/891805922","heartbeat_back_addr":"172.21.15.112:6831/891805922","heartbeat_front_addr":"172.21.15.112:6829/891805922","state":["exists","up"]},{"osd":11,"uuid":"fd63b7fd-0db5-46d5-82fa-9707bbd0b495","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":82,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6824","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6825","nonce":938436244}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6826","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6827","nonce":938436244}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6830","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6831","nonce":938436244}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6828","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6829","nonce":938436244}]},"public_addr":"172.21.15.115:6825/938436244","cluster_addr":"172.21.15.115:6827/938436244","heartbeat_back_addr":"172.21.15.115:6831/938436244","heartbeat_front_addr":"172.21.15.115:6829/938436244","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:33:39.036822+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:34:03.339566+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:34:27.149273+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:34:51.315561+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:35:15.396031+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:35:38.957168+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:36:03.213914+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:36:27.763699+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:36:51.688670+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:37:17.280165+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:37:42.554633+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:38:07.438039+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.99:6801/1328245934":"2024-03-20T01:32:06.661002+0000","172.21.15.99:0/1618965837":"2024-03-20T01:32:06.661002+0000","172.21.15.99:0/2491429851":"2024-03-20T01:32:06.661002+0000","172.21.15.99:6800/1328245934":"2024-03-20T01:32:06.661002+0000","172.21.15.99:0/2112231766":"2024-03-20T01:31:40.161751+0000","172.21.15.99:0/3341836545":"2024-03-20T01:31:40.161751+0000","172.21.15.99:6800/4223048293":"2024-03-20T01:31:40.161751+0000","172.21.15.99:6801/4223048293":"2024-03-20T01:31:40.161751+0000","172.21.15.99:0/2377621863":"2024-03-20T01:31:40.161751+0000","172.21.15.99:0/1904400935":"2024-03-20T01:32:06.661002+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-19T01:38:18.955 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:18.954+0000 7f77c0b53640 1 -- 172.21.15.99:0/3113912413 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f77a40a2880 msgr2=0x7f77a40a4d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:18.955 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:18.954+0000 7f77c0b53640 1 --2- 172.21.15.99:0/3113912413 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f77a40a2880 0x7f77a40a4d40 secure :-1 s=READY pgs=127 cs=0 l=1 rev1=1 crypto rx=0x7f77b0000c00 tx=0x7f77b0005fd0 comp rx=0 tx=0).stop 2024-03-19T01:38:18.955 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:18.954+0000 7f77c0b53640 1 -- 172.21.15.99:0/3113912413 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f77bc1022f0 msgr2=0x7f77bc109c40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:18.955 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:18.954+0000 7f77c0b53640 1 --2- 172.21.15.99:0/3113912413 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f77bc1022f0 0x7f77bc109c40 secure :-1 s=READY pgs=149 cs=0 l=1 rev1=1 crypto rx=0x7f77b400cad0 tx=0x7f77b400cf90 comp rx=0 tx=0).stop 2024-03-19T01:38:18.955 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:18.954+0000 7f77c0b53640 1 -- 172.21.15.99:0/3113912413 shutdown_connections 2024-03-19T01:38:18.956 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:18.954+0000 7f77c0b53640 1 --2- 172.21.15.99:0/3113912413 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f77a40a2880 0x7f77a40a4d40 unknown :-1 s=CLOSED pgs=127 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:18.956 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:18.954+0000 7f77c0b53640 1 --2- 172.21.15.99:0/3113912413 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f77bc101930 0x7f77bc109700 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:18.956 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:18.954+0000 7f77c0b53640 1 --2- 172.21.15.99:0/3113912413 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f77bc1022f0 0x7f77bc109c40 unknown :-1 s=CLOSED pgs=149 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:18.956 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:18.954+0000 7f77c0b53640 1 --2- 172.21.15.99:0/3113912413 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f77bc100740 0x7f77bc10b0c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:18.956 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:18.954+0000 7f77c0b53640 1 -- 172.21.15.99:0/3113912413 >> 172.21.15.99:0/3113912413 conn(0x7f77bc0fbcb0 msgr2=0x7f77bc104cb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:18.956 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:18.954+0000 7f77c0b53640 1 -- 172.21.15.99:0/3113912413 shutdown_connections 2024-03-19T01:38:18.956 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:18.954+0000 7f77c0b53640 1 -- 172.21.15.99:0/3113912413 wait complete. 2024-03-19T01:38:18.965 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:18 smithi099 ceph-mon[62951]: from='client.24394 -' entity='client.admin' cmd=[{"prefix": "orch apply mds", "fs_name": "all", "placement": "12;smithi099=a;smithi099=d;smithi099=g;smithi099=j;smithi112=b;smithi112=e;smithi112=h;smithi112=k;smithi115=c;smithi115=f;smithi115=i;smithi115=l", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:38:18.965 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:18 smithi099 ceph-mon[62951]: Saving service mds.all spec with placement smithi099=a;smithi099=d;smithi099=g;smithi099=j;smithi112=b;smithi112=e;smithi112=h;smithi112=k;smithi115=c;smithi115=f;smithi115=i;smithi115=l;count:12 2024-03-19T01:38:18.965 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:18 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:38:18.965 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:18 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:38:18.965 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:18 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:18.965 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:18 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.l", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-19T01:38:18.965 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:18 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.l", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-19T01:38:18.965 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:18 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:38:18.967 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-19T01:34:31.001309+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 12, 'score_stable': 12, 'optimal_score': 0.25, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-19T01:38:18.967 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-19T01:38:19.073 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.070+0000 7f5ea1d3b640 1 -- 172.21.15.99:0/280553880 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5e9c100720 msgr2=0x7f5e9c100b40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:19.073 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.070+0000 7f5ea1d3b640 1 --2- 172.21.15.99:0/280553880 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5e9c100720 0x7f5e9c100b40 secure :-1 s=READY pgs=150 cs=0 l=1 rev1=1 crypto rx=0x7f5e8c009900 tx=0x7f5e8c02bc30 comp rx=0 tx=0).stop 2024-03-19T01:38:19.073 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.070+0000 7f5ea1d3b640 1 -- 172.21.15.99:0/280553880 shutdown_connections 2024-03-19T01:38:19.074 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.070+0000 7f5ea1d3b640 1 --2- 172.21.15.99:0/280553880 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5e9c1022d0 0x7f5e9c108b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:19.074 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.070+0000 7f5ea1d3b640 1 --2- 172.21.15.99:0/280553880 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5e9c101910 0x7f5e9c101d90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:19.074 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.070+0000 7f5ea1d3b640 1 --2- 172.21.15.99:0/280553880 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5e9c100720 0x7f5e9c100b40 unknown :-1 s=CLOSED pgs=150 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:19.074 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.070+0000 7f5ea1d3b640 1 -- 172.21.15.99:0/280553880 >> 172.21.15.99:0/280553880 conn(0x7f5e9c0fbcb0 msgr2=0x7f5e9c0fe0d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:19.074 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.070+0000 7f5ea1d3b640 1 -- 172.21.15.99:0/280553880 shutdown_connections 2024-03-19T01:38:19.074 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.070+0000 7f5ea1d3b640 1 -- 172.21.15.99:0/280553880 wait complete. 2024-03-19T01:38:19.074 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.070+0000 7f5ea1d3b640 1 Processor -- start 2024-03-19T01:38:19.074 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.070+0000 7f5ea1d3b640 1 -- start start 2024-03-19T01:38:19.074 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.071+0000 7f5ea1d3b640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5e9c100720 0x7f5e9c1046b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:19.074 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.071+0000 7f5ea1d3b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5e9c101910 0x7f5e9c102cf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:19.074 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.071+0000 7f5ea1d3b640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5e9c1022d0 0x7f5e9c103230 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:19.074 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.071+0000 7f5ea1d3b640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f5e9c10c390 con 0x7f5e9c101910 2024-03-19T01:38:19.074 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.071+0000 7f5ea1d3b640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f5e9c10c210 con 0x7f5e9c1022d0 2024-03-19T01:38:19.075 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.071+0000 7f5ea1d3b640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f5e9c10c510 con 0x7f5e9c100720 2024-03-19T01:38:19.075 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.071+0000 7f5e92ffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5e9c101910 0x7f5e9c102cf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:19.075 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.071+0000 7f5e9b7fe640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5e9c100720 0x7f5e9c1046b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:19.075 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.071+0000 7f5e92ffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5e9c101910 0x7f5e9c102cf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:32966/0 (socket says 172.21.15.99:32966) 2024-03-19T01:38:19.075 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.071+0000 7f5e92ffd640 1 -- 172.21.15.99:0/2340797535 learned_addr learned my addr 172.21.15.99:0/2340797535 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:38:19.075 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.071+0000 7f5e9bfff640 1 --2- 172.21.15.99:0/2340797535 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5e9c1022d0 0x7f5e9c103230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:19.075 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.071+0000 7f5e92ffd640 1 -- 172.21.15.99:0/2340797535 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5e9c100720 msgr2=0x7f5e9c1046b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:19.075 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.071+0000 7f5e92ffd640 1 --2- 172.21.15.99:0/2340797535 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5e9c100720 0x7f5e9c1046b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:19.075 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.071+0000 7f5e92ffd640 1 -- 172.21.15.99:0/2340797535 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5e9c1022d0 msgr2=0x7f5e9c103230 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:19.075 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.071+0000 7f5e92ffd640 1 --2- 172.21.15.99:0/2340797535 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5e9c1022d0 0x7f5e9c103230 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:19.075 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.071+0000 7f5e92ffd640 1 -- 172.21.15.99:0/2340797535 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f5e9c103ac0 con 0x7f5e9c101910 2024-03-19T01:38:19.075 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.071+0000 7f5e92ffd640 1 --2- 172.21.15.99:0/2340797535 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5e9c101910 0x7f5e9c102cf0 secure :-1 s=READY pgs=151 cs=0 l=1 rev1=1 crypto rx=0x7f5e8400d770 tx=0x7f5e8400dc30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:19.075 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.071+0000 7f5e997fa640 1 -- 172.21.15.99:0/2340797535 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5e84017070 con 0x7f5e9c101910 2024-03-19T01:38:19.075 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.071+0000 7f5e997fa640 1 -- 172.21.15.99:0/2340797535 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f5e84005500 con 0x7f5e9c101910 2024-03-19T01:38:19.076 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.071+0000 7f5e997fa640 1 -- 172.21.15.99:0/2340797535 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5e84004e00 con 0x7f5e9c101910 2024-03-19T01:38:19.076 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.071+0000 7f5ea1d3b640 1 -- 172.21.15.99:0/2340797535 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f5e9c1a1cd0 con 0x7f5e9c101910 2024-03-19T01:38:19.076 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.072+0000 7f5ea1d3b640 1 -- 172.21.15.99:0/2340797535 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f5e9c1a21d0 con 0x7f5e9c101910 2024-03-19T01:38:19.076 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.072+0000 7f5ea1d3b640 1 -- 172.21.15.99:0/2340797535 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f5e9c1a2400 con 0x7f5e9c101910 2024-03-19T01:38:19.077 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.073+0000 7f5e997fa640 1 -- 172.21.15.99:0/2340797535 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 16) ==== 137290+0+0 (secure 0 0 0) 0x7f5e84005000 con 0x7f5e9c101910 2024-03-19T01:38:19.077 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.074+0000 7f5e997fa640 1 --2- 172.21.15.99:0/2340797535 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f5e880a2880 0x7f5e880a4d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:19.077 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.074+0000 7f5e9b7fe640 1 --2- 172.21.15.99:0/2340797535 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f5e880a2880 0x7f5e880a4d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:19.077 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.074+0000 7f5e997fa640 1 -- 172.21.15.99:0/2340797535 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(83..83 src has 1..83) ==== 7127+0+0 (secure 0 0 0) 0x7f5e840cded0 con 0x7f5e9c101910 2024-03-19T01:38:19.077 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.074+0000 7f5e9b7fe640 1 --2- 172.21.15.99:0/2340797535 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f5e880a2880 0x7f5e880a4d40 secure :-1 s=READY pgs=128 cs=0 l=1 rev1=1 crypto rx=0x7f5e8c000c00 tx=0x7f5e8c005fd0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:19.080 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.078+0000 7f5e997fa640 1 -- 172.21.15.99:0/2340797535 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f5e8409adc0 con 0x7f5e9c101910 2024-03-19T01:38:19.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:18 smithi112 ceph-mon[66819]: from='client.24394 -' entity='client.admin' cmd=[{"prefix": "orch apply mds", "fs_name": "all", "placement": "12;smithi099=a;smithi099=d;smithi099=g;smithi099=j;smithi112=b;smithi112=e;smithi112=h;smithi112=k;smithi115=c;smithi115=f;smithi115=i;smithi115=l", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:38:19.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:18 smithi112 ceph-mon[66819]: Saving service mds.all spec with placement smithi099=a;smithi099=d;smithi099=g;smithi099=j;smithi112=b;smithi112=e;smithi112=h;smithi112=k;smithi115=c;smithi115=f;smithi115=i;smithi115=l;count:12 2024-03-19T01:38:19.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:18 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:38:19.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:18 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:38:19.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:18 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:19.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:18 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.l", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-19T01:38:19.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:18 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.l", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-19T01:38:19.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:18 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:38:19.254 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.252+0000 7f5ea1d3b640 1 -- 172.21.15.99:0/2340797535 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) -- 0x7f5e9c1a26f0 con 0x7f5e9c101910 2024-03-19T01:38:19.254 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.253+0000 7f5e997fa640 1 -- 172.21.15.99:0/2340797535 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v83) ==== 93+0+10 (secure 0 0 0) 0x7f5e8409ee60 con 0x7f5e9c101910 2024-03-19T01:38:19.254 INFO:teuthology.orchestra.run.smithi099.stdout:pg_num: 1 2024-03-19T01:38:19.256 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.255+0000 7f5ea1d3b640 1 -- 172.21.15.99:0/2340797535 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f5e880a2880 msgr2=0x7f5e880a4d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:19.256 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.255+0000 7f5ea1d3b640 1 --2- 172.21.15.99:0/2340797535 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f5e880a2880 0x7f5e880a4d40 secure :-1 s=READY pgs=128 cs=0 l=1 rev1=1 crypto rx=0x7f5e8c000c00 tx=0x7f5e8c005fd0 comp rx=0 tx=0).stop 2024-03-19T01:38:19.256 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.255+0000 7f5ea1d3b640 1 -- 172.21.15.99:0/2340797535 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5e9c101910 msgr2=0x7f5e9c102cf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:19.256 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.255+0000 7f5ea1d3b640 1 --2- 172.21.15.99:0/2340797535 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5e9c101910 0x7f5e9c102cf0 secure :-1 s=READY pgs=151 cs=0 l=1 rev1=1 crypto rx=0x7f5e8400d770 tx=0x7f5e8400dc30 comp rx=0 tx=0).stop 2024-03-19T01:38:19.256 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.255+0000 7f5ea1d3b640 1 -- 172.21.15.99:0/2340797535 shutdown_connections 2024-03-19T01:38:19.257 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.255+0000 7f5ea1d3b640 1 --2- 172.21.15.99:0/2340797535 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f5e880a2880 0x7f5e880a4d40 unknown :-1 s=CLOSED pgs=128 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:19.257 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.255+0000 7f5ea1d3b640 1 --2- 172.21.15.99:0/2340797535 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5e9c1022d0 0x7f5e9c103230 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:19.257 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.255+0000 7f5ea1d3b640 1 --2- 172.21.15.99:0/2340797535 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5e9c101910 0x7f5e9c102cf0 unknown :-1 s=CLOSED pgs=151 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:19.257 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.255+0000 7f5ea1d3b640 1 --2- 172.21.15.99:0/2340797535 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5e9c100720 0x7f5e9c1046b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:19.257 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.255+0000 7f5ea1d3b640 1 -- 172.21.15.99:0/2340797535 >> 172.21.15.99:0/2340797535 conn(0x7f5e9c0fbcb0 msgr2=0x7f5e9c0fd7b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:19.257 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.255+0000 7f5ea1d3b640 1 -- 172.21.15.99:0/2340797535 shutdown_connections 2024-03-19T01:38:19.257 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.255+0000 7f5ea1d3b640 1 -- 172.21.15.99:0/2340797535 wait complete. 2024-03-19T01:38:19.267 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-19T01:38:19.370 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.368+0000 7fa05e427640 1 -- 172.21.15.99:0/3148082129 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa058100740 msgr2=0x7fa058100b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:19.370 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.368+0000 7fa05e427640 1 --2- 172.21.15.99:0/3148082129 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa058100740 0x7fa058100b60 secure :-1 s=READY pgs=152 cs=0 l=1 rev1=1 crypto rx=0x7fa048009900 tx=0x7fa04802bc30 comp rx=0 tx=0).stop 2024-03-19T01:38:19.370 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.368+0000 7fa05e427640 1 -- 172.21.15.99:0/3148082129 shutdown_connections 2024-03-19T01:38:19.370 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.368+0000 7fa05e427640 1 --2- 172.21.15.99:0/3148082129 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa058101930 0x7fa058101db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:19.370 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.368+0000 7fa05e427640 1 --2- 172.21.15.99:0/3148082129 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa0581022f0 0x7fa058108b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:19.370 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.368+0000 7fa05e427640 1 --2- 172.21.15.99:0/3148082129 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa058100740 0x7fa058100b60 unknown :-1 s=CLOSED pgs=152 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:19.370 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.368+0000 7fa05e427640 1 -- 172.21.15.99:0/3148082129 >> 172.21.15.99:0/3148082129 conn(0x7fa0580fbcb0 msgr2=0x7fa0580fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:19.370 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.368+0000 7fa05e427640 1 -- 172.21.15.99:0/3148082129 shutdown_connections 2024-03-19T01:38:19.370 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.368+0000 7fa05e427640 1 -- 172.21.15.99:0/3148082129 wait complete. 2024-03-19T01:38:19.370 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.368+0000 7fa05e427640 1 Processor -- start 2024-03-19T01:38:19.371 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.368+0000 7fa05e427640 1 -- start start 2024-03-19T01:38:19.373 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.369+0000 7fa05e427640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa058100740 0x7fa058196a00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:19.373 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.369+0000 7fa05e427640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa058101930 0x7fa058196f40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:19.373 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.369+0000 7fa05e427640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa0581022f0 0x7fa05819dfb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:19.373 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.369+0000 7fa05e427640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fa05810c3c0 con 0x7fa058101930 2024-03-19T01:38:19.373 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.369+0000 7fa05e427640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fa05810c240 con 0x7fa058100740 2024-03-19T01:38:19.373 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.369+0000 7fa05e427640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fa05810c540 con 0x7fa0581022f0 2024-03-19T01:38:19.373 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.369+0000 7fa0577fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa058101930 0x7fa058196f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:19.373 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.369+0000 7fa0577fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa058101930 0x7fa058196f40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:32984/0 (socket says 172.21.15.99:32984) 2024-03-19T01:38:19.373 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.369+0000 7fa0577fe640 1 -- 172.21.15.99:0/1768276552 learned_addr learned my addr 172.21.15.99:0/1768276552 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:38:19.374 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.369+0000 7fa057fff640 1 --2- 172.21.15.99:0/1768276552 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa058100740 0x7fa058196a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:19.374 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.369+0000 7fa05c9c4640 1 --2- 172.21.15.99:0/1768276552 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa0581022f0 0x7fa05819dfb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:19.374 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.369+0000 7fa0577fe640 1 -- 172.21.15.99:0/1768276552 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa0581022f0 msgr2=0x7fa05819dfb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:19.374 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.369+0000 7fa0577fe640 1 --2- 172.21.15.99:0/1768276552 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa0581022f0 0x7fa05819dfb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:19.374 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.369+0000 7fa0577fe640 1 -- 172.21.15.99:0/1768276552 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa058100740 msgr2=0x7fa058196a00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:19.374 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.369+0000 7fa0577fe640 1 --2- 172.21.15.99:0/1768276552 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa058100740 0x7fa058196a00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:19.374 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.369+0000 7fa0577fe640 1 -- 172.21.15.99:0/1768276552 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa05819e620 con 0x7fa058101930 2024-03-19T01:38:19.374 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.369+0000 7fa05c9c4640 1 --2- 172.21.15.99:0/1768276552 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa0581022f0 0x7fa05819dfb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-19T01:38:19.374 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.369+0000 7fa0577fe640 1 --2- 172.21.15.99:0/1768276552 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa058101930 0x7fa058196f40 secure :-1 s=READY pgs=153 cs=0 l=1 rev1=1 crypto rx=0x7fa04c007b50 tx=0x7fa04c00e420 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:19.374 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.369+0000 7fa0557fa640 1 -- 172.21.15.99:0/1768276552 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa04c007c40 con 0x7fa058101930 2024-03-19T01:38:19.374 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.370+0000 7fa0557fa640 1 -- 172.21.15.99:0/1768276552 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa04c007de0 con 0x7fa058101930 2024-03-19T01:38:19.374 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.370+0000 7fa0557fa640 1 -- 172.21.15.99:0/1768276552 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa04c00ae60 con 0x7fa058101930 2024-03-19T01:38:19.374 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.370+0000 7fa05e427640 1 -- 172.21.15.99:0/1768276552 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa058069ef0 con 0x7fa058101930 2024-03-19T01:38:19.374 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.370+0000 7fa05e427640 1 -- 172.21.15.99:0/1768276552 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fa05806a440 con 0x7fa058101930 2024-03-19T01:38:19.375 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.371+0000 7fa05e427640 1 -- 172.21.15.99:0/1768276552 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fa058068a60 con 0x7fa058101930 2024-03-19T01:38:19.375 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.371+0000 7fa0557fa640 1 -- 172.21.15.99:0/1768276552 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 16) ==== 137290+0+0 (secure 0 0 0) 0x7fa04c00a440 con 0x7fa058101930 2024-03-19T01:38:19.375 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.372+0000 7fa0557fa640 1 --2- 172.21.15.99:0/1768276552 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa0400a2800 0x7fa0400a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:19.375 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.372+0000 7fa057fff640 1 --2- 172.21.15.99:0/1768276552 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa0400a2800 0x7fa0400a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:19.375 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.372+0000 7fa0557fa640 1 -- 172.21.15.99:0/1768276552 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(83..83 src has 1..83) ==== 7127+0+0 (secure 0 0 0) 0x7fa04c0cfaf0 con 0x7fa058101930 2024-03-19T01:38:19.375 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.373+0000 7fa057fff640 1 --2- 172.21.15.99:0/1768276552 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa0400a2800 0x7fa0400a4cc0 secure :-1 s=READY pgs=129 cs=0 l=1 rev1=1 crypto rx=0x7fa048000c00 tx=0x7fa048004f10 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:19.378 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.376+0000 7fa0557fa640 1 -- 172.21.15.99:0/1768276552 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7fa04c023060 con 0x7fa058101930 2024-03-19T01:38:19.593 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.592+0000 7fa05e427640 1 -- 172.21.15.99:0/1768276552 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7fa0581075e0 con 0x7fa058101930 2024-03-19T01:38:19.593 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.592+0000 7fa0557fa640 1 -- 172.21.15.99:0/1768276552 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 1 v1) ==== 93+0+569 (secure 0 0 0) 0x7fa04c09d0c0 con 0x7fa058101930 2024-03-19T01:38:19.594 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:38:19.594 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":1,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[]} 2024-03-19T01:38:19.594 INFO:teuthology.orchestra.run.smithi099.stderr:dumped fsmap epoch 1 2024-03-19T01:38:19.596 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.594+0000 7fa05e427640 1 -- 172.21.15.99:0/1768276552 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa0400a2800 msgr2=0x7fa0400a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:19.596 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.594+0000 7fa05e427640 1 --2- 172.21.15.99:0/1768276552 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa0400a2800 0x7fa0400a4cc0 secure :-1 s=READY pgs=129 cs=0 l=1 rev1=1 crypto rx=0x7fa048000c00 tx=0x7fa048004f10 comp rx=0 tx=0).stop 2024-03-19T01:38:19.596 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.594+0000 7fa05e427640 1 -- 172.21.15.99:0/1768276552 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa058101930 msgr2=0x7fa058196f40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:19.596 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.594+0000 7fa05e427640 1 --2- 172.21.15.99:0/1768276552 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa058101930 0x7fa058196f40 secure :-1 s=READY pgs=153 cs=0 l=1 rev1=1 crypto rx=0x7fa04c007b50 tx=0x7fa04c00e420 comp rx=0 tx=0).stop 2024-03-19T01:38:19.596 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.594+0000 7fa05e427640 1 -- 172.21.15.99:0/1768276552 shutdown_connections 2024-03-19T01:38:19.596 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.594+0000 7fa05e427640 1 --2- 172.21.15.99:0/1768276552 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa0581022f0 0x7fa05819dfb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:19.596 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.594+0000 7fa05e427640 1 --2- 172.21.15.99:0/1768276552 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa058101930 0x7fa058196f40 unknown :-1 s=CLOSED pgs=153 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:19.596 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.594+0000 7fa05e427640 1 --2- 172.21.15.99:0/1768276552 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa0400a2800 0x7fa0400a4cc0 unknown :-1 s=CLOSED pgs=129 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:19.596 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.594+0000 7fa05e427640 1 --2- 172.21.15.99:0/1768276552 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa058100740 0x7fa058196a00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:19.596 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.594+0000 7fa05e427640 1 -- 172.21.15.99:0/1768276552 >> 172.21.15.99:0/1768276552 conn(0x7fa0580fbcb0 msgr2=0x7fa058104bd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:19.596 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.594+0000 7fa05e427640 1 -- 172.21.15.99:0/1768276552 shutdown_connections 2024-03-19T01:38:19.597 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:19.594+0000 7fa05e427640 1 -- 172.21.15.99:0/1768276552 wait complete. 2024-03-19T01:38:20.051 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:19 smithi115 ceph-mon[67332]: Deploying daemon mds.l on smithi115 2024-03-19T01:38:20.052 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:19 smithi115 ceph-mon[67332]: pgmap v259: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:20.052 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:19 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/3113912413' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-19T01:38:20.052 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:19 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/2340797535' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-03-19T01:38:20.052 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:19 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/1768276552' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:38:20.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:19 smithi099 ceph-mon[62951]: Deploying daemon mds.l on smithi115 2024-03-19T01:38:20.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:19 smithi099 ceph-mon[62951]: pgmap v259: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:20.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:19 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/3113912413' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-19T01:38:20.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:19 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/2340797535' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-03-19T01:38:20.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:19 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/1768276552' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:38:20.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:19 smithi112 ceph-mon[66819]: Deploying daemon mds.l on smithi115 2024-03-19T01:38:20.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:19 smithi112 ceph-mon[66819]: pgmap v259: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:20.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:19 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/3113912413' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-19T01:38:20.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:19 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/2340797535' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-03-19T01:38:20.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:19 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/1768276552' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:38:20.307 INFO:journalctl@ceph.mds.l.smithi115.stdout:Mar 19 01:38:20 smithi115 systemd[1]: Starting Ceph mds.l for 438fb69c-e590-11ee-95c9-87774f69a715... 2024-03-19T01:38:20.731 INFO:journalctl@ceph.mds.l.smithi115.stdout:Mar 19 01:38:20 smithi115 podman[96521]: 2024-03-19 01:38:20.347498438 +0000 UTC m=+0.017845948 image pull 2312c8e42d0069a2dbb2364bddb00264db4b800e7f18a7e7e4eb8f3ecab2ff1c quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639 2024-03-19T01:38:20.731 INFO:journalctl@ceph.mds.l.smithi115.stdout:Mar 19 01:38:20 smithi115 podman[96521]: 2024-03-19 01:38:20.503885633 +0000 UTC m=+0.174233121 container create 5547b1cc5bf1dc8362bce22f02cc601bb7ab6a1f75399567269f53ed1c811e88 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-l, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240307, org.label-schema.license=GPLv2) 2024-03-19T01:38:20.731 INFO:journalctl@ceph.mds.l.smithi115.stdout:Mar 19 01:38:20 smithi115 podman[96521]: 2024-03-19 01:38:20.731253288 +0000 UTC m=+0.401600778 container init 5547b1cc5bf1dc8362bce22f02cc601bb7ab6a1f75399567269f53ed1c811e88 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-l, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240307, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.vendor=CentOS) 2024-03-19T01:38:21.057 INFO:journalctl@ceph.mds.l.smithi115.stdout:Mar 19 01:38:20 smithi115 podman[96521]: 2024-03-19 01:38:20.734104075 +0000 UTC m=+0.404451572 container start 5547b1cc5bf1dc8362bce22f02cc601bb7ab6a1f75399567269f53ed1c811e88 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-l, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240307, GIT_BRANCH=HEAD, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-03-19T01:38:21.057 INFO:journalctl@ceph.mds.l.smithi115.stdout:Mar 19 01:38:20 smithi115 ceph-mds[96540]: -- 172.21.15.115:0/806816037 <== mon.1 v2:172.21.15.115:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55624772ba40 con 0x5562476ff000 2024-03-19T01:38:21.057 INFO:journalctl@ceph.mds.l.smithi115.stdout:Mar 19 01:38:20 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-l[96536]: starting mds.l at 2024-03-19T01:38:21.057 INFO:journalctl@ceph.mds.l.smithi115.stdout:Mar 19 01:38:20 smithi115 bash[96521]: 5547b1cc5bf1dc8362bce22f02cc601bb7ab6a1f75399567269f53ed1c811e88 2024-03-19T01:38:21.057 INFO:journalctl@ceph.mds.l.smithi115.stdout:Mar 19 01:38:20 smithi115 systemd[1]: Started Ceph mds.l for 438fb69c-e590-11ee-95c9-87774f69a715. 2024-03-19T01:38:21.607 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-19T01:38:21.723 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:21.720+0000 7f9902cc6640 1 -- 172.21.15.99:0/1145286820 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f98fc1022f0 msgr2=0x7f98fc108b80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:21.723 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:21.720+0000 7f9902cc6640 1 --2- 172.21.15.99:0/1145286820 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f98fc1022f0 0x7f98fc108b80 secure :-1 s=READY pgs=154 cs=0 l=1 rev1=1 crypto rx=0x7f98ec009900 tx=0x7f98ec02bc30 comp rx=0 tx=0).stop 2024-03-19T01:38:21.723 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:21.720+0000 7f9902cc6640 1 -- 172.21.15.99:0/1145286820 shutdown_connections 2024-03-19T01:38:21.723 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:21.720+0000 7f9902cc6640 1 --2- 172.21.15.99:0/1145286820 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f98fc1022f0 0x7f98fc108b80 unknown :-1 s=CLOSED pgs=154 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:21.723 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:21.720+0000 7f9902cc6640 1 --2- 172.21.15.99:0/1145286820 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f98fc101930 0x7f98fc101db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:21.723 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:21.720+0000 7f9902cc6640 1 --2- 172.21.15.99:0/1145286820 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f98fc100740 0x7f98fc100b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:21.723 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:21.720+0000 7f9902cc6640 1 -- 172.21.15.99:0/1145286820 >> 172.21.15.99:0/1145286820 conn(0x7f98fc0fbcb0 msgr2=0x7f98fc0fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:21.724 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:21.720+0000 7f9902cc6640 1 -- 172.21.15.99:0/1145286820 shutdown_connections 2024-03-19T01:38:21.724 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:21.720+0000 7f9902cc6640 1 -- 172.21.15.99:0/1145286820 wait complete. 2024-03-19T01:38:21.724 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:21.720+0000 7f9902cc6640 1 Processor -- start 2024-03-19T01:38:21.724 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:21.720+0000 7f9902cc6640 1 -- start start 2024-03-19T01:38:21.724 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:21.720+0000 7f9902cc6640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f98fc100740 0x7f98fc197a10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:21.724 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:21.720+0000 7f9902cc6640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f98fc101930 0x7f98fc197f50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:21.724 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:21.720+0000 7f9902cc6640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f98fc1022f0 0x7f98fc19ef70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:21.724 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:21.720+0000 7f9902cc6640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f98fc10c3a0 con 0x7f98fc100740 2024-03-19T01:38:21.724 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:21.720+0000 7f9902cc6640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f98fc10c220 con 0x7f98fc101930 2024-03-19T01:38:21.724 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:21.720+0000 7f9902cc6640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f98fc10c520 con 0x7f98fc1022f0 2024-03-19T01:38:21.724 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:21.721+0000 7f9900a62640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f98fc100740 0x7f98fc197a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:21.724 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:21.721+0000 7f9900a62640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f98fc100740 0x7f98fc197a10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:33006/0 (socket says 172.21.15.99:33006) 2024-03-19T01:38:21.725 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:21.721+0000 7f9900a62640 1 -- 172.21.15.99:0/2262053013 learned_addr learned my addr 172.21.15.99:0/2262053013 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:38:21.725 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:21.721+0000 7f9901263640 1 --2- 172.21.15.99:0/2262053013 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f98fc1022f0 0x7f98fc19ef70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:21.725 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:21.721+0000 7f9900a62640 1 -- 172.21.15.99:0/2262053013 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f98fc1022f0 msgr2=0x7f98fc19ef70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:21.725 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:21.721+0000 7f9900a62640 1 --2- 172.21.15.99:0/2262053013 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f98fc1022f0 0x7f98fc19ef70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:21.725 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:21.721+0000 7f98fbfff640 1 --2- 172.21.15.99:0/2262053013 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f98fc101930 0x7f98fc197f50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:21.725 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:21.721+0000 7f9900a62640 1 -- 172.21.15.99:0/2262053013 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f98fc101930 msgr2=0x7f98fc197f50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:21.725 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:21.721+0000 7f9900a62640 1 --2- 172.21.15.99:0/2262053013 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f98fc101930 0x7f98fc197f50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:21.725 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:21.721+0000 7f9900a62640 1 -- 172.21.15.99:0/2262053013 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f98fc19f5e0 con 0x7f98fc100740 2024-03-19T01:38:21.725 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:21.721+0000 7f9901263640 1 --2- 172.21.15.99:0/2262053013 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f98fc1022f0 0x7f98fc19ef70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-19T01:38:21.725 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:21.721+0000 7f98fbfff640 1 --2- 172.21.15.99:0/2262053013 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f98fc101930 0x7f98fc197f50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-19T01:38:21.725 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:21.721+0000 7f9900a62640 1 --2- 172.21.15.99:0/2262053013 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f98fc100740 0x7f98fc197a10 secure :-1 s=READY pgs=155 cs=0 l=1 rev1=1 crypto rx=0x7f98f400ccb0 tx=0x7f98f400a620 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:21.725 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:21.721+0000 7f98f9ffb640 1 -- 172.21.15.99:0/2262053013 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f98f4003b70 con 0x7f98fc100740 2024-03-19T01:38:21.725 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:21.721+0000 7f98f9ffb640 1 -- 172.21.15.99:0/2262053013 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f98f400ae50 con 0x7f98fc100740 2024-03-19T01:38:21.726 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:21.721+0000 7f9902cc6640 1 -- 172.21.15.99:0/2262053013 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f98fc19f8d0 con 0x7f98fc100740 2024-03-19T01:38:21.726 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:21.721+0000 7f98f9ffb640 1 -- 172.21.15.99:0/2262053013 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f98f400f810 con 0x7f98fc100740 2024-03-19T01:38:21.726 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:21.721+0000 7f9902cc6640 1 -- 172.21.15.99:0/2262053013 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f98fc19fe20 con 0x7f98fc100740 2024-03-19T01:38:21.726 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:21.722+0000 7f9902cc6640 1 -- 172.21.15.99:0/2262053013 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f98fc1a00b0 con 0x7f98fc100740 2024-03-19T01:38:21.726 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:21.723+0000 7f98f9ffb640 1 -- 172.21.15.99:0/2262053013 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 16) ==== 137290+0+0 (secure 0 0 0) 0x7f98f400f9b0 con 0x7f98fc100740 2024-03-19T01:38:21.727 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:21.724+0000 7f98f9ffb640 1 --2- 172.21.15.99:0/2262053013 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f98e40a2800 0x7f98e40a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:21.727 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:21.724+0000 7f98f9ffb640 1 -- 172.21.15.99:0/2262053013 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(83..83 src has 1..83) ==== 7127+0+0 (secure 0 0 0) 0x7f98f40ce7e0 con 0x7f98fc100740 2024-03-19T01:38:21.727 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:21.724+0000 7f98fbfff640 1 --2- 172.21.15.99:0/2262053013 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f98e40a2800 0x7f98e40a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:21.727 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:21.724+0000 7f98fbfff640 1 --2- 172.21.15.99:0/2262053013 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f98e40a2800 0x7f98e40a4cc0 secure :-1 s=READY pgs=131 cs=0 l=1 rev1=1 crypto rx=0x7f98fc198ed0 tx=0x7f98f0005d90 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:21.729 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:21.728+0000 7f98f9ffb640 1 -- 172.21.15.99:0/2262053013 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f98f4014070 con 0x7f98fc100740 2024-03-19T01:38:21.823 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:21 smithi112 ceph-mon[66819]: pgmap v260: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:21.823 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:21 smithi112 ceph-mon[66819]: mds.? [v2:172.21.15.115:6832/2219320481,v1:172.21.15.115:6833/2219320481] up:boot 2024-03-19T01:38:21.823 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:21 smithi112 ceph-mon[66819]: fsmap 1 up:standby 2024-03-19T01:38:21.823 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:21 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "l"}]: dispatch 2024-03-19T01:38:21.823 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:21 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:21.823 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:21 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:21.823 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:21 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:21.823 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:21 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.k", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-19T01:38:21.823 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:21 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.k", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-19T01:38:21.823 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:21 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:38:21.943 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:21.943+0000 7f9902cc6640 1 -- 172.21.15.99:0/2262053013 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f98fc1a0280 con 0x7f98fc100740 2024-03-19T01:38:21.944 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:21.943+0000 7f98f9ffb640 1 -- 172.21.15.99:0/2262053013 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 2 v2) ==== 93+0+1396 (secure 0 0 0) 0x7f98f409bd20 con 0x7f98fc100740 2024-03-19T01:38:21.944 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:38:21.944 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":2,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":24415,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.115:6833/2219320481","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6832","nonce":2219320481},{"type":"v1","addr":"172.21.15.115:6833","nonce":2219320481}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2}],"filesystems":[]} 2024-03-19T01:38:21.944 INFO:teuthology.orchestra.run.smithi099.stderr:dumped fsmap epoch 2 2024-03-19T01:38:21.946 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:21.945+0000 7f9902cc6640 1 -- 172.21.15.99:0/2262053013 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f98e40a2800 msgr2=0x7f98e40a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:21.946 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:21.945+0000 7f9902cc6640 1 --2- 172.21.15.99:0/2262053013 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f98e40a2800 0x7f98e40a4cc0 secure :-1 s=READY pgs=131 cs=0 l=1 rev1=1 crypto rx=0x7f98fc198ed0 tx=0x7f98f0005d90 comp rx=0 tx=0).stop 2024-03-19T01:38:21.946 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:21.945+0000 7f9902cc6640 1 -- 172.21.15.99:0/2262053013 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f98fc100740 msgr2=0x7f98fc197a10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:21.946 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:21.945+0000 7f9902cc6640 1 --2- 172.21.15.99:0/2262053013 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f98fc100740 0x7f98fc197a10 secure :-1 s=READY pgs=155 cs=0 l=1 rev1=1 crypto rx=0x7f98f400ccb0 tx=0x7f98f400a620 comp rx=0 tx=0).stop 2024-03-19T01:38:21.946 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:21.945+0000 7f9902cc6640 1 -- 172.21.15.99:0/2262053013 shutdown_connections 2024-03-19T01:38:21.946 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:21.945+0000 7f9902cc6640 1 --2- 172.21.15.99:0/2262053013 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f98e40a2800 0x7f98e40a4cc0 unknown :-1 s=CLOSED pgs=131 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:21.947 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:21.945+0000 7f9902cc6640 1 --2- 172.21.15.99:0/2262053013 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f98fc101930 0x7f98fc197f50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:21.947 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:21.945+0000 7f9902cc6640 1 --2- 172.21.15.99:0/2262053013 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f98fc1022f0 0x7f98fc19ef70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:21.947 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:21.945+0000 7f9902cc6640 1 --2- 172.21.15.99:0/2262053013 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f98fc100740 0x7f98fc197a10 unknown :-1 s=CLOSED pgs=155 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:21.947 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:21.945+0000 7f9902cc6640 1 -- 172.21.15.99:0/2262053013 >> 172.21.15.99:0/2262053013 conn(0x7f98fc0fbcb0 msgr2=0x7f98fc0fd630 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:21.947 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:21.945+0000 7f9902cc6640 1 -- 172.21.15.99:0/2262053013 shutdown_connections 2024-03-19T01:38:21.947 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:21.945+0000 7f9902cc6640 1 -- 172.21.15.99:0/2262053013 wait complete. 2024-03-19T01:38:21.956 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:21 smithi099 ceph-mon[62951]: pgmap v260: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:21.956 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:21 smithi099 ceph-mon[62951]: mds.? [v2:172.21.15.115:6832/2219320481,v1:172.21.15.115:6833/2219320481] up:boot 2024-03-19T01:38:21.956 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:21 smithi099 ceph-mon[62951]: fsmap 1 up:standby 2024-03-19T01:38:21.956 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:21 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "l"}]: dispatch 2024-03-19T01:38:21.956 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:21 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:21.956 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:21 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:21.957 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:21 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:21.957 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:21 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.k", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-19T01:38:21.957 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:21 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.k", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-19T01:38:21.957 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:21 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:38:22.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:21 smithi115 ceph-mon[67332]: pgmap v260: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:22.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:21 smithi115 ceph-mon[67332]: mds.? [v2:172.21.15.115:6832/2219320481,v1:172.21.15.115:6833/2219320481] up:boot 2024-03-19T01:38:22.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:21 smithi115 ceph-mon[67332]: fsmap 1 up:standby 2024-03-19T01:38:22.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:21 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "l"}]: dispatch 2024-03-19T01:38:22.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:21 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:22.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:21 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:22.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:21 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:22.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:21 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.k", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-19T01:38:22.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:21 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.k", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-19T01:38:22.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:21 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:38:23.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:22 smithi115 ceph-mon[67332]: Deploying daemon mds.k on smithi112 2024-03-19T01:38:23.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:22 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/2262053013' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:38:23.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:22 smithi099 ceph-mon[62951]: Deploying daemon mds.k on smithi112 2024-03-19T01:38:23.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:22 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/2262053013' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:38:23.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:22 smithi112 ceph-mon[66819]: Deploying daemon mds.k on smithi112 2024-03-19T01:38:23.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:22 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/2262053013' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:38:23.112 INFO:journalctl@ceph.mds.k.smithi112.stdout:Mar 19 01:38:22 smithi112 systemd[1]: Starting Ceph mds.k for 438fb69c-e590-11ee-95c9-87774f69a715... 2024-03-19T01:38:23.414 INFO:journalctl@ceph.mds.k.smithi112.stdout:Mar 19 01:38:23 smithi112 podman[96451]: 2024-03-19 01:38:23.154339058 +0000 UTC m=+0.109121579 container create 20eddeea6ac326bfd03b1e87140250fd370b351ab039147839bfadf141893f8e (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-k, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240307, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.vendor=CentOS) 2024-03-19T01:38:23.415 INFO:journalctl@ceph.mds.k.smithi112.stdout:Mar 19 01:38:23 smithi112 podman[96451]: 2024-03-19 01:38:23.062521099 +0000 UTC m=+0.017303621 image pull 2312c8e42d0069a2dbb2364bddb00264db4b800e7f18a7e7e4eb8f3ecab2ff1c quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639 2024-03-19T01:38:23.415 INFO:journalctl@ceph.mds.k.smithi112.stdout:Mar 19 01:38:23 smithi112 podman[96451]: 2024-03-19 01:38:23.356521 +0000 UTC m=+0.311303523 container init 20eddeea6ac326bfd03b1e87140250fd370b351ab039147839bfadf141893f8e (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-k, CEPH_POINT_RELEASE=, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240307, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.schema-version=1.0) 2024-03-19T01:38:23.415 INFO:journalctl@ceph.mds.k.smithi112.stdout:Mar 19 01:38:23 smithi112 podman[96451]: 2024-03-19 01:38:23.359133527 +0000 UTC m=+0.313916043 container start 20eddeea6ac326bfd03b1e87140250fd370b351ab039147839bfadf141893f8e (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-k, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240307, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, GIT_BRANCH=HEAD) 2024-03-19T01:38:23.415 INFO:journalctl@ceph.mds.k.smithi112.stdout:Mar 19 01:38:23 smithi112 ceph-mds[96470]: -- 172.21.15.112:0/3911568218 <== mon.2 v2:172.21.15.112:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x559eed659a40 con 0x559eed62c800 2024-03-19T01:38:23.415 INFO:journalctl@ceph.mds.k.smithi112.stdout:Mar 19 01:38:23 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-k[96466]: starting mds.k at 2024-03-19T01:38:23.415 INFO:journalctl@ceph.mds.k.smithi112.stdout:Mar 19 01:38:23 smithi112 bash[96451]: 20eddeea6ac326bfd03b1e87140250fd370b351ab039147839bfadf141893f8e 2024-03-19T01:38:23.783 INFO:journalctl@ceph.mds.k.smithi112.stdout:Mar 19 01:38:23 smithi112 systemd[1]: Started Ceph mds.k for 438fb69c-e590-11ee-95c9-87774f69a715. 2024-03-19T01:38:23.959 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-19T01:38:24.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:23 smithi115 ceph-mon[67332]: pgmap v261: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:24.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:23 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:24.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:23 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:24.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:23 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:24.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:23 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.j", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-19T01:38:24.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:23 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.j", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-19T01:38:24.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:23 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:38:24.074 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:24.071+0000 7ff8e3085640 1 -- 172.21.15.99:0/3728222564 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff8dc101930 msgr2=0x7ff8dc101db0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:24.074 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:24.071+0000 7ff8e3085640 1 --2- 172.21.15.99:0/3728222564 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff8dc101930 0x7ff8dc101db0 secure :-1 s=READY pgs=156 cs=0 l=1 rev1=1 crypto rx=0x7ff8c8009900 tx=0x7ff8c802bc30 comp rx=0 tx=0).stop 2024-03-19T01:38:24.074 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:24.072+0000 7ff8e3085640 1 -- 172.21.15.99:0/3728222564 shutdown_connections 2024-03-19T01:38:24.074 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:24.072+0000 7ff8e3085640 1 --2- 172.21.15.99:0/3728222564 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff8dc1022f0 0x7ff8dc108b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:24.074 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:24.072+0000 7ff8e3085640 1 --2- 172.21.15.99:0/3728222564 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff8dc101930 0x7ff8dc101db0 unknown :-1 s=CLOSED pgs=156 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:24.074 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:24.072+0000 7ff8e3085640 1 --2- 172.21.15.99:0/3728222564 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff8dc100740 0x7ff8dc100b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:24.074 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:24.072+0000 7ff8e3085640 1 -- 172.21.15.99:0/3728222564 >> 172.21.15.99:0/3728222564 conn(0x7ff8dc0fbcb0 msgr2=0x7ff8dc0fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:24.075 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:24.072+0000 7ff8e3085640 1 -- 172.21.15.99:0/3728222564 shutdown_connections 2024-03-19T01:38:24.075 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:24.072+0000 7ff8e3085640 1 -- 172.21.15.99:0/3728222564 wait complete. 2024-03-19T01:38:24.075 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:24.072+0000 7ff8e3085640 1 Processor -- start 2024-03-19T01:38:24.075 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:24.072+0000 7ff8e3085640 1 -- start start 2024-03-19T01:38:24.075 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:24.072+0000 7ff8e3085640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff8dc100740 0x7ff8dc10b0c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:24.075 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:24.072+0000 7ff8e3085640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff8dc101930 0x7ff8dc109700 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:24.075 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:24.072+0000 7ff8e3085640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff8dc1022f0 0x7ff8dc109c40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:24.075 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:24.072+0000 7ff8e3085640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7ff8dc10d4e0 con 0x7ff8dc1022f0 2024-03-19T01:38:24.075 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:24.072+0000 7ff8e3085640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7ff8dc10d360 con 0x7ff8dc101930 2024-03-19T01:38:24.075 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:24.072+0000 7ff8e3085640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7ff8dc10d660 con 0x7ff8dc100740 2024-03-19T01:38:24.075 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:24.072+0000 7ff8e1622640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff8dc1022f0 0x7ff8dc109c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:24.075 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:24.072+0000 7ff8e1622640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff8dc1022f0 0x7ff8dc109c40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:48770/0 (socket says 172.21.15.99:48770) 2024-03-19T01:38:24.075 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:24.072+0000 7ff8e1622640 1 -- 172.21.15.99:0/230936909 learned_addr learned my addr 172.21.15.99:0/230936909 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:38:24.075 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:24.073+0000 7ff8dbfff640 1 --2- 172.21.15.99:0/230936909 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff8dc101930 0x7ff8dc109700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:24.076 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:24.073+0000 7ff8e0e21640 1 --2- 172.21.15.99:0/230936909 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff8dc100740 0x7ff8dc10b0c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:24.076 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:24.073+0000 7ff8e1622640 1 -- 172.21.15.99:0/230936909 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff8dc100740 msgr2=0x7ff8dc10b0c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:24.076 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:24.073+0000 7ff8e1622640 1 --2- 172.21.15.99:0/230936909 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff8dc100740 0x7ff8dc10b0c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:24.076 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:24.073+0000 7ff8e1622640 1 -- 172.21.15.99:0/230936909 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff8dc101930 msgr2=0x7ff8dc109700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:24.076 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:24.073+0000 7ff8e1622640 1 --2- 172.21.15.99:0/230936909 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff8dc101930 0x7ff8dc109700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:24.076 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:24.073+0000 7ff8e1622640 1 -- 172.21.15.99:0/230936909 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ff8dc10a4d0 con 0x7ff8dc1022f0 2024-03-19T01:38:24.076 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:24.073+0000 7ff8e1622640 1 --2- 172.21.15.99:0/230936909 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff8dc1022f0 0x7ff8dc109c40 secure :-1 s=READY pgs=157 cs=0 l=1 rev1=1 crypto rx=0x7ff8d400cad0 tx=0x7ff8d400cf90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:24.076 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:24.073+0000 7ff8d9ffb640 1 -- 172.21.15.99:0/230936909 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff8d400ae20 con 0x7ff8dc1022f0 2024-03-19T01:38:24.076 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:24.073+0000 7ff8d9ffb640 1 -- 172.21.15.99:0/230936909 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7ff8d4005020 con 0x7ff8dc1022f0 2024-03-19T01:38:24.076 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:24.073+0000 7ff8e3085640 1 -- 172.21.15.99:0/230936909 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ff8dc1a1ca0 con 0x7ff8dc1022f0 2024-03-19T01:38:24.076 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:24.073+0000 7ff8d9ffb640 1 -- 172.21.15.99:0/230936909 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff8d400f6f0 con 0x7ff8dc1022f0 2024-03-19T01:38:24.076 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:24.073+0000 7ff8e3085640 1 -- 172.21.15.99:0/230936909 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7ff8dc1a2170 con 0x7ff8dc1022f0 2024-03-19T01:38:24.077 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:24.074+0000 7ff8e3085640 1 -- 172.21.15.99:0/230936909 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7ff8dc068610 con 0x7ff8dc1022f0 2024-03-19T01:38:24.077 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:24.075+0000 7ff8d9ffb640 1 -- 172.21.15.99:0/230936909 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 16) ==== 137290+0+0 (secure 0 0 0) 0x7ff8d40051d0 con 0x7ff8dc1022f0 2024-03-19T01:38:24.077 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:24.076+0000 7ff8d9ffb640 1 --2- 172.21.15.99:0/230936909 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff8cc0a2800 0x7ff8cc0a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:24.077 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:24.076+0000 7ff8e0e21640 1 --2- 172.21.15.99:0/230936909 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff8cc0a2800 0x7ff8cc0a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:24.077 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:24.076+0000 7ff8d9ffb640 1 -- 172.21.15.99:0/230936909 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(83..83 src has 1..83) ==== 7127+0+0 (secure 0 0 0) 0x7ff8d40ce680 con 0x7ff8dc1022f0 2024-03-19T01:38:24.078 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:24.076+0000 7ff8e0e21640 1 --2- 172.21.15.99:0/230936909 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff8cc0a2800 0x7ff8cc0a4cc0 secure :-1 s=READY pgs=133 cs=0 l=1 rev1=1 crypto rx=0x7ff8d00058c0 tx=0x7ff8d000a4c0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:24.080 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:24.080+0000 7ff8d9ffb640 1 -- 172.21.15.99:0/230936909 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7ff8d4014030 con 0x7ff8dc1022f0 2024-03-19T01:38:24.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:23 smithi099 ceph-mon[62951]: pgmap v261: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:24.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:23 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:24.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:23 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:24.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:23 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:24.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:23 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.j", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-19T01:38:24.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:23 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.j", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-19T01:38:24.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:23 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:38:24.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:23 smithi112 ceph-mon[66819]: pgmap v261: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:24.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:23 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:24.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:23 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:24.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:23 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:24.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:23 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.j", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-19T01:38:24.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:23 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.j", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-19T01:38:24.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:23 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:38:24.305 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:24.304+0000 7ff8e3085640 1 -- 172.21.15.99:0/230936909 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7ff8dc02d1f0 con 0x7ff8dc1022f0 2024-03-19T01:38:24.305 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:24.305+0000 7ff8d9ffb640 1 -- 172.21.15.99:0/230936909 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 3 v3) ==== 93+0+2224 (secure 0 0 0) 0x7ff8d409bbc0 con 0x7ff8dc1022f0 2024-03-19T01:38:24.306 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:38:24.306 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":3,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":24383,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.112:6833/2969810977","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6832","nonce":2969810977},{"type":"v1","addr":"172.21.15.112:6833","nonce":2969810977}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3},{"gid":24415,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.115:6833/2219320481","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6832","nonce":2219320481},{"type":"v1","addr":"172.21.15.115:6833","nonce":2219320481}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2}],"filesystems":[]} 2024-03-19T01:38:24.306 INFO:teuthology.orchestra.run.smithi099.stderr:dumped fsmap epoch 3 2024-03-19T01:38:24.308 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:24.307+0000 7ff8e3085640 1 -- 172.21.15.99:0/230936909 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff8cc0a2800 msgr2=0x7ff8cc0a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:24.309 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:24.307+0000 7ff8e3085640 1 --2- 172.21.15.99:0/230936909 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff8cc0a2800 0x7ff8cc0a4cc0 secure :-1 s=READY pgs=133 cs=0 l=1 rev1=1 crypto rx=0x7ff8d00058c0 tx=0x7ff8d000a4c0 comp rx=0 tx=0).stop 2024-03-19T01:38:24.309 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:24.307+0000 7ff8e3085640 1 -- 172.21.15.99:0/230936909 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff8dc1022f0 msgr2=0x7ff8dc109c40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:24.309 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:24.307+0000 7ff8e3085640 1 --2- 172.21.15.99:0/230936909 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff8dc1022f0 0x7ff8dc109c40 secure :-1 s=READY pgs=157 cs=0 l=1 rev1=1 crypto rx=0x7ff8d400cad0 tx=0x7ff8d400cf90 comp rx=0 tx=0).stop 2024-03-19T01:38:24.309 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:24.307+0000 7ff8e3085640 1 -- 172.21.15.99:0/230936909 shutdown_connections 2024-03-19T01:38:24.310 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:24.307+0000 7ff8e3085640 1 --2- 172.21.15.99:0/230936909 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff8cc0a2800 0x7ff8cc0a4cc0 unknown :-1 s=CLOSED pgs=133 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:24.310 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:24.307+0000 7ff8e3085640 1 --2- 172.21.15.99:0/230936909 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff8dc101930 0x7ff8dc109700 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:24.310 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:24.307+0000 7ff8e3085640 1 --2- 172.21.15.99:0/230936909 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff8dc1022f0 0x7ff8dc109c40 unknown :-1 s=CLOSED pgs=157 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:24.310 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:24.307+0000 7ff8e3085640 1 --2- 172.21.15.99:0/230936909 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff8dc100740 0x7ff8dc10b0c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:24.310 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:24.307+0000 7ff8e3085640 1 -- 172.21.15.99:0/230936909 >> 172.21.15.99:0/230936909 conn(0x7ff8dc0fbcb0 msgr2=0x7ff8dc104d00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:24.311 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:24.307+0000 7ff8e3085640 1 -- 172.21.15.99:0/230936909 shutdown_connections 2024-03-19T01:38:24.311 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:24.307+0000 7ff8e3085640 1 -- 172.21.15.99:0/230936909 wait complete. 2024-03-19T01:38:25.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:24 smithi115 ceph-mon[67332]: Deploying daemon mds.j on smithi099 2024-03-19T01:38:25.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:24 smithi115 ceph-mon[67332]: mds.? [v2:172.21.15.112:6832/2969810977,v1:172.21.15.112:6833/2969810977] up:boot 2024-03-19T01:38:25.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:24 smithi115 ceph-mon[67332]: fsmap 2 up:standby 2024-03-19T01:38:25.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:24 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "k"}]: dispatch 2024-03-19T01:38:25.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:24 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/230936909' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:38:25.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:24 smithi099 ceph-mon[62951]: Deploying daemon mds.j on smithi099 2024-03-19T01:38:25.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:24 smithi099 ceph-mon[62951]: mds.? [v2:172.21.15.112:6832/2969810977,v1:172.21.15.112:6833/2969810977] up:boot 2024-03-19T01:38:25.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:24 smithi099 ceph-mon[62951]: fsmap 2 up:standby 2024-03-19T01:38:25.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:24 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "k"}]: dispatch 2024-03-19T01:38:25.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:24 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/230936909' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:38:25.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:24 smithi112 ceph-mon[66819]: Deploying daemon mds.j on smithi099 2024-03-19T01:38:25.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:24 smithi112 ceph-mon[66819]: mds.? [v2:172.21.15.112:6832/2969810977,v1:172.21.15.112:6833/2969810977] up:boot 2024-03-19T01:38:25.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:24 smithi112 ceph-mon[66819]: fsmap 2 up:standby 2024-03-19T01:38:25.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:24 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "k"}]: dispatch 2024-03-19T01:38:25.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:24 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/230936909' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:38:25.797 INFO:journalctl@ceph.mds.j.smithi099.stdout:Mar 19 01:38:25 smithi099 systemd[1]: Starting Ceph mds.j for 438fb69c-e590-11ee-95c9-87774f69a715... 2024-03-19T01:38:26.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:25 smithi115 ceph-mon[67332]: pgmap v262: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:26.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:25 smithi099 ceph-mon[62951]: pgmap v262: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:26.067 INFO:journalctl@ceph.mds.j.smithi099.stdout:Mar 19 01:38:25 smithi099 podman[103419]: 2024-03-19 01:38:25.705330021 +0000 UTC m=+0.018045339 image pull 2312c8e42d0069a2dbb2364bddb00264db4b800e7f18a7e7e4eb8f3ecab2ff1c quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639 2024-03-19T01:38:26.067 INFO:journalctl@ceph.mds.j.smithi099.stdout:Mar 19 01:38:25 smithi099 podman[103419]: 2024-03-19 01:38:25.814375478 +0000 UTC m=+0.127090799 container create ca9adb08a0032a36ab7bf0f5ba80051f7f04a9b2d23aba5bb3fa647b91cb0345 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-j, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, io.buildah.version=1.33.5, ceph=True, org.label-schema.build-date=20240307, CEPH_POINT_RELEASE=) 2024-03-19T01:38:26.068 INFO:journalctl@ceph.mds.j.smithi099.stdout:Mar 19 01:38:26 smithi099 podman[103419]: 2024-03-19 01:38:26.042520703 +0000 UTC m=+0.355236030 container init ca9adb08a0032a36ab7bf0f5ba80051f7f04a9b2d23aba5bb3fa647b91cb0345 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-j, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240307, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5) 2024-03-19T01:38:26.068 INFO:journalctl@ceph.mds.j.smithi099.stdout:Mar 19 01:38:26 smithi099 podman[103419]: 2024-03-19 01:38:26.044972972 +0000 UTC m=+0.357688291 container start ca9adb08a0032a36ab7bf0f5ba80051f7f04a9b2d23aba5bb3fa647b91cb0345 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-j, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240307) 2024-03-19T01:38:26.068 INFO:journalctl@ceph.mds.j.smithi099.stdout:Mar 19 01:38:26 smithi099 ceph-mds[103438]: -- 172.21.15.99:0/2892021322 <== mon.0 v2:172.21.15.99:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x559a19df1a40 con 0x559a19dc5000 2024-03-19T01:38:26.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:25 smithi112 ceph-mon[66819]: pgmap v262: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:26.319 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-19T01:38:26.341 INFO:journalctl@ceph.mds.j.smithi099.stdout:Mar 19 01:38:26 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-j[103434]: starting mds.j at 2024-03-19T01:38:26.341 INFO:journalctl@ceph.mds.j.smithi099.stdout:Mar 19 01:38:26 smithi099 bash[103419]: ca9adb08a0032a36ab7bf0f5ba80051f7f04a9b2d23aba5bb3fa647b91cb0345 2024-03-19T01:38:26.341 INFO:journalctl@ceph.mds.j.smithi099.stdout:Mar 19 01:38:26 smithi099 systemd[1]: Started Ceph mds.j for 438fb69c-e590-11ee-95c9-87774f69a715. 2024-03-19T01:38:26.429 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:26.426+0000 7fb3ed6f8640 1 -- 172.21.15.99:0/1438369616 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb3e8100a30 msgr2=0x7fb3e8100eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:26.429 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:26.426+0000 7fb3ed6f8640 1 --2- 172.21.15.99:0/1438369616 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb3e8100a30 0x7fb3e8100eb0 secure :-1 s=READY pgs=160 cs=0 l=1 rev1=1 crypto rx=0x7fb3dc009900 tx=0x7fb3dc02bc30 comp rx=0 tx=0).stop 2024-03-19T01:38:26.429 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:26.426+0000 7fb3ed6f8640 1 -- 172.21.15.99:0/1438369616 shutdown_connections 2024-03-19T01:38:26.429 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:26.426+0000 7fb3ed6f8640 1 --2- 172.21.15.99:0/1438369616 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb3e81013f0 0x7fb3e8108c90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:26.429 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:26.426+0000 7fb3ed6f8640 1 --2- 172.21.15.99:0/1438369616 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb3e8100a30 0x7fb3e8100eb0 unknown :-1 s=CLOSED pgs=160 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:26.429 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:26.426+0000 7fb3ed6f8640 1 --2- 172.21.15.99:0/1438369616 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb3e8101d00 0x7fb3e8102120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:26.429 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:26.426+0000 7fb3ed6f8640 1 -- 172.21.15.99:0/1438369616 >> 172.21.15.99:0/1438369616 conn(0x7fb3e80fb460 msgr2=0x7fb3e80fd8a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:26.429 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:26.426+0000 7fb3ed6f8640 1 -- 172.21.15.99:0/1438369616 shutdown_connections 2024-03-19T01:38:26.429 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:26.426+0000 7fb3ed6f8640 1 -- 172.21.15.99:0/1438369616 wait complete. 2024-03-19T01:38:26.429 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:26.427+0000 7fb3ed6f8640 1 Processor -- start 2024-03-19T01:38:26.430 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:26.427+0000 7fb3ed6f8640 1 -- start start 2024-03-19T01:38:26.430 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:26.427+0000 7fb3ed6f8640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb3e8100a30 0x7fb3e8196aa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:26.430 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:26.427+0000 7fb3ed6f8640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb3e81013f0 0x7fb3e8196fe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:26.430 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:26.427+0000 7fb3ed6f8640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb3e8101d00 0x7fb3e819e050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:26.430 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:26.427+0000 7fb3ed6f8640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fb3e810c400 con 0x7fb3e81013f0 2024-03-19T01:38:26.430 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:26.427+0000 7fb3ed6f8640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fb3e810c280 con 0x7fb3e8101d00 2024-03-19T01:38:26.430 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:26.427+0000 7fb3ed6f8640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fb3e810c580 con 0x7fb3e8100a30 2024-03-19T01:38:26.430 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:26.427+0000 7fb3e67fc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb3e81013f0 0x7fb3e8196fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:26.430 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:26.427+0000 7fb3e67fc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb3e81013f0 0x7fb3e8196fe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:48802/0 (socket says 172.21.15.99:48802) 2024-03-19T01:38:26.430 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:26.427+0000 7fb3e67fc640 1 -- 172.21.15.99:0/3435102580 learned_addr learned my addr 172.21.15.99:0/3435102580 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:38:26.430 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:26.427+0000 7fb3e6ffd640 1 --2- 172.21.15.99:0/3435102580 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb3e8100a30 0x7fb3e8196aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:26.430 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:26.427+0000 7fb3e77fe640 1 --2- 172.21.15.99:0/3435102580 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb3e8101d00 0x7fb3e819e050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:26.430 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:26.427+0000 7fb3e67fc640 1 -- 172.21.15.99:0/3435102580 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb3e8100a30 msgr2=0x7fb3e8196aa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:26.430 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:26.427+0000 7fb3e67fc640 1 --2- 172.21.15.99:0/3435102580 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb3e8100a30 0x7fb3e8196aa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:26.431 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:26.427+0000 7fb3e67fc640 1 -- 172.21.15.99:0/3435102580 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb3e8101d00 msgr2=0x7fb3e819e050 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:26.431 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:26.427+0000 7fb3e67fc640 1 --2- 172.21.15.99:0/3435102580 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb3e8101d00 0x7fb3e819e050 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:26.431 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:26.427+0000 7fb3e67fc640 1 -- 172.21.15.99:0/3435102580 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb3e819e6c0 con 0x7fb3e81013f0 2024-03-19T01:38:26.431 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:26.428+0000 7fb3e67fc640 1 --2- 172.21.15.99:0/3435102580 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb3e81013f0 0x7fb3e8196fe0 secure :-1 s=READY pgs=161 cs=0 l=1 rev1=1 crypto rx=0x7fb3dc02efd0 tx=0x7fb3dc02cdc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:26.431 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:26.428+0000 7fb3cffff640 1 -- 172.21.15.99:0/3435102580 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb3dc041070 con 0x7fb3e81013f0 2024-03-19T01:38:26.431 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:26.428+0000 7fb3ed6f8640 1 -- 172.21.15.99:0/3435102580 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb3e819e950 con 0x7fb3e81013f0 2024-03-19T01:38:26.431 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:26.428+0000 7fb3cffff640 1 -- 172.21.15.99:0/3435102580 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fb3dc0054e0 con 0x7fb3e81013f0 2024-03-19T01:38:26.431 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:26.428+0000 7fb3ed6f8640 1 -- 172.21.15.99:0/3435102580 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fb3e819ee90 con 0x7fb3e81013f0 2024-03-19T01:38:26.431 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:26.428+0000 7fb3cffff640 1 -- 172.21.15.99:0/3435102580 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb3dc0346a0 con 0x7fb3e81013f0 2024-03-19T01:38:26.432 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:26.429+0000 7fb3ed6f8640 1 -- 172.21.15.99:0/3435102580 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fb3e8107b70 con 0x7fb3e81013f0 2024-03-19T01:38:26.432 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:26.431+0000 7fb3cffff640 1 -- 172.21.15.99:0/3435102580 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 16) ==== 137290+0+0 (secure 0 0 0) 0x7fb3dc044070 con 0x7fb3e81013f0 2024-03-19T01:38:26.432 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:26.431+0000 7fb3cffff640 1 --2- 172.21.15.99:0/3435102580 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb3d00a2800 0x7fb3d00a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:26.433 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:26.431+0000 7fb3cffff640 1 -- 172.21.15.99:0/3435102580 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(83..83 src has 1..83) ==== 7127+0+0 (secure 0 0 0) 0x7fb3dc0f0450 con 0x7fb3e81013f0 2024-03-19T01:38:26.433 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:26.431+0000 7fb3e6ffd640 1 --2- 172.21.15.99:0/3435102580 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb3d00a2800 0x7fb3d00a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:26.433 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:26.432+0000 7fb3e6ffd640 1 --2- 172.21.15.99:0/3435102580 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb3d00a2800 0x7fb3d00a4cc0 secure :-1 s=READY pgs=134 cs=0 l=1 rev1=1 crypto rx=0x7fb3d8005fd0 tx=0x7fb3d800b390 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:26.435 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:26.435+0000 7fb3cffff640 1 -- 172.21.15.99:0/3435102580 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7fb3dc0bd990 con 0x7fb3e81013f0 2024-03-19T01:38:26.657 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:26.657+0000 7fb3ed6f8640 1 -- 172.21.15.99:0/3435102580 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7fb3e802d1f0 con 0x7fb3e81013f0 2024-03-19T01:38:26.658 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:26.657+0000 7fb3cffff640 1 -- 172.21.15.99:0/3435102580 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 3 v3) ==== 93+0+2224 (secure 0 0 0) 0x7fb3dc0bd290 con 0x7fb3e81013f0 2024-03-19T01:38:26.658 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:38:26.658 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":3,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":24383,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.112:6833/2969810977","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6832","nonce":2969810977},{"type":"v1","addr":"172.21.15.112:6833","nonce":2969810977}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3},{"gid":24415,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.115:6833/2219320481","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6832","nonce":2219320481},{"type":"v1","addr":"172.21.15.115:6833","nonce":2219320481}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2}],"filesystems":[]} 2024-03-19T01:38:26.658 INFO:teuthology.orchestra.run.smithi099.stderr:dumped fsmap epoch 3 2024-03-19T01:38:26.661 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:26.660+0000 7fb3ed6f8640 1 -- 172.21.15.99:0/3435102580 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb3d00a2800 msgr2=0x7fb3d00a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:26.661 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:26.660+0000 7fb3ed6f8640 1 --2- 172.21.15.99:0/3435102580 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb3d00a2800 0x7fb3d00a4cc0 secure :-1 s=READY pgs=134 cs=0 l=1 rev1=1 crypto rx=0x7fb3d8005fd0 tx=0x7fb3d800b390 comp rx=0 tx=0).stop 2024-03-19T01:38:26.661 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:26.660+0000 7fb3ed6f8640 1 -- 172.21.15.99:0/3435102580 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb3e81013f0 msgr2=0x7fb3e8196fe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:26.661 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:26.660+0000 7fb3ed6f8640 1 --2- 172.21.15.99:0/3435102580 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb3e81013f0 0x7fb3e8196fe0 secure :-1 s=READY pgs=161 cs=0 l=1 rev1=1 crypto rx=0x7fb3dc02efd0 tx=0x7fb3dc02cdc0 comp rx=0 tx=0).stop 2024-03-19T01:38:26.661 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:26.660+0000 7fb3ed6f8640 1 -- 172.21.15.99:0/3435102580 shutdown_connections 2024-03-19T01:38:26.661 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:26.660+0000 7fb3ed6f8640 1 --2- 172.21.15.99:0/3435102580 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb3d00a2800 0x7fb3d00a4cc0 unknown :-1 s=CLOSED pgs=134 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:26.661 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:26.660+0000 7fb3ed6f8640 1 --2- 172.21.15.99:0/3435102580 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb3e8101d00 0x7fb3e819e050 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:26.661 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:26.660+0000 7fb3ed6f8640 1 --2- 172.21.15.99:0/3435102580 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb3e81013f0 0x7fb3e8196fe0 unknown :-1 s=CLOSED pgs=161 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:26.661 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:26.660+0000 7fb3ed6f8640 1 --2- 172.21.15.99:0/3435102580 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb3e8100a30 0x7fb3e8196aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:26.661 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:26.660+0000 7fb3ed6f8640 1 -- 172.21.15.99:0/3435102580 >> 172.21.15.99:0/3435102580 conn(0x7fb3e80fb460 msgr2=0x7fb3e8104d40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:26.661 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:26.660+0000 7fb3ed6f8640 1 -- 172.21.15.99:0/3435102580 shutdown_connections 2024-03-19T01:38:26.661 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:26.660+0000 7fb3ed6f8640 1 -- 172.21.15.99:0/3435102580 wait complete. 2024-03-19T01:38:27.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:27 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:27.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:27 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:27.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:27 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:27.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:27 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.d", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-19T01:38:27.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:27 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.d", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-19T01:38:27.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:27 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:38:27.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:27 smithi099 ceph-mon[62951]: Deploying daemon mds.d on smithi099 2024-03-19T01:38:27.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:27 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/3435102580' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:38:27.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:27 smithi099 ceph-mon[62951]: pgmap v263: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:27.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:27 smithi099 ceph-mon[62951]: mds.? [v2:172.21.15.99:6834/1631815621,v1:172.21.15.99:6835/1631815621] up:boot 2024-03-19T01:38:27.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:27 smithi099 ceph-mon[62951]: fsmap 3 up:standby 2024-03-19T01:38:27.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:27 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "j"}]: dispatch 2024-03-19T01:38:27.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:27 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:27.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:27 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:27.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:27 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:27.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:27 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.d", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-19T01:38:27.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:27 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.d", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-19T01:38:27.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:27 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:38:27.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:27 smithi115 ceph-mon[67332]: Deploying daemon mds.d on smithi099 2024-03-19T01:38:27.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:27 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/3435102580' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:38:27.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:27 smithi115 ceph-mon[67332]: pgmap v263: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:27.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:27 smithi115 ceph-mon[67332]: mds.? [v2:172.21.15.99:6834/1631815621,v1:172.21.15.99:6835/1631815621] up:boot 2024-03-19T01:38:27.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:27 smithi115 ceph-mon[67332]: fsmap 3 up:standby 2024-03-19T01:38:27.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:27 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "j"}]: dispatch 2024-03-19T01:38:27.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:27 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:27.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:27 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:27.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:27 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:27.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:27 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.d", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-19T01:38:27.613 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:27 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.d", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-19T01:38:27.613 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:27 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:38:27.613 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:27 smithi112 ceph-mon[66819]: Deploying daemon mds.d on smithi099 2024-03-19T01:38:27.613 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:27 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/3435102580' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:38:27.613 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:27 smithi112 ceph-mon[66819]: pgmap v263: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:27.613 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:27 smithi112 ceph-mon[66819]: mds.? [v2:172.21.15.99:6834/1631815621,v1:172.21.15.99:6835/1631815621] up:boot 2024-03-19T01:38:27.613 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:27 smithi112 ceph-mon[66819]: fsmap 3 up:standby 2024-03-19T01:38:27.613 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:27 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "j"}]: dispatch 2024-03-19T01:38:28.341 INFO:journalctl@ceph.mds.d.smithi099.stdout:Mar 19 01:38:28 smithi099 systemd[1]: Starting Ceph mds.d for 438fb69c-e590-11ee-95c9-87774f69a715... 2024-03-19T01:38:28.673 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-19T01:38:28.762 INFO:journalctl@ceph.mds.d.smithi099.stdout:Mar 19 01:38:28 smithi099 podman[103766]: 2024-03-19 01:38:28.47637837 +0000 UTC m=+0.110716710 container create 3051f64272da14623a4fdedb0aea3b46e2dc28e0e793865d93e6baf863e49fcf (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-d, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240307, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, CEPH_POINT_RELEASE=) 2024-03-19T01:38:28.762 INFO:journalctl@ceph.mds.d.smithi099.stdout:Mar 19 01:38:28 smithi099 podman[103766]: 2024-03-19 01:38:28.384053631 +0000 UTC m=+0.018391984 image pull 2312c8e42d0069a2dbb2364bddb00264db4b800e7f18a7e7e4eb8f3ecab2ff1c quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639 2024-03-19T01:38:28.783 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:28.781+0000 7fd397d53640 1 -- 172.21.15.99:0/2568163152 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd390102300 msgr2=0x7fd390108b90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:28.783 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:28.781+0000 7fd397d53640 1 --2- 172.21.15.99:0/2568163152 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd390102300 0x7fd390108b90 secure :-1 s=READY pgs=162 cs=0 l=1 rev1=1 crypto rx=0x7fd388009ca0 tx=0x7fd38802c0f0 comp rx=0 tx=0).stop 2024-03-19T01:38:28.784 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:28.781+0000 7fd397d53640 1 -- 172.21.15.99:0/2568163152 shutdown_connections 2024-03-19T01:38:28.784 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:28.781+0000 7fd397d53640 1 --2- 172.21.15.99:0/2568163152 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd390102300 0x7fd390108b90 unknown :-1 s=CLOSED pgs=162 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:28.784 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:28.781+0000 7fd397d53640 1 --2- 172.21.15.99:0/2568163152 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd390101940 0x7fd390101dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:28.784 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:28.781+0000 7fd397d53640 1 --2- 172.21.15.99:0/2568163152 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd390100750 0x7fd390100b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:28.784 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:28.781+0000 7fd397d53640 1 -- 172.21.15.99:0/2568163152 >> 172.21.15.99:0/2568163152 conn(0x7fd3900fbd00 msgr2=0x7fd3900fe120 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:28.784 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:28.781+0000 7fd397d53640 1 -- 172.21.15.99:0/2568163152 shutdown_connections 2024-03-19T01:38:28.784 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:28.781+0000 7fd397d53640 1 -- 172.21.15.99:0/2568163152 wait complete. 2024-03-19T01:38:28.784 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:28.782+0000 7fd397d53640 1 Processor -- start 2024-03-19T01:38:28.784 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:28.782+0000 7fd397d53640 1 -- start start 2024-03-19T01:38:28.784 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:28.782+0000 7fd397d53640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd390100750 0x7fd390104750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:28.784 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:28.782+0000 7fd397d53640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd390101940 0x7fd390102d90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:28.784 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:28.782+0000 7fd397d53640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd390102300 0x7fd3901032d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:28.784 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:28.782+0000 7fd397d53640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fd39010c3d0 con 0x7fd390102300 2024-03-19T01:38:28.784 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:28.782+0000 7fd397d53640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fd39010c250 con 0x7fd390101940 2024-03-19T01:38:28.785 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:28.782+0000 7fd397d53640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fd39010c550 con 0x7fd390100750 2024-03-19T01:38:28.785 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:28.782+0000 7fd3962f0640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd390102300 0x7fd3901032d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:28.785 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:28.782+0000 7fd3962f0640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd390102300 0x7fd3901032d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:48826/0 (socket says 172.21.15.99:48826) 2024-03-19T01:38:28.785 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:28.782+0000 7fd3962f0640 1 -- 172.21.15.99:0/3542394813 learned_addr learned my addr 172.21.15.99:0/3542394813 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:38:28.785 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:28.782+0000 7fd395aef640 1 --2- 172.21.15.99:0/3542394813 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd390100750 0x7fd390104750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:28.785 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:28.782+0000 7fd3952ee640 1 --2- 172.21.15.99:0/3542394813 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd390101940 0x7fd390102d90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:28.785 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:28.782+0000 7fd3962f0640 1 -- 172.21.15.99:0/3542394813 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd390100750 msgr2=0x7fd390104750 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:28.785 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:28.782+0000 7fd3962f0640 1 --2- 172.21.15.99:0/3542394813 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd390100750 0x7fd390104750 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:28.785 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:28.782+0000 7fd3962f0640 1 -- 172.21.15.99:0/3542394813 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd390101940 msgr2=0x7fd390102d90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:28.785 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:28.782+0000 7fd3962f0640 1 --2- 172.21.15.99:0/3542394813 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd390101940 0x7fd390102d90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:28.785 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:28.782+0000 7fd3962f0640 1 -- 172.21.15.99:0/3542394813 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd390103b60 con 0x7fd390102300 2024-03-19T01:38:28.785 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:28.783+0000 7fd3962f0640 1 --2- 172.21.15.99:0/3542394813 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd390102300 0x7fd3901032d0 secure :-1 s=READY pgs=163 cs=0 l=1 rev1=1 crypto rx=0x7fd38802c700 tx=0x7fd388032f20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:28.785 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:28.783+0000 7fd386ffd640 1 -- 172.21.15.99:0/3542394813 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd388039040 con 0x7fd390102300 2024-03-19T01:38:28.786 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:28.783+0000 7fd386ffd640 1 -- 172.21.15.99:0/3542394813 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd388044070 con 0x7fd390102300 2024-03-19T01:38:28.786 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:28.783+0000 7fd397d53640 1 -- 172.21.15.99:0/3542394813 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd3901a1bd0 con 0x7fd390102300 2024-03-19T01:38:28.786 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:28.783+0000 7fd386ffd640 1 -- 172.21.15.99:0/3542394813 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd3880042a0 con 0x7fd390102300 2024-03-19T01:38:28.786 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:28.783+0000 7fd397d53640 1 -- 172.21.15.99:0/3542394813 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fd3901a2120 con 0x7fd390102300 2024-03-19T01:38:28.787 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:28.784+0000 7fd397d53640 1 -- 172.21.15.99:0/3542394813 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fd3901a22f0 con 0x7fd390102300 2024-03-19T01:38:28.787 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:28.785+0000 7fd386ffd640 1 -- 172.21.15.99:0/3542394813 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 16) ==== 137290+0+0 (secure 0 0 0) 0x7fd388003970 con 0x7fd390102300 2024-03-19T01:38:28.787 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:28.785+0000 7fd386ffd640 1 --2- 172.21.15.99:0/3542394813 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd37c0a2800 0x7fd37c0a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:28.787 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:28.786+0000 7fd395aef640 1 --2- 172.21.15.99:0/3542394813 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd37c0a2800 0x7fd37c0a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:28.787 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:28.786+0000 7fd386ffd640 1 -- 172.21.15.99:0/3542394813 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(83..83 src has 1..83) ==== 7127+0+0 (secure 0 0 0) 0x7fd3880f0a00 con 0x7fd390102300 2024-03-19T01:38:28.787 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:28.786+0000 7fd395aef640 1 --2- 172.21.15.99:0/3542394813 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd37c0a2800 0x7fd37c0a4cc0 secure :-1 s=READY pgs=136 cs=0 l=1 rev1=1 crypto rx=0x7fd3901017a0 tx=0x7fd38000a2d0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:28.793 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:28.792+0000 7fd386ffd640 1 -- 172.21.15.99:0/3542394813 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7fd3880bdfd0 con 0x7fd390102300 2024-03-19T01:38:29.013 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:29.013+0000 7fd397d53640 1 -- 172.21.15.99:0/3542394813 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7fd3901a2680 con 0x7fd390102300 2024-03-19T01:38:29.014 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:29.013+0000 7fd386ffd640 1 -- 172.21.15.99:0/3542394813 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 4 v4) ==== 93+0+3049 (secure 0 0 0) 0x7fd3880bd8d0 con 0x7fd390102300 2024-03-19T01:38:29.014 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:38:29.014 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":4,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14562,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6835/1631815621","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6834","nonce":1631815621},{"type":"v1","addr":"172.21.15.99:6835","nonce":1631815621}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":4},{"gid":24383,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.112:6833/2969810977","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6832","nonce":2969810977},{"type":"v1","addr":"172.21.15.112:6833","nonce":2969810977}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3},{"gid":24415,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.115:6833/2219320481","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6832","nonce":2219320481},{"type":"v1","addr":"172.21.15.115:6833","nonce":2219320481}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2}],"filesystems":[]} 2024-03-19T01:38:29.014 INFO:teuthology.orchestra.run.smithi099.stderr:dumped fsmap epoch 4 2024-03-19T01:38:29.018 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:29.015+0000 7fd397d53640 1 -- 172.21.15.99:0/3542394813 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd37c0a2800 msgr2=0x7fd37c0a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:29.018 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:29.015+0000 7fd397d53640 1 --2- 172.21.15.99:0/3542394813 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd37c0a2800 0x7fd37c0a4cc0 secure :-1 s=READY pgs=136 cs=0 l=1 rev1=1 crypto rx=0x7fd3901017a0 tx=0x7fd38000a2d0 comp rx=0 tx=0).stop 2024-03-19T01:38:29.018 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:29.015+0000 7fd397d53640 1 -- 172.21.15.99:0/3542394813 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd390102300 msgr2=0x7fd3901032d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:29.018 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:29.015+0000 7fd397d53640 1 --2- 172.21.15.99:0/3542394813 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd390102300 0x7fd3901032d0 secure :-1 s=READY pgs=163 cs=0 l=1 rev1=1 crypto rx=0x7fd38802c700 tx=0x7fd388032f20 comp rx=0 tx=0).stop 2024-03-19T01:38:29.018 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:29.015+0000 7fd397d53640 1 -- 172.21.15.99:0/3542394813 shutdown_connections 2024-03-19T01:38:29.018 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:29.015+0000 7fd397d53640 1 --2- 172.21.15.99:0/3542394813 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd37c0a2800 0x7fd37c0a4cc0 unknown :-1 s=CLOSED pgs=136 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:29.018 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:29.015+0000 7fd397d53640 1 --2- 172.21.15.99:0/3542394813 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd390101940 0x7fd390102d90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:29.018 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:29.015+0000 7fd397d53640 1 --2- 172.21.15.99:0/3542394813 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd390102300 0x7fd3901032d0 unknown :-1 s=CLOSED pgs=163 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:29.018 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:29.015+0000 7fd397d53640 1 --2- 172.21.15.99:0/3542394813 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd390100750 0x7fd390104750 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:29.018 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:29.015+0000 7fd397d53640 1 -- 172.21.15.99:0/3542394813 >> 172.21.15.99:0/3542394813 conn(0x7fd3900fbd00 msgr2=0x7fd390069e90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:29.019 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:29.015+0000 7fd397d53640 1 -- 172.21.15.99:0/3542394813 shutdown_connections 2024-03-19T01:38:29.019 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:29.015+0000 7fd397d53640 1 -- 172.21.15.99:0/3542394813 wait complete. 2024-03-19T01:38:29.026 INFO:journalctl@ceph.mds.d.smithi099.stdout:Mar 19 01:38:28 smithi099 podman[103766]: 2024-03-19 01:38:28.762747573 +0000 UTC m=+0.397085933 container init 3051f64272da14623a4fdedb0aea3b46e2dc28e0e793865d93e6baf863e49fcf (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-d, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, CEPH_POINT_RELEASE=, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240307, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD) 2024-03-19T01:38:29.026 INFO:journalctl@ceph.mds.d.smithi099.stdout:Mar 19 01:38:28 smithi099 podman[103766]: 2024-03-19 01:38:28.767105644 +0000 UTC m=+0.401443999 container start 3051f64272da14623a4fdedb0aea3b46e2dc28e0e793865d93e6baf863e49fcf (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-d, GIT_BRANCH=HEAD, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, ceph=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240307, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-03-19T01:38:29.026 INFO:journalctl@ceph.mds.d.smithi099.stdout:Mar 19 01:38:28 smithi099 ceph-mds[103805]: -- 172.21.15.99:0/2844019747 <== mon.0 v2:172.21.15.99:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56511d719a40 con 0x56511d6ec800 2024-03-19T01:38:29.026 INFO:journalctl@ceph.mds.d.smithi099.stdout:Mar 19 01:38:28 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-d[103781]: starting mds.d at 2024-03-19T01:38:29.026 INFO:journalctl@ceph.mds.d.smithi099.stdout:Mar 19 01:38:28 smithi099 bash[103766]: 3051f64272da14623a4fdedb0aea3b46e2dc28e0e793865d93e6baf863e49fcf 2024-03-19T01:38:29.026 INFO:journalctl@ceph.mds.d.smithi099.stdout:Mar 19 01:38:28 smithi099 systemd[1]: Started Ceph mds.d for 438fb69c-e590-11ee-95c9-87774f69a715. 2024-03-19T01:38:29.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:29 smithi112 ceph-mon[66819]: pgmap v264: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:29.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:29 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:29.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:29 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:29.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:29 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:29.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:29 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.e", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-19T01:38:29.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:29 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.e", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-19T01:38:29.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:29 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:38:29.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:29 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/3542394813' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:38:30.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:29 smithi115 ceph-mon[67332]: pgmap v264: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:30.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:29 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:30.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:29 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:30.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:29 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:30.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:29 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.e", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-19T01:38:30.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:29 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.e", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-19T01:38:30.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:29 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:38:30.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:29 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/3542394813' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:38:30.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:29 smithi099 ceph-mon[62951]: pgmap v264: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:30.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:29 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:30.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:29 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:30.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:29 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:30.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:29 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.e", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-19T01:38:30.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:29 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.e", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-19T01:38:30.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:29 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:38:30.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:29 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/3542394813' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:38:31.029 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-19T01:38:31.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:30 smithi115 ceph-mon[67332]: Deploying daemon mds.e on smithi112 2024-03-19T01:38:31.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:30 smithi115 ceph-mon[67332]: mds.? [v2:172.21.15.99:6836/2038634879,v1:172.21.15.99:6837/2038634879] up:boot 2024-03-19T01:38:31.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:30 smithi115 ceph-mon[67332]: fsmap 4 up:standby 2024-03-19T01:38:31.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:30 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "d"}]: dispatch 2024-03-19T01:38:31.057 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:30 smithi099 ceph-mon[62951]: Deploying daemon mds.e on smithi112 2024-03-19T01:38:31.058 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:30 smithi099 ceph-mon[62951]: mds.? [v2:172.21.15.99:6836/2038634879,v1:172.21.15.99:6837/2038634879] up:boot 2024-03-19T01:38:31.058 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:30 smithi099 ceph-mon[62951]: fsmap 4 up:standby 2024-03-19T01:38:31.058 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:30 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "d"}]: dispatch 2024-03-19T01:38:31.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:30 smithi112 ceph-mon[66819]: Deploying daemon mds.e on smithi112 2024-03-19T01:38:31.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:30 smithi112 ceph-mon[66819]: mds.? [v2:172.21.15.99:6836/2038634879,v1:172.21.15.99:6837/2038634879] up:boot 2024-03-19T01:38:31.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:30 smithi112 ceph-mon[66819]: fsmap 4 up:standby 2024-03-19T01:38:31.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:30 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "d"}]: dispatch 2024-03-19T01:38:31.113 INFO:journalctl@ceph.mds.e.smithi112.stdout:Mar 19 01:38:30 smithi112 systemd[1]: Starting Ceph mds.e for 438fb69c-e590-11ee-95c9-87774f69a715... 2024-03-19T01:38:31.145 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:31.143+0000 7f6409345640 1 -- 172.21.15.99:0/4181206484 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6404101950 msgr2=0x7f6404101dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:31.146 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:31.143+0000 7f6409345640 1 --2- 172.21.15.99:0/4181206484 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6404101950 0x7f6404101dd0 secure :-1 s=READY pgs=165 cs=0 l=1 rev1=1 crypto rx=0x7f63f4009900 tx=0x7f63f402bc30 comp rx=0 tx=0).stop 2024-03-19T01:38:31.146 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:31.143+0000 7f6409345640 1 -- 172.21.15.99:0/4181206484 shutdown_connections 2024-03-19T01:38:31.146 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:31.143+0000 7f6409345640 1 --2- 172.21.15.99:0/4181206484 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f6404102310 0x7f6404108ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:31.146 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:31.143+0000 7f6409345640 1 --2- 172.21.15.99:0/4181206484 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6404101950 0x7f6404101dd0 unknown :-1 s=CLOSED pgs=165 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:31.146 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:31.143+0000 7f6409345640 1 --2- 172.21.15.99:0/4181206484 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6404100760 0x7f6404100b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:31.146 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:31.143+0000 7f6409345640 1 -- 172.21.15.99:0/4181206484 >> 172.21.15.99:0/4181206484 conn(0x7f64040fbcd0 msgr2=0x7f64040fe110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:31.146 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:31.143+0000 7f6409345640 1 -- 172.21.15.99:0/4181206484 shutdown_connections 2024-03-19T01:38:31.146 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:31.143+0000 7f6409345640 1 -- 172.21.15.99:0/4181206484 wait complete. 2024-03-19T01:38:31.146 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:31.143+0000 7f6409345640 1 Processor -- start 2024-03-19T01:38:31.146 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:31.143+0000 7f6409345640 1 -- start start 2024-03-19T01:38:31.146 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:31.144+0000 7f6409345640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6404100760 0x7f6404196a30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:31.146 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:31.144+0000 7f6409345640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f6404101950 0x7f6404196f70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:31.147 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:31.144+0000 7f6409345640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6404102310 0x7f640419dfe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:31.147 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:31.144+0000 7f6409345640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f640410c3c0 con 0x7f6404100760 2024-03-19T01:38:31.147 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:31.144+0000 7f6409345640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f640410c240 con 0x7f6404102310 2024-03-19T01:38:31.147 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:31.144+0000 7f6409345640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f640410c540 con 0x7f6404101950 2024-03-19T01:38:31.147 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:31.144+0000 7f6402ffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6404100760 0x7f6404196a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:31.147 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:31.144+0000 7f6402ffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6404100760 0x7f6404196a30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:48864/0 (socket says 172.21.15.99:48864) 2024-03-19T01:38:31.147 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:31.144+0000 7f6402ffd640 1 -- 172.21.15.99:0/3187640446 learned_addr learned my addr 172.21.15.99:0/3187640446 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:38:31.147 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:31.144+0000 7f64037fe640 1 --2- 172.21.15.99:0/3187640446 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6404102310 0x7f640419dfe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:31.147 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:31.144+0000 7f6402ffd640 1 -- 172.21.15.99:0/3187640446 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f6404101950 msgr2=0x7f6404196f70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:31.147 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:31.144+0000 7f6402ffd640 1 --2- 172.21.15.99:0/3187640446 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f6404101950 0x7f6404196f70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:31.147 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:31.144+0000 7f6402ffd640 1 -- 172.21.15.99:0/3187640446 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6404102310 msgr2=0x7f640419dfe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:31.147 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:31.144+0000 7f6402ffd640 1 --2- 172.21.15.99:0/3187640446 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6404102310 0x7f640419dfe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:31.147 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:31.144+0000 7f6402ffd640 1 -- 172.21.15.99:0/3187640446 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f640419e650 con 0x7f6404100760 2024-03-19T01:38:31.148 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:31.144+0000 7f64037fe640 1 --2- 172.21.15.99:0/3187640446 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6404102310 0x7f640419dfe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-19T01:38:31.148 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:31.144+0000 7f6402ffd640 1 --2- 172.21.15.99:0/3187640446 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6404100760 0x7f6404196a30 secure :-1 s=READY pgs=166 cs=0 l=1 rev1=1 crypto rx=0x7f63ec00e9c0 tx=0x7f63ec00ee80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:31.148 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:31.144+0000 7f63fbfff640 1 -- 172.21.15.99:0/3187640446 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f63ec00cd80 con 0x7f6404100760 2024-03-19T01:38:31.148 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:31.144+0000 7f6409345640 1 -- 172.21.15.99:0/3187640446 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f640419e940 con 0x7f6404100760 2024-03-19T01:38:31.148 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:31.144+0000 7f63fbfff640 1 -- 172.21.15.99:0/3187640446 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f63ec0035f0 con 0x7f6404100760 2024-03-19T01:38:31.148 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:31.144+0000 7f63fbfff640 1 -- 172.21.15.99:0/3187640446 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f63ec010690 con 0x7f6404100760 2024-03-19T01:38:31.148 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:31.144+0000 7f6409345640 1 -- 172.21.15.99:0/3187640446 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f640419eee0 con 0x7f6404100760 2024-03-19T01:38:31.148 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:31.145+0000 7f6409345640 1 -- 172.21.15.99:0/3187640446 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f6404107b00 con 0x7f6404100760 2024-03-19T01:38:31.149 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:31.146+0000 7f63fbfff640 1 -- 172.21.15.99:0/3187640446 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 16) ==== 137290+0+0 (secure 0 0 0) 0x7f63ec0037a0 con 0x7f6404100760 2024-03-19T01:38:31.149 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:31.147+0000 7f63fbfff640 1 --2- 172.21.15.99:0/3187640446 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f63f00a2800 0x7f63f00a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:31.149 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:31.147+0000 7f64027fc640 1 --2- 172.21.15.99:0/3187640446 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f63f00a2800 0x7f63f00a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:31.149 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:31.147+0000 7f63fbfff640 1 -- 172.21.15.99:0/3187640446 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(83..83 src has 1..83) ==== 7127+0+0 (secure 0 0 0) 0x7f63ec0ceb30 con 0x7f6404100760 2024-03-19T01:38:31.149 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:31.148+0000 7f64027fc640 1 --2- 172.21.15.99:0/3187640446 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f63f00a2800 0x7f63f00a4cc0 secure :-1 s=READY pgs=138 cs=0 l=1 rev1=1 crypto rx=0x7f63f4036040 tx=0x7f63f4000bc0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:31.151 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:31.151+0000 7f63fbfff640 1 -- 172.21.15.99:0/3187640446 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f63ec09c100 con 0x7f6404100760 2024-03-19T01:38:31.365 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:31.365+0000 7f6409345640 1 -- 172.21.15.99:0/3187640446 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f640402d450 con 0x7f6404100760 2024-03-19T01:38:31.366 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:31.365+0000 7f63fbfff640 1 -- 172.21.15.99:0/3187640446 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 5 v5) ==== 93+0+3874 (secure 0 0 0) 0x7f63ec09ba00 con 0x7f6404100760 2024-03-19T01:38:31.366 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:38:31.366 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":5,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14562,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6835/1631815621","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6834","nonce":1631815621},{"type":"v1","addr":"172.21.15.99:6835","nonce":1631815621}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":4},{"gid":24383,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.112:6833/2969810977","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6832","nonce":2969810977},{"type":"v1","addr":"172.21.15.112:6833","nonce":2969810977}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3},{"gid":24404,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6837/2038634879","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6836","nonce":2038634879},{"type":"v1","addr":"172.21.15.99:6837","nonce":2038634879}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":24415,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.115:6833/2219320481","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6832","nonce":2219320481},{"type":"v1","addr":"172.21.15.115:6833","nonce":2219320481}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2}],"filesystems":[]} 2024-03-19T01:38:31.367 INFO:teuthology.orchestra.run.smithi099.stderr:dumped fsmap epoch 5 2024-03-19T01:38:31.368 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:31.367+0000 7f6409345640 1 -- 172.21.15.99:0/3187640446 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f63f00a2800 msgr2=0x7f63f00a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:31.368 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:31.367+0000 7f6409345640 1 --2- 172.21.15.99:0/3187640446 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f63f00a2800 0x7f63f00a4cc0 secure :-1 s=READY pgs=138 cs=0 l=1 rev1=1 crypto rx=0x7f63f4036040 tx=0x7f63f4000bc0 comp rx=0 tx=0).stop 2024-03-19T01:38:31.369 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:31.367+0000 7f6409345640 1 -- 172.21.15.99:0/3187640446 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6404100760 msgr2=0x7f6404196a30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:31.369 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:31.367+0000 7f6409345640 1 --2- 172.21.15.99:0/3187640446 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6404100760 0x7f6404196a30 secure :-1 s=READY pgs=166 cs=0 l=1 rev1=1 crypto rx=0x7f63ec00e9c0 tx=0x7f63ec00ee80 comp rx=0 tx=0).stop 2024-03-19T01:38:31.369 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:31.367+0000 7f6409345640 1 -- 172.21.15.99:0/3187640446 shutdown_connections 2024-03-19T01:38:31.369 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:31.367+0000 7f6409345640 1 --2- 172.21.15.99:0/3187640446 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f63f00a2800 0x7f63f00a4cc0 unknown :-1 s=CLOSED pgs=138 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:31.369 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:31.367+0000 7f6409345640 1 --2- 172.21.15.99:0/3187640446 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6404102310 0x7f640419dfe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:31.370 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:31.367+0000 7f6409345640 1 --2- 172.21.15.99:0/3187640446 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f6404101950 0x7f6404196f70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:31.370 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:31.367+0000 7f6409345640 1 --2- 172.21.15.99:0/3187640446 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6404100760 0x7f6404196a30 unknown :-1 s=CLOSED pgs=166 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:31.370 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:31.367+0000 7f6409345640 1 -- 172.21.15.99:0/3187640446 >> 172.21.15.99:0/3187640446 conn(0x7f64040fbcd0 msgr2=0x7f6404104cd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:31.370 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:31.367+0000 7f6409345640 1 -- 172.21.15.99:0/3187640446 shutdown_connections 2024-03-19T01:38:31.370 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:31.367+0000 7f6409345640 1 -- 172.21.15.99:0/3187640446 wait complete. 2024-03-19T01:38:31.418 INFO:journalctl@ceph.mds.e.smithi112.stdout:Mar 19 01:38:31 smithi112 podman[96758]: 2024-03-19 01:38:31.148263544 +0000 UTC m=+0.099606766 container create 3378ef045bc81c8fad3b97487361d96b709ccb0f59097b080e6356d3ad8ca902 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-e, org.label-schema.build-date=20240307, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-19T01:38:31.418 INFO:journalctl@ceph.mds.e.smithi112.stdout:Mar 19 01:38:31 smithi112 podman[96758]: 2024-03-19 01:38:31.0660555 +0000 UTC m=+0.017398723 image pull 2312c8e42d0069a2dbb2364bddb00264db4b800e7f18a7e7e4eb8f3ecab2ff1c quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639 2024-03-19T01:38:31.418 INFO:journalctl@ceph.mds.e.smithi112.stdout:Mar 19 01:38:31 smithi112 podman[96758]: 2024-03-19 01:38:31.359096611 +0000 UTC m=+0.310439837 container init 3378ef045bc81c8fad3b97487361d96b709ccb0f59097b080e6356d3ad8ca902 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-e, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240307, org.label-schema.vendor=CentOS) 2024-03-19T01:38:31.418 INFO:journalctl@ceph.mds.e.smithi112.stdout:Mar 19 01:38:31 smithi112 podman[96758]: 2024-03-19 01:38:31.361682265 +0000 UTC m=+0.313025480 container start 3378ef045bc81c8fad3b97487361d96b709ccb0f59097b080e6356d3ad8ca902 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-e, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240307, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_BRANCH=HEAD) 2024-03-19T01:38:31.418 INFO:journalctl@ceph.mds.e.smithi112.stdout:Mar 19 01:38:31 smithi112 ceph-mds[96777]: -- 172.21.15.112:0/120661537 <== mon.2 v2:172.21.15.112:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5618bdf6da40 con 0x5618bdf41000 2024-03-19T01:38:31.418 INFO:journalctl@ceph.mds.e.smithi112.stdout:Mar 19 01:38:31 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-e[96773]: starting mds.e at 2024-03-19T01:38:31.418 INFO:journalctl@ceph.mds.e.smithi112.stdout:Mar 19 01:38:31 smithi112 bash[96758]: 3378ef045bc81c8fad3b97487361d96b709ccb0f59097b080e6356d3ad8ca902 2024-03-19T01:38:31.803 INFO:journalctl@ceph.mds.e.smithi112.stdout:Mar 19 01:38:31 smithi112 systemd[1]: Started Ceph mds.e for 438fb69c-e590-11ee-95c9-87774f69a715. 2024-03-19T01:38:32.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:31 smithi115 ceph-mon[67332]: pgmap v265: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:32.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:31 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/3187640446' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:38:32.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:31 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:32.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:31 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:32.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:31 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:32.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:31 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.f", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-19T01:38:32.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:31 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.f", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-19T01:38:32.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:31 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:38:32.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:31 smithi099 ceph-mon[62951]: pgmap v265: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:32.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:31 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/3187640446' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:38:32.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:31 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:32.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:31 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:32.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:31 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:32.093 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:31 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.f", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-19T01:38:32.093 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:31 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.f", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-19T01:38:32.093 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:31 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:38:32.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:31 smithi112 ceph-mon[66819]: pgmap v265: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:32.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:31 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/3187640446' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:38:32.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:31 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:32.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:31 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:32.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:31 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:32.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:31 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.f", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-19T01:38:32.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:31 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.f", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-19T01:38:32.113 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:31 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:38:33.084 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:32 smithi115 ceph-mon[67332]: Deploying daemon mds.f on smithi115 2024-03-19T01:38:33.084 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:32 smithi115 ceph-mon[67332]: mds.? [v2:172.21.15.112:6834/3690587528,v1:172.21.15.112:6835/3690587528] up:boot 2024-03-19T01:38:33.084 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:32 smithi115 ceph-mon[67332]: fsmap 5 up:standby 2024-03-19T01:38:33.084 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:32 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "e"}]: dispatch 2024-03-19T01:38:33.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:32 smithi099 ceph-mon[62951]: Deploying daemon mds.f on smithi115 2024-03-19T01:38:33.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:32 smithi099 ceph-mon[62951]: mds.? [v2:172.21.15.112:6834/3690587528,v1:172.21.15.112:6835/3690587528] up:boot 2024-03-19T01:38:33.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:32 smithi099 ceph-mon[62951]: fsmap 5 up:standby 2024-03-19T01:38:33.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:32 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "e"}]: dispatch 2024-03-19T01:38:33.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:32 smithi112 ceph-mon[66819]: Deploying daemon mds.f on smithi115 2024-03-19T01:38:33.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:32 smithi112 ceph-mon[66819]: mds.? [v2:172.21.15.112:6834/3690587528,v1:172.21.15.112:6835/3690587528] up:boot 2024-03-19T01:38:33.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:32 smithi112 ceph-mon[66819]: fsmap 5 up:standby 2024-03-19T01:38:33.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:32 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "e"}]: dispatch 2024-03-19T01:38:33.380 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-19T01:38:33.461 INFO:journalctl@ceph.mds.f.smithi115.stdout:Mar 19 01:38:33 smithi115 systemd[1]: Starting Ceph mds.f for 438fb69c-e590-11ee-95c9-87774f69a715... 2024-03-19T01:38:33.496 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:33.494+0000 7fca0e6a7640 1 -- 172.21.15.99:0/3324725939 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fca08100760 msgr2=0x7fca08100b80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:33.496 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:33.494+0000 7fca0e6a7640 1 --2- 172.21.15.99:0/3324725939 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fca08100760 0x7fca08100b80 secure :-1 s=READY pgs=167 cs=0 l=1 rev1=1 crypto rx=0x7fc9f8009900 tx=0x7fc9f802bc30 comp rx=0 tx=0).stop 2024-03-19T01:38:33.496 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:33.494+0000 7fca0e6a7640 1 -- 172.21.15.99:0/3324725939 shutdown_connections 2024-03-19T01:38:33.496 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:33.494+0000 7fca0e6a7640 1 --2- 172.21.15.99:0/3324725939 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fca08102310 0x7fca08108ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:33.496 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:33.494+0000 7fca0e6a7640 1 --2- 172.21.15.99:0/3324725939 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fca08101950 0x7fca08101dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:33.496 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:33.494+0000 7fca0e6a7640 1 --2- 172.21.15.99:0/3324725939 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fca08100760 0x7fca08100b80 unknown :-1 s=CLOSED pgs=167 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:33.497 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:33.494+0000 7fca0e6a7640 1 -- 172.21.15.99:0/3324725939 >> 172.21.15.99:0/3324725939 conn(0x7fca080fbcd0 msgr2=0x7fca080fe110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:33.497 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:33.494+0000 7fca0e6a7640 1 -- 172.21.15.99:0/3324725939 shutdown_connections 2024-03-19T01:38:33.497 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:33.494+0000 7fca0e6a7640 1 -- 172.21.15.99:0/3324725939 wait complete. 2024-03-19T01:38:33.497 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:33.494+0000 7fca0e6a7640 1 Processor -- start 2024-03-19T01:38:33.497 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:33.494+0000 7fca0e6a7640 1 -- start start 2024-03-19T01:38:33.497 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:33.495+0000 7fca0e6a7640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fca08100760 0x7fca08197c20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:33.497 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:33.495+0000 7fca0e6a7640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fca08101950 0x7fca08198160 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:33.497 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:33.495+0000 7fca0e6a7640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fca08102310 0x7fca0819f180 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:33.497 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:33.495+0000 7fca0e6a7640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fca0810c3b0 con 0x7fca08100760 2024-03-19T01:38:33.497 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:33.495+0000 7fca0e6a7640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fca0810c230 con 0x7fca08101950 2024-03-19T01:38:33.497 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:33.495+0000 7fca0e6a7640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fca0810c530 con 0x7fca08102310 2024-03-19T01:38:33.497 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:33.495+0000 7fca07fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fca08100760 0x7fca08197c20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:33.497 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:33.495+0000 7fca07fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fca08100760 0x7fca08197c20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:51250/0 (socket says 172.21.15.99:51250) 2024-03-19T01:38:33.498 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:33.495+0000 7fca07fff640 1 -- 172.21.15.99:0/603476246 learned_addr learned my addr 172.21.15.99:0/603476246 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:38:33.498 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:33.495+0000 7fca077fe640 1 --2- 172.21.15.99:0/603476246 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fca08101950 0x7fca08198160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:33.498 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:33.495+0000 7fca0cc44640 1 --2- 172.21.15.99:0/603476246 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fca08102310 0x7fca0819f180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:33.498 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:33.495+0000 7fca07fff640 1 -- 172.21.15.99:0/603476246 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fca08102310 msgr2=0x7fca0819f180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:33.498 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:33.495+0000 7fca07fff640 1 --2- 172.21.15.99:0/603476246 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fca08102310 0x7fca0819f180 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:33.498 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:33.495+0000 7fca07fff640 1 -- 172.21.15.99:0/603476246 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fca08101950 msgr2=0x7fca08198160 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:33.498 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:33.495+0000 7fca07fff640 1 --2- 172.21.15.99:0/603476246 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fca08101950 0x7fca08198160 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:33.498 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:33.495+0000 7fca07fff640 1 -- 172.21.15.99:0/603476246 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fca0819f7f0 con 0x7fca08100760 2024-03-19T01:38:33.498 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:33.495+0000 7fca0cc44640 1 --2- 172.21.15.99:0/603476246 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fca08102310 0x7fca0819f180 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-19T01:38:33.498 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:33.495+0000 7fca07fff640 1 --2- 172.21.15.99:0/603476246 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fca08100760 0x7fca08197c20 secure :-1 s=READY pgs=168 cs=0 l=1 rev1=1 crypto rx=0x7fc9f8000c00 tx=0x7fc9f8005d50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:33.498 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:33.495+0000 7fca057fa640 1 -- 172.21.15.99:0/603476246 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc9f8041070 con 0x7fca08100760 2024-03-19T01:38:33.498 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:33.495+0000 7fca0e6a7640 1 -- 172.21.15.99:0/603476246 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fca0819fa80 con 0x7fca08100760 2024-03-19T01:38:33.499 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:33.495+0000 7fca057fa640 1 -- 172.21.15.99:0/603476246 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fc9f80044d0 con 0x7fca08100760 2024-03-19T01:38:33.499 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:33.495+0000 7fca057fa640 1 -- 172.21.15.99:0/603476246 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc9f8034440 con 0x7fca08100760 2024-03-19T01:38:33.499 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:33.495+0000 7fca0e6a7640 1 -- 172.21.15.99:0/603476246 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fca0819ffc0 con 0x7fca08100760 2024-03-19T01:38:33.500 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:33.497+0000 7fca0e6a7640 1 -- 172.21.15.99:0/603476246 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fca081a01f0 con 0x7fca08100760 2024-03-19T01:38:33.500 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:33.498+0000 7fca057fa640 1 -- 172.21.15.99:0/603476246 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 16) ==== 137290+0+0 (secure 0 0 0) 0x7fc9f8044070 con 0x7fca08100760 2024-03-19T01:38:33.500 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:33.498+0000 7fca057fa640 1 --2- 172.21.15.99:0/603476246 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fc9f40a2800 0x7fc9f40a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:33.500 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:33.498+0000 7fca077fe640 1 --2- 172.21.15.99:0/603476246 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fc9f40a2800 0x7fc9f40a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:33.500 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:33.498+0000 7fca057fa640 1 -- 172.21.15.99:0/603476246 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(83..83 src has 1..83) ==== 7127+0+0 (secure 0 0 0) 0x7fc9f80f0160 con 0x7fca08100760 2024-03-19T01:38:33.500 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:33.499+0000 7fca077fe640 1 --2- 172.21.15.99:0/603476246 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fc9f40a2800 0x7fc9f40a4cc0 secure :-1 s=READY pgs=140 cs=0 l=1 rev1=1 crypto rx=0x7fca081990e0 tx=0x7fc9f0005f90 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:33.503 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:33.503+0000 7fca057fa640 1 -- 172.21.15.99:0/603476246 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7fc9f80bd6a0 con 0x7fca08100760 2024-03-19T01:38:33.712 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:33.712+0000 7fca0e6a7640 1 -- 172.21.15.99:0/603476246 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7fca081a03c0 con 0x7fca08100760 2024-03-19T01:38:33.713 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:33.712+0000 7fca057fa640 1 -- 172.21.15.99:0/603476246 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 6 v6) ==== 93+0+4702 (secure 0 0 0) 0x7fc9f80bcfa0 con 0x7fca08100760 2024-03-19T01:38:33.713 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:38:33.713 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":6,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14562,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6835/1631815621","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6834","nonce":1631815621},{"type":"v1","addr":"172.21.15.99:6835","nonce":1631815621}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":4},{"gid":24383,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.112:6833/2969810977","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6832","nonce":2969810977},{"type":"v1","addr":"172.21.15.112:6833","nonce":2969810977}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3},{"gid":24404,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6837/2038634879","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6836","nonce":2038634879},{"type":"v1","addr":"172.21.15.99:6837","nonce":2038634879}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":24415,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.115:6833/2219320481","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6832","nonce":2219320481},{"type":"v1","addr":"172.21.15.115:6833","nonce":2219320481}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2},{"gid":24416,"name":"e","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.112:6835/3690587528","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6834","nonce":3690587528},{"type":"v1","addr":"172.21.15.112:6835","nonce":3690587528}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":6}],"filesystems":[]} 2024-03-19T01:38:33.714 INFO:teuthology.orchestra.run.smithi099.stderr:dumped fsmap epoch 6 2024-03-19T01:38:33.715 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:33.714+0000 7fca0e6a7640 1 -- 172.21.15.99:0/603476246 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fc9f40a2800 msgr2=0x7fc9f40a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:33.715 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:33.714+0000 7fca0e6a7640 1 --2- 172.21.15.99:0/603476246 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fc9f40a2800 0x7fc9f40a4cc0 secure :-1 s=READY pgs=140 cs=0 l=1 rev1=1 crypto rx=0x7fca081990e0 tx=0x7fc9f0005f90 comp rx=0 tx=0).stop 2024-03-19T01:38:33.715 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:33.714+0000 7fca0e6a7640 1 -- 172.21.15.99:0/603476246 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fca08100760 msgr2=0x7fca08197c20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:33.715 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:33.714+0000 7fca0e6a7640 1 --2- 172.21.15.99:0/603476246 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fca08100760 0x7fca08197c20 secure :-1 s=READY pgs=168 cs=0 l=1 rev1=1 crypto rx=0x7fc9f8000c00 tx=0x7fc9f8005d50 comp rx=0 tx=0).stop 2024-03-19T01:38:33.715 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:33.714+0000 7fca0e6a7640 1 -- 172.21.15.99:0/603476246 shutdown_connections 2024-03-19T01:38:33.715 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:33.714+0000 7fca0e6a7640 1 --2- 172.21.15.99:0/603476246 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fc9f40a2800 0x7fc9f40a4cc0 unknown :-1 s=CLOSED pgs=140 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:33.715 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:33.714+0000 7fca0e6a7640 1 --2- 172.21.15.99:0/603476246 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fca08101950 0x7fca08198160 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:33.715 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:33.714+0000 7fca0e6a7640 1 --2- 172.21.15.99:0/603476246 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fca08102310 0x7fca0819f180 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:33.715 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:33.714+0000 7fca0e6a7640 1 --2- 172.21.15.99:0/603476246 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fca08100760 0x7fca08197c20 unknown :-1 s=CLOSED pgs=168 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:33.715 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:33.714+0000 7fca0e6a7640 1 -- 172.21.15.99:0/603476246 >> 172.21.15.99:0/603476246 conn(0x7fca080fbcd0 msgr2=0x7fca08069e50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:33.716 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:33.714+0000 7fca0e6a7640 1 -- 172.21.15.99:0/603476246 shutdown_connections 2024-03-19T01:38:33.716 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:33.714+0000 7fca0e6a7640 1 -- 172.21.15.99:0/603476246 wait complete. 2024-03-19T01:38:33.771 INFO:journalctl@ceph.mds.f.smithi115.stdout:Mar 19 01:38:33 smithi115 podman[96830]: 2024-03-19 01:38:33.578034463 +0000 UTC m=+0.016751368 image pull 2312c8e42d0069a2dbb2364bddb00264db4b800e7f18a7e7e4eb8f3ecab2ff1c quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639 2024-03-19T01:38:33.771 INFO:journalctl@ceph.mds.f.smithi115.stdout:Mar 19 01:38:33 smithi115 podman[96830]: 2024-03-19 01:38:33.77191262 +0000 UTC m=+0.210629512 container create 8b886dd7bfadaf5c445cd81d1af10c7064a66a24cfe43d7ddc0e456b87a80206 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-f, maintainer=Guillaume Abrioux , GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240307, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True) 2024-03-19T01:38:34.042 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:33 smithi115 ceph-mon[67332]: pgmap v266: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:34.042 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:33 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/603476246' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:38:34.042 INFO:journalctl@ceph.mds.f.smithi115.stdout:Mar 19 01:38:34 smithi115 podman[96830]: 2024-03-19 01:38:34.016675864 +0000 UTC m=+0.455392771 container init 8b886dd7bfadaf5c445cd81d1af10c7064a66a24cfe43d7ddc0e456b87a80206 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-f, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240307) 2024-03-19T01:38:34.043 INFO:journalctl@ceph.mds.f.smithi115.stdout:Mar 19 01:38:34 smithi115 podman[96830]: 2024-03-19 01:38:34.019521261 +0000 UTC m=+0.458238160 container start 8b886dd7bfadaf5c445cd81d1af10c7064a66a24cfe43d7ddc0e456b87a80206 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-f, org.label-schema.build-date=20240307, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.schema-version=1.0, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2) 2024-03-19T01:38:34.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:33 smithi099 ceph-mon[62951]: pgmap v266: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:34.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:33 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/603476246' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:38:34.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:33 smithi112 ceph-mon[66819]: pgmap v266: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:34.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:33 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/603476246' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:38:34.306 INFO:journalctl@ceph.mds.f.smithi115.stdout:Mar 19 01:38:34 smithi115 ceph-mds[96849]: -- 172.21.15.115:0/109092441 <== mon.1 v2:172.21.15.115:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55b83488fa40 con 0x55b834863000 2024-03-19T01:38:34.306 INFO:journalctl@ceph.mds.f.smithi115.stdout:Mar 19 01:38:34 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-f[96845]: starting mds.f at 2024-03-19T01:38:34.307 INFO:journalctl@ceph.mds.f.smithi115.stdout:Mar 19 01:38:34 smithi115 bash[96830]: 8b886dd7bfadaf5c445cd81d1af10c7064a66a24cfe43d7ddc0e456b87a80206 2024-03-19T01:38:34.307 INFO:journalctl@ceph.mds.f.smithi115.stdout:Mar 19 01:38:34 smithi115 systemd[1]: Started Ceph mds.f for 438fb69c-e590-11ee-95c9-87774f69a715. 2024-03-19T01:38:35.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:35 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:35.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:35 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:35.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:35 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:35.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:35 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.c", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-19T01:38:35.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:35 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.c", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-19T01:38:35.558 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:35 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:38:35.558 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:35 smithi115 ceph-mon[67332]: Deploying daemon mds.c on smithi115 2024-03-19T01:38:35.558 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:35 smithi115 ceph-mon[67332]: pgmap v267: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:35.558 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:35 smithi115 ceph-mon[67332]: mds.? [v2:172.21.15.115:6834/591078413,v1:172.21.15.115:6835/591078413] up:boot 2024-03-19T01:38:35.558 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:35 smithi115 ceph-mon[67332]: fsmap 6 up:standby 2024-03-19T01:38:35.559 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:35 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "f"}]: dispatch 2024-03-19T01:38:35.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:35 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:35.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:35 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:35.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:35 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:35.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:35 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.c", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-19T01:38:35.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:35 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.c", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-19T01:38:35.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:35 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:38:35.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:35 smithi099 ceph-mon[62951]: Deploying daemon mds.c on smithi115 2024-03-19T01:38:35.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:35 smithi099 ceph-mon[62951]: pgmap v267: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:35.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:35 smithi099 ceph-mon[62951]: mds.? [v2:172.21.15.115:6834/591078413,v1:172.21.15.115:6835/591078413] up:boot 2024-03-19T01:38:35.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:35 smithi099 ceph-mon[62951]: fsmap 6 up:standby 2024-03-19T01:38:35.593 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:35 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "f"}]: dispatch 2024-03-19T01:38:35.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:35 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:35.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:35 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:35.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:35 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:35.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:35 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.c", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-19T01:38:35.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:35 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.c", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-19T01:38:35.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:35 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:38:35.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:35 smithi112 ceph-mon[66819]: Deploying daemon mds.c on smithi115 2024-03-19T01:38:35.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:35 smithi112 ceph-mon[66819]: pgmap v267: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:35.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:35 smithi112 ceph-mon[66819]: mds.? [v2:172.21.15.115:6834/591078413,v1:172.21.15.115:6835/591078413] up:boot 2024-03-19T01:38:35.613 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:35 smithi112 ceph-mon[66819]: fsmap 6 up:standby 2024-03-19T01:38:35.613 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:35 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "f"}]: dispatch 2024-03-19T01:38:35.727 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-19T01:38:35.840 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:35.839+0000 7f94c1fa9640 1 -- 172.21.15.99:0/2222974066 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f94bc100740 msgr2=0x7f94bc100b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:35.840 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:35.839+0000 7f94c1fa9640 1 --2- 172.21.15.99:0/2222974066 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f94bc100740 0x7f94bc100b60 secure :-1 s=READY pgs=169 cs=0 l=1 rev1=1 crypto rx=0x7f94ac009900 tx=0x7f94ac02bc30 comp rx=0 tx=0).stop 2024-03-19T01:38:35.841 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:35.840+0000 7f94c1fa9640 1 -- 172.21.15.99:0/2222974066 shutdown_connections 2024-03-19T01:38:35.841 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:35.840+0000 7f94c1fa9640 1 --2- 172.21.15.99:0/2222974066 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f94bc101930 0x7f94bc101db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:35.841 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:35.840+0000 7f94c1fa9640 1 --2- 172.21.15.99:0/2222974066 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f94bc1022f0 0x7f94bc108b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:35.841 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:35.840+0000 7f94c1fa9640 1 --2- 172.21.15.99:0/2222974066 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f94bc100740 0x7f94bc100b60 unknown :-1 s=CLOSED pgs=169 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:35.841 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:35.840+0000 7f94c1fa9640 1 -- 172.21.15.99:0/2222974066 >> 172.21.15.99:0/2222974066 conn(0x7f94bc0fbcb0 msgr2=0x7f94bc0fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:35.841 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:35.840+0000 7f94c1fa9640 1 -- 172.21.15.99:0/2222974066 shutdown_connections 2024-03-19T01:38:35.841 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:35.840+0000 7f94c1fa9640 1 -- 172.21.15.99:0/2222974066 wait complete. 2024-03-19T01:38:35.841 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:35.840+0000 7f94c1fa9640 1 Processor -- start 2024-03-19T01:38:35.841 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:35.840+0000 7f94c1fa9640 1 -- start start 2024-03-19T01:38:35.843 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:35.841+0000 7f94c1fa9640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f94bc100740 0x7f94bc196a30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:35.844 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:35.841+0000 7f94c1fa9640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f94bc101930 0x7f94bc196f70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:35.844 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:35.841+0000 7f94c1fa9640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f94bc1022f0 0x7f94bc19dfe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:35.844 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:35.841+0000 7f94c1fa9640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f94bc10c3c0 con 0x7f94bc1022f0 2024-03-19T01:38:35.844 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:35.841+0000 7f94c1fa9640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f94bc10c240 con 0x7f94bc100740 2024-03-19T01:38:35.844 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:35.841+0000 7f94c1fa9640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f94bc10c540 con 0x7f94bc101930 2024-03-19T01:38:35.844 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:35.841+0000 7f94bbfff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f94bc1022f0 0x7f94bc19dfe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:35.844 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:35.841+0000 7f94bbfff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f94bc1022f0 0x7f94bc19dfe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:51270/0 (socket says 172.21.15.99:51270) 2024-03-19T01:38:35.844 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:35.841+0000 7f94bbfff640 1 -- 172.21.15.99:0/3994152984 learned_addr learned my addr 172.21.15.99:0/3994152984 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:38:35.844 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:35.841+0000 7f94b2ffd640 1 --2- 172.21.15.99:0/3994152984 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f94bc101930 0x7f94bc196f70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:35.844 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:35.841+0000 7f94bb7fe640 1 --2- 172.21.15.99:0/3994152984 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f94bc100740 0x7f94bc196a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:35.844 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:35.841+0000 7f94bbfff640 1 -- 172.21.15.99:0/3994152984 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f94bc101930 msgr2=0x7f94bc196f70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:35.844 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:35.841+0000 7f94bbfff640 1 --2- 172.21.15.99:0/3994152984 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f94bc101930 0x7f94bc196f70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:35.845 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:35.841+0000 7f94bbfff640 1 -- 172.21.15.99:0/3994152984 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f94bc100740 msgr2=0x7f94bc196a30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:35.845 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:35.841+0000 7f94bbfff640 1 --2- 172.21.15.99:0/3994152984 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f94bc100740 0x7f94bc196a30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:35.845 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:35.841+0000 7f94bbfff640 1 -- 172.21.15.99:0/3994152984 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f94bc19e650 con 0x7f94bc1022f0 2024-03-19T01:38:35.845 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:35.841+0000 7f94bb7fe640 1 --2- 172.21.15.99:0/3994152984 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f94bc100740 0x7f94bc196a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-19T01:38:35.845 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:35.841+0000 7f94bbfff640 1 --2- 172.21.15.99:0/3994152984 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f94bc1022f0 0x7f94bc19dfe0 secure :-1 s=READY pgs=170 cs=0 l=1 rev1=1 crypto rx=0x7f94b400eaa0 tx=0x7f94b400ef60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:35.845 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:35.841+0000 7f94b97fa640 1 -- 172.21.15.99:0/3994152984 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f94b400cdb0 con 0x7f94bc1022f0 2024-03-19T01:38:35.845 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:35.841+0000 7f94b97fa640 1 -- 172.21.15.99:0/3994152984 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f94b4005020 con 0x7f94bc1022f0 2024-03-19T01:38:35.845 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:35.841+0000 7f94c1fa9640 1 -- 172.21.15.99:0/3994152984 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f94bc19e940 con 0x7f94bc1022f0 2024-03-19T01:38:35.845 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:35.841+0000 7f94b97fa640 1 -- 172.21.15.99:0/3994152984 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f94b40116b0 con 0x7f94bc1022f0 2024-03-19T01:38:35.845 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:35.841+0000 7f94c1fa9640 1 -- 172.21.15.99:0/3994152984 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f94bc19eee0 con 0x7f94bc1022f0 2024-03-19T01:38:35.845 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:35.842+0000 7f94c1fa9640 1 -- 172.21.15.99:0/3994152984 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f94bc107a60 con 0x7f94bc1022f0 2024-03-19T01:38:35.845 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:35.843+0000 7f94b97fa640 1 -- 172.21.15.99:0/3994152984 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 16) ==== 137290+0+0 (secure 0 0 0) 0x7f94b4005560 con 0x7f94bc1022f0 2024-03-19T01:38:35.846 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:35.844+0000 7f94b97fa640 1 --2- 172.21.15.99:0/3994152984 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f94a80a2880 0x7f94a80a4d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:35.846 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:35.844+0000 7f94bb7fe640 1 --2- 172.21.15.99:0/3994152984 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f94a80a2880 0x7f94a80a4d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:35.846 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:35.844+0000 7f94b97fa640 1 -- 172.21.15.99:0/3994152984 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(83..83 src has 1..83) ==== 7127+0+0 (secure 0 0 0) 0x7f94b40ce760 con 0x7f94bc1022f0 2024-03-19T01:38:35.846 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:35.844+0000 7f94bb7fe640 1 --2- 172.21.15.99:0/3994152984 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f94a80a2880 0x7f94a80a4d40 secure :-1 s=READY pgs=142 cs=0 l=1 rev1=1 crypto rx=0x7f94ac02c7b0 tx=0x7f94ac005fd0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:35.849 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:35.849+0000 7f94b97fa640 1 -- 172.21.15.99:0/3994152984 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f94b4016030 con 0x7f94bc1022f0 2024-03-19T01:38:36.061 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:36.061+0000 7f94c1fa9640 1 -- 172.21.15.99:0/3994152984 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f94bc02d1f0 con 0x7f94bc1022f0 2024-03-19T01:38:36.061 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:36.061+0000 7f94b97fa640 1 -- 172.21.15.99:0/3994152984 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 7 v7) ==== 93+0+5527 (secure 0 0 0) 0x7f94b409bca0 con 0x7f94bc1022f0 2024-03-19T01:38:36.062 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:38:36.062 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":7,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14562,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6835/1631815621","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6834","nonce":1631815621},{"type":"v1","addr":"172.21.15.99:6835","nonce":1631815621}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":4},{"gid":24383,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.112:6833/2969810977","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6832","nonce":2969810977},{"type":"v1","addr":"172.21.15.112:6833","nonce":2969810977}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3},{"gid":24404,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6837/2038634879","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6836","nonce":2038634879},{"type":"v1","addr":"172.21.15.99:6837","nonce":2038634879}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":24415,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.115:6833/2219320481","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6832","nonce":2219320481},{"type":"v1","addr":"172.21.15.115:6833","nonce":2219320481}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2},{"gid":24416,"name":"e","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.112:6835/3690587528","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6834","nonce":3690587528},{"type":"v1","addr":"172.21.15.112:6835","nonce":3690587528}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":6},{"gid":24460,"name":"f","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.115:6835/591078413","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6834","nonce":591078413},{"type":"v1","addr":"172.21.15.115:6835","nonce":591078413}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":7}],"filesystems":[]} 2024-03-19T01:38:36.062 INFO:teuthology.orchestra.run.smithi099.stderr:dumped fsmap epoch 7 2024-03-19T01:38:36.064 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:36.063+0000 7f94c1fa9640 1 -- 172.21.15.99:0/3994152984 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f94a80a2880 msgr2=0x7f94a80a4d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:36.064 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:36.063+0000 7f94c1fa9640 1 --2- 172.21.15.99:0/3994152984 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f94a80a2880 0x7f94a80a4d40 secure :-1 s=READY pgs=142 cs=0 l=1 rev1=1 crypto rx=0x7f94ac02c7b0 tx=0x7f94ac005fd0 comp rx=0 tx=0).stop 2024-03-19T01:38:36.064 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:36.063+0000 7f94c1fa9640 1 -- 172.21.15.99:0/3994152984 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f94bc1022f0 msgr2=0x7f94bc19dfe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:36.064 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:36.063+0000 7f94c1fa9640 1 --2- 172.21.15.99:0/3994152984 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f94bc1022f0 0x7f94bc19dfe0 secure :-1 s=READY pgs=170 cs=0 l=1 rev1=1 crypto rx=0x7f94b400eaa0 tx=0x7f94b400ef60 comp rx=0 tx=0).stop 2024-03-19T01:38:36.064 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:36.063+0000 7f94c1fa9640 1 -- 172.21.15.99:0/3994152984 shutdown_connections 2024-03-19T01:38:36.064 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:36.063+0000 7f94c1fa9640 1 --2- 172.21.15.99:0/3994152984 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f94bc1022f0 0x7f94bc19dfe0 unknown :-1 s=CLOSED pgs=170 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:36.064 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:36.063+0000 7f94c1fa9640 1 --2- 172.21.15.99:0/3994152984 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f94bc101930 0x7f94bc196f70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:36.064 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:36.063+0000 7f94c1fa9640 1 --2- 172.21.15.99:0/3994152984 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f94a80a2880 0x7f94a80a4d40 unknown :-1 s=CLOSED pgs=142 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:36.064 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:36.063+0000 7f94c1fa9640 1 --2- 172.21.15.99:0/3994152984 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f94bc100740 0x7f94bc196a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:36.065 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:36.063+0000 7f94c1fa9640 1 -- 172.21.15.99:0/3994152984 >> 172.21.15.99:0/3994152984 conn(0x7f94bc0fbcb0 msgr2=0x7f94bc104c30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:36.065 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:36.063+0000 7f94c1fa9640 1 -- 172.21.15.99:0/3994152984 shutdown_connections 2024-03-19T01:38:36.065 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:36.063+0000 7f94c1fa9640 1 -- 172.21.15.99:0/3994152984 wait complete. 2024-03-19T01:38:36.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:36 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/3994152984' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:38:36.307 INFO:journalctl@ceph.mds.c.smithi115.stdout:Mar 19 01:38:36 smithi115 systemd[1]: Starting Ceph mds.c for 438fb69c-e590-11ee-95c9-87774f69a715... 2024-03-19T01:38:36.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:36 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/3994152984' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:38:36.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:36 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/3994152984' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:38:36.669 INFO:journalctl@ceph.mds.c.smithi115.stdout:Mar 19 01:38:36 smithi115 podman[97138]: 2024-03-19 01:38:36.299714026 +0000 UTC m=+0.017618181 image pull 2312c8e42d0069a2dbb2364bddb00264db4b800e7f18a7e7e4eb8f3ecab2ff1c quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639 2024-03-19T01:38:36.670 INFO:journalctl@ceph.mds.c.smithi115.stdout:Mar 19 01:38:36 smithi115 podman[97138]: 2024-03-19 01:38:36.425250912 +0000 UTC m=+0.143155050 container create ff3f5e67fed9cbb582e3ab4f836771004c3f1a5c0c76d93d802b3b3610630173 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-c, ceph=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, io.buildah.version=1.33.5, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240307, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-03-19T01:38:36.670 INFO:journalctl@ceph.mds.c.smithi115.stdout:Mar 19 01:38:36 smithi115 podman[97138]: 2024-03-19 01:38:36.644230474 +0000 UTC m=+0.362134627 container init ff3f5e67fed9cbb582e3ab4f836771004c3f1a5c0c76d93d802b3b3610630173 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-c, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240307, GIT_BRANCH=HEAD) 2024-03-19T01:38:36.670 INFO:journalctl@ceph.mds.c.smithi115.stdout:Mar 19 01:38:36 smithi115 podman[97138]: 2024-03-19 01:38:36.646920533 +0000 UTC m=+0.364824673 container start ff3f5e67fed9cbb582e3ab4f836771004c3f1a5c0c76d93d802b3b3610630173 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-c, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, maintainer=Guillaume Abrioux , RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240307, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2) 2024-03-19T01:38:36.670 INFO:journalctl@ceph.mds.c.smithi115.stdout:Mar 19 01:38:36 smithi115 ceph-mds[97157]: -- 172.21.15.115:0/305109217 <== mon.1 v2:172.21.15.115:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x556d95127a40 con 0x556d950fb000 2024-03-19T01:38:37.056 INFO:journalctl@ceph.mds.c.smithi115.stdout:Mar 19 01:38:36 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-c[97153]: starting mds.c at 2024-03-19T01:38:37.057 INFO:journalctl@ceph.mds.c.smithi115.stdout:Mar 19 01:38:36 smithi115 bash[97138]: ff3f5e67fed9cbb582e3ab4f836771004c3f1a5c0c76d93d802b3b3610630173 2024-03-19T01:38:37.057 INFO:journalctl@ceph.mds.c.smithi115.stdout:Mar 19 01:38:36 smithi115 systemd[1]: Started Ceph mds.c for 438fb69c-e590-11ee-95c9-87774f69a715. 2024-03-19T01:38:38.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:37 smithi115 ceph-mon[67332]: pgmap v268: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:38.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:37 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:38.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:37 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:38.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:37 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:38.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:37 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.h", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-19T01:38:38.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:37 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.h", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-19T01:38:38.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:37 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:38:38.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:37 smithi115 ceph-mon[67332]: Deploying daemon mds.h on smithi112 2024-03-19T01:38:38.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:37 smithi115 ceph-mon[67332]: mds.? [v2:172.21.15.115:6836/1368988191,v1:172.21.15.115:6837/1368988191] up:boot 2024-03-19T01:38:38.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:37 smithi115 ceph-mon[67332]: fsmap 7 up:standby 2024-03-19T01:38:38.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:37 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "c"}]: dispatch 2024-03-19T01:38:38.076 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-19T01:38:38.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:37 smithi099 ceph-mon[62951]: pgmap v268: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:38.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:37 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:38.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:37 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:38.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:37 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:38.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:37 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.h", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-19T01:38:38.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:37 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.h", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-19T01:38:38.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:37 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:38:38.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:37 smithi099 ceph-mon[62951]: Deploying daemon mds.h on smithi112 2024-03-19T01:38:38.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:37 smithi099 ceph-mon[62951]: mds.? [v2:172.21.15.115:6836/1368988191,v1:172.21.15.115:6837/1368988191] up:boot 2024-03-19T01:38:38.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:37 smithi099 ceph-mon[62951]: fsmap 7 up:standby 2024-03-19T01:38:38.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:37 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "c"}]: dispatch 2024-03-19T01:38:38.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:37 smithi112 ceph-mon[66819]: pgmap v268: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:38.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:37 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:38.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:37 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:38.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:37 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:38.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:37 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.h", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-19T01:38:38.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:37 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.h", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-19T01:38:38.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:37 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:38:38.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:37 smithi112 ceph-mon[66819]: Deploying daemon mds.h on smithi112 2024-03-19T01:38:38.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:37 smithi112 ceph-mon[66819]: mds.? [v2:172.21.15.115:6836/1368988191,v1:172.21.15.115:6837/1368988191] up:boot 2024-03-19T01:38:38.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:37 smithi112 ceph-mon[66819]: fsmap 7 up:standby 2024-03-19T01:38:38.113 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:37 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "c"}]: dispatch 2024-03-19T01:38:38.192 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:38.189+0000 7fd601436640 1 -- 172.21.15.99:0/3374248810 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd5fc1075a0 msgr2=0x7fd5fc109990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:38.192 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:38.189+0000 7fd601436640 1 --2- 172.21.15.99:0/3374248810 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd5fc1075a0 0x7fd5fc109990 secure :-1 s=READY pgs=171 cs=0 l=1 rev1=1 crypto rx=0x7fd5f4009ca0 tx=0x7fd5f402c0f0 comp rx=0 tx=0).stop 2024-03-19T01:38:38.192 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:38.189+0000 7fd601436640 1 -- 172.21.15.99:0/3374248810 shutdown_connections 2024-03-19T01:38:38.192 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:38.189+0000 7fd601436640 1 --2- 172.21.15.99:0/3374248810 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd5fc104c70 0x7fd5fc107060 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:38.192 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:38.189+0000 7fd601436640 1 --2- 172.21.15.99:0/3374248810 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd5fc1075a0 0x7fd5fc109990 unknown :-1 s=CLOSED pgs=171 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:38.192 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:38.189+0000 7fd601436640 1 --2- 172.21.15.99:0/3374248810 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd5fc102340 0x7fd5fc104730 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:38.192 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:38.189+0000 7fd601436640 1 -- 172.21.15.99:0/3374248810 >> 172.21.15.99:0/3374248810 conn(0x7fd5fc0fbcd0 msgr2=0x7fd5fc0fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:38.192 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:38.189+0000 7fd601436640 1 -- 172.21.15.99:0/3374248810 shutdown_connections 2024-03-19T01:38:38.192 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:38.189+0000 7fd601436640 1 -- 172.21.15.99:0/3374248810 wait complete. 2024-03-19T01:38:38.193 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:38.189+0000 7fd601436640 1 Processor -- start 2024-03-19T01:38:38.193 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:38.189+0000 7fd601436640 1 -- start start 2024-03-19T01:38:38.193 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:38.190+0000 7fd601436640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd5fc102340 0x7fd5fc197b70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:38.193 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:38.190+0000 7fd601436640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd5fc104c70 0x7fd5fc1980b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:38.193 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:38.190+0000 7fd601436640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd5fc1075a0 0x7fd5fc19f120 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:38.193 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:38.190+0000 7fd601436640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fd5fc10d530 con 0x7fd5fc104c70 2024-03-19T01:38:38.193 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:38.190+0000 7fd601436640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fd5fc10d3b0 con 0x7fd5fc102340 2024-03-19T01:38:38.193 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:38.190+0000 7fd601436640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fd5fc10d6b0 con 0x7fd5fc1075a0 2024-03-19T01:38:38.193 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:38.190+0000 7fd5fa7fc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd5fc104c70 0x7fd5fc1980b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:38.193 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:38.190+0000 7fd5fa7fc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd5fc104c70 0x7fd5fc1980b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:51280/0 (socket says 172.21.15.99:51280) 2024-03-19T01:38:38.193 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:38.190+0000 7fd5fa7fc640 1 -- 172.21.15.99:0/2580018804 learned_addr learned my addr 172.21.15.99:0/2580018804 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:38:38.194 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:38.190+0000 7fd5faffd640 1 --2- 172.21.15.99:0/2580018804 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd5fc102340 0x7fd5fc197b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:38.194 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:38.190+0000 7fd5fb7fe640 1 --2- 172.21.15.99:0/2580018804 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd5fc1075a0 0x7fd5fc19f120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:38.194 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:38.190+0000 7fd5fa7fc640 1 -- 172.21.15.99:0/2580018804 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd5fc1075a0 msgr2=0x7fd5fc19f120 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:38.194 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:38.190+0000 7fd5fa7fc640 1 --2- 172.21.15.99:0/2580018804 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd5fc1075a0 0x7fd5fc19f120 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:38.194 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:38.190+0000 7fd5fa7fc640 1 -- 172.21.15.99:0/2580018804 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd5fc102340 msgr2=0x7fd5fc197b70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:38.194 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:38.190+0000 7fd5fa7fc640 1 --2- 172.21.15.99:0/2580018804 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd5fc102340 0x7fd5fc197b70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:38.194 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:38.190+0000 7fd5fa7fc640 1 -- 172.21.15.99:0/2580018804 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd5fc19f790 con 0x7fd5fc104c70 2024-03-19T01:38:38.194 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:38.190+0000 7fd5fb7fe640 1 --2- 172.21.15.99:0/2580018804 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd5fc1075a0 0x7fd5fc19f120 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-19T01:38:38.194 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:38.190+0000 7fd5fa7fc640 1 --2- 172.21.15.99:0/2580018804 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd5fc104c70 0x7fd5fc1980b0 secure :-1 s=READY pgs=172 cs=0 l=1 rev1=1 crypto rx=0x7fd5ec00d770 tx=0x7fd5ec00db70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:38.194 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:38.190+0000 7fd5e3fff640 1 -- 172.21.15.99:0/2580018804 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd5ec017070 con 0x7fd5fc104c70 2024-03-19T01:38:38.194 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:38.190+0000 7fd5e3fff640 1 -- 172.21.15.99:0/2580018804 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd5ec010460 con 0x7fd5fc104c70 2024-03-19T01:38:38.194 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:38.190+0000 7fd601436640 1 -- 172.21.15.99:0/2580018804 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd5fc19fa80 con 0x7fd5fc104c70 2024-03-19T01:38:38.194 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:38.190+0000 7fd5e3fff640 1 -- 172.21.15.99:0/2580018804 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd5ec00f620 con 0x7fd5fc104c70 2024-03-19T01:38:38.194 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:38.190+0000 7fd601436640 1 -- 172.21.15.99:0/2580018804 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fd5fc19ffd0 con 0x7fd5fc104c70 2024-03-19T01:38:38.195 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:38.192+0000 7fd601436640 1 -- 172.21.15.99:0/2580018804 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fd5fc0fd7c0 con 0x7fd5fc104c70 2024-03-19T01:38:38.195 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:38.193+0000 7fd5e3fff640 1 -- 172.21.15.99:0/2580018804 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 16) ==== 137290+0+0 (secure 0 0 0) 0x7fd5ec01a070 con 0x7fd5fc104c70 2024-03-19T01:38:38.195 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:38.193+0000 7fd5e3fff640 1 --2- 172.21.15.99:0/2580018804 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd5e40a2800 0x7fd5e40a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:38.195 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:38.193+0000 7fd5faffd640 1 --2- 172.21.15.99:0/2580018804 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd5e40a2800 0x7fd5e40a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:38.196 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:38.193+0000 7fd5e3fff640 1 -- 172.21.15.99:0/2580018804 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(83..83 src has 1..83) ==== 7127+0+0 (secure 0 0 0) 0x7fd5ec0ce010 con 0x7fd5fc104c70 2024-03-19T01:38:38.196 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:38.194+0000 7fd5faffd640 1 --2- 172.21.15.99:0/2580018804 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd5e40a2800 0x7fd5e40a4cc0 secure :-1 s=READY pgs=144 cs=0 l=1 rev1=1 crypto rx=0x7fd5f0009990 tx=0x7fd5f0009110 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:38.197 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:38.197+0000 7fd5e3fff640 1 -- 172.21.15.99:0/2580018804 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7fd5ec09b550 con 0x7fd5fc104c70 2024-03-19T01:38:38.410 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:38.410+0000 7fd601436640 1 -- 172.21.15.99:0/2580018804 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7fd5fc02d1f0 con 0x7fd5fc104c70 2024-03-19T01:38:38.411 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:38.410+0000 7fd5e3fff640 1 -- 172.21.15.99:0/2580018804 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 8 v8) ==== 93+0+6355 (secure 0 0 0) 0x7fd5ec09ae50 con 0x7fd5fc104c70 2024-03-19T01:38:38.411 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:38:38.411 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":8,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14562,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6835/1631815621","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6834","nonce":1631815621},{"type":"v1","addr":"172.21.15.99:6835","nonce":1631815621}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":4},{"gid":24383,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.112:6833/2969810977","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6832","nonce":2969810977},{"type":"v1","addr":"172.21.15.112:6833","nonce":2969810977}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3},{"gid":24404,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6837/2038634879","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6836","nonce":2038634879},{"type":"v1","addr":"172.21.15.99:6837","nonce":2038634879}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":24415,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.115:6833/2219320481","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6832","nonce":2219320481},{"type":"v1","addr":"172.21.15.115:6833","nonce":2219320481}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2},{"gid":24416,"name":"e","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.112:6835/3690587528","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6834","nonce":3690587528},{"type":"v1","addr":"172.21.15.112:6835","nonce":3690587528}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":6},{"gid":24460,"name":"f","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.115:6835/591078413","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6834","nonce":591078413},{"type":"v1","addr":"172.21.15.115:6835","nonce":591078413}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":7},{"gid":24472,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.115:6837/1368988191","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6836","nonce":1368988191},{"type":"v1","addr":"172.21.15.115:6837","nonce":1368988191}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":8}],"filesystems":[]} 2024-03-19T01:38:38.411 INFO:teuthology.orchestra.run.smithi099.stderr:dumped fsmap epoch 8 2024-03-19T01:38:38.413 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:38.412+0000 7fd601436640 1 -- 172.21.15.99:0/2580018804 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd5e40a2800 msgr2=0x7fd5e40a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:38.413 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:38.412+0000 7fd601436640 1 --2- 172.21.15.99:0/2580018804 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd5e40a2800 0x7fd5e40a4cc0 secure :-1 s=READY pgs=144 cs=0 l=1 rev1=1 crypto rx=0x7fd5f0009990 tx=0x7fd5f0009110 comp rx=0 tx=0).stop 2024-03-19T01:38:38.413 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:38.412+0000 7fd601436640 1 -- 172.21.15.99:0/2580018804 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd5fc104c70 msgr2=0x7fd5fc1980b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:38.413 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:38.412+0000 7fd601436640 1 --2- 172.21.15.99:0/2580018804 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd5fc104c70 0x7fd5fc1980b0 secure :-1 s=READY pgs=172 cs=0 l=1 rev1=1 crypto rx=0x7fd5ec00d770 tx=0x7fd5ec00db70 comp rx=0 tx=0).stop 2024-03-19T01:38:38.413 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:38.412+0000 7fd601436640 1 -- 172.21.15.99:0/2580018804 shutdown_connections 2024-03-19T01:38:38.413 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:38.412+0000 7fd601436640 1 --2- 172.21.15.99:0/2580018804 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd5fc1075a0 0x7fd5fc19f120 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:38.413 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:38.412+0000 7fd601436640 1 --2- 172.21.15.99:0/2580018804 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd5fc104c70 0x7fd5fc1980b0 unknown :-1 s=CLOSED pgs=172 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:38.414 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:38.412+0000 7fd601436640 1 --2- 172.21.15.99:0/2580018804 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd5e40a2800 0x7fd5e40a4cc0 unknown :-1 s=CLOSED pgs=144 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:38.414 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:38.412+0000 7fd601436640 1 --2- 172.21.15.99:0/2580018804 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd5fc102340 0x7fd5fc197b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:38.414 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:38.412+0000 7fd601436640 1 -- 172.21.15.99:0/2580018804 >> 172.21.15.99:0/2580018804 conn(0x7fd5fc0fbcd0 msgr2=0x7fd5fc102c80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:38.414 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:38.413+0000 7fd601436640 1 -- 172.21.15.99:0/2580018804 shutdown_connections 2024-03-19T01:38:38.414 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:38.413+0000 7fd601436640 1 -- 172.21.15.99:0/2580018804 wait complete. 2024-03-19T01:38:38.784 INFO:journalctl@ceph.mds.h.smithi112.stdout:Mar 19 01:38:38 smithi112 systemd[1]: Starting Ceph mds.h for 438fb69c-e590-11ee-95c9-87774f69a715... 2024-03-19T01:38:39.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:38 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/2580018804' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:38:39.085 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:38 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/2580018804' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:38:39.086 INFO:journalctl@ceph.mds.h.smithi112.stdout:Mar 19 01:38:38 smithi112 podman[97065]: 2024-03-19 01:38:38.900646961 +0000 UTC m=+0.106178074 container create 1b8f536d04b2ecd12906ef8b2dfaf3f65b8543ba65829a593a2e204f706f91d0 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-h, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240307, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, maintainer=Guillaume Abrioux ) 2024-03-19T01:38:39.086 INFO:journalctl@ceph.mds.h.smithi112.stdout:Mar 19 01:38:38 smithi112 podman[97065]: 2024-03-19 01:38:38.812059164 +0000 UTC m=+0.017590292 image pull 2312c8e42d0069a2dbb2364bddb00264db4b800e7f18a7e7e4eb8f3ecab2ff1c quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639 2024-03-19T01:38:39.086 INFO:journalctl@ceph.mds.h.smithi112.stdout:Mar 19 01:38:39 smithi112 podman[97065]: 2024-03-19 01:38:39.086231624 +0000 UTC m=+0.291762742 container init 1b8f536d04b2ecd12906ef8b2dfaf3f65b8543ba65829a593a2e204f706f91d0 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-h, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.build-date=20240307, io.buildah.version=1.33.5, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-03-19T01:38:39.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:38 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/2580018804' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:38:39.362 INFO:journalctl@ceph.mds.h.smithi112.stdout:Mar 19 01:38:39 smithi112 podman[97065]: 2024-03-19 01:38:39.088970274 +0000 UTC m=+0.294501398 container start 1b8f536d04b2ecd12906ef8b2dfaf3f65b8543ba65829a593a2e204f706f91d0 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-h, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, GIT_CLEAN=True, org.label-schema.build-date=20240307, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-03-19T01:38:39.362 INFO:journalctl@ceph.mds.h.smithi112.stdout:Mar 19 01:38:39 smithi112 ceph-mds[97084]: -- 172.21.15.112:0/4093418194 <== mon.2 v2:172.21.15.112:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55f746f9ba40 con 0x55f746f6ec00 2024-03-19T01:38:39.362 INFO:journalctl@ceph.mds.h.smithi112.stdout:Mar 19 01:38:39 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-h[97080]: starting mds.h at 2024-03-19T01:38:39.362 INFO:journalctl@ceph.mds.h.smithi112.stdout:Mar 19 01:38:39 smithi112 bash[97065]: 1b8f536d04b2ecd12906ef8b2dfaf3f65b8543ba65829a593a2e204f706f91d0 2024-03-19T01:38:39.362 INFO:journalctl@ceph.mds.h.smithi112.stdout:Mar 19 01:38:39 smithi112 systemd[1]: Started Ceph mds.h for 438fb69c-e590-11ee-95c9-87774f69a715. 2024-03-19T01:38:40.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:39 smithi115 ceph-mon[67332]: pgmap v269: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:40.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:39 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:40.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:39 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:40.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:39 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:40.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:39 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.i", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-19T01:38:40.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:39 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.i", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-19T01:38:40.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:39 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:38:40.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:39 smithi099 ceph-mon[62951]: pgmap v269: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:40.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:39 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:40.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:39 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:40.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:39 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:40.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:39 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.i", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-19T01:38:40.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:39 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.i", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-19T01:38:40.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:39 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:38:40.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:39 smithi112 ceph-mon[66819]: pgmap v269: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:40.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:39 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:40.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:39 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:40.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:39 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:40.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:39 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.i", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-19T01:38:40.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:39 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.i", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-19T01:38:40.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:39 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:38:40.425 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-19T01:38:40.531 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:40.528+0000 7f5e5aa12640 1 -- 172.21.15.99:0/1398761289 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5e54102e70 msgr2=0x7f5e54103290 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:40.531 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:40.528+0000 7f5e5aa12640 1 --2- 172.21.15.99:0/1398761289 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5e54102e70 0x7f5e54103290 secure :-1 s=READY pgs=173 cs=0 l=1 rev1=1 crypto rx=0x7f5e44009900 tx=0x7f5e4402bc30 comp rx=0 tx=0).stop 2024-03-19T01:38:40.531 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:40.528+0000 7f5e5aa12640 1 -- 172.21.15.99:0/1398761289 shutdown_connections 2024-03-19T01:38:40.531 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:40.528+0000 7f5e5aa12640 1 --2- 172.21.15.99:0/1398761289 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5e541002a0 0x7f5e54109e50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:40.531 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:40.528+0000 7f5e5aa12640 1 --2- 172.21.15.99:0/1398761289 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5e540ff8e0 0x7f5e540ffd60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:40.531 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:40.528+0000 7f5e5aa12640 1 --2- 172.21.15.99:0/1398761289 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5e54102e70 0x7f5e54103290 unknown :-1 s=CLOSED pgs=173 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:40.531 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:40.528+0000 7f5e5aa12640 1 -- 172.21.15.99:0/1398761289 >> 172.21.15.99:0/1398761289 conn(0x7f5e540fb440 msgr2=0x7f5e540fd880 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:40.531 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:40.528+0000 7f5e5aa12640 1 -- 172.21.15.99:0/1398761289 shutdown_connections 2024-03-19T01:38:40.531 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:40.528+0000 7f5e5aa12640 1 -- 172.21.15.99:0/1398761289 wait complete. 2024-03-19T01:38:40.532 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:40.528+0000 7f5e5aa12640 1 Processor -- start 2024-03-19T01:38:40.532 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:40.528+0000 7f5e5aa12640 1 -- start start 2024-03-19T01:38:40.532 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:40.529+0000 7f5e5aa12640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5e540ff8e0 0x7f5e54197bc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:40.532 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:40.529+0000 7f5e5aa12640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5e541002a0 0x7f5e54198100 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:40.532 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:40.529+0000 7f5e5aa12640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5e54102e70 0x7f5e5419f170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:40.532 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:40.529+0000 7f5e5aa12640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f5e5410d4c0 con 0x7f5e540ff8e0 2024-03-19T01:38:40.532 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:40.529+0000 7f5e5aa12640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f5e5410d340 con 0x7f5e54102e70 2024-03-19T01:38:40.532 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:40.529+0000 7f5e5aa12640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f5e5410d640 con 0x7f5e541002a0 2024-03-19T01:38:40.532 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:40.529+0000 7f5e53fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5e540ff8e0 0x7f5e54197bc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:40.532 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:40.529+0000 7f5e53fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5e540ff8e0 0x7f5e54197bc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:51304/0 (socket says 172.21.15.99:51304) 2024-03-19T01:38:40.532 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:40.529+0000 7f5e53fff640 1 -- 172.21.15.99:0/1917126871 learned_addr learned my addr 172.21.15.99:0/1917126871 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:38:40.532 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:40.529+0000 7f5e537fe640 1 --2- 172.21.15.99:0/1917126871 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5e541002a0 0x7f5e54198100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:40.533 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:40.529+0000 7f5e53fff640 1 -- 172.21.15.99:0/1917126871 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5e541002a0 msgr2=0x7f5e54198100 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:40.533 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:40.529+0000 7f5e53fff640 1 --2- 172.21.15.99:0/1917126871 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5e541002a0 0x7f5e54198100 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:40.533 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:40.529+0000 7f5e53fff640 1 -- 172.21.15.99:0/1917126871 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5e54102e70 msgr2=0x7f5e5419f170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:40.533 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:40.529+0000 7f5e53fff640 1 --2- 172.21.15.99:0/1917126871 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5e54102e70 0x7f5e5419f170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:40.533 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:40.529+0000 7f5e53fff640 1 -- 172.21.15.99:0/1917126871 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f5e5419f7e0 con 0x7f5e540ff8e0 2024-03-19T01:38:40.533 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:40.529+0000 7f5e53fff640 1 --2- 172.21.15.99:0/1917126871 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5e540ff8e0 0x7f5e54197bc0 secure :-1 s=READY pgs=174 cs=0 l=1 rev1=1 crypto rx=0x7f5e44005ab0 tx=0x7f5e44005dc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:40.533 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:40.529+0000 7f5e517fa640 1 -- 172.21.15.99:0/1917126871 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5e44041070 con 0x7f5e540ff8e0 2024-03-19T01:38:40.533 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:40.529+0000 7f5e5aa12640 1 -- 172.21.15.99:0/1917126871 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f5e5419fa70 con 0x7f5e540ff8e0 2024-03-19T01:38:40.533 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:40.529+0000 7f5e517fa640 1 -- 172.21.15.99:0/1917126871 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f5e440054e0 con 0x7f5e540ff8e0 2024-03-19T01:38:40.533 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:40.529+0000 7f5e517fa640 1 -- 172.21.15.99:0/1917126871 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5e44034440 con 0x7f5e540ff8e0 2024-03-19T01:38:40.533 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:40.529+0000 7f5e5aa12640 1 -- 172.21.15.99:0/1917126871 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f5e5419ff60 con 0x7f5e540ff8e0 2024-03-19T01:38:40.533 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:40.530+0000 7f5e5aa12640 1 -- 172.21.15.99:0/1917126871 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f5e54108d50 con 0x7f5e540ff8e0 2024-03-19T01:38:40.534 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:40.531+0000 7f5e517fa640 1 -- 172.21.15.99:0/1917126871 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 16) ==== 137290+0+0 (secure 0 0 0) 0x7f5e44044070 con 0x7f5e540ff8e0 2024-03-19T01:38:40.534 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:40.532+0000 7f5e517fa640 1 --2- 172.21.15.99:0/1917126871 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f5e3c0a2800 0x7f5e3c0a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:40.534 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:40.532+0000 7f5e537fe640 1 --2- 172.21.15.99:0/1917126871 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f5e3c0a2800 0x7f5e3c0a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:40.535 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:40.532+0000 7f5e517fa640 1 -- 172.21.15.99:0/1917126871 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(83..83 src has 1..83) ==== 7127+0+0 (secure 0 0 0) 0x7f5e440f0230 con 0x7f5e540ff8e0 2024-03-19T01:38:40.535 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:40.532+0000 7f5e537fe640 1 --2- 172.21.15.99:0/1917126871 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f5e3c0a2800 0x7f5e3c0a4cc0 secure :-1 s=READY pgs=146 cs=0 l=1 rev1=1 crypto rx=0x7f5e541990d0 tx=0x7f5e4800b390 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:40.537 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:40.536+0000 7f5e517fa640 1 -- 172.21.15.99:0/1917126871 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f5e440bd770 con 0x7f5e540ff8e0 2024-03-19T01:38:40.751 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:40.750+0000 7f5e5aa12640 1 -- 172.21.15.99:0/1917126871 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f5e5402d3c0 con 0x7f5e540ff8e0 2024-03-19T01:38:40.751 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:40.751+0000 7f5e517fa640 1 -- 172.21.15.99:0/1917126871 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 9 v9) ==== 93+0+7183 (secure 0 0 0) 0x7f5e440bd070 con 0x7f5e540ff8e0 2024-03-19T01:38:40.752 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:38:40.752 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":9,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14562,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6835/1631815621","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6834","nonce":1631815621},{"type":"v1","addr":"172.21.15.99:6835","nonce":1631815621}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":4},{"gid":24383,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.112:6833/2969810977","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6832","nonce":2969810977},{"type":"v1","addr":"172.21.15.112:6833","nonce":2969810977}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3},{"gid":24404,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6837/2038634879","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6836","nonce":2038634879},{"type":"v1","addr":"172.21.15.99:6837","nonce":2038634879}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":24415,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.115:6833/2219320481","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6832","nonce":2219320481},{"type":"v1","addr":"172.21.15.115:6833","nonce":2219320481}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2},{"gid":24416,"name":"e","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.112:6835/3690587528","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6834","nonce":3690587528},{"type":"v1","addr":"172.21.15.112:6835","nonce":3690587528}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":6},{"gid":24440,"name":"h","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.112:6837/2339823159","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6836","nonce":2339823159},{"type":"v1","addr":"172.21.15.112:6837","nonce":2339823159}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":9},{"gid":24460,"name":"f","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.115:6835/591078413","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6834","nonce":591078413},{"type":"v1","addr":"172.21.15.115:6835","nonce":591078413}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":7},{"gid":24472,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.115:6837/1368988191","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6836","nonce":1368988191},{"type":"v1","addr":"172.21.15.115:6837","nonce":1368988191}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":8}],"filesystems":[]} 2024-03-19T01:38:40.752 INFO:teuthology.orchestra.run.smithi099.stderr:dumped fsmap epoch 9 2024-03-19T01:38:40.753 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:40.753+0000 7f5e5aa12640 1 -- 172.21.15.99:0/1917126871 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f5e3c0a2800 msgr2=0x7f5e3c0a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:40.754 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:40.753+0000 7f5e5aa12640 1 --2- 172.21.15.99:0/1917126871 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f5e3c0a2800 0x7f5e3c0a4cc0 secure :-1 s=READY pgs=146 cs=0 l=1 rev1=1 crypto rx=0x7f5e541990d0 tx=0x7f5e4800b390 comp rx=0 tx=0).stop 2024-03-19T01:38:40.754 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:40.753+0000 7f5e5aa12640 1 -- 172.21.15.99:0/1917126871 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5e540ff8e0 msgr2=0x7f5e54197bc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:40.754 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:40.753+0000 7f5e5aa12640 1 --2- 172.21.15.99:0/1917126871 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5e540ff8e0 0x7f5e54197bc0 secure :-1 s=READY pgs=174 cs=0 l=1 rev1=1 crypto rx=0x7f5e44005ab0 tx=0x7f5e44005dc0 comp rx=0 tx=0).stop 2024-03-19T01:38:40.754 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:40.753+0000 7f5e5aa12640 1 -- 172.21.15.99:0/1917126871 shutdown_connections 2024-03-19T01:38:40.754 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:40.753+0000 7f5e5aa12640 1 --2- 172.21.15.99:0/1917126871 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f5e3c0a2800 0x7f5e3c0a4cc0 unknown :-1 s=CLOSED pgs=146 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:40.754 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:40.753+0000 7f5e5aa12640 1 --2- 172.21.15.99:0/1917126871 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5e54102e70 0x7f5e5419f170 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:40.754 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:40.753+0000 7f5e5aa12640 1 --2- 172.21.15.99:0/1917126871 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5e541002a0 0x7f5e54198100 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:40.754 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:40.753+0000 7f5e5aa12640 1 --2- 172.21.15.99:0/1917126871 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5e540ff8e0 0x7f5e54197bc0 unknown :-1 s=CLOSED pgs=174 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:40.754 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:40.753+0000 7f5e5aa12640 1 -- 172.21.15.99:0/1917126871 >> 172.21.15.99:0/1917126871 conn(0x7f5e540fb440 msgr2=0x7f5e54105f20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:40.754 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:40.753+0000 7f5e5aa12640 1 -- 172.21.15.99:0/1917126871 shutdown_connections 2024-03-19T01:38:40.755 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:40.753+0000 7f5e5aa12640 1 -- 172.21.15.99:0/1917126871 wait complete. 2024-03-19T01:38:41.045 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:40 smithi115 ceph-mon[67332]: Deploying daemon mds.i on smithi115 2024-03-19T01:38:41.045 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:40 smithi115 ceph-mon[67332]: mds.? [v2:172.21.15.112:6836/2339823159,v1:172.21.15.112:6837/2339823159] up:boot 2024-03-19T01:38:41.045 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:40 smithi115 ceph-mon[67332]: fsmap 8 up:standby 2024-03-19T01:38:41.045 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:40 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "h"}]: dispatch 2024-03-19T01:38:41.045 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:40 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/1917126871' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:38:41.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:40 smithi099 ceph-mon[62951]: Deploying daemon mds.i on smithi115 2024-03-19T01:38:41.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:40 smithi099 ceph-mon[62951]: mds.? [v2:172.21.15.112:6836/2339823159,v1:172.21.15.112:6837/2339823159] up:boot 2024-03-19T01:38:41.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:40 smithi099 ceph-mon[62951]: fsmap 8 up:standby 2024-03-19T01:38:41.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:40 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "h"}]: dispatch 2024-03-19T01:38:41.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:40 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/1917126871' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:38:41.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:40 smithi112 ceph-mon[66819]: Deploying daemon mds.i on smithi115 2024-03-19T01:38:41.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:40 smithi112 ceph-mon[66819]: mds.? [v2:172.21.15.112:6836/2339823159,v1:172.21.15.112:6837/2339823159] up:boot 2024-03-19T01:38:41.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:40 smithi112 ceph-mon[66819]: fsmap 8 up:standby 2024-03-19T01:38:41.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:40 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "h"}]: dispatch 2024-03-19T01:38:41.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:40 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/1917126871' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:38:41.306 INFO:journalctl@ceph.mds.i.smithi115.stdout:Mar 19 01:38:41 smithi115 systemd[1]: Starting Ceph mds.i for 438fb69c-e590-11ee-95c9-87774f69a715... 2024-03-19T01:38:41.730 INFO:journalctl@ceph.mds.i.smithi115.stdout:Mar 19 01:38:41 smithi115 podman[97444]: 2024-03-19 01:38:41.345412767 +0000 UTC m=+0.017878367 image pull 2312c8e42d0069a2dbb2364bddb00264db4b800e7f18a7e7e4eb8f3ecab2ff1c quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639 2024-03-19T01:38:41.730 INFO:journalctl@ceph.mds.i.smithi115.stdout:Mar 19 01:38:41 smithi115 podman[97444]: 2024-03-19 01:38:41.452857278 +0000 UTC m=+0.125322928 container create 03bb723802445487840469d87aff49344f197edfb94d992c33eff8a8ca3c3c65 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-i, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.build-date=20240307, CEPH_POINT_RELEASE=) 2024-03-19T01:38:41.730 INFO:journalctl@ceph.mds.i.smithi115.stdout:Mar 19 01:38:41 smithi115 podman[97444]: 2024-03-19 01:38:41.655158467 +0000 UTC m=+0.327624054 container init 03bb723802445487840469d87aff49344f197edfb94d992c33eff8a8ca3c3c65 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-i, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240307, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.33.5, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.license=GPLv2, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, CEPH_POINT_RELEASE=) 2024-03-19T01:38:41.730 INFO:journalctl@ceph.mds.i.smithi115.stdout:Mar 19 01:38:41 smithi115 podman[97444]: 2024-03-19 01:38:41.657755394 +0000 UTC m=+0.330220974 container start 03bb723802445487840469d87aff49344f197edfb94d992c33eff8a8ca3c3c65 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-i, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240307, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-03-19T01:38:41.730 INFO:journalctl@ceph.mds.i.smithi115.stdout:Mar 19 01:38:41 smithi115 ceph-mds[97463]: -- 172.21.15.115:0/2142813186 <== mon.1 v2:172.21.15.115:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55f85d3fba40 con 0x55f85d3cec00 2024-03-19T01:38:41.730 INFO:journalctl@ceph.mds.i.smithi115.stdout:Mar 19 01:38:41 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-i[97459]: starting mds.i at 2024-03-19T01:38:41.730 INFO:journalctl@ceph.mds.i.smithi115.stdout:Mar 19 01:38:41 smithi115 bash[97444]: 03bb723802445487840469d87aff49344f197edfb94d992c33eff8a8ca3c3c65 2024-03-19T01:38:42.056 INFO:journalctl@ceph.mds.i.smithi115.stdout:Mar 19 01:38:41 smithi115 systemd[1]: Started Ceph mds.i for 438fb69c-e590-11ee-95c9-87774f69a715. 2024-03-19T01:38:42.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:41 smithi115 ceph-mon[67332]: pgmap v270: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:42.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:41 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:42.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:41 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:42.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:41 smithi099 ceph-mon[62951]: pgmap v270: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:42.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:41 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:42.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:41 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:42.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:41 smithi112 ceph-mon[66819]: pgmap v270: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:42.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:41 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:42.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:41 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:42.765 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-19T01:38:42.873 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:42.870+0000 7f622e587640 1 -- 172.21.15.99:0/1961969378 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6228100760 msgr2=0x7f6228100b80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:42.873 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:42.870+0000 7f622e587640 1 --2- 172.21.15.99:0/1961969378 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6228100760 0x7f6228100b80 secure :-1 s=READY pgs=175 cs=0 l=1 rev1=1 crypto rx=0x7f6218009900 tx=0x7f621802bc30 comp rx=0 tx=0).stop 2024-03-19T01:38:42.873 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:42.870+0000 7f622e587640 1 -- 172.21.15.99:0/1961969378 shutdown_connections 2024-03-19T01:38:42.873 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:42.870+0000 7f622e587640 1 --2- 172.21.15.99:0/1961969378 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6228102310 0x7f6228108ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:42.874 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:42.870+0000 7f622e587640 1 --2- 172.21.15.99:0/1961969378 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f6228101950 0x7f6228101dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:42.874 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:42.870+0000 7f622e587640 1 --2- 172.21.15.99:0/1961969378 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6228100760 0x7f6228100b80 unknown :-1 s=CLOSED pgs=175 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:42.874 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:42.870+0000 7f622e587640 1 -- 172.21.15.99:0/1961969378 >> 172.21.15.99:0/1961969378 conn(0x7f62280fbcd0 msgr2=0x7f62280fe110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:42.874 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:42.870+0000 7f622e587640 1 -- 172.21.15.99:0/1961969378 shutdown_connections 2024-03-19T01:38:42.874 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:42.870+0000 7f622e587640 1 -- 172.21.15.99:0/1961969378 wait complete. 2024-03-19T01:38:42.874 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:42.871+0000 7f622e587640 1 Processor -- start 2024-03-19T01:38:42.874 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:42.871+0000 7f622e587640 1 -- start start 2024-03-19T01:38:42.874 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:42.871+0000 7f622e587640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f6228100760 0x7f6228197bf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:42.874 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:42.871+0000 7f622e587640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6228101950 0x7f6228198130 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:42.874 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:42.871+0000 7f622e587640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6228102310 0x7f622819f1a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:42.874 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:42.871+0000 7f622e587640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f622810d540 con 0x7f6228102310 2024-03-19T01:38:42.874 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:42.871+0000 7f622e587640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f622810d3c0 con 0x7f6228101950 2024-03-19T01:38:42.875 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:42.871+0000 7f622e587640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f622810d6c0 con 0x7f6228100760 2024-03-19T01:38:42.875 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:42.871+0000 7f622cb24640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6228102310 0x7f622819f1a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:42.875 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:42.871+0000 7f622cb24640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6228102310 0x7f622819f1a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:57426/0 (socket says 172.21.15.99:57426) 2024-03-19T01:38:42.875 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:42.871+0000 7f622cb24640 1 -- 172.21.15.99:0/477712731 learned_addr learned my addr 172.21.15.99:0/477712731 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:38:42.875 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:42.871+0000 7f6227fff640 1 --2- 172.21.15.99:0/477712731 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f6228100760 0x7f6228197bf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:42.875 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:42.871+0000 7f62277fe640 1 --2- 172.21.15.99:0/477712731 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6228101950 0x7f6228198130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:42.875 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:42.871+0000 7f622cb24640 1 -- 172.21.15.99:0/477712731 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f6228100760 msgr2=0x7f6228197bf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:42.875 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:42.871+0000 7f622cb24640 1 --2- 172.21.15.99:0/477712731 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f6228100760 0x7f6228197bf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:42.875 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:42.871+0000 7f622cb24640 1 -- 172.21.15.99:0/477712731 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6228101950 msgr2=0x7f6228198130 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:42.875 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:42.871+0000 7f622cb24640 1 --2- 172.21.15.99:0/477712731 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6228101950 0x7f6228198130 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:42.875 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:42.871+0000 7f622cb24640 1 -- 172.21.15.99:0/477712731 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f622819f810 con 0x7f6228102310 2024-03-19T01:38:42.875 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:42.871+0000 7f62277fe640 1 --2- 172.21.15.99:0/477712731 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6228101950 0x7f6228198130 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-19T01:38:42.876 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:42.872+0000 7f622cb24640 1 --2- 172.21.15.99:0/477712731 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6228102310 0x7f622819f1a0 secure :-1 s=READY pgs=176 cs=0 l=1 rev1=1 crypto rx=0x7f622000eaa0 tx=0x7f622000ef60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:42.876 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:42.872+0000 7f62257fa640 1 -- 172.21.15.99:0/477712731 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f622000cdb0 con 0x7f6228102310 2024-03-19T01:38:42.876 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:42.872+0000 7f622e587640 1 -- 172.21.15.99:0/477712731 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f622819fb00 con 0x7f6228102310 2024-03-19T01:38:42.876 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:42.872+0000 7f62257fa640 1 -- 172.21.15.99:0/477712731 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f6220005020 con 0x7f6228102310 2024-03-19T01:38:42.876 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:42.872+0000 7f62257fa640 1 -- 172.21.15.99:0/477712731 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f62200116b0 con 0x7f6228102310 2024-03-19T01:38:42.876 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:42.872+0000 7f622e587640 1 -- 172.21.15.99:0/477712731 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f62281a0000 con 0x7f6228102310 2024-03-19T01:38:42.876 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:42.873+0000 7f622e587640 1 -- 172.21.15.99:0/477712731 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f6228107a80 con 0x7f6228102310 2024-03-19T01:38:42.877 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:42.874+0000 7f62257fa640 1 -- 172.21.15.99:0/477712731 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 16) ==== 137290+0+0 (secure 0 0 0) 0x7f6220005560 con 0x7f6228102310 2024-03-19T01:38:42.877 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:42.875+0000 7f62257fa640 1 --2- 172.21.15.99:0/477712731 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f62100a2800 0x7f62100a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:42.877 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:42.875+0000 7f6227fff640 1 --2- 172.21.15.99:0/477712731 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f62100a2800 0x7f62100a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:42.877 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:42.875+0000 7f62257fa640 1 -- 172.21.15.99:0/477712731 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(83..83 src has 1..83) ==== 7127+0+0 (secure 0 0 0) 0x7f62200ce830 con 0x7f6228102310 2024-03-19T01:38:42.877 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:42.875+0000 7f6227fff640 1 --2- 172.21.15.99:0/477712731 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f62100a2800 0x7f62100a4cc0 secure :-1 s=READY pgs=148 cs=0 l=1 rev1=1 crypto rx=0x7f6218000c00 tx=0x7f6218005fd0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:42.879 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:42.879+0000 7f62257fa640 1 -- 172.21.15.99:0/477712731 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f6220016030 con 0x7f6228102310 2024-03-19T01:38:43.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:42 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:43.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:42 smithi099 ceph-mon[62951]: mds.? [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] up:boot 2024-03-19T01:38:43.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:42 smithi099 ceph-mon[62951]: fsmap 9 up:standby 2024-03-19T01:38:43.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:42 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "i"}]: dispatch 2024-03-19T01:38:43.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:42 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.g", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-19T01:38:43.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:42 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.g", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-19T01:38:43.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:42 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:38:43.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:42 smithi099 ceph-mon[62951]: Deploying daemon mds.g on smithi099 2024-03-19T01:38:43.096 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:43.096+0000 7f622e587640 1 -- 172.21.15.99:0/477712731 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f622802d5f0 con 0x7f6228102310 2024-03-19T01:38:43.097 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:43.097+0000 7f62257fa640 1 -- 172.21.15.99:0/477712731 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 10 v10) ==== 94+0+8013 (secure 0 0 0) 0x7f622009bd70 con 0x7f6228102310 2024-03-19T01:38:43.098 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:38:43.098 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":10,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14562,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6835/1631815621","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6834","nonce":1631815621},{"type":"v1","addr":"172.21.15.99:6835","nonce":1631815621}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":4},{"gid":24383,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.112:6833/2969810977","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6832","nonce":2969810977},{"type":"v1","addr":"172.21.15.112:6833","nonce":2969810977}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3},{"gid":24404,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6837/2038634879","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6836","nonce":2038634879},{"type":"v1","addr":"172.21.15.99:6837","nonce":2038634879}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":24415,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.115:6833/2219320481","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6832","nonce":2219320481},{"type":"v1","addr":"172.21.15.115:6833","nonce":2219320481}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2},{"gid":24416,"name":"e","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.112:6835/3690587528","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6834","nonce":3690587528},{"type":"v1","addr":"172.21.15.112:6835","nonce":3690587528}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":6},{"gid":24440,"name":"h","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.112:6837/2339823159","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6836","nonce":2339823159},{"type":"v1","addr":"172.21.15.112:6837","nonce":2339823159}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":9},{"gid":24460,"name":"f","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.115:6835/591078413","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6834","nonce":591078413},{"type":"v1","addr":"172.21.15.115:6835","nonce":591078413}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":7},{"gid":24472,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.115:6837/1368988191","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6836","nonce":1368988191},{"type":"v1","addr":"172.21.15.115:6837","nonce":1368988191}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":8},{"gid":24487,"name":"i","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.115:6839/2743761968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6838","nonce":2743761968},{"type":"v1","addr":"172.21.15.115:6839","nonce":2743761968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":10}],"filesystems":[]} 2024-03-19T01:38:43.098 INFO:teuthology.orchestra.run.smithi099.stderr:dumped fsmap epoch 10 2024-03-19T01:38:43.100 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:43.099+0000 7f622e587640 1 -- 172.21.15.99:0/477712731 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f62100a2800 msgr2=0x7f62100a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:43.100 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:43.099+0000 7f622e587640 1 --2- 172.21.15.99:0/477712731 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f62100a2800 0x7f62100a4cc0 secure :-1 s=READY pgs=148 cs=0 l=1 rev1=1 crypto rx=0x7f6218000c00 tx=0x7f6218005fd0 comp rx=0 tx=0).stop 2024-03-19T01:38:43.100 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:43.099+0000 7f622e587640 1 -- 172.21.15.99:0/477712731 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6228102310 msgr2=0x7f622819f1a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:43.100 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:43.099+0000 7f622e587640 1 --2- 172.21.15.99:0/477712731 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6228102310 0x7f622819f1a0 secure :-1 s=READY pgs=176 cs=0 l=1 rev1=1 crypto rx=0x7f622000eaa0 tx=0x7f622000ef60 comp rx=0 tx=0).stop 2024-03-19T01:38:43.100 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:43.100+0000 7f622e587640 1 -- 172.21.15.99:0/477712731 shutdown_connections 2024-03-19T01:38:43.100 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:43.100+0000 7f622e587640 1 --2- 172.21.15.99:0/477712731 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f62100a2800 0x7f62100a4cc0 unknown :-1 s=CLOSED pgs=148 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:43.101 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:43.100+0000 7f622e587640 1 --2- 172.21.15.99:0/477712731 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6228101950 0x7f6228198130 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:43.101 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:43.100+0000 7f622e587640 1 --2- 172.21.15.99:0/477712731 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6228102310 0x7f622819f1a0 unknown :-1 s=CLOSED pgs=176 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:43.101 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:43.100+0000 7f622e587640 1 --2- 172.21.15.99:0/477712731 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f6228100760 0x7f6228197bf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:43.101 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:43.100+0000 7f622e587640 1 -- 172.21.15.99:0/477712731 >> 172.21.15.99:0/477712731 conn(0x7f62280fbcd0 msgr2=0x7f6228104c50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:43.101 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:43.100+0000 7f622e587640 1 -- 172.21.15.99:0/477712731 shutdown_connections 2024-03-19T01:38:43.101 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:43.100+0000 7f622e587640 1 -- 172.21.15.99:0/477712731 wait complete. 2024-03-19T01:38:43.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:42 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:43.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:42 smithi112 ceph-mon[66819]: mds.? [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] up:boot 2024-03-19T01:38:43.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:42 smithi112 ceph-mon[66819]: fsmap 9 up:standby 2024-03-19T01:38:43.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:42 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "i"}]: dispatch 2024-03-19T01:38:43.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:42 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.g", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-19T01:38:43.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:42 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.g", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-19T01:38:43.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:42 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:38:43.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:42 smithi112 ceph-mon[66819]: Deploying daemon mds.g on smithi099 2024-03-19T01:38:43.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:42 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:43.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:42 smithi115 ceph-mon[67332]: mds.? [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] up:boot 2024-03-19T01:38:43.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:42 smithi115 ceph-mon[67332]: fsmap 9 up:standby 2024-03-19T01:38:43.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:42 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "i"}]: dispatch 2024-03-19T01:38:43.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:42 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.g", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-19T01:38:43.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:42 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.g", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-19T01:38:43.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:42 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:38:43.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:42 smithi115 ceph-mon[67332]: Deploying daemon mds.g on smithi099 2024-03-19T01:38:44.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:43 smithi099 ceph-mon[62951]: pgmap v271: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:44.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:43 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/477712731' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:38:44.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:43 smithi112 ceph-mon[66819]: pgmap v271: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:44.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:43 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/477712731' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:38:44.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:43 smithi115 ceph-mon[67332]: pgmap v271: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:44.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:43 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/477712731' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:38:44.841 INFO:journalctl@ceph.mds.g.smithi099.stdout:Mar 19 01:38:44 smithi099 systemd[1]: Started Ceph mds.g for 438fb69c-e590-11ee-95c9-87774f69a715. 2024-03-19T01:38:45.112 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-19T01:38:45.225 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:45.222+0000 7f8c2ff6f640 1 -- 172.21.15.99:0/590203262 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8c28100740 msgr2=0x7f8c28100b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:45.225 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:45.222+0000 7f8c2ff6f640 1 --2- 172.21.15.99:0/590203262 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8c28100740 0x7f8c28100b60 secure :-1 s=READY pgs=179 cs=0 l=1 rev1=1 crypto rx=0x7f8c24009900 tx=0x7f8c2402bc30 comp rx=0 tx=0).stop 2024-03-19T01:38:45.225 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:45.222+0000 7f8c2ff6f640 1 -- 172.21.15.99:0/590203262 shutdown_connections 2024-03-19T01:38:45.225 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:45.222+0000 7f8c2ff6f640 1 --2- 172.21.15.99:0/590203262 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8c28101930 0x7f8c28101db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:45.225 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:45.222+0000 7f8c2ff6f640 1 --2- 172.21.15.99:0/590203262 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8c281022f0 0x7f8c28108b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:45.225 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:45.222+0000 7f8c2ff6f640 1 --2- 172.21.15.99:0/590203262 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8c28100740 0x7f8c28100b60 unknown :-1 s=CLOSED pgs=179 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:45.225 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:45.222+0000 7f8c2ff6f640 1 -- 172.21.15.99:0/590203262 >> 172.21.15.99:0/590203262 conn(0x7f8c280fbcb0 msgr2=0x7f8c280fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:45.225 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:45.222+0000 7f8c2ff6f640 1 -- 172.21.15.99:0/590203262 shutdown_connections 2024-03-19T01:38:45.225 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:45.223+0000 7f8c2ff6f640 1 -- 172.21.15.99:0/590203262 wait complete. 2024-03-19T01:38:45.225 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:45.223+0000 7f8c2ff6f640 1 Processor -- start 2024-03-19T01:38:45.226 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:45.223+0000 7f8c2ff6f640 1 -- start start 2024-03-19T01:38:45.226 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:45.223+0000 7f8c2ff6f640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8c28100740 0x7f8c28196a90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:45.226 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:45.223+0000 7f8c2ff6f640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8c28101930 0x7f8c28196fd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:45.226 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:45.223+0000 7f8c2ff6f640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8c281022f0 0x7f8c2819e040 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:45.226 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:45.223+0000 7f8c2ff6f640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f8c2810c390 con 0x7f8c281022f0 2024-03-19T01:38:45.226 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:45.223+0000 7f8c2ff6f640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f8c2810c210 con 0x7f8c28101930 2024-03-19T01:38:45.226 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:45.223+0000 7f8c2ff6f640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f8c2810c510 con 0x7f8c28100740 2024-03-19T01:38:45.226 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:45.223+0000 7f8c2e50c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8c281022f0 0x7f8c2819e040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:45.226 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:45.223+0000 7f8c2e50c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8c281022f0 0x7f8c2819e040 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:57448/0 (socket says 172.21.15.99:57448) 2024-03-19T01:38:45.226 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:45.223+0000 7f8c2e50c640 1 -- 172.21.15.99:0/3077041202 learned_addr learned my addr 172.21.15.99:0/3077041202 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:38:45.226 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:45.223+0000 7f8c2dd0b640 1 --2- 172.21.15.99:0/3077041202 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8c28100740 0x7f8c28196a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:45.226 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:45.224+0000 7f8c2d50a640 1 --2- 172.21.15.99:0/3077041202 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8c28101930 0x7f8c28196fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:45.226 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:45.224+0000 7f8c2e50c640 1 -- 172.21.15.99:0/3077041202 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8c28100740 msgr2=0x7f8c28196a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:45.227 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:45.224+0000 7f8c2e50c640 1 --2- 172.21.15.99:0/3077041202 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8c28100740 0x7f8c28196a90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:45.227 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:45.224+0000 7f8c2e50c640 1 -- 172.21.15.99:0/3077041202 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8c28101930 msgr2=0x7f8c28196fd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:45.227 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:45.224+0000 7f8c2e50c640 1 --2- 172.21.15.99:0/3077041202 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8c28101930 0x7f8c28196fd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:45.227 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:45.224+0000 7f8c2e50c640 1 -- 172.21.15.99:0/3077041202 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f8c2819e6b0 con 0x7f8c281022f0 2024-03-19T01:38:45.227 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:45.224+0000 7f8c2e50c640 1 --2- 172.21.15.99:0/3077041202 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8c281022f0 0x7f8c2819e040 secure :-1 s=READY pgs=180 cs=0 l=1 rev1=1 crypto rx=0x7f8c20009fd0 tx=0x7f8c2000b260 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:45.227 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:45.224+0000 7f8c1effd640 1 -- 172.21.15.99:0/3077041202 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8c20017070 con 0x7f8c281022f0 2024-03-19T01:38:45.227 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:45.224+0000 7f8c2ff6f640 1 -- 172.21.15.99:0/3077041202 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f8c2819e9a0 con 0x7f8c281022f0 2024-03-19T01:38:45.227 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:45.224+0000 7f8c1effd640 1 -- 172.21.15.99:0/3077041202 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f8c20003e50 con 0x7f8c281022f0 2024-03-19T01:38:45.227 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:45.224+0000 7f8c1effd640 1 -- 172.21.15.99:0/3077041202 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8c200106c0 con 0x7f8c281022f0 2024-03-19T01:38:45.227 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:45.224+0000 7f8c2ff6f640 1 -- 172.21.15.99:0/3077041202 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f8c2819eea0 con 0x7f8c281022f0 2024-03-19T01:38:45.228 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:45.225+0000 7f8c2ff6f640 1 -- 172.21.15.99:0/3077041202 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f8c28107ae0 con 0x7f8c281022f0 2024-03-19T01:38:45.228 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:45.227+0000 7f8c1effd640 1 -- 172.21.15.99:0/3077041202 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 16) ==== 137290+0+0 (secure 0 0 0) 0x7f8c20003970 con 0x7f8c281022f0 2024-03-19T01:38:45.228 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:45.228+0000 7f8c1effd640 1 --2- 172.21.15.99:0/3077041202 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f8c140a2800 0x7f8c140a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:45.228 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:45.228+0000 7f8c2dd0b640 1 --2- 172.21.15.99:0/3077041202 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f8c140a2800 0x7f8c140a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:45.228 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:45.228+0000 7f8c1effd640 1 -- 172.21.15.99:0/3077041202 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(83..83 src has 1..83) ==== 7127+0+0 (secure 0 0 0) 0x7f8c200ce2a0 con 0x7f8c281022f0 2024-03-19T01:38:45.229 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:45.228+0000 7f8c2dd0b640 1 --2- 172.21.15.99:0/3077041202 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f8c140a2800 0x7f8c140a4cc0 secure :-1 s=READY pgs=150 cs=0 l=1 rev1=1 crypto rx=0x7f8c24000c00 tx=0x7f8c24005fd0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:45.231 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:45.231+0000 7f8c1effd640 1 -- 172.21.15.99:0/3077041202 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f8c2009b190 con 0x7f8c281022f0 2024-03-19T01:38:45.461 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:45.460+0000 7f8c2ff6f640 1 -- 172.21.15.99:0/3077041202 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f8c2802d1f0 con 0x7f8c281022f0 2024-03-19T01:38:45.461 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:45.461+0000 7f8c1effd640 1 -- 172.21.15.99:0/3077041202 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 11 v11) ==== 94+0+8839 (secure 0 0 0) 0x7f8c2009f230 con 0x7f8c281022f0 2024-03-19T01:38:45.462 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:38:45.462 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":11,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14562,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6835/1631815621","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6834","nonce":1631815621},{"type":"v1","addr":"172.21.15.99:6835","nonce":1631815621}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":4},{"gid":14646,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6839/2236145144","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6838","nonce":2236145144},{"type":"v1","addr":"172.21.15.99:6839","nonce":2236145144}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11},{"gid":24383,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.112:6833/2969810977","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6832","nonce":2969810977},{"type":"v1","addr":"172.21.15.112:6833","nonce":2969810977}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3},{"gid":24404,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6837/2038634879","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6836","nonce":2038634879},{"type":"v1","addr":"172.21.15.99:6837","nonce":2038634879}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":24415,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.115:6833/2219320481","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6832","nonce":2219320481},{"type":"v1","addr":"172.21.15.115:6833","nonce":2219320481}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2},{"gid":24416,"name":"e","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.112:6835/3690587528","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6834","nonce":3690587528},{"type":"v1","addr":"172.21.15.112:6835","nonce":3690587528}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":6},{"gid":24440,"name":"h","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.112:6837/2339823159","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6836","nonce":2339823159},{"type":"v1","addr":"172.21.15.112:6837","nonce":2339823159}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":9},{"gid":24460,"name":"f","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.115:6835/591078413","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6834","nonce":591078413},{"type":"v1","addr":"172.21.15.115:6835","nonce":591078413}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":7},{"gid":24472,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.115:6837/1368988191","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6836","nonce":1368988191},{"type":"v1","addr":"172.21.15.115:6837","nonce":1368988191}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":8},{"gid":24487,"name":"i","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.115:6839/2743761968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6838","nonce":2743761968},{"type":"v1","addr":"172.21.15.115:6839","nonce":2743761968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":10}],"filesystems":[]} 2024-03-19T01:38:45.462 INFO:teuthology.orchestra.run.smithi099.stderr:dumped fsmap epoch 11 2024-03-19T01:38:45.464 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:45.463+0000 7f8c2ff6f640 1 -- 172.21.15.99:0/3077041202 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f8c140a2800 msgr2=0x7f8c140a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:45.464 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:45.463+0000 7f8c2ff6f640 1 --2- 172.21.15.99:0/3077041202 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f8c140a2800 0x7f8c140a4cc0 secure :-1 s=READY pgs=150 cs=0 l=1 rev1=1 crypto rx=0x7f8c24000c00 tx=0x7f8c24005fd0 comp rx=0 tx=0).stop 2024-03-19T01:38:45.464 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:45.463+0000 7f8c2ff6f640 1 -- 172.21.15.99:0/3077041202 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8c281022f0 msgr2=0x7f8c2819e040 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:45.464 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:45.463+0000 7f8c2ff6f640 1 --2- 172.21.15.99:0/3077041202 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8c281022f0 0x7f8c2819e040 secure :-1 s=READY pgs=180 cs=0 l=1 rev1=1 crypto rx=0x7f8c20009fd0 tx=0x7f8c2000b260 comp rx=0 tx=0).stop 2024-03-19T01:38:45.464 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:45.463+0000 7f8c2ff6f640 1 -- 172.21.15.99:0/3077041202 shutdown_connections 2024-03-19T01:38:45.464 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:45.463+0000 7f8c2ff6f640 1 --2- 172.21.15.99:0/3077041202 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f8c140a2800 0x7f8c140a4cc0 unknown :-1 s=CLOSED pgs=150 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:45.464 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:45.463+0000 7f8c2ff6f640 1 --2- 172.21.15.99:0/3077041202 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8c28101930 0x7f8c28196fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:45.464 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:45.463+0000 7f8c2ff6f640 1 --2- 172.21.15.99:0/3077041202 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8c281022f0 0x7f8c2819e040 unknown :-1 s=CLOSED pgs=180 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:45.464 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:45.463+0000 7f8c2ff6f640 1 --2- 172.21.15.99:0/3077041202 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8c28100740 0x7f8c28196a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:45.464 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:45.463+0000 7f8c2ff6f640 1 -- 172.21.15.99:0/3077041202 >> 172.21.15.99:0/3077041202 conn(0x7f8c280fbcb0 msgr2=0x7f8c28104cb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:45.464 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:45.463+0000 7f8c2ff6f640 1 -- 172.21.15.99:0/3077041202 shutdown_connections 2024-03-19T01:38:45.464 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:45.463+0000 7f8c2ff6f640 1 -- 172.21.15.99:0/3077041202 wait complete. 2024-03-19T01:38:45.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:45 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:45.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:45 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:45.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:45 smithi099 ceph-mon[62951]: pgmap v272: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:45.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:45 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:45.842 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:45 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-19T01:38:45.842 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:45 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-19T01:38:45.842 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:45 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:38:45.842 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:45 smithi099 ceph-mon[62951]: Deploying daemon mds.a on smithi099 2024-03-19T01:38:45.842 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:45 smithi099 ceph-mon[62951]: mds.? [v2:172.21.15.99:6838/2236145144,v1:172.21.15.99:6839/2236145144] up:boot 2024-03-19T01:38:45.842 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:45 smithi099 ceph-mon[62951]: fsmap 10 up:standby 2024-03-19T01:38:45.842 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:45 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "g"}]: dispatch 2024-03-19T01:38:45.842 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:45 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/3077041202' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:38:46.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:45 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:46.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:45 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:46.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:45 smithi115 ceph-mon[67332]: pgmap v272: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:46.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:45 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:46.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:45 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-19T01:38:46.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:45 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-19T01:38:46.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:45 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:38:46.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:45 smithi115 ceph-mon[67332]: Deploying daemon mds.a on smithi099 2024-03-19T01:38:46.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:45 smithi115 ceph-mon[67332]: mds.? [v2:172.21.15.99:6838/2236145144,v1:172.21.15.99:6839/2236145144] up:boot 2024-03-19T01:38:46.058 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:45 smithi115 ceph-mon[67332]: fsmap 10 up:standby 2024-03-19T01:38:46.058 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:45 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "g"}]: dispatch 2024-03-19T01:38:46.058 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:45 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/3077041202' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:38:46.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:45 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:46.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:45 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:46.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:45 smithi112 ceph-mon[66819]: pgmap v272: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:46.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:45 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:46.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:45 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-19T01:38:46.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:45 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-19T01:38:46.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:45 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:38:46.113 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:45 smithi112 ceph-mon[66819]: Deploying daemon mds.a on smithi099 2024-03-19T01:38:46.113 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:45 smithi112 ceph-mon[66819]: mds.? [v2:172.21.15.99:6838/2236145144,v1:172.21.15.99:6839/2236145144] up:boot 2024-03-19T01:38:46.113 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:45 smithi112 ceph-mon[66819]: fsmap 10 up:standby 2024-03-19T01:38:46.113 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:45 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "g"}]: dispatch 2024-03-19T01:38:46.113 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:45 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/3077041202' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:38:47.022 INFO:journalctl@ceph.mds.a.smithi099.stdout:Mar 19 01:38:46 smithi099 systemd[1]: Starting Ceph mds.a for 438fb69c-e590-11ee-95c9-87774f69a715... 2024-03-19T01:38:47.294 INFO:journalctl@ceph.mds.a.smithi099.stdout:Mar 19 01:38:47 smithi099 podman[104719]: 2024-03-19 01:38:46.926751238 +0000 UTC m=+0.018759019 image pull 2312c8e42d0069a2dbb2364bddb00264db4b800e7f18a7e7e4eb8f3ecab2ff1c quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639 2024-03-19T01:38:47.294 INFO:journalctl@ceph.mds.a.smithi099.stdout:Mar 19 01:38:47 smithi099 podman[104719]: 2024-03-19 01:38:47.02377219 +0000 UTC m=+0.115779962 container create 4b3805d58da22fca839a8e7fa803a6033ce20cac387521647211070b8f5d9c2d (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-a, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.license=GPLv2, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.build-date=20240307, org.label-schema.vendor=CentOS) 2024-03-19T01:38:47.294 INFO:journalctl@ceph.mds.a.smithi099.stdout:Mar 19 01:38:47 smithi099 podman[104719]: 2024-03-19 01:38:47.268187934 +0000 UTC m=+0.360195716 container init 4b3805d58da22fca839a8e7fa803a6033ce20cac387521647211070b8f5d9c2d (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-a, GIT_CLEAN=True, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240307, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30) 2024-03-19T01:38:47.295 INFO:journalctl@ceph.mds.a.smithi099.stdout:Mar 19 01:38:47 smithi099 podman[104719]: 2024-03-19 01:38:47.27090535 +0000 UTC m=+0.362913125 container start 4b3805d58da22fca839a8e7fa803a6033ce20cac387521647211070b8f5d9c2d (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-a, org.label-schema.build-date=20240307, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.license=GPLv2, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5) 2024-03-19T01:38:47.476 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-19T01:38:47.581 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:47.580+0000 7ffa70d90640 1 -- 172.21.15.99:0/1831541664 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ffa6c102a30 msgr2=0x7ffa6c102eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:47.581 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:47.580+0000 7ffa70d90640 1 --2- 172.21.15.99:0/1831541664 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ffa6c102a30 0x7ffa6c102eb0 secure :-1 s=READY pgs=183 cs=0 l=1 rev1=1 crypto rx=0x7ffa54009900 tx=0x7ffa5402bc30 comp rx=0 tx=0).stop 2024-03-19T01:38:47.581 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:47.580+0000 7ffa70d90640 1 -- 172.21.15.99:0/1831541664 shutdown_connections 2024-03-19T01:38:47.581 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:47.580+0000 7ffa70d90640 1 --2- 172.21.15.99:0/1831541664 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ffa6c1033f0 0x7ffa6c109c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:47.581 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:47.580+0000 7ffa70d90640 1 --2- 172.21.15.99:0/1831541664 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ffa6c102a30 0x7ffa6c102eb0 unknown :-1 s=CLOSED pgs=183 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:47.581 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:47.580+0000 7ffa70d90640 1 --2- 172.21.15.99:0/1831541664 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ffa6c101840 0x7ffa6c101c60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:47.582 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:47.580+0000 7ffa70d90640 1 -- 172.21.15.99:0/1831541664 >> 172.21.15.99:0/1831541664 conn(0x7ffa6c0fcdd0 msgr2=0x7ffa6c0ff1f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:47.582 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:47.581+0000 7ffa70d90640 1 -- 172.21.15.99:0/1831541664 shutdown_connections 2024-03-19T01:38:47.582 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:47.581+0000 7ffa70d90640 1 -- 172.21.15.99:0/1831541664 wait complete. 2024-03-19T01:38:47.584 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:47.581+0000 7ffa70d90640 1 Processor -- start 2024-03-19T01:38:47.584 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:47.581+0000 7ffa70d90640 1 -- start start 2024-03-19T01:38:47.584 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:47.582+0000 7ffa70d90640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ffa6c101840 0x7ffa6c197b60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:47.584 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:47.582+0000 7ffa70d90640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ffa6c102a30 0x7ffa6c1980a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:47.584 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:47.582+0000 7ffa70d90640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ffa6c1033f0 0x7ffa6c19f110 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:47.584 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:47.582+0000 7ffa70d90640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7ffa6c10d4c0 con 0x7ffa6c1033f0 2024-03-19T01:38:47.584 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:47.582+0000 7ffa70d90640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7ffa6c10d340 con 0x7ffa6c102a30 2024-03-19T01:38:47.584 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:47.582+0000 7ffa70d90640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7ffa6c10d640 con 0x7ffa6c101840 2024-03-19T01:38:47.584 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:47.582+0000 7ffa6ad9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ffa6c1033f0 0x7ffa6c19f110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:47.584 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:47.582+0000 7ffa6ad9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ffa6c1033f0 0x7ffa6c19f110 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:57488/0 (socket says 172.21.15.99:57488) 2024-03-19T01:38:47.584 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:47.582+0000 7ffa6ad9d640 1 -- 172.21.15.99:0/3975228543 learned_addr learned my addr 172.21.15.99:0/3975228543 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:38:47.584 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:47.582+0000 7ffa63fff640 1 --2- 172.21.15.99:0/3975228543 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ffa6c102a30 0x7ffa6c1980a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:47.584 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:47.582+0000 7ffa6ad9d640 1 -- 172.21.15.99:0/3975228543 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ffa6c101840 msgr2=0x7ffa6c197b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:47.585 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:47.582+0000 7ffa6ad9d640 1 --2- 172.21.15.99:0/3975228543 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ffa6c101840 0x7ffa6c197b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:47.585 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:47.582+0000 7ffa6ad9d640 1 -- 172.21.15.99:0/3975228543 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ffa6c102a30 msgr2=0x7ffa6c1980a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:47.585 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:47.582+0000 7ffa6ad9d640 1 --2- 172.21.15.99:0/3975228543 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ffa6c102a30 0x7ffa6c1980a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:47.585 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:47.582+0000 7ffa6ad9d640 1 -- 172.21.15.99:0/3975228543 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ffa6c19f780 con 0x7ffa6c1033f0 2024-03-19T01:38:47.585 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:47.582+0000 7ffa6ad9d640 1 --2- 172.21.15.99:0/3975228543 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ffa6c1033f0 0x7ffa6c19f110 secure :-1 s=READY pgs=184 cs=0 l=1 rev1=1 crypto rx=0x7ffa6400eb10 tx=0x7ffa6400c2d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:47.585 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:47.582+0000 7ffa637fe640 1 -- 172.21.15.99:0/3975228543 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffa64004b20 con 0x7ffa6c1033f0 2024-03-19T01:38:47.585 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:47.582+0000 7ffa70d90640 1 -- 172.21.15.99:0/3975228543 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ffa6c19fa70 con 0x7ffa6c1033f0 2024-03-19T01:38:47.585 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:47.582+0000 7ffa637fe640 1 -- 172.21.15.99:0/3975228543 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7ffa64004cc0 con 0x7ffa6c1033f0 2024-03-19T01:38:47.585 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:47.582+0000 7ffa637fe640 1 -- 172.21.15.99:0/3975228543 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffa64010760 con 0x7ffa6c1033f0 2024-03-19T01:38:47.585 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:47.582+0000 7ffa70d90640 1 -- 172.21.15.99:0/3975228543 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7ffa6c19ff90 con 0x7ffa6c1033f0 2024-03-19T01:38:47.585 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:47.583+0000 7ffa70d90640 1 -- 172.21.15.99:0/3975228543 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7ffa6c108b60 con 0x7ffa6c1033f0 2024-03-19T01:38:47.586 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:47.585+0000 7ffa637fe640 1 -- 172.21.15.99:0/3975228543 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 16) ==== 137290+0+0 (secure 0 0 0) 0x7ffa64010900 con 0x7ffa6c1033f0 2024-03-19T01:38:47.586 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:47.585+0000 7ffa637fe640 1 --2- 172.21.15.99:0/3975228543 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ffa580a2800 0x7ffa580a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:47.586 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:47.585+0000 7ffa6a59c640 1 --2- 172.21.15.99:0/3975228543 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ffa580a2800 0x7ffa580a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:47.586 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:47.586+0000 7ffa637fe640 1 -- 172.21.15.99:0/3975228543 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(83..83 src has 1..83) ==== 7127+0+0 (secure 0 0 0) 0x7ffa640cee10 con 0x7ffa6c1033f0 2024-03-19T01:38:47.586 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:47.586+0000 7ffa6a59c640 1 --2- 172.21.15.99:0/3975228543 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ffa580a2800 0x7ffa580a4cc0 secure :-1 s=READY pgs=151 cs=0 l=1 rev1=1 crypto rx=0x7ffa5c005fd0 tx=0x7ffa5c00b410 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:47.589 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:47.589+0000 7ffa637fe640 1 -- 172.21.15.99:0/3975228543 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7ffa64053740 con 0x7ffa6c1033f0 2024-03-19T01:38:47.591 INFO:journalctl@ceph.mds.a.smithi099.stdout:Mar 19 01:38:47 smithi099 ceph-mds[104738]: -- 172.21.15.99:0/2486559442 <== mon.0 v2:172.21.15.99:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x557ea023fa40 con 0x557ea0213000 2024-03-19T01:38:47.591 INFO:journalctl@ceph.mds.a.smithi099.stdout:Mar 19 01:38:47 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-a[104734]: starting mds.a at 2024-03-19T01:38:47.591 INFO:journalctl@ceph.mds.a.smithi099.stdout:Mar 19 01:38:47 smithi099 bash[104719]: 4b3805d58da22fca839a8e7fa803a6033ce20cac387521647211070b8f5d9c2d 2024-03-19T01:38:47.591 INFO:journalctl@ceph.mds.a.smithi099.stdout:Mar 19 01:38:47 smithi099 systemd[1]: Started Ceph mds.a for 438fb69c-e590-11ee-95c9-87774f69a715. 2024-03-19T01:38:47.803 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:47.803+0000 7ffa70d90640 1 -- 172.21.15.99:0/3975228543 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7ffa6c02d1f0 con 0x7ffa6c1033f0 2024-03-19T01:38:47.804 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:47.804+0000 7ffa637fe640 1 -- 172.21.15.99:0/3975228543 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 12 v12) ==== 94+0+9665 (secure 0 0 0) 0x7ffa6409c350 con 0x7ffa6c1033f0 2024-03-19T01:38:47.804 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:38:47.805 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":12,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14562,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6835/1631815621","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6834","nonce":1631815621},{"type":"v1","addr":"172.21.15.99:6835","nonce":1631815621}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":4},{"gid":14646,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6839/2236145144","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6838","nonce":2236145144},{"type":"v1","addr":"172.21.15.99:6839","nonce":2236145144}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11},{"gid":14658,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6841/1660990159","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6840","nonce":1660990159},{"type":"v1","addr":"172.21.15.99:6841","nonce":1660990159}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":12},{"gid":24383,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.112:6833/2969810977","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6832","nonce":2969810977},{"type":"v1","addr":"172.21.15.112:6833","nonce":2969810977}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3},{"gid":24404,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6837/2038634879","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6836","nonce":2038634879},{"type":"v1","addr":"172.21.15.99:6837","nonce":2038634879}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":24415,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.115:6833/2219320481","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6832","nonce":2219320481},{"type":"v1","addr":"172.21.15.115:6833","nonce":2219320481}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2},{"gid":24416,"name":"e","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.112:6835/3690587528","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6834","nonce":3690587528},{"type":"v1","addr":"172.21.15.112:6835","nonce":3690587528}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":6},{"gid":24440,"name":"h","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.112:6837/2339823159","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6836","nonce":2339823159},{"type":"v1","addr":"172.21.15.112:6837","nonce":2339823159}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":9},{"gid":24460,"name":"f","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.115:6835/591078413","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6834","nonce":591078413},{"type":"v1","addr":"172.21.15.115:6835","nonce":591078413}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":7},{"gid":24472,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.115:6837/1368988191","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6836","nonce":1368988191},{"type":"v1","addr":"172.21.15.115:6837","nonce":1368988191}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":8},{"gid":24487,"name":"i","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.115:6839/2743761968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6838","nonce":2743761968},{"type":"v1","addr":"172.21.15.115:6839","nonce":2743761968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":10}],"filesystems":[]} 2024-03-19T01:38:47.805 INFO:teuthology.orchestra.run.smithi099.stderr:dumped fsmap epoch 12 2024-03-19T01:38:47.807 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:47.806+0000 7ffa70d90640 1 -- 172.21.15.99:0/3975228543 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ffa580a2800 msgr2=0x7ffa580a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:47.807 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:47.806+0000 7ffa70d90640 1 --2- 172.21.15.99:0/3975228543 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ffa580a2800 0x7ffa580a4cc0 secure :-1 s=READY pgs=151 cs=0 l=1 rev1=1 crypto rx=0x7ffa5c005fd0 tx=0x7ffa5c00b410 comp rx=0 tx=0).stop 2024-03-19T01:38:47.807 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:47.806+0000 7ffa70d90640 1 -- 172.21.15.99:0/3975228543 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ffa6c1033f0 msgr2=0x7ffa6c19f110 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:47.807 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:47.806+0000 7ffa70d90640 1 --2- 172.21.15.99:0/3975228543 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ffa6c1033f0 0x7ffa6c19f110 secure :-1 s=READY pgs=184 cs=0 l=1 rev1=1 crypto rx=0x7ffa6400eb10 tx=0x7ffa6400c2d0 comp rx=0 tx=0).stop 2024-03-19T01:38:47.807 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:47.806+0000 7ffa70d90640 1 -- 172.21.15.99:0/3975228543 shutdown_connections 2024-03-19T01:38:47.807 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:47.806+0000 7ffa70d90640 1 --2- 172.21.15.99:0/3975228543 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ffa580a2800 0x7ffa580a4cc0 unknown :-1 s=CLOSED pgs=151 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:47.807 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:47.806+0000 7ffa70d90640 1 --2- 172.21.15.99:0/3975228543 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ffa6c102a30 0x7ffa6c1980a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:47.807 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:47.806+0000 7ffa70d90640 1 --2- 172.21.15.99:0/3975228543 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ffa6c1033f0 0x7ffa6c19f110 unknown :-1 s=CLOSED pgs=184 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:47.807 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:47.806+0000 7ffa70d90640 1 --2- 172.21.15.99:0/3975228543 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ffa6c101840 0x7ffa6c197b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:47.807 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:47.806+0000 7ffa70d90640 1 -- 172.21.15.99:0/3975228543 >> 172.21.15.99:0/3975228543 conn(0x7ffa6c0fcdd0 msgr2=0x7ffa6c105d30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:47.807 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:47.806+0000 7ffa70d90640 1 -- 172.21.15.99:0/3975228543 shutdown_connections 2024-03-19T01:38:47.807 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:47.806+0000 7ffa70d90640 1 -- 172.21.15.99:0/3975228543 wait complete. 2024-03-19T01:38:48.050 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:47 smithi112 ceph-mon[66819]: pgmap v273: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:48.050 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:47 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:48.051 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:47 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:48.051 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:47 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:48.051 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:47 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-19T01:38:48.051 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:47 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-19T01:38:48.051 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:47 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:38:48.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:47 smithi115 ceph-mon[67332]: pgmap v273: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:48.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:47 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:48.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:47 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:48.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:47 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:48.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:47 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-19T01:38:48.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:47 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-19T01:38:48.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:47 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:38:48.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:47 smithi099 ceph-mon[62951]: pgmap v273: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:48.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:47 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:48.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:47 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:48.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:47 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:48.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:47 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-19T01:38:48.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:47 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-19T01:38:48.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:47 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:38:49.051 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:48 smithi112 ceph-mon[66819]: Deploying daemon mds.b on smithi112 2024-03-19T01:38:49.051 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:48 smithi112 ceph-mon[66819]: mds.? [v2:172.21.15.99:6840/1660990159,v1:172.21.15.99:6841/1660990159] up:boot 2024-03-19T01:38:49.052 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:48 smithi112 ceph-mon[66819]: fsmap 11 up:standby 2024-03-19T01:38:49.052 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:48 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-03-19T01:38:49.052 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:48 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/3975228543' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:38:49.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:48 smithi115 ceph-mon[67332]: Deploying daemon mds.b on smithi112 2024-03-19T01:38:49.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:48 smithi115 ceph-mon[67332]: mds.? [v2:172.21.15.99:6840/1660990159,v1:172.21.15.99:6841/1660990159] up:boot 2024-03-19T01:38:49.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:48 smithi115 ceph-mon[67332]: fsmap 11 up:standby 2024-03-19T01:38:49.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:48 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-03-19T01:38:49.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:48 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/3975228543' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:38:49.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:48 smithi099 ceph-mon[62951]: Deploying daemon mds.b on smithi112 2024-03-19T01:38:49.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:48 smithi099 ceph-mon[62951]: mds.? [v2:172.21.15.99:6840/1660990159,v1:172.21.15.99:6841/1660990159] up:boot 2024-03-19T01:38:49.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:48 smithi099 ceph-mon[62951]: fsmap 11 up:standby 2024-03-19T01:38:49.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:48 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-03-19T01:38:49.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:48 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/3975228543' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:38:49.612 INFO:journalctl@ceph.mds.b.smithi112.stdout:Mar 19 01:38:49 smithi112 systemd[1]: Starting Ceph mds.b for 438fb69c-e590-11ee-95c9-87774f69a715... 2024-03-19T01:38:49.818 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-19T01:38:49.936 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:49.933+0000 7f13e21e8640 1 -- 172.21.15.99:0/2625103463 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f13dc102a30 msgr2=0x7f13dc102eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:49.936 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:49.933+0000 7f13e21e8640 1 --2- 172.21.15.99:0/2625103463 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f13dc102a30 0x7f13dc102eb0 secure :-1 s=READY pgs=185 cs=0 l=1 rev1=1 crypto rx=0x7f13d0009900 tx=0x7f13d002bc30 comp rx=0 tx=0).stop 2024-03-19T01:38:49.936 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:49.933+0000 7f13e21e8640 1 -- 172.21.15.99:0/2625103463 shutdown_connections 2024-03-19T01:38:49.936 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:49.933+0000 7f13e21e8640 1 --2- 172.21.15.99:0/2625103463 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f13dc1033f0 0x7f13dc109c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:49.936 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:49.933+0000 7f13e21e8640 1 --2- 172.21.15.99:0/2625103463 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f13dc102a30 0x7f13dc102eb0 unknown :-1 s=CLOSED pgs=185 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:49.936 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:49.933+0000 7f13e21e8640 1 --2- 172.21.15.99:0/2625103463 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f13dc101840 0x7f13dc101c60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:49.936 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:49.933+0000 7f13e21e8640 1 -- 172.21.15.99:0/2625103463 >> 172.21.15.99:0/2625103463 conn(0x7f13dc0fcdd0 msgr2=0x7f13dc0ff1f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:49.936 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:49.933+0000 7f13e21e8640 1 -- 172.21.15.99:0/2625103463 shutdown_connections 2024-03-19T01:38:49.936 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:49.933+0000 7f13e21e8640 1 -- 172.21.15.99:0/2625103463 wait complete. 2024-03-19T01:38:49.936 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:49.934+0000 7f13e21e8640 1 Processor -- start 2024-03-19T01:38:49.936 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:49.934+0000 7f13e21e8640 1 -- start start 2024-03-19T01:38:49.937 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:49.934+0000 7f13e21e8640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f13dc101840 0x7f13dc199d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:49.937 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:49.934+0000 7f13e21e8640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f13dc102a30 0x7f13dc19a280 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:49.937 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:49.934+0000 7f13e21e8640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f13dc1033f0 0x7f13dc1a12b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:49.937 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:49.934+0000 7f13e21e8640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f13dc10d510 con 0x7f13dc1033f0 2024-03-19T01:38:49.937 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:49.934+0000 7f13e21e8640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f13dc10d390 con 0x7f13dc101840 2024-03-19T01:38:49.937 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:49.934+0000 7f13e21e8640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f13dc10d690 con 0x7f13dc102a30 2024-03-19T01:38:49.937 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:49.934+0000 7f13dbfff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f13dc1033f0 0x7f13dc1a12b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:49.937 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:49.934+0000 7f13dbfff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f13dc1033f0 0x7f13dc1a12b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:57508/0 (socket says 172.21.15.99:57508) 2024-03-19T01:38:49.937 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:49.934+0000 7f13dbfff640 1 -- 172.21.15.99:0/1859135288 learned_addr learned my addr 172.21.15.99:0/1859135288 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:38:49.937 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:49.934+0000 7f13db7fe640 1 --2- 172.21.15.99:0/1859135288 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f13dc101840 0x7f13dc199d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:49.937 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:49.934+0000 7f13daffd640 1 --2- 172.21.15.99:0/1859135288 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f13dc102a30 0x7f13dc19a280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:49.937 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:49.934+0000 7f13dbfff640 1 -- 172.21.15.99:0/1859135288 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f13dc102a30 msgr2=0x7f13dc19a280 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:49.937 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:49.934+0000 7f13dbfff640 1 --2- 172.21.15.99:0/1859135288 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f13dc102a30 0x7f13dc19a280 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:49.938 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:49.934+0000 7f13dbfff640 1 -- 172.21.15.99:0/1859135288 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f13dc101840 msgr2=0x7f13dc199d40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:49.938 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:49.934+0000 7f13dbfff640 1 --2- 172.21.15.99:0/1859135288 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f13dc101840 0x7f13dc199d40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:49.938 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:49.934+0000 7f13dbfff640 1 -- 172.21.15.99:0/1859135288 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f13dc1a1950 con 0x7f13dc1033f0 2024-03-19T01:38:49.938 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:49.935+0000 7f13dbfff640 1 --2- 172.21.15.99:0/1859135288 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f13dc1033f0 0x7f13dc1a12b0 secure :-1 s=READY pgs=186 cs=0 l=1 rev1=1 crypto rx=0x7f13d400eb10 tx=0x7f13d400c2d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:49.938 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:49.935+0000 7f13d8ff9640 1 -- 172.21.15.99:0/1859135288 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f13d4004b20 con 0x7f13dc1033f0 2024-03-19T01:38:49.938 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:49.935+0000 7f13e21e8640 1 -- 172.21.15.99:0/1859135288 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f13dc1a1c40 con 0x7f13dc1033f0 2024-03-19T01:38:49.938 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:49.935+0000 7f13d8ff9640 1 -- 172.21.15.99:0/1859135288 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f13d4004cc0 con 0x7f13dc1033f0 2024-03-19T01:38:49.938 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:49.935+0000 7f13d8ff9640 1 -- 172.21.15.99:0/1859135288 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f13d4010760 con 0x7f13dc1033f0 2024-03-19T01:38:49.938 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:49.935+0000 7f13e21e8640 1 -- 172.21.15.99:0/1859135288 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f13dc1a2190 con 0x7f13dc1033f0 2024-03-19T01:38:49.939 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:49.936+0000 7f13e21e8640 1 -- 172.21.15.99:0/1859135288 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f13dc1a2360 con 0x7f13dc1033f0 2024-03-19T01:38:49.939 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:49.937+0000 7f13d8ff9640 1 -- 172.21.15.99:0/1859135288 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 16) ==== 137290+0+0 (secure 0 0 0) 0x7f13d4005020 con 0x7f13dc1033f0 2024-03-19T01:38:49.939 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:49.937+0000 7f13d8ff9640 1 --2- 172.21.15.99:0/1859135288 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f13c40a2800 0x7f13c40a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:49.939 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:49.937+0000 7f13db7fe640 1 --2- 172.21.15.99:0/1859135288 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f13c40a2800 0x7f13c40a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:49.939 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:49.938+0000 7f13d8ff9640 1 -- 172.21.15.99:0/1859135288 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(83..83 src has 1..83) ==== 7127+0+0 (secure 0 0 0) 0x7f13d40ced10 con 0x7f13dc1033f0 2024-03-19T01:38:49.939 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:49.938+0000 7f13db7fe640 1 --2- 172.21.15.99:0/1859135288 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f13c40a2800 0x7f13c40a4cc0 secure :-1 s=READY pgs=153 cs=0 l=1 rev1=1 crypto rx=0x7f13dc19b210 tx=0x7f13cc005f90 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:49.942 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:49.942+0000 7f13d8ff9640 1 -- 172.21.15.99:0/1859135288 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f13d409c250 con 0x7f13dc1033f0 2024-03-19T01:38:50.006 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:49 smithi112 ceph-mon[66819]: pgmap v274: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:50.006 INFO:journalctl@ceph.mds.b.smithi112.stdout:Mar 19 01:38:49 smithi112 podman[97387]: 2024-03-19 01:38:49.652871465 +0000 UTC m=+0.018086286 image pull 2312c8e42d0069a2dbb2364bddb00264db4b800e7f18a7e7e4eb8f3ecab2ff1c quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639 2024-03-19T01:38:50.006 INFO:journalctl@ceph.mds.b.smithi112.stdout:Mar 19 01:38:49 smithi112 podman[97387]: 2024-03-19 01:38:49.77062336 +0000 UTC m=+0.135838174 container create de00165d1e05d16e711ff811983f3ce64e232cb4953d8b1ef7eda6e813b1ae81 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-b, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240307) 2024-03-19T01:38:50.007 INFO:journalctl@ceph.mds.b.smithi112.stdout:Mar 19 01:38:49 smithi112 podman[97387]: 2024-03-19 01:38:49.981613367 +0000 UTC m=+0.346828197 container init de00165d1e05d16e711ff811983f3ce64e232cb4953d8b1ef7eda6e813b1ae81 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-b, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.build-date=20240307, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=wip-batrick-testing-20240318.181317-b6ae249, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-19T01:38:50.007 INFO:journalctl@ceph.mds.b.smithi112.stdout:Mar 19 01:38:49 smithi112 podman[97387]: 2024-03-19 01:38:49.984243816 +0000 UTC m=+0.349458633 container start de00165d1e05d16e711ff811983f3ce64e232cb4953d8b1ef7eda6e813b1ae81 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-b, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240307, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_BRANCH=HEAD) 2024-03-19T01:38:50.007 INFO:journalctl@ceph.mds.b.smithi112.stdout:Mar 19 01:38:50 smithi112 ceph-mds[97407]: -- 172.21.15.112:0/954418336 <== mon.2 v2:172.21.15.112:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5653a9183a40 con 0x5653a9157000 2024-03-19T01:38:50.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:49 smithi115 ceph-mon[67332]: pgmap v274: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:50.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:49 smithi099 ceph-mon[62951]: pgmap v274: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:50.159 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:50.159+0000 7f13e21e8640 1 -- 172.21.15.99:0/1859135288 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f13dc1a2530 con 0x7f13dc1033f0 2024-03-19T01:38:50.160 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:50.160+0000 7f13d8ff9640 1 -- 172.21.15.99:0/1859135288 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 12 v12) ==== 94+0+9665 (secure 0 0 0) 0x7f13d409bb50 con 0x7f13dc1033f0 2024-03-19T01:38:50.160 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:38:50.160 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":12,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14562,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6835/1631815621","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6834","nonce":1631815621},{"type":"v1","addr":"172.21.15.99:6835","nonce":1631815621}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":4},{"gid":14646,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6839/2236145144","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6838","nonce":2236145144},{"type":"v1","addr":"172.21.15.99:6839","nonce":2236145144}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11},{"gid":14658,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6841/1660990159","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6840","nonce":1660990159},{"type":"v1","addr":"172.21.15.99:6841","nonce":1660990159}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":12},{"gid":24383,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.112:6833/2969810977","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6832","nonce":2969810977},{"type":"v1","addr":"172.21.15.112:6833","nonce":2969810977}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3},{"gid":24404,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6837/2038634879","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6836","nonce":2038634879},{"type":"v1","addr":"172.21.15.99:6837","nonce":2038634879}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":24415,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.115:6833/2219320481","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6832","nonce":2219320481},{"type":"v1","addr":"172.21.15.115:6833","nonce":2219320481}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2},{"gid":24416,"name":"e","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.112:6835/3690587528","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6834","nonce":3690587528},{"type":"v1","addr":"172.21.15.112:6835","nonce":3690587528}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":6},{"gid":24440,"name":"h","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.112:6837/2339823159","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6836","nonce":2339823159},{"type":"v1","addr":"172.21.15.112:6837","nonce":2339823159}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":9},{"gid":24460,"name":"f","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.115:6835/591078413","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6834","nonce":591078413},{"type":"v1","addr":"172.21.15.115:6835","nonce":591078413}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":7},{"gid":24472,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.115:6837/1368988191","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6836","nonce":1368988191},{"type":"v1","addr":"172.21.15.115:6837","nonce":1368988191}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":8},{"gid":24487,"name":"i","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.115:6839/2743761968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6838","nonce":2743761968},{"type":"v1","addr":"172.21.15.115:6839","nonce":2743761968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":10}],"filesystems":[]} 2024-03-19T01:38:50.161 INFO:teuthology.orchestra.run.smithi099.stderr:dumped fsmap epoch 12 2024-03-19T01:38:50.162 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:50.162+0000 7f13e21e8640 1 -- 172.21.15.99:0/1859135288 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f13c40a2800 msgr2=0x7f13c40a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:50.162 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:50.162+0000 7f13e21e8640 1 --2- 172.21.15.99:0/1859135288 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f13c40a2800 0x7f13c40a4cc0 secure :-1 s=READY pgs=153 cs=0 l=1 rev1=1 crypto rx=0x7f13dc19b210 tx=0x7f13cc005f90 comp rx=0 tx=0).stop 2024-03-19T01:38:50.163 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:50.162+0000 7f13e21e8640 1 -- 172.21.15.99:0/1859135288 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f13dc1033f0 msgr2=0x7f13dc1a12b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:50.163 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:50.162+0000 7f13e21e8640 1 --2- 172.21.15.99:0/1859135288 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f13dc1033f0 0x7f13dc1a12b0 secure :-1 s=READY pgs=186 cs=0 l=1 rev1=1 crypto rx=0x7f13d400eb10 tx=0x7f13d400c2d0 comp rx=0 tx=0).stop 2024-03-19T01:38:50.163 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:50.162+0000 7f13e21e8640 1 -- 172.21.15.99:0/1859135288 shutdown_connections 2024-03-19T01:38:50.163 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:50.162+0000 7f13e21e8640 1 --2- 172.21.15.99:0/1859135288 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f13dc1033f0 0x7f13dc1a12b0 unknown :-1 s=CLOSED pgs=186 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:50.163 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:50.162+0000 7f13e21e8640 1 --2- 172.21.15.99:0/1859135288 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f13dc102a30 0x7f13dc19a280 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:50.163 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:50.162+0000 7f13e21e8640 1 --2- 172.21.15.99:0/1859135288 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f13c40a2800 0x7f13c40a4cc0 unknown :-1 s=CLOSED pgs=153 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:50.163 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:50.162+0000 7f13e21e8640 1 --2- 172.21.15.99:0/1859135288 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f13dc101840 0x7f13dc199d40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:50.163 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:50.162+0000 7f13e21e8640 1 -- 172.21.15.99:0/1859135288 >> 172.21.15.99:0/1859135288 conn(0x7f13dc0fcdd0 msgr2=0x7f13dc073a50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:50.163 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:50.162+0000 7f13e21e8640 1 -- 172.21.15.99:0/1859135288 shutdown_connections 2024-03-19T01:38:50.163 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:50.162+0000 7f13e21e8640 1 -- 172.21.15.99:0/1859135288 wait complete. 2024-03-19T01:38:50.361 INFO:journalctl@ceph.mds.b.smithi112.stdout:Mar 19 01:38:50 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-b[97403]: starting mds.b at 2024-03-19T01:38:50.362 INFO:journalctl@ceph.mds.b.smithi112.stdout:Mar 19 01:38:50 smithi112 bash[97387]: de00165d1e05d16e711ff811983f3ce64e232cb4953d8b1ef7eda6e813b1ae81 2024-03-19T01:38:50.362 INFO:journalctl@ceph.mds.b.smithi112.stdout:Mar 19 01:38:50 smithi112 systemd[1]: Started Ceph mds.b for 438fb69c-e590-11ee-95c9-87774f69a715. 2024-03-19T01:38:51.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:51 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:51.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:51 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:51.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:51 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:51.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:51 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:51.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:51 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:51.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:51 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/1859135288' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:38:51.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:51 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:38:51.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:51 smithi099 ceph-mon[62951]: pgmap v275: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:51.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:51 smithi099 ceph-mon[62951]: mds.? [v2:172.21.15.112:6838/986249680,v1:172.21.15.112:6839/986249680] up:boot 2024-03-19T01:38:51.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:51 smithi099 ceph-mon[62951]: fsmap 12 up:standby 2024-03-19T01:38:51.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:51 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-03-19T01:38:51.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:51 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:51.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:51 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:51.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:51 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:51.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:51 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:51.363 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:51 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:51.363 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:51 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/1859135288' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:38:51.363 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:51 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:38:51.363 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:51 smithi112 ceph-mon[66819]: pgmap v275: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:51.363 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:51 smithi112 ceph-mon[66819]: mds.? [v2:172.21.15.112:6838/986249680,v1:172.21.15.112:6839/986249680] up:boot 2024-03-19T01:38:51.363 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:51 smithi112 ceph-mon[66819]: fsmap 12 up:standby 2024-03-19T01:38:51.363 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:51 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-03-19T01:38:51.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:51 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:51.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:51 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:51.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:51 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:51.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:51 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:51.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:51 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:51.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:51 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/1859135288' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:38:51.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:51 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:38:51.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:51 smithi115 ceph-mon[67332]: pgmap v275: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:51.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:51 smithi115 ceph-mon[67332]: mds.? [v2:172.21.15.112:6838/986249680,v1:172.21.15.112:6839/986249680] up:boot 2024-03-19T01:38:51.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:51 smithi115 ceph-mon[67332]: fsmap 12 up:standby 2024-03-19T01:38:51.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:51 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-03-19T01:38:52.175 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-19T01:38:52.283 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.281+0000 7fe693e0b640 1 -- 172.21.15.99:0/1749745542 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe68c102b00 msgr2=0x7fe68c102f80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:52.284 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.281+0000 7fe693e0b640 1 --2- 172.21.15.99:0/1749745542 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe68c102b00 0x7fe68c102f80 secure :-1 s=READY pgs=187 cs=0 l=1 rev1=1 crypto rx=0x7fe688009930 tx=0x7fe68802bc30 comp rx=0 tx=0).stop 2024-03-19T01:38:52.284 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.281+0000 7fe693e0b640 1 -- 172.21.15.99:0/1749745542 shutdown_connections 2024-03-19T01:38:52.284 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.281+0000 7fe693e0b640 1 --2- 172.21.15.99:0/1749745542 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fe68c1034c0 0x7fe68c109d50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:52.284 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.281+0000 7fe693e0b640 1 --2- 172.21.15.99:0/1749745542 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe68c102b00 0x7fe68c102f80 unknown :-1 s=CLOSED pgs=187 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:52.284 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.281+0000 7fe693e0b640 1 --2- 172.21.15.99:0/1749745542 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fe68c0ffb30 0x7fe68c0fff50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:52.284 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.281+0000 7fe693e0b640 1 -- 172.21.15.99:0/1749745542 >> 172.21.15.99:0/1749745542 conn(0x7fe68c0fb460 msgr2=0x7fe68c0fd8a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:52.284 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.281+0000 7fe693e0b640 1 -- 172.21.15.99:0/1749745542 shutdown_connections 2024-03-19T01:38:52.284 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.281+0000 7fe693e0b640 1 -- 172.21.15.99:0/1749745542 wait complete. 2024-03-19T01:38:52.284 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.281+0000 7fe693e0b640 1 Processor -- start 2024-03-19T01:38:52.284 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.281+0000 7fe693e0b640 1 -- start start 2024-03-19T01:38:52.284 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.281+0000 7fe693e0b640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fe68c0ffb30 0x7fe68c197b50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:52.284 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.281+0000 7fe693e0b640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fe68c102b00 0x7fe68c198090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:52.284 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.282+0000 7fe693e0b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe68c1034c0 0x7fe68c19f100 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:52.284 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.282+0000 7fe693e0b640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fe68c10d510 con 0x7fe68c1034c0 2024-03-19T01:38:52.285 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.282+0000 7fe693e0b640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fe68c10d390 con 0x7fe68c0ffb30 2024-03-19T01:38:52.285 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.282+0000 7fe693e0b640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fe68c10d690 con 0x7fe68c102b00 2024-03-19T01:38:52.285 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.282+0000 7fe6923a8640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe68c1034c0 0x7fe68c19f100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:52.285 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.282+0000 7fe6913a6640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fe68c102b00 0x7fe68c198090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:52.285 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.282+0000 7fe6923a8640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe68c1034c0 0x7fe68c19f100 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:57526/0 (socket says 172.21.15.99:57526) 2024-03-19T01:38:52.285 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.282+0000 7fe6923a8640 1 -- 172.21.15.99:0/2419907817 learned_addr learned my addr 172.21.15.99:0/2419907817 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:38:52.285 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.282+0000 7fe691ba7640 1 --2- 172.21.15.99:0/2419907817 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fe68c0ffb30 0x7fe68c197b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:52.285 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.282+0000 7fe6923a8640 1 -- 172.21.15.99:0/2419907817 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fe68c102b00 msgr2=0x7fe68c198090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:52.285 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.282+0000 7fe6923a8640 1 --2- 172.21.15.99:0/2419907817 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fe68c102b00 0x7fe68c198090 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:52.285 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.282+0000 7fe6923a8640 1 -- 172.21.15.99:0/2419907817 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fe68c0ffb30 msgr2=0x7fe68c197b50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:52.285 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.282+0000 7fe6923a8640 1 --2- 172.21.15.99:0/2419907817 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fe68c0ffb30 0x7fe68c197b50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:52.285 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.282+0000 7fe6923a8640 1 -- 172.21.15.99:0/2419907817 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fe68c19f770 con 0x7fe68c1034c0 2024-03-19T01:38:52.285 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.282+0000 7fe6923a8640 1 --2- 172.21.15.99:0/2419907817 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe68c1034c0 0x7fe68c19f100 secure :-1 s=READY pgs=188 cs=0 l=1 rev1=1 crypto rx=0x7fe68400cce0 tx=0x7fe68400a620 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:52.285 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.282+0000 7fe682ffd640 1 -- 172.21.15.99:0/2419907817 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe68400f450 con 0x7fe68c1034c0 2024-03-19T01:38:52.286 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.282+0000 7fe693e0b640 1 -- 172.21.15.99:0/2419907817 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fe68c19fa60 con 0x7fe68c1034c0 2024-03-19T01:38:52.286 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.282+0000 7fe682ffd640 1 -- 172.21.15.99:0/2419907817 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe68400fad0 con 0x7fe68c1034c0 2024-03-19T01:38:52.286 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.282+0000 7fe682ffd640 1 -- 172.21.15.99:0/2419907817 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe684018c10 con 0x7fe68c1034c0 2024-03-19T01:38:52.286 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.282+0000 7fe693e0b640 1 -- 172.21.15.99:0/2419907817 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fe68c19ff30 con 0x7fe68c1034c0 2024-03-19T01:38:52.286 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.283+0000 7fe693e0b640 1 -- 172.21.15.99:0/2419907817 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fe68c108c30 con 0x7fe68c1034c0 2024-03-19T01:38:52.287 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.284+0000 7fe682ffd640 1 -- 172.21.15.99:0/2419907817 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 16) ==== 137290+0+0 (secure 0 0 0) 0x7fe68400f5f0 con 0x7fe68c1034c0 2024-03-19T01:38:52.287 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.285+0000 7fe682ffd640 1 --2- 172.21.15.99:0/2419907817 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fe6780a2800 0x7fe6780a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:52.287 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.285+0000 7fe691ba7640 1 --2- 172.21.15.99:0/2419907817 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fe6780a2800 0x7fe6780a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:52.287 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.285+0000 7fe682ffd640 1 -- 172.21.15.99:0/2419907817 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(83..83 src has 1..83) ==== 7127+0+0 (secure 0 0 0) 0x7fe6840cec00 con 0x7fe68c1034c0 2024-03-19T01:38:52.287 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.285+0000 7fe691ba7640 1 --2- 172.21.15.99:0/2419907817 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fe6780a2800 0x7fe6780a4cc0 secure :-1 s=READY pgs=155 cs=0 l=1 rev1=1 crypto rx=0x7fe68c069c70 tx=0x7fe67c005f90 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:52.289 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.289+0000 7fe682ffd640 1 -- 172.21.15.99:0/2419907817 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7fe6840d5080 con 0x7fe68c1034c0 2024-03-19T01:38:52.507 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.506+0000 7fe693e0b640 1 -- 172.21.15.99:0/2419907817 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7fe68c02d1f0 con 0x7fe68c1034c0 2024-03-19T01:38:52.507 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.507+0000 7fe682ffd640 1 -- 172.21.15.99:0/2419907817 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 13 v13) ==== 94+0+10491 (secure 0 0 0) 0x7fe6840afe20 con 0x7fe68c1034c0 2024-03-19T01:38:52.507 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:38:52.507 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":13,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14562,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6835/1631815621","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6834","nonce":1631815621},{"type":"v1","addr":"172.21.15.99:6835","nonce":1631815621}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":4},{"gid":14646,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6839/2236145144","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6838","nonce":2236145144},{"type":"v1","addr":"172.21.15.99:6839","nonce":2236145144}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11},{"gid":14658,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6841/1660990159","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6840","nonce":1660990159},{"type":"v1","addr":"172.21.15.99:6841","nonce":1660990159}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":12},{"gid":24383,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.112:6833/2969810977","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6832","nonce":2969810977},{"type":"v1","addr":"172.21.15.112:6833","nonce":2969810977}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3},{"gid":24404,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6837/2038634879","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6836","nonce":2038634879},{"type":"v1","addr":"172.21.15.99:6837","nonce":2038634879}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":24415,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.115:6833/2219320481","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6832","nonce":2219320481},{"type":"v1","addr":"172.21.15.115:6833","nonce":2219320481}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2},{"gid":24416,"name":"e","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.112:6835/3690587528","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6834","nonce":3690587528},{"type":"v1","addr":"172.21.15.112:6835","nonce":3690587528}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":6},{"gid":24440,"name":"h","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.112:6837/2339823159","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6836","nonce":2339823159},{"type":"v1","addr":"172.21.15.112:6837","nonce":2339823159}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":9},{"gid":24460,"name":"f","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.115:6835/591078413","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6834","nonce":591078413},{"type":"v1","addr":"172.21.15.115:6835","nonce":591078413}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":7},{"gid":24472,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.115:6837/1368988191","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6836","nonce":1368988191},{"type":"v1","addr":"172.21.15.115:6837","nonce":1368988191}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":8},{"gid":24479,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.112:6839/986249680","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6838","nonce":986249680},{"type":"v1","addr":"172.21.15.112:6839","nonce":986249680}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":13},{"gid":24487,"name":"i","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.115:6839/2743761968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6838","nonce":2743761968},{"type":"v1","addr":"172.21.15.115:6839","nonce":2743761968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":10}],"filesystems":[]} 2024-03-19T01:38:52.508 INFO:teuthology.orchestra.run.smithi099.stderr:dumped fsmap epoch 13 2024-03-19T01:38:52.510 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.509+0000 7fe693e0b640 1 -- 172.21.15.99:0/2419907817 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fe6780a2800 msgr2=0x7fe6780a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:52.510 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.509+0000 7fe693e0b640 1 --2- 172.21.15.99:0/2419907817 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fe6780a2800 0x7fe6780a4cc0 secure :-1 s=READY pgs=155 cs=0 l=1 rev1=1 crypto rx=0x7fe68c069c70 tx=0x7fe67c005f90 comp rx=0 tx=0).stop 2024-03-19T01:38:52.510 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.509+0000 7fe693e0b640 1 -- 172.21.15.99:0/2419907817 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe68c1034c0 msgr2=0x7fe68c19f100 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:52.510 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.509+0000 7fe693e0b640 1 --2- 172.21.15.99:0/2419907817 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe68c1034c0 0x7fe68c19f100 secure :-1 s=READY pgs=188 cs=0 l=1 rev1=1 crypto rx=0x7fe68400cce0 tx=0x7fe68400a620 comp rx=0 tx=0).stop 2024-03-19T01:38:52.510 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.509+0000 7fe693e0b640 1 -- 172.21.15.99:0/2419907817 shutdown_connections 2024-03-19T01:38:52.510 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.509+0000 7fe693e0b640 1 --2- 172.21.15.99:0/2419907817 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe68c1034c0 0x7fe68c19f100 unknown :-1 s=CLOSED pgs=188 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:52.510 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.509+0000 7fe693e0b640 1 --2- 172.21.15.99:0/2419907817 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fe68c102b00 0x7fe68c198090 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:52.510 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.509+0000 7fe693e0b640 1 --2- 172.21.15.99:0/2419907817 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fe6780a2800 0x7fe6780a4cc0 unknown :-1 s=CLOSED pgs=155 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:52.510 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.509+0000 7fe693e0b640 1 --2- 172.21.15.99:0/2419907817 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fe68c0ffb30 0x7fe68c197b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:52.511 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.509+0000 7fe693e0b640 1 -- 172.21.15.99:0/2419907817 >> 172.21.15.99:0/2419907817 conn(0x7fe68c0fb460 msgr2=0x7fe68c105e00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:52.511 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.509+0000 7fe693e0b640 1 -- 172.21.15.99:0/2419907817 shutdown_connections 2024-03-19T01:38:52.511 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.509+0000 7fe693e0b640 1 -- 172.21.15.99:0/2419907817 wait complete. 2024-03-19T01:38:52.521 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-19T01:38:52.630 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.627+0000 7f8711405640 1 -- 172.21.15.99:0/3389239815 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f870c1000d0 msgr2=0x7f870c102500 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:52.630 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.627+0000 7f8711405640 1 --2- 172.21.15.99:0/3389239815 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f870c1000d0 0x7f870c102500 secure :-1 s=READY pgs=189 cs=0 l=1 rev1=1 crypto rx=0x7f86fc009900 tx=0x7f86fc02bc30 comp rx=0 tx=0).stop 2024-03-19T01:38:52.630 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.627+0000 7f8711405640 1 -- 172.21.15.99:0/3389239815 shutdown_connections 2024-03-19T01:38:52.630 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.627+0000 7f8711405640 1 --2- 172.21.15.99:0/3389239815 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f870c1053f0 0x7f870c105870 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:52.631 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.627+0000 7f8711405640 1 --2- 172.21.15.99:0/3389239815 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f870c102a40 0x7f870c104eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:52.631 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.627+0000 7f8711405640 1 --2- 172.21.15.99:0/3389239815 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f870c1000d0 0x7f870c102500 unknown :-1 s=CLOSED pgs=189 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:52.631 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.627+0000 7f8711405640 1 -- 172.21.15.99:0/3389239815 >> 172.21.15.99:0/3389239815 conn(0x7f870c0fbcd0 msgr2=0x7f870c0fe110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:52.631 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.627+0000 7f8711405640 1 -- 172.21.15.99:0/3389239815 shutdown_connections 2024-03-19T01:38:52.631 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.627+0000 7f8711405640 1 -- 172.21.15.99:0/3389239815 wait complete. 2024-03-19T01:38:52.631 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.628+0000 7f8711405640 1 Processor -- start 2024-03-19T01:38:52.631 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.628+0000 7f8711405640 1 -- start start 2024-03-19T01:38:52.631 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.628+0000 7f8711405640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f870c1000d0 0x7f870c19e210 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:52.631 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.628+0000 7f8711405640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f870c102a40 0x7f870c19e750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:52.631 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.628+0000 7f8711405640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f870c1053f0 0x7f870c1a5770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:52.631 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.628+0000 7f8711405640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f870c113ad0 con 0x7f870c102a40 2024-03-19T01:38:52.631 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.628+0000 7f8711405640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f870c113950 con 0x7f870c1000d0 2024-03-19T01:38:52.631 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.628+0000 7f8711405640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f870c113c50 con 0x7f870c1053f0 2024-03-19T01:38:52.632 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.628+0000 7f8703fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f870c102a40 0x7f870c19e750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:52.632 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.628+0000 7f8703fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f870c102a40 0x7f870c19e750 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58306/0 (socket says 172.21.15.99:58306) 2024-03-19T01:38:52.632 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.628+0000 7f8703fff640 1 -- 172.21.15.99:0/2357188144 learned_addr learned my addr 172.21.15.99:0/2357188144 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:38:52.632 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.628+0000 7f870affd640 1 --2- 172.21.15.99:0/2357188144 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f870c1000d0 0x7f870c19e210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:52.632 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.628+0000 7f870b7fe640 1 --2- 172.21.15.99:0/2357188144 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f870c1053f0 0x7f870c1a5770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:52.632 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.628+0000 7f8703fff640 1 -- 172.21.15.99:0/2357188144 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f870c1053f0 msgr2=0x7f870c1a5770 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:52.632 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.628+0000 7f8703fff640 1 --2- 172.21.15.99:0/2357188144 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f870c1053f0 0x7f870c1a5770 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:52.632 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.628+0000 7f8703fff640 1 -- 172.21.15.99:0/2357188144 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f870c1000d0 msgr2=0x7f870c19e210 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:52.632 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.628+0000 7f8703fff640 1 --2- 172.21.15.99:0/2357188144 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f870c1000d0 0x7f870c19e210 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:52.632 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.628+0000 7f8703fff640 1 -- 172.21.15.99:0/2357188144 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f870c1a5de0 con 0x7f870c102a40 2024-03-19T01:38:52.632 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.628+0000 7f8703fff640 1 --2- 172.21.15.99:0/2357188144 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f870c102a40 0x7f870c19e750 secure :-1 s=READY pgs=190 cs=0 l=1 rev1=1 crypto rx=0x7f86f400e9f0 tx=0x7f86f400edf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:52.632 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.628+0000 7f8708ff9640 1 -- 172.21.15.99:0/2357188144 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f86f400cd80 con 0x7f870c102a40 2024-03-19T01:38:52.632 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.629+0000 7f8708ff9640 1 -- 172.21.15.99:0/2357188144 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f86f4005500 con 0x7f870c102a40 2024-03-19T01:38:52.633 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.629+0000 7f8711405640 1 -- 172.21.15.99:0/2357188144 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f870c1a60d0 con 0x7f870c102a40 2024-03-19T01:38:52.633 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.629+0000 7f8708ff9640 1 -- 172.21.15.99:0/2357188144 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f86f4010690 con 0x7f870c102a40 2024-03-19T01:38:52.633 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.629+0000 7f8711405640 1 -- 172.21.15.99:0/2357188144 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f870c1a65d0 con 0x7f870c102a40 2024-03-19T01:38:52.633 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.629+0000 7f8711405640 1 -- 172.21.15.99:0/2357188144 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f870c0fd7c0 con 0x7f870c102a40 2024-03-19T01:38:52.634 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.630+0000 7f8708ff9640 1 -- 172.21.15.99:0/2357188144 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 16) ==== 137290+0+0 (secure 0 0 0) 0x7f86f40046c0 con 0x7f870c102a40 2024-03-19T01:38:52.634 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.631+0000 7f8708ff9640 1 --2- 172.21.15.99:0/2357188144 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f86f80a2800 0x7f86f80a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:52.634 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.631+0000 7f870affd640 1 --2- 172.21.15.99:0/2357188144 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f86f80a2800 0x7f86f80a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:52.634 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.631+0000 7f8708ff9640 1 -- 172.21.15.99:0/2357188144 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(83..83 src has 1..83) ==== 7127+0+0 (secure 0 0 0) 0x7f86f40cea60 con 0x7f870c102a40 2024-03-19T01:38:52.634 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.631+0000 7f870affd640 1 --2- 172.21.15.99:0/2357188144 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f86f80a2800 0x7f86f80a4cc0 secure :-1 s=READY pgs=156 cs=0 l=1 rev1=1 crypto rx=0x7f86fc009230 tx=0x7f86fc005fd0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:52.635 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.635+0000 7f8708ff9640 1 -- 172.21.15.99:0/2357188144 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f86f409bfa0 con 0x7f870c102a40 2024-03-19T01:38:52.826 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.825+0000 7f8711405640 1 -- 172.21.15.99:0/2357188144 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f870c02d410 con 0x7f870c102a40 2024-03-19T01:38:52.827 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.826+0000 7f8708ff9640 1 -- 172.21.15.99:0/2357188144 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v83) ==== 74+0+18470 (secure 0 0 0) 0x7f86f409b8a0 con 0x7f870c102a40 2024-03-19T01:38:52.827 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:38:52.827 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":83,"fsid":"438fb69c-e590-11ee-95c9-87774f69a715","created":"2024-03-19T01:31:10.165073+0000","modified":"2024-03-19T01:38:10.088747+0000","last_up_change":"2024-03-19T01:38:09.082688+0000","last_in_change":"2024-03-19T01:37:49.115793+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-19T01:34:31.001309+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"0c407a0e-a40e-43d7-ad9a-bb8f3cd4a9ba","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6803","nonce":2935260112}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6805","nonce":2935260112}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6808","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6809","nonce":2935260112}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6807","nonce":2935260112}]},"public_addr":"172.21.15.99:6803/2935260112","cluster_addr":"172.21.15.99:6805/2935260112","heartbeat_back_addr":"172.21.15.99:6809/2935260112","heartbeat_front_addr":"172.21.15.99:6807/2935260112","state":["exists","up"]},{"osd":1,"uuid":"a48f9173-4a06-49f5-bca2-2c9047b8f666","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6800","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6801","nonce":1468854289}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6802","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6803","nonce":1468854289}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6806","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6807","nonce":1468854289}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6804","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6805","nonce":1468854289}]},"public_addr":"172.21.15.112:6801/1468854289","cluster_addr":"172.21.15.112:6803/1468854289","heartbeat_back_addr":"172.21.15.112:6807/1468854289","heartbeat_front_addr":"172.21.15.112:6805/1468854289","state":["exists","up"]},{"osd":2,"uuid":"b7715fdd-b6c9-4348-bd16-5eb61b427eef","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6800","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6801","nonce":3766690472}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6802","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6803","nonce":3766690472}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6806","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6807","nonce":3766690472}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6804","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6805","nonce":3766690472}]},"public_addr":"172.21.15.115:6801/3766690472","cluster_addr":"172.21.15.115:6803/3766690472","heartbeat_back_addr":"172.21.15.115:6807/3766690472","heartbeat_front_addr":"172.21.15.115:6805/3766690472","state":["exists","up"]},{"osd":3,"uuid":"6697ed1b-96bf-42de-8440-d17e9785b359","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6810","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6811","nonce":3756732895}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6812","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6813","nonce":3756732895}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6816","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6817","nonce":3756732895}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6814","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6815","nonce":3756732895}]},"public_addr":"172.21.15.99:6811/3756732895","cluster_addr":"172.21.15.99:6813/3756732895","heartbeat_back_addr":"172.21.15.99:6817/3756732895","heartbeat_front_addr":"172.21.15.99:6815/3756732895","state":["exists","up"]},{"osd":4,"uuid":"d958d6b9-67f6-43d0-b704-aa55dc9df83a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":34,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6808","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6809","nonce":3717986391}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6810","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6811","nonce":3717986391}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6814","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6815","nonce":3717986391}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6812","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6813","nonce":3717986391}]},"public_addr":"172.21.15.112:6809/3717986391","cluster_addr":"172.21.15.112:6811/3717986391","heartbeat_back_addr":"172.21.15.112:6815/3717986391","heartbeat_front_addr":"172.21.15.112:6813/3717986391","state":["exists","up"]},{"osd":5,"uuid":"a73eb49a-7925-49ba-bc4f-515e17f8b8ae","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6808","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6809","nonce":1971806408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6810","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6811","nonce":1971806408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6814","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6815","nonce":1971806408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6812","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6813","nonce":1971806408}]},"public_addr":"172.21.15.115:6809/1971806408","cluster_addr":"172.21.15.115:6811/1971806408","heartbeat_back_addr":"172.21.15.115:6815/1971806408","heartbeat_front_addr":"172.21.15.115:6813/1971806408","state":["exists","up"]},{"osd":6,"uuid":"caa718cd-e08b-48de-832a-29546d71e508","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6818","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6819","nonce":1130832772}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6820","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6821","nonce":1130832772}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6824","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6825","nonce":1130832772}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6822","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6823","nonce":1130832772}]},"public_addr":"172.21.15.99:6819/1130832772","cluster_addr":"172.21.15.99:6821/1130832772","heartbeat_back_addr":"172.21.15.99:6825/1130832772","heartbeat_front_addr":"172.21.15.99:6823/1130832772","state":["exists","up"]},{"osd":7,"uuid":"dbee19a1-2315-40d3-88e7-876fcde28903","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":53,"up_thru":77,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6816","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6817","nonce":3610356576}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6818","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6819","nonce":3610356576}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6822","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6823","nonce":3610356576}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6820","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6821","nonce":3610356576}]},"public_addr":"172.21.15.112:6817/3610356576","cluster_addr":"172.21.15.112:6819/3610356576","heartbeat_back_addr":"172.21.15.112:6823/3610356576","heartbeat_front_addr":"172.21.15.112:6821/3610356576","state":["exists","up"]},{"osd":8,"uuid":"14a866dc-5ce3-42fc-8bcf-9088b68726cb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":60,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6816","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6817","nonce":4255803111}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6818","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6819","nonce":4255803111}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6822","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6823","nonce":4255803111}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6820","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6821","nonce":4255803111}]},"public_addr":"172.21.15.115:6817/4255803111","cluster_addr":"172.21.15.115:6819/4255803111","heartbeat_back_addr":"172.21.15.115:6823/4255803111","heartbeat_front_addr":"172.21.15.115:6821/4255803111","state":["exists","up"]},{"osd":9,"uuid":"1db93652-6035-4906-91ae-d64d5a909502","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":68,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6826","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6827","nonce":1410921523}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6828","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6829","nonce":1410921523}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6832","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6833","nonce":1410921523}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6830","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6831","nonce":1410921523}]},"public_addr":"172.21.15.99:6827/1410921523","cluster_addr":"172.21.15.99:6829/1410921523","heartbeat_back_addr":"172.21.15.99:6833/1410921523","heartbeat_front_addr":"172.21.15.99:6831/1410921523","state":["exists","up"]},{"osd":10,"uuid":"6c50a8a7-3345-4f4a-af0e-5606c7aca453","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":74,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6824","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6825","nonce":891805922}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6826","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6827","nonce":891805922}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6830","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6831","nonce":891805922}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6828","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6829","nonce":891805922}]},"public_addr":"172.21.15.112:6825/891805922","cluster_addr":"172.21.15.112:6827/891805922","heartbeat_back_addr":"172.21.15.112:6831/891805922","heartbeat_front_addr":"172.21.15.112:6829/891805922","state":["exists","up"]},{"osd":11,"uuid":"fd63b7fd-0db5-46d5-82fa-9707bbd0b495","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":82,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6824","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6825","nonce":938436244}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6826","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6827","nonce":938436244}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6830","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6831","nonce":938436244}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6828","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6829","nonce":938436244}]},"public_addr":"172.21.15.115:6825/938436244","cluster_addr":"172.21.15.115:6827/938436244","heartbeat_back_addr":"172.21.15.115:6831/938436244","heartbeat_front_addr":"172.21.15.115:6829/938436244","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:33:39.036822+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:34:03.339566+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:34:27.149273+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:34:51.315561+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:35:15.396031+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:35:38.957168+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:36:03.213914+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:36:27.763699+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:36:51.688670+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:37:17.280165+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:37:42.554633+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:38:07.438039+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.99:6801/1328245934":"2024-03-20T01:32:06.661002+0000","172.21.15.99:0/1618965837":"2024-03-20T01:32:06.661002+0000","172.21.15.99:0/2491429851":"2024-03-20T01:32:06.661002+0000","172.21.15.99:6800/1328245934":"2024-03-20T01:32:06.661002+0000","172.21.15.99:0/2112231766":"2024-03-20T01:31:40.161751+0000","172.21.15.99:0/3341836545":"2024-03-20T01:31:40.161751+0000","172.21.15.99:6800/4223048293":"2024-03-20T01:31:40.161751+0000","172.21.15.99:6801/4223048293":"2024-03-20T01:31:40.161751+0000","172.21.15.99:0/2377621863":"2024-03-20T01:31:40.161751+0000","172.21.15.99:0/1904400935":"2024-03-20T01:32:06.661002+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-19T01:38:52.829 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.828+0000 7f8711405640 1 -- 172.21.15.99:0/2357188144 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f86f80a2800 msgr2=0x7f86f80a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:52.829 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.828+0000 7f8711405640 1 --2- 172.21.15.99:0/2357188144 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f86f80a2800 0x7f86f80a4cc0 secure :-1 s=READY pgs=156 cs=0 l=1 rev1=1 crypto rx=0x7f86fc009230 tx=0x7f86fc005fd0 comp rx=0 tx=0).stop 2024-03-19T01:38:52.829 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.828+0000 7f8711405640 1 -- 172.21.15.99:0/2357188144 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f870c102a40 msgr2=0x7f870c19e750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:52.829 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.828+0000 7f8711405640 1 --2- 172.21.15.99:0/2357188144 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f870c102a40 0x7f870c19e750 secure :-1 s=READY pgs=190 cs=0 l=1 rev1=1 crypto rx=0x7f86f400e9f0 tx=0x7f86f400edf0 comp rx=0 tx=0).stop 2024-03-19T01:38:52.829 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.828+0000 7f8711405640 1 -- 172.21.15.99:0/2357188144 shutdown_connections 2024-03-19T01:38:52.829 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.828+0000 7f8711405640 1 --2- 172.21.15.99:0/2357188144 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f870c1053f0 0x7f870c1a5770 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:52.829 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.829+0000 7f8711405640 1 --2- 172.21.15.99:0/2357188144 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f870c102a40 0x7f870c19e750 unknown :-1 s=CLOSED pgs=190 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:52.829 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.829+0000 7f8711405640 1 --2- 172.21.15.99:0/2357188144 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f86f80a2800 0x7f86f80a4cc0 unknown :-1 s=CLOSED pgs=156 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:52.829 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.829+0000 7f8711405640 1 --2- 172.21.15.99:0/2357188144 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f870c1000d0 0x7f870c19e210 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:52.830 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.829+0000 7f8711405640 1 -- 172.21.15.99:0/2357188144 >> 172.21.15.99:0/2357188144 conn(0x7f870c0fbcd0 msgr2=0x7f870c103490 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:52.830 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.829+0000 7f8711405640 1 -- 172.21.15.99:0/2357188144 shutdown_connections 2024-03-19T01:38:52.830 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.829+0000 7f8711405640 1 -- 172.21.15.99:0/2357188144 wait complete. 2024-03-19T01:38:52.839 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:52 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:52.839 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:52 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:52.839 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:52 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:52.839 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:52 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:52.839 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:52 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:52.839 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:52 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:52.839 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:52 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:52.839 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:52 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:38:52.839 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:52 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:38:52.840 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:52 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:52.840 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:52 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/2419907817' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:38:52.840 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:52 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:38:52.841 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-19T01:34:31.001309+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 12, 'score_stable': 12, 'optimal_score': 0.25, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-19T01:38:52.841 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-19T01:38:52.950 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.947+0000 7f9d9b8ea640 1 -- 172.21.15.99:0/580913364 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9d94100760 msgr2=0x7f9d94100b80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:52.950 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.947+0000 7f9d9b8ea640 1 --2- 172.21.15.99:0/580913364 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9d94100760 0x7f9d94100b80 secure :-1 s=READY pgs=191 cs=0 l=1 rev1=1 crypto rx=0x7f9d90009900 tx=0x7f9d9002bc30 comp rx=0 tx=0).stop 2024-03-19T01:38:52.950 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.947+0000 7f9d9b8ea640 1 -- 172.21.15.99:0/580913364 shutdown_connections 2024-03-19T01:38:52.950 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.947+0000 7f9d9b8ea640 1 --2- 172.21.15.99:0/580913364 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9d94101950 0x7f9d94101dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:52.950 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.947+0000 7f9d9b8ea640 1 --2- 172.21.15.99:0/580913364 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9d94102310 0x7f9d94108ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:52.950 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.947+0000 7f9d9b8ea640 1 --2- 172.21.15.99:0/580913364 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9d94100760 0x7f9d94100b80 unknown :-1 s=CLOSED pgs=191 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:52.950 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.947+0000 7f9d9b8ea640 1 -- 172.21.15.99:0/580913364 >> 172.21.15.99:0/580913364 conn(0x7f9d940fbcd0 msgr2=0x7f9d940fe110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:52.950 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.947+0000 7f9d9b8ea640 1 -- 172.21.15.99:0/580913364 shutdown_connections 2024-03-19T01:38:52.950 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.947+0000 7f9d9b8ea640 1 -- 172.21.15.99:0/580913364 wait complete. 2024-03-19T01:38:52.950 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.948+0000 7f9d9b8ea640 1 Processor -- start 2024-03-19T01:38:52.951 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.948+0000 7f9d9b8ea640 1 -- start start 2024-03-19T01:38:52.951 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.948+0000 7f9d9b8ea640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9d94100760 0x7f9d9406a6d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:52.951 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.948+0000 7f9d9b8ea640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9d94101950 0x7f9d9419c910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:52.951 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.948+0000 7f9d9b8ea640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9d94102310 0x7f9d9419ef50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:52.951 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.948+0000 7f9d9b8ea640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f9d9410c3b0 con 0x7f9d94102310 2024-03-19T01:38:52.951 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.948+0000 7f9d9b8ea640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f9d9410c230 con 0x7f9d94100760 2024-03-19T01:38:52.951 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.948+0000 7f9d9b8ea640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f9d9410c530 con 0x7f9d94101950 2024-03-19T01:38:52.951 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.948+0000 7f9d99e87640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9d94102310 0x7f9d9419ef50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:52.951 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.948+0000 7f9d99e87640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9d94102310 0x7f9d9419ef50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58318/0 (socket says 172.21.15.99:58318) 2024-03-19T01:38:52.951 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.948+0000 7f9d99e87640 1 -- 172.21.15.99:0/131876912 learned_addr learned my addr 172.21.15.99:0/131876912 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:38:52.951 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.948+0000 7f9d98e85640 1 --2- 172.21.15.99:0/131876912 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9d94101950 0x7f9d9419c910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:52.951 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.948+0000 7f9d99686640 1 --2- 172.21.15.99:0/131876912 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9d94100760 0x7f9d9406a6d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:52.951 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.948+0000 7f9d99e87640 1 -- 172.21.15.99:0/131876912 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9d94101950 msgr2=0x7f9d9419c910 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:52.952 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.948+0000 7f9d99e87640 1 --2- 172.21.15.99:0/131876912 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9d94101950 0x7f9d9419c910 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:52.952 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.948+0000 7f9d99e87640 1 -- 172.21.15.99:0/131876912 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9d94100760 msgr2=0x7f9d9406a6d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:52.952 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.948+0000 7f9d99e87640 1 --2- 172.21.15.99:0/131876912 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9d94100760 0x7f9d9406a6d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:52.952 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.948+0000 7f9d99e87640 1 -- 172.21.15.99:0/131876912 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f9d9419f810 con 0x7f9d94102310 2024-03-19T01:38:52.952 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.949+0000 7f9d99e87640 1 --2- 172.21.15.99:0/131876912 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9d94102310 0x7f9d9419ef50 secure :-1 s=READY pgs=192 cs=0 l=1 rev1=1 crypto rx=0x7f9d8c009fd0 tx=0x7f9d8c00b260 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:52.952 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.949+0000 7f9d8a7fc640 1 -- 172.21.15.99:0/131876912 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9d8c017070 con 0x7f9d94102310 2024-03-19T01:38:52.952 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.949+0000 7f9d8a7fc640 1 -- 172.21.15.99:0/131876912 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f9d8c003e50 con 0x7f9d94102310 2024-03-19T01:38:52.952 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.949+0000 7f9d9b8ea640 1 -- 172.21.15.99:0/131876912 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f9d9419fb00 con 0x7f9d94102310 2024-03-19T01:38:52.952 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.949+0000 7f9d8a7fc640 1 -- 172.21.15.99:0/131876912 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9d8c0106c0 con 0x7f9d94102310 2024-03-19T01:38:52.952 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.949+0000 7f9d9b8ea640 1 -- 172.21.15.99:0/131876912 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f9d941a0050 con 0x7f9d94102310 2024-03-19T01:38:52.953 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.950+0000 7f9d9b8ea640 1 -- 172.21.15.99:0/131876912 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f9d94107ba0 con 0x7f9d94102310 2024-03-19T01:38:52.954 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.951+0000 7f9d8a7fc640 1 -- 172.21.15.99:0/131876912 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 16) ==== 137290+0+0 (secure 0 0 0) 0x7f9d8c003970 con 0x7f9d94102310 2024-03-19T01:38:52.954 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.952+0000 7f9d8a7fc640 1 --2- 172.21.15.99:0/131876912 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f9d800a2880 0x7f9d800a4d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:52.954 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.952+0000 7f9d99686640 1 --2- 172.21.15.99:0/131876912 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f9d800a2880 0x7f9d800a4d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:52.954 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.952+0000 7f9d8a7fc640 1 -- 172.21.15.99:0/131876912 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(83..83 src has 1..83) ==== 7127+0+0 (secure 0 0 0) 0x7f9d8c0ce180 con 0x7f9d94102310 2024-03-19T01:38:52.954 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.952+0000 7f9d99686640 1 --2- 172.21.15.99:0/131876912 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f9d800a2880 0x7f9d800a4d40 secure :-1 s=READY pgs=157 cs=0 l=1 rev1=1 crypto rx=0x7f9d90005ab0 tx=0x7f9d90005fd0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:52.956 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:52.956+0000 7f9d8a7fc640 1 -- 172.21.15.99:0/131876912 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f9d8c09b0d0 con 0x7f9d94102310 2024-03-19T01:38:53.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:52 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:53.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:52 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:53.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:52 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:53.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:52 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:53.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:52 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:53.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:52 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:53.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:52 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:53.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:52 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:38:53.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:52 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:38:53.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:52 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:53.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:52 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/2419907817' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:38:53.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:52 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:38:53.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:52 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:53.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:52 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:53.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:52 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:53.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:52 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:53.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:52 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:53.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:52 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:53.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:52 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:53.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:52 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:38:53.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:52 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:38:53.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:52 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:53.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:52 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/2419907817' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:38:53.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:52 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:38:53.140 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.140+0000 7f9d9b8ea640 1 -- 172.21.15.99:0/131876912 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) -- 0x7f9d9402d3f0 con 0x7f9d94102310 2024-03-19T01:38:53.141 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.141+0000 7f9d8a7fc640 1 -- 172.21.15.99:0/131876912 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v83) ==== 93+0+10 (secure 0 0 0) 0x7f9d8c09f170 con 0x7f9d94102310 2024-03-19T01:38:53.141 INFO:teuthology.orchestra.run.smithi099.stdout:pg_num: 1 2024-03-19T01:38:53.144 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.143+0000 7f9d9b8ea640 1 -- 172.21.15.99:0/131876912 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f9d800a2880 msgr2=0x7f9d800a4d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:53.144 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.143+0000 7f9d9b8ea640 1 --2- 172.21.15.99:0/131876912 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f9d800a2880 0x7f9d800a4d40 secure :-1 s=READY pgs=157 cs=0 l=1 rev1=1 crypto rx=0x7f9d90005ab0 tx=0x7f9d90005fd0 comp rx=0 tx=0).stop 2024-03-19T01:38:53.144 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.143+0000 7f9d9b8ea640 1 -- 172.21.15.99:0/131876912 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9d94102310 msgr2=0x7f9d9419ef50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:53.144 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.143+0000 7f9d9b8ea640 1 --2- 172.21.15.99:0/131876912 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9d94102310 0x7f9d9419ef50 secure :-1 s=READY pgs=192 cs=0 l=1 rev1=1 crypto rx=0x7f9d8c009fd0 tx=0x7f9d8c00b260 comp rx=0 tx=0).stop 2024-03-19T01:38:53.144 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.143+0000 7f9d9b8ea640 1 -- 172.21.15.99:0/131876912 shutdown_connections 2024-03-19T01:38:53.144 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.143+0000 7f9d9b8ea640 1 --2- 172.21.15.99:0/131876912 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9d94102310 0x7f9d9419ef50 unknown :-1 s=CLOSED pgs=192 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:53.144 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.143+0000 7f9d9b8ea640 1 --2- 172.21.15.99:0/131876912 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9d94101950 0x7f9d9419c910 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:53.145 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.143+0000 7f9d9b8ea640 1 --2- 172.21.15.99:0/131876912 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f9d800a2880 0x7f9d800a4d40 unknown :-1 s=CLOSED pgs=157 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:53.145 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.143+0000 7f9d9b8ea640 1 --2- 172.21.15.99:0/131876912 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9d94100760 0x7f9d9406a6d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:53.145 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.143+0000 7f9d9b8ea640 1 -- 172.21.15.99:0/131876912 >> 172.21.15.99:0/131876912 conn(0x7f9d940fbcd0 msgr2=0x7f9d94104d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:53.145 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.144+0000 7f9d9b8ea640 1 -- 172.21.15.99:0/131876912 shutdown_connections 2024-03-19T01:38:53.145 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.144+0000 7f9d9b8ea640 1 -- 172.21.15.99:0/131876912 wait complete. 2024-03-19T01:38:53.157 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --format=json-pretty osd lspools 2024-03-19T01:38:53.272 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.270+0000 7fd20db90640 1 -- 172.21.15.99:0/2162007534 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd208103480 msgr2=0x7fd208109d10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:53.272 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.270+0000 7fd20db90640 1 --2- 172.21.15.99:0/2162007534 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd208103480 0x7fd208109d10 secure :-1 s=READY pgs=193 cs=0 l=1 rev1=1 crypto rx=0x7fd200009cd0 tx=0x7fd20002c0f0 comp rx=0 tx=0).stop 2024-03-19T01:38:53.272 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.270+0000 7fd20db90640 1 -- 172.21.15.99:0/2162007534 shutdown_connections 2024-03-19T01:38:53.272 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.270+0000 7fd20db90640 1 --2- 172.21.15.99:0/2162007534 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd208103480 0x7fd208109d10 unknown :-1 s=CLOSED pgs=193 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:53.272 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.270+0000 7fd20db90640 1 --2- 172.21.15.99:0/2162007534 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd208102ac0 0x7fd208102f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:53.272 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.270+0000 7fd20db90640 1 --2- 172.21.15.99:0/2162007534 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd2080ffaf0 0x7fd2080fff10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:53.273 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.270+0000 7fd20db90640 1 -- 172.21.15.99:0/2162007534 >> 172.21.15.99:0/2162007534 conn(0x7fd2080fb420 msgr2=0x7fd2080fd860 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:53.273 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.270+0000 7fd20db90640 1 -- 172.21.15.99:0/2162007534 shutdown_connections 2024-03-19T01:38:53.273 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.270+0000 7fd20db90640 1 -- 172.21.15.99:0/2162007534 wait complete. 2024-03-19T01:38:53.273 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.270+0000 7fd20db90640 1 Processor -- start 2024-03-19T01:38:53.273 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.271+0000 7fd20db90640 1 -- start start 2024-03-19T01:38:53.273 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.271+0000 7fd20db90640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd2080ffaf0 0x7fd208102430 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:53.273 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.271+0000 7fd20db90640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd208102ac0 0x7fd208100a70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:53.273 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.271+0000 7fd20db90640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd208103480 0x7fd208100fb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:53.273 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.271+0000 7fd20db90640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fd20810d4d0 con 0x7fd2080ffaf0 2024-03-19T01:38:53.273 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.271+0000 7fd20db90640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fd20810d350 con 0x7fd208103480 2024-03-19T01:38:53.273 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.271+0000 7fd20db90640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fd20810d650 con 0x7fd208102ac0 2024-03-19T01:38:53.273 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.271+0000 7fd2077fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd2080ffaf0 0x7fd208102430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:53.273 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.271+0000 7fd2077fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd2080ffaf0 0x7fd208102430 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58334/0 (socket says 172.21.15.99:58334) 2024-03-19T01:38:53.273 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.271+0000 7fd2077fe640 1 -- 172.21.15.99:0/1680464628 learned_addr learned my addr 172.21.15.99:0/1680464628 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:38:53.274 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.271+0000 7fd206ffd640 1 --2- 172.21.15.99:0/1680464628 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd208102ac0 0x7fd208100a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:53.274 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.271+0000 7fd207fff640 1 --2- 172.21.15.99:0/1680464628 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd208103480 0x7fd208100fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:53.274 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.271+0000 7fd2077fe640 1 -- 172.21.15.99:0/1680464628 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd208102ac0 msgr2=0x7fd208100a70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:53.274 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.271+0000 7fd2077fe640 1 --2- 172.21.15.99:0/1680464628 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd208102ac0 0x7fd208100a70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:53.274 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.271+0000 7fd2077fe640 1 -- 172.21.15.99:0/1680464628 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd208103480 msgr2=0x7fd208100fb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:53.274 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.271+0000 7fd2077fe640 1 --2- 172.21.15.99:0/1680464628 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd208103480 0x7fd208100fb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:53.274 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.271+0000 7fd2077fe640 1 -- 172.21.15.99:0/1680464628 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd208101840 con 0x7fd2080ffaf0 2024-03-19T01:38:53.274 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.271+0000 7fd2077fe640 1 --2- 172.21.15.99:0/1680464628 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd2080ffaf0 0x7fd208102430 secure :-1 s=READY pgs=194 cs=0 l=1 rev1=1 crypto rx=0x7fd1f800c970 tx=0x7fd1f800ce30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:53.274 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.272+0000 7fd204ff9640 1 -- 172.21.15.99:0/1680464628 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd1f8007c10 con 0x7fd2080ffaf0 2024-03-19T01:38:53.275 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.272+0000 7fd20db90640 1 -- 172.21.15.99:0/1680464628 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd2081a1ba0 con 0x7fd2080ffaf0 2024-03-19T01:38:53.275 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.272+0000 7fd204ff9640 1 -- 172.21.15.99:0/1680464628 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd1f8007db0 con 0x7fd2080ffaf0 2024-03-19T01:38:53.275 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.272+0000 7fd204ff9640 1 -- 172.21.15.99:0/1680464628 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd1f800f8a0 con 0x7fd2080ffaf0 2024-03-19T01:38:53.275 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.272+0000 7fd20db90640 1 -- 172.21.15.99:0/1680464628 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fd2081a2140 con 0x7fd2080ffaf0 2024-03-19T01:38:53.275 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.273+0000 7fd20db90640 1 -- 172.21.15.99:0/1680464628 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fd1f40052c0 con 0x7fd2080ffaf0 2024-03-19T01:38:53.276 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.274+0000 7fd204ff9640 1 -- 172.21.15.99:0/1680464628 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 16) ==== 137290+0+0 (secure 0 0 0) 0x7fd1f800fa80 con 0x7fd2080ffaf0 2024-03-19T01:38:53.276 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.274+0000 7fd204ff9640 1 --2- 172.21.15.99:0/1680464628 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd1f00a2880 0x7fd1f00a4d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:53.276 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.275+0000 7fd206ffd640 1 --2- 172.21.15.99:0/1680464628 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd1f00a2880 0x7fd1f00a4d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:53.276 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.275+0000 7fd204ff9640 1 -- 172.21.15.99:0/1680464628 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(83..83 src has 1..83) ==== 7127+0+0 (secure 0 0 0) 0x7fd1f80ce6d0 con 0x7fd2080ffaf0 2024-03-19T01:38:53.276 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.275+0000 7fd206ffd640 1 --2- 172.21.15.99:0/1680464628 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd1f00a2880 0x7fd1f00a4d40 secure :-1 s=READY pgs=158 cs=0 l=1 rev1=1 crypto rx=0x7fd2081021a0 tx=0x7fd1fc00b390 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:53.279 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.279+0000 7fd204ff9640 1 -- 172.21.15.99:0/1680464628 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7fd1f809bca0 con 0x7fd2080ffaf0 2024-03-19T01:38:53.457 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.457+0000 7fd20db90640 1 -- 172.21.15.99:0/1680464628 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "osd lspools", "format": "json-pretty"} v 0) -- 0x7fd1f4005880 con 0x7fd2080ffaf0 2024-03-19T01:38:53.458 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.457+0000 7fd204ff9640 1 -- 172.21.15.99:0/1680464628 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "osd lspools", "format": "json-pretty"}]=0 v83) ==== 84+0+65 (secure 0 0 0) 0x7fd1f809b5a0 con 0x7fd2080ffaf0 2024-03-19T01:38:53.458 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:38:53.458 INFO:teuthology.orchestra.run.smithi099.stdout:[ 2024-03-19T01:38:53.458 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T01:38:53.458 INFO:teuthology.orchestra.run.smithi099.stdout: "poolnum": 1, 2024-03-19T01:38:53.458 INFO:teuthology.orchestra.run.smithi099.stdout: "poolname": ".mgr" 2024-03-19T01:38:53.458 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T01:38:53.458 INFO:teuthology.orchestra.run.smithi099.stdout:] 2024-03-19T01:38:53.460 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.459+0000 7fd20db90640 1 -- 172.21.15.99:0/1680464628 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd1f00a2880 msgr2=0x7fd1f00a4d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:53.460 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.459+0000 7fd20db90640 1 --2- 172.21.15.99:0/1680464628 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd1f00a2880 0x7fd1f00a4d40 secure :-1 s=READY pgs=158 cs=0 l=1 rev1=1 crypto rx=0x7fd2081021a0 tx=0x7fd1fc00b390 comp rx=0 tx=0).stop 2024-03-19T01:38:53.460 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.459+0000 7fd20db90640 1 -- 172.21.15.99:0/1680464628 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd2080ffaf0 msgr2=0x7fd208102430 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:53.460 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.459+0000 7fd20db90640 1 --2- 172.21.15.99:0/1680464628 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd2080ffaf0 0x7fd208102430 secure :-1 s=READY pgs=194 cs=0 l=1 rev1=1 crypto rx=0x7fd1f800c970 tx=0x7fd1f800ce30 comp rx=0 tx=0).stop 2024-03-19T01:38:53.460 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.459+0000 7fd20db90640 1 -- 172.21.15.99:0/1680464628 shutdown_connections 2024-03-19T01:38:53.460 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.459+0000 7fd20db90640 1 --2- 172.21.15.99:0/1680464628 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd1f00a2880 0x7fd1f00a4d40 unknown :-1 s=CLOSED pgs=158 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:53.461 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.459+0000 7fd20db90640 1 --2- 172.21.15.99:0/1680464628 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd208103480 0x7fd208100fb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:53.461 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.459+0000 7fd20db90640 1 --2- 172.21.15.99:0/1680464628 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd208102ac0 0x7fd208100a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:53.461 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.459+0000 7fd20db90640 1 --2- 172.21.15.99:0/1680464628 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd2080ffaf0 0x7fd208102430 unknown :-1 s=CLOSED pgs=194 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:53.461 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.459+0000 7fd20db90640 1 -- 172.21.15.99:0/1680464628 >> 172.21.15.99:0/1680464628 conn(0x7fd2080fb420 msgr2=0x7fd208105d60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:53.461 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.459+0000 7fd20db90640 1 -- 172.21.15.99:0/1680464628 shutdown_connections 2024-03-19T01:38:53.461 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.459+0000 7fd20db90640 1 -- 172.21.15.99:0/1680464628 wait complete. 2024-03-19T01:38:53.471 DEBUG:tasks.cephfs.filesystem:Creating filesystem 'cephfs' 2024-03-19T01:38:53.471 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool create cephfs_metadata --pg_num_min 64 2024-03-19T01:38:53.580 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.579+0000 7f6d4888c640 1 -- 172.21.15.99:0/2804461365 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6d44102340 msgr2=0x7f6d44104730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:53.580 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.579+0000 7f6d4888c640 1 --2- 172.21.15.99:0/2804461365 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6d44102340 0x7f6d44104730 secure :-1 s=READY pgs=195 cs=0 l=1 rev1=1 crypto rx=0x7f6d34009900 tx=0x7f6d3402bc30 comp rx=0 tx=0).stop 2024-03-19T01:38:53.580 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.579+0000 7f6d4888c640 1 -- 172.21.15.99:0/2804461365 shutdown_connections 2024-03-19T01:38:53.580 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.579+0000 7f6d4888c640 1 --2- 172.21.15.99:0/2804461365 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6d441075a0 0x7f6d44109990 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:53.580 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.579+0000 7f6d4888c640 1 --2- 172.21.15.99:0/2804461365 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f6d44104c70 0x7f6d44107060 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:53.580 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.579+0000 7f6d4888c640 1 --2- 172.21.15.99:0/2804461365 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6d44102340 0x7f6d44104730 unknown :-1 s=CLOSED pgs=195 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:53.581 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.579+0000 7f6d4888c640 1 -- 172.21.15.99:0/2804461365 >> 172.21.15.99:0/2804461365 conn(0x7f6d440fbcd0 msgr2=0x7f6d440fe110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:53.581 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.579+0000 7f6d4888c640 1 -- 172.21.15.99:0/2804461365 shutdown_connections 2024-03-19T01:38:53.581 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.579+0000 7f6d4888c640 1 -- 172.21.15.99:0/2804461365 wait complete. 2024-03-19T01:38:53.581 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.580+0000 7f6d4888c640 1 Processor -- start 2024-03-19T01:38:53.583 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.580+0000 7f6d4888c640 1 -- start start 2024-03-19T01:38:53.583 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.580+0000 7f6d4888c640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f6d44102340 0x7f6d44101cb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:53.583 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.580+0000 7f6d4888c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6d44104c70 0x7f6d441002f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:53.583 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.580+0000 7f6d4888c640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6d441075a0 0x7f6d44100830 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:53.583 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.580+0000 7f6d4888c640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f6d4410d4d0 con 0x7f6d44104c70 2024-03-19T01:38:53.583 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.580+0000 7f6d4888c640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f6d4410d350 con 0x7f6d441075a0 2024-03-19T01:38:53.583 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.580+0000 7f6d4888c640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f6d4410d650 con 0x7f6d44102340 2024-03-19T01:38:53.584 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.580+0000 7f6d39d9b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6d44104c70 0x7f6d441002f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:53.584 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.581+0000 7f6d39d9b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6d44104c70 0x7f6d441002f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58358/0 (socket says 172.21.15.99:58358) 2024-03-19T01:38:53.584 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.581+0000 7f6d39d9b640 1 -- 172.21.15.99:0/4294476713 learned_addr learned my addr 172.21.15.99:0/4294476713 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:38:53.584 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.581+0000 7f6d4259c640 1 --2- 172.21.15.99:0/4294476713 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f6d44102340 0x7f6d44101cb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:53.584 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.581+0000 7f6d42d9d640 1 --2- 172.21.15.99:0/4294476713 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6d441075a0 0x7f6d44100830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:53.584 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.581+0000 7f6d39d9b640 1 -- 172.21.15.99:0/4294476713 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f6d44102340 msgr2=0x7f6d44101cb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:53.584 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.581+0000 7f6d39d9b640 1 --2- 172.21.15.99:0/4294476713 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f6d44102340 0x7f6d44101cb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:53.584 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.581+0000 7f6d39d9b640 1 -- 172.21.15.99:0/4294476713 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6d441075a0 msgr2=0x7f6d44100830 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:53.584 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.581+0000 7f6d39d9b640 1 --2- 172.21.15.99:0/4294476713 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6d441075a0 0x7f6d44100830 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:53.584 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.581+0000 7f6d39d9b640 1 -- 172.21.15.99:0/4294476713 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f6d441010c0 con 0x7f6d44104c70 2024-03-19T01:38:53.584 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.581+0000 7f6d39d9b640 1 --2- 172.21.15.99:0/4294476713 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6d44104c70 0x7f6d441002f0 secure :-1 s=READY pgs=196 cs=0 l=1 rev1=1 crypto rx=0x7f6d2c00d770 tx=0x7f6d2c00dc30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:53.584 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.581+0000 7f6d3bfff640 1 -- 172.21.15.99:0/4294476713 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6d2c017070 con 0x7f6d44104c70 2024-03-19T01:38:53.584 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.581+0000 7f6d4888c640 1 -- 172.21.15.99:0/4294476713 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f6d441a1d50 con 0x7f6d44104c70 2024-03-19T01:38:53.584 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.581+0000 7f6d3bfff640 1 -- 172.21.15.99:0/4294476713 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f6d2c009c30 con 0x7f6d44104c70 2024-03-19T01:38:53.585 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.581+0000 7f6d4888c640 1 -- 172.21.15.99:0/4294476713 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f6d441a2260 con 0x7f6d44104c70 2024-03-19T01:38:53.585 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.582+0000 7f6d3bfff640 1 -- 172.21.15.99:0/4294476713 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6d2c00f620 con 0x7f6d44104c70 2024-03-19T01:38:53.585 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.583+0000 7f6d4888c640 1 -- 172.21.15.99:0/4294476713 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f6d440fd7c0 con 0x7f6d44104c70 2024-03-19T01:38:53.586 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.584+0000 7f6d3bfff640 1 -- 172.21.15.99:0/4294476713 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 16) ==== 137290+0+0 (secure 0 0 0) 0x7f6d2c01a070 con 0x7f6d44104c70 2024-03-19T01:38:53.586 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.584+0000 7f6d3bfff640 1 --2- 172.21.15.99:0/4294476713 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f6d300a2800 0x7f6d300a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:53.586 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.584+0000 7f6d3bfff640 1 -- 172.21.15.99:0/4294476713 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(83..83 src has 1..83) ==== 7127+0+0 (secure 0 0 0) 0x7f6d2c0ce440 con 0x7f6d44104c70 2024-03-19T01:38:53.586 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.584+0000 7f6d4259c640 1 --2- 172.21.15.99:0/4294476713 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f6d300a2800 0x7f6d300a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:53.586 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.585+0000 7f6d4259c640 1 --2- 172.21.15.99:0/4294476713 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f6d300a2800 0x7f6d300a4cc0 secure :-1 s=READY pgs=159 cs=0 l=1 rev1=1 crypto rx=0x7f6d34009230 tx=0x7f6d34005fd0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:53.589 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.589+0000 7f6d3bfff640 1 -- 172.21.15.99:0/4294476713 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f6d2c009de0 con 0x7f6d44104c70 2024-03-19T01:38:53.770 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:53.770+0000 7f6d4888c640 1 -- 172.21.15.99:0/4294476713 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64} v 0) -- 0x7f6d4402d440 con 0x7f6d44104c70 2024-03-19T01:38:54.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:53 smithi115 ceph-mon[67332]: pgmap v276: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:54.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:53 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/2357188144' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-19T01:38:54.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:53 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/131876912' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-03-19T01:38:54.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:53 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/1680464628' entity='client.admin' cmd=[{"prefix": "osd lspools", "format": "json-pretty"}]: dispatch 2024-03-19T01:38:54.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:53 smithi099 ceph-mon[62951]: pgmap v276: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:54.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:53 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/2357188144' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-19T01:38:54.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:53 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/131876912' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-03-19T01:38:54.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:53 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/1680464628' entity='client.admin' cmd=[{"prefix": "osd lspools", "format": "json-pretty"}]: dispatch 2024-03-19T01:38:54.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:53 smithi112 ceph-mon[66819]: pgmap v276: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:54.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:53 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/2357188144' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-19T01:38:54.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:53 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/131876912' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-03-19T01:38:54.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:53 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/1680464628' entity='client.admin' cmd=[{"prefix": "osd lspools", "format": "json-pretty"}]: dispatch 2024-03-19T01:38:54.768 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:54.767+0000 7f6d3bfff640 1 -- 172.21.15.99:0/4294476713 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]=0 pool 'cephfs_metadata' created v84) ==== 138+0+0 (secure 0 0 0) 0x7f6d2c09b980 con 0x7f6d44104c70 2024-03-19T01:38:54.768 INFO:teuthology.orchestra.run.smithi099.stderr:pool 'cephfs_metadata' created 2024-03-19T01:38:54.771 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:54.770+0000 7f6d4888c640 1 -- 172.21.15.99:0/4294476713 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f6d300a2800 msgr2=0x7f6d300a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:54.771 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:54.770+0000 7f6d4888c640 1 --2- 172.21.15.99:0/4294476713 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f6d300a2800 0x7f6d300a4cc0 secure :-1 s=READY pgs=159 cs=0 l=1 rev1=1 crypto rx=0x7f6d34009230 tx=0x7f6d34005fd0 comp rx=0 tx=0).stop 2024-03-19T01:38:54.771 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:54.770+0000 7f6d4888c640 1 -- 172.21.15.99:0/4294476713 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6d44104c70 msgr2=0x7f6d441002f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:54.771 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:54.770+0000 7f6d4888c640 1 --2- 172.21.15.99:0/4294476713 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6d44104c70 0x7f6d441002f0 secure :-1 s=READY pgs=196 cs=0 l=1 rev1=1 crypto rx=0x7f6d2c00d770 tx=0x7f6d2c00dc30 comp rx=0 tx=0).stop 2024-03-19T01:38:54.771 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:54.770+0000 7f6d4888c640 1 -- 172.21.15.99:0/4294476713 shutdown_connections 2024-03-19T01:38:54.771 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:54.770+0000 7f6d4888c640 1 --2- 172.21.15.99:0/4294476713 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f6d300a2800 0x7f6d300a4cc0 unknown :-1 s=CLOSED pgs=159 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:54.771 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:54.770+0000 7f6d4888c640 1 --2- 172.21.15.99:0/4294476713 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6d441075a0 0x7f6d44100830 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:54.771 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:54.770+0000 7f6d4888c640 1 --2- 172.21.15.99:0/4294476713 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6d44104c70 0x7f6d441002f0 unknown :-1 s=CLOSED pgs=196 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:54.772 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:54.770+0000 7f6d4888c640 1 --2- 172.21.15.99:0/4294476713 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f6d44102340 0x7f6d44101cb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:54.772 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:54.770+0000 7f6d4888c640 1 -- 172.21.15.99:0/4294476713 >> 172.21.15.99:0/4294476713 conn(0x7f6d440fbcd0 msgr2=0x7f6d441041d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:54.772 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:54.770+0000 7f6d4888c640 1 -- 172.21.15.99:0/4294476713 shutdown_connections 2024-03-19T01:38:54.772 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:54.770+0000 7f6d4888c640 1 -- 172.21.15.99:0/4294476713 wait complete. 2024-03-19T01:38:54.784 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool create cephfs_data 64 --pg_num_min 64 --target_size_ratio 0.9 2024-03-19T01:38:54.897 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:54.894+0000 7ff7db29a640 1 -- 172.21.15.99:0/3508860022 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff7d4100730 msgr2=0x7ff7d4100b50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:54.898 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:54.894+0000 7ff7db29a640 1 --2- 172.21.15.99:0/3508860022 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff7d4100730 0x7ff7d4100b50 secure :-1 s=READY pgs=197 cs=0 l=1 rev1=1 crypto rx=0x7ff7cc009900 tx=0x7ff7cc02bc30 comp rx=0 tx=0).stop 2024-03-19T01:38:54.898 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:54.894+0000 7ff7db29a640 1 -- 172.21.15.99:0/3508860022 shutdown_connections 2024-03-19T01:38:54.898 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:54.894+0000 7ff7db29a640 1 --2- 172.21.15.99:0/3508860022 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff7d4101920 0x7ff7d4101da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:54.898 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:54.894+0000 7ff7db29a640 1 --2- 172.21.15.99:0/3508860022 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff7d41022e0 0x7ff7d4108b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:54.898 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:54.894+0000 7ff7db29a640 1 --2- 172.21.15.99:0/3508860022 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff7d4100730 0x7ff7d4100b50 unknown :-1 s=CLOSED pgs=197 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:54.898 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:54.894+0000 7ff7db29a640 1 -- 172.21.15.99:0/3508860022 >> 172.21.15.99:0/3508860022 conn(0x7ff7d40fbcc0 msgr2=0x7ff7d40fe0e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:54.898 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:54.894+0000 7ff7db29a640 1 -- 172.21.15.99:0/3508860022 shutdown_connections 2024-03-19T01:38:54.898 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:54.894+0000 7ff7db29a640 1 -- 172.21.15.99:0/3508860022 wait complete. 2024-03-19T01:38:54.898 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:54.894+0000 7ff7db29a640 1 Processor -- start 2024-03-19T01:38:54.898 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:54.894+0000 7ff7db29a640 1 -- start start 2024-03-19T01:38:54.898 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:54.895+0000 7ff7db29a640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff7d4100730 0x7ff7d406a9a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:54.898 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:54.895+0000 7ff7db29a640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff7d4101920 0x7ff7d419c8b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:54.899 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:54.895+0000 7ff7db29a640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff7d41022e0 0x7ff7d419eef0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:54.899 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:54.895+0000 7ff7db29a640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7ff7d410c380 con 0x7ff7d4101920 2024-03-19T01:38:54.899 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:54.895+0000 7ff7db29a640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7ff7d410c200 con 0x7ff7d4100730 2024-03-19T01:38:54.899 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:54.895+0000 7ff7db29a640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7ff7d410c500 con 0x7ff7d41022e0 2024-03-19T01:38:54.899 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:54.895+0000 7ff7d8835640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff7d4101920 0x7ff7d419c8b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:54.899 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:54.895+0000 7ff7d9036640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff7d4100730 0x7ff7d406a9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:54.899 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:54.895+0000 7ff7d8835640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff7d4101920 0x7ff7d419c8b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58390/0 (socket says 172.21.15.99:58390) 2024-03-19T01:38:54.899 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:54.895+0000 7ff7d8835640 1 -- 172.21.15.99:0/1693594958 learned_addr learned my addr 172.21.15.99:0/1693594958 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:38:54.899 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:54.895+0000 7ff7d9837640 1 --2- 172.21.15.99:0/1693594958 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff7d41022e0 0x7ff7d419eef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:54.899 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:54.895+0000 7ff7d8835640 1 -- 172.21.15.99:0/1693594958 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff7d41022e0 msgr2=0x7ff7d419eef0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:54.899 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:54.895+0000 7ff7d8835640 1 --2- 172.21.15.99:0/1693594958 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff7d41022e0 0x7ff7d419eef0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:54.900 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:54.895+0000 7ff7d8835640 1 -- 172.21.15.99:0/1693594958 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff7d4100730 msgr2=0x7ff7d406a9a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:54.900 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:54.895+0000 7ff7d8835640 1 --2- 172.21.15.99:0/1693594958 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff7d4100730 0x7ff7d406a9a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:54.900 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:54.895+0000 7ff7d8835640 1 -- 172.21.15.99:0/1693594958 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ff7d419f7b0 con 0x7ff7d4101920 2024-03-19T01:38:54.900 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:54.895+0000 7ff7d8835640 1 --2- 172.21.15.99:0/1693594958 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff7d4101920 0x7ff7d419c8b0 secure :-1 s=READY pgs=198 cs=0 l=1 rev1=1 crypto rx=0x7ff7c800d950 tx=0x7ff7c800de10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:54.900 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:54.895+0000 7ff7d27fc640 1 -- 172.21.15.99:0/1693594958 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff7c8017030 con 0x7ff7d4101920 2024-03-19T01:38:54.900 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:54.895+0000 7ff7d27fc640 1 -- 172.21.15.99:0/1693594958 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7ff7c8004020 con 0x7ff7d4101920 2024-03-19T01:38:54.900 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:54.895+0000 7ff7db29a640 1 -- 172.21.15.99:0/1693594958 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ff7d419faa0 con 0x7ff7d4101920 2024-03-19T01:38:54.900 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:54.895+0000 7ff7d27fc640 1 -- 172.21.15.99:0/1693594958 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff7c80047a0 con 0x7ff7d4101920 2024-03-19T01:38:54.900 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:54.895+0000 7ff7db29a640 1 -- 172.21.15.99:0/1693594958 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7ff7d41a0040 con 0x7ff7d4101920 2024-03-19T01:38:54.900 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:54.896+0000 7ff7db29a640 1 -- 172.21.15.99:0/1693594958 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7ff7d4107ac0 con 0x7ff7d4101920 2024-03-19T01:38:54.900 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:54.897+0000 7ff7d27fc640 1 -- 172.21.15.99:0/1693594958 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 16) ==== 137290+0+0 (secure 0 0 0) 0x7ff7c80041d0 con 0x7ff7d4101920 2024-03-19T01:38:54.901 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:54.898+0000 7ff7d27fc640 1 --2- 172.21.15.99:0/1693594958 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff7bc0a2880 0x7ff7bc0a4d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:54.901 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:54.898+0000 7ff7d9036640 1 --2- 172.21.15.99:0/1693594958 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff7bc0a2880 0x7ff7bc0a4d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:54.901 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:54.898+0000 7ff7d27fc640 1 -- 172.21.15.99:0/1693594958 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(84..84 src has 1..84) ==== 7497+0+0 (secure 0 0 0) 0x7ff7c80ce6d0 con 0x7ff7d4101920 2024-03-19T01:38:54.901 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:54.898+0000 7ff7d9036640 1 --2- 172.21.15.99:0/1693594958 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff7bc0a2880 0x7ff7bc0a4d40 secure :-1 s=READY pgs=160 cs=0 l=1 rev1=1 crypto rx=0x7ff7cc000c00 tx=0x7ff7cc005fd0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:54.902 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:54.902+0000 7ff7d27fc640 1 -- 172.21.15.99:0/1693594958 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7ff7c809bb20 con 0x7ff7d4101920 2024-03-19T01:38:55.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:54 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/4294476713' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2024-03-19T01:38:55.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:54 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:55.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:54 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:55.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:54 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:55.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:54 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:55.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:54 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:55.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:54 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:55.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:54 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:38:55.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:54 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:38:55.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:54 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:55.080 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:55.080+0000 7ff7db29a640 1 -- 172.21.15.99:0/1693594958 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9} v 0) -- 0x7ff7d402d1f0 con 0x7ff7d4101920 2024-03-19T01:38:55.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:54 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/4294476713' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2024-03-19T01:38:55.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:54 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:55.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:54 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:55.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:54 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:55.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:54 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:55.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:54 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:55.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:54 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:55.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:54 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:38:55.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:54 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:38:55.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:54 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:55.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:54 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/4294476713' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2024-03-19T01:38:55.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:54 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:55.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:54 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:55.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:54 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:55.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:54 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:55.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:54 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:55.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:54 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:55.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:54 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:38:55.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:54 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:38:55.113 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:54 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:38:55.775 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:55.775+0000 7ff7d27fc640 1 -- 172.21.15.99:0/1693594958 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]=0 pool 'cephfs_data' created v85) ==== 170+0+0 (secure 0 0 0) 0x7ff7c809b420 con 0x7ff7d4101920 2024-03-19T01:38:55.775 INFO:teuthology.orchestra.run.smithi099.stderr:pool 'cephfs_data' created 2024-03-19T01:38:55.777 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:55.777+0000 7ff7db29a640 1 -- 172.21.15.99:0/1693594958 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff7bc0a2880 msgr2=0x7ff7bc0a4d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:55.777 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:55.777+0000 7ff7db29a640 1 --2- 172.21.15.99:0/1693594958 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff7bc0a2880 0x7ff7bc0a4d40 secure :-1 s=READY pgs=160 cs=0 l=1 rev1=1 crypto rx=0x7ff7cc000c00 tx=0x7ff7cc005fd0 comp rx=0 tx=0).stop 2024-03-19T01:38:55.778 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:55.777+0000 7ff7db29a640 1 -- 172.21.15.99:0/1693594958 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff7d4101920 msgr2=0x7ff7d419c8b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:55.778 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:55.777+0000 7ff7db29a640 1 --2- 172.21.15.99:0/1693594958 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff7d4101920 0x7ff7d419c8b0 secure :-1 s=READY pgs=198 cs=0 l=1 rev1=1 crypto rx=0x7ff7c800d950 tx=0x7ff7c800de10 comp rx=0 tx=0).stop 2024-03-19T01:38:55.778 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:55.777+0000 7ff7db29a640 1 -- 172.21.15.99:0/1693594958 shutdown_connections 2024-03-19T01:38:55.778 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:55.777+0000 7ff7db29a640 1 --2- 172.21.15.99:0/1693594958 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff7d41022e0 0x7ff7d419eef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:55.778 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:55.777+0000 7ff7db29a640 1 --2- 172.21.15.99:0/1693594958 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff7d4101920 0x7ff7d419c8b0 unknown :-1 s=CLOSED pgs=198 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:55.778 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:55.777+0000 7ff7db29a640 1 --2- 172.21.15.99:0/1693594958 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff7bc0a2880 0x7ff7bc0a4d40 unknown :-1 s=CLOSED pgs=160 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:55.778 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:55.777+0000 7ff7db29a640 1 --2- 172.21.15.99:0/1693594958 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff7d4100730 0x7ff7d406a9a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:55.778 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:55.777+0000 7ff7db29a640 1 -- 172.21.15.99:0/1693594958 >> 172.21.15.99:0/1693594958 conn(0x7ff7d40fbcc0 msgr2=0x7ff7d4104c90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:55.778 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:55.777+0000 7ff7db29a640 1 -- 172.21.15.99:0/1693594958 shutdown_connections 2024-03-19T01:38:55.778 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:55.777+0000 7ff7db29a640 1 -- 172.21.15.99:0/1693594958 wait complete. 2024-03-19T01:38:55.796 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs new cephfs cephfs_metadata cephfs_data 2024-03-19T01:38:55.904 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:55.902+0000 7fa1378dc640 1 -- 172.21.15.99:0/509497637 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa1301075c0 msgr2=0x7fa1301099b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:55.904 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:55.902+0000 7fa1378dc640 1 --2- 172.21.15.99:0/509497637 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa1301075c0 0x7fa1301099b0 secure :-1 s=READY pgs=199 cs=0 l=1 rev1=1 crypto rx=0x7fa128009ca0 tx=0x7fa12802c0f0 comp rx=0 tx=0).stop 2024-03-19T01:38:55.905 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:55.902+0000 7fa1378dc640 1 -- 172.21.15.99:0/509497637 shutdown_connections 2024-03-19T01:38:55.905 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:55.902+0000 7fa1378dc640 1 --2- 172.21.15.99:0/509497637 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa130104c90 0x7fa130107080 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:55.905 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:55.902+0000 7fa1378dc640 1 --2- 172.21.15.99:0/509497637 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa1301075c0 0x7fa1301099b0 unknown :-1 s=CLOSED pgs=199 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:55.905 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:55.902+0000 7fa1378dc640 1 --2- 172.21.15.99:0/509497637 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa130102360 0x7fa130104750 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:55.905 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:55.902+0000 7fa1378dc640 1 -- 172.21.15.99:0/509497637 >> 172.21.15.99:0/509497637 conn(0x7fa1300fbcb0 msgr2=0x7fa1300fe0d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:55.905 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:55.902+0000 7fa1378dc640 1 -- 172.21.15.99:0/509497637 shutdown_connections 2024-03-19T01:38:55.905 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:55.902+0000 7fa1378dc640 1 -- 172.21.15.99:0/509497637 wait complete. 2024-03-19T01:38:55.905 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:55.902+0000 7fa1378dc640 1 Processor -- start 2024-03-19T01:38:55.905 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:55.902+0000 7fa1378dc640 1 -- start start 2024-03-19T01:38:55.905 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:55.903+0000 7fa1378dc640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa130102360 0x7fa130197a30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:55.905 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:55.903+0000 7fa1378dc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa130104c90 0x7fa130197f70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:55.905 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:55.903+0000 7fa1378dc640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa1301075c0 0x7fa13019efe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:55.905 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:55.903+0000 7fa1378dc640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fa13010d4b0 con 0x7fa130104c90 2024-03-19T01:38:55.906 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:55.903+0000 7fa1378dc640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fa13010d330 con 0x7fa130102360 2024-03-19T01:38:55.906 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:55.903+0000 7fa1378dc640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fa13010d630 con 0x7fa1301075c0 2024-03-19T01:38:55.906 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:55.903+0000 7fa134e77640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa130104c90 0x7fa130197f70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:55.906 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:55.903+0000 7fa134e77640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa130104c90 0x7fa130197f70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58404/0 (socket says 172.21.15.99:58404) 2024-03-19T01:38:55.906 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:55.903+0000 7fa134e77640 1 -- 172.21.15.99:0/3022342202 learned_addr learned my addr 172.21.15.99:0/3022342202 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:38:55.906 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:55.903+0000 7fa135e79640 1 --2- 172.21.15.99:0/3022342202 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa1301075c0 0x7fa13019efe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:55.906 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:55.903+0000 7fa134e77640 1 -- 172.21.15.99:0/3022342202 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa1301075c0 msgr2=0x7fa13019efe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:55.906 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:55.903+0000 7fa134e77640 1 --2- 172.21.15.99:0/3022342202 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa1301075c0 0x7fa13019efe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:55.906 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:55.903+0000 7fa134e77640 1 -- 172.21.15.99:0/3022342202 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa130102360 msgr2=0x7fa130197a30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:55.906 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:55.903+0000 7fa135678640 1 --2- 172.21.15.99:0/3022342202 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa130102360 0x7fa130197a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:55.906 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:55.903+0000 7fa134e77640 1 --2- 172.21.15.99:0/3022342202 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa130102360 0x7fa130197a30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:55.906 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:55.903+0000 7fa134e77640 1 -- 172.21.15.99:0/3022342202 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa13019f650 con 0x7fa130104c90 2024-03-19T01:38:55.906 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:55.903+0000 7fa135678640 1 --2- 172.21.15.99:0/3022342202 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa130102360 0x7fa130197a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-19T01:38:55.906 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:55.903+0000 7fa134e77640 1 --2- 172.21.15.99:0/3022342202 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa130104c90 0x7fa130197f70 secure :-1 s=READY pgs=200 cs=0 l=1 rev1=1 crypto rx=0x7fa12000c970 tx=0x7fa12000ce30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:55.907 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:55.903+0000 7fa1267fc640 1 -- 172.21.15.99:0/3022342202 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa120014070 con 0x7fa130104c90 2024-03-19T01:38:55.907 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:55.903+0000 7fa1378dc640 1 -- 172.21.15.99:0/3022342202 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa13019f940 con 0x7fa130104c90 2024-03-19T01:38:55.907 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:55.903+0000 7fa1267fc640 1 -- 172.21.15.99:0/3022342202 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa120004440 con 0x7fa130104c90 2024-03-19T01:38:55.907 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:55.903+0000 7fa1267fc640 1 -- 172.21.15.99:0/3022342202 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa12000f450 con 0x7fa130104c90 2024-03-19T01:38:55.907 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:55.903+0000 7fa1378dc640 1 -- 172.21.15.99:0/3022342202 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fa13019fe90 con 0x7fa130104c90 2024-03-19T01:38:55.907 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:55.904+0000 7fa1378dc640 1 -- 172.21.15.99:0/3022342202 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fa1300fd7a0 con 0x7fa130104c90 2024-03-19T01:38:55.908 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:55.905+0000 7fa1267fc640 1 -- 172.21.15.99:0/3022342202 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 16) ==== 137290+0+0 (secure 0 0 0) 0x7fa120017070 con 0x7fa130104c90 2024-03-19T01:38:55.908 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:55.906+0000 7fa1267fc640 1 --2- 172.21.15.99:0/3022342202 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa11c0a2800 0x7fa11c0a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:55.908 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:55.906+0000 7fa135678640 1 --2- 172.21.15.99:0/3022342202 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa11c0a2800 0x7fa11c0a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:55.908 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:55.906+0000 7fa1267fc640 1 -- 172.21.15.99:0/3022342202 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(85..85 src has 1..85) ==== 7879+0+0 (secure 0 0 0) 0x7fa1200cebf0 con 0x7fa130104c90 2024-03-19T01:38:55.908 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:55.906+0000 7fa135678640 1 --2- 172.21.15.99:0/3022342202 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa11c0a2800 0x7fa11c0a4cc0 secure :-1 s=READY pgs=161 cs=0 l=1 rev1=1 crypto rx=0x7fa12c00baa0 tx=0x7fa12c005d90 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:55.910 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:55.910+0000 7fa1267fc640 1 -- 172.21.15.99:0/3022342202 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7fa12001e020 con 0x7fa130104c90 2024-03-19T01:38:56.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:55 smithi115 ceph-mon[67332]: pgmap v277: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:56.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:55 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/4294476713' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]': finished 2024-03-19T01:38:56.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:55 smithi115 ceph-mon[67332]: osdmap e84: 12 total, 12 up, 12 in 2024-03-19T01:38:56.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:55 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/1693594958' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]: dispatch 2024-03-19T01:38:56.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:55 smithi099 ceph-mon[62951]: pgmap v277: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:56.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:55 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/4294476713' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]': finished 2024-03-19T01:38:56.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:55 smithi099 ceph-mon[62951]: osdmap e84: 12 total, 12 up, 12 in 2024-03-19T01:38:56.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:55 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/1693594958' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]: dispatch 2024-03-19T01:38:56.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:55 smithi112 ceph-mon[66819]: pgmap v277: 1 pgs: 1 active+clean; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:56.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:55 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/4294476713' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]': finished 2024-03-19T01:38:56.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:55 smithi112 ceph-mon[66819]: osdmap e84: 12 total, 12 up, 12 in 2024-03-19T01:38:56.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:55 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/1693594958' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]: dispatch 2024-03-19T01:38:56.122 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:56.122+0000 7fa1378dc640 1 -- 172.21.15.99:0/3022342202 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"} v 0) -- 0x7fa13002d1f0 con 0x7fa130104c90 2024-03-19T01:38:56.786 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:56.785+0000 7fa1267fc640 1 -- 172.21.15.99:0/3022342202 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]=0 new fs with metadata pool 2 and data pool 3 v14) ==== 172+0+0 (secure 0 0 0) 0x7fa12009be20 con 0x7fa130104c90 2024-03-19T01:38:56.786 INFO:teuthology.orchestra.run.smithi099.stderr:new fs with metadata pool 2 and data pool 3 2024-03-19T01:38:56.790 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:56.789+0000 7fa1378dc640 1 -- 172.21.15.99:0/3022342202 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa11c0a2800 msgr2=0x7fa11c0a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:56.790 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:56.789+0000 7fa1378dc640 1 --2- 172.21.15.99:0/3022342202 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa11c0a2800 0x7fa11c0a4cc0 secure :-1 s=READY pgs=161 cs=0 l=1 rev1=1 crypto rx=0x7fa12c00baa0 tx=0x7fa12c005d90 comp rx=0 tx=0).stop 2024-03-19T01:38:56.790 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:56.789+0000 7fa1378dc640 1 -- 172.21.15.99:0/3022342202 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa130104c90 msgr2=0x7fa130197f70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:56.790 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:56.789+0000 7fa1378dc640 1 --2- 172.21.15.99:0/3022342202 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa130104c90 0x7fa130197f70 secure :-1 s=READY pgs=200 cs=0 l=1 rev1=1 crypto rx=0x7fa12000c970 tx=0x7fa12000ce30 comp rx=0 tx=0).stop 2024-03-19T01:38:56.790 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:56.790+0000 7fa1378dc640 1 -- 172.21.15.99:0/3022342202 shutdown_connections 2024-03-19T01:38:56.790 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:56.790+0000 7fa1378dc640 1 --2- 172.21.15.99:0/3022342202 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa1301075c0 0x7fa13019efe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:56.790 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:56.790+0000 7fa1378dc640 1 --2- 172.21.15.99:0/3022342202 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa130104c90 0x7fa130197f70 unknown :-1 s=CLOSED pgs=200 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:56.790 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:56.790+0000 7fa1378dc640 1 --2- 172.21.15.99:0/3022342202 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa11c0a2800 0x7fa11c0a4cc0 unknown :-1 s=CLOSED pgs=161 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:56.790 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:56.790+0000 7fa1378dc640 1 --2- 172.21.15.99:0/3022342202 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa130102360 0x7fa130197a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:56.791 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:56.790+0000 7fa1378dc640 1 -- 172.21.15.99:0/3022342202 >> 172.21.15.99:0/3022342202 conn(0x7fa1300fbcb0 msgr2=0x7fa130102c60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:56.791 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:56.790+0000 7fa1378dc640 1 -- 172.21.15.99:0/3022342202 shutdown_connections 2024-03-19T01:38:56.791 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:56.790+0000 7fa1378dc640 1 -- 172.21.15.99:0/3022342202 wait complete. 2024-03-19T01:38:56.808 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-19T01:38:56.912 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:56.909+0000 7f5e8d177640 1 -- 172.21.15.99:0/1805947744 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5e881034c0 msgr2=0x7f5e88109d50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:56.912 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:56.909+0000 7f5e8d177640 1 --2- 172.21.15.99:0/1805947744 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5e881034c0 0x7f5e88109d50 secure :-1 s=READY pgs=201 cs=0 l=1 rev1=1 crypto rx=0x7f5e78009930 tx=0x7f5e7802bc30 comp rx=0 tx=0).stop 2024-03-19T01:38:56.912 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:56.909+0000 7f5e8d177640 1 -- 172.21.15.99:0/1805947744 shutdown_connections 2024-03-19T01:38:56.912 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:56.909+0000 7f5e8d177640 1 --2- 172.21.15.99:0/1805947744 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5e88102b00 0x7f5e88102f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:56.912 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:56.909+0000 7f5e8d177640 1 --2- 172.21.15.99:0/1805947744 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5e881034c0 0x7f5e88109d50 unknown :-1 s=CLOSED pgs=201 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:56.913 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:56.909+0000 7f5e8d177640 1 --2- 172.21.15.99:0/1805947744 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5e880ffb30 0x7f5e880fff50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:56.913 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:56.909+0000 7f5e8d177640 1 -- 172.21.15.99:0/1805947744 >> 172.21.15.99:0/1805947744 conn(0x7f5e880fb460 msgr2=0x7f5e880fd8a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:56.913 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:56.910+0000 7f5e8d177640 1 -- 172.21.15.99:0/1805947744 shutdown_connections 2024-03-19T01:38:56.913 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:56.910+0000 7f5e8d177640 1 -- 172.21.15.99:0/1805947744 wait complete. 2024-03-19T01:38:56.913 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:56.910+0000 7f5e8d177640 1 Processor -- start 2024-03-19T01:38:56.913 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:56.910+0000 7f5e8d177640 1 -- start start 2024-03-19T01:38:56.913 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:56.910+0000 7f5e8d177640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5e880ffb30 0x7f5e88102470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:56.913 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:56.910+0000 7f5e8d177640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5e88102b00 0x7f5e88100ab0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:56.913 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:56.910+0000 7f5e8d177640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5e881034c0 0x7f5e88100ff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:56.913 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:56.910+0000 7f5e8d177640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f5e8810d580 con 0x7f5e881034c0 2024-03-19T01:38:56.913 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:56.910+0000 7f5e8d177640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f5e8810d400 con 0x7f5e880ffb30 2024-03-19T01:38:56.913 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:56.910+0000 7f5e8d177640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f5e8810d700 con 0x7f5e88102b00 2024-03-19T01:38:56.913 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:56.911+0000 7f5e8759e640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5e881034c0 0x7f5e88100ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:56.913 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:56.911+0000 7f5e86d9d640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5e880ffb30 0x7f5e88102470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:56.914 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:56.911+0000 7f5e8759e640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5e881034c0 0x7f5e88100ff0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58414/0 (socket says 172.21.15.99:58414) 2024-03-19T01:38:56.914 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:56.911+0000 7f5e8759e640 1 -- 172.21.15.99:0/2993675700 learned_addr learned my addr 172.21.15.99:0/2993675700 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:38:56.914 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:56.911+0000 7f5e8659c640 1 --2- 172.21.15.99:0/2993675700 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5e88102b00 0x7f5e88100ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:56.914 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:56.911+0000 7f5e8759e640 1 -- 172.21.15.99:0/2993675700 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5e88102b00 msgr2=0x7f5e88100ab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:56.914 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:56.911+0000 7f5e8759e640 1 --2- 172.21.15.99:0/2993675700 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5e88102b00 0x7f5e88100ab0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:56.914 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:56.911+0000 7f5e8759e640 1 -- 172.21.15.99:0/2993675700 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5e880ffb30 msgr2=0x7f5e88102470 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:56.914 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:56.911+0000 7f5e8759e640 1 --2- 172.21.15.99:0/2993675700 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5e880ffb30 0x7f5e88102470 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:56.914 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:56.911+0000 7f5e8759e640 1 -- 172.21.15.99:0/2993675700 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f5e88101880 con 0x7f5e881034c0 2024-03-19T01:38:56.914 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:56.911+0000 7f5e8759e640 1 --2- 172.21.15.99:0/2993675700 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5e881034c0 0x7f5e88100ff0 secure :-1 s=READY pgs=202 cs=0 l=1 rev1=1 crypto rx=0x7f5e7802c8e0 tx=0x7f5e78004040 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:56.914 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:56.911+0000 7f5e7ffff640 1 -- 172.21.15.99:0/2993675700 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5e78041070 con 0x7f5e881034c0 2024-03-19T01:38:56.914 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:56.911+0000 7f5e8d177640 1 -- 172.21.15.99:0/2993675700 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f5e881a1cb0 con 0x7f5e881034c0 2024-03-19T01:38:56.914 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:56.911+0000 7f5e7ffff640 1 -- 172.21.15.99:0/2993675700 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f5e780054e0 con 0x7f5e881034c0 2024-03-19T01:38:56.914 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:56.911+0000 7f5e7ffff640 1 -- 172.21.15.99:0/2993675700 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5e780346a0 con 0x7f5e881034c0 2024-03-19T01:38:56.915 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:56.911+0000 7f5e8d177640 1 -- 172.21.15.99:0/2993675700 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f5e881a21a0 con 0x7f5e881034c0 2024-03-19T01:38:56.915 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:56.912+0000 7f5e8d177640 1 -- 172.21.15.99:0/2993675700 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f5e88108cb0 con 0x7f5e881034c0 2024-03-19T01:38:56.915 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:56.913+0000 7f5e7ffff640 1 -- 172.21.15.99:0/2993675700 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 16) ==== 137290+0+0 (secure 0 0 0) 0x7f5e78044070 con 0x7f5e881034c0 2024-03-19T01:38:56.916 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:56.914+0000 7f5e7ffff640 1 --2- 172.21.15.99:0/2993675700 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f5e740a2800 0x7f5e740a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:56.916 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:56.914+0000 7f5e86d9d640 1 --2- 172.21.15.99:0/2993675700 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f5e740a2800 0x7f5e740a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:56.916 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:56.914+0000 7f5e7ffff640 1 -- 172.21.15.99:0/2993675700 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(86..86 src has 1..86) ==== 7979+0+0 (secure 0 0 0) 0x7f5e780f0a20 con 0x7f5e881034c0 2024-03-19T01:38:56.916 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:56.914+0000 7f5e86d9d640 1 --2- 172.21.15.99:0/2993675700 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f5e740a2800 0x7f5e740a4cc0 secure :-1 s=READY pgs=162 cs=0 l=1 rev1=1 crypto rx=0x7f5e88069c70 tx=0x7f5e8000d040 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:56.917 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:56.917+0000 7f5e7ffff640 1 -- 172.21.15.99:0/2993675700 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f5e780bdc10 con 0x7f5e881034c0 2024-03-19T01:38:57.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:56 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/1693594958' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]': finished 2024-03-19T01:38:57.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:56 smithi115 ceph-mon[67332]: osdmap e85: 12 total, 12 up, 12 in 2024-03-19T01:38:57.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:56 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/3022342202' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2024-03-19T01:38:57.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:56 smithi115 ceph-mon[67332]: daemon mds.i assigned to filesystem cephfs as rank 0 2024-03-19T01:38:57.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:56 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/1693594958' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]': finished 2024-03-19T01:38:57.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:56 smithi099 ceph-mon[62951]: osdmap e85: 12 total, 12 up, 12 in 2024-03-19T01:38:57.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:56 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/3022342202' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2024-03-19T01:38:57.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:56 smithi099 ceph-mon[62951]: daemon mds.i assigned to filesystem cephfs as rank 0 2024-03-19T01:38:57.095 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.095+0000 7f5e8d177640 1 -- 172.21.15.99:0/2993675700 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f5e8802d440 con 0x7f5e881034c0 2024-03-19T01:38:57.097 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.097+0000 7f5e7ffff640 1 -- 172.21.15.99:0/2993675700 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v86) ==== 74+0+22246 (secure 0 0 0) 0x7f5e780bd510 con 0x7f5e881034c0 2024-03-19T01:38:57.097 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:38:57.097 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":86,"fsid":"438fb69c-e590-11ee-95c9-87774f69a715","created":"2024-03-19T01:31:10.165073+0000","modified":"2024-03-19T01:38:56.124164+0000","last_up_change":"2024-03-19T01:38:09.082688+0000","last_in_change":"2024-03-19T01:37:49.115793+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-19T01:34:31.001309+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-19T01:38:54.018623+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"86","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-19T01:38:55.319036+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"86","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"0c407a0e-a40e-43d7-ad9a-bb8f3cd4a9ba","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6803","nonce":2935260112}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6805","nonce":2935260112}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6808","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6809","nonce":2935260112}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6807","nonce":2935260112}]},"public_addr":"172.21.15.99:6803/2935260112","cluster_addr":"172.21.15.99:6805/2935260112","heartbeat_back_addr":"172.21.15.99:6809/2935260112","heartbeat_front_addr":"172.21.15.99:6807/2935260112","state":["exists","up"]},{"osd":1,"uuid":"a48f9173-4a06-49f5-bca2-2c9047b8f666","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6800","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6801","nonce":1468854289}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6802","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6803","nonce":1468854289}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6806","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6807","nonce":1468854289}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6804","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6805","nonce":1468854289}]},"public_addr":"172.21.15.112:6801/1468854289","cluster_addr":"172.21.15.112:6803/1468854289","heartbeat_back_addr":"172.21.15.112:6807/1468854289","heartbeat_front_addr":"172.21.15.112:6805/1468854289","state":["exists","up"]},{"osd":2,"uuid":"b7715fdd-b6c9-4348-bd16-5eb61b427eef","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6800","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6801","nonce":3766690472}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6802","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6803","nonce":3766690472}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6806","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6807","nonce":3766690472}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6804","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6805","nonce":3766690472}]},"public_addr":"172.21.15.115:6801/3766690472","cluster_addr":"172.21.15.115:6803/3766690472","heartbeat_back_addr":"172.21.15.115:6807/3766690472","heartbeat_front_addr":"172.21.15.115:6805/3766690472","state":["exists","up"]},{"osd":3,"uuid":"6697ed1b-96bf-42de-8440-d17e9785b359","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6810","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6811","nonce":3756732895}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6812","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6813","nonce":3756732895}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6816","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6817","nonce":3756732895}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6814","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6815","nonce":3756732895}]},"public_addr":"172.21.15.99:6811/3756732895","cluster_addr":"172.21.15.99:6813/3756732895","heartbeat_back_addr":"172.21.15.99:6817/3756732895","heartbeat_front_addr":"172.21.15.99:6815/3756732895","state":["exists","up"]},{"osd":4,"uuid":"d958d6b9-67f6-43d0-b704-aa55dc9df83a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":34,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6808","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6809","nonce":3717986391}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6810","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6811","nonce":3717986391}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6814","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6815","nonce":3717986391}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6812","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6813","nonce":3717986391}]},"public_addr":"172.21.15.112:6809/3717986391","cluster_addr":"172.21.15.112:6811/3717986391","heartbeat_back_addr":"172.21.15.112:6815/3717986391","heartbeat_front_addr":"172.21.15.112:6813/3717986391","state":["exists","up"]},{"osd":5,"uuid":"a73eb49a-7925-49ba-bc4f-515e17f8b8ae","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6808","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6809","nonce":1971806408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6810","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6811","nonce":1971806408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6814","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6815","nonce":1971806408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6812","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6813","nonce":1971806408}]},"public_addr":"172.21.15.115:6809/1971806408","cluster_addr":"172.21.15.115:6811/1971806408","heartbeat_back_addr":"172.21.15.115:6815/1971806408","heartbeat_front_addr":"172.21.15.115:6813/1971806408","state":["exists","up"]},{"osd":6,"uuid":"caa718cd-e08b-48de-832a-29546d71e508","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6818","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6819","nonce":1130832772}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6820","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6821","nonce":1130832772}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6824","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6825","nonce":1130832772}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6822","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6823","nonce":1130832772}]},"public_addr":"172.21.15.99:6819/1130832772","cluster_addr":"172.21.15.99:6821/1130832772","heartbeat_back_addr":"172.21.15.99:6825/1130832772","heartbeat_front_addr":"172.21.15.99:6823/1130832772","state":["exists","up"]},{"osd":7,"uuid":"dbee19a1-2315-40d3-88e7-876fcde28903","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":53,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6816","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6817","nonce":3610356576}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6818","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6819","nonce":3610356576}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6822","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6823","nonce":3610356576}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6820","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6821","nonce":3610356576}]},"public_addr":"172.21.15.112:6817/3610356576","cluster_addr":"172.21.15.112:6819/3610356576","heartbeat_back_addr":"172.21.15.112:6823/3610356576","heartbeat_front_addr":"172.21.15.112:6821/3610356576","state":["exists","up"]},{"osd":8,"uuid":"14a866dc-5ce3-42fc-8bcf-9088b68726cb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":60,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6816","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6817","nonce":4255803111}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6818","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6819","nonce":4255803111}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6822","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6823","nonce":4255803111}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6820","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6821","nonce":4255803111}]},"public_addr":"172.21.15.115:6817/4255803111","cluster_addr":"172.21.15.115:6819/4255803111","heartbeat_back_addr":"172.21.15.115:6823/4255803111","heartbeat_front_addr":"172.21.15.115:6821/4255803111","state":["exists","up"]},{"osd":9,"uuid":"1db93652-6035-4906-91ae-d64d5a909502","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":68,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6826","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6827","nonce":1410921523}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6828","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6829","nonce":1410921523}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6832","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6833","nonce":1410921523}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6830","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6831","nonce":1410921523}]},"public_addr":"172.21.15.99:6827/1410921523","cluster_addr":"172.21.15.99:6829/1410921523","heartbeat_back_addr":"172.21.15.99:6833/1410921523","heartbeat_front_addr":"172.21.15.99:6831/1410921523","state":["exists","up"]},{"osd":10,"uuid":"6c50a8a7-3345-4f4a-af0e-5606c7aca453","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":74,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6824","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6825","nonce":891805922}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6826","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6827","nonce":891805922}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6830","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6831","nonce":891805922}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6828","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6829","nonce":891805922}]},"public_addr":"172.21.15.112:6825/891805922","cluster_addr":"172.21.15.112:6827/891805922","heartbeat_back_addr":"172.21.15.112:6831/891805922","heartbeat_front_addr":"172.21.15.112:6829/891805922","state":["exists","up"]},{"osd":11,"uuid":"fd63b7fd-0db5-46d5-82fa-9707bbd0b495","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":82,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6824","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6825","nonce":938436244}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6826","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6827","nonce":938436244}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6830","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6831","nonce":938436244}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6828","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6829","nonce":938436244}]},"public_addr":"172.21.15.115:6825/938436244","cluster_addr":"172.21.15.115:6827/938436244","heartbeat_back_addr":"172.21.15.115:6831/938436244","heartbeat_front_addr":"172.21.15.115:6829/938436244","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:33:39.036822+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:34:03.339566+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:34:27.149273+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:34:51.315561+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:35:15.396031+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:35:38.957168+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:36:03.213914+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:36:27.763699+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:36:51.688670+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:37:17.280165+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:37:42.554633+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:38:07.438039+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.99:6801/1328245934":"2024-03-20T01:32:06.661002+0000","172.21.15.99:0/1618965837":"2024-03-20T01:32:06.661002+0000","172.21.15.99:0/2491429851":"2024-03-20T01:32:06.661002+0000","172.21.15.99:6800/1328245934":"2024-03-20T01:32:06.661002+0000","172.21.15.99:0/2112231766":"2024-03-20T01:31:40.161751+0000","172.21.15.99:0/3341836545":"2024-03-20T01:31:40.161751+0000","172.21.15.99:6800/4223048293":"2024-03-20T01:31:40.161751+0000","172.21.15.99:6801/4223048293":"2024-03-20T01:31:40.161751+0000","172.21.15.99:0/2377621863":"2024-03-20T01:31:40.161751+0000","172.21.15.99:0/1904400935":"2024-03-20T01:32:06.661002+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-19T01:38:57.099 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.099+0000 7f5e8d177640 1 -- 172.21.15.99:0/2993675700 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f5e740a2800 msgr2=0x7f5e740a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:57.099 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.099+0000 7f5e8d177640 1 --2- 172.21.15.99:0/2993675700 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f5e740a2800 0x7f5e740a4cc0 secure :-1 s=READY pgs=162 cs=0 l=1 rev1=1 crypto rx=0x7f5e88069c70 tx=0x7f5e8000d040 comp rx=0 tx=0).stop 2024-03-19T01:38:57.099 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.099+0000 7f5e8d177640 1 -- 172.21.15.99:0/2993675700 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5e881034c0 msgr2=0x7f5e88100ff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:57.100 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.099+0000 7f5e8d177640 1 --2- 172.21.15.99:0/2993675700 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5e881034c0 0x7f5e88100ff0 secure :-1 s=READY pgs=202 cs=0 l=1 rev1=1 crypto rx=0x7f5e7802c8e0 tx=0x7f5e78004040 comp rx=0 tx=0).stop 2024-03-19T01:38:57.100 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.099+0000 7f5e8d177640 1 -- 172.21.15.99:0/2993675700 shutdown_connections 2024-03-19T01:38:57.100 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.099+0000 7f5e8d177640 1 --2- 172.21.15.99:0/2993675700 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5e881034c0 0x7f5e88100ff0 unknown :-1 s=CLOSED pgs=202 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:57.100 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.099+0000 7f5e8d177640 1 --2- 172.21.15.99:0/2993675700 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5e88102b00 0x7f5e88100ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:57.100 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.099+0000 7f5e8d177640 1 --2- 172.21.15.99:0/2993675700 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f5e740a2800 0x7f5e740a4cc0 unknown :-1 s=CLOSED pgs=162 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:57.100 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.099+0000 7f5e8d177640 1 --2- 172.21.15.99:0/2993675700 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5e880ffb30 0x7f5e88102470 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:57.100 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.099+0000 7f5e8d177640 1 -- 172.21.15.99:0/2993675700 >> 172.21.15.99:0/2993675700 conn(0x7f5e880fb460 msgr2=0x7f5e88105e80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:57.100 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.099+0000 7f5e8d177640 1 -- 172.21.15.99:0/2993675700 shutdown_connections 2024-03-19T01:38:57.100 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.099+0000 7f5e8d177640 1 -- 172.21.15.99:0/2993675700 wait complete. 2024-03-19T01:38:57.111 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-19T01:38:57.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:56 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/1693594958' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]': finished 2024-03-19T01:38:57.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:56 smithi112 ceph-mon[66819]: osdmap e85: 12 total, 12 up, 12 in 2024-03-19T01:38:57.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:56 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/3022342202' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2024-03-19T01:38:57.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:56 smithi112 ceph-mon[66819]: daemon mds.i assigned to filesystem cephfs as rank 0 2024-03-19T01:38:57.215 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.212+0000 7f211cce8640 1 -- 172.21.15.99:0/3200677415 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2118103a60 msgr2=0x7f2118103e80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:57.215 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.212+0000 7f211cce8640 1 --2- 172.21.15.99:0/3200677415 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2118103a60 0x7f2118103e80 secure :-1 s=READY pgs=203 cs=0 l=1 rev1=1 crypto rx=0x7f2108009ca0 tx=0x7f210802c0f0 comp rx=0 tx=0).stop 2024-03-19T01:38:57.215 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.212+0000 7f211cce8640 1 -- 172.21.15.99:0/3200677415 shutdown_connections 2024-03-19T01:38:57.215 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.212+0000 7f211cce8640 1 --2- 172.21.15.99:0/3200677415 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f2118104c50 0x7f21181050d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:57.215 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.212+0000 7f211cce8640 1 --2- 172.21.15.99:0/3200677415 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f2118105610 0x7f211810bea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:57.215 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.212+0000 7f211cce8640 1 --2- 172.21.15.99:0/3200677415 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2118103a60 0x7f2118103e80 unknown :-1 s=CLOSED pgs=203 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:57.216 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.212+0000 7f211cce8640 1 -- 172.21.15.99:0/3200677415 >> 172.21.15.99:0/3200677415 conn(0x7f21180fefd0 msgr2=0x7f2118101410 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:57.216 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.212+0000 7f211cce8640 1 -- 172.21.15.99:0/3200677415 shutdown_connections 2024-03-19T01:38:57.216 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.212+0000 7f211cce8640 1 -- 172.21.15.99:0/3200677415 wait complete. 2024-03-19T01:38:57.216 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.213+0000 7f211cce8640 1 Processor -- start 2024-03-19T01:38:57.216 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.213+0000 7f211cce8640 1 -- start start 2024-03-19T01:38:57.216 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.213+0000 7f211cce8640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f2118103a60 0x7f2118199e00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:57.216 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.213+0000 7f211cce8640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f2118104c50 0x7f211819a340 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:57.216 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.213+0000 7f211cce8640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2118105610 0x7f21181a13b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:57.216 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.213+0000 7f211cce8640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f211810f730 con 0x7f2118105610 2024-03-19T01:38:57.216 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.213+0000 7f211cce8640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f211810f5b0 con 0x7f2118104c50 2024-03-19T01:38:57.216 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.213+0000 7f211cce8640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f211810f8b0 con 0x7f2118103a60 2024-03-19T01:38:57.216 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.213+0000 7f2116d9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2118105610 0x7f21181a13b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:57.216 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.213+0000 7f2116d9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2118105610 0x7f21181a13b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58432/0 (socket says 172.21.15.99:58432) 2024-03-19T01:38:57.216 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.213+0000 7f2116d9d640 1 -- 172.21.15.99:0/197101494 learned_addr learned my addr 172.21.15.99:0/197101494 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:38:57.217 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.213+0000 7f211659c640 1 --2- 172.21.15.99:0/197101494 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f2118103a60 0x7f2118199e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:57.217 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.214+0000 7f2115d9b640 1 --2- 172.21.15.99:0/197101494 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f2118104c50 0x7f211819a340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:57.217 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.214+0000 7f2116d9d640 1 -- 172.21.15.99:0/197101494 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f2118103a60 msgr2=0x7f2118199e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:57.217 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.214+0000 7f2116d9d640 1 --2- 172.21.15.99:0/197101494 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f2118103a60 0x7f2118199e00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:57.217 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.214+0000 7f2116d9d640 1 -- 172.21.15.99:0/197101494 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f2118104c50 msgr2=0x7f211819a340 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:57.217 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.214+0000 7f2116d9d640 1 --2- 172.21.15.99:0/197101494 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f2118104c50 0x7f211819a340 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:57.217 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.214+0000 7f2116d9d640 1 -- 172.21.15.99:0/197101494 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f21181a1a20 con 0x7f2118105610 2024-03-19T01:38:57.217 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.214+0000 7f2116d9d640 1 --2- 172.21.15.99:0/197101494 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2118105610 0x7f21181a13b0 secure :-1 s=READY pgs=204 cs=0 l=1 rev1=1 crypto rx=0x7f2110009fd0 tx=0x7f211000b8e0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:57.217 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.214+0000 7f21077fe640 1 -- 172.21.15.99:0/197101494 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2110011830 con 0x7f2118105610 2024-03-19T01:38:57.217 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.214+0000 7f21077fe640 1 -- 172.21.15.99:0/197101494 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f2110016070 con 0x7f2118105610 2024-03-19T01:38:57.217 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.214+0000 7f211cce8640 1 -- 172.21.15.99:0/197101494 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f21181a1d10 con 0x7f2118105610 2024-03-19T01:38:57.217 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.214+0000 7f21077fe640 1 -- 172.21.15.99:0/197101494 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f211000f2e0 con 0x7f2118105610 2024-03-19T01:38:57.217 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.214+0000 7f211cce8640 1 -- 172.21.15.99:0/197101494 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f21181a2200 con 0x7f2118105610 2024-03-19T01:38:57.218 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.215+0000 7f211cce8640 1 -- 172.21.15.99:0/197101494 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f211810ae50 con 0x7f2118105610 2024-03-19T01:38:57.218 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.216+0000 7f21077fe640 1 -- 172.21.15.99:0/197101494 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 16) ==== 137290+0+0 (secure 0 0 0) 0x7f21100119d0 con 0x7f2118105610 2024-03-19T01:38:57.218 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.216+0000 7f21077fe640 1 --2- 172.21.15.99:0/197101494 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f21000a2800 0x7f21000a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:57.219 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.217+0000 7f211659c640 1 --2- 172.21.15.99:0/197101494 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f21000a2800 0x7f21000a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:57.219 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.217+0000 7f21077fe640 1 -- 172.21.15.99:0/197101494 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(86..86 src has 1..86) ==== 7979+0+0 (secure 0 0 0) 0x7f21100cdf40 con 0x7f2118105610 2024-03-19T01:38:57.219 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.217+0000 7f211659c640 1 --2- 172.21.15.99:0/197101494 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f21000a2800 0x7f21000a4cc0 secure :-1 s=READY pgs=163 cs=0 l=1 rev1=1 crypto rx=0x7f210802c700 tx=0x7f2108032590 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:57.221 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.221+0000 7f21077fe640 1 -- 172.21.15.99:0/197101494 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f211009b130 con 0x7f2118105610 2024-03-19T01:38:57.398 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.397+0000 7f211cce8640 1 -- 172.21.15.99:0/197101494 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f21180624c0 con 0x7f2118105610 2024-03-19T01:38:57.399 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.399+0000 7f21077fe640 1 -- 172.21.15.99:0/197101494 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v86) ==== 74+0+22246 (secure 0 0 0) 0x7f211009aa30 con 0x7f2118105610 2024-03-19T01:38:57.400 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:38:57.400 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":86,"fsid":"438fb69c-e590-11ee-95c9-87774f69a715","created":"2024-03-19T01:31:10.165073+0000","modified":"2024-03-19T01:38:56.124164+0000","last_up_change":"2024-03-19T01:38:09.082688+0000","last_in_change":"2024-03-19T01:37:49.115793+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-19T01:34:31.001309+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-19T01:38:54.018623+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"86","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-19T01:38:55.319036+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"86","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"0c407a0e-a40e-43d7-ad9a-bb8f3cd4a9ba","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6803","nonce":2935260112}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6805","nonce":2935260112}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6808","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6809","nonce":2935260112}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6807","nonce":2935260112}]},"public_addr":"172.21.15.99:6803/2935260112","cluster_addr":"172.21.15.99:6805/2935260112","heartbeat_back_addr":"172.21.15.99:6809/2935260112","heartbeat_front_addr":"172.21.15.99:6807/2935260112","state":["exists","up"]},{"osd":1,"uuid":"a48f9173-4a06-49f5-bca2-2c9047b8f666","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6800","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6801","nonce":1468854289}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6802","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6803","nonce":1468854289}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6806","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6807","nonce":1468854289}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6804","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6805","nonce":1468854289}]},"public_addr":"172.21.15.112:6801/1468854289","cluster_addr":"172.21.15.112:6803/1468854289","heartbeat_back_addr":"172.21.15.112:6807/1468854289","heartbeat_front_addr":"172.21.15.112:6805/1468854289","state":["exists","up"]},{"osd":2,"uuid":"b7715fdd-b6c9-4348-bd16-5eb61b427eef","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6800","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6801","nonce":3766690472}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6802","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6803","nonce":3766690472}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6806","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6807","nonce":3766690472}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6804","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6805","nonce":3766690472}]},"public_addr":"172.21.15.115:6801/3766690472","cluster_addr":"172.21.15.115:6803/3766690472","heartbeat_back_addr":"172.21.15.115:6807/3766690472","heartbeat_front_addr":"172.21.15.115:6805/3766690472","state":["exists","up"]},{"osd":3,"uuid":"6697ed1b-96bf-42de-8440-d17e9785b359","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6810","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6811","nonce":3756732895}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6812","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6813","nonce":3756732895}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6816","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6817","nonce":3756732895}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6814","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6815","nonce":3756732895}]},"public_addr":"172.21.15.99:6811/3756732895","cluster_addr":"172.21.15.99:6813/3756732895","heartbeat_back_addr":"172.21.15.99:6817/3756732895","heartbeat_front_addr":"172.21.15.99:6815/3756732895","state":["exists","up"]},{"osd":4,"uuid":"d958d6b9-67f6-43d0-b704-aa55dc9df83a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":34,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6808","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6809","nonce":3717986391}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6810","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6811","nonce":3717986391}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6814","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6815","nonce":3717986391}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6812","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6813","nonce":3717986391}]},"public_addr":"172.21.15.112:6809/3717986391","cluster_addr":"172.21.15.112:6811/3717986391","heartbeat_back_addr":"172.21.15.112:6815/3717986391","heartbeat_front_addr":"172.21.15.112:6813/3717986391","state":["exists","up"]},{"osd":5,"uuid":"a73eb49a-7925-49ba-bc4f-515e17f8b8ae","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6808","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6809","nonce":1971806408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6810","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6811","nonce":1971806408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6814","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6815","nonce":1971806408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6812","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6813","nonce":1971806408}]},"public_addr":"172.21.15.115:6809/1971806408","cluster_addr":"172.21.15.115:6811/1971806408","heartbeat_back_addr":"172.21.15.115:6815/1971806408","heartbeat_front_addr":"172.21.15.115:6813/1971806408","state":["exists","up"]},{"osd":6,"uuid":"caa718cd-e08b-48de-832a-29546d71e508","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6818","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6819","nonce":1130832772}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6820","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6821","nonce":1130832772}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6824","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6825","nonce":1130832772}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6822","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6823","nonce":1130832772}]},"public_addr":"172.21.15.99:6819/1130832772","cluster_addr":"172.21.15.99:6821/1130832772","heartbeat_back_addr":"172.21.15.99:6825/1130832772","heartbeat_front_addr":"172.21.15.99:6823/1130832772","state":["exists","up"]},{"osd":7,"uuid":"dbee19a1-2315-40d3-88e7-876fcde28903","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":53,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6816","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6817","nonce":3610356576}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6818","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6819","nonce":3610356576}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6822","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6823","nonce":3610356576}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6820","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6821","nonce":3610356576}]},"public_addr":"172.21.15.112:6817/3610356576","cluster_addr":"172.21.15.112:6819/3610356576","heartbeat_back_addr":"172.21.15.112:6823/3610356576","heartbeat_front_addr":"172.21.15.112:6821/3610356576","state":["exists","up"]},{"osd":8,"uuid":"14a866dc-5ce3-42fc-8bcf-9088b68726cb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":60,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6816","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6817","nonce":4255803111}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6818","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6819","nonce":4255803111}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6822","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6823","nonce":4255803111}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6820","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6821","nonce":4255803111}]},"public_addr":"172.21.15.115:6817/4255803111","cluster_addr":"172.21.15.115:6819/4255803111","heartbeat_back_addr":"172.21.15.115:6823/4255803111","heartbeat_front_addr":"172.21.15.115:6821/4255803111","state":["exists","up"]},{"osd":9,"uuid":"1db93652-6035-4906-91ae-d64d5a909502","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":68,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6826","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6827","nonce":1410921523}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6828","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6829","nonce":1410921523}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6832","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6833","nonce":1410921523}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6830","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6831","nonce":1410921523}]},"public_addr":"172.21.15.99:6827/1410921523","cluster_addr":"172.21.15.99:6829/1410921523","heartbeat_back_addr":"172.21.15.99:6833/1410921523","heartbeat_front_addr":"172.21.15.99:6831/1410921523","state":["exists","up"]},{"osd":10,"uuid":"6c50a8a7-3345-4f4a-af0e-5606c7aca453","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":74,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6824","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6825","nonce":891805922}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6826","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6827","nonce":891805922}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6830","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6831","nonce":891805922}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6828","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6829","nonce":891805922}]},"public_addr":"172.21.15.112:6825/891805922","cluster_addr":"172.21.15.112:6827/891805922","heartbeat_back_addr":"172.21.15.112:6831/891805922","heartbeat_front_addr":"172.21.15.112:6829/891805922","state":["exists","up"]},{"osd":11,"uuid":"fd63b7fd-0db5-46d5-82fa-9707bbd0b495","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":82,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6824","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6825","nonce":938436244}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6826","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6827","nonce":938436244}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6830","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6831","nonce":938436244}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6828","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6829","nonce":938436244}]},"public_addr":"172.21.15.115:6825/938436244","cluster_addr":"172.21.15.115:6827/938436244","heartbeat_back_addr":"172.21.15.115:6831/938436244","heartbeat_front_addr":"172.21.15.115:6829/938436244","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:33:39.036822+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:34:03.339566+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:34:27.149273+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:34:51.315561+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:35:15.396031+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:35:38.957168+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:36:03.213914+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:36:27.763699+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:36:51.688670+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:37:17.280165+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:37:42.554633+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:38:07.438039+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.99:6801/1328245934":"2024-03-20T01:32:06.661002+0000","172.21.15.99:0/1618965837":"2024-03-20T01:32:06.661002+0000","172.21.15.99:0/2491429851":"2024-03-20T01:32:06.661002+0000","172.21.15.99:6800/1328245934":"2024-03-20T01:32:06.661002+0000","172.21.15.99:0/2112231766":"2024-03-20T01:31:40.161751+0000","172.21.15.99:0/3341836545":"2024-03-20T01:31:40.161751+0000","172.21.15.99:6800/4223048293":"2024-03-20T01:31:40.161751+0000","172.21.15.99:6801/4223048293":"2024-03-20T01:31:40.161751+0000","172.21.15.99:0/2377621863":"2024-03-20T01:31:40.161751+0000","172.21.15.99:0/1904400935":"2024-03-20T01:32:06.661002+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-19T01:38:57.401 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.401+0000 7f211cce8640 1 -- 172.21.15.99:0/197101494 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f21000a2800 msgr2=0x7f21000a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:57.401 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.401+0000 7f211cce8640 1 --2- 172.21.15.99:0/197101494 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f21000a2800 0x7f21000a4cc0 secure :-1 s=READY pgs=163 cs=0 l=1 rev1=1 crypto rx=0x7f210802c700 tx=0x7f2108032590 comp rx=0 tx=0).stop 2024-03-19T01:38:57.401 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.401+0000 7f211cce8640 1 -- 172.21.15.99:0/197101494 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2118105610 msgr2=0x7f21181a13b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:57.402 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.401+0000 7f211cce8640 1 --2- 172.21.15.99:0/197101494 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2118105610 0x7f21181a13b0 secure :-1 s=READY pgs=204 cs=0 l=1 rev1=1 crypto rx=0x7f2110009fd0 tx=0x7f211000b8e0 comp rx=0 tx=0).stop 2024-03-19T01:38:57.402 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.401+0000 7f211cce8640 1 -- 172.21.15.99:0/197101494 shutdown_connections 2024-03-19T01:38:57.402 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.401+0000 7f211cce8640 1 --2- 172.21.15.99:0/197101494 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f21000a2800 0x7f21000a4cc0 unknown :-1 s=CLOSED pgs=163 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:57.402 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.401+0000 7f211cce8640 1 --2- 172.21.15.99:0/197101494 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f2118104c50 0x7f211819a340 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:57.402 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.401+0000 7f211cce8640 1 --2- 172.21.15.99:0/197101494 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2118105610 0x7f21181a13b0 unknown :-1 s=CLOSED pgs=204 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:57.402 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.401+0000 7f211cce8640 1 --2- 172.21.15.99:0/197101494 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f2118103a60 0x7f2118199e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:57.402 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.401+0000 7f211cce8640 1 -- 172.21.15.99:0/197101494 >> 172.21.15.99:0/197101494 conn(0x7f21180fefd0 msgr2=0x7f2118108020 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:57.402 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.401+0000 7f211cce8640 1 -- 172.21.15.99:0/197101494 shutdown_connections 2024-03-19T01:38:57.402 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.401+0000 7f211cce8640 1 -- 172.21.15.99:0/197101494 wait complete. 2024-03-19T01:38:57.413 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs standby_count_wanted 0 2024-03-19T01:38:57.519 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.516+0000 7fb4087e6640 1 -- 172.21.15.99:0/3133405352 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb400102a20 msgr2=0x7fb400102ea0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:57.519 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.516+0000 7fb4087e6640 1 --2- 172.21.15.99:0/3133405352 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb400102a20 0x7fb400102ea0 secure :-1 s=READY pgs=205 cs=0 l=1 rev1=1 crypto rx=0x7fb3f0009900 tx=0x7fb3f002bc30 comp rx=0 tx=0).stop 2024-03-19T01:38:57.519 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.516+0000 7fb4087e6640 1 -- 172.21.15.99:0/3133405352 shutdown_connections 2024-03-19T01:38:57.519 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.516+0000 7fb4087e6640 1 --2- 172.21.15.99:0/3133405352 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb4001033e0 0x7fb400109c70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:57.520 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.516+0000 7fb4087e6640 1 --2- 172.21.15.99:0/3133405352 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb400102a20 0x7fb400102ea0 unknown :-1 s=CLOSED pgs=205 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:57.520 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.516+0000 7fb4087e6640 1 --2- 172.21.15.99:0/3133405352 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb400101830 0x7fb400101c50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:57.520 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.516+0000 7fb4087e6640 1 -- 172.21.15.99:0/3133405352 >> 172.21.15.99:0/3133405352 conn(0x7fb4000fcdc0 msgr2=0x7fb4000ff1e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:57.520 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.516+0000 7fb4087e6640 1 -- 172.21.15.99:0/3133405352 shutdown_connections 2024-03-19T01:38:57.520 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.516+0000 7fb4087e6640 1 -- 172.21.15.99:0/3133405352 wait complete. 2024-03-19T01:38:57.520 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.517+0000 7fb4087e6640 1 Processor -- start 2024-03-19T01:38:57.520 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.517+0000 7fb4087e6640 1 -- start start 2024-03-19T01:38:57.520 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.517+0000 7fb4087e6640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb400101830 0x7fb400197b60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:57.520 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.517+0000 7fb4087e6640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb400102a20 0x7fb4001980a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:57.520 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.517+0000 7fb4087e6640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb4001033e0 0x7fb40019f110 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:57.520 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.517+0000 7fb4087e6640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fb40010d490 con 0x7fb400101830 2024-03-19T01:38:57.520 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.517+0000 7fb4087e6640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fb40010d310 con 0x7fb4001033e0 2024-03-19T01:38:57.520 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.517+0000 7fb4087e6640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fb40010d610 con 0x7fb400102a20 2024-03-19T01:38:57.521 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.517+0000 7fb406582640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb400101830 0x7fb400197b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:57.521 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.517+0000 7fb406582640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb400101830 0x7fb400197b60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58454/0 (socket says 172.21.15.99:58454) 2024-03-19T01:38:57.521 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.517+0000 7fb406582640 1 -- 172.21.15.99:0/497189000 learned_addr learned my addr 172.21.15.99:0/497189000 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:38:57.521 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.517+0000 7fb406d83640 1 --2- 172.21.15.99:0/497189000 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb4001033e0 0x7fb40019f110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:57.521 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.517+0000 7fb405d81640 1 --2- 172.21.15.99:0/497189000 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb400102a20 0x7fb4001980a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:57.521 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.517+0000 7fb406582640 1 -- 172.21.15.99:0/497189000 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb400102a20 msgr2=0x7fb4001980a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:57.521 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.517+0000 7fb406582640 1 --2- 172.21.15.99:0/497189000 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb400102a20 0x7fb4001980a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:57.521 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.517+0000 7fb406582640 1 -- 172.21.15.99:0/497189000 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb4001033e0 msgr2=0x7fb40019f110 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:57.521 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.517+0000 7fb406582640 1 --2- 172.21.15.99:0/497189000 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb4001033e0 0x7fb40019f110 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:57.521 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.517+0000 7fb406582640 1 -- 172.21.15.99:0/497189000 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb40019f780 con 0x7fb400101830 2024-03-19T01:38:57.521 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.517+0000 7fb406d83640 1 --2- 172.21.15.99:0/497189000 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb4001033e0 0x7fb40019f110 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-19T01:38:57.521 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.518+0000 7fb406582640 1 --2- 172.21.15.99:0/497189000 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb400101830 0x7fb400197b60 secure :-1 s=READY pgs=206 cs=0 l=1 rev1=1 crypto rx=0x7fb3fc00d770 tx=0x7fb3fc00dc30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:57.521 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.518+0000 7fb3f77fe640 1 -- 172.21.15.99:0/497189000 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb3fc017070 con 0x7fb400101830 2024-03-19T01:38:57.521 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.518+0000 7fb3f77fe640 1 -- 172.21.15.99:0/497189000 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fb3fc009c30 con 0x7fb400101830 2024-03-19T01:38:57.522 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.518+0000 7fb3f77fe640 1 -- 172.21.15.99:0/497189000 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb3fc003dd0 con 0x7fb400101830 2024-03-19T01:38:57.522 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.518+0000 7fb4087e6640 1 -- 172.21.15.99:0/497189000 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb40019fa70 con 0x7fb400101830 2024-03-19T01:38:57.522 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.518+0000 7fb4087e6640 1 -- 172.21.15.99:0/497189000 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fb40019ff70 con 0x7fb400101830 2024-03-19T01:38:57.522 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.519+0000 7fb4087e6640 1 -- 172.21.15.99:0/497189000 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fb400108bd0 con 0x7fb400101830 2024-03-19T01:38:57.523 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.520+0000 7fb3f77fe640 1 -- 172.21.15.99:0/497189000 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 16) ==== 137290+0+0 (secure 0 0 0) 0x7fb3fc003f70 con 0x7fb400101830 2024-03-19T01:38:57.523 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.520+0000 7fb3f77fe640 1 --2- 172.21.15.99:0/497189000 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb3ec0a2800 0x7fb3ec0a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:57.523 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.520+0000 7fb405d81640 1 --2- 172.21.15.99:0/497189000 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb3ec0a2800 0x7fb3ec0a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:57.523 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.520+0000 7fb3f77fe640 1 -- 172.21.15.99:0/497189000 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(86..86 src has 1..86) ==== 7979+0+0 (secure 0 0 0) 0x7fb3fc0ce350 con 0x7fb400101830 2024-03-19T01:38:57.523 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.521+0000 7fb405d81640 1 --2- 172.21.15.99:0/497189000 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb3ec0a2800 0x7fb3ec0a4cc0 secure :-1 s=READY pgs=164 cs=0 l=1 rev1=1 crypto rx=0x7fb3f0036040 tx=0x7fb3f0005fd0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:57.525 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.525+0000 7fb3f77fe640 1 -- 172.21.15.99:0/497189000 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7fb3fc09b540 con 0x7fb400101830 2024-03-19T01:38:57.743 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.742+0000 7fb4087e6640 1 -- 172.21.15.99:0/497189000 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"} v 0) -- 0x7fb40002d4d0 con 0x7fb400101830 2024-03-19T01:38:57.786 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.786+0000 7fb3f77fe640 1 -- 172.21.15.99:0/497189000 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]=0 v15) ==== 118+0+0 (secure 0 0 0) 0x7fb3fc009de0 con 0x7fb400101830 2024-03-19T01:38:57.788 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.788+0000 7fb4087e6640 1 -- 172.21.15.99:0/497189000 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb3ec0a2800 msgr2=0x7fb3ec0a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:57.789 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.788+0000 7fb4087e6640 1 --2- 172.21.15.99:0/497189000 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb3ec0a2800 0x7fb3ec0a4cc0 secure :-1 s=READY pgs=164 cs=0 l=1 rev1=1 crypto rx=0x7fb3f0036040 tx=0x7fb3f0005fd0 comp rx=0 tx=0).stop 2024-03-19T01:38:57.789 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.788+0000 7fb4087e6640 1 -- 172.21.15.99:0/497189000 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb400101830 msgr2=0x7fb400197b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:57.789 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.788+0000 7fb4087e6640 1 --2- 172.21.15.99:0/497189000 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb400101830 0x7fb400197b60 secure :-1 s=READY pgs=206 cs=0 l=1 rev1=1 crypto rx=0x7fb3fc00d770 tx=0x7fb3fc00dc30 comp rx=0 tx=0).stop 2024-03-19T01:38:57.789 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.788+0000 7fb4087e6640 1 -- 172.21.15.99:0/497189000 shutdown_connections 2024-03-19T01:38:57.789 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.788+0000 7fb4087e6640 1 --2- 172.21.15.99:0/497189000 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb3ec0a2800 0x7fb3ec0a4cc0 unknown :-1 s=CLOSED pgs=164 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:57.789 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.788+0000 7fb4087e6640 1 --2- 172.21.15.99:0/497189000 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb4001033e0 0x7fb40019f110 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:57.789 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.788+0000 7fb4087e6640 1 --2- 172.21.15.99:0/497189000 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb400102a20 0x7fb4001980a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:57.789 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.788+0000 7fb4087e6640 1 --2- 172.21.15.99:0/497189000 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb400101830 0x7fb400197b60 unknown :-1 s=CLOSED pgs=206 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:57.789 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.788+0000 7fb4087e6640 1 -- 172.21.15.99:0/497189000 >> 172.21.15.99:0/497189000 conn(0x7fb4000fcdc0 msgr2=0x7fb400105da0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:57.789 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.788+0000 7fb4087e6640 1 -- 172.21.15.99:0/497189000 shutdown_connections 2024-03-19T01:38:57.789 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.788+0000 7fb4087e6640 1 -- 172.21.15.99:0/497189000 wait complete. 2024-03-19T01:38:57.801 DEBUG:tasks.cephfs.filesystem:fs_config: {'max_mds': 3, 'session_timeout': 300, 'standby_replay': True, 'subvols': {'create': 2}} 2024-03-19T01:38:57.802 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs max_mds 3 2024-03-19T01:38:57.908 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.906+0000 7f70fc681640 1 -- 172.21.15.99:0/867344793 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f70f4101830 msgr2=0x7f70f4101c50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:57.909 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.906+0000 7f70fc681640 1 --2- 172.21.15.99:0/867344793 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f70f4101830 0x7f70f4101c50 secure :-1 s=READY pgs=207 cs=0 l=1 rev1=1 crypto rx=0x7f70f0009900 tx=0x7f70f002bc30 comp rx=0 tx=0).stop 2024-03-19T01:38:57.909 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.906+0000 7f70fc681640 1 -- 172.21.15.99:0/867344793 shutdown_connections 2024-03-19T01:38:57.909 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.906+0000 7f70fc681640 1 --2- 172.21.15.99:0/867344793 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f70f41033e0 0x7f70f4109c70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:57.909 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.906+0000 7f70fc681640 1 --2- 172.21.15.99:0/867344793 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f70f4102a20 0x7f70f4102ea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:57.909 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.906+0000 7f70fc681640 1 --2- 172.21.15.99:0/867344793 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f70f4101830 0x7f70f4101c50 unknown :-1 s=CLOSED pgs=207 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:57.909 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.906+0000 7f70fc681640 1 -- 172.21.15.99:0/867344793 >> 172.21.15.99:0/867344793 conn(0x7f70f40fcdc0 msgr2=0x7f70f40ff1e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:57.909 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.906+0000 7f70fc681640 1 -- 172.21.15.99:0/867344793 shutdown_connections 2024-03-19T01:38:57.909 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.906+0000 7f70fc681640 1 -- 172.21.15.99:0/867344793 wait complete. 2024-03-19T01:38:57.909 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.907+0000 7f70fc681640 1 Processor -- start 2024-03-19T01:38:57.909 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.907+0000 7f70fc681640 1 -- start start 2024-03-19T01:38:57.909 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.907+0000 7f70fc681640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f70f4101830 0x7f70f4197ba0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:57.909 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.907+0000 7f70fc681640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f70f4102a20 0x7f70f41980e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:57.910 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.907+0000 7f70fc681640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f70f41033e0 0x7f70f419f150 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:57.910 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.907+0000 7f70fc681640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f70f410d4a0 con 0x7f70f4102a20 2024-03-19T01:38:57.910 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.907+0000 7f70fc681640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f70f410d320 con 0x7f70f4101830 2024-03-19T01:38:57.910 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.907+0000 7f70fc681640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f70f410d620 con 0x7f70f41033e0 2024-03-19T01:38:57.910 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.907+0000 7f70f9c1c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f70f4102a20 0x7f70f41980e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:57.910 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.907+0000 7f70f9c1c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f70f4102a20 0x7f70f41980e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58476/0 (socket says 172.21.15.99:58476) 2024-03-19T01:38:57.910 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.907+0000 7f70f9c1c640 1 -- 172.21.15.99:0/553489765 learned_addr learned my addr 172.21.15.99:0/553489765 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:38:57.910 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.907+0000 7f70fac1e640 1 --2- 172.21.15.99:0/553489765 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f70f41033e0 0x7f70f419f150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:57.910 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.907+0000 7f70fa41d640 1 --2- 172.21.15.99:0/553489765 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f70f4101830 0x7f70f4197ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:57.911 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.907+0000 7f70f9c1c640 1 -- 172.21.15.99:0/553489765 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f70f41033e0 msgr2=0x7f70f419f150 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:57.911 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.907+0000 7f70f9c1c640 1 --2- 172.21.15.99:0/553489765 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f70f41033e0 0x7f70f419f150 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:57.911 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.907+0000 7f70f9c1c640 1 -- 172.21.15.99:0/553489765 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f70f4101830 msgr2=0x7f70f4197ba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:57.911 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.907+0000 7f70f9c1c640 1 --2- 172.21.15.99:0/553489765 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f70f4101830 0x7f70f4197ba0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:57.911 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.907+0000 7f70f9c1c640 1 -- 172.21.15.99:0/553489765 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f70f419f7c0 con 0x7f70f4102a20 2024-03-19T01:38:57.911 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.907+0000 7f70fa41d640 1 --2- 172.21.15.99:0/553489765 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f70f4101830 0x7f70f4197ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-19T01:38:57.911 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.907+0000 7f70f9c1c640 1 --2- 172.21.15.99:0/553489765 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f70f4102a20 0x7f70f41980e0 secure :-1 s=READY pgs=208 cs=0 l=1 rev1=1 crypto rx=0x7f70e400e9f0 tx=0x7f70e400edf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:57.911 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.907+0000 7f70eb7fe640 1 -- 172.21.15.99:0/553489765 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f70e400cd80 con 0x7f70f4102a20 2024-03-19T01:38:57.911 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.907+0000 7f70eb7fe640 1 -- 172.21.15.99:0/553489765 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f70e4005500 con 0x7f70f4102a20 2024-03-19T01:38:57.911 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.907+0000 7f70fc681640 1 -- 172.21.15.99:0/553489765 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f70f419fab0 con 0x7f70f4102a20 2024-03-19T01:38:57.911 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.907+0000 7f70eb7fe640 1 -- 172.21.15.99:0/553489765 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f70e4010690 con 0x7f70f4102a20 2024-03-19T01:38:57.911 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.907+0000 7f70fc681640 1 -- 172.21.15.99:0/553489765 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f70f419ffb0 con 0x7f70f4102a20 2024-03-19T01:38:57.911 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.908+0000 7f70fc681640 1 -- 172.21.15.99:0/553489765 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f70f4108bc0 con 0x7f70f4102a20 2024-03-19T01:38:57.912 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.909+0000 7f70eb7fe640 1 -- 172.21.15.99:0/553489765 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 16) ==== 137290+0+0 (secure 0 0 0) 0x7f70e40046c0 con 0x7f70f4102a20 2024-03-19T01:38:57.912 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.910+0000 7f70eb7fe640 1 --2- 172.21.15.99:0/553489765 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f70e00a2800 0x7f70e00a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:57.912 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.910+0000 7f70fa41d640 1 --2- 172.21.15.99:0/553489765 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f70e00a2800 0x7f70e00a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:57.912 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.910+0000 7f70eb7fe640 1 -- 172.21.15.99:0/553489765 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(87..87 src has 1..87) ==== 7979+0+0 (secure 0 0 0) 0x7f70e40cedb0 con 0x7f70f4102a20 2024-03-19T01:38:57.912 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.911+0000 7f70fa41d640 1 --2- 172.21.15.99:0/553489765 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f70e00a2800 0x7f70e00a4cc0 secure :-1 s=READY pgs=165 cs=0 l=1 rev1=1 crypto rx=0x7f70f0036040 tx=0x7f70f0000bc0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:57.915 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:57.915+0000 7f70eb7fe640 1 -- 172.21.15.99:0/553489765 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f70e409bfa0 con 0x7f70f4102a20 2024-03-19T01:38:58.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:57 smithi115 ceph-mon[67332]: pgmap v280: 97 pgs: 8 active+clean, 24 creating+peering, 65 unknown; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:58.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:57 smithi115 ceph-mon[67332]: Health check failed: 2 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-19T01:38:58.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:57 smithi115 ceph-mon[67332]: osdmap e86: 12 total, 12 up, 12 in 2024-03-19T01:38:58.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:57 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/3022342202' entity='client.admin' cmd='[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]': finished 2024-03-19T01:38:58.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:57 smithi115 ceph-mon[67332]: fsmap cephfs:1 {0=i=up:creating} 11 up:standby 2024-03-19T01:38:58.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:57 smithi115 ceph-mon[67332]: daemon mds.i is now active in filesystem cephfs as rank 0 2024-03-19T01:38:58.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:57 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/2993675700' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-19T01:38:58.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:57 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/197101494' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-19T01:38:58.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:57 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/497189000' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]: dispatch 2024-03-19T01:38:58.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:57 smithi099 ceph-mon[62951]: pgmap v280: 97 pgs: 8 active+clean, 24 creating+peering, 65 unknown; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:58.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:57 smithi099 ceph-mon[62951]: Health check failed: 2 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-19T01:38:58.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:57 smithi099 ceph-mon[62951]: osdmap e86: 12 total, 12 up, 12 in 2024-03-19T01:38:58.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:57 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/3022342202' entity='client.admin' cmd='[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]': finished 2024-03-19T01:38:58.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:57 smithi099 ceph-mon[62951]: fsmap cephfs:1 {0=i=up:creating} 11 up:standby 2024-03-19T01:38:58.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:57 smithi099 ceph-mon[62951]: daemon mds.i is now active in filesystem cephfs as rank 0 2024-03-19T01:38:58.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:57 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/2993675700' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-19T01:38:58.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:57 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/197101494' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-19T01:38:58.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:57 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/497189000' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]: dispatch 2024-03-19T01:38:58.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:57 smithi112 ceph-mon[66819]: pgmap v280: 97 pgs: 8 active+clean, 24 creating+peering, 65 unknown; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail 2024-03-19T01:38:58.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:57 smithi112 ceph-mon[66819]: Health check failed: 2 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-19T01:38:58.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:57 smithi112 ceph-mon[66819]: osdmap e86: 12 total, 12 up, 12 in 2024-03-19T01:38:58.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:57 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/3022342202' entity='client.admin' cmd='[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]': finished 2024-03-19T01:38:58.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:57 smithi112 ceph-mon[66819]: fsmap cephfs:1 {0=i=up:creating} 11 up:standby 2024-03-19T01:38:58.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:57 smithi112 ceph-mon[66819]: daemon mds.i is now active in filesystem cephfs as rank 0 2024-03-19T01:38:58.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:57 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/2993675700' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-19T01:38:58.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:57 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/197101494' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-19T01:38:58.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:57 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/497189000' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]: dispatch 2024-03-19T01:38:58.129 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:58.129+0000 7f70fc681640 1 -- 172.21.15.99:0/553489765 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "max_mds", "val": "3"} v 0) -- 0x7f70f402d3f0 con 0x7f70f4102a20 2024-03-19T01:38:58.795 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:58.795+0000 7f70eb7fe640 1 -- 172.21.15.99:0/553489765 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "max_mds", "val": "3"}]=0 v16) ==== 105+0+0 (secure 0 0 0) 0x7f70e409b8a0 con 0x7f70f4102a20 2024-03-19T01:38:58.798 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:58.797+0000 7f70fc681640 1 -- 172.21.15.99:0/553489765 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f70e00a2800 msgr2=0x7f70e00a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:58.798 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:58.797+0000 7f70fc681640 1 --2- 172.21.15.99:0/553489765 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f70e00a2800 0x7f70e00a4cc0 secure :-1 s=READY pgs=165 cs=0 l=1 rev1=1 crypto rx=0x7f70f0036040 tx=0x7f70f0000bc0 comp rx=0 tx=0).stop 2024-03-19T01:38:58.798 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:58.797+0000 7f70fc681640 1 -- 172.21.15.99:0/553489765 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f70f4102a20 msgr2=0x7f70f41980e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:58.798 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:58.797+0000 7f70fc681640 1 --2- 172.21.15.99:0/553489765 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f70f4102a20 0x7f70f41980e0 secure :-1 s=READY pgs=208 cs=0 l=1 rev1=1 crypto rx=0x7f70e400e9f0 tx=0x7f70e400edf0 comp rx=0 tx=0).stop 2024-03-19T01:38:58.798 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:58.797+0000 7f70fc681640 1 -- 172.21.15.99:0/553489765 shutdown_connections 2024-03-19T01:38:58.798 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:58.797+0000 7f70fc681640 1 --2- 172.21.15.99:0/553489765 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f70f41033e0 0x7f70f419f150 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:58.798 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:58.797+0000 7f70fc681640 1 --2- 172.21.15.99:0/553489765 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f70f4102a20 0x7f70f41980e0 unknown :-1 s=CLOSED pgs=208 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:58.798 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:58.797+0000 7f70fc681640 1 --2- 172.21.15.99:0/553489765 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f70e00a2800 0x7f70e00a4cc0 unknown :-1 s=CLOSED pgs=165 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:58.798 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:58.797+0000 7f70fc681640 1 --2- 172.21.15.99:0/553489765 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f70f4101830 0x7f70f4197ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:58.798 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:58.797+0000 7f70fc681640 1 -- 172.21.15.99:0/553489765 >> 172.21.15.99:0/553489765 conn(0x7f70f40fcdc0 msgr2=0x7f70f4105d90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:58.799 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:58.797+0000 7f70fc681640 1 -- 172.21.15.99:0/553489765 shutdown_connections 2024-03-19T01:38:58.799 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:58.797+0000 7f70fc681640 1 -- 172.21.15.99:0/553489765 wait complete. 2024-03-19T01:38:58.813 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs allow_standby_replay true 2024-03-19T01:38:58.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:58 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/497189000' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]': finished 2024-03-19T01:38:58.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:58 smithi112 ceph-mon[66819]: mds.? [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] up:active 2024-03-19T01:38:58.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:58 smithi112 ceph-mon[66819]: fsmap cephfs:1 {0=i=up:active} 11 up:standby 2024-03-19T01:38:58.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:58 smithi112 ceph-mon[66819]: osdmap e87: 12 total, 12 up, 12 in 2024-03-19T01:38:58.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:58 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/553489765' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "max_mds", "val": "3"}]: dispatch 2024-03-19T01:38:58.922 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:58.919+0000 7f05f51c4640 1 -- 172.21.15.99:0/2256996818 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f05f0101910 msgr2=0x7f05f0101d90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:58.922 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:58.919+0000 7f05f51c4640 1 --2- 172.21.15.99:0/2256996818 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f05f0101910 0x7f05f0101d90 secure :-1 s=READY pgs=209 cs=0 l=1 rev1=1 crypto rx=0x7f05d8009900 tx=0x7f05d802bc30 comp rx=0 tx=0).stop 2024-03-19T01:38:58.922 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:58.919+0000 7f05f51c4640 1 -- 172.21.15.99:0/2256996818 shutdown_connections 2024-03-19T01:38:58.922 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:58.919+0000 7f05f51c4640 1 --2- 172.21.15.99:0/2256996818 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f05f01022d0 0x7f05f0108b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:58.922 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:58.919+0000 7f05f51c4640 1 --2- 172.21.15.99:0/2256996818 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f05f0101910 0x7f05f0101d90 unknown :-1 s=CLOSED pgs=209 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:58.923 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:58.919+0000 7f05f51c4640 1 --2- 172.21.15.99:0/2256996818 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f05f0100720 0x7f05f0100b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:58.923 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:58.919+0000 7f05f51c4640 1 -- 172.21.15.99:0/2256996818 >> 172.21.15.99:0/2256996818 conn(0x7f05f00fbcb0 msgr2=0x7f05f00fe0d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:58.923 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:58.919+0000 7f05f51c4640 1 -- 172.21.15.99:0/2256996818 shutdown_connections 2024-03-19T01:38:58.923 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:58.919+0000 7f05f51c4640 1 -- 172.21.15.99:0/2256996818 wait complete. 2024-03-19T01:38:58.923 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:58.920+0000 7f05f51c4640 1 Processor -- start 2024-03-19T01:38:58.923 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:58.920+0000 7f05f51c4640 1 -- start start 2024-03-19T01:38:58.923 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:58.920+0000 7f05f51c4640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f05f0100720 0x7f05f0197b30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:58.923 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:58.920+0000 7f05f51c4640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f05f0101910 0x7f05f0198070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:58.923 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:58.920+0000 7f05f51c4640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f05f01022d0 0x7f05f019f070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:58.923 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:58.920+0000 7f05f51c4640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f05f010c3a0 con 0x7f05f0101910 2024-03-19T01:38:58.923 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:58.920+0000 7f05f51c4640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f05f010c220 con 0x7f05f0100720 2024-03-19T01:38:58.923 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:58.920+0000 7f05f51c4640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f05f010c520 con 0x7f05f01022d0 2024-03-19T01:38:58.923 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:58.920+0000 7f05e7fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f05f0101910 0x7f05f0198070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:58.924 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:58.920+0000 7f05e7fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f05f0101910 0x7f05f0198070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58492/0 (socket says 172.21.15.99:58492) 2024-03-19T01:38:58.924 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:58.920+0000 7f05e7fff640 1 -- 172.21.15.99:0/1250711503 learned_addr learned my addr 172.21.15.99:0/1250711503 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:38:58.924 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:58.921+0000 7f05ef59e640 1 --2- 172.21.15.99:0/1250711503 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f05f01022d0 0x7f05f019f070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:58.924 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:58.921+0000 7f05eed9d640 1 --2- 172.21.15.99:0/1250711503 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f05f0100720 0x7f05f0197b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:58.924 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:58.921+0000 7f05e7fff640 1 -- 172.21.15.99:0/1250711503 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f05f01022d0 msgr2=0x7f05f019f070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:58.924 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:58.921+0000 7f05e7fff640 1 --2- 172.21.15.99:0/1250711503 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f05f01022d0 0x7f05f019f070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:58.924 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:58.921+0000 7f05e7fff640 1 -- 172.21.15.99:0/1250711503 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f05f0100720 msgr2=0x7f05f0197b30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:58.924 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:58.921+0000 7f05e7fff640 1 --2- 172.21.15.99:0/1250711503 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f05f0100720 0x7f05f0197b30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:58.924 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:58.921+0000 7f05e7fff640 1 -- 172.21.15.99:0/1250711503 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f05f019f6e0 con 0x7f05f0101910 2024-03-19T01:38:58.924 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:58.921+0000 7f05eed9d640 1 --2- 172.21.15.99:0/1250711503 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f05f0100720 0x7f05f0197b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-19T01:38:58.924 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:58.921+0000 7f05e7fff640 1 --2- 172.21.15.99:0/1250711503 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f05f0101910 0x7f05f0198070 secure :-1 s=READY pgs=210 cs=0 l=1 rev1=1 crypto rx=0x7f05d8009230 tx=0x7f05d8005d50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:58.924 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:58.921+0000 7f05ecd99640 1 -- 172.21.15.99:0/1250711503 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f05d8041070 con 0x7f05f0101910 2024-03-19T01:38:58.924 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:58.921+0000 7f05ecd99640 1 -- 172.21.15.99:0/1250711503 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f05d80044d0 con 0x7f05f0101910 2024-03-19T01:38:58.924 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:58.921+0000 7f05f51c4640 1 -- 172.21.15.99:0/1250711503 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f05f019f970 con 0x7f05f0101910 2024-03-19T01:38:58.924 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:58.921+0000 7f05ecd99640 1 -- 172.21.15.99:0/1250711503 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f05d8034440 con 0x7f05f0101910 2024-03-19T01:38:58.925 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:58.921+0000 7f05f51c4640 1 -- 172.21.15.99:0/1250711503 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f05f019fe60 con 0x7f05f0101910 2024-03-19T01:38:58.925 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:58.922+0000 7f05f51c4640 1 -- 172.21.15.99:0/1250711503 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f05f01a0030 con 0x7f05f0101910 2024-03-19T01:38:58.925 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:58.923+0000 7f05ecd99640 1 -- 172.21.15.99:0/1250711503 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 16) ==== 137290+0+0 (secure 0 0 0) 0x7f05d8044070 con 0x7f05f0101910 2024-03-19T01:38:58.926 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:58.924+0000 7f05ecd99640 1 --2- 172.21.15.99:0/1250711503 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f05dc0a2800 0x7f05dc0a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:58.926 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:58.924+0000 7f05eed9d640 1 --2- 172.21.15.99:0/1250711503 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f05dc0a2800 0x7f05dc0a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:58.926 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:58.924+0000 7f05ecd99640 1 -- 172.21.15.99:0/1250711503 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(87..87 src has 1..87) ==== 7979+0+0 (secure 0 0 0) 0x7f05d80f0590 con 0x7f05f0101910 2024-03-19T01:38:58.926 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:58.924+0000 7f05eed9d640 1 --2- 172.21.15.99:0/1250711503 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f05dc0a2800 0x7f05dc0a4cc0 secure :-1 s=READY pgs=166 cs=0 l=1 rev1=1 crypto rx=0x7f05f0101770 tx=0x7f05e0005db0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:58.928 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:58.928+0000 7f05ecd99640 1 -- 172.21.15.99:0/1250711503 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f05d80bd780 con 0x7f05f0101910 2024-03-19T01:38:59.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:58 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/497189000' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]': finished 2024-03-19T01:38:59.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:58 smithi115 ceph-mon[67332]: mds.? [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] up:active 2024-03-19T01:38:59.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:58 smithi115 ceph-mon[67332]: fsmap cephfs:1 {0=i=up:active} 11 up:standby 2024-03-19T01:38:59.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:58 smithi115 ceph-mon[67332]: osdmap e87: 12 total, 12 up, 12 in 2024-03-19T01:38:59.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:58 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/553489765' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "max_mds", "val": "3"}]: dispatch 2024-03-19T01:38:59.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:58 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/497189000' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]': finished 2024-03-19T01:38:59.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:58 smithi099 ceph-mon[62951]: mds.? [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] up:active 2024-03-19T01:38:59.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:58 smithi099 ceph-mon[62951]: fsmap cephfs:1 {0=i=up:active} 11 up:standby 2024-03-19T01:38:59.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:58 smithi099 ceph-mon[62951]: osdmap e87: 12 total, 12 up, 12 in 2024-03-19T01:38:59.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:58 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/553489765' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "max_mds", "val": "3"}]: dispatch 2024-03-19T01:38:59.142 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:59.141+0000 7f05f51c4640 1 -- 172.21.15.99:0/1250711503 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "true"} v 0) -- 0x7f05f01a0420 con 0x7f05f0101910 2024-03-19T01:38:59.802 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:59.802+0000 7f05ecd99640 1 -- 172.21.15.99:0/1250711503 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "true"}]=0 v18) ==== 121+0+0 (secure 0 0 0) 0x7f05d80bd080 con 0x7f05f0101910 2024-03-19T01:38:59.805 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:59.804+0000 7f05f51c4640 1 -- 172.21.15.99:0/1250711503 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f05dc0a2800 msgr2=0x7f05dc0a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:59.805 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:59.804+0000 7f05f51c4640 1 --2- 172.21.15.99:0/1250711503 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f05dc0a2800 0x7f05dc0a4cc0 secure :-1 s=READY pgs=166 cs=0 l=1 rev1=1 crypto rx=0x7f05f0101770 tx=0x7f05e0005db0 comp rx=0 tx=0).stop 2024-03-19T01:38:59.805 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:59.804+0000 7f05f51c4640 1 -- 172.21.15.99:0/1250711503 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f05f0101910 msgr2=0x7f05f0198070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:59.805 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:59.804+0000 7f05f51c4640 1 --2- 172.21.15.99:0/1250711503 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f05f0101910 0x7f05f0198070 secure :-1 s=READY pgs=210 cs=0 l=1 rev1=1 crypto rx=0x7f05d8009230 tx=0x7f05d8005d50 comp rx=0 tx=0).stop 2024-03-19T01:38:59.805 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:59.804+0000 7f05f51c4640 1 -- 172.21.15.99:0/1250711503 shutdown_connections 2024-03-19T01:38:59.805 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:59.804+0000 7f05f51c4640 1 --2- 172.21.15.99:0/1250711503 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f05f01022d0 0x7f05f019f070 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:59.805 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:59.804+0000 7f05f51c4640 1 --2- 172.21.15.99:0/1250711503 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f05f0101910 0x7f05f0198070 unknown :-1 s=CLOSED pgs=210 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:59.805 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:59.804+0000 7f05f51c4640 1 --2- 172.21.15.99:0/1250711503 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f05dc0a2800 0x7f05dc0a4cc0 unknown :-1 s=CLOSED pgs=166 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:59.805 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:59.804+0000 7f05f51c4640 1 --2- 172.21.15.99:0/1250711503 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f05f0100720 0x7f05f0197b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:59.805 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:59.804+0000 7f05f51c4640 1 -- 172.21.15.99:0/1250711503 >> 172.21.15.99:0/1250711503 conn(0x7f05f00fbcb0 msgr2=0x7f05f00fd760 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:59.805 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:59.804+0000 7f05f51c4640 1 -- 172.21.15.99:0/1250711503 shutdown_connections 2024-03-19T01:38:59.806 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:59.804+0000 7f05f51c4640 1 -- 172.21.15.99:0/1250711503 wait complete. 2024-03-19T01:38:59.819 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs session_timeout 300 2024-03-19T01:38:59.927 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:59.925+0000 7fd2458de640 1 -- 172.21.15.99:0/983668019 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd240102350 msgr2=0x7fd240104740 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:59.927 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:59.925+0000 7fd2458de640 1 --2- 172.21.15.99:0/983668019 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd240102350 0x7fd240104740 secure :-1 s=READY pgs=211 cs=0 l=1 rev1=1 crypto rx=0x7fd230009900 tx=0x7fd23002bc30 comp rx=0 tx=0).stop 2024-03-19T01:38:59.928 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:59.925+0000 7fd2458de640 1 -- 172.21.15.99:0/983668019 shutdown_connections 2024-03-19T01:38:59.928 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:59.925+0000 7fd2458de640 1 --2- 172.21.15.99:0/983668019 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd2401075b0 0x7fd2401099a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:59.928 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:59.925+0000 7fd2458de640 1 --2- 172.21.15.99:0/983668019 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd240104c80 0x7fd240107070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:59.928 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:59.925+0000 7fd2458de640 1 --2- 172.21.15.99:0/983668019 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd240102350 0x7fd240104740 unknown :-1 s=CLOSED pgs=211 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:59.928 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:59.925+0000 7fd2458de640 1 -- 172.21.15.99:0/983668019 >> 172.21.15.99:0/983668019 conn(0x7fd2400fbcc0 msgr2=0x7fd2400fe0e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:38:59.928 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:59.925+0000 7fd2458de640 1 -- 172.21.15.99:0/983668019 shutdown_connections 2024-03-19T01:38:59.928 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:59.925+0000 7fd2458de640 1 -- 172.21.15.99:0/983668019 wait complete. 2024-03-19T01:38:59.928 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:59.925+0000 7fd2458de640 1 Processor -- start 2024-03-19T01:38:59.928 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:59.926+0000 7fd2458de640 1 -- start start 2024-03-19T01:38:59.928 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:59.926+0000 7fd2458de640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd240102350 0x7fd240101cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:59.928 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:59.926+0000 7fd2458de640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd240104c80 0x7fd240100300 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:59.928 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:59.926+0000 7fd2458de640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd2401075b0 0x7fd240100840 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:59.928 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:59.926+0000 7fd2458de640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fd24010d4d0 con 0x7fd2401075b0 2024-03-19T01:38:59.928 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:59.926+0000 7fd2458de640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fd24010d350 con 0x7fd240104c80 2024-03-19T01:38:59.929 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:59.926+0000 7fd2458de640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fd24010d650 con 0x7fd240102350 2024-03-19T01:38:59.929 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:59.926+0000 7fd23f7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd2401075b0 0x7fd240100840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:59.929 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:59.926+0000 7fd23f7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd2401075b0 0x7fd240100840 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58508/0 (socket says 172.21.15.99:58508) 2024-03-19T01:38:59.929 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:59.926+0000 7fd23f7fe640 1 -- 172.21.15.99:0/1971893420 learned_addr learned my addr 172.21.15.99:0/1971893420 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:38:59.929 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:59.926+0000 7fd23effd640 1 --2- 172.21.15.99:0/1971893420 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd240102350 0x7fd240101cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:59.929 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:59.926+0000 7fd23e7fc640 1 --2- 172.21.15.99:0/1971893420 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd240104c80 0x7fd240100300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:59.929 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:59.926+0000 7fd23f7fe640 1 -- 172.21.15.99:0/1971893420 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd240102350 msgr2=0x7fd240101cc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:59.929 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:59.926+0000 7fd23f7fe640 1 --2- 172.21.15.99:0/1971893420 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd240102350 0x7fd240101cc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:59.929 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:59.926+0000 7fd23f7fe640 1 -- 172.21.15.99:0/1971893420 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd240104c80 msgr2=0x7fd240100300 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:38:59.929 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:59.926+0000 7fd23f7fe640 1 --2- 172.21.15.99:0/1971893420 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd240104c80 0x7fd240100300 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:38:59.929 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:59.926+0000 7fd23f7fe640 1 -- 172.21.15.99:0/1971893420 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd2401010d0 con 0x7fd2401075b0 2024-03-19T01:38:59.929 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:59.926+0000 7fd23f7fe640 1 --2- 172.21.15.99:0/1971893420 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd2401075b0 0x7fd240100840 secure :-1 s=READY pgs=212 cs=0 l=1 rev1=1 crypto rx=0x7fd23800eb10 tx=0x7fd23800c2d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:59.929 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:59.927+0000 7fd2448dc640 1 -- 172.21.15.99:0/1971893420 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd238004b20 con 0x7fd2401075b0 2024-03-19T01:38:59.930 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:59.927+0000 7fd2458de640 1 -- 172.21.15.99:0/1971893420 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd2401a1d00 con 0x7fd2401075b0 2024-03-19T01:38:59.930 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:59.927+0000 7fd2448dc640 1 -- 172.21.15.99:0/1971893420 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd238004cc0 con 0x7fd2401075b0 2024-03-19T01:38:59.930 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:59.927+0000 7fd2448dc640 1 -- 172.21.15.99:0/1971893420 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd238010760 con 0x7fd2401075b0 2024-03-19T01:38:59.930 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:59.927+0000 7fd2458de640 1 -- 172.21.15.99:0/1971893420 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fd2401a21a0 con 0x7fd2401075b0 2024-03-19T01:38:59.931 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:59.928+0000 7fd2458de640 1 -- 172.21.15.99:0/1971893420 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fd2400fd7b0 con 0x7fd2401075b0 2024-03-19T01:38:59.931 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:59.929+0000 7fd2448dc640 1 -- 172.21.15.99:0/1971893420 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 16) ==== 137290+0+0 (secure 0 0 0) 0x7fd238010900 con 0x7fd2401075b0 2024-03-19T01:38:59.931 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:59.929+0000 7fd2448dc640 1 --2- 172.21.15.99:0/1971893420 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd2280a2800 0x7fd2280a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:38:59.931 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:59.929+0000 7fd23effd640 1 --2- 172.21.15.99:0/1971893420 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd2280a2800 0x7fd2280a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:38:59.931 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:59.929+0000 7fd2448dc640 1 -- 172.21.15.99:0/1971893420 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(87..87 src has 1..87) ==== 7979+0+0 (secure 0 0 0) 0x7fd2380cf0d0 con 0x7fd2401075b0 2024-03-19T01:38:59.931 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:59.930+0000 7fd23effd640 1 --2- 172.21.15.99:0/1971893420 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd2280a2800 0x7fd2280a4cc0 secure :-1 s=READY pgs=167 cs=0 l=1 rev1=1 crypto rx=0x7fd23002c7b0 tx=0x7fd230005fd0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:38:59.933 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:38:59.933+0000 7fd2448dc640 1 -- 172.21.15.99:0/1971893420 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7fd238053740 con 0x7fd2401075b0 2024-03-19T01:39:00.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:59 smithi115 ceph-mon[67332]: pgmap v283: 97 pgs: 50 active+clean, 24 creating+peering, 23 unknown; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail; 511 B/s wr, 7 op/s 2024-03-19T01:39:00.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:59 smithi115 ceph-mon[67332]: Health check cleared: POOL_APP_NOT_ENABLED (was: 2 pool(s) do not have an application enabled) 2024-03-19T01:39:00.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:59 smithi115 ceph-mon[67332]: Cluster is now healthy 2024-03-19T01:39:00.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:59 smithi115 ceph-mon[67332]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-03-19T01:39:00.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:59 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/553489765' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "max_mds", "val": "3"}]': finished 2024-03-19T01:39:00.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:59 smithi115 ceph-mon[67332]: daemon mds.b assigned to filesystem cephfs as rank 1 (now has 2 ranks) 2024-03-19T01:39:00.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:59 smithi115 ceph-mon[67332]: fsmap cephfs:1 {0=i=up:active} 11 up:standby 2024-03-19T01:39:00.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:59 smithi115 ceph-mon[67332]: fsmap cephfs:2 {0=i=up:active,1=b=up:creating} 10 up:standby 2024-03-19T01:39:00.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:59 smithi115 ceph-mon[67332]: daemon mds.b is now active in filesystem cephfs as rank 1 2024-03-19T01:39:00.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:38:59 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/1250711503' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "true"}]: dispatch 2024-03-19T01:39:00.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:59 smithi099 ceph-mon[62951]: pgmap v283: 97 pgs: 50 active+clean, 24 creating+peering, 23 unknown; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail; 511 B/s wr, 7 op/s 2024-03-19T01:39:00.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:59 smithi099 ceph-mon[62951]: Health check cleared: POOL_APP_NOT_ENABLED (was: 2 pool(s) do not have an application enabled) 2024-03-19T01:39:00.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:59 smithi099 ceph-mon[62951]: Cluster is now healthy 2024-03-19T01:39:00.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:59 smithi099 ceph-mon[62951]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-03-19T01:39:00.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:59 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/553489765' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "max_mds", "val": "3"}]': finished 2024-03-19T01:39:00.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:59 smithi099 ceph-mon[62951]: daemon mds.b assigned to filesystem cephfs as rank 1 (now has 2 ranks) 2024-03-19T01:39:00.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:59 smithi099 ceph-mon[62951]: fsmap cephfs:1 {0=i=up:active} 11 up:standby 2024-03-19T01:39:00.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:59 smithi099 ceph-mon[62951]: fsmap cephfs:2 {0=i=up:active,1=b=up:creating} 10 up:standby 2024-03-19T01:39:00.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:59 smithi099 ceph-mon[62951]: daemon mds.b is now active in filesystem cephfs as rank 1 2024-03-19T01:39:00.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:38:59 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/1250711503' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "true"}]: dispatch 2024-03-19T01:39:00.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:59 smithi112 ceph-mon[66819]: pgmap v283: 97 pgs: 50 active+clean, 24 creating+peering, 23 unknown; 577 KiB data, 365 MiB used, 1.0 TiB / 1.0 TiB avail; 511 B/s wr, 7 op/s 2024-03-19T01:39:00.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:59 smithi112 ceph-mon[66819]: Health check cleared: POOL_APP_NOT_ENABLED (was: 2 pool(s) do not have an application enabled) 2024-03-19T01:39:00.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:59 smithi112 ceph-mon[66819]: Cluster is now healthy 2024-03-19T01:39:00.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:59 smithi112 ceph-mon[66819]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-03-19T01:39:00.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:59 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/553489765' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "max_mds", "val": "3"}]': finished 2024-03-19T01:39:00.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:59 smithi112 ceph-mon[66819]: daemon mds.b assigned to filesystem cephfs as rank 1 (now has 2 ranks) 2024-03-19T01:39:00.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:59 smithi112 ceph-mon[66819]: fsmap cephfs:1 {0=i=up:active} 11 up:standby 2024-03-19T01:39:00.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:59 smithi112 ceph-mon[66819]: fsmap cephfs:2 {0=i=up:active,1=b=up:creating} 10 up:standby 2024-03-19T01:39:00.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:59 smithi112 ceph-mon[66819]: daemon mds.b is now active in filesystem cephfs as rank 1 2024-03-19T01:39:00.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:38:59 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/1250711503' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "true"}]: dispatch 2024-03-19T01:39:00.147 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.147+0000 7fd2458de640 1 -- 172.21.15.99:0/1971893420 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "session_timeout", "val": "300"} v 0) -- 0x7fd24002d1f0 con 0x7fd2401075b0 2024-03-19T01:39:00.324 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.324+0000 7fd2448dc640 1 -- 172.21.15.99:0/1971893420 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "session_timeout", "val": "300"}]=0 v20) ==== 115+0+0 (secure 0 0 0) 0x7fd23809c2c0 con 0x7fd2401075b0 2024-03-19T01:39:00.326 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.326+0000 7fd2458de640 1 -- 172.21.15.99:0/1971893420 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd2280a2800 msgr2=0x7fd2280a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:00.326 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.326+0000 7fd2458de640 1 --2- 172.21.15.99:0/1971893420 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd2280a2800 0x7fd2280a4cc0 secure :-1 s=READY pgs=167 cs=0 l=1 rev1=1 crypto rx=0x7fd23002c7b0 tx=0x7fd230005fd0 comp rx=0 tx=0).stop 2024-03-19T01:39:00.326 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.326+0000 7fd2458de640 1 -- 172.21.15.99:0/1971893420 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd2401075b0 msgr2=0x7fd240100840 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:00.326 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.326+0000 7fd2458de640 1 --2- 172.21.15.99:0/1971893420 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd2401075b0 0x7fd240100840 secure :-1 s=READY pgs=212 cs=0 l=1 rev1=1 crypto rx=0x7fd23800eb10 tx=0x7fd23800c2d0 comp rx=0 tx=0).stop 2024-03-19T01:39:00.327 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.326+0000 7fd2458de640 1 -- 172.21.15.99:0/1971893420 shutdown_connections 2024-03-19T01:39:00.327 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.326+0000 7fd2458de640 1 --2- 172.21.15.99:0/1971893420 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd2280a2800 0x7fd2280a4cc0 unknown :-1 s=CLOSED pgs=167 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:00.327 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.326+0000 7fd2458de640 1 --2- 172.21.15.99:0/1971893420 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd240104c80 0x7fd240100300 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:00.327 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.326+0000 7fd2458de640 1 --2- 172.21.15.99:0/1971893420 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd2401075b0 0x7fd240100840 unknown :-1 s=CLOSED pgs=212 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:00.327 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.326+0000 7fd2458de640 1 --2- 172.21.15.99:0/1971893420 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd240102350 0x7fd240101cc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:00.327 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.326+0000 7fd2458de640 1 -- 172.21.15.99:0/1971893420 >> 172.21.15.99:0/1971893420 conn(0x7fd2400fbcc0 msgr2=0x7fd240102e10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:00.327 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.326+0000 7fd2458de640 1 -- 172.21.15.99:0/1971893420 shutdown_connections 2024-03-19T01:39:00.327 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.326+0000 7fd2458de640 1 -- 172.21.15.99:0/1971893420 wait complete. 2024-03-19T01:39:00.338 DEBUG:tasks.cephfs.filesystem:Creating {'create': 2} subvols for filesystem 'cephfs' 2024-03-19T01:39:00.338 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs subvolumegroup create cephfs qa 2024-03-19T01:39:00.444 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.442+0000 7f8a07e19640 1 -- 172.21.15.99:0/582375334 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8a001000b0 msgr2=0x7f8a001024e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:00.445 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.442+0000 7f8a07e19640 1 --2- 172.21.15.99:0/582375334 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8a001000b0 0x7f8a001024e0 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 crypto rx=0x7f89fc009860 tx=0x7f89fc02bb70 comp rx=0 tx=0).stop 2024-03-19T01:39:00.445 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.442+0000 7f8a07e19640 1 -- 172.21.15.99:0/582375334 shutdown_connections 2024-03-19T01:39:00.445 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.442+0000 7f8a07e19640 1 --2- 172.21.15.99:0/582375334 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8a001053d0 0x7f8a00105850 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:00.445 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.442+0000 7f8a07e19640 1 --2- 172.21.15.99:0/582375334 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8a00102a20 0x7f8a00104e90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:00.445 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.442+0000 7f8a07e19640 1 --2- 172.21.15.99:0/582375334 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8a001000b0 0x7f8a001024e0 unknown :-1 s=CLOSED pgs=54 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:00.445 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.442+0000 7f8a07e19640 1 -- 172.21.15.99:0/582375334 >> 172.21.15.99:0/582375334 conn(0x7f8a000fbcd0 msgr2=0x7f8a000fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:00.445 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.442+0000 7f8a07e19640 1 -- 172.21.15.99:0/582375334 shutdown_connections 2024-03-19T01:39:00.445 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.442+0000 7f8a07e19640 1 -- 172.21.15.99:0/582375334 wait complete. 2024-03-19T01:39:00.445 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.443+0000 7f8a07e19640 1 Processor -- start 2024-03-19T01:39:00.445 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.443+0000 7f8a07e19640 1 -- start start 2024-03-19T01:39:00.445 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.443+0000 7f8a07e19640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8a001000b0 0x7f8a0019e180 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:00.446 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.443+0000 7f8a07e19640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8a00102a20 0x7f8a0019e6c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:00.446 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.443+0000 7f8a07e19640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8a001053d0 0x7f8a001a5730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:00.446 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.443+0000 7f8a07e19640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f8a00113ab0 con 0x7f8a001053d0 2024-03-19T01:39:00.446 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.443+0000 7f8a07e19640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f8a00113930 con 0x7f8a001000b0 2024-03-19T01:39:00.446 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.443+0000 7f8a07e19640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f8a00113c30 con 0x7f8a00102a20 2024-03-19T01:39:00.446 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.443+0000 7f8a063b6640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8a001053d0 0x7f8a001a5730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:00.446 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.443+0000 7f8a063b6640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8a001053d0 0x7f8a001a5730 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58528/0 (socket says 172.21.15.99:58528) 2024-03-19T01:39:00.446 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.443+0000 7f8a053b4640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8a00102a20 0x7f8a0019e6c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:00.446 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.443+0000 7f8a063b6640 1 -- 172.21.15.99:0/1798763529 learned_addr learned my addr 172.21.15.99:0/1798763529 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:39:00.446 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.443+0000 7f8a05bb5640 1 --2- 172.21.15.99:0/1798763529 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8a001000b0 0x7f8a0019e180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:00.446 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.443+0000 7f8a063b6640 1 -- 172.21.15.99:0/1798763529 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8a00102a20 msgr2=0x7f8a0019e6c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:00.446 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.443+0000 7f8a063b6640 1 --2- 172.21.15.99:0/1798763529 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8a00102a20 0x7f8a0019e6c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:00.446 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.443+0000 7f8a063b6640 1 -- 172.21.15.99:0/1798763529 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8a001000b0 msgr2=0x7f8a0019e180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:00.447 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.443+0000 7f8a063b6640 1 --2- 172.21.15.99:0/1798763529 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8a001000b0 0x7f8a0019e180 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:00.447 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.443+0000 7f8a063b6640 1 -- 172.21.15.99:0/1798763529 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f8a001a5da0 con 0x7f8a001053d0 2024-03-19T01:39:00.447 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.444+0000 7f8a063b6640 1 --2- 172.21.15.99:0/1798763529 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8a001053d0 0x7f8a001a5730 secure :-1 s=READY pgs=213 cs=0 l=1 rev1=1 crypto rx=0x7f89f800eb10 tx=0x7f89f800c2d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:00.447 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.444+0000 7f89f6ffd640 1 -- 172.21.15.99:0/1798763529 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f89f8004b20 con 0x7f8a001053d0 2024-03-19T01:39:00.447 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.444+0000 7f8a07e19640 1 -- 172.21.15.99:0/1798763529 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f8a001a6090 con 0x7f8a001053d0 2024-03-19T01:39:00.447 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.444+0000 7f89f6ffd640 1 -- 172.21.15.99:0/1798763529 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f89f8004cc0 con 0x7f8a001053d0 2024-03-19T01:39:00.447 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.444+0000 7f8a07e19640 1 -- 172.21.15.99:0/1798763529 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f8a001a6630 con 0x7f8a001053d0 2024-03-19T01:39:00.447 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.444+0000 7f89f6ffd640 1 -- 172.21.15.99:0/1798763529 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f89f8010760 con 0x7f8a001053d0 2024-03-19T01:39:00.448 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.445+0000 7f8a07e19640 1 -- 172.21.15.99:0/1798763529 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f8a000fd7c0 con 0x7f8a001053d0 2024-03-19T01:39:00.448 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.446+0000 7f89f6ffd640 1 -- 172.21.15.99:0/1798763529 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 16) ==== 137290+0+0 (secure 0 0 0) 0x7f89f8005000 con 0x7f8a001053d0 2024-03-19T01:39:00.448 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.446+0000 7f89f6ffd640 1 --2- 172.21.15.99:0/1798763529 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f89ec0a2800 0x7f89ec0a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:00.448 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.447+0000 7f8a05bb5640 1 --2- 172.21.15.99:0/1798763529 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f89ec0a2800 0x7f89ec0a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:00.448 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.447+0000 7f89f6ffd640 1 -- 172.21.15.99:0/1798763529 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(87..87 src has 1..87) ==== 7979+0+0 (secure 0 0 0) 0x7f89f80cf690 con 0x7f8a001053d0 2024-03-19T01:39:00.448 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.447+0000 7f8a05bb5640 1 --2- 172.21.15.99:0/1798763529 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f89ec0a2800 0x7f89ec0a4cc0 secure :-1 s=READY pgs=168 cs=0 l=1 rev1=1 crypto rx=0x7f89fc02c180 tx=0x7f89fc034ad0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:00.451 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.450+0000 7f89f6ffd640 1 -- 172.21.15.99:0/1798763529 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f89f809c7e0 con 0x7f8a001053d0 2024-03-19T01:39:00.653 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.653+0000 7f8a07e19640 1 -- 172.21.15.99:0/1798763529 --> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] -- mgr_command(tid 0: {"prefix": "fs subvolumegroup create", "vol_name": "cephfs", "group_name": "qa", "target": ["mon-mgr", ""]}) -- 0x7f8a00061c60 con 0x7f89ec0a2800 2024-03-19T01:39:00.674 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.674+0000 7f89f6ffd640 1 -- 172.21.15.99:0/1798763529 <== mgr.14150 v2:172.21.15.99:6800/4272410295 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+0 (secure 0 0 0) 0x7f8a00061c60 con 0x7f89ec0a2800 2024-03-19T01:39:00.677 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.676+0000 7f8a07e19640 1 -- 172.21.15.99:0/1798763529 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f89ec0a2800 msgr2=0x7f89ec0a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:00.677 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.676+0000 7f8a07e19640 1 --2- 172.21.15.99:0/1798763529 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f89ec0a2800 0x7f89ec0a4cc0 secure :-1 s=READY pgs=168 cs=0 l=1 rev1=1 crypto rx=0x7f89fc02c180 tx=0x7f89fc034ad0 comp rx=0 tx=0).stop 2024-03-19T01:39:00.677 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.676+0000 7f8a07e19640 1 -- 172.21.15.99:0/1798763529 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8a001053d0 msgr2=0x7f8a001a5730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:00.677 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.676+0000 7f8a07e19640 1 --2- 172.21.15.99:0/1798763529 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8a001053d0 0x7f8a001a5730 secure :-1 s=READY pgs=213 cs=0 l=1 rev1=1 crypto rx=0x7f89f800eb10 tx=0x7f89f800c2d0 comp rx=0 tx=0).stop 2024-03-19T01:39:00.677 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.676+0000 7f8a07e19640 1 -- 172.21.15.99:0/1798763529 shutdown_connections 2024-03-19T01:39:00.677 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.676+0000 7f8a07e19640 1 --2- 172.21.15.99:0/1798763529 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8a001053d0 0x7f8a001a5730 unknown :-1 s=CLOSED pgs=213 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:00.677 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.676+0000 7f8a07e19640 1 --2- 172.21.15.99:0/1798763529 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8a00102a20 0x7f8a0019e6c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:00.677 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.676+0000 7f8a07e19640 1 --2- 172.21.15.99:0/1798763529 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f89ec0a2800 0x7f89ec0a4cc0 unknown :-1 s=CLOSED pgs=168 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:00.677 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.676+0000 7f8a07e19640 1 --2- 172.21.15.99:0/1798763529 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8a001000b0 0x7f8a0019e180 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:00.677 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.676+0000 7f8a07e19640 1 -- 172.21.15.99:0/1798763529 >> 172.21.15.99:0/1798763529 conn(0x7f8a000fbcd0 msgr2=0x7f8a001009f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:00.678 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.676+0000 7f8a07e19640 1 -- 172.21.15.99:0/1798763529 shutdown_connections 2024-03-19T01:39:00.678 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.676+0000 7f8a07e19640 1 -- 172.21.15.99:0/1798763529 wait complete. 2024-03-19T01:39:00.688 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs subvolume create cephfs sv_0 --group_name qa 2024-03-19T01:39:00.797 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.793+0000 7f583015b640 1 -- 172.21.15.99:0/890734530 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5828102b90 msgr2=0x7f5828103010 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:00.797 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.793+0000 7f583015b640 1 --2- 172.21.15.99:0/890734530 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5828102b90 0x7f5828103010 secure :-1 s=READY pgs=216 cs=0 l=1 rev1=1 crypto rx=0x7f5818009930 tx=0x7f581802bc30 comp rx=0 tx=0).stop 2024-03-19T01:39:00.797 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.793+0000 7f583015b640 1 -- 172.21.15.99:0/890734530 shutdown_connections 2024-03-19T01:39:00.797 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.793+0000 7f583015b640 1 --2- 172.21.15.99:0/890734530 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5828103550 0x7f5828109de0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:00.797 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.793+0000 7f583015b640 1 --2- 172.21.15.99:0/890734530 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5828102b90 0x7f5828103010 unknown :-1 s=CLOSED pgs=216 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:00.797 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.793+0000 7f583015b640 1 --2- 172.21.15.99:0/890734530 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f58280ffb20 0x7f58280fff40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:00.797 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.793+0000 7f583015b640 1 -- 172.21.15.99:0/890734530 >> 172.21.15.99:0/890734530 conn(0x7f58280fb430 msgr2=0x7f58280fd870 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:00.797 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.793+0000 7f583015b640 1 -- 172.21.15.99:0/890734530 shutdown_connections 2024-03-19T01:39:00.797 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.793+0000 7f583015b640 1 -- 172.21.15.99:0/890734530 wait complete. 2024-03-19T01:39:00.798 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.794+0000 7f583015b640 1 Processor -- start 2024-03-19T01:39:00.798 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.794+0000 7f583015b640 1 -- start start 2024-03-19T01:39:00.798 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.794+0000 7f583015b640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f58280ffb20 0x7f5828197b50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:00.798 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.794+0000 7f583015b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5828102b90 0x7f5828198090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:00.798 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.794+0000 7f583015b640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5828103550 0x7f582819f100 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:00.798 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.794+0000 7f583015b640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f582810d4d0 con 0x7f5828102b90 2024-03-19T01:39:00.798 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.794+0000 7f583015b640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f582810d350 con 0x7f58280ffb20 2024-03-19T01:39:00.798 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.794+0000 7f583015b640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f582810d650 con 0x7f5828103550 2024-03-19T01:39:00.798 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.794+0000 7f582d6f6640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5828102b90 0x7f5828198090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:00.798 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.794+0000 7f582d6f6640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5828102b90 0x7f5828198090 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58560/0 (socket says 172.21.15.99:58560) 2024-03-19T01:39:00.798 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.794+0000 7f582d6f6640 1 -- 172.21.15.99:0/1616765018 learned_addr learned my addr 172.21.15.99:0/1616765018 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:39:00.798 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.794+0000 7f582def7640 1 --2- 172.21.15.99:0/1616765018 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f58280ffb20 0x7f5828197b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:00.798 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.794+0000 7f582e6f8640 1 --2- 172.21.15.99:0/1616765018 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5828103550 0x7f582819f100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:00.799 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.794+0000 7f582d6f6640 1 -- 172.21.15.99:0/1616765018 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5828103550 msgr2=0x7f582819f100 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:00.799 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.794+0000 7f582d6f6640 1 --2- 172.21.15.99:0/1616765018 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5828103550 0x7f582819f100 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:00.799 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.794+0000 7f582d6f6640 1 -- 172.21.15.99:0/1616765018 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f58280ffb20 msgr2=0x7f5828197b50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:00.799 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.794+0000 7f582d6f6640 1 --2- 172.21.15.99:0/1616765018 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f58280ffb20 0x7f5828197b50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:00.799 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.794+0000 7f582d6f6640 1 -- 172.21.15.99:0/1616765018 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f582819f770 con 0x7f5828102b90 2024-03-19T01:39:00.799 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.794+0000 7f582d6f6640 1 --2- 172.21.15.99:0/1616765018 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5828102b90 0x7f5828198090 secure :-1 s=READY pgs=217 cs=0 l=1 rev1=1 crypto rx=0x7f5818000c00 tx=0x7f581802c640 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:00.799 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.795+0000 7f581effd640 1 -- 172.21.15.99:0/1616765018 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5818041070 con 0x7f5828102b90 2024-03-19T01:39:00.799 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.795+0000 7f581effd640 1 -- 172.21.15.99:0/1616765018 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f5818004450 con 0x7f5828102b90 2024-03-19T01:39:00.799 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.795+0000 7f583015b640 1 -- 172.21.15.99:0/1616765018 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f582819fa00 con 0x7f5828102b90 2024-03-19T01:39:00.799 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.795+0000 7f581effd640 1 -- 172.21.15.99:0/1616765018 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5818034440 con 0x7f5828102b90 2024-03-19T01:39:00.799 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.795+0000 7f583015b640 1 -- 172.21.15.99:0/1616765018 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f582819ff40 con 0x7f5828102b90 2024-03-19T01:39:00.800 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.796+0000 7f583015b640 1 -- 172.21.15.99:0/1616765018 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f5828108cc0 con 0x7f5828102b90 2024-03-19T01:39:00.800 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.797+0000 7f581effd640 1 -- 172.21.15.99:0/1616765018 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 16) ==== 137290+0+0 (secure 0 0 0) 0x7f5818044070 con 0x7f5828102b90 2024-03-19T01:39:00.800 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.797+0000 7f581effd640 1 --2- 172.21.15.99:0/1616765018 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f58140a2880 0x7f58140a4d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:00.800 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.798+0000 7f582def7640 1 --2- 172.21.15.99:0/1616765018 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f58140a2880 0x7f58140a4d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:00.800 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.798+0000 7f581effd640 1 -- 172.21.15.99:0/1616765018 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(87..87 src has 1..87) ==== 7979+0+0 (secure 0 0 0) 0x7f58180f0340 con 0x7f5828102b90 2024-03-19T01:39:00.800 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.798+0000 7f582def7640 1 --2- 172.21.15.99:0/1616765018 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f58140a2880 0x7f58140a4d40 secure :-1 s=READY pgs=169 cs=0 l=1 rev1=1 crypto rx=0x7f5828069b50 tx=0x7f5824005f90 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:00.803 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:00.802+0000 7f581effd640 1 -- 172.21.15.99:0/1616765018 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f58180bcf50 con 0x7f5828102b90 2024-03-19T01:39:01.008 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.007+0000 7f583015b640 1 -- 172.21.15.99:0/1616765018 --> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] -- mgr_command(tid 0: {"prefix": "fs subvolume create", "vol_name": "cephfs", "sub_name": "sv_0", "group_name": "qa", "target": ["mon-mgr", ""]}) -- 0x7f5828061cf0 con 0x7f58140a2880 2024-03-19T01:39:01.028 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.026+0000 7f581effd640 1 -- 172.21.15.99:0/1616765018 <== mgr.14150 v2:172.21.15.99:6800/4272410295 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+0 (secure 0 0 0) 0x7f5828061cf0 con 0x7f58140a2880 2024-03-19T01:39:01.030 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.028+0000 7f583015b640 1 -- 172.21.15.99:0/1616765018 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f58140a2880 msgr2=0x7f58140a4d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:01.030 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.028+0000 7f583015b640 1 --2- 172.21.15.99:0/1616765018 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f58140a2880 0x7f58140a4d40 secure :-1 s=READY pgs=169 cs=0 l=1 rev1=1 crypto rx=0x7f5828069b50 tx=0x7f5824005f90 comp rx=0 tx=0).stop 2024-03-19T01:39:01.030 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.028+0000 7f583015b640 1 -- 172.21.15.99:0/1616765018 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5828102b90 msgr2=0x7f5828198090 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:01.030 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.028+0000 7f583015b640 1 --2- 172.21.15.99:0/1616765018 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5828102b90 0x7f5828198090 secure :-1 s=READY pgs=217 cs=0 l=1 rev1=1 crypto rx=0x7f5818000c00 tx=0x7f581802c640 comp rx=0 tx=0).stop 2024-03-19T01:39:01.030 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.029+0000 7f583015b640 1 -- 172.21.15.99:0/1616765018 shutdown_connections 2024-03-19T01:39:01.031 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.029+0000 7f583015b640 1 --2- 172.21.15.99:0/1616765018 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5828103550 0x7f582819f100 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:01.031 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.029+0000 7f583015b640 1 --2- 172.21.15.99:0/1616765018 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5828102b90 0x7f5828198090 unknown :-1 s=CLOSED pgs=217 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:01.031 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.029+0000 7f583015b640 1 --2- 172.21.15.99:0/1616765018 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f58140a2880 0x7f58140a4d40 unknown :-1 s=CLOSED pgs=169 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:01.031 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.029+0000 7f583015b640 1 --2- 172.21.15.99:0/1616765018 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f58280ffb20 0x7f5828197b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:01.031 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.029+0000 7f583015b640 1 -- 172.21.15.99:0/1616765018 >> 172.21.15.99:0/1616765018 conn(0x7f58280fb430 msgr2=0x7f5828105e90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:01.031 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.029+0000 7f583015b640 1 -- 172.21.15.99:0/1616765018 shutdown_connections 2024-03-19T01:39:01.031 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.029+0000 7f583015b640 1 -- 172.21.15.99:0/1616765018 wait complete. 2024-03-19T01:39:01.040 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:39:00 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:39:00.658+0000 7f67c3a45700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-19T01:39:01.040 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:39:00 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:39:00.658+0000 7f67c3a45700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-19T01:39:01.040 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:39:00 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:39:00.658+0000 7f67c3a45700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-19T01:39:01.040 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:39:00 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:39:00.658+0000 7f67c3a45700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-19T01:39:01.040 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:39:00 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:39:00.658+0000 7f67c3a45700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-19T01:39:01.041 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:00 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/1250711503' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "true"}]': finished 2024-03-19T01:39:01.041 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:00 smithi099 ceph-mon[62951]: mds.? [v2:172.21.15.112:6838/986249680,v1:172.21.15.112:6839/986249680] up:active 2024-03-19T01:39:01.041 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:00 smithi099 ceph-mon[62951]: daemon mds.c assigned to filesystem cephfs as rank 2 (now has 3 ranks) 2024-03-19T01:39:01.041 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:00 smithi099 ceph-mon[62951]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-03-19T01:39:01.041 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:00 smithi099 ceph-mon[62951]: Cluster is now healthy 2024-03-19T01:39:01.041 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:00 smithi099 ceph-mon[62951]: fsmap cephfs:2 {0=i=up:active,1=b=up:active} 10 up:standby 2024-03-19T01:39:01.041 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:00 smithi099 ceph-mon[62951]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:creating} 9 up:standby 2024-03-19T01:39:01.041 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:00 smithi099 ceph-mon[62951]: daemon mds.c is now active in filesystem cephfs as rank 2 2024-03-19T01:39:01.041 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:00 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/1971893420' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "session_timeout", "val": "300"}]: dispatch 2024-03-19T01:39:01.041 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:00 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/1971893420' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "session_timeout", "val": "300"}]': finished 2024-03-19T01:39:01.041 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:00 smithi099 ceph-mon[62951]: mds.? [v2:172.21.15.115:6836/1368988191,v1:172.21.15.115:6837/1368988191] up:active 2024-03-19T01:39:01.041 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:00 smithi099 ceph-mon[62951]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T01:39:01.042 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs subvolume getpath cephfs --group_name qa sv_0 2024-03-19T01:39:01.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:00 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/1250711503' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "true"}]': finished 2024-03-19T01:39:01.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:00 smithi115 ceph-mon[67332]: mds.? [v2:172.21.15.112:6838/986249680,v1:172.21.15.112:6839/986249680] up:active 2024-03-19T01:39:01.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:00 smithi115 ceph-mon[67332]: daemon mds.c assigned to filesystem cephfs as rank 2 (now has 3 ranks) 2024-03-19T01:39:01.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:00 smithi115 ceph-mon[67332]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-03-19T01:39:01.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:00 smithi115 ceph-mon[67332]: Cluster is now healthy 2024-03-19T01:39:01.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:00 smithi115 ceph-mon[67332]: fsmap cephfs:2 {0=i=up:active,1=b=up:active} 10 up:standby 2024-03-19T01:39:01.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:00 smithi115 ceph-mon[67332]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:creating} 9 up:standby 2024-03-19T01:39:01.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:00 smithi115 ceph-mon[67332]: daemon mds.c is now active in filesystem cephfs as rank 2 2024-03-19T01:39:01.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:00 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/1971893420' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "session_timeout", "val": "300"}]: dispatch 2024-03-19T01:39:01.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:00 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/1971893420' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "session_timeout", "val": "300"}]': finished 2024-03-19T01:39:01.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:00 smithi115 ceph-mon[67332]: mds.? [v2:172.21.15.115:6836/1368988191,v1:172.21.15.115:6837/1368988191] up:active 2024-03-19T01:39:01.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:00 smithi115 ceph-mon[67332]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T01:39:01.059 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:00 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/1250711503' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "true"}]': finished 2024-03-19T01:39:01.059 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:00 smithi112 ceph-mon[66819]: mds.? [v2:172.21.15.112:6838/986249680,v1:172.21.15.112:6839/986249680] up:active 2024-03-19T01:39:01.059 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:00 smithi112 ceph-mon[66819]: daemon mds.c assigned to filesystem cephfs as rank 2 (now has 3 ranks) 2024-03-19T01:39:01.059 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:00 smithi112 ceph-mon[66819]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-03-19T01:39:01.059 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:00 smithi112 ceph-mon[66819]: Cluster is now healthy 2024-03-19T01:39:01.060 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:00 smithi112 ceph-mon[66819]: fsmap cephfs:2 {0=i=up:active,1=b=up:active} 10 up:standby 2024-03-19T01:39:01.060 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:00 smithi112 ceph-mon[66819]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:creating} 9 up:standby 2024-03-19T01:39:01.060 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:00 smithi112 ceph-mon[66819]: daemon mds.c is now active in filesystem cephfs as rank 2 2024-03-19T01:39:01.060 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:00 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/1971893420' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "session_timeout", "val": "300"}]: dispatch 2024-03-19T01:39:01.060 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:00 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/1971893420' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "session_timeout", "val": "300"}]': finished 2024-03-19T01:39:01.060 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:00 smithi112 ceph-mon[66819]: mds.? [v2:172.21.15.115:6836/1368988191,v1:172.21.15.115:6837/1368988191] up:active 2024-03-19T01:39:01.060 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:00 smithi112 ceph-mon[66819]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T01:39:01.152 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.147+0000 7fa35b0e8640 1 -- 172.21.15.99:0/1904633438 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa3541033e0 msgr2=0x7fa354109c70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:01.152 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.148+0000 7fa35b0e8640 1 --2- 172.21.15.99:0/1904633438 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa3541033e0 0x7fa354109c70 secure :-1 s=READY pgs=218 cs=0 l=1 rev1=1 crypto rx=0x7fa34c009ca0 tx=0x7fa34c02c0f0 comp rx=0 tx=0).stop 2024-03-19T01:39:01.152 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.148+0000 7fa35b0e8640 1 -- 172.21.15.99:0/1904633438 shutdown_connections 2024-03-19T01:39:01.152 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.148+0000 7fa35b0e8640 1 --2- 172.21.15.99:0/1904633438 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa3541033e0 0x7fa354109c70 unknown :-1 s=CLOSED pgs=218 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:01.152 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.148+0000 7fa35b0e8640 1 --2- 172.21.15.99:0/1904633438 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa354102a20 0x7fa354102ea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:01.152 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.148+0000 7fa35b0e8640 1 --2- 172.21.15.99:0/1904633438 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa354101830 0x7fa354101c50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:01.153 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.148+0000 7fa35b0e8640 1 -- 172.21.15.99:0/1904633438 >> 172.21.15.99:0/1904633438 conn(0x7fa3540fcdc0 msgr2=0x7fa3540ff1e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:01.153 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.148+0000 7fa35b0e8640 1 -- 172.21.15.99:0/1904633438 shutdown_connections 2024-03-19T01:39:01.153 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.148+0000 7fa35b0e8640 1 -- 172.21.15.99:0/1904633438 wait complete. 2024-03-19T01:39:01.153 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.148+0000 7fa35b0e8640 1 Processor -- start 2024-03-19T01:39:01.153 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.148+0000 7fa35b0e8640 1 -- start start 2024-03-19T01:39:01.153 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.148+0000 7fa35b0e8640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa354101830 0x7fa354197af0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:01.153 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.148+0000 7fa35b0e8640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa354102a20 0x7fa354198030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:01.153 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.148+0000 7fa35b0e8640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa3541033e0 0x7fa35419f0a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:01.153 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.148+0000 7fa35b0e8640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fa35410d4b0 con 0x7fa354102a20 2024-03-19T01:39:01.153 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.148+0000 7fa35b0e8640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fa35410d330 con 0x7fa3541033e0 2024-03-19T01:39:01.153 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.148+0000 7fa35b0e8640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fa35410d630 con 0x7fa354101830 2024-03-19T01:39:01.153 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.148+0000 7fa353fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa354102a20 0x7fa354198030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:01.153 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.148+0000 7fa353fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa354102a20 0x7fa354198030 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58578/0 (socket says 172.21.15.99:58578) 2024-03-19T01:39:01.154 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.148+0000 7fa353fff640 1 -- 172.21.15.99:0/2302560315 learned_addr learned my addr 172.21.15.99:0/2302560315 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:39:01.154 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.148+0000 7fa358e84640 1 --2- 172.21.15.99:0/2302560315 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa354101830 0x7fa354197af0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:01.154 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.148+0000 7fa359685640 1 --2- 172.21.15.99:0/2302560315 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa3541033e0 0x7fa35419f0a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:01.154 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.149+0000 7fa353fff640 1 -- 172.21.15.99:0/2302560315 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa354101830 msgr2=0x7fa354197af0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:01.154 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.149+0000 7fa353fff640 1 --2- 172.21.15.99:0/2302560315 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa354101830 0x7fa354197af0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:01.154 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.149+0000 7fa353fff640 1 -- 172.21.15.99:0/2302560315 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa3541033e0 msgr2=0x7fa35419f0a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:01.154 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.149+0000 7fa353fff640 1 --2- 172.21.15.99:0/2302560315 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa3541033e0 0x7fa35419f0a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:01.154 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.149+0000 7fa353fff640 1 -- 172.21.15.99:0/2302560315 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa35419f710 con 0x7fa354102a20 2024-03-19T01:39:01.154 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.149+0000 7fa353fff640 1 --2- 172.21.15.99:0/2302560315 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa354102a20 0x7fa354198030 secure :-1 s=READY pgs=219 cs=0 l=1 rev1=1 crypto rx=0x7fa34000e9f0 tx=0x7fa34000edf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:01.154 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.149+0000 7fa351ffb640 1 -- 172.21.15.99:0/2302560315 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa34000cd80 con 0x7fa354102a20 2024-03-19T01:39:01.154 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.149+0000 7fa351ffb640 1 -- 172.21.15.99:0/2302560315 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa340005500 con 0x7fa354102a20 2024-03-19T01:39:01.154 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.149+0000 7fa35b0e8640 1 -- 172.21.15.99:0/2302560315 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa35419fa00 con 0x7fa354102a20 2024-03-19T01:39:01.154 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.149+0000 7fa351ffb640 1 -- 172.21.15.99:0/2302560315 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa340010690 con 0x7fa354102a20 2024-03-19T01:39:01.155 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.149+0000 7fa35b0e8640 1 -- 172.21.15.99:0/2302560315 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fa35419ff00 con 0x7fa354102a20 2024-03-19T01:39:01.155 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.150+0000 7fa35b0e8640 1 -- 172.21.15.99:0/2302560315 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fa354108b50 con 0x7fa354102a20 2024-03-19T01:39:01.155 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.151+0000 7fa351ffb640 1 -- 172.21.15.99:0/2302560315 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 16) ==== 137290+0+0 (secure 0 0 0) 0x7fa3400046c0 con 0x7fa354102a20 2024-03-19T01:39:01.155 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.152+0000 7fa351ffb640 1 --2- 172.21.15.99:0/2302560315 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa3440a2800 0x7fa3440a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:01.156 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.152+0000 7fa358e84640 1 --2- 172.21.15.99:0/2302560315 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa3440a2800 0x7fa3440a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:01.156 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.152+0000 7fa351ffb640 1 -- 172.21.15.99:0/2302560315 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(87..87 src has 1..87) ==== 7979+0+0 (secure 0 0 0) 0x7fa3400cecf0 con 0x7fa354102a20 2024-03-19T01:39:01.156 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.152+0000 7fa358e84640 1 --2- 172.21.15.99:0/2302560315 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa3440a2800 0x7fa3440a4cc0 secure :-1 s=READY pgs=170 cs=0 l=1 rev1=1 crypto rx=0x7fa354102880 tx=0x7fa348005f90 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:01.157 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.156+0000 7fa351ffb640 1 -- 172.21.15.99:0/2302560315 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7fa3400d3070 con 0x7fa354102a20 2024-03-19T01:39:01.365 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.363+0000 7fa35b0e8640 1 -- 172.21.15.99:0/2302560315 --> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] -- mgr_command(tid 0: {"prefix": "fs subvolume getpath", "vol_name": "cephfs", "group_name": "qa", "sub_name": "sv_0", "target": ["mon-mgr", ""]}) -- 0x7fa354061c60 con 0x7fa3440a2800 2024-03-19T01:39:01.368 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.367+0000 7fa351ffb640 1 -- 172.21.15.99:0/2302560315 <== mgr.14150 v2:172.21.15.99:6800/4272410295 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+54 (secure 0 0 0) 0x7fa354061c60 con 0x7fa3440a2800 2024-03-19T01:39:01.369 INFO:teuthology.orchestra.run.smithi099.stdout:/volumes/qa/sv_0/fc72b5c6-4406-45fc-8e24-aba76e567c80 2024-03-19T01:39:01.371 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.369+0000 7fa35b0e8640 1 -- 172.21.15.99:0/2302560315 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa3440a2800 msgr2=0x7fa3440a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:01.371 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.369+0000 7fa35b0e8640 1 --2- 172.21.15.99:0/2302560315 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa3440a2800 0x7fa3440a4cc0 secure :-1 s=READY pgs=170 cs=0 l=1 rev1=1 crypto rx=0x7fa354102880 tx=0x7fa348005f90 comp rx=0 tx=0).stop 2024-03-19T01:39:01.371 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.369+0000 7fa35b0e8640 1 -- 172.21.15.99:0/2302560315 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa354102a20 msgr2=0x7fa354198030 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:01.371 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.369+0000 7fa35b0e8640 1 --2- 172.21.15.99:0/2302560315 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa354102a20 0x7fa354198030 secure :-1 s=READY pgs=219 cs=0 l=1 rev1=1 crypto rx=0x7fa34000e9f0 tx=0x7fa34000edf0 comp rx=0 tx=0).stop 2024-03-19T01:39:01.371 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.369+0000 7fa35b0e8640 1 -- 172.21.15.99:0/2302560315 shutdown_connections 2024-03-19T01:39:01.371 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.369+0000 7fa35b0e8640 1 --2- 172.21.15.99:0/2302560315 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa3440a2800 0x7fa3440a4cc0 unknown :-1 s=CLOSED pgs=170 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:01.371 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.369+0000 7fa35b0e8640 1 --2- 172.21.15.99:0/2302560315 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa3541033e0 0x7fa35419f0a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:01.371 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.369+0000 7fa35b0e8640 1 --2- 172.21.15.99:0/2302560315 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa354102a20 0x7fa354198030 unknown :-1 s=CLOSED pgs=219 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:01.372 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.369+0000 7fa35b0e8640 1 --2- 172.21.15.99:0/2302560315 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa354101830 0x7fa354197af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:01.372 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.369+0000 7fa35b0e8640 1 -- 172.21.15.99:0/2302560315 >> 172.21.15.99:0/2302560315 conn(0x7fa3540fcdc0 msgr2=0x7fa354105d20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:01.372 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.369+0000 7fa35b0e8640 1 -- 172.21.15.99:0/2302560315 shutdown_connections 2024-03-19T01:39:01.372 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.369+0000 7fa35b0e8640 1 -- 172.21.15.99:0/2302560315 wait complete. 2024-03-19T01:39:01.382 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs subvolume create cephfs sv_1 --group_name qa 2024-03-19T01:39:01.490 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.486+0000 7f85b517e640 1 -- 172.21.15.99:0/1038784652 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f85b01022d0 msgr2=0x7f85b0108b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:01.490 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.486+0000 7f85b517e640 1 --2- 172.21.15.99:0/1038784652 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f85b01022d0 0x7f85b0108b60 secure :-1 s=READY pgs=220 cs=0 l=1 rev1=1 crypto rx=0x7f85a8009ca0 tx=0x7f85a802c0f0 comp rx=0 tx=0).stop 2024-03-19T01:39:01.490 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.486+0000 7f85b517e640 1 -- 172.21.15.99:0/1038784652 shutdown_connections 2024-03-19T01:39:01.490 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.486+0000 7f85b517e640 1 --2- 172.21.15.99:0/1038784652 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f85b01022d0 0x7f85b0108b60 unknown :-1 s=CLOSED pgs=220 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:01.490 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.486+0000 7f85b517e640 1 --2- 172.21.15.99:0/1038784652 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f85b0101910 0x7f85b0101d90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:01.490 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.486+0000 7f85b517e640 1 --2- 172.21.15.99:0/1038784652 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f85b0100720 0x7f85b0100b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:01.490 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.486+0000 7f85b517e640 1 -- 172.21.15.99:0/1038784652 >> 172.21.15.99:0/1038784652 conn(0x7f85b00fbcb0 msgr2=0x7f85b00fe0d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:01.490 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.486+0000 7f85b517e640 1 -- 172.21.15.99:0/1038784652 shutdown_connections 2024-03-19T01:39:01.490 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.486+0000 7f85b517e640 1 -- 172.21.15.99:0/1038784652 wait complete. 2024-03-19T01:39:01.490 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.487+0000 7f85b517e640 1 Processor -- start 2024-03-19T01:39:01.490 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.487+0000 7f85b517e640 1 -- start start 2024-03-19T01:39:01.491 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.487+0000 7f85b517e640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f85b0100720 0x7f85b0196a20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:01.491 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.487+0000 7f85b517e640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f85b0101910 0x7f85b0196f60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:01.491 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.487+0000 7f85b517e640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f85b01022d0 0x7f85b019dfd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:01.491 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.487+0000 7f85b517e640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f85b010c380 con 0x7f85b0100720 2024-03-19T01:39:01.491 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.487+0000 7f85b517e640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f85b010c200 con 0x7f85b0101910 2024-03-19T01:39:01.491 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.487+0000 7f85b517e640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f85b010c500 con 0x7f85b01022d0 2024-03-19T01:39:01.491 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.487+0000 7f85aed9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f85b0100720 0x7f85b0196a20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:01.491 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.487+0000 7f85aed9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f85b0100720 0x7f85b0196a20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58604/0 (socket says 172.21.15.99:58604) 2024-03-19T01:39:01.491 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.487+0000 7f85aed9d640 1 -- 172.21.15.99:0/187348541 learned_addr learned my addr 172.21.15.99:0/187348541 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:39:01.491 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.487+0000 7f85a7fff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f85b0101910 0x7f85b0196f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:01.491 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.487+0000 7f85af59e640 1 --2- 172.21.15.99:0/187348541 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f85b01022d0 0x7f85b019dfd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:01.491 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.487+0000 7f85aed9d640 1 -- 172.21.15.99:0/187348541 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f85b01022d0 msgr2=0x7f85b019dfd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:01.491 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.487+0000 7f85aed9d640 1 --2- 172.21.15.99:0/187348541 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f85b01022d0 0x7f85b019dfd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:01.491 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.487+0000 7f85aed9d640 1 -- 172.21.15.99:0/187348541 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f85b0101910 msgr2=0x7f85b0196f60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:01.492 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.487+0000 7f85aed9d640 1 --2- 172.21.15.99:0/187348541 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f85b0101910 0x7f85b0196f60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:01.492 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.487+0000 7f85aed9d640 1 -- 172.21.15.99:0/187348541 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f85b019e640 con 0x7f85b0100720 2024-03-19T01:39:01.492 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.487+0000 7f85af59e640 1 --2- 172.21.15.99:0/187348541 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f85b01022d0 0x7f85b019dfd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:39:01.492 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.488+0000 7f85aed9d640 1 --2- 172.21.15.99:0/187348541 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f85b0100720 0x7f85b0196a20 secure :-1 s=READY pgs=221 cs=0 l=1 rev1=1 crypto rx=0x7f85a000e9c0 tx=0x7f85a000ee80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:01.492 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.488+0000 7f85acd99640 1 -- 172.21.15.99:0/187348541 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f85a000cd80 con 0x7f85b0100720 2024-03-19T01:39:01.492 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.488+0000 7f85acd99640 1 -- 172.21.15.99:0/187348541 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f85a00035f0 con 0x7f85b0100720 2024-03-19T01:39:01.492 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.488+0000 7f85b517e640 1 -- 172.21.15.99:0/187348541 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f85b019e930 con 0x7f85b0100720 2024-03-19T01:39:01.492 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.488+0000 7f85acd99640 1 -- 172.21.15.99:0/187348541 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f85a0010690 con 0x7f85b0100720 2024-03-19T01:39:01.492 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.488+0000 7f85b517e640 1 -- 172.21.15.99:0/187348541 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f85b019ee80 con 0x7f85b0100720 2024-03-19T01:39:01.492 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.489+0000 7f85b517e640 1 -- 172.21.15.99:0/187348541 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f85b0107a40 con 0x7f85b0100720 2024-03-19T01:39:01.493 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.490+0000 7f85acd99640 1 -- 172.21.15.99:0/187348541 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 16) ==== 137290+0+0 (secure 0 0 0) 0x7f85a00037a0 con 0x7f85b0100720 2024-03-19T01:39:01.493 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.490+0000 7f85acd99640 1 --2- 172.21.15.99:0/187348541 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f859c0a2800 0x7f859c0a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:01.493 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.490+0000 7f85a7fff640 1 --2- 172.21.15.99:0/187348541 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f859c0a2800 0x7f859c0a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:01.493 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.490+0000 7f85acd99640 1 -- 172.21.15.99:0/187348541 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(87..87 src has 1..87) ==== 7979+0+0 (secure 0 0 0) 0x7f85a00cecc0 con 0x7f85b0100720 2024-03-19T01:39:01.493 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.491+0000 7f85a7fff640 1 --2- 172.21.15.99:0/187348541 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f859c0a2800 0x7f859c0a4cc0 secure :-1 s=READY pgs=171 cs=0 l=1 rev1=1 crypto rx=0x7f85b0197f30 tx=0x7f8598005f90 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:01.496 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.495+0000 7f85acd99640 1 -- 172.21.15.99:0/187348541 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f85a009beb0 con 0x7f85b0100720 2024-03-19T01:39:01.702 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.701+0000 7f85b517e640 1 -- 172.21.15.99:0/187348541 --> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] -- mgr_command(tid 0: {"prefix": "fs subvolume create", "vol_name": "cephfs", "sub_name": "sv_1", "group_name": "qa", "target": ["mon-mgr", ""]}) -- 0x7f85b0061c60 con 0x7f859c0a2800 2024-03-19T01:39:01.723 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.722+0000 7f85acd99640 1 -- 172.21.15.99:0/187348541 <== mgr.14150 v2:172.21.15.99:6800/4272410295 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+0 (secure 0 0 0) 0x7f85b0061c60 con 0x7f859c0a2800 2024-03-19T01:39:01.726 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.724+0000 7f85b517e640 1 -- 172.21.15.99:0/187348541 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f859c0a2800 msgr2=0x7f859c0a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:01.726 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.724+0000 7f85b517e640 1 --2- 172.21.15.99:0/187348541 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f859c0a2800 0x7f859c0a4cc0 secure :-1 s=READY pgs=171 cs=0 l=1 rev1=1 crypto rx=0x7f85b0197f30 tx=0x7f8598005f90 comp rx=0 tx=0).stop 2024-03-19T01:39:01.726 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.724+0000 7f85b517e640 1 -- 172.21.15.99:0/187348541 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f85b0100720 msgr2=0x7f85b0196a20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:01.726 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.724+0000 7f85b517e640 1 --2- 172.21.15.99:0/187348541 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f85b0100720 0x7f85b0196a20 secure :-1 s=READY pgs=221 cs=0 l=1 rev1=1 crypto rx=0x7f85a000e9c0 tx=0x7f85a000ee80 comp rx=0 tx=0).stop 2024-03-19T01:39:01.726 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.725+0000 7f85b517e640 1 -- 172.21.15.99:0/187348541 shutdown_connections 2024-03-19T01:39:01.726 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.725+0000 7f85b517e640 1 --2- 172.21.15.99:0/187348541 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f859c0a2800 0x7f859c0a4cc0 unknown :-1 s=CLOSED pgs=171 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:01.727 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.725+0000 7f85b517e640 1 --2- 172.21.15.99:0/187348541 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f85b0101910 0x7f85b0196f60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:01.727 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.725+0000 7f85b517e640 1 --2- 172.21.15.99:0/187348541 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f85b01022d0 0x7f85b019dfd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:01.727 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.725+0000 7f85b517e640 1 --2- 172.21.15.99:0/187348541 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f85b0100720 0x7f85b0196a20 unknown :-1 s=CLOSED pgs=221 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:01.727 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.725+0000 7f85b517e640 1 -- 172.21.15.99:0/187348541 >> 172.21.15.99:0/187348541 conn(0x7f85b00fbcb0 msgr2=0x7f85b0104c10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:01.727 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.725+0000 7f85b517e640 1 -- 172.21.15.99:0/187348541 shutdown_connections 2024-03-19T01:39:01.727 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.725+0000 7f85b517e640 1 -- 172.21.15.99:0/187348541 wait complete. 2024-03-19T01:39:01.737 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs subvolume getpath cephfs --group_name qa sv_1 2024-03-19T01:39:01.852 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.848+0000 7f5dfda61640 1 -- 172.21.15.99:0/1998424833 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5df8101950 msgr2=0x7f5df8101dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:01.852 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.848+0000 7f5dfda61640 1 --2- 172.21.15.99:0/1998424833 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5df8101950 0x7f5df8101dd0 secure :-1 s=READY pgs=222 cs=0 l=1 rev1=1 crypto rx=0x7f5de0009900 tx=0x7f5de002bc30 comp rx=0 tx=0).stop 2024-03-19T01:39:01.852 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.848+0000 7f5dfda61640 1 -- 172.21.15.99:0/1998424833 shutdown_connections 2024-03-19T01:39:01.852 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.848+0000 7f5dfda61640 1 --2- 172.21.15.99:0/1998424833 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5df8102310 0x7f5df8108ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:01.852 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.848+0000 7f5dfda61640 1 --2- 172.21.15.99:0/1998424833 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5df8101950 0x7f5df8101dd0 unknown :-1 s=CLOSED pgs=222 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:01.852 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.848+0000 7f5dfda61640 1 --2- 172.21.15.99:0/1998424833 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5df8100760 0x7f5df8100b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:01.853 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.848+0000 7f5dfda61640 1 -- 172.21.15.99:0/1998424833 >> 172.21.15.99:0/1998424833 conn(0x7f5df80fbcd0 msgr2=0x7f5df80fe110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:01.853 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.848+0000 7f5dfda61640 1 -- 172.21.15.99:0/1998424833 shutdown_connections 2024-03-19T01:39:01.853 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.848+0000 7f5dfda61640 1 -- 172.21.15.99:0/1998424833 wait complete. 2024-03-19T01:39:01.853 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.849+0000 7f5dfda61640 1 Processor -- start 2024-03-19T01:39:01.853 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.849+0000 7f5dfda61640 1 -- start start 2024-03-19T01:39:01.853 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.849+0000 7f5dfda61640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5df8100760 0x7f5df81046f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:01.853 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.849+0000 7f5dfda61640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5df8101950 0x7f5df8102d30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:01.853 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.849+0000 7f5dfda61640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5df8102310 0x7f5df8103270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:01.853 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.849+0000 7f5dfda61640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f5df810c3e0 con 0x7f5df8100760 2024-03-19T01:39:01.853 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.849+0000 7f5dfda61640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f5df810c260 con 0x7f5df8101950 2024-03-19T01:39:01.853 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.849+0000 7f5dfda61640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f5df810c560 con 0x7f5df8102310 2024-03-19T01:39:01.853 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.849+0000 7f5df6ffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5df8100760 0x7f5df81046f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:01.853 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.849+0000 7f5df6ffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5df8100760 0x7f5df81046f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58618/0 (socket says 172.21.15.99:58618) 2024-03-19T01:39:01.854 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.849+0000 7f5df6ffd640 1 -- 172.21.15.99:0/618235931 learned_addr learned my addr 172.21.15.99:0/618235931 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:39:01.854 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.849+0000 7f5df77fe640 1 --2- 172.21.15.99:0/618235931 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5df8102310 0x7f5df8103270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:01.854 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.849+0000 7f5deffff640 1 --2- 172.21.15.99:0/618235931 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5df8101950 0x7f5df8102d30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:01.854 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.849+0000 7f5df6ffd640 1 -- 172.21.15.99:0/618235931 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5df8102310 msgr2=0x7f5df8103270 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:01.854 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.849+0000 7f5df6ffd640 1 --2- 172.21.15.99:0/618235931 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5df8102310 0x7f5df8103270 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:01.854 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.849+0000 7f5df6ffd640 1 -- 172.21.15.99:0/618235931 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5df8101950 msgr2=0x7f5df8102d30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:01.854 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.849+0000 7f5df6ffd640 1 --2- 172.21.15.99:0/618235931 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5df8101950 0x7f5df8102d30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:01.854 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.849+0000 7f5df6ffd640 1 -- 172.21.15.99:0/618235931 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f5df8103b00 con 0x7f5df8100760 2024-03-19T01:39:01.854 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.850+0000 7f5df6ffd640 1 --2- 172.21.15.99:0/618235931 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5df8100760 0x7f5df81046f0 secure :-1 s=READY pgs=223 cs=0 l=1 rev1=1 crypto rx=0x7f5de800e9c0 tx=0x7f5de800ee80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:01.854 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.850+0000 7f5dfca5f640 1 -- 172.21.15.99:0/618235931 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5de800cde0 con 0x7f5df8100760 2024-03-19T01:39:01.854 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.850+0000 7f5dfca5f640 1 -- 172.21.15.99:0/618235931 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f5de8010910 con 0x7f5df8100760 2024-03-19T01:39:01.854 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.850+0000 7f5dfda61640 1 -- 172.21.15.99:0/618235931 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f5df81a1cd0 con 0x7f5df8100760 2024-03-19T01:39:01.855 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.850+0000 7f5dfca5f640 1 -- 172.21.15.99:0/618235931 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5de80189c0 con 0x7f5df8100760 2024-03-19T01:39:01.855 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.850+0000 7f5dfda61640 1 -- 172.21.15.99:0/618235931 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f5df81a21a0 con 0x7f5df8100760 2024-03-19T01:39:01.855 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.851+0000 7f5dfda61640 1 -- 172.21.15.99:0/618235931 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f5df81a2370 con 0x7f5df8100760 2024-03-19T01:39:01.855 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.852+0000 7f5dfca5f640 1 -- 172.21.15.99:0/618235931 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 17) ==== 137341+0+0 (secure 0 0 0) 0x7f5de8010ac0 con 0x7f5df8100760 2024-03-19T01:39:01.856 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.853+0000 7f5dfca5f640 1 --2- 172.21.15.99:0/618235931 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f5de40a2890 0x7f5de40a4d50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:01.856 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.853+0000 7f5deffff640 1 --2- 172.21.15.99:0/618235931 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f5de40a2890 0x7f5de40a4d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:01.856 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.853+0000 7f5dfca5f640 1 -- 172.21.15.99:0/618235931 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(87..87 src has 1..87) ==== 7979+0+0 (secure 0 0 0) 0x7f5de80cef90 con 0x7f5df8100760 2024-03-19T01:39:01.856 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.853+0000 7f5deffff640 1 --2- 172.21.15.99:0/618235931 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f5de40a2890 0x7f5de40a4d50 secure :-1 s=READY pgs=172 cs=0 l=1 rev1=1 crypto rx=0x7f5de0009230 tx=0x7f5de0005fd0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:01.859 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:01.857+0000 7f5dfca5f640 1 -- 172.21.15.99:0/618235931 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f5de8053740 con 0x7f5df8100760 2024-03-19T01:39:02.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:01 smithi115 ceph-mon[67332]: from='client.14760 -' entity='client.admin' cmd=[{"prefix": "fs subvolumegroup create", "vol_name": "cephfs", "group_name": "qa", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:39:02.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:01 smithi115 ceph-mon[67332]: pgmap v284: 97 pgs: 81 active+clean, 16 creating+peering; 579 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 685 B/s rd, 2.5 KiB/s wr, 11 op/s 2024-03-19T01:39:02.057 INFO:journalctl@ceph.mds.i.smithi115.stdout:Mar 19 01:39:01 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-i[97459]: 2024-03-19T01:39:01.814+0000 7fcc9fa12700 -1 mds.pinger is_rank_lagging: rank=0 was never sent ping request. 2024-03-19T01:39:02.057 INFO:journalctl@ceph.mds.i.smithi115.stdout:Mar 19 01:39:01 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-i[97459]: 2024-03-19T01:39:01.814+0000 7fcc9fa12700 -1 mds.pinger is_rank_lagging: rank=1 was never sent ping request. 2024-03-19T01:39:02.057 INFO:journalctl@ceph.mds.i.smithi115.stdout:Mar 19 01:39:01 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-i[97459]: 2024-03-19T01:39:01.814+0000 7fcc9fa12700 -1 mds.pinger is_rank_lagging: rank=2 was never sent ping request. 2024-03-19T01:39:02.064 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.063+0000 7f5dfda61640 1 -- 172.21.15.99:0/618235931 --> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] -- mgr_command(tid 0: {"prefix": "fs subvolume getpath", "vol_name": "cephfs", "group_name": "qa", "sub_name": "sv_1", "target": ["mon-mgr", ""]}) -- 0x7f5df80fdee0 con 0x7f5de40a2890 2024-03-19T01:39:02.067 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.066+0000 7f5dfca5f640 1 -- 172.21.15.99:0/618235931 <== mgr.14150 v2:172.21.15.99:6800/4272410295 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+54 (secure 0 0 0) 0x7f5df80fdee0 con 0x7f5de40a2890 2024-03-19T01:39:02.068 INFO:teuthology.orchestra.run.smithi099.stdout:/volumes/qa/sv_1/6b9b1809-7324-48f8-a03e-5dbb285d942e 2024-03-19T01:39:02.070 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.069+0000 7f5dfda61640 1 -- 172.21.15.99:0/618235931 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f5de40a2890 msgr2=0x7f5de40a4d50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:02.070 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.069+0000 7f5dfda61640 1 --2- 172.21.15.99:0/618235931 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f5de40a2890 0x7f5de40a4d50 secure :-1 s=READY pgs=172 cs=0 l=1 rev1=1 crypto rx=0x7f5de0009230 tx=0x7f5de0005fd0 comp rx=0 tx=0).stop 2024-03-19T01:39:02.070 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.069+0000 7f5dfda61640 1 -- 172.21.15.99:0/618235931 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5df8100760 msgr2=0x7f5df81046f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:02.070 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.069+0000 7f5dfda61640 1 --2- 172.21.15.99:0/618235931 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5df8100760 0x7f5df81046f0 secure :-1 s=READY pgs=223 cs=0 l=1 rev1=1 crypto rx=0x7f5de800e9c0 tx=0x7f5de800ee80 comp rx=0 tx=0).stop 2024-03-19T01:39:02.071 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.069+0000 7f5dfda61640 1 -- 172.21.15.99:0/618235931 shutdown_connections 2024-03-19T01:39:02.071 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.069+0000 7f5dfda61640 1 --2- 172.21.15.99:0/618235931 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f5de40a2890 0x7f5de40a4d50 unknown :-1 s=CLOSED pgs=172 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:02.071 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.069+0000 7f5dfda61640 1 --2- 172.21.15.99:0/618235931 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5df8101950 0x7f5df8102d30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:02.071 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.069+0000 7f5dfda61640 1 --2- 172.21.15.99:0/618235931 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5df8102310 0x7f5df8103270 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:02.071 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.069+0000 7f5dfda61640 1 --2- 172.21.15.99:0/618235931 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5df8100760 0x7f5df81046f0 unknown :-1 s=CLOSED pgs=223 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:02.071 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.069+0000 7f5dfda61640 1 -- 172.21.15.99:0/618235931 >> 172.21.15.99:0/618235931 conn(0x7f5df80fbcd0 msgr2=0x7f5df80fd7d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:02.071 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.069+0000 7f5dfda61640 1 -- 172.21.15.99:0/618235931 shutdown_connections 2024-03-19T01:39:02.071 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.069+0000 7f5dfda61640 1 -- 172.21.15.99:0/618235931 wait complete. 2024-03-19T01:39:02.080 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:01 smithi099 ceph-mon[62951]: from='client.14760 -' entity='client.admin' cmd=[{"prefix": "fs subvolumegroup create", "vol_name": "cephfs", "group_name": "qa", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:39:02.080 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:01 smithi099 ceph-mon[62951]: pgmap v284: 97 pgs: 81 active+clean, 16 creating+peering; 579 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 685 B/s rd, 2.5 KiB/s wr, 11 op/s 2024-03-19T01:39:02.081 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-19T01:39:02.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:01 smithi112 ceph-mon[66819]: from='client.14760 -' entity='client.admin' cmd=[{"prefix": "fs subvolumegroup create", "vol_name": "cephfs", "group_name": "qa", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:39:02.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:01 smithi112 ceph-mon[66819]: pgmap v284: 97 pgs: 81 active+clean, 16 creating+peering; 579 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 685 B/s rd, 2.5 KiB/s wr, 11 op/s 2024-03-19T01:39:02.185 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.181+0000 7f412b081640 1 -- 172.21.15.99:0/282851760 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4124100740 msgr2=0x7f4124100b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:02.185 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.181+0000 7f412b081640 1 --2- 172.21.15.99:0/282851760 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4124100740 0x7f4124100b60 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 crypto rx=0x7f4120009860 tx=0x7f412002bb70 comp rx=0 tx=0).stop 2024-03-19T01:39:02.185 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.181+0000 7f412b081640 1 -- 172.21.15.99:0/282851760 shutdown_connections 2024-03-19T01:39:02.185 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.181+0000 7f412b081640 1 --2- 172.21.15.99:0/282851760 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f41241022f0 0x7f4124108b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:02.185 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.181+0000 7f412b081640 1 --2- 172.21.15.99:0/282851760 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4124101930 0x7f4124101db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:02.186 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.181+0000 7f412b081640 1 --2- 172.21.15.99:0/282851760 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4124100740 0x7f4124100b60 unknown :-1 s=CLOSED pgs=58 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:02.186 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.181+0000 7f412b081640 1 -- 172.21.15.99:0/282851760 >> 172.21.15.99:0/282851760 conn(0x7f41240fbcb0 msgr2=0x7f41240fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:02.186 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.181+0000 7f412b081640 1 -- 172.21.15.99:0/282851760 shutdown_connections 2024-03-19T01:39:02.186 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.181+0000 7f412b081640 1 -- 172.21.15.99:0/282851760 wait complete. 2024-03-19T01:39:02.186 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.182+0000 7f412b081640 1 Processor -- start 2024-03-19T01:39:02.186 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.182+0000 7f412b081640 1 -- start start 2024-03-19T01:39:02.186 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.182+0000 7f412b081640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4124100740 0x7f4124197b60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:02.186 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.182+0000 7f412b081640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4124101930 0x7f41241980a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:02.186 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.182+0000 7f412b081640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f41241022f0 0x7f412419f110 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:02.186 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.182+0000 7f412b081640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f412410d4e0 con 0x7f41241022f0 2024-03-19T01:39:02.186 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.182+0000 7f412b081640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f412410d360 con 0x7f4124101930 2024-03-19T01:39:02.186 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.182+0000 7f412b081640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f412410d660 con 0x7f4124100740 2024-03-19T01:39:02.187 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.182+0000 7f412961e640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f41241022f0 0x7f412419f110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:02.187 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.182+0000 7f412961e640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f41241022f0 0x7f412419f110 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58624/0 (socket says 172.21.15.99:58624) 2024-03-19T01:39:02.187 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.182+0000 7f412961e640 1 -- 172.21.15.99:0/954980700 learned_addr learned my addr 172.21.15.99:0/954980700 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:39:02.187 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.182+0000 7f4128e1d640 1 --2- 172.21.15.99:0/954980700 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4124100740 0x7f4124197b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:02.187 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.182+0000 7f411bfff640 1 --2- 172.21.15.99:0/954980700 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4124101930 0x7f41241980a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:02.187 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.182+0000 7f412961e640 1 -- 172.21.15.99:0/954980700 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4124100740 msgr2=0x7f4124197b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:02.187 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.182+0000 7f412961e640 1 --2- 172.21.15.99:0/954980700 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4124100740 0x7f4124197b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:02.187 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.182+0000 7f412961e640 1 -- 172.21.15.99:0/954980700 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4124101930 msgr2=0x7f41241980a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:02.187 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.182+0000 7f412961e640 1 --2- 172.21.15.99:0/954980700 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4124101930 0x7f41241980a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:02.187 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.183+0000 7f412961e640 1 -- 172.21.15.99:0/954980700 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f412419f780 con 0x7f41241022f0 2024-03-19T01:39:02.187 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.183+0000 7f4128e1d640 1 --2- 172.21.15.99:0/954980700 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4124100740 0x7f4124197b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:39:02.187 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.183+0000 7f412961e640 1 --2- 172.21.15.99:0/954980700 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f41241022f0 0x7f412419f110 secure :-1 s=READY pgs=224 cs=0 l=1 rev1=1 crypto rx=0x7f411c00eb10 tx=0x7f411c00c2d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:02.187 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.183+0000 7f4119ffb640 1 -- 172.21.15.99:0/954980700 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f411c004b20 con 0x7f41241022f0 2024-03-19T01:39:02.188 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.183+0000 7f4119ffb640 1 -- 172.21.15.99:0/954980700 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f411c004cc0 con 0x7f41241022f0 2024-03-19T01:39:02.188 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.183+0000 7f412b081640 1 -- 172.21.15.99:0/954980700 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f412419fa70 con 0x7f41241022f0 2024-03-19T01:39:02.188 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.183+0000 7f4119ffb640 1 -- 172.21.15.99:0/954980700 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f411c010760 con 0x7f41241022f0 2024-03-19T01:39:02.188 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.183+0000 7f412b081640 1 -- 172.21.15.99:0/954980700 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f412419ff70 con 0x7f41241022f0 2024-03-19T01:39:02.188 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.184+0000 7f412b081640 1 -- 172.21.15.99:0/954980700 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f4124107a60 con 0x7f41241022f0 2024-03-19T01:39:02.189 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.185+0000 7f4119ffb640 1 -- 172.21.15.99:0/954980700 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 17) ==== 137341+0+0 (secure 0 0 0) 0x7f411c005020 con 0x7f41241022f0 2024-03-19T01:39:02.189 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.186+0000 7f4119ffb640 1 --2- 172.21.15.99:0/954980700 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f410c0a2890 0x7f410c0a4d50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:02.189 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.186+0000 7f4128e1d640 1 --2- 172.21.15.99:0/954980700 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f410c0a2890 0x7f410c0a4d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:02.189 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.186+0000 7f4119ffb640 1 -- 172.21.15.99:0/954980700 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(87..87 src has 1..87) ==== 7979+0+0 (secure 0 0 0) 0x7f411c0cefb0 con 0x7f41241022f0 2024-03-19T01:39:02.189 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.186+0000 7f4128e1d640 1 --2- 172.21.15.99:0/954980700 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f410c0a2890 0x7f410c0a4d50 secure :-1 s=READY pgs=173 cs=0 l=1 rev1=1 crypto rx=0x7f4120009830 tx=0x7f412002e040 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:02.191 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.190+0000 7f4119ffb640 1 -- 172.21.15.99:0/954980700 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f411c09c0a0 con 0x7f41241022f0 2024-03-19T01:39:02.408 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.407+0000 7f412b081640 1 -- 172.21.15.99:0/954980700 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f412402d1f0 con 0x7f41241022f0 2024-03-19T01:39:02.409 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.407+0000 7f4119ffb640 1 -- 172.21.15.99:0/954980700 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 20 v20) ==== 94+0+11878 (secure 0 0 0) 0x7f411c09b9a0 con 0x7f41241022f0 2024-03-19T01:39:02.409 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:39:02.409 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":20,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14562,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6835/1631815621","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6834","nonce":1631815621},{"type":"v1","addr":"172.21.15.99:6835","nonce":1631815621}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":4},{"gid":14646,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6839/2236145144","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6838","nonce":2236145144},{"type":"v1","addr":"172.21.15.99:6839","nonce":2236145144}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11},{"gid":14658,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6841/1660990159","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6840","nonce":1660990159},{"type":"v1","addr":"172.21.15.99:6841","nonce":1660990159}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":12},{"gid":24383,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.112:6833/2969810977","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6832","nonce":2969810977},{"type":"v1","addr":"172.21.15.112:6833","nonce":2969810977}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3},{"gid":24404,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6837/2038634879","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6836","nonce":2038634879},{"type":"v1","addr":"172.21.15.99:6837","nonce":2038634879}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":24415,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.115:6833/2219320481","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6832","nonce":2219320481},{"type":"v1","addr":"172.21.15.115:6833","nonce":2219320481}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":20,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-03-19T01:38:56.124529+0000","modified":"2024-03-19T01:39:00.323326+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24487,"mds_1":24479,"mds_2":24472},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24416":{"gid":24416,"name":"e","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.112:6835/3690587528","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6834","nonce":3690587528},{"type":"v1","addr":"172.21.15.112:6835","nonce":3690587528}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24440":{"gid":24440,"name":"h","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.112:6837/2339823159","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6836","nonce":2339823159},{"type":"v1","addr":"172.21.15.112:6837","nonce":2339823159}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24460":{"gid":24460,"name":"f","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.115:6835/591078413","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6834","nonce":591078413},{"type":"v1","addr":"172.21.15.115:6835","nonce":591078413}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24472":{"gid":24472,"name":"c","rank":2,"incarnation":19,"state":"up:active","state_seq":7,"addr":"172.21.15.115:6837/1368988191","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6836","nonce":1368988191},{"type":"v1","addr":"172.21.15.115:6837","nonce":1368988191}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24479":{"gid":24479,"name":"b","rank":1,"incarnation":17,"state":"up:active","state_seq":4,"addr":"172.21.15.112:6839/986249680","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6838","nonce":986249680},{"type":"v1","addr":"172.21.15.112:6839","nonce":986249680}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24487":{"gid":24487,"name":"i","rank":0,"incarnation":14,"state":"up:active","state_seq":5,"addr":"172.21.15.115:6839/2743761968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6838","nonce":2743761968},{"type":"v1","addr":"172.21.15.115:6839","nonce":2743761968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24487,"qdb_cluster":[24487,24479,24472]},"id":1}]} 2024-03-19T01:39:02.409 INFO:teuthology.orchestra.run.smithi099.stderr:dumped fsmap epoch 20 2024-03-19T01:39:02.411 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.409+0000 7f412b081640 1 -- 172.21.15.99:0/954980700 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f410c0a2890 msgr2=0x7f410c0a4d50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:02.411 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.409+0000 7f412b081640 1 --2- 172.21.15.99:0/954980700 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f410c0a2890 0x7f410c0a4d50 secure :-1 s=READY pgs=173 cs=0 l=1 rev1=1 crypto rx=0x7f4120009830 tx=0x7f412002e040 comp rx=0 tx=0).stop 2024-03-19T01:39:02.411 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.409+0000 7f412b081640 1 -- 172.21.15.99:0/954980700 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f41241022f0 msgr2=0x7f412419f110 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:02.411 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.409+0000 7f412b081640 1 --2- 172.21.15.99:0/954980700 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f41241022f0 0x7f412419f110 secure :-1 s=READY pgs=224 cs=0 l=1 rev1=1 crypto rx=0x7f411c00eb10 tx=0x7f411c00c2d0 comp rx=0 tx=0).stop 2024-03-19T01:39:02.411 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.409+0000 7f412b081640 1 -- 172.21.15.99:0/954980700 shutdown_connections 2024-03-19T01:39:02.411 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.410+0000 7f412b081640 1 --2- 172.21.15.99:0/954980700 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f410c0a2890 0x7f410c0a4d50 unknown :-1 s=CLOSED pgs=173 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:02.411 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.410+0000 7f412b081640 1 --2- 172.21.15.99:0/954980700 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4124101930 0x7f41241980a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:02.411 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.410+0000 7f412b081640 1 --2- 172.21.15.99:0/954980700 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f41241022f0 0x7f412419f110 unknown :-1 s=CLOSED pgs=224 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:02.412 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.410+0000 7f412b081640 1 --2- 172.21.15.99:0/954980700 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4124100740 0x7f4124197b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:02.412 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.410+0000 7f412b081640 1 -- 172.21.15.99:0/954980700 >> 172.21.15.99:0/954980700 conn(0x7f41240fbcb0 msgr2=0x7f4124104c30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:02.412 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.410+0000 7f412b081640 1 -- 172.21.15.99:0/954980700 shutdown_connections 2024-03-19T01:39:02.412 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.410+0000 7f412b081640 1 -- 172.21.15.99:0/954980700 wait complete. 2024-03-19T01:39:02.422 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-19T01:39:02.526 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.522+0000 7fbe797cd640 1 -- 172.21.15.99:0/1593270057 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbe74100760 msgr2=0x7fbe74100b80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:02.526 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.522+0000 7fbe797cd640 1 --2- 172.21.15.99:0/1593270057 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbe74100760 0x7fbe74100b80 secure :-1 s=READY pgs=225 cs=0 l=1 rev1=1 crypto rx=0x7fbe6c009900 tx=0x7fbe6c02bc30 comp rx=0 tx=0).stop 2024-03-19T01:39:02.526 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.522+0000 7fbe797cd640 1 -- 172.21.15.99:0/1593270057 shutdown_connections 2024-03-19T01:39:02.526 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.522+0000 7fbe797cd640 1 --2- 172.21.15.99:0/1593270057 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fbe74102310 0x7fbe74108ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:02.526 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.522+0000 7fbe797cd640 1 --2- 172.21.15.99:0/1593270057 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fbe74101950 0x7fbe74101dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:02.526 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.522+0000 7fbe797cd640 1 --2- 172.21.15.99:0/1593270057 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbe74100760 0x7fbe74100b80 unknown :-1 s=CLOSED pgs=225 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:02.526 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.522+0000 7fbe797cd640 1 -- 172.21.15.99:0/1593270057 >> 172.21.15.99:0/1593270057 conn(0x7fbe740fbcd0 msgr2=0x7fbe740fe110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:02.527 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.522+0000 7fbe797cd640 1 -- 172.21.15.99:0/1593270057 shutdown_connections 2024-03-19T01:39:02.527 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.522+0000 7fbe797cd640 1 -- 172.21.15.99:0/1593270057 wait complete. 2024-03-19T01:39:02.527 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.522+0000 7fbe797cd640 1 Processor -- start 2024-03-19T01:39:02.527 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.523+0000 7fbe797cd640 1 -- start start 2024-03-19T01:39:02.527 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.523+0000 7fbe797cd640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fbe74100760 0x7fbe74196b30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:02.527 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.523+0000 7fbe797cd640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fbe74101950 0x7fbe74197070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:02.527 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.523+0000 7fbe797cd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbe74102310 0x7fbe7419e0e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:02.527 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.523+0000 7fbe797cd640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fbe7410c3b0 con 0x7fbe74102310 2024-03-19T01:39:02.527 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.523+0000 7fbe797cd640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fbe7410c230 con 0x7fbe74100760 2024-03-19T01:39:02.527 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.523+0000 7fbe797cd640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fbe7410c530 con 0x7fbe74101950 2024-03-19T01:39:02.527 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.523+0000 7fbe737fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbe74102310 0x7fbe7419e0e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:02.528 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.523+0000 7fbe737fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbe74102310 0x7fbe7419e0e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58648/0 (socket says 172.21.15.99:58648) 2024-03-19T01:39:02.528 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.523+0000 7fbe737fe640 1 -- 172.21.15.99:0/3068977748 learned_addr learned my addr 172.21.15.99:0/3068977748 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:39:02.528 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.523+0000 7fbe72ffd640 1 --2- 172.21.15.99:0/3068977748 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fbe74100760 0x7fbe74196b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:02.528 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.523+0000 7fbe727fc640 1 --2- 172.21.15.99:0/3068977748 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fbe74101950 0x7fbe74197070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:02.528 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.523+0000 7fbe737fe640 1 -- 172.21.15.99:0/3068977748 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fbe74101950 msgr2=0x7fbe74197070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:02.528 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.523+0000 7fbe737fe640 1 --2- 172.21.15.99:0/3068977748 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fbe74101950 0x7fbe74197070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:02.528 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.523+0000 7fbe737fe640 1 -- 172.21.15.99:0/3068977748 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fbe74100760 msgr2=0x7fbe74196b30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:02.528 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.523+0000 7fbe737fe640 1 --2- 172.21.15.99:0/3068977748 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fbe74100760 0x7fbe74196b30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:02.528 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.523+0000 7fbe737fe640 1 -- 172.21.15.99:0/3068977748 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fbe7419e750 con 0x7fbe74102310 2024-03-19T01:39:02.528 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.523+0000 7fbe737fe640 1 --2- 172.21.15.99:0/3068977748 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbe74102310 0x7fbe7419e0e0 secure :-1 s=READY pgs=226 cs=0 l=1 rev1=1 crypto rx=0x7fbe6400e9b0 tx=0x7fbe6400ee70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:02.528 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.523+0000 7fbe6bfff640 1 -- 172.21.15.99:0/3068977748 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbe6400cd80 con 0x7fbe74102310 2024-03-19T01:39:02.528 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.523+0000 7fbe797cd640 1 -- 172.21.15.99:0/3068977748 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fbe7419ea40 con 0x7fbe74102310 2024-03-19T01:39:02.528 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.523+0000 7fbe6bfff640 1 -- 172.21.15.99:0/3068977748 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fbe640035f0 con 0x7fbe74102310 2024-03-19T01:39:02.529 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.523+0000 7fbe6bfff640 1 -- 172.21.15.99:0/3068977748 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbe64018630 con 0x7fbe74102310 2024-03-19T01:39:02.529 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.523+0000 7fbe797cd640 1 -- 172.21.15.99:0/3068977748 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fbe7419ef90 con 0x7fbe74102310 2024-03-19T01:39:02.529 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.524+0000 7fbe797cd640 1 -- 172.21.15.99:0/3068977748 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fbe74107b70 con 0x7fbe74102310 2024-03-19T01:39:02.530 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.526+0000 7fbe6bfff640 1 -- 172.21.15.99:0/3068977748 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 17) ==== 137341+0+0 (secure 0 0 0) 0x7fbe640037a0 con 0x7fbe74102310 2024-03-19T01:39:02.530 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.526+0000 7fbe6bfff640 1 --2- 172.21.15.99:0/3068977748 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fbe600a2890 0x7fbe600a4d50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:02.530 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.526+0000 7fbe72ffd640 1 --2- 172.21.15.99:0/3068977748 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fbe600a2890 0x7fbe600a4d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:02.530 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.526+0000 7fbe6bfff640 1 -- 172.21.15.99:0/3068977748 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(87..87 src has 1..87) ==== 7979+0+0 (secure 0 0 0) 0x7fbe640ceec0 con 0x7fbe74102310 2024-03-19T01:39:02.530 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.527+0000 7fbe72ffd640 1 --2- 172.21.15.99:0/3068977748 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fbe600a2890 0x7fbe600a4d50 secure :-1 s=READY pgs=174 cs=0 l=1 rev1=1 crypto rx=0x7fbe6c005ab0 tx=0x7fbe6c005fd0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:02.531 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.530+0000 7fbe6bfff640 1 -- 172.21.15.99:0/3068977748 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7fbe64053740 con 0x7fbe74102310 2024-03-19T01:39:02.707 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.705+0000 7fbe797cd640 1 -- 172.21.15.99:0/3068977748 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7fbe7402d440 con 0x7fbe74102310 2024-03-19T01:39:02.708 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.707+0000 7fbe6bfff640 1 -- 172.21.15.99:0/3068977748 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v87) ==== 74+0+22233 (secure 0 0 0) 0x7fbe6409bfb0 con 0x7fbe74102310 2024-03-19T01:39:02.708 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:39:02.709 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":87,"fsid":"438fb69c-e590-11ee-95c9-87774f69a715","created":"2024-03-19T01:31:10.165073+0000","modified":"2024-03-19T01:38:57.785798+0000","last_up_change":"2024-03-19T01:38:09.082688+0000","last_in_change":"2024-03-19T01:37:49.115793+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-19T01:34:31.001309+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-19T01:38:54.018623+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"86","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-19T01:38:55.319036+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"87","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"0c407a0e-a40e-43d7-ad9a-bb8f3cd4a9ba","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6803","nonce":2935260112}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6805","nonce":2935260112}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6808","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6809","nonce":2935260112}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6807","nonce":2935260112}]},"public_addr":"172.21.15.99:6803/2935260112","cluster_addr":"172.21.15.99:6805/2935260112","heartbeat_back_addr":"172.21.15.99:6809/2935260112","heartbeat_front_addr":"172.21.15.99:6807/2935260112","state":["exists","up"]},{"osd":1,"uuid":"a48f9173-4a06-49f5-bca2-2c9047b8f666","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6800","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6801","nonce":1468854289}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6802","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6803","nonce":1468854289}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6806","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6807","nonce":1468854289}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6804","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6805","nonce":1468854289}]},"public_addr":"172.21.15.112:6801/1468854289","cluster_addr":"172.21.15.112:6803/1468854289","heartbeat_back_addr":"172.21.15.112:6807/1468854289","heartbeat_front_addr":"172.21.15.112:6805/1468854289","state":["exists","up"]},{"osd":2,"uuid":"b7715fdd-b6c9-4348-bd16-5eb61b427eef","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6800","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6801","nonce":3766690472}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6802","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6803","nonce":3766690472}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6806","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6807","nonce":3766690472}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6804","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6805","nonce":3766690472}]},"public_addr":"172.21.15.115:6801/3766690472","cluster_addr":"172.21.15.115:6803/3766690472","heartbeat_back_addr":"172.21.15.115:6807/3766690472","heartbeat_front_addr":"172.21.15.115:6805/3766690472","state":["exists","up"]},{"osd":3,"uuid":"6697ed1b-96bf-42de-8440-d17e9785b359","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6810","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6811","nonce":3756732895}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6812","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6813","nonce":3756732895}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6816","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6817","nonce":3756732895}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6814","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6815","nonce":3756732895}]},"public_addr":"172.21.15.99:6811/3756732895","cluster_addr":"172.21.15.99:6813/3756732895","heartbeat_back_addr":"172.21.15.99:6817/3756732895","heartbeat_front_addr":"172.21.15.99:6815/3756732895","state":["exists","up"]},{"osd":4,"uuid":"d958d6b9-67f6-43d0-b704-aa55dc9df83a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":34,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6808","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6809","nonce":3717986391}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6810","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6811","nonce":3717986391}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6814","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6815","nonce":3717986391}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6812","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6813","nonce":3717986391}]},"public_addr":"172.21.15.112:6809/3717986391","cluster_addr":"172.21.15.112:6811/3717986391","heartbeat_back_addr":"172.21.15.112:6815/3717986391","heartbeat_front_addr":"172.21.15.112:6813/3717986391","state":["exists","up"]},{"osd":5,"uuid":"a73eb49a-7925-49ba-bc4f-515e17f8b8ae","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6808","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6809","nonce":1971806408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6810","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6811","nonce":1971806408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6814","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6815","nonce":1971806408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6812","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6813","nonce":1971806408}]},"public_addr":"172.21.15.115:6809/1971806408","cluster_addr":"172.21.15.115:6811/1971806408","heartbeat_back_addr":"172.21.15.115:6815/1971806408","heartbeat_front_addr":"172.21.15.115:6813/1971806408","state":["exists","up"]},{"osd":6,"uuid":"caa718cd-e08b-48de-832a-29546d71e508","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6818","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6819","nonce":1130832772}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6820","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6821","nonce":1130832772}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6824","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6825","nonce":1130832772}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6822","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6823","nonce":1130832772}]},"public_addr":"172.21.15.99:6819/1130832772","cluster_addr":"172.21.15.99:6821/1130832772","heartbeat_back_addr":"172.21.15.99:6825/1130832772","heartbeat_front_addr":"172.21.15.99:6823/1130832772","state":["exists","up"]},{"osd":7,"uuid":"dbee19a1-2315-40d3-88e7-876fcde28903","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":53,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6816","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6817","nonce":3610356576}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6818","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6819","nonce":3610356576}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6822","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6823","nonce":3610356576}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6820","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6821","nonce":3610356576}]},"public_addr":"172.21.15.112:6817/3610356576","cluster_addr":"172.21.15.112:6819/3610356576","heartbeat_back_addr":"172.21.15.112:6823/3610356576","heartbeat_front_addr":"172.21.15.112:6821/3610356576","state":["exists","up"]},{"osd":8,"uuid":"14a866dc-5ce3-42fc-8bcf-9088b68726cb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":60,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6816","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6817","nonce":4255803111}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6818","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6819","nonce":4255803111}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6822","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6823","nonce":4255803111}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6820","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6821","nonce":4255803111}]},"public_addr":"172.21.15.115:6817/4255803111","cluster_addr":"172.21.15.115:6819/4255803111","heartbeat_back_addr":"172.21.15.115:6823/4255803111","heartbeat_front_addr":"172.21.15.115:6821/4255803111","state":["exists","up"]},{"osd":9,"uuid":"1db93652-6035-4906-91ae-d64d5a909502","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":68,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6826","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6827","nonce":1410921523}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6828","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6829","nonce":1410921523}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6832","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6833","nonce":1410921523}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6830","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6831","nonce":1410921523}]},"public_addr":"172.21.15.99:6827/1410921523","cluster_addr":"172.21.15.99:6829/1410921523","heartbeat_back_addr":"172.21.15.99:6833/1410921523","heartbeat_front_addr":"172.21.15.99:6831/1410921523","state":["exists","up"]},{"osd":10,"uuid":"6c50a8a7-3345-4f4a-af0e-5606c7aca453","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":74,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6824","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6825","nonce":891805922}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6826","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6827","nonce":891805922}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6830","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6831","nonce":891805922}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6828","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6829","nonce":891805922}]},"public_addr":"172.21.15.112:6825/891805922","cluster_addr":"172.21.15.112:6827/891805922","heartbeat_back_addr":"172.21.15.112:6831/891805922","heartbeat_front_addr":"172.21.15.112:6829/891805922","state":["exists","up"]},{"osd":11,"uuid":"fd63b7fd-0db5-46d5-82fa-9707bbd0b495","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":82,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6824","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6825","nonce":938436244}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6826","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6827","nonce":938436244}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6830","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6831","nonce":938436244}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6828","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6829","nonce":938436244}]},"public_addr":"172.21.15.115:6825/938436244","cluster_addr":"172.21.15.115:6827/938436244","heartbeat_back_addr":"172.21.15.115:6831/938436244","heartbeat_front_addr":"172.21.15.115:6829/938436244","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:33:39.036822+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:34:03.339566+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:34:27.149273+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:34:51.315561+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:35:15.396031+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:35:38.957168+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:36:03.213914+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:36:27.763699+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:36:51.688670+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:37:17.280165+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:37:42.554633+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:38:07.438039+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.99:6801/1328245934":"2024-03-20T01:32:06.661002+0000","172.21.15.99:0/1618965837":"2024-03-20T01:32:06.661002+0000","172.21.15.99:0/2491429851":"2024-03-20T01:32:06.661002+0000","172.21.15.99:6800/1328245934":"2024-03-20T01:32:06.661002+0000","172.21.15.99:0/2112231766":"2024-03-20T01:31:40.161751+0000","172.21.15.99:0/3341836545":"2024-03-20T01:31:40.161751+0000","172.21.15.99:6800/4223048293":"2024-03-20T01:31:40.161751+0000","172.21.15.99:6801/4223048293":"2024-03-20T01:31:40.161751+0000","172.21.15.99:0/2377621863":"2024-03-20T01:31:40.161751+0000","172.21.15.99:0/1904400935":"2024-03-20T01:32:06.661002+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-19T01:39:02.711 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.709+0000 7fbe797cd640 1 -- 172.21.15.99:0/3068977748 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fbe600a2890 msgr2=0x7fbe600a4d50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:02.711 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.709+0000 7fbe797cd640 1 --2- 172.21.15.99:0/3068977748 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fbe600a2890 0x7fbe600a4d50 secure :-1 s=READY pgs=174 cs=0 l=1 rev1=1 crypto rx=0x7fbe6c005ab0 tx=0x7fbe6c005fd0 comp rx=0 tx=0).stop 2024-03-19T01:39:02.711 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.709+0000 7fbe797cd640 1 -- 172.21.15.99:0/3068977748 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbe74102310 msgr2=0x7fbe7419e0e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:02.712 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.709+0000 7fbe797cd640 1 --2- 172.21.15.99:0/3068977748 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbe74102310 0x7fbe7419e0e0 secure :-1 s=READY pgs=226 cs=0 l=1 rev1=1 crypto rx=0x7fbe6400e9b0 tx=0x7fbe6400ee70 comp rx=0 tx=0).stop 2024-03-19T01:39:02.712 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.709+0000 7fbe797cd640 1 -- 172.21.15.99:0/3068977748 shutdown_connections 2024-03-19T01:39:02.712 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.709+0000 7fbe797cd640 1 --2- 172.21.15.99:0/3068977748 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbe74102310 0x7fbe7419e0e0 unknown :-1 s=CLOSED pgs=226 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:02.712 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.709+0000 7fbe797cd640 1 --2- 172.21.15.99:0/3068977748 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fbe74101950 0x7fbe74197070 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:02.712 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.709+0000 7fbe797cd640 1 --2- 172.21.15.99:0/3068977748 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fbe600a2890 0x7fbe600a4d50 unknown :-1 s=CLOSED pgs=174 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:02.712 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.709+0000 7fbe797cd640 1 --2- 172.21.15.99:0/3068977748 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fbe74100760 0x7fbe74196b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:02.712 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.709+0000 7fbe797cd640 1 -- 172.21.15.99:0/3068977748 >> 172.21.15.99:0/3068977748 conn(0x7fbe740fbcd0 msgr2=0x7fbe74104d40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:02.712 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.709+0000 7fbe797cd640 1 -- 172.21.15.99:0/3068977748 shutdown_connections 2024-03-19T01:39:02.712 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.709+0000 7fbe797cd640 1 -- 172.21.15.99:0/3068977748 wait complete. 2024-03-19T01:39:02.722 INFO:tasks.cephfs.filesystem.ceph_manager:waiting for clean 2024-03-19T01:39:02.722 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-03-19T01:39:02.826 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.822+0000 7f43f0297640 1 -- 172.21.15.99:0/2019176059 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f43e81022f0 msgr2=0x7f43e8108b80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:02.826 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.822+0000 7f43f0297640 1 --2- 172.21.15.99:0/2019176059 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f43e81022f0 0x7f43e8108b80 secure :-1 s=READY pgs=227 cs=0 l=1 rev1=1 crypto rx=0x7f43e0009ca0 tx=0x7f43e002c0f0 comp rx=0 tx=0).stop 2024-03-19T01:39:02.826 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.822+0000 7f43f0297640 1 -- 172.21.15.99:0/2019176059 shutdown_connections 2024-03-19T01:39:02.826 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.822+0000 7f43f0297640 1 --2- 172.21.15.99:0/2019176059 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f43e81022f0 0x7f43e8108b80 unknown :-1 s=CLOSED pgs=227 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:02.826 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.822+0000 7f43f0297640 1 --2- 172.21.15.99:0/2019176059 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f43e8101930 0x7f43e8101db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:02.826 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.822+0000 7f43f0297640 1 --2- 172.21.15.99:0/2019176059 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f43e8100740 0x7f43e8100b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:02.826 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.822+0000 7f43f0297640 1 -- 172.21.15.99:0/2019176059 >> 172.21.15.99:0/2019176059 conn(0x7f43e80fbcb0 msgr2=0x7f43e80fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:02.827 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.822+0000 7f43f0297640 1 -- 172.21.15.99:0/2019176059 shutdown_connections 2024-03-19T01:39:02.827 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.822+0000 7f43f0297640 1 -- 172.21.15.99:0/2019176059 wait complete. 2024-03-19T01:39:02.829 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.823+0000 7f43f0297640 1 Processor -- start 2024-03-19T01:39:02.829 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.823+0000 7f43f0297640 1 -- start start 2024-03-19T01:39:02.830 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.823+0000 7f43f0297640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f43e8100740 0x7f43e8197af0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:02.830 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.823+0000 7f43f0297640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f43e8101930 0x7f43e8198030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:02.830 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.823+0000 7f43f0297640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f43e81022f0 0x7f43e819f0a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:02.830 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.823+0000 7f43f0297640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f43e810c3c0 con 0x7f43e8100740 2024-03-19T01:39:02.830 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.823+0000 7f43f0297640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f43e810c240 con 0x7f43e81022f0 2024-03-19T01:39:02.830 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.823+0000 7f43f0297640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f43e810c540 con 0x7f43e8101930 2024-03-19T01:39:02.830 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.823+0000 7f43ee033640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f43e8100740 0x7f43e8197af0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:02.830 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.823+0000 7f43ee033640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f43e8100740 0x7f43e8197af0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:52016/0 (socket says 172.21.15.99:52016) 2024-03-19T01:39:02.830 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.823+0000 7f43ee033640 1 -- 172.21.15.99:0/108915896 learned_addr learned my addr 172.21.15.99:0/108915896 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:39:02.830 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.823+0000 7f43ed832640 1 --2- 172.21.15.99:0/108915896 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f43e8101930 0x7f43e8198030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:02.830 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.823+0000 7f43ee834640 1 --2- 172.21.15.99:0/108915896 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f43e81022f0 0x7f43e819f0a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:02.830 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.823+0000 7f43ee033640 1 -- 172.21.15.99:0/108915896 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f43e8101930 msgr2=0x7f43e8198030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:02.830 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.823+0000 7f43ee033640 1 --2- 172.21.15.99:0/108915896 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f43e8101930 0x7f43e8198030 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:02.831 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.823+0000 7f43ee033640 1 -- 172.21.15.99:0/108915896 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f43e81022f0 msgr2=0x7f43e819f0a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:02.831 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.823+0000 7f43ee033640 1 --2- 172.21.15.99:0/108915896 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f43e81022f0 0x7f43e819f0a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:02.831 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.823+0000 7f43ee033640 1 -- 172.21.15.99:0/108915896 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f43e819f710 con 0x7f43e8100740 2024-03-19T01:39:02.831 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.823+0000 7f43ee834640 1 --2- 172.21.15.99:0/108915896 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f43e81022f0 0x7f43e819f0a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:39:02.831 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.823+0000 7f43ee033640 1 --2- 172.21.15.99:0/108915896 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f43e8100740 0x7f43e8197af0 secure :-1 s=READY pgs=228 cs=0 l=1 rev1=1 crypto rx=0x7f43e400e9c0 tx=0x7f43e400ee80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:02.831 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.823+0000 7f43df7fe640 1 -- 172.21.15.99:0/108915896 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f43e400cd80 con 0x7f43e8100740 2024-03-19T01:39:02.831 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.824+0000 7f43df7fe640 1 -- 172.21.15.99:0/108915896 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f43e40035f0 con 0x7f43e8100740 2024-03-19T01:39:02.831 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.824+0000 7f43f0297640 1 -- 172.21.15.99:0/108915896 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f43e819fa00 con 0x7f43e8100740 2024-03-19T01:39:02.831 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.824+0000 7f43df7fe640 1 -- 172.21.15.99:0/108915896 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f43e4010690 con 0x7f43e8100740 2024-03-19T01:39:02.831 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.824+0000 7f43f0297640 1 -- 172.21.15.99:0/108915896 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f43e819ff50 con 0x7f43e8100740 2024-03-19T01:39:02.831 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.825+0000 7f43f0297640 1 -- 172.21.15.99:0/108915896 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f43e81a0180 con 0x7f43e8100740 2024-03-19T01:39:02.832 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.826+0000 7f43df7fe640 1 -- 172.21.15.99:0/108915896 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 17) ==== 137341+0+0 (secure 0 0 0) 0x7f43e40037a0 con 0x7f43e8100740 2024-03-19T01:39:02.832 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.826+0000 7f43df7fe640 1 --2- 172.21.15.99:0/108915896 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f43d40a2890 0x7f43d40a4d50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:02.832 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.826+0000 7f43ed832640 1 --2- 172.21.15.99:0/108915896 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f43d40a2890 0x7f43d40a4d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:02.832 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.826+0000 7f43df7fe640 1 -- 172.21.15.99:0/108915896 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(87..87 src has 1..87) ==== 7979+0+0 (secure 0 0 0) 0x7f43e40cedf0 con 0x7f43e8100740 2024-03-19T01:39:02.832 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.827+0000 7f43ed832640 1 --2- 172.21.15.99:0/108915896 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f43d40a2890 0x7f43d40a4d50 secure :-1 s=READY pgs=175 cs=0 l=1 rev1=1 crypto rx=0x7f43e8199000 tx=0x7f43d8005f90 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:02.832 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:02.830+0000 7f43df7fe640 1 -- 172.21.15.99:0/108915896 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f43e409bee0 con 0x7f43e8100740 2024-03-19T01:39:03.005 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.004+0000 7f43f0297640 1 -- 172.21.15.99:0/108915896 --> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7f43e81a0350 con 0x7f43d40a2890 2024-03-19T01:39:03.012 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.011+0000 7f43df7fe640 1 -- 172.21.15.99:0/108915896 <== mgr.14150 v2:172.21.15.99:6800/4272410295 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+258767 (secure 0 0 0) 0x7f43e81a0350 con 0x7f43d40a2890 2024-03-19T01:39:03.016 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:39:03.018 INFO:teuthology.orchestra.run.smithi099.stderr:dumped all 2024-03-19T01:39:03.020 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.015+0000 7f43f0297640 1 -- 172.21.15.99:0/108915896 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f43d40a2890 msgr2=0x7f43d40a4d50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:03.020 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.015+0000 7f43f0297640 1 --2- 172.21.15.99:0/108915896 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f43d40a2890 0x7f43d40a4d50 secure :-1 s=READY pgs=175 cs=0 l=1 rev1=1 crypto rx=0x7f43e8199000 tx=0x7f43d8005f90 comp rx=0 tx=0).stop 2024-03-19T01:39:03.020 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.015+0000 7f43f0297640 1 -- 172.21.15.99:0/108915896 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f43e8100740 msgr2=0x7f43e8197af0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:03.020 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.015+0000 7f43f0297640 1 --2- 172.21.15.99:0/108915896 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f43e8100740 0x7f43e8197af0 secure :-1 s=READY pgs=228 cs=0 l=1 rev1=1 crypto rx=0x7f43e400e9c0 tx=0x7f43e400ee80 comp rx=0 tx=0).stop 2024-03-19T01:39:03.020 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.015+0000 7f43f0297640 1 -- 172.21.15.99:0/108915896 shutdown_connections 2024-03-19T01:39:03.020 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.015+0000 7f43f0297640 1 --2- 172.21.15.99:0/108915896 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f43d40a2890 0x7f43d40a4d50 unknown :-1 s=CLOSED pgs=175 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:03.020 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.015+0000 7f43f0297640 1 --2- 172.21.15.99:0/108915896 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f43e81022f0 0x7f43e819f0a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:03.020 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.015+0000 7f43f0297640 1 --2- 172.21.15.99:0/108915896 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f43e8101930 0x7f43e8198030 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:03.020 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.015+0000 7f43f0297640 1 --2- 172.21.15.99:0/108915896 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f43e8100740 0x7f43e8197af0 unknown :-1 s=CLOSED pgs=228 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:03.020 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.015+0000 7f43f0297640 1 -- 172.21.15.99:0/108915896 >> 172.21.15.99:0/108915896 conn(0x7f43e80fbcb0 msgr2=0x7f43e80fd700 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:03.021 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.015+0000 7f43f0297640 1 -- 172.21.15.99:0/108915896 shutdown_connections 2024-03-19T01:39:03.021 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.015+0000 7f43f0297640 1 -- 172.21.15.99:0/108915896 wait complete. 2024-03-19T01:39:03.026 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:02 smithi099 ceph-mon[62951]: from='client.14772 -' entity='client.admin' cmd=[{"prefix": "fs subvolume create", "vol_name": "cephfs", "sub_name": "sv_0", "group_name": "qa", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:39:03.026 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:02 smithi099 ceph-mon[62951]: from='client.14778 -' entity='client.admin' cmd=[{"prefix": "fs subvolume getpath", "vol_name": "cephfs", "group_name": "qa", "sub_name": "sv_0", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:39:03.026 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:02 smithi099 ceph-mon[62951]: from='client.14784 -' entity='client.admin' cmd=[{"prefix": "fs subvolume create", "vol_name": "cephfs", "sub_name": "sv_1", "group_name": "qa", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:39:03.027 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:02 smithi099 ceph-mon[62951]: mgrmap e17: x(active, since 6m), standbys: z, y 2024-03-19T01:39:03.027 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:02 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/954980700' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:39:03.027 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:02 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/3068977748' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-19T01:39:03.028 INFO:teuthology.orchestra.run.smithi099.stdout:{"pg_ready":true,"pg_map":{"version":285,"stamp":"2024-03-19T01:39:02.740784+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":596238,"num_objects":54,"num_object_clones":0,"num_object_copies":162,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":54,"num_whiteouts":0,"num_read":150,"num_read_kb":128,"num_write":295,"num_write_kb":2970,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":35,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":236,"ondisk_log_size":236,"up":291,"acting":291,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":217,"num_osds":12,"num_per_pool_osds":12,"num_per_pool_omap_osds":12,"kb":1124941824,"kb_used":374524,"kb_used_data":12860,"kb_used_omap":18,"kb_used_meta":361517,"kb_avail":1124567300,"statfs":{"total":1151940427776,"available":1151556915200,"internally_reserved":0,"allocated":13168640,"data_stored":7147113,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19139,"internal_metadata":370193725},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":5870,"num_objects":52,"num_object_clones":0,"num_object_copies":156,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":52,"num_whiteouts":0,"num_read":24,"num_read_kb":19,"num_write":114,"num_write_kb":32,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":35,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.962183"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.298802+0000","last_change":"2024-03-19T01:38:56.809406+0000","last_active":"2024-03-19T01:38:58.298802+0000","last_peered":"2024-03-19T01:38:58.298802+0000","last_clean":"2024-03-19T01:38:58.298802+0000","last_became_active":"2024-03-19T01:38:56.808971+0000","last_became_peered":"2024-03-19T01:38:56.808971+0000","last_unstale":"2024-03-19T01:38:58.298802+0000","last_undegraded":"2024-03-19T01:38:58.298802+0000","last_fullsized":"2024-03-19T01:38:58.298802+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T11:07:58.551704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.793364+0000","last_change":"2024-03-19T01:38:56.798847+0000","last_active":"2024-03-19T01:38:57.793364+0000","last_peered":"2024-03-19T01:38:57.793364+0000","last_clean":"2024-03-19T01:38:57.793364+0000","last_became_active":"2024-03-19T01:38:56.795442+0000","last_became_peered":"2024-03-19T01:38:56.795442+0000","last_unstale":"2024-03-19T01:38:57.793364+0000","last_undegraded":"2024-03-19T01:38:57.793364+0000","last_fullsized":"2024-03-19T01:38:57.793364+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T12:05:55.721962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,11,10],"acting":[0,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.793067+0000","last_change":"2024-03-19T01:38:56.806317+0000","last_active":"2024-03-19T01:38:57.793067+0000","last_peered":"2024-03-19T01:38:57.793067+0000","last_clean":"2024-03-19T01:38:57.793067+0000","last_became_active":"2024-03-19T01:38:56.806001+0000","last_became_peered":"2024-03-19T01:38:56.806001+0000","last_unstale":"2024-03-19T01:38:57.793067+0000","last_undegraded":"2024-03-19T01:38:57.793067+0000","last_fullsized":"2024-03-19T01:38:57.793067+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T05:19:45.501431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.298565+0000","last_change":"2024-03-19T01:38:56.803611+0000","last_active":"2024-03-19T01:38:58.298565+0000","last_peered":"2024-03-19T01:38:58.298565+0000","last_clean":"2024-03-19T01:38:58.298565+0000","last_became_active":"2024-03-19T01:38:56.803300+0000","last_became_peered":"2024-03-19T01:38:56.803300+0000","last_unstale":"2024-03-19T01:38:58.298565+0000","last_undegraded":"2024-03-19T01:38:58.298565+0000","last_fullsized":"2024-03-19T01:38:58.298565+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T06:05:49.287266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,8,2],"acting":[4,8,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.793673+0000","last_change":"2024-03-19T01:38:56.799910+0000","last_active":"2024-03-19T01:38:57.793673+0000","last_peered":"2024-03-19T01:38:57.793673+0000","last_clean":"2024-03-19T01:38:57.793673+0000","last_became_active":"2024-03-19T01:38:56.799567+0000","last_became_peered":"2024-03-19T01:38:56.799567+0000","last_unstale":"2024-03-19T01:38:57.793673+0000","last_undegraded":"2024-03-19T01:38:57.793673+0000","last_fullsized":"2024-03-19T01:38:57.793673+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T05:59:42.563083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,3,6],"acting":[8,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.922736+0000","last_change":"2024-03-19T01:38:56.806713+0000","last_active":"2024-03-19T01:38:57.922736+0000","last_peered":"2024-03-19T01:38:57.922736+0000","last_clean":"2024-03-19T01:38:57.922736+0000","last_became_active":"2024-03-19T01:38:56.804692+0000","last_became_peered":"2024-03-19T01:38:56.804692+0000","last_unstale":"2024-03-19T01:38:57.922736+0000","last_undegraded":"2024-03-19T01:38:57.922736+0000","last_fullsized":"2024-03-19T01:38:57.922736+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T11:45:02.994981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,9,4],"acting":[10,9,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.794545+0000","last_change":"2024-03-19T01:38:56.804091+0000","last_active":"2024-03-19T01:38:57.794545+0000","last_peered":"2024-03-19T01:38:57.794545+0000","last_clean":"2024-03-19T01:38:57.794545+0000","last_became_active":"2024-03-19T01:38:56.802308+0000","last_became_peered":"2024-03-19T01:38:56.802308+0000","last_unstale":"2024-03-19T01:38:57.794545+0000","last_undegraded":"2024-03-19T01:38:57.794545+0000","last_fullsized":"2024-03-19T01:38:57.794545+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T07:35:42.952521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,9,5],"acting":[7,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.799578+0000","last_change":"2024-03-19T01:38:56.796475+0000","last_active":"2024-03-19T01:38:57.799578+0000","last_peered":"2024-03-19T01:38:57.799578+0000","last_clean":"2024-03-19T01:38:57.799578+0000","last_became_active":"2024-03-19T01:38:56.796116+0000","last_became_peered":"2024-03-19T01:38:56.796116+0000","last_unstale":"2024-03-19T01:38:57.799578+0000","last_undegraded":"2024-03-19T01:38:57.799578+0000","last_fullsized":"2024-03-19T01:38:57.799578+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T04:35:19.821098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,10],"acting":[3,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.298585+0000","last_change":"2024-03-19T01:38:56.804197+0000","last_active":"2024-03-19T01:38:58.298585+0000","last_peered":"2024-03-19T01:38:58.298585+0000","last_clean":"2024-03-19T01:38:58.298585+0000","last_became_active":"2024-03-19T01:38:56.803209+0000","last_became_peered":"2024-03-19T01:38:56.803209+0000","last_unstale":"2024-03-19T01:38:58.298585+0000","last_undegraded":"2024-03-19T01:38:58.298585+0000","last_fullsized":"2024-03-19T01:38:58.298585+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T06:26:30.303689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,11],"acting":[1,5,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.923082+0000","last_change":"2024-03-19T01:38:56.797973+0000","last_active":"2024-03-19T01:38:57.923082+0000","last_peered":"2024-03-19T01:38:57.923082+0000","last_clean":"2024-03-19T01:38:57.923082+0000","last_became_active":"2024-03-19T01:38:56.797396+0000","last_became_peered":"2024-03-19T01:38:56.797396+0000","last_unstale":"2024-03-19T01:38:57.923082+0000","last_undegraded":"2024-03-19T01:38:57.923082+0000","last_fullsized":"2024-03-19T01:38:57.923082+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T11:46:32.867919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,2,4],"acting":[10,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.794962+0000","last_change":"2024-03-19T01:38:56.802211+0000","last_active":"2024-03-19T01:38:57.794962+0000","last_peered":"2024-03-19T01:38:57.794962+0000","last_clean":"2024-03-19T01:38:57.794962+0000","last_became_active":"2024-03-19T01:38:56.801776+0000","last_became_peered":"2024-03-19T01:38:56.801776+0000","last_unstale":"2024-03-19T01:38:57.794962+0000","last_undegraded":"2024-03-19T01:38:57.794962+0000","last_fullsized":"2024-03-19T01:38:57.794962+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T10:09:36.983771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,8,1],"acting":[7,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.298327+0000","last_change":"2024-03-19T01:38:56.798463+0000","last_active":"2024-03-19T01:38:58.298327+0000","last_peered":"2024-03-19T01:38:58.298327+0000","last_clean":"2024-03-19T01:38:58.298327+0000","last_became_active":"2024-03-19T01:38:56.797187+0000","last_became_peered":"2024-03-19T01:38:56.797187+0000","last_unstale":"2024-03-19T01:38:58.298327+0000","last_undegraded":"2024-03-19T01:38:58.298327+0000","last_fullsized":"2024-03-19T01:38:58.298327+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T11:10:04.349393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,8],"acting":[4,10,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.297712+0000","last_change":"2024-03-19T01:38:56.801650+0000","last_active":"2024-03-19T01:38:58.297712+0000","last_peered":"2024-03-19T01:38:58.297712+0000","last_clean":"2024-03-19T01:38:58.297712+0000","last_became_active":"2024-03-19T01:38:56.801323+0000","last_became_peered":"2024-03-19T01:38:56.801323+0000","last_unstale":"2024-03-19T01:38:58.297712+0000","last_undegraded":"2024-03-19T01:38:58.297712+0000","last_fullsized":"2024-03-19T01:38:58.297712+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T06:10:28.518843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,10,5],"acting":[6,10,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.793142+0000","last_change":"2024-03-19T01:38:56.798633+0000","last_active":"2024-03-19T01:38:57.793142+0000","last_peered":"2024-03-19T01:38:57.793142+0000","last_clean":"2024-03-19T01:38:57.793142+0000","last_became_active":"2024-03-19T01:38:56.798342+0000","last_became_peered":"2024-03-19T01:38:56.798342+0000","last_unstale":"2024-03-19T01:38:57.793142+0000","last_undegraded":"2024-03-19T01:38:57.793142+0000","last_fullsized":"2024-03-19T01:38:57.793142+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T03:21:57.350930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,8,10],"acting":[0,8,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.793138+0000","last_change":"2024-03-19T01:38:56.801516+0000","last_active":"2024-03-19T01:38:57.793138+0000","last_peered":"2024-03-19T01:38:57.793138+0000","last_clean":"2024-03-19T01:38:57.793138+0000","last_became_active":"2024-03-19T01:38:56.799602+0000","last_became_peered":"2024-03-19T01:38:56.799602+0000","last_unstale":"2024-03-19T01:38:57.793138+0000","last_undegraded":"2024-03-19T01:38:57.793138+0000","last_fullsized":"2024-03-19T01:38:57.793138+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T11:14:44.289556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.297811+0000","last_change":"2024-03-19T01:38:56.796556+0000","last_active":"2024-03-19T01:38:58.297811+0000","last_peered":"2024-03-19T01:38:58.297811+0000","last_clean":"2024-03-19T01:38:58.297811+0000","last_became_active":"2024-03-19T01:38:56.796215+0000","last_became_peered":"2024-03-19T01:38:56.796215+0000","last_unstale":"2024-03-19T01:38:58.297811+0000","last_undegraded":"2024-03-19T01:38:58.297811+0000","last_fullsized":"2024-03-19T01:38:58.297811+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T09:12:00.826877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.298955+0000","last_change":"2024-03-19T01:38:56.808706+0000","last_active":"2024-03-19T01:38:58.298955+0000","last_peered":"2024-03-19T01:38:58.298955+0000","last_clean":"2024-03-19T01:38:58.298955+0000","last_became_active":"2024-03-19T01:38:56.808065+0000","last_became_peered":"2024-03-19T01:38:56.808065+0000","last_unstale":"2024-03-19T01:38:58.298955+0000","last_undegraded":"2024-03-19T01:38:58.298955+0000","last_fullsized":"2024-03-19T01:38:58.298955+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T08:40:28.256172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.797935+0000","last_change":"2024-03-19T01:38:56.799501+0000","last_active":"2024-03-19T01:38:57.797935+0000","last_peered":"2024-03-19T01:38:57.797935+0000","last_clean":"2024-03-19T01:38:57.797935+0000","last_became_active":"2024-03-19T01:38:56.799202+0000","last_became_peered":"2024-03-19T01:38:56.799202+0000","last_unstale":"2024-03-19T01:38:57.797935+0000","last_undegraded":"2024-03-19T01:38:57.797935+0000","last_fullsized":"2024-03-19T01:38:57.797935+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T09:42:06.386165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,1,10],"acting":[9,1,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.297593+0000","last_change":"2024-03-19T01:38:56.810697+0000","last_active":"2024-03-19T01:38:58.297593+0000","last_peered":"2024-03-19T01:38:58.297593+0000","last_clean":"2024-03-19T01:38:58.297593+0000","last_became_active":"2024-03-19T01:38:56.810202+0000","last_became_peered":"2024-03-19T01:38:56.810202+0000","last_unstale":"2024-03-19T01:38:58.297593+0000","last_undegraded":"2024-03-19T01:38:58.297593+0000","last_fullsized":"2024-03-19T01:38:58.297593+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T06:22:11.807807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1f","version":"87'9","reported_seq":25,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:59.817756+0000","last_change":"2024-03-19T01:38:55.791479+0000","last_active":"2024-03-19T01:38:59.817756+0000","last_peered":"2024-03-19T01:38:59.817756+0000","last_clean":"2024-03-19T01:38:59.817756+0000","last_became_active":"2024-03-19T01:38:55.791150+0000","last_became_peered":"2024-03-19T01:38:55.791150+0000","last_unstale":"2024-03-19T01:38:59.817756+0000","last_undegraded":"2024-03-19T01:38:59.817756+0000","last_fullsized":"2024-03-19T01:38:59.817756+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T09:56:07.953602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,7,8],"acting":[9,7,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.800064+0000","last_change":"2024-03-19T01:38:56.802404+0000","last_active":"2024-03-19T01:38:57.800064+0000","last_peered":"2024-03-19T01:38:57.800064+0000","last_clean":"2024-03-19T01:38:57.800064+0000","last_became_active":"2024-03-19T01:38:56.802060+0000","last_became_peered":"2024-03-19T01:38:56.802060+0000","last_unstale":"2024-03-19T01:38:57.800064+0000","last_undegraded":"2024-03-19T01:38:57.800064+0000","last_fullsized":"2024-03-19T01:38:57.800064+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T07:45:34.287073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1e","version":"87'2","reported_seq":17,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.800156+0000","last_change":"2024-03-19T01:38:55.798249+0000","last_active":"2024-03-19T01:38:57.800156+0000","last_peered":"2024-03-19T01:38:57.800156+0000","last_clean":"2024-03-19T01:38:57.800156+0000","last_became_active":"2024-03-19T01:38:55.797952+0000","last_became_peered":"2024-03-19T01:38:55.797952+0000","last_unstale":"2024-03-19T01:38:57.800156+0000","last_undegraded":"2024-03-19T01:38:57.800156+0000","last_fullsized":"2024-03-19T01:38:57.800156+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T12:12:40.925462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.793449+0000","last_change":"2024-03-19T01:38:56.805992+0000","last_active":"2024-03-19T01:38:57.793449+0000","last_peered":"2024-03-19T01:38:57.793449+0000","last_clean":"2024-03-19T01:38:57.793449+0000","last_became_active":"2024-03-19T01:38:56.804997+0000","last_became_peered":"2024-03-19T01:38:56.804997+0000","last_unstale":"2024-03-19T01:38:57.793449+0000","last_undegraded":"2024-03-19T01:38:57.793449+0000","last_fullsized":"2024-03-19T01:38:57.793449+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T10:33:39.003792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,9],"acting":[0,1,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1d","version":"86'2","reported_seq":17,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.794052+0000","last_change":"2024-03-19T01:38:55.795296+0000","last_active":"2024-03-19T01:38:57.794052+0000","last_peered":"2024-03-19T01:38:57.794052+0000","last_clean":"2024-03-19T01:38:57.794052+0000","last_became_active":"2024-03-19T01:38:55.794321+0000","last_became_peered":"2024-03-19T01:38:55.794321+0000","last_unstale":"2024-03-19T01:38:57.794052+0000","last_undegraded":"2024-03-19T01:38:57.794052+0000","last_fullsized":"2024-03-19T01:38:57.794052+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T12:26:12.914801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.298260+0000","last_change":"2024-03-19T01:38:56.802337+0000","last_active":"2024-03-19T01:38:58.298260+0000","last_peered":"2024-03-19T01:38:58.298260+0000","last_clean":"2024-03-19T01:38:58.298260+0000","last_became_active":"2024-03-19T01:38:56.802032+0000","last_became_peered":"2024-03-19T01:38:56.802032+0000","last_unstale":"2024-03-19T01:38:58.298260+0000","last_undegraded":"2024-03-19T01:38:58.298260+0000","last_fullsized":"2024-03-19T01:38:58.298260+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T07:57:13.512027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,6],"acting":[1,10,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1c","version":"86'2","reported_seq":17,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.794040+0000","last_change":"2024-03-19T01:38:55.793796+0000","last_active":"2024-03-19T01:38:57.794040+0000","last_peered":"2024-03-19T01:38:57.794040+0000","last_clean":"2024-03-19T01:38:57.794040+0000","last_became_active":"2024-03-19T01:38:55.792818+0000","last_became_peered":"2024-03-19T01:38:55.792818+0000","last_unstale":"2024-03-19T01:38:57.794040+0000","last_undegraded":"2024-03-19T01:38:57.794040+0000","last_fullsized":"2024-03-19T01:38:57.794040+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T09:44:17.403644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,1,5],"acting":[8,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.297664+0000","last_change":"2024-03-19T01:38:56.801280+0000","last_active":"2024-03-19T01:38:58.297664+0000","last_peered":"2024-03-19T01:38:58.297664+0000","last_clean":"2024-03-19T01:38:58.297664+0000","last_became_active":"2024-03-19T01:38:56.800944+0000","last_became_peered":"2024-03-19T01:38:56.800944+0000","last_unstale":"2024-03-19T01:38:58.297664+0000","last_undegraded":"2024-03-19T01:38:58.297664+0000","last_fullsized":"2024-03-19T01:38:58.297664+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T03:09:45.373387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1b","version":"87'6","reported_seq":26,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:39:01.333431+0000","last_change":"2024-03-19T01:38:55.792563+0000","last_active":"2024-03-19T01:39:01.333431+0000","last_peered":"2024-03-19T01:39:01.333431+0000","last_clean":"2024-03-19T01:39:01.333431+0000","last_became_active":"2024-03-19T01:38:55.792098+0000","last_became_peered":"2024-03-19T01:38:55.792098+0000","last_unstale":"2024-03-19T01:39:01.333431+0000","last_undegraded":"2024-03-19T01:39:01.333431+0000","last_fullsized":"2024-03-19T01:39:01.333431+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T08:40:54.804413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1013,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":5,"num_read_kb":3,"num_write":3,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.793857+0000","last_change":"2024-03-19T01:38:56.810155+0000","last_active":"2024-03-19T01:38:57.793857+0000","last_peered":"2024-03-19T01:38:57.793857+0000","last_clean":"2024-03-19T01:38:57.793857+0000","last_became_active":"2024-03-19T01:38:56.809820+0000","last_became_peered":"2024-03-19T01:38:56.809820+0000","last_unstale":"2024-03-19T01:38:57.793857+0000","last_undegraded":"2024-03-19T01:38:57.793857+0000","last_fullsized":"2024-03-19T01:38:57.793857+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T12:44:30.463663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,9,5],"acting":[8,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.1a","version":"87'3","reported_seq":18,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:59.817422+0000","last_change":"2024-03-19T01:38:55.797324+0000","last_active":"2024-03-19T01:38:59.817422+0000","last_peered":"2024-03-19T01:38:59.817422+0000","last_clean":"2024-03-19T01:38:59.817422+0000","last_became_active":"2024-03-19T01:38:55.796997+0000","last_became_peered":"2024-03-19T01:38:55.796997+0000","last_unstale":"2024-03-19T01:38:59.817422+0000","last_undegraded":"2024-03-19T01:38:59.817422+0000","last_fullsized":"2024-03-19T01:38:59.817422+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T08:40:37.023902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,8,6],"acting":[10,8,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.297845+0000","last_change":"2024-03-19T01:38:56.785995+0000","last_active":"2024-03-19T01:38:58.297845+0000","last_peered":"2024-03-19T01:38:58.297845+0000","last_clean":"2024-03-19T01:38:58.297845+0000","last_became_active":"2024-03-19T01:38:56.785659+0000","last_became_peered":"2024-03-19T01:38:56.785659+0000","last_unstale":"2024-03-19T01:38:58.297845+0000","last_undegraded":"2024-03-19T01:38:58.297845+0000","last_fullsized":"2024-03-19T01:38:58.297845+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T06:40:33.389038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.19","version":"87'2","reported_seq":17,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:59.817555+0000","last_change":"2024-03-19T01:38:55.790034+0000","last_active":"2024-03-19T01:38:59.817555+0000","last_peered":"2024-03-19T01:38:59.817555+0000","last_clean":"2024-03-19T01:38:59.817555+0000","last_became_active":"2024-03-19T01:38:55.786369+0000","last_became_peered":"2024-03-19T01:38:55.786369+0000","last_unstale":"2024-03-19T01:38:59.817555+0000","last_undegraded":"2024-03-19T01:38:59.817555+0000","last_fullsized":"2024-03-19T01:38:59.817555+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T06:13:52.925644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,9],"acting":[6,4,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.799603+0000","last_change":"2024-03-19T01:38:56.791253+0000","last_active":"2024-03-19T01:38:57.799603+0000","last_peered":"2024-03-19T01:38:57.799603+0000","last_clean":"2024-03-19T01:38:57.799603+0000","last_became_active":"2024-03-19T01:38:56.790793+0000","last_became_peered":"2024-03-19T01:38:56.790793+0000","last_unstale":"2024-03-19T01:38:57.799603+0000","last_undegraded":"2024-03-19T01:38:57.799603+0000","last_fullsized":"2024-03-19T01:38:57.799603+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T05:14:15.277513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,6],"acting":[3,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"87'1","reported_seq":16,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.297684+0000","last_change":"2024-03-19T01:38:55.793379+0000","last_active":"2024-03-19T01:38:58.297684+0000","last_peered":"2024-03-19T01:38:58.297684+0000","last_clean":"2024-03-19T01:38:58.297684+0000","last_became_active":"2024-03-19T01:38:55.793035+0000","last_became_peered":"2024-03-19T01:38:55.793035+0000","last_unstale":"2024-03-19T01:38:58.297684+0000","last_undegraded":"2024-03-19T01:38:58.297684+0000","last_fullsized":"2024-03-19T01:38:58.297684+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T05:01:52.122826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.797836+0000","last_change":"2024-03-19T01:38:56.800540+0000","last_active":"2024-03-19T01:38:57.797836+0000","last_peered":"2024-03-19T01:38:57.797836+0000","last_clean":"2024-03-19T01:38:57.797836+0000","last_became_active":"2024-03-19T01:38:56.800199+0000","last_became_peered":"2024-03-19T01:38:56.800199+0000","last_unstale":"2024-03-19T01:38:57.797836+0000","last_undegraded":"2024-03-19T01:38:57.797836+0000","last_fullsized":"2024-03-19T01:38:57.797836+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T05:17:57.722253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,3,10],"acting":[9,3,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.17","version":"87'3","reported_seq":19,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.811349+0000","last_change":"2024-03-19T01:38:55.792930+0000","last_active":"2024-03-19T01:38:58.811349+0000","last_peered":"2024-03-19T01:38:58.811349+0000","last_clean":"2024-03-19T01:38:58.811349+0000","last_became_active":"2024-03-19T01:38:55.792602+0000","last_became_peered":"2024-03-19T01:38:55.792602+0000","last_unstale":"2024-03-19T01:38:58.811349+0000","last_undegraded":"2024-03-19T01:38:58.811349+0000","last_fullsized":"2024-03-19T01:38:58.811349+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T12:28:15.399311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,5,9],"acting":[10,5,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.298685+0000","last_change":"2024-03-19T01:38:56.801228+0000","last_active":"2024-03-19T01:38:58.298685+0000","last_peered":"2024-03-19T01:38:58.298685+0000","last_clean":"2024-03-19T01:38:58.298685+0000","last_became_active":"2024-03-19T01:38:56.800656+0000","last_became_peered":"2024-03-19T01:38:56.800656+0000","last_unstale":"2024-03-19T01:38:58.298685+0000","last_undegraded":"2024-03-19T01:38:58.298685+0000","last_fullsized":"2024-03-19T01:38:58.298685+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T05:43:32.565608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,9],"acting":[1,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.16","version":"87'2","reported_seq":17,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.811067+0000","last_change":"2024-03-19T01:38:55.795899+0000","last_active":"2024-03-19T01:38:58.811067+0000","last_peered":"2024-03-19T01:38:58.811067+0000","last_clean":"2024-03-19T01:38:58.811067+0000","last_became_active":"2024-03-19T01:38:55.795434+0000","last_became_peered":"2024-03-19T01:38:55.795434+0000","last_unstale":"2024-03-19T01:38:58.811067+0000","last_undegraded":"2024-03-19T01:38:58.811067+0000","last_fullsized":"2024-03-19T01:38:58.811067+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T02:32:39.376935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.793443+0000","last_change":"2024-03-19T01:38:56.794349+0000","last_active":"2024-03-19T01:38:57.793443+0000","last_peered":"2024-03-19T01:38:57.793443+0000","last_clean":"2024-03-19T01:38:57.793443+0000","last_became_active":"2024-03-19T01:38:56.794038+0000","last_became_peered":"2024-03-19T01:38:56.794038+0000","last_unstale":"2024-03-19T01:38:57.793443+0000","last_undegraded":"2024-03-19T01:38:57.793443+0000","last_fullsized":"2024-03-19T01:38:57.793443+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T09:44:17.534659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"87'4","reported_seq":20,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.810182+0000","last_change":"2024-03-19T01:38:55.793769+0000","last_active":"2024-03-19T01:38:58.810182+0000","last_peered":"2024-03-19T01:38:58.810182+0000","last_clean":"2024-03-19T01:38:58.810182+0000","last_became_active":"2024-03-19T01:38:55.793423+0000","last_became_peered":"2024-03-19T01:38:55.793423+0000","last_unstale":"2024-03-19T01:38:58.810182+0000","last_undegraded":"2024-03-19T01:38:58.810182+0000","last_fullsized":"2024-03-19T01:38:58.810182+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T12:56:37.933791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":672,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,10],"acting":[5,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.298491+0000","last_change":"2024-03-19T01:38:56.794855+0000","last_active":"2024-03-19T01:38:58.298491+0000","last_peered":"2024-03-19T01:38:58.298491+0000","last_clean":"2024-03-19T01:38:58.298491+0000","last_became_active":"2024-03-19T01:38:56.794553+0000","last_became_peered":"2024-03-19T01:38:56.794553+0000","last_unstale":"2024-03-19T01:38:58.298491+0000","last_undegraded":"2024-03-19T01:38:58.298491+0000","last_fullsized":"2024-03-19T01:38:58.298491+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T08:33:20.473620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,8,4],"acting":[1,8,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.14","version":"87'8","reported_seq":27,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:39:00.331846+0000","last_change":"2024-03-19T01:38:55.794317+0000","last_active":"2024-03-19T01:39:00.331846+0000","last_peered":"2024-03-19T01:39:00.331846+0000","last_clean":"2024-03-19T01:39:00.331846+0000","last_became_active":"2024-03-19T01:38:55.793970+0000","last_became_peered":"2024-03-19T01:38:55.793970+0000","last_unstale":"2024-03-19T01:39:00.331846+0000","last_undegraded":"2024-03-19T01:39:00.331846+0000","last_fullsized":"2024-03-19T01:39:00.331846+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T02:25:09.584906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1612,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":4,"num_read_kb":3,"num_write":9,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,1,2],"acting":[10,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.793864+0000","last_change":"2024-03-19T01:38:56.794400+0000","last_active":"2024-03-19T01:38:57.793864+0000","last_peered":"2024-03-19T01:38:57.793864+0000","last_clean":"2024-03-19T01:38:57.793864+0000","last_became_active":"2024-03-19T01:38:56.794027+0000","last_became_peered":"2024-03-19T01:38:56.794027+0000","last_unstale":"2024-03-19T01:38:57.793864+0000","last_undegraded":"2024-03-19T01:38:57.793864+0000","last_fullsized":"2024-03-19T01:38:57.793864+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T06:33:51.398239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,7],"acting":[8,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.13","version":"87'6","reported_seq":21,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:59.816398+0000","last_change":"2024-03-19T01:38:55.792812+0000","last_active":"2024-03-19T01:38:59.816398+0000","last_peered":"2024-03-19T01:38:59.816398+0000","last_clean":"2024-03-19T01:38:59.816398+0000","last_became_active":"2024-03-19T01:38:55.792422+0000","last_became_peered":"2024-03-19T01:38:55.792422+0000","last_unstale":"2024-03-19T01:38:59.816398+0000","last_undegraded":"2024-03-19T01:38:59.816398+0000","last_fullsized":"2024-03-19T01:38:59.816398+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T11:04:58.863405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,10,3],"acting":[2,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.297574+0000","last_change":"2024-03-19T01:38:56.794156+0000","last_active":"2024-03-19T01:38:58.297574+0000","last_peered":"2024-03-19T01:38:58.297574+0000","last_clean":"2024-03-19T01:38:58.297574+0000","last_became_active":"2024-03-19T01:38:56.793812+0000","last_became_peered":"2024-03-19T01:38:56.793812+0000","last_unstale":"2024-03-19T01:38:58.297574+0000","last_undegraded":"2024-03-19T01:38:58.297574+0000","last_fullsized":"2024-03-19T01:38:58.297574+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T05:00:53.379548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.12","version":"87'3","reported_seq":18,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.810237+0000","last_change":"2024-03-19T01:38:55.798220+0000","last_active":"2024-03-19T01:38:58.810237+0000","last_peered":"2024-03-19T01:38:58.810237+0000","last_clean":"2024-03-19T01:38:58.810237+0000","last_became_active":"2024-03-19T01:38:55.797901+0000","last_became_peered":"2024-03-19T01:38:55.797901+0000","last_unstale":"2024-03-19T01:38:58.810237+0000","last_undegraded":"2024-03-19T01:38:58.810237+0000","last_fullsized":"2024-03-19T01:38:58.810237+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T11:45:20.233596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.794988+0000","last_change":"2024-03-19T01:38:56.805603+0000","last_active":"2024-03-19T01:38:57.794988+0000","last_peered":"2024-03-19T01:38:57.794988+0000","last_clean":"2024-03-19T01:38:57.794988+0000","last_became_active":"2024-03-19T01:38:56.805248+0000","last_became_peered":"2024-03-19T01:38:56.805248+0000","last_unstale":"2024-03-19T01:38:57.794988+0000","last_undegraded":"2024-03-19T01:38:57.794988+0000","last_fullsized":"2024-03-19T01:38:57.794988+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T05:28:41.503673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"87'4","reported_seq":24,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:39:01.333924+0000","last_change":"2024-03-19T01:38:55.793254+0000","last_active":"2024-03-19T01:39:01.333924+0000","last_peered":"2024-03-19T01:39:01.333924+0000","last_clean":"2024-03-19T01:39:01.333924+0000","last_became_active":"2024-03-19T01:38:55.792925+0000","last_became_peered":"2024-03-19T01:38:55.792925+0000","last_unstale":"2024-03-19T01:39:01.333924+0000","last_undegraded":"2024-03-19T01:39:01.333924+0000","last_fullsized":"2024-03-19T01:39:01.333924+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T05:17:25.353050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":979,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":6,"num_read_kb":4,"num_write":3,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,10],"acting":[4,7,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.794465+0000","last_change":"2024-03-19T01:38:56.807845+0000","last_active":"2024-03-19T01:38:57.794465+0000","last_peered":"2024-03-19T01:38:57.794465+0000","last_clean":"2024-03-19T01:38:57.794465+0000","last_became_active":"2024-03-19T01:38:56.807175+0000","last_became_peered":"2024-03-19T01:38:56.807175+0000","last_unstale":"2024-03-19T01:38:57.794465+0000","last_undegraded":"2024-03-19T01:38:57.794465+0000","last_fullsized":"2024-03-19T01:38:57.794465+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T10:45:37.669659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.10","version":"86'8","reported_seq":23,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.793597+0000","last_change":"2024-03-19T01:38:55.793320+0000","last_active":"2024-03-19T01:38:57.793597+0000","last_peered":"2024-03-19T01:38:57.793597+0000","last_clean":"2024-03-19T01:38:57.793597+0000","last_became_active":"2024-03-19T01:38:55.793030+0000","last_became_peered":"2024-03-19T01:38:55.793030+0000","last_unstale":"2024-03-19T01:38:57.793597+0000","last_undegraded":"2024-03-19T01:38:57.793597+0000","last_fullsized":"2024-03-19T01:38:57.793597+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T04:01:31.028364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,2],"acting":[8,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.297443+0000","last_change":"2024-03-19T01:38:56.796338+0000","last_active":"2024-03-19T01:38:58.297443+0000","last_peered":"2024-03-19T01:38:58.297443+0000","last_clean":"2024-03-19T01:38:58.297443+0000","last_became_active":"2024-03-19T01:38:56.795660+0000","last_became_peered":"2024-03-19T01:38:56.795660+0000","last_unstale":"2024-03-19T01:38:58.297443+0000","last_undegraded":"2024-03-19T01:38:58.297443+0000","last_fullsized":"2024-03-19T01:38:58.297443+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T01:56:03.144907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.f","version":"86'1","reported_seq":16,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.794069+0000","last_change":"2024-03-19T01:38:55.791670+0000","last_active":"2024-03-19T01:38:57.794069+0000","last_peered":"2024-03-19T01:38:57.794069+0000","last_clean":"2024-03-19T01:38:57.794069+0000","last_became_active":"2024-03-19T01:38:55.790859+0000","last_became_peered":"2024-03-19T01:38:55.790859+0000","last_unstale":"2024-03-19T01:38:57.794069+0000","last_undegraded":"2024-03-19T01:38:57.794069+0000","last_fullsized":"2024-03-19T01:38:57.794069+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T08:13:16.770134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,5,4],"acting":[8,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.298742+0000","last_change":"2024-03-19T01:38:56.806291+0000","last_active":"2024-03-19T01:38:58.298742+0000","last_peered":"2024-03-19T01:38:58.298742+0000","last_clean":"2024-03-19T01:38:58.298742+0000","last_became_active":"2024-03-19T01:38:56.805972+0000","last_became_peered":"2024-03-19T01:38:56.805972+0000","last_unstale":"2024-03-19T01:38:58.298742+0000","last_undegraded":"2024-03-19T01:38:58.298742+0000","last_fullsized":"2024-03-19T01:38:58.298742+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T05:28:19.575710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,5,4],"acting":[11,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":15,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.797967+0000","last_change":"2024-03-19T01:38:55.798331+0000","last_active":"2024-03-19T01:38:57.797967+0000","last_peered":"2024-03-19T01:38:57.797967+0000","last_clean":"2024-03-19T01:38:57.797967+0000","last_became_active":"2024-03-19T01:38:55.797948+0000","last_became_peered":"2024-03-19T01:38:55.797948+0000","last_unstale":"2024-03-19T01:38:57.797967+0000","last_undegraded":"2024-03-19T01:38:57.797967+0000","last_fullsized":"2024-03-19T01:38:57.797967+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T04:35:09.036490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,2,0],"acting":[9,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.922859+0000","last_change":"2024-03-19T01:38:56.810162+0000","last_active":"2024-03-19T01:38:57.922859+0000","last_peered":"2024-03-19T01:38:57.922859+0000","last_clean":"2024-03-19T01:38:57.922859+0000","last_became_active":"2024-03-19T01:38:56.809888+0000","last_became_peered":"2024-03-19T01:38:56.809888+0000","last_unstale":"2024-03-19T01:38:57.922859+0000","last_undegraded":"2024-03-19T01:38:57.922859+0000","last_fullsized":"2024-03-19T01:38:57.922859+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T05:49:35.606065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,3,6],"acting":[10,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"2.2","version":"87'1","reported_seq":16,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.794773+0000","last_change":"2024-03-19T01:38:55.796629+0000","last_active":"2024-03-19T01:38:57.794773+0000","last_peered":"2024-03-19T01:38:57.794773+0000","last_clean":"2024-03-19T01:38:57.794773+0000","last_became_active":"2024-03-19T01:38:55.796232+0000","last_became_peered":"2024-03-19T01:38:55.796232+0000","last_unstale":"2024-03-19T01:38:57.794773+0000","last_undegraded":"2024-03-19T01:38:57.794773+0000","last_fullsized":"2024-03-19T01:38:57.794773+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T09:38:58.853129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.922918+0000","last_change":"2024-03-19T01:38:56.798227+0000","last_active":"2024-03-19T01:38:57.922918+0000","last_peered":"2024-03-19T01:38:57.922918+0000","last_clean":"2024-03-19T01:38:57.922918+0000","last_became_active":"2024-03-19T01:38:56.797302+0000","last_became_peered":"2024-03-19T01:38:56.797302+0000","last_unstale":"2024-03-19T01:38:57.922918+0000","last_undegraded":"2024-03-19T01:38:57.922918+0000","last_fullsized":"2024-03-19T01:38:57.922918+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T12:35:56.975118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,4,0],"acting":[10,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.297785+0000","last_change":"2024-03-19T01:38:56.802272+0000","last_active":"2024-03-19T01:38:58.297785+0000","last_peered":"2024-03-19T01:38:58.297785+0000","last_clean":"2024-03-19T01:38:58.297785+0000","last_became_active":"2024-03-19T01:38:56.801854+0000","last_became_peered":"2024-03-19T01:38:56.801854+0000","last_unstale":"2024-03-19T01:38:58.297785+0000","last_undegraded":"2024-03-19T01:38:58.297785+0000","last_fullsized":"2024-03-19T01:38:58.297785+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T11:32:23.363896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"87'4","reported_seq":21,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:39:00.330850+0000","last_change":"2024-03-19T01:38:55.785957+0000","last_active":"2024-03-19T01:39:00.330850+0000","last_peered":"2024-03-19T01:39:00.330850+0000","last_clean":"2024-03-19T01:39:00.330850+0000","last_became_active":"2024-03-19T01:38:55.785527+0000","last_became_peered":"2024-03-19T01:38:55.785527+0000","last_unstale":"2024-03-19T01:39:00.330850+0000","last_undegraded":"2024-03-19T01:39:00.330850+0000","last_fullsized":"2024-03-19T01:39:00.330850+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T09:33:55.329126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.800040+0000","last_change":"2024-03-19T01:38:56.798213+0000","last_active":"2024-03-19T01:38:57.800040+0000","last_peered":"2024-03-19T01:38:57.800040+0000","last_clean":"2024-03-19T01:38:57.800040+0000","last_became_active":"2024-03-19T01:38:56.797475+0000","last_became_peered":"2024-03-19T01:38:56.797475+0000","last_unstale":"2024-03-19T01:38:57.800040+0000","last_undegraded":"2024-03-19T01:38:57.800040+0000","last_fullsized":"2024-03-19T01:38:57.800040+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T07:41:42.021626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"87'3","reported_seq":20,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:39:00.329317+0000","last_change":"2024-03-19T01:38:55.792521+0000","last_active":"2024-03-19T01:39:00.329317+0000","last_peered":"2024-03-19T01:39:00.329317+0000","last_clean":"2024-03-19T01:39:00.329317+0000","last_became_active":"2024-03-19T01:38:55.792166+0000","last_became_peered":"2024-03-19T01:38:55.792166+0000","last_unstale":"2024-03-19T01:39:00.329317+0000","last_undegraded":"2024-03-19T01:39:00.329317+0000","last_fullsized":"2024-03-19T01:39:00.329317+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T08:15:12.216641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":4,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,9],"acting":[4,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.0","version":"23'137","reported_seq":340,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.794520+0000","last_change":"2024-03-19T01:37:48.171712+0000","last_active":"2024-03-19T01:38:57.794520+0000","last_peered":"2024-03-19T01:38:57.794520+0000","last_clean":"2024-03-19T01:38:57.794520+0000","last_became_active":"2024-03-19T01:37:48.171323+0000","last_became_peered":"2024-03-19T01:37:48.171323+0000","last_unstale":"2024-03-19T01:38:57.794520+0000","last_undegraded":"2024-03-19T01:38:57.794520+0000","last_fullsized":"2024-03-19T01:38:57.794520+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:34:31.088679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:34:31.088679+0000","last_clean_scrub_stamp":"2024-03-19T01:34:31.088679+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T02:22:49.477567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":109,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.297458+0000","last_change":"2024-03-19T01:38:56.797868+0000","last_active":"2024-03-19T01:38:58.297458+0000","last_peered":"2024-03-19T01:38:58.297458+0000","last_clean":"2024-03-19T01:38:58.297458+0000","last_became_active":"2024-03-19T01:38:56.796855+0000","last_became_peered":"2024-03-19T01:38:56.796855+0000","last_unstale":"2024-03-19T01:38:58.297458+0000","last_undegraded":"2024-03-19T01:38:58.297458+0000","last_fullsized":"2024-03-19T01:38:58.297458+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T01:53:09.887368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"87'3","reported_seq":18,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:59.816889+0000","last_change":"2024-03-19T01:38:55.797507+0000","last_active":"2024-03-19T01:38:59.816889+0000","last_peered":"2024-03-19T01:38:59.816889+0000","last_clean":"2024-03-19T01:38:59.816889+0000","last_became_active":"2024-03-19T01:38:55.797167+0000","last_became_peered":"2024-03-19T01:38:55.797167+0000","last_unstale":"2024-03-19T01:38:59.816889+0000","last_undegraded":"2024-03-19T01:38:59.816889+0000","last_fullsized":"2024-03-19T01:38:59.816889+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T01:45:06.037491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,8],"acting":[7,5,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":15,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.793854+0000","last_change":"2024-03-19T01:38:55.795464+0000","last_active":"2024-03-19T01:38:57.793854+0000","last_peered":"2024-03-19T01:38:57.793854+0000","last_clean":"2024-03-19T01:38:57.793854+0000","last_became_active":"2024-03-19T01:38:55.793827+0000","last_became_peered":"2024-03-19T01:38:55.793827+0000","last_unstale":"2024-03-19T01:38:57.793854+0000","last_undegraded":"2024-03-19T01:38:57.793854+0000","last_fullsized":"2024-03-19T01:38:57.793854+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T13:03:56.862269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,9],"acting":[8,0,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.297755+0000","last_change":"2024-03-19T01:38:56.795569+0000","last_active":"2024-03-19T01:38:58.297755+0000","last_peered":"2024-03-19T01:38:58.297755+0000","last_clean":"2024-03-19T01:38:58.297755+0000","last_became_active":"2024-03-19T01:38:56.795281+0000","last_became_peered":"2024-03-19T01:38:56.795281+0000","last_unstale":"2024-03-19T01:38:58.297755+0000","last_undegraded":"2024-03-19T01:38:58.297755+0000","last_fullsized":"2024-03-19T01:38:58.297755+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T03:18:00.599307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,9],"acting":[6,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"87'4","reported_seq":19,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:59.817344+0000","last_change":"2024-03-19T01:38:55.792306+0000","last_active":"2024-03-19T01:38:59.817344+0000","last_peered":"2024-03-19T01:38:59.817344+0000","last_clean":"2024-03-19T01:38:59.817344+0000","last_became_active":"2024-03-19T01:38:55.792019+0000","last_became_peered":"2024-03-19T01:38:55.792019+0000","last_unstale":"2024-03-19T01:38:59.817344+0000","last_undegraded":"2024-03-19T01:38:59.817344+0000","last_fullsized":"2024-03-19T01:38:59.817344+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T06:33:06.869054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,8,3],"acting":[9,8,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.800042+0000","last_change":"2024-03-19T01:38:56.798314+0000","last_active":"2024-03-19T01:38:57.800042+0000","last_peered":"2024-03-19T01:38:57.800042+0000","last_clean":"2024-03-19T01:38:57.800042+0000","last_became_active":"2024-03-19T01:38:56.797999+0000","last_became_peered":"2024-03-19T01:38:56.797999+0000","last_unstale":"2024-03-19T01:38:57.800042+0000","last_undegraded":"2024-03-19T01:38:57.800042+0000","last_fullsized":"2024-03-19T01:38:57.800042+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T06:23:36.861238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,7],"acting":[3,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"87'4","reported_seq":19,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.809263+0000","last_change":"2024-03-19T01:38:55.790866+0000","last_active":"2024-03-19T01:38:58.809263+0000","last_peered":"2024-03-19T01:38:58.809263+0000","last_clean":"2024-03-19T01:38:58.809263+0000","last_became_active":"2024-03-19T01:38:55.790363+0000","last_became_peered":"2024-03-19T01:38:55.790363+0000","last_unstale":"2024-03-19T01:38:58.809263+0000","last_undegraded":"2024-03-19T01:38:58.809263+0000","last_fullsized":"2024-03-19T01:38:58.809263+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T09:00:27.595598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,11,6],"acting":[4,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.793331+0000","last_change":"2024-03-19T01:38:56.793369+0000","last_active":"2024-03-19T01:38:57.793331+0000","last_peered":"2024-03-19T01:38:57.793331+0000","last_clean":"2024-03-19T01:38:57.793331+0000","last_became_active":"2024-03-19T01:38:56.793068+0000","last_became_peered":"2024-03-19T01:38:56.793068+0000","last_unstale":"2024-03-19T01:38:57.793331+0000","last_undegraded":"2024-03-19T01:38:57.793331+0000","last_fullsized":"2024-03-19T01:38:57.793331+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T13:12:57.954100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"87'2","reported_seq":17,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.810496+0000","last_change":"2024-03-19T01:38:55.793869+0000","last_active":"2024-03-19T01:38:58.810496+0000","last_peered":"2024-03-19T01:38:58.810496+0000","last_clean":"2024-03-19T01:38:58.810496+0000","last_became_active":"2024-03-19T01:38:55.791841+0000","last_became_peered":"2024-03-19T01:38:55.791841+0000","last_unstale":"2024-03-19T01:38:58.810496+0000","last_undegraded":"2024-03-19T01:38:58.810496+0000","last_fullsized":"2024-03-19T01:38:58.810496+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T06:57:21.802060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.793600+0000","last_change":"2024-03-19T01:38:56.802272+0000","last_active":"2024-03-19T01:38:57.793600+0000","last_peered":"2024-03-19T01:38:57.793600+0000","last_clean":"2024-03-19T01:38:57.793600+0000","last_became_active":"2024-03-19T01:38:56.801071+0000","last_became_peered":"2024-03-19T01:38:56.801071+0000","last_unstale":"2024-03-19T01:38:57.793600+0000","last_undegraded":"2024-03-19T01:38:57.793600+0000","last_fullsized":"2024-03-19T01:38:57.793600+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T06:58:12.762975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,10],"acting":[8,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.9","version":"87'3","reported_seq":18,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.810245+0000","last_change":"2024-03-19T01:38:55.795186+0000","last_active":"2024-03-19T01:38:58.810245+0000","last_peered":"2024-03-19T01:38:58.810245+0000","last_clean":"2024-03-19T01:38:58.810245+0000","last_became_active":"2024-03-19T01:38:55.794635+0000","last_became_peered":"2024-03-19T01:38:55.794635+0000","last_unstale":"2024-03-19T01:38:58.810245+0000","last_undegraded":"2024-03-19T01:38:58.810245+0000","last_fullsized":"2024-03-19T01:38:58.810245+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T03:55:56.322716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,9,10],"acting":[3,9,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.298693+0000","last_change":"2024-03-19T01:38:56.812656+0000","last_active":"2024-03-19T01:38:58.298693+0000","last_peered":"2024-03-19T01:38:58.298693+0000","last_clean":"2024-03-19T01:38:58.298693+0000","last_became_active":"2024-03-19T01:38:56.811824+0000","last_became_peered":"2024-03-19T01:38:56.811824+0000","last_unstale":"2024-03-19T01:38:58.298693+0000","last_undegraded":"2024-03-19T01:38:58.298693+0000","last_fullsized":"2024-03-19T01:38:58.298693+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T04:28:11.866892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,0,2],"acting":[11,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.a","version":"87'1","reported_seq":16,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.297486+0000","last_change":"2024-03-19T01:38:55.791193+0000","last_active":"2024-03-19T01:38:58.297486+0000","last_peered":"2024-03-19T01:38:58.297486+0000","last_clean":"2024-03-19T01:38:58.297486+0000","last_became_active":"2024-03-19T01:38:55.790820+0000","last_became_peered":"2024-03-19T01:38:55.790820+0000","last_unstale":"2024-03-19T01:38:58.297486+0000","last_undegraded":"2024-03-19T01:38:58.297486+0000","last_fullsized":"2024-03-19T01:38:58.297486+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T10:19:09.564798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,8],"acting":[2,3,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.799870+0000","last_change":"2024-03-19T01:38:56.803287+0000","last_active":"2024-03-19T01:38:57.799870+0000","last_peered":"2024-03-19T01:38:57.799870+0000","last_clean":"2024-03-19T01:38:57.799870+0000","last_became_active":"2024-03-19T01:38:56.802924+0000","last_became_peered":"2024-03-19T01:38:56.802924+0000","last_unstale":"2024-03-19T01:38:57.799870+0000","last_undegraded":"2024-03-19T01:38:57.799870+0000","last_fullsized":"2024-03-19T01:38:57.799870+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T07:45:23.726378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,10],"acting":[3,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"86'2","reported_seq":17,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.793690+0000","last_change":"2024-03-19T01:38:55.795320+0000","last_active":"2024-03-19T01:38:57.793690+0000","last_peered":"2024-03-19T01:38:57.793690+0000","last_clean":"2024-03-19T01:38:57.793690+0000","last_became_active":"2024-03-19T01:38:55.794478+0000","last_became_peered":"2024-03-19T01:38:55.794478+0000","last_unstale":"2024-03-19T01:38:57.793690+0000","last_undegraded":"2024-03-19T01:38:57.793690+0000","last_fullsized":"2024-03-19T01:38:57.793690+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T05:03:45.867009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.794801+0000","last_change":"2024-03-19T01:38:56.797272+0000","last_active":"2024-03-19T01:38:57.794801+0000","last_peered":"2024-03-19T01:38:57.794801+0000","last_clean":"2024-03-19T01:38:57.794801+0000","last_became_active":"2024-03-19T01:38:56.796944+0000","last_became_peered":"2024-03-19T01:38:56.796944+0000","last_unstale":"2024-03-19T01:38:57.794801+0000","last_undegraded":"2024-03-19T01:38:57.794801+0000","last_fullsized":"2024-03-19T01:38:57.794801+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T06:14:58.147093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,6],"acting":[7,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.c","version":"87'3","reported_seq":18,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:59.817159+0000","last_change":"2024-03-19T01:38:55.795642+0000","last_active":"2024-03-19T01:38:59.817159+0000","last_peered":"2024-03-19T01:38:59.817159+0000","last_clean":"2024-03-19T01:38:59.817159+0000","last_became_active":"2024-03-19T01:38:55.795285+0000","last_became_peered":"2024-03-19T01:38:55.795285+0000","last_unstale":"2024-03-19T01:38:59.817159+0000","last_undegraded":"2024-03-19T01:38:59.817159+0000","last_fullsized":"2024-03-19T01:38:59.817159+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T10:09:18.888802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,11,0],"acting":[2,11,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.795015+0000","last_change":"2024-03-19T01:38:56.801802+0000","last_active":"2024-03-19T01:38:57.795015+0000","last_peered":"2024-03-19T01:38:57.795015+0000","last_clean":"2024-03-19T01:38:57.795015+0000","last_became_active":"2024-03-19T01:38:56.801130+0000","last_became_peered":"2024-03-19T01:38:56.801130+0000","last_unstale":"2024-03-19T01:38:57.795015+0000","last_undegraded":"2024-03-19T01:38:57.795015+0000","last_fullsized":"2024-03-19T01:38:57.795015+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T10:36:45.509217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,11],"acting":[7,1,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"86'1","reported_seq":17,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.297606+0000","last_change":"2024-03-19T01:38:55.795085+0000","last_active":"2024-03-19T01:38:58.297606+0000","last_peered":"2024-03-19T01:38:58.297606+0000","last_clean":"2024-03-19T01:38:58.297606+0000","last_became_active":"2024-03-19T01:38:55.794626+0000","last_became_peered":"2024-03-19T01:38:55.794626+0000","last_unstale":"2024-03-19T01:38:58.297606+0000","last_undegraded":"2024-03-19T01:38:58.297606+0000","last_fullsized":"2024-03-19T01:38:58.297606+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T05:51:50.428331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,10,3],"acting":[5,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.794739+0000","last_change":"2024-03-19T01:38:56.802141+0000","last_active":"2024-03-19T01:38:57.794739+0000","last_peered":"2024-03-19T01:38:57.794739+0000","last_clean":"2024-03-19T01:38:57.794739+0000","last_became_active":"2024-03-19T01:38:56.801788+0000","last_became_peered":"2024-03-19T01:38:56.801788+0000","last_unstale":"2024-03-19T01:38:57.794739+0000","last_undegraded":"2024-03-19T01:38:57.794739+0000","last_fullsized":"2024-03-19T01:38:57.794739+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T10:13:39.264272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"86'2","reported_seq":17,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.297633+0000","last_change":"2024-03-19T01:38:55.792956+0000","last_active":"2024-03-19T01:38:58.297633+0000","last_peered":"2024-03-19T01:38:58.297633+0000","last_clean":"2024-03-19T01:38:58.297633+0000","last_became_active":"2024-03-19T01:38:55.792633+0000","last_became_peered":"2024-03-19T01:38:55.792633+0000","last_unstale":"2024-03-19T01:38:58.297633+0000","last_undegraded":"2024-03-19T01:38:58.297633+0000","last_fullsized":"2024-03-19T01:38:58.297633+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T11:17:14.276294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.794986+0000","last_change":"2024-03-19T01:38:56.794428+0000","last_active":"2024-03-19T01:38:57.794986+0000","last_peered":"2024-03-19T01:38:57.794986+0000","last_clean":"2024-03-19T01:38:57.794986+0000","last_became_active":"2024-03-19T01:38:56.794051+0000","last_became_peered":"2024-03-19T01:38:56.794051+0000","last_unstale":"2024-03-19T01:38:57.794986+0000","last_undegraded":"2024-03-19T01:38:57.794986+0000","last_fullsized":"2024-03-19T01:38:57.794986+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T02:07:20.707276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,11,8],"acting":[7,11,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.298850+0000","last_change":"2024-03-19T01:38:56.812596+0000","last_active":"2024-03-19T01:38:58.298850+0000","last_peered":"2024-03-19T01:38:58.298850+0000","last_clean":"2024-03-19T01:38:58.298850+0000","last_became_active":"2024-03-19T01:38:56.812268+0000","last_became_peered":"2024-03-19T01:38:56.812268+0000","last_unstale":"2024-03-19T01:38:58.298850+0000","last_undegraded":"2024-03-19T01:38:58.298850+0000","last_fullsized":"2024-03-19T01:38:58.298850+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T08:54:51.350908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,2,5],"acting":[11,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.297648+0000","last_change":"2024-03-19T01:38:56.804280+0000","last_active":"2024-03-19T01:38:58.297648+0000","last_peered":"2024-03-19T01:38:58.297648+0000","last_clean":"2024-03-19T01:38:58.297648+0000","last_became_active":"2024-03-19T01:38:56.801928+0000","last_became_peered":"2024-03-19T01:38:56.801928+0000","last_unstale":"2024-03-19T01:38:58.297648+0000","last_undegraded":"2024-03-19T01:38:58.297648+0000","last_fullsized":"2024-03-19T01:38:58.297648+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T11:49:29.778949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.297370+0000","last_change":"2024-03-19T01:38:56.798719+0000","last_active":"2024-03-19T01:38:58.297370+0000","last_peered":"2024-03-19T01:38:58.297370+0000","last_clean":"2024-03-19T01:38:58.297370+0000","last_became_active":"2024-03-19T01:38:56.798253+0000","last_became_peered":"2024-03-19T01:38:56.798253+0000","last_unstale":"2024-03-19T01:38:58.297370+0000","last_undegraded":"2024-03-19T01:38:58.297370+0000","last_fullsized":"2024-03-19T01:38:58.297370+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T06:27:56.754679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.793136+0000","last_change":"2024-03-19T01:38:56.795221+0000","last_active":"2024-03-19T01:38:57.793136+0000","last_peered":"2024-03-19T01:38:57.793136+0000","last_clean":"2024-03-19T01:38:57.793136+0000","last_became_active":"2024-03-19T01:38:56.794756+0000","last_became_peered":"2024-03-19T01:38:56.794756+0000","last_unstale":"2024-03-19T01:38:57.793136+0000","last_undegraded":"2024-03-19T01:38:57.793136+0000","last_fullsized":"2024-03-19T01:38:57.793136+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T10:05:54.394976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.297599+0000","last_change":"2024-03-19T01:38:56.794482+0000","last_active":"2024-03-19T01:38:58.297599+0000","last_peered":"2024-03-19T01:38:58.297599+0000","last_clean":"2024-03-19T01:38:58.297599+0000","last_became_active":"2024-03-19T01:38:56.794140+0000","last_became_peered":"2024-03-19T01:38:56.794140+0000","last_unstale":"2024-03-19T01:38:58.297599+0000","last_undegraded":"2024-03-19T01:38:58.297599+0000","last_fullsized":"2024-03-19T01:38:58.297599+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T04:29:38.985047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.298663+0000","last_change":"2024-03-19T01:38:56.802020+0000","last_active":"2024-03-19T01:38:58.298663+0000","last_peered":"2024-03-19T01:38:58.298663+0000","last_clean":"2024-03-19T01:38:58.298663+0000","last_became_active":"2024-03-19T01:38:56.801668+0000","last_became_peered":"2024-03-19T01:38:56.801668+0000","last_unstale":"2024-03-19T01:38:58.298663+0000","last_undegraded":"2024-03-19T01:38:58.298663+0000","last_fullsized":"2024-03-19T01:38:58.298663+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T04:13:55.199279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,8,5],"acting":[11,8,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.297975+0000","last_change":"2024-03-19T01:38:56.809100+0000","last_active":"2024-03-19T01:38:58.297975+0000","last_peered":"2024-03-19T01:38:58.297975+0000","last_clean":"2024-03-19T01:38:58.297975+0000","last_became_active":"2024-03-19T01:38:56.807985+0000","last_became_peered":"2024-03-19T01:38:56.807985+0000","last_unstale":"2024-03-19T01:38:58.297975+0000","last_undegraded":"2024-03-19T01:38:58.297975+0000","last_fullsized":"2024-03-19T01:38:58.297975+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T02:58:25.235837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,8,1],"acting":[2,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.799942+0000","last_change":"2024-03-19T01:38:56.799707+0000","last_active":"2024-03-19T01:38:57.799942+0000","last_peered":"2024-03-19T01:38:57.799942+0000","last_clean":"2024-03-19T01:38:57.799942+0000","last_became_active":"2024-03-19T01:38:56.799388+0000","last_became_peered":"2024-03-19T01:38:56.799388+0000","last_unstale":"2024-03-19T01:38:57.799942+0000","last_undegraded":"2024-03-19T01:38:57.799942+0000","last_fullsized":"2024-03-19T01:38:57.799942+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T11:10:18.360180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.297772+0000","last_change":"2024-03-19T01:38:56.786807+0000","last_active":"2024-03-19T01:38:58.297772+0000","last_peered":"2024-03-19T01:38:58.297772+0000","last_clean":"2024-03-19T01:38:58.297772+0000","last_became_active":"2024-03-19T01:38:56.786395+0000","last_became_peered":"2024-03-19T01:38:56.786395+0000","last_unstale":"2024-03-19T01:38:58.297772+0000","last_undegraded":"2024-03-19T01:38:58.297772+0000","last_fullsized":"2024-03-19T01:38:58.297772+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T05:19:20.228926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,9,3],"acting":[5,9,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.799636+0000","last_change":"2024-03-19T01:38:56.800759+0000","last_active":"2024-03-19T01:38:57.799636+0000","last_peered":"2024-03-19T01:38:57.799636+0000","last_clean":"2024-03-19T01:38:57.799636+0000","last_became_active":"2024-03-19T01:38:56.800462+0000","last_became_peered":"2024-03-19T01:38:56.800462+0000","last_unstale":"2024-03-19T01:38:57.799636+0000","last_undegraded":"2024-03-19T01:38:57.799636+0000","last_fullsized":"2024-03-19T01:38:57.799636+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T02:13:19.070148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,11],"acting":[3,2,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.298221+0000","last_change":"2024-03-19T01:38:56.802653+0000","last_active":"2024-03-19T01:38:58.298221+0000","last_peered":"2024-03-19T01:38:58.298221+0000","last_clean":"2024-03-19T01:38:58.298221+0000","last_became_active":"2024-03-19T01:38:56.802284+0000","last_became_peered":"2024-03-19T01:38:56.802284+0000","last_unstale":"2024-03-19T01:38:58.298221+0000","last_undegraded":"2024-03-19T01:38:58.298221+0000","last_fullsized":"2024-03-19T01:38:58.298221+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T06:17:13.936359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.297697+0000","last_change":"2024-03-19T01:38:56.794668+0000","last_active":"2024-03-19T01:38:58.297697+0000","last_peered":"2024-03-19T01:38:58.297697+0000","last_clean":"2024-03-19T01:38:58.297697+0000","last_became_active":"2024-03-19T01:38:56.794375+0000","last_became_peered":"2024-03-19T01:38:56.794375+0000","last_unstale":"2024-03-19T01:38:58.297697+0000","last_undegraded":"2024-03-19T01:38:58.297697+0000","last_fullsized":"2024-03-19T01:38:58.297697+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T08:26:20.528825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.298601+0000","last_change":"2024-03-19T01:38:56.805535+0000","last_active":"2024-03-19T01:38:58.298601+0000","last_peered":"2024-03-19T01:38:58.298601+0000","last_clean":"2024-03-19T01:38:58.298601+0000","last_became_active":"2024-03-19T01:38:56.805211+0000","last_became_peered":"2024-03-19T01:38:56.805211+0000","last_unstale":"2024-03-19T01:38:58.298601+0000","last_undegraded":"2024-03-19T01:38:58.298601+0000","last_fullsized":"2024-03-19T01:38:58.298601+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T04:47:28.688137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,4,8],"acting":[11,4,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":234,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":192,"acting":192,"num_store_stats":12},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":109,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4157440,"data_stored":4132576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":137,"ondisk_log_size":137,"up":3,"acting":3,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":5870,"num_objects":52,"num_object_clones":0,"num_object_copies":156,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":52,"num_whiteouts":0,"num_read":24,"num_read_kb":19,"num_write":114,"num_write_kb":32,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":35,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":225280,"data_stored":34991,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":99,"ondisk_log_size":99,"up":96,"acting":96,"num_store_stats":12}],"osd_stats":[{"osd":11,"up_from":82,"seq":352187318285,"num_pgs":18,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27932,"kb_used_data":912,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717220,"statfs":{"total":95995035648,"available":95966433280,"internally_reserved":0,"allocated":933888,"data_stored":447510,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1596,"internal_metadata":27654596},"hb_peers":[0,1,2,4,5,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":10,"up_from":74,"seq":317827579923,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32712,"kb_used_data":1536,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93712440,"statfs":{"total":95995035648,"available":95961538560,"internally_reserved":0,"allocated":1572864,"data_stored":1041068,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1600,"internal_metadata":31914432},"hb_peers":[0,1,2,3,4,5,6,8,9,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":9,"up_from":68,"seq":292057776152,"num_pgs":17,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32108,"kb_used_data":932,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93713044,"statfs":{"total":95995035648,"available":95962157056,"internally_reserved":0,"allocated":954368,"data_stored":448328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1599,"internal_metadata":31914433},"hb_peers":[0,1,2,3,4,5,6,7,8,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":8,"up_from":60,"seq":257698037787,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27940,"kb_used_data":920,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717212,"statfs":{"total":95995035648,"available":95966425088,"internally_reserved":0,"allocated":942080,"data_stored":448114,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,2,3,4,5,6,7,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":7,"up_from":53,"seq":227633266722,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32684,"kb_used_data":1508,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93712468,"statfs":{"total":95995035648,"available":95961567232,"internally_reserved":0,"allocated":1544192,"data_stored":1040452,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":31914443},"hb_peers":[1,2,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":6,"up_from":47,"seq":201863462951,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32096,"kb_used_data":920,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93713056,"statfs":{"total":95995035648,"available":95962169344,"internally_reserved":0,"allocated":942080,"data_stored":447717,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":31914440},"hb_peers":[0,1,2,3,4,5,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":13,"seq":55834574910,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27944,"kb_used_data":924,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717208,"statfs":{"total":95995035648,"available":95966420992,"internally_reserved":0,"allocated":946176,"data_stored":449122,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2,3,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738435,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36248,"kb_used_data":904,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":93708904,"statfs":{"total":95995035648,"available":95957917696,"internally_reserved":0,"allocated":925696,"data_stored":439030,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":36174277},"hb_peers":[1,2,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":18,"seq":77309411385,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27956,"kb_used_data":936,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717196,"statfs":{"total":95995035648,"available":95966408704,"internally_reserved":0,"allocated":958464,"data_stored":449778,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,3,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":27,"seq":115964117045,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32108,"kb_used_data":924,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93713044,"statfs":{"total":95995035648,"available":95962157056,"internally_reserved":0,"allocated":946176,"data_stored":447704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31914439},"hb_peers":[0,1,2,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":34,"seq":146028888111,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27940,"kb_used_data":924,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717212,"statfs":{"total":95995035648,"available":95966425088,"internally_reserved":0,"allocated":946176,"data_stored":448545,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2,3,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":40,"seq":171798691883,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36856,"kb_used_data":1520,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":93708296,"statfs":{"total":95995035648,"available":95957295104,"internally_reserved":0,"allocated":1556480,"data_stored":1039745,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":36174277},"hb_peers":[0,1,2,3,4,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":20895,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":36864,"data_stored":2962,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1035,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1867,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":2574,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":818,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":45056,"data_stored":3190,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-19T01:39:03.032 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-03-19T01:39:03.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:02 smithi112 ceph-mon[66819]: from='client.14772 -' entity='client.admin' cmd=[{"prefix": "fs subvolume create", "vol_name": "cephfs", "sub_name": "sv_0", "group_name": "qa", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:39:03.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:02 smithi112 ceph-mon[66819]: from='client.14778 -' entity='client.admin' cmd=[{"prefix": "fs subvolume getpath", "vol_name": "cephfs", "group_name": "qa", "sub_name": "sv_0", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:39:03.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:02 smithi112 ceph-mon[66819]: from='client.14784 -' entity='client.admin' cmd=[{"prefix": "fs subvolume create", "vol_name": "cephfs", "sub_name": "sv_1", "group_name": "qa", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:39:03.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:02 smithi112 ceph-mon[66819]: mgrmap e17: x(active, since 6m), standbys: z, y 2024-03-19T01:39:03.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:02 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/954980700' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:39:03.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:02 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/3068977748' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-19T01:39:03.172 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.169+0000 7f834471b640 1 -- 172.21.15.99:0/1321350389 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f833c0ffb30 msgr2=0x7f833c0fff50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:03.173 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.169+0000 7f834471b640 1 --2- 172.21.15.99:0/1321350389 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f833c0ffb30 0x7f833c0fff50 secure :-1 s=READY pgs=229 cs=0 l=1 rev1=1 crypto rx=0x7f832c009930 tx=0x7f832c02bc30 comp rx=0 tx=0).stop 2024-03-19T01:39:03.173 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.169+0000 7f834471b640 1 -- 172.21.15.99:0/1321350389 shutdown_connections 2024-03-19T01:39:03.173 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.169+0000 7f834471b640 1 --2- 172.21.15.99:0/1321350389 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f833c1034c0 0x7f833c109d50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:03.173 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.169+0000 7f834471b640 1 --2- 172.21.15.99:0/1321350389 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f833c102b00 0x7f833c102f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:03.173 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.169+0000 7f834471b640 1 --2- 172.21.15.99:0/1321350389 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f833c0ffb30 0x7f833c0fff50 unknown :-1 s=CLOSED pgs=229 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:03.173 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.169+0000 7f834471b640 1 -- 172.21.15.99:0/1321350389 >> 172.21.15.99:0/1321350389 conn(0x7f833c0fb460 msgr2=0x7f833c0fd8a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:03.173 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.169+0000 7f834471b640 1 -- 172.21.15.99:0/1321350389 shutdown_connections 2024-03-19T01:39:03.173 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.169+0000 7f834471b640 1 -- 172.21.15.99:0/1321350389 wait complete. 2024-03-19T01:39:03.173 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.169+0000 7f834471b640 1 Processor -- start 2024-03-19T01:39:03.173 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.169+0000 7f834471b640 1 -- start start 2024-03-19T01:39:03.173 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.170+0000 7f834471b640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f833c0ffb30 0x7f833c197c40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:03.173 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.170+0000 7f834471b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f833c102b00 0x7f833c198180 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:03.173 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.170+0000 7f834471b640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f833c1034c0 0x7f833c19f1b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:03.174 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.170+0000 7f834471b640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f833c10d550 con 0x7f833c102b00 2024-03-19T01:39:03.174 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.170+0000 7f834471b640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f833c10d3d0 con 0x7f833c1034c0 2024-03-19T01:39:03.174 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.170+0000 7f834471b640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f833c10d6d0 con 0x7f833c0ffb30 2024-03-19T01:39:03.174 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.170+0000 7f8341cb6640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f833c102b00 0x7f833c198180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:03.174 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.170+0000 7f8341cb6640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f833c102b00 0x7f833c198180 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:52030/0 (socket says 172.21.15.99:52030) 2024-03-19T01:39:03.174 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.170+0000 7f8341cb6640 1 -- 172.21.15.99:0/1840016108 learned_addr learned my addr 172.21.15.99:0/1840016108 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:39:03.174 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.170+0000 7f83424b7640 1 --2- 172.21.15.99:0/1840016108 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f833c0ffb30 0x7f833c197c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:03.174 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.170+0000 7f8342cb8640 1 --2- 172.21.15.99:0/1840016108 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f833c1034c0 0x7f833c19f1b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:03.174 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.170+0000 7f8341cb6640 1 -- 172.21.15.99:0/1840016108 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f833c0ffb30 msgr2=0x7f833c197c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:03.174 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.170+0000 7f8341cb6640 1 --2- 172.21.15.99:0/1840016108 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f833c0ffb30 0x7f833c197c40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:03.174 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.170+0000 7f8341cb6640 1 -- 172.21.15.99:0/1840016108 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f833c1034c0 msgr2=0x7f833c19f1b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:03.174 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.170+0000 7f8341cb6640 1 --2- 172.21.15.99:0/1840016108 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f833c1034c0 0x7f833c19f1b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:03.174 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.170+0000 7f8341cb6640 1 -- 172.21.15.99:0/1840016108 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f833c19f850 con 0x7f833c102b00 2024-03-19T01:39:03.174 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.170+0000 7f8342cb8640 1 --2- 172.21.15.99:0/1840016108 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f833c1034c0 0x7f833c19f1b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-19T01:39:03.175 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.170+0000 7f8341cb6640 1 --2- 172.21.15.99:0/1840016108 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f833c102b00 0x7f833c198180 secure :-1 s=READY pgs=230 cs=0 l=1 rev1=1 crypto rx=0x7f833800e9f0 tx=0x7f833800edf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:03.175 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.170+0000 7f83337fe640 1 -- 172.21.15.99:0/1840016108 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f833800cd80 con 0x7f833c102b00 2024-03-19T01:39:03.175 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.170+0000 7f83337fe640 1 -- 172.21.15.99:0/1840016108 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f8338005500 con 0x7f833c102b00 2024-03-19T01:39:03.175 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.170+0000 7f834471b640 1 -- 172.21.15.99:0/1840016108 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f833c19fb40 con 0x7f833c102b00 2024-03-19T01:39:03.175 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.170+0000 7f83337fe640 1 -- 172.21.15.99:0/1840016108 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8338010690 con 0x7f833c102b00 2024-03-19T01:39:03.175 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.170+0000 7f834471b640 1 -- 172.21.15.99:0/1840016108 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f833c1a0010 con 0x7f833c102b00 2024-03-19T01:39:03.175 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.171+0000 7f834471b640 1 -- 172.21.15.99:0/1840016108 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f833c108c30 con 0x7f833c102b00 2024-03-19T01:39:03.176 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.172+0000 7f83337fe640 1 -- 172.21.15.99:0/1840016108 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 17) ==== 137341+0+0 (secure 0 0 0) 0x7f83380046c0 con 0x7f833c102b00 2024-03-19T01:39:03.176 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.173+0000 7f83337fe640 1 --2- 172.21.15.99:0/1840016108 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f83280a2890 0x7f83280a4d50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:03.176 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.173+0000 7f83424b7640 1 --2- 172.21.15.99:0/1840016108 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f83280a2890 0x7f83280a4d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:03.176 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.173+0000 7f83337fe640 1 -- 172.21.15.99:0/1840016108 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(87..87 src has 1..87) ==== 7979+0+0 (secure 0 0 0) 0x7f83380cef70 con 0x7f833c102b00 2024-03-19T01:39:03.176 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.173+0000 7f83424b7640 1 --2- 172.21.15.99:0/1840016108 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f83280a2890 0x7f83280a4d50 secure :-1 s=READY pgs=176 cs=0 l=1 rev1=1 crypto rx=0x7f832c009230 tx=0x7f832c02df90 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:03.178 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.177+0000 7f83337fe640 1 -- 172.21.15.99:0/1840016108 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f833809c060 con 0x7f833c102b00 2024-03-19T01:39:03.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:02 smithi115 ceph-mon[67332]: from='client.14772 -' entity='client.admin' cmd=[{"prefix": "fs subvolume create", "vol_name": "cephfs", "sub_name": "sv_0", "group_name": "qa", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:39:03.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:02 smithi115 ceph-mon[67332]: from='client.14778 -' entity='client.admin' cmd=[{"prefix": "fs subvolume getpath", "vol_name": "cephfs", "group_name": "qa", "sub_name": "sv_0", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:39:03.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:02 smithi115 ceph-mon[67332]: from='client.14784 -' entity='client.admin' cmd=[{"prefix": "fs subvolume create", "vol_name": "cephfs", "sub_name": "sv_1", "group_name": "qa", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:39:03.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:02 smithi115 ceph-mon[67332]: mgrmap e17: x(active, since 6m), standbys: z, y 2024-03-19T01:39:03.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:02 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/954980700' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:39:03.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:02 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/3068977748' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-19T01:39:03.354 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.353+0000 7f834471b640 1 -- 172.21.15.99:0/1840016108 --> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7f833c061c60 con 0x7f83280a2890 2024-03-19T01:39:03.361 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.360+0000 7f83337fe640 1 -- 172.21.15.99:0/1840016108 <== mgr.14150 v2:172.21.15.99:6800/4272410295 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+258767 (secure 0 0 0) 0x7f833c061c60 con 0x7f83280a2890 2024-03-19T01:39:03.364 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:39:03.365 INFO:teuthology.orchestra.run.smithi099.stderr:dumped all 2024-03-19T01:39:03.367 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.363+0000 7f834471b640 1 -- 172.21.15.99:0/1840016108 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f83280a2890 msgr2=0x7f83280a4d50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:03.367 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.363+0000 7f834471b640 1 --2- 172.21.15.99:0/1840016108 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f83280a2890 0x7f83280a4d50 secure :-1 s=READY pgs=176 cs=0 l=1 rev1=1 crypto rx=0x7f832c009230 tx=0x7f832c02df90 comp rx=0 tx=0).stop 2024-03-19T01:39:03.367 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.363+0000 7f834471b640 1 -- 172.21.15.99:0/1840016108 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f833c102b00 msgr2=0x7f833c198180 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:03.367 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.363+0000 7f834471b640 1 --2- 172.21.15.99:0/1840016108 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f833c102b00 0x7f833c198180 secure :-1 s=READY pgs=230 cs=0 l=1 rev1=1 crypto rx=0x7f833800e9f0 tx=0x7f833800edf0 comp rx=0 tx=0).stop 2024-03-19T01:39:03.367 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.363+0000 7f834471b640 1 -- 172.21.15.99:0/1840016108 shutdown_connections 2024-03-19T01:39:03.367 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.363+0000 7f834471b640 1 --2- 172.21.15.99:0/1840016108 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f83280a2890 0x7f83280a4d50 unknown :-1 s=CLOSED pgs=176 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:03.367 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.363+0000 7f834471b640 1 --2- 172.21.15.99:0/1840016108 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f833c1034c0 0x7f833c19f1b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:03.368 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.363+0000 7f834471b640 1 --2- 172.21.15.99:0/1840016108 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f833c102b00 0x7f833c198180 unknown :-1 s=CLOSED pgs=230 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:03.368 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.363+0000 7f834471b640 1 --2- 172.21.15.99:0/1840016108 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f833c0ffb30 0x7f833c197c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:03.368 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.363+0000 7f834471b640 1 -- 172.21.15.99:0/1840016108 >> 172.21.15.99:0/1840016108 conn(0x7f833c0fb460 msgr2=0x7f833c105e00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:03.368 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.363+0000 7f834471b640 1 -- 172.21.15.99:0/1840016108 shutdown_connections 2024-03-19T01:39:03.368 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:03.363+0000 7f834471b640 1 -- 172.21.15.99:0/1840016108 wait complete. 2024-03-19T01:39:03.375 INFO:teuthology.orchestra.run.smithi099.stdout:{"pg_ready":true,"pg_map":{"version":285,"stamp":"2024-03-19T01:39:02.740784+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":596238,"num_objects":54,"num_object_clones":0,"num_object_copies":162,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":54,"num_whiteouts":0,"num_read":150,"num_read_kb":128,"num_write":295,"num_write_kb":2970,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":35,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":236,"ondisk_log_size":236,"up":291,"acting":291,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":217,"num_osds":12,"num_per_pool_osds":12,"num_per_pool_omap_osds":12,"kb":1124941824,"kb_used":374524,"kb_used_data":12860,"kb_used_omap":18,"kb_used_meta":361517,"kb_avail":1124567300,"statfs":{"total":1151940427776,"available":1151556915200,"internally_reserved":0,"allocated":13168640,"data_stored":7147113,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19139,"internal_metadata":370193725},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":5870,"num_objects":52,"num_object_clones":0,"num_object_copies":156,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":52,"num_whiteouts":0,"num_read":24,"num_read_kb":19,"num_write":114,"num_write_kb":32,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":35,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.962183"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.298802+0000","last_change":"2024-03-19T01:38:56.809406+0000","last_active":"2024-03-19T01:38:58.298802+0000","last_peered":"2024-03-19T01:38:58.298802+0000","last_clean":"2024-03-19T01:38:58.298802+0000","last_became_active":"2024-03-19T01:38:56.808971+0000","last_became_peered":"2024-03-19T01:38:56.808971+0000","last_unstale":"2024-03-19T01:38:58.298802+0000","last_undegraded":"2024-03-19T01:38:58.298802+0000","last_fullsized":"2024-03-19T01:38:58.298802+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T11:07:58.551704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.793364+0000","last_change":"2024-03-19T01:38:56.798847+0000","last_active":"2024-03-19T01:38:57.793364+0000","last_peered":"2024-03-19T01:38:57.793364+0000","last_clean":"2024-03-19T01:38:57.793364+0000","last_became_active":"2024-03-19T01:38:56.795442+0000","last_became_peered":"2024-03-19T01:38:56.795442+0000","last_unstale":"2024-03-19T01:38:57.793364+0000","last_undegraded":"2024-03-19T01:38:57.793364+0000","last_fullsized":"2024-03-19T01:38:57.793364+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T12:05:55.721962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,11,10],"acting":[0,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.793067+0000","last_change":"2024-03-19T01:38:56.806317+0000","last_active":"2024-03-19T01:38:57.793067+0000","last_peered":"2024-03-19T01:38:57.793067+0000","last_clean":"2024-03-19T01:38:57.793067+0000","last_became_active":"2024-03-19T01:38:56.806001+0000","last_became_peered":"2024-03-19T01:38:56.806001+0000","last_unstale":"2024-03-19T01:38:57.793067+0000","last_undegraded":"2024-03-19T01:38:57.793067+0000","last_fullsized":"2024-03-19T01:38:57.793067+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T05:19:45.501431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.298565+0000","last_change":"2024-03-19T01:38:56.803611+0000","last_active":"2024-03-19T01:38:58.298565+0000","last_peered":"2024-03-19T01:38:58.298565+0000","last_clean":"2024-03-19T01:38:58.298565+0000","last_became_active":"2024-03-19T01:38:56.803300+0000","last_became_peered":"2024-03-19T01:38:56.803300+0000","last_unstale":"2024-03-19T01:38:58.298565+0000","last_undegraded":"2024-03-19T01:38:58.298565+0000","last_fullsized":"2024-03-19T01:38:58.298565+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T06:05:49.287266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,8,2],"acting":[4,8,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.793673+0000","last_change":"2024-03-19T01:38:56.799910+0000","last_active":"2024-03-19T01:38:57.793673+0000","last_peered":"2024-03-19T01:38:57.793673+0000","last_clean":"2024-03-19T01:38:57.793673+0000","last_became_active":"2024-03-19T01:38:56.799567+0000","last_became_peered":"2024-03-19T01:38:56.799567+0000","last_unstale":"2024-03-19T01:38:57.793673+0000","last_undegraded":"2024-03-19T01:38:57.793673+0000","last_fullsized":"2024-03-19T01:38:57.793673+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T05:59:42.563083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,3,6],"acting":[8,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.922736+0000","last_change":"2024-03-19T01:38:56.806713+0000","last_active":"2024-03-19T01:38:57.922736+0000","last_peered":"2024-03-19T01:38:57.922736+0000","last_clean":"2024-03-19T01:38:57.922736+0000","last_became_active":"2024-03-19T01:38:56.804692+0000","last_became_peered":"2024-03-19T01:38:56.804692+0000","last_unstale":"2024-03-19T01:38:57.922736+0000","last_undegraded":"2024-03-19T01:38:57.922736+0000","last_fullsized":"2024-03-19T01:38:57.922736+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T11:45:02.994981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,9,4],"acting":[10,9,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.794545+0000","last_change":"2024-03-19T01:38:56.804091+0000","last_active":"2024-03-19T01:38:57.794545+0000","last_peered":"2024-03-19T01:38:57.794545+0000","last_clean":"2024-03-19T01:38:57.794545+0000","last_became_active":"2024-03-19T01:38:56.802308+0000","last_became_peered":"2024-03-19T01:38:56.802308+0000","last_unstale":"2024-03-19T01:38:57.794545+0000","last_undegraded":"2024-03-19T01:38:57.794545+0000","last_fullsized":"2024-03-19T01:38:57.794545+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T07:35:42.952521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,9,5],"acting":[7,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.799578+0000","last_change":"2024-03-19T01:38:56.796475+0000","last_active":"2024-03-19T01:38:57.799578+0000","last_peered":"2024-03-19T01:38:57.799578+0000","last_clean":"2024-03-19T01:38:57.799578+0000","last_became_active":"2024-03-19T01:38:56.796116+0000","last_became_peered":"2024-03-19T01:38:56.796116+0000","last_unstale":"2024-03-19T01:38:57.799578+0000","last_undegraded":"2024-03-19T01:38:57.799578+0000","last_fullsized":"2024-03-19T01:38:57.799578+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T04:35:19.821098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,10],"acting":[3,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.298585+0000","last_change":"2024-03-19T01:38:56.804197+0000","last_active":"2024-03-19T01:38:58.298585+0000","last_peered":"2024-03-19T01:38:58.298585+0000","last_clean":"2024-03-19T01:38:58.298585+0000","last_became_active":"2024-03-19T01:38:56.803209+0000","last_became_peered":"2024-03-19T01:38:56.803209+0000","last_unstale":"2024-03-19T01:38:58.298585+0000","last_undegraded":"2024-03-19T01:38:58.298585+0000","last_fullsized":"2024-03-19T01:38:58.298585+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T06:26:30.303689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,11],"acting":[1,5,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.923082+0000","last_change":"2024-03-19T01:38:56.797973+0000","last_active":"2024-03-19T01:38:57.923082+0000","last_peered":"2024-03-19T01:38:57.923082+0000","last_clean":"2024-03-19T01:38:57.923082+0000","last_became_active":"2024-03-19T01:38:56.797396+0000","last_became_peered":"2024-03-19T01:38:56.797396+0000","last_unstale":"2024-03-19T01:38:57.923082+0000","last_undegraded":"2024-03-19T01:38:57.923082+0000","last_fullsized":"2024-03-19T01:38:57.923082+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T11:46:32.867919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,2,4],"acting":[10,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.794962+0000","last_change":"2024-03-19T01:38:56.802211+0000","last_active":"2024-03-19T01:38:57.794962+0000","last_peered":"2024-03-19T01:38:57.794962+0000","last_clean":"2024-03-19T01:38:57.794962+0000","last_became_active":"2024-03-19T01:38:56.801776+0000","last_became_peered":"2024-03-19T01:38:56.801776+0000","last_unstale":"2024-03-19T01:38:57.794962+0000","last_undegraded":"2024-03-19T01:38:57.794962+0000","last_fullsized":"2024-03-19T01:38:57.794962+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T10:09:36.983771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,8,1],"acting":[7,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.298327+0000","last_change":"2024-03-19T01:38:56.798463+0000","last_active":"2024-03-19T01:38:58.298327+0000","last_peered":"2024-03-19T01:38:58.298327+0000","last_clean":"2024-03-19T01:38:58.298327+0000","last_became_active":"2024-03-19T01:38:56.797187+0000","last_became_peered":"2024-03-19T01:38:56.797187+0000","last_unstale":"2024-03-19T01:38:58.298327+0000","last_undegraded":"2024-03-19T01:38:58.298327+0000","last_fullsized":"2024-03-19T01:38:58.298327+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T11:10:04.349393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,8],"acting":[4,10,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.297712+0000","last_change":"2024-03-19T01:38:56.801650+0000","last_active":"2024-03-19T01:38:58.297712+0000","last_peered":"2024-03-19T01:38:58.297712+0000","last_clean":"2024-03-19T01:38:58.297712+0000","last_became_active":"2024-03-19T01:38:56.801323+0000","last_became_peered":"2024-03-19T01:38:56.801323+0000","last_unstale":"2024-03-19T01:38:58.297712+0000","last_undegraded":"2024-03-19T01:38:58.297712+0000","last_fullsized":"2024-03-19T01:38:58.297712+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T06:10:28.518843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,10,5],"acting":[6,10,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.793142+0000","last_change":"2024-03-19T01:38:56.798633+0000","last_active":"2024-03-19T01:38:57.793142+0000","last_peered":"2024-03-19T01:38:57.793142+0000","last_clean":"2024-03-19T01:38:57.793142+0000","last_became_active":"2024-03-19T01:38:56.798342+0000","last_became_peered":"2024-03-19T01:38:56.798342+0000","last_unstale":"2024-03-19T01:38:57.793142+0000","last_undegraded":"2024-03-19T01:38:57.793142+0000","last_fullsized":"2024-03-19T01:38:57.793142+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T03:21:57.350930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,8,10],"acting":[0,8,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.793138+0000","last_change":"2024-03-19T01:38:56.801516+0000","last_active":"2024-03-19T01:38:57.793138+0000","last_peered":"2024-03-19T01:38:57.793138+0000","last_clean":"2024-03-19T01:38:57.793138+0000","last_became_active":"2024-03-19T01:38:56.799602+0000","last_became_peered":"2024-03-19T01:38:56.799602+0000","last_unstale":"2024-03-19T01:38:57.793138+0000","last_undegraded":"2024-03-19T01:38:57.793138+0000","last_fullsized":"2024-03-19T01:38:57.793138+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T11:14:44.289556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.297811+0000","last_change":"2024-03-19T01:38:56.796556+0000","last_active":"2024-03-19T01:38:58.297811+0000","last_peered":"2024-03-19T01:38:58.297811+0000","last_clean":"2024-03-19T01:38:58.297811+0000","last_became_active":"2024-03-19T01:38:56.796215+0000","last_became_peered":"2024-03-19T01:38:56.796215+0000","last_unstale":"2024-03-19T01:38:58.297811+0000","last_undegraded":"2024-03-19T01:38:58.297811+0000","last_fullsized":"2024-03-19T01:38:58.297811+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T09:12:00.826877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.298955+0000","last_change":"2024-03-19T01:38:56.808706+0000","last_active":"2024-03-19T01:38:58.298955+0000","last_peered":"2024-03-19T01:38:58.298955+0000","last_clean":"2024-03-19T01:38:58.298955+0000","last_became_active":"2024-03-19T01:38:56.808065+0000","last_became_peered":"2024-03-19T01:38:56.808065+0000","last_unstale":"2024-03-19T01:38:58.298955+0000","last_undegraded":"2024-03-19T01:38:58.298955+0000","last_fullsized":"2024-03-19T01:38:58.298955+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T08:40:28.256172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.797935+0000","last_change":"2024-03-19T01:38:56.799501+0000","last_active":"2024-03-19T01:38:57.797935+0000","last_peered":"2024-03-19T01:38:57.797935+0000","last_clean":"2024-03-19T01:38:57.797935+0000","last_became_active":"2024-03-19T01:38:56.799202+0000","last_became_peered":"2024-03-19T01:38:56.799202+0000","last_unstale":"2024-03-19T01:38:57.797935+0000","last_undegraded":"2024-03-19T01:38:57.797935+0000","last_fullsized":"2024-03-19T01:38:57.797935+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T09:42:06.386165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,1,10],"acting":[9,1,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.297593+0000","last_change":"2024-03-19T01:38:56.810697+0000","last_active":"2024-03-19T01:38:58.297593+0000","last_peered":"2024-03-19T01:38:58.297593+0000","last_clean":"2024-03-19T01:38:58.297593+0000","last_became_active":"2024-03-19T01:38:56.810202+0000","last_became_peered":"2024-03-19T01:38:56.810202+0000","last_unstale":"2024-03-19T01:38:58.297593+0000","last_undegraded":"2024-03-19T01:38:58.297593+0000","last_fullsized":"2024-03-19T01:38:58.297593+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T06:22:11.807807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1f","version":"87'9","reported_seq":25,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:59.817756+0000","last_change":"2024-03-19T01:38:55.791479+0000","last_active":"2024-03-19T01:38:59.817756+0000","last_peered":"2024-03-19T01:38:59.817756+0000","last_clean":"2024-03-19T01:38:59.817756+0000","last_became_active":"2024-03-19T01:38:55.791150+0000","last_became_peered":"2024-03-19T01:38:55.791150+0000","last_unstale":"2024-03-19T01:38:59.817756+0000","last_undegraded":"2024-03-19T01:38:59.817756+0000","last_fullsized":"2024-03-19T01:38:59.817756+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T09:56:07.953602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,7,8],"acting":[9,7,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.800064+0000","last_change":"2024-03-19T01:38:56.802404+0000","last_active":"2024-03-19T01:38:57.800064+0000","last_peered":"2024-03-19T01:38:57.800064+0000","last_clean":"2024-03-19T01:38:57.800064+0000","last_became_active":"2024-03-19T01:38:56.802060+0000","last_became_peered":"2024-03-19T01:38:56.802060+0000","last_unstale":"2024-03-19T01:38:57.800064+0000","last_undegraded":"2024-03-19T01:38:57.800064+0000","last_fullsized":"2024-03-19T01:38:57.800064+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T07:45:34.287073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1e","version":"87'2","reported_seq":17,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.800156+0000","last_change":"2024-03-19T01:38:55.798249+0000","last_active":"2024-03-19T01:38:57.800156+0000","last_peered":"2024-03-19T01:38:57.800156+0000","last_clean":"2024-03-19T01:38:57.800156+0000","last_became_active":"2024-03-19T01:38:55.797952+0000","last_became_peered":"2024-03-19T01:38:55.797952+0000","last_unstale":"2024-03-19T01:38:57.800156+0000","last_undegraded":"2024-03-19T01:38:57.800156+0000","last_fullsized":"2024-03-19T01:38:57.800156+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T12:12:40.925462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.793449+0000","last_change":"2024-03-19T01:38:56.805992+0000","last_active":"2024-03-19T01:38:57.793449+0000","last_peered":"2024-03-19T01:38:57.793449+0000","last_clean":"2024-03-19T01:38:57.793449+0000","last_became_active":"2024-03-19T01:38:56.804997+0000","last_became_peered":"2024-03-19T01:38:56.804997+0000","last_unstale":"2024-03-19T01:38:57.793449+0000","last_undegraded":"2024-03-19T01:38:57.793449+0000","last_fullsized":"2024-03-19T01:38:57.793449+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T10:33:39.003792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,9],"acting":[0,1,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1d","version":"86'2","reported_seq":17,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.794052+0000","last_change":"2024-03-19T01:38:55.795296+0000","last_active":"2024-03-19T01:38:57.794052+0000","last_peered":"2024-03-19T01:38:57.794052+0000","last_clean":"2024-03-19T01:38:57.794052+0000","last_became_active":"2024-03-19T01:38:55.794321+0000","last_became_peered":"2024-03-19T01:38:55.794321+0000","last_unstale":"2024-03-19T01:38:57.794052+0000","last_undegraded":"2024-03-19T01:38:57.794052+0000","last_fullsized":"2024-03-19T01:38:57.794052+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T12:26:12.914801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.298260+0000","last_change":"2024-03-19T01:38:56.802337+0000","last_active":"2024-03-19T01:38:58.298260+0000","last_peered":"2024-03-19T01:38:58.298260+0000","last_clean":"2024-03-19T01:38:58.298260+0000","last_became_active":"2024-03-19T01:38:56.802032+0000","last_became_peered":"2024-03-19T01:38:56.802032+0000","last_unstale":"2024-03-19T01:38:58.298260+0000","last_undegraded":"2024-03-19T01:38:58.298260+0000","last_fullsized":"2024-03-19T01:38:58.298260+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T07:57:13.512027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,6],"acting":[1,10,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1c","version":"86'2","reported_seq":17,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.794040+0000","last_change":"2024-03-19T01:38:55.793796+0000","last_active":"2024-03-19T01:38:57.794040+0000","last_peered":"2024-03-19T01:38:57.794040+0000","last_clean":"2024-03-19T01:38:57.794040+0000","last_became_active":"2024-03-19T01:38:55.792818+0000","last_became_peered":"2024-03-19T01:38:55.792818+0000","last_unstale":"2024-03-19T01:38:57.794040+0000","last_undegraded":"2024-03-19T01:38:57.794040+0000","last_fullsized":"2024-03-19T01:38:57.794040+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T09:44:17.403644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,1,5],"acting":[8,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.297664+0000","last_change":"2024-03-19T01:38:56.801280+0000","last_active":"2024-03-19T01:38:58.297664+0000","last_peered":"2024-03-19T01:38:58.297664+0000","last_clean":"2024-03-19T01:38:58.297664+0000","last_became_active":"2024-03-19T01:38:56.800944+0000","last_became_peered":"2024-03-19T01:38:56.800944+0000","last_unstale":"2024-03-19T01:38:58.297664+0000","last_undegraded":"2024-03-19T01:38:58.297664+0000","last_fullsized":"2024-03-19T01:38:58.297664+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T03:09:45.373387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1b","version":"87'6","reported_seq":26,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:39:01.333431+0000","last_change":"2024-03-19T01:38:55.792563+0000","last_active":"2024-03-19T01:39:01.333431+0000","last_peered":"2024-03-19T01:39:01.333431+0000","last_clean":"2024-03-19T01:39:01.333431+0000","last_became_active":"2024-03-19T01:38:55.792098+0000","last_became_peered":"2024-03-19T01:38:55.792098+0000","last_unstale":"2024-03-19T01:39:01.333431+0000","last_undegraded":"2024-03-19T01:39:01.333431+0000","last_fullsized":"2024-03-19T01:39:01.333431+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T08:40:54.804413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1013,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":5,"num_read_kb":3,"num_write":3,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.793857+0000","last_change":"2024-03-19T01:38:56.810155+0000","last_active":"2024-03-19T01:38:57.793857+0000","last_peered":"2024-03-19T01:38:57.793857+0000","last_clean":"2024-03-19T01:38:57.793857+0000","last_became_active":"2024-03-19T01:38:56.809820+0000","last_became_peered":"2024-03-19T01:38:56.809820+0000","last_unstale":"2024-03-19T01:38:57.793857+0000","last_undegraded":"2024-03-19T01:38:57.793857+0000","last_fullsized":"2024-03-19T01:38:57.793857+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T12:44:30.463663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,9,5],"acting":[8,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.1a","version":"87'3","reported_seq":18,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:59.817422+0000","last_change":"2024-03-19T01:38:55.797324+0000","last_active":"2024-03-19T01:38:59.817422+0000","last_peered":"2024-03-19T01:38:59.817422+0000","last_clean":"2024-03-19T01:38:59.817422+0000","last_became_active":"2024-03-19T01:38:55.796997+0000","last_became_peered":"2024-03-19T01:38:55.796997+0000","last_unstale":"2024-03-19T01:38:59.817422+0000","last_undegraded":"2024-03-19T01:38:59.817422+0000","last_fullsized":"2024-03-19T01:38:59.817422+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T08:40:37.023902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,8,6],"acting":[10,8,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.297845+0000","last_change":"2024-03-19T01:38:56.785995+0000","last_active":"2024-03-19T01:38:58.297845+0000","last_peered":"2024-03-19T01:38:58.297845+0000","last_clean":"2024-03-19T01:38:58.297845+0000","last_became_active":"2024-03-19T01:38:56.785659+0000","last_became_peered":"2024-03-19T01:38:56.785659+0000","last_unstale":"2024-03-19T01:38:58.297845+0000","last_undegraded":"2024-03-19T01:38:58.297845+0000","last_fullsized":"2024-03-19T01:38:58.297845+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T06:40:33.389038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.19","version":"87'2","reported_seq":17,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:59.817555+0000","last_change":"2024-03-19T01:38:55.790034+0000","last_active":"2024-03-19T01:38:59.817555+0000","last_peered":"2024-03-19T01:38:59.817555+0000","last_clean":"2024-03-19T01:38:59.817555+0000","last_became_active":"2024-03-19T01:38:55.786369+0000","last_became_peered":"2024-03-19T01:38:55.786369+0000","last_unstale":"2024-03-19T01:38:59.817555+0000","last_undegraded":"2024-03-19T01:38:59.817555+0000","last_fullsized":"2024-03-19T01:38:59.817555+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T06:13:52.925644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,9],"acting":[6,4,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.799603+0000","last_change":"2024-03-19T01:38:56.791253+0000","last_active":"2024-03-19T01:38:57.799603+0000","last_peered":"2024-03-19T01:38:57.799603+0000","last_clean":"2024-03-19T01:38:57.799603+0000","last_became_active":"2024-03-19T01:38:56.790793+0000","last_became_peered":"2024-03-19T01:38:56.790793+0000","last_unstale":"2024-03-19T01:38:57.799603+0000","last_undegraded":"2024-03-19T01:38:57.799603+0000","last_fullsized":"2024-03-19T01:38:57.799603+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T05:14:15.277513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,6],"acting":[3,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"87'1","reported_seq":16,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.297684+0000","last_change":"2024-03-19T01:38:55.793379+0000","last_active":"2024-03-19T01:38:58.297684+0000","last_peered":"2024-03-19T01:38:58.297684+0000","last_clean":"2024-03-19T01:38:58.297684+0000","last_became_active":"2024-03-19T01:38:55.793035+0000","last_became_peered":"2024-03-19T01:38:55.793035+0000","last_unstale":"2024-03-19T01:38:58.297684+0000","last_undegraded":"2024-03-19T01:38:58.297684+0000","last_fullsized":"2024-03-19T01:38:58.297684+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T05:01:52.122826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.797836+0000","last_change":"2024-03-19T01:38:56.800540+0000","last_active":"2024-03-19T01:38:57.797836+0000","last_peered":"2024-03-19T01:38:57.797836+0000","last_clean":"2024-03-19T01:38:57.797836+0000","last_became_active":"2024-03-19T01:38:56.800199+0000","last_became_peered":"2024-03-19T01:38:56.800199+0000","last_unstale":"2024-03-19T01:38:57.797836+0000","last_undegraded":"2024-03-19T01:38:57.797836+0000","last_fullsized":"2024-03-19T01:38:57.797836+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T05:17:57.722253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,3,10],"acting":[9,3,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.17","version":"87'3","reported_seq":19,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.811349+0000","last_change":"2024-03-19T01:38:55.792930+0000","last_active":"2024-03-19T01:38:58.811349+0000","last_peered":"2024-03-19T01:38:58.811349+0000","last_clean":"2024-03-19T01:38:58.811349+0000","last_became_active":"2024-03-19T01:38:55.792602+0000","last_became_peered":"2024-03-19T01:38:55.792602+0000","last_unstale":"2024-03-19T01:38:58.811349+0000","last_undegraded":"2024-03-19T01:38:58.811349+0000","last_fullsized":"2024-03-19T01:38:58.811349+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T12:28:15.399311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,5,9],"acting":[10,5,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.298685+0000","last_change":"2024-03-19T01:38:56.801228+0000","last_active":"2024-03-19T01:38:58.298685+0000","last_peered":"2024-03-19T01:38:58.298685+0000","last_clean":"2024-03-19T01:38:58.298685+0000","last_became_active":"2024-03-19T01:38:56.800656+0000","last_became_peered":"2024-03-19T01:38:56.800656+0000","last_unstale":"2024-03-19T01:38:58.298685+0000","last_undegraded":"2024-03-19T01:38:58.298685+0000","last_fullsized":"2024-03-19T01:38:58.298685+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T05:43:32.565608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,9],"acting":[1,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.16","version":"87'2","reported_seq":17,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.811067+0000","last_change":"2024-03-19T01:38:55.795899+0000","last_active":"2024-03-19T01:38:58.811067+0000","last_peered":"2024-03-19T01:38:58.811067+0000","last_clean":"2024-03-19T01:38:58.811067+0000","last_became_active":"2024-03-19T01:38:55.795434+0000","last_became_peered":"2024-03-19T01:38:55.795434+0000","last_unstale":"2024-03-19T01:38:58.811067+0000","last_undegraded":"2024-03-19T01:38:58.811067+0000","last_fullsized":"2024-03-19T01:38:58.811067+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T02:32:39.376935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.793443+0000","last_change":"2024-03-19T01:38:56.794349+0000","last_active":"2024-03-19T01:38:57.793443+0000","last_peered":"2024-03-19T01:38:57.793443+0000","last_clean":"2024-03-19T01:38:57.793443+0000","last_became_active":"2024-03-19T01:38:56.794038+0000","last_became_peered":"2024-03-19T01:38:56.794038+0000","last_unstale":"2024-03-19T01:38:57.793443+0000","last_undegraded":"2024-03-19T01:38:57.793443+0000","last_fullsized":"2024-03-19T01:38:57.793443+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T09:44:17.534659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"87'4","reported_seq":20,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.810182+0000","last_change":"2024-03-19T01:38:55.793769+0000","last_active":"2024-03-19T01:38:58.810182+0000","last_peered":"2024-03-19T01:38:58.810182+0000","last_clean":"2024-03-19T01:38:58.810182+0000","last_became_active":"2024-03-19T01:38:55.793423+0000","last_became_peered":"2024-03-19T01:38:55.793423+0000","last_unstale":"2024-03-19T01:38:58.810182+0000","last_undegraded":"2024-03-19T01:38:58.810182+0000","last_fullsized":"2024-03-19T01:38:58.810182+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T12:56:37.933791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":672,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,10],"acting":[5,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.298491+0000","last_change":"2024-03-19T01:38:56.794855+0000","last_active":"2024-03-19T01:38:58.298491+0000","last_peered":"2024-03-19T01:38:58.298491+0000","last_clean":"2024-03-19T01:38:58.298491+0000","last_became_active":"2024-03-19T01:38:56.794553+0000","last_became_peered":"2024-03-19T01:38:56.794553+0000","last_unstale":"2024-03-19T01:38:58.298491+0000","last_undegraded":"2024-03-19T01:38:58.298491+0000","last_fullsized":"2024-03-19T01:38:58.298491+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T08:33:20.473620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,8,4],"acting":[1,8,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.14","version":"87'8","reported_seq":27,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:39:00.331846+0000","last_change":"2024-03-19T01:38:55.794317+0000","last_active":"2024-03-19T01:39:00.331846+0000","last_peered":"2024-03-19T01:39:00.331846+0000","last_clean":"2024-03-19T01:39:00.331846+0000","last_became_active":"2024-03-19T01:38:55.793970+0000","last_became_peered":"2024-03-19T01:38:55.793970+0000","last_unstale":"2024-03-19T01:39:00.331846+0000","last_undegraded":"2024-03-19T01:39:00.331846+0000","last_fullsized":"2024-03-19T01:39:00.331846+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T02:25:09.584906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1612,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":4,"num_read_kb":3,"num_write":9,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,1,2],"acting":[10,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.793864+0000","last_change":"2024-03-19T01:38:56.794400+0000","last_active":"2024-03-19T01:38:57.793864+0000","last_peered":"2024-03-19T01:38:57.793864+0000","last_clean":"2024-03-19T01:38:57.793864+0000","last_became_active":"2024-03-19T01:38:56.794027+0000","last_became_peered":"2024-03-19T01:38:56.794027+0000","last_unstale":"2024-03-19T01:38:57.793864+0000","last_undegraded":"2024-03-19T01:38:57.793864+0000","last_fullsized":"2024-03-19T01:38:57.793864+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T06:33:51.398239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,7],"acting":[8,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.13","version":"87'6","reported_seq":21,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:59.816398+0000","last_change":"2024-03-19T01:38:55.792812+0000","last_active":"2024-03-19T01:38:59.816398+0000","last_peered":"2024-03-19T01:38:59.816398+0000","last_clean":"2024-03-19T01:38:59.816398+0000","last_became_active":"2024-03-19T01:38:55.792422+0000","last_became_peered":"2024-03-19T01:38:55.792422+0000","last_unstale":"2024-03-19T01:38:59.816398+0000","last_undegraded":"2024-03-19T01:38:59.816398+0000","last_fullsized":"2024-03-19T01:38:59.816398+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T11:04:58.863405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,10,3],"acting":[2,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.297574+0000","last_change":"2024-03-19T01:38:56.794156+0000","last_active":"2024-03-19T01:38:58.297574+0000","last_peered":"2024-03-19T01:38:58.297574+0000","last_clean":"2024-03-19T01:38:58.297574+0000","last_became_active":"2024-03-19T01:38:56.793812+0000","last_became_peered":"2024-03-19T01:38:56.793812+0000","last_unstale":"2024-03-19T01:38:58.297574+0000","last_undegraded":"2024-03-19T01:38:58.297574+0000","last_fullsized":"2024-03-19T01:38:58.297574+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T05:00:53.379548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.12","version":"87'3","reported_seq":18,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.810237+0000","last_change":"2024-03-19T01:38:55.798220+0000","last_active":"2024-03-19T01:38:58.810237+0000","last_peered":"2024-03-19T01:38:58.810237+0000","last_clean":"2024-03-19T01:38:58.810237+0000","last_became_active":"2024-03-19T01:38:55.797901+0000","last_became_peered":"2024-03-19T01:38:55.797901+0000","last_unstale":"2024-03-19T01:38:58.810237+0000","last_undegraded":"2024-03-19T01:38:58.810237+0000","last_fullsized":"2024-03-19T01:38:58.810237+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T11:45:20.233596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.794988+0000","last_change":"2024-03-19T01:38:56.805603+0000","last_active":"2024-03-19T01:38:57.794988+0000","last_peered":"2024-03-19T01:38:57.794988+0000","last_clean":"2024-03-19T01:38:57.794988+0000","last_became_active":"2024-03-19T01:38:56.805248+0000","last_became_peered":"2024-03-19T01:38:56.805248+0000","last_unstale":"2024-03-19T01:38:57.794988+0000","last_undegraded":"2024-03-19T01:38:57.794988+0000","last_fullsized":"2024-03-19T01:38:57.794988+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T05:28:41.503673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"87'4","reported_seq":24,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:39:01.333924+0000","last_change":"2024-03-19T01:38:55.793254+0000","last_active":"2024-03-19T01:39:01.333924+0000","last_peered":"2024-03-19T01:39:01.333924+0000","last_clean":"2024-03-19T01:39:01.333924+0000","last_became_active":"2024-03-19T01:38:55.792925+0000","last_became_peered":"2024-03-19T01:38:55.792925+0000","last_unstale":"2024-03-19T01:39:01.333924+0000","last_undegraded":"2024-03-19T01:39:01.333924+0000","last_fullsized":"2024-03-19T01:39:01.333924+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T05:17:25.353050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":979,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":6,"num_read_kb":4,"num_write":3,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,10],"acting":[4,7,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.794465+0000","last_change":"2024-03-19T01:38:56.807845+0000","last_active":"2024-03-19T01:38:57.794465+0000","last_peered":"2024-03-19T01:38:57.794465+0000","last_clean":"2024-03-19T01:38:57.794465+0000","last_became_active":"2024-03-19T01:38:56.807175+0000","last_became_peered":"2024-03-19T01:38:56.807175+0000","last_unstale":"2024-03-19T01:38:57.794465+0000","last_undegraded":"2024-03-19T01:38:57.794465+0000","last_fullsized":"2024-03-19T01:38:57.794465+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T10:45:37.669659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.10","version":"86'8","reported_seq":23,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.793597+0000","last_change":"2024-03-19T01:38:55.793320+0000","last_active":"2024-03-19T01:38:57.793597+0000","last_peered":"2024-03-19T01:38:57.793597+0000","last_clean":"2024-03-19T01:38:57.793597+0000","last_became_active":"2024-03-19T01:38:55.793030+0000","last_became_peered":"2024-03-19T01:38:55.793030+0000","last_unstale":"2024-03-19T01:38:57.793597+0000","last_undegraded":"2024-03-19T01:38:57.793597+0000","last_fullsized":"2024-03-19T01:38:57.793597+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T04:01:31.028364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,2],"acting":[8,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.297443+0000","last_change":"2024-03-19T01:38:56.796338+0000","last_active":"2024-03-19T01:38:58.297443+0000","last_peered":"2024-03-19T01:38:58.297443+0000","last_clean":"2024-03-19T01:38:58.297443+0000","last_became_active":"2024-03-19T01:38:56.795660+0000","last_became_peered":"2024-03-19T01:38:56.795660+0000","last_unstale":"2024-03-19T01:38:58.297443+0000","last_undegraded":"2024-03-19T01:38:58.297443+0000","last_fullsized":"2024-03-19T01:38:58.297443+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T01:56:03.144907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.f","version":"86'1","reported_seq":16,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.794069+0000","last_change":"2024-03-19T01:38:55.791670+0000","last_active":"2024-03-19T01:38:57.794069+0000","last_peered":"2024-03-19T01:38:57.794069+0000","last_clean":"2024-03-19T01:38:57.794069+0000","last_became_active":"2024-03-19T01:38:55.790859+0000","last_became_peered":"2024-03-19T01:38:55.790859+0000","last_unstale":"2024-03-19T01:38:57.794069+0000","last_undegraded":"2024-03-19T01:38:57.794069+0000","last_fullsized":"2024-03-19T01:38:57.794069+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T08:13:16.770134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,5,4],"acting":[8,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.298742+0000","last_change":"2024-03-19T01:38:56.806291+0000","last_active":"2024-03-19T01:38:58.298742+0000","last_peered":"2024-03-19T01:38:58.298742+0000","last_clean":"2024-03-19T01:38:58.298742+0000","last_became_active":"2024-03-19T01:38:56.805972+0000","last_became_peered":"2024-03-19T01:38:56.805972+0000","last_unstale":"2024-03-19T01:38:58.298742+0000","last_undegraded":"2024-03-19T01:38:58.298742+0000","last_fullsized":"2024-03-19T01:38:58.298742+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T05:28:19.575710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,5,4],"acting":[11,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":15,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.797967+0000","last_change":"2024-03-19T01:38:55.798331+0000","last_active":"2024-03-19T01:38:57.797967+0000","last_peered":"2024-03-19T01:38:57.797967+0000","last_clean":"2024-03-19T01:38:57.797967+0000","last_became_active":"2024-03-19T01:38:55.797948+0000","last_became_peered":"2024-03-19T01:38:55.797948+0000","last_unstale":"2024-03-19T01:38:57.797967+0000","last_undegraded":"2024-03-19T01:38:57.797967+0000","last_fullsized":"2024-03-19T01:38:57.797967+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T04:35:09.036490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,2,0],"acting":[9,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.922859+0000","last_change":"2024-03-19T01:38:56.810162+0000","last_active":"2024-03-19T01:38:57.922859+0000","last_peered":"2024-03-19T01:38:57.922859+0000","last_clean":"2024-03-19T01:38:57.922859+0000","last_became_active":"2024-03-19T01:38:56.809888+0000","last_became_peered":"2024-03-19T01:38:56.809888+0000","last_unstale":"2024-03-19T01:38:57.922859+0000","last_undegraded":"2024-03-19T01:38:57.922859+0000","last_fullsized":"2024-03-19T01:38:57.922859+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T05:49:35.606065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,3,6],"acting":[10,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"2.2","version":"87'1","reported_seq":16,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.794773+0000","last_change":"2024-03-19T01:38:55.796629+0000","last_active":"2024-03-19T01:38:57.794773+0000","last_peered":"2024-03-19T01:38:57.794773+0000","last_clean":"2024-03-19T01:38:57.794773+0000","last_became_active":"2024-03-19T01:38:55.796232+0000","last_became_peered":"2024-03-19T01:38:55.796232+0000","last_unstale":"2024-03-19T01:38:57.794773+0000","last_undegraded":"2024-03-19T01:38:57.794773+0000","last_fullsized":"2024-03-19T01:38:57.794773+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T09:38:58.853129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.922918+0000","last_change":"2024-03-19T01:38:56.798227+0000","last_active":"2024-03-19T01:38:57.922918+0000","last_peered":"2024-03-19T01:38:57.922918+0000","last_clean":"2024-03-19T01:38:57.922918+0000","last_became_active":"2024-03-19T01:38:56.797302+0000","last_became_peered":"2024-03-19T01:38:56.797302+0000","last_unstale":"2024-03-19T01:38:57.922918+0000","last_undegraded":"2024-03-19T01:38:57.922918+0000","last_fullsized":"2024-03-19T01:38:57.922918+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T12:35:56.975118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,4,0],"acting":[10,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.297785+0000","last_change":"2024-03-19T01:38:56.802272+0000","last_active":"2024-03-19T01:38:58.297785+0000","last_peered":"2024-03-19T01:38:58.297785+0000","last_clean":"2024-03-19T01:38:58.297785+0000","last_became_active":"2024-03-19T01:38:56.801854+0000","last_became_peered":"2024-03-19T01:38:56.801854+0000","last_unstale":"2024-03-19T01:38:58.297785+0000","last_undegraded":"2024-03-19T01:38:58.297785+0000","last_fullsized":"2024-03-19T01:38:58.297785+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T11:32:23.363896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"87'4","reported_seq":21,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:39:00.330850+0000","last_change":"2024-03-19T01:38:55.785957+0000","last_active":"2024-03-19T01:39:00.330850+0000","last_peered":"2024-03-19T01:39:00.330850+0000","last_clean":"2024-03-19T01:39:00.330850+0000","last_became_active":"2024-03-19T01:38:55.785527+0000","last_became_peered":"2024-03-19T01:38:55.785527+0000","last_unstale":"2024-03-19T01:39:00.330850+0000","last_undegraded":"2024-03-19T01:39:00.330850+0000","last_fullsized":"2024-03-19T01:39:00.330850+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T09:33:55.329126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.800040+0000","last_change":"2024-03-19T01:38:56.798213+0000","last_active":"2024-03-19T01:38:57.800040+0000","last_peered":"2024-03-19T01:38:57.800040+0000","last_clean":"2024-03-19T01:38:57.800040+0000","last_became_active":"2024-03-19T01:38:56.797475+0000","last_became_peered":"2024-03-19T01:38:56.797475+0000","last_unstale":"2024-03-19T01:38:57.800040+0000","last_undegraded":"2024-03-19T01:38:57.800040+0000","last_fullsized":"2024-03-19T01:38:57.800040+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T07:41:42.021626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"87'3","reported_seq":20,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:39:00.329317+0000","last_change":"2024-03-19T01:38:55.792521+0000","last_active":"2024-03-19T01:39:00.329317+0000","last_peered":"2024-03-19T01:39:00.329317+0000","last_clean":"2024-03-19T01:39:00.329317+0000","last_became_active":"2024-03-19T01:38:55.792166+0000","last_became_peered":"2024-03-19T01:38:55.792166+0000","last_unstale":"2024-03-19T01:39:00.329317+0000","last_undegraded":"2024-03-19T01:39:00.329317+0000","last_fullsized":"2024-03-19T01:39:00.329317+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T08:15:12.216641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":4,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,9],"acting":[4,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.0","version":"23'137","reported_seq":340,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.794520+0000","last_change":"2024-03-19T01:37:48.171712+0000","last_active":"2024-03-19T01:38:57.794520+0000","last_peered":"2024-03-19T01:38:57.794520+0000","last_clean":"2024-03-19T01:38:57.794520+0000","last_became_active":"2024-03-19T01:37:48.171323+0000","last_became_peered":"2024-03-19T01:37:48.171323+0000","last_unstale":"2024-03-19T01:38:57.794520+0000","last_undegraded":"2024-03-19T01:38:57.794520+0000","last_fullsized":"2024-03-19T01:38:57.794520+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:34:31.088679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:34:31.088679+0000","last_clean_scrub_stamp":"2024-03-19T01:34:31.088679+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T02:22:49.477567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":109,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.297458+0000","last_change":"2024-03-19T01:38:56.797868+0000","last_active":"2024-03-19T01:38:58.297458+0000","last_peered":"2024-03-19T01:38:58.297458+0000","last_clean":"2024-03-19T01:38:58.297458+0000","last_became_active":"2024-03-19T01:38:56.796855+0000","last_became_peered":"2024-03-19T01:38:56.796855+0000","last_unstale":"2024-03-19T01:38:58.297458+0000","last_undegraded":"2024-03-19T01:38:58.297458+0000","last_fullsized":"2024-03-19T01:38:58.297458+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T01:53:09.887368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"87'3","reported_seq":18,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:59.816889+0000","last_change":"2024-03-19T01:38:55.797507+0000","last_active":"2024-03-19T01:38:59.816889+0000","last_peered":"2024-03-19T01:38:59.816889+0000","last_clean":"2024-03-19T01:38:59.816889+0000","last_became_active":"2024-03-19T01:38:55.797167+0000","last_became_peered":"2024-03-19T01:38:55.797167+0000","last_unstale":"2024-03-19T01:38:59.816889+0000","last_undegraded":"2024-03-19T01:38:59.816889+0000","last_fullsized":"2024-03-19T01:38:59.816889+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T01:45:06.037491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,8],"acting":[7,5,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":15,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.793854+0000","last_change":"2024-03-19T01:38:55.795464+0000","last_active":"2024-03-19T01:38:57.793854+0000","last_peered":"2024-03-19T01:38:57.793854+0000","last_clean":"2024-03-19T01:38:57.793854+0000","last_became_active":"2024-03-19T01:38:55.793827+0000","last_became_peered":"2024-03-19T01:38:55.793827+0000","last_unstale":"2024-03-19T01:38:57.793854+0000","last_undegraded":"2024-03-19T01:38:57.793854+0000","last_fullsized":"2024-03-19T01:38:57.793854+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T13:03:56.862269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,9],"acting":[8,0,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.297755+0000","last_change":"2024-03-19T01:38:56.795569+0000","last_active":"2024-03-19T01:38:58.297755+0000","last_peered":"2024-03-19T01:38:58.297755+0000","last_clean":"2024-03-19T01:38:58.297755+0000","last_became_active":"2024-03-19T01:38:56.795281+0000","last_became_peered":"2024-03-19T01:38:56.795281+0000","last_unstale":"2024-03-19T01:38:58.297755+0000","last_undegraded":"2024-03-19T01:38:58.297755+0000","last_fullsized":"2024-03-19T01:38:58.297755+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T03:18:00.599307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,9],"acting":[6,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"87'4","reported_seq":19,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:59.817344+0000","last_change":"2024-03-19T01:38:55.792306+0000","last_active":"2024-03-19T01:38:59.817344+0000","last_peered":"2024-03-19T01:38:59.817344+0000","last_clean":"2024-03-19T01:38:59.817344+0000","last_became_active":"2024-03-19T01:38:55.792019+0000","last_became_peered":"2024-03-19T01:38:55.792019+0000","last_unstale":"2024-03-19T01:38:59.817344+0000","last_undegraded":"2024-03-19T01:38:59.817344+0000","last_fullsized":"2024-03-19T01:38:59.817344+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T06:33:06.869054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,8,3],"acting":[9,8,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.800042+0000","last_change":"2024-03-19T01:38:56.798314+0000","last_active":"2024-03-19T01:38:57.800042+0000","last_peered":"2024-03-19T01:38:57.800042+0000","last_clean":"2024-03-19T01:38:57.800042+0000","last_became_active":"2024-03-19T01:38:56.797999+0000","last_became_peered":"2024-03-19T01:38:56.797999+0000","last_unstale":"2024-03-19T01:38:57.800042+0000","last_undegraded":"2024-03-19T01:38:57.800042+0000","last_fullsized":"2024-03-19T01:38:57.800042+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T06:23:36.861238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,7],"acting":[3,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"87'4","reported_seq":19,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.809263+0000","last_change":"2024-03-19T01:38:55.790866+0000","last_active":"2024-03-19T01:38:58.809263+0000","last_peered":"2024-03-19T01:38:58.809263+0000","last_clean":"2024-03-19T01:38:58.809263+0000","last_became_active":"2024-03-19T01:38:55.790363+0000","last_became_peered":"2024-03-19T01:38:55.790363+0000","last_unstale":"2024-03-19T01:38:58.809263+0000","last_undegraded":"2024-03-19T01:38:58.809263+0000","last_fullsized":"2024-03-19T01:38:58.809263+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T09:00:27.595598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,11,6],"acting":[4,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.793331+0000","last_change":"2024-03-19T01:38:56.793369+0000","last_active":"2024-03-19T01:38:57.793331+0000","last_peered":"2024-03-19T01:38:57.793331+0000","last_clean":"2024-03-19T01:38:57.793331+0000","last_became_active":"2024-03-19T01:38:56.793068+0000","last_became_peered":"2024-03-19T01:38:56.793068+0000","last_unstale":"2024-03-19T01:38:57.793331+0000","last_undegraded":"2024-03-19T01:38:57.793331+0000","last_fullsized":"2024-03-19T01:38:57.793331+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T13:12:57.954100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"87'2","reported_seq":17,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.810496+0000","last_change":"2024-03-19T01:38:55.793869+0000","last_active":"2024-03-19T01:38:58.810496+0000","last_peered":"2024-03-19T01:38:58.810496+0000","last_clean":"2024-03-19T01:38:58.810496+0000","last_became_active":"2024-03-19T01:38:55.791841+0000","last_became_peered":"2024-03-19T01:38:55.791841+0000","last_unstale":"2024-03-19T01:38:58.810496+0000","last_undegraded":"2024-03-19T01:38:58.810496+0000","last_fullsized":"2024-03-19T01:38:58.810496+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T06:57:21.802060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.793600+0000","last_change":"2024-03-19T01:38:56.802272+0000","last_active":"2024-03-19T01:38:57.793600+0000","last_peered":"2024-03-19T01:38:57.793600+0000","last_clean":"2024-03-19T01:38:57.793600+0000","last_became_active":"2024-03-19T01:38:56.801071+0000","last_became_peered":"2024-03-19T01:38:56.801071+0000","last_unstale":"2024-03-19T01:38:57.793600+0000","last_undegraded":"2024-03-19T01:38:57.793600+0000","last_fullsized":"2024-03-19T01:38:57.793600+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T06:58:12.762975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,10],"acting":[8,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.9","version":"87'3","reported_seq":18,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.810245+0000","last_change":"2024-03-19T01:38:55.795186+0000","last_active":"2024-03-19T01:38:58.810245+0000","last_peered":"2024-03-19T01:38:58.810245+0000","last_clean":"2024-03-19T01:38:58.810245+0000","last_became_active":"2024-03-19T01:38:55.794635+0000","last_became_peered":"2024-03-19T01:38:55.794635+0000","last_unstale":"2024-03-19T01:38:58.810245+0000","last_undegraded":"2024-03-19T01:38:58.810245+0000","last_fullsized":"2024-03-19T01:38:58.810245+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T03:55:56.322716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,9,10],"acting":[3,9,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.298693+0000","last_change":"2024-03-19T01:38:56.812656+0000","last_active":"2024-03-19T01:38:58.298693+0000","last_peered":"2024-03-19T01:38:58.298693+0000","last_clean":"2024-03-19T01:38:58.298693+0000","last_became_active":"2024-03-19T01:38:56.811824+0000","last_became_peered":"2024-03-19T01:38:56.811824+0000","last_unstale":"2024-03-19T01:38:58.298693+0000","last_undegraded":"2024-03-19T01:38:58.298693+0000","last_fullsized":"2024-03-19T01:38:58.298693+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T04:28:11.866892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,0,2],"acting":[11,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.a","version":"87'1","reported_seq":16,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.297486+0000","last_change":"2024-03-19T01:38:55.791193+0000","last_active":"2024-03-19T01:38:58.297486+0000","last_peered":"2024-03-19T01:38:58.297486+0000","last_clean":"2024-03-19T01:38:58.297486+0000","last_became_active":"2024-03-19T01:38:55.790820+0000","last_became_peered":"2024-03-19T01:38:55.790820+0000","last_unstale":"2024-03-19T01:38:58.297486+0000","last_undegraded":"2024-03-19T01:38:58.297486+0000","last_fullsized":"2024-03-19T01:38:58.297486+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T10:19:09.564798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,8],"acting":[2,3,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.799870+0000","last_change":"2024-03-19T01:38:56.803287+0000","last_active":"2024-03-19T01:38:57.799870+0000","last_peered":"2024-03-19T01:38:57.799870+0000","last_clean":"2024-03-19T01:38:57.799870+0000","last_became_active":"2024-03-19T01:38:56.802924+0000","last_became_peered":"2024-03-19T01:38:56.802924+0000","last_unstale":"2024-03-19T01:38:57.799870+0000","last_undegraded":"2024-03-19T01:38:57.799870+0000","last_fullsized":"2024-03-19T01:38:57.799870+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T07:45:23.726378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,10],"acting":[3,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"86'2","reported_seq":17,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.793690+0000","last_change":"2024-03-19T01:38:55.795320+0000","last_active":"2024-03-19T01:38:57.793690+0000","last_peered":"2024-03-19T01:38:57.793690+0000","last_clean":"2024-03-19T01:38:57.793690+0000","last_became_active":"2024-03-19T01:38:55.794478+0000","last_became_peered":"2024-03-19T01:38:55.794478+0000","last_unstale":"2024-03-19T01:38:57.793690+0000","last_undegraded":"2024-03-19T01:38:57.793690+0000","last_fullsized":"2024-03-19T01:38:57.793690+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T05:03:45.867009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.794801+0000","last_change":"2024-03-19T01:38:56.797272+0000","last_active":"2024-03-19T01:38:57.794801+0000","last_peered":"2024-03-19T01:38:57.794801+0000","last_clean":"2024-03-19T01:38:57.794801+0000","last_became_active":"2024-03-19T01:38:56.796944+0000","last_became_peered":"2024-03-19T01:38:56.796944+0000","last_unstale":"2024-03-19T01:38:57.794801+0000","last_undegraded":"2024-03-19T01:38:57.794801+0000","last_fullsized":"2024-03-19T01:38:57.794801+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T06:14:58.147093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,6],"acting":[7,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.c","version":"87'3","reported_seq":18,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:59.817159+0000","last_change":"2024-03-19T01:38:55.795642+0000","last_active":"2024-03-19T01:38:59.817159+0000","last_peered":"2024-03-19T01:38:59.817159+0000","last_clean":"2024-03-19T01:38:59.817159+0000","last_became_active":"2024-03-19T01:38:55.795285+0000","last_became_peered":"2024-03-19T01:38:55.795285+0000","last_unstale":"2024-03-19T01:38:59.817159+0000","last_undegraded":"2024-03-19T01:38:59.817159+0000","last_fullsized":"2024-03-19T01:38:59.817159+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T10:09:18.888802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,11,0],"acting":[2,11,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.795015+0000","last_change":"2024-03-19T01:38:56.801802+0000","last_active":"2024-03-19T01:38:57.795015+0000","last_peered":"2024-03-19T01:38:57.795015+0000","last_clean":"2024-03-19T01:38:57.795015+0000","last_became_active":"2024-03-19T01:38:56.801130+0000","last_became_peered":"2024-03-19T01:38:56.801130+0000","last_unstale":"2024-03-19T01:38:57.795015+0000","last_undegraded":"2024-03-19T01:38:57.795015+0000","last_fullsized":"2024-03-19T01:38:57.795015+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T10:36:45.509217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,11],"acting":[7,1,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"86'1","reported_seq":17,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.297606+0000","last_change":"2024-03-19T01:38:55.795085+0000","last_active":"2024-03-19T01:38:58.297606+0000","last_peered":"2024-03-19T01:38:58.297606+0000","last_clean":"2024-03-19T01:38:58.297606+0000","last_became_active":"2024-03-19T01:38:55.794626+0000","last_became_peered":"2024-03-19T01:38:55.794626+0000","last_unstale":"2024-03-19T01:38:58.297606+0000","last_undegraded":"2024-03-19T01:38:58.297606+0000","last_fullsized":"2024-03-19T01:38:58.297606+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T05:51:50.428331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,10,3],"acting":[5,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.794739+0000","last_change":"2024-03-19T01:38:56.802141+0000","last_active":"2024-03-19T01:38:57.794739+0000","last_peered":"2024-03-19T01:38:57.794739+0000","last_clean":"2024-03-19T01:38:57.794739+0000","last_became_active":"2024-03-19T01:38:56.801788+0000","last_became_peered":"2024-03-19T01:38:56.801788+0000","last_unstale":"2024-03-19T01:38:57.794739+0000","last_undegraded":"2024-03-19T01:38:57.794739+0000","last_fullsized":"2024-03-19T01:38:57.794739+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T10:13:39.264272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"86'2","reported_seq":17,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.297633+0000","last_change":"2024-03-19T01:38:55.792956+0000","last_active":"2024-03-19T01:38:58.297633+0000","last_peered":"2024-03-19T01:38:58.297633+0000","last_clean":"2024-03-19T01:38:58.297633+0000","last_became_active":"2024-03-19T01:38:55.792633+0000","last_became_peered":"2024-03-19T01:38:55.792633+0000","last_unstale":"2024-03-19T01:38:58.297633+0000","last_undegraded":"2024-03-19T01:38:58.297633+0000","last_fullsized":"2024-03-19T01:38:58.297633+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T11:17:14.276294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.794986+0000","last_change":"2024-03-19T01:38:56.794428+0000","last_active":"2024-03-19T01:38:57.794986+0000","last_peered":"2024-03-19T01:38:57.794986+0000","last_clean":"2024-03-19T01:38:57.794986+0000","last_became_active":"2024-03-19T01:38:56.794051+0000","last_became_peered":"2024-03-19T01:38:56.794051+0000","last_unstale":"2024-03-19T01:38:57.794986+0000","last_undegraded":"2024-03-19T01:38:57.794986+0000","last_fullsized":"2024-03-19T01:38:57.794986+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T02:07:20.707276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,11,8],"acting":[7,11,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.298850+0000","last_change":"2024-03-19T01:38:56.812596+0000","last_active":"2024-03-19T01:38:58.298850+0000","last_peered":"2024-03-19T01:38:58.298850+0000","last_clean":"2024-03-19T01:38:58.298850+0000","last_became_active":"2024-03-19T01:38:56.812268+0000","last_became_peered":"2024-03-19T01:38:56.812268+0000","last_unstale":"2024-03-19T01:38:58.298850+0000","last_undegraded":"2024-03-19T01:38:58.298850+0000","last_fullsized":"2024-03-19T01:38:58.298850+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T08:54:51.350908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,2,5],"acting":[11,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.297648+0000","last_change":"2024-03-19T01:38:56.804280+0000","last_active":"2024-03-19T01:38:58.297648+0000","last_peered":"2024-03-19T01:38:58.297648+0000","last_clean":"2024-03-19T01:38:58.297648+0000","last_became_active":"2024-03-19T01:38:56.801928+0000","last_became_peered":"2024-03-19T01:38:56.801928+0000","last_unstale":"2024-03-19T01:38:58.297648+0000","last_undegraded":"2024-03-19T01:38:58.297648+0000","last_fullsized":"2024-03-19T01:38:58.297648+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T11:49:29.778949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.297370+0000","last_change":"2024-03-19T01:38:56.798719+0000","last_active":"2024-03-19T01:38:58.297370+0000","last_peered":"2024-03-19T01:38:58.297370+0000","last_clean":"2024-03-19T01:38:58.297370+0000","last_became_active":"2024-03-19T01:38:56.798253+0000","last_became_peered":"2024-03-19T01:38:56.798253+0000","last_unstale":"2024-03-19T01:38:58.297370+0000","last_undegraded":"2024-03-19T01:38:58.297370+0000","last_fullsized":"2024-03-19T01:38:58.297370+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T06:27:56.754679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.793136+0000","last_change":"2024-03-19T01:38:56.795221+0000","last_active":"2024-03-19T01:38:57.793136+0000","last_peered":"2024-03-19T01:38:57.793136+0000","last_clean":"2024-03-19T01:38:57.793136+0000","last_became_active":"2024-03-19T01:38:56.794756+0000","last_became_peered":"2024-03-19T01:38:56.794756+0000","last_unstale":"2024-03-19T01:38:57.793136+0000","last_undegraded":"2024-03-19T01:38:57.793136+0000","last_fullsized":"2024-03-19T01:38:57.793136+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T10:05:54.394976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.297599+0000","last_change":"2024-03-19T01:38:56.794482+0000","last_active":"2024-03-19T01:38:58.297599+0000","last_peered":"2024-03-19T01:38:58.297599+0000","last_clean":"2024-03-19T01:38:58.297599+0000","last_became_active":"2024-03-19T01:38:56.794140+0000","last_became_peered":"2024-03-19T01:38:56.794140+0000","last_unstale":"2024-03-19T01:38:58.297599+0000","last_undegraded":"2024-03-19T01:38:58.297599+0000","last_fullsized":"2024-03-19T01:38:58.297599+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T04:29:38.985047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.298663+0000","last_change":"2024-03-19T01:38:56.802020+0000","last_active":"2024-03-19T01:38:58.298663+0000","last_peered":"2024-03-19T01:38:58.298663+0000","last_clean":"2024-03-19T01:38:58.298663+0000","last_became_active":"2024-03-19T01:38:56.801668+0000","last_became_peered":"2024-03-19T01:38:56.801668+0000","last_unstale":"2024-03-19T01:38:58.298663+0000","last_undegraded":"2024-03-19T01:38:58.298663+0000","last_fullsized":"2024-03-19T01:38:58.298663+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T04:13:55.199279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,8,5],"acting":[11,8,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.297975+0000","last_change":"2024-03-19T01:38:56.809100+0000","last_active":"2024-03-19T01:38:58.297975+0000","last_peered":"2024-03-19T01:38:58.297975+0000","last_clean":"2024-03-19T01:38:58.297975+0000","last_became_active":"2024-03-19T01:38:56.807985+0000","last_became_peered":"2024-03-19T01:38:56.807985+0000","last_unstale":"2024-03-19T01:38:58.297975+0000","last_undegraded":"2024-03-19T01:38:58.297975+0000","last_fullsized":"2024-03-19T01:38:58.297975+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T02:58:25.235837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,8,1],"acting":[2,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.799942+0000","last_change":"2024-03-19T01:38:56.799707+0000","last_active":"2024-03-19T01:38:57.799942+0000","last_peered":"2024-03-19T01:38:57.799942+0000","last_clean":"2024-03-19T01:38:57.799942+0000","last_became_active":"2024-03-19T01:38:56.799388+0000","last_became_peered":"2024-03-19T01:38:56.799388+0000","last_unstale":"2024-03-19T01:38:57.799942+0000","last_undegraded":"2024-03-19T01:38:57.799942+0000","last_fullsized":"2024-03-19T01:38:57.799942+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T11:10:18.360180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.297772+0000","last_change":"2024-03-19T01:38:56.786807+0000","last_active":"2024-03-19T01:38:58.297772+0000","last_peered":"2024-03-19T01:38:58.297772+0000","last_clean":"2024-03-19T01:38:58.297772+0000","last_became_active":"2024-03-19T01:38:56.786395+0000","last_became_peered":"2024-03-19T01:38:56.786395+0000","last_unstale":"2024-03-19T01:38:58.297772+0000","last_undegraded":"2024-03-19T01:38:58.297772+0000","last_fullsized":"2024-03-19T01:38:58.297772+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T05:19:20.228926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,9,3],"acting":[5,9,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:57.799636+0000","last_change":"2024-03-19T01:38:56.800759+0000","last_active":"2024-03-19T01:38:57.799636+0000","last_peered":"2024-03-19T01:38:57.799636+0000","last_clean":"2024-03-19T01:38:57.799636+0000","last_became_active":"2024-03-19T01:38:56.800462+0000","last_became_peered":"2024-03-19T01:38:56.800462+0000","last_unstale":"2024-03-19T01:38:57.799636+0000","last_undegraded":"2024-03-19T01:38:57.799636+0000","last_fullsized":"2024-03-19T01:38:57.799636+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T02:13:19.070148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,11],"acting":[3,2,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.298221+0000","last_change":"2024-03-19T01:38:56.802653+0000","last_active":"2024-03-19T01:38:58.298221+0000","last_peered":"2024-03-19T01:38:58.298221+0000","last_clean":"2024-03-19T01:38:58.298221+0000","last_became_active":"2024-03-19T01:38:56.802284+0000","last_became_peered":"2024-03-19T01:38:56.802284+0000","last_unstale":"2024-03-19T01:38:58.298221+0000","last_undegraded":"2024-03-19T01:38:58.298221+0000","last_fullsized":"2024-03-19T01:38:58.298221+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T06:17:13.936359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.297697+0000","last_change":"2024-03-19T01:38:56.794668+0000","last_active":"2024-03-19T01:38:58.297697+0000","last_peered":"2024-03-19T01:38:58.297697+0000","last_clean":"2024-03-19T01:38:58.297697+0000","last_became_active":"2024-03-19T01:38:56.794375+0000","last_became_peered":"2024-03-19T01:38:56.794375+0000","last_unstale":"2024-03-19T01:38:58.297697+0000","last_undegraded":"2024-03-19T01:38:58.297697+0000","last_fullsized":"2024-03-19T01:38:58.297697+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T08:26:20.528825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":13,"reported_epoch":87,"state":"active+clean","last_fresh":"2024-03-19T01:38:58.298601+0000","last_change":"2024-03-19T01:38:56.805535+0000","last_active":"2024-03-19T01:38:58.298601+0000","last_peered":"2024-03-19T01:38:58.298601+0000","last_clean":"2024-03-19T01:38:58.298601+0000","last_became_active":"2024-03-19T01:38:56.805211+0000","last_became_peered":"2024-03-19T01:38:56.805211+0000","last_unstale":"2024-03-19T01:38:58.298601+0000","last_undegraded":"2024-03-19T01:38:58.298601+0000","last_fullsized":"2024-03-19T01:38:58.298601+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T04:47:28.688137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,4,8],"acting":[11,4,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":234,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":192,"acting":192,"num_store_stats":12},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":109,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4157440,"data_stored":4132576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":137,"ondisk_log_size":137,"up":3,"acting":3,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":5870,"num_objects":52,"num_object_clones":0,"num_object_copies":156,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":52,"num_whiteouts":0,"num_read":24,"num_read_kb":19,"num_write":114,"num_write_kb":32,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":35,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":225280,"data_stored":34991,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":99,"ondisk_log_size":99,"up":96,"acting":96,"num_store_stats":12}],"osd_stats":[{"osd":11,"up_from":82,"seq":352187318285,"num_pgs":18,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27932,"kb_used_data":912,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717220,"statfs":{"total":95995035648,"available":95966433280,"internally_reserved":0,"allocated":933888,"data_stored":447510,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1596,"internal_metadata":27654596},"hb_peers":[0,1,2,4,5,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":10,"up_from":74,"seq":317827579923,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32712,"kb_used_data":1536,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93712440,"statfs":{"total":95995035648,"available":95961538560,"internally_reserved":0,"allocated":1572864,"data_stored":1041068,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1600,"internal_metadata":31914432},"hb_peers":[0,1,2,3,4,5,6,8,9,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":9,"up_from":68,"seq":292057776152,"num_pgs":17,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32108,"kb_used_data":932,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93713044,"statfs":{"total":95995035648,"available":95962157056,"internally_reserved":0,"allocated":954368,"data_stored":448328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1599,"internal_metadata":31914433},"hb_peers":[0,1,2,3,4,5,6,7,8,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":8,"up_from":60,"seq":257698037787,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27940,"kb_used_data":920,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717212,"statfs":{"total":95995035648,"available":95966425088,"internally_reserved":0,"allocated":942080,"data_stored":448114,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,2,3,4,5,6,7,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":7,"up_from":53,"seq":227633266722,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32684,"kb_used_data":1508,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93712468,"statfs":{"total":95995035648,"available":95961567232,"internally_reserved":0,"allocated":1544192,"data_stored":1040452,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":31914443},"hb_peers":[1,2,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":6,"up_from":47,"seq":201863462951,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32096,"kb_used_data":920,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93713056,"statfs":{"total":95995035648,"available":95962169344,"internally_reserved":0,"allocated":942080,"data_stored":447717,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":31914440},"hb_peers":[0,1,2,3,4,5,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":13,"seq":55834574910,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27944,"kb_used_data":924,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717208,"statfs":{"total":95995035648,"available":95966420992,"internally_reserved":0,"allocated":946176,"data_stored":449122,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2,3,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738435,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36248,"kb_used_data":904,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":93708904,"statfs":{"total":95995035648,"available":95957917696,"internally_reserved":0,"allocated":925696,"data_stored":439030,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":36174277},"hb_peers":[1,2,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":18,"seq":77309411385,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27956,"kb_used_data":936,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717196,"statfs":{"total":95995035648,"available":95966408704,"internally_reserved":0,"allocated":958464,"data_stored":449778,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,3,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":27,"seq":115964117045,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32108,"kb_used_data":924,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93713044,"statfs":{"total":95995035648,"available":95962157056,"internally_reserved":0,"allocated":946176,"data_stored":447704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31914439},"hb_peers":[0,1,2,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":34,"seq":146028888111,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27940,"kb_used_data":924,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717212,"statfs":{"total":95995035648,"available":95966425088,"internally_reserved":0,"allocated":946176,"data_stored":448545,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2,3,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":40,"seq":171798691883,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36856,"kb_used_data":1520,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":93708296,"statfs":{"total":95995035648,"available":95957295104,"internally_reserved":0,"allocated":1556480,"data_stored":1039745,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":36174277},"hb_peers":[0,1,2,3,4,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":20895,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":36864,"data_stored":2962,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1035,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1867,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":2574,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":818,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":45056,"data_stored":3190,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-19T01:39:03.382 INFO:tasks.cephfs.filesystem.ceph_manager:clean! 2024-03-19T01:39:03.382 INFO:tasks.cephadm:Setting up client nodes... 2024-03-19T01:39:03.382 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-03-19T01:39:03.653 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:39:04.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:03 smithi099 ceph-mon[62951]: from='client.14790 -' entity='client.admin' cmd=[{"prefix": "fs subvolume getpath", "vol_name": "cephfs", "group_name": "qa", "sub_name": "sv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:39:04.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:03 smithi099 ceph-mon[62951]: pgmap v285: 97 pgs: 97 active+clean; 582 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 KiB/s rd, 4.6 KiB/s wr, 19 op/s 2024-03-19T01:39:04.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:03 smithi112 ceph-mon[66819]: from='client.14790 -' entity='client.admin' cmd=[{"prefix": "fs subvolume getpath", "vol_name": "cephfs", "group_name": "qa", "sub_name": "sv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:39:04.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:03 smithi112 ceph-mon[66819]: pgmap v285: 97 pgs: 97 active+clean; 582 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 KiB/s rd, 4.6 KiB/s wr, 19 op/s 2024-03-19T01:39:04.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:03 smithi115 ceph-mon[67332]: from='client.14790 -' entity='client.admin' cmd=[{"prefix": "fs subvolume getpath", "vol_name": "cephfs", "group_name": "qa", "sub_name": "sv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:39:04.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:03 smithi115 ceph-mon[67332]: pgmap v285: 97 pgs: 97 active+clean; 582 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 KiB/s rd, 4.6 KiB/s wr, 19 op/s 2024-03-19T01:39:05.009 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:04 smithi099 ceph-mon[62951]: from='client.14808 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-19T01:39:05.009 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:04 smithi099 ceph-mon[62951]: from='client.14814 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-19T01:39:05.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:04 smithi112 ceph-mon[66819]: from='client.14808 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-19T01:39:05.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:04 smithi112 ceph-mon[66819]: from='client.14814 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-19T01:39:05.246 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:05.242+0000 7fbe4fbd0700 1 -- 172.21.15.99:0/4170318267 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbe48101270 msgr2=0x7fbe48101690 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:05.246 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:05.242+0000 7fbe4fbd0700 1 --2- 172.21.15.99:0/4170318267 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbe48101270 0x7fbe48101690 secure :-1 s=READY pgs=231 cs=0 l=1 rev1=1 crypto rx=0x7fbe440099c0 tx=0x7fbe44009cd0 comp rx=0 tx=0).stop 2024-03-19T01:39:05.246 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:05.242+0000 7fbe4fbd0700 1 -- 172.21.15.99:0/4170318267 shutdown_connections 2024-03-19T01:39:05.247 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:05.242+0000 7fbe4fbd0700 1 --2- 172.21.15.99:0/4170318267 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fbe48102e10 0x7fbe481096c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:05.247 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:05.242+0000 7fbe4fbd0700 1 --2- 172.21.15.99:0/4170318267 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbe48101270 0x7fbe48101690 unknown :-1 s=CLOSED pgs=231 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:05.247 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:05.242+0000 7fbe4fbd0700 1 --2- 172.21.15.99:0/4170318267 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fbe48102470 0x7fbe481028d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:05.247 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:05.242+0000 7fbe4fbd0700 1 -- 172.21.15.99:0/4170318267 >> 172.21.15.99:0/4170318267 conn(0x7fbe480fc830 msgr2=0x7fbe480fec50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:05.247 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:05.242+0000 7fbe4fbd0700 1 -- 172.21.15.99:0/4170318267 shutdown_connections 2024-03-19T01:39:05.247 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:05.242+0000 7fbe4fbd0700 1 -- 172.21.15.99:0/4170318267 wait complete. 2024-03-19T01:39:05.247 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:05.243+0000 7fbe4fbd0700 1 Processor -- start 2024-03-19T01:39:05.247 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:05.243+0000 7fbe4fbd0700 1 -- start start 2024-03-19T01:39:05.247 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:05.243+0000 7fbe4fbd0700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fbe48101270 0x7fbe48197420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:05.247 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:05.243+0000 7fbe4fbd0700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbe48102470 0x7fbe48197960 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:05.248 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:05.243+0000 7fbe4fbd0700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fbe48102e10 0x7fbe4819e9e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:05.248 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:05.243+0000 7fbe4fbd0700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fbe4810ce60 con 0x7fbe48102470 2024-03-19T01:39:05.248 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:05.243+0000 7fbe4fbd0700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fbe4810cce0 con 0x7fbe48101270 2024-03-19T01:39:05.248 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:05.243+0000 7fbe4fbd0700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fbe4810cfe0 con 0x7fbe48102e10 2024-03-19T01:39:05.248 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:05.244+0000 7fbe4d16b700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbe48102470 0x7fbe48197960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:05.248 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:05.244+0000 7fbe4d16b700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbe48102470 0x7fbe48197960 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:52052/0 (socket says 172.21.15.99:52052) 2024-03-19T01:39:05.248 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:05.244+0000 7fbe4d16b700 1 -- 172.21.15.99:0/2703114276 learned_addr learned my addr 172.21.15.99:0/2703114276 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:39:05.248 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:05.244+0000 7fbe4d96c700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fbe48101270 0x7fbe48197420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:05.248 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:05.244+0000 7fbe4e16d700 1 --2- 172.21.15.99:0/2703114276 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fbe48102e10 0x7fbe4819e9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:05.248 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:05.244+0000 7fbe4d16b700 1 -- 172.21.15.99:0/2703114276 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fbe48102e10 msgr2=0x7fbe4819e9e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:05.248 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:05.244+0000 7fbe4d16b700 1 --2- 172.21.15.99:0/2703114276 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fbe48102e10 0x7fbe4819e9e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:05.248 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:05.244+0000 7fbe4d16b700 1 -- 172.21.15.99:0/2703114276 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fbe48101270 msgr2=0x7fbe48197420 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:05.248 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:05.244+0000 7fbe4d16b700 1 --2- 172.21.15.99:0/2703114276 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fbe48101270 0x7fbe48197420 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:05.249 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:05.244+0000 7fbe4d16b700 1 -- 172.21.15.99:0/2703114276 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fbe4819f080 con 0x7fbe48102470 2024-03-19T01:39:05.249 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:05.244+0000 7fbe4d96c700 1 --2- 172.21.15.99:0/2703114276 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fbe48101270 0x7fbe48197420 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:39:05.249 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:05.244+0000 7fbe4d16b700 1 --2- 172.21.15.99:0/2703114276 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbe48102470 0x7fbe48197960 secure :-1 s=READY pgs=232 cs=0 l=1 rev1=1 crypto rx=0x7fbe3800e9d0 tx=0x7fbe3800ece0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:05.249 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:05.244+0000 7fbe3effd700 1 -- 172.21.15.99:0/2703114276 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbe3800cbd0 con 0x7fbe48102470 2024-03-19T01:39:05.249 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:05.244+0000 7fbe4fbd0700 1 -- 172.21.15.99:0/2703114276 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fbe4819f370 con 0x7fbe48102470 2024-03-19T01:39:05.249 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:05.244+0000 7fbe3effd700 1 -- 172.21.15.99:0/2703114276 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fbe3800ee70 con 0x7fbe48102470 2024-03-19T01:39:05.249 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:05.244+0000 7fbe3effd700 1 -- 172.21.15.99:0/2703114276 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbe38018680 con 0x7fbe48102470 2024-03-19T01:39:05.249 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:05.244+0000 7fbe4fbd0700 1 -- 172.21.15.99:0/2703114276 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fbe4806aa50 con 0x7fbe48102470 2024-03-19T01:39:05.249 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:05.245+0000 7fbe4fbd0700 1 -- 172.21.15.99:0/2703114276 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fbe48050820 con 0x7fbe48102470 2024-03-19T01:39:05.250 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:05.246+0000 7fbe3effd700 1 -- 172.21.15.99:0/2703114276 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 17) ==== 137341+0+0 (secure 0 0 0) 0x7fbe380188e0 con 0x7fbe48102470 2024-03-19T01:39:05.250 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:05.247+0000 7fbe3effd700 1 --2- 172.21.15.99:0/2703114276 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fbe340a4be0 0x7fbe340a70a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:05.250 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:05.247+0000 7fbe4d96c700 1 --2- 172.21.15.99:0/2703114276 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fbe340a4be0 0x7fbe340a70a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:05.251 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:05.247+0000 7fbe3effd700 1 -- 172.21.15.99:0/2703114276 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(87..87 src has 1..87) ==== 7979+0+0 (secure 0 0 0) 0x7fbe380cec30 con 0x7fbe48102470 2024-03-19T01:39:05.251 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:05.247+0000 7fbe4d96c700 1 --2- 172.21.15.99:0/2703114276 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fbe340a4be0 0x7fbe340a70a0 secure :-1 s=READY pgs=177 cs=0 l=1 rev1=1 crypto rx=0x7fbe4400b580 tx=0x7fbe44005fb0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:05.252 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:05.251+0000 7fbe3effd700 1 -- 172.21.15.99:0/2703114276 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7fbe3809bd20 con 0x7fbe48102470 2024-03-19T01:39:05.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:04 smithi115 ceph-mon[67332]: from='client.14808 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-19T01:39:05.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:04 smithi115 ceph-mon[67332]: from='client.14814 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-19T01:39:05.529 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:05.528+0000 7fbe4fbd0700 1 -- 172.21.15.99:0/2703114276 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]} v 0) -- 0x7fbe4806b730 con 0x7fbe48102470 2024-03-19T01:39:05.531 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:05.530+0000 7fbe3effd700 1 -- 172.21.15.99:0/2703114276 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]=0 v31) ==== 170+0+59 (secure 0 0 0) 0x7fbe4806b730 con 0x7fbe48102470 2024-03-19T01:39:05.531 INFO:teuthology.orchestra.run.smithi099.stdout:[client.0] 2024-03-19T01:39:05.532 INFO:teuthology.orchestra.run.smithi099.stdout: key = AQC57PhlUROeHxAA6NRfwHNd5QhsqNbbi+wi9g== 2024-03-19T01:39:05.534 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:05.532+0000 7fbe4fbd0700 1 -- 172.21.15.99:0/2703114276 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fbe340a4be0 msgr2=0x7fbe340a70a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:05.534 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:05.532+0000 7fbe4fbd0700 1 --2- 172.21.15.99:0/2703114276 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fbe340a4be0 0x7fbe340a70a0 secure :-1 s=READY pgs=177 cs=0 l=1 rev1=1 crypto rx=0x7fbe4400b580 tx=0x7fbe44005fb0 comp rx=0 tx=0).stop 2024-03-19T01:39:05.534 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:05.532+0000 7fbe4fbd0700 1 -- 172.21.15.99:0/2703114276 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbe48102470 msgr2=0x7fbe48197960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:05.534 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:05.532+0000 7fbe4fbd0700 1 --2- 172.21.15.99:0/2703114276 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbe48102470 0x7fbe48197960 secure :-1 s=READY pgs=232 cs=0 l=1 rev1=1 crypto rx=0x7fbe3800e9d0 tx=0x7fbe3800ece0 comp rx=0 tx=0).stop 2024-03-19T01:39:05.534 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:05.532+0000 7fbe4fbd0700 1 -- 172.21.15.99:0/2703114276 shutdown_connections 2024-03-19T01:39:05.535 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:05.532+0000 7fbe4fbd0700 1 --2- 172.21.15.99:0/2703114276 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fbe340a4be0 0x7fbe340a70a0 unknown :-1 s=CLOSED pgs=177 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:05.535 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:05.532+0000 7fbe4fbd0700 1 --2- 172.21.15.99:0/2703114276 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fbe48102e10 0x7fbe4819e9e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:05.535 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:05.532+0000 7fbe4fbd0700 1 --2- 172.21.15.99:0/2703114276 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fbe48101270 0x7fbe48197420 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:05.535 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:05.532+0000 7fbe4fbd0700 1 --2- 172.21.15.99:0/2703114276 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbe48102470 0x7fbe48197960 unknown :-1 s=CLOSED pgs=232 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:05.535 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:05.532+0000 7fbe4fbd0700 1 -- 172.21.15.99:0/2703114276 >> 172.21.15.99:0/2703114276 conn(0x7fbe480fc830 msgr2=0x7fbe48105770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:05.535 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:05.533+0000 7fbe4fbd0700 1 -- 172.21.15.99:0/2703114276 shutdown_connections 2024-03-19T01:39:05.535 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:05.533+0000 7fbe4fbd0700 1 -- 172.21.15.99:0/2703114276 wait complete. 2024-03-19T01:39:06.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:05 smithi099 ceph-mon[62951]: pgmap v286: 97 pgs: 97 active+clean; 582 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 3.0 KiB/s rd, 5.1 KiB/s wr, 19 op/s 2024-03-19T01:39:06.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:05 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/2703114276' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-19T01:39:06.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:05 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/2703114276' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-03-19T01:39:06.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:05 smithi112 ceph-mon[66819]: pgmap v286: 97 pgs: 97 active+clean; 582 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 3.0 KiB/s rd, 5.1 KiB/s wr, 19 op/s 2024-03-19T01:39:06.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:05 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/2703114276' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-19T01:39:06.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:05 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/2703114276' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-03-19T01:39:06.208 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2024-03-19T01:39:06.208 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-03-19T01:39:06.208 DEBUG:teuthology.orchestra.run.smithi099:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-03-19T01:39:06.251 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-03-19T01:39:06.251 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-03-19T01:39:06.251 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph mgr dump --format=json 2024-03-19T01:39:06.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:05 smithi115 ceph-mon[67332]: pgmap v286: 97 pgs: 97 active+clean; 582 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 3.0 KiB/s rd, 5.1 KiB/s wr, 19 op/s 2024-03-19T01:39:06.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:05 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/2703114276' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-19T01:39:06.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:05 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/2703114276' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-03-19T01:39:06.529 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:39:06.934 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:06 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.0", "id": [8, 1]}]: dispatch 2024-03-19T01:39:06.934 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:06 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.8", "id": [2, 11]}]: dispatch 2024-03-19T01:39:06.934 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:06 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.b", "id": [8, 11]}]: dispatch 2024-03-19T01:39:06.934 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:06 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.e", "id": [2, 4]}]: dispatch 2024-03-19T01:39:06.935 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:06 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.0", "id": [5, 9]}]: dispatch 2024-03-19T01:39:06.935 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:06 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.2", "id": [3, 9]}]: dispatch 2024-03-19T01:39:06.935 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:06 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.17", "id": [4, 8]}]: dispatch 2024-03-19T01:39:06.935 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:06 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.31", "id": [4, 8]}]: dispatch 2024-03-19T01:39:06.935 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:06 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.37", "id": [5, 7]}]: dispatch 2024-03-19T01:39:06.935 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:06 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3a", "id": [4, 6]}]: dispatch 2024-03-19T01:39:07.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:06 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.0", "id": [8, 1]}]: dispatch 2024-03-19T01:39:07.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:06 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.8", "id": [2, 11]}]: dispatch 2024-03-19T01:39:07.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:06 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.b", "id": [8, 11]}]: dispatch 2024-03-19T01:39:07.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:06 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.e", "id": [2, 4]}]: dispatch 2024-03-19T01:39:07.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:06 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.0", "id": [5, 9]}]: dispatch 2024-03-19T01:39:07.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:06 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.2", "id": [3, 9]}]: dispatch 2024-03-19T01:39:07.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:06 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.17", "id": [4, 8]}]: dispatch 2024-03-19T01:39:07.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:06 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.31", "id": [4, 8]}]: dispatch 2024-03-19T01:39:07.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:06 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.37", "id": [5, 7]}]: dispatch 2024-03-19T01:39:07.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:06 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3a", "id": [4, 6]}]: dispatch 2024-03-19T01:39:07.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:06 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.0", "id": [8, 1]}]: dispatch 2024-03-19T01:39:07.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:06 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.8", "id": [2, 11]}]: dispatch 2024-03-19T01:39:07.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:06 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.b", "id": [8, 11]}]: dispatch 2024-03-19T01:39:07.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:06 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.e", "id": [2, 4]}]: dispatch 2024-03-19T01:39:07.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:06 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.0", "id": [5, 9]}]: dispatch 2024-03-19T01:39:07.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:06 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.2", "id": [3, 9]}]: dispatch 2024-03-19T01:39:07.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:06 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.17", "id": [4, 8]}]: dispatch 2024-03-19T01:39:07.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:06 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.31", "id": [4, 8]}]: dispatch 2024-03-19T01:39:07.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:06 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.37", "id": [5, 7]}]: dispatch 2024-03-19T01:39:07.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:06 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3a", "id": [4, 6]}]: dispatch 2024-03-19T01:39:08.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:07 smithi099 ceph-mon[62951]: pgmap v287: 97 pgs: 97 active+clean; 618 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 7.2 KiB/s rd, 8.3 KiB/s wr, 19 op/s 2024-03-19T01:39:08.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:07 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.0", "id": [8, 1]}]': finished 2024-03-19T01:39:08.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:07 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.8", "id": [2, 11]}]': finished 2024-03-19T01:39:08.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:07 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.b", "id": [8, 11]}]': finished 2024-03-19T01:39:08.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:07 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.e", "id": [2, 4]}]': finished 2024-03-19T01:39:08.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:07 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.0", "id": [5, 9]}]': finished 2024-03-19T01:39:08.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:07 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.2", "id": [3, 9]}]': finished 2024-03-19T01:39:08.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:07 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.17", "id": [4, 8]}]': finished 2024-03-19T01:39:08.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:07 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.31", "id": [4, 8]}]': finished 2024-03-19T01:39:08.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:07 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.37", "id": [5, 7]}]': finished 2024-03-19T01:39:08.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:07 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3a", "id": [4, 6]}]': finished 2024-03-19T01:39:08.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:07 smithi099 ceph-mon[62951]: osdmap e88: 12 total, 12 up, 12 in 2024-03-19T01:39:08.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:07 smithi112 ceph-mon[66819]: pgmap v287: 97 pgs: 97 active+clean; 618 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 7.2 KiB/s rd, 8.3 KiB/s wr, 19 op/s 2024-03-19T01:39:08.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:07 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.0", "id": [8, 1]}]': finished 2024-03-19T01:39:08.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:07 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.8", "id": [2, 11]}]': finished 2024-03-19T01:39:08.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:07 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.b", "id": [8, 11]}]': finished 2024-03-19T01:39:08.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:07 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.e", "id": [2, 4]}]': finished 2024-03-19T01:39:08.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:07 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.0", "id": [5, 9]}]': finished 2024-03-19T01:39:08.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:07 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.2", "id": [3, 9]}]': finished 2024-03-19T01:39:08.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:07 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.17", "id": [4, 8]}]': finished 2024-03-19T01:39:08.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:07 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.31", "id": [4, 8]}]': finished 2024-03-19T01:39:08.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:07 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.37", "id": [5, 7]}]': finished 2024-03-19T01:39:08.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:07 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3a", "id": [4, 6]}]': finished 2024-03-19T01:39:08.113 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:07 smithi112 ceph-mon[66819]: osdmap e88: 12 total, 12 up, 12 in 2024-03-19T01:39:08.291 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:08.290+0000 7fab46985700 1 -- 172.21.15.99:0/684897418 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fab40106530 msgr2=0x7fab4010e100 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:08.291 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:08.290+0000 7fab46985700 1 --2- 172.21.15.99:0/684897418 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fab40106530 0x7fab4010e100 secure :-1 s=READY pgs=233 cs=0 l=1 rev1=1 crypto rx=0x7fab38009f90 tx=0x7fab3800b330 comp rx=0 tx=0).stop 2024-03-19T01:39:08.291 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:08.290+0000 7fab46985700 1 -- 172.21.15.99:0/684897418 shutdown_connections 2024-03-19T01:39:08.291 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:08.290+0000 7fab46985700 1 --2- 172.21.15.99:0/684897418 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fab40106530 0x7fab4010e100 unknown :-1 s=CLOSED pgs=233 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:08.291 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:08.290+0000 7fab46985700 1 --2- 172.21.15.99:0/684897418 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fab401014a0 0x7fab40101880 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:08.291 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:08.290+0000 7fab46985700 1 --2- 172.21.15.99:0/684897418 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fab40101e50 0x7fab40105e50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:08.292 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:08.290+0000 7fab46985700 1 -- 172.21.15.99:0/684897418 >> 172.21.15.99:0/684897418 conn(0x7fab40077230 msgr2=0x7fab40077640 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:08.292 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:08.290+0000 7fab46985700 1 -- 172.21.15.99:0/684897418 shutdown_connections 2024-03-19T01:39:08.292 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:08.290+0000 7fab46985700 1 -- 172.21.15.99:0/684897418 wait complete. 2024-03-19T01:39:08.292 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:08.291+0000 7fab46985700 1 Processor -- start 2024-03-19T01:39:08.292 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:08.291+0000 7fab46985700 1 -- start start 2024-03-19T01:39:08.293 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:08.291+0000 7fab46985700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fab401014a0 0x7fab4019d600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:08.293 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:08.291+0000 7fab46985700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fab40101e50 0x7fab4019db40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:08.293 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:08.291+0000 7fab46985700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fab40106530 0x7fab401977d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:08.293 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:08.291+0000 7fab46985700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fab40111510 con 0x7fab401014a0 2024-03-19T01:39:08.294 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:08.291+0000 7fab46985700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fab40111390 con 0x7fab40101e50 2024-03-19T01:39:08.294 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:08.291+0000 7fab46985700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fab40111690 con 0x7fab40106530 2024-03-19T01:39:08.294 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:08.291+0000 7fab3ffff700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fab401014a0 0x7fab4019d600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:08.294 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:08.291+0000 7fab3ffff700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fab401014a0 0x7fab4019d600 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:52068/0 (socket says 172.21.15.99:52068) 2024-03-19T01:39:08.294 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:08.291+0000 7fab3ffff700 1 -- 172.21.15.99:0/806524342 learned_addr learned my addr 172.21.15.99:0/806524342 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:39:08.294 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:08.292+0000 7fab3f7fe700 1 --2- 172.21.15.99:0/806524342 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fab40101e50 0x7fab4019db40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:08.294 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:08.292+0000 7fab44f22700 1 --2- 172.21.15.99:0/806524342 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fab40106530 0x7fab401977d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:08.294 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:08.292+0000 7fab3f7fe700 1 -- 172.21.15.99:0/806524342 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fab40106530 msgr2=0x7fab401977d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:08.294 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:08.292+0000 7fab3f7fe700 1 --2- 172.21.15.99:0/806524342 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fab40106530 0x7fab401977d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:08.294 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:08.292+0000 7fab3f7fe700 1 -- 172.21.15.99:0/806524342 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fab401014a0 msgr2=0x7fab4019d600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:08.294 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:08.292+0000 7fab3f7fe700 1 --2- 172.21.15.99:0/806524342 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fab401014a0 0x7fab4019d600 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:08.294 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:08.292+0000 7fab3f7fe700 1 -- 172.21.15.99:0/806524342 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fab40197fb0 con 0x7fab40101e50 2024-03-19T01:39:08.294 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:08.292+0000 7fab3ffff700 1 --2- 172.21.15.99:0/806524342 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fab401014a0 0x7fab4019d600 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:39:08.295 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:08.292+0000 7fab3f7fe700 1 --2- 172.21.15.99:0/806524342 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fab40101e50 0x7fab4019db40 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7fab3000e9c0 tx=0x7fab3000ecd0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:08.295 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:08.292+0000 7fab3d7fa700 1 -- 172.21.15.99:0/806524342 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fab30004220 con 0x7fab40101e50 2024-03-19T01:39:08.295 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:08.292+0000 7fab3d7fa700 1 -- 172.21.15.99:0/806524342 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fab30004d10 con 0x7fab40101e50 2024-03-19T01:39:08.295 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:08.292+0000 7fab46985700 1 -- 172.21.15.99:0/806524342 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fab401982a0 con 0x7fab40101e50 2024-03-19T01:39:08.295 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:08.292+0000 7fab3d7fa700 1 -- 172.21.15.99:0/806524342 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fab30005020 con 0x7fab40101e50 2024-03-19T01:39:08.295 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:08.292+0000 7fab46985700 1 -- 172.21.15.99:0/806524342 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fab401a4460 con 0x7fab40101e50 2024-03-19T01:39:08.295 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:08.293+0000 7fab46985700 1 -- 172.21.15.99:0/806524342 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fab40102930 con 0x7fab40101e50 2024-03-19T01:39:08.296 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:08.294+0000 7fab3d7fa700 1 -- 172.21.15.99:0/806524342 <== mon.2 v2:172.21.15.112:3300/0 4 ==== mgrmap(e 17) ==== 137341+0+0 (secure 0 0 0) 0x7fab30004750 con 0x7fab40101e50 2024-03-19T01:39:08.296 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:08.295+0000 7fab3d7fa700 1 --2- 172.21.15.99:0/806524342 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fab280a2920 0x7fab280a4de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:08.296 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:08.295+0000 7fab3ffff700 1 --2- 172.21.15.99:0/806524342 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fab280a2920 0x7fab280a4de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:08.296 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:08.295+0000 7fab3d7fa700 1 -- 172.21.15.99:0/806524342 <== mon.2 v2:172.21.15.112:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8269+0+0 (secure 0 0 0) 0x7fab300cea30 con 0x7fab40101e50 2024-03-19T01:39:08.296 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:08.295+0000 7fab3ffff700 1 --2- 172.21.15.99:0/806524342 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fab280a2920 0x7fab280a4de0 secure :-1 s=READY pgs=178 cs=0 l=1 rev1=1 crypto rx=0x7fab34009750 tx=0x7fab34009310 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:08.298 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:08.298+0000 7fab3d7fa700 1 -- 172.21.15.99:0/806524342 <== mon.2 v2:172.21.15.112:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7fab3009b950 con 0x7fab40101e50 2024-03-19T01:39:08.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:07 smithi115 ceph-mon[67332]: pgmap v287: 97 pgs: 97 active+clean; 618 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 7.2 KiB/s rd, 8.3 KiB/s wr, 19 op/s 2024-03-19T01:39:08.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:07 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.0", "id": [8, 1]}]': finished 2024-03-19T01:39:08.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:07 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.8", "id": [2, 11]}]': finished 2024-03-19T01:39:08.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:07 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.b", "id": [8, 11]}]': finished 2024-03-19T01:39:08.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:07 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.e", "id": [2, 4]}]': finished 2024-03-19T01:39:08.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:07 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.0", "id": [5, 9]}]': finished 2024-03-19T01:39:08.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:07 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.2", "id": [3, 9]}]': finished 2024-03-19T01:39:08.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:07 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.17", "id": [4, 8]}]': finished 2024-03-19T01:39:08.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:07 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.31", "id": [4, 8]}]': finished 2024-03-19T01:39:08.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:07 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.37", "id": [5, 7]}]': finished 2024-03-19T01:39:08.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:07 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3a", "id": [4, 6]}]': finished 2024-03-19T01:39:08.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:07 smithi115 ceph-mon[67332]: osdmap e88: 12 total, 12 up, 12 in 2024-03-19T01:39:08.546 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:08.546+0000 7fab46985700 1 -- 172.21.15.99:0/806524342 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_command({"prefix": "mgr dump", "format": "json"} v 0) -- 0x7fab40198ce0 con 0x7fab40101e50 2024-03-19T01:39:08.551 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:08.551+0000 7fab3d7fa700 1 -- 172.21.15.99:0/806524342 <== mon.2 v2:172.21.15.112:3300/0 7 ==== mon_command_ack([{"prefix": "mgr dump", "format": "json"}]=0 v17) ==== 74+0+262658 (secure 0 0 0) 0x7fab40198ce0 con 0x7fab40101e50 2024-03-19T01:39:08.554 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:39:08.556 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:08.553+0000 7fab46985700 1 -- 172.21.15.99:0/806524342 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fab280a2920 msgr2=0x7fab280a4de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:08.556 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:08.553+0000 7fab46985700 1 --2- 172.21.15.99:0/806524342 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fab280a2920 0x7fab280a4de0 secure :-1 s=READY pgs=178 cs=0 l=1 rev1=1 crypto rx=0x7fab34009750 tx=0x7fab34009310 comp rx=0 tx=0).stop 2024-03-19T01:39:08.556 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:08.553+0000 7fab46985700 1 -- 172.21.15.99:0/806524342 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fab40101e50 msgr2=0x7fab4019db40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:08.556 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:08.553+0000 7fab46985700 1 --2- 172.21.15.99:0/806524342 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fab40101e50 0x7fab4019db40 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7fab3000e9c0 tx=0x7fab3000ecd0 comp rx=0 tx=0).stop 2024-03-19T01:39:08.556 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:08.553+0000 7fab46985700 1 -- 172.21.15.99:0/806524342 shutdown_connections 2024-03-19T01:39:08.557 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:08.553+0000 7fab46985700 1 --2- 172.21.15.99:0/806524342 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fab280a2920 0x7fab280a4de0 unknown :-1 s=CLOSED pgs=178 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:08.557 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:08.554+0000 7fab46985700 1 --2- 172.21.15.99:0/806524342 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fab40106530 0x7fab401977d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:08.557 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:08.554+0000 7fab46985700 1 --2- 172.21.15.99:0/806524342 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fab401014a0 0x7fab4019d600 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:08.557 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:08.554+0000 7fab46985700 1 --2- 172.21.15.99:0/806524342 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fab40101e50 0x7fab4019db40 unknown :-1 s=CLOSED pgs=55 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:08.557 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:08.554+0000 7fab46985700 1 -- 172.21.15.99:0/806524342 >> 172.21.15.99:0/806524342 conn(0x7fab40077230 msgr2=0x7fab400ff290 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:08.557 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:08.554+0000 7fab46985700 1 -- 172.21.15.99:0/806524342 shutdown_connections 2024-03-19T01:39:08.557 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:08.554+0000 7fab46985700 1 -- 172.21.15.99:0/806524342 wait complete. 2024-03-19T01:39:09.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:08 smithi099 ceph-mon[62951]: osdmap e89: 12 total, 12 up, 12 in 2024-03-19T01:39:09.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:08 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/806524342' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-03-19T01:39:09.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:08 smithi112 ceph-mon[66819]: osdmap e89: 12 total, 12 up, 12 in 2024-03-19T01:39:09.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:08 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/806524342' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-03-19T01:39:09.182 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":17,"flags":0,"active_gid":14150,"active_name":"x","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6800","nonce":4272410295},{"type":"v1","addr":"172.21.15.99:6801","nonce":4272410295}]},"active_addr":"172.21.15.99:6801/4272410295","active_change":"2024-03-19T01:32:06.661109+0000","active_mgr_features":4540701547738038271,"available":true,"standbys":[{"gid":24106,"name":"z","mgr_features":4540701547738038271,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.12","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]},{"gid":24110,"name":"y","mgr_features":4540701547738038271,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.12","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.12","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.99:8443/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":3,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.99:0","nonce":121881045}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.99:0","nonce":750780629}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.99:0","nonce":223376036}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.99:0","nonce":1928788970}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.99:0","nonce":1300716747}]}]} 2024-03-19T01:39:09.187 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-03-19T01:39:09.187 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-03-19T01:39:09.187 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph osd dump --format=json 2024-03-19T01:39:09.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:08 smithi115 ceph-mon[67332]: osdmap e89: 12 total, 12 up, 12 in 2024-03-19T01:39:09.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:08 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/806524342' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-03-19T01:39:09.428 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:39:10.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:09 smithi099 ceph-mon[62951]: pgmap v290: 97 pgs: 1 peering, 96 active+clean; 618 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 9.1 KiB/s rd, 8.5 KiB/s wr, 17 op/s 2024-03-19T01:39:10.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:09 smithi112 ceph-mon[66819]: pgmap v290: 97 pgs: 1 peering, 96 active+clean; 618 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 9.1 KiB/s rd, 8.5 KiB/s wr, 17 op/s 2024-03-19T01:39:10.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:09 smithi115 ceph-mon[67332]: pgmap v290: 97 pgs: 1 peering, 96 active+clean; 618 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 9.1 KiB/s rd, 8.5 KiB/s wr, 17 op/s 2024-03-19T01:39:11.036 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:11.034+0000 7fda8dab8700 1 -- 172.21.15.99:0/1434436160 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fda88101bb0 msgr2=0x7fda8810e090 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:11.037 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:11.034+0000 7fda8dab8700 1 --2- 172.21.15.99:0/1434436160 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fda88101bb0 0x7fda8810e090 secure :-1 s=READY pgs=234 cs=0 l=1 rev1=1 crypto rx=0x7fda8000b0a0 tx=0x7fda8000b3b0 comp rx=0 tx=0).stop 2024-03-19T01:39:11.037 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:11.034+0000 7fda8dab8700 1 -- 172.21.15.99:0/1434436160 shutdown_connections 2024-03-19T01:39:11.037 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:11.034+0000 7fda8dab8700 1 --2- 172.21.15.99:0/1434436160 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fda88101bb0 0x7fda8810e090 unknown :-1 s=CLOSED pgs=234 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:11.037 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:11.034+0000 7fda8dab8700 1 --2- 172.21.15.99:0/1434436160 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fda88107250 0x7fda88107630 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:11.037 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:11.034+0000 7fda8dab8700 1 --2- 172.21.15.99:0/1434436160 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fda88101210 0x7fda88101670 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:11.037 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:11.034+0000 7fda8dab8700 1 -- 172.21.15.99:0/1434436160 >> 172.21.15.99:0/1434436160 conn(0x7fda88078140 msgr2=0x7fda88078550 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:11.037 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:11.034+0000 7fda8dab8700 1 -- 172.21.15.99:0/1434436160 shutdown_connections 2024-03-19T01:39:11.037 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:11.034+0000 7fda8dab8700 1 -- 172.21.15.99:0/1434436160 wait complete. 2024-03-19T01:39:11.037 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:11.035+0000 7fda8dab8700 1 Processor -- start 2024-03-19T01:39:11.037 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:11.035+0000 7fda8dab8700 1 -- start start 2024-03-19T01:39:11.037 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:11.035+0000 7fda8dab8700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fda88101210 0x7fda88199890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:11.038 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:11.035+0000 7fda8dab8700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fda88101bb0 0x7fda88199dd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:11.038 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:11.035+0000 7fda8dab8700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fda88107250 0x7fda881a4910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:11.038 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:11.035+0000 7fda8dab8700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fda881114d0 con 0x7fda88101210 2024-03-19T01:39:11.038 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:11.036+0000 7fda8dab8700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fda88111350 con 0x7fda88101bb0 2024-03-19T01:39:11.038 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:11.036+0000 7fda8dab8700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fda88111650 con 0x7fda88107250 2024-03-19T01:39:11.038 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:11.036+0000 7fda877fe700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fda88101210 0x7fda88199890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:11.038 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:11.036+0000 7fda87fff700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fda88107250 0x7fda881a4910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:11.038 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:11.036+0000 7fda877fe700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fda88101210 0x7fda88199890 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:52086/0 (socket says 172.21.15.99:52086) 2024-03-19T01:39:11.038 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:11.036+0000 7fda877fe700 1 -- 172.21.15.99:0/2719856178 learned_addr learned my addr 172.21.15.99:0/2719856178 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:39:11.038 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:11.036+0000 7fda7ffff700 1 --2- 172.21.15.99:0/2719856178 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fda88101bb0 0x7fda88199dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:11.038 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:11.036+0000 7fda7ffff700 1 -- 172.21.15.99:0/2719856178 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fda88107250 msgr2=0x7fda881a4910 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:11.038 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:11.036+0000 7fda7ffff700 1 --2- 172.21.15.99:0/2719856178 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fda88107250 0x7fda881a4910 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:11.038 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:11.036+0000 7fda7ffff700 1 -- 172.21.15.99:0/2719856178 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fda88101210 msgr2=0x7fda88199890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:11.038 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:11.036+0000 7fda7ffff700 1 --2- 172.21.15.99:0/2719856178 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fda88101210 0x7fda88199890 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:11.039 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:11.036+0000 7fda7ffff700 1 -- 172.21.15.99:0/2719856178 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fda881a4e50 con 0x7fda88101bb0 2024-03-19T01:39:11.039 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:11.036+0000 7fda87fff700 1 --2- 172.21.15.99:0/2719856178 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fda88107250 0x7fda881a4910 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:39:11.039 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:11.036+0000 7fda7ffff700 1 --2- 172.21.15.99:0/2719856178 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fda88101bb0 0x7fda88199dd0 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7fda7000e9f0 tx=0x7fda7000ed00 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:11.039 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:11.036+0000 7fda857fa700 1 -- 172.21.15.99:0/2719856178 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fda70004220 con 0x7fda88101bb0 2024-03-19T01:39:11.039 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:11.036+0000 7fda857fa700 1 -- 172.21.15.99:0/2719856178 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fda70004d10 con 0x7fda88101bb0 2024-03-19T01:39:11.039 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:11.036+0000 7fda857fa700 1 -- 172.21.15.99:0/2719856178 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fda70005020 con 0x7fda88101bb0 2024-03-19T01:39:11.039 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:11.036+0000 7fda8dab8700 1 -- 172.21.15.99:0/2719856178 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fda881a5080 con 0x7fda88101bb0 2024-03-19T01:39:11.039 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:11.036+0000 7fda8dab8700 1 -- 172.21.15.99:0/2719856178 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fda881a5570 con 0x7fda88101bb0 2024-03-19T01:39:11.040 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:11.037+0000 7fda8dab8700 1 -- 172.21.15.99:0/2719856178 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fda88068fd0 con 0x7fda88101bb0 2024-03-19T01:39:11.040 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:11.038+0000 7fda857fa700 1 -- 172.21.15.99:0/2719856178 <== mon.2 v2:172.21.15.112:3300/0 4 ==== mgrmap(e 17) ==== 137341+0+0 (secure 0 0 0) 0x7fda70004750 con 0x7fda88101bb0 2024-03-19T01:39:11.040 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:11.039+0000 7fda857fa700 1 --2- 172.21.15.99:0/2719856178 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fda740a2920 0x7fda740a4de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:11.040 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:11.039+0000 7fda877fe700 1 --2- 172.21.15.99:0/2719856178 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fda740a2920 0x7fda740a4de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:11.040 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:11.039+0000 7fda857fa700 1 -- 172.21.15.99:0/2719856178 <== mon.2 v2:172.21.15.112:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8269+0+0 (secure 0 0 0) 0x7fda700ceb00 con 0x7fda88101bb0 2024-03-19T01:39:11.040 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:11.039+0000 7fda877fe700 1 --2- 172.21.15.99:0/2719856178 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fda740a2920 0x7fda740a4de0 secure :-1 s=READY pgs=179 cs=0 l=1 rev1=1 crypto rx=0x7fda78009ca0 tx=0x7fda78009260 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:11.043 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:11.043+0000 7fda857fa700 1 -- 172.21.15.99:0/2719856178 <== mon.2 v2:172.21.15.112:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7fda7009b9c0 con 0x7fda88101bb0 2024-03-19T01:39:11.248 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:11.248+0000 7fda8dab8700 1 -- 172.21.15.99:0/2719856178 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7fda8802dbb0 con 0x7fda88101bb0 2024-03-19T01:39:11.249 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:11.249+0000 7fda857fa700 1 -- 172.21.15.99:0/2719856178 <== mon.2 v2:172.21.15.112:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v89) ==== 74+0+22757 (secure 0 0 0) 0x7fda7009b2c0 con 0x7fda88101bb0 2024-03-19T01:39:11.250 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:39:11.250 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":89,"fsid":"438fb69c-e590-11ee-95c9-87774f69a715","created":"2024-03-19T01:31:10.165073+0000","modified":"2024-03-19T01:39:07.825595+0000","last_up_change":"2024-03-19T01:38:09.082688+0000","last_in_change":"2024-03-19T01:37:49.115793+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-19T01:34:31.001309+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-19T01:38:54.018623+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"86","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-19T01:38:55.319036+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"87","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"0c407a0e-a40e-43d7-ad9a-bb8f3cd4a9ba","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6803","nonce":2935260112}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6805","nonce":2935260112}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6808","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6809","nonce":2935260112}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6807","nonce":2935260112}]},"public_addr":"172.21.15.99:6803/2935260112","cluster_addr":"172.21.15.99:6805/2935260112","heartbeat_back_addr":"172.21.15.99:6809/2935260112","heartbeat_front_addr":"172.21.15.99:6807/2935260112","state":["exists","up"]},{"osd":1,"uuid":"a48f9173-4a06-49f5-bca2-2c9047b8f666","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6800","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6801","nonce":1468854289}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6802","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6803","nonce":1468854289}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6806","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6807","nonce":1468854289}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6804","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6805","nonce":1468854289}]},"public_addr":"172.21.15.112:6801/1468854289","cluster_addr":"172.21.15.112:6803/1468854289","heartbeat_back_addr":"172.21.15.112:6807/1468854289","heartbeat_front_addr":"172.21.15.112:6805/1468854289","state":["exists","up"]},{"osd":2,"uuid":"b7715fdd-b6c9-4348-bd16-5eb61b427eef","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6800","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6801","nonce":3766690472}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6802","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6803","nonce":3766690472}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6806","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6807","nonce":3766690472}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6804","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6805","nonce":3766690472}]},"public_addr":"172.21.15.115:6801/3766690472","cluster_addr":"172.21.15.115:6803/3766690472","heartbeat_back_addr":"172.21.15.115:6807/3766690472","heartbeat_front_addr":"172.21.15.115:6805/3766690472","state":["exists","up"]},{"osd":3,"uuid":"6697ed1b-96bf-42de-8440-d17e9785b359","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6810","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6811","nonce":3756732895}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6812","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6813","nonce":3756732895}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6816","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6817","nonce":3756732895}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6814","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6815","nonce":3756732895}]},"public_addr":"172.21.15.99:6811/3756732895","cluster_addr":"172.21.15.99:6813/3756732895","heartbeat_back_addr":"172.21.15.99:6817/3756732895","heartbeat_front_addr":"172.21.15.99:6815/3756732895","state":["exists","up"]},{"osd":4,"uuid":"d958d6b9-67f6-43d0-b704-aa55dc9df83a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":34,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6808","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6809","nonce":3717986391}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6810","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6811","nonce":3717986391}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6814","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6815","nonce":3717986391}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6812","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6813","nonce":3717986391}]},"public_addr":"172.21.15.112:6809/3717986391","cluster_addr":"172.21.15.112:6811/3717986391","heartbeat_back_addr":"172.21.15.112:6815/3717986391","heartbeat_front_addr":"172.21.15.112:6813/3717986391","state":["exists","up"]},{"osd":5,"uuid":"a73eb49a-7925-49ba-bc4f-515e17f8b8ae","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6808","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6809","nonce":1971806408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6810","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6811","nonce":1971806408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6814","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6815","nonce":1971806408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6812","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6813","nonce":1971806408}]},"public_addr":"172.21.15.115:6809/1971806408","cluster_addr":"172.21.15.115:6811/1971806408","heartbeat_back_addr":"172.21.15.115:6815/1971806408","heartbeat_front_addr":"172.21.15.115:6813/1971806408","state":["exists","up"]},{"osd":6,"uuid":"caa718cd-e08b-48de-832a-29546d71e508","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6818","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6819","nonce":1130832772}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6820","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6821","nonce":1130832772}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6824","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6825","nonce":1130832772}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6822","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6823","nonce":1130832772}]},"public_addr":"172.21.15.99:6819/1130832772","cluster_addr":"172.21.15.99:6821/1130832772","heartbeat_back_addr":"172.21.15.99:6825/1130832772","heartbeat_front_addr":"172.21.15.99:6823/1130832772","state":["exists","up"]},{"osd":7,"uuid":"dbee19a1-2315-40d3-88e7-876fcde28903","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":53,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6816","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6817","nonce":3610356576}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6818","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6819","nonce":3610356576}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6822","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6823","nonce":3610356576}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6820","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6821","nonce":3610356576}]},"public_addr":"172.21.15.112:6817/3610356576","cluster_addr":"172.21.15.112:6819/3610356576","heartbeat_back_addr":"172.21.15.112:6823/3610356576","heartbeat_front_addr":"172.21.15.112:6821/3610356576","state":["exists","up"]},{"osd":8,"uuid":"14a866dc-5ce3-42fc-8bcf-9088b68726cb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":60,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6816","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6817","nonce":4255803111}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6818","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6819","nonce":4255803111}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6822","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6823","nonce":4255803111}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6820","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6821","nonce":4255803111}]},"public_addr":"172.21.15.115:6817/4255803111","cluster_addr":"172.21.15.115:6819/4255803111","heartbeat_back_addr":"172.21.15.115:6823/4255803111","heartbeat_front_addr":"172.21.15.115:6821/4255803111","state":["exists","up"]},{"osd":9,"uuid":"1db93652-6035-4906-91ae-d64d5a909502","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":68,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6826","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6827","nonce":1410921523}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6828","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6829","nonce":1410921523}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6832","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6833","nonce":1410921523}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6830","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6831","nonce":1410921523}]},"public_addr":"172.21.15.99:6827/1410921523","cluster_addr":"172.21.15.99:6829/1410921523","heartbeat_back_addr":"172.21.15.99:6833/1410921523","heartbeat_front_addr":"172.21.15.99:6831/1410921523","state":["exists","up"]},{"osd":10,"uuid":"6c50a8a7-3345-4f4a-af0e-5606c7aca453","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":74,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6824","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6825","nonce":891805922}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6826","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6827","nonce":891805922}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6830","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6831","nonce":891805922}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6828","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6829","nonce":891805922}]},"public_addr":"172.21.15.112:6825/891805922","cluster_addr":"172.21.15.112:6827/891805922","heartbeat_back_addr":"172.21.15.112:6831/891805922","heartbeat_front_addr":"172.21.15.112:6829/891805922","state":["exists","up"]},{"osd":11,"uuid":"fd63b7fd-0db5-46d5-82fa-9707bbd0b495","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":82,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6824","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6825","nonce":938436244}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6826","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6827","nonce":938436244}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6830","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6831","nonce":938436244}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6828","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6829","nonce":938436244}]},"public_addr":"172.21.15.115:6825/938436244","cluster_addr":"172.21.15.115:6827/938436244","heartbeat_back_addr":"172.21.15.115:6831/938436244","heartbeat_front_addr":"172.21.15.115:6829/938436244","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:33:39.036822+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:34:03.339566+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:34:27.149273+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:34:51.315561+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:35:15.396031+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:35:38.957168+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:36:03.213914+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:36:27.763699+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:36:51.688670+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:37:17.280165+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:37:42.554633+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:38:07.438039+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.0","mappings":[{"from":8,"to":1}]},{"pgid":"2.8","mappings":[{"from":2,"to":11}]},{"pgid":"2.b","mappings":[{"from":8,"to":11}]},{"pgid":"2.e","mappings":[{"from":2,"to":4}]},{"pgid":"3.0","mappings":[{"from":5,"to":9}]},{"pgid":"3.2","mappings":[{"from":3,"to":9}]},{"pgid":"3.17","mappings":[{"from":4,"to":8}]},{"pgid":"3.31","mappings":[{"from":4,"to":8}]},{"pgid":"3.37","mappings":[{"from":5,"to":7}]},{"pgid":"3.3a","mappings":[{"from":4,"to":6}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.99:6801/1328245934":"2024-03-20T01:32:06.661002+0000","172.21.15.99:0/1618965837":"2024-03-20T01:32:06.661002+0000","172.21.15.99:0/2491429851":"2024-03-20T01:32:06.661002+0000","172.21.15.99:6800/1328245934":"2024-03-20T01:32:06.661002+0000","172.21.15.99:0/2112231766":"2024-03-20T01:31:40.161751+0000","172.21.15.99:0/3341836545":"2024-03-20T01:31:40.161751+0000","172.21.15.99:6800/4223048293":"2024-03-20T01:31:40.161751+0000","172.21.15.99:6801/4223048293":"2024-03-20T01:31:40.161751+0000","172.21.15.99:0/2377621863":"2024-03-20T01:31:40.161751+0000","172.21.15.99:0/1904400935":"2024-03-20T01:32:06.661002+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-19T01:39:11.252 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:11.252+0000 7fda8dab8700 1 -- 172.21.15.99:0/2719856178 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fda740a2920 msgr2=0x7fda740a4de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:11.252 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:11.252+0000 7fda8dab8700 1 --2- 172.21.15.99:0/2719856178 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fda740a2920 0x7fda740a4de0 secure :-1 s=READY pgs=179 cs=0 l=1 rev1=1 crypto rx=0x7fda78009ca0 tx=0x7fda78009260 comp rx=0 tx=0).stop 2024-03-19T01:39:11.252 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:11.252+0000 7fda8dab8700 1 -- 172.21.15.99:0/2719856178 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fda88101bb0 msgr2=0x7fda88199dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:11.252 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:11.252+0000 7fda8dab8700 1 --2- 172.21.15.99:0/2719856178 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fda88101bb0 0x7fda88199dd0 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7fda7000e9f0 tx=0x7fda7000ed00 comp rx=0 tx=0).stop 2024-03-19T01:39:11.252 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:11.252+0000 7fda8dab8700 1 -- 172.21.15.99:0/2719856178 shutdown_connections 2024-03-19T01:39:11.252 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:11.252+0000 7fda8dab8700 1 --2- 172.21.15.99:0/2719856178 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fda740a2920 0x7fda740a4de0 unknown :-1 s=CLOSED pgs=179 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:11.252 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:11.252+0000 7fda8dab8700 1 --2- 172.21.15.99:0/2719856178 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fda88107250 0x7fda881a4910 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:11.252 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:11.252+0000 7fda8dab8700 1 --2- 172.21.15.99:0/2719856178 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fda88101210 0x7fda88199890 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:11.253 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:11.252+0000 7fda8dab8700 1 --2- 172.21.15.99:0/2719856178 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fda88101bb0 0x7fda88199dd0 unknown :-1 s=CLOSED pgs=56 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:11.253 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:11.252+0000 7fda8dab8700 1 -- 172.21.15.99:0/2719856178 >> 172.21.15.99:0/2719856178 conn(0x7fda88078140 msgr2=0x7fda880ffe30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:11.253 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:11.252+0000 7fda8dab8700 1 -- 172.21.15.99:0/2719856178 shutdown_connections 2024-03-19T01:39:11.253 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:11.252+0000 7fda8dab8700 1 -- 172.21.15.99:0/2719856178 wait complete. 2024-03-19T01:39:11.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:11 smithi099 ceph-mon[62951]: pgmap v291: 97 pgs: 1 peering, 96 active+clean; 631 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 10 KiB/s rd, 8.4 KiB/s wr, 10 op/s; 0 B/s, 0 objects/s recovering 2024-03-19T01:39:11.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:11 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/2719856178' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-19T01:39:11.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:11 smithi112 ceph-mon[66819]: pgmap v291: 97 pgs: 1 peering, 96 active+clean; 631 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 10 KiB/s rd, 8.4 KiB/s wr, 10 op/s; 0 B/s, 0 objects/s recovering 2024-03-19T01:39:11.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:11 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/2719856178' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-19T01:39:11.806 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:11 smithi115 ceph-mon[67332]: pgmap v291: 97 pgs: 1 peering, 96 active+clean; 631 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 10 KiB/s rd, 8.4 KiB/s wr, 10 op/s; 0 B/s, 0 objects/s recovering 2024-03-19T01:39:11.806 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:11 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/2719856178' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-19T01:39:11.911 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-03-19T01:39:11.912 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph osd dump --format=json 2024-03-19T01:39:12.155 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:39:13.721 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:13.719+0000 7fec33f6b700 1 -- 172.21.15.99:0/4246276821 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fec2c069fc0 msgr2=0x7fec2c06a420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:13.721 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:13.719+0000 7fec33f6b700 1 --2- 172.21.15.99:0/4246276821 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fec2c069fc0 0x7fec2c06a420 secure :-1 s=READY pgs=235 cs=0 l=1 rev1=1 crypto rx=0x7fec1c0099c0 tx=0x7fec1c009cd0 comp rx=0 tx=0).stop 2024-03-19T01:39:13.721 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:13.719+0000 7fec33f6b700 1 -- 172.21.15.99:0/4246276821 shutdown_connections 2024-03-19T01:39:13.722 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:13.719+0000 7fec33f6b700 1 --2- 172.21.15.99:0/4246276821 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fec2c06a960 0x7fec2c10f1a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:13.722 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:13.719+0000 7fec33f6b700 1 --2- 172.21.15.99:0/4246276821 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fec2c108360 0x7fec2c108740 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:13.722 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:13.719+0000 7fec33f6b700 1 --2- 172.21.15.99:0/4246276821 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fec2c069fc0 0x7fec2c06a420 unknown :-1 s=CLOSED pgs=235 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:13.722 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:13.719+0000 7fec33f6b700 1 -- 172.21.15.99:0/4246276821 >> 172.21.15.99:0/4246276821 conn(0x7fec2c078140 msgr2=0x7fec2c078550 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:13.722 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:13.719+0000 7fec33f6b700 1 -- 172.21.15.99:0/4246276821 shutdown_connections 2024-03-19T01:39:13.722 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:13.719+0000 7fec33f6b700 1 -- 172.21.15.99:0/4246276821 wait complete. 2024-03-19T01:39:13.722 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:13.720+0000 7fec33f6b700 1 Processor -- start 2024-03-19T01:39:13.722 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:13.720+0000 7fec33f6b700 1 -- start start 2024-03-19T01:39:13.722 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:13.720+0000 7fec33f6b700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fec2c069fc0 0x7fec2c1012e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:13.722 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:13.720+0000 7fec33f6b700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fec2c06a960 0x7fec2c101820 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:13.722 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:13.720+0000 7fec33f6b700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fec2c108360 0x7fec2c101d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:13.722 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:13.720+0000 7fec33f6b700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fec2c1125c0 con 0x7fec2c06a960 2024-03-19T01:39:13.722 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:13.720+0000 7fec33f6b700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fec2c112440 con 0x7fec2c108360 2024-03-19T01:39:13.723 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:13.720+0000 7fec33f6b700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fec2c112740 con 0x7fec2c069fc0 2024-03-19T01:39:13.723 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:13.720+0000 7fec31d07700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fec2c069fc0 0x7fec2c1012e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:13.723 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:13.720+0000 7fec31506700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fec2c06a960 0x7fec2c101820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:13.723 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:13.720+0000 7fec31d07700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fec2c069fc0 0x7fec2c1012e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:50742/0 (socket says 172.21.15.99:50742) 2024-03-19T01:39:13.723 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:13.720+0000 7fec31d07700 1 -- 172.21.15.99:0/2468205808 learned_addr learned my addr 172.21.15.99:0/2468205808 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:39:13.723 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:13.720+0000 7fec32508700 1 --2- 172.21.15.99:0/2468205808 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fec2c108360 0x7fec2c101d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:13.723 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:13.720+0000 7fec31d07700 1 -- 172.21.15.99:0/2468205808 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fec2c108360 msgr2=0x7fec2c101d60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:13.723 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:13.720+0000 7fec31d07700 1 --2- 172.21.15.99:0/2468205808 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fec2c108360 0x7fec2c101d60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:13.723 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:13.720+0000 7fec31d07700 1 -- 172.21.15.99:0/2468205808 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fec2c06a960 msgr2=0x7fec2c101820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:13.723 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:13.721+0000 7fec31d07700 1 --2- 172.21.15.99:0/2468205808 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fec2c06a960 0x7fec2c101820 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:13.723 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:13.721+0000 7fec31d07700 1 -- 172.21.15.99:0/2468205808 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fec2c1a6fa0 con 0x7fec2c069fc0 2024-03-19T01:39:13.723 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:13.721+0000 7fec31506700 1 --2- 172.21.15.99:0/2468205808 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fec2c06a960 0x7fec2c101820 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:39:13.723 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:13.721+0000 7fec31d07700 1 --2- 172.21.15.99:0/2468205808 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fec2c069fc0 0x7fec2c1012e0 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7fec2800c370 tx=0x7fec2800c680 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:13.723 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:13.721+0000 7fec22ffd700 1 -- 172.21.15.99:0/2468205808 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fec280092e0 con 0x7fec2c069fc0 2024-03-19T01:39:13.724 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:13.721+0000 7fec33f6b700 1 -- 172.21.15.99:0/2468205808 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fec2c1a7230 con 0x7fec2c069fc0 2024-03-19T01:39:13.724 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:13.721+0000 7fec22ffd700 1 -- 172.21.15.99:0/2468205808 <== mon.1 v2:172.21.15.115:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fec2800cbe0 con 0x7fec2c069fc0 2024-03-19T01:39:13.724 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:13.721+0000 7fec22ffd700 1 -- 172.21.15.99:0/2468205808 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fec280047e0 con 0x7fec2c069fc0 2024-03-19T01:39:13.724 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:13.721+0000 7fec33f6b700 1 -- 172.21.15.99:0/2468205808 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fec2c1a7790 con 0x7fec2c069fc0 2024-03-19T01:39:13.725 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:13.722+0000 7fec33f6b700 1 -- 172.21.15.99:0/2468205808 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fec2c068fd0 con 0x7fec2c069fc0 2024-03-19T01:39:13.725 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:13.723+0000 7fec22ffd700 1 -- 172.21.15.99:0/2468205808 <== mon.1 v2:172.21.15.115:3300/0 4 ==== mgrmap(e 17) ==== 137341+0+0 (secure 0 0 0) 0x7fec2801a020 con 0x7fec2c069fc0 2024-03-19T01:39:13.725 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:13.723+0000 7fec22ffd700 1 --2- 172.21.15.99:0/2468205808 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fec180a2920 0x7fec180a4de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:13.725 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:13.724+0000 7fec31506700 1 --2- 172.21.15.99:0/2468205808 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fec180a2920 0x7fec180a4de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:13.725 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:13.724+0000 7fec22ffd700 1 -- 172.21.15.99:0/2468205808 <== mon.1 v2:172.21.15.115:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8269+0+0 (secure 0 0 0) 0x7fec280ceb60 con 0x7fec2c069fc0 2024-03-19T01:39:13.725 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:13.724+0000 7fec31506700 1 --2- 172.21.15.99:0/2468205808 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fec180a2920 0x7fec180a4de0 secure :-1 s=READY pgs=180 cs=0 l=1 rev1=1 crypto rx=0x7fec2c102d70 tx=0x7fec1c005fb0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:13.732 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:13.732+0000 7fec22ffd700 1 -- 172.21.15.99:0/2468205808 <== mon.1 v2:172.21.15.115:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7fec280cee20 con 0x7fec2c069fc0 2024-03-19T01:39:13.928 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:13.928+0000 7fec33f6b700 1 -- 172.21.15.99:0/2468205808 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7fec2c02dbb0 con 0x7fec2c069fc0 2024-03-19T01:39:13.930 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:13.930+0000 7fec22ffd700 1 -- 172.21.15.99:0/2468205808 <== mon.1 v2:172.21.15.115:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v89) ==== 74+0+22757 (secure 0 0 0) 0x7fec28010040 con 0x7fec2c069fc0 2024-03-19T01:39:13.931 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:39:13.931 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":89,"fsid":"438fb69c-e590-11ee-95c9-87774f69a715","created":"2024-03-19T01:31:10.165073+0000","modified":"2024-03-19T01:39:07.825595+0000","last_up_change":"2024-03-19T01:38:09.082688+0000","last_in_change":"2024-03-19T01:37:49.115793+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-19T01:34:31.001309+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-19T01:38:54.018623+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"86","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-19T01:38:55.319036+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"87","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"0c407a0e-a40e-43d7-ad9a-bb8f3cd4a9ba","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6803","nonce":2935260112}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6805","nonce":2935260112}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6808","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6809","nonce":2935260112}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6807","nonce":2935260112}]},"public_addr":"172.21.15.99:6803/2935260112","cluster_addr":"172.21.15.99:6805/2935260112","heartbeat_back_addr":"172.21.15.99:6809/2935260112","heartbeat_front_addr":"172.21.15.99:6807/2935260112","state":["exists","up"]},{"osd":1,"uuid":"a48f9173-4a06-49f5-bca2-2c9047b8f666","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6800","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6801","nonce":1468854289}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6802","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6803","nonce":1468854289}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6806","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6807","nonce":1468854289}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6804","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6805","nonce":1468854289}]},"public_addr":"172.21.15.112:6801/1468854289","cluster_addr":"172.21.15.112:6803/1468854289","heartbeat_back_addr":"172.21.15.112:6807/1468854289","heartbeat_front_addr":"172.21.15.112:6805/1468854289","state":["exists","up"]},{"osd":2,"uuid":"b7715fdd-b6c9-4348-bd16-5eb61b427eef","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6800","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6801","nonce":3766690472}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6802","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6803","nonce":3766690472}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6806","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6807","nonce":3766690472}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6804","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6805","nonce":3766690472}]},"public_addr":"172.21.15.115:6801/3766690472","cluster_addr":"172.21.15.115:6803/3766690472","heartbeat_back_addr":"172.21.15.115:6807/3766690472","heartbeat_front_addr":"172.21.15.115:6805/3766690472","state":["exists","up"]},{"osd":3,"uuid":"6697ed1b-96bf-42de-8440-d17e9785b359","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6810","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6811","nonce":3756732895}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6812","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6813","nonce":3756732895}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6816","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6817","nonce":3756732895}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6814","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6815","nonce":3756732895}]},"public_addr":"172.21.15.99:6811/3756732895","cluster_addr":"172.21.15.99:6813/3756732895","heartbeat_back_addr":"172.21.15.99:6817/3756732895","heartbeat_front_addr":"172.21.15.99:6815/3756732895","state":["exists","up"]},{"osd":4,"uuid":"d958d6b9-67f6-43d0-b704-aa55dc9df83a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":34,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6808","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6809","nonce":3717986391}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6810","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6811","nonce":3717986391}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6814","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6815","nonce":3717986391}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6812","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6813","nonce":3717986391}]},"public_addr":"172.21.15.112:6809/3717986391","cluster_addr":"172.21.15.112:6811/3717986391","heartbeat_back_addr":"172.21.15.112:6815/3717986391","heartbeat_front_addr":"172.21.15.112:6813/3717986391","state":["exists","up"]},{"osd":5,"uuid":"a73eb49a-7925-49ba-bc4f-515e17f8b8ae","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6808","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6809","nonce":1971806408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6810","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6811","nonce":1971806408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6814","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6815","nonce":1971806408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6812","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6813","nonce":1971806408}]},"public_addr":"172.21.15.115:6809/1971806408","cluster_addr":"172.21.15.115:6811/1971806408","heartbeat_back_addr":"172.21.15.115:6815/1971806408","heartbeat_front_addr":"172.21.15.115:6813/1971806408","state":["exists","up"]},{"osd":6,"uuid":"caa718cd-e08b-48de-832a-29546d71e508","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6818","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6819","nonce":1130832772}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6820","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6821","nonce":1130832772}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6824","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6825","nonce":1130832772}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6822","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6823","nonce":1130832772}]},"public_addr":"172.21.15.99:6819/1130832772","cluster_addr":"172.21.15.99:6821/1130832772","heartbeat_back_addr":"172.21.15.99:6825/1130832772","heartbeat_front_addr":"172.21.15.99:6823/1130832772","state":["exists","up"]},{"osd":7,"uuid":"dbee19a1-2315-40d3-88e7-876fcde28903","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":53,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6816","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6817","nonce":3610356576}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6818","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6819","nonce":3610356576}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6822","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6823","nonce":3610356576}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6820","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6821","nonce":3610356576}]},"public_addr":"172.21.15.112:6817/3610356576","cluster_addr":"172.21.15.112:6819/3610356576","heartbeat_back_addr":"172.21.15.112:6823/3610356576","heartbeat_front_addr":"172.21.15.112:6821/3610356576","state":["exists","up"]},{"osd":8,"uuid":"14a866dc-5ce3-42fc-8bcf-9088b68726cb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":60,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6816","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6817","nonce":4255803111}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6818","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6819","nonce":4255803111}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6822","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6823","nonce":4255803111}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6820","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6821","nonce":4255803111}]},"public_addr":"172.21.15.115:6817/4255803111","cluster_addr":"172.21.15.115:6819/4255803111","heartbeat_back_addr":"172.21.15.115:6823/4255803111","heartbeat_front_addr":"172.21.15.115:6821/4255803111","state":["exists","up"]},{"osd":9,"uuid":"1db93652-6035-4906-91ae-d64d5a909502","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":68,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6826","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6827","nonce":1410921523}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6828","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6829","nonce":1410921523}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6832","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6833","nonce":1410921523}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6830","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6831","nonce":1410921523}]},"public_addr":"172.21.15.99:6827/1410921523","cluster_addr":"172.21.15.99:6829/1410921523","heartbeat_back_addr":"172.21.15.99:6833/1410921523","heartbeat_front_addr":"172.21.15.99:6831/1410921523","state":["exists","up"]},{"osd":10,"uuid":"6c50a8a7-3345-4f4a-af0e-5606c7aca453","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":74,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6824","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6825","nonce":891805922}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6826","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6827","nonce":891805922}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6830","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6831","nonce":891805922}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6828","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6829","nonce":891805922}]},"public_addr":"172.21.15.112:6825/891805922","cluster_addr":"172.21.15.112:6827/891805922","heartbeat_back_addr":"172.21.15.112:6831/891805922","heartbeat_front_addr":"172.21.15.112:6829/891805922","state":["exists","up"]},{"osd":11,"uuid":"fd63b7fd-0db5-46d5-82fa-9707bbd0b495","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":82,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6824","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6825","nonce":938436244}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6826","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6827","nonce":938436244}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6830","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6831","nonce":938436244}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6828","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6829","nonce":938436244}]},"public_addr":"172.21.15.115:6825/938436244","cluster_addr":"172.21.15.115:6827/938436244","heartbeat_back_addr":"172.21.15.115:6831/938436244","heartbeat_front_addr":"172.21.15.115:6829/938436244","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:33:39.036822+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:34:03.339566+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:34:27.149273+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:34:51.315561+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:35:15.396031+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:35:38.957168+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:36:03.213914+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:36:27.763699+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:36:51.688670+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:37:17.280165+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:37:42.554633+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:38:07.438039+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.0","mappings":[{"from":8,"to":1}]},{"pgid":"2.8","mappings":[{"from":2,"to":11}]},{"pgid":"2.b","mappings":[{"from":8,"to":11}]},{"pgid":"2.e","mappings":[{"from":2,"to":4}]},{"pgid":"3.0","mappings":[{"from":5,"to":9}]},{"pgid":"3.2","mappings":[{"from":3,"to":9}]},{"pgid":"3.17","mappings":[{"from":4,"to":8}]},{"pgid":"3.31","mappings":[{"from":4,"to":8}]},{"pgid":"3.37","mappings":[{"from":5,"to":7}]},{"pgid":"3.3a","mappings":[{"from":4,"to":6}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.99:6801/1328245934":"2024-03-20T01:32:06.661002+0000","172.21.15.99:0/1618965837":"2024-03-20T01:32:06.661002+0000","172.21.15.99:0/2491429851":"2024-03-20T01:32:06.661002+0000","172.21.15.99:6800/1328245934":"2024-03-20T01:32:06.661002+0000","172.21.15.99:0/2112231766":"2024-03-20T01:31:40.161751+0000","172.21.15.99:0/3341836545":"2024-03-20T01:31:40.161751+0000","172.21.15.99:6800/4223048293":"2024-03-20T01:31:40.161751+0000","172.21.15.99:6801/4223048293":"2024-03-20T01:31:40.161751+0000","172.21.15.99:0/2377621863":"2024-03-20T01:31:40.161751+0000","172.21.15.99:0/1904400935":"2024-03-20T01:32:06.661002+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-19T01:39:13.933 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:13.932+0000 7fec33f6b700 1 -- 172.21.15.99:0/2468205808 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fec180a2920 msgr2=0x7fec180a4de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:13.933 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:13.932+0000 7fec33f6b700 1 --2- 172.21.15.99:0/2468205808 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fec180a2920 0x7fec180a4de0 secure :-1 s=READY pgs=180 cs=0 l=1 rev1=1 crypto rx=0x7fec2c102d70 tx=0x7fec1c005fb0 comp rx=0 tx=0).stop 2024-03-19T01:39:13.933 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:13.932+0000 7fec33f6b700 1 -- 172.21.15.99:0/2468205808 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fec2c069fc0 msgr2=0x7fec2c1012e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:13.933 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:13.932+0000 7fec33f6b700 1 --2- 172.21.15.99:0/2468205808 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fec2c069fc0 0x7fec2c1012e0 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7fec2800c370 tx=0x7fec2800c680 comp rx=0 tx=0).stop 2024-03-19T01:39:13.933 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:13.932+0000 7fec33f6b700 1 -- 172.21.15.99:0/2468205808 shutdown_connections 2024-03-19T01:39:13.933 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:13.932+0000 7fec33f6b700 1 --2- 172.21.15.99:0/2468205808 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fec180a2920 0x7fec180a4de0 unknown :-1 s=CLOSED pgs=180 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:13.933 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:13.932+0000 7fec33f6b700 1 --2- 172.21.15.99:0/2468205808 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fec2c069fc0 0x7fec2c1012e0 unknown :-1 s=CLOSED pgs=59 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:13.933 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:13.932+0000 7fec33f6b700 1 --2- 172.21.15.99:0/2468205808 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fec2c108360 0x7fec2c101d60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:13.934 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:13.932+0000 7fec33f6b700 1 --2- 172.21.15.99:0/2468205808 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fec2c06a960 0x7fec2c101820 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:13.934 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:13.932+0000 7fec33f6b700 1 -- 172.21.15.99:0/2468205808 >> 172.21.15.99:0/2468205808 conn(0x7fec2c078140 msgr2=0x7fec2c0ffe50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:13.934 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:13.933+0000 7fec33f6b700 1 -- 172.21.15.99:0/2468205808 shutdown_connections 2024-03-19T01:39:13.934 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:13.933+0000 7fec33f6b700 1 -- 172.21.15.99:0/2468205808 wait complete. 2024-03-19T01:39:14.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:13 smithi115 ceph-mon[67332]: pgmap v292: 97 pgs: 97 active+clean; 631 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 10 KiB/s rd, 7.4 KiB/s wr, 10 op/s; 0 B/s, 0 objects/s recovering 2024-03-19T01:39:14.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:13 smithi099 ceph-mon[62951]: pgmap v292: 97 pgs: 97 active+clean; 631 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 10 KiB/s rd, 7.4 KiB/s wr, 10 op/s; 0 B/s, 0 objects/s recovering 2024-03-19T01:39:14.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:13 smithi112 ceph-mon[66819]: pgmap v292: 97 pgs: 97 active+clean; 631 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 10 KiB/s rd, 7.4 KiB/s wr, 10 op/s; 0 B/s, 0 objects/s recovering 2024-03-19T01:39:14.544 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph tell osd.0 flush_pg_stats 2024-03-19T01:39:14.544 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph tell osd.1 flush_pg_stats 2024-03-19T01:39:14.544 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph tell osd.2 flush_pg_stats 2024-03-19T01:39:14.545 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph tell osd.3 flush_pg_stats 2024-03-19T01:39:14.545 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph tell osd.4 flush_pg_stats 2024-03-19T01:39:14.546 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph tell osd.5 flush_pg_stats 2024-03-19T01:39:14.546 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph tell osd.6 flush_pg_stats 2024-03-19T01:39:14.547 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph tell osd.7 flush_pg_stats 2024-03-19T01:39:14.547 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph tell osd.8 flush_pg_stats 2024-03-19T01:39:14.547 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph tell osd.9 flush_pg_stats 2024-03-19T01:39:14.548 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph tell osd.10 flush_pg_stats 2024-03-19T01:39:14.548 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph tell osd.11 flush_pg_stats 2024-03-19T01:39:15.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:14 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/2468205808' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-19T01:39:15.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:14 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/2468205808' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-19T01:39:15.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:14 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/2468205808' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-19T01:39:15.220 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:39:15.515 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:39:15.515 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:39:15.517 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:39:15.518 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:39:15.520 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:39:15.520 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:39:15.521 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:39:15.526 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:39:15.529 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:39:15.533 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:39:15.539 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:39:16.005 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:15 smithi099 ceph-mon[62951]: pgmap v293: 97 pgs: 97 active+clean; 631 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 4.0 KiB/s rd, 2.1 KiB/s wr, 6 op/s; 0 B/s, 0 objects/s recovering 2024-03-19T01:39:16.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:15 smithi115 ceph-mon[67332]: pgmap v293: 97 pgs: 97 active+clean; 631 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 4.0 KiB/s rd, 2.1 KiB/s wr, 6 op/s; 0 B/s, 0 objects/s recovering 2024-03-19T01:39:16.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:15 smithi112 ceph-mon[66819]: pgmap v293: 97 pgs: 97 active+clean; 631 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 4.0 KiB/s rd, 2.1 KiB/s wr, 6 op/s; 0 B/s, 0 objects/s recovering 2024-03-19T01:39:18.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:17 smithi099 ceph-mon[62951]: pgmap v294: 97 pgs: 97 active+clean; 636 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 4.7 KiB/s rd, 2.2 KiB/s wr, 6 op/s; 0 B/s, 0 objects/s recovering 2024-03-19T01:39:18.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:17 smithi112 ceph-mon[66819]: pgmap v294: 97 pgs: 97 active+clean; 636 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 4.7 KiB/s rd, 2.2 KiB/s wr, 6 op/s; 0 B/s, 0 objects/s recovering 2024-03-19T01:39:18.134 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:17 smithi115 ceph-mon[67332]: pgmap v294: 97 pgs: 97 active+clean; 636 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 4.7 KiB/s rd, 2.2 KiB/s wr, 6 op/s; 0 B/s, 0 objects/s recovering 2024-03-19T01:39:20.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:19 smithi099 ceph-mon[62951]: pgmap v295: 97 pgs: 97 active+clean; 636 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 4.3 KiB/s rd, 2.0 KiB/s wr, 6 op/s; 0 B/s, 0 objects/s recovering 2024-03-19T01:39:20.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:19 smithi112 ceph-mon[66819]: pgmap v295: 97 pgs: 97 active+clean; 636 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 4.3 KiB/s rd, 2.0 KiB/s wr, 6 op/s; 0 B/s, 0 objects/s recovering 2024-03-19T01:39:20.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:19 smithi115 ceph-mon[67332]: pgmap v295: 97 pgs: 97 active+clean; 636 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 4.3 KiB/s rd, 2.0 KiB/s wr, 6 op/s; 0 B/s, 0 objects/s recovering 2024-03-19T01:39:21.966 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:21 smithi099 ceph-mon[62951]: pgmap v296: 97 pgs: 97 active+clean; 638 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 5.0 KiB/s rd, 2.2 KiB/s wr, 7 op/s; 0 B/s, 0 objects/s recovering 2024-03-19T01:39:22.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:21 smithi112 ceph-mon[66819]: pgmap v296: 97 pgs: 97 active+clean; 638 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 5.0 KiB/s rd, 2.2 KiB/s wr, 7 op/s; 0 B/s, 0 objects/s recovering 2024-03-19T01:39:22.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:21 smithi115 ceph-mon[67332]: pgmap v296: 97 pgs: 97 active+clean; 638 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 5.0 KiB/s rd, 2.2 KiB/s wr, 7 op/s; 0 B/s, 0 objects/s recovering 2024-03-19T01:39:22.637 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.637+0000 7f740bf01700 1 -- 172.21.15.99:0/3246076236 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7404100900 msgr2=0x7f7404100d80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:22.638 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.637+0000 7f740bf01700 1 --2- 172.21.15.99:0/3246076236 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7404100900 0x7f7404100d80 secure :-1 s=READY pgs=236 cs=0 l=1 rev1=1 crypto rx=0x7f73f4009990 tx=0x7f73f4009ca0 comp rx=0 tx=0).stop 2024-03-19T01:39:22.638 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.637+0000 7f740bf01700 1 -- 172.21.15.99:0/3246076236 shutdown_connections 2024-03-19T01:39:22.638 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.637+0000 7f740bf01700 1 --2- 172.21.15.99:0/3246076236 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f74041012c0 0x7f740410f220 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:22.638 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.637+0000 7f740bf01700 1 --2- 172.21.15.99:0/3246076236 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f7404107380 0x7f7404107760 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:22.638 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.637+0000 7f740bf01700 1 --2- 172.21.15.99:0/3246076236 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7404100900 0x7f7404100d80 unknown :-1 s=CLOSED pgs=236 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:22.638 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.637+0000 7f740bf01700 1 -- 172.21.15.99:0/3246076236 >> 172.21.15.99:0/3246076236 conn(0x7f7404077810 msgr2=0x7f74040fec30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:22.638 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.637+0000 7f740bf01700 1 -- 172.21.15.99:0/3246076236 shutdown_connections 2024-03-19T01:39:22.638 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.637+0000 7f740bf01700 1 -- 172.21.15.99:0/3246076236 wait complete. 2024-03-19T01:39:22.640 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.638+0000 7f740bf01700 1 Processor -- start 2024-03-19T01:39:22.640 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.638+0000 7f740bf01700 1 -- start start 2024-03-19T01:39:22.640 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.638+0000 7f740bf01700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f7404100900 0x7f7404101ae0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:22.640 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.638+0000 7f740bf01700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f74041012c0 0x7f7404102020 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:22.640 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.638+0000 7f740bf01700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7404107380 0x7f7404102560 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:22.640 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.638+0000 7f740bf01700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f7404112740 con 0x7f7404107380 2024-03-19T01:39:22.640 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.638+0000 7f740bf01700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f74041125c0 con 0x7f74041012c0 2024-03-19T01:39:22.640 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.638+0000 7f740bf01700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f74041128c0 con 0x7f7404100900 2024-03-19T01:39:22.640 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.638+0000 7f740a49e700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7404107380 0x7f7404102560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:22.641 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.638+0000 7f740a49e700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7404107380 0x7f7404102560 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:35444/0 (socket says 172.21.15.99:35444) 2024-03-19T01:39:22.641 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.638+0000 7f740a49e700 1 -- 172.21.15.99:0/3995956347 learned_addr learned my addr 172.21.15.99:0/3995956347 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:39:22.641 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.638+0000 7f7409c9d700 1 --2- 172.21.15.99:0/3995956347 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f7404100900 0x7f7404101ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:22.641 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.638+0000 7f740949c700 1 --2- 172.21.15.99:0/3995956347 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f74041012c0 0x7f7404102020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:22.641 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.638+0000 7f7409c9d700 1 -- 172.21.15.99:0/3995956347 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f74041012c0 msgr2=0x7f7404102020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:22.641 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.638+0000 7f7409c9d700 1 --2- 172.21.15.99:0/3995956347 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f74041012c0 0x7f7404102020 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:22.641 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.638+0000 7f7409c9d700 1 -- 172.21.15.99:0/3995956347 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7404107380 msgr2=0x7f7404102560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:22.641 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.638+0000 7f7409c9d700 1 --2- 172.21.15.99:0/3995956347 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7404107380 0x7f7404102560 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:22.641 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.638+0000 7f7409c9d700 1 -- 172.21.15.99:0/3995956347 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f74041a7100 con 0x7f7404100900 2024-03-19T01:39:22.641 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.638+0000 7f740949c700 1 --2- 172.21.15.99:0/3995956347 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f74041012c0 0x7f7404102020 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:39:22.641 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.638+0000 7f740a49e700 1 --2- 172.21.15.99:0/3995956347 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7404107380 0x7f7404102560 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:39:22.641 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.639+0000 7f7409c9d700 1 --2- 172.21.15.99:0/3995956347 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f7404100900 0x7f7404101ae0 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7f73fc00e9f0 tx=0x7f73fc00ed00 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:22.642 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.639+0000 7f73faffd700 1 -- 172.21.15.99:0/3995956347 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f73fc00c9c0 con 0x7f7404100900 2024-03-19T01:39:22.642 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.639+0000 7f740bf01700 1 -- 172.21.15.99:0/3995956347 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f74041a73f0 con 0x7f7404100900 2024-03-19T01:39:22.642 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.639+0000 7f73faffd700 1 -- 172.21.15.99:0/3995956347 <== mon.1 v2:172.21.15.115:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f73fc00cb60 con 0x7f7404100900 2024-03-19T01:39:22.642 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.639+0000 7f73faffd700 1 -- 172.21.15.99:0/3995956347 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f73fc018680 con 0x7f7404100900 2024-03-19T01:39:22.642 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.639+0000 7f740bf01700 1 -- 172.21.15.99:0/3995956347 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f74041a7900 con 0x7f7404100900 2024-03-19T01:39:22.642 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.639+0000 7f740bf01700 1 -- 172.21.15.99:0/3995956347 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f7404062a20 con 0x7f7404100900 2024-03-19T01:39:22.643 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.641+0000 7f73faffd700 1 -- 172.21.15.99:0/3995956347 <== mon.1 v2:172.21.15.115:3300/0 4 ==== mgrmap(e 17) ==== 137341+0+0 (secure 0 0 0) 0x7f73fc010430 con 0x7f7404100900 2024-03-19T01:39:22.643 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.641+0000 7f73faffd700 1 --2- 172.21.15.99:0/3995956347 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f73f00a2920 0x7f73f00a4de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:22.643 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.641+0000 7f740949c700 1 --2- 172.21.15.99:0/3995956347 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f73f00a2920 0x7f73f00a4de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:22.643 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.642+0000 7f73faffd700 1 -- 172.21.15.99:0/3995956347 <== mon.1 v2:172.21.15.115:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8269+0+0 (secure 0 0 0) 0x7f73fc0cf1b0 con 0x7f7404100900 2024-03-19T01:39:22.643 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.642+0000 7f73faffd700 1 --2- 172.21.15.99:0/3995956347 >> [v2:172.21.15.112:6800/1468854289,v1:172.21.15.112:6801/1468854289] conn(0x7f73f00ac560 0x7f73f00aea00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:22.644 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.642+0000 7f73faffd700 1 -- 172.21.15.99:0/3995956347 --> [v2:172.21.15.112:6800/1468854289,v1:172.21.15.112:6801/1468854289] -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f73fc01bec0 con 0x7f73f00ac560 2024-03-19T01:39:22.644 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.642+0000 7f73faffd700 1 -- 172.21.15.99:0/3995956347 <== mon.1 v2:172.21.15.115:3300/0 6 ==== mon_get_version_reply(handle=1 version=89) ==== 24+0+0 (secure 0 0 0) 0x7f73fc0cf560 con 0x7f7404100900 2024-03-19T01:39:22.644 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.642+0000 7f740949c700 1 --2- 172.21.15.99:0/3995956347 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f73f00a2920 0x7f73f00a4de0 secure :-1 s=READY pgs=181 cs=0 l=1 rev1=1 crypto rx=0x7f73f400b2e0 tx=0x7f73f4005960 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:22.644 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.642+0000 7f740a49e700 1 --2- 172.21.15.99:0/3995956347 >> [v2:172.21.15.112:6800/1468854289,v1:172.21.15.112:6801/1468854289] conn(0x7f73f00ac560 0x7f73f00aea00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:22.644 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.642+0000 7f740a49e700 1 --2- 172.21.15.99:0/3995956347 >> [v2:172.21.15.112:6800/1468854289,v1:172.21.15.112:6801/1468854289] conn(0x7f73f00ac560 0x7f73f00aea00 crc :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:22.644 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.643+0000 7f73faffd700 1 -- 172.21.15.99:0/3995956347 <== osd.1 v2:172.21.15.112:6800/1468854289 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (crc 0 0 0) 0x7f73fc01bec0 con 0x7f73f00ac560 2024-03-19T01:39:22.659 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.659+0000 7f740bf01700 1 -- 172.21.15.99:0/3995956347 --> [v2:172.21.15.112:6800/1468854289,v1:172.21.15.112:6801/1468854289] -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7f7404062e20 con 0x7f73f00ac560 2024-03-19T01:39:22.659 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.659+0000 7f73faffd700 1 -- 172.21.15.99:0/3995956347 <== osd.1 v2:172.21.15.112:6800/1468854289 2 ==== command_reply(tid 2: 0 ) ==== 8+0+11 (crc 0 0 0) 0x7f7404062e20 con 0x7f73f00ac560 2024-03-19T01:39:22.660 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.659+0000 7f740bf01700 1 -- 172.21.15.99:0/3995956347 >> [v2:172.21.15.112:6800/1468854289,v1:172.21.15.112:6801/1468854289] conn(0x7f73f00ac560 msgr2=0x7f73f00aea00 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:22.660 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.659+0000 7f740bf01700 1 --2- 172.21.15.99:0/3995956347 >> [v2:172.21.15.112:6800/1468854289,v1:172.21.15.112:6801/1468854289] conn(0x7f73f00ac560 0x7f73f00aea00 crc :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:22.660 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.659+0000 7f740bf01700 1 -- 172.21.15.99:0/3995956347 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f73f00a2920 msgr2=0x7f73f00a4de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:22.661 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.659+0000 7f740bf01700 1 --2- 172.21.15.99:0/3995956347 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f73f00a2920 0x7f73f00a4de0 secure :-1 s=READY pgs=181 cs=0 l=1 rev1=1 crypto rx=0x7f73f400b2e0 tx=0x7f73f4005960 comp rx=0 tx=0).stop 2024-03-19T01:39:22.661 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.659+0000 7f740bf01700 1 -- 172.21.15.99:0/3995956347 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f7404100900 msgr2=0x7f7404101ae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:22.661 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.659+0000 7f740bf01700 1 --2- 172.21.15.99:0/3995956347 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f7404100900 0x7f7404101ae0 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7f73fc00e9f0 tx=0x7f73fc00ed00 comp rx=0 tx=0).stop 2024-03-19T01:39:22.661 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.659+0000 7f740a49e700 1 -- 172.21.15.99:0/3995956347 reap_dead start 2024-03-19T01:39:22.661 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.659+0000 7f740bf01700 1 -- 172.21.15.99:0/3995956347 shutdown_connections 2024-03-19T01:39:22.661 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.659+0000 7f740bf01700 1 -- 172.21.15.99:0/3995956347 >> 172.21.15.99:0/3995956347 conn(0x7f7404077810 msgr2=0x7f7404105360 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:22.661 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.660+0000 7f740bf01700 1 -- 172.21.15.99:0/3995956347 shutdown_connections 2024-03-19T01:39:22.661 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:22.660+0000 7f740bf01700 1 -- 172.21.15.99:0/3995956347 wait complete. 2024-03-19T01:39:24.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:23 smithi099 ceph-mon[62951]: pgmap v297: 97 pgs: 97 active+clean; 638 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 3.6 KiB/s rd, 938 B/s wr, 7 op/s; 0 B/s, 0 objects/s recovering 2024-03-19T01:39:24.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:23 smithi112 ceph-mon[66819]: pgmap v297: 97 pgs: 97 active+clean; 638 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 3.6 KiB/s rd, 938 B/s wr, 7 op/s; 0 B/s, 0 objects/s recovering 2024-03-19T01:39:24.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:23 smithi115 ceph-mon[67332]: pgmap v297: 97 pgs: 97 active+clean; 638 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 3.6 KiB/s rd, 938 B/s wr, 7 op/s; 0 B/s, 0 objects/s recovering 2024-03-19T01:39:26.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:25 smithi099 ceph-mon[62951]: pgmap v298: 97 pgs: 97 active+clean; 638 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 3.2 KiB/s rd, 853 B/s wr, 5 op/s 2024-03-19T01:39:26.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:25 smithi112 ceph-mon[66819]: pgmap v298: 97 pgs: 97 active+clean; 638 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 3.2 KiB/s rd, 853 B/s wr, 5 op/s 2024-03-19T01:39:26.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:25 smithi115 ceph-mon[67332]: pgmap v298: 97 pgs: 97 active+clean; 638 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 3.2 KiB/s rd, 853 B/s wr, 5 op/s 2024-03-19T01:39:27.607 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.597+0000 7fd3440ad700 1 -- 172.21.15.99:0/1028285087 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd33c101e50 msgr2=0x7fd33c105e10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:27.608 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.597+0000 7fd3440ad700 1 --2- 172.21.15.99:0/1028285087 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd33c101e50 0x7fd33c105e10 secure :-1 s=READY pgs=239 cs=0 l=1 rev1=1 crypto rx=0x7fd32c0099c0 tx=0x7fd32c009cd0 comp rx=0 tx=0).stop 2024-03-19T01:39:27.608 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.597+0000 7fd3440ad700 1 -- 172.21.15.99:0/1028285087 shutdown_connections 2024-03-19T01:39:27.608 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.597+0000 7fd3440ad700 1 --2- 172.21.15.99:0/1028285087 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd33c1014a0 0x7fd33c101880 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:27.608 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.597+0000 7fd3440ad700 1 --2- 172.21.15.99:0/1028285087 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd33c1064f0 0x7fd33c10e0c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:27.608 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.597+0000 7fd3440ad700 1 --2- 172.21.15.99:0/1028285087 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd33c101e50 0x7fd33c105e10 unknown :-1 s=CLOSED pgs=239 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:27.608 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.597+0000 7fd3440ad700 1 -- 172.21.15.99:0/1028285087 >> 172.21.15.99:0/1028285087 conn(0x7fd33c077260 msgr2=0x7fd33c077670 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:27.608 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.597+0000 7fd3440ad700 1 -- 172.21.15.99:0/1028285087 shutdown_connections 2024-03-19T01:39:27.608 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.597+0000 7fd3440ad700 1 -- 172.21.15.99:0/1028285087 wait complete. 2024-03-19T01:39:27.608 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.598+0000 7fd3440ad700 1 Processor -- start 2024-03-19T01:39:27.608 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.598+0000 7fd3440ad700 1 -- start start 2024-03-19T01:39:27.608 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.598+0000 7fd3440ad700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd33c1014a0 0x7fd33c1998d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:27.609 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.598+0000 7fd3440ad700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd33c101e50 0x7fd33c199e10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:27.609 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.598+0000 7fd3440ad700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd33c1064f0 0x7fd33c1a4950 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:27.609 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.598+0000 7fd3440ad700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fd33c1114d0 con 0x7fd33c1014a0 2024-03-19T01:39:27.609 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.598+0000 7fd3440ad700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fd33c111350 con 0x7fd33c101e50 2024-03-19T01:39:27.609 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.598+0000 7fd3440ad700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fd33c111650 con 0x7fd33c1064f0 2024-03-19T01:39:27.609 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.598+0000 7fd341e49700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd33c1014a0 0x7fd33c1998d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:27.609 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.598+0000 7fd341e49700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd33c1014a0 0x7fd33c1998d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:35484/0 (socket says 172.21.15.99:35484) 2024-03-19T01:39:27.609 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.598+0000 7fd341e49700 1 -- 172.21.15.99:0/1916646908 learned_addr learned my addr 172.21.15.99:0/1916646908 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:39:27.609 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.598+0000 7fd341648700 1 --2- 172.21.15.99:0/1916646908 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd33c101e50 0x7fd33c199e10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:27.609 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.598+0000 7fd34264a700 1 --2- 172.21.15.99:0/1916646908 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd33c1064f0 0x7fd33c1a4950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:27.609 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.598+0000 7fd341e49700 1 -- 172.21.15.99:0/1916646908 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd33c1064f0 msgr2=0x7fd33c1a4950 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:27.609 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.598+0000 7fd341e49700 1 --2- 172.21.15.99:0/1916646908 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd33c1064f0 0x7fd33c1a4950 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:27.609 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.598+0000 7fd341e49700 1 -- 172.21.15.99:0/1916646908 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd33c101e50 msgr2=0x7fd33c199e10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:27.610 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.598+0000 7fd341e49700 1 --2- 172.21.15.99:0/1916646908 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd33c101e50 0x7fd33c199e10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:27.610 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.598+0000 7fd341e49700 1 -- 172.21.15.99:0/1916646908 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd33c1a4e90 con 0x7fd33c1014a0 2024-03-19T01:39:27.610 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.598+0000 7fd341e49700 1 --2- 172.21.15.99:0/1916646908 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd33c1014a0 0x7fd33c1998d0 secure :-1 s=READY pgs=240 cs=0 l=1 rev1=1 crypto rx=0x7fd33800ef30 tx=0x7fd33800c3a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:27.610 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.599+0000 7fd332ffd700 1 -- 172.21.15.99:0/1916646908 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd3380092e0 con 0x7fd33c1014a0 2024-03-19T01:39:27.610 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.599+0000 7fd332ffd700 1 -- 172.21.15.99:0/1916646908 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd338004bb0 con 0x7fd33c1014a0 2024-03-19T01:39:27.610 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.599+0000 7fd332ffd700 1 -- 172.21.15.99:0/1916646908 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd338004280 con 0x7fd33c1014a0 2024-03-19T01:39:27.610 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.599+0000 7fd3440ad700 1 -- 172.21.15.99:0/1916646908 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd33c1a5060 con 0x7fd33c1014a0 2024-03-19T01:39:27.610 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.599+0000 7fd3440ad700 1 -- 172.21.15.99:0/1916646908 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fd33c1a5570 con 0x7fd33c1014a0 2024-03-19T01:39:27.610 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.600+0000 7fd3440ad700 1 -- 172.21.15.99:0/1916646908 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=osdmap handle=1) -- 0x7fd33c062a20 con 0x7fd33c1014a0 2024-03-19T01:39:27.610 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.601+0000 7fd332ffd700 1 -- 172.21.15.99:0/1916646908 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 17) ==== 137341+0+0 (secure 0 0 0) 0x7fd33801b030 con 0x7fd33c1014a0 2024-03-19T01:39:27.610 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.601+0000 7fd332ffd700 1 --2- 172.21.15.99:0/1916646908 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd3280a29a0 0x7fd3280a4e60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:27.610 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.601+0000 7fd341648700 1 --2- 172.21.15.99:0/1916646908 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd3280a29a0 0x7fd3280a4e60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:27.610 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.601+0000 7fd332ffd700 1 -- 172.21.15.99:0/1916646908 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8269+0+0 (secure 0 0 0) 0x7fd3380cf020 con 0x7fd33c1014a0 2024-03-19T01:39:27.610 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.601+0000 7fd332ffd700 1 --2- 172.21.15.99:0/1916646908 >> [v2:172.21.15.112:6824/891805922,v1:172.21.15.112:6825/891805922] conn(0x7fd3280ac5e0 0x7fd3280aea80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:27.611 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.601+0000 7fd332ffd700 1 -- 172.21.15.99:0/1916646908 --> [v2:172.21.15.112:6824/891805922,v1:172.21.15.112:6825/891805922] -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7fd33801fe50 con 0x7fd3280ac5e0 2024-03-19T01:39:27.611 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.601+0000 7fd332ffd700 1 -- 172.21.15.99:0/1916646908 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_get_version_reply(handle=1 version=89) ==== 24+0+0 (secure 0 0 0) 0x7fd3380cf470 con 0x7fd33c1014a0 2024-03-19T01:39:27.611 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.602+0000 7fd341648700 1 --2- 172.21.15.99:0/1916646908 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd3280a29a0 0x7fd3280a4e60 secure :-1 s=READY pgs=183 cs=0 l=1 rev1=1 crypto rx=0x7fd32c000c00 tx=0x7fd32c005fb0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:27.611 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.602+0000 7fd34264a700 1 --2- 172.21.15.99:0/1916646908 >> [v2:172.21.15.112:6824/891805922,v1:172.21.15.112:6825/891805922] conn(0x7fd3280ac5e0 0x7fd3280aea80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:27.611 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.602+0000 7fd34264a700 1 --2- 172.21.15.99:0/1916646908 >> [v2:172.21.15.112:6824/891805922,v1:172.21.15.112:6825/891805922] conn(0x7fd3280ac5e0 0x7fd3280aea80 crc :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.10 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:27.611 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.603+0000 7fd332ffd700 1 -- 172.21.15.99:0/1916646908 <== osd.10 v2:172.21.15.112:6824/891805922 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (crc 0 0 0) 0x7fd33801fe50 con 0x7fd3280ac5e0 2024-03-19T01:39:27.619 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.619+0000 7fd3440ad700 1 -- 172.21.15.99:0/1916646908 --> [v2:172.21.15.112:6824/891805922,v1:172.21.15.112:6825/891805922] -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7fd33c062ea0 con 0x7fd3280ac5e0 2024-03-19T01:39:27.620 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.619+0000 7fd332ffd700 1 -- 172.21.15.99:0/1916646908 <== osd.10 v2:172.21.15.112:6824/891805922 2 ==== command_reply(tid 2: 0 ) ==== 8+0+12 (crc 0 0 0) 0x7fd33c062ea0 con 0x7fd3280ac5e0 2024-03-19T01:39:27.620 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.620+0000 7fd3440ad700 1 -- 172.21.15.99:0/1916646908 >> [v2:172.21.15.112:6824/891805922,v1:172.21.15.112:6825/891805922] conn(0x7fd3280ac5e0 msgr2=0x7fd3280aea80 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:27.620 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.620+0000 7fd3440ad700 1 --2- 172.21.15.99:0/1916646908 >> [v2:172.21.15.112:6824/891805922,v1:172.21.15.112:6825/891805922] conn(0x7fd3280ac5e0 0x7fd3280aea80 crc :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:27.621 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.620+0000 7fd3440ad700 1 -- 172.21.15.99:0/1916646908 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd3280a29a0 msgr2=0x7fd3280a4e60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:27.621 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.620+0000 7fd3440ad700 1 --2- 172.21.15.99:0/1916646908 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd3280a29a0 0x7fd3280a4e60 secure :-1 s=READY pgs=183 cs=0 l=1 rev1=1 crypto rx=0x7fd32c000c00 tx=0x7fd32c005fb0 comp rx=0 tx=0).stop 2024-03-19T01:39:27.621 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.620+0000 7fd3440ad700 1 -- 172.21.15.99:0/1916646908 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd33c1014a0 msgr2=0x7fd33c1998d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:27.621 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.620+0000 7fd3440ad700 1 --2- 172.21.15.99:0/1916646908 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd33c1014a0 0x7fd33c1998d0 secure :-1 s=READY pgs=240 cs=0 l=1 rev1=1 crypto rx=0x7fd33800ef30 tx=0x7fd33800c3a0 comp rx=0 tx=0).stop 2024-03-19T01:39:27.621 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.620+0000 7fd34264a700 1 -- 172.21.15.99:0/1916646908 reap_dead start 2024-03-19T01:39:27.621 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.620+0000 7fd3440ad700 1 -- 172.21.15.99:0/1916646908 shutdown_connections 2024-03-19T01:39:27.621 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.620+0000 7fd3440ad700 1 -- 172.21.15.99:0/1916646908 >> 172.21.15.99:0/1916646908 conn(0x7fd33c077260 msgr2=0x7fd33c0ff290 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:27.621 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.620+0000 7fd3440ad700 1 -- 172.21.15.99:0/1916646908 shutdown_connections 2024-03-19T01:39:27.621 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.620+0000 7fd3440ad700 1 -- 172.21.15.99:0/1916646908 wait complete. 2024-03-19T01:39:27.984 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.980+0000 7ff46cf90700 1 -- 172.21.15.99:0/161407054 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff468100eb0 msgr2=0x7ff468101290 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:27.985 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.981+0000 7ff46cf90700 1 --2- 172.21.15.99:0/161407054 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff468100eb0 0x7ff468101290 secure :-1 s=READY pgs=241 cs=0 l=1 rev1=1 crypto rx=0x7ff45c0099c0 tx=0x7ff45c009cd0 comp rx=0 tx=0).stop 2024-03-19T01:39:27.985 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.981+0000 7ff46cf90700 1 -- 172.21.15.99:0/161407054 shutdown_connections 2024-03-19T01:39:27.985 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.981+0000 7ff46cf90700 1 --2- 172.21.15.99:0/161407054 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff4681076d0 0x7ff46810f250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:27.985 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.981+0000 7ff46cf90700 1 --2- 172.21.15.99:0/161407054 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff468100eb0 0x7ff468101290 unknown :-1 s=CLOSED pgs=241 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:27.985 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.981+0000 7ff46cf90700 1 --2- 172.21.15.99:0/161407054 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff468101860 0x7ff468106fa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:27.985 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.981+0000 7ff46cf90700 1 -- 172.21.15.99:0/161407054 >> 172.21.15.99:0/161407054 conn(0x7ff468076570 msgr2=0x7ff468076980 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:27.986 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.981+0000 7ff46cf90700 1 -- 172.21.15.99:0/161407054 shutdown_connections 2024-03-19T01:39:27.986 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.981+0000 7ff46cf90700 1 -- 172.21.15.99:0/161407054 wait complete. 2024-03-19T01:39:27.986 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.981+0000 7ff46cf90700 1 Processor -- start 2024-03-19T01:39:27.986 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.981+0000 7ff46cf90700 1 -- start start 2024-03-19T01:39:27.986 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.982+0000 7ff46cf90700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff468100eb0 0x7ff468101e30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:27.987 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.982+0000 7ff46cf90700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff468101860 0x7ff468102370 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:27.987 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.982+0000 7ff46cf90700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff4681076d0 0x7ff4681028b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:27.987 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.982+0000 7ff46cf90700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7ff468112660 con 0x7ff468100eb0 2024-03-19T01:39:27.987 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.982+0000 7ff46cf90700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7ff4681124e0 con 0x7ff468101860 2024-03-19T01:39:27.987 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.982+0000 7ff46cf90700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7ff4681127e0 con 0x7ff4681076d0 2024-03-19T01:39:27.987 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.982+0000 7ff46659c700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff468100eb0 0x7ff468101e30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:27.988 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.982+0000 7ff46659c700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff468100eb0 0x7ff468101e30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:35500/0 (socket says 172.21.15.99:35500) 2024-03-19T01:39:27.988 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.982+0000 7ff46659c700 1 -- 172.21.15.99:0/2618538349 learned_addr learned my addr 172.21.15.99:0/2618538349 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:39:27.988 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.982+0000 7ff466d9d700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff4681076d0 0x7ff4681028b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:27.988 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.982+0000 7ff465d9b700 1 --2- 172.21.15.99:0/2618538349 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff468101860 0x7ff468102370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:27.988 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.982+0000 7ff46659c700 1 -- 172.21.15.99:0/2618538349 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff4681076d0 msgr2=0x7ff4681028b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:27.989 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.982+0000 7ff46659c700 1 --2- 172.21.15.99:0/2618538349 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff4681076d0 0x7ff4681028b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:27.989 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.982+0000 7ff46659c700 1 -- 172.21.15.99:0/2618538349 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff468101860 msgr2=0x7ff468102370 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:27.989 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.982+0000 7ff46659c700 1 --2- 172.21.15.99:0/2618538349 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff468101860 0x7ff468102370 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:27.989 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.982+0000 7ff46659c700 1 -- 172.21.15.99:0/2618538349 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ff4681a7030 con 0x7ff468100eb0 2024-03-19T01:39:27.989 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.982+0000 7ff465d9b700 1 --2- 172.21.15.99:0/2618538349 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff468101860 0x7ff468102370 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:39:27.989 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.982+0000 7ff46659c700 1 --2- 172.21.15.99:0/2618538349 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff468100eb0 0x7ff468101e30 secure :-1 s=READY pgs=242 cs=0 l=1 rev1=1 crypto rx=0x7ff45c006010 tx=0x7ff45c00f690 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:27.989 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.982+0000 7ff4577fe700 1 -- 172.21.15.99:0/2618538349 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff45c003680 con 0x7ff468100eb0 2024-03-19T01:39:27.989 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.982+0000 7ff46cf90700 1 -- 172.21.15.99:0/2618538349 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ff4681a72c0 con 0x7ff468100eb0 2024-03-19T01:39:27.990 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.982+0000 7ff4577fe700 1 -- 172.21.15.99:0/2618538349 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7ff45c0050d0 con 0x7ff468100eb0 2024-03-19T01:39:27.990 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.982+0000 7ff4577fe700 1 -- 172.21.15.99:0/2618538349 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff45c003db0 con 0x7ff468100eb0 2024-03-19T01:39:27.990 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.983+0000 7ff46cf90700 1 -- 172.21.15.99:0/2618538349 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7ff4681a7710 con 0x7ff468100eb0 2024-03-19T01:39:27.990 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.983+0000 7ff46cf90700 1 -- 172.21.15.99:0/2618538349 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=osdmap handle=1) -- 0x7ff468062a20 con 0x7ff468100eb0 2024-03-19T01:39:27.992 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.985+0000 7ff4577fe700 1 -- 172.21.15.99:0/2618538349 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 17) ==== 137341+0+0 (secure 0 0 0) 0x7ff45c005280 con 0x7ff468100eb0 2024-03-19T01:39:27.992 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.985+0000 7ff4577fe700 1 --2- 172.21.15.99:0/2618538349 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff4500a2ac0 0x7ff4500a4f80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:27.992 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.985+0000 7ff465d9b700 1 --2- 172.21.15.99:0/2618538349 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff4500a2ac0 0x7ff4500a4f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:27.992 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.985+0000 7ff4577fe700 1 -- 172.21.15.99:0/2618538349 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8269+0+0 (secure 0 0 0) 0x7ff45c0cfca0 con 0x7ff468100eb0 2024-03-19T01:39:27.992 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.986+0000 7ff4577fe700 1 --2- 172.21.15.99:0/2618538349 >> [v2:172.21.15.112:6816/3610356576,v1:172.21.15.112:6817/3610356576] conn(0x7ff4500ac700 0x7ff4500aeba0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:27.992 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.986+0000 7ff4577fe700 1 -- 172.21.15.99:0/2618538349 --> [v2:172.21.15.112:6816/3610356576,v1:172.21.15.112:6817/3610356576] -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7ff45c00fe50 con 0x7ff4500ac700 2024-03-19T01:39:27.992 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.986+0000 7ff4577fe700 1 -- 172.21.15.99:0/2618538349 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_get_version_reply(handle=1 version=89) ==== 24+0+0 (secure 0 0 0) 0x7ff45c0d00f0 con 0x7ff468100eb0 2024-03-19T01:39:27.993 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.986+0000 7ff466d9d700 1 --2- 172.21.15.99:0/2618538349 >> [v2:172.21.15.112:6816/3610356576,v1:172.21.15.112:6817/3610356576] conn(0x7ff4500ac700 0x7ff4500aeba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:27.993 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.986+0000 7ff465d9b700 1 --2- 172.21.15.99:0/2618538349 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff4500a2ac0 0x7ff4500a4f80 secure :-1 s=READY pgs=184 cs=0 l=1 rev1=1 crypto rx=0x7ff458009680 tx=0x7ff458009260 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:27.993 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.986+0000 7ff466d9d700 1 --2- 172.21.15.99:0/2618538349 >> [v2:172.21.15.112:6816/3610356576,v1:172.21.15.112:6817/3610356576] conn(0x7ff4500ac700 0x7ff4500aeba0 crc :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.7 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:27.993 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:27.987+0000 7ff4577fe700 1 -- 172.21.15.99:0/2618538349 <== osd.7 v2:172.21.15.112:6816/3610356576 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (crc 0 0 0) 0x7ff45c00fe50 con 0x7ff4500ac700 2024-03-19T01:39:28.003 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:28.003+0000 7ff46cf90700 1 -- 172.21.15.99:0/2618538349 --> [v2:172.21.15.112:6816/3610356576,v1:172.21.15.112:6817/3610356576] -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7ff468062ea0 con 0x7ff4500ac700 2024-03-19T01:39:28.003 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:28.003+0000 7ff4577fe700 1 -- 172.21.15.99:0/2618538349 <== osd.7 v2:172.21.15.112:6816/3610356576 2 ==== command_reply(tid 2: 0 ) ==== 8+0+12 (crc 0 0 0) 0x7ff468062ea0 con 0x7ff4500ac700 2024-03-19T01:39:28.004 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:28.003+0000 7ff46cf90700 1 -- 172.21.15.99:0/2618538349 >> [v2:172.21.15.112:6816/3610356576,v1:172.21.15.112:6817/3610356576] conn(0x7ff4500ac700 msgr2=0x7ff4500aeba0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:28.004 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:28.003+0000 7ff46cf90700 1 --2- 172.21.15.99:0/2618538349 >> [v2:172.21.15.112:6816/3610356576,v1:172.21.15.112:6817/3610356576] conn(0x7ff4500ac700 0x7ff4500aeba0 crc :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:28.004 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:28.003+0000 7ff46cf90700 1 -- 172.21.15.99:0/2618538349 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff4500a2ac0 msgr2=0x7ff4500a4f80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:28.005 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:28.004+0000 7ff46cf90700 1 --2- 172.21.15.99:0/2618538349 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff4500a2ac0 0x7ff4500a4f80 secure :-1 s=READY pgs=184 cs=0 l=1 rev1=1 crypto rx=0x7ff458009680 tx=0x7ff458009260 comp rx=0 tx=0).stop 2024-03-19T01:39:28.005 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:28.004+0000 7ff46cf90700 1 -- 172.21.15.99:0/2618538349 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff468100eb0 msgr2=0x7ff468101e30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:28.005 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:28.004+0000 7ff46cf90700 1 --2- 172.21.15.99:0/2618538349 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff468100eb0 0x7ff468101e30 secure :-1 s=READY pgs=242 cs=0 l=1 rev1=1 crypto rx=0x7ff45c006010 tx=0x7ff45c00f690 comp rx=0 tx=0).stop 2024-03-19T01:39:28.005 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:28.004+0000 7ff466d9d700 1 -- 172.21.15.99:0/2618538349 reap_dead start 2024-03-19T01:39:28.005 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:28.004+0000 7ff46cf90700 1 -- 172.21.15.99:0/2618538349 shutdown_connections 2024-03-19T01:39:28.005 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:28.004+0000 7ff46cf90700 1 -- 172.21.15.99:0/2618538349 >> 172.21.15.99:0/2618538349 conn(0x7ff468076570 msgr2=0x7ff4681003d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:28.005 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:28.004+0000 7ff46cf90700 1 -- 172.21.15.99:0/2618538349 shutdown_connections 2024-03-19T01:39:28.005 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:28.004+0000 7ff46cf90700 1 -- 172.21.15.99:0/2618538349 wait complete. 2024-03-19T01:39:28.014 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:27 smithi099 ceph-mon[62951]: pgmap v299: 97 pgs: 97 active+clean; 638 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 4.0 KiB/s rd, 853 B/s wr, 6 op/s 2024-03-19T01:39:28.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:27 smithi112 ceph-mon[66819]: pgmap v299: 97 pgs: 97 active+clean; 638 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 4.0 KiB/s rd, 853 B/s wr, 6 op/s 2024-03-19T01:39:28.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:27 smithi115 ceph-mon[67332]: pgmap v299: 97 pgs: 97 active+clean; 638 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 4.0 KiB/s rd, 853 B/s wr, 6 op/s 2024-03-19T01:39:28.754 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.815+0000 7f9a45990700 1 -- 172.21.15.99:0/398287438 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9a40101e90 msgr2=0x7f9a40105ee0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:28.754 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.815+0000 7f9a45990700 1 --2- 172.21.15.99:0/398287438 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9a40101e90 0x7f9a40105ee0 secure :-1 s=READY pgs=237 cs=0 l=1 rev1=1 crypto rx=0x7f9a280099c0 tx=0x7f9a28009cd0 comp rx=0 tx=0).stop 2024-03-19T01:39:28.754 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.815+0000 7f9a45990700 1 -- 172.21.15.99:0/398287438 shutdown_connections 2024-03-19T01:39:28.754 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.815+0000 7f9a45990700 1 --2- 172.21.15.99:0/398287438 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9a401065c0 0x7f9a4010e190 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:28.754 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.815+0000 7f9a45990700 1 --2- 172.21.15.99:0/398287438 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9a401014e0 0x7f9a401018c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:28.754 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.815+0000 7f9a45990700 1 --2- 172.21.15.99:0/398287438 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9a40101e90 0x7f9a40105ee0 unknown :-1 s=CLOSED pgs=237 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:28.755 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.815+0000 7f9a45990700 1 -- 172.21.15.99:0/398287438 >> 172.21.15.99:0/398287438 conn(0x7f9a40078340 msgr2=0x7f9a40078750 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:28.755 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.815+0000 7f9a45990700 1 -- 172.21.15.99:0/398287438 shutdown_connections 2024-03-19T01:39:28.755 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.815+0000 7f9a45990700 1 -- 172.21.15.99:0/398287438 wait complete. 2024-03-19T01:39:28.755 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.815+0000 7f9a45990700 1 Processor -- start 2024-03-19T01:39:28.755 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.816+0000 7f9a45990700 1 -- start start 2024-03-19T01:39:28.755 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.816+0000 7f9a45990700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9a401014e0 0x7f9a40199940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:28.755 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.816+0000 7f9a45990700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9a40101e90 0x7f9a40199e80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:28.755 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.816+0000 7f9a45990700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9a401065c0 0x7f9a401a49c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:28.755 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.816+0000 7f9a45990700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f9a401115a0 con 0x7f9a40101e90 2024-03-19T01:39:28.755 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.816+0000 7f9a45990700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f9a40111420 con 0x7f9a401065c0 2024-03-19T01:39:28.755 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.816+0000 7f9a45990700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f9a40111720 con 0x7f9a401014e0 2024-03-19T01:39:28.755 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.816+0000 7f9a3e7fc700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9a40101e90 0x7f9a40199e80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:28.755 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.816+0000 7f9a3effd700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9a401014e0 0x7f9a40199940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:28.755 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.816+0000 7f9a3e7fc700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9a40101e90 0x7f9a40199e80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:35464/0 (socket says 172.21.15.99:35464) 2024-03-19T01:39:28.756 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.816+0000 7f9a3e7fc700 1 -- 172.21.15.99:0/3493928123 learned_addr learned my addr 172.21.15.99:0/3493928123 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:39:28.756 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.816+0000 7f9a3f7fe700 1 --2- 172.21.15.99:0/3493928123 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9a401065c0 0x7f9a401a49c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:28.756 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.816+0000 7f9a3e7fc700 1 -- 172.21.15.99:0/3493928123 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9a401014e0 msgr2=0x7f9a40199940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:28.756 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.816+0000 7f9a3e7fc700 1 --2- 172.21.15.99:0/3493928123 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9a401014e0 0x7f9a40199940 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:28.756 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.816+0000 7f9a3e7fc700 1 -- 172.21.15.99:0/3493928123 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9a401065c0 msgr2=0x7f9a401a49c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:28.756 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.816+0000 7f9a3e7fc700 1 --2- 172.21.15.99:0/3493928123 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9a401065c0 0x7f9a401a49c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:28.756 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.816+0000 7f9a3e7fc700 1 -- 172.21.15.99:0/3493928123 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f9a401a4f00 con 0x7f9a40101e90 2024-03-19T01:39:28.756 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.816+0000 7f9a3e7fc700 1 --2- 172.21.15.99:0/3493928123 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9a40101e90 0x7f9a40199e80 secure :-1 s=READY pgs=238 cs=0 l=1 rev1=1 crypto rx=0x7f9a2800f690 tx=0x7f9a28005f10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:28.756 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.816+0000 7f9a4498e700 1 -- 172.21.15.99:0/3493928123 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9a28027070 con 0x7f9a40101e90 2024-03-19T01:39:28.757 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.817+0000 7f9a4498e700 1 -- 172.21.15.99:0/3493928123 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f9a28003680 con 0x7f9a40101e90 2024-03-19T01:39:28.757 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.817+0000 7f9a45990700 1 -- 172.21.15.99:0/3493928123 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f9a401a50d0 con 0x7f9a40101e90 2024-03-19T01:39:28.757 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.817+0000 7f9a4498e700 1 -- 172.21.15.99:0/3493928123 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9a280173f0 con 0x7f9a40101e90 2024-03-19T01:39:28.757 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.817+0000 7f9a45990700 1 -- 172.21.15.99:0/3493928123 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f9a401a5520 con 0x7f9a40101e90 2024-03-19T01:39:28.757 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.817+0000 7f9a45990700 1 -- 172.21.15.99:0/3493928123 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f9a40062a20 con 0x7f9a40101e90 2024-03-19T01:39:28.757 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.819+0000 7f9a4498e700 1 -- 172.21.15.99:0/3493928123 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 17) ==== 137341+0+0 (secure 0 0 0) 0x7f9a2802a070 con 0x7f9a40101e90 2024-03-19T01:39:28.757 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.819+0000 7f9a4498e700 1 --2- 172.21.15.99:0/3493928123 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f9a2c0a2920 0x7f9a2c0a4de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:28.757 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.819+0000 7f9a3effd700 1 --2- 172.21.15.99:0/3493928123 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f9a2c0a2920 0x7f9a2c0a4de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:28.757 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.819+0000 7f9a4498e700 1 -- 172.21.15.99:0/3493928123 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8269+0+0 (secure 0 0 0) 0x7f9a280d01d0 con 0x7f9a40101e90 2024-03-19T01:39:28.757 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.819+0000 7f9a4498e700 1 --2- 172.21.15.99:0/3493928123 >> [v2:172.21.15.99:6818/1130832772,v1:172.21.15.99:6819/1130832772] conn(0x7f9a2c0ac560 0x7f9a2c0aea00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:28.757 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.819+0000 7f9a4498e700 1 -- 172.21.15.99:0/3493928123 --> [v2:172.21.15.99:6818/1130832772,v1:172.21.15.99:6819/1130832772] -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f9a28024ec0 con 0x7f9a2c0ac560 2024-03-19T01:39:28.757 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.820+0000 7f9a3f7fe700 1 --2- 172.21.15.99:0/3493928123 >> [v2:172.21.15.99:6818/1130832772,v1:172.21.15.99:6819/1130832772] conn(0x7f9a2c0ac560 0x7f9a2c0aea00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:28.758 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.820+0000 7f9a3effd700 1 --2- 172.21.15.99:0/3493928123 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f9a2c0a2920 0x7f9a2c0a4de0 secure :-1 s=READY pgs=182 cs=0 l=1 rev1=1 crypto rx=0x7f9a38009680 tx=0x7f9a38009260 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:28.758 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.820+0000 7f9a3f7fe700 1 --2- 172.21.15.99:0/3493928123 >> [v2:172.21.15.99:6818/1130832772,v1:172.21.15.99:6819/1130832772] conn(0x7f9a2c0ac560 0x7f9a2c0aea00 crc :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.6 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:28.758 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.820+0000 7f9a4498e700 1 -- 172.21.15.99:0/3493928123 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_get_version_reply(handle=1 version=89) ==== 24+0+0 (secure 0 0 0) 0x7f9a28098e30 con 0x7f9a40101e90 2024-03-19T01:39:28.758 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.821+0000 7f9a4498e700 1 -- 172.21.15.99:0/3493928123 <== osd.6 v2:172.21.15.99:6818/1130832772 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (crc 0 0 0) 0x7f9a28024ec0 con 0x7f9a2c0ac560 2024-03-19T01:39:28.758 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.837+0000 7f9a45990700 1 -- 172.21.15.99:0/3493928123 --> [v2:172.21.15.99:6818/1130832772,v1:172.21.15.99:6819/1130832772] -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7f9a40062e20 con 0x7f9a2c0ac560 2024-03-19T01:39:28.758 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.837+0000 7f9a4498e700 1 -- 172.21.15.99:0/3493928123 <== osd.6 v2:172.21.15.99:6818/1130832772 2 ==== command_reply(tid 2: 0 ) ==== 8+0+12 (crc 0 0 0) 0x7f9a40062e20 con 0x7f9a2c0ac560 2024-03-19T01:39:28.758 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.837+0000 7f9a45990700 1 -- 172.21.15.99:0/3493928123 >> [v2:172.21.15.99:6818/1130832772,v1:172.21.15.99:6819/1130832772] conn(0x7f9a2c0ac560 msgr2=0x7f9a2c0aea00 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:28.758 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.837+0000 7f9a45990700 1 --2- 172.21.15.99:0/3493928123 >> [v2:172.21.15.99:6818/1130832772,v1:172.21.15.99:6819/1130832772] conn(0x7f9a2c0ac560 0x7f9a2c0aea00 crc :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:28.758 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.837+0000 7f9a45990700 1 -- 172.21.15.99:0/3493928123 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f9a2c0a2920 msgr2=0x7f9a2c0a4de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:28.758 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.837+0000 7f9a45990700 1 --2- 172.21.15.99:0/3493928123 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f9a2c0a2920 0x7f9a2c0a4de0 secure :-1 s=READY pgs=182 cs=0 l=1 rev1=1 crypto rx=0x7f9a38009680 tx=0x7f9a38009260 comp rx=0 tx=0).stop 2024-03-19T01:39:28.758 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.837+0000 7f9a45990700 1 -- 172.21.15.99:0/3493928123 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9a40101e90 msgr2=0x7f9a40199e80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:28.758 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.837+0000 7f9a45990700 1 --2- 172.21.15.99:0/3493928123 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9a40101e90 0x7f9a40199e80 secure :-1 s=READY pgs=238 cs=0 l=1 rev1=1 crypto rx=0x7f9a2800f690 tx=0x7f9a28005f10 comp rx=0 tx=0).stop 2024-03-19T01:39:28.758 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.837+0000 7f9a3f7fe700 1 -- 172.21.15.99:0/3493928123 reap_dead start 2024-03-19T01:39:28.758 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.838+0000 7f9a45990700 1 -- 172.21.15.99:0/3493928123 shutdown_connections 2024-03-19T01:39:28.759 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.838+0000 7f9a45990700 1 -- 172.21.15.99:0/3493928123 >> 172.21.15.99:0/3493928123 conn(0x7f9a40078340 msgr2=0x7f9a400ff1c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:28.759 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.838+0000 7f9a45990700 1 -- 172.21.15.99:0/3493928123 shutdown_connections 2024-03-19T01:39:28.759 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:26.838+0000 7f9a45990700 1 -- 172.21.15.99:0/3493928123 wait complete. 2024-03-19T01:39:29.586 INFO:teuthology.orchestra.run.smithi099.stdout:201863462956 2024-03-19T01:39:29.586 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph osd last-stat-seq osd.6 2024-03-19T01:39:29.795 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.792+0000 7f4487681700 1 -- 172.21.15.99:0/2050165322 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4480102d20 msgr2=0x7f448010f160 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:29.795 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.792+0000 7f4487681700 1 --2- 172.21.15.99:0/2050165322 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4480102d20 0x7f448010f160 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7f447c009990 tx=0x7f447c009ca0 comp rx=0 tx=0).stop 2024-03-19T01:39:29.795 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.792+0000 7f4487681700 1 -- 172.21.15.99:0/2050165322 shutdown_connections 2024-03-19T01:39:29.796 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.792+0000 7f4487681700 1 --2- 172.21.15.99:0/2050165322 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4480102d20 0x7f448010f160 unknown :-1 s=CLOSED pgs=61 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:29.796 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.792+0000 7f4487681700 1 --2- 172.21.15.99:0/2050165322 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4480108340 0x7f4480108720 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:29.796 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.792+0000 7f4487681700 1 --2- 172.21.15.99:0/2050165322 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4480102360 0x7f44801027e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:29.796 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.792+0000 7f4487681700 1 -- 172.21.15.99:0/2050165322 >> 172.21.15.99:0/2050165322 conn(0x7f4480079440 msgr2=0x7f4480079850 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:29.796 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.792+0000 7f4487681700 1 -- 172.21.15.99:0/2050165322 shutdown_connections 2024-03-19T01:39:29.796 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.792+0000 7f4487681700 1 -- 172.21.15.99:0/2050165322 wait complete. 2024-03-19T01:39:29.796 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.793+0000 7f4487681700 1 Processor -- start 2024-03-19T01:39:29.796 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.793+0000 7f4487681700 1 -- start start 2024-03-19T01:39:29.796 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.793+0000 7f4487681700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4480102360 0x7f4480074610 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:29.796 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.793+0000 7f4487681700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4480102d20 0x7f4480074b50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:29.796 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.793+0000 7f4487681700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4480108340 0x7f4480075090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:29.797 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.793+0000 7f4487681700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f44801125c0 con 0x7f4480108340 2024-03-19T01:39:29.797 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.793+0000 7f4487681700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f4480112440 con 0x7f4480102360 2024-03-19T01:39:29.797 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.793+0000 7f4485c1e700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4480108340 0x7f4480075090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:29.797 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.793+0000 7f448541d700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4480102360 0x7f4480074610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:29.797 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.793+0000 7f4484c1c700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4480102d20 0x7f4480074b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:29.797 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.793+0000 7f448541d700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4480102360 0x7f4480074610 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:53018/0 (socket says 172.21.15.99:53018) 2024-03-19T01:39:29.797 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.793+0000 7f4485c1e700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4480108340 0x7f4480075090 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:35516/0 (socket says 172.21.15.99:35516) 2024-03-19T01:39:29.797 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.793+0000 7f448541d700 1 -- 172.21.15.99:0/368687304 learned_addr learned my addr 172.21.15.99:0/368687304 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:39:29.797 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.794+0000 7f4487681700 1 -- 172.21.15.99:0/368687304 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f4480112740 con 0x7f4480102d20 2024-03-19T01:39:29.797 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.794+0000 7f4484c1c700 1 -- 172.21.15.99:0/368687304 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4480102360 msgr2=0x7f4480074610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:29.797 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.794+0000 7f4484c1c700 1 --2- 172.21.15.99:0/368687304 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4480102360 0x7f4480074610 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:29.797 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.794+0000 7f4484c1c700 1 -- 172.21.15.99:0/368687304 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4480108340 msgr2=0x7f4480075090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:29.797 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.794+0000 7f4484c1c700 1 --2- 172.21.15.99:0/368687304 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4480108340 0x7f4480075090 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:29.797 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.794+0000 7f4484c1c700 1 -- 172.21.15.99:0/368687304 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f44801a6ee0 con 0x7f4480102d20 2024-03-19T01:39:29.798 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.794+0000 7f448541d700 1 --2- 172.21.15.99:0/368687304 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4480102360 0x7f4480074610 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:39:29.798 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.794+0000 7f4485c1e700 1 --2- 172.21.15.99:0/368687304 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4480108340 0x7f4480075090 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:39:29.798 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.794+0000 7f4484c1c700 1 --2- 172.21.15.99:0/368687304 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4480102d20 0x7f4480074b50 secure :-1 s=READY pgs=62 cs=0 l=1 rev1=1 crypto rx=0x7f447000e9f0 tx=0x7f447000ed00 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:29.798 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.794+0000 7f44767fc700 1 -- 172.21.15.99:0/368687304 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f447000c9c0 con 0x7f4480102d20 2024-03-19T01:39:29.798 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.794+0000 7f44767fc700 1 -- 172.21.15.99:0/368687304 <== mon.1 v2:172.21.15.115:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f447000cb60 con 0x7f4480102d20 2024-03-19T01:39:29.798 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.794+0000 7f4487681700 1 -- 172.21.15.99:0/368687304 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f44801a71d0 con 0x7f4480102d20 2024-03-19T01:39:29.798 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.794+0000 7f44767fc700 1 -- 172.21.15.99:0/368687304 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4470018680 con 0x7f4480102d20 2024-03-19T01:39:29.798 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.794+0000 7f4487681700 1 -- 172.21.15.99:0/368687304 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f44801a7760 con 0x7f4480102d20 2024-03-19T01:39:29.799 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.795+0000 7f4487681700 1 -- 172.21.15.99:0/368687304 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f4480062aa0 con 0x7f4480102d20 2024-03-19T01:39:29.799 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.797+0000 7f44767fc700 1 -- 172.21.15.99:0/368687304 <== mon.1 v2:172.21.15.115:3300/0 4 ==== mgrmap(e 17) ==== 137341+0+0 (secure 0 0 0) 0x7f4470010430 con 0x7f4480102d20 2024-03-19T01:39:29.799 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.797+0000 7f44767fc700 1 --2- 172.21.15.99:0/368687304 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f446c0a29a0 0x7f446c0a4e60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:29.799 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.798+0000 7f448541d700 1 --2- 172.21.15.99:0/368687304 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f446c0a29a0 0x7f446c0a4e60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:29.800 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.798+0000 7f44767fc700 1 -- 172.21.15.99:0/368687304 <== mon.1 v2:172.21.15.115:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8269+0+0 (secure 0 0 0) 0x7f44700cf180 con 0x7f4480102d20 2024-03-19T01:39:29.800 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.798+0000 7f44767fc700 1 --2- 172.21.15.99:0/368687304 >> [v2:172.21.15.112:6808/3717986391,v1:172.21.15.112:6809/3717986391] conn(0x7f446c0ac5e0 0x7f446c0aea80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:29.800 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.798+0000 7f44767fc700 1 -- 172.21.15.99:0/368687304 --> [v2:172.21.15.112:6808/3717986391,v1:172.21.15.112:6809/3717986391] -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f447001bec0 con 0x7f446c0ac5e0 2024-03-19T01:39:29.800 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.798+0000 7f44767fc700 1 -- 172.21.15.99:0/368687304 <== mon.1 v2:172.21.15.115:3300/0 6 ==== mon_get_version_reply(handle=1 version=89) ==== 24+0+0 (secure 0 0 0) 0x7f44700cf530 con 0x7f4480102d20 2024-03-19T01:39:29.800 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.798+0000 7f4485c1e700 1 --2- 172.21.15.99:0/368687304 >> [v2:172.21.15.112:6808/3717986391,v1:172.21.15.112:6809/3717986391] conn(0x7f446c0ac5e0 0x7f446c0aea80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:29.800 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.798+0000 7f448541d700 1 --2- 172.21.15.99:0/368687304 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f446c0a29a0 0x7f446c0a4e60 secure :-1 s=READY pgs=185 cs=0 l=1 rev1=1 crypto rx=0x7f4478009480 tx=0x7f447800b040 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:29.800 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.798+0000 7f4485c1e700 1 --2- 172.21.15.99:0/368687304 >> [v2:172.21.15.112:6808/3717986391,v1:172.21.15.112:6809/3717986391] conn(0x7f446c0ac5e0 0x7f446c0aea80 crc :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.4 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:29.800 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.799+0000 7f44767fc700 1 -- 172.21.15.99:0/368687304 <== osd.4 v2:172.21.15.112:6808/3717986391 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (crc 0 0 0) 0x7f447001bec0 con 0x7f446c0ac5e0 2024-03-19T01:39:29.816 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.815+0000 7f4487681700 1 -- 172.21.15.99:0/368687304 --> [v2:172.21.15.112:6808/3717986391,v1:172.21.15.112:6809/3717986391] -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7f4480062ea0 con 0x7f446c0ac5e0 2024-03-19T01:39:29.816 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.816+0000 7f44767fc700 1 -- 172.21.15.99:0/368687304 <== osd.4 v2:172.21.15.112:6808/3717986391 2 ==== command_reply(tid 2: 0 ) ==== 8+0+12 (crc 0 0 0) 0x7f4480062ea0 con 0x7f446c0ac5e0 2024-03-19T01:39:29.817 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.816+0000 7f4487681700 1 -- 172.21.15.99:0/368687304 >> [v2:172.21.15.112:6808/3717986391,v1:172.21.15.112:6809/3717986391] conn(0x7f446c0ac5e0 msgr2=0x7f446c0aea80 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:29.817 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.816+0000 7f4487681700 1 --2- 172.21.15.99:0/368687304 >> [v2:172.21.15.112:6808/3717986391,v1:172.21.15.112:6809/3717986391] conn(0x7f446c0ac5e0 0x7f446c0aea80 crc :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:29.817 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.816+0000 7f4487681700 1 -- 172.21.15.99:0/368687304 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f446c0a29a0 msgr2=0x7f446c0a4e60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:29.817 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.816+0000 7f4487681700 1 --2- 172.21.15.99:0/368687304 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f446c0a29a0 0x7f446c0a4e60 secure :-1 s=READY pgs=185 cs=0 l=1 rev1=1 crypto rx=0x7f4478009480 tx=0x7f447800b040 comp rx=0 tx=0).stop 2024-03-19T01:39:29.817 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.816+0000 7f4487681700 1 -- 172.21.15.99:0/368687304 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4480102d20 msgr2=0x7f4480074b50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:29.817 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.816+0000 7f4487681700 1 --2- 172.21.15.99:0/368687304 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4480102d20 0x7f4480074b50 secure :-1 s=READY pgs=62 cs=0 l=1 rev1=1 crypto rx=0x7f447000e9f0 tx=0x7f447000ed00 comp rx=0 tx=0).stop 2024-03-19T01:39:29.817 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.816+0000 7f4485c1e700 1 -- 172.21.15.99:0/368687304 reap_dead start 2024-03-19T01:39:29.818 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.816+0000 7f4487681700 1 -- 172.21.15.99:0/368687304 shutdown_connections 2024-03-19T01:39:29.818 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.816+0000 7f4487681700 1 -- 172.21.15.99:0/368687304 >> 172.21.15.99:0/368687304 conn(0x7f4480079440 msgr2=0x7f4480100f00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:29.818 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.816+0000 7f4487681700 1 -- 172.21.15.99:0/368687304 shutdown_connections 2024-03-19T01:39:29.818 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:29.816+0000 7f4487681700 1 -- 172.21.15.99:0/368687304 wait complete. 2024-03-19T01:39:29.861 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:29 smithi112 ceph-mon[66819]: pgmap v300: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 597 B/s wr, 7 op/s 2024-03-19T01:39:30.093 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:29 smithi099 ceph-mon[62951]: pgmap v300: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 597 B/s wr, 7 op/s 2024-03-19T01:39:30.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:29 smithi115 ceph-mon[67332]: pgmap v300: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 597 B/s wr, 7 op/s 2024-03-19T01:39:31.029 INFO:teuthology.orchestra.run.smithi099.stdout:227633266728 2024-03-19T01:39:31.029 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph osd last-stat-seq osd.7 2024-03-19T01:39:32.078 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:31 smithi099 ceph-mon[62951]: pgmap v301: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 597 B/s wr, 6 op/s 2024-03-19T01:39:32.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:31 smithi112 ceph-mon[66819]: pgmap v301: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 597 B/s wr, 6 op/s 2024-03-19T01:39:32.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:31 smithi115 ceph-mon[67332]: pgmap v301: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 597 B/s wr, 6 op/s 2024-03-19T01:39:32.768 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.543+0000 7f368f0b4700 1 -- 172.21.15.99:0/276084738 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3688102360 msgr2=0x7f36881027e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:32.768 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.543+0000 7f368f0b4700 1 --2- 172.21.15.99:0/276084738 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3688102360 0x7f36881027e0 secure :-1 s=READY pgs=243 cs=0 l=1 rev1=1 crypto rx=0x7f36740099c0 tx=0x7f3674009cd0 comp rx=0 tx=0).stop 2024-03-19T01:39:32.768 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.543+0000 7f368f0b4700 1 -- 172.21.15.99:0/276084738 shutdown_connections 2024-03-19T01:39:32.768 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.543+0000 7f368f0b4700 1 --2- 172.21.15.99:0/276084738 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f3688102d20 0x7f368810f160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:32.768 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.543+0000 7f368f0b4700 1 --2- 172.21.15.99:0/276084738 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3688108340 0x7f3688108720 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:32.768 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.543+0000 7f368f0b4700 1 --2- 172.21.15.99:0/276084738 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3688102360 0x7f36881027e0 unknown :-1 s=CLOSED pgs=243 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:32.768 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.543+0000 7f368f0b4700 1 -- 172.21.15.99:0/276084738 >> 172.21.15.99:0/276084738 conn(0x7f3688079440 msgr2=0x7f3688079850 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:32.768 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.543+0000 7f368f0b4700 1 -- 172.21.15.99:0/276084738 shutdown_connections 2024-03-19T01:39:32.769 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.543+0000 7f368f0b4700 1 -- 172.21.15.99:0/276084738 wait complete. 2024-03-19T01:39:32.769 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.544+0000 7f368f0b4700 1 Processor -- start 2024-03-19T01:39:32.769 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.544+0000 7f368f0b4700 1 -- start start 2024-03-19T01:39:32.769 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.544+0000 7f368f0b4700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f3688102360 0x7f368819e720 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:32.769 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.544+0000 7f368f0b4700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3688102d20 0x7f368819ec60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:32.769 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.544+0000 7f368f0b4700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3688108340 0x7f3688198930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:32.769 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.544+0000 7f368f0b4700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f36881125f0 con 0x7f3688102d20 2024-03-19T01:39:32.769 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.544+0000 7f368f0b4700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f3688112470 con 0x7f3688108340 2024-03-19T01:39:32.769 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.544+0000 7f368f0b4700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f3688112770 con 0x7f3688102360 2024-03-19T01:39:32.769 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.544+0000 7f3687fff700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3688102d20 0x7f368819ec60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:32.769 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.544+0000 7f3687fff700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3688102d20 0x7f368819ec60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:35536/0 (socket says 172.21.15.99:35536) 2024-03-19T01:39:32.769 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.544+0000 7f3687fff700 1 -- 172.21.15.99:0/1260040643 learned_addr learned my addr 172.21.15.99:0/1260040643 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:39:32.769 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.544+0000 7f368ce50700 1 --2- 172.21.15.99:0/1260040643 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f3688102360 0x7f368819e720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:32.770 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.544+0000 7f368d651700 1 --2- 172.21.15.99:0/1260040643 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3688108340 0x7f3688198930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:32.770 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.545+0000 7f3687fff700 1 -- 172.21.15.99:0/1260040643 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f3688102360 msgr2=0x7f368819e720 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:32.770 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.545+0000 7f3687fff700 1 --2- 172.21.15.99:0/1260040643 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f3688102360 0x7f368819e720 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:32.770 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.545+0000 7f3687fff700 1 -- 172.21.15.99:0/1260040643 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3688108340 msgr2=0x7f3688198930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:32.770 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.545+0000 7f3687fff700 1 --2- 172.21.15.99:0/1260040643 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3688108340 0x7f3688198930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:32.770 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.545+0000 7f3687fff700 1 -- 172.21.15.99:0/1260040643 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f3688199090 con 0x7f3688102d20 2024-03-19T01:39:32.770 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.545+0000 7f368ce50700 1 --2- 172.21.15.99:0/1260040643 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f3688102360 0x7f368819e720 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:39:32.770 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.545+0000 7f3687fff700 1 --2- 172.21.15.99:0/1260040643 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3688102d20 0x7f368819ec60 secure :-1 s=READY pgs=244 cs=0 l=1 rev1=1 crypto rx=0x7f3674006010 tx=0x7f3674003a90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:32.770 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.545+0000 7f3685ffb700 1 -- 172.21.15.99:0/1260040643 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3674003680 con 0x7f3688102d20 2024-03-19T01:39:32.770 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.545+0000 7f3685ffb700 1 -- 172.21.15.99:0/1260040643 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f3674003e50 con 0x7f3688102d20 2024-03-19T01:39:32.770 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.545+0000 7f368f0b4700 1 -- 172.21.15.99:0/1260040643 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f3688199320 con 0x7f3688102d20 2024-03-19T01:39:32.770 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.545+0000 7f3685ffb700 1 -- 172.21.15.99:0/1260040643 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f367400fe10 con 0x7f3688102d20 2024-03-19T01:39:32.771 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.545+0000 7f368f0b4700 1 -- 172.21.15.99:0/1260040643 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f36881a55f0 con 0x7f3688102d20 2024-03-19T01:39:32.771 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.546+0000 7f368f0b4700 1 -- 172.21.15.99:0/1260040643 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f3688062a60 con 0x7f3688102d20 2024-03-19T01:39:32.771 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.547+0000 7f3685ffb700 1 -- 172.21.15.99:0/1260040643 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 17) ==== 137341+0+0 (secure 0 0 0) 0x7f367400f460 con 0x7f3688102d20 2024-03-19T01:39:32.771 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.548+0000 7f3685ffb700 1 --2- 172.21.15.99:0/1260040643 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f36780a2b40 0x7f36780a5000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:32.771 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.548+0000 7f368ce50700 1 --2- 172.21.15.99:0/1260040643 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f36780a2b40 0x7f36780a5000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:32.771 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.548+0000 7f3685ffb700 1 -- 172.21.15.99:0/1260040643 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8269+0+0 (secure 0 0 0) 0x7f36740cfa00 con 0x7f3688102d20 2024-03-19T01:39:32.771 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.548+0000 7f3685ffb700 1 --2- 172.21.15.99:0/1260040643 >> [v2:172.21.15.99:6826/1410921523,v1:172.21.15.99:6827/1410921523] conn(0x7f36780ac780 0x7f36780aec20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:32.772 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.548+0000 7f3685ffb700 1 -- 172.21.15.99:0/1260040643 --> [v2:172.21.15.99:6826/1410921523,v1:172.21.15.99:6827/1410921523] -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f3674001d30 con 0x7f36780ac780 2024-03-19T01:39:32.772 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.548+0000 7f3685ffb700 1 -- 172.21.15.99:0/1260040643 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_get_version_reply(handle=1 version=89) ==== 24+0+0 (secure 0 0 0) 0x7f36740cfe50 con 0x7f3688102d20 2024-03-19T01:39:32.772 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.548+0000 7f368d651700 1 --2- 172.21.15.99:0/1260040643 >> [v2:172.21.15.99:6826/1410921523,v1:172.21.15.99:6827/1410921523] conn(0x7f36780ac780 0x7f36780aec20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:32.772 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.548+0000 7f368ce50700 1 --2- 172.21.15.99:0/1260040643 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f36780a2b40 0x7f36780a5000 secure :-1 s=READY pgs=186 cs=0 l=1 rev1=1 crypto rx=0x7f3688100990 tx=0x7f367c009260 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:32.772 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.549+0000 7f368d651700 1 --2- 172.21.15.99:0/1260040643 >> [v2:172.21.15.99:6826/1410921523,v1:172.21.15.99:6827/1410921523] conn(0x7f36780ac780 0x7f36780aec20 crc :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.9 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:32.772 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.549+0000 7f3685ffb700 1 -- 172.21.15.99:0/1260040643 <== osd.9 v2:172.21.15.99:6826/1410921523 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (crc 0 0 0) 0x7f3674001d30 con 0x7f36780ac780 2024-03-19T01:39:32.772 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.565+0000 7f368f0b4700 1 -- 172.21.15.99:0/1260040643 --> [v2:172.21.15.99:6826/1410921523,v1:172.21.15.99:6827/1410921523] -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7f3688062e60 con 0x7f36780ac780 2024-03-19T01:39:32.772 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.565+0000 7f3685ffb700 1 -- 172.21.15.99:0/1260040643 <== osd.9 v2:172.21.15.99:6826/1410921523 2 ==== command_reply(tid 2: 0 ) ==== 8+0+12 (crc 0 0 0) 0x7f3688062e60 con 0x7f36780ac780 2024-03-19T01:39:32.772 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.566+0000 7f368f0b4700 1 -- 172.21.15.99:0/1260040643 >> [v2:172.21.15.99:6826/1410921523,v1:172.21.15.99:6827/1410921523] conn(0x7f36780ac780 msgr2=0x7f36780aec20 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:32.772 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.566+0000 7f368f0b4700 1 --2- 172.21.15.99:0/1260040643 >> [v2:172.21.15.99:6826/1410921523,v1:172.21.15.99:6827/1410921523] conn(0x7f36780ac780 0x7f36780aec20 crc :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:32.772 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.566+0000 7f368f0b4700 1 -- 172.21.15.99:0/1260040643 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f36780a2b40 msgr2=0x7f36780a5000 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:32.772 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.566+0000 7f368f0b4700 1 --2- 172.21.15.99:0/1260040643 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f36780a2b40 0x7f36780a5000 secure :-1 s=READY pgs=186 cs=0 l=1 rev1=1 crypto rx=0x7f3688100990 tx=0x7f367c009260 comp rx=0 tx=0).stop 2024-03-19T01:39:32.772 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.566+0000 7f368f0b4700 1 -- 172.21.15.99:0/1260040643 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3688102d20 msgr2=0x7f368819ec60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:32.773 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.566+0000 7f368f0b4700 1 --2- 172.21.15.99:0/1260040643 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3688102d20 0x7f368819ec60 secure :-1 s=READY pgs=244 cs=0 l=1 rev1=1 crypto rx=0x7f3674006010 tx=0x7f3674003a90 comp rx=0 tx=0).stop 2024-03-19T01:39:32.773 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.566+0000 7f368d651700 1 -- 172.21.15.99:0/1260040643 reap_dead start 2024-03-19T01:39:32.773 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.566+0000 7f368f0b4700 1 -- 172.21.15.99:0/1260040643 shutdown_connections 2024-03-19T01:39:32.773 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.566+0000 7f368f0b4700 1 -- 172.21.15.99:0/1260040643 >> 172.21.15.99:0/1260040643 conn(0x7f3688079440 msgr2=0x7f3688101030 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:32.773 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.566+0000 7f368f0b4700 1 -- 172.21.15.99:0/1260040643 shutdown_connections 2024-03-19T01:39:32.773 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:30.566+0000 7f368f0b4700 1 -- 172.21.15.99:0/1260040643 wait complete. 2024-03-19T01:39:33.346 INFO:teuthology.orchestra.run.smithi099.stdout:292057776158 2024-03-19T01:39:33.346 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph osd last-stat-seq osd.9 2024-03-19T01:39:33.810 INFO:teuthology.orchestra.run.smithi099.stdout:146028888117 2024-03-19T01:39:33.811 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph osd last-stat-seq osd.4 2024-03-19T01:39:34.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:33 smithi099 ceph-mon[62951]: pgmap v302: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 3.1 KiB/s rd, 170 B/s wr, 6 op/s 2024-03-19T01:39:34.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:33 smithi112 ceph-mon[66819]: pgmap v302: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 3.1 KiB/s rd, 170 B/s wr, 6 op/s 2024-03-19T01:39:34.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:33 smithi115 ceph-mon[67332]: pgmap v302: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 3.1 KiB/s rd, 170 B/s wr, 6 op/s 2024-03-19T01:39:34.313 INFO:teuthology.orchestra.run.smithi099.stdout:55834574915 2024-03-19T01:39:34.313 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph osd last-stat-seq osd.1 2024-03-19T01:39:34.862 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.860+0000 7f6fa44f1700 1 -- 172.21.15.99:0/2773619311 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6f9c106590 msgr2=0x7f6f9c10e160 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:34.862 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.860+0000 7f6fa44f1700 1 --2- 172.21.15.99:0/2773619311 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6f9c106590 0x7f6f9c10e160 secure :-1 s=READY pgs=246 cs=0 l=1 rev1=1 crypto rx=0x7f6f940099c0 tx=0x7f6f94009cd0 comp rx=0 tx=0).stop 2024-03-19T01:39:34.862 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.860+0000 7f6fa44f1700 1 -- 172.21.15.99:0/2773619311 shutdown_connections 2024-03-19T01:39:34.863 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.860+0000 7f6fa44f1700 1 --2- 172.21.15.99:0/2773619311 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6f9c106590 0x7f6f9c10e160 unknown :-1 s=CLOSED pgs=246 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:34.863 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.860+0000 7f6fa44f1700 1 --2- 172.21.15.99:0/2773619311 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6f9c101c70 0x7f6f9c102050 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:34.863 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.860+0000 7f6fa44f1700 1 --2- 172.21.15.99:0/2773619311 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f6f9c102620 0x7f6f9c105eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:34.863 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.860+0000 7f6fa44f1700 1 -- 172.21.15.99:0/2773619311 >> 172.21.15.99:0/2773619311 conn(0x7f6f9c077810 msgr2=0x7f6f9c0fec30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:34.863 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.860+0000 7f6fa44f1700 1 -- 172.21.15.99:0/2773619311 shutdown_connections 2024-03-19T01:39:34.863 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.860+0000 7f6fa44f1700 1 -- 172.21.15.99:0/2773619311 wait complete. 2024-03-19T01:39:34.863 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.861+0000 7f6fa44f1700 1 Processor -- start 2024-03-19T01:39:34.863 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.861+0000 7f6fa44f1700 1 -- start start 2024-03-19T01:39:34.863 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.861+0000 7f6fa44f1700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6f9c101c70 0x7f6f9c1998e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:34.863 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.861+0000 7f6fa44f1700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f6f9c102620 0x7f6f9c199e20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:34.863 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.861+0000 7f6fa44f1700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6f9c106590 0x7f6f9c1a4960 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:34.863 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.861+0000 7f6fa44f1700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f6f9c111580 con 0x7f6f9c106590 2024-03-19T01:39:34.864 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.861+0000 7f6fa44f1700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f6f9c111400 con 0x7f6f9c101c70 2024-03-19T01:39:34.864 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.861+0000 7f6fa44f1700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f6f9c111700 con 0x7f6f9c102620 2024-03-19T01:39:34.864 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.861+0000 7f6fa1a8c700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f6f9c102620 0x7f6f9c199e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:34.864 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.861+0000 7f6fa228d700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6f9c101c70 0x7f6f9c1998e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:34.864 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.861+0000 7f6fa2a8e700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6f9c106590 0x7f6f9c1a4960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:34.864 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.861+0000 7f6fa1a8c700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f6f9c102620 0x7f6f9c199e20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:38766/0 (socket says 172.21.15.99:38766) 2024-03-19T01:39:34.864 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.861+0000 7f6fa1a8c700 1 -- 172.21.15.99:0/96284397 learned_addr learned my addr 172.21.15.99:0/96284397 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:39:34.864 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.861+0000 7f6fa228d700 1 -- 172.21.15.99:0/96284397 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f6f9c102620 msgr2=0x7f6f9c199e20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:34.864 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.861+0000 7f6fa228d700 1 --2- 172.21.15.99:0/96284397 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f6f9c102620 0x7f6f9c199e20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:34.864 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.861+0000 7f6fa228d700 1 -- 172.21.15.99:0/96284397 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6f9c106590 msgr2=0x7f6f9c1a4960 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:34.864 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.861+0000 7f6fa228d700 1 --2- 172.21.15.99:0/96284397 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6f9c106590 0x7f6f9c1a4960 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:34.864 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.861+0000 7f6fa228d700 1 -- 172.21.15.99:0/96284397 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f6f9c1a4ea0 con 0x7f6f9c101c70 2024-03-19T01:39:34.865 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.862+0000 7f6fa228d700 1 --2- 172.21.15.99:0/96284397 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6f9c101c70 0x7f6f9c1998e0 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7f6f8c00ef10 tx=0x7f6f8c00c3a0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:34.865 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.862+0000 7f6f9b7fe700 1 -- 172.21.15.99:0/96284397 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6f8c00c9c0 con 0x7f6f9c101c70 2024-03-19T01:39:34.865 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.862+0000 7f6fa44f1700 1 -- 172.21.15.99:0/96284397 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f6f9c1a5070 con 0x7f6f9c101c70 2024-03-19T01:39:34.865 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.862+0000 7f6f9b7fe700 1 -- 172.21.15.99:0/96284397 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f6f8c019070 con 0x7f6f9c101c70 2024-03-19T01:39:34.865 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.862+0000 7f6f9b7fe700 1 -- 172.21.15.99:0/96284397 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6f8c0146e0 con 0x7f6f9c101c70 2024-03-19T01:39:34.865 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.862+0000 7f6fa44f1700 1 -- 172.21.15.99:0/96284397 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f6f9c1a5580 con 0x7f6f9c101c70 2024-03-19T01:39:34.866 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.862+0000 7f6fa44f1700 1 -- 172.21.15.99:0/96284397 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f6f9c062a20 con 0x7f6f9c101c70 2024-03-19T01:39:34.866 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.864+0000 7f6f9b7fe700 1 -- 172.21.15.99:0/96284397 <== mon.2 v2:172.21.15.112:3300/0 4 ==== mgrmap(e 17) ==== 137341+0+0 (secure 0 0 0) 0x7f6f8c012050 con 0x7f6f9c101c70 2024-03-19T01:39:34.867 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.864+0000 7f6f9b7fe700 1 --2- 172.21.15.99:0/96284397 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f6f840a2920 0x7f6f840a4de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:34.867 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.864+0000 7f6fa1a8c700 1 --2- 172.21.15.99:0/96284397 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f6f840a2920 0x7f6f840a4de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:34.867 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.864+0000 7f6f9b7fe700 1 -- 172.21.15.99:0/96284397 <== mon.2 v2:172.21.15.112:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8269+0+0 (secure 0 0 0) 0x7f6f8c0cf670 con 0x7f6f9c101c70 2024-03-19T01:39:34.867 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.864+0000 7f6f9b7fe700 1 --2- 172.21.15.99:0/96284397 >> [v2:172.21.15.115:6824/938436244,v1:172.21.15.115:6825/938436244] conn(0x7f6f840ac560 0x7f6f840aea00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:34.867 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.864+0000 7f6f9b7fe700 1 -- 172.21.15.99:0/96284397 --> [v2:172.21.15.115:6824/938436244,v1:172.21.15.115:6825/938436244] -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f6f8c00be50 con 0x7f6f840ac560 2024-03-19T01:39:34.867 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.864+0000 7f6f9b7fe700 1 -- 172.21.15.99:0/96284397 <== mon.2 v2:172.21.15.112:3300/0 6 ==== mon_get_version_reply(handle=1 version=89) ==== 24+0+0 (secure 0 0 0) 0x7f6f8c0cfa20 con 0x7f6f9c101c70 2024-03-19T01:39:34.867 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.865+0000 7f6fa1a8c700 1 --2- 172.21.15.99:0/96284397 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f6f840a2920 0x7f6f840a4de0 secure :-1 s=READY pgs=188 cs=0 l=1 rev1=1 crypto rx=0x7f6f9c104150 tx=0x7f6f90009260 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:34.867 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.865+0000 7f6fa2a8e700 1 --2- 172.21.15.99:0/96284397 >> [v2:172.21.15.115:6824/938436244,v1:172.21.15.115:6825/938436244] conn(0x7f6f840ac560 0x7f6f840aea00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:34.867 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.865+0000 7f6fa2a8e700 1 --2- 172.21.15.99:0/96284397 >> [v2:172.21.15.115:6824/938436244,v1:172.21.15.115:6825/938436244] conn(0x7f6f840ac560 0x7f6f840aea00 crc :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.11 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:34.867 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.866+0000 7f6f9b7fe700 1 -- 172.21.15.99:0/96284397 <== osd.11 v2:172.21.15.115:6824/938436244 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (crc 0 0 0) 0x7f6f8c00be50 con 0x7f6f840ac560 2024-03-19T01:39:34.882 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.882+0000 7f6fa44f1700 1 -- 172.21.15.99:0/96284397 --> [v2:172.21.15.115:6824/938436244,v1:172.21.15.115:6825/938436244] -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7f6f9c062e20 con 0x7f6f840ac560 2024-03-19T01:39:34.882 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.882+0000 7f6f9b7fe700 1 -- 172.21.15.99:0/96284397 <== osd.11 v2:172.21.15.115:6824/938436244 2 ==== command_reply(tid 2: 0 ) ==== 8+0+12 (crc 0 0 0) 0x7f6f9c062e20 con 0x7f6f840ac560 2024-03-19T01:39:34.883 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.882+0000 7f6fa44f1700 1 -- 172.21.15.99:0/96284397 >> [v2:172.21.15.115:6824/938436244,v1:172.21.15.115:6825/938436244] conn(0x7f6f840ac560 msgr2=0x7f6f840aea00 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:34.883 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.882+0000 7f6fa44f1700 1 --2- 172.21.15.99:0/96284397 >> [v2:172.21.15.115:6824/938436244,v1:172.21.15.115:6825/938436244] conn(0x7f6f840ac560 0x7f6f840aea00 crc :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:34.883 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.883+0000 7f6fa44f1700 1 -- 172.21.15.99:0/96284397 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f6f840a2920 msgr2=0x7f6f840a4de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:34.884 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.883+0000 7f6fa44f1700 1 --2- 172.21.15.99:0/96284397 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f6f840a2920 0x7f6f840a4de0 secure :-1 s=READY pgs=188 cs=0 l=1 rev1=1 crypto rx=0x7f6f9c104150 tx=0x7f6f90009260 comp rx=0 tx=0).stop 2024-03-19T01:39:34.884 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.883+0000 7f6fa44f1700 1 -- 172.21.15.99:0/96284397 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6f9c101c70 msgr2=0x7f6f9c1998e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:34.884 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.883+0000 7f6fa44f1700 1 --2- 172.21.15.99:0/96284397 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6f9c101c70 0x7f6f9c1998e0 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7f6f8c00ef10 tx=0x7f6f8c00c3a0 comp rx=0 tx=0).stop 2024-03-19T01:39:34.884 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.883+0000 7f6fa2a8e700 1 -- 172.21.15.99:0/96284397 reap_dead start 2024-03-19T01:39:34.884 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.883+0000 7f6fa44f1700 1 -- 172.21.15.99:0/96284397 shutdown_connections 2024-03-19T01:39:34.884 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.883+0000 7f6fa44f1700 1 -- 172.21.15.99:0/96284397 >> 172.21.15.99:0/96284397 conn(0x7f6f9c077810 msgr2=0x7f6f9c104d30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:34.884 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.883+0000 7f6fa44f1700 1 -- 172.21.15.99:0/96284397 shutdown_connections 2024-03-19T01:39:34.884 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:34.883+0000 7f6fa44f1700 1 -- 172.21.15.99:0/96284397 wait complete. 2024-03-19T01:39:35.441 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.438+0000 7f86c28ff700 1 -- 172.21.15.99:0/4167951441 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f86bc101460 msgr2=0x7f86bc101840 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:35.441 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.438+0000 7f86c28ff700 1 --2- 172.21.15.99:0/4167951441 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f86bc101460 0x7f86bc101840 secure :-1 s=READY pgs=249 cs=0 l=1 rev1=1 crypto rx=0x7f86ac009990 tx=0x7f86ac009ca0 comp rx=0 tx=0).stop 2024-03-19T01:39:35.441 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.438+0000 7f86c28ff700 1 -- 172.21.15.99:0/4167951441 shutdown_connections 2024-03-19T01:39:35.441 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.438+0000 7f86c28ff700 1 --2- 172.21.15.99:0/4167951441 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f86bc106520 0x7f86bc10e0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:35.442 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.438+0000 7f86c28ff700 1 --2- 172.21.15.99:0/4167951441 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f86bc101460 0x7f86bc101840 unknown :-1 s=CLOSED pgs=249 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:35.442 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.438+0000 7f86c28ff700 1 --2- 172.21.15.99:0/4167951441 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f86bc101e10 0x7f86bc105e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:35.442 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.438+0000 7f86c28ff700 1 -- 172.21.15.99:0/4167951441 >> 172.21.15.99:0/4167951441 conn(0x7f86bc078180 msgr2=0x7f86bc078590 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:35.442 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.439+0000 7f86c28ff700 1 -- 172.21.15.99:0/4167951441 shutdown_connections 2024-03-19T01:39:35.442 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.439+0000 7f86c28ff700 1 -- 172.21.15.99:0/4167951441 wait complete. 2024-03-19T01:39:35.442 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.439+0000 7f86c28ff700 1 Processor -- start 2024-03-19T01:39:35.442 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.439+0000 7f86c28ff700 1 -- start start 2024-03-19T01:39:35.442 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.440+0000 7f86c28ff700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f86bc101460 0x7f86bc19d640 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:35.442 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.440+0000 7f86c28ff700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f86bc101e10 0x7f86bc19db80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:35.442 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.440+0000 7f86c28ff700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f86bc106520 0x7f86bc1977c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:35.442 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.440+0000 7f86c28ff700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f86bc111500 con 0x7f86bc101e10 2024-03-19T01:39:35.443 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.440+0000 7f86c28ff700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f86bc111380 con 0x7f86bc101460 2024-03-19T01:39:35.443 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.440+0000 7f86c28ff700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f86bc111680 con 0x7f86bc106520 2024-03-19T01:39:35.443 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.440+0000 7f86bb7fe700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f86bc101e10 0x7f86bc19db80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:35.443 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.440+0000 7f86bb7fe700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f86bc101e10 0x7f86bc19db80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:47300/0 (socket says 172.21.15.99:47300) 2024-03-19T01:39:35.443 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.440+0000 7f86bb7fe700 1 -- 172.21.15.99:0/2773127460 learned_addr learned my addr 172.21.15.99:0/2773127460 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:39:35.443 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.440+0000 7f86c0e9c700 1 --2- 172.21.15.99:0/2773127460 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f86bc106520 0x7f86bc1977c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:35.443 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.440+0000 7f86bbfff700 1 --2- 172.21.15.99:0/2773127460 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f86bc101460 0x7f86bc19d640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:35.443 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.440+0000 7f86bb7fe700 1 -- 172.21.15.99:0/2773127460 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f86bc106520 msgr2=0x7f86bc1977c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:35.443 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.440+0000 7f86bb7fe700 1 --2- 172.21.15.99:0/2773127460 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f86bc106520 0x7f86bc1977c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:35.443 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.440+0000 7f86bb7fe700 1 -- 172.21.15.99:0/2773127460 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f86bc101460 msgr2=0x7f86bc19d640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:35.444 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.440+0000 7f86bb7fe700 1 --2- 172.21.15.99:0/2773127460 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f86bc101460 0x7f86bc19d640 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:35.444 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.440+0000 7f86bb7fe700 1 -- 172.21.15.99:0/2773127460 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f86bc197fa0 con 0x7f86bc101e10 2024-03-19T01:39:35.444 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.440+0000 7f86c0e9c700 1 --2- 172.21.15.99:0/2773127460 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f86bc106520 0x7f86bc1977c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:39:35.444 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.440+0000 7f86bbfff700 1 --2- 172.21.15.99:0/2773127460 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f86bc101460 0x7f86bc19d640 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:39:35.444 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.440+0000 7f86bb7fe700 1 --2- 172.21.15.99:0/2773127460 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f86bc101e10 0x7f86bc19db80 secure :-1 s=READY pgs=250 cs=0 l=1 rev1=1 crypto rx=0x7f86b000e9f0 tx=0x7f86b000ed00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:35.444 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.440+0000 7f86b97fa700 1 -- 172.21.15.99:0/2773127460 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f86b0004a30 con 0x7f86bc101e10 2024-03-19T01:39:35.444 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.441+0000 7f86b97fa700 1 -- 172.21.15.99:0/2773127460 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f86b000cca0 con 0x7f86bc101e10 2024-03-19T01:39:35.444 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.441+0000 7f86c28ff700 1 -- 172.21.15.99:0/2773127460 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f86bc198290 con 0x7f86bc101e10 2024-03-19T01:39:35.444 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.441+0000 7f86b97fa700 1 -- 172.21.15.99:0/2773127460 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f86b00056c0 con 0x7f86bc101e10 2024-03-19T01:39:35.444 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.441+0000 7f86c28ff700 1 -- 172.21.15.99:0/2773127460 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f86bc06a9b0 con 0x7f86bc101e10 2024-03-19T01:39:35.445 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.441+0000 7f86c28ff700 1 -- 172.21.15.99:0/2773127460 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f86bc06afc0 con 0x7f86bc101e10 2024-03-19T01:39:35.446 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.443+0000 7f86b97fa700 1 -- 172.21.15.99:0/2773127460 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 17) ==== 137341+0+0 (secure 0 0 0) 0x7f86b000c7c0 con 0x7f86bc101e10 2024-03-19T01:39:35.446 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.443+0000 7f86b97fa700 1 --2- 172.21.15.99:0/2773127460 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f86a40a2920 0x7f86a40a4de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:35.446 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.443+0000 7f86bbfff700 1 --2- 172.21.15.99:0/2773127460 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f86a40a2920 0x7f86a40a4de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:35.446 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.443+0000 7f86b97fa700 1 -- 172.21.15.99:0/2773127460 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8269+0+0 (secure 0 0 0) 0x7f86b00cf240 con 0x7f86bc101e10 2024-03-19T01:39:35.446 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.444+0000 7f86b97fa700 1 --2- 172.21.15.99:0/2773127460 >> [v2:172.21.15.115:6800/3766690472,v1:172.21.15.115:6801/3766690472] conn(0x7f86a40ac560 0x7f86a40aea00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:35.446 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.444+0000 7f86b97fa700 1 -- 172.21.15.99:0/2773127460 --> [v2:172.21.15.115:6800/3766690472,v1:172.21.15.115:6801/3766690472] -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f86b0004500 con 0x7f86a40ac560 2024-03-19T01:39:35.446 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.444+0000 7f86b97fa700 1 -- 172.21.15.99:0/2773127460 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_get_version_reply(handle=1 version=89) ==== 24+0+0 (secure 0 0 0) 0x7f86b00cf690 con 0x7f86bc101e10 2024-03-19T01:39:35.446 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.444+0000 7f86bbfff700 1 --2- 172.21.15.99:0/2773127460 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f86a40a2920 0x7f86a40a4de0 secure :-1 s=READY pgs=190 cs=0 l=1 rev1=1 crypto rx=0x7f86ac0098e0 tx=0x7f86ac005960 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:35.446 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.444+0000 7f86c0e9c700 1 --2- 172.21.15.99:0/2773127460 >> [v2:172.21.15.115:6800/3766690472,v1:172.21.15.115:6801/3766690472] conn(0x7f86a40ac560 0x7f86a40aea00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:35.446 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.444+0000 7f86c0e9c700 1 --2- 172.21.15.99:0/2773127460 >> [v2:172.21.15.115:6800/3766690472,v1:172.21.15.115:6801/3766690472] conn(0x7f86a40ac560 0x7f86a40aea00 crc :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:35.446 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.445+0000 7f86b97fa700 1 -- 172.21.15.99:0/2773127460 <== osd.2 v2:172.21.15.115:6800/3766690472 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (crc 0 0 0) 0x7f86b0004500 con 0x7f86a40ac560 2024-03-19T01:39:35.461 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.461+0000 7f86c28ff700 1 -- 172.21.15.99:0/2773127460 --> [v2:172.21.15.115:6800/3766690472,v1:172.21.15.115:6801/3766690472] -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7f86bc06b440 con 0x7f86a40ac560 2024-03-19T01:39:35.462 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.462+0000 7f86b97fa700 1 -- 172.21.15.99:0/2773127460 <== osd.2 v2:172.21.15.115:6800/3766690472 2 ==== command_reply(tid 2: 0 ) ==== 8+0+11 (crc 0 0 0) 0x7f86bc06b440 con 0x7f86a40ac560 2024-03-19T01:39:35.463 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.462+0000 7f86c28ff700 1 -- 172.21.15.99:0/2773127460 >> [v2:172.21.15.115:6800/3766690472,v1:172.21.15.115:6801/3766690472] conn(0x7f86a40ac560 msgr2=0x7f86a40aea00 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:35.463 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.462+0000 7f86c28ff700 1 --2- 172.21.15.99:0/2773127460 >> [v2:172.21.15.115:6800/3766690472,v1:172.21.15.115:6801/3766690472] conn(0x7f86a40ac560 0x7f86a40aea00 crc :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:35.463 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.462+0000 7f86c28ff700 1 -- 172.21.15.99:0/2773127460 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f86a40a2920 msgr2=0x7f86a40a4de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:35.463 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.462+0000 7f86c28ff700 1 --2- 172.21.15.99:0/2773127460 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f86a40a2920 0x7f86a40a4de0 secure :-1 s=READY pgs=190 cs=0 l=1 rev1=1 crypto rx=0x7f86ac0098e0 tx=0x7f86ac005960 comp rx=0 tx=0).stop 2024-03-19T01:39:35.463 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.462+0000 7f86c28ff700 1 -- 172.21.15.99:0/2773127460 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f86bc101e10 msgr2=0x7f86bc19db80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:35.463 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.462+0000 7f86c28ff700 1 --2- 172.21.15.99:0/2773127460 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f86bc101e10 0x7f86bc19db80 secure :-1 s=READY pgs=250 cs=0 l=1 rev1=1 crypto rx=0x7f86b000e9f0 tx=0x7f86b000ed00 comp rx=0 tx=0).stop 2024-03-19T01:39:35.463 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.462+0000 7f86c0e9c700 1 -- 172.21.15.99:0/2773127460 reap_dead start 2024-03-19T01:39:35.463 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.462+0000 7f86c28ff700 1 -- 172.21.15.99:0/2773127460 shutdown_connections 2024-03-19T01:39:35.463 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.462+0000 7f86c28ff700 1 -- 172.21.15.99:0/2773127460 >> 172.21.15.99:0/2773127460 conn(0x7f86bc078180 msgr2=0x7f86bc0ff530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:35.463 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.462+0000 7f86c28ff700 1 -- 172.21.15.99:0/2773127460 shutdown_connections 2024-03-19T01:39:35.464 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.462+0000 7f86c28ff700 1 -- 172.21.15.99:0/2773127460 wait complete. 2024-03-19T01:39:35.595 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.311+0000 7f7cad741700 1 -- 172.21.15.99:0/3905777259 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7ca8102360 msgr2=0x7f7ca81027e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:35.595 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.311+0000 7f7cad741700 1 --2- 172.21.15.99:0/3905777259 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7ca8102360 0x7f7ca81027e0 secure :-1 s=READY pgs=247 cs=0 l=1 rev1=1 crypto rx=0x7f7c9c0099c0 tx=0x7f7c9c009cd0 comp rx=0 tx=0).stop 2024-03-19T01:39:35.596 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.311+0000 7f7cad741700 1 -- 172.21.15.99:0/3905777259 shutdown_connections 2024-03-19T01:39:35.596 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.311+0000 7f7cad741700 1 --2- 172.21.15.99:0/3905777259 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f7ca8102d20 0x7f7ca810f160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:35.596 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.311+0000 7f7cad741700 1 --2- 172.21.15.99:0/3905777259 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f7ca8108340 0x7f7ca8108720 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:35.596 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.311+0000 7f7cad741700 1 --2- 172.21.15.99:0/3905777259 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7ca8102360 0x7f7ca81027e0 unknown :-1 s=CLOSED pgs=247 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:35.596 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.311+0000 7f7cad741700 1 -- 172.21.15.99:0/3905777259 >> 172.21.15.99:0/3905777259 conn(0x7f7ca8079440 msgr2=0x7f7ca8079850 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:35.596 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.311+0000 7f7cad741700 1 -- 172.21.15.99:0/3905777259 shutdown_connections 2024-03-19T01:39:35.596 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.311+0000 7f7cad741700 1 -- 172.21.15.99:0/3905777259 wait complete. 2024-03-19T01:39:35.596 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.312+0000 7f7cad741700 1 Processor -- start 2024-03-19T01:39:35.596 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.312+0000 7f7cad741700 1 -- start start 2024-03-19T01:39:35.596 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.312+0000 7f7cad741700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f7ca8102360 0x7f7ca819e720 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:35.596 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.312+0000 7f7cad741700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f7ca8102d20 0x7f7ca819ec60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:35.596 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.312+0000 7f7cad741700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7ca8108340 0x7f7ca8198930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:35.596 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.312+0000 7f7cad741700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f7ca81125f0 con 0x7f7ca8108340 2024-03-19T01:39:35.597 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.312+0000 7f7cad741700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f7ca8112470 con 0x7f7ca8102d20 2024-03-19T01:39:35.597 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.312+0000 7f7cad741700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f7ca8112770 con 0x7f7ca8102360 2024-03-19T01:39:35.597 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.312+0000 7f7ca77fe700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7ca8108340 0x7f7ca8198930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:35.597 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.312+0000 7f7ca77fe700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7ca8108340 0x7f7ca8198930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:47278/0 (socket says 172.21.15.99:47278) 2024-03-19T01:39:35.597 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.312+0000 7f7ca77fe700 1 -- 172.21.15.99:0/303793517 learned_addr learned my addr 172.21.15.99:0/303793517 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:39:35.597 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.312+0000 7f7ca6ffd700 1 --2- 172.21.15.99:0/303793517 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f7ca8102360 0x7f7ca819e720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:35.597 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.312+0000 7f7ca67fc700 1 --2- 172.21.15.99:0/303793517 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f7ca8102d20 0x7f7ca819ec60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:35.597 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.313+0000 7f7ca77fe700 1 -- 172.21.15.99:0/303793517 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f7ca8102360 msgr2=0x7f7ca819e720 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:35.597 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.313+0000 7f7ca77fe700 1 --2- 172.21.15.99:0/303793517 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f7ca8102360 0x7f7ca819e720 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:35.598 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.313+0000 7f7ca77fe700 1 -- 172.21.15.99:0/303793517 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f7ca8102d20 msgr2=0x7f7ca819ec60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:35.598 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.313+0000 7f7ca77fe700 1 --2- 172.21.15.99:0/303793517 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f7ca8102d20 0x7f7ca819ec60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:35.598 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.313+0000 7f7ca77fe700 1 -- 172.21.15.99:0/303793517 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7ca8199090 con 0x7f7ca8108340 2024-03-19T01:39:35.598 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.313+0000 7f7ca6ffd700 1 --2- 172.21.15.99:0/303793517 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f7ca8102360 0x7f7ca819e720 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:39:35.598 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.313+0000 7f7ca67fc700 1 --2- 172.21.15.99:0/303793517 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f7ca8102d20 0x7f7ca819ec60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:39:35.598 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.313+0000 7f7ca77fe700 1 --2- 172.21.15.99:0/303793517 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7ca8108340 0x7f7ca8198930 secure :-1 s=READY pgs=248 cs=0 l=1 rev1=1 crypto rx=0x7f7ca000ef30 tx=0x7f7ca000c3a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:35.598 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.313+0000 7f7c8ffff700 1 -- 172.21.15.99:0/303793517 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7ca00092e0 con 0x7f7ca8108340 2024-03-19T01:39:35.598 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.313+0000 7f7cad741700 1 -- 172.21.15.99:0/303793517 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f7ca8199320 con 0x7f7ca8108340 2024-03-19T01:39:35.598 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.313+0000 7f7c8ffff700 1 -- 172.21.15.99:0/303793517 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7ca0004bb0 con 0x7f7ca8108340 2024-03-19T01:39:35.599 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.313+0000 7f7c8ffff700 1 -- 172.21.15.99:0/303793517 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7ca0004280 con 0x7f7ca8108340 2024-03-19T01:39:35.599 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.313+0000 7f7cad741700 1 -- 172.21.15.99:0/303793517 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f7ca81a55f0 con 0x7f7ca8108340 2024-03-19T01:39:35.599 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.314+0000 7f7cad741700 1 -- 172.21.15.99:0/303793517 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f7ca8062a60 con 0x7f7ca8108340 2024-03-19T01:39:35.599 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.315+0000 7f7c8ffff700 1 -- 172.21.15.99:0/303793517 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 17) ==== 137341+0+0 (secure 0 0 0) 0x7f7ca001b030 con 0x7f7ca8108340 2024-03-19T01:39:35.599 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.315+0000 7f7c8ffff700 1 --2- 172.21.15.99:0/303793517 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f7c900a2920 0x7f7c900a4de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:35.599 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.316+0000 7f7ca6ffd700 1 --2- 172.21.15.99:0/303793517 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f7c900a2920 0x7f7c900a4de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:35.599 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.316+0000 7f7c8ffff700 1 -- 172.21.15.99:0/303793517 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8269+0+0 (secure 0 0 0) 0x7f7ca00cefc0 con 0x7f7ca8108340 2024-03-19T01:39:35.599 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.316+0000 7f7c8ffff700 1 --2- 172.21.15.99:0/303793517 >> [v2:172.21.15.99:6810/3756732895,v1:172.21.15.99:6811/3756732895] conn(0x7f7c900ac560 0x7f7c900aea00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:35.599 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.316+0000 7f7c8ffff700 1 -- 172.21.15.99:0/303793517 --> [v2:172.21.15.99:6810/3756732895,v1:172.21.15.99:6811/3756732895] -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f7ca001fe50 con 0x7f7c900ac560 2024-03-19T01:39:35.599 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.316+0000 7f7c8ffff700 1 -- 172.21.15.99:0/303793517 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_get_version_reply(handle=1 version=89) ==== 24+0+0 (secure 0 0 0) 0x7f7ca00cf4a0 con 0x7f7ca8108340 2024-03-19T01:39:35.599 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.316+0000 7f7ca67fc700 1 --2- 172.21.15.99:0/303793517 >> [v2:172.21.15.99:6810/3756732895,v1:172.21.15.99:6811/3756732895] conn(0x7f7c900ac560 0x7f7c900aea00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:35.599 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.316+0000 7f7ca6ffd700 1 --2- 172.21.15.99:0/303793517 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f7c900a2920 0x7f7c900a4de0 secure :-1 s=READY pgs=189 cs=0 l=1 rev1=1 crypto rx=0x7f7ca8100990 tx=0x7f7c98009260 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:35.599 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.316+0000 7f7ca67fc700 1 --2- 172.21.15.99:0/303793517 >> [v2:172.21.15.99:6810/3756732895,v1:172.21.15.99:6811/3756732895] conn(0x7f7c900ac560 0x7f7c900aea00 crc :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.3 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:35.599 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.317+0000 7f7c8ffff700 1 -- 172.21.15.99:0/303793517 <== osd.3 v2:172.21.15.99:6810/3756732895 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (crc 0 0 0) 0x7f7ca001fe50 con 0x7f7c900ac560 2024-03-19T01:39:35.600 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.333+0000 7f7cad741700 1 -- 172.21.15.99:0/303793517 --> [v2:172.21.15.99:6810/3756732895,v1:172.21.15.99:6811/3756732895] -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7f7ca8062e60 con 0x7f7c900ac560 2024-03-19T01:39:35.600 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.334+0000 7f7c8ffff700 1 -- 172.21.15.99:0/303793517 <== osd.3 v2:172.21.15.99:6810/3756732895 2 ==== command_reply(tid 2: 0 ) ==== 8+0+12 (crc 0 0 0) 0x7f7ca8062e60 con 0x7f7c900ac560 2024-03-19T01:39:35.600 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.334+0000 7f7cad741700 1 -- 172.21.15.99:0/303793517 >> [v2:172.21.15.99:6810/3756732895,v1:172.21.15.99:6811/3756732895] conn(0x7f7c900ac560 msgr2=0x7f7c900aea00 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:35.600 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.334+0000 7f7cad741700 1 --2- 172.21.15.99:0/303793517 >> [v2:172.21.15.99:6810/3756732895,v1:172.21.15.99:6811/3756732895] conn(0x7f7c900ac560 0x7f7c900aea00 crc :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:35.600 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.334+0000 7f7cad741700 1 -- 172.21.15.99:0/303793517 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f7c900a2920 msgr2=0x7f7c900a4de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:35.600 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.334+0000 7f7cad741700 1 --2- 172.21.15.99:0/303793517 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f7c900a2920 0x7f7c900a4de0 secure :-1 s=READY pgs=189 cs=0 l=1 rev1=1 crypto rx=0x7f7ca8100990 tx=0x7f7c98009260 comp rx=0 tx=0).stop 2024-03-19T01:39:35.600 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.334+0000 7f7cad741700 1 -- 172.21.15.99:0/303793517 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7ca8108340 msgr2=0x7f7ca8198930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:35.600 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.334+0000 7f7cad741700 1 --2- 172.21.15.99:0/303793517 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7ca8108340 0x7f7ca8198930 secure :-1 s=READY pgs=248 cs=0 l=1 rev1=1 crypto rx=0x7f7ca000ef30 tx=0x7f7ca000c3a0 comp rx=0 tx=0).stop 2024-03-19T01:39:35.600 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.334+0000 7f7ca77fe700 1 -- 172.21.15.99:0/303793517 reap_dead start 2024-03-19T01:39:35.600 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.334+0000 7f7cad741700 1 -- 172.21.15.99:0/303793517 shutdown_connections 2024-03-19T01:39:35.600 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.334+0000 7f7cad741700 1 -- 172.21.15.99:0/303793517 >> 172.21.15.99:0/303793517 conn(0x7f7ca8079440 msgr2=0x7f7ca8101030 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:35.600 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.334+0000 7f7cad741700 1 -- 172.21.15.99:0/303793517 shutdown_connections 2024-03-19T01:39:35.600 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:35.334+0000 7f7cad741700 1 -- 172.21.15.99:0/303793517 wait complete. 2024-03-19T01:39:36.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:35 smithi099 ceph-mon[62951]: pgmap v303: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 KiB/s rd, 170 B/s wr, 5 op/s 2024-03-19T01:39:36.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:35 smithi112 ceph-mon[66819]: pgmap v303: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 KiB/s rd, 170 B/s wr, 5 op/s 2024-03-19T01:39:36.142 INFO:teuthology.orchestra.run.smithi099.stdout:317827579929 2024-03-19T01:39:36.143 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph osd last-stat-seq osd.10 2024-03-19T01:39:36.182 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.685+0000 7f607533b700 1 -- 172.21.15.99:0/1816099911 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6070102d20 msgr2=0x7f607010f160 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:36.183 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.685+0000 7f607533b700 1 --2- 172.21.15.99:0/1816099911 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6070102d20 0x7f607010f160 secure :-1 s=READY pgs=245 cs=0 l=1 rev1=1 crypto rx=0x7f606800b0d0 tx=0x7f606800b3e0 comp rx=0 tx=0).stop 2024-03-19T01:39:36.183 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.685+0000 7f607533b700 1 -- 172.21.15.99:0/1816099911 shutdown_connections 2024-03-19T01:39:36.183 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.685+0000 7f607533b700 1 --2- 172.21.15.99:0/1816099911 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6070102d20 0x7f607010f160 unknown :-1 s=CLOSED pgs=245 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:36.183 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.685+0000 7f607533b700 1 --2- 172.21.15.99:0/1816099911 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6070108340 0x7f6070108720 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:36.183 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.685+0000 7f607533b700 1 --2- 172.21.15.99:0/1816099911 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f6070102360 0x7f60701027e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:36.183 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.685+0000 7f607533b700 1 -- 172.21.15.99:0/1816099911 >> 172.21.15.99:0/1816099911 conn(0x7f6070079440 msgr2=0x7f6070079850 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:36.183 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.685+0000 7f607533b700 1 -- 172.21.15.99:0/1816099911 shutdown_connections 2024-03-19T01:39:36.183 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.685+0000 7f607533b700 1 -- 172.21.15.99:0/1816099911 wait complete. 2024-03-19T01:39:36.183 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.686+0000 7f607533b700 1 Processor -- start 2024-03-19T01:39:36.183 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.686+0000 7f607533b700 1 -- start start 2024-03-19T01:39:36.183 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.686+0000 7f607533b700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f6070102360 0x7f607019e720 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:36.183 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.686+0000 7f607533b700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6070102d20 0x7f607019ec60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:36.183 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.686+0000 7f607533b700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6070108340 0x7f6070198930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:36.184 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.686+0000 7f607533b700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f60701125f0 con 0x7f6070108340 2024-03-19T01:39:36.184 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.686+0000 7f607533b700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f6070112470 con 0x7f6070102d20 2024-03-19T01:39:36.184 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.686+0000 7f607533b700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f6070112770 con 0x7f6070102360 2024-03-19T01:39:36.184 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.686+0000 7f606effd700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f6070102360 0x7f607019e720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:36.184 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.686+0000 7f60667fc700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6070102d20 0x7f607019ec60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:36.184 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.686+0000 7f606f7fe700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6070108340 0x7f6070198930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:36.184 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.686+0000 7f606effd700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f6070102360 0x7f607019e720 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:38754/0 (socket says 172.21.15.99:38754) 2024-03-19T01:39:36.184 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.686+0000 7f606effd700 1 -- 172.21.15.99:0/119070113 learned_addr learned my addr 172.21.15.99:0/119070113 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:39:36.184 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.686+0000 7f60667fc700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6070102d20 0x7f607019ec60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:39318/0 (socket says 172.21.15.99:39318) 2024-03-19T01:39:36.184 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.687+0000 7f606effd700 1 -- 172.21.15.99:0/119070113 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6070102d20 msgr2=0x7f607019ec60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:36.184 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.687+0000 7f606effd700 1 --2- 172.21.15.99:0/119070113 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6070102d20 0x7f607019ec60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:36.184 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.687+0000 7f606effd700 1 -- 172.21.15.99:0/119070113 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6070108340 msgr2=0x7f6070198930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:36.185 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.687+0000 7f606effd700 1 --2- 172.21.15.99:0/119070113 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6070108340 0x7f6070198930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:36.185 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.687+0000 7f606effd700 1 -- 172.21.15.99:0/119070113 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f6070199090 con 0x7f6070102360 2024-03-19T01:39:36.185 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.687+0000 7f60667fc700 1 --2- 172.21.15.99:0/119070113 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6070102d20 0x7f607019ec60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:39:36.185 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.687+0000 7f606f7fe700 1 --2- 172.21.15.99:0/119070113 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6070108340 0x7f6070198930 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:39:36.185 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.687+0000 7f606effd700 1 --2- 172.21.15.99:0/119070113 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f6070102360 0x7f607019e720 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 crypto rx=0x7f606000e9a0 tx=0x7f606000ecb0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:36.185 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.687+0000 7f606cff9700 1 -- 172.21.15.99:0/119070113 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f606000ca10 con 0x7f6070102360 2024-03-19T01:39:36.185 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.687+0000 7f607533b700 1 -- 172.21.15.99:0/119070113 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f6070199380 con 0x7f6070102360 2024-03-19T01:39:36.185 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.687+0000 7f606cff9700 1 -- 172.21.15.99:0/119070113 <== mon.1 v2:172.21.15.115:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f606000cbb0 con 0x7f6070102360 2024-03-19T01:39:36.185 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.687+0000 7f607533b700 1 -- 172.21.15.99:0/119070113 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f60701a55f0 con 0x7f6070102360 2024-03-19T01:39:36.185 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.687+0000 7f606cff9700 1 -- 172.21.15.99:0/119070113 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6060018620 con 0x7f6070102360 2024-03-19T01:39:36.186 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.688+0000 7f607533b700 1 -- 172.21.15.99:0/119070113 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f60700629e0 con 0x7f6070102360 2024-03-19T01:39:36.186 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.689+0000 7f606cff9700 1 -- 172.21.15.99:0/119070113 <== mon.1 v2:172.21.15.115:3300/0 4 ==== mgrmap(e 17) ==== 137341+0+0 (secure 0 0 0) 0x7f6060010430 con 0x7f6070102360 2024-03-19T01:39:36.186 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.689+0000 7f606cff9700 1 --2- 172.21.15.99:0/119070113 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f605c0a2920 0x7f605c0a4de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:36.186 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.690+0000 7f60667fc700 1 --2- 172.21.15.99:0/119070113 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f605c0a2920 0x7f605c0a4de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:36.186 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.690+0000 7f606cff9700 1 -- 172.21.15.99:0/119070113 <== mon.1 v2:172.21.15.115:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8269+0+0 (secure 0 0 0) 0x7f60600cf2b0 con 0x7f6070102360 2024-03-19T01:39:36.186 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.690+0000 7f606cff9700 1 --2- 172.21.15.99:0/119070113 >> [v2:172.21.15.115:6808/1971806408,v1:172.21.15.115:6809/1971806408] conn(0x7f605c0ac560 0x7f605c0aea00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:36.186 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.690+0000 7f606cff9700 1 -- 172.21.15.99:0/119070113 --> [v2:172.21.15.115:6808/1971806408,v1:172.21.15.115:6809/1971806408] -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f606001bec0 con 0x7f605c0ac560 2024-03-19T01:39:36.186 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.690+0000 7f606cff9700 1 -- 172.21.15.99:0/119070113 <== mon.1 v2:172.21.15.115:3300/0 6 ==== mon_get_version_reply(handle=1 version=89) ==== 24+0+0 (secure 0 0 0) 0x7f60600cf660 con 0x7f6070102360 2024-03-19T01:39:36.186 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.690+0000 7f606f7fe700 1 --2- 172.21.15.99:0/119070113 >> [v2:172.21.15.115:6808/1971806408,v1:172.21.15.115:6809/1971806408] conn(0x7f605c0ac560 0x7f605c0aea00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:36.186 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.690+0000 7f60667fc700 1 --2- 172.21.15.99:0/119070113 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f605c0a2920 0x7f605c0a4de0 secure :-1 s=READY pgs=187 cs=0 l=1 rev1=1 crypto rx=0x7f6070079a20 tx=0x7f6058009260 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:36.186 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.690+0000 7f606f7fe700 1 --2- 172.21.15.99:0/119070113 >> [v2:172.21.15.115:6808/1971806408,v1:172.21.15.115:6809/1971806408] conn(0x7f605c0ac560 0x7f605c0aea00 crc :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.5 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:36.186 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.691+0000 7f606cff9700 1 -- 172.21.15.99:0/119070113 <== osd.5 v2:172.21.15.115:6808/1971806408 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (crc 0 0 0) 0x7f606001bec0 con 0x7f605c0ac560 2024-03-19T01:39:36.186 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.708+0000 7f607533b700 1 -- 172.21.15.99:0/119070113 --> [v2:172.21.15.115:6808/1971806408,v1:172.21.15.115:6809/1971806408] -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7f6070062e10 con 0x7f605c0ac560 2024-03-19T01:39:36.187 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.708+0000 7f606cff9700 1 -- 172.21.15.99:0/119070113 <== osd.5 v2:172.21.15.115:6808/1971806408 2 ==== command_reply(tid 2: 0 ) ==== 8+0+12 (crc 0 0 0) 0x7f6070062e10 con 0x7f605c0ac560 2024-03-19T01:39:36.187 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.709+0000 7f607533b700 1 -- 172.21.15.99:0/119070113 >> [v2:172.21.15.115:6808/1971806408,v1:172.21.15.115:6809/1971806408] conn(0x7f605c0ac560 msgr2=0x7f605c0aea00 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:36.187 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.709+0000 7f607533b700 1 --2- 172.21.15.99:0/119070113 >> [v2:172.21.15.115:6808/1971806408,v1:172.21.15.115:6809/1971806408] conn(0x7f605c0ac560 0x7f605c0aea00 crc :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:36.187 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.709+0000 7f607533b700 1 -- 172.21.15.99:0/119070113 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f605c0a2920 msgr2=0x7f605c0a4de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:36.187 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.709+0000 7f607533b700 1 --2- 172.21.15.99:0/119070113 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f605c0a2920 0x7f605c0a4de0 secure :-1 s=READY pgs=187 cs=0 l=1 rev1=1 crypto rx=0x7f6070079a20 tx=0x7f6058009260 comp rx=0 tx=0).stop 2024-03-19T01:39:36.187 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.709+0000 7f607533b700 1 -- 172.21.15.99:0/119070113 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f6070102360 msgr2=0x7f607019e720 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:36.187 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.709+0000 7f607533b700 1 --2- 172.21.15.99:0/119070113 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f6070102360 0x7f607019e720 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 crypto rx=0x7f606000e9a0 tx=0x7f606000ecb0 comp rx=0 tx=0).stop 2024-03-19T01:39:36.187 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.709+0000 7f606f7fe700 1 -- 172.21.15.99:0/119070113 reap_dead start 2024-03-19T01:39:36.187 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.709+0000 7f607533b700 1 -- 172.21.15.99:0/119070113 shutdown_connections 2024-03-19T01:39:36.187 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.709+0000 7f607533b700 1 -- 172.21.15.99:0/119070113 >> 172.21.15.99:0/119070113 conn(0x7f6070079440 msgr2=0x7f6070100fc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:36.187 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.709+0000 7f607533b700 1 -- 172.21.15.99:0/119070113 shutdown_connections 2024-03-19T01:39:36.187 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:32.709+0000 7f607533b700 1 -- 172.21.15.99:0/119070113 wait complete. 2024-03-19T01:39:36.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:35 smithi115 ceph-mon[67332]: pgmap v303: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 KiB/s rd, 170 B/s wr, 5 op/s 2024-03-19T01:39:36.777 INFO:teuthology.orchestra.run.smithi099.stdout:171798691890 2024-03-19T01:39:36.778 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph osd last-stat-seq osd.5 2024-03-19T01:39:37.269 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.266+0000 7f326c32f700 1 -- 172.21.15.99:0/793835003 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3264102d20 msgr2=0x7f326410f160 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:37.269 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.266+0000 7f326c32f700 1 --2- 172.21.15.99:0/793835003 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3264102d20 0x7f326410f160 secure :-1 s=READY pgs=251 cs=0 l=1 rev1=1 crypto rx=0x7f325c00b0d0 tx=0x7f325c00b3e0 comp rx=0 tx=0).stop 2024-03-19T01:39:37.269 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.266+0000 7f326c32f700 1 -- 172.21.15.99:0/793835003 shutdown_connections 2024-03-19T01:39:37.269 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.266+0000 7f326c32f700 1 --2- 172.21.15.99:0/793835003 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3264102d20 0x7f326410f160 unknown :-1 s=CLOSED pgs=251 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:37.269 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.266+0000 7f326c32f700 1 --2- 172.21.15.99:0/793835003 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3264108340 0x7f3264108720 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:37.270 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.266+0000 7f326c32f700 1 --2- 172.21.15.99:0/793835003 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f3264102360 0x7f32641027e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:37.270 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.266+0000 7f326c32f700 1 -- 172.21.15.99:0/793835003 >> 172.21.15.99:0/793835003 conn(0x7f3264079440 msgr2=0x7f3264079850 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:37.270 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.267+0000 7f326c32f700 1 -- 172.21.15.99:0/793835003 shutdown_connections 2024-03-19T01:39:37.270 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.267+0000 7f326c32f700 1 -- 172.21.15.99:0/793835003 wait complete. 2024-03-19T01:39:37.270 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.267+0000 7f326c32f700 1 Processor -- start 2024-03-19T01:39:37.270 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.267+0000 7f326c32f700 1 -- start start 2024-03-19T01:39:37.270 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.267+0000 7f326c32f700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f3264102360 0x7f326419e720 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:37.270 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.267+0000 7f326c32f700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3264102d20 0x7f326419ec60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:37.270 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.267+0000 7f326c32f700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3264108340 0x7f3264198930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:37.270 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.267+0000 7f326c32f700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f32641125f0 con 0x7f3264108340 2024-03-19T01:39:37.270 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.267+0000 7f326c32f700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f3264112470 con 0x7f3264102d20 2024-03-19T01:39:37.270 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.267+0000 7f326c32f700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f3264112770 con 0x7f3264102360 2024-03-19T01:39:37.270 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.267+0000 7f326a8cc700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3264108340 0x7f3264198930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:37.271 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.267+0000 7f326a0cb700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f3264102360 0x7f326419e720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:37.271 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.267+0000 7f326a8cc700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3264108340 0x7f3264198930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:47320/0 (socket says 172.21.15.99:47320) 2024-03-19T01:39:37.271 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.268+0000 7f326a8cc700 1 -- 172.21.15.99:0/4176199008 learned_addr learned my addr 172.21.15.99:0/4176199008 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:39:37.271 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.268+0000 7f32698ca700 1 --2- 172.21.15.99:0/4176199008 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3264102d20 0x7f326419ec60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:37.271 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.268+0000 7f326a8cc700 1 -- 172.21.15.99:0/4176199008 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f3264102360 msgr2=0x7f326419e720 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:37.271 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.268+0000 7f326a8cc700 1 --2- 172.21.15.99:0/4176199008 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f3264102360 0x7f326419e720 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:37.271 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.268+0000 7f326a8cc700 1 -- 172.21.15.99:0/4176199008 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3264102d20 msgr2=0x7f326419ec60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:37.271 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.268+0000 7f326a8cc700 1 --2- 172.21.15.99:0/4176199008 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3264102d20 0x7f326419ec60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:37.271 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.268+0000 7f326a8cc700 1 -- 172.21.15.99:0/4176199008 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f3264199090 con 0x7f3264108340 2024-03-19T01:39:37.271 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.268+0000 7f326a0cb700 1 --2- 172.21.15.99:0/4176199008 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f3264102360 0x7f326419e720 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:39:37.271 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.268+0000 7f326a8cc700 1 --2- 172.21.15.99:0/4176199008 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3264108340 0x7f3264198930 secure :-1 s=READY pgs=252 cs=0 l=1 rev1=1 crypto rx=0x7f325c006070 tx=0x7f325c007a20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:37.271 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.268+0000 7f325b7fe700 1 -- 172.21.15.99:0/4176199008 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f325c003c00 con 0x7f3264108340 2024-03-19T01:39:37.271 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.268+0000 7f326c32f700 1 -- 172.21.15.99:0/4176199008 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f3264199320 con 0x7f3264108340 2024-03-19T01:39:37.272 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.268+0000 7f325b7fe700 1 -- 172.21.15.99:0/4176199008 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f325c022070 con 0x7f3264108340 2024-03-19T01:39:37.272 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.268+0000 7f325b7fe700 1 -- 172.21.15.99:0/4176199008 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f325c01b6c0 con 0x7f3264108340 2024-03-19T01:39:37.272 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.268+0000 7f326c32f700 1 -- 172.21.15.99:0/4176199008 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f32641a55f0 con 0x7f3264108340 2024-03-19T01:39:37.272 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.269+0000 7f326c32f700 1 -- 172.21.15.99:0/4176199008 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f32640629e0 con 0x7f3264108340 2024-03-19T01:39:37.273 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.270+0000 7f325b7fe700 1 -- 172.21.15.99:0/4176199008 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 17) ==== 137341+0+0 (secure 0 0 0) 0x7f325c01d470 con 0x7f3264108340 2024-03-19T01:39:37.273 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.271+0000 7f325b7fe700 1 --2- 172.21.15.99:0/4176199008 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f32500a2920 0x7f32500a4de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:37.273 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.271+0000 7f326a0cb700 1 --2- 172.21.15.99:0/4176199008 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f32500a2920 0x7f32500a4de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:37.273 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.271+0000 7f325b7fe700 1 -- 172.21.15.99:0/4176199008 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8269+0+0 (secure 0 0 0) 0x7f325c00f020 con 0x7f3264108340 2024-03-19T01:39:37.273 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.271+0000 7f325b7fe700 1 --2- 172.21.15.99:0/4176199008 >> [v2:172.21.15.99:6802/2935260112,v1:172.21.15.99:6803/2935260112] conn(0x7f32500ac560 0x7f32500aea00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:37.273 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.271+0000 7f325b7fe700 1 -- 172.21.15.99:0/4176199008 --> [v2:172.21.15.99:6802/2935260112,v1:172.21.15.99:6803/2935260112] -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f325c028e70 con 0x7f32500ac560 2024-03-19T01:39:37.274 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.271+0000 7f325b7fe700 1 -- 172.21.15.99:0/4176199008 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_get_version_reply(handle=1 version=89) ==== 24+0+0 (secure 0 0 0) 0x7f325c0cfe40 con 0x7f3264108340 2024-03-19T01:39:37.274 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.271+0000 7f32698ca700 1 --2- 172.21.15.99:0/4176199008 >> [v2:172.21.15.99:6802/2935260112,v1:172.21.15.99:6803/2935260112] conn(0x7f32500ac560 0x7f32500aea00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:37.274 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.271+0000 7f326a0cb700 1 --2- 172.21.15.99:0/4176199008 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f32500a2920 0x7f32500a4de0 secure :-1 s=READY pgs=191 cs=0 l=1 rev1=1 crypto rx=0x7f3264100b70 tx=0x7f3260009260 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:37.274 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.272+0000 7f32698ca700 1 --2- 172.21.15.99:0/4176199008 >> [v2:172.21.15.99:6802/2935260112,v1:172.21.15.99:6803/2935260112] conn(0x7f32500ac560 0x7f32500aea00 crc :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:37.274 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.272+0000 7f325b7fe700 1 -- 172.21.15.99:0/4176199008 <== osd.0 v2:172.21.15.99:6802/2935260112 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (crc 0 0 0) 0x7f325c028e70 con 0x7f32500ac560 2024-03-19T01:39:37.288 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.288+0000 7f326c32f700 1 -- 172.21.15.99:0/4176199008 --> [v2:172.21.15.99:6802/2935260112,v1:172.21.15.99:6803/2935260112] -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7f3264062e10 con 0x7f32500ac560 2024-03-19T01:39:37.289 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.289+0000 7f325b7fe700 1 -- 172.21.15.99:0/4176199008 <== osd.0 v2:172.21.15.99:6802/2935260112 2 ==== command_reply(tid 2: 0 ) ==== 8+0+11 (crc 0 0 0) 0x7f3264062e10 con 0x7f32500ac560 2024-03-19T01:39:37.290 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.289+0000 7f326c32f700 1 -- 172.21.15.99:0/4176199008 >> [v2:172.21.15.99:6802/2935260112,v1:172.21.15.99:6803/2935260112] conn(0x7f32500ac560 msgr2=0x7f32500aea00 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:37.290 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.289+0000 7f326c32f700 1 --2- 172.21.15.99:0/4176199008 >> [v2:172.21.15.99:6802/2935260112,v1:172.21.15.99:6803/2935260112] conn(0x7f32500ac560 0x7f32500aea00 crc :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:37.290 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.289+0000 7f326c32f700 1 -- 172.21.15.99:0/4176199008 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f32500a2920 msgr2=0x7f32500a4de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:37.290 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.289+0000 7f326c32f700 1 --2- 172.21.15.99:0/4176199008 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f32500a2920 0x7f32500a4de0 secure :-1 s=READY pgs=191 cs=0 l=1 rev1=1 crypto rx=0x7f3264100b70 tx=0x7f3260009260 comp rx=0 tx=0).stop 2024-03-19T01:39:37.290 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.289+0000 7f326c32f700 1 -- 172.21.15.99:0/4176199008 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3264108340 msgr2=0x7f3264198930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:37.290 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.289+0000 7f326c32f700 1 --2- 172.21.15.99:0/4176199008 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3264108340 0x7f3264198930 secure :-1 s=READY pgs=252 cs=0 l=1 rev1=1 crypto rx=0x7f325c006070 tx=0x7f325c007a20 comp rx=0 tx=0).stop 2024-03-19T01:39:37.290 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.289+0000 7f326a8cc700 1 -- 172.21.15.99:0/4176199008 reap_dead start 2024-03-19T01:39:37.290 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.289+0000 7f326c32f700 1 -- 172.21.15.99:0/4176199008 shutdown_connections 2024-03-19T01:39:37.291 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.289+0000 7f326c32f700 1 -- 172.21.15.99:0/4176199008 >> 172.21.15.99:0/4176199008 conn(0x7f3264079440 msgr2=0x7f3264100fc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:37.291 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.289+0000 7f326c32f700 1 -- 172.21.15.99:0/4176199008 shutdown_connections 2024-03-19T01:39:37.291 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.290+0000 7f326c32f700 1 -- 172.21.15.99:0/4176199008 wait complete. 2024-03-19T01:39:37.485 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.483+0000 7f4042dfc700 1 -- 172.21.15.99:0/251675460 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f403c106570 msgr2=0x7f403c10e140 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:37.485 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.483+0000 7f4042dfc700 1 --2- 172.21.15.99:0/251675460 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f403c106570 0x7f403c10e140 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 crypto rx=0x7f4034009fc0 tx=0x7f403400b330 comp rx=0 tx=0).stop 2024-03-19T01:39:37.485 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.483+0000 7f4042dfc700 1 -- 172.21.15.99:0/251675460 shutdown_connections 2024-03-19T01:39:37.485 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.483+0000 7f4042dfc700 1 --2- 172.21.15.99:0/251675460 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f403c106570 0x7f403c10e140 unknown :-1 s=CLOSED pgs=64 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:37.485 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.483+0000 7f4042dfc700 1 --2- 172.21.15.99:0/251675460 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f403c101490 0x7f403c101870 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:37.485 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.483+0000 7f4042dfc700 1 --2- 172.21.15.99:0/251675460 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f403c101e40 0x7f403c105e90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:37.485 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.483+0000 7f4042dfc700 1 -- 172.21.15.99:0/251675460 >> 172.21.15.99:0/251675460 conn(0x7f403c078270 msgr2=0x7f403c078680 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:37.485 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.483+0000 7f4042dfc700 1 -- 172.21.15.99:0/251675460 shutdown_connections 2024-03-19T01:39:37.485 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.483+0000 7f4042dfc700 1 -- 172.21.15.99:0/251675460 wait complete. 2024-03-19T01:39:37.485 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.483+0000 7f4042dfc700 1 Processor -- start 2024-03-19T01:39:37.485 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.483+0000 7f4042dfc700 1 -- start start 2024-03-19T01:39:37.485 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.484+0000 7f4042dfc700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f403c101490 0x7f403c19e720 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:37.486 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.484+0000 7f4042dfc700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f403c101e40 0x7f403c19ec60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:37.486 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.484+0000 7f4042dfc700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f403c106570 0x7f403c1988e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:37.486 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.484+0000 7f4042dfc700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f403c111560 con 0x7f403c101490 2024-03-19T01:39:37.486 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.484+0000 7f4042dfc700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f403c1113e0 con 0x7f403c106570 2024-03-19T01:39:37.486 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.484+0000 7f4042dfc700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f403c1116e0 con 0x7f403c101e40 2024-03-19T01:39:37.486 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.484+0000 7f4040b98700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f403c101490 0x7f403c19e720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:37.486 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.484+0000 7f4040b98700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f403c101490 0x7f403c19e720 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:47332/0 (socket says 172.21.15.99:47332) 2024-03-19T01:39:37.486 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.484+0000 7f4040b98700 1 -- 172.21.15.99:0/2617812660 learned_addr learned my addr 172.21.15.99:0/2617812660 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:39:37.486 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.484+0000 7f403bfff700 1 --2- 172.21.15.99:0/2617812660 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f403c101e40 0x7f403c19ec60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:37.486 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.484+0000 7f4041399700 1 --2- 172.21.15.99:0/2617812660 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f403c106570 0x7f403c1988e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:37.486 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.484+0000 7f4040b98700 1 -- 172.21.15.99:0/2617812660 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f403c101e40 msgr2=0x7f403c19ec60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:37.486 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.484+0000 7f4040b98700 1 --2- 172.21.15.99:0/2617812660 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f403c101e40 0x7f403c19ec60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:37.486 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.484+0000 7f4040b98700 1 -- 172.21.15.99:0/2617812660 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f403c106570 msgr2=0x7f403c1988e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:37.486 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.484+0000 7f4040b98700 1 --2- 172.21.15.99:0/2617812660 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f403c106570 0x7f403c1988e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:37.487 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.484+0000 7f4040b98700 1 -- 172.21.15.99:0/2617812660 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f403c199040 con 0x7f403c101490 2024-03-19T01:39:37.487 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.484+0000 7f403bfff700 1 --2- 172.21.15.99:0/2617812660 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f403c101e40 0x7f403c19ec60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:39:37.487 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.484+0000 7f4040b98700 1 --2- 172.21.15.99:0/2617812660 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f403c101490 0x7f403c19e720 secure :-1 s=READY pgs=253 cs=0 l=1 rev1=1 crypto rx=0x7f403000ec10 tx=0x7f403000ef20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:37.487 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.484+0000 7f4039ffb700 1 -- 172.21.15.99:0/2617812660 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f403000cc90 con 0x7f403c101490 2024-03-19T01:39:37.487 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.484+0000 7f4042dfc700 1 -- 172.21.15.99:0/2617812660 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f403c199330 con 0x7f403c101490 2024-03-19T01:39:37.487 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.484+0000 7f4039ffb700 1 -- 172.21.15.99:0/2617812660 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f403000ce30 con 0x7f403c101490 2024-03-19T01:39:37.487 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.484+0000 7f4039ffb700 1 -- 172.21.15.99:0/2617812660 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4030010690 con 0x7f403c101490 2024-03-19T01:39:37.487 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.485+0000 7f4042dfc700 1 -- 172.21.15.99:0/2617812660 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f403c1a55f0 con 0x7f403c101490 2024-03-19T01:39:37.488 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.485+0000 7f4042dfc700 1 -- 172.21.15.99:0/2617812660 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f403c062a20 con 0x7f403c101490 2024-03-19T01:39:37.488 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.486+0000 7f4039ffb700 1 -- 172.21.15.99:0/2617812660 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 17) ==== 137341+0+0 (secure 0 0 0) 0x7f4030003680 con 0x7f403c101490 2024-03-19T01:39:37.488 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.487+0000 7f4039ffb700 1 --2- 172.21.15.99:0/2617812660 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f402c0a2920 0x7f402c0a4de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:37.488 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.487+0000 7f403bfff700 1 --2- 172.21.15.99:0/2617812660 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f402c0a2920 0x7f402c0a4de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:37.488 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.487+0000 7f4039ffb700 1 -- 172.21.15.99:0/2617812660 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8269+0+0 (secure 0 0 0) 0x7f40300cf200 con 0x7f403c101490 2024-03-19T01:39:37.488 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.487+0000 7f4039ffb700 1 --2- 172.21.15.99:0/2617812660 >> [v2:172.21.15.115:6816/4255803111,v1:172.21.15.115:6817/4255803111] conn(0x7f402c0ac560 0x7f402c0aea00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:37.488 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.487+0000 7f4039ffb700 1 -- 172.21.15.99:0/2617812660 --> [v2:172.21.15.115:6816/4255803111,v1:172.21.15.115:6817/4255803111] -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f4030012ec0 con 0x7f402c0ac560 2024-03-19T01:39:37.489 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.487+0000 7f4039ffb700 1 -- 172.21.15.99:0/2617812660 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_get_version_reply(handle=1 version=89) ==== 24+0+0 (secure 0 0 0) 0x7f40300cf650 con 0x7f403c101490 2024-03-19T01:39:37.489 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.487+0000 7f4041399700 1 --2- 172.21.15.99:0/2617812660 >> [v2:172.21.15.115:6816/4255803111,v1:172.21.15.115:6817/4255803111] conn(0x7f402c0ac560 0x7f402c0aea00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:37.489 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.487+0000 7f403bfff700 1 --2- 172.21.15.99:0/2617812660 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f402c0a2920 0x7f402c0a4de0 secure :-1 s=READY pgs=192 cs=0 l=1 rev1=1 crypto rx=0x7f403c107b10 tx=0x7f4028009260 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:37.489 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.488+0000 7f4041399700 1 --2- 172.21.15.99:0/2617812660 >> [v2:172.21.15.115:6816/4255803111,v1:172.21.15.115:6817/4255803111] conn(0x7f402c0ac560 0x7f402c0aea00 crc :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.8 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:37.489 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.489+0000 7f4039ffb700 1 -- 172.21.15.99:0/2617812660 <== osd.8 v2:172.21.15.115:6816/4255803111 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (crc 0 0 0) 0x7f4030012ec0 con 0x7f402c0ac560 2024-03-19T01:39:37.505 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.504+0000 7f4042dfc700 1 -- 172.21.15.99:0/2617812660 --> [v2:172.21.15.115:6816/4255803111,v1:172.21.15.115:6817/4255803111] -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7f403c062ea0 con 0x7f402c0ac560 2024-03-19T01:39:37.505 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.505+0000 7f4039ffb700 1 -- 172.21.15.99:0/2617812660 <== osd.8 v2:172.21.15.115:6816/4255803111 2 ==== command_reply(tid 2: 0 ) ==== 8+0+12 (crc 0 0 0) 0x7f403c062ea0 con 0x7f402c0ac560 2024-03-19T01:39:37.506 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.505+0000 7f4042dfc700 1 -- 172.21.15.99:0/2617812660 >> [v2:172.21.15.115:6816/4255803111,v1:172.21.15.115:6817/4255803111] conn(0x7f402c0ac560 msgr2=0x7f402c0aea00 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:37.506 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.505+0000 7f4042dfc700 1 --2- 172.21.15.99:0/2617812660 >> [v2:172.21.15.115:6816/4255803111,v1:172.21.15.115:6817/4255803111] conn(0x7f402c0ac560 0x7f402c0aea00 crc :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:37.506 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.505+0000 7f4042dfc700 1 -- 172.21.15.99:0/2617812660 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f402c0a2920 msgr2=0x7f402c0a4de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:37.506 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.505+0000 7f4042dfc700 1 --2- 172.21.15.99:0/2617812660 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f402c0a2920 0x7f402c0a4de0 secure :-1 s=READY pgs=192 cs=0 l=1 rev1=1 crypto rx=0x7f403c107b10 tx=0x7f4028009260 comp rx=0 tx=0).stop 2024-03-19T01:39:37.506 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.505+0000 7f4042dfc700 1 -- 172.21.15.99:0/2617812660 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f403c101490 msgr2=0x7f403c19e720 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:37.506 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.505+0000 7f4042dfc700 1 --2- 172.21.15.99:0/2617812660 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f403c101490 0x7f403c19e720 secure :-1 s=READY pgs=253 cs=0 l=1 rev1=1 crypto rx=0x7f403000ec10 tx=0x7f403000ef20 comp rx=0 tx=0).stop 2024-03-19T01:39:37.506 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.505+0000 7f4041399700 1 -- 172.21.15.99:0/2617812660 reap_dead start 2024-03-19T01:39:37.506 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.505+0000 7f4042dfc700 1 -- 172.21.15.99:0/2617812660 shutdown_connections 2024-03-19T01:39:37.506 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.505+0000 7f4042dfc700 1 -- 172.21.15.99:0/2617812660 >> 172.21.15.99:0/2617812660 conn(0x7f403c078270 msgr2=0x7f403c06acf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:37.506 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.505+0000 7f4042dfc700 1 -- 172.21.15.99:0/2617812660 shutdown_connections 2024-03-19T01:39:37.507 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:37.505+0000 7f4042dfc700 1 -- 172.21.15.99:0/2617812660 wait complete. 2024-03-19T01:39:38.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:37 smithi099 ceph-mon[62951]: pgmap v304: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 3.5 KiB/s rd, 170 B/s wr, 6 op/s 2024-03-19T01:39:38.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:37 smithi112 ceph-mon[66819]: pgmap v304: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 3.5 KiB/s rd, 170 B/s wr, 6 op/s 2024-03-19T01:39:38.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:37 smithi115 ceph-mon[67332]: pgmap v304: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 3.5 KiB/s rd, 170 B/s wr, 6 op/s 2024-03-19T01:39:38.749 INFO:teuthology.orchestra.run.smithi099.stdout:352187318292 2024-03-19T01:39:38.749 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph osd last-stat-seq osd.11 2024-03-19T01:39:39.190 INFO:teuthology.orchestra.run.smithi099.stdout:77309411393 2024-03-19T01:39:39.190 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph osd last-stat-seq osd.2 2024-03-19T01:39:39.799 INFO:teuthology.orchestra.run.smithi099.stdout:34359738443 2024-03-19T01:39:39.799 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph osd last-stat-seq osd.0 2024-03-19T01:39:40.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:39 smithi099 ceph-mon[62951]: pgmap v305: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 3.1 KiB/s rd, 170 B/s wr, 6 op/s 2024-03-19T01:39:40.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:39 smithi112 ceph-mon[66819]: pgmap v305: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 3.1 KiB/s rd, 170 B/s wr, 6 op/s 2024-03-19T01:39:40.158 INFO:teuthology.orchestra.run.smithi099.stdout:115964117053 2024-03-19T01:39:40.158 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph osd last-stat-seq osd.3 2024-03-19T01:39:40.240 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:39:40.243 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:39:40.243 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:39:40.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:39 smithi115 ceph-mon[67332]: pgmap v305: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 3.1 KiB/s rd, 170 B/s wr, 6 op/s 2024-03-19T01:39:40.754 INFO:teuthology.orchestra.run.smithi099.stdout:257698037795 2024-03-19T01:39:40.755 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph osd last-stat-seq osd.8 2024-03-19T01:39:40.781 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:39:40.782 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:39:40.788 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:39:42.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:41 smithi099 ceph-mon[62951]: pgmap v306: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 3.0 KiB/s rd, 5 op/s 2024-03-19T01:39:42.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:41 smithi112 ceph-mon[66819]: pgmap v306: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 3.0 KiB/s rd, 5 op/s 2024-03-19T01:39:42.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:41 smithi115 ceph-mon[67332]: pgmap v306: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 3.0 KiB/s rd, 5 op/s 2024-03-19T01:39:44.066 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:43 smithi099 ceph-mon[62951]: pgmap v307: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 5 op/s 2024-03-19T01:39:44.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:43 smithi112 ceph-mon[66819]: pgmap v307: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 5 op/s 2024-03-19T01:39:44.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:43 smithi115 ceph-mon[67332]: pgmap v307: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 5 op/s 2024-03-19T01:39:44.831 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.831+0000 7f4f88737700 1 -- 172.21.15.99:0/1584896566 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4f8006a960 msgr2=0x7f4f8010f1a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:44.832 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.831+0000 7f4f88737700 1 --2- 172.21.15.99:0/1584896566 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4f8006a960 0x7f4f8010f1a0 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 crypto rx=0x7f4f78009f90 tx=0x7f4f7800b330 comp rx=0 tx=0).stop 2024-03-19T01:39:44.832 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.831+0000 7f4f88737700 1 -- 172.21.15.99:0/1584896566 shutdown_connections 2024-03-19T01:39:44.832 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.831+0000 7f4f88737700 1 --2- 172.21.15.99:0/1584896566 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4f8006a960 0x7f4f8010f1a0 unknown :-1 s=CLOSED pgs=58 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:44.832 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.831+0000 7f4f88737700 1 --2- 172.21.15.99:0/1584896566 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4f80108360 0x7f4f80108740 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:44.832 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.831+0000 7f4f88737700 1 --2- 172.21.15.99:0/1584896566 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4f80069fc0 0x7f4f8006a420 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:44.832 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.831+0000 7f4f88737700 1 -- 172.21.15.99:0/1584896566 >> 172.21.15.99:0/1584896566 conn(0x7f4f80078140 msgr2=0x7f4f80078550 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:44.833 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.831+0000 7f4f88737700 1 -- 172.21.15.99:0/1584896566 shutdown_connections 2024-03-19T01:39:44.833 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.831+0000 7f4f88737700 1 -- 172.21.15.99:0/1584896566 wait complete. 2024-03-19T01:39:44.837 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.832+0000 7f4f88737700 1 Processor -- start 2024-03-19T01:39:44.837 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.832+0000 7f4f88737700 1 -- start start 2024-03-19T01:39:44.837 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.832+0000 7f4f88737700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4f80069fc0 0x7f4f8019e700 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:44.837 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.832+0000 7f4f88737700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4f8006a960 0x7f4f8019ec40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:44.837 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.832+0000 7f4f88737700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4f80108360 0x7f4f80198910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:44.837 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.832+0000 7f4f88737700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f4f801125b0 con 0x7f4f8006a960 2024-03-19T01:39:44.837 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.832+0000 7f4f88737700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f4f80112430 con 0x7f4f80108360 2024-03-19T01:39:44.838 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.832+0000 7f4f88737700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f4f80112730 con 0x7f4f80069fc0 2024-03-19T01:39:44.838 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.832+0000 7f4f86cd4700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4f80108360 0x7f4f80198910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:44.838 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.832+0000 7f4f85cd2700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4f8006a960 0x7f4f8019ec40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:44.838 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.832+0000 7f4f864d3700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4f80069fc0 0x7f4f8019e700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:44.838 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.832+0000 7f4f86cd4700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4f80108360 0x7f4f80198910 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:44694/0 (socket says 172.21.15.99:44694) 2024-03-19T01:39:44.838 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.832+0000 7f4f85cd2700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4f8006a960 0x7f4f8019ec40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:52718/0 (socket says 172.21.15.99:52718) 2024-03-19T01:39:44.838 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.832+0000 7f4f86cd4700 1 -- 172.21.15.99:0/2429696883 learned_addr learned my addr 172.21.15.99:0/2429696883 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:39:44.839 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.832+0000 7f4f864d3700 1 --2- 172.21.15.99:0/2429696883 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4f80069fc0 0x7f4f8019e700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:57670/0 (socket says 172.21.15.99:57670) 2024-03-19T01:39:44.839 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.832+0000 7f4f86cd4700 1 -- 172.21.15.99:0/2429696883 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4f80069fc0 msgr2=0x7f4f8019e700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:44.839 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.832+0000 7f4f86cd4700 1 --2- 172.21.15.99:0/2429696883 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4f80069fc0 0x7f4f8019e700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:44.839 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.832+0000 7f4f86cd4700 1 -- 172.21.15.99:0/2429696883 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4f8006a960 msgr2=0x7f4f8019ec40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:44.839 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.832+0000 7f4f86cd4700 1 --2- 172.21.15.99:0/2429696883 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4f8006a960 0x7f4f8019ec40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:44.839 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.832+0000 7f4f86cd4700 1 -- 172.21.15.99:0/2429696883 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f4f80199070 con 0x7f4f80108360 2024-03-19T01:39:44.839 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.832+0000 7f4f85cd2700 1 --2- 172.21.15.99:0/2429696883 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4f8006a960 0x7f4f8019ec40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:39:44.839 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.833+0000 7f4f86cd4700 1 --2- 172.21.15.99:0/2429696883 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4f80108360 0x7f4f80198910 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7f4f7800d010 tx=0x7f4f7800bca0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:44.840 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.833+0000 7f4f777fe700 1 -- 172.21.15.99:0/2429696883 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4f78021070 con 0x7f4f80108360 2024-03-19T01:39:44.840 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.833+0000 7f4f777fe700 1 -- 172.21.15.99:0/2429696883 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f4f7801d450 con 0x7f4f80108360 2024-03-19T01:39:44.840 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.833+0000 7f4f777fe700 1 -- 172.21.15.99:0/2429696883 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4f780052a0 con 0x7f4f80108360 2024-03-19T01:39:44.840 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.833+0000 7f4f88737700 1 -- 172.21.15.99:0/2429696883 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f4f80199300 con 0x7f4f80108360 2024-03-19T01:39:44.840 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.833+0000 7f4f88737700 1 -- 172.21.15.99:0/2429696883 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f4f80101210 con 0x7f4f80108360 2024-03-19T01:39:44.840 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.834+0000 7f4f88737700 1 -- 172.21.15.99:0/2429696883 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f4f80103a10 con 0x7f4f80108360 2024-03-19T01:39:44.840 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.835+0000 7f4f777fe700 1 -- 172.21.15.99:0/2429696883 <== mon.2 v2:172.21.15.112:3300/0 4 ==== mgrmap(e 17) ==== 137341+0+0 (secure 0 0 0) 0x7f4f78024070 con 0x7f4f80108360 2024-03-19T01:39:44.840 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.835+0000 7f4f777fe700 1 --2- 172.21.15.99:0/2429696883 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f4f6c0a29a0 0x7f4f6c0a4e60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:44.840 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.836+0000 7f4f864d3700 1 --2- 172.21.15.99:0/2429696883 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f4f6c0a29a0 0x7f4f6c0a4e60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:44.841 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.836+0000 7f4f777fe700 1 -- 172.21.15.99:0/2429696883 <== mon.2 v2:172.21.15.112:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8269+0+0 (secure 0 0 0) 0x7f4f780cfe10 con 0x7f4f80108360 2024-03-19T01:39:44.841 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.836+0000 7f4f864d3700 1 --2- 172.21.15.99:0/2429696883 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f4f6c0a29a0 0x7f4f6c0a4e60 secure :-1 s=READY pgs=194 cs=0 l=1 rev1=1 crypto rx=0x7f4f70009750 tx=0x7f4f70009310 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:44.841 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.839+0000 7f4f777fe700 1 -- 172.21.15.99:0/2429696883 <== mon.2 v2:172.21.15.112:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f4f7809cd60 con 0x7f4f80108360 2024-03-19T01:39:45.055 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:45.055+0000 7f4f88737700 1 -- 172.21.15.99:0/2429696883 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 4} v 0) -- 0x7f4f8002dbb0 con 0x7f4f80108360 2024-03-19T01:39:45.055 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:45.055+0000 7f4f777fe700 1 -- 172.21.15.99:0/2429696883 <== mon.2 v2:172.21.15.112:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 4}]=0 v0) ==== 74+0+13 (secure 0 0 0) 0x7f4f7809c660 con 0x7f4f80108360 2024-03-19T01:39:45.055 INFO:teuthology.orchestra.run.smithi099.stdout:146028888120 2024-03-19T01:39:45.057 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:45.057+0000 7f4f88737700 1 -- 172.21.15.99:0/2429696883 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f4f6c0a29a0 msgr2=0x7f4f6c0a4e60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:45.058 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:45.057+0000 7f4f88737700 1 --2- 172.21.15.99:0/2429696883 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f4f6c0a29a0 0x7f4f6c0a4e60 secure :-1 s=READY pgs=194 cs=0 l=1 rev1=1 crypto rx=0x7f4f70009750 tx=0x7f4f70009310 comp rx=0 tx=0).stop 2024-03-19T01:39:45.058 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:45.057+0000 7f4f88737700 1 -- 172.21.15.99:0/2429696883 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4f80108360 msgr2=0x7f4f80198910 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:45.058 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:45.057+0000 7f4f88737700 1 --2- 172.21.15.99:0/2429696883 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4f80108360 0x7f4f80198910 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7f4f7800d010 tx=0x7f4f7800bca0 comp rx=0 tx=0).stop 2024-03-19T01:39:45.058 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:45.057+0000 7f4f88737700 1 -- 172.21.15.99:0/2429696883 shutdown_connections 2024-03-19T01:39:45.058 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:45.057+0000 7f4f88737700 1 --2- 172.21.15.99:0/2429696883 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f4f6c0a29a0 0x7f4f6c0a4e60 unknown :-1 s=CLOSED pgs=194 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:45.058 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:45.057+0000 7f4f88737700 1 --2- 172.21.15.99:0/2429696883 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4f80069fc0 0x7f4f8019e700 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:45.058 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:45.057+0000 7f4f88737700 1 --2- 172.21.15.99:0/2429696883 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4f80108360 0x7f4f80198910 unknown :-1 s=CLOSED pgs=59 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:45.058 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:45.057+0000 7f4f88737700 1 --2- 172.21.15.99:0/2429696883 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4f8006a960 0x7f4f8019ec40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:45.058 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:45.057+0000 7f4f88737700 1 -- 172.21.15.99:0/2429696883 >> 172.21.15.99:0/2429696883 conn(0x7f4f80078140 msgr2=0x7f4f800ffe20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:45.058 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:45.057+0000 7f4f88737700 1 -- 172.21.15.99:0/2429696883 shutdown_connections 2024-03-19T01:39:45.058 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:45.057+0000 7f4f88737700 1 -- 172.21.15.99:0/2429696883 wait complete. 2024-03-19T01:39:45.897 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:45 smithi099 ceph-mon[62951]: pgmap v308: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-03-19T01:39:45.897 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:45 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/2440021828' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 10}]: dispatch 2024-03-19T01:39:45.897 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:45 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/2429696883' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-03-19T01:39:46.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:45 smithi112 ceph-mon[66819]: pgmap v308: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-03-19T01:39:46.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:45 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/2440021828' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 10}]: dispatch 2024-03-19T01:39:46.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:45 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/2429696883' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-03-19T01:39:46.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:45 smithi115 ceph-mon[67332]: pgmap v308: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-03-19T01:39:46.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:45 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/2440021828' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 10}]: dispatch 2024-03-19T01:39:46.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:45 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/2429696883' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-03-19T01:39:47.053 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.732+0000 7f6793530700 1 -- 172.21.15.99:0/1281632832 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f678c101e50 msgr2=0x7f678c105e10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:47.053 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.732+0000 7f6793530700 1 --2- 172.21.15.99:0/1281632832 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f678c101e50 0x7f678c105e10 secure :-1 s=READY pgs=254 cs=0 l=1 rev1=1 crypto rx=0x7f677c0099c0 tx=0x7f677c009cd0 comp rx=0 tx=0).stop 2024-03-19T01:39:47.053 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.732+0000 7f6793530700 1 -- 172.21.15.99:0/1281632832 shutdown_connections 2024-03-19T01:39:47.053 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.732+0000 7f6793530700 1 --2- 172.21.15.99:0/1281632832 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f678c1064f0 0x7f678c10e0c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:47.053 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.732+0000 7f6793530700 1 --2- 172.21.15.99:0/1281632832 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f678c1014a0 0x7f678c101880 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:47.053 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.732+0000 7f6793530700 1 --2- 172.21.15.99:0/1281632832 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f678c101e50 0x7f678c105e10 unknown :-1 s=CLOSED pgs=254 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:47.053 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.732+0000 7f6793530700 1 -- 172.21.15.99:0/1281632832 >> 172.21.15.99:0/1281632832 conn(0x7f678c077260 msgr2=0x7f678c077670 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:47.054 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.732+0000 7f6793530700 1 -- 172.21.15.99:0/1281632832 shutdown_connections 2024-03-19T01:39:47.054 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.732+0000 7f6793530700 1 -- 172.21.15.99:0/1281632832 wait complete. 2024-03-19T01:39:47.054 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.733+0000 7f6793530700 1 Processor -- start 2024-03-19T01:39:47.054 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.733+0000 7f6793530700 1 -- start start 2024-03-19T01:39:47.054 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.733+0000 7f6793530700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f678c1014a0 0x7f678c1998b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:47.054 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.733+0000 7f6793530700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f678c101e50 0x7f678c199df0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:47.054 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.733+0000 7f6793530700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f678c1064f0 0x7f678c1a4930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:47.054 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.733+0000 7f6793530700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f678c1114d0 con 0x7f678c1064f0 2024-03-19T01:39:47.054 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.733+0000 7f6793530700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f678c111350 con 0x7f678c101e50 2024-03-19T01:39:47.054 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.733+0000 7f6793530700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f678c111650 con 0x7f678c1014a0 2024-03-19T01:39:47.054 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.733+0000 7f6791acd700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f678c1064f0 0x7f678c1a4930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:47.054 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.733+0000 7f6791acd700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f678c1064f0 0x7f678c1a4930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:52702/0 (socket says 172.21.15.99:52702) 2024-03-19T01:39:47.054 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.733+0000 7f6791acd700 1 -- 172.21.15.99:0/2440021828 learned_addr learned my addr 172.21.15.99:0/2440021828 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:39:47.055 INFO:teuthology.orchestra.run.smithi099.stdout:317827579932 2024-03-19T01:39:47.055 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.733+0000 7f6790acb700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f678c101e50 0x7f678c199df0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:47.055 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.733+0000 7f67912cc700 1 --2- 172.21.15.99:0/2440021828 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f678c1014a0 0x7f678c1998b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:47.055 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.733+0000 7f6791acd700 1 -- 172.21.15.99:0/2440021828 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f678c1014a0 msgr2=0x7f678c1998b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:47.055 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.733+0000 7f6791acd700 1 --2- 172.21.15.99:0/2440021828 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f678c1014a0 0x7f678c1998b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:47.055 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.733+0000 7f6791acd700 1 -- 172.21.15.99:0/2440021828 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f678c101e50 msgr2=0x7f678c199df0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:47.055 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.733+0000 7f6791acd700 1 --2- 172.21.15.99:0/2440021828 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f678c101e50 0x7f678c199df0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:47.056 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.733+0000 7f6791acd700 1 -- 172.21.15.99:0/2440021828 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f678c1a4e70 con 0x7f678c1064f0 2024-03-19T01:39:47.056 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.733+0000 7f6791acd700 1 --2- 172.21.15.99:0/2440021828 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f678c1064f0 0x7f678c1a4930 secure :-1 s=READY pgs=255 cs=0 l=1 rev1=1 crypto rx=0x7f678800e940 tx=0x7f678800ec50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:47.056 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.734+0000 7f67827fc700 1 -- 172.21.15.99:0/2440021828 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6788004a30 con 0x7f678c1064f0 2024-03-19T01:39:47.056 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.734+0000 7f67827fc700 1 -- 172.21.15.99:0/2440021828 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f6788005500 con 0x7f678c1064f0 2024-03-19T01:39:47.056 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.734+0000 7f67827fc700 1 -- 172.21.15.99:0/2440021828 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f678800ce20 con 0x7f678c1064f0 2024-03-19T01:39:47.056 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.734+0000 7f6793530700 1 -- 172.21.15.99:0/2440021828 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f678c1a5100 con 0x7f678c1064f0 2024-03-19T01:39:47.056 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.734+0000 7f6793530700 1 -- 172.21.15.99:0/2440021828 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f678c1a5610 con 0x7f678c1064f0 2024-03-19T01:39:47.056 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.735+0000 7f6793530700 1 -- 172.21.15.99:0/2440021828 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f678c068fd0 con 0x7f678c1064f0 2024-03-19T01:39:47.056 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.736+0000 7f67827fc700 1 -- 172.21.15.99:0/2440021828 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 17) ==== 137341+0+0 (secure 0 0 0) 0x7f6788005000 con 0x7f678c1064f0 2024-03-19T01:39:47.056 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.736+0000 7f67827fc700 1 --2- 172.21.15.99:0/2440021828 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f67780a2920 0x7f67780a4de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:47.056 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.737+0000 7f67912cc700 1 --2- 172.21.15.99:0/2440021828 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f67780a2920 0x7f67780a4de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:47.056 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.737+0000 7f67827fc700 1 -- 172.21.15.99:0/2440021828 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8269+0+0 (secure 0 0 0) 0x7f67880ceeb0 con 0x7f678c1064f0 2024-03-19T01:39:47.057 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.737+0000 7f67912cc700 1 --2- 172.21.15.99:0/2440021828 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f67780a2920 0x7f67780a4de0 secure :-1 s=READY pgs=193 cs=0 l=1 rev1=1 crypto rx=0x7f678400f540 tx=0x7f678400b040 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:47.057 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.744+0000 7f67827fc700 1 -- 172.21.15.99:0/2440021828 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f6788053740 con 0x7f678c1064f0 2024-03-19T01:39:47.057 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.954+0000 7f6793530700 1 -- 172.21.15.99:0/2440021828 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 10} v 0) -- 0x7f678c02dd70 con 0x7f678c1064f0 2024-03-19T01:39:47.057 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.955+0000 7f67827fc700 1 -- 172.21.15.99:0/2440021828 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 10}]=0 v0) ==== 75+0+13 (secure 0 0 0) 0x7f678809bdd0 con 0x7f678c1064f0 2024-03-19T01:39:47.057 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.957+0000 7f6793530700 1 -- 172.21.15.99:0/2440021828 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f67780a2920 msgr2=0x7f67780a4de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:47.057 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.957+0000 7f6793530700 1 --2- 172.21.15.99:0/2440021828 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f67780a2920 0x7f67780a4de0 secure :-1 s=READY pgs=193 cs=0 l=1 rev1=1 crypto rx=0x7f678400f540 tx=0x7f678400b040 comp rx=0 tx=0).stop 2024-03-19T01:39:47.057 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.957+0000 7f6793530700 1 -- 172.21.15.99:0/2440021828 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f678c1064f0 msgr2=0x7f678c1a4930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:47.057 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.957+0000 7f6793530700 1 --2- 172.21.15.99:0/2440021828 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f678c1064f0 0x7f678c1a4930 secure :-1 s=READY pgs=255 cs=0 l=1 rev1=1 crypto rx=0x7f678800e940 tx=0x7f678800ec50 comp rx=0 tx=0).stop 2024-03-19T01:39:47.057 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.957+0000 7f6793530700 1 -- 172.21.15.99:0/2440021828 shutdown_connections 2024-03-19T01:39:47.057 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.957+0000 7f6793530700 1 --2- 172.21.15.99:0/2440021828 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f67780a2920 0x7f67780a4de0 unknown :-1 s=CLOSED pgs=193 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:47.057 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.957+0000 7f6793530700 1 --2- 172.21.15.99:0/2440021828 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f678c1014a0 0x7f678c1998b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:47.057 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.957+0000 7f6793530700 1 --2- 172.21.15.99:0/2440021828 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f678c1064f0 0x7f678c1a4930 unknown :-1 s=CLOSED pgs=255 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:47.057 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.957+0000 7f6793530700 1 --2- 172.21.15.99:0/2440021828 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f678c101e50 0x7f678c199df0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:47.057 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.957+0000 7f6793530700 1 -- 172.21.15.99:0/2440021828 >> 172.21.15.99:0/2440021828 conn(0x7f678c077260 msgr2=0x7f678c0ff250 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:47.057 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.957+0000 7f6793530700 1 -- 172.21.15.99:0/2440021828 shutdown_connections 2024-03-19T01:39:47.058 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:44.957+0000 7f6793530700 1 -- 172.21.15.99:0/2440021828 wait complete. 2024-03-19T01:39:47.670 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:39:47.673 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:39:47.680 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:39:47.720 INFO:tasks.cephadm.ceph_manager.ceph:need seq 317827579929 got 317827579932 for osd.10 2024-03-19T01:39:47.720 DEBUG:teuthology.parallel:result is None 2024-03-19T01:39:48.002 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:47 smithi099 ceph-mon[62951]: pgmap v309: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 6 op/s 2024-03-19T01:39:48.108 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:47 smithi115 ceph-mon[67332]: pgmap v309: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 6 op/s 2024-03-19T01:39:48.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:47 smithi112 ceph-mon[66819]: pgmap v309: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 6 op/s 2024-03-19T01:39:48.585 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:48.582+0000 7f17f24ee700 1 -- 172.21.15.99:0/2232761906 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f17ec108360 msgr2=0x7f17ec108740 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:48.585 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:48.582+0000 7f17f24ee700 1 --2- 172.21.15.99:0/2232761906 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f17ec108360 0x7f17ec108740 secure :-1 s=READY pgs=256 cs=0 l=1 rev1=1 crypto rx=0x7f17e00099c0 tx=0x7f17e0009cd0 comp rx=0 tx=0).stop 2024-03-19T01:39:48.585 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:48.582+0000 7f17f24ee700 1 -- 172.21.15.99:0/2232761906 shutdown_connections 2024-03-19T01:39:48.585 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:48.582+0000 7f17f24ee700 1 --2- 172.21.15.99:0/2232761906 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f17ec06a960 0x7f17ec10f1a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:48.585 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:48.582+0000 7f17f24ee700 1 --2- 172.21.15.99:0/2232761906 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f17ec108360 0x7f17ec108740 unknown :-1 s=CLOSED pgs=256 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:48.585 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:48.582+0000 7f17f24ee700 1 --2- 172.21.15.99:0/2232761906 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f17ec069fc0 0x7f17ec06a420 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:48.585 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:48.582+0000 7f17f24ee700 1 -- 172.21.15.99:0/2232761906 >> 172.21.15.99:0/2232761906 conn(0x7f17ec078140 msgr2=0x7f17ec078550 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:48.585 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:48.582+0000 7f17f24ee700 1 -- 172.21.15.99:0/2232761906 shutdown_connections 2024-03-19T01:39:48.585 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:48.582+0000 7f17f24ee700 1 -- 172.21.15.99:0/2232761906 wait complete. 2024-03-19T01:39:48.585 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:48.583+0000 7f17f24ee700 1 Processor -- start 2024-03-19T01:39:48.586 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:48.583+0000 7f17f24ee700 1 -- start start 2024-03-19T01:39:48.586 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:48.583+0000 7f17f24ee700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f17ec069fc0 0x7f17ec1012e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:48.586 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:48.583+0000 7f17f24ee700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f17ec06a960 0x7f17ec101820 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:48.586 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:48.583+0000 7f17f24ee700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f17ec108360 0x7f17ec101d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:48.586 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:48.583+0000 7f17f24ee700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f17ec1125a0 con 0x7f17ec108360 2024-03-19T01:39:48.586 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:48.583+0000 7f17f24ee700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f17ec112420 con 0x7f17ec069fc0 2024-03-19T01:39:48.586 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:48.583+0000 7f17f24ee700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f17ec112720 con 0x7f17ec06a960 2024-03-19T01:39:48.586 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:48.583+0000 7f17ebfff700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f17ec069fc0 0x7f17ec1012e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:48.586 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:48.583+0000 7f17ebfff700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f17ec069fc0 0x7f17ec1012e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:44714/0 (socket says 172.21.15.99:44714) 2024-03-19T01:39:48.586 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:48.583+0000 7f17ebfff700 1 -- 172.21.15.99:0/1839783288 learned_addr learned my addr 172.21.15.99:0/1839783288 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:39:48.586 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:48.583+0000 7f17eb7fe700 1 --2- 172.21.15.99:0/1839783288 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f17ec06a960 0x7f17ec101820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:48.586 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:48.584+0000 7f17f0a8b700 1 --2- 172.21.15.99:0/1839783288 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f17ec108360 0x7f17ec101d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:48.587 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:48.584+0000 7f17ebfff700 1 -- 172.21.15.99:0/1839783288 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f17ec06a960 msgr2=0x7f17ec101820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:48.587 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:48.584+0000 7f17ebfff700 1 --2- 172.21.15.99:0/1839783288 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f17ec06a960 0x7f17ec101820 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:48.587 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:48.584+0000 7f17ebfff700 1 -- 172.21.15.99:0/1839783288 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f17ec108360 msgr2=0x7f17ec101d60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:48.587 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:48.584+0000 7f17ebfff700 1 --2- 172.21.15.99:0/1839783288 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f17ec108360 0x7f17ec101d60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:48.587 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:48.584+0000 7f17ebfff700 1 -- 172.21.15.99:0/1839783288 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f17ec1a6fd0 con 0x7f17ec069fc0 2024-03-19T01:39:48.587 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:48.584+0000 7f17f0a8b700 1 --2- 172.21.15.99:0/1839783288 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f17ec108360 0x7f17ec101d60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-19T01:39:48.587 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:48.584+0000 7f17ebfff700 1 --2- 172.21.15.99:0/1839783288 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f17ec069fc0 0x7f17ec1012e0 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7f17e00059d0 tx=0x7f17e00038b0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:48.587 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:48.584+0000 7f17e97fa700 1 -- 172.21.15.99:0/1839783288 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f17e0027070 con 0x7f17ec069fc0 2024-03-19T01:39:48.587 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:48.584+0000 7f17f24ee700 1 -- 172.21.15.99:0/1839783288 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f17ec1a7260 con 0x7f17ec069fc0 2024-03-19T01:39:48.587 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:48.584+0000 7f17e97fa700 1 -- 172.21.15.99:0/1839783288 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f17e0004050 con 0x7f17ec069fc0 2024-03-19T01:39:48.587 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:48.584+0000 7f17f24ee700 1 -- 172.21.15.99:0/1839783288 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f17ec1a7710 con 0x7f17ec069fc0 2024-03-19T01:39:48.587 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:48.584+0000 7f17e97fa700 1 -- 172.21.15.99:0/1839783288 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f17e0022d50 con 0x7f17ec069fc0 2024-03-19T01:39:48.588 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:48.585+0000 7f17f24ee700 1 -- 172.21.15.99:0/1839783288 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f17ec068fd0 con 0x7f17ec069fc0 2024-03-19T01:39:48.588 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:48.586+0000 7f17e97fa700 1 -- 172.21.15.99:0/1839783288 <== mon.2 v2:172.21.15.112:3300/0 4 ==== mgrmap(e 17) ==== 137341+0+0 (secure 0 0 0) 0x7f17e002a070 con 0x7f17ec069fc0 2024-03-19T01:39:48.589 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:48.587+0000 7f17e97fa700 1 --2- 172.21.15.99:0/1839783288 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f17d40a2920 0x7f17d40a4de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:48.589 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:48.587+0000 7f17eb7fe700 1 --2- 172.21.15.99:0/1839783288 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f17d40a2920 0x7f17d40a4de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:48.589 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:48.587+0000 7f17e97fa700 1 -- 172.21.15.99:0/1839783288 <== mon.2 v2:172.21.15.112:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8269+0+0 (secure 0 0 0) 0x7f17e00cfc70 con 0x7f17ec069fc0 2024-03-19T01:39:48.589 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:48.587+0000 7f17eb7fe700 1 --2- 172.21.15.99:0/1839783288 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f17d40a2920 0x7f17d40a4de0 secure :-1 s=READY pgs=195 cs=0 l=1 rev1=1 crypto rx=0x7f17ec0787b0 tx=0x7f17dc0073d0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:48.590 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:48.590+0000 7f17e97fa700 1 -- 172.21.15.99:0/1839783288 <== mon.2 v2:172.21.15.112:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f17e0013070 con 0x7f17ec069fc0 2024-03-19T01:39:48.798 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:48.798+0000 7f17f24ee700 1 -- 172.21.15.99:0/1839783288 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 6} v 0) -- 0x7f17ec02dbb0 con 0x7f17ec069fc0 2024-03-19T01:39:48.799 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:48.798+0000 7f17e97fa700 1 -- 172.21.15.99:0/1839783288 <== mon.2 v2:172.21.15.112:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 6}]=0 v0) ==== 74+0+13 (secure 0 0 0) 0x7f17e009ccd0 con 0x7f17ec069fc0 2024-03-19T01:39:48.801 INFO:teuthology.orchestra.run.smithi099.stdout:201863462960 2024-03-19T01:39:48.801 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:48.801+0000 7f17f24ee700 1 -- 172.21.15.99:0/1839783288 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f17d40a2920 msgr2=0x7f17d40a4de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:48.801 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:48.801+0000 7f17f24ee700 1 --2- 172.21.15.99:0/1839783288 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f17d40a2920 0x7f17d40a4de0 secure :-1 s=READY pgs=195 cs=0 l=1 rev1=1 crypto rx=0x7f17ec0787b0 tx=0x7f17dc0073d0 comp rx=0 tx=0).stop 2024-03-19T01:39:48.801 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:48.801+0000 7f17f24ee700 1 -- 172.21.15.99:0/1839783288 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f17ec069fc0 msgr2=0x7f17ec1012e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:48.801 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:48.801+0000 7f17f24ee700 1 --2- 172.21.15.99:0/1839783288 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f17ec069fc0 0x7f17ec1012e0 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7f17e00059d0 tx=0x7f17e00038b0 comp rx=0 tx=0).stop 2024-03-19T01:39:48.803 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:48.801+0000 7f17f24ee700 1 -- 172.21.15.99:0/1839783288 shutdown_connections 2024-03-19T01:39:48.803 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:48.801+0000 7f17f24ee700 1 --2- 172.21.15.99:0/1839783288 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f17d40a2920 0x7f17d40a4de0 unknown :-1 s=CLOSED pgs=195 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:48.803 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:48.801+0000 7f17f24ee700 1 --2- 172.21.15.99:0/1839783288 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f17ec108360 0x7f17ec101d60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:48.804 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:48.801+0000 7f17f24ee700 1 --2- 172.21.15.99:0/1839783288 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f17ec069fc0 0x7f17ec1012e0 unknown :-1 s=CLOSED pgs=60 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:48.804 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:48.801+0000 7f17f24ee700 1 --2- 172.21.15.99:0/1839783288 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f17ec06a960 0x7f17ec101820 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:48.804 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:48.801+0000 7f17f24ee700 1 -- 172.21.15.99:0/1839783288 >> 172.21.15.99:0/1839783288 conn(0x7f17ec078140 msgr2=0x7f17ec0ffe60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:48.804 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:48.801+0000 7f17f24ee700 1 -- 172.21.15.99:0/1839783288 shutdown_connections 2024-03-19T01:39:48.804 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:48.801+0000 7f17f24ee700 1 -- 172.21.15.99:0/1839783288 wait complete. 2024-03-19T01:39:49.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:48 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/1839783288' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-03-19T01:39:49.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:48 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/1839783288' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-03-19T01:39:49.213 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:49.210+0000 7fe7db0d0700 1 -- 172.21.15.99:0/383635432 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe7d4104d90 msgr2=0x7fe7d41051f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:49.213 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:49.210+0000 7fe7db0d0700 1 --2- 172.21.15.99:0/383635432 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe7d4104d90 0x7fe7d41051f0 secure :-1 s=READY pgs=257 cs=0 l=1 rev1=1 crypto rx=0x7fe7cc00b0a0 tx=0x7fe7cc00b3b0 comp rx=0 tx=0).stop 2024-03-19T01:39:49.213 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:49.210+0000 7fe7db0d0700 1 -- 172.21.15.99:0/383635432 shutdown_connections 2024-03-19T01:39:49.213 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:49.210+0000 7fe7db0d0700 1 --2- 172.21.15.99:0/383635432 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fe7d41072e0 0x7fe7d406aa90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:49.214 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:49.210+0000 7fe7db0d0700 1 --2- 172.21.15.99:0/383635432 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe7d4104d90 0x7fe7d41051f0 unknown :-1 s=CLOSED pgs=257 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:49.214 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:49.210+0000 7fe7db0d0700 1 --2- 172.21.15.99:0/383635432 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fe7d406afd0 0x7fe7d406b450 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:49.214 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:49.210+0000 7fe7db0d0700 1 -- 172.21.15.99:0/383635432 >> 172.21.15.99:0/383635432 conn(0x7fe7d4078270 msgr2=0x7fe7d4078680 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:49.214 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:49.211+0000 7fe7db0d0700 1 -- 172.21.15.99:0/383635432 shutdown_connections 2024-03-19T01:39:49.214 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:49.211+0000 7fe7db0d0700 1 -- 172.21.15.99:0/383635432 wait complete. 2024-03-19T01:39:49.214 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:49.211+0000 7fe7db0d0700 1 Processor -- start 2024-03-19T01:39:49.214 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:49.211+0000 7fe7db0d0700 1 -- start start 2024-03-19T01:39:49.214 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:49.211+0000 7fe7db0d0700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fe7d406afd0 0x7fe7d419f210 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:49.214 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:49.211+0000 7fe7db0d0700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe7d4104d90 0x7fe7d419f750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:49.214 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:49.211+0000 7fe7db0d0700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fe7d41072e0 0x7fe7d41a3ae0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:49.214 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:49.211+0000 7fe7db0d0700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fe7d4118cc0 con 0x7fe7d4104d90 2024-03-19T01:39:49.214 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:49.211+0000 7fe7db0d0700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fe7d4118b40 con 0x7fe7d406afd0 2024-03-19T01:39:49.215 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:49.211+0000 7fe7db0d0700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fe7d4118e40 con 0x7fe7d41072e0 2024-03-19T01:39:49.215 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:49.211+0000 7fe7d3fff700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe7d4104d90 0x7fe7d419f750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:49.215 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:49.211+0000 7fe7d3fff700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe7d4104d90 0x7fe7d419f750 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:52756/0 (socket says 172.21.15.99:52756) 2024-03-19T01:39:49.215 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:49.211+0000 7fe7d3fff700 1 -- 172.21.15.99:0/3630493165 learned_addr learned my addr 172.21.15.99:0/3630493165 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:39:49.215 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:49.211+0000 7fe7d8e6c700 1 --2- 172.21.15.99:0/3630493165 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fe7d406afd0 0x7fe7d419f210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:49.215 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:49.212+0000 7fe7d966d700 1 --2- 172.21.15.99:0/3630493165 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fe7d41072e0 0x7fe7d41a3ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:49.215 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:49.212+0000 7fe7d3fff700 1 -- 172.21.15.99:0/3630493165 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fe7d41072e0 msgr2=0x7fe7d41a3ae0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:49.215 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:49.212+0000 7fe7d3fff700 1 --2- 172.21.15.99:0/3630493165 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fe7d41072e0 0x7fe7d41a3ae0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:49.215 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:49.212+0000 7fe7d3fff700 1 -- 172.21.15.99:0/3630493165 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fe7d406afd0 msgr2=0x7fe7d419f210 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:49.215 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:49.212+0000 7fe7d3fff700 1 --2- 172.21.15.99:0/3630493165 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fe7d406afd0 0x7fe7d419f210 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:49.215 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:49.212+0000 7fe7d3fff700 1 -- 172.21.15.99:0/3630493165 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fe7d41a4220 con 0x7fe7d4104d90 2024-03-19T01:39:49.215 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:49.212+0000 7fe7d8e6c700 1 --2- 172.21.15.99:0/3630493165 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fe7d406afd0 0x7fe7d419f210 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:39:49.215 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:49.212+0000 7fe7d3fff700 1 --2- 172.21.15.99:0/3630493165 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe7d4104d90 0x7fe7d419f750 secure :-1 s=READY pgs=258 cs=0 l=1 rev1=1 crypto rx=0x7fe7c000e9a0 tx=0x7fe7c000ecb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:49.215 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:49.212+0000 7fe7d1ffb700 1 -- 172.21.15.99:0/3630493165 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe7c000ca10 con 0x7fe7d4104d90 2024-03-19T01:39:49.216 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:49.212+0000 7fe7db0d0700 1 -- 172.21.15.99:0/3630493165 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fe7d41a4510 con 0x7fe7d4104d90 2024-03-19T01:39:49.216 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:49.212+0000 7fe7d1ffb700 1 -- 172.21.15.99:0/3630493165 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe7c000cbb0 con 0x7fe7d4104d90 2024-03-19T01:39:49.216 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:49.212+0000 7fe7d1ffb700 1 -- 172.21.15.99:0/3630493165 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe7c0018680 con 0x7fe7d4104d90 2024-03-19T01:39:49.216 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:49.212+0000 7fe7db0d0700 1 -- 172.21.15.99:0/3630493165 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fe7d41abd90 con 0x7fe7d4104d90 2024-03-19T01:39:49.216 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:49.213+0000 7fe7db0d0700 1 -- 172.21.15.99:0/3630493165 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fe7d4068fd0 con 0x7fe7d4104d90 2024-03-19T01:39:49.217 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:49.214+0000 7fe7d1ffb700 1 -- 172.21.15.99:0/3630493165 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 17) ==== 137341+0+0 (secure 0 0 0) 0x7fe7c0010430 con 0x7fe7d4104d90 2024-03-19T01:39:49.217 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:49.215+0000 7fe7d1ffb700 1 --2- 172.21.15.99:0/3630493165 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fe7c40a2920 0x7fe7c40a4de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:49.217 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:49.215+0000 7fe7d8e6c700 1 --2- 172.21.15.99:0/3630493165 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fe7c40a2920 0x7fe7c40a4de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:49.217 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:49.215+0000 7fe7d1ffb700 1 -- 172.21.15.99:0/3630493165 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8269+0+0 (secure 0 0 0) 0x7fe7c00cf2e0 con 0x7fe7d4104d90 2024-03-19T01:39:49.217 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:49.215+0000 7fe7d8e6c700 1 --2- 172.21.15.99:0/3630493165 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fe7c40a2920 0x7fe7c40a4de0 secure :-1 s=READY pgs=196 cs=0 l=1 rev1=1 crypto rx=0x7fe7c8009fd0 tx=0x7fe7c8009260 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:49.219 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:49.219+0000 7fe7d1ffb700 1 -- 172.21.15.99:0/3630493165 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7fe7c009c200 con 0x7fe7d4104d90 2024-03-19T01:39:49.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:48 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/1839783288' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-03-19T01:39:49.420 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:49.420+0000 7fe7db0d0700 1 -- 172.21.15.99:0/3630493165 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 7} v 0) -- 0x7fe7d41abf60 con 0x7fe7d4104d90 2024-03-19T01:39:49.421 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:49.420+0000 7fe7d1ffb700 1 -- 172.21.15.99:0/3630493165 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 7}]=0 v0) ==== 74+0+13 (secure 0 0 0) 0x7fe7c009bb00 con 0x7fe7d4104d90 2024-03-19T01:39:49.421 INFO:teuthology.orchestra.run.smithi099.stdout:227633266732 2024-03-19T01:39:49.423 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:49.422+0000 7fe7db0d0700 1 -- 172.21.15.99:0/3630493165 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fe7c40a2920 msgr2=0x7fe7c40a4de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:49.423 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:49.422+0000 7fe7db0d0700 1 --2- 172.21.15.99:0/3630493165 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fe7c40a2920 0x7fe7c40a4de0 secure :-1 s=READY pgs=196 cs=0 l=1 rev1=1 crypto rx=0x7fe7c8009fd0 tx=0x7fe7c8009260 comp rx=0 tx=0).stop 2024-03-19T01:39:49.423 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:49.422+0000 7fe7db0d0700 1 -- 172.21.15.99:0/3630493165 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe7d4104d90 msgr2=0x7fe7d419f750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:49.423 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:49.422+0000 7fe7db0d0700 1 --2- 172.21.15.99:0/3630493165 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe7d4104d90 0x7fe7d419f750 secure :-1 s=READY pgs=258 cs=0 l=1 rev1=1 crypto rx=0x7fe7c000e9a0 tx=0x7fe7c000ecb0 comp rx=0 tx=0).stop 2024-03-19T01:39:49.423 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:49.422+0000 7fe7db0d0700 1 -- 172.21.15.99:0/3630493165 shutdown_connections 2024-03-19T01:39:49.423 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:49.422+0000 7fe7db0d0700 1 --2- 172.21.15.99:0/3630493165 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fe7c40a2920 0x7fe7c40a4de0 unknown :-1 s=CLOSED pgs=196 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:49.423 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:49.422+0000 7fe7db0d0700 1 --2- 172.21.15.99:0/3630493165 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fe7d41072e0 0x7fe7d41a3ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:49.423 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:49.422+0000 7fe7db0d0700 1 --2- 172.21.15.99:0/3630493165 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fe7d406afd0 0x7fe7d419f210 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:49.424 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:49.422+0000 7fe7db0d0700 1 --2- 172.21.15.99:0/3630493165 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe7d4104d90 0x7fe7d419f750 unknown :-1 s=CLOSED pgs=258 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:49.424 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:49.422+0000 7fe7db0d0700 1 -- 172.21.15.99:0/3630493165 >> 172.21.15.99:0/3630493165 conn(0x7fe7d4078270 msgr2=0x7fe7d41001e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:49.424 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:49.423+0000 7fe7db0d0700 1 -- 172.21.15.99:0/3630493165 shutdown_connections 2024-03-19T01:39:49.424 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:49.423+0000 7fe7db0d0700 1 -- 172.21.15.99:0/3630493165 wait complete. 2024-03-19T01:39:50.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:49 smithi099 ceph-mon[62951]: pgmap v310: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 6 op/s 2024-03-19T01:39:50.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:49 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/3630493165' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-03-19T01:39:50.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:49 smithi112 ceph-mon[66819]: pgmap v310: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 6 op/s 2024-03-19T01:39:50.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:49 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/3630493165' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-03-19T01:39:50.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:49 smithi115 ceph-mon[67332]: pgmap v310: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 6 op/s 2024-03-19T01:39:50.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:49 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/3630493165' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-03-19T01:39:51.017 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.016+0000 7fcc02b95700 1 -- 172.21.15.99:0/1641334698 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fcbfc108340 msgr2=0x7fcbfc108720 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:51.017 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.016+0000 7fcc02b95700 1 --2- 172.21.15.99:0/1641334698 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fcbfc108340 0x7fcbfc108720 secure :-1 s=READY pgs=259 cs=0 l=1 rev1=1 crypto rx=0x7fcbf00099c0 tx=0x7fcbf0009cd0 comp rx=0 tx=0).stop 2024-03-19T01:39:51.017 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.016+0000 7fcc02b95700 1 -- 172.21.15.99:0/1641334698 shutdown_connections 2024-03-19T01:39:51.018 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.016+0000 7fcc02b95700 1 --2- 172.21.15.99:0/1641334698 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fcbfc102d20 0x7fcbfc10f160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:51.018 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.016+0000 7fcc02b95700 1 --2- 172.21.15.99:0/1641334698 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fcbfc108340 0x7fcbfc108720 unknown :-1 s=CLOSED pgs=259 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:51.018 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.016+0000 7fcc02b95700 1 --2- 172.21.15.99:0/1641334698 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fcbfc102360 0x7fcbfc1027e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:51.018 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.016+0000 7fcc02b95700 1 -- 172.21.15.99:0/1641334698 >> 172.21.15.99:0/1641334698 conn(0x7fcbfc079440 msgr2=0x7fcbfc079850 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:51.018 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.016+0000 7fcc02b95700 1 -- 172.21.15.99:0/1641334698 shutdown_connections 2024-03-19T01:39:51.018 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.016+0000 7fcc02b95700 1 -- 172.21.15.99:0/1641334698 wait complete. 2024-03-19T01:39:51.018 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.016+0000 7fcc02b95700 1 Processor -- start 2024-03-19T01:39:51.018 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.016+0000 7fcc02b95700 1 -- start start 2024-03-19T01:39:51.020 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.016+0000 7fcc02b95700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fcbfc102360 0x7fcbfc19e670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:51.020 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.016+0000 7fcc02b95700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fcbfc102d20 0x7fcbfc19ebb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:51.020 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.016+0000 7fcc02b95700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fcbfc108340 0x7fcbfc198840 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:51.020 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.016+0000 7fcc02b95700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fcbfc1125c0 con 0x7fcbfc108340 2024-03-19T01:39:51.020 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.016+0000 7fcc02b95700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fcbfc112440 con 0x7fcbfc102360 2024-03-19T01:39:51.020 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.016+0000 7fcc02b95700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fcbfc112740 con 0x7fcbfc102d20 2024-03-19T01:39:51.021 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.017+0000 7fcc00931700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fcbfc102360 0x7fcbfc19e670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:51.021 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.017+0000 7fcc01132700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fcbfc108340 0x7fcbfc198840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:51.021 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.017+0000 7fcc01132700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fcbfc108340 0x7fcbfc198840 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:52774/0 (socket says 172.21.15.99:52774) 2024-03-19T01:39:51.021 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.017+0000 7fcbfbfff700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fcbfc102d20 0x7fcbfc19ebb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:51.021 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.017+0000 7fcc00931700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fcbfc102360 0x7fcbfc19e670 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:44760/0 (socket says 172.21.15.99:44760) 2024-03-19T01:39:51.021 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.017+0000 7fcc01132700 1 -- 172.21.15.99:0/1195229782 learned_addr learned my addr 172.21.15.99:0/1195229782 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:39:51.021 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.017+0000 7fcc01132700 1 -- 172.21.15.99:0/1195229782 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fcbfc102d20 msgr2=0x7fcbfc19ebb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:51.021 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.017+0000 7fcc01132700 1 --2- 172.21.15.99:0/1195229782 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fcbfc102d20 0x7fcbfc19ebb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:51.021 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.017+0000 7fcc01132700 1 -- 172.21.15.99:0/1195229782 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fcbfc102360 msgr2=0x7fcbfc19e670 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:51.021 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.017+0000 7fcc01132700 1 --2- 172.21.15.99:0/1195229782 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fcbfc102360 0x7fcbfc19e670 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:51.021 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.017+0000 7fcc01132700 1 -- 172.21.15.99:0/1195229782 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fcbfc199020 con 0x7fcbfc108340 2024-03-19T01:39:51.022 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.017+0000 7fcc00931700 1 --2- 172.21.15.99:0/1195229782 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fcbfc102360 0x7fcbfc19e670 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:39:51.022 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.017+0000 7fcbfbfff700 1 --2- 172.21.15.99:0/1195229782 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fcbfc102d20 0x7fcbfc19ebb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:39:51.022 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.017+0000 7fcc01132700 1 --2- 172.21.15.99:0/1195229782 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fcbfc108340 0x7fcbfc198840 secure :-1 s=READY pgs=260 cs=0 l=1 rev1=1 crypto rx=0x7fcbf400ef30 tx=0x7fcbf400c3a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:51.022 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.017+0000 7fcbf9ffb700 1 -- 172.21.15.99:0/1195229782 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fcbf40092e0 con 0x7fcbfc108340 2024-03-19T01:39:51.022 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.017+0000 7fcbf9ffb700 1 -- 172.21.15.99:0/1195229782 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fcbf4004bb0 con 0x7fcbfc108340 2024-03-19T01:39:51.022 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.017+0000 7fcbf9ffb700 1 -- 172.21.15.99:0/1195229782 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fcbf4004280 con 0x7fcbfc108340 2024-03-19T01:39:51.022 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.017+0000 7fcc02b95700 1 -- 172.21.15.99:0/1195229782 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fcbfc1992b0 con 0x7fcbfc108340 2024-03-19T01:39:51.022 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.018+0000 7fcc02b95700 1 -- 172.21.15.99:0/1195229782 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fcbfc1a54d0 con 0x7fcbfc108340 2024-03-19T01:39:51.022 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.019+0000 7fcc02b95700 1 -- 172.21.15.99:0/1195229782 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fcbfc1039f0 con 0x7fcbfc108340 2024-03-19T01:39:51.023 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.020+0000 7fcbf9ffb700 1 -- 172.21.15.99:0/1195229782 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 17) ==== 137341+0+0 (secure 0 0 0) 0x7fcbf401b030 con 0x7fcbfc108340 2024-03-19T01:39:51.023 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.020+0000 7fcbf9ffb700 1 --2- 172.21.15.99:0/1195229782 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fcbec0a29a0 0x7fcbec0a4e60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:51.023 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.020+0000 7fcc00931700 1 --2- 172.21.15.99:0/1195229782 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fcbec0a29a0 0x7fcbec0a4e60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:51.023 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.020+0000 7fcbf9ffb700 1 -- 172.21.15.99:0/1195229782 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8269+0+0 (secure 0 0 0) 0x7fcbf40ce660 con 0x7fcbfc108340 2024-03-19T01:39:51.024 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.021+0000 7fcc00931700 1 --2- 172.21.15.99:0/1195229782 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fcbec0a29a0 0x7fcbec0a4e60 secure :-1 s=READY pgs=197 cs=0 l=1 rev1=1 crypto rx=0x7fcbf0000c00 tx=0x7fcbf0005fb0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:51.026 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.025+0000 7fcbf9ffb700 1 -- 172.21.15.99:0/1195229782 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7fcbf4022020 con 0x7fcbfc108340 2024-03-19T01:39:51.229 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.227+0000 7fcc02b95700 1 -- 172.21.15.99:0/1195229782 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 9} v 0) -- 0x7fcbfc02dbb0 con 0x7fcbfc108340 2024-03-19T01:39:51.229 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.227+0000 7fcbf9ffb700 1 -- 172.21.15.99:0/1195229782 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 9}]=0 v0) ==== 74+0+13 (secure 0 0 0) 0x7fcbf4010040 con 0x7fcbfc108340 2024-03-19T01:39:51.229 INFO:teuthology.orchestra.run.smithi099.stdout:292057776162 2024-03-19T01:39:51.231 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.230+0000 7fcc02b95700 1 -- 172.21.15.99:0/1195229782 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fcbec0a29a0 msgr2=0x7fcbec0a4e60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:51.231 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.230+0000 7fcc02b95700 1 --2- 172.21.15.99:0/1195229782 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fcbec0a29a0 0x7fcbec0a4e60 secure :-1 s=READY pgs=197 cs=0 l=1 rev1=1 crypto rx=0x7fcbf0000c00 tx=0x7fcbf0005fb0 comp rx=0 tx=0).stop 2024-03-19T01:39:51.231 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.230+0000 7fcc02b95700 1 -- 172.21.15.99:0/1195229782 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fcbfc108340 msgr2=0x7fcbfc198840 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:51.231 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.230+0000 7fcc02b95700 1 --2- 172.21.15.99:0/1195229782 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fcbfc108340 0x7fcbfc198840 secure :-1 s=READY pgs=260 cs=0 l=1 rev1=1 crypto rx=0x7fcbf400ef30 tx=0x7fcbf400c3a0 comp rx=0 tx=0).stop 2024-03-19T01:39:51.231 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.230+0000 7fcc02b95700 1 -- 172.21.15.99:0/1195229782 shutdown_connections 2024-03-19T01:39:51.231 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.230+0000 7fcc02b95700 1 --2- 172.21.15.99:0/1195229782 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fcbec0a29a0 0x7fcbec0a4e60 unknown :-1 s=CLOSED pgs=197 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:51.231 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.230+0000 7fcc02b95700 1 --2- 172.21.15.99:0/1195229782 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fcbfc108340 0x7fcbfc198840 unknown :-1 s=CLOSED pgs=260 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:51.231 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.230+0000 7fcc02b95700 1 --2- 172.21.15.99:0/1195229782 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fcbfc102360 0x7fcbfc19e670 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:51.231 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.230+0000 7fcc02b95700 1 --2- 172.21.15.99:0/1195229782 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fcbfc102d20 0x7fcbfc19ebb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:51.232 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.230+0000 7fcc02b95700 1 -- 172.21.15.99:0/1195229782 >> 172.21.15.99:0/1195229782 conn(0x7fcbfc079440 msgr2=0x7fcbfc100fa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:51.232 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.230+0000 7fcc02b95700 1 -- 172.21.15.99:0/1195229782 shutdown_connections 2024-03-19T01:39:51.232 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:51.230+0000 7fcc02b95700 1 -- 172.21.15.99:0/1195229782 wait complete. 2024-03-19T01:39:52.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:51 smithi112 ceph-mon[66819]: pgmap v311: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 6 op/s 2024-03-19T01:39:52.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:51 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/1195229782' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 9}]: dispatch 2024-03-19T01:39:52.226 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:51 smithi099 ceph-mon[62951]: pgmap v311: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 6 op/s 2024-03-19T01:39:52.226 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:51 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/1195229782' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 9}]: dispatch 2024-03-19T01:39:52.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:51 smithi115 ceph-mon[67332]: pgmap v311: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 6 op/s 2024-03-19T01:39:52.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:51 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/1195229782' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 9}]: dispatch 2024-03-19T01:39:53.143 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:53.139+0000 7f0a9b977700 1 -- 172.21.15.99:0/1906290739 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0a94101bb0 msgr2=0x7f0a9410e090 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:53.143 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:53.139+0000 7f0a9b977700 1 --2- 172.21.15.99:0/1906290739 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0a94101bb0 0x7f0a9410e090 secure :-1 s=READY pgs=261 cs=0 l=1 rev1=1 crypto rx=0x7f0a8c00b0a0 tx=0x7f0a8c00b3b0 comp rx=0 tx=0).stop 2024-03-19T01:39:53.143 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:53.139+0000 7f0a9b977700 1 -- 172.21.15.99:0/1906290739 shutdown_connections 2024-03-19T01:39:53.143 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:53.139+0000 7f0a9b977700 1 --2- 172.21.15.99:0/1906290739 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f0a94107250 0x7f0a94107630 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:53.143 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:53.139+0000 7f0a9b977700 1 --2- 172.21.15.99:0/1906290739 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0a94101bb0 0x7f0a9410e090 unknown :-1 s=CLOSED pgs=261 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:53.143 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:53.139+0000 7f0a9b977700 1 --2- 172.21.15.99:0/1906290739 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f0a94101210 0x7f0a94101670 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:53.143 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:53.139+0000 7f0a9b977700 1 -- 172.21.15.99:0/1906290739 >> 172.21.15.99:0/1906290739 conn(0x7f0a94078140 msgr2=0x7f0a94078550 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:53.143 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:53.139+0000 7f0a9b977700 1 -- 172.21.15.99:0/1906290739 shutdown_connections 2024-03-19T01:39:53.143 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:53.139+0000 7f0a9b977700 1 -- 172.21.15.99:0/1906290739 wait complete. 2024-03-19T01:39:53.144 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:53.140+0000 7f0a9b977700 1 Processor -- start 2024-03-19T01:39:53.144 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:53.140+0000 7f0a9b977700 1 -- start start 2024-03-19T01:39:53.144 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:53.140+0000 7f0a9b977700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f0a94101210 0x7f0a94199880 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:53.144 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:53.140+0000 7f0a9b977700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f0a94101bb0 0x7f0a94199dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:53.144 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:53.140+0000 7f0a9b977700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0a94107250 0x7f0a941a4900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:53.144 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:53.140+0000 7f0a9b977700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f0a941114a0 con 0x7f0a94107250 2024-03-19T01:39:53.144 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:53.140+0000 7f0a9b977700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f0a94111320 con 0x7f0a94101210 2024-03-19T01:39:53.144 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:53.140+0000 7f0a9b977700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f0a94111620 con 0x7f0a94101bb0 2024-03-19T01:39:53.144 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:53.140+0000 7f0a99713700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f0a94101210 0x7f0a94199880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:53.144 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:53.140+0000 7f0a99713700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f0a94101210 0x7f0a94199880 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:50460/0 (socket says 172.21.15.99:50460) 2024-03-19T01:39:53.144 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:53.140+0000 7f0a99713700 1 -- 172.21.15.99:0/3996451834 learned_addr learned my addr 172.21.15.99:0/3996451834 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:39:53.144 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:53.140+0000 7f0a98f12700 1 --2- 172.21.15.99:0/3996451834 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f0a94101bb0 0x7f0a94199dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:53.144 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:53.140+0000 7f0a99f14700 1 --2- 172.21.15.99:0/3996451834 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0a94107250 0x7f0a941a4900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:53.144 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:53.140+0000 7f0a99713700 1 -- 172.21.15.99:0/3996451834 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f0a94101bb0 msgr2=0x7f0a94199dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:53.145 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:53.140+0000 7f0a99713700 1 --2- 172.21.15.99:0/3996451834 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f0a94101bb0 0x7f0a94199dc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:53.145 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:53.140+0000 7f0a99713700 1 -- 172.21.15.99:0/3996451834 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0a94107250 msgr2=0x7f0a941a4900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:53.145 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:53.140+0000 7f0a99713700 1 --2- 172.21.15.99:0/3996451834 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0a94107250 0x7f0a941a4900 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:53.145 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:53.140+0000 7f0a99713700 1 -- 172.21.15.99:0/3996451834 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f0a941a4e40 con 0x7f0a94101210 2024-03-19T01:39:53.145 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:53.140+0000 7f0a98f12700 1 --2- 172.21.15.99:0/3996451834 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f0a94101bb0 0x7f0a94199dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:39:53.145 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:53.140+0000 7f0a99f14700 1 --2- 172.21.15.99:0/3996451834 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0a94107250 0x7f0a941a4900 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-19T01:39:53.145 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:53.141+0000 7f0a99713700 1 --2- 172.21.15.99:0/3996451834 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f0a94101210 0x7f0a94199880 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7f0a9000e9a0 tx=0x7f0a9000ecb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:53.145 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:53.141+0000 7f0a8a7fc700 1 -- 172.21.15.99:0/3996451834 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0a9000cbd0 con 0x7f0a94101210 2024-03-19T01:39:53.145 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:53.141+0000 7f0a9b977700 1 -- 172.21.15.99:0/3996451834 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f0a941a5070 con 0x7f0a94101210 2024-03-19T01:39:53.145 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:53.141+0000 7f0a8a7fc700 1 -- 172.21.15.99:0/3996451834 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f0a9000cd70 con 0x7f0a94101210 2024-03-19T01:39:53.145 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:53.141+0000 7f0a9b977700 1 -- 172.21.15.99:0/3996451834 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f0a941a55d0 con 0x7f0a94101210 2024-03-19T01:39:53.145 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:53.141+0000 7f0a8a7fc700 1 -- 172.21.15.99:0/3996451834 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0a90018710 con 0x7f0a94101210 2024-03-19T01:39:53.145 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:53.142+0000 7f0a9b977700 1 -- 172.21.15.99:0/3996451834 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f0a78005330 con 0x7f0a94101210 2024-03-19T01:39:53.146 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:53.143+0000 7f0a8a7fc700 1 -- 172.21.15.99:0/3996451834 <== mon.2 v2:172.21.15.112:3300/0 4 ==== mgrmap(e 17) ==== 137341+0+0 (secure 0 0 0) 0x7f0a900188b0 con 0x7f0a94101210 2024-03-19T01:39:53.146 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:53.143+0000 7f0a8a7fc700 1 --2- 172.21.15.99:0/3996451834 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f0a800a2920 0x7f0a800a4de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:53.146 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:53.143+0000 7f0a98f12700 1 --2- 172.21.15.99:0/3996451834 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f0a800a2920 0x7f0a800a4de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:53.146 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:53.144+0000 7f0a8a7fc700 1 -- 172.21.15.99:0/3996451834 <== mon.2 v2:172.21.15.112:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8269+0+0 (secure 0 0 0) 0x7f0a900cf2a0 con 0x7f0a94101210 2024-03-19T01:39:53.146 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:53.144+0000 7f0a98f12700 1 --2- 172.21.15.99:0/3996451834 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f0a800a2920 0x7f0a800a4de0 secure :-1 s=READY pgs=198 cs=0 l=1 rev1=1 crypto rx=0x7f0a84009680 tx=0x7f0a84009260 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:53.152 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:53.151+0000 7f0a8a7fc700 1 -- 172.21.15.99:0/3996451834 <== mon.2 v2:172.21.15.112:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f0a9009c160 con 0x7f0a94101210 2024-03-19T01:39:53.359 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:53.358+0000 7f0a9b977700 1 -- 172.21.15.99:0/3996451834 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 1} v 0) -- 0x7f0a780058f0 con 0x7f0a94101210 2024-03-19T01:39:53.360 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:53.358+0000 7f0a8a7fc700 1 -- 172.21.15.99:0/3996451834 <== mon.2 v2:172.21.15.112:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 1}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7f0a9009ba60 con 0x7f0a94101210 2024-03-19T01:39:53.360 INFO:teuthology.orchestra.run.smithi099.stdout:55834574921 2024-03-19T01:39:53.362 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:53.360+0000 7f0a9b977700 1 -- 172.21.15.99:0/3996451834 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f0a800a2920 msgr2=0x7f0a800a4de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:53.362 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:53.360+0000 7f0a9b977700 1 --2- 172.21.15.99:0/3996451834 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f0a800a2920 0x7f0a800a4de0 secure :-1 s=READY pgs=198 cs=0 l=1 rev1=1 crypto rx=0x7f0a84009680 tx=0x7f0a84009260 comp rx=0 tx=0).stop 2024-03-19T01:39:53.362 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:53.360+0000 7f0a9b977700 1 -- 172.21.15.99:0/3996451834 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f0a94101210 msgr2=0x7f0a94199880 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:53.362 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:53.360+0000 7f0a9b977700 1 --2- 172.21.15.99:0/3996451834 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f0a94101210 0x7f0a94199880 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7f0a9000e9a0 tx=0x7f0a9000ecb0 comp rx=0 tx=0).stop 2024-03-19T01:39:53.362 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:53.360+0000 7f0a9b977700 1 -- 172.21.15.99:0/3996451834 shutdown_connections 2024-03-19T01:39:53.362 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:53.360+0000 7f0a9b977700 1 --2- 172.21.15.99:0/3996451834 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f0a800a2920 0x7f0a800a4de0 unknown :-1 s=CLOSED pgs=198 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:53.362 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:53.360+0000 7f0a9b977700 1 --2- 172.21.15.99:0/3996451834 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0a94107250 0x7f0a941a4900 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:53.362 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:53.360+0000 7f0a9b977700 1 --2- 172.21.15.99:0/3996451834 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f0a94101210 0x7f0a94199880 unknown :-1 s=CLOSED pgs=61 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:53.363 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:53.360+0000 7f0a9b977700 1 --2- 172.21.15.99:0/3996451834 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f0a94101bb0 0x7f0a94199dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:53.363 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:53.361+0000 7f0a9b977700 1 -- 172.21.15.99:0/3996451834 >> 172.21.15.99:0/3996451834 conn(0x7f0a94078140 msgr2=0x7f0a940ffe20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:53.363 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:53.361+0000 7f0a9b977700 1 -- 172.21.15.99:0/3996451834 shutdown_connections 2024-03-19T01:39:53.363 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:53.361+0000 7f0a9b977700 1 -- 172.21.15.99:0/3996451834 wait complete. 2024-03-19T01:39:53.736 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462956 got 201863462960 for osd.6 2024-03-19T01:39:53.737 DEBUG:teuthology.parallel:result is None 2024-03-19T01:39:54.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:53 smithi099 ceph-mon[62951]: pgmap v312: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 6 op/s 2024-03-19T01:39:54.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:53 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/3996451834' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-03-19T01:39:54.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:53 smithi112 ceph-mon[66819]: pgmap v312: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 6 op/s 2024-03-19T01:39:54.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:53 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/3996451834' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-03-19T01:39:54.179 INFO:tasks.cephadm.ceph_manager.ceph:need seq 146028888117 got 146028888120 for osd.4 2024-03-19T01:39:54.179 DEBUG:teuthology.parallel:result is None 2024-03-19T01:39:54.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:53 smithi115 ceph-mon[67332]: pgmap v312: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 6 op/s 2024-03-19T01:39:54.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:53 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/3996451834' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-03-19T01:39:54.766 INFO:tasks.cephadm.ceph_manager.ceph:need seq 227633266728 got 227633266732 for osd.7 2024-03-19T01:39:54.767 DEBUG:teuthology.parallel:result is None 2024-03-19T01:39:55.050 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:54 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:39:55.247 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:54 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:39:55.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:54 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:39:55.343 INFO:tasks.cephadm.ceph_manager.ceph:need seq 292057776158 got 292057776162 for osd.9 2024-03-19T01:39:55.343 DEBUG:teuthology.parallel:result is None 2024-03-19T01:39:56.215 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:55 smithi099 ceph-mon[62951]: pgmap v313: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-03-19T01:39:56.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:55 smithi115 ceph-mon[67332]: pgmap v313: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-03-19T01:39:56.318 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:56.316+0000 7f8daae19700 1 -- 172.21.15.99:0/2906891022 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8da4106530 msgr2=0x7f8da410e100 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:56.318 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:56.316+0000 7f8daae19700 1 --2- 172.21.15.99:0/2906891022 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8da4106530 0x7f8da410e100 secure :-1 s=READY pgs=262 cs=0 l=1 rev1=1 crypto rx=0x7f8d9c00b0a0 tx=0x7f8d9c00b3b0 comp rx=0 tx=0).stop 2024-03-19T01:39:56.318 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:56.316+0000 7f8daae19700 1 -- 172.21.15.99:0/2906891022 shutdown_connections 2024-03-19T01:39:56.318 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:56.316+0000 7f8daae19700 1 --2- 172.21.15.99:0/2906891022 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8da41014a0 0x7f8da4101880 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:56.318 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:56.316+0000 7f8daae19700 1 --2- 172.21.15.99:0/2906891022 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8da4106530 0x7f8da410e100 unknown :-1 s=CLOSED pgs=262 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:56.319 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:56.316+0000 7f8daae19700 1 --2- 172.21.15.99:0/2906891022 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8da4101e50 0x7f8da4105e50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:56.319 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:56.316+0000 7f8daae19700 1 -- 172.21.15.99:0/2906891022 >> 172.21.15.99:0/2906891022 conn(0x7f8da4077230 msgr2=0x7f8da4077640 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:56.319 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:56.316+0000 7f8daae19700 1 -- 172.21.15.99:0/2906891022 shutdown_connections 2024-03-19T01:39:56.319 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:56.316+0000 7f8daae19700 1 -- 172.21.15.99:0/2906891022 wait complete. 2024-03-19T01:39:56.319 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:56.317+0000 7f8daae19700 1 Processor -- start 2024-03-19T01:39:56.319 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:56.317+0000 7f8daae19700 1 -- start start 2024-03-19T01:39:56.319 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:56.317+0000 7f8daae19700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8da41014a0 0x7f8da4197700 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:56.319 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:56.317+0000 7f8daae19700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8da4101e50 0x7f8da4197c40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:56.319 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:56.317+0000 7f8daae19700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8da4106530 0x7f8da4198180 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:56.319 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:56.317+0000 7f8daae19700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f8da4111510 con 0x7f8da4101e50 2024-03-19T01:39:56.319 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:56.317+0000 7f8daae19700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f8da4111390 con 0x7f8da41014a0 2024-03-19T01:39:56.319 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:56.317+0000 7f8daae19700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f8da4111690 con 0x7f8da4106530 2024-03-19T01:39:56.319 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:56.317+0000 7f8da8bb5700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8da41014a0 0x7f8da4197700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:56.319 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:56.317+0000 7f8da3fff700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8da4101e50 0x7f8da4197c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:56.320 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:56.317+0000 7f8da8bb5700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8da41014a0 0x7f8da4197700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:50486/0 (socket says 172.21.15.99:50486) 2024-03-19T01:39:56.320 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:56.317+0000 7f8da8bb5700 1 -- 172.21.15.99:0/1997330721 learned_addr learned my addr 172.21.15.99:0/1997330721 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:39:56.320 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:56.317+0000 7f8da93b6700 1 --2- 172.21.15.99:0/1997330721 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8da4106530 0x7f8da4198180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:56.320 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:56.317+0000 7f8da8bb5700 1 -- 172.21.15.99:0/1997330721 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8da4106530 msgr2=0x7f8da4198180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:56.320 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:56.317+0000 7f8da8bb5700 1 --2- 172.21.15.99:0/1997330721 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8da4106530 0x7f8da4198180 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:56.320 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:56.317+0000 7f8da8bb5700 1 -- 172.21.15.99:0/1997330721 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8da4101e50 msgr2=0x7f8da4197c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:56.320 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:56.317+0000 7f8da8bb5700 1 --2- 172.21.15.99:0/1997330721 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8da4101e50 0x7f8da4197c40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:56.320 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:56.317+0000 7f8da8bb5700 1 -- 172.21.15.99:0/1997330721 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f8da41a6f70 con 0x7f8da41014a0 2024-03-19T01:39:56.320 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:56.317+0000 7f8da3fff700 1 --2- 172.21.15.99:0/1997330721 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8da4101e50 0x7f8da4197c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:39:56.320 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:56.318+0000 7f8da8bb5700 1 --2- 172.21.15.99:0/1997330721 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8da41014a0 0x7f8da4197700 secure :-1 s=READY pgs=62 cs=0 l=1 rev1=1 crypto rx=0x7f8d9800e9c0 tx=0x7f8d9800ecd0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:56.320 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:56.318+0000 7f8da1ffb700 1 -- 172.21.15.99:0/1997330721 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8d9800cbd0 con 0x7f8da41014a0 2024-03-19T01:39:56.320 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:56.318+0000 7f8daae19700 1 -- 172.21.15.99:0/1997330721 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f8da41a7260 con 0x7f8da41014a0 2024-03-19T01:39:56.320 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:56.318+0000 7f8da1ffb700 1 -- 172.21.15.99:0/1997330721 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f8d9800cd70 con 0x7f8da41014a0 2024-03-19T01:39:56.320 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:56.318+0000 7f8da1ffb700 1 -- 172.21.15.99:0/1997330721 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8d98010430 con 0x7f8da41014a0 2024-03-19T01:39:56.321 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:56.318+0000 7f8daae19700 1 -- 172.21.15.99:0/1997330721 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f8da41a77a0 con 0x7f8da41014a0 2024-03-19T01:39:56.321 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:56.319+0000 7f8daae19700 1 -- 172.21.15.99:0/1997330721 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f8da4102930 con 0x7f8da41014a0 2024-03-19T01:39:56.321 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:56.320+0000 7f8da1ffb700 1 -- 172.21.15.99:0/1997330721 <== mon.2 v2:172.21.15.112:3300/0 4 ==== mgrmap(e 17) ==== 137341+0+0 (secure 0 0 0) 0x7f8d98004750 con 0x7f8da41014a0 2024-03-19T01:39:56.322 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:56.321+0000 7f8da1ffb700 1 --2- 172.21.15.99:0/1997330721 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f8d940a2920 0x7f8d940a4de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:56.322 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:56.321+0000 7f8da3fff700 1 --2- 172.21.15.99:0/1997330721 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f8d940a2920 0x7f8d940a4de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:56.322 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:56.321+0000 7f8da1ffb700 1 -- 172.21.15.99:0/1997330721 <== mon.2 v2:172.21.15.112:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8269+0+0 (secure 0 0 0) 0x7f8d980cf0f0 con 0x7f8da41014a0 2024-03-19T01:39:56.322 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:56.321+0000 7f8da3fff700 1 --2- 172.21.15.99:0/1997330721 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f8d940a2920 0x7f8d940a4de0 secure :-1 s=READY pgs=199 cs=0 l=1 rev1=1 crypto rx=0x7f8da4199860 tx=0x7f8d90009260 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:56.328 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:56.328+0000 7f8da1ffb700 1 -- 172.21.15.99:0/1997330721 <== mon.2 v2:172.21.15.112:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f8d98053740 con 0x7f8da41014a0 2024-03-19T01:39:56.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:55 smithi112 ceph-mon[66819]: pgmap v313: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-03-19T01:39:56.529 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:56.528+0000 7f8daae19700 1 -- 172.21.15.99:0/1997330721 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 2} v 0) -- 0x7f8da406b000 con 0x7f8da41014a0 2024-03-19T01:39:56.529 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:56.529+0000 7f8da1ffb700 1 -- 172.21.15.99:0/1997330721 <== mon.2 v2:172.21.15.112:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 2}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7f8d9809c010 con 0x7f8da41014a0 2024-03-19T01:39:56.530 INFO:teuthology.orchestra.run.smithi099.stdout:77309411396 2024-03-19T01:39:56.532 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:56.531+0000 7f8daae19700 1 -- 172.21.15.99:0/1997330721 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f8d940a2920 msgr2=0x7f8d940a4de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:56.532 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:56.531+0000 7f8daae19700 1 --2- 172.21.15.99:0/1997330721 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f8d940a2920 0x7f8d940a4de0 secure :-1 s=READY pgs=199 cs=0 l=1 rev1=1 crypto rx=0x7f8da4199860 tx=0x7f8d90009260 comp rx=0 tx=0).stop 2024-03-19T01:39:56.532 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:56.531+0000 7f8daae19700 1 -- 172.21.15.99:0/1997330721 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8da41014a0 msgr2=0x7f8da4197700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:56.532 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:56.531+0000 7f8daae19700 1 --2- 172.21.15.99:0/1997330721 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8da41014a0 0x7f8da4197700 secure :-1 s=READY pgs=62 cs=0 l=1 rev1=1 crypto rx=0x7f8d9800e9c0 tx=0x7f8d9800ecd0 comp rx=0 tx=0).stop 2024-03-19T01:39:56.532 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:56.531+0000 7f8daae19700 1 -- 172.21.15.99:0/1997330721 shutdown_connections 2024-03-19T01:39:56.532 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:56.531+0000 7f8daae19700 1 --2- 172.21.15.99:0/1997330721 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f8d940a2920 0x7f8d940a4de0 unknown :-1 s=CLOSED pgs=199 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:56.533 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:56.531+0000 7f8daae19700 1 --2- 172.21.15.99:0/1997330721 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8da4106530 0x7f8da4198180 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:56.533 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:56.531+0000 7f8daae19700 1 --2- 172.21.15.99:0/1997330721 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8da41014a0 0x7f8da4197700 unknown :-1 s=CLOSED pgs=62 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:56.533 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:56.531+0000 7f8daae19700 1 --2- 172.21.15.99:0/1997330721 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8da4101e50 0x7f8da4197c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:56.533 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:56.531+0000 7f8daae19700 1 -- 172.21.15.99:0/1997330721 >> 172.21.15.99:0/1997330721 conn(0x7f8da4077230 msgr2=0x7f8da40ff2f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:56.533 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:56.531+0000 7f8daae19700 1 -- 172.21.15.99:0/1997330721 shutdown_connections 2024-03-19T01:39:56.533 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:56.531+0000 7f8daae19700 1 -- 172.21.15.99:0/1997330721 wait complete. 2024-03-19T01:39:57.130 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:57 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:39:57.131 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:57 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:39:57.131 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:57 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:39:57.131 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:57 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:39:57.131 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:57 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/1997330721' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-03-19T01:39:57.131 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:57 smithi099 ceph-mon[62951]: pgmap v314: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 6 op/s 2024-03-19T01:39:57.132 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:57.132+0000 7ff53fb85700 1 -- 172.21.15.99:0/2525828900 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff538106520 msgr2=0x7ff53810e0f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:57.133 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:57.132+0000 7ff53fb85700 1 --2- 172.21.15.99:0/2525828900 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff538106520 0x7ff53810e0f0 secure :-1 s=READY pgs=263 cs=0 l=1 rev1=1 crypto rx=0x7ff53000b0a0 tx=0x7ff53000b3b0 comp rx=0 tx=0).stop 2024-03-19T01:39:57.133 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:57.132+0000 7ff53fb85700 1 -- 172.21.15.99:0/2525828900 shutdown_connections 2024-03-19T01:39:57.133 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:57.132+0000 7ff53fb85700 1 --2- 172.21.15.99:0/2525828900 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff538106520 0x7ff53810e0f0 unknown :-1 s=CLOSED pgs=263 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:57.133 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:57.132+0000 7ff53fb85700 1 --2- 172.21.15.99:0/2525828900 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff538101460 0x7ff538101840 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:57.133 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:57.132+0000 7ff53fb85700 1 --2- 172.21.15.99:0/2525828900 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff538101e10 0x7ff538105e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:57.133 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:57.132+0000 7ff53fb85700 1 -- 172.21.15.99:0/2525828900 >> 172.21.15.99:0/2525828900 conn(0x7ff538078180 msgr2=0x7ff538078590 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:57.133 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:57.132+0000 7ff53fb85700 1 -- 172.21.15.99:0/2525828900 shutdown_connections 2024-03-19T01:39:57.133 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:57.132+0000 7ff53fb85700 1 -- 172.21.15.99:0/2525828900 wait complete. 2024-03-19T01:39:57.135 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:57.133+0000 7ff53fb85700 1 Processor -- start 2024-03-19T01:39:57.135 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:57.133+0000 7ff53fb85700 1 -- start start 2024-03-19T01:39:57.135 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:57.133+0000 7ff53fb85700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff538101460 0x7ff53819e6d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:57.135 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:57.133+0000 7ff53fb85700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff538101e10 0x7ff53819ec10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:57.135 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:57.133+0000 7ff53fb85700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff538106520 0x7ff538198890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:57.135 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:57.133+0000 7ff53fb85700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7ff538111500 con 0x7ff538101e10 2024-03-19T01:39:57.135 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:57.133+0000 7ff53fb85700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7ff538111380 con 0x7ff538106520 2024-03-19T01:39:57.135 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:57.133+0000 7ff53fb85700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7ff538111680 con 0x7ff538101460 2024-03-19T01:39:57.135 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:57.133+0000 7ff53d120700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff538101e10 0x7ff53819ec10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:57.136 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:57.133+0000 7ff53d120700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff538101e10 0x7ff53819ec10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:50478/0 (socket says 172.21.15.99:50478) 2024-03-19T01:39:57.136 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:57.133+0000 7ff53d120700 1 -- 172.21.15.99:0/3664609717 learned_addr learned my addr 172.21.15.99:0/3664609717 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:39:57.136 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:57.133+0000 7ff53d921700 1 --2- 172.21.15.99:0/3664609717 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff538101460 0x7ff53819e6d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:57.136 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:57.133+0000 7ff53e122700 1 --2- 172.21.15.99:0/3664609717 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff538106520 0x7ff538198890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:57.136 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:57.134+0000 7ff53d120700 1 -- 172.21.15.99:0/3664609717 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff538101460 msgr2=0x7ff53819e6d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:57.136 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:57.134+0000 7ff53d120700 1 --2- 172.21.15.99:0/3664609717 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff538101460 0x7ff53819e6d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:57.136 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:57.134+0000 7ff53d120700 1 -- 172.21.15.99:0/3664609717 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff538106520 msgr2=0x7ff538198890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:57.136 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:57.134+0000 7ff53d120700 1 --2- 172.21.15.99:0/3664609717 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff538106520 0x7ff538198890 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:57.136 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:57.134+0000 7ff53d120700 1 -- 172.21.15.99:0/3664609717 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ff538198ff0 con 0x7ff538101e10 2024-03-19T01:39:57.136 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:57.134+0000 7ff53d120700 1 --2- 172.21.15.99:0/3664609717 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff538101e10 0x7ff53819ec10 secure :-1 s=READY pgs=264 cs=0 l=1 rev1=1 crypto rx=0x7ff52800e9f0 tx=0x7ff52800ed00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:57.136 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:57.134+0000 7ff52effd700 1 -- 172.21.15.99:0/3664609717 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff528004a30 con 0x7ff538101e10 2024-03-19T01:39:57.136 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:57.134+0000 7ff52effd700 1 -- 172.21.15.99:0/3664609717 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7ff52800cca0 con 0x7ff538101e10 2024-03-19T01:39:57.136 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:57.134+0000 7ff52effd700 1 -- 172.21.15.99:0/3664609717 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff5280056c0 con 0x7ff538101e10 2024-03-19T01:39:57.137 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:57.134+0000 7ff53fb85700 1 -- 172.21.15.99:0/3664609717 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ff5381992e0 con 0x7ff538101e10 2024-03-19T01:39:57.137 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:57.134+0000 7ff53fb85700 1 -- 172.21.15.99:0/3664609717 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7ff5381a55a0 con 0x7ff538101e10 2024-03-19T01:39:57.137 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:57.135+0000 7ff53fb85700 1 -- 172.21.15.99:0/3664609717 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7ff538102920 con 0x7ff538101e10 2024-03-19T01:39:57.137 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:57.136+0000 7ff52effd700 1 -- 172.21.15.99:0/3664609717 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 17) ==== 137341+0+0 (secure 0 0 0) 0x7ff52800c7c0 con 0x7ff538101e10 2024-03-19T01:39:57.138 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:57.137+0000 7ff52effd700 1 --2- 172.21.15.99:0/3664609717 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff5240a29a0 0x7ff5240a4e60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:57.138 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:57.137+0000 7ff53d921700 1 --2- 172.21.15.99:0/3664609717 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff5240a29a0 0x7ff5240a4e60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:57.138 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:57.137+0000 7ff52effd700 1 -- 172.21.15.99:0/3664609717 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8269+0+0 (secure 0 0 0) 0x7ff5280ceef0 con 0x7ff538101e10 2024-03-19T01:39:57.138 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:57.137+0000 7ff53d921700 1 --2- 172.21.15.99:0/3664609717 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff5240a29a0 0x7ff5240a4e60 secure :-1 s=READY pgs=200 cs=0 l=1 rev1=1 crypto rx=0x7ff5381999d0 tx=0x7ff534009260 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:57.141 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:57.141+0000 7ff52effd700 1 -- 172.21.15.99:0/3664609717 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7ff528053740 con 0x7ff538101e10 2024-03-19T01:39:57.346 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:57.346+0000 7ff53fb85700 1 -- 172.21.15.99:0/3664609717 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 11} v 0) -- 0x7ff53802dbb0 con 0x7ff538101e10 2024-03-19T01:39:57.346 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:57.346+0000 7ff52effd700 1 -- 172.21.15.99:0/3664609717 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 11}]=0 v0) ==== 75+0+13 (secure 0 0 0) 0x7ff52809be10 con 0x7ff538101e10 2024-03-19T01:39:57.347 INFO:teuthology.orchestra.run.smithi099.stdout:352187318297 2024-03-19T01:39:57.348 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:57.348+0000 7ff53fb85700 1 -- 172.21.15.99:0/3664609717 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff5240a29a0 msgr2=0x7ff5240a4e60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:57.349 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:57.348+0000 7ff53fb85700 1 --2- 172.21.15.99:0/3664609717 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff5240a29a0 0x7ff5240a4e60 secure :-1 s=READY pgs=200 cs=0 l=1 rev1=1 crypto rx=0x7ff5381999d0 tx=0x7ff534009260 comp rx=0 tx=0).stop 2024-03-19T01:39:57.349 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:57.348+0000 7ff53fb85700 1 -- 172.21.15.99:0/3664609717 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff538101e10 msgr2=0x7ff53819ec10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:57.349 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:57.348+0000 7ff53fb85700 1 --2- 172.21.15.99:0/3664609717 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff538101e10 0x7ff53819ec10 secure :-1 s=READY pgs=264 cs=0 l=1 rev1=1 crypto rx=0x7ff52800e9f0 tx=0x7ff52800ed00 comp rx=0 tx=0).stop 2024-03-19T01:39:57.349 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:57.348+0000 7ff53fb85700 1 -- 172.21.15.99:0/3664609717 shutdown_connections 2024-03-19T01:39:57.349 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:57.348+0000 7ff53fb85700 1 --2- 172.21.15.99:0/3664609717 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff5240a29a0 0x7ff5240a4e60 unknown :-1 s=CLOSED pgs=200 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:57.349 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:57.348+0000 7ff53fb85700 1 --2- 172.21.15.99:0/3664609717 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff538101460 0x7ff53819e6d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:57.349 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:57.348+0000 7ff53fb85700 1 --2- 172.21.15.99:0/3664609717 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff538106520 0x7ff538198890 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:57.349 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:57.348+0000 7ff53fb85700 1 --2- 172.21.15.99:0/3664609717 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff538101e10 0x7ff53819ec10 unknown :-1 s=CLOSED pgs=264 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:57.349 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:57.348+0000 7ff53fb85700 1 -- 172.21.15.99:0/3664609717 >> 172.21.15.99:0/3664609717 conn(0x7ff538078180 msgr2=0x7ff53806ac50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:57.349 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:57.348+0000 7ff53fb85700 1 -- 172.21.15.99:0/3664609717 shutdown_connections 2024-03-19T01:39:57.349 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:57.348+0000 7ff53fb85700 1 -- 172.21.15.99:0/3664609717 wait complete. 2024-03-19T01:39:57.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:57 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:39:57.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:57 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:39:57.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:57 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:39:57.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:57 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:39:57.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:57 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/1997330721' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-03-19T01:39:57.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:57 smithi112 ceph-mon[66819]: pgmap v314: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 6 op/s 2024-03-19T01:39:57.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:57 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:39:57.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:57 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:39:57.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:57 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:39:57.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:57 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:39:57.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:57 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/1997330721' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-03-19T01:39:57.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:57 smithi115 ceph-mon[67332]: pgmap v314: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 6 op/s 2024-03-19T01:39:58.072 INFO:tasks.cephadm.ceph_manager.ceph:need seq 352187318292 got 352187318297 for osd.11 2024-03-19T01:39:58.072 DEBUG:teuthology.parallel:result is None 2024-03-19T01:39:58.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:58 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/3664609717' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 11}]: dispatch 2024-03-19T01:39:58.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:58 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/3664609717' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 11}]: dispatch 2024-03-19T01:39:58.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:58 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/3664609717' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 11}]: dispatch 2024-03-19T01:39:58.741 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574915 got 55834574921 for osd.1 2024-03-19T01:39:58.741 DEBUG:teuthology.parallel:result is None 2024-03-19T01:39:58.920 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:58.918+0000 7fda6b9b5700 1 -- 172.21.15.99:0/1670682311 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fda64108340 msgr2=0x7fda64108720 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:58.920 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:58.918+0000 7fda6b9b5700 1 --2- 172.21.15.99:0/1670682311 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fda64108340 0x7fda64108720 secure :-1 s=READY pgs=265 cs=0 l=1 rev1=1 crypto rx=0x7fda600099c0 tx=0x7fda60009cd0 comp rx=0 tx=0).stop 2024-03-19T01:39:58.920 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:58.918+0000 7fda6b9b5700 1 -- 172.21.15.99:0/1670682311 shutdown_connections 2024-03-19T01:39:58.920 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:58.918+0000 7fda6b9b5700 1 --2- 172.21.15.99:0/1670682311 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fda64102d20 0x7fda6410f160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:58.920 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:58.918+0000 7fda6b9b5700 1 --2- 172.21.15.99:0/1670682311 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fda64108340 0x7fda64108720 unknown :-1 s=CLOSED pgs=265 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:58.920 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:58.918+0000 7fda6b9b5700 1 --2- 172.21.15.99:0/1670682311 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fda64102360 0x7fda641027e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:58.920 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:58.918+0000 7fda6b9b5700 1 -- 172.21.15.99:0/1670682311 >> 172.21.15.99:0/1670682311 conn(0x7fda64079440 msgr2=0x7fda64079850 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:58.920 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:58.918+0000 7fda6b9b5700 1 -- 172.21.15.99:0/1670682311 shutdown_connections 2024-03-19T01:39:58.920 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:58.918+0000 7fda6b9b5700 1 -- 172.21.15.99:0/1670682311 wait complete. 2024-03-19T01:39:58.921 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:58.919+0000 7fda6b9b5700 1 Processor -- start 2024-03-19T01:39:58.921 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:58.919+0000 7fda6b9b5700 1 -- start start 2024-03-19T01:39:58.921 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:58.919+0000 7fda6b9b5700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fda64102360 0x7fda6419e680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:58.921 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:58.919+0000 7fda6b9b5700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fda64102d20 0x7fda6419ebc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:58.921 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:58.919+0000 7fda6b9b5700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fda64108340 0x7fda64198800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:58.921 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:58.919+0000 7fda6b9b5700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fda641125c0 con 0x7fda64102360 2024-03-19T01:39:58.921 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:58.919+0000 7fda6b9b5700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fda64112440 con 0x7fda64102d20 2024-03-19T01:39:58.921 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:58.919+0000 7fda6b9b5700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fda64112740 con 0x7fda64108340 2024-03-19T01:39:58.921 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:58.919+0000 7fda69751700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fda64102360 0x7fda6419e680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:58.921 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:58.919+0000 7fda68f50700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fda64102d20 0x7fda6419ebc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:58.921 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:58.919+0000 7fda69751700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fda64102360 0x7fda6419e680 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:50492/0 (socket says 172.21.15.99:50492) 2024-03-19T01:39:58.921 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:58.919+0000 7fda69751700 1 -- 172.21.15.99:0/714348590 learned_addr learned my addr 172.21.15.99:0/714348590 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:39:58.922 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:58.919+0000 7fda68f50700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fda64102d20 0x7fda6419ebc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:50512/0 (socket says 172.21.15.99:50512) 2024-03-19T01:39:58.922 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:58.919+0000 7fda69f52700 1 --2- 172.21.15.99:0/714348590 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fda64108340 0x7fda64198800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:58.922 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:58.919+0000 7fda69751700 1 -- 172.21.15.99:0/714348590 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fda64108340 msgr2=0x7fda64198800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:58.922 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:58.919+0000 7fda69751700 1 --2- 172.21.15.99:0/714348590 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fda64108340 0x7fda64198800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:58.922 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:58.919+0000 7fda69751700 1 -- 172.21.15.99:0/714348590 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fda64102d20 msgr2=0x7fda6419ebc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:58.922 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:58.919+0000 7fda69751700 1 --2- 172.21.15.99:0/714348590 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fda64102d20 0x7fda6419ebc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:58.922 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:58.919+0000 7fda69751700 1 -- 172.21.15.99:0/714348590 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fda64198fe0 con 0x7fda64102360 2024-03-19T01:39:58.922 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:58.919+0000 7fda68f50700 1 --2- 172.21.15.99:0/714348590 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fda64102d20 0x7fda6419ebc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:39:58.922 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:58.919+0000 7fda69751700 1 --2- 172.21.15.99:0/714348590 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fda64102360 0x7fda6419e680 secure :-1 s=READY pgs=266 cs=0 l=1 rev1=1 crypto rx=0x7fda60006010 tx=0x7fda6000f690 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:58.922 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:58.919+0000 7fda5a7fc700 1 -- 172.21.15.99:0/714348590 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fda60003680 con 0x7fda64102360 2024-03-19T01:39:58.922 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:58.920+0000 7fda5a7fc700 1 -- 172.21.15.99:0/714348590 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fda600050d0 con 0x7fda64102360 2024-03-19T01:39:58.922 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:58.920+0000 7fda6b9b5700 1 -- 172.21.15.99:0/714348590 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fda64199270 con 0x7fda64102360 2024-03-19T01:39:58.922 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:58.920+0000 7fda5a7fc700 1 -- 172.21.15.99:0/714348590 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fda60003db0 con 0x7fda64102360 2024-03-19T01:39:58.923 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:58.920+0000 7fda6b9b5700 1 -- 172.21.15.99:0/714348590 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fda641a54e0 con 0x7fda64102360 2024-03-19T01:39:58.924 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:58.921+0000 7fda6b9b5700 1 -- 172.21.15.99:0/714348590 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fda641039f0 con 0x7fda64102360 2024-03-19T01:39:58.924 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:58.922+0000 7fda5a7fc700 1 -- 172.21.15.99:0/714348590 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 17) ==== 137341+0+0 (secure 0 0 0) 0x7fda60005280 con 0x7fda64102360 2024-03-19T01:39:58.924 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:58.923+0000 7fda5a7fc700 1 --2- 172.21.15.99:0/714348590 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fda500a2ac0 0x7fda500a4f80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:39:58.924 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:58.923+0000 7fda68f50700 1 --2- 172.21.15.99:0/714348590 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fda500a2ac0 0x7fda500a4f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:39:58.924 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:58.923+0000 7fda5a7fc700 1 -- 172.21.15.99:0/714348590 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8269+0+0 (secure 0 0 0) 0x7fda600cfae0 con 0x7fda64102360 2024-03-19T01:39:58.924 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:58.923+0000 7fda68f50700 1 --2- 172.21.15.99:0/714348590 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fda500a2ac0 0x7fda500a4f80 secure :-1 s=READY pgs=201 cs=0 l=1 rev1=1 crypto rx=0x7fda640799d0 tx=0x7fda54009260 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:39:58.926 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:58.926+0000 7fda5a7fc700 1 -- 172.21.15.99:0/714348590 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7fda6005d740 con 0x7fda64102360 2024-03-19T01:39:59.130 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:59.129+0000 7fda6b9b5700 1 -- 172.21.15.99:0/714348590 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 5} v 0) -- 0x7fda6402dbb0 con 0x7fda64102360 2024-03-19T01:39:59.130 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:59.130+0000 7fda5a7fc700 1 -- 172.21.15.99:0/714348590 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 5}]=0 v0) ==== 74+0+13 (secure 0 0 0) 0x7fda60013070 con 0x7fda64102360 2024-03-19T01:39:59.130 INFO:teuthology.orchestra.run.smithi099.stdout:171798691894 2024-03-19T01:39:59.132 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:59.132+0000 7fda6b9b5700 1 -- 172.21.15.99:0/714348590 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fda500a2ac0 msgr2=0x7fda500a4f80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:59.132 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:59.132+0000 7fda6b9b5700 1 --2- 172.21.15.99:0/714348590 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fda500a2ac0 0x7fda500a4f80 secure :-1 s=READY pgs=201 cs=0 l=1 rev1=1 crypto rx=0x7fda640799d0 tx=0x7fda54009260 comp rx=0 tx=0).stop 2024-03-19T01:39:59.132 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:59.132+0000 7fda6b9b5700 1 -- 172.21.15.99:0/714348590 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fda64102360 msgr2=0x7fda6419e680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:39:59.132 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:59.132+0000 7fda6b9b5700 1 --2- 172.21.15.99:0/714348590 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fda64102360 0x7fda6419e680 secure :-1 s=READY pgs=266 cs=0 l=1 rev1=1 crypto rx=0x7fda60006010 tx=0x7fda6000f690 comp rx=0 tx=0).stop 2024-03-19T01:39:59.132 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:59.132+0000 7fda6b9b5700 1 -- 172.21.15.99:0/714348590 shutdown_connections 2024-03-19T01:39:59.132 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:59.132+0000 7fda6b9b5700 1 --2- 172.21.15.99:0/714348590 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fda500a2ac0 0x7fda500a4f80 unknown :-1 s=CLOSED pgs=201 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:59.133 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:59.132+0000 7fda6b9b5700 1 --2- 172.21.15.99:0/714348590 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fda64108340 0x7fda64198800 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:59.133 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:59.132+0000 7fda6b9b5700 1 --2- 172.21.15.99:0/714348590 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fda64102360 0x7fda6419e680 unknown :-1 s=CLOSED pgs=266 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:59.133 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:59.132+0000 7fda6b9b5700 1 --2- 172.21.15.99:0/714348590 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fda64102d20 0x7fda6419ebc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:39:59.133 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:59.132+0000 7fda6b9b5700 1 -- 172.21.15.99:0/714348590 >> 172.21.15.99:0/714348590 conn(0x7fda64079440 msgr2=0x7fda64100f20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:39:59.133 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:59.132+0000 7fda6b9b5700 1 -- 172.21.15.99:0/714348590 shutdown_connections 2024-03-19T01:39:59.133 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:39:59.132+0000 7fda6b9b5700 1 -- 172.21.15.99:0/714348590 wait complete. 2024-03-19T01:39:59.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:59 smithi099 ceph-mon[62951]: pgmap v315: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 6 op/s 2024-03-19T01:39:59.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:39:59 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/714348590' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-03-19T01:39:59.389 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411393 got 77309411396 for osd.2 2024-03-19T01:39:59.390 DEBUG:teuthology.parallel:result is None 2024-03-19T01:39:59.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:59 smithi115 ceph-mon[67332]: pgmap v315: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 6 op/s 2024-03-19T01:39:59.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:39:59 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/714348590' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-03-19T01:39:59.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:59 smithi112 ceph-mon[66819]: pgmap v315: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 6 op/s 2024-03-19T01:39:59.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:39:59 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/714348590' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-03-19T01:39:59.996 INFO:tasks.cephadm.ceph_manager.ceph:need seq 171798691890 got 171798691894 for osd.5 2024-03-19T01:39:59.996 DEBUG:teuthology.parallel:result is None 2024-03-19T01:40:00.046 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:40:00.046 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:40:00.046 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:40:00.216 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:00 smithi099 ceph-mon[62951]: overall HEALTH_OK 2024-03-19T01:40:00.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:00 smithi115 ceph-mon[67332]: overall HEALTH_OK 2024-03-19T01:40:00.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:00 smithi112 ceph-mon[66819]: overall HEALTH_OK 2024-03-19T01:40:01.458 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:01 smithi099 ceph-mon[62951]: pgmap v316: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 6 op/s 2024-03-19T01:40:01.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:01 smithi115 ceph-mon[67332]: pgmap v316: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 6 op/s 2024-03-19T01:40:01.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:01 smithi112 ceph-mon[66819]: pgmap v316: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 6 op/s 2024-03-19T01:40:03.236 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:03.235+0000 7ff236eb8700 1 -- 172.21.15.99:0/3796374355 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff2301011f0 msgr2=0x7ff230101670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:03.236 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:03.235+0000 7ff236eb8700 1 --2- 172.21.15.99:0/3796374355 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff2301011f0 0x7ff230101670 secure :-1 s=READY pgs=267 cs=0 l=1 rev1=1 crypto rx=0x7ff21c009990 tx=0x7ff21c009ca0 comp rx=0 tx=0).stop 2024-03-19T01:40:03.236 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:03.235+0000 7ff236eb8700 1 -- 172.21.15.99:0/3796374355 shutdown_connections 2024-03-19T01:40:03.236 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:03.235+0000 7ff236eb8700 1 --2- 172.21.15.99:0/3796374355 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff230101bb0 0x7ff23010e050 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:03.236 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:03.235+0000 7ff236eb8700 1 --2- 172.21.15.99:0/3796374355 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff2301072a0 0x7ff230107680 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:03.236 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:03.235+0000 7ff236eb8700 1 --2- 172.21.15.99:0/3796374355 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff2301011f0 0x7ff230101670 unknown :-1 s=CLOSED pgs=267 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:03.236 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:03.235+0000 7ff236eb8700 1 -- 172.21.15.99:0/3796374355 >> 172.21.15.99:0/3796374355 conn(0x7ff230078220 msgr2=0x7ff230078630 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:03.236 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:03.235+0000 7ff236eb8700 1 -- 172.21.15.99:0/3796374355 shutdown_connections 2024-03-19T01:40:03.236 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:03.235+0000 7ff236eb8700 1 -- 172.21.15.99:0/3796374355 wait complete. 2024-03-19T01:40:03.238 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:03.236+0000 7ff236eb8700 1 Processor -- start 2024-03-19T01:40:03.238 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:03.236+0000 7ff236eb8700 1 -- start start 2024-03-19T01:40:03.239 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:03.236+0000 7ff236eb8700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff2301011f0 0x7ff2301998a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:03.239 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:03.236+0000 7ff236eb8700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff230101bb0 0x7ff230199de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:03.239 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:03.236+0000 7ff236eb8700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff2301072a0 0x7ff2301a4920 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:03.239 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:03.236+0000 7ff236eb8700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7ff2301114e0 con 0x7ff230101bb0 2024-03-19T01:40:03.239 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:03.236+0000 7ff236eb8700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7ff230111360 con 0x7ff2301072a0 2024-03-19T01:40:03.239 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:03.236+0000 7ff236eb8700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7ff230111660 con 0x7ff2301011f0 2024-03-19T01:40:03.239 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:03.236+0000 7ff22ffff700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff230101bb0 0x7ff230199de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:03.239 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:03.236+0000 7ff234c54700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff2301011f0 0x7ff2301998a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:03.239 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:03.236+0000 7ff22ffff700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff230101bb0 0x7ff230199de0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:46026/0 (socket says 172.21.15.99:46026) 2024-03-19T01:40:03.239 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:03.236+0000 7ff234c54700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff2301011f0 0x7ff2301998a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:47116/0 (socket says 172.21.15.99:47116) 2024-03-19T01:40:03.239 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:03.236+0000 7ff22ffff700 1 -- 172.21.15.99:0/368348876 learned_addr learned my addr 172.21.15.99:0/368348876 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:03.239 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:03.236+0000 7ff235455700 1 --2- 172.21.15.99:0/368348876 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff2301072a0 0x7ff2301a4920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:03.240 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:03.237+0000 7ff234c54700 1 -- 172.21.15.99:0/368348876 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff2301072a0 msgr2=0x7ff2301a4920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:03.240 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:03.237+0000 7ff234c54700 1 --2- 172.21.15.99:0/368348876 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff2301072a0 0x7ff2301a4920 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:03.240 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:03.237+0000 7ff234c54700 1 -- 172.21.15.99:0/368348876 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff230101bb0 msgr2=0x7ff230199de0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:03.240 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:03.237+0000 7ff234c54700 1 --2- 172.21.15.99:0/368348876 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff230101bb0 0x7ff230199de0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:03.240 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:03.237+0000 7ff234c54700 1 -- 172.21.15.99:0/368348876 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ff2301a4e60 con 0x7ff2301011f0 2024-03-19T01:40:03.240 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:03.237+0000 7ff22ffff700 1 --2- 172.21.15.99:0/368348876 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff230101bb0 0x7ff230199de0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:40:03.240 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:03.237+0000 7ff234c54700 1 --2- 172.21.15.99:0/368348876 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff2301011f0 0x7ff2301998a0 secure :-1 s=READY pgs=65 cs=0 l=1 rev1=1 crypto rx=0x7ff22400e9f0 tx=0x7ff22400ed00 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:03.240 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:03.237+0000 7ff22dffb700 1 -- 172.21.15.99:0/368348876 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff22400c9c0 con 0x7ff2301011f0 2024-03-19T01:40:03.240 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:03.237+0000 7ff22dffb700 1 -- 172.21.15.99:0/368348876 <== mon.1 v2:172.21.15.115:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7ff22400cb60 con 0x7ff2301011f0 2024-03-19T01:40:03.240 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:03.237+0000 7ff236eb8700 1 -- 172.21.15.99:0/368348876 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ff2301a5090 con 0x7ff2301011f0 2024-03-19T01:40:03.240 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:03.237+0000 7ff22dffb700 1 -- 172.21.15.99:0/368348876 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff224018680 con 0x7ff2301011f0 2024-03-19T01:40:03.240 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:03.237+0000 7ff236eb8700 1 -- 172.21.15.99:0/368348876 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7ff2301a55a0 con 0x7ff2301011f0 2024-03-19T01:40:03.240 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:03.238+0000 7ff236eb8700 1 -- 172.21.15.99:0/368348876 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7ff230068fd0 con 0x7ff2301011f0 2024-03-19T01:40:03.241 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:03.239+0000 7ff22dffb700 1 -- 172.21.15.99:0/368348876 <== mon.1 v2:172.21.15.115:3300/0 4 ==== mgrmap(e 17) ==== 137341+0+0 (secure 0 0 0) 0x7ff224010430 con 0x7ff2301011f0 2024-03-19T01:40:03.241 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:03.240+0000 7ff22dffb700 1 --2- 172.21.15.99:0/368348876 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff2200a2920 0x7ff2200a4de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:03.241 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:03.240+0000 7ff22ffff700 1 --2- 172.21.15.99:0/368348876 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff2200a2920 0x7ff2200a4de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:03.241 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:03.240+0000 7ff22dffb700 1 -- 172.21.15.99:0/368348876 <== mon.1 v2:172.21.15.115:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8269+0+0 (secure 0 0 0) 0x7ff2240ced10 con 0x7ff2301011f0 2024-03-19T01:40:03.241 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:03.240+0000 7ff22ffff700 1 --2- 172.21.15.99:0/368348876 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff2200a2920 0x7ff2200a4de0 secure :-1 s=READY pgs=202 cs=0 l=1 rev1=1 crypto rx=0x7ff230078960 tx=0x7ff21c009590 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:03.245 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:03.245+0000 7ff22dffb700 1 -- 172.21.15.99:0/368348876 <== mon.1 v2:172.21.15.115:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7ff2240d4020 con 0x7ff2301011f0 2024-03-19T01:40:03.445 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:03.445+0000 7ff236eb8700 1 -- 172.21.15.99:0/368348876 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 8} v 0) -- 0x7ff23002dbb0 con 0x7ff2301011f0 2024-03-19T01:40:03.446 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:03.445+0000 7ff22dffb700 1 -- 172.21.15.99:0/368348876 <== mon.1 v2:172.21.15.115:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 8}]=0 v0) ==== 74+0+13 (secure 0 0 0) 0x7ff22409bbd0 con 0x7ff2301011f0 2024-03-19T01:40:03.446 INFO:teuthology.orchestra.run.smithi099.stdout:257698037800 2024-03-19T01:40:03.448 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:03.447+0000 7ff236eb8700 1 -- 172.21.15.99:0/368348876 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff2200a2920 msgr2=0x7ff2200a4de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:03.448 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:03.447+0000 7ff236eb8700 1 --2- 172.21.15.99:0/368348876 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff2200a2920 0x7ff2200a4de0 secure :-1 s=READY pgs=202 cs=0 l=1 rev1=1 crypto rx=0x7ff230078960 tx=0x7ff21c009590 comp rx=0 tx=0).stop 2024-03-19T01:40:03.448 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:03.447+0000 7ff236eb8700 1 -- 172.21.15.99:0/368348876 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff2301011f0 msgr2=0x7ff2301998a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:03.448 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:03.447+0000 7ff236eb8700 1 --2- 172.21.15.99:0/368348876 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff2301011f0 0x7ff2301998a0 secure :-1 s=READY pgs=65 cs=0 l=1 rev1=1 crypto rx=0x7ff22400e9f0 tx=0x7ff22400ed00 comp rx=0 tx=0).stop 2024-03-19T01:40:03.448 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:03.447+0000 7ff236eb8700 1 -- 172.21.15.99:0/368348876 shutdown_connections 2024-03-19T01:40:03.448 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:03.447+0000 7ff236eb8700 1 --2- 172.21.15.99:0/368348876 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff2200a2920 0x7ff2200a4de0 unknown :-1 s=CLOSED pgs=202 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:03.448 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:03.447+0000 7ff236eb8700 1 --2- 172.21.15.99:0/368348876 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff2301011f0 0x7ff2301998a0 unknown :-1 s=CLOSED pgs=65 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:03.448 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:03.447+0000 7ff236eb8700 1 --2- 172.21.15.99:0/368348876 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff2301072a0 0x7ff2301a4920 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:03.449 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:03.447+0000 7ff236eb8700 1 --2- 172.21.15.99:0/368348876 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff230101bb0 0x7ff230199de0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:03.449 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:03.448+0000 7ff236eb8700 1 -- 172.21.15.99:0/368348876 >> 172.21.15.99:0/368348876 conn(0x7ff230078220 msgr2=0x7ff2301000c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:03.449 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:03.448+0000 7ff236eb8700 1 -- 172.21.15.99:0/368348876 shutdown_connections 2024-03-19T01:40:03.449 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:03.448+0000 7ff236eb8700 1 -- 172.21.15.99:0/368348876 wait complete. 2024-03-19T01:40:03.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:03 smithi099 ceph-mon[62951]: pgmap v317: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 6 op/s 2024-03-19T01:40:03.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:03 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/368348876' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 8}]: dispatch 2024-03-19T01:40:04.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:03 smithi112 ceph-mon[66819]: pgmap v317: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 6 op/s 2024-03-19T01:40:04.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:03 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/368348876' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 8}]: dispatch 2024-03-19T01:40:04.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:03 smithi115 ceph-mon[67332]: pgmap v317: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 6 op/s 2024-03-19T01:40:04.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:03 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/368348876' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 8}]: dispatch 2024-03-19T01:40:04.348 INFO:tasks.cephadm.ceph_manager.ceph:need seq 257698037795 got 257698037800 for osd.8 2024-03-19T01:40:04.348 DEBUG:teuthology.parallel:result is None 2024-03-19T01:40:04.780 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:04.764+0000 7fab751c2700 1 -- 172.21.15.99:0/262621328 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fab701009d0 msgr2=0x7fab70100e50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:04.780 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:04.764+0000 7fab751c2700 1 --2- 172.21.15.99:0/262621328 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fab701009d0 0x7fab70100e50 secure :-1 s=READY pgs=66 cs=0 l=1 rev1=1 crypto rx=0x7fab6400aa00 tx=0x7fab6400ad10 comp rx=0 tx=0).stop 2024-03-19T01:40:04.780 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:04.764+0000 7fab751c2700 1 -- 172.21.15.99:0/262621328 shutdown_connections 2024-03-19T01:40:04.780 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:04.764+0000 7fab751c2700 1 --2- 172.21.15.99:0/262621328 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fab70101390 0x7fab7010d8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:04.780 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:04.764+0000 7fab751c2700 1 --2- 172.21.15.99:0/262621328 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fab701069d0 0x7fab70106db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:04.780 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:04.764+0000 7fab751c2700 1 --2- 172.21.15.99:0/262621328 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fab701009d0 0x7fab70100e50 unknown :-1 s=CLOSED pgs=66 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:04.780 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:04.764+0000 7fab751c2700 1 -- 172.21.15.99:0/262621328 >> 172.21.15.99:0/262621328 conn(0x7fab70077810 msgr2=0x7fab700fec30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:04.780 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:04.764+0000 7fab751c2700 1 -- 172.21.15.99:0/262621328 shutdown_connections 2024-03-19T01:40:04.780 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:04.764+0000 7fab751c2700 1 -- 172.21.15.99:0/262621328 wait complete. 2024-03-19T01:40:04.780 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:04.765+0000 7fab751c2700 1 Processor -- start 2024-03-19T01:40:04.780 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:04.765+0000 7fab751c2700 1 -- start start 2024-03-19T01:40:04.780 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:04.765+0000 7fab751c2700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fab701009d0 0x7fab701a0940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:04.781 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:04.765+0000 7fab751c2700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fab70101390 0x7fab701a0e80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:04.781 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:04.765+0000 7fab751c2700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fab701069d0 0x7fab7019ab00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:04.781 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:04.765+0000 7fab751c2700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fab701104a0 con 0x7fab70101390 2024-03-19T01:40:04.781 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:04.765+0000 7fab751c2700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fab70110320 con 0x7fab701069d0 2024-03-19T01:40:04.781 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:04.765+0000 7fab751c2700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fab70110620 con 0x7fab701009d0 2024-03-19T01:40:04.781 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:04.765+0000 7fab6ed9d700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fab701009d0 0x7fab701a0940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:04.781 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:04.765+0000 7fab6e59c700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fab70101390 0x7fab701a0e80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:04.781 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:04.765+0000 7fab6ed9d700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fab701009d0 0x7fab701a0940 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:47128/0 (socket says 172.21.15.99:47128) 2024-03-19T01:40:04.781 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:04.765+0000 7fab6f59e700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fab701069d0 0x7fab7019ab00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:04.781 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:04.765+0000 7fab6ed9d700 1 -- 172.21.15.99:0/2868733896 learned_addr learned my addr 172.21.15.99:0/2868733896 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:04.781 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:04.766+0000 7fab6ed9d700 1 -- 172.21.15.99:0/2868733896 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fab701069d0 msgr2=0x7fab7019ab00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:04.781 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:04.766+0000 7fab6ed9d700 1 --2- 172.21.15.99:0/2868733896 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fab701069d0 0x7fab7019ab00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:04.781 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:04.766+0000 7fab6ed9d700 1 -- 172.21.15.99:0/2868733896 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fab70101390 msgr2=0x7fab701a0e80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:04.782 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:04.766+0000 7fab6ed9d700 1 --2- 172.21.15.99:0/2868733896 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fab70101390 0x7fab701a0e80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:04.782 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:04.766+0000 7fab6ed9d700 1 -- 172.21.15.99:0/2868733896 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fab7019b260 con 0x7fab701009d0 2024-03-19T01:40:04.782 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:04.766+0000 7fab6e59c700 1 --2- 172.21.15.99:0/2868733896 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fab70101390 0x7fab701a0e80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:40:04.782 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:04.766+0000 7fab6ed9d700 1 --2- 172.21.15.99:0/2868733896 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fab701009d0 0x7fab701a0940 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 crypto rx=0x7fab6000ec10 tx=0x7fab6000ef20 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:04.782 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:04.766+0000 7fab57fff700 1 -- 172.21.15.99:0/2868733896 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fab6000cc90 con 0x7fab701009d0 2024-03-19T01:40:04.782 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:04.766+0000 7fab57fff700 1 -- 172.21.15.99:0/2868733896 <== mon.1 v2:172.21.15.115:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fab6000ce30 con 0x7fab701009d0 2024-03-19T01:40:04.782 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:04.766+0000 7fab57fff700 1 -- 172.21.15.99:0/2868733896 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fab60010690 con 0x7fab701009d0 2024-03-19T01:40:04.782 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:04.766+0000 7fab751c2700 1 -- 172.21.15.99:0/2868733896 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fab7019b550 con 0x7fab701009d0 2024-03-19T01:40:04.782 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:04.766+0000 7fab751c2700 1 -- 172.21.15.99:0/2868733896 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fab701a7810 con 0x7fab701009d0 2024-03-19T01:40:04.782 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:04.767+0000 7fab751c2700 1 -- 172.21.15.99:0/2868733896 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fab70102110 con 0x7fab701009d0 2024-03-19T01:40:04.782 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:04.768+0000 7fab57fff700 1 -- 172.21.15.99:0/2868733896 <== mon.1 v2:172.21.15.115:3300/0 4 ==== mgrmap(e 17) ==== 137341+0+0 (secure 0 0 0) 0x7fab60003680 con 0x7fab701009d0 2024-03-19T01:40:04.782 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:04.768+0000 7fab57fff700 1 --2- 172.21.15.99:0/2868733896 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fab580a29a0 0x7fab580a4e60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:04.782 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:04.769+0000 7fab6e59c700 1 --2- 172.21.15.99:0/2868733896 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fab580a29a0 0x7fab580a4e60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:04.783 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:04.769+0000 7fab57fff700 1 -- 172.21.15.99:0/2868733896 <== mon.1 v2:172.21.15.115:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8269+0+0 (secure 0 0 0) 0x7fab600ceee0 con 0x7fab701009d0 2024-03-19T01:40:04.783 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:04.769+0000 7fab6e59c700 1 --2- 172.21.15.99:0/2868733896 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fab580a29a0 0x7fab580a4e60 secure :-1 s=READY pgs=203 cs=0 l=1 rev1=1 crypto rx=0x7fab6400a920 tx=0x7fab6400c650 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:04.783 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:04.772+0000 7fab57fff700 1 -- 172.21.15.99:0/2868733896 <== mon.1 v2:172.21.15.115:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7fab6009be00 con 0x7fab701009d0 2024-03-19T01:40:04.979 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:04.979+0000 7fab751c2700 1 -- 172.21.15.99:0/2868733896 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 3} v 0) -- 0x7fab701a7aa0 con 0x7fab701009d0 2024-03-19T01:40:04.979 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:04.979+0000 7fab57fff700 1 -- 172.21.15.99:0/2868733896 <== mon.1 v2:172.21.15.115:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 3}]=0 v0) ==== 74+0+13 (secure 0 0 0) 0x7fab6009b700 con 0x7fab701009d0 2024-03-19T01:40:04.980 INFO:teuthology.orchestra.run.smithi099.stdout:115964117058 2024-03-19T01:40:04.982 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:04.982+0000 7fab751c2700 1 -- 172.21.15.99:0/2868733896 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fab580a29a0 msgr2=0x7fab580a4e60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:04.982 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:04.982+0000 7fab751c2700 1 --2- 172.21.15.99:0/2868733896 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fab580a29a0 0x7fab580a4e60 secure :-1 s=READY pgs=203 cs=0 l=1 rev1=1 crypto rx=0x7fab6400a920 tx=0x7fab6400c650 comp rx=0 tx=0).stop 2024-03-19T01:40:04.982 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:04.982+0000 7fab751c2700 1 -- 172.21.15.99:0/2868733896 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fab701009d0 msgr2=0x7fab701a0940 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:04.982 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:04.982+0000 7fab751c2700 1 --2- 172.21.15.99:0/2868733896 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fab701009d0 0x7fab701a0940 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 crypto rx=0x7fab6000ec10 tx=0x7fab6000ef20 comp rx=0 tx=0).stop 2024-03-19T01:40:04.982 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:04.982+0000 7fab751c2700 1 -- 172.21.15.99:0/2868733896 shutdown_connections 2024-03-19T01:40:04.982 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:04.982+0000 7fab751c2700 1 --2- 172.21.15.99:0/2868733896 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fab580a29a0 0x7fab580a4e60 unknown :-1 s=CLOSED pgs=203 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:04.982 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:04.982+0000 7fab751c2700 1 --2- 172.21.15.99:0/2868733896 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fab701009d0 0x7fab701a0940 unknown :-1 s=CLOSED pgs=67 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:04.982 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:04.982+0000 7fab751c2700 1 --2- 172.21.15.99:0/2868733896 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fab701069d0 0x7fab7019ab00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:04.982 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:04.982+0000 7fab751c2700 1 --2- 172.21.15.99:0/2868733896 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fab70101390 0x7fab701a0e80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:04.983 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:04.982+0000 7fab751c2700 1 -- 172.21.15.99:0/2868733896 >> 172.21.15.99:0/2868733896 conn(0x7fab70077810 msgr2=0x7fab700fef20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:04.983 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:04.982+0000 7fab751c2700 1 -- 172.21.15.99:0/2868733896 shutdown_connections 2024-03-19T01:40:04.983 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:04.982+0000 7fab751c2700 1 -- 172.21.15.99:0/2868733896 wait complete. 2024-03-19T01:40:05.493 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:05.493+0000 7f58f6add700 1 -- 172.21.15.99:0/2321144103 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f58f0101bb0 msgr2=0x7f58f010e050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:05.493 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:05.493+0000 7f58f6add700 1 --2- 172.21.15.99:0/2321144103 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f58f0101bb0 0x7f58f010e050 secure :-1 s=READY pgs=268 cs=0 l=1 rev1=1 crypto rx=0x7f58e800b0a0 tx=0x7f58e800b3b0 comp rx=0 tx=0).stop 2024-03-19T01:40:05.494 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:05.493+0000 7f58f6add700 1 -- 172.21.15.99:0/2321144103 shutdown_connections 2024-03-19T01:40:05.494 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:05.493+0000 7f58f6add700 1 --2- 172.21.15.99:0/2321144103 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f58f0101bb0 0x7f58f010e050 unknown :-1 s=CLOSED pgs=268 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:05.494 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:05.493+0000 7f58f6add700 1 --2- 172.21.15.99:0/2321144103 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f58f01072a0 0x7f58f0107680 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:05.494 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:05.493+0000 7f58f6add700 1 --2- 172.21.15.99:0/2321144103 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f58f01011f0 0x7f58f0101670 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:05.494 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:05.493+0000 7f58f6add700 1 -- 172.21.15.99:0/2321144103 >> 172.21.15.99:0/2321144103 conn(0x7f58f0078220 msgr2=0x7f58f0078630 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:05.494 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:05.493+0000 7f58f6add700 1 -- 172.21.15.99:0/2321144103 shutdown_connections 2024-03-19T01:40:05.494 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:05.493+0000 7f58f6add700 1 -- 172.21.15.99:0/2321144103 wait complete. 2024-03-19T01:40:05.494 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:05.493+0000 7f58f6add700 1 Processor -- start 2024-03-19T01:40:05.494 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:05.493+0000 7f58f6add700 1 -- start start 2024-03-19T01:40:05.495 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:05.494+0000 7f58f6add700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f58f01011f0 0x7f58f0102ea0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:05.495 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:05.494+0000 7f58f6add700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f58f0101bb0 0x7f58f01033e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:05.495 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:05.494+0000 7f58f6add700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f58f01072a0 0x7f58f0103920 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:05.495 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:05.494+0000 7f58f6add700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f58f01114e0 con 0x7f58f01011f0 2024-03-19T01:40:05.496 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:05.494+0000 7f58f6add700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f58f0111360 con 0x7f58f01072a0 2024-03-19T01:40:05.496 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:05.494+0000 7f58f6add700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f58f0111660 con 0x7f58f0101bb0 2024-03-19T01:40:05.496 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:05.494+0000 7f58f4879700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f58f01011f0 0x7f58f0102ea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:05.496 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:05.494+0000 7f58f4879700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f58f01011f0 0x7f58f0102ea0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:46066/0 (socket says 172.21.15.99:46066) 2024-03-19T01:40:05.496 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:05.494+0000 7f58f4879700 1 -- 172.21.15.99:0/2382233256 learned_addr learned my addr 172.21.15.99:0/2382233256 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:05.496 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:05.494+0000 7f58e7fff700 1 --2- 172.21.15.99:0/2382233256 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f58f0101bb0 0x7f58f01033e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:05.496 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:05.494+0000 7f58f507a700 1 --2- 172.21.15.99:0/2382233256 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f58f01072a0 0x7f58f0103920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:05.496 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:05.494+0000 7f58f4879700 1 -- 172.21.15.99:0/2382233256 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f58f0101bb0 msgr2=0x7f58f01033e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:05.496 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:05.494+0000 7f58f4879700 1 --2- 172.21.15.99:0/2382233256 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f58f0101bb0 0x7f58f01033e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:05.496 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:05.494+0000 7f58f4879700 1 -- 172.21.15.99:0/2382233256 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f58f01072a0 msgr2=0x7f58f0103920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:05.496 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:05.494+0000 7f58f4879700 1 --2- 172.21.15.99:0/2382233256 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f58f01072a0 0x7f58f0103920 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:05.496 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:05.494+0000 7f58f4879700 1 -- 172.21.15.99:0/2382233256 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f58f01a6e90 con 0x7f58f01011f0 2024-03-19T01:40:05.496 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:05.494+0000 7f58e7fff700 1 --2- 172.21.15.99:0/2382233256 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f58f0101bb0 0x7f58f01033e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:40:05.496 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:05.494+0000 7f58f4879700 1 --2- 172.21.15.99:0/2382233256 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f58f01011f0 0x7f58f0102ea0 secure :-1 s=READY pgs=269 cs=0 l=1 rev1=1 crypto rx=0x7f58ec00e9a0 tx=0x7f58ec00ecb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:05.496 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:05.494+0000 7f58e5ffb700 1 -- 172.21.15.99:0/2382233256 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f58ec00ca10 con 0x7f58f01011f0 2024-03-19T01:40:05.497 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:05.494+0000 7f58e5ffb700 1 -- 172.21.15.99:0/2382233256 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f58ec00cbb0 con 0x7f58f01011f0 2024-03-19T01:40:05.497 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:05.494+0000 7f58f6add700 1 -- 172.21.15.99:0/2382233256 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f58f01a7180 con 0x7f58f01011f0 2024-03-19T01:40:05.497 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:05.494+0000 7f58e5ffb700 1 -- 172.21.15.99:0/2382233256 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f58ec018680 con 0x7f58f01011f0 2024-03-19T01:40:05.497 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:05.495+0000 7f58f6add700 1 -- 172.21.15.99:0/2382233256 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f58f01a7710 con 0x7f58f01011f0 2024-03-19T01:40:05.498 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:05.495+0000 7f58f6add700 1 -- 172.21.15.99:0/2382233256 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f58f0068fd0 con 0x7f58f01011f0 2024-03-19T01:40:05.498 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:05.497+0000 7f58e5ffb700 1 -- 172.21.15.99:0/2382233256 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 17) ==== 137341+0+0 (secure 0 0 0) 0x7f58ec010430 con 0x7f58f01011f0 2024-03-19T01:40:05.498 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:05.497+0000 7f58e5ffb700 1 --2- 172.21.15.99:0/2382233256 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f58d80a29a0 0x7f58d80a4e60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:05.498 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:05.497+0000 7f58e7fff700 1 --2- 172.21.15.99:0/2382233256 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f58d80a29a0 0x7f58d80a4e60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:05.498 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:05.497+0000 7f58e5ffb700 1 -- 172.21.15.99:0/2382233256 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8269+0+0 (secure 0 0 0) 0x7f58ec0cf1f0 con 0x7f58f01011f0 2024-03-19T01:40:05.498 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:05.498+0000 7f58e7fff700 1 --2- 172.21.15.99:0/2382233256 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f58d80a29a0 0x7f58d80a4e60 secure :-1 s=READY pgs=204 cs=0 l=1 rev1=1 crypto rx=0x7f58f00ff9d0 tx=0x7f58e0009260 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:05.501 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:05.501+0000 7f58e5ffb700 1 -- 172.21.15.99:0/2382233256 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f58ec09c110 con 0x7f58f01011f0 2024-03-19T01:40:05.702 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:05.701+0000 7f58f6add700 1 -- 172.21.15.99:0/2382233256 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 0} v 0) -- 0x7f58f01a7a70 con 0x7f58f01011f0 2024-03-19T01:40:05.702 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:05.702+0000 7f58e5ffb700 1 -- 172.21.15.99:0/2382233256 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 0}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7f58ec09ba10 con 0x7f58f01011f0 2024-03-19T01:40:05.702 INFO:teuthology.orchestra.run.smithi099.stdout:34359738449 2024-03-19T01:40:05.704 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:05.704+0000 7f58f6add700 1 -- 172.21.15.99:0/2382233256 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f58d80a29a0 msgr2=0x7f58d80a4e60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:05.704 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:05.704+0000 7f58f6add700 1 --2- 172.21.15.99:0/2382233256 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f58d80a29a0 0x7f58d80a4e60 secure :-1 s=READY pgs=204 cs=0 l=1 rev1=1 crypto rx=0x7f58f00ff9d0 tx=0x7f58e0009260 comp rx=0 tx=0).stop 2024-03-19T01:40:05.704 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:05.704+0000 7f58f6add700 1 -- 172.21.15.99:0/2382233256 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f58f01011f0 msgr2=0x7f58f0102ea0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:05.705 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:05.704+0000 7f58f6add700 1 --2- 172.21.15.99:0/2382233256 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f58f01011f0 0x7f58f0102ea0 secure :-1 s=READY pgs=269 cs=0 l=1 rev1=1 crypto rx=0x7f58ec00e9a0 tx=0x7f58ec00ecb0 comp rx=0 tx=0).stop 2024-03-19T01:40:05.705 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:05.704+0000 7f58f6add700 1 -- 172.21.15.99:0/2382233256 shutdown_connections 2024-03-19T01:40:05.705 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:05.704+0000 7f58f6add700 1 --2- 172.21.15.99:0/2382233256 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f58d80a29a0 0x7f58d80a4e60 unknown :-1 s=CLOSED pgs=204 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:05.705 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:05.704+0000 7f58f6add700 1 --2- 172.21.15.99:0/2382233256 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f58f01072a0 0x7f58f0103920 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:05.705 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:05.704+0000 7f58f6add700 1 --2- 172.21.15.99:0/2382233256 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f58f01011f0 0x7f58f0102ea0 unknown :-1 s=CLOSED pgs=269 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:05.705 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:05.704+0000 7f58f6add700 1 --2- 172.21.15.99:0/2382233256 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f58f0101bb0 0x7f58f01033e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:05.705 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:05.704+0000 7f58f6add700 1 -- 172.21.15.99:0/2382233256 >> 172.21.15.99:0/2382233256 conn(0x7f58f0078220 msgr2=0x7f58f01004b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:05.705 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:05.704+0000 7f58f6add700 1 -- 172.21.15.99:0/2382233256 shutdown_connections 2024-03-19T01:40:05.705 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:05.704+0000 7f58f6add700 1 -- 172.21.15.99:0/2382233256 wait complete. 2024-03-19T01:40:06.006 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:05 smithi099 ceph-mon[62951]: pgmap v318: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-03-19T01:40:06.006 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:05 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/2868733896' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-03-19T01:40:06.006 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:05 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/2382233256' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-03-19T01:40:06.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:05 smithi112 ceph-mon[66819]: pgmap v318: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-03-19T01:40:06.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:05 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/2868733896' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-03-19T01:40:06.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:05 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/2382233256' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-03-19T01:40:06.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:05 smithi115 ceph-mon[67332]: pgmap v318: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-03-19T01:40:06.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:05 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/2868733896' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-03-19T01:40:06.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:05 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/2382233256' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-03-19T01:40:06.821 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738443 got 34359738449 for osd.0 2024-03-19T01:40:06.821 DEBUG:teuthology.parallel:result is None 2024-03-19T01:40:07.248 INFO:tasks.cephadm.ceph_manager.ceph:need seq 115964117053 got 115964117058 for osd.3 2024-03-19T01:40:07.249 DEBUG:teuthology.parallel:result is None 2024-03-19T01:40:07.249 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-03-19T01:40:07.249 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph pg dump --format=json 2024-03-19T01:40:07.508 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:40:07.959 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:07 smithi099 ceph-mon[62951]: pgmap v319: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 6 op/s 2024-03-19T01:40:07.959 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:07 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [1, 9]}]: dispatch 2024-03-19T01:40:08.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:07 smithi112 ceph-mon[66819]: pgmap v319: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 6 op/s 2024-03-19T01:40:08.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:07 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [1, 9]}]: dispatch 2024-03-19T01:40:08.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:07 smithi115 ceph-mon[67332]: pgmap v319: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 6 op/s 2024-03-19T01:40:08.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:07 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [1, 9]}]: dispatch 2024-03-19T01:40:08.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:08 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [1, 9]}]': finished 2024-03-19T01:40:08.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:08 smithi099 ceph-mon[62951]: osdmap e90: 12 total, 12 up, 12 in 2024-03-19T01:40:08.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:08 smithi099 ceph-mon[62951]: mgrmap e18: x(active, since 8m), standbys: z, y 2024-03-19T01:40:08.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:08 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:40:08.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:08 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:40:09.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:08 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [1, 9]}]': finished 2024-03-19T01:40:09.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:08 smithi112 ceph-mon[66819]: osdmap e90: 12 total, 12 up, 12 in 2024-03-19T01:40:09.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:08 smithi112 ceph-mon[66819]: mgrmap e18: x(active, since 8m), standbys: z, y 2024-03-19T01:40:09.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:08 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:40:09.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:08 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:40:09.225 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:09.225+0000 7f90e75cd700 1 -- 172.21.15.99:0/2538426717 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f90e0101bb0 msgr2=0x7f90e010e090 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:09.226 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:09.225+0000 7f90e75cd700 1 --2- 172.21.15.99:0/2538426717 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f90e0101bb0 0x7f90e010e090 secure :-1 s=READY pgs=270 cs=0 l=1 rev1=1 crypto rx=0x7f90d800b0a0 tx=0x7f90d800b3b0 comp rx=0 tx=0).stop 2024-03-19T01:40:09.226 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:09.225+0000 7f90e75cd700 1 -- 172.21.15.99:0/2538426717 shutdown_connections 2024-03-19T01:40:09.226 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:09.225+0000 7f90e75cd700 1 --2- 172.21.15.99:0/2538426717 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f90e0101bb0 0x7f90e010e090 unknown :-1 s=CLOSED pgs=270 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:09.226 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:09.225+0000 7f90e75cd700 1 --2- 172.21.15.99:0/2538426717 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f90e0107250 0x7f90e0107630 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:09.226 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:09.225+0000 7f90e75cd700 1 --2- 172.21.15.99:0/2538426717 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f90e0101210 0x7f90e0101670 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:09.226 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:09.225+0000 7f90e75cd700 1 -- 172.21.15.99:0/2538426717 >> 172.21.15.99:0/2538426717 conn(0x7f90e0078140 msgr2=0x7f90e0078550 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:09.226 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:09.225+0000 7f90e75cd700 1 -- 172.21.15.99:0/2538426717 shutdown_connections 2024-03-19T01:40:09.226 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:09.225+0000 7f90e75cd700 1 -- 172.21.15.99:0/2538426717 wait complete. 2024-03-19T01:40:09.228 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:09.226+0000 7f90e75cd700 1 Processor -- start 2024-03-19T01:40:09.228 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:09.226+0000 7f90e75cd700 1 -- start start 2024-03-19T01:40:09.228 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:09.226+0000 7f90e75cd700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f90e0101210 0x7f90e019e660 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:09.228 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:09.226+0000 7f90e75cd700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f90e0101bb0 0x7f90e019eba0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:09.228 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:09.226+0000 7f90e75cd700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f90e0107250 0x7f90e01987e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:09.228 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:09.226+0000 7f90e75cd700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f90e01114a0 con 0x7f90e0101210 2024-03-19T01:40:09.229 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:09.226+0000 7f90e75cd700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f90e0111320 con 0x7f90e0107250 2024-03-19T01:40:09.229 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:09.226+0000 7f90e75cd700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f90e0111620 con 0x7f90e0101bb0 2024-03-19T01:40:09.229 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:09.226+0000 7f90e5369700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f90e0101210 0x7f90e019e660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:09.229 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:09.226+0000 7f90e5369700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f90e0101210 0x7f90e019e660 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:46088/0 (socket says 172.21.15.99:46088) 2024-03-19T01:40:09.229 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:09.226+0000 7f90e5369700 1 -- 172.21.15.99:0/3889548598 learned_addr learned my addr 172.21.15.99:0/3889548598 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:09.229 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:09.226+0000 7f90e4b68700 1 --2- 172.21.15.99:0/3889548598 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f90e0101bb0 0x7f90e019eba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:09.229 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:09.226+0000 7f90e5b6a700 1 --2- 172.21.15.99:0/3889548598 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f90e0107250 0x7f90e01987e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:09.229 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:09.227+0000 7f90e4b68700 1 -- 172.21.15.99:0/3889548598 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f90e0107250 msgr2=0x7f90e01987e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:09.229 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:09.227+0000 7f90e4b68700 1 --2- 172.21.15.99:0/3889548598 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f90e0107250 0x7f90e01987e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:09.229 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:09.227+0000 7f90e4b68700 1 -- 172.21.15.99:0/3889548598 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f90e0101210 msgr2=0x7f90e019e660 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:09.229 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:09.227+0000 7f90e4b68700 1 --2- 172.21.15.99:0/3889548598 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f90e0101210 0x7f90e019e660 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:09.229 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:09.227+0000 7f90e4b68700 1 -- 172.21.15.99:0/3889548598 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f90e0198fc0 con 0x7f90e0101bb0 2024-03-19T01:40:09.229 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:09.227+0000 7f90e5369700 1 --2- 172.21.15.99:0/3889548598 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f90e0101210 0x7f90e019e660 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:40:09.229 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:09.227+0000 7f90e4b68700 1 --2- 172.21.15.99:0/3889548598 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f90e0101bb0 0x7f90e019eba0 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7f90dc00e9f0 tx=0x7f90dc00ed00 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:09.229 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:09.227+0000 7f90d67fc700 1 -- 172.21.15.99:0/3889548598 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f90dc00c9c0 con 0x7f90e0101bb0 2024-03-19T01:40:09.230 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:09.227+0000 7f90e75cd700 1 -- 172.21.15.99:0/3889548598 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f90e01992b0 con 0x7f90e0101bb0 2024-03-19T01:40:09.230 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:09.227+0000 7f90d67fc700 1 -- 172.21.15.99:0/3889548598 <== mon.1 v2:172.21.15.115:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f90dc00cb60 con 0x7f90e0101bb0 2024-03-19T01:40:09.230 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:09.227+0000 7f90d67fc700 1 -- 172.21.15.99:0/3889548598 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f90dc018680 con 0x7f90e0101bb0 2024-03-19T01:40:09.230 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:09.227+0000 7f90e75cd700 1 -- 172.21.15.99:0/3889548598 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f90e01a54c0 con 0x7f90e0101bb0 2024-03-19T01:40:09.230 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:09.228+0000 7f90e75cd700 1 -- 172.21.15.99:0/3889548598 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f90e0102900 con 0x7f90e0101bb0 2024-03-19T01:40:09.231 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:09.230+0000 7f90d67fc700 1 -- 172.21.15.99:0/3889548598 <== mon.1 v2:172.21.15.115:3300/0 4 ==== mgrmap(e 18) ==== 137290+0+0 (secure 0 0 0) 0x7f90dc010430 con 0x7f90e0101bb0 2024-03-19T01:40:09.231 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:09.230+0000 7f90d67fc700 1 --2- 172.21.15.99:0/3889548598 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f90cc0a2890 0x7f90cc0a4d50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:09.231 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:09.230+0000 7f90e5369700 1 --2- 172.21.15.99:0/3889548598 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f90cc0a2890 0x7f90cc0a4d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:09.231 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:09.231+0000 7f90d67fc700 1 -- 172.21.15.99:0/3889548598 <== mon.1 v2:172.21.15.115:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f90dc0cebd0 con 0x7f90e0101bb0 2024-03-19T01:40:09.231 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:09.231+0000 7f90e5369700 1 --2- 172.21.15.99:0/3889548598 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f90cc0a2890 0x7f90cc0a4d50 secure :-1 s=READY pgs=205 cs=0 l=1 rev1=1 crypto rx=0x7f90d0009680 tx=0x7f90d0009260 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:09.238 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:09.238+0000 7f90d67fc700 1 -- 172.21.15.99:0/3889548598 <== mon.1 v2:172.21.15.115:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f90dc09bc30 con 0x7f90e0101bb0 2024-03-19T01:40:09.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:08 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [1, 9]}]': finished 2024-03-19T01:40:09.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:08 smithi115 ceph-mon[67332]: osdmap e90: 12 total, 12 up, 12 in 2024-03-19T01:40:09.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:08 smithi115 ceph-mon[67332]: mgrmap e18: x(active, since 8m), standbys: z, y 2024-03-19T01:40:09.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:08 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:40:09.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:08 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:40:09.439 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:09.439+0000 7f90e75cd700 1 -- 172.21.15.99:0/3889548598 --> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7f90e0105000 con 0x7f90cc0a2890 2024-03-19T01:40:09.446 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:09.446+0000 7f90d67fc700 1 -- 172.21.15.99:0/3889548598 <== mgr.14150 v2:172.21.15.99:6800/4272410295 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+258786 (secure 0 0 0) 0x7f90e0105000 con 0x7f90cc0a2890 2024-03-19T01:40:09.451 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:40:09.453 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:09.450+0000 7f90e75cd700 1 -- 172.21.15.99:0/3889548598 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f90cc0a2890 msgr2=0x7f90cc0a4d50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:09.453 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:09.450+0000 7f90e75cd700 1 --2- 172.21.15.99:0/3889548598 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f90cc0a2890 0x7f90cc0a4d50 secure :-1 s=READY pgs=205 cs=0 l=1 rev1=1 crypto rx=0x7f90d0009680 tx=0x7f90d0009260 comp rx=0 tx=0).stop 2024-03-19T01:40:09.453 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:09.450+0000 7f90e75cd700 1 -- 172.21.15.99:0/3889548598 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f90e0101bb0 msgr2=0x7f90e019eba0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:09.453 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:09.450+0000 7f90e75cd700 1 --2- 172.21.15.99:0/3889548598 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f90e0101bb0 0x7f90e019eba0 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7f90dc00e9f0 tx=0x7f90dc00ed00 comp rx=0 tx=0).stop 2024-03-19T01:40:09.453 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:09.450+0000 7f90e75cd700 1 -- 172.21.15.99:0/3889548598 shutdown_connections 2024-03-19T01:40:09.453 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:09.450+0000 7f90e75cd700 1 --2- 172.21.15.99:0/3889548598 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f90cc0a2890 0x7f90cc0a4d50 unknown :-1 s=CLOSED pgs=205 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:09.453 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:09.450+0000 7f90e75cd700 1 --2- 172.21.15.99:0/3889548598 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f90e0107250 0x7f90e01987e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:09.453 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:09.450+0000 7f90e75cd700 1 --2- 172.21.15.99:0/3889548598 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f90e0101210 0x7f90e019e660 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:09.453 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:09.450+0000 7f90e75cd700 1 --2- 172.21.15.99:0/3889548598 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f90e0101bb0 0x7f90e019eba0 unknown :-1 s=CLOSED pgs=68 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:09.453 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:09.450+0000 7f90e75cd700 1 -- 172.21.15.99:0/3889548598 >> 172.21.15.99:0/3889548598 conn(0x7f90e0078140 msgr2=0x7f90e0100470 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:09.453 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:09.451+0000 7f90e75cd700 1 -- 172.21.15.99:0/3889548598 shutdown_connections 2024-03-19T01:40:09.454 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:09.451+0000 7f90e75cd700 1 -- 172.21.15.99:0/3889548598 wait complete. 2024-03-19T01:40:09.454 INFO:teuthology.orchestra.run.smithi099.stderr:dumped all 2024-03-19T01:40:10.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:09 smithi099 ceph-mon[62951]: pgmap v321: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 3.0 KiB/s rd, 307 B/s wr, 6 op/s 2024-03-19T01:40:10.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:09 smithi099 ceph-mon[62951]: osdmap e91: 12 total, 12 up, 12 in 2024-03-19T01:40:10.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:09 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:40:10.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:09 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:40:10.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:09 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:40:10.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:09 smithi112 ceph-mon[66819]: pgmap v321: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 3.0 KiB/s rd, 307 B/s wr, 6 op/s 2024-03-19T01:40:10.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:09 smithi112 ceph-mon[66819]: osdmap e91: 12 total, 12 up, 12 in 2024-03-19T01:40:10.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:09 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:40:10.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:09 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:40:10.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:09 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:40:10.142 INFO:teuthology.orchestra.run.smithi099.stdout:{"pg_ready":true,"pg_map":{"version":322,"stamp":"2024-03-19T01:40:08.823026+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":655299,"num_objects":64,"num_object_clones":0,"num_object_copies":192,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":64,"num_whiteouts":0,"num_read":551,"num_read_kb":388,"num_write":343,"num_write_kb":3053,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":40,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":449,"ondisk_log_size":449,"up":291,"acting":291,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":291,"num_osds":12,"num_per_pool_osds":12,"num_per_pool_omap_osds":12,"kb":1124941824,"kb_used":375188,"kb_used_data":13516,"kb_used_omap":18,"kb_used_meta":361517,"kb_avail":1124566636,"statfs":{"total":1151940427776,"available":1151556235264,"internally_reserved":0,"allocated":13840384,"data_stored":7551247,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19139,"internal_metadata":370193725},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":282,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":20,"num_write":3,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.066474"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":17,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.831563+0000","last_change":"2024-03-19T01:38:56.809406+0000","last_active":"2024-03-19T01:39:07.831563+0000","last_peered":"2024-03-19T01:39:07.831563+0000","last_clean":"2024-03-19T01:39:07.831563+0000","last_became_active":"2024-03-19T01:38:56.808971+0000","last_became_peered":"2024-03-19T01:38:56.808971+0000","last_unstale":"2024-03-19T01:39:07.831563+0000","last_undegraded":"2024-03-19T01:39:07.831563+0000","last_fullsized":"2024-03-19T01:39:07.831563+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T11:07:58.551704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.3e","version":"89'4","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:40:06.772429+0000","last_change":"2024-03-19T01:38:56.798847+0000","last_active":"2024-03-19T01:40:06.772429+0000","last_peered":"2024-03-19T01:40:06.772429+0000","last_clean":"2024-03-19T01:40:06.772429+0000","last_became_active":"2024-03-19T01:38:56.795442+0000","last_became_peered":"2024-03-19T01:38:56.795442+0000","last_unstale":"2024-03-19T01:40:06.772429+0000","last_undegraded":"2024-03-19T01:40:06.772429+0000","last_fullsized":"2024-03-19T01:40:06.772429+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T12:05:55.721962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,11,10],"acting":[0,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3d","version":"89'7","reported_seq":24,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:40:06.777513+0000","last_change":"2024-03-19T01:38:56.806317+0000","last_active":"2024-03-19T01:40:06.777513+0000","last_peered":"2024-03-19T01:40:06.777513+0000","last_clean":"2024-03-19T01:40:06.777513+0000","last_became_active":"2024-03-19T01:38:56.806001+0000","last_became_peered":"2024-03-19T01:38:56.806001+0000","last_unstale":"2024-03-19T01:40:06.777513+0000","last_undegraded":"2024-03-19T01:40:06.777513+0000","last_fullsized":"2024-03-19T01:40:06.777513+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T05:19:45.501431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"89'8","reported_seq":25,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:40:06.766516+0000","last_change":"2024-03-19T01:38:56.803611+0000","last_active":"2024-03-19T01:40:06.766516+0000","last_peered":"2024-03-19T01:40:06.766516+0000","last_clean":"2024-03-19T01:40:06.766516+0000","last_became_active":"2024-03-19T01:38:56.803300+0000","last_became_peered":"2024-03-19T01:38:56.803300+0000","last_unstale":"2024-03-19T01:40:06.766516+0000","last_undegraded":"2024-03-19T01:40:06.766516+0000","last_fullsized":"2024-03-19T01:40:06.766516+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T06:05:49.287266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,8,2],"acting":[4,8,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.3b","version":"89'4","reported_seq":23,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-19T01:40:07.829176+0000","last_change":"2024-03-19T01:38:56.799910+0000","last_active":"2024-03-19T01:40:07.829176+0000","last_peered":"2024-03-19T01:40:07.829176+0000","last_clean":"2024-03-19T01:40:07.829176+0000","last_became_active":"2024-03-19T01:38:56.799567+0000","last_became_peered":"2024-03-19T01:38:56.799567+0000","last_unstale":"2024-03-19T01:40:07.829176+0000","last_undegraded":"2024-03-19T01:40:07.829176+0000","last_fullsized":"2024-03-19T01:40:07.829176+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T05:59:42.563083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,3,6],"acting":[8,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":23,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.833027+0000","last_change":"2024-03-19T01:39:07.833027+0000","last_active":"2024-03-19T01:39:07.833027+0000","last_peered":"2024-03-19T01:39:07.833027+0000","last_clean":"2024-03-19T01:39:07.833027+0000","last_became_active":"2024-03-19T01:39:07.832700+0000","last_became_peered":"2024-03-19T01:39:07.832700+0000","last_unstale":"2024-03-19T01:39:07.833027+0000","last_undegraded":"2024-03-19T01:39:07.833027+0000","last_fullsized":"2024-03-19T01:39:07.833027+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T10:40:44.588730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,9,6],"acting":[10,9,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":17,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.828627+0000","last_change":"2024-03-19T01:38:56.804091+0000","last_active":"2024-03-19T01:39:07.828627+0000","last_peered":"2024-03-19T01:39:07.828627+0000","last_clean":"2024-03-19T01:39:07.828627+0000","last_became_active":"2024-03-19T01:38:56.802308+0000","last_became_peered":"2024-03-19T01:38:56.802308+0000","last_unstale":"2024-03-19T01:39:07.828627+0000","last_undegraded":"2024-03-19T01:39:07.828627+0000","last_fullsized":"2024-03-19T01:39:07.828627+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T07:35:42.952521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,9,5],"acting":[7,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":17,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.834116+0000","last_change":"2024-03-19T01:38:56.796475+0000","last_active":"2024-03-19T01:39:07.834116+0000","last_peered":"2024-03-19T01:39:07.834116+0000","last_clean":"2024-03-19T01:39:07.834116+0000","last_became_active":"2024-03-19T01:38:56.796116+0000","last_became_peered":"2024-03-19T01:38:56.796116+0000","last_unstale":"2024-03-19T01:39:07.834116+0000","last_undegraded":"2024-03-19T01:39:07.834116+0000","last_fullsized":"2024-03-19T01:39:07.834116+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T04:35:19.821098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,10],"acting":[3,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":23,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.836670+0000","last_change":"2024-03-19T01:39:07.836670+0000","last_active":"2024-03-19T01:39:07.836670+0000","last_peered":"2024-03-19T01:39:07.836670+0000","last_clean":"2024-03-19T01:39:07.836670+0000","last_became_active":"2024-03-19T01:39:07.836247+0000","last_became_peered":"2024-03-19T01:39:07.836247+0000","last_unstale":"2024-03-19T01:39:07.836670+0000","last_undegraded":"2024-03-19T01:39:07.836670+0000","last_fullsized":"2024-03-19T01:39:07.836670+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T11:26:39.713658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,11],"acting":[1,7,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":17,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.831645+0000","last_change":"2024-03-19T01:38:56.797973+0000","last_active":"2024-03-19T01:39:07.831645+0000","last_peered":"2024-03-19T01:39:07.831645+0000","last_clean":"2024-03-19T01:39:07.831645+0000","last_became_active":"2024-03-19T01:38:56.797396+0000","last_became_peered":"2024-03-19T01:38:56.797396+0000","last_unstale":"2024-03-19T01:39:07.831645+0000","last_undegraded":"2024-03-19T01:39:07.831645+0000","last_fullsized":"2024-03-19T01:39:07.831645+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T11:46:32.867919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,2,4],"acting":[10,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":17,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.829087+0000","last_change":"2024-03-19T01:38:56.802211+0000","last_active":"2024-03-19T01:39:07.829087+0000","last_peered":"2024-03-19T01:39:07.829087+0000","last_clean":"2024-03-19T01:39:07.829087+0000","last_became_active":"2024-03-19T01:38:56.801776+0000","last_became_peered":"2024-03-19T01:38:56.801776+0000","last_unstale":"2024-03-19T01:39:07.829087+0000","last_undegraded":"2024-03-19T01:39:07.829087+0000","last_fullsized":"2024-03-19T01:39:07.829087+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T10:09:36.983771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,8,1],"acting":[7,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.34","version":"89'7","reported_seq":24,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:40:06.766409+0000","last_change":"2024-03-19T01:38:56.798463+0000","last_active":"2024-03-19T01:40:06.766409+0000","last_peered":"2024-03-19T01:40:06.766409+0000","last_clean":"2024-03-19T01:40:06.766409+0000","last_became_active":"2024-03-19T01:38:56.797187+0000","last_became_peered":"2024-03-19T01:38:56.797187+0000","last_unstale":"2024-03-19T01:40:06.766409+0000","last_undegraded":"2024-03-19T01:40:06.766409+0000","last_fullsized":"2024-03-19T01:40:06.766409+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T11:10:04.349393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,8],"acting":[4,10,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"89'10","reported_seq":27,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:40:06.773337+0000","last_change":"2024-03-19T01:38:56.801650+0000","last_active":"2024-03-19T01:40:06.773337+0000","last_peered":"2024-03-19T01:40:06.773337+0000","last_clean":"2024-03-19T01:40:06.773337+0000","last_became_active":"2024-03-19T01:38:56.801323+0000","last_became_peered":"2024-03-19T01:38:56.801323+0000","last_unstale":"2024-03-19T01:40:06.773337+0000","last_undegraded":"2024-03-19T01:40:06.773337+0000","last_fullsized":"2024-03-19T01:40:06.773337+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T06:10:28.518843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,10,5],"acting":[6,10,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.32","version":"89'7","reported_seq":24,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:40:06.776031+0000","last_change":"2024-03-19T01:38:56.798633+0000","last_active":"2024-03-19T01:40:06.776031+0000","last_peered":"2024-03-19T01:40:06.776031+0000","last_clean":"2024-03-19T01:40:06.776031+0000","last_became_active":"2024-03-19T01:38:56.798342+0000","last_became_peered":"2024-03-19T01:38:56.798342+0000","last_unstale":"2024-03-19T01:40:06.776031+0000","last_undegraded":"2024-03-19T01:40:06.776031+0000","last_fullsized":"2024-03-19T01:40:06.776031+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T03:21:57.350930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,8,10],"acting":[0,8,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"89'7","reported_seq":30,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:40:06.776914+0000","last_change":"2024-03-19T01:39:07.835127+0000","last_active":"2024-03-19T01:40:06.776914+0000","last_peered":"2024-03-19T01:40:06.776914+0000","last_clean":"2024-03-19T01:40:06.776914+0000","last_became_active":"2024-03-19T01:39:07.834731+0000","last_became_peered":"2024-03-19T01:39:07.834731+0000","last_unstale":"2024-03-19T01:40:06.776914+0000","last_undegraded":"2024-03-19T01:40:06.776914+0000","last_fullsized":"2024-03-19T01:40:06.776914+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T11:37:14.867184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,8,2],"acting":[0,8,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":17,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.832062+0000","last_change":"2024-03-19T01:38:56.796556+0000","last_active":"2024-03-19T01:39:07.832062+0000","last_peered":"2024-03-19T01:39:07.832062+0000","last_clean":"2024-03-19T01:39:07.832062+0000","last_became_active":"2024-03-19T01:38:56.796215+0000","last_became_peered":"2024-03-19T01:38:56.796215+0000","last_unstale":"2024-03-19T01:39:07.832062+0000","last_undegraded":"2024-03-19T01:39:07.832062+0000","last_fullsized":"2024-03-19T01:39:07.832062+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T09:12:00.826877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":17,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.831671+0000","last_change":"2024-03-19T01:38:56.808706+0000","last_active":"2024-03-19T01:39:07.831671+0000","last_peered":"2024-03-19T01:39:07.831671+0000","last_clean":"2024-03-19T01:39:07.831671+0000","last_became_active":"2024-03-19T01:38:56.808065+0000","last_became_peered":"2024-03-19T01:38:56.808065+0000","last_unstale":"2024-03-19T01:39:07.831671+0000","last_undegraded":"2024-03-19T01:39:07.831671+0000","last_fullsized":"2024-03-19T01:39:07.831671+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T08:40:28.256172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":17,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.829173+0000","last_change":"2024-03-19T01:38:56.799501+0000","last_active":"2024-03-19T01:39:07.829173+0000","last_peered":"2024-03-19T01:39:07.829173+0000","last_clean":"2024-03-19T01:39:07.829173+0000","last_became_active":"2024-03-19T01:38:56.799202+0000","last_became_peered":"2024-03-19T01:38:56.799202+0000","last_unstale":"2024-03-19T01:39:07.829173+0000","last_undegraded":"2024-03-19T01:39:07.829173+0000","last_fullsized":"2024-03-19T01:39:07.829173+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T09:42:06.386165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,1,10],"acting":[9,1,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":17,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.829847+0000","last_change":"2024-03-19T01:38:56.810697+0000","last_active":"2024-03-19T01:39:07.829847+0000","last_peered":"2024-03-19T01:39:07.829847+0000","last_clean":"2024-03-19T01:39:07.829847+0000","last_became_active":"2024-03-19T01:38:56.810202+0000","last_became_peered":"2024-03-19T01:38:56.810202+0000","last_unstale":"2024-03-19T01:39:07.829847+0000","last_undegraded":"2024-03-19T01:39:07.829847+0000","last_fullsized":"2024-03-19T01:39:07.829847+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T06:22:11.807807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1f","version":"87'9","reported_seq":29,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.829094+0000","last_change":"2024-03-19T01:38:55.791479+0000","last_active":"2024-03-19T01:39:07.829094+0000","last_peered":"2024-03-19T01:39:07.829094+0000","last_clean":"2024-03-19T01:39:07.829094+0000","last_became_active":"2024-03-19T01:38:55.791150+0000","last_became_peered":"2024-03-19T01:38:55.791150+0000","last_unstale":"2024-03-19T01:39:07.829094+0000","last_undegraded":"2024-03-19T01:39:07.829094+0000","last_fullsized":"2024-03-19T01:39:07.829094+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T09:56:07.953602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,7,8],"acting":[9,7,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":17,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.833686+0000","last_change":"2024-03-19T01:38:56.802404+0000","last_active":"2024-03-19T01:39:07.833686+0000","last_peered":"2024-03-19T01:39:07.833686+0000","last_clean":"2024-03-19T01:39:07.833686+0000","last_became_active":"2024-03-19T01:38:56.802060+0000","last_became_peered":"2024-03-19T01:38:56.802060+0000","last_unstale":"2024-03-19T01:39:07.833686+0000","last_undegraded":"2024-03-19T01:39:07.833686+0000","last_fullsized":"2024-03-19T01:39:07.833686+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T07:45:34.287073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1e","version":"87'2","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.834184+0000","last_change":"2024-03-19T01:38:55.798249+0000","last_active":"2024-03-19T01:39:07.834184+0000","last_peered":"2024-03-19T01:39:07.834184+0000","last_clean":"2024-03-19T01:39:07.834184+0000","last_became_active":"2024-03-19T01:38:55.797952+0000","last_became_peered":"2024-03-19T01:38:55.797952+0000","last_unstale":"2024-03-19T01:39:07.834184+0000","last_undegraded":"2024-03-19T01:39:07.834184+0000","last_fullsized":"2024-03-19T01:39:07.834184+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T12:12:40.925462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"89'6","reported_seq":23,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:40:06.775843+0000","last_change":"2024-03-19T01:38:56.805992+0000","last_active":"2024-03-19T01:40:06.775843+0000","last_peered":"2024-03-19T01:40:06.775843+0000","last_clean":"2024-03-19T01:40:06.775843+0000","last_became_active":"2024-03-19T01:38:56.804997+0000","last_became_peered":"2024-03-19T01:38:56.804997+0000","last_unstale":"2024-03-19T01:40:06.775843+0000","last_undegraded":"2024-03-19T01:40:06.775843+0000","last_fullsized":"2024-03-19T01:40:06.775843+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T10:33:39.003792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,9],"acting":[0,1,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1d","version":"87'4","reported_seq":25,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-19T01:40:07.829025+0000","last_change":"2024-03-19T01:38:55.795296+0000","last_active":"2024-03-19T01:40:07.829025+0000","last_peered":"2024-03-19T01:40:07.829025+0000","last_clean":"2024-03-19T01:40:07.829025+0000","last_became_active":"2024-03-19T01:38:55.794321+0000","last_became_peered":"2024-03-19T01:38:55.794321+0000","last_unstale":"2024-03-19T01:40:07.829025+0000","last_undegraded":"2024-03-19T01:40:07.829025+0000","last_fullsized":"2024-03-19T01:40:07.829025+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T12:26:12.914801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":17,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.832218+0000","last_change":"2024-03-19T01:38:56.802337+0000","last_active":"2024-03-19T01:39:07.832218+0000","last_peered":"2024-03-19T01:39:07.832218+0000","last_clean":"2024-03-19T01:39:07.832218+0000","last_became_active":"2024-03-19T01:38:56.802032+0000","last_became_peered":"2024-03-19T01:38:56.802032+0000","last_unstale":"2024-03-19T01:39:07.832218+0000","last_undegraded":"2024-03-19T01:39:07.832218+0000","last_fullsized":"2024-03-19T01:39:07.832218+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T07:57:13.512027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,6],"acting":[1,10,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1c","version":"86'2","reported_seq":23,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-19T01:40:07.828793+0000","last_change":"2024-03-19T01:38:55.793796+0000","last_active":"2024-03-19T01:40:07.828793+0000","last_peered":"2024-03-19T01:40:07.828793+0000","last_clean":"2024-03-19T01:40:07.828793+0000","last_became_active":"2024-03-19T01:38:55.792818+0000","last_became_peered":"2024-03-19T01:38:55.792818+0000","last_unstale":"2024-03-19T01:40:07.828793+0000","last_undegraded":"2024-03-19T01:40:07.828793+0000","last_fullsized":"2024-03-19T01:40:07.828793+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T09:44:17.403644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,1,5],"acting":[8,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1d","version":"89'15","reported_seq":32,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:40:06.779461+0000","last_change":"2024-03-19T01:38:56.801280+0000","last_active":"2024-03-19T01:40:06.779461+0000","last_peered":"2024-03-19T01:40:06.779461+0000","last_clean":"2024-03-19T01:40:06.779461+0000","last_became_active":"2024-03-19T01:38:56.800944+0000","last_became_peered":"2024-03-19T01:38:56.800944+0000","last_unstale":"2024-03-19T01:40:06.779461+0000","last_undegraded":"2024-03-19T01:40:06.779461+0000","last_fullsized":"2024-03-19T01:40:06.779461+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T03:09:45.373387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1b","version":"89'7","reported_seq":96,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:40:06.399876+0000","last_change":"2024-03-19T01:38:55.792563+0000","last_active":"2024-03-19T01:40:06.399876+0000","last_peered":"2024-03-19T01:40:06.399876+0000","last_clean":"2024-03-19T01:40:06.399876+0000","last_became_active":"2024-03-19T01:38:55.792098+0000","last_became_peered":"2024-03-19T01:38:55.792098+0000","last_unstale":"2024-03-19T01:40:06.399876+0000","last_undegraded":"2024-03-19T01:40:06.399876+0000","last_fullsized":"2024-03-19T01:40:06.399876+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T08:40:54.804413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1154,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":70,"num_read_kb":3,"num_write":4,"num_write_kb":4,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"89'7","reported_seq":26,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-19T01:40:07.829406+0000","last_change":"2024-03-19T01:38:56.810155+0000","last_active":"2024-03-19T01:40:07.829406+0000","last_peered":"2024-03-19T01:40:07.829406+0000","last_clean":"2024-03-19T01:40:07.829406+0000","last_became_active":"2024-03-19T01:38:56.809820+0000","last_became_peered":"2024-03-19T01:38:56.809820+0000","last_unstale":"2024-03-19T01:40:07.829406+0000","last_undegraded":"2024-03-19T01:40:07.829406+0000","last_fullsized":"2024-03-19T01:40:07.829406+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T12:44:30.463663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,9,5],"acting":[8,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.1a","version":"87'3","reported_seq":22,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.831323+0000","last_change":"2024-03-19T01:38:55.797324+0000","last_active":"2024-03-19T01:39:07.831323+0000","last_peered":"2024-03-19T01:39:07.831323+0000","last_clean":"2024-03-19T01:39:07.831323+0000","last_became_active":"2024-03-19T01:38:55.796997+0000","last_became_peered":"2024-03-19T01:38:55.796997+0000","last_unstale":"2024-03-19T01:39:07.831323+0000","last_undegraded":"2024-03-19T01:39:07.831323+0000","last_fullsized":"2024-03-19T01:39:07.831323+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T08:40:37.023902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,8,6],"acting":[10,8,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.1b","version":"89'8","reported_seq":25,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:40:06.771086+0000","last_change":"2024-03-19T01:38:56.785995+0000","last_active":"2024-03-19T01:40:06.771086+0000","last_peered":"2024-03-19T01:40:06.771086+0000","last_clean":"2024-03-19T01:40:06.771086+0000","last_became_active":"2024-03-19T01:38:56.785659+0000","last_became_peered":"2024-03-19T01:38:56.785659+0000","last_unstale":"2024-03-19T01:40:06.771086+0000","last_undegraded":"2024-03-19T01:40:06.771086+0000","last_fullsized":"2024-03-19T01:40:06.771086+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T06:40:33.389038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.19","version":"87'2","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.831084+0000","last_change":"2024-03-19T01:38:55.790034+0000","last_active":"2024-03-19T01:39:07.831084+0000","last_peered":"2024-03-19T01:39:07.831084+0000","last_clean":"2024-03-19T01:39:07.831084+0000","last_became_active":"2024-03-19T01:38:55.786369+0000","last_became_peered":"2024-03-19T01:38:55.786369+0000","last_unstale":"2024-03-19T01:39:07.831084+0000","last_undegraded":"2024-03-19T01:39:07.831084+0000","last_fullsized":"2024-03-19T01:39:07.831084+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T06:13:52.925644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,9],"acting":[6,4,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":17,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.834269+0000","last_change":"2024-03-19T01:38:56.791253+0000","last_active":"2024-03-19T01:39:07.834269+0000","last_peered":"2024-03-19T01:39:07.834269+0000","last_clean":"2024-03-19T01:39:07.834269+0000","last_became_active":"2024-03-19T01:38:56.790793+0000","last_became_peered":"2024-03-19T01:38:56.790793+0000","last_unstale":"2024-03-19T01:39:07.834269+0000","last_undegraded":"2024-03-19T01:39:07.834269+0000","last_fullsized":"2024-03-19T01:39:07.834269+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T05:14:15.277513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,6],"acting":[3,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"87'1","reported_seq":20,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.833264+0000","last_change":"2024-03-19T01:38:55.793379+0000","last_active":"2024-03-19T01:39:07.833264+0000","last_peered":"2024-03-19T01:39:07.833264+0000","last_clean":"2024-03-19T01:39:07.833264+0000","last_became_active":"2024-03-19T01:38:55.793035+0000","last_became_peered":"2024-03-19T01:38:55.793035+0000","last_unstale":"2024-03-19T01:39:07.833264+0000","last_undegraded":"2024-03-19T01:39:07.833264+0000","last_fullsized":"2024-03-19T01:39:07.833264+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T05:01:52.122826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":17,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.828816+0000","last_change":"2024-03-19T01:38:56.800540+0000","last_active":"2024-03-19T01:39:07.828816+0000","last_peered":"2024-03-19T01:39:07.828816+0000","last_clean":"2024-03-19T01:39:07.828816+0000","last_became_active":"2024-03-19T01:38:56.800199+0000","last_became_peered":"2024-03-19T01:38:56.800199+0000","last_unstale":"2024-03-19T01:39:07.828816+0000","last_undegraded":"2024-03-19T01:39:07.828816+0000","last_fullsized":"2024-03-19T01:39:07.828816+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T05:17:57.722253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,3,10],"acting":[9,3,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.17","version":"87'3","reported_seq":88,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:40:04.397518+0000","last_change":"2024-03-19T01:38:55.792930+0000","last_active":"2024-03-19T01:40:04.397518+0000","last_peered":"2024-03-19T01:40:04.397518+0000","last_clean":"2024-03-19T01:40:04.397518+0000","last_became_active":"2024-03-19T01:38:55.792602+0000","last_became_peered":"2024-03-19T01:38:55.792602+0000","last_unstale":"2024-03-19T01:40:04.397518+0000","last_undegraded":"2024-03-19T01:40:04.397518+0000","last_fullsized":"2024-03-19T01:40:04.397518+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T12:28:15.399311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":66,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,5,9],"acting":[10,5,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":17,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.832330+0000","last_change":"2024-03-19T01:38:56.801228+0000","last_active":"2024-03-19T01:39:07.832330+0000","last_peered":"2024-03-19T01:39:07.832330+0000","last_clean":"2024-03-19T01:39:07.832330+0000","last_became_active":"2024-03-19T01:38:56.800656+0000","last_became_peered":"2024-03-19T01:38:56.800656+0000","last_unstale":"2024-03-19T01:39:07.832330+0000","last_undegraded":"2024-03-19T01:39:07.832330+0000","last_fullsized":"2024-03-19T01:39:07.832330+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T05:43:32.565608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,9],"acting":[1,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.16","version":"87'2","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.829002+0000","last_change":"2024-03-19T01:38:55.795899+0000","last_active":"2024-03-19T01:39:07.829002+0000","last_peered":"2024-03-19T01:39:07.829002+0000","last_clean":"2024-03-19T01:39:07.829002+0000","last_became_active":"2024-03-19T01:38:55.795434+0000","last_became_peered":"2024-03-19T01:38:55.795434+0000","last_unstale":"2024-03-19T01:39:07.829002+0000","last_undegraded":"2024-03-19T01:39:07.829002+0000","last_fullsized":"2024-03-19T01:39:07.829002+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T02:32:39.376935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"89'9","reported_seq":32,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:40:06.773386+0000","last_change":"2024-03-19T01:39:07.837847+0000","last_active":"2024-03-19T01:40:06.773386+0000","last_peered":"2024-03-19T01:40:06.773386+0000","last_clean":"2024-03-19T01:40:06.773386+0000","last_became_active":"2024-03-19T01:39:07.837514+0000","last_became_peered":"2024-03-19T01:39:07.837514+0000","last_unstale":"2024-03-19T01:40:06.773386+0000","last_undegraded":"2024-03-19T01:40:06.773386+0000","last_fullsized":"2024-03-19T01:40:06.773386+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T06:28:54.164949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,8,5],"acting":[0,8,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"87'4","reported_seq":24,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.833137+0000","last_change":"2024-03-19T01:38:55.793769+0000","last_active":"2024-03-19T01:39:07.833137+0000","last_peered":"2024-03-19T01:39:07.833137+0000","last_clean":"2024-03-19T01:39:07.833137+0000","last_became_active":"2024-03-19T01:38:55.793423+0000","last_became_peered":"2024-03-19T01:38:55.793423+0000","last_unstale":"2024-03-19T01:39:07.833137+0000","last_undegraded":"2024-03-19T01:39:07.833137+0000","last_fullsized":"2024-03-19T01:39:07.833137+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T12:56:37.933791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":672,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,10],"acting":[5,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":17,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.832181+0000","last_change":"2024-03-19T01:38:56.794855+0000","last_active":"2024-03-19T01:39:07.832181+0000","last_peered":"2024-03-19T01:39:07.832181+0000","last_clean":"2024-03-19T01:39:07.832181+0000","last_became_active":"2024-03-19T01:38:56.794553+0000","last_became_peered":"2024-03-19T01:38:56.794553+0000","last_unstale":"2024-03-19T01:39:07.832181+0000","last_undegraded":"2024-03-19T01:39:07.832181+0000","last_fullsized":"2024-03-19T01:39:07.832181+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T08:33:20.473620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,8,4],"acting":[1,8,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.14","version":"89'29","reported_seq":188,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:40:05.398606+0000","last_change":"2024-03-19T01:38:55.794317+0000","last_active":"2024-03-19T01:40:05.398606+0000","last_peered":"2024-03-19T01:40:05.398606+0000","last_clean":"2024-03-19T01:40:05.398606+0000","last_became_active":"2024-03-19T01:38:55.793970+0000","last_became_peered":"2024-03-19T01:38:55.793970+0000","last_unstale":"2024-03-19T01:40:05.398606+0000","last_undegraded":"2024-03-19T01:40:05.398606+0000","last_fullsized":"2024-03-19T01:40:05.398606+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":29,"log_dups_size":0,"ondisk_log_size":29,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T02:25:09.584906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60023,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":140,"num_read_kb":128,"num_write":30,"num_write_kb":70,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,1,2],"acting":[10,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.15","version":"89'6","reported_seq":25,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-19T01:40:07.828797+0000","last_change":"2024-03-19T01:38:56.794400+0000","last_active":"2024-03-19T01:40:07.828797+0000","last_peered":"2024-03-19T01:40:07.828797+0000","last_clean":"2024-03-19T01:40:07.828797+0000","last_became_active":"2024-03-19T01:38:56.794027+0000","last_became_peered":"2024-03-19T01:38:56.794027+0000","last_unstale":"2024-03-19T01:40:07.828797+0000","last_undegraded":"2024-03-19T01:40:07.828797+0000","last_fullsized":"2024-03-19T01:40:07.828797+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T06:33:51.398239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,7],"acting":[8,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.13","version":"87'6","reported_seq":26,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.829583+0000","last_change":"2024-03-19T01:38:55.792812+0000","last_active":"2024-03-19T01:39:07.829583+0000","last_peered":"2024-03-19T01:39:07.829583+0000","last_clean":"2024-03-19T01:39:07.829583+0000","last_became_active":"2024-03-19T01:38:55.792422+0000","last_became_peered":"2024-03-19T01:38:55.792422+0000","last_unstale":"2024-03-19T01:39:07.829583+0000","last_undegraded":"2024-03-19T01:39:07.829583+0000","last_fullsized":"2024-03-19T01:39:07.829583+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T11:04:58.863405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,10,3],"acting":[2,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.12","version":"89'9","reported_seq":26,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:40:06.779007+0000","last_change":"2024-03-19T01:38:56.794156+0000","last_active":"2024-03-19T01:40:06.779007+0000","last_peered":"2024-03-19T01:40:06.779007+0000","last_clean":"2024-03-19T01:40:06.779007+0000","last_became_active":"2024-03-19T01:38:56.793812+0000","last_became_peered":"2024-03-19T01:38:56.793812+0000","last_unstale":"2024-03-19T01:40:06.779007+0000","last_undegraded":"2024-03-19T01:40:06.779007+0000","last_fullsized":"2024-03-19T01:40:06.779007+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T05:00:53.379548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.12","version":"87'3","reported_seq":22,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.833710+0000","last_change":"2024-03-19T01:38:55.798220+0000","last_active":"2024-03-19T01:39:07.833710+0000","last_peered":"2024-03-19T01:39:07.833710+0000","last_clean":"2024-03-19T01:39:07.833710+0000","last_became_active":"2024-03-19T01:38:55.797901+0000","last_became_peered":"2024-03-19T01:38:55.797901+0000","last_unstale":"2024-03-19T01:39:07.833710+0000","last_undegraded":"2024-03-19T01:39:07.833710+0000","last_fullsized":"2024-03-19T01:39:07.833710+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T11:45:20.233596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":17,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.828950+0000","last_change":"2024-03-19T01:38:56.805603+0000","last_active":"2024-03-19T01:39:07.828950+0000","last_peered":"2024-03-19T01:39:07.828950+0000","last_clean":"2024-03-19T01:39:07.828950+0000","last_became_active":"2024-03-19T01:38:56.805248+0000","last_became_peered":"2024-03-19T01:38:56.805248+0000","last_unstale":"2024-03-19T01:39:07.828950+0000","last_undegraded":"2024-03-19T01:39:07.828950+0000","last_fullsized":"2024-03-19T01:39:07.828950+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T05:28:41.503673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"89'5","reported_seq":94,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:40:06.418298+0000","last_change":"2024-03-19T01:38:55.793254+0000","last_active":"2024-03-19T01:40:06.418298+0000","last_peered":"2024-03-19T01:40:06.418298+0000","last_clean":"2024-03-19T01:40:06.418298+0000","last_became_active":"2024-03-19T01:38:55.792925+0000","last_became_peered":"2024-03-19T01:38:55.792925+0000","last_unstale":"2024-03-19T01:40:06.418298+0000","last_undegraded":"2024-03-19T01:40:06.418298+0000","last_fullsized":"2024-03-19T01:40:06.418298+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T05:17:25.353050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1120,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":71,"num_read_kb":4,"num_write":4,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,10],"acting":[4,7,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":17,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.828561+0000","last_change":"2024-03-19T01:38:56.807845+0000","last_active":"2024-03-19T01:39:07.828561+0000","last_peered":"2024-03-19T01:39:07.828561+0000","last_clean":"2024-03-19T01:39:07.828561+0000","last_became_active":"2024-03-19T01:38:56.807175+0000","last_became_peered":"2024-03-19T01:38:56.807175+0000","last_unstale":"2024-03-19T01:39:07.828561+0000","last_undegraded":"2024-03-19T01:39:07.828561+0000","last_fullsized":"2024-03-19T01:39:07.828561+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T10:45:37.669659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.10","version":"87'15","reported_seq":37,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-19T01:40:07.828910+0000","last_change":"2024-03-19T01:38:55.793320+0000","last_active":"2024-03-19T01:40:07.828910+0000","last_peered":"2024-03-19T01:40:07.828910+0000","last_clean":"2024-03-19T01:40:07.828910+0000","last_became_active":"2024-03-19T01:38:55.793030+0000","last_became_peered":"2024-03-19T01:38:55.793030+0000","last_unstale":"2024-03-19T01:40:07.828910+0000","last_undegraded":"2024-03-19T01:40:07.828910+0000","last_fullsized":"2024-03-19T01:40:07.828910+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T04:01:31.028364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":8,"num_object_clones":0,"num_object_copies":24,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":20,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":6,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,2],"acting":[8,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.11","version":"89'6","reported_seq":23,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:40:06.780226+0000","last_change":"2024-03-19T01:38:56.796338+0000","last_active":"2024-03-19T01:40:06.780226+0000","last_peered":"2024-03-19T01:40:06.780226+0000","last_clean":"2024-03-19T01:40:06.780226+0000","last_became_active":"2024-03-19T01:38:56.795660+0000","last_became_peered":"2024-03-19T01:38:56.795660+0000","last_unstale":"2024-03-19T01:40:06.780226+0000","last_undegraded":"2024-03-19T01:40:06.780226+0000","last_fullsized":"2024-03-19T01:40:06.780226+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T01:56:03.144907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.f","version":"87'2","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-19T01:40:07.828540+0000","last_change":"2024-03-19T01:38:55.791670+0000","last_active":"2024-03-19T01:40:07.828540+0000","last_peered":"2024-03-19T01:40:07.828540+0000","last_clean":"2024-03-19T01:40:07.828540+0000","last_became_active":"2024-03-19T01:38:55.790859+0000","last_became_peered":"2024-03-19T01:38:55.790859+0000","last_unstale":"2024-03-19T01:40:07.828540+0000","last_undegraded":"2024-03-19T01:40:07.828540+0000","last_fullsized":"2024-03-19T01:40:07.828540+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T08:13:16.770134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":2,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,5,4],"acting":[8,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":17,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.831192+0000","last_change":"2024-03-19T01:38:56.806291+0000","last_active":"2024-03-19T01:39:07.831192+0000","last_peered":"2024-03-19T01:39:07.831192+0000","last_clean":"2024-03-19T01:39:07.831192+0000","last_became_active":"2024-03-19T01:38:56.805972+0000","last_became_peered":"2024-03-19T01:38:56.805972+0000","last_unstale":"2024-03-19T01:39:07.831192+0000","last_undegraded":"2024-03-19T01:39:07.831192+0000","last_fullsized":"2024-03-19T01:39:07.831192+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T05:28:19.575710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,5,4],"acting":[11,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.829075+0000","last_change":"2024-03-19T01:38:55.798331+0000","last_active":"2024-03-19T01:39:07.829075+0000","last_peered":"2024-03-19T01:39:07.829075+0000","last_clean":"2024-03-19T01:39:07.829075+0000","last_became_active":"2024-03-19T01:38:55.797948+0000","last_became_peered":"2024-03-19T01:38:55.797948+0000","last_unstale":"2024-03-19T01:39:07.829075+0000","last_undegraded":"2024-03-19T01:39:07.829075+0000","last_fullsized":"2024-03-19T01:39:07.829075+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T04:35:09.036490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,2,0],"acting":[9,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.5","version":"87'1","reported_seq":18,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.831702+0000","last_change":"2024-03-19T01:38:56.810162+0000","last_active":"2024-03-19T01:39:07.831702+0000","last_peered":"2024-03-19T01:39:07.831702+0000","last_clean":"2024-03-19T01:39:07.831702+0000","last_became_active":"2024-03-19T01:38:56.809888+0000","last_became_peered":"2024-03-19T01:38:56.809888+0000","last_unstale":"2024-03-19T01:39:07.831702+0000","last_undegraded":"2024-03-19T01:39:07.831702+0000","last_fullsized":"2024-03-19T01:39:07.831702+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T05:49:35.606065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,3,6],"acting":[10,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"2.2","version":"87'1","reported_seq":20,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.828770+0000","last_change":"2024-03-19T01:38:55.796629+0000","last_active":"2024-03-19T01:39:07.828770+0000","last_peered":"2024-03-19T01:39:07.828770+0000","last_clean":"2024-03-19T01:39:07.828770+0000","last_became_active":"2024-03-19T01:38:55.796232+0000","last_became_peered":"2024-03-19T01:38:55.796232+0000","last_unstale":"2024-03-19T01:39:07.828770+0000","last_undegraded":"2024-03-19T01:39:07.828770+0000","last_fullsized":"2024-03-19T01:39:07.828770+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T09:38:58.853129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":17,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.831539+0000","last_change":"2024-03-19T01:38:56.798227+0000","last_active":"2024-03-19T01:39:07.831539+0000","last_peered":"2024-03-19T01:39:07.831539+0000","last_clean":"2024-03-19T01:39:07.831539+0000","last_became_active":"2024-03-19T01:38:56.797302+0000","last_became_peered":"2024-03-19T01:38:56.797302+0000","last_unstale":"2024-03-19T01:39:07.831539+0000","last_undegraded":"2024-03-19T01:39:07.831539+0000","last_fullsized":"2024-03-19T01:39:07.831539+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T12:35:56.975118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,4,0],"acting":[10,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":11,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.931227+0000","last_change":"2024-03-19T01:39:07.930840+0000","last_active":"2024-03-19T01:39:07.931227+0000","last_peered":"2024-03-19T01:39:07.931227+0000","last_clean":"2024-03-19T01:39:07.931227+0000","last_became_active":"2024-03-19T01:39:07.930581+0000","last_became_peered":"2024-03-19T01:39:07.930581+0000","last_unstale":"2024-03-19T01:39:07.931227+0000","last_undegraded":"2024-03-19T01:39:07.931227+0000","last_fullsized":"2024-03-19T01:39:07.931227+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T04:24:25.476025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,2,4],"acting":[9,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.1","version":"89'5","reported_seq":91,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:40:06.417906+0000","last_change":"2024-03-19T01:38:55.785957+0000","last_active":"2024-03-19T01:40:06.417906+0000","last_peered":"2024-03-19T01:40:06.417906+0000","last_clean":"2024-03-19T01:40:06.417906+0000","last_became_active":"2024-03-19T01:38:55.785527+0000","last_became_peered":"2024-03-19T01:38:55.785527+0000","last_unstale":"2024-03-19T01:40:06.417906+0000","last_undegraded":"2024-03-19T01:40:06.417906+0000","last_fullsized":"2024-03-19T01:40:06.417906+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T09:33:55.329126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":67,"num_read_kb":67,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":11,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.837652+0000","last_change":"2024-03-19T01:39:07.835329+0000","last_active":"2024-03-19T01:39:07.837652+0000","last_peered":"2024-03-19T01:39:07.837652+0000","last_clean":"2024-03-19T01:39:07.837652+0000","last_became_active":"2024-03-19T01:39:07.834744+0000","last_became_peered":"2024-03-19T01:39:07.834744+0000","last_unstale":"2024-03-19T01:39:07.837652+0000","last_undegraded":"2024-03-19T01:39:07.837652+0000","last_fullsized":"2024-03-19T01:39:07.837652+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T07:13:22.760031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,5,1],"acting":[9,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.3","version":"87'3","reported_seq":24,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.930581+0000","last_change":"2024-03-19T01:38:55.792521+0000","last_active":"2024-03-19T01:39:07.930581+0000","last_peered":"2024-03-19T01:39:07.930581+0000","last_clean":"2024-03-19T01:39:07.930581+0000","last_became_active":"2024-03-19T01:38:55.792166+0000","last_became_peered":"2024-03-19T01:38:55.792166+0000","last_unstale":"2024-03-19T01:39:07.930581+0000","last_undegraded":"2024-03-19T01:39:07.930581+0000","last_fullsized":"2024-03-19T01:39:07.930581+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T08:15:12.216641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":4,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,9],"acting":[4,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.0","version":"23'137","reported_seq":344,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.828610+0000","last_change":"2024-03-19T01:37:48.171712+0000","last_active":"2024-03-19T01:39:07.828610+0000","last_peered":"2024-03-19T01:39:07.828610+0000","last_clean":"2024-03-19T01:39:07.828610+0000","last_became_active":"2024-03-19T01:37:48.171323+0000","last_became_peered":"2024-03-19T01:37:48.171323+0000","last_unstale":"2024-03-19T01:39:07.828610+0000","last_undegraded":"2024-03-19T01:39:07.828610+0000","last_fullsized":"2024-03-19T01:39:07.828610+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:34:31.088679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:34:31.088679+0000","last_clean_scrub_stamp":"2024-03-19T01:34:31.088679+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T02:22:49.477567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":109,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"89'5","reported_seq":22,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:40:06.780848+0000","last_change":"2024-03-19T01:38:56.797868+0000","last_active":"2024-03-19T01:40:06.780848+0000","last_peered":"2024-03-19T01:40:06.780848+0000","last_clean":"2024-03-19T01:40:06.780848+0000","last_became_active":"2024-03-19T01:38:56.796855+0000","last_became_peered":"2024-03-19T01:38:56.796855+0000","last_unstale":"2024-03-19T01:40:06.780848+0000","last_undegraded":"2024-03-19T01:40:06.780848+0000","last_fullsized":"2024-03-19T01:40:06.780848+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T01:53:09.887368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"87'3","reported_seq":32,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.839838+0000","last_change":"2024-03-19T01:39:07.839838+0000","last_active":"2024-03-19T01:39:07.839838+0000","last_peered":"2024-03-19T01:39:07.839838+0000","last_clean":"2024-03-19T01:39:07.839838+0000","last_became_active":"2024-03-19T01:39:07.836581+0000","last_became_peered":"2024-03-19T01:39:07.836581+0000","last_unstale":"2024-03-19T01:39:07.839838+0000","last_undegraded":"2024-03-19T01:39:07.839838+0000","last_fullsized":"2024-03-19T01:39:07.839838+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T11:38:01.087677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"87'1","reported_seq":23,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-19T01:40:07.828901+0000","last_change":"2024-03-19T01:38:55.795464+0000","last_active":"2024-03-19T01:40:07.828901+0000","last_peered":"2024-03-19T01:40:07.828901+0000","last_clean":"2024-03-19T01:40:07.828901+0000","last_became_active":"2024-03-19T01:38:55.793827+0000","last_became_peered":"2024-03-19T01:38:55.793827+0000","last_unstale":"2024-03-19T01:40:07.828901+0000","last_undegraded":"2024-03-19T01:40:07.828901+0000","last_fullsized":"2024-03-19T01:40:07.828901+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T13:03:56.862269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,9],"acting":[8,0,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.4","version":"89'9","reported_seq":26,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:40:06.781029+0000","last_change":"2024-03-19T01:38:56.795569+0000","last_active":"2024-03-19T01:40:06.781029+0000","last_peered":"2024-03-19T01:40:06.781029+0000","last_clean":"2024-03-19T01:40:06.781029+0000","last_became_active":"2024-03-19T01:38:56.795281+0000","last_became_peered":"2024-03-19T01:38:56.795281+0000","last_unstale":"2024-03-19T01:40:06.781029+0000","last_undegraded":"2024-03-19T01:40:06.781029+0000","last_fullsized":"2024-03-19T01:40:06.781029+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T03:18:00.599307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,9],"acting":[6,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"88'7","reported_seq":26,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.829297+0000","last_change":"2024-03-19T01:38:55.792306+0000","last_active":"2024-03-19T01:39:07.829297+0000","last_peered":"2024-03-19T01:39:07.829297+0000","last_clean":"2024-03-19T01:39:07.829297+0000","last_became_active":"2024-03-19T01:38:55.792019+0000","last_became_peered":"2024-03-19T01:38:55.792019+0000","last_unstale":"2024-03-19T01:39:07.829297+0000","last_undegraded":"2024-03-19T01:39:07.829297+0000","last_fullsized":"2024-03-19T01:39:07.829297+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T06:33:06.869054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,8,3],"acting":[9,8,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":17,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.833784+0000","last_change":"2024-03-19T01:38:56.798314+0000","last_active":"2024-03-19T01:39:07.833784+0000","last_peered":"2024-03-19T01:39:07.833784+0000","last_clean":"2024-03-19T01:39:07.833784+0000","last_became_active":"2024-03-19T01:38:56.797999+0000","last_became_peered":"2024-03-19T01:38:56.797999+0000","last_unstale":"2024-03-19T01:39:07.833784+0000","last_undegraded":"2024-03-19T01:39:07.833784+0000","last_fullsized":"2024-03-19T01:39:07.833784+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T06:23:36.861238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,7],"acting":[3,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"87'4","reported_seq":23,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.930691+0000","last_change":"2024-03-19T01:38:55.790866+0000","last_active":"2024-03-19T01:39:07.930691+0000","last_peered":"2024-03-19T01:39:07.930691+0000","last_clean":"2024-03-19T01:39:07.930691+0000","last_became_active":"2024-03-19T01:38:55.790363+0000","last_became_peered":"2024-03-19T01:38:55.790363+0000","last_unstale":"2024-03-19T01:39:07.930691+0000","last_undegraded":"2024-03-19T01:39:07.930691+0000","last_fullsized":"2024-03-19T01:39:07.930691+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T09:00:27.595598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,11,6],"acting":[4,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"89'7","reported_seq":24,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:40:06.779387+0000","last_change":"2024-03-19T01:38:56.793369+0000","last_active":"2024-03-19T01:40:06.779387+0000","last_peered":"2024-03-19T01:40:06.779387+0000","last_clean":"2024-03-19T01:40:06.779387+0000","last_became_active":"2024-03-19T01:38:56.793068+0000","last_became_peered":"2024-03-19T01:38:56.793068+0000","last_unstale":"2024-03-19T01:40:06.779387+0000","last_undegraded":"2024-03-19T01:40:06.779387+0000","last_fullsized":"2024-03-19T01:40:06.779387+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T13:12:57.954100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"87'2","reported_seq":31,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.839372+0000","last_change":"2024-03-19T01:39:07.839372+0000","last_active":"2024-03-19T01:39:07.839372+0000","last_peered":"2024-03-19T01:39:07.839372+0000","last_clean":"2024-03-19T01:39:07.839372+0000","last_became_active":"2024-03-19T01:39:07.836229+0000","last_became_peered":"2024-03-19T01:39:07.836229+0000","last_unstale":"2024-03-19T01:39:07.839372+0000","last_undegraded":"2024-03-19T01:39:07.839372+0000","last_fullsized":"2024-03-19T01:39:07.839372+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T12:28:33.102856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,11,0],"acting":[1,11,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"89'2","reported_seq":21,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-19T01:40:07.828339+0000","last_change":"2024-03-19T01:38:56.802272+0000","last_active":"2024-03-19T01:40:07.828339+0000","last_peered":"2024-03-19T01:40:07.828339+0000","last_clean":"2024-03-19T01:40:07.828339+0000","last_became_active":"2024-03-19T01:38:56.801071+0000","last_became_peered":"2024-03-19T01:38:56.801071+0000","last_unstale":"2024-03-19T01:40:07.828339+0000","last_undegraded":"2024-03-19T01:40:07.828339+0000","last_fullsized":"2024-03-19T01:40:07.828339+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T06:58:12.762975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,10],"acting":[8,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.9","version":"87'3","reported_seq":22,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.834006+0000","last_change":"2024-03-19T01:38:55.795186+0000","last_active":"2024-03-19T01:39:07.834006+0000","last_peered":"2024-03-19T01:39:07.834006+0000","last_clean":"2024-03-19T01:39:07.834006+0000","last_became_active":"2024-03-19T01:38:55.794635+0000","last_became_peered":"2024-03-19T01:38:55.794635+0000","last_unstale":"2024-03-19T01:39:07.834006+0000","last_undegraded":"2024-03-19T01:39:07.834006+0000","last_fullsized":"2024-03-19T01:39:07.834006+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T03:55:56.322716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,9,10],"acting":[3,9,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":17,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.830652+0000","last_change":"2024-03-19T01:38:56.812656+0000","last_active":"2024-03-19T01:39:07.830652+0000","last_peered":"2024-03-19T01:39:07.830652+0000","last_clean":"2024-03-19T01:39:07.830652+0000","last_became_active":"2024-03-19T01:38:56.811824+0000","last_became_peered":"2024-03-19T01:38:56.811824+0000","last_unstale":"2024-03-19T01:39:07.830652+0000","last_undegraded":"2024-03-19T01:39:07.830652+0000","last_fullsized":"2024-03-19T01:39:07.830652+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T04:28:11.866892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,0,2],"acting":[11,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.a","version":"87'1","reported_seq":20,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.829668+0000","last_change":"2024-03-19T01:38:55.791193+0000","last_active":"2024-03-19T01:39:07.829668+0000","last_peered":"2024-03-19T01:39:07.829668+0000","last_clean":"2024-03-19T01:39:07.829668+0000","last_became_active":"2024-03-19T01:38:55.790820+0000","last_became_peered":"2024-03-19T01:38:55.790820+0000","last_unstale":"2024-03-19T01:39:07.829668+0000","last_undegraded":"2024-03-19T01:39:07.829668+0000","last_fullsized":"2024-03-19T01:39:07.829668+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T10:19:09.564798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,8],"acting":[2,3,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":17,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.833991+0000","last_change":"2024-03-19T01:38:56.803287+0000","last_active":"2024-03-19T01:39:07.833991+0000","last_peered":"2024-03-19T01:39:07.833991+0000","last_clean":"2024-03-19T01:39:07.833991+0000","last_became_active":"2024-03-19T01:38:56.802924+0000","last_became_peered":"2024-03-19T01:38:56.802924+0000","last_unstale":"2024-03-19T01:39:07.833991+0000","last_undegraded":"2024-03-19T01:39:07.833991+0000","last_fullsized":"2024-03-19T01:39:07.833991+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T07:45:23.726378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,10],"acting":[3,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"87'4","reported_seq":17,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.840733+0000","last_change":"2024-03-19T01:39:07.840234+0000","last_active":"2024-03-19T01:39:07.840733+0000","last_peered":"2024-03-19T01:39:07.840733+0000","last_clean":"2024-03-19T01:39:07.840733+0000","last_became_active":"2024-03-19T01:39:07.835111+0000","last_became_peered":"2024-03-19T01:39:07.835111+0000","last_unstale":"2024-03-19T01:39:07.840733+0000","last_undegraded":"2024-03-19T01:39:07.840733+0000","last_fullsized":"2024-03-19T01:39:07.840733+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T03:30:22.558494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,2,7],"acting":[11,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":17,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.828877+0000","last_change":"2024-03-19T01:38:56.797272+0000","last_active":"2024-03-19T01:39:07.828877+0000","last_peered":"2024-03-19T01:39:07.828877+0000","last_clean":"2024-03-19T01:39:07.828877+0000","last_became_active":"2024-03-19T01:38:56.796944+0000","last_became_peered":"2024-03-19T01:38:56.796944+0000","last_unstale":"2024-03-19T01:39:07.828877+0000","last_undegraded":"2024-03-19T01:39:07.828877+0000","last_fullsized":"2024-03-19T01:39:07.828877+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T06:14:58.147093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,6],"acting":[7,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.c","version":"87'3","reported_seq":23,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.830064+0000","last_change":"2024-03-19T01:38:55.795642+0000","last_active":"2024-03-19T01:39:07.830064+0000","last_peered":"2024-03-19T01:39:07.830064+0000","last_clean":"2024-03-19T01:39:07.830064+0000","last_became_active":"2024-03-19T01:38:55.795285+0000","last_became_peered":"2024-03-19T01:38:55.795285+0000","last_unstale":"2024-03-19T01:39:07.830064+0000","last_undegraded":"2024-03-19T01:39:07.830064+0000","last_fullsized":"2024-03-19T01:39:07.830064+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T10:09:18.888802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,11,0],"acting":[2,11,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":17,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.829092+0000","last_change":"2024-03-19T01:38:56.801802+0000","last_active":"2024-03-19T01:39:07.829092+0000","last_peered":"2024-03-19T01:39:07.829092+0000","last_clean":"2024-03-19T01:39:07.829092+0000","last_became_active":"2024-03-19T01:38:56.801130+0000","last_became_peered":"2024-03-19T01:38:56.801130+0000","last_unstale":"2024-03-19T01:39:07.829092+0000","last_undegraded":"2024-03-19T01:39:07.829092+0000","last_fullsized":"2024-03-19T01:39:07.829092+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T10:36:45.509217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,11],"acting":[7,1,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"86'1","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.832928+0000","last_change":"2024-03-19T01:38:55.795085+0000","last_active":"2024-03-19T01:39:07.832928+0000","last_peered":"2024-03-19T01:39:07.832928+0000","last_clean":"2024-03-19T01:39:07.832928+0000","last_became_active":"2024-03-19T01:38:55.794626+0000","last_became_peered":"2024-03-19T01:38:55.794626+0000","last_unstale":"2024-03-19T01:39:07.832928+0000","last_undegraded":"2024-03-19T01:39:07.832928+0000","last_fullsized":"2024-03-19T01:39:07.832928+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T05:51:50.428331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,10,3],"acting":[5,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"87'1","reported_seq":18,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.829148+0000","last_change":"2024-03-19T01:38:56.802141+0000","last_active":"2024-03-19T01:39:07.829148+0000","last_peered":"2024-03-19T01:39:07.829148+0000","last_clean":"2024-03-19T01:39:07.829148+0000","last_became_active":"2024-03-19T01:38:56.801788+0000","last_became_peered":"2024-03-19T01:38:56.801788+0000","last_unstale":"2024-03-19T01:39:07.829148+0000","last_undegraded":"2024-03-19T01:39:07.829148+0000","last_fullsized":"2024-03-19T01:39:07.829148+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T10:13:39.264272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"86'2","reported_seq":31,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.933658+0000","last_change":"2024-03-19T01:39:07.933658+0000","last_active":"2024-03-19T01:39:07.933658+0000","last_peered":"2024-03-19T01:39:07.933658+0000","last_clean":"2024-03-19T01:39:07.933658+0000","last_became_active":"2024-03-19T01:39:07.931377+0000","last_became_peered":"2024-03-19T01:39:07.931377+0000","last_unstale":"2024-03-19T01:39:07.933658+0000","last_undegraded":"2024-03-19T01:39:07.933658+0000","last_fullsized":"2024-03-19T01:39:07.933658+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T10:09:37.509763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":17,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.829226+0000","last_change":"2024-03-19T01:38:56.794428+0000","last_active":"2024-03-19T01:39:07.829226+0000","last_peered":"2024-03-19T01:39:07.829226+0000","last_clean":"2024-03-19T01:39:07.829226+0000","last_became_active":"2024-03-19T01:38:56.794051+0000","last_became_peered":"2024-03-19T01:38:56.794051+0000","last_unstale":"2024-03-19T01:39:07.829226+0000","last_undegraded":"2024-03-19T01:39:07.829226+0000","last_fullsized":"2024-03-19T01:39:07.829226+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T02:07:20.707276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,11,8],"acting":[7,11,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":17,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.830755+0000","last_change":"2024-03-19T01:38:56.812596+0000","last_active":"2024-03-19T01:39:07.830755+0000","last_peered":"2024-03-19T01:39:07.830755+0000","last_clean":"2024-03-19T01:39:07.830755+0000","last_became_active":"2024-03-19T01:38:56.812268+0000","last_became_peered":"2024-03-19T01:38:56.812268+0000","last_unstale":"2024-03-19T01:39:07.830755+0000","last_undegraded":"2024-03-19T01:39:07.830755+0000","last_fullsized":"2024-03-19T01:39:07.830755+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T08:54:51.350908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,2,5],"acting":[11,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":17,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.829411+0000","last_change":"2024-03-19T01:38:56.804280+0000","last_active":"2024-03-19T01:39:07.829411+0000","last_peered":"2024-03-19T01:39:07.829411+0000","last_clean":"2024-03-19T01:39:07.829411+0000","last_became_active":"2024-03-19T01:38:56.801928+0000","last_became_peered":"2024-03-19T01:38:56.801928+0000","last_unstale":"2024-03-19T01:39:07.829411+0000","last_undegraded":"2024-03-19T01:39:07.829411+0000","last_fullsized":"2024-03-19T01:39:07.829411+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T11:49:29.778949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":17,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.830015+0000","last_change":"2024-03-19T01:38:56.798719+0000","last_active":"2024-03-19T01:39:07.830015+0000","last_peered":"2024-03-19T01:39:07.830015+0000","last_clean":"2024-03-19T01:39:07.830015+0000","last_became_active":"2024-03-19T01:38:56.798253+0000","last_became_peered":"2024-03-19T01:38:56.798253+0000","last_unstale":"2024-03-19T01:39:07.830015+0000","last_undegraded":"2024-03-19T01:39:07.830015+0000","last_fullsized":"2024-03-19T01:39:07.830015+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T06:27:56.754679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.23","version":"89'7","reported_seq":24,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:40:06.776859+0000","last_change":"2024-03-19T01:38:56.795221+0000","last_active":"2024-03-19T01:40:06.776859+0000","last_peered":"2024-03-19T01:40:06.776859+0000","last_clean":"2024-03-19T01:40:06.776859+0000","last_became_active":"2024-03-19T01:38:56.794756+0000","last_became_peered":"2024-03-19T01:38:56.794756+0000","last_unstale":"2024-03-19T01:40:06.776859+0000","last_undegraded":"2024-03-19T01:40:06.776859+0000","last_fullsized":"2024-03-19T01:40:06.776859+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T10:05:54.394976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.24","version":"89'4","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:40:06.777488+0000","last_change":"2024-03-19T01:38:56.794482+0000","last_active":"2024-03-19T01:40:06.777488+0000","last_peered":"2024-03-19T01:40:06.777488+0000","last_clean":"2024-03-19T01:40:06.777488+0000","last_became_active":"2024-03-19T01:38:56.794140+0000","last_became_peered":"2024-03-19T01:38:56.794140+0000","last_unstale":"2024-03-19T01:40:06.777488+0000","last_undegraded":"2024-03-19T01:40:06.777488+0000","last_fullsized":"2024-03-19T01:40:06.777488+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T04:29:38.985047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":17,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.831118+0000","last_change":"2024-03-19T01:38:56.802020+0000","last_active":"2024-03-19T01:39:07.831118+0000","last_peered":"2024-03-19T01:39:07.831118+0000","last_clean":"2024-03-19T01:39:07.831118+0000","last_became_active":"2024-03-19T01:38:56.801668+0000","last_became_peered":"2024-03-19T01:38:56.801668+0000","last_unstale":"2024-03-19T01:39:07.831118+0000","last_undegraded":"2024-03-19T01:39:07.831118+0000","last_fullsized":"2024-03-19T01:39:07.831118+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T04:13:55.199279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,8,5],"acting":[11,8,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":17,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.829784+0000","last_change":"2024-03-19T01:38:56.809100+0000","last_active":"2024-03-19T01:39:07.829784+0000","last_peered":"2024-03-19T01:39:07.829784+0000","last_clean":"2024-03-19T01:39:07.829784+0000","last_became_active":"2024-03-19T01:38:56.807985+0000","last_became_peered":"2024-03-19T01:38:56.807985+0000","last_unstale":"2024-03-19T01:39:07.829784+0000","last_undegraded":"2024-03-19T01:39:07.829784+0000","last_fullsized":"2024-03-19T01:39:07.829784+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T02:58:25.235837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,8,1],"acting":[2,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":17,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.833738+0000","last_change":"2024-03-19T01:38:56.799707+0000","last_active":"2024-03-19T01:39:07.833738+0000","last_peered":"2024-03-19T01:39:07.833738+0000","last_clean":"2024-03-19T01:39:07.833738+0000","last_became_active":"2024-03-19T01:38:56.799388+0000","last_became_peered":"2024-03-19T01:38:56.799388+0000","last_unstale":"2024-03-19T01:39:07.833738+0000","last_undegraded":"2024-03-19T01:39:07.833738+0000","last_fullsized":"2024-03-19T01:39:07.833738+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T11:10:18.360180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"89'6","reported_seq":23,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:40:06.772406+0000","last_change":"2024-03-19T01:38:56.786807+0000","last_active":"2024-03-19T01:40:06.772406+0000","last_peered":"2024-03-19T01:40:06.772406+0000","last_clean":"2024-03-19T01:40:06.772406+0000","last_became_active":"2024-03-19T01:38:56.786395+0000","last_became_peered":"2024-03-19T01:38:56.786395+0000","last_unstale":"2024-03-19T01:40:06.772406+0000","last_undegraded":"2024-03-19T01:40:06.772406+0000","last_fullsized":"2024-03-19T01:40:06.772406+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T05:19:20.228926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,9,3],"acting":[5,9,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":17,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.833971+0000","last_change":"2024-03-19T01:38:56.800759+0000","last_active":"2024-03-19T01:39:07.833971+0000","last_peered":"2024-03-19T01:39:07.833971+0000","last_clean":"2024-03-19T01:39:07.833971+0000","last_became_active":"2024-03-19T01:38:56.800462+0000","last_became_peered":"2024-03-19T01:38:56.800462+0000","last_unstale":"2024-03-19T01:39:07.833971+0000","last_undegraded":"2024-03-19T01:39:07.833971+0000","last_fullsized":"2024-03-19T01:39:07.833971+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T02:13:19.070148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,11],"acting":[3,2,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":17,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.831737+0000","last_change":"2024-03-19T01:38:56.802653+0000","last_active":"2024-03-19T01:39:07.831737+0000","last_peered":"2024-03-19T01:39:07.831737+0000","last_clean":"2024-03-19T01:39:07.831737+0000","last_became_active":"2024-03-19T01:38:56.802284+0000","last_became_peered":"2024-03-19T01:38:56.802284+0000","last_unstale":"2024-03-19T01:39:07.831737+0000","last_undegraded":"2024-03-19T01:39:07.831737+0000","last_fullsized":"2024-03-19T01:39:07.831737+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T06:17:13.936359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2b","version":"89'11","reported_seq":28,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:40:06.778680+0000","last_change":"2024-03-19T01:38:56.794668+0000","last_active":"2024-03-19T01:40:06.778680+0000","last_peered":"2024-03-19T01:40:06.778680+0000","last_clean":"2024-03-19T01:40:06.778680+0000","last_became_active":"2024-03-19T01:38:56.794375+0000","last_became_peered":"2024-03-19T01:38:56.794375+0000","last_unstale":"2024-03-19T01:40:06.778680+0000","last_undegraded":"2024-03-19T01:40:06.778680+0000","last_fullsized":"2024-03-19T01:40:06.778680+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T08:26:20.528825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":17,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.830773+0000","last_change":"2024-03-19T01:38:56.805535+0000","last_active":"2024-03-19T01:39:07.830773+0000","last_peered":"2024-03-19T01:39:07.830773+0000","last_clean":"2024-03-19T01:39:07.830773+0000","last_became_active":"2024-03-19T01:38:56.805211+0000","last_became_peered":"2024-03-19T01:38:56.805211+0000","last_unstale":"2024-03-19T01:39:07.830773+0000","last_undegraded":"2024-03-19T01:39:07.830773+0000","last_fullsized":"2024-03-19T01:39:07.830773+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T04:47:28.688137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,4,8],"acting":[11,4,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":234,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":702,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":173,"ondisk_log_size":173,"up":192,"acting":192,"num_store_stats":12},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":109,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4157440,"data_stored":4132576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":137,"ondisk_log_size":137,"up":3,"acting":3,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":64697,"num_objects":60,"num_object_clones":0,"num_object_copies":180,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":60,"num_whiteouts":0,"num_read":425,"num_read_kb":279,"num_write":160,"num_write_kb":113,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":40,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":417792,"data_stored":193527,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":96,"acting":96,"num_store_stats":12}],"osd_stats":[{"osd":11,"up_from":82,"seq":352187318299,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27972,"kb_used_data":948,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717180,"statfs":{"total":95995035648,"available":95966392320,"internally_reserved":0,"allocated":970752,"data_stored":465376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1596,"internal_metadata":27654596},"hb_peers":[0,1,2,4,5,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":10,"up_from":74,"seq":317827579937,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32804,"kb_used_data":1628,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93712348,"statfs":{"total":95995035648,"available":95961444352,"internally_reserved":0,"allocated":1667072,"data_stored":1117271,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1600,"internal_metadata":31914432},"hb_peers":[0,1,2,3,4,5,6,8,9,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":9,"up_from":68,"seq":292057776166,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32144,"kb_used_data":964,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93713008,"statfs":{"total":95995035648,"available":95962120192,"internally_reserved":0,"allocated":987136,"data_stored":465612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1599,"internal_metadata":31914433},"hb_peers":[0,1,2,3,4,5,6,7,8,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":8,"up_from":60,"seq":257698037802,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28000,"kb_used_data":980,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717152,"statfs":{"total":95995035648,"available":95966363648,"internally_reserved":0,"allocated":1003520,"data_stored":474059,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,2,3,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":7,"up_from":53,"seq":227633266736,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32728,"kb_used_data":1544,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93712424,"statfs":{"total":95995035648,"available":95961522176,"internally_reserved":0,"allocated":1581056,"data_stored":1057853,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":31914443},"hb_peers":[1,2,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863462965,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32132,"kb_used_data":952,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93713020,"statfs":{"total":95995035648,"available":95962132480,"internally_reserved":0,"allocated":974848,"data_stored":465001,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":31914440},"hb_peers":[0,1,2,3,4,5,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":1,"up_from":13,"seq":55834574924,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28028,"kb_used_data":1012,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717124,"statfs":{"total":95995035648,"available":95966334976,"internally_reserved":0,"allocated":1036288,"data_stored":524817,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2,3,4,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738450,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36308,"kb_used_data":972,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":93708844,"statfs":{"total":95995035648,"available":95957856256,"internally_reserved":0,"allocated":995328,"data_stored":466272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":36174277},"hb_peers":[1,2,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":18,"seq":77309411399,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28052,"kb_used_data":1036,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717100,"statfs":{"total":95995035648,"available":95966310400,"internally_reserved":0,"allocated":1060864,"data_stored":526558,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":27,"seq":115964117059,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32152,"kb_used_data":968,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93713000,"statfs":{"total":95995035648,"available":95962112000,"internally_reserved":0,"allocated":991232,"data_stored":465312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31914439},"hb_peers":[0,1,2,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":34,"seq":146028888125,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27972,"kb_used_data":956,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717180,"statfs":{"total":95995035648,"available":95966392320,"internally_reserved":0,"allocated":978944,"data_stored":465970,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2,3,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":40,"seq":171798691897,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36896,"kb_used_data":1556,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":93708256,"statfs":{"total":95995035648,"available":95957254144,"internally_reserved":0,"allocated":1593344,"data_stored":1057146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":36174277},"hb_peers":[0,1,2,3,4,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1478,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":69632,"data_stored":60023,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":94208,"data_stored":61764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":2008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":2574,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":738,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":818,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":61992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":234,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-19T01:40:10.146 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph pg dump --format=json 2024-03-19T01:40:10.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:09 smithi115 ceph-mon[67332]: pgmap v321: 97 pgs: 97 active+clean; 640 KiB data, 366 MiB used, 1.0 TiB / 1.0 TiB avail; 3.0 KiB/s rd, 307 B/s wr, 6 op/s 2024-03-19T01:40:10.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:09 smithi115 ceph-mon[67332]: osdmap e91: 12 total, 12 up, 12 in 2024-03-19T01:40:10.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:09 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:40:10.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:09 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:40:10.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:09 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:40:10.393 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:40:11.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:10 smithi099 ceph-mon[62951]: from='client.24811 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-19T01:40:11.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:10 smithi112 ceph-mon[66819]: from='client.24811 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-19T01:40:11.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:10 smithi115 ceph-mon[67332]: from='client.24811 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-19T01:40:12.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:11 smithi099 ceph-mon[62951]: pgmap v323: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.2 KiB/s rd, 895 B/s wr, 6 op/s 2024-03-19T01:40:12.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:11 smithi112 ceph-mon[66819]: pgmap v323: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.2 KiB/s rd, 895 B/s wr, 6 op/s 2024-03-19T01:40:12.134 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:12.133+0000 7f177d129700 1 -- 172.21.15.99:0/166360263 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f177810edf0 msgr2=0x7f17781111e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:12.134 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:12.133+0000 7f177d129700 1 --2- 172.21.15.99:0/166360263 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f177810edf0 0x7f17781111e0 secure :-1 s=READY pgs=271 cs=0 l=1 rev1=1 crypto rx=0x7f177000b0a0 tx=0x7f177000b3b0 comp rx=0 tx=0).stop 2024-03-19T01:40:12.134 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:12.133+0000 7f177d129700 1 -- 172.21.15.99:0/166360263 shutdown_connections 2024-03-19T01:40:12.134 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:12.133+0000 7f177d129700 1 --2- 172.21.15.99:0/166360263 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f177806aa70 0x7f177806ae50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:12.134 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:12.133+0000 7f177d129700 1 --2- 172.21.15.99:0/166360263 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f177810edf0 0x7f17781111e0 unknown :-1 s=CLOSED pgs=271 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:12.134 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:12.133+0000 7f177d129700 1 --2- 172.21.15.99:0/166360263 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f177806b390 0x7f177810e8b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:12.134 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:12.133+0000 7f177d129700 1 -- 172.21.15.99:0/166360263 >> 172.21.15.99:0/166360263 conn(0x7f1778077250 msgr2=0x7f1778077660 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:12.134 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:12.133+0000 7f177d129700 1 -- 172.21.15.99:0/166360263 shutdown_connections 2024-03-19T01:40:12.135 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:12.133+0000 7f177d129700 1 -- 172.21.15.99:0/166360263 wait complete. 2024-03-19T01:40:12.136 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:12.134+0000 7f177d129700 1 Processor -- start 2024-03-19T01:40:12.137 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:12.134+0000 7f177d129700 1 -- start start 2024-03-19T01:40:12.137 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:12.135+0000 7f177d129700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f177806aa70 0x7f17781a08e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:12.137 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:12.135+0000 7f177d129700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f177806b390 0x7f17781a0e20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:12.137 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:12.135+0000 7f177d129700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f177810edf0 0x7f177819aaa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:12.137 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:12.135+0000 7f177d129700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f1778114800 con 0x7f177810edf0 2024-03-19T01:40:12.137 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:12.135+0000 7f177d129700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f1778114680 con 0x7f177806b390 2024-03-19T01:40:12.137 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:12.135+0000 7f177d129700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f1778114980 con 0x7f177806aa70 2024-03-19T01:40:12.137 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:12.135+0000 7f1776d9d700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f177806aa70 0x7f17781a08e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:12.137 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:12.135+0000 7f177759e700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f177810edf0 0x7f177819aaa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:12.137 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:12.135+0000 7f1776d9d700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f177806aa70 0x7f17781a08e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:47186/0 (socket says 172.21.15.99:47186) 2024-03-19T01:40:12.137 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:12.135+0000 7f1776d9d700 1 -- 172.21.15.99:0/1172721400 learned_addr learned my addr 172.21.15.99:0/1172721400 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:12.137 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:12.135+0000 7f177659c700 1 --2- 172.21.15.99:0/1172721400 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f177806b390 0x7f17781a0e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:12.137 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:12.135+0000 7f1776d9d700 1 -- 172.21.15.99:0/1172721400 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f177806b390 msgr2=0x7f17781a0e20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:12.138 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:12.135+0000 7f1776d9d700 1 --2- 172.21.15.99:0/1172721400 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f177806b390 0x7f17781a0e20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:12.138 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:12.135+0000 7f1776d9d700 1 -- 172.21.15.99:0/1172721400 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f177810edf0 msgr2=0x7f177819aaa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:12.138 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:12.135+0000 7f1776d9d700 1 --2- 172.21.15.99:0/1172721400 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f177810edf0 0x7f177819aaa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:12.138 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:12.135+0000 7f1776d9d700 1 -- 172.21.15.99:0/1172721400 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f177819b200 con 0x7f177806aa70 2024-03-19T01:40:12.138 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:12.135+0000 7f177659c700 1 --2- 172.21.15.99:0/1172721400 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f177806b390 0x7f17781a0e20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:40:12.138 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:12.135+0000 7f1776d9d700 1 --2- 172.21.15.99:0/1172721400 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f177806aa70 0x7f17781a08e0 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x7f176800e9c0 tx=0x7f176800ecd0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:12.138 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:12.135+0000 7f175ffff700 1 -- 172.21.15.99:0/1172721400 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f176800cbd0 con 0x7f177806aa70 2024-03-19T01:40:12.138 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:12.135+0000 7f175ffff700 1 -- 172.21.15.99:0/1172721400 <== mon.1 v2:172.21.15.115:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f176800ee60 con 0x7f177806aa70 2024-03-19T01:40:12.138 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:12.135+0000 7f177d129700 1 -- 172.21.15.99:0/1172721400 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f177819b4f0 con 0x7f177806aa70 2024-03-19T01:40:12.138 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:12.135+0000 7f175ffff700 1 -- 172.21.15.99:0/1172721400 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1768005630 con 0x7f177806aa70 2024-03-19T01:40:12.138 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:12.136+0000 7f177d129700 1 -- 172.21.15.99:0/1172721400 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f17781a77b0 con 0x7f177806aa70 2024-03-19T01:40:12.138 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:12.136+0000 7f177d129700 1 -- 172.21.15.99:0/1172721400 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f1778068fd0 con 0x7f177806aa70 2024-03-19T01:40:12.139 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:12.138+0000 7f175ffff700 1 -- 172.21.15.99:0/1172721400 <== mon.1 v2:172.21.15.115:3300/0 4 ==== mgrmap(e 18) ==== 137290+0+0 (secure 0 0 0) 0x7f1768004750 con 0x7f177806aa70 2024-03-19T01:40:12.139 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:12.138+0000 7f175ffff700 1 --2- 172.21.15.99:0/1172721400 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f17600a2890 0x7f17600a4d50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:12.139 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:12.138+0000 7f177659c700 1 --2- 172.21.15.99:0/1172721400 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f17600a2890 0x7f17600a4d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:12.139 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:12.138+0000 7f175ffff700 1 -- 172.21.15.99:0/1172721400 <== mon.1 v2:172.21.15.115:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f17680cefb0 con 0x7f177806aa70 2024-03-19T01:40:12.139 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:12.138+0000 7f177659c700 1 --2- 172.21.15.99:0/1172721400 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f17600a2890 0x7f17600a4d50 secure :-1 s=READY pgs=206 cs=0 l=1 rev1=1 crypto rx=0x7f1778100f70 tx=0x7f176c009260 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:12.142 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:12.142+0000 7f175ffff700 1 -- 172.21.15.99:0/1172721400 <== mon.1 v2:172.21.15.115:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f1768053740 con 0x7f177806aa70 2024-03-19T01:40:12.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:11 smithi115 ceph-mon[67332]: pgmap v323: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.2 KiB/s rd, 895 B/s wr, 6 op/s 2024-03-19T01:40:12.333 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:12.333+0000 7f177d129700 1 -- 172.21.15.99:0/1172721400 --> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7f1778062750 con 0x7f17600a2890 2024-03-19T01:40:12.340 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:12.340+0000 7f175ffff700 1 -- 172.21.15.99:0/1172721400 <== mgr.14150 v2:172.21.15.99:6800/4272410295 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+258867 (secure 0 0 0) 0x7f1778062750 con 0x7f17600a2890 2024-03-19T01:40:12.346 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:40:12.350 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:12.343+0000 7f177d129700 1 -- 172.21.15.99:0/1172721400 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f17600a2890 msgr2=0x7f17600a4d50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:12.350 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:12.343+0000 7f177d129700 1 --2- 172.21.15.99:0/1172721400 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f17600a2890 0x7f17600a4d50 secure :-1 s=READY pgs=206 cs=0 l=1 rev1=1 crypto rx=0x7f1778100f70 tx=0x7f176c009260 comp rx=0 tx=0).stop 2024-03-19T01:40:12.350 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:12.343+0000 7f177d129700 1 -- 172.21.15.99:0/1172721400 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f177806aa70 msgr2=0x7f17781a08e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:12.350 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:12.343+0000 7f177d129700 1 --2- 172.21.15.99:0/1172721400 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f177806aa70 0x7f17781a08e0 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x7f176800e9c0 tx=0x7f176800ecd0 comp rx=0 tx=0).stop 2024-03-19T01:40:12.351 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:12.343+0000 7f177d129700 1 -- 172.21.15.99:0/1172721400 shutdown_connections 2024-03-19T01:40:12.351 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:12.343+0000 7f177d129700 1 --2- 172.21.15.99:0/1172721400 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f17600a2890 0x7f17600a4d50 unknown :-1 s=CLOSED pgs=206 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:12.351 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:12.343+0000 7f177d129700 1 --2- 172.21.15.99:0/1172721400 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f177806aa70 0x7f17781a08e0 unknown :-1 s=CLOSED pgs=69 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:12.351 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:12.343+0000 7f177d129700 1 --2- 172.21.15.99:0/1172721400 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f177810edf0 0x7f177819aaa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:12.351 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:12.343+0000 7f177d129700 1 --2- 172.21.15.99:0/1172721400 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f177806b390 0x7f17781a0e20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:12.351 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:12.343+0000 7f177d129700 1 -- 172.21.15.99:0/1172721400 >> 172.21.15.99:0/1172721400 conn(0x7f1778077250 msgr2=0x7f1778110590 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:12.351 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:12.343+0000 7f177d129700 1 -- 172.21.15.99:0/1172721400 shutdown_connections 2024-03-19T01:40:12.352 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:12.343+0000 7f177d129700 1 -- 172.21.15.99:0/1172721400 wait complete. 2024-03-19T01:40:12.352 INFO:teuthology.orchestra.run.smithi099.stderr:dumped all 2024-03-19T01:40:13.056 INFO:teuthology.orchestra.run.smithi099.stdout:{"pg_ready":true,"pg_map":{"version":323,"stamp":"2024-03-19T01:40:10.759718+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":655548,"num_objects":64,"num_object_clones":0,"num_object_copies":192,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":64,"num_whiteouts":0,"num_read":567,"num_read_kb":399,"num_write":347,"num_write_kb":3057,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":40,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":626,"ondisk_log_size":626,"up":291,"acting":291,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":291,"num_osds":12,"num_per_pool_osds":12,"num_per_pool_omap_osds":12,"kb":1124941824,"kb_used":375332,"kb_used_data":13644,"kb_used_omap":18,"kb_used_meta":361517,"kb_avail":1124566492,"statfs":{"total":1151940427776,"available":1151556087808,"internally_reserved":0,"allocated":13971456,"data_stored":7620159,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19139,"internal_metadata":370193725},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":531,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":26,"num_write":7,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002563"},"pg_stats":[{"pgid":"3.3f","version":"89'8","reported_seq":29,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-03-19T01:40:08.988006+0000","last_change":"2024-03-19T01:38:56.809406+0000","last_active":"2024-03-19T01:40:08.988006+0000","last_peered":"2024-03-19T01:40:08.988006+0000","last_clean":"2024-03-19T01:40:08.988006+0000","last_became_active":"2024-03-19T01:38:56.808971+0000","last_became_peered":"2024-03-19T01:38:56.808971+0000","last_unstale":"2024-03-19T01:40:08.988006+0000","last_undegraded":"2024-03-19T01:40:08.988006+0000","last_fullsized":"2024-03-19T01:40:08.988006+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T11:07:58.551704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.3e","version":"89'4","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:40:06.772429+0000","last_change":"2024-03-19T01:38:56.798847+0000","last_active":"2024-03-19T01:40:06.772429+0000","last_peered":"2024-03-19T01:40:06.772429+0000","last_clean":"2024-03-19T01:40:06.772429+0000","last_became_active":"2024-03-19T01:38:56.795442+0000","last_became_peered":"2024-03-19T01:38:56.795442+0000","last_unstale":"2024-03-19T01:40:06.772429+0000","last_undegraded":"2024-03-19T01:40:06.772429+0000","last_fullsized":"2024-03-19T01:40:06.772429+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T12:05:55.721962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,11,10],"acting":[0,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3d","version":"89'7","reported_seq":24,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:40:06.777513+0000","last_change":"2024-03-19T01:38:56.806317+0000","last_active":"2024-03-19T01:40:06.777513+0000","last_peered":"2024-03-19T01:40:06.777513+0000","last_clean":"2024-03-19T01:40:06.777513+0000","last_became_active":"2024-03-19T01:38:56.806001+0000","last_became_peered":"2024-03-19T01:38:56.806001+0000","last_unstale":"2024-03-19T01:40:06.777513+0000","last_undegraded":"2024-03-19T01:40:06.777513+0000","last_fullsized":"2024-03-19T01:40:06.777513+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T05:19:45.501431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"89'8","reported_seq":25,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:40:06.766516+0000","last_change":"2024-03-19T01:38:56.803611+0000","last_active":"2024-03-19T01:40:06.766516+0000","last_peered":"2024-03-19T01:40:06.766516+0000","last_clean":"2024-03-19T01:40:06.766516+0000","last_became_active":"2024-03-19T01:38:56.803300+0000","last_became_peered":"2024-03-19T01:38:56.803300+0000","last_unstale":"2024-03-19T01:40:06.766516+0000","last_undegraded":"2024-03-19T01:40:06.766516+0000","last_fullsized":"2024-03-19T01:40:06.766516+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T06:05:49.287266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,8,2],"acting":[4,8,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.3b","version":"89'4","reported_seq":23,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-19T01:40:07.829176+0000","last_change":"2024-03-19T01:38:56.799910+0000","last_active":"2024-03-19T01:40:07.829176+0000","last_peered":"2024-03-19T01:40:07.829176+0000","last_clean":"2024-03-19T01:40:07.829176+0000","last_became_active":"2024-03-19T01:38:56.799567+0000","last_became_peered":"2024-03-19T01:38:56.799567+0000","last_unstale":"2024-03-19T01:40:07.829176+0000","last_undegraded":"2024-03-19T01:40:07.829176+0000","last_fullsized":"2024-03-19T01:40:07.829176+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T05:59:42.563083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,3,6],"acting":[8,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.3a","version":"89'6","reported_seq":33,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-03-19T01:40:08.987894+0000","last_change":"2024-03-19T01:39:07.833027+0000","last_active":"2024-03-19T01:40:08.987894+0000","last_peered":"2024-03-19T01:40:08.987894+0000","last_clean":"2024-03-19T01:40:08.987894+0000","last_became_active":"2024-03-19T01:39:07.832700+0000","last_became_peered":"2024-03-19T01:39:07.832700+0000","last_unstale":"2024-03-19T01:40:08.987894+0000","last_undegraded":"2024-03-19T01:40:08.987894+0000","last_fullsized":"2024-03-19T01:40:08.987894+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T10:40:44.588730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,9,6],"acting":[10,9,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":17,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.828627+0000","last_change":"2024-03-19T01:38:56.804091+0000","last_active":"2024-03-19T01:39:07.828627+0000","last_peered":"2024-03-19T01:39:07.828627+0000","last_clean":"2024-03-19T01:39:07.828627+0000","last_became_active":"2024-03-19T01:38:56.802308+0000","last_became_peered":"2024-03-19T01:38:56.802308+0000","last_unstale":"2024-03-19T01:39:07.828627+0000","last_undegraded":"2024-03-19T01:39:07.828627+0000","last_fullsized":"2024-03-19T01:39:07.828627+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T07:35:42.952521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,9,5],"acting":[7,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"89'9","reported_seq":30,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-03-19T01:40:08.987641+0000","last_change":"2024-03-19T01:38:56.796475+0000","last_active":"2024-03-19T01:40:08.987641+0000","last_peered":"2024-03-19T01:40:08.987641+0000","last_clean":"2024-03-19T01:40:08.987641+0000","last_became_active":"2024-03-19T01:38:56.796116+0000","last_became_peered":"2024-03-19T01:38:56.796116+0000","last_unstale":"2024-03-19T01:40:08.987641+0000","last_undegraded":"2024-03-19T01:40:08.987641+0000","last_fullsized":"2024-03-19T01:40:08.987641+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T04:35:19.821098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,10],"acting":[3,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":23,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.836670+0000","last_change":"2024-03-19T01:39:07.836670+0000","last_active":"2024-03-19T01:39:07.836670+0000","last_peered":"2024-03-19T01:39:07.836670+0000","last_clean":"2024-03-19T01:39:07.836670+0000","last_became_active":"2024-03-19T01:39:07.836247+0000","last_became_peered":"2024-03-19T01:39:07.836247+0000","last_unstale":"2024-03-19T01:39:07.836670+0000","last_undegraded":"2024-03-19T01:39:07.836670+0000","last_fullsized":"2024-03-19T01:39:07.836670+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T11:26:39.713658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,11],"acting":[1,7,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.36","version":"89'16","reported_seq":37,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-03-19T01:40:08.988179+0000","last_change":"2024-03-19T01:38:56.797973+0000","last_active":"2024-03-19T01:40:08.988179+0000","last_peered":"2024-03-19T01:40:08.988179+0000","last_clean":"2024-03-19T01:40:08.988179+0000","last_became_active":"2024-03-19T01:38:56.797396+0000","last_became_peered":"2024-03-19T01:38:56.797396+0000","last_unstale":"2024-03-19T01:40:08.988179+0000","last_undegraded":"2024-03-19T01:40:08.988179+0000","last_fullsized":"2024-03-19T01:40:08.988179+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":16,"log_dups_size":0,"ondisk_log_size":16,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T11:46:32.867919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,2,4],"acting":[10,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":17,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.829087+0000","last_change":"2024-03-19T01:38:56.802211+0000","last_active":"2024-03-19T01:39:07.829087+0000","last_peered":"2024-03-19T01:39:07.829087+0000","last_clean":"2024-03-19T01:39:07.829087+0000","last_became_active":"2024-03-19T01:38:56.801776+0000","last_became_peered":"2024-03-19T01:38:56.801776+0000","last_unstale":"2024-03-19T01:39:07.829087+0000","last_undegraded":"2024-03-19T01:39:07.829087+0000","last_fullsized":"2024-03-19T01:39:07.829087+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T10:09:36.983771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,8,1],"acting":[7,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.34","version":"89'7","reported_seq":24,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:40:06.766409+0000","last_change":"2024-03-19T01:38:56.798463+0000","last_active":"2024-03-19T01:40:06.766409+0000","last_peered":"2024-03-19T01:40:06.766409+0000","last_clean":"2024-03-19T01:40:06.766409+0000","last_became_active":"2024-03-19T01:38:56.797187+0000","last_became_peered":"2024-03-19T01:38:56.797187+0000","last_unstale":"2024-03-19T01:40:06.766409+0000","last_undegraded":"2024-03-19T01:40:06.766409+0000","last_fullsized":"2024-03-19T01:40:06.766409+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T11:10:04.349393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,8],"acting":[4,10,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"89'10","reported_seq":27,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:40:06.773337+0000","last_change":"2024-03-19T01:38:56.801650+0000","last_active":"2024-03-19T01:40:06.773337+0000","last_peered":"2024-03-19T01:40:06.773337+0000","last_clean":"2024-03-19T01:40:06.773337+0000","last_became_active":"2024-03-19T01:38:56.801323+0000","last_became_peered":"2024-03-19T01:38:56.801323+0000","last_unstale":"2024-03-19T01:40:06.773337+0000","last_undegraded":"2024-03-19T01:40:06.773337+0000","last_fullsized":"2024-03-19T01:40:06.773337+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T06:10:28.518843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,10,5],"acting":[6,10,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.32","version":"89'7","reported_seq":24,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:40:06.776031+0000","last_change":"2024-03-19T01:38:56.798633+0000","last_active":"2024-03-19T01:40:06.776031+0000","last_peered":"2024-03-19T01:40:06.776031+0000","last_clean":"2024-03-19T01:40:06.776031+0000","last_became_active":"2024-03-19T01:38:56.798342+0000","last_became_peered":"2024-03-19T01:38:56.798342+0000","last_unstale":"2024-03-19T01:40:06.776031+0000","last_undegraded":"2024-03-19T01:40:06.776031+0000","last_fullsized":"2024-03-19T01:40:06.776031+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T03:21:57.350930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,8,10],"acting":[0,8,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"89'7","reported_seq":30,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:40:06.776914+0000","last_change":"2024-03-19T01:39:07.835127+0000","last_active":"2024-03-19T01:40:06.776914+0000","last_peered":"2024-03-19T01:40:06.776914+0000","last_clean":"2024-03-19T01:40:06.776914+0000","last_became_active":"2024-03-19T01:39:07.834731+0000","last_became_peered":"2024-03-19T01:39:07.834731+0000","last_unstale":"2024-03-19T01:40:06.776914+0000","last_undegraded":"2024-03-19T01:40:06.776914+0000","last_fullsized":"2024-03-19T01:40:06.776914+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T11:37:14.867184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,8,2],"acting":[0,8,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":17,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.832062+0000","last_change":"2024-03-19T01:38:56.796556+0000","last_active":"2024-03-19T01:39:07.832062+0000","last_peered":"2024-03-19T01:39:07.832062+0000","last_clean":"2024-03-19T01:39:07.832062+0000","last_became_active":"2024-03-19T01:38:56.796215+0000","last_became_peered":"2024-03-19T01:38:56.796215+0000","last_unstale":"2024-03-19T01:39:07.832062+0000","last_undegraded":"2024-03-19T01:39:07.832062+0000","last_fullsized":"2024-03-19T01:39:07.832062+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T09:12:00.826877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2f","version":"89'8","reported_seq":29,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-03-19T01:40:08.988213+0000","last_change":"2024-03-19T01:38:56.808706+0000","last_active":"2024-03-19T01:40:08.988213+0000","last_peered":"2024-03-19T01:40:08.988213+0000","last_clean":"2024-03-19T01:40:08.988213+0000","last_became_active":"2024-03-19T01:38:56.808065+0000","last_became_peered":"2024-03-19T01:38:56.808065+0000","last_unstale":"2024-03-19T01:40:08.988213+0000","last_undegraded":"2024-03-19T01:40:08.988213+0000","last_fullsized":"2024-03-19T01:40:08.988213+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T08:40:28.256172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":17,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.829173+0000","last_change":"2024-03-19T01:38:56.799501+0000","last_active":"2024-03-19T01:39:07.829173+0000","last_peered":"2024-03-19T01:39:07.829173+0000","last_clean":"2024-03-19T01:39:07.829173+0000","last_became_active":"2024-03-19T01:38:56.799202+0000","last_became_peered":"2024-03-19T01:38:56.799202+0000","last_unstale":"2024-03-19T01:39:07.829173+0000","last_undegraded":"2024-03-19T01:39:07.829173+0000","last_fullsized":"2024-03-19T01:39:07.829173+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T09:42:06.386165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,1,10],"acting":[9,1,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.2d","version":"89'7","reported_seq":28,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-03-19T01:40:08.824465+0000","last_change":"2024-03-19T01:38:56.810697+0000","last_active":"2024-03-19T01:40:08.824465+0000","last_peered":"2024-03-19T01:40:08.824465+0000","last_clean":"2024-03-19T01:40:08.824465+0000","last_became_active":"2024-03-19T01:38:56.810202+0000","last_became_peered":"2024-03-19T01:38:56.810202+0000","last_unstale":"2024-03-19T01:40:08.824465+0000","last_undegraded":"2024-03-19T01:40:08.824465+0000","last_fullsized":"2024-03-19T01:40:08.824465+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T06:22:11.807807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1f","version":"87'9","reported_seq":29,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.829094+0000","last_change":"2024-03-19T01:38:55.791479+0000","last_active":"2024-03-19T01:39:07.829094+0000","last_peered":"2024-03-19T01:39:07.829094+0000","last_clean":"2024-03-19T01:39:07.829094+0000","last_became_active":"2024-03-19T01:38:55.791150+0000","last_became_peered":"2024-03-19T01:38:55.791150+0000","last_unstale":"2024-03-19T01:39:07.829094+0000","last_undegraded":"2024-03-19T01:39:07.829094+0000","last_fullsized":"2024-03-19T01:39:07.829094+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T09:56:07.953602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,7,8],"acting":[9,7,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.1e","version":"89'5","reported_seq":26,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-03-19T01:40:08.988175+0000","last_change":"2024-03-19T01:38:56.802404+0000","last_active":"2024-03-19T01:40:08.988175+0000","last_peered":"2024-03-19T01:40:08.988175+0000","last_clean":"2024-03-19T01:40:08.988175+0000","last_became_active":"2024-03-19T01:38:56.802060+0000","last_became_peered":"2024-03-19T01:38:56.802060+0000","last_unstale":"2024-03-19T01:40:08.988175+0000","last_undegraded":"2024-03-19T01:40:08.988175+0000","last_fullsized":"2024-03-19T01:40:08.988175+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T07:45:34.287073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1e","version":"87'2","reported_seq":25,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-03-19T01:40:08.988326+0000","last_change":"2024-03-19T01:38:55.798249+0000","last_active":"2024-03-19T01:40:08.988326+0000","last_peered":"2024-03-19T01:40:08.988326+0000","last_clean":"2024-03-19T01:40:08.988326+0000","last_became_active":"2024-03-19T01:38:55.797952+0000","last_became_peered":"2024-03-19T01:38:55.797952+0000","last_unstale":"2024-03-19T01:40:08.988326+0000","last_undegraded":"2024-03-19T01:40:08.988326+0000","last_fullsized":"2024-03-19T01:40:08.988326+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T12:12:40.925462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"89'6","reported_seq":23,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:40:06.775843+0000","last_change":"2024-03-19T01:38:56.805992+0000","last_active":"2024-03-19T01:40:06.775843+0000","last_peered":"2024-03-19T01:40:06.775843+0000","last_clean":"2024-03-19T01:40:06.775843+0000","last_became_active":"2024-03-19T01:38:56.804997+0000","last_became_peered":"2024-03-19T01:38:56.804997+0000","last_unstale":"2024-03-19T01:40:06.775843+0000","last_undegraded":"2024-03-19T01:40:06.775843+0000","last_fullsized":"2024-03-19T01:40:06.775843+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T10:33:39.003792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,9],"acting":[0,1,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1d","version":"87'4","reported_seq":25,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-19T01:40:07.829025+0000","last_change":"2024-03-19T01:38:55.795296+0000","last_active":"2024-03-19T01:40:07.829025+0000","last_peered":"2024-03-19T01:40:07.829025+0000","last_clean":"2024-03-19T01:40:07.829025+0000","last_became_active":"2024-03-19T01:38:55.794321+0000","last_became_peered":"2024-03-19T01:38:55.794321+0000","last_unstale":"2024-03-19T01:40:07.829025+0000","last_undegraded":"2024-03-19T01:40:07.829025+0000","last_fullsized":"2024-03-19T01:40:07.829025+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T12:26:12.914801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":17,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.832218+0000","last_change":"2024-03-19T01:38:56.802337+0000","last_active":"2024-03-19T01:39:07.832218+0000","last_peered":"2024-03-19T01:39:07.832218+0000","last_clean":"2024-03-19T01:39:07.832218+0000","last_became_active":"2024-03-19T01:38:56.802032+0000","last_became_peered":"2024-03-19T01:38:56.802032+0000","last_unstale":"2024-03-19T01:39:07.832218+0000","last_undegraded":"2024-03-19T01:39:07.832218+0000","last_fullsized":"2024-03-19T01:39:07.832218+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T07:57:13.512027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,6],"acting":[1,10,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1c","version":"86'2","reported_seq":23,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-19T01:40:07.828793+0000","last_change":"2024-03-19T01:38:55.793796+0000","last_active":"2024-03-19T01:40:07.828793+0000","last_peered":"2024-03-19T01:40:07.828793+0000","last_clean":"2024-03-19T01:40:07.828793+0000","last_became_active":"2024-03-19T01:38:55.792818+0000","last_became_peered":"2024-03-19T01:38:55.792818+0000","last_unstale":"2024-03-19T01:40:07.828793+0000","last_undegraded":"2024-03-19T01:40:07.828793+0000","last_fullsized":"2024-03-19T01:40:07.828793+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T09:44:17.403644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,1,5],"acting":[8,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1d","version":"89'15","reported_seq":32,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:40:06.779461+0000","last_change":"2024-03-19T01:38:56.801280+0000","last_active":"2024-03-19T01:40:06.779461+0000","last_peered":"2024-03-19T01:40:06.779461+0000","last_clean":"2024-03-19T01:40:06.779461+0000","last_became_active":"2024-03-19T01:38:56.800944+0000","last_became_peered":"2024-03-19T01:38:56.800944+0000","last_unstale":"2024-03-19T01:40:06.779461+0000","last_undegraded":"2024-03-19T01:40:06.779461+0000","last_fullsized":"2024-03-19T01:40:06.779461+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T03:09:45.373387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1b","version":"89'7","reported_seq":96,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:40:06.399876+0000","last_change":"2024-03-19T01:38:55.792563+0000","last_active":"2024-03-19T01:40:06.399876+0000","last_peered":"2024-03-19T01:40:06.399876+0000","last_clean":"2024-03-19T01:40:06.399876+0000","last_became_active":"2024-03-19T01:38:55.792098+0000","last_became_peered":"2024-03-19T01:38:55.792098+0000","last_unstale":"2024-03-19T01:40:06.399876+0000","last_undegraded":"2024-03-19T01:40:06.399876+0000","last_fullsized":"2024-03-19T01:40:06.399876+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T08:40:54.804413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1154,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":70,"num_read_kb":3,"num_write":4,"num_write_kb":4,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"89'7","reported_seq":26,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-19T01:40:07.829406+0000","last_change":"2024-03-19T01:38:56.810155+0000","last_active":"2024-03-19T01:40:07.829406+0000","last_peered":"2024-03-19T01:40:07.829406+0000","last_clean":"2024-03-19T01:40:07.829406+0000","last_became_active":"2024-03-19T01:38:56.809820+0000","last_became_peered":"2024-03-19T01:38:56.809820+0000","last_unstale":"2024-03-19T01:40:07.829406+0000","last_undegraded":"2024-03-19T01:40:07.829406+0000","last_fullsized":"2024-03-19T01:40:07.829406+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T12:44:30.463663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,9,5],"acting":[8,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.1a","version":"87'3","reported_seq":26,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-03-19T01:40:08.987779+0000","last_change":"2024-03-19T01:38:55.797324+0000","last_active":"2024-03-19T01:40:08.987779+0000","last_peered":"2024-03-19T01:40:08.987779+0000","last_clean":"2024-03-19T01:40:08.987779+0000","last_became_active":"2024-03-19T01:38:55.796997+0000","last_became_peered":"2024-03-19T01:38:55.796997+0000","last_unstale":"2024-03-19T01:40:08.987779+0000","last_undegraded":"2024-03-19T01:40:08.987779+0000","last_fullsized":"2024-03-19T01:40:08.987779+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T08:40:37.023902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,8,6],"acting":[10,8,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.1b","version":"89'8","reported_seq":25,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:40:06.771086+0000","last_change":"2024-03-19T01:38:56.785995+0000","last_active":"2024-03-19T01:40:06.771086+0000","last_peered":"2024-03-19T01:40:06.771086+0000","last_clean":"2024-03-19T01:40:06.771086+0000","last_became_active":"2024-03-19T01:38:56.785659+0000","last_became_peered":"2024-03-19T01:38:56.785659+0000","last_unstale":"2024-03-19T01:40:06.771086+0000","last_undegraded":"2024-03-19T01:40:06.771086+0000","last_fullsized":"2024-03-19T01:40:06.771086+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T06:40:33.389038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.19","version":"87'2","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.831084+0000","last_change":"2024-03-19T01:38:55.790034+0000","last_active":"2024-03-19T01:39:07.831084+0000","last_peered":"2024-03-19T01:39:07.831084+0000","last_clean":"2024-03-19T01:39:07.831084+0000","last_became_active":"2024-03-19T01:38:55.786369+0000","last_became_peered":"2024-03-19T01:38:55.786369+0000","last_unstale":"2024-03-19T01:39:07.831084+0000","last_undegraded":"2024-03-19T01:39:07.831084+0000","last_fullsized":"2024-03-19T01:39:07.831084+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T06:13:52.925644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,9],"acting":[6,4,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"89'8","reported_seq":29,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-03-19T01:40:08.987650+0000","last_change":"2024-03-19T01:38:56.791253+0000","last_active":"2024-03-19T01:40:08.987650+0000","last_peered":"2024-03-19T01:40:08.987650+0000","last_clean":"2024-03-19T01:40:08.987650+0000","last_became_active":"2024-03-19T01:38:56.790793+0000","last_became_peered":"2024-03-19T01:38:56.790793+0000","last_unstale":"2024-03-19T01:40:08.987650+0000","last_undegraded":"2024-03-19T01:40:08.987650+0000","last_fullsized":"2024-03-19T01:40:08.987650+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T05:14:15.277513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,6],"acting":[3,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"87'1","reported_seq":20,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.833264+0000","last_change":"2024-03-19T01:38:55.793379+0000","last_active":"2024-03-19T01:39:07.833264+0000","last_peered":"2024-03-19T01:39:07.833264+0000","last_clean":"2024-03-19T01:39:07.833264+0000","last_became_active":"2024-03-19T01:38:55.793035+0000","last_became_peered":"2024-03-19T01:38:55.793035+0000","last_unstale":"2024-03-19T01:39:07.833264+0000","last_undegraded":"2024-03-19T01:39:07.833264+0000","last_fullsized":"2024-03-19T01:39:07.833264+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T05:01:52.122826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":17,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.828816+0000","last_change":"2024-03-19T01:38:56.800540+0000","last_active":"2024-03-19T01:39:07.828816+0000","last_peered":"2024-03-19T01:39:07.828816+0000","last_clean":"2024-03-19T01:39:07.828816+0000","last_became_active":"2024-03-19T01:38:56.800199+0000","last_became_peered":"2024-03-19T01:38:56.800199+0000","last_unstale":"2024-03-19T01:39:07.828816+0000","last_undegraded":"2024-03-19T01:39:07.828816+0000","last_fullsized":"2024-03-19T01:39:07.828816+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T05:17:57.722253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,3,10],"acting":[9,3,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.17","version":"89'4","reported_seq":98,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-03-19T01:40:09.405094+0000","last_change":"2024-03-19T01:38:55.792930+0000","last_active":"2024-03-19T01:40:09.405094+0000","last_peered":"2024-03-19T01:40:09.405094+0000","last_clean":"2024-03-19T01:40:09.405094+0000","last_became_active":"2024-03-19T01:38:55.792602+0000","last_became_peered":"2024-03-19T01:38:55.792602+0000","last_unstale":"2024-03-19T01:40:09.405094+0000","last_undegraded":"2024-03-19T01:40:09.405094+0000","last_fullsized":"2024-03-19T01:40:09.405094+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T12:28:15.399311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":71,"num_read_kb":71,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,5,9],"acting":[10,5,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":17,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.832330+0000","last_change":"2024-03-19T01:38:56.801228+0000","last_active":"2024-03-19T01:39:07.832330+0000","last_peered":"2024-03-19T01:39:07.832330+0000","last_clean":"2024-03-19T01:39:07.832330+0000","last_became_active":"2024-03-19T01:38:56.800656+0000","last_became_peered":"2024-03-19T01:38:56.800656+0000","last_unstale":"2024-03-19T01:39:07.832330+0000","last_undegraded":"2024-03-19T01:39:07.832330+0000","last_fullsized":"2024-03-19T01:39:07.832330+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T05:43:32.565608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,9],"acting":[1,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.16","version":"87'2","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.829002+0000","last_change":"2024-03-19T01:38:55.795899+0000","last_active":"2024-03-19T01:39:07.829002+0000","last_peered":"2024-03-19T01:39:07.829002+0000","last_clean":"2024-03-19T01:39:07.829002+0000","last_became_active":"2024-03-19T01:38:55.795434+0000","last_became_peered":"2024-03-19T01:38:55.795434+0000","last_unstale":"2024-03-19T01:39:07.829002+0000","last_undegraded":"2024-03-19T01:39:07.829002+0000","last_fullsized":"2024-03-19T01:39:07.829002+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T02:32:39.376935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"89'9","reported_seq":32,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:40:06.773386+0000","last_change":"2024-03-19T01:39:07.837847+0000","last_active":"2024-03-19T01:40:06.773386+0000","last_peered":"2024-03-19T01:40:06.773386+0000","last_clean":"2024-03-19T01:40:06.773386+0000","last_became_active":"2024-03-19T01:39:07.837514+0000","last_became_peered":"2024-03-19T01:39:07.837514+0000","last_unstale":"2024-03-19T01:40:06.773386+0000","last_undegraded":"2024-03-19T01:40:06.773386+0000","last_fullsized":"2024-03-19T01:40:06.773386+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T06:28:54.164949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,8,5],"acting":[0,8,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"87'4","reported_seq":24,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.833137+0000","last_change":"2024-03-19T01:38:55.793769+0000","last_active":"2024-03-19T01:39:07.833137+0000","last_peered":"2024-03-19T01:39:07.833137+0000","last_clean":"2024-03-19T01:39:07.833137+0000","last_became_active":"2024-03-19T01:38:55.793423+0000","last_became_peered":"2024-03-19T01:38:55.793423+0000","last_unstale":"2024-03-19T01:39:07.833137+0000","last_undegraded":"2024-03-19T01:39:07.833137+0000","last_fullsized":"2024-03-19T01:39:07.833137+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T12:56:37.933791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":672,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,10],"acting":[5,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":17,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.832181+0000","last_change":"2024-03-19T01:38:56.794855+0000","last_active":"2024-03-19T01:39:07.832181+0000","last_peered":"2024-03-19T01:39:07.832181+0000","last_clean":"2024-03-19T01:39:07.832181+0000","last_became_active":"2024-03-19T01:38:56.794553+0000","last_became_peered":"2024-03-19T01:38:56.794553+0000","last_unstale":"2024-03-19T01:39:07.832181+0000","last_undegraded":"2024-03-19T01:39:07.832181+0000","last_fullsized":"2024-03-19T01:39:07.832181+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T08:33:20.473620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,8,4],"acting":[1,8,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.14","version":"89'32","reported_seq":206,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-03-19T01:40:10.406371+0000","last_change":"2024-03-19T01:38:55.794317+0000","last_active":"2024-03-19T01:40:10.406371+0000","last_peered":"2024-03-19T01:40:10.406371+0000","last_clean":"2024-03-19T01:40:10.406371+0000","last_became_active":"2024-03-19T01:38:55.793970+0000","last_became_peered":"2024-03-19T01:38:55.793970+0000","last_unstale":"2024-03-19T01:40:10.406371+0000","last_undegraded":"2024-03-19T01:40:10.406371+0000","last_fullsized":"2024-03-19T01:40:10.406371+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":32,"log_dups_size":0,"ondisk_log_size":32,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T02:25:09.584906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60272,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":151,"num_read_kb":134,"num_write":33,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,1,2],"acting":[10,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.15","version":"89'6","reported_seq":25,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-19T01:40:07.828797+0000","last_change":"2024-03-19T01:38:56.794400+0000","last_active":"2024-03-19T01:40:07.828797+0000","last_peered":"2024-03-19T01:40:07.828797+0000","last_clean":"2024-03-19T01:40:07.828797+0000","last_became_active":"2024-03-19T01:38:56.794027+0000","last_became_peered":"2024-03-19T01:38:56.794027+0000","last_unstale":"2024-03-19T01:40:07.828797+0000","last_undegraded":"2024-03-19T01:40:07.828797+0000","last_fullsized":"2024-03-19T01:40:07.828797+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T06:33:51.398239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,7],"acting":[8,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.13","version":"87'6","reported_seq":30,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-03-19T01:40:08.824336+0000","last_change":"2024-03-19T01:38:55.792812+0000","last_active":"2024-03-19T01:40:08.824336+0000","last_peered":"2024-03-19T01:40:08.824336+0000","last_clean":"2024-03-19T01:40:08.824336+0000","last_became_active":"2024-03-19T01:38:55.792422+0000","last_became_peered":"2024-03-19T01:38:55.792422+0000","last_unstale":"2024-03-19T01:40:08.824336+0000","last_undegraded":"2024-03-19T01:40:08.824336+0000","last_fullsized":"2024-03-19T01:40:08.824336+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T11:04:58.863405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,10,3],"acting":[2,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.12","version":"89'9","reported_seq":26,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:40:06.779007+0000","last_change":"2024-03-19T01:38:56.794156+0000","last_active":"2024-03-19T01:40:06.779007+0000","last_peered":"2024-03-19T01:40:06.779007+0000","last_clean":"2024-03-19T01:40:06.779007+0000","last_became_active":"2024-03-19T01:38:56.793812+0000","last_became_peered":"2024-03-19T01:38:56.793812+0000","last_unstale":"2024-03-19T01:40:06.779007+0000","last_undegraded":"2024-03-19T01:40:06.779007+0000","last_fullsized":"2024-03-19T01:40:06.779007+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T05:00:53.379548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.12","version":"87'3","reported_seq":22,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.833710+0000","last_change":"2024-03-19T01:38:55.798220+0000","last_active":"2024-03-19T01:39:07.833710+0000","last_peered":"2024-03-19T01:39:07.833710+0000","last_clean":"2024-03-19T01:39:07.833710+0000","last_became_active":"2024-03-19T01:38:55.797901+0000","last_became_peered":"2024-03-19T01:38:55.797901+0000","last_unstale":"2024-03-19T01:39:07.833710+0000","last_undegraded":"2024-03-19T01:39:07.833710+0000","last_fullsized":"2024-03-19T01:39:07.833710+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T11:45:20.233596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":17,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.828950+0000","last_change":"2024-03-19T01:38:56.805603+0000","last_active":"2024-03-19T01:39:07.828950+0000","last_peered":"2024-03-19T01:39:07.828950+0000","last_clean":"2024-03-19T01:39:07.828950+0000","last_became_active":"2024-03-19T01:38:56.805248+0000","last_became_peered":"2024-03-19T01:38:56.805248+0000","last_unstale":"2024-03-19T01:39:07.828950+0000","last_undegraded":"2024-03-19T01:39:07.828950+0000","last_fullsized":"2024-03-19T01:39:07.828950+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T05:28:41.503673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"89'5","reported_seq":94,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:40:06.418298+0000","last_change":"2024-03-19T01:38:55.793254+0000","last_active":"2024-03-19T01:40:06.418298+0000","last_peered":"2024-03-19T01:40:06.418298+0000","last_clean":"2024-03-19T01:40:06.418298+0000","last_became_active":"2024-03-19T01:38:55.792925+0000","last_became_peered":"2024-03-19T01:38:55.792925+0000","last_unstale":"2024-03-19T01:40:06.418298+0000","last_undegraded":"2024-03-19T01:40:06.418298+0000","last_fullsized":"2024-03-19T01:40:06.418298+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T05:17:25.353050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1120,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":71,"num_read_kb":4,"num_write":4,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,10],"acting":[4,7,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":17,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.828561+0000","last_change":"2024-03-19T01:38:56.807845+0000","last_active":"2024-03-19T01:39:07.828561+0000","last_peered":"2024-03-19T01:39:07.828561+0000","last_clean":"2024-03-19T01:39:07.828561+0000","last_became_active":"2024-03-19T01:38:56.807175+0000","last_became_peered":"2024-03-19T01:38:56.807175+0000","last_unstale":"2024-03-19T01:39:07.828561+0000","last_undegraded":"2024-03-19T01:39:07.828561+0000","last_fullsized":"2024-03-19T01:39:07.828561+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T10:45:37.669659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.10","version":"87'15","reported_seq":37,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-19T01:40:07.828910+0000","last_change":"2024-03-19T01:38:55.793320+0000","last_active":"2024-03-19T01:40:07.828910+0000","last_peered":"2024-03-19T01:40:07.828910+0000","last_clean":"2024-03-19T01:40:07.828910+0000","last_became_active":"2024-03-19T01:38:55.793030+0000","last_became_peered":"2024-03-19T01:38:55.793030+0000","last_unstale":"2024-03-19T01:40:07.828910+0000","last_undegraded":"2024-03-19T01:40:07.828910+0000","last_fullsized":"2024-03-19T01:40:07.828910+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T04:01:31.028364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":8,"num_object_clones":0,"num_object_copies":24,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":20,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":6,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,2],"acting":[8,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.11","version":"89'6","reported_seq":23,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:40:06.780226+0000","last_change":"2024-03-19T01:38:56.796338+0000","last_active":"2024-03-19T01:40:06.780226+0000","last_peered":"2024-03-19T01:40:06.780226+0000","last_clean":"2024-03-19T01:40:06.780226+0000","last_became_active":"2024-03-19T01:38:56.795660+0000","last_became_peered":"2024-03-19T01:38:56.795660+0000","last_unstale":"2024-03-19T01:40:06.780226+0000","last_undegraded":"2024-03-19T01:40:06.780226+0000","last_fullsized":"2024-03-19T01:40:06.780226+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T01:56:03.144907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.f","version":"87'2","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-19T01:40:07.828540+0000","last_change":"2024-03-19T01:38:55.791670+0000","last_active":"2024-03-19T01:40:07.828540+0000","last_peered":"2024-03-19T01:40:07.828540+0000","last_clean":"2024-03-19T01:40:07.828540+0000","last_became_active":"2024-03-19T01:38:55.790859+0000","last_became_peered":"2024-03-19T01:38:55.790859+0000","last_unstale":"2024-03-19T01:40:07.828540+0000","last_undegraded":"2024-03-19T01:40:07.828540+0000","last_fullsized":"2024-03-19T01:40:07.828540+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T08:13:16.770134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":2,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,5,4],"acting":[8,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.e","version":"89'5","reported_seq":26,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-03-19T01:40:08.987577+0000","last_change":"2024-03-19T01:38:56.806291+0000","last_active":"2024-03-19T01:40:08.987577+0000","last_peered":"2024-03-19T01:40:08.987577+0000","last_clean":"2024-03-19T01:40:08.987577+0000","last_became_active":"2024-03-19T01:38:56.805972+0000","last_became_peered":"2024-03-19T01:38:56.805972+0000","last_unstale":"2024-03-19T01:40:08.987577+0000","last_undegraded":"2024-03-19T01:40:08.987577+0000","last_fullsized":"2024-03-19T01:40:08.987577+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T05:28:19.575710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,5,4],"acting":[11,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.829075+0000","last_change":"2024-03-19T01:38:55.798331+0000","last_active":"2024-03-19T01:39:07.829075+0000","last_peered":"2024-03-19T01:39:07.829075+0000","last_clean":"2024-03-19T01:39:07.829075+0000","last_became_active":"2024-03-19T01:38:55.797948+0000","last_became_peered":"2024-03-19T01:38:55.797948+0000","last_unstale":"2024-03-19T01:39:07.829075+0000","last_undegraded":"2024-03-19T01:39:07.829075+0000","last_fullsized":"2024-03-19T01:39:07.829075+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T04:35:09.036490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,2,0],"acting":[9,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.5","version":"89'5","reported_seq":26,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-03-19T01:40:08.987794+0000","last_change":"2024-03-19T01:38:56.810162+0000","last_active":"2024-03-19T01:40:08.987794+0000","last_peered":"2024-03-19T01:40:08.987794+0000","last_clean":"2024-03-19T01:40:08.987794+0000","last_became_active":"2024-03-19T01:38:56.809888+0000","last_became_peered":"2024-03-19T01:38:56.809888+0000","last_unstale":"2024-03-19T01:40:08.987794+0000","last_undegraded":"2024-03-19T01:40:08.987794+0000","last_fullsized":"2024-03-19T01:40:08.987794+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T05:49:35.606065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,3,6],"acting":[10,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"2.2","version":"87'1","reported_seq":20,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.828770+0000","last_change":"2024-03-19T01:38:55.796629+0000","last_active":"2024-03-19T01:39:07.828770+0000","last_peered":"2024-03-19T01:39:07.828770+0000","last_clean":"2024-03-19T01:39:07.828770+0000","last_became_active":"2024-03-19T01:38:55.796232+0000","last_became_peered":"2024-03-19T01:38:55.796232+0000","last_unstale":"2024-03-19T01:39:07.828770+0000","last_undegraded":"2024-03-19T01:39:07.828770+0000","last_fullsized":"2024-03-19T01:39:07.828770+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T09:38:58.853129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3","version":"89'7","reported_seq":28,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-03-19T01:40:08.987896+0000","last_change":"2024-03-19T01:38:56.798227+0000","last_active":"2024-03-19T01:40:08.987896+0000","last_peered":"2024-03-19T01:40:08.987896+0000","last_clean":"2024-03-19T01:40:08.987896+0000","last_became_active":"2024-03-19T01:38:56.797302+0000","last_became_peered":"2024-03-19T01:38:56.797302+0000","last_unstale":"2024-03-19T01:40:08.987896+0000","last_undegraded":"2024-03-19T01:40:08.987896+0000","last_fullsized":"2024-03-19T01:40:08.987896+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T12:35:56.975118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,4,0],"acting":[10,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":11,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.931227+0000","last_change":"2024-03-19T01:39:07.930840+0000","last_active":"2024-03-19T01:39:07.931227+0000","last_peered":"2024-03-19T01:39:07.931227+0000","last_clean":"2024-03-19T01:39:07.931227+0000","last_became_active":"2024-03-19T01:39:07.930581+0000","last_became_peered":"2024-03-19T01:39:07.930581+0000","last_unstale":"2024-03-19T01:39:07.931227+0000","last_undegraded":"2024-03-19T01:39:07.931227+0000","last_fullsized":"2024-03-19T01:39:07.931227+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T04:24:25.476025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,2,4],"acting":[9,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.1","version":"89'5","reported_seq":91,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:40:06.417906+0000","last_change":"2024-03-19T01:38:55.785957+0000","last_active":"2024-03-19T01:40:06.417906+0000","last_peered":"2024-03-19T01:40:06.417906+0000","last_clean":"2024-03-19T01:40:06.417906+0000","last_became_active":"2024-03-19T01:38:55.785527+0000","last_became_peered":"2024-03-19T01:38:55.785527+0000","last_unstale":"2024-03-19T01:40:06.417906+0000","last_undegraded":"2024-03-19T01:40:06.417906+0000","last_fullsized":"2024-03-19T01:40:06.417906+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T09:33:55.329126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":67,"num_read_kb":67,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":11,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.837652+0000","last_change":"2024-03-19T01:39:07.835329+0000","last_active":"2024-03-19T01:39:07.837652+0000","last_peered":"2024-03-19T01:39:07.837652+0000","last_clean":"2024-03-19T01:39:07.837652+0000","last_became_active":"2024-03-19T01:39:07.834744+0000","last_became_peered":"2024-03-19T01:39:07.834744+0000","last_unstale":"2024-03-19T01:39:07.837652+0000","last_undegraded":"2024-03-19T01:39:07.837652+0000","last_fullsized":"2024-03-19T01:39:07.837652+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T07:13:22.760031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,5,1],"acting":[9,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.3","version":"87'3","reported_seq":24,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.930581+0000","last_change":"2024-03-19T01:38:55.792521+0000","last_active":"2024-03-19T01:39:07.930581+0000","last_peered":"2024-03-19T01:39:07.930581+0000","last_clean":"2024-03-19T01:39:07.930581+0000","last_became_active":"2024-03-19T01:38:55.792166+0000","last_became_peered":"2024-03-19T01:38:55.792166+0000","last_unstale":"2024-03-19T01:39:07.930581+0000","last_undegraded":"2024-03-19T01:39:07.930581+0000","last_fullsized":"2024-03-19T01:39:07.930581+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T08:15:12.216641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":4,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,9],"acting":[4,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.0","version":"23'137","reported_seq":344,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.828610+0000","last_change":"2024-03-19T01:37:48.171712+0000","last_active":"2024-03-19T01:39:07.828610+0000","last_peered":"2024-03-19T01:39:07.828610+0000","last_clean":"2024-03-19T01:39:07.828610+0000","last_became_active":"2024-03-19T01:37:48.171323+0000","last_became_peered":"2024-03-19T01:37:48.171323+0000","last_unstale":"2024-03-19T01:39:07.828610+0000","last_undegraded":"2024-03-19T01:39:07.828610+0000","last_fullsized":"2024-03-19T01:39:07.828610+0000","mapping_epoch":77,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":78,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:34:31.088679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:34:31.088679+0000","last_clean_scrub_stamp":"2024-03-19T01:34:31.088679+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T02:22:49.477567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":109,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"89'5","reported_seq":22,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:40:06.780848+0000","last_change":"2024-03-19T01:38:56.797868+0000","last_active":"2024-03-19T01:40:06.780848+0000","last_peered":"2024-03-19T01:40:06.780848+0000","last_clean":"2024-03-19T01:40:06.780848+0000","last_became_active":"2024-03-19T01:38:56.796855+0000","last_became_peered":"2024-03-19T01:38:56.796855+0000","last_unstale":"2024-03-19T01:40:06.780848+0000","last_undegraded":"2024-03-19T01:40:06.780848+0000","last_fullsized":"2024-03-19T01:40:06.780848+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T01:53:09.887368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"87'3","reported_seq":32,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.839838+0000","last_change":"2024-03-19T01:39:07.839838+0000","last_active":"2024-03-19T01:39:07.839838+0000","last_peered":"2024-03-19T01:39:07.839838+0000","last_clean":"2024-03-19T01:39:07.839838+0000","last_became_active":"2024-03-19T01:39:07.836581+0000","last_became_peered":"2024-03-19T01:39:07.836581+0000","last_unstale":"2024-03-19T01:39:07.839838+0000","last_undegraded":"2024-03-19T01:39:07.839838+0000","last_fullsized":"2024-03-19T01:39:07.839838+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T11:38:01.087677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"87'1","reported_seq":23,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-19T01:40:07.828901+0000","last_change":"2024-03-19T01:38:55.795464+0000","last_active":"2024-03-19T01:40:07.828901+0000","last_peered":"2024-03-19T01:40:07.828901+0000","last_clean":"2024-03-19T01:40:07.828901+0000","last_became_active":"2024-03-19T01:38:55.793827+0000","last_became_peered":"2024-03-19T01:38:55.793827+0000","last_unstale":"2024-03-19T01:40:07.828901+0000","last_undegraded":"2024-03-19T01:40:07.828901+0000","last_fullsized":"2024-03-19T01:40:07.828901+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T13:03:56.862269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,9],"acting":[8,0,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.4","version":"89'9","reported_seq":26,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:40:06.781029+0000","last_change":"2024-03-19T01:38:56.795569+0000","last_active":"2024-03-19T01:40:06.781029+0000","last_peered":"2024-03-19T01:40:06.781029+0000","last_clean":"2024-03-19T01:40:06.781029+0000","last_became_active":"2024-03-19T01:38:56.795281+0000","last_became_peered":"2024-03-19T01:38:56.795281+0000","last_unstale":"2024-03-19T01:40:06.781029+0000","last_undegraded":"2024-03-19T01:40:06.781029+0000","last_fullsized":"2024-03-19T01:40:06.781029+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T03:18:00.599307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,9],"acting":[6,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"88'7","reported_seq":26,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.829297+0000","last_change":"2024-03-19T01:38:55.792306+0000","last_active":"2024-03-19T01:39:07.829297+0000","last_peered":"2024-03-19T01:39:07.829297+0000","last_clean":"2024-03-19T01:39:07.829297+0000","last_became_active":"2024-03-19T01:38:55.792019+0000","last_became_peered":"2024-03-19T01:38:55.792019+0000","last_unstale":"2024-03-19T01:39:07.829297+0000","last_undegraded":"2024-03-19T01:39:07.829297+0000","last_fullsized":"2024-03-19T01:39:07.829297+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T06:33:06.869054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,8,3],"acting":[9,8,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.7","version":"89'5","reported_seq":26,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-03-19T01:40:08.988401+0000","last_change":"2024-03-19T01:38:56.798314+0000","last_active":"2024-03-19T01:40:08.988401+0000","last_peered":"2024-03-19T01:40:08.988401+0000","last_clean":"2024-03-19T01:40:08.988401+0000","last_became_active":"2024-03-19T01:38:56.797999+0000","last_became_peered":"2024-03-19T01:38:56.797999+0000","last_unstale":"2024-03-19T01:40:08.988401+0000","last_undegraded":"2024-03-19T01:40:08.988401+0000","last_fullsized":"2024-03-19T01:40:08.988401+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T06:23:36.861238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,7],"acting":[3,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"87'4","reported_seq":23,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.930691+0000","last_change":"2024-03-19T01:38:55.790866+0000","last_active":"2024-03-19T01:39:07.930691+0000","last_peered":"2024-03-19T01:39:07.930691+0000","last_clean":"2024-03-19T01:39:07.930691+0000","last_became_active":"2024-03-19T01:38:55.790363+0000","last_became_peered":"2024-03-19T01:38:55.790363+0000","last_unstale":"2024-03-19T01:39:07.930691+0000","last_undegraded":"2024-03-19T01:39:07.930691+0000","last_fullsized":"2024-03-19T01:39:07.930691+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T09:00:27.595598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,11,6],"acting":[4,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"89'7","reported_seq":24,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:40:06.779387+0000","last_change":"2024-03-19T01:38:56.793369+0000","last_active":"2024-03-19T01:40:06.779387+0000","last_peered":"2024-03-19T01:40:06.779387+0000","last_clean":"2024-03-19T01:40:06.779387+0000","last_became_active":"2024-03-19T01:38:56.793068+0000","last_became_peered":"2024-03-19T01:38:56.793068+0000","last_unstale":"2024-03-19T01:40:06.779387+0000","last_undegraded":"2024-03-19T01:40:06.779387+0000","last_fullsized":"2024-03-19T01:40:06.779387+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T13:12:57.954100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"87'2","reported_seq":31,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.839372+0000","last_change":"2024-03-19T01:39:07.839372+0000","last_active":"2024-03-19T01:39:07.839372+0000","last_peered":"2024-03-19T01:39:07.839372+0000","last_clean":"2024-03-19T01:39:07.839372+0000","last_became_active":"2024-03-19T01:39:07.836229+0000","last_became_peered":"2024-03-19T01:39:07.836229+0000","last_unstale":"2024-03-19T01:39:07.839372+0000","last_undegraded":"2024-03-19T01:39:07.839372+0000","last_fullsized":"2024-03-19T01:39:07.839372+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T12:28:33.102856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,11,0],"acting":[1,11,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"89'2","reported_seq":21,"reported_epoch":90,"state":"active+clean","last_fresh":"2024-03-19T01:40:07.828339+0000","last_change":"2024-03-19T01:38:56.802272+0000","last_active":"2024-03-19T01:40:07.828339+0000","last_peered":"2024-03-19T01:40:07.828339+0000","last_clean":"2024-03-19T01:40:07.828339+0000","last_became_active":"2024-03-19T01:38:56.801071+0000","last_became_peered":"2024-03-19T01:38:56.801071+0000","last_unstale":"2024-03-19T01:40:07.828339+0000","last_undegraded":"2024-03-19T01:40:07.828339+0000","last_fullsized":"2024-03-19T01:40:07.828339+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T06:58:12.762975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,10],"acting":[8,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.9","version":"87'3","reported_seq":26,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-03-19T01:40:08.987855+0000","last_change":"2024-03-19T01:38:55.795186+0000","last_active":"2024-03-19T01:40:08.987855+0000","last_peered":"2024-03-19T01:40:08.987855+0000","last_clean":"2024-03-19T01:40:08.987855+0000","last_became_active":"2024-03-19T01:38:55.794635+0000","last_became_peered":"2024-03-19T01:38:55.794635+0000","last_unstale":"2024-03-19T01:40:08.987855+0000","last_undegraded":"2024-03-19T01:40:08.987855+0000","last_fullsized":"2024-03-19T01:40:08.987855+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T03:55:56.322716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,9,10],"acting":[3,9,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"89'9","reported_seq":30,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-03-19T01:40:08.987398+0000","last_change":"2024-03-19T01:38:56.812656+0000","last_active":"2024-03-19T01:40:08.987398+0000","last_peered":"2024-03-19T01:40:08.987398+0000","last_clean":"2024-03-19T01:40:08.987398+0000","last_became_active":"2024-03-19T01:38:56.811824+0000","last_became_peered":"2024-03-19T01:38:56.811824+0000","last_unstale":"2024-03-19T01:40:08.987398+0000","last_undegraded":"2024-03-19T01:40:08.987398+0000","last_fullsized":"2024-03-19T01:40:08.987398+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T04:28:11.866892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,0,2],"acting":[11,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.a","version":"87'1","reported_seq":24,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-03-19T01:40:08.824175+0000","last_change":"2024-03-19T01:38:55.791193+0000","last_active":"2024-03-19T01:40:08.824175+0000","last_peered":"2024-03-19T01:40:08.824175+0000","last_clean":"2024-03-19T01:40:08.824175+0000","last_became_active":"2024-03-19T01:38:55.790820+0000","last_became_peered":"2024-03-19T01:38:55.790820+0000","last_unstale":"2024-03-19T01:40:08.824175+0000","last_undegraded":"2024-03-19T01:40:08.824175+0000","last_fullsized":"2024-03-19T01:40:08.824175+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T10:19:09.564798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,8],"acting":[2,3,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"89'8","reported_seq":29,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-03-19T01:40:08.988041+0000","last_change":"2024-03-19T01:38:56.803287+0000","last_active":"2024-03-19T01:40:08.988041+0000","last_peered":"2024-03-19T01:40:08.988041+0000","last_clean":"2024-03-19T01:40:08.988041+0000","last_became_active":"2024-03-19T01:38:56.802924+0000","last_became_peered":"2024-03-19T01:38:56.802924+0000","last_unstale":"2024-03-19T01:40:08.988041+0000","last_undegraded":"2024-03-19T01:40:08.988041+0000","last_fullsized":"2024-03-19T01:40:08.988041+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T07:45:23.726378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,10],"acting":[3,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"87'4","reported_seq":21,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-03-19T01:40:08.987527+0000","last_change":"2024-03-19T01:39:07.840234+0000","last_active":"2024-03-19T01:40:08.987527+0000","last_peered":"2024-03-19T01:40:08.987527+0000","last_clean":"2024-03-19T01:40:08.987527+0000","last_became_active":"2024-03-19T01:39:07.835111+0000","last_became_peered":"2024-03-19T01:39:07.835111+0000","last_unstale":"2024-03-19T01:40:08.987527+0000","last_undegraded":"2024-03-19T01:40:08.987527+0000","last_fullsized":"2024-03-19T01:40:08.987527+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T03:30:22.558494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,2,7],"acting":[11,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":17,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.828877+0000","last_change":"2024-03-19T01:38:56.797272+0000","last_active":"2024-03-19T01:39:07.828877+0000","last_peered":"2024-03-19T01:39:07.828877+0000","last_clean":"2024-03-19T01:39:07.828877+0000","last_became_active":"2024-03-19T01:38:56.796944+0000","last_became_peered":"2024-03-19T01:38:56.796944+0000","last_unstale":"2024-03-19T01:39:07.828877+0000","last_undegraded":"2024-03-19T01:39:07.828877+0000","last_fullsized":"2024-03-19T01:39:07.828877+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T06:14:58.147093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,6],"acting":[7,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.c","version":"87'3","reported_seq":27,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-03-19T01:40:08.824470+0000","last_change":"2024-03-19T01:38:55.795642+0000","last_active":"2024-03-19T01:40:08.824470+0000","last_peered":"2024-03-19T01:40:08.824470+0000","last_clean":"2024-03-19T01:40:08.824470+0000","last_became_active":"2024-03-19T01:38:55.795285+0000","last_became_peered":"2024-03-19T01:38:55.795285+0000","last_unstale":"2024-03-19T01:40:08.824470+0000","last_undegraded":"2024-03-19T01:40:08.824470+0000","last_fullsized":"2024-03-19T01:40:08.824470+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T10:09:18.888802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,11,0],"acting":[2,11,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":17,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.829092+0000","last_change":"2024-03-19T01:38:56.801802+0000","last_active":"2024-03-19T01:39:07.829092+0000","last_peered":"2024-03-19T01:39:07.829092+0000","last_clean":"2024-03-19T01:39:07.829092+0000","last_became_active":"2024-03-19T01:38:56.801130+0000","last_became_peered":"2024-03-19T01:38:56.801130+0000","last_unstale":"2024-03-19T01:39:07.829092+0000","last_undegraded":"2024-03-19T01:39:07.829092+0000","last_fullsized":"2024-03-19T01:39:07.829092+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T10:36:45.509217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,11],"acting":[7,1,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"86'1","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.832928+0000","last_change":"2024-03-19T01:38:55.795085+0000","last_active":"2024-03-19T01:39:07.832928+0000","last_peered":"2024-03-19T01:39:07.832928+0000","last_clean":"2024-03-19T01:39:07.832928+0000","last_became_active":"2024-03-19T01:38:55.794626+0000","last_became_peered":"2024-03-19T01:38:55.794626+0000","last_unstale":"2024-03-19T01:39:07.832928+0000","last_undegraded":"2024-03-19T01:39:07.832928+0000","last_fullsized":"2024-03-19T01:39:07.832928+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T05:51:50.428331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,10,3],"acting":[5,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"87'1","reported_seq":18,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.829148+0000","last_change":"2024-03-19T01:38:56.802141+0000","last_active":"2024-03-19T01:39:07.829148+0000","last_peered":"2024-03-19T01:39:07.829148+0000","last_clean":"2024-03-19T01:39:07.829148+0000","last_became_active":"2024-03-19T01:38:56.801788+0000","last_became_peered":"2024-03-19T01:38:56.801788+0000","last_unstale":"2024-03-19T01:39:07.829148+0000","last_undegraded":"2024-03-19T01:39:07.829148+0000","last_fullsized":"2024-03-19T01:39:07.829148+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T10:13:39.264272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"86'2","reported_seq":31,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.933658+0000","last_change":"2024-03-19T01:39:07.933658+0000","last_active":"2024-03-19T01:39:07.933658+0000","last_peered":"2024-03-19T01:39:07.933658+0000","last_clean":"2024-03-19T01:39:07.933658+0000","last_became_active":"2024-03-19T01:39:07.931377+0000","last_became_peered":"2024-03-19T01:39:07.931377+0000","last_unstale":"2024-03-19T01:39:07.933658+0000","last_undegraded":"2024-03-19T01:39:07.933658+0000","last_fullsized":"2024-03-19T01:39:07.933658+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":84,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:54.761593+0000","last_clean_scrub_stamp":"2024-03-19T01:38:54.761593+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T10:09:37.509763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":17,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.829226+0000","last_change":"2024-03-19T01:38:56.794428+0000","last_active":"2024-03-19T01:39:07.829226+0000","last_peered":"2024-03-19T01:39:07.829226+0000","last_clean":"2024-03-19T01:39:07.829226+0000","last_became_active":"2024-03-19T01:38:56.794051+0000","last_became_peered":"2024-03-19T01:38:56.794051+0000","last_unstale":"2024-03-19T01:39:07.829226+0000","last_undegraded":"2024-03-19T01:39:07.829226+0000","last_fullsized":"2024-03-19T01:39:07.829226+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T02:07:20.707276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,11,8],"acting":[7,11,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.20","version":"89'9","reported_seq":30,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-03-19T01:40:08.987423+0000","last_change":"2024-03-19T01:38:56.812596+0000","last_active":"2024-03-19T01:40:08.987423+0000","last_peered":"2024-03-19T01:40:08.987423+0000","last_clean":"2024-03-19T01:40:08.987423+0000","last_became_active":"2024-03-19T01:38:56.812268+0000","last_became_peered":"2024-03-19T01:38:56.812268+0000","last_unstale":"2024-03-19T01:40:08.987423+0000","last_undegraded":"2024-03-19T01:40:08.987423+0000","last_fullsized":"2024-03-19T01:40:08.987423+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T08:54:51.350908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,2,5],"acting":[11,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.21","version":"89'12","reported_seq":33,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-03-19T01:40:08.824210+0000","last_change":"2024-03-19T01:38:56.804280+0000","last_active":"2024-03-19T01:40:08.824210+0000","last_peered":"2024-03-19T01:40:08.824210+0000","last_clean":"2024-03-19T01:40:08.824210+0000","last_became_active":"2024-03-19T01:38:56.801928+0000","last_became_peered":"2024-03-19T01:38:56.801928+0000","last_unstale":"2024-03-19T01:40:08.824210+0000","last_undegraded":"2024-03-19T01:40:08.824210+0000","last_fullsized":"2024-03-19T01:40:08.824210+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":12,"log_dups_size":0,"ondisk_log_size":12,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T11:49:29.778949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.22","version":"89'8","reported_seq":29,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-03-19T01:40:08.824171+0000","last_change":"2024-03-19T01:38:56.798719+0000","last_active":"2024-03-19T01:40:08.824171+0000","last_peered":"2024-03-19T01:40:08.824171+0000","last_clean":"2024-03-19T01:40:08.824171+0000","last_became_active":"2024-03-19T01:38:56.798253+0000","last_became_peered":"2024-03-19T01:38:56.798253+0000","last_unstale":"2024-03-19T01:40:08.824171+0000","last_undegraded":"2024-03-19T01:40:08.824171+0000","last_fullsized":"2024-03-19T01:40:08.824171+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T06:27:56.754679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.23","version":"89'7","reported_seq":24,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:40:06.776859+0000","last_change":"2024-03-19T01:38:56.795221+0000","last_active":"2024-03-19T01:40:06.776859+0000","last_peered":"2024-03-19T01:40:06.776859+0000","last_clean":"2024-03-19T01:40:06.776859+0000","last_became_active":"2024-03-19T01:38:56.794756+0000","last_became_peered":"2024-03-19T01:38:56.794756+0000","last_unstale":"2024-03-19T01:40:06.776859+0000","last_undegraded":"2024-03-19T01:40:06.776859+0000","last_fullsized":"2024-03-19T01:40:06.776859+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T10:05:54.394976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.24","version":"89'4","reported_seq":21,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:40:06.777488+0000","last_change":"2024-03-19T01:38:56.794482+0000","last_active":"2024-03-19T01:40:06.777488+0000","last_peered":"2024-03-19T01:40:06.777488+0000","last_clean":"2024-03-19T01:40:06.777488+0000","last_became_active":"2024-03-19T01:38:56.794140+0000","last_became_peered":"2024-03-19T01:38:56.794140+0000","last_unstale":"2024-03-19T01:40:06.777488+0000","last_undegraded":"2024-03-19T01:40:06.777488+0000","last_fullsized":"2024-03-19T01:40:06.777488+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T04:29:38.985047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"89'6","reported_seq":27,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-03-19T01:40:08.987849+0000","last_change":"2024-03-19T01:38:56.802020+0000","last_active":"2024-03-19T01:40:08.987849+0000","last_peered":"2024-03-19T01:40:08.987849+0000","last_clean":"2024-03-19T01:40:08.987849+0000","last_became_active":"2024-03-19T01:38:56.801668+0000","last_became_peered":"2024-03-19T01:38:56.801668+0000","last_unstale":"2024-03-19T01:40:08.987849+0000","last_undegraded":"2024-03-19T01:40:08.987849+0000","last_fullsized":"2024-03-19T01:40:08.987849+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T04:13:55.199279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,8,5],"acting":[11,8,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.26","version":"89'15","reported_seq":36,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-03-19T01:40:08.824653+0000","last_change":"2024-03-19T01:38:56.809100+0000","last_active":"2024-03-19T01:40:08.824653+0000","last_peered":"2024-03-19T01:40:08.824653+0000","last_clean":"2024-03-19T01:40:08.824653+0000","last_became_active":"2024-03-19T01:38:56.807985+0000","last_became_peered":"2024-03-19T01:38:56.807985+0000","last_unstale":"2024-03-19T01:40:08.824653+0000","last_undegraded":"2024-03-19T01:40:08.824653+0000","last_fullsized":"2024-03-19T01:40:08.824653+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T02:58:25.235837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,8,1],"acting":[2,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.27","version":"89'2","reported_seq":23,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-03-19T01:40:08.988288+0000","last_change":"2024-03-19T01:38:56.799707+0000","last_active":"2024-03-19T01:40:08.988288+0000","last_peered":"2024-03-19T01:40:08.988288+0000","last_clean":"2024-03-19T01:40:08.988288+0000","last_became_active":"2024-03-19T01:38:56.799388+0000","last_became_peered":"2024-03-19T01:38:56.799388+0000","last_unstale":"2024-03-19T01:40:08.988288+0000","last_undegraded":"2024-03-19T01:40:08.988288+0000","last_fullsized":"2024-03-19T01:40:08.988288+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T11:10:18.360180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"89'6","reported_seq":23,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:40:06.772406+0000","last_change":"2024-03-19T01:38:56.786807+0000","last_active":"2024-03-19T01:40:06.772406+0000","last_peered":"2024-03-19T01:40:06.772406+0000","last_clean":"2024-03-19T01:40:06.772406+0000","last_became_active":"2024-03-19T01:38:56.786395+0000","last_became_peered":"2024-03-19T01:38:56.786395+0000","last_unstale":"2024-03-19T01:40:06.772406+0000","last_undegraded":"2024-03-19T01:40:06.772406+0000","last_fullsized":"2024-03-19T01:40:06.772406+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T05:19:20.228926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,9,3],"acting":[5,9,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"89'8","reported_seq":29,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-03-19T01:40:08.987754+0000","last_change":"2024-03-19T01:38:56.800759+0000","last_active":"2024-03-19T01:40:08.987754+0000","last_peered":"2024-03-19T01:40:08.987754+0000","last_clean":"2024-03-19T01:40:08.987754+0000","last_became_active":"2024-03-19T01:38:56.800462+0000","last_became_peered":"2024-03-19T01:38:56.800462+0000","last_unstale":"2024-03-19T01:40:08.987754+0000","last_undegraded":"2024-03-19T01:40:08.987754+0000","last_fullsized":"2024-03-19T01:40:08.987754+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T02:13:19.070148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,11],"acting":[3,2,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":17,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:39:07.831737+0000","last_change":"2024-03-19T01:38:56.802653+0000","last_active":"2024-03-19T01:39:07.831737+0000","last_peered":"2024-03-19T01:39:07.831737+0000","last_clean":"2024-03-19T01:39:07.831737+0000","last_became_active":"2024-03-19T01:38:56.802284+0000","last_became_peered":"2024-03-19T01:38:56.802284+0000","last_unstale":"2024-03-19T01:39:07.831737+0000","last_undegraded":"2024-03-19T01:39:07.831737+0000","last_fullsized":"2024-03-19T01:39:07.831737+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T06:17:13.936359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2b","version":"89'11","reported_seq":28,"reported_epoch":89,"state":"active+clean","last_fresh":"2024-03-19T01:40:06.778680+0000","last_change":"2024-03-19T01:38:56.794668+0000","last_active":"2024-03-19T01:40:06.778680+0000","last_peered":"2024-03-19T01:40:06.778680+0000","last_clean":"2024-03-19T01:40:06.778680+0000","last_became_active":"2024-03-19T01:38:56.794375+0000","last_became_peered":"2024-03-19T01:38:56.794375+0000","last_unstale":"2024-03-19T01:40:06.778680+0000","last_undegraded":"2024-03-19T01:40:06.778680+0000","last_fullsized":"2024-03-19T01:40:06.778680+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T08:26:20.528825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2c","version":"89'8","reported_seq":29,"reported_epoch":91,"state":"active+clean","last_fresh":"2024-03-19T01:40:08.987543+0000","last_change":"2024-03-19T01:38:56.805535+0000","last_active":"2024-03-19T01:40:08.987543+0000","last_peered":"2024-03-19T01:40:08.987543+0000","last_clean":"2024-03-19T01:40:08.987543+0000","last_became_active":"2024-03-19T01:38:56.805211+0000","last_became_peered":"2024-03-19T01:38:56.805211+0000","last_unstale":"2024-03-19T01:40:08.987543+0000","last_undegraded":"2024-03-19T01:40:08.987543+0000","last_fullsized":"2024-03-19T01:40:08.987543+0000","mapping_epoch":85,"log_start":"0'0","ondisk_log_start":"0'0","created":85,"last_epoch_clean":86,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-19T01:38:55.768886+0000","last_clean_scrub_stamp":"2024-03-19T01:38:55.768886+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-20T04:47:28.688137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,4,8],"acting":[11,4,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":234,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":702,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":346,"ondisk_log_size":346,"up":192,"acting":192,"num_store_stats":12},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":109,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4157440,"data_stored":4132576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":137,"ondisk_log_size":137,"up":3,"acting":3,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":64946,"num_objects":60,"num_object_clones":0,"num_object_copies":180,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":60,"num_whiteouts":0,"num_read":441,"num_read_kb":290,"num_write":164,"num_write_kb":117,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":40,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":417792,"data_stored":194307,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":143,"ondisk_log_size":143,"up":96,"acting":96,"num_store_stats":12}],"osd_stats":[{"osd":11,"up_from":82,"seq":352187318300,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28004,"kb_used_data":980,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717148,"statfs":{"total":95995035648,"available":95966359552,"internally_reserved":0,"allocated":1003520,"data_stored":482409,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1596,"internal_metadata":27654596},"hb_peers":[0,1,2,4,5,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":10,"up_from":74,"seq":317827579938,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32844,"kb_used_data":1660,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93712308,"statfs":{"total":95995035648,"available":95961403392,"internally_reserved":0,"allocated":1699840,"data_stored":1134694,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1600,"internal_metadata":31914432},"hb_peers":[0,1,2,3,4,5,6,8,9,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":9,"up_from":68,"seq":292057776166,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32144,"kb_used_data":964,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93713008,"statfs":{"total":95995035648,"available":95962120192,"internally_reserved":0,"allocated":987136,"data_stored":465612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1599,"internal_metadata":31914433},"hb_peers":[0,1,2,3,4,5,6,7,8,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":8,"up_from":60,"seq":257698037802,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28000,"kb_used_data":980,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717152,"statfs":{"total":95995035648,"available":95966363648,"internally_reserved":0,"allocated":1003520,"data_stored":474059,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,2,3,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":7,"up_from":53,"seq":227633266736,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32728,"kb_used_data":1544,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93712424,"statfs":{"total":95995035648,"available":95961522176,"internally_reserved":0,"allocated":1581056,"data_stored":1057853,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":31914443},"hb_peers":[1,2,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863462965,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32132,"kb_used_data":952,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93713020,"statfs":{"total":95995035648,"available":95962132480,"internally_reserved":0,"allocated":974848,"data_stored":465001,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":31914440},"hb_peers":[0,1,2,3,4,5,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":1,"up_from":13,"seq":55834574924,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28028,"kb_used_data":1012,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717124,"statfs":{"total":95995035648,"available":95966334976,"internally_reserved":0,"allocated":1036288,"data_stored":524817,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2,3,4,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738450,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36308,"kb_used_data":972,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":93708844,"statfs":{"total":95995035648,"available":95957856256,"internally_reserved":0,"allocated":995328,"data_stored":466272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":36174277},"hb_peers":[1,2,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":18,"seq":77309411400,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28092,"kb_used_data":1068,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717060,"statfs":{"total":95995035648,"available":95966269440,"internally_reserved":0,"allocated":1093632,"data_stored":543981,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":27,"seq":115964117060,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32184,"kb_used_data":1000,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93712968,"statfs":{"total":95995035648,"available":95962079232,"internally_reserved":0,"allocated":1024000,"data_stored":482345,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31914439},"hb_peers":[0,1,2,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":34,"seq":146028888125,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27972,"kb_used_data":956,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717180,"statfs":{"total":95995035648,"available":95966392320,"internally_reserved":0,"allocated":978944,"data_stored":465970,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2,3,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":40,"seq":171798691897,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36896,"kb_used_data":1556,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":93708256,"statfs":{"total":95995035648,"available":95957254144,"internally_reserved":0,"allocated":1593344,"data_stored":1057146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":36174277},"hb_peers":[0,1,2,3,4,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1478,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":69632,"data_stored":60023,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":94208,"data_stored":62154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":2008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":2574,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":738,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":818,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":62382,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":234,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-19T01:40:13.063 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-03-19T01:40:13.063 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-03-19T01:40:13.063 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-03-19T01:40:13.063 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph health --format=json 2024-03-19T01:40:13.303 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:40:14.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:13 smithi099 ceph-mon[62951]: from='client.24817 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-19T01:40:14.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:13 smithi099 ceph-mon[62951]: pgmap v324: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 4.1 KiB/s rd, 895 B/s wr, 8 op/s 2024-03-19T01:40:14.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:13 smithi112 ceph-mon[66819]: from='client.24817 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-19T01:40:14.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:13 smithi112 ceph-mon[66819]: pgmap v324: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 4.1 KiB/s rd, 895 B/s wr, 8 op/s 2024-03-19T01:40:14.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:13 smithi115 ceph-mon[67332]: from='client.24817 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-19T01:40:14.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:13 smithi115 ceph-mon[67332]: pgmap v324: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 4.1 KiB/s rd, 895 B/s wr, 8 op/s 2024-03-19T01:40:14.932 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:14.931+0000 7f9d1d009700 1 -- 172.21.15.99:0/2905753980 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9d18101d50 msgr2=0x7f9d181021d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:14.932 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:14.931+0000 7f9d1d009700 1 --2- 172.21.15.99:0/2905753980 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9d18101d50 0x7f9d181021d0 secure :-1 s=READY pgs=272 cs=0 l=1 rev1=1 crypto rx=0x7f9d0c009990 tx=0x7f9d0c009ca0 comp rx=0 tx=0).stop 2024-03-19T01:40:14.932 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:14.931+0000 7f9d1d009700 1 -- 172.21.15.99:0/2905753980 shutdown_connections 2024-03-19T01:40:14.932 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:14.931+0000 7f9d1d009700 1 --2- 172.21.15.99:0/2905753980 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9d18106240 0x7f9d18106620 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:14.932 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:14.931+0000 7f9d1d009700 1 --2- 172.21.15.99:0/2905753980 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9d18102710 0x7f9d1810e0e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:14.932 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:14.931+0000 7f9d1d009700 1 --2- 172.21.15.99:0/2905753980 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9d18101d50 0x7f9d181021d0 unknown :-1 s=CLOSED pgs=272 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:14.932 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:14.931+0000 7f9d1d009700 1 -- 172.21.15.99:0/2905753980 >> 172.21.15.99:0/2905753980 conn(0x7f9d18076540 msgr2=0x7f9d18076950 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:14.932 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:14.931+0000 7f9d1d009700 1 -- 172.21.15.99:0/2905753980 shutdown_connections 2024-03-19T01:40:14.932 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:14.931+0000 7f9d1d009700 1 -- 172.21.15.99:0/2905753980 wait complete. 2024-03-19T01:40:14.934 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:14.932+0000 7f9d1d009700 1 Processor -- start 2024-03-19T01:40:14.934 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:14.932+0000 7f9d1d009700 1 -- start start 2024-03-19T01:40:14.934 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:14.933+0000 7f9d1d009700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9d18101d50 0x7f9d1819d680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:14.934 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:14.933+0000 7f9d1d009700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9d18102710 0x7f9d1819dbc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:14.934 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:14.933+0000 7f9d1d009700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9d18106240 0x7f9d18197800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:14.935 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:14.933+0000 7f9d1d009700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f9d18111540 con 0x7f9d18106240 2024-03-19T01:40:14.935 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:14.933+0000 7f9d1d009700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f9d181113c0 con 0x7f9d18101d50 2024-03-19T01:40:14.935 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:14.933+0000 7f9d1d009700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f9d181116c0 con 0x7f9d18102710 2024-03-19T01:40:14.935 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:14.933+0000 7f9d1759e700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9d18106240 0x7f9d18197800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:14.935 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:14.933+0000 7f9d16d9d700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9d18101d50 0x7f9d1819d680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:14.935 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:14.933+0000 7f9d1759e700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9d18106240 0x7f9d18197800 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:52698/0 (socket says 172.21.15.99:52698) 2024-03-19T01:40:14.935 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:14.933+0000 7f9d1759e700 1 -- 172.21.15.99:0/4173248580 learned_addr learned my addr 172.21.15.99:0/4173248580 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:14.935 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:14.933+0000 7f9d1659c700 1 --2- 172.21.15.99:0/4173248580 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9d18102710 0x7f9d1819dbc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:14.935 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:14.933+0000 7f9d16d9d700 1 -- 172.21.15.99:0/4173248580 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9d18102710 msgr2=0x7f9d1819dbc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:14.935 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:14.933+0000 7f9d16d9d700 1 --2- 172.21.15.99:0/4173248580 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9d18102710 0x7f9d1819dbc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:14.935 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:14.933+0000 7f9d16d9d700 1 -- 172.21.15.99:0/4173248580 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9d18106240 msgr2=0x7f9d18197800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:14.935 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:14.933+0000 7f9d16d9d700 1 --2- 172.21.15.99:0/4173248580 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9d18106240 0x7f9d18197800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:14.935 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:14.933+0000 7f9d16d9d700 1 -- 172.21.15.99:0/4173248580 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f9d18197fe0 con 0x7f9d18101d50 2024-03-19T01:40:14.935 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:14.933+0000 7f9d1759e700 1 --2- 172.21.15.99:0/4173248580 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9d18106240 0x7f9d18197800 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:40:14.935 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:14.933+0000 7f9d16d9d700 1 --2- 172.21.15.99:0/4173248580 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9d18101d50 0x7f9d1819d680 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 crypto rx=0x7f9d0800d750 tx=0x7f9d0800da60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:14.937 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:14.933+0000 7f9cfffff700 1 -- 172.21.15.99:0/4173248580 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9d08014030 con 0x7f9d18101d50 2024-03-19T01:40:14.937 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:14.933+0000 7f9cfffff700 1 -- 172.21.15.99:0/4173248580 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f9d08004500 con 0x7f9d18101d50 2024-03-19T01:40:14.937 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:14.933+0000 7f9d1d009700 1 -- 172.21.15.99:0/4173248580 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f9d181982d0 con 0x7f9d18101d50 2024-03-19T01:40:14.937 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:14.934+0000 7f9cfffff700 1 -- 172.21.15.99:0/4173248580 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9d08005420 con 0x7f9d18101d50 2024-03-19T01:40:14.937 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:14.934+0000 7f9d1d009700 1 -- 172.21.15.99:0/4173248580 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f9d181a44e0 con 0x7f9d18101d50 2024-03-19T01:40:14.938 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:14.935+0000 7f9d1d009700 1 -- 172.21.15.99:0/4173248580 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f9d1806a620 con 0x7f9d18101d50 2024-03-19T01:40:14.938 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:14.935+0000 7f9cfffff700 1 -- 172.21.15.99:0/4173248580 <== mon.2 v2:172.21.15.112:3300/0 4 ==== mgrmap(e 18) ==== 137290+0+0 (secure 0 0 0) 0x7f9d08004020 con 0x7f9d18101d50 2024-03-19T01:40:14.938 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:14.936+0000 7f9cfffff700 1 --2- 172.21.15.99:0/4173248580 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f9d000a2890 0x7f9d000a4d50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:14.938 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:14.936+0000 7f9d1659c700 1 --2- 172.21.15.99:0/4173248580 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f9d000a2890 0x7f9d000a4d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:14.938 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:14.936+0000 7f9cfffff700 1 -- 172.21.15.99:0/4173248580 <== mon.2 v2:172.21.15.112:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f9d080ce670 con 0x7f9d18101d50 2024-03-19T01:40:14.938 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:14.936+0000 7f9d1659c700 1 --2- 172.21.15.99:0/4173248580 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f9d000a2890 0x7f9d000a4d50 secure :-1 s=READY pgs=207 cs=0 l=1 rev1=1 crypto rx=0x7f9d0c0098e0 tx=0x7f9d0c005960 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:14.941 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:14.941+0000 7f9cfffff700 1 -- 172.21.15.99:0/4173248580 <== mon.2 v2:172.21.15.112:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f9d0809b6d0 con 0x7f9d18101d50 2024-03-19T01:40:15.180 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:15.180+0000 7f9d1d009700 1 -- 172.21.15.99:0/4173248580 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_command({"prefix": "health", "format": "json"} v 0) -- 0x7f9d1802dbb0 con 0x7f9d18101d50 2024-03-19T01:40:15.181 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:15.180+0000 7f9cfffff700 1 -- 172.21.15.99:0/4173248580 <== mon.2 v2:172.21.15.112:3300/0 7 ==== mon_command_ack([{"prefix": "health", "format": "json"}]=0 v0) ==== 72+0+46 (secure 0 0 0) 0x7f9d0809afd0 con 0x7f9d18101d50 2024-03-19T01:40:15.181 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:40:15.181 INFO:teuthology.orchestra.run.smithi099.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-03-19T01:40:15.182 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:15.182+0000 7f9d1d009700 1 -- 172.21.15.99:0/4173248580 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f9d000a2890 msgr2=0x7f9d000a4d50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:15.183 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:15.182+0000 7f9d1d009700 1 --2- 172.21.15.99:0/4173248580 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f9d000a2890 0x7f9d000a4d50 secure :-1 s=READY pgs=207 cs=0 l=1 rev1=1 crypto rx=0x7f9d0c0098e0 tx=0x7f9d0c005960 comp rx=0 tx=0).stop 2024-03-19T01:40:15.183 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:15.182+0000 7f9d1d009700 1 -- 172.21.15.99:0/4173248580 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9d18101d50 msgr2=0x7f9d1819d680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:15.183 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:15.182+0000 7f9d1d009700 1 --2- 172.21.15.99:0/4173248580 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9d18101d50 0x7f9d1819d680 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 crypto rx=0x7f9d0800d750 tx=0x7f9d0800da60 comp rx=0 tx=0).stop 2024-03-19T01:40:15.183 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:15.182+0000 7f9d1d009700 1 -- 172.21.15.99:0/4173248580 shutdown_connections 2024-03-19T01:40:15.183 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:15.182+0000 7f9d1d009700 1 --2- 172.21.15.99:0/4173248580 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f9d000a2890 0x7f9d000a4d50 unknown :-1 s=CLOSED pgs=207 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:15.183 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:15.182+0000 7f9d1d009700 1 --2- 172.21.15.99:0/4173248580 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9d18106240 0x7f9d18197800 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:15.183 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:15.182+0000 7f9d1d009700 1 --2- 172.21.15.99:0/4173248580 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9d18101d50 0x7f9d1819d680 unknown :-1 s=CLOSED pgs=63 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:15.183 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:15.182+0000 7f9d1d009700 1 --2- 172.21.15.99:0/4173248580 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9d18102710 0x7f9d1819dbc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:15.183 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:15.182+0000 7f9d1d009700 1 -- 172.21.15.99:0/4173248580 >> 172.21.15.99:0/4173248580 conn(0x7f9d18076540 msgr2=0x7f9d180febf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:15.183 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:15.183+0000 7f9d1d009700 1 -- 172.21.15.99:0/4173248580 shutdown_connections 2024-03-19T01:40:15.184 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:15.183+0000 7f9d1d009700 1 -- 172.21.15.99:0/4173248580 wait complete. 2024-03-19T01:40:15.771 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-03-19T01:40:15.772 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-19T01:40:15.880 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:15.878+0000 7f516b0f8640 1 -- 172.21.15.99:0/944261263 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5164101950 msgr2=0x7f5164101dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:15.881 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:15.878+0000 7f516b0f8640 1 --2- 172.21.15.99:0/944261263 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5164101950 0x7f5164101dd0 secure :-1 s=READY pgs=273 cs=0 l=1 rev1=1 crypto rx=0x7f5154009900 tx=0x7f515402bc30 comp rx=0 tx=0).stop 2024-03-19T01:40:15.881 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:15.878+0000 7f516b0f8640 1 -- 172.21.15.99:0/944261263 shutdown_connections 2024-03-19T01:40:15.881 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:15.878+0000 7f516b0f8640 1 --2- 172.21.15.99:0/944261263 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5164102310 0x7f5164108ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:15.881 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:15.878+0000 7f516b0f8640 1 --2- 172.21.15.99:0/944261263 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5164101950 0x7f5164101dd0 unknown :-1 s=CLOSED pgs=273 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:15.881 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:15.878+0000 7f516b0f8640 1 --2- 172.21.15.99:0/944261263 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5164100760 0x7f5164100b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:15.881 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:15.878+0000 7f516b0f8640 1 -- 172.21.15.99:0/944261263 >> 172.21.15.99:0/944261263 conn(0x7f51640fbcd0 msgr2=0x7f51640fe110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:15.881 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:15.878+0000 7f516b0f8640 1 -- 172.21.15.99:0/944261263 shutdown_connections 2024-03-19T01:40:15.881 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:15.878+0000 7f516b0f8640 1 -- 172.21.15.99:0/944261263 wait complete. 2024-03-19T01:40:15.881 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:15.879+0000 7f516b0f8640 1 Processor -- start 2024-03-19T01:40:15.881 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:15.879+0000 7f516b0f8640 1 -- start start 2024-03-19T01:40:15.881 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:15.879+0000 7f516b0f8640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5164100760 0x7f5164196a80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:15.881 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:15.879+0000 7f516b0f8640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5164101950 0x7f5164196fc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:15.882 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:15.879+0000 7f516b0f8640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5164102310 0x7f516419e030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:15.882 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:15.879+0000 7f516b0f8640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f516410c3e0 con 0x7f5164100760 2024-03-19T01:40:15.882 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:15.879+0000 7f516b0f8640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f516410c260 con 0x7f5164101950 2024-03-19T01:40:15.882 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:15.879+0000 7f516b0f8640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f516410c560 con 0x7f5164102310 2024-03-19T01:40:15.882 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:15.879+0000 7f5168e94640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5164100760 0x7f5164196a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:15.882 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:15.879+0000 7f5168e94640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5164100760 0x7f5164196a80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:52712/0 (socket says 172.21.15.99:52712) 2024-03-19T01:40:15.882 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:15.879+0000 7f5168e94640 1 -- 172.21.15.99:0/3336642981 learned_addr learned my addr 172.21.15.99:0/3336642981 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:15.882 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:15.879+0000 7f5169695640 1 --2- 172.21.15.99:0/3336642981 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5164102310 0x7f516419e030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:15.882 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:15.879+0000 7f515bfff640 1 --2- 172.21.15.99:0/3336642981 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5164101950 0x7f5164196fc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:15.882 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:15.879+0000 7f5168e94640 1 -- 172.21.15.99:0/3336642981 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5164102310 msgr2=0x7f516419e030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:15.882 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:15.879+0000 7f5168e94640 1 --2- 172.21.15.99:0/3336642981 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5164102310 0x7f516419e030 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:15.882 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:15.879+0000 7f5168e94640 1 -- 172.21.15.99:0/3336642981 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5164101950 msgr2=0x7f5164196fc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:15.882 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:15.879+0000 7f5168e94640 1 --2- 172.21.15.99:0/3336642981 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5164101950 0x7f5164196fc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:15.883 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:15.879+0000 7f5168e94640 1 -- 172.21.15.99:0/3336642981 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f516419e6a0 con 0x7f5164100760 2024-03-19T01:40:15.883 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:15.879+0000 7f5168e94640 1 --2- 172.21.15.99:0/3336642981 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5164100760 0x7f5164196a80 secure :-1 s=READY pgs=274 cs=0 l=1 rev1=1 crypto rx=0x7f516000e9c0 tx=0x7f516000edc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:15.883 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:15.879+0000 7f5159ffb640 1 -- 172.21.15.99:0/3336642981 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f516000ccc0 con 0x7f5164100760 2024-03-19T01:40:15.883 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:15.879+0000 7f516b0f8640 1 -- 172.21.15.99:0/3336642981 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f516419e990 con 0x7f5164100760 2024-03-19T01:40:15.883 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:15.879+0000 7f5159ffb640 1 -- 172.21.15.99:0/3336642981 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f51600035f0 con 0x7f5164100760 2024-03-19T01:40:15.883 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:15.880+0000 7f5159ffb640 1 -- 172.21.15.99:0/3336642981 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f516000ce60 con 0x7f5164100760 2024-03-19T01:40:15.883 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:15.880+0000 7f516b0f8640 1 -- 172.21.15.99:0/3336642981 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f516419ee60 con 0x7f5164100760 2024-03-19T01:40:15.883 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:15.881+0000 7f516b0f8640 1 -- 172.21.15.99:0/3336642981 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f5164107a80 con 0x7f5164100760 2024-03-19T01:40:15.884 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:15.882+0000 7f5159ffb640 1 -- 172.21.15.99:0/3336642981 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 18) ==== 137290+0+0 (secure 0 0 0) 0x7f51600037a0 con 0x7f5164100760 2024-03-19T01:40:15.884 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:15.882+0000 7f5159ffb640 1 --2- 172.21.15.99:0/3336642981 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f514c0a2800 0x7f514c0a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:15.884 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:15.882+0000 7f515bfff640 1 --2- 172.21.15.99:0/3336642981 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f514c0a2800 0x7f514c0a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:15.884 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:15.882+0000 7f5159ffb640 1 -- 172.21.15.99:0/3336642981 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f51600cee00 con 0x7f5164100760 2024-03-19T01:40:15.884 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:15.883+0000 7f515bfff640 1 --2- 172.21.15.99:0/3336642981 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f514c0a2800 0x7f514c0a4cc0 secure :-1 s=READY pgs=208 cs=0 l=1 rev1=1 crypto rx=0x7f515402c7b0 tx=0x7f5154005fd0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:15.887 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:15.887+0000 7f5159ffb640 1 -- 172.21.15.99:0/3336642981 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f516009be60 con 0x7f5164100760 2024-03-19T01:40:16.064 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.064+0000 7f516b0f8640 1 -- 172.21.15.99:0/3336642981 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f5164062420 con 0x7f5164100760 2024-03-19T01:40:16.066 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.066+0000 7f5159ffb640 1 -- 172.21.15.99:0/3336642981 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v91) ==== 74+0+22804 (secure 0 0 0) 0x7f516009b760 con 0x7f5164100760 2024-03-19T01:40:16.066 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:40:16.066 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":91,"fsid":"438fb69c-e590-11ee-95c9-87774f69a715","created":"2024-03-19T01:31:10.165073+0000","modified":"2024-03-19T01:40:08.821469+0000","last_up_change":"2024-03-19T01:38:09.082688+0000","last_in_change":"2024-03-19T01:37:49.115793+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-19T01:34:31.001309+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-19T01:38:54.018623+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"86","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-19T01:38:55.319036+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"87","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"0c407a0e-a40e-43d7-ad9a-bb8f3cd4a9ba","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6803","nonce":2935260112}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6805","nonce":2935260112}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6808","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6809","nonce":2935260112}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6807","nonce":2935260112}]},"public_addr":"172.21.15.99:6803/2935260112","cluster_addr":"172.21.15.99:6805/2935260112","heartbeat_back_addr":"172.21.15.99:6809/2935260112","heartbeat_front_addr":"172.21.15.99:6807/2935260112","state":["exists","up"]},{"osd":1,"uuid":"a48f9173-4a06-49f5-bca2-2c9047b8f666","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6800","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6801","nonce":1468854289}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6802","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6803","nonce":1468854289}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6806","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6807","nonce":1468854289}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6804","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6805","nonce":1468854289}]},"public_addr":"172.21.15.112:6801/1468854289","cluster_addr":"172.21.15.112:6803/1468854289","heartbeat_back_addr":"172.21.15.112:6807/1468854289","heartbeat_front_addr":"172.21.15.112:6805/1468854289","state":["exists","up"]},{"osd":2,"uuid":"b7715fdd-b6c9-4348-bd16-5eb61b427eef","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6800","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6801","nonce":3766690472}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6802","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6803","nonce":3766690472}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6806","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6807","nonce":3766690472}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6804","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6805","nonce":3766690472}]},"public_addr":"172.21.15.115:6801/3766690472","cluster_addr":"172.21.15.115:6803/3766690472","heartbeat_back_addr":"172.21.15.115:6807/3766690472","heartbeat_front_addr":"172.21.15.115:6805/3766690472","state":["exists","up"]},{"osd":3,"uuid":"6697ed1b-96bf-42de-8440-d17e9785b359","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6810","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6811","nonce":3756732895}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6812","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6813","nonce":3756732895}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6816","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6817","nonce":3756732895}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6814","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6815","nonce":3756732895}]},"public_addr":"172.21.15.99:6811/3756732895","cluster_addr":"172.21.15.99:6813/3756732895","heartbeat_back_addr":"172.21.15.99:6817/3756732895","heartbeat_front_addr":"172.21.15.99:6815/3756732895","state":["exists","up"]},{"osd":4,"uuid":"d958d6b9-67f6-43d0-b704-aa55dc9df83a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":34,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6808","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6809","nonce":3717986391}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6810","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6811","nonce":3717986391}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6814","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6815","nonce":3717986391}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6812","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6813","nonce":3717986391}]},"public_addr":"172.21.15.112:6809/3717986391","cluster_addr":"172.21.15.112:6811/3717986391","heartbeat_back_addr":"172.21.15.112:6815/3717986391","heartbeat_front_addr":"172.21.15.112:6813/3717986391","state":["exists","up"]},{"osd":5,"uuid":"a73eb49a-7925-49ba-bc4f-515e17f8b8ae","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6808","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6809","nonce":1971806408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6810","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6811","nonce":1971806408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6814","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6815","nonce":1971806408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6812","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6813","nonce":1971806408}]},"public_addr":"172.21.15.115:6809/1971806408","cluster_addr":"172.21.15.115:6811/1971806408","heartbeat_back_addr":"172.21.15.115:6815/1971806408","heartbeat_front_addr":"172.21.15.115:6813/1971806408","state":["exists","up"]},{"osd":6,"uuid":"caa718cd-e08b-48de-832a-29546d71e508","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6818","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6819","nonce":1130832772}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6820","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6821","nonce":1130832772}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6824","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6825","nonce":1130832772}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6822","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6823","nonce":1130832772}]},"public_addr":"172.21.15.99:6819/1130832772","cluster_addr":"172.21.15.99:6821/1130832772","heartbeat_back_addr":"172.21.15.99:6825/1130832772","heartbeat_front_addr":"172.21.15.99:6823/1130832772","state":["exists","up"]},{"osd":7,"uuid":"dbee19a1-2315-40d3-88e7-876fcde28903","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":53,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6816","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6817","nonce":3610356576}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6818","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6819","nonce":3610356576}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6822","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6823","nonce":3610356576}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6820","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6821","nonce":3610356576}]},"public_addr":"172.21.15.112:6817/3610356576","cluster_addr":"172.21.15.112:6819/3610356576","heartbeat_back_addr":"172.21.15.112:6823/3610356576","heartbeat_front_addr":"172.21.15.112:6821/3610356576","state":["exists","up"]},{"osd":8,"uuid":"14a866dc-5ce3-42fc-8bcf-9088b68726cb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":60,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6816","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6817","nonce":4255803111}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6818","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6819","nonce":4255803111}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6822","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6823","nonce":4255803111}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6820","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6821","nonce":4255803111}]},"public_addr":"172.21.15.115:6817/4255803111","cluster_addr":"172.21.15.115:6819/4255803111","heartbeat_back_addr":"172.21.15.115:6823/4255803111","heartbeat_front_addr":"172.21.15.115:6821/4255803111","state":["exists","up"]},{"osd":9,"uuid":"1db93652-6035-4906-91ae-d64d5a909502","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":68,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6826","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6827","nonce":1410921523}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6828","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6829","nonce":1410921523}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6832","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6833","nonce":1410921523}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6830","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6831","nonce":1410921523}]},"public_addr":"172.21.15.99:6827/1410921523","cluster_addr":"172.21.15.99:6829/1410921523","heartbeat_back_addr":"172.21.15.99:6833/1410921523","heartbeat_front_addr":"172.21.15.99:6831/1410921523","state":["exists","up"]},{"osd":10,"uuid":"6c50a8a7-3345-4f4a-af0e-5606c7aca453","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":74,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6824","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6825","nonce":891805922}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6826","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6827","nonce":891805922}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6830","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6831","nonce":891805922}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6828","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6829","nonce":891805922}]},"public_addr":"172.21.15.112:6825/891805922","cluster_addr":"172.21.15.112:6827/891805922","heartbeat_back_addr":"172.21.15.112:6831/891805922","heartbeat_front_addr":"172.21.15.112:6829/891805922","state":["exists","up"]},{"osd":11,"uuid":"fd63b7fd-0db5-46d5-82fa-9707bbd0b495","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":82,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6824","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6825","nonce":938436244}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6826","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6827","nonce":938436244}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6830","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6831","nonce":938436244}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6828","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6829","nonce":938436244}]},"public_addr":"172.21.15.115:6825/938436244","cluster_addr":"172.21.15.115:6827/938436244","heartbeat_back_addr":"172.21.15.115:6831/938436244","heartbeat_front_addr":"172.21.15.115:6829/938436244","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:33:39.036822+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:34:03.339566+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:34:27.149273+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:34:51.315561+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:35:15.396031+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:35:38.957168+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:36:03.213914+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:36:27.763699+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:36:51.688670+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:37:17.280165+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:37:42.554633+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:38:07.438039+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.0","mappings":[{"from":8,"to":1}]},{"pgid":"2.8","mappings":[{"from":2,"to":11}]},{"pgid":"2.b","mappings":[{"from":8,"to":11}]},{"pgid":"2.e","mappings":[{"from":2,"to":4}]},{"pgid":"3.0","mappings":[{"from":5,"to":9}]},{"pgid":"3.2","mappings":[{"from":3,"to":9}]},{"pgid":"3.14","mappings":[{"from":1,"to":9}]},{"pgid":"3.17","mappings":[{"from":4,"to":8}]},{"pgid":"3.31","mappings":[{"from":4,"to":8}]},{"pgid":"3.37","mappings":[{"from":5,"to":7}]},{"pgid":"3.3a","mappings":[{"from":4,"to":6}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.99:6801/1328245934":"2024-03-20T01:32:06.661002+0000","172.21.15.99:0/1618965837":"2024-03-20T01:32:06.661002+0000","172.21.15.99:0/2491429851":"2024-03-20T01:32:06.661002+0000","172.21.15.99:6800/1328245934":"2024-03-20T01:32:06.661002+0000","172.21.15.99:0/2112231766":"2024-03-20T01:31:40.161751+0000","172.21.15.99:0/3341836545":"2024-03-20T01:31:40.161751+0000","172.21.15.99:6800/4223048293":"2024-03-20T01:31:40.161751+0000","172.21.15.99:6801/4223048293":"2024-03-20T01:31:40.161751+0000","172.21.15.99:0/2377621863":"2024-03-20T01:31:40.161751+0000","172.21.15.99:0/1904400935":"2024-03-20T01:32:06.661002+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-19T01:40:16.068 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.068+0000 7f516b0f8640 1 -- 172.21.15.99:0/3336642981 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f514c0a2800 msgr2=0x7f514c0a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:16.068 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.068+0000 7f516b0f8640 1 --2- 172.21.15.99:0/3336642981 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f514c0a2800 0x7f514c0a4cc0 secure :-1 s=READY pgs=208 cs=0 l=1 rev1=1 crypto rx=0x7f515402c7b0 tx=0x7f5154005fd0 comp rx=0 tx=0).stop 2024-03-19T01:40:16.068 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.068+0000 7f516b0f8640 1 -- 172.21.15.99:0/3336642981 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5164100760 msgr2=0x7f5164196a80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:16.068 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.068+0000 7f516b0f8640 1 --2- 172.21.15.99:0/3336642981 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5164100760 0x7f5164196a80 secure :-1 s=READY pgs=274 cs=0 l=1 rev1=1 crypto rx=0x7f516000e9c0 tx=0x7f516000edc0 comp rx=0 tx=0).stop 2024-03-19T01:40:16.068 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.068+0000 7f516b0f8640 1 -- 172.21.15.99:0/3336642981 shutdown_connections 2024-03-19T01:40:16.069 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.068+0000 7f516b0f8640 1 --2- 172.21.15.99:0/3336642981 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f514c0a2800 0x7f514c0a4cc0 unknown :-1 s=CLOSED pgs=208 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:16.069 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.068+0000 7f516b0f8640 1 --2- 172.21.15.99:0/3336642981 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5164101950 0x7f5164196fc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:16.069 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.068+0000 7f516b0f8640 1 --2- 172.21.15.99:0/3336642981 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5164102310 0x7f516419e030 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:16.069 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.068+0000 7f516b0f8640 1 --2- 172.21.15.99:0/3336642981 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5164100760 0x7f5164196a80 unknown :-1 s=CLOSED pgs=274 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:16.069 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.068+0000 7f516b0f8640 1 -- 172.21.15.99:0/3336642981 >> 172.21.15.99:0/3336642981 conn(0x7f51640fbcd0 msgr2=0x7f5164104c50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:16.069 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.068+0000 7f516b0f8640 1 -- 172.21.15.99:0/3336642981 shutdown_connections 2024-03-19T01:40:16.069 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.068+0000 7f516b0f8640 1 -- 172.21.15.99:0/3336642981 wait complete. 2024-03-19T01:40:16.078 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:15 smithi099 ceph-mon[62951]: pgmap v325: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 895 B/s wr, 7 op/s 2024-03-19T01:40:16.078 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:15 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/4173248580' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-19T01:40:16.080 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-19T01:34:31.001309+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 12, 'score_stable': 12, 'optimal_score': 0.25, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-19T01:38:54.018623+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '86', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.8799999952316284, 'score_stable': 1.8799999952316284, 'optimal_score': 1, 'raw_score_acting': 1.8799999952316284, 'raw_score_stable': 1.8799999952316284, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-03-19T01:38:55.319036+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '87', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-19T01:40:16.080 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-19T01:40:16.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:15 smithi112 ceph-mon[66819]: pgmap v325: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 895 B/s wr, 7 op/s 2024-03-19T01:40:16.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:15 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/4173248580' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-19T01:40:16.188 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.185+0000 7ff9c84ea640 1 -- 172.21.15.99:0/3526202561 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff9c01022c0 msgr2=0x7ff9c01046b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:16.188 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.185+0000 7ff9c84ea640 1 --2- 172.21.15.99:0/3526202561 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff9c01022c0 0x7ff9c01046b0 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 crypto rx=0x7ff9bc009860 tx=0x7ff9bc02bb70 comp rx=0 tx=0).stop 2024-03-19T01:40:16.188 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.185+0000 7ff9c84ea640 1 -- 172.21.15.99:0/3526202561 shutdown_connections 2024-03-19T01:40:16.188 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.185+0000 7ff9c84ea640 1 --2- 172.21.15.99:0/3526202561 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff9c0107520 0x7ff9c0109970 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:16.188 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.185+0000 7ff9c84ea640 1 --2- 172.21.15.99:0/3526202561 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff9c0104bf0 0x7ff9c0106fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:16.188 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.185+0000 7ff9c84ea640 1 --2- 172.21.15.99:0/3526202561 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff9c01022c0 0x7ff9c01046b0 unknown :-1 s=CLOSED pgs=64 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:16.188 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.185+0000 7ff9c84ea640 1 -- 172.21.15.99:0/3526202561 >> 172.21.15.99:0/3526202561 conn(0x7ff9c00fbcd0 msgr2=0x7ff9c00fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:16.188 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.185+0000 7ff9c84ea640 1 -- 172.21.15.99:0/3526202561 shutdown_connections 2024-03-19T01:40:16.188 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.185+0000 7ff9c84ea640 1 -- 172.21.15.99:0/3526202561 wait complete. 2024-03-19T01:40:16.188 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.186+0000 7ff9c84ea640 1 Processor -- start 2024-03-19T01:40:16.188 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.186+0000 7ff9c84ea640 1 -- start start 2024-03-19T01:40:16.189 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.186+0000 7ff9c84ea640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff9c01022c0 0x7ff9c0197bc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:16.189 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.186+0000 7ff9c84ea640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff9c0104bf0 0x7ff9c0198100 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:16.189 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.186+0000 7ff9c84ea640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff9c0107520 0x7ff9c019f170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:16.189 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.186+0000 7ff9c84ea640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7ff9c010d4c0 con 0x7ff9c01022c0 2024-03-19T01:40:16.189 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.186+0000 7ff9c84ea640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7ff9c010d340 con 0x7ff9c0104bf0 2024-03-19T01:40:16.189 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.186+0000 7ff9c84ea640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7ff9c010d640 con 0x7ff9c0107520 2024-03-19T01:40:16.189 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.186+0000 7ff9c6286640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff9c01022c0 0x7ff9c0197bc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:16.189 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.186+0000 7ff9c6286640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff9c01022c0 0x7ff9c0197bc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:52736/0 (socket says 172.21.15.99:52736) 2024-03-19T01:40:16.189 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.186+0000 7ff9c6286640 1 -- 172.21.15.99:0/444389581 learned_addr learned my addr 172.21.15.99:0/444389581 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:16.189 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.186+0000 7ff9c5a85640 1 --2- 172.21.15.99:0/444389581 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff9c0104bf0 0x7ff9c0198100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:16.189 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.186+0000 7ff9c6a87640 1 --2- 172.21.15.99:0/444389581 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff9c0107520 0x7ff9c019f170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:16.189 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.186+0000 7ff9c6286640 1 -- 172.21.15.99:0/444389581 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff9c0107520 msgr2=0x7ff9c019f170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:16.189 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.186+0000 7ff9c6286640 1 --2- 172.21.15.99:0/444389581 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff9c0107520 0x7ff9c019f170 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:16.189 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.186+0000 7ff9c6286640 1 -- 172.21.15.99:0/444389581 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff9c0104bf0 msgr2=0x7ff9c0198100 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:16.190 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.186+0000 7ff9c6286640 1 --2- 172.21.15.99:0/444389581 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff9c0104bf0 0x7ff9c0198100 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:16.190 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.186+0000 7ff9c6286640 1 -- 172.21.15.99:0/444389581 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ff9c019f7e0 con 0x7ff9c01022c0 2024-03-19T01:40:16.190 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.186+0000 7ff9c5a85640 1 --2- 172.21.15.99:0/444389581 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff9c0104bf0 0x7ff9c0198100 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:40:16.190 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.187+0000 7ff9c6286640 1 --2- 172.21.15.99:0/444389581 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff9c01022c0 0x7ff9c0197bc0 secure :-1 s=READY pgs=275 cs=0 l=1 rev1=1 crypto rx=0x7ff9bc005940 tx=0x7ff9bc032470 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:16.190 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.187+0000 7ff9b77fe640 1 -- 172.21.15.99:0/444389581 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff9bc041020 con 0x7ff9c01022c0 2024-03-19T01:40:16.190 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.187+0000 7ff9c84ea640 1 -- 172.21.15.99:0/444389581 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ff9c019fa70 con 0x7ff9c01022c0 2024-03-19T01:40:16.190 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.187+0000 7ff9b77fe640 1 -- 172.21.15.99:0/444389581 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7ff9bc02c9f0 con 0x7ff9c01022c0 2024-03-19T01:40:16.190 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.187+0000 7ff9b77fe640 1 -- 172.21.15.99:0/444389581 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff9bc039040 con 0x7ff9c01022c0 2024-03-19T01:40:16.190 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.187+0000 7ff9c84ea640 1 -- 172.21.15.99:0/444389581 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7ff9c019ffb0 con 0x7ff9c01022c0 2024-03-19T01:40:16.190 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.188+0000 7ff9c84ea640 1 -- 172.21.15.99:0/444389581 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7ff9c00fd7c0 con 0x7ff9c01022c0 2024-03-19T01:40:16.191 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.189+0000 7ff9b77fe640 1 -- 172.21.15.99:0/444389581 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 18) ==== 137290+0+0 (secure 0 0 0) 0x7ff9bc044020 con 0x7ff9c01022c0 2024-03-19T01:40:16.191 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.189+0000 7ff9b77fe640 1 --2- 172.21.15.99:0/444389581 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff9ac0a2800 0x7ff9ac0a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:16.191 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.189+0000 7ff9c5a85640 1 --2- 172.21.15.99:0/444389581 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff9ac0a2800 0x7ff9ac0a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:16.191 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.189+0000 7ff9b77fe640 1 -- 172.21.15.99:0/444389581 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7ff9bc0f03f0 con 0x7ff9c01022c0 2024-03-19T01:40:16.191 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.190+0000 7ff9c5a85640 1 --2- 172.21.15.99:0/444389581 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff9ac0a2800 0x7ff9ac0a4cc0 secure :-1 s=READY pgs=209 cs=0 l=1 rev1=1 crypto rx=0x7ff9c01990d0 tx=0x7ff9b0005f90 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:16.193 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.194+0000 7ff9b77fe640 1 -- 172.21.15.99:0/444389581 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7ff9bc04b020 con 0x7ff9c01022c0 2024-03-19T01:40:16.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:15 smithi115 ceph-mon[67332]: pgmap v325: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 895 B/s wr, 7 op/s 2024-03-19T01:40:16.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:15 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/4173248580' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-19T01:40:16.370 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.370+0000 7ff9c84ea640 1 -- 172.21.15.99:0/444389581 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) -- 0x7ff9c0062420 con 0x7ff9c01022c0 2024-03-19T01:40:16.370 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.370+0000 7ff9b77fe640 1 -- 172.21.15.99:0/444389581 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v91) ==== 93+0+10 (secure 0 0 0) 0x7ff9bc0bd450 con 0x7ff9c01022c0 2024-03-19T01:40:16.371 INFO:teuthology.orchestra.run.smithi099.stdout:pg_num: 1 2024-03-19T01:40:16.373 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.372+0000 7ff9c84ea640 1 -- 172.21.15.99:0/444389581 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff9ac0a2800 msgr2=0x7ff9ac0a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:16.374 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.372+0000 7ff9c84ea640 1 --2- 172.21.15.99:0/444389581 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff9ac0a2800 0x7ff9ac0a4cc0 secure :-1 s=READY pgs=209 cs=0 l=1 rev1=1 crypto rx=0x7ff9c01990d0 tx=0x7ff9b0005f90 comp rx=0 tx=0).stop 2024-03-19T01:40:16.374 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.372+0000 7ff9c84ea640 1 -- 172.21.15.99:0/444389581 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff9c01022c0 msgr2=0x7ff9c0197bc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:16.374 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.372+0000 7ff9c84ea640 1 --2- 172.21.15.99:0/444389581 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff9c01022c0 0x7ff9c0197bc0 secure :-1 s=READY pgs=275 cs=0 l=1 rev1=1 crypto rx=0x7ff9bc005940 tx=0x7ff9bc032470 comp rx=0 tx=0).stop 2024-03-19T01:40:16.374 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.372+0000 7ff9c84ea640 1 -- 172.21.15.99:0/444389581 shutdown_connections 2024-03-19T01:40:16.374 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.372+0000 7ff9c84ea640 1 --2- 172.21.15.99:0/444389581 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff9ac0a2800 0x7ff9ac0a4cc0 unknown :-1 s=CLOSED pgs=209 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:16.374 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.372+0000 7ff9c84ea640 1 --2- 172.21.15.99:0/444389581 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff9c0104bf0 0x7ff9c0198100 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:16.374 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.372+0000 7ff9c84ea640 1 --2- 172.21.15.99:0/444389581 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff9c0107520 0x7ff9c019f170 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:16.374 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.372+0000 7ff9c84ea640 1 --2- 172.21.15.99:0/444389581 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff9c01022c0 0x7ff9c0197bc0 unknown :-1 s=CLOSED pgs=275 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:16.374 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.372+0000 7ff9c84ea640 1 -- 172.21.15.99:0/444389581 >> 172.21.15.99:0/444389581 conn(0x7ff9c00fbcd0 msgr2=0x7ff9c0102ce0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:16.374 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.372+0000 7ff9c84ea640 1 -- 172.21.15.99:0/444389581 shutdown_connections 2024-03-19T01:40:16.374 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.372+0000 7ff9c84ea640 1 -- 172.21.15.99:0/444389581 wait complete. 2024-03-19T01:40:16.384 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-19T01:40:16.490 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.488+0000 7fd9f3863640 1 -- 172.21.15.99:0/2123136505 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd9ec100730 msgr2=0x7fd9ec100b50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:16.491 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.488+0000 7fd9f3863640 1 --2- 172.21.15.99:0/2123136505 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd9ec100730 0x7fd9ec100b50 secure :-1 s=READY pgs=276 cs=0 l=1 rev1=1 crypto rx=0x7fd9e8009900 tx=0x7fd9e802bc30 comp rx=0 tx=0).stop 2024-03-19T01:40:16.491 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.488+0000 7fd9f3863640 1 -- 172.21.15.99:0/2123136505 shutdown_connections 2024-03-19T01:40:16.491 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.488+0000 7fd9f3863640 1 --2- 172.21.15.99:0/2123136505 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd9ec101920 0x7fd9ec101da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:16.491 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.488+0000 7fd9f3863640 1 --2- 172.21.15.99:0/2123136505 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd9ec1022e0 0x7fd9ec108b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:16.491 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.488+0000 7fd9f3863640 1 --2- 172.21.15.99:0/2123136505 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd9ec100730 0x7fd9ec100b50 unknown :-1 s=CLOSED pgs=276 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:16.491 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.488+0000 7fd9f3863640 1 -- 172.21.15.99:0/2123136505 >> 172.21.15.99:0/2123136505 conn(0x7fd9ec0fbcc0 msgr2=0x7fd9ec0fe0e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:16.491 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.488+0000 7fd9f3863640 1 -- 172.21.15.99:0/2123136505 shutdown_connections 2024-03-19T01:40:16.491 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.488+0000 7fd9f3863640 1 -- 172.21.15.99:0/2123136505 wait complete. 2024-03-19T01:40:16.491 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.489+0000 7fd9f3863640 1 Processor -- start 2024-03-19T01:40:16.491 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.489+0000 7fd9f3863640 1 -- start start 2024-03-19T01:40:16.491 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.489+0000 7fd9f3863640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd9ec100730 0x7fd9ec06a9a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:16.492 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.489+0000 7fd9f3863640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd9ec101920 0x7fd9ec19c8c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:16.492 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.489+0000 7fd9f3863640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd9ec1022e0 0x7fd9ec19ef00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:16.492 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.489+0000 7fd9f3863640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fd9ec10c3b0 con 0x7fd9ec1022e0 2024-03-19T01:40:16.492 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.489+0000 7fd9f3863640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fd9ec10c230 con 0x7fd9ec100730 2024-03-19T01:40:16.492 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.489+0000 7fd9f3863640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fd9ec10c530 con 0x7fd9ec101920 2024-03-19T01:40:16.492 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.489+0000 7fd9f15ff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd9ec100730 0x7fd9ec06a9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:16.492 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.489+0000 7fd9f1e00640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd9ec1022e0 0x7fd9ec19ef00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:16.492 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.489+0000 7fd9f15ff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd9ec100730 0x7fd9ec06a9a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:45014/0 (socket says 172.21.15.99:45014) 2024-03-19T01:40:16.492 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.489+0000 7fd9f15ff640 1 -- 172.21.15.99:0/743216750 learned_addr learned my addr 172.21.15.99:0/743216750 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:16.492 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.489+0000 7fd9f0dfe640 1 --2- 172.21.15.99:0/743216750 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd9ec101920 0x7fd9ec19c8c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:16.492 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.489+0000 7fd9f1e00640 1 -- 172.21.15.99:0/743216750 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd9ec101920 msgr2=0x7fd9ec19c8c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:16.492 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.489+0000 7fd9f1e00640 1 --2- 172.21.15.99:0/743216750 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd9ec101920 0x7fd9ec19c8c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:16.492 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.489+0000 7fd9f1e00640 1 -- 172.21.15.99:0/743216750 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd9ec100730 msgr2=0x7fd9ec06a9a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:16.492 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.489+0000 7fd9f1e00640 1 --2- 172.21.15.99:0/743216750 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd9ec100730 0x7fd9ec06a9a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:16.493 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.489+0000 7fd9f1e00640 1 -- 172.21.15.99:0/743216750 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd9ec19f7c0 con 0x7fd9ec1022e0 2024-03-19T01:40:16.493 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.489+0000 7fd9f15ff640 1 --2- 172.21.15.99:0/743216750 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd9ec100730 0x7fd9ec06a9a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:40:16.493 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.489+0000 7fd9f1e00640 1 --2- 172.21.15.99:0/743216750 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd9ec1022e0 0x7fd9ec19ef00 secure :-1 s=READY pgs=277 cs=0 l=1 rev1=1 crypto rx=0x7fd9e4009fd0 tx=0x7fd9e400ef10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:16.493 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.490+0000 7fd9e27fc640 1 -- 172.21.15.99:0/743216750 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd9e400ce20 con 0x7fd9ec1022e0 2024-03-19T01:40:16.493 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.490+0000 7fd9f3863640 1 -- 172.21.15.99:0/743216750 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd9ec19fab0 con 0x7fd9ec1022e0 2024-03-19T01:40:16.493 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.490+0000 7fd9e27fc640 1 -- 172.21.15.99:0/743216750 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd9e4005020 con 0x7fd9ec1022e0 2024-03-19T01:40:16.493 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.490+0000 7fd9e27fc640 1 -- 172.21.15.99:0/743216750 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd9e4010720 con 0x7fd9ec1022e0 2024-03-19T01:40:16.493 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.490+0000 7fd9f3863640 1 -- 172.21.15.99:0/743216750 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fd9ec1a0000 con 0x7fd9ec1022e0 2024-03-19T01:40:16.493 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.491+0000 7fd9f3863640 1 -- 172.21.15.99:0/743216750 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fd9ec107a70 con 0x7fd9ec1022e0 2024-03-19T01:40:16.494 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.492+0000 7fd9e27fc640 1 -- 172.21.15.99:0/743216750 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 18) ==== 137290+0+0 (secure 0 0 0) 0x7fd9e4005540 con 0x7fd9ec1022e0 2024-03-19T01:40:16.494 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.492+0000 7fd9e27fc640 1 --2- 172.21.15.99:0/743216750 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd9d80a2800 0x7fd9d80a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:16.494 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.492+0000 7fd9f15ff640 1 --2- 172.21.15.99:0/743216750 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd9d80a2800 0x7fd9d80a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:16.494 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.492+0000 7fd9e27fc640 1 -- 172.21.15.99:0/743216750 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7fd9e40cf160 con 0x7fd9ec1022e0 2024-03-19T01:40:16.494 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.493+0000 7fd9f15ff640 1 --2- 172.21.15.99:0/743216750 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd9d80a2800 0x7fd9d80a4cc0 secure :-1 s=READY pgs=210 cs=0 l=1 rev1=1 crypto rx=0x7fd9e802c8e0 tx=0x7fd9e8005b50 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:16.496 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.496+0000 7fd9e27fc640 1 -- 172.21.15.99:0/743216750 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7fd9e409c250 con 0x7fd9ec1022e0 2024-03-19T01:40:16.670 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.669+0000 7fd9f3863640 1 -- 172.21.15.99:0/743216750 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) -- 0x7fd9ec02d3f0 con 0x7fd9ec1022e0 2024-03-19T01:40:16.670 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.670+0000 7fd9e27fc640 1 -- 172.21.15.99:0/743216750 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v91) ==== 104+0+11 (secure 0 0 0) 0x7fd9e409bb50 con 0x7fd9ec1022e0 2024-03-19T01:40:16.670 INFO:teuthology.orchestra.run.smithi099.stdout:pg_num: 32 2024-03-19T01:40:16.673 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.672+0000 7fd9f3863640 1 -- 172.21.15.99:0/743216750 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd9d80a2800 msgr2=0x7fd9d80a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:16.673 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.672+0000 7fd9f3863640 1 --2- 172.21.15.99:0/743216750 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd9d80a2800 0x7fd9d80a4cc0 secure :-1 s=READY pgs=210 cs=0 l=1 rev1=1 crypto rx=0x7fd9e802c8e0 tx=0x7fd9e8005b50 comp rx=0 tx=0).stop 2024-03-19T01:40:16.673 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.672+0000 7fd9f3863640 1 -- 172.21.15.99:0/743216750 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd9ec1022e0 msgr2=0x7fd9ec19ef00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:16.673 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.672+0000 7fd9f3863640 1 --2- 172.21.15.99:0/743216750 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd9ec1022e0 0x7fd9ec19ef00 secure :-1 s=READY pgs=277 cs=0 l=1 rev1=1 crypto rx=0x7fd9e4009fd0 tx=0x7fd9e400ef10 comp rx=0 tx=0).stop 2024-03-19T01:40:16.673 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.672+0000 7fd9f3863640 1 -- 172.21.15.99:0/743216750 shutdown_connections 2024-03-19T01:40:16.673 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.672+0000 7fd9f3863640 1 --2- 172.21.15.99:0/743216750 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd9ec1022e0 0x7fd9ec19ef00 unknown :-1 s=CLOSED pgs=277 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:16.673 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.672+0000 7fd9f3863640 1 --2- 172.21.15.99:0/743216750 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd9ec101920 0x7fd9ec19c8c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:16.673 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.672+0000 7fd9f3863640 1 --2- 172.21.15.99:0/743216750 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd9d80a2800 0x7fd9d80a4cc0 unknown :-1 s=CLOSED pgs=210 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:16.673 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.672+0000 7fd9f3863640 1 --2- 172.21.15.99:0/743216750 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd9ec100730 0x7fd9ec06a9a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:16.673 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.672+0000 7fd9f3863640 1 -- 172.21.15.99:0/743216750 >> 172.21.15.99:0/743216750 conn(0x7fd9ec0fbcc0 msgr2=0x7fd9ec104c40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:16.673 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.672+0000 7fd9f3863640 1 -- 172.21.15.99:0/743216750 shutdown_connections 2024-03-19T01:40:16.673 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.672+0000 7fd9f3863640 1 -- 172.21.15.99:0/743216750 wait complete. 2024-03-19T01:40:16.684 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-19T01:40:16.792 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.790+0000 7f575bda9640 1 -- 172.21.15.99:0/3105993432 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5754101910 msgr2=0x7f5754101d90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:16.792 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.790+0000 7f575bda9640 1 --2- 172.21.15.99:0/3105993432 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5754101910 0x7f5754101d90 secure :-1 s=READY pgs=278 cs=0 l=1 rev1=1 crypto rx=0x7f5750009900 tx=0x7f575002bc30 comp rx=0 tx=0).stop 2024-03-19T01:40:16.793 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.790+0000 7f575bda9640 1 -- 172.21.15.99:0/3105993432 shutdown_connections 2024-03-19T01:40:16.793 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.790+0000 7f575bda9640 1 --2- 172.21.15.99:0/3105993432 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f57541022d0 0x7f5754108b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:16.793 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.790+0000 7f575bda9640 1 --2- 172.21.15.99:0/3105993432 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5754101910 0x7f5754101d90 unknown :-1 s=CLOSED pgs=278 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:16.793 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.790+0000 7f575bda9640 1 --2- 172.21.15.99:0/3105993432 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5754100720 0x7f5754100b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:16.793 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.790+0000 7f575bda9640 1 -- 172.21.15.99:0/3105993432 >> 172.21.15.99:0/3105993432 conn(0x7f57540fbcb0 msgr2=0x7f57540fe0d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:16.793 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.790+0000 7f575bda9640 1 -- 172.21.15.99:0/3105993432 shutdown_connections 2024-03-19T01:40:16.793 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.790+0000 7f575bda9640 1 -- 172.21.15.99:0/3105993432 wait complete. 2024-03-19T01:40:16.793 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.790+0000 7f575bda9640 1 Processor -- start 2024-03-19T01:40:16.793 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.790+0000 7f575bda9640 1 -- start start 2024-03-19T01:40:16.793 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.791+0000 7f575bda9640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5754100720 0x7f5754196aa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:16.793 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.791+0000 7f575bda9640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5754101910 0x7f5754196fe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:16.793 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.791+0000 7f575bda9640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f57541022d0 0x7f575419e050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:16.793 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.791+0000 7f575bda9640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f575410c380 con 0x7f57541022d0 2024-03-19T01:40:16.793 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.791+0000 7f575bda9640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f575410c200 con 0x7f5754101910 2024-03-19T01:40:16.794 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.791+0000 7f575bda9640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f575410c500 con 0x7f5754100720 2024-03-19T01:40:16.794 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.791+0000 7f575a346640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f57541022d0 0x7f575419e050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:16.794 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.791+0000 7f575a346640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f57541022d0 0x7f575419e050 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:52792/0 (socket says 172.21.15.99:52792) 2024-03-19T01:40:16.794 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.791+0000 7f575a346640 1 -- 172.21.15.99:0/1310734870 learned_addr learned my addr 172.21.15.99:0/1310734870 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:16.794 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.791+0000 7f5759344640 1 --2- 172.21.15.99:0/1310734870 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5754101910 0x7f5754196fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:16.794 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.791+0000 7f5759b45640 1 --2- 172.21.15.99:0/1310734870 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5754100720 0x7f5754196aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:16.794 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.791+0000 7f575a346640 1 -- 172.21.15.99:0/1310734870 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5754100720 msgr2=0x7f5754196aa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:16.794 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.791+0000 7f575a346640 1 --2- 172.21.15.99:0/1310734870 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5754100720 0x7f5754196aa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:16.794 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.791+0000 7f575a346640 1 -- 172.21.15.99:0/1310734870 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5754101910 msgr2=0x7f5754196fe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:16.794 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.791+0000 7f575a346640 1 --2- 172.21.15.99:0/1310734870 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5754101910 0x7f5754196fe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:16.794 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.791+0000 7f575a346640 1 -- 172.21.15.99:0/1310734870 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f575419e6c0 con 0x7f57541022d0 2024-03-19T01:40:16.794 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.791+0000 7f575a346640 1 --2- 172.21.15.99:0/1310734870 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f57541022d0 0x7f575419e050 secure :-1 s=READY pgs=279 cs=0 l=1 rev1=1 crypto rx=0x7f574c00efd0 tx=0x7f574c00c6a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:16.795 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.791+0000 7f574affd640 1 -- 172.21.15.99:0/1310734870 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f574c003b70 con 0x7f57541022d0 2024-03-19T01:40:16.795 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.791+0000 7f574affd640 1 -- 172.21.15.99:0/1310734870 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f574c003d10 con 0x7f57541022d0 2024-03-19T01:40:16.795 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.791+0000 7f575bda9640 1 -- 172.21.15.99:0/1310734870 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f575419e9b0 con 0x7f57541022d0 2024-03-19T01:40:16.795 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.791+0000 7f574affd640 1 -- 172.21.15.99:0/1310734870 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f574c0109f0 con 0x7f57541022d0 2024-03-19T01:40:16.795 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.791+0000 7f575bda9640 1 -- 172.21.15.99:0/1310734870 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f575419ee80 con 0x7f57541022d0 2024-03-19T01:40:16.795 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.792+0000 7f575bda9640 1 -- 172.21.15.99:0/1310734870 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f5754107b60 con 0x7f57541022d0 2024-03-19T01:40:16.795 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.793+0000 7f574affd640 1 -- 172.21.15.99:0/1310734870 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 18) ==== 137290+0+0 (secure 0 0 0) 0x7f574c004020 con 0x7f57541022d0 2024-03-19T01:40:16.796 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.794+0000 7f574affd640 1 --2- 172.21.15.99:0/1310734870 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f57400a2800 0x7f57400a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:16.796 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.794+0000 7f5759b45640 1 --2- 172.21.15.99:0/1310734870 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f57400a2800 0x7f57400a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:16.796 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.794+0000 7f574affd640 1 -- 172.21.15.99:0/1310734870 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f574c0cef10 con 0x7f57541022d0 2024-03-19T01:40:16.796 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.795+0000 7f5759b45640 1 --2- 172.21.15.99:0/1310734870 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f57400a2800 0x7f57400a4cc0 secure :-1 s=READY pgs=211 cs=0 l=1 rev1=1 crypto rx=0x7f57440095f0 tx=0x7f5744005f90 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:16.799 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.799+0000 7f574affd640 1 -- 172.21.15.99:0/1310734870 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f574c09bf70 con 0x7f57541022d0 2024-03-19T01:40:16.978 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.978+0000 7f575bda9640 1 -- 172.21.15.99:0/1310734870 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) -- 0x7f575402d1f0 con 0x7f57541022d0 2024-03-19T01:40:16.978 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.978+0000 7f574affd640 1 -- 172.21.15.99:0/1310734870 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v91) ==== 100+0+11 (secure 0 0 0) 0x7f574c09b870 con 0x7f57541022d0 2024-03-19T01:40:16.979 INFO:teuthology.orchestra.run.smithi099.stdout:pg_num: 64 2024-03-19T01:40:16.981 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.980+0000 7f575bda9640 1 -- 172.21.15.99:0/1310734870 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f57400a2800 msgr2=0x7f57400a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:16.981 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.980+0000 7f575bda9640 1 --2- 172.21.15.99:0/1310734870 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f57400a2800 0x7f57400a4cc0 secure :-1 s=READY pgs=211 cs=0 l=1 rev1=1 crypto rx=0x7f57440095f0 tx=0x7f5744005f90 comp rx=0 tx=0).stop 2024-03-19T01:40:16.981 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.980+0000 7f575bda9640 1 -- 172.21.15.99:0/1310734870 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f57541022d0 msgr2=0x7f575419e050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:16.981 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.980+0000 7f575bda9640 1 --2- 172.21.15.99:0/1310734870 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f57541022d0 0x7f575419e050 secure :-1 s=READY pgs=279 cs=0 l=1 rev1=1 crypto rx=0x7f574c00efd0 tx=0x7f574c00c6a0 comp rx=0 tx=0).stop 2024-03-19T01:40:16.981 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.980+0000 7f575bda9640 1 -- 172.21.15.99:0/1310734870 shutdown_connections 2024-03-19T01:40:16.981 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.980+0000 7f575bda9640 1 --2- 172.21.15.99:0/1310734870 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f57400a2800 0x7f57400a4cc0 unknown :-1 s=CLOSED pgs=211 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:16.981 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.980+0000 7f575bda9640 1 --2- 172.21.15.99:0/1310734870 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5754101910 0x7f5754196fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:16.981 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.980+0000 7f575bda9640 1 --2- 172.21.15.99:0/1310734870 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f57541022d0 0x7f575419e050 unknown :-1 s=CLOSED pgs=279 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:16.981 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.980+0000 7f575bda9640 1 --2- 172.21.15.99:0/1310734870 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5754100720 0x7f5754196aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:16.982 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.980+0000 7f575bda9640 1 -- 172.21.15.99:0/1310734870 >> 172.21.15.99:0/1310734870 conn(0x7f57540fbcb0 msgr2=0x7f5754104d30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:16.982 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.980+0000 7f575bda9640 1 -- 172.21.15.99:0/1310734870 shutdown_connections 2024-03-19T01:40:16.982 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:16.981+0000 7f575bda9640 1 -- 172.21.15.99:0/1310734870 wait complete. 2024-03-19T01:40:16.991 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:16 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/3336642981' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-19T01:40:16.991 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:16 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/444389581' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-03-19T01:40:16.991 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:16 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/743216750' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-03-19T01:40:16.992 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-19T01:40:17.095 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.092+0000 7fd37f59e640 1 -- 172.21.15.99:0/1447752365 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd380102850 msgr2=0x7fd380102cd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:17.095 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.092+0000 7fd37f59e640 1 --2- 172.21.15.99:0/1447752365 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd380102850 0x7fd380102cd0 secure :-1 s=READY pgs=280 cs=0 l=1 rev1=1 crypto rx=0x7fd374009900 tx=0x7fd37402bc30 comp rx=0 tx=0).stop 2024-03-19T01:40:17.095 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.092+0000 7fd37f59e640 1 -- 172.21.15.99:0/1447752365 shutdown_connections 2024-03-19T01:40:17.095 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.092+0000 7fd37f59e640 1 --2- 172.21.15.99:0/1447752365 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd380103210 0x7fd380109aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:17.095 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.092+0000 7fd37f59e640 1 --2- 172.21.15.99:0/1447752365 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd380102850 0x7fd380102cd0 unknown :-1 s=CLOSED pgs=280 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:17.095 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.092+0000 7fd37f59e640 1 --2- 172.21.15.99:0/1447752365 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd380101660 0x7fd380101a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:17.096 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.092+0000 7fd37f59e640 1 -- 172.21.15.99:0/1447752365 >> 172.21.15.99:0/1447752365 conn(0x7fd3800fcbb0 msgr2=0x7fd3800ff010 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:17.096 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.092+0000 7fd37f59e640 1 -- 172.21.15.99:0/1447752365 shutdown_connections 2024-03-19T01:40:17.096 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.092+0000 7fd37f59e640 1 -- 172.21.15.99:0/1447752365 wait complete. 2024-03-19T01:40:17.096 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.093+0000 7fd37f59e640 1 Processor -- start 2024-03-19T01:40:17.096 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.093+0000 7fd37f59e640 1 -- start start 2024-03-19T01:40:17.096 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.093+0000 7fd37f59e640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd380101660 0x7fd380197990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:17.096 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.093+0000 7fd37f59e640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd380102850 0x7fd380197ed0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:17.096 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.093+0000 7fd37f59e640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd380103210 0x7fd38019ef40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:17.096 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.093+0000 7fd37f59e640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fd38010d2f0 con 0x7fd380102850 2024-03-19T01:40:17.096 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.093+0000 7fd37f59e640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fd38010d170 con 0x7fd380103210 2024-03-19T01:40:17.097 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.093+0000 7fd37f59e640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fd38010d470 con 0x7fd380101660 2024-03-19T01:40:17.097 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.093+0000 7fd37dd9b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd380102850 0x7fd380197ed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:17.097 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.093+0000 7fd37dd9b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd380102850 0x7fd380197ed0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:52814/0 (socket says 172.21.15.99:52814) 2024-03-19T01:40:17.097 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.093+0000 7fd37dd9b640 1 -- 172.21.15.99:0/3609521473 learned_addr learned my addr 172.21.15.99:0/3609521473 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:17.097 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.093+0000 7fd37e59c640 1 --2- 172.21.15.99:0/3609521473 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd380101660 0x7fd380197990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:17.097 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.093+0000 7fd37ed9d640 1 --2- 172.21.15.99:0/3609521473 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd380103210 0x7fd38019ef40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:17.097 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.093+0000 7fd37dd9b640 1 -- 172.21.15.99:0/3609521473 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd380101660 msgr2=0x7fd380197990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:17.097 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.093+0000 7fd37dd9b640 1 --2- 172.21.15.99:0/3609521473 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd380101660 0x7fd380197990 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:17.097 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.093+0000 7fd37dd9b640 1 -- 172.21.15.99:0/3609521473 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd380103210 msgr2=0x7fd38019ef40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:17.097 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.093+0000 7fd37dd9b640 1 --2- 172.21.15.99:0/3609521473 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd380103210 0x7fd38019ef40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:17.097 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.093+0000 7fd37dd9b640 1 -- 172.21.15.99:0/3609521473 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd38019f5b0 con 0x7fd380102850 2024-03-19T01:40:17.097 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.093+0000 7fd37dd9b640 1 --2- 172.21.15.99:0/3609521473 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd380102850 0x7fd380197ed0 secure :-1 s=READY pgs=281 cs=0 l=1 rev1=1 crypto rx=0x7fd374009230 tx=0x7fd37402c650 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:17.097 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.094+0000 7fd36f7fe640 1 -- 172.21.15.99:0/3609521473 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd374041070 con 0x7fd380102850 2024-03-19T01:40:17.097 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.094+0000 7fd36f7fe640 1 -- 172.21.15.99:0/3609521473 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd374004450 con 0x7fd380102850 2024-03-19T01:40:17.097 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.094+0000 7fd37f59e640 1 -- 172.21.15.99:0/3609521473 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd38019f840 con 0x7fd380102850 2024-03-19T01:40:17.098 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.094+0000 7fd36f7fe640 1 -- 172.21.15.99:0/3609521473 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd374003e20 con 0x7fd380102850 2024-03-19T01:40:17.098 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.094+0000 7fd37f59e640 1 -- 172.21.15.99:0/3609521473 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fd38019fd30 con 0x7fd380102850 2024-03-19T01:40:17.098 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.095+0000 7fd37f59e640 1 -- 172.21.15.99:0/3609521473 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fd380108980 con 0x7fd380102850 2024-03-19T01:40:17.099 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.096+0000 7fd36f7fe640 1 -- 172.21.15.99:0/3609521473 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 18) ==== 137290+0+0 (secure 0 0 0) 0x7fd374044070 con 0x7fd380102850 2024-03-19T01:40:17.099 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.096+0000 7fd36f7fe640 1 --2- 172.21.15.99:0/3609521473 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd3680a2800 0x7fd3680a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:17.099 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.096+0000 7fd37e59c640 1 --2- 172.21.15.99:0/3609521473 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd3680a2800 0x7fd3680a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:17.099 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.096+0000 7fd36f7fe640 1 -- 172.21.15.99:0/3609521473 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7fd3740f0630 con 0x7fd380102850 2024-03-19T01:40:17.099 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.097+0000 7fd37e59c640 1 --2- 172.21.15.99:0/3609521473 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd3680a2800 0x7fd3680a4cc0 secure :-1 s=READY pgs=212 cs=0 l=1 rev1=1 crypto rx=0x7fd3801026b0 tx=0x7fd378009500 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:17.100 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.101+0000 7fd36f7fe640 1 -- 172.21.15.99:0/3609521473 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7fd3740bd040 con 0x7fd380102850 2024-03-19T01:40:17.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:16 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/3336642981' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-19T01:40:17.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:16 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/444389581' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-03-19T01:40:17.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:16 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/743216750' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-03-19T01:40:17.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:16 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/3336642981' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-19T01:40:17.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:16 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/444389581' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-03-19T01:40:17.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:16 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/743216750' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-03-19T01:40:17.311 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.311+0000 7fd37f59e640 1 -- 172.21.15.99:0/3609521473 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7fd380062420 con 0x7fd380102850 2024-03-19T01:40:17.312 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.312+0000 7fd36f7fe640 1 -- 172.21.15.99:0/3609521473 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 20 v20) ==== 94+0+11878 (secure 0 0 0) 0x7fd3740c10e0 con 0x7fd380102850 2024-03-19T01:40:17.312 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:40:17.312 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":20,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14562,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6835/1631815621","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6834","nonce":1631815621},{"type":"v1","addr":"172.21.15.99:6835","nonce":1631815621}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":4},{"gid":14646,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6839/2236145144","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6838","nonce":2236145144},{"type":"v1","addr":"172.21.15.99:6839","nonce":2236145144}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11},{"gid":14658,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6841/1660990159","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6840","nonce":1660990159},{"type":"v1","addr":"172.21.15.99:6841","nonce":1660990159}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":12},{"gid":24383,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.112:6833/2969810977","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6832","nonce":2969810977},{"type":"v1","addr":"172.21.15.112:6833","nonce":2969810977}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3},{"gid":24404,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6837/2038634879","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6836","nonce":2038634879},{"type":"v1","addr":"172.21.15.99:6837","nonce":2038634879}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":24415,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.115:6833/2219320481","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6832","nonce":2219320481},{"type":"v1","addr":"172.21.15.115:6833","nonce":2219320481}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":20,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-03-19T01:38:56.124529+0000","modified":"2024-03-19T01:39:00.323326+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24487,"mds_1":24479,"mds_2":24472},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24416":{"gid":24416,"name":"e","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.112:6835/3690587528","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6834","nonce":3690587528},{"type":"v1","addr":"172.21.15.112:6835","nonce":3690587528}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24440":{"gid":24440,"name":"h","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.112:6837/2339823159","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6836","nonce":2339823159},{"type":"v1","addr":"172.21.15.112:6837","nonce":2339823159}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24460":{"gid":24460,"name":"f","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.115:6835/591078413","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6834","nonce":591078413},{"type":"v1","addr":"172.21.15.115:6835","nonce":591078413}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24472":{"gid":24472,"name":"c","rank":2,"incarnation":19,"state":"up:active","state_seq":7,"addr":"172.21.15.115:6837/1368988191","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6836","nonce":1368988191},{"type":"v1","addr":"172.21.15.115:6837","nonce":1368988191}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24479":{"gid":24479,"name":"b","rank":1,"incarnation":17,"state":"up:active","state_seq":4,"addr":"172.21.15.112:6839/986249680","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6838","nonce":986249680},{"type":"v1","addr":"172.21.15.112:6839","nonce":986249680}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24487":{"gid":24487,"name":"i","rank":0,"incarnation":14,"state":"up:active","state_seq":5,"addr":"172.21.15.115:6839/2743761968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6838","nonce":2743761968},{"type":"v1","addr":"172.21.15.115:6839","nonce":2743761968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24487,"qdb_cluster":[24487,24479,24472]},"id":1}]} 2024-03-19T01:40:17.313 INFO:teuthology.orchestra.run.smithi099.stderr:dumped fsmap epoch 20 2024-03-19T01:40:17.314 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.314+0000 7fd37f59e640 1 -- 172.21.15.99:0/3609521473 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd3680a2800 msgr2=0x7fd3680a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:17.314 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.314+0000 7fd37f59e640 1 --2- 172.21.15.99:0/3609521473 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd3680a2800 0x7fd3680a4cc0 secure :-1 s=READY pgs=212 cs=0 l=1 rev1=1 crypto rx=0x7fd3801026b0 tx=0x7fd378009500 comp rx=0 tx=0).stop 2024-03-19T01:40:17.314 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.314+0000 7fd37f59e640 1 -- 172.21.15.99:0/3609521473 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd380102850 msgr2=0x7fd380197ed0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:17.314 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.314+0000 7fd37f59e640 1 --2- 172.21.15.99:0/3609521473 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd380102850 0x7fd380197ed0 secure :-1 s=READY pgs=281 cs=0 l=1 rev1=1 crypto rx=0x7fd374009230 tx=0x7fd37402c650 comp rx=0 tx=0).stop 2024-03-19T01:40:17.314 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.314+0000 7fd37f59e640 1 -- 172.21.15.99:0/3609521473 shutdown_connections 2024-03-19T01:40:17.315 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.314+0000 7fd37f59e640 1 --2- 172.21.15.99:0/3609521473 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd3680a2800 0x7fd3680a4cc0 unknown :-1 s=CLOSED pgs=212 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:17.315 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.314+0000 7fd37f59e640 1 --2- 172.21.15.99:0/3609521473 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd380103210 0x7fd38019ef40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:17.315 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.314+0000 7fd37f59e640 1 --2- 172.21.15.99:0/3609521473 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd380102850 0x7fd380197ed0 unknown :-1 s=CLOSED pgs=281 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:17.315 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.314+0000 7fd37f59e640 1 --2- 172.21.15.99:0/3609521473 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd380101660 0x7fd380197990 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:17.315 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.314+0000 7fd37f59e640 1 -- 172.21.15.99:0/3609521473 >> 172.21.15.99:0/3609521473 conn(0x7fd3800fcbb0 msgr2=0x7fd380105b50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:17.315 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.314+0000 7fd37f59e640 1 -- 172.21.15.99:0/3609521473 shutdown_connections 2024-03-19T01:40:17.315 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.314+0000 7fd37f59e640 1 -- 172.21.15.99:0/3609521473 wait complete. 2024-03-19T01:40:17.326 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-19T01:40:17.429 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.426+0000 7fa556559640 1 -- 172.21.15.99:0/465117324 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa550101930 msgr2=0x7fa550101db0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:17.429 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.426+0000 7fa556559640 1 --2- 172.21.15.99:0/465117324 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa550101930 0x7fa550101db0 secure :-1 s=READY pgs=65 cs=0 l=1 rev1=1 crypto rx=0x7fa544009900 tx=0x7fa54402bc30 comp rx=0 tx=0).stop 2024-03-19T01:40:17.429 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.426+0000 7fa556559640 1 -- 172.21.15.99:0/465117324 shutdown_connections 2024-03-19T01:40:17.429 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.426+0000 7fa556559640 1 --2- 172.21.15.99:0/465117324 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa550101930 0x7fa550101db0 unknown :-1 s=CLOSED pgs=65 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:17.429 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.426+0000 7fa556559640 1 --2- 172.21.15.99:0/465117324 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa5501022f0 0x7fa550108b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:17.429 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.426+0000 7fa556559640 1 --2- 172.21.15.99:0/465117324 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa550100740 0x7fa550100b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:17.429 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.426+0000 7fa556559640 1 -- 172.21.15.99:0/465117324 >> 172.21.15.99:0/465117324 conn(0x7fa5500fbcb0 msgr2=0x7fa5500fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:17.429 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.427+0000 7fa556559640 1 -- 172.21.15.99:0/465117324 shutdown_connections 2024-03-19T01:40:17.430 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.427+0000 7fa556559640 1 -- 172.21.15.99:0/465117324 wait complete. 2024-03-19T01:40:17.430 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.427+0000 7fa556559640 1 Processor -- start 2024-03-19T01:40:17.430 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.427+0000 7fa556559640 1 -- start start 2024-03-19T01:40:17.430 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.427+0000 7fa556559640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa550100740 0x7fa550196a00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:17.430 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.427+0000 7fa556559640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa550101930 0x7fa550196f40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:17.430 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.427+0000 7fa556559640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa5501022f0 0x7fa55019dfb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:17.430 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.427+0000 7fa556559640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fa55010c3c0 con 0x7fa5501022f0 2024-03-19T01:40:17.430 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.427+0000 7fa556559640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fa55010c240 con 0x7fa550100740 2024-03-19T01:40:17.430 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.427+0000 7fa556559640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fa55010c540 con 0x7fa550101930 2024-03-19T01:40:17.430 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.428+0000 7fa554af6640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa5501022f0 0x7fa55019dfb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:17.430 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.428+0000 7fa554af6640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa5501022f0 0x7fa55019dfb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:52836/0 (socket says 172.21.15.99:52836) 2024-03-19T01:40:17.430 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.428+0000 7fa554af6640 1 -- 172.21.15.99:0/2036460928 learned_addr learned my addr 172.21.15.99:0/2036460928 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:17.430 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.428+0000 7fa54ffff640 1 --2- 172.21.15.99:0/2036460928 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa550100740 0x7fa550196a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:17.430 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.428+0000 7fa54f7fe640 1 --2- 172.21.15.99:0/2036460928 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa550101930 0x7fa550196f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:17.431 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.428+0000 7fa554af6640 1 -- 172.21.15.99:0/2036460928 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa550101930 msgr2=0x7fa550196f40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:17.431 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.428+0000 7fa554af6640 1 --2- 172.21.15.99:0/2036460928 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa550101930 0x7fa550196f40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:17.431 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.428+0000 7fa554af6640 1 -- 172.21.15.99:0/2036460928 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa550100740 msgr2=0x7fa550196a00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:17.431 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.428+0000 7fa554af6640 1 --2- 172.21.15.99:0/2036460928 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa550100740 0x7fa550196a00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:17.431 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.428+0000 7fa554af6640 1 -- 172.21.15.99:0/2036460928 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa55019e620 con 0x7fa5501022f0 2024-03-19T01:40:17.431 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.428+0000 7fa554af6640 1 --2- 172.21.15.99:0/2036460928 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa5501022f0 0x7fa55019dfb0 secure :-1 s=READY pgs=282 cs=0 l=1 rev1=1 crypto rx=0x7fa54800eb10 tx=0x7fa54800c2d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:17.431 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.428+0000 7fa54d7fa640 1 -- 172.21.15.99:0/2036460928 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa548004b20 con 0x7fa5501022f0 2024-03-19T01:40:17.431 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.428+0000 7fa556559640 1 -- 172.21.15.99:0/2036460928 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa55019e910 con 0x7fa5501022f0 2024-03-19T01:40:17.431 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.428+0000 7fa54d7fa640 1 -- 172.21.15.99:0/2036460928 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa548004cc0 con 0x7fa5501022f0 2024-03-19T01:40:17.431 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.428+0000 7fa54d7fa640 1 -- 172.21.15.99:0/2036460928 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa548010760 con 0x7fa5501022f0 2024-03-19T01:40:17.431 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.428+0000 7fa556559640 1 -- 172.21.15.99:0/2036460928 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fa55019ee30 con 0x7fa5501022f0 2024-03-19T01:40:17.432 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.429+0000 7fa556559640 1 -- 172.21.15.99:0/2036460928 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fa550107a60 con 0x7fa5501022f0 2024-03-19T01:40:17.432 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.430+0000 7fa54d7fa640 1 -- 172.21.15.99:0/2036460928 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 18) ==== 137290+0+0 (secure 0 0 0) 0x7fa548005020 con 0x7fa5501022f0 2024-03-19T01:40:17.432 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.431+0000 7fa54d7fa640 1 --2- 172.21.15.99:0/2036460928 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa5380a2800 0x7fa5380a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:17.432 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.431+0000 7fa54ffff640 1 --2- 172.21.15.99:0/2036460928 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa5380a2800 0x7fa5380a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:17.433 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.431+0000 7fa54d7fa640 1 -- 172.21.15.99:0/2036460928 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7fa5480cf180 con 0x7fa5501022f0 2024-03-19T01:40:17.433 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.431+0000 7fa54ffff640 1 --2- 172.21.15.99:0/2036460928 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa5380a2800 0x7fa5380a4cc0 secure :-1 s=READY pgs=213 cs=0 l=1 rev1=1 crypto rx=0x7fa5400095f0 tx=0x7fa540005f90 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:17.435 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.435+0000 7fa54d7fa640 1 -- 172.21.15.99:0/2036460928 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7fa54809c1e0 con 0x7fa5501022f0 2024-03-19T01:40:17.613 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.613+0000 7fa556559640 1 -- 172.21.15.99:0/2036460928 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7fa55002d1f0 con 0x7fa5501022f0 2024-03-19T01:40:17.614 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.614+0000 7fa54d7fa640 1 -- 172.21.15.99:0/2036460928 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v91) ==== 74+0+22804 (secure 0 0 0) 0x7fa54809bae0 con 0x7fa5501022f0 2024-03-19T01:40:17.615 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:40:17.615 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":91,"fsid":"438fb69c-e590-11ee-95c9-87774f69a715","created":"2024-03-19T01:31:10.165073+0000","modified":"2024-03-19T01:40:08.821469+0000","last_up_change":"2024-03-19T01:38:09.082688+0000","last_in_change":"2024-03-19T01:37:49.115793+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-19T01:34:31.001309+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-19T01:38:54.018623+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"86","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-19T01:38:55.319036+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"87","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"0c407a0e-a40e-43d7-ad9a-bb8f3cd4a9ba","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6803","nonce":2935260112}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6805","nonce":2935260112}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6808","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6809","nonce":2935260112}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6807","nonce":2935260112}]},"public_addr":"172.21.15.99:6803/2935260112","cluster_addr":"172.21.15.99:6805/2935260112","heartbeat_back_addr":"172.21.15.99:6809/2935260112","heartbeat_front_addr":"172.21.15.99:6807/2935260112","state":["exists","up"]},{"osd":1,"uuid":"a48f9173-4a06-49f5-bca2-2c9047b8f666","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6800","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6801","nonce":1468854289}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6802","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6803","nonce":1468854289}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6806","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6807","nonce":1468854289}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6804","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6805","nonce":1468854289}]},"public_addr":"172.21.15.112:6801/1468854289","cluster_addr":"172.21.15.112:6803/1468854289","heartbeat_back_addr":"172.21.15.112:6807/1468854289","heartbeat_front_addr":"172.21.15.112:6805/1468854289","state":["exists","up"]},{"osd":2,"uuid":"b7715fdd-b6c9-4348-bd16-5eb61b427eef","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6800","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6801","nonce":3766690472}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6802","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6803","nonce":3766690472}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6806","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6807","nonce":3766690472}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6804","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6805","nonce":3766690472}]},"public_addr":"172.21.15.115:6801/3766690472","cluster_addr":"172.21.15.115:6803/3766690472","heartbeat_back_addr":"172.21.15.115:6807/3766690472","heartbeat_front_addr":"172.21.15.115:6805/3766690472","state":["exists","up"]},{"osd":3,"uuid":"6697ed1b-96bf-42de-8440-d17e9785b359","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6810","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6811","nonce":3756732895}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6812","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6813","nonce":3756732895}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6816","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6817","nonce":3756732895}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6814","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6815","nonce":3756732895}]},"public_addr":"172.21.15.99:6811/3756732895","cluster_addr":"172.21.15.99:6813/3756732895","heartbeat_back_addr":"172.21.15.99:6817/3756732895","heartbeat_front_addr":"172.21.15.99:6815/3756732895","state":["exists","up"]},{"osd":4,"uuid":"d958d6b9-67f6-43d0-b704-aa55dc9df83a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":34,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6808","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6809","nonce":3717986391}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6810","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6811","nonce":3717986391}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6814","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6815","nonce":3717986391}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6812","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6813","nonce":3717986391}]},"public_addr":"172.21.15.112:6809/3717986391","cluster_addr":"172.21.15.112:6811/3717986391","heartbeat_back_addr":"172.21.15.112:6815/3717986391","heartbeat_front_addr":"172.21.15.112:6813/3717986391","state":["exists","up"]},{"osd":5,"uuid":"a73eb49a-7925-49ba-bc4f-515e17f8b8ae","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6808","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6809","nonce":1971806408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6810","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6811","nonce":1971806408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6814","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6815","nonce":1971806408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6812","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6813","nonce":1971806408}]},"public_addr":"172.21.15.115:6809/1971806408","cluster_addr":"172.21.15.115:6811/1971806408","heartbeat_back_addr":"172.21.15.115:6815/1971806408","heartbeat_front_addr":"172.21.15.115:6813/1971806408","state":["exists","up"]},{"osd":6,"uuid":"caa718cd-e08b-48de-832a-29546d71e508","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6818","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6819","nonce":1130832772}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6820","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6821","nonce":1130832772}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6824","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6825","nonce":1130832772}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6822","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6823","nonce":1130832772}]},"public_addr":"172.21.15.99:6819/1130832772","cluster_addr":"172.21.15.99:6821/1130832772","heartbeat_back_addr":"172.21.15.99:6825/1130832772","heartbeat_front_addr":"172.21.15.99:6823/1130832772","state":["exists","up"]},{"osd":7,"uuid":"dbee19a1-2315-40d3-88e7-876fcde28903","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":53,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6816","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6817","nonce":3610356576}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6818","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6819","nonce":3610356576}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6822","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6823","nonce":3610356576}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6820","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6821","nonce":3610356576}]},"public_addr":"172.21.15.112:6817/3610356576","cluster_addr":"172.21.15.112:6819/3610356576","heartbeat_back_addr":"172.21.15.112:6823/3610356576","heartbeat_front_addr":"172.21.15.112:6821/3610356576","state":["exists","up"]},{"osd":8,"uuid":"14a866dc-5ce3-42fc-8bcf-9088b68726cb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":60,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6816","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6817","nonce":4255803111}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6818","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6819","nonce":4255803111}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6822","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6823","nonce":4255803111}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6820","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6821","nonce":4255803111}]},"public_addr":"172.21.15.115:6817/4255803111","cluster_addr":"172.21.15.115:6819/4255803111","heartbeat_back_addr":"172.21.15.115:6823/4255803111","heartbeat_front_addr":"172.21.15.115:6821/4255803111","state":["exists","up"]},{"osd":9,"uuid":"1db93652-6035-4906-91ae-d64d5a909502","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":68,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6826","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6827","nonce":1410921523}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6828","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6829","nonce":1410921523}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6832","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6833","nonce":1410921523}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6830","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6831","nonce":1410921523}]},"public_addr":"172.21.15.99:6827/1410921523","cluster_addr":"172.21.15.99:6829/1410921523","heartbeat_back_addr":"172.21.15.99:6833/1410921523","heartbeat_front_addr":"172.21.15.99:6831/1410921523","state":["exists","up"]},{"osd":10,"uuid":"6c50a8a7-3345-4f4a-af0e-5606c7aca453","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":74,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6824","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6825","nonce":891805922}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6826","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6827","nonce":891805922}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6830","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6831","nonce":891805922}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6828","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6829","nonce":891805922}]},"public_addr":"172.21.15.112:6825/891805922","cluster_addr":"172.21.15.112:6827/891805922","heartbeat_back_addr":"172.21.15.112:6831/891805922","heartbeat_front_addr":"172.21.15.112:6829/891805922","state":["exists","up"]},{"osd":11,"uuid":"fd63b7fd-0db5-46d5-82fa-9707bbd0b495","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":82,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6824","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6825","nonce":938436244}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6826","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6827","nonce":938436244}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6830","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6831","nonce":938436244}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6828","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6829","nonce":938436244}]},"public_addr":"172.21.15.115:6825/938436244","cluster_addr":"172.21.15.115:6827/938436244","heartbeat_back_addr":"172.21.15.115:6831/938436244","heartbeat_front_addr":"172.21.15.115:6829/938436244","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:33:39.036822+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:34:03.339566+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:34:27.149273+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:34:51.315561+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:35:15.396031+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:35:38.957168+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:36:03.213914+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:36:27.763699+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:36:51.688670+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:37:17.280165+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:37:42.554633+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:38:07.438039+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.0","mappings":[{"from":8,"to":1}]},{"pgid":"2.8","mappings":[{"from":2,"to":11}]},{"pgid":"2.b","mappings":[{"from":8,"to":11}]},{"pgid":"2.e","mappings":[{"from":2,"to":4}]},{"pgid":"3.0","mappings":[{"from":5,"to":9}]},{"pgid":"3.2","mappings":[{"from":3,"to":9}]},{"pgid":"3.14","mappings":[{"from":1,"to":9}]},{"pgid":"3.17","mappings":[{"from":4,"to":8}]},{"pgid":"3.31","mappings":[{"from":4,"to":8}]},{"pgid":"3.37","mappings":[{"from":5,"to":7}]},{"pgid":"3.3a","mappings":[{"from":4,"to":6}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.99:6801/1328245934":"2024-03-20T01:32:06.661002+0000","172.21.15.99:0/1618965837":"2024-03-20T01:32:06.661002+0000","172.21.15.99:0/2491429851":"2024-03-20T01:32:06.661002+0000","172.21.15.99:6800/1328245934":"2024-03-20T01:32:06.661002+0000","172.21.15.99:0/2112231766":"2024-03-20T01:31:40.161751+0000","172.21.15.99:0/3341836545":"2024-03-20T01:31:40.161751+0000","172.21.15.99:6800/4223048293":"2024-03-20T01:31:40.161751+0000","172.21.15.99:6801/4223048293":"2024-03-20T01:31:40.161751+0000","172.21.15.99:0/2377621863":"2024-03-20T01:31:40.161751+0000","172.21.15.99:0/1904400935":"2024-03-20T01:32:06.661002+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-19T01:40:17.617 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.616+0000 7fa556559640 1 -- 172.21.15.99:0/2036460928 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa5380a2800 msgr2=0x7fa5380a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:17.617 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.616+0000 7fa556559640 1 --2- 172.21.15.99:0/2036460928 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa5380a2800 0x7fa5380a4cc0 secure :-1 s=READY pgs=213 cs=0 l=1 rev1=1 crypto rx=0x7fa5400095f0 tx=0x7fa540005f90 comp rx=0 tx=0).stop 2024-03-19T01:40:17.617 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.616+0000 7fa556559640 1 -- 172.21.15.99:0/2036460928 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa5501022f0 msgr2=0x7fa55019dfb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:17.617 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.616+0000 7fa556559640 1 --2- 172.21.15.99:0/2036460928 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa5501022f0 0x7fa55019dfb0 secure :-1 s=READY pgs=282 cs=0 l=1 rev1=1 crypto rx=0x7fa54800eb10 tx=0x7fa54800c2d0 comp rx=0 tx=0).stop 2024-03-19T01:40:17.617 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.617+0000 7fa556559640 1 -- 172.21.15.99:0/2036460928 shutdown_connections 2024-03-19T01:40:17.617 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.617+0000 7fa556559640 1 --2- 172.21.15.99:0/2036460928 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa5501022f0 0x7fa55019dfb0 unknown :-1 s=CLOSED pgs=282 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:17.617 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.617+0000 7fa556559640 1 --2- 172.21.15.99:0/2036460928 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa550101930 0x7fa550196f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:17.617 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.617+0000 7fa556559640 1 --2- 172.21.15.99:0/2036460928 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa5380a2800 0x7fa5380a4cc0 unknown :-1 s=CLOSED pgs=213 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:17.617 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.617+0000 7fa556559640 1 --2- 172.21.15.99:0/2036460928 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa550100740 0x7fa550196a00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:17.617 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.617+0000 7fa556559640 1 -- 172.21.15.99:0/2036460928 >> 172.21.15.99:0/2036460928 conn(0x7fa5500fbcb0 msgr2=0x7fa550104c30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:17.618 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.617+0000 7fa556559640 1 -- 172.21.15.99:0/2036460928 shutdown_connections 2024-03-19T01:40:17.618 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.617+0000 7fa556559640 1 -- 172.21.15.99:0/2036460928 wait complete. 2024-03-19T01:40:17.628 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-19T01:34:31.001309+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 12, 'score_stable': 12, 'optimal_score': 0.25, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-19T01:38:54.018623+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '86', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.8799999952316284, 'score_stable': 1.8799999952316284, 'optimal_score': 1, 'raw_score_acting': 1.8799999952316284, 'raw_score_stable': 1.8799999952316284, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-03-19T01:38:55.319036+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '87', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-19T01:40:17.629 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-19T01:40:17.734 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.733+0000 7f236d8bf640 1 -- 172.21.15.99:0/2379166300 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2368101910 msgr2=0x7f2368101d90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:17.734 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.733+0000 7f236d8bf640 1 --2- 172.21.15.99:0/2379166300 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2368101910 0x7f2368101d90 secure :-1 s=READY pgs=283 cs=0 l=1 rev1=1 crypto rx=0x7f2350009900 tx=0x7f235002bc30 comp rx=0 tx=0).stop 2024-03-19T01:40:17.734 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.733+0000 7f236d8bf640 1 -- 172.21.15.99:0/2379166300 shutdown_connections 2024-03-19T01:40:17.734 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.733+0000 7f236d8bf640 1 --2- 172.21.15.99:0/2379166300 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f23681022d0 0x7f2368108b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:17.734 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.733+0000 7f236d8bf640 1 --2- 172.21.15.99:0/2379166300 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2368101910 0x7f2368101d90 unknown :-1 s=CLOSED pgs=283 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:17.734 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.733+0000 7f236d8bf640 1 --2- 172.21.15.99:0/2379166300 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f2368100720 0x7f2368100b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:17.734 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.733+0000 7f236d8bf640 1 -- 172.21.15.99:0/2379166300 >> 172.21.15.99:0/2379166300 conn(0x7f23680fbcd0 msgr2=0x7f23680fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:17.734 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.733+0000 7f236d8bf640 1 -- 172.21.15.99:0/2379166300 shutdown_connections 2024-03-19T01:40:17.734 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.733+0000 7f236d8bf640 1 -- 172.21.15.99:0/2379166300 wait complete. 2024-03-19T01:40:17.734 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.734+0000 7f236d8bf640 1 Processor -- start 2024-03-19T01:40:17.734 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.734+0000 7f236d8bf640 1 -- start start 2024-03-19T01:40:17.736 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.734+0000 7f236d8bf640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f2368100720 0x7f236806a9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:17.736 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.734+0000 7f236d8bf640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f2368101910 0x7f236819c8d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:17.736 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.734+0000 7f236d8bf640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f23681022d0 0x7f236819eef0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:17.737 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.734+0000 7f236d8bf640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f236810c380 con 0x7f23681022d0 2024-03-19T01:40:17.737 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.734+0000 7f236d8bf640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f236810c200 con 0x7f2368100720 2024-03-19T01:40:17.737 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.734+0000 7f236d8bf640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f236810c500 con 0x7f2368101910 2024-03-19T01:40:17.737 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.734+0000 7f23677fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f23681022d0 0x7f236819eef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:17.737 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.734+0000 7f2366ffd640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f2368100720 0x7f236806a9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:17.737 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.734+0000 7f23677fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f23681022d0 0x7f236819eef0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:52854/0 (socket says 172.21.15.99:52854) 2024-03-19T01:40:17.737 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.734+0000 7f23677fe640 1 -- 172.21.15.99:0/859528443 learned_addr learned my addr 172.21.15.99:0/859528443 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:17.737 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.734+0000 7f235e7fc640 1 --2- 172.21.15.99:0/859528443 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f2368101910 0x7f236819c8d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:17.737 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.734+0000 7f23677fe640 1 -- 172.21.15.99:0/859528443 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f2368101910 msgr2=0x7f236819c8d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:17.737 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.734+0000 7f23677fe640 1 --2- 172.21.15.99:0/859528443 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f2368101910 0x7f236819c8d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:17.737 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.734+0000 7f23677fe640 1 -- 172.21.15.99:0/859528443 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f2368100720 msgr2=0x7f236806a9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:17.737 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.734+0000 7f23677fe640 1 --2- 172.21.15.99:0/859528443 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f2368100720 0x7f236806a9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:17.737 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.734+0000 7f23677fe640 1 -- 172.21.15.99:0/859528443 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f236819f800 con 0x7f23681022d0 2024-03-19T01:40:17.737 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.735+0000 7f23677fe640 1 --2- 172.21.15.99:0/859528443 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f23681022d0 0x7f236819eef0 secure :-1 s=READY pgs=284 cs=0 l=1 rev1=1 crypto rx=0x7f236000efd0 tx=0x7f236000c6a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:17.737 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.735+0000 7f236c8bd640 1 -- 172.21.15.99:0/859528443 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2360003b70 con 0x7f23681022d0 2024-03-19T01:40:17.738 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.735+0000 7f236d8bf640 1 -- 172.21.15.99:0/859528443 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f236819fa90 con 0x7f23681022d0 2024-03-19T01:40:17.738 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.735+0000 7f236c8bd640 1 -- 172.21.15.99:0/859528443 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f2360003d10 con 0x7f23681022d0 2024-03-19T01:40:17.738 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.735+0000 7f236c8bd640 1 -- 172.21.15.99:0/859528443 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2360010990 con 0x7f23681022d0 2024-03-19T01:40:17.738 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.735+0000 7f236d8bf640 1 -- 172.21.15.99:0/859528443 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f236819ffb0 con 0x7f23681022d0 2024-03-19T01:40:17.738 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.736+0000 7f236d8bf640 1 -- 172.21.15.99:0/859528443 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f2368107ac0 con 0x7f23681022d0 2024-03-19T01:40:17.739 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.737+0000 7f236c8bd640 1 -- 172.21.15.99:0/859528443 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 18) ==== 137290+0+0 (secure 0 0 0) 0x7f2360004000 con 0x7f23681022d0 2024-03-19T01:40:17.739 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.737+0000 7f236c8bd640 1 --2- 172.21.15.99:0/859528443 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f23540a2800 0x7f23540a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:17.739 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.738+0000 7f2366ffd640 1 --2- 172.21.15.99:0/859528443 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f23540a2800 0x7f23540a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:17.739 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.738+0000 7f236c8bd640 1 -- 172.21.15.99:0/859528443 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f23600cf290 con 0x7f23681022d0 2024-03-19T01:40:17.739 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.738+0000 7f2366ffd640 1 --2- 172.21.15.99:0/859528443 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f23540a2800 0x7f23540a4cc0 secure :-1 s=READY pgs=214 cs=0 l=1 rev1=1 crypto rx=0x7f23580095f0 tx=0x7f2358005f90 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:17.742 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.742+0000 7f236c8bd640 1 -- 172.21.15.99:0/859528443 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f236009c2f0 con 0x7f23681022d0 2024-03-19T01:40:17.920 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.919+0000 7f236d8bf640 1 -- 172.21.15.99:0/859528443 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) -- 0x7f2368062420 con 0x7f23681022d0 2024-03-19T01:40:17.920 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.920+0000 7f236c8bd640 1 -- 172.21.15.99:0/859528443 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v91) ==== 93+0+10 (secure 0 0 0) 0x7f236009bbf0 con 0x7f23681022d0 2024-03-19T01:40:17.920 INFO:teuthology.orchestra.run.smithi099.stdout:pg_num: 1 2024-03-19T01:40:17.922 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.922+0000 7f236d8bf640 1 -- 172.21.15.99:0/859528443 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f23540a2800 msgr2=0x7f23540a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:17.922 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.922+0000 7f236d8bf640 1 --2- 172.21.15.99:0/859528443 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f23540a2800 0x7f23540a4cc0 secure :-1 s=READY pgs=214 cs=0 l=1 rev1=1 crypto rx=0x7f23580095f0 tx=0x7f2358005f90 comp rx=0 tx=0).stop 2024-03-19T01:40:17.922 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.922+0000 7f236d8bf640 1 -- 172.21.15.99:0/859528443 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f23681022d0 msgr2=0x7f236819eef0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:17.922 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.922+0000 7f236d8bf640 1 --2- 172.21.15.99:0/859528443 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f23681022d0 0x7f236819eef0 secure :-1 s=READY pgs=284 cs=0 l=1 rev1=1 crypto rx=0x7f236000efd0 tx=0x7f236000c6a0 comp rx=0 tx=0).stop 2024-03-19T01:40:17.922 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.922+0000 7f236d8bf640 1 -- 172.21.15.99:0/859528443 shutdown_connections 2024-03-19T01:40:17.922 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.922+0000 7f236d8bf640 1 --2- 172.21.15.99:0/859528443 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f23681022d0 0x7f236819eef0 unknown :-1 s=CLOSED pgs=284 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:17.923 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.922+0000 7f236d8bf640 1 --2- 172.21.15.99:0/859528443 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f2368101910 0x7f236819c8d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:17.923 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.922+0000 7f236d8bf640 1 --2- 172.21.15.99:0/859528443 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f23540a2800 0x7f23540a4cc0 unknown :-1 s=CLOSED pgs=214 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:17.923 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.922+0000 7f236d8bf640 1 --2- 172.21.15.99:0/859528443 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f2368100720 0x7f236806a9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:17.923 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.922+0000 7f236d8bf640 1 -- 172.21.15.99:0/859528443 >> 172.21.15.99:0/859528443 conn(0x7f23680fbcd0 msgr2=0x7f2368104c90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:17.923 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.922+0000 7f236d8bf640 1 -- 172.21.15.99:0/859528443 shutdown_connections 2024-03-19T01:40:17.923 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:17.922+0000 7f236d8bf640 1 -- 172.21.15.99:0/859528443 wait complete. 2024-03-19T01:40:17.932 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:17 smithi099 ceph-mon[62951]: pgmap v326: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.1 KiB/s rd, 458 B/s wr, 6 op/s 2024-03-19T01:40:17.932 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:17 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/1310734870' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-03-19T01:40:17.932 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:17 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/3609521473' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:40:17.933 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:17 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/2036460928' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-19T01:40:17.934 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-19T01:40:18.040 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.038+0000 7fbbce3c4640 1 -- 172.21.15.99:0/3766496981 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbbc8101920 msgr2=0x7fbbc8101da0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:18.040 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.038+0000 7fbbce3c4640 1 --2- 172.21.15.99:0/3766496981 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbbc8101920 0x7fbbc8101da0 secure :-1 s=READY pgs=285 cs=0 l=1 rev1=1 crypto rx=0x7fbbbc009900 tx=0x7fbbbc02bc30 comp rx=0 tx=0).stop 2024-03-19T01:40:18.041 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.038+0000 7fbbce3c4640 1 -- 172.21.15.99:0/3766496981 shutdown_connections 2024-03-19T01:40:18.041 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.038+0000 7fbbce3c4640 1 --2- 172.21.15.99:0/3766496981 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fbbc81022e0 0x7fbbc8108b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:18.041 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.038+0000 7fbbce3c4640 1 --2- 172.21.15.99:0/3766496981 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbbc8101920 0x7fbbc8101da0 unknown :-1 s=CLOSED pgs=285 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:18.041 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.038+0000 7fbbce3c4640 1 --2- 172.21.15.99:0/3766496981 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fbbc8100730 0x7fbbc8100b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:18.041 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.038+0000 7fbbce3c4640 1 -- 172.21.15.99:0/3766496981 >> 172.21.15.99:0/3766496981 conn(0x7fbbc80fbcc0 msgr2=0x7fbbc80fe0e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:18.041 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.038+0000 7fbbce3c4640 1 -- 172.21.15.99:0/3766496981 shutdown_connections 2024-03-19T01:40:18.041 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.038+0000 7fbbce3c4640 1 -- 172.21.15.99:0/3766496981 wait complete. 2024-03-19T01:40:18.041 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.038+0000 7fbbce3c4640 1 Processor -- start 2024-03-19T01:40:18.041 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.038+0000 7fbbce3c4640 1 -- start start 2024-03-19T01:40:18.041 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.039+0000 7fbbce3c4640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fbbc8100730 0x7fbbc8197b80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:18.041 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.039+0000 7fbbce3c4640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fbbc8101920 0x7fbbc81980c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:18.041 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.039+0000 7fbbce3c4640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbbc81022e0 0x7fbbc819f130 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:18.042 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.039+0000 7fbbce3c4640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fbbc810d4d0 con 0x7fbbc81022e0 2024-03-19T01:40:18.042 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.039+0000 7fbbce3c4640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fbbc810d350 con 0x7fbbc8101920 2024-03-19T01:40:18.042 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.039+0000 7fbbce3c4640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fbbc810d650 con 0x7fbbc8100730 2024-03-19T01:40:18.042 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.039+0000 7fbbcc961640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbbc81022e0 0x7fbbc819f130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:18.042 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.039+0000 7fbbcc961640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbbc81022e0 0x7fbbc819f130 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:52866/0 (socket says 172.21.15.99:52866) 2024-03-19T01:40:18.042 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.039+0000 7fbbcc961640 1 -- 172.21.15.99:0/1686719112 learned_addr learned my addr 172.21.15.99:0/1686719112 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:18.042 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.039+0000 7fbbc77fe640 1 --2- 172.21.15.99:0/1686719112 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fbbc8101920 0x7fbbc81980c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:18.042 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.039+0000 7fbbc7fff640 1 --2- 172.21.15.99:0/1686719112 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fbbc8100730 0x7fbbc8197b80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:18.042 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.039+0000 7fbbcc961640 1 -- 172.21.15.99:0/1686719112 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fbbc8100730 msgr2=0x7fbbc8197b80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:18.042 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.039+0000 7fbbcc961640 1 --2- 172.21.15.99:0/1686719112 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fbbc8100730 0x7fbbc8197b80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:18.042 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.039+0000 7fbbcc961640 1 -- 172.21.15.99:0/1686719112 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fbbc8101920 msgr2=0x7fbbc81980c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:18.042 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.039+0000 7fbbcc961640 1 --2- 172.21.15.99:0/1686719112 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fbbc8101920 0x7fbbc81980c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:18.042 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.039+0000 7fbbcc961640 1 -- 172.21.15.99:0/1686719112 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fbbc819f7a0 con 0x7fbbc81022e0 2024-03-19T01:40:18.043 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.039+0000 7fbbc7fff640 1 --2- 172.21.15.99:0/1686719112 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fbbc8100730 0x7fbbc8197b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:40:18.043 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.039+0000 7fbbc77fe640 1 --2- 172.21.15.99:0/1686719112 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fbbc8101920 0x7fbbc81980c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:40:18.043 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.039+0000 7fbbcc961640 1 --2- 172.21.15.99:0/1686719112 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbbc81022e0 0x7fbbc819f130 secure :-1 s=READY pgs=286 cs=0 l=1 rev1=1 crypto rx=0x7fbbc000eb10 tx=0x7fbbc000c2d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:18.043 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.039+0000 7fbbc57fa640 1 -- 172.21.15.99:0/1686719112 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbbc0004b20 con 0x7fbbc81022e0 2024-03-19T01:40:18.043 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.039+0000 7fbbce3c4640 1 -- 172.21.15.99:0/1686719112 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fbbc819fa90 con 0x7fbbc81022e0 2024-03-19T01:40:18.043 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.039+0000 7fbbc57fa640 1 -- 172.21.15.99:0/1686719112 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fbbc0004cc0 con 0x7fbbc81022e0 2024-03-19T01:40:18.043 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.039+0000 7fbbc57fa640 1 -- 172.21.15.99:0/1686719112 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbbc0010760 con 0x7fbbc81022e0 2024-03-19T01:40:18.043 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.039+0000 7fbbce3c4640 1 -- 172.21.15.99:0/1686719112 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fbbc819ff60 con 0x7fbbc81022e0 2024-03-19T01:40:18.043 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.040+0000 7fbbce3c4640 1 -- 172.21.15.99:0/1686719112 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fbbc8107af0 con 0x7fbbc81022e0 2024-03-19T01:40:18.044 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.041+0000 7fbbc57fa640 1 -- 172.21.15.99:0/1686719112 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 18) ==== 137290+0+0 (secure 0 0 0) 0x7fbbc0005020 con 0x7fbbc81022e0 2024-03-19T01:40:18.044 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.042+0000 7fbbc57fa640 1 --2- 172.21.15.99:0/1686719112 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fbbb00a2800 0x7fbbb00a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:18.044 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.042+0000 7fbbc7fff640 1 --2- 172.21.15.99:0/1686719112 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fbbb00a2800 0x7fbbb00a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:18.044 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.042+0000 7fbbc57fa640 1 -- 172.21.15.99:0/1686719112 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7fbbc00cf010 con 0x7fbbc81022e0 2024-03-19T01:40:18.044 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.042+0000 7fbbc7fff640 1 --2- 172.21.15.99:0/1686719112 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fbbb00a2800 0x7fbbb00a4cc0 secure :-1 s=READY pgs=215 cs=0 l=1 rev1=1 crypto rx=0x7fbbc8101780 tx=0x7fbbb8005f90 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:18.047 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.047+0000 7fbbc57fa640 1 -- 172.21.15.99:0/1686719112 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7fbbc009c070 con 0x7fbbc81022e0 2024-03-19T01:40:18.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:17 smithi112 ceph-mon[66819]: pgmap v326: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.1 KiB/s rd, 458 B/s wr, 6 op/s 2024-03-19T01:40:18.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:17 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/1310734870' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-03-19T01:40:18.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:17 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/3609521473' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:40:18.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:17 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/2036460928' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-19T01:40:18.207 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:17 smithi115 ceph-mon[67332]: pgmap v326: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.1 KiB/s rd, 458 B/s wr, 6 op/s 2024-03-19T01:40:18.207 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:17 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/1310734870' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-03-19T01:40:18.207 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:17 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/3609521473' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:40:18.207 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:17 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/2036460928' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-19T01:40:18.222 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.221+0000 7fbbce3c4640 1 -- 172.21.15.99:0/1686719112 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) -- 0x7fbbc8062420 con 0x7fbbc81022e0 2024-03-19T01:40:18.222 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.221+0000 7fbbc57fa640 1 -- 172.21.15.99:0/1686719112 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v91) ==== 104+0+11 (secure 0 0 0) 0x7fbbc009b970 con 0x7fbbc81022e0 2024-03-19T01:40:18.222 INFO:teuthology.orchestra.run.smithi099.stdout:pg_num: 32 2024-03-19T01:40:18.223 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.223+0000 7fbbce3c4640 1 -- 172.21.15.99:0/1686719112 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fbbb00a2800 msgr2=0x7fbbb00a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:18.223 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.223+0000 7fbbce3c4640 1 --2- 172.21.15.99:0/1686719112 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fbbb00a2800 0x7fbbb00a4cc0 secure :-1 s=READY pgs=215 cs=0 l=1 rev1=1 crypto rx=0x7fbbc8101780 tx=0x7fbbb8005f90 comp rx=0 tx=0).stop 2024-03-19T01:40:18.224 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.223+0000 7fbbce3c4640 1 -- 172.21.15.99:0/1686719112 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbbc81022e0 msgr2=0x7fbbc819f130 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:18.224 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.223+0000 7fbbce3c4640 1 --2- 172.21.15.99:0/1686719112 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbbc81022e0 0x7fbbc819f130 secure :-1 s=READY pgs=286 cs=0 l=1 rev1=1 crypto rx=0x7fbbc000eb10 tx=0x7fbbc000c2d0 comp rx=0 tx=0).stop 2024-03-19T01:40:18.224 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.223+0000 7fbbce3c4640 1 -- 172.21.15.99:0/1686719112 shutdown_connections 2024-03-19T01:40:18.224 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.223+0000 7fbbce3c4640 1 --2- 172.21.15.99:0/1686719112 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fbbb00a2800 0x7fbbb00a4cc0 unknown :-1 s=CLOSED pgs=215 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:18.224 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.223+0000 7fbbce3c4640 1 --2- 172.21.15.99:0/1686719112 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fbbc8101920 0x7fbbc81980c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:18.224 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.223+0000 7fbbce3c4640 1 --2- 172.21.15.99:0/1686719112 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbbc81022e0 0x7fbbc819f130 unknown :-1 s=CLOSED pgs=286 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:18.224 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.223+0000 7fbbce3c4640 1 --2- 172.21.15.99:0/1686719112 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fbbc8100730 0x7fbbc8197b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:18.224 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.223+0000 7fbbce3c4640 1 -- 172.21.15.99:0/1686719112 >> 172.21.15.99:0/1686719112 conn(0x7fbbc80fbcc0 msgr2=0x7fbbc8104cc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:18.224 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.223+0000 7fbbce3c4640 1 -- 172.21.15.99:0/1686719112 shutdown_connections 2024-03-19T01:40:18.224 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.223+0000 7fbbce3c4640 1 -- 172.21.15.99:0/1686719112 wait complete. 2024-03-19T01:40:18.235 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-19T01:40:18.341 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.338+0000 7fed6b7b3640 1 -- 172.21.15.99:0/3869114962 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fed64102a40 msgr2=0x7fed64102ec0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:18.341 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.338+0000 7fed6b7b3640 1 --2- 172.21.15.99:0/3869114962 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fed64102a40 0x7fed64102ec0 secure :-1 s=READY pgs=287 cs=0 l=1 rev1=1 crypto rx=0x7fed54009900 tx=0x7fed5402bc30 comp rx=0 tx=0).stop 2024-03-19T01:40:18.341 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.338+0000 7fed6b7b3640 1 -- 172.21.15.99:0/3869114962 shutdown_connections 2024-03-19T01:40:18.341 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.338+0000 7fed6b7b3640 1 --2- 172.21.15.99:0/3869114962 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fed64103400 0x7fed64109c90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:18.341 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.338+0000 7fed6b7b3640 1 --2- 172.21.15.99:0/3869114962 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fed64102a40 0x7fed64102ec0 unknown :-1 s=CLOSED pgs=287 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:18.341 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.338+0000 7fed6b7b3640 1 --2- 172.21.15.99:0/3869114962 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fed64101850 0x7fed64101c70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:18.341 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.338+0000 7fed6b7b3640 1 -- 172.21.15.99:0/3869114962 >> 172.21.15.99:0/3869114962 conn(0x7fed640fcdc0 msgr2=0x7fed640ff200 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:18.341 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.339+0000 7fed6b7b3640 1 -- 172.21.15.99:0/3869114962 shutdown_connections 2024-03-19T01:40:18.341 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.339+0000 7fed6b7b3640 1 -- 172.21.15.99:0/3869114962 wait complete. 2024-03-19T01:40:18.341 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.339+0000 7fed6b7b3640 1 Processor -- start 2024-03-19T01:40:18.342 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.339+0000 7fed6b7b3640 1 -- start start 2024-03-19T01:40:18.342 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.339+0000 7fed6b7b3640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fed64101850 0x7fed64197b50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:18.342 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.339+0000 7fed6b7b3640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fed64102a40 0x7fed64198090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:18.342 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.339+0000 7fed6b7b3640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fed64103400 0x7fed6419f100 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:18.342 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.339+0000 7fed6b7b3640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fed6410d4b0 con 0x7fed64103400 2024-03-19T01:40:18.342 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.339+0000 7fed6b7b3640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fed6410d330 con 0x7fed64101850 2024-03-19T01:40:18.342 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.339+0000 7fed6b7b3640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fed6410d630 con 0x7fed64102a40 2024-03-19T01:40:18.342 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.339+0000 7fed69d50640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fed64103400 0x7fed6419f100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:18.342 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.339+0000 7fed69d50640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fed64103400 0x7fed6419f100 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:52888/0 (socket says 172.21.15.99:52888) 2024-03-19T01:40:18.342 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.339+0000 7fed69d50640 1 -- 172.21.15.99:0/964676406 learned_addr learned my addr 172.21.15.99:0/964676406 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:18.342 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.339+0000 7fed68d4e640 1 --2- 172.21.15.99:0/964676406 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fed64102a40 0x7fed64198090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:18.342 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.339+0000 7fed6954f640 1 --2- 172.21.15.99:0/964676406 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fed64101850 0x7fed64197b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:18.342 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.340+0000 7fed69d50640 1 -- 172.21.15.99:0/964676406 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fed64102a40 msgr2=0x7fed64198090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:18.342 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.340+0000 7fed69d50640 1 --2- 172.21.15.99:0/964676406 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fed64102a40 0x7fed64198090 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:18.343 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.340+0000 7fed69d50640 1 -- 172.21.15.99:0/964676406 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fed64101850 msgr2=0x7fed64197b50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:18.343 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.340+0000 7fed69d50640 1 --2- 172.21.15.99:0/964676406 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fed64101850 0x7fed64197b50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:18.343 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.340+0000 7fed69d50640 1 -- 172.21.15.99:0/964676406 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fed6419f770 con 0x7fed64103400 2024-03-19T01:40:18.343 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.340+0000 7fed69d50640 1 --2- 172.21.15.99:0/964676406 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fed64103400 0x7fed6419f100 secure :-1 s=READY pgs=288 cs=0 l=1 rev1=1 crypto rx=0x7fed5c00e970 tx=0x7fed5c00ed70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:18.343 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.340+0000 7fed5a7fc640 1 -- 172.21.15.99:0/964676406 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fed5c00ccc0 con 0x7fed64103400 2024-03-19T01:40:18.343 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.340+0000 7fed6b7b3640 1 -- 172.21.15.99:0/964676406 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fed6419fa60 con 0x7fed64103400 2024-03-19T01:40:18.343 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.340+0000 7fed5a7fc640 1 -- 172.21.15.99:0/964676406 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fed5c00ce60 con 0x7fed64103400 2024-03-19T01:40:18.343 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.340+0000 7fed5a7fc640 1 -- 172.21.15.99:0/964676406 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fed5c010490 con 0x7fed64103400 2024-03-19T01:40:18.343 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.340+0000 7fed6b7b3640 1 -- 172.21.15.99:0/964676406 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fed6419ff80 con 0x7fed64103400 2024-03-19T01:40:18.343 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.341+0000 7fed6b7b3640 1 -- 172.21.15.99:0/964676406 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fed64108bf0 con 0x7fed64103400 2024-03-19T01:40:18.344 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.342+0000 7fed5a7fc640 1 -- 172.21.15.99:0/964676406 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 18) ==== 137290+0+0 (secure 0 0 0) 0x7fed5c005020 con 0x7fed64103400 2024-03-19T01:40:18.344 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.342+0000 7fed5a7fc640 1 --2- 172.21.15.99:0/964676406 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fed500a2800 0x7fed500a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:18.344 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.343+0000 7fed6954f640 1 --2- 172.21.15.99:0/964676406 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fed500a2800 0x7fed500a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:18.344 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.343+0000 7fed5a7fc640 1 -- 172.21.15.99:0/964676406 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7fed5c0cef00 con 0x7fed64103400 2024-03-19T01:40:18.344 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.343+0000 7fed6954f640 1 --2- 172.21.15.99:0/964676406 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fed500a2800 0x7fed500a4cc0 secure :-1 s=READY pgs=216 cs=0 l=1 rev1=1 crypto rx=0x7fed641028a0 tx=0x7fed60009500 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:18.347 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.347+0000 7fed5a7fc640 1 -- 172.21.15.99:0/964676406 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7fed5c09bf60 con 0x7fed64103400 2024-03-19T01:40:18.523 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.523+0000 7fed6b7b3640 1 -- 172.21.15.99:0/964676406 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) -- 0x7fed64062420 con 0x7fed64103400 2024-03-19T01:40:18.523 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.523+0000 7fed5a7fc640 1 -- 172.21.15.99:0/964676406 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v91) ==== 100+0+11 (secure 0 0 0) 0x7fed5c09b860 con 0x7fed64103400 2024-03-19T01:40:18.523 INFO:teuthology.orchestra.run.smithi099.stdout:pg_num: 64 2024-03-19T01:40:18.525 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.525+0000 7fed6b7b3640 1 -- 172.21.15.99:0/964676406 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fed500a2800 msgr2=0x7fed500a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:18.525 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.525+0000 7fed6b7b3640 1 --2- 172.21.15.99:0/964676406 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fed500a2800 0x7fed500a4cc0 secure :-1 s=READY pgs=216 cs=0 l=1 rev1=1 crypto rx=0x7fed641028a0 tx=0x7fed60009500 comp rx=0 tx=0).stop 2024-03-19T01:40:18.525 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.525+0000 7fed6b7b3640 1 -- 172.21.15.99:0/964676406 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fed64103400 msgr2=0x7fed6419f100 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:18.525 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.525+0000 7fed6b7b3640 1 --2- 172.21.15.99:0/964676406 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fed64103400 0x7fed6419f100 secure :-1 s=READY pgs=288 cs=0 l=1 rev1=1 crypto rx=0x7fed5c00e970 tx=0x7fed5c00ed70 comp rx=0 tx=0).stop 2024-03-19T01:40:18.525 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.525+0000 7fed6b7b3640 1 -- 172.21.15.99:0/964676406 shutdown_connections 2024-03-19T01:40:18.526 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.525+0000 7fed6b7b3640 1 --2- 172.21.15.99:0/964676406 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fed64103400 0x7fed6419f100 unknown :-1 s=CLOSED pgs=288 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:18.526 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.525+0000 7fed6b7b3640 1 --2- 172.21.15.99:0/964676406 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fed64102a40 0x7fed64198090 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:18.526 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.525+0000 7fed6b7b3640 1 --2- 172.21.15.99:0/964676406 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fed500a2800 0x7fed500a4cc0 unknown :-1 s=CLOSED pgs=216 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:18.526 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.525+0000 7fed6b7b3640 1 --2- 172.21.15.99:0/964676406 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fed64101850 0x7fed64197b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:18.526 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.525+0000 7fed6b7b3640 1 -- 172.21.15.99:0/964676406 >> 172.21.15.99:0/964676406 conn(0x7fed640fcdc0 msgr2=0x7fed64105dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:18.526 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.525+0000 7fed6b7b3640 1 -- 172.21.15.99:0/964676406 shutdown_connections 2024-03-19T01:40:18.526 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.525+0000 7fed6b7b3640 1 -- 172.21.15.99:0/964676406 wait complete. 2024-03-19T01:40:18.536 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-19T01:40:18.640 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.638+0000 7ff808bb3640 1 -- 172.21.15.99:0/2156959396 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff804101930 msgr2=0x7ff804101db0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:18.640 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.638+0000 7ff808bb3640 1 --2- 172.21.15.99:0/2156959396 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff804101930 0x7ff804101db0 secure :-1 s=READY pgs=66 cs=0 l=1 rev1=1 crypto rx=0x7ff7f4009860 tx=0x7ff7f402bb70 comp rx=0 tx=0).stop 2024-03-19T01:40:18.641 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.638+0000 7ff808bb3640 1 -- 172.21.15.99:0/2156959396 shutdown_connections 2024-03-19T01:40:18.641 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.638+0000 7ff808bb3640 1 --2- 172.21.15.99:0/2156959396 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff804101930 0x7ff804101db0 unknown :-1 s=CLOSED pgs=66 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:18.641 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.638+0000 7ff808bb3640 1 --2- 172.21.15.99:0/2156959396 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff8041022f0 0x7ff804108b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:18.641 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.638+0000 7ff808bb3640 1 --2- 172.21.15.99:0/2156959396 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff804100740 0x7ff804100b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:18.641 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.638+0000 7ff808bb3640 1 -- 172.21.15.99:0/2156959396 >> 172.21.15.99:0/2156959396 conn(0x7ff8040fbcb0 msgr2=0x7ff8040fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:18.641 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.638+0000 7ff808bb3640 1 -- 172.21.15.99:0/2156959396 shutdown_connections 2024-03-19T01:40:18.641 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.638+0000 7ff808bb3640 1 -- 172.21.15.99:0/2156959396 wait complete. 2024-03-19T01:40:18.641 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.639+0000 7ff808bb3640 1 Processor -- start 2024-03-19T01:40:18.641 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.639+0000 7ff808bb3640 1 -- start start 2024-03-19T01:40:18.641 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.639+0000 7ff808bb3640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff804100740 0x7ff804069510 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:18.641 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.639+0000 7ff808bb3640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff804101930 0x7ff80419c8c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:18.641 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.639+0000 7ff808bb3640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff8041022f0 0x7ff80419ef00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:18.641 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.639+0000 7ff808bb3640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7ff80410c3c0 con 0x7ff804101930 2024-03-19T01:40:18.642 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.639+0000 7ff808bb3640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7ff80410c240 con 0x7ff8041022f0 2024-03-19T01:40:18.642 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.639+0000 7ff808bb3640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7ff80410c540 con 0x7ff804100740 2024-03-19T01:40:18.642 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.639+0000 7ff801d9b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff804101930 0x7ff80419c8c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:18.642 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.639+0000 7ff801d9b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff804101930 0x7ff80419c8c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:52906/0 (socket says 172.21.15.99:52906) 2024-03-19T01:40:18.642 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.639+0000 7ff801d9b640 1 -- 172.21.15.99:0/90418591 learned_addr learned my addr 172.21.15.99:0/90418591 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:18.642 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.639+0000 7ff80259c640 1 --2- 172.21.15.99:0/90418591 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff804100740 0x7ff804069510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:18.642 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.639+0000 7ff802d9d640 1 --2- 172.21.15.99:0/90418591 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff8041022f0 0x7ff80419ef00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:18.642 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.639+0000 7ff801d9b640 1 -- 172.21.15.99:0/90418591 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff804100740 msgr2=0x7ff804069510 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:18.642 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.639+0000 7ff801d9b640 1 --2- 172.21.15.99:0/90418591 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff804100740 0x7ff804069510 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:18.642 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.639+0000 7ff801d9b640 1 -- 172.21.15.99:0/90418591 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff8041022f0 msgr2=0x7ff80419ef00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:18.642 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.639+0000 7ff801d9b640 1 --2- 172.21.15.99:0/90418591 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff8041022f0 0x7ff80419ef00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:18.642 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.639+0000 7ff801d9b640 1 -- 172.21.15.99:0/90418591 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ff80419f7c0 con 0x7ff804101930 2024-03-19T01:40:18.642 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.639+0000 7ff801d9b640 1 --2- 172.21.15.99:0/90418591 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff804101930 0x7ff80419c8c0 secure :-1 s=READY pgs=289 cs=0 l=1 rev1=1 crypto rx=0x7ff7f4005940 tx=0x7ff7f4032470 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:18.643 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.639+0000 7ff7f37fe640 1 -- 172.21.15.99:0/90418591 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff7f4041020 con 0x7ff804101930 2024-03-19T01:40:18.643 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.639+0000 7ff808bb3640 1 -- 172.21.15.99:0/90418591 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ff80419fa50 con 0x7ff804101930 2024-03-19T01:40:18.643 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.639+0000 7ff7f37fe640 1 -- 172.21.15.99:0/90418591 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7ff7f402c9f0 con 0x7ff804101930 2024-03-19T01:40:18.643 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.639+0000 7ff7f37fe640 1 -- 172.21.15.99:0/90418591 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff7f4039040 con 0x7ff804101930 2024-03-19T01:40:18.643 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.640+0000 7ff808bb3640 1 -- 172.21.15.99:0/90418591 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7ff80419ff40 con 0x7ff804101930 2024-03-19T01:40:18.644 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.641+0000 7ff808bb3640 1 -- 172.21.15.99:0/90418591 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7ff804107ab0 con 0x7ff804101930 2024-03-19T01:40:18.644 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.642+0000 7ff7f37fe640 1 -- 172.21.15.99:0/90418591 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 18) ==== 137290+0+0 (secure 0 0 0) 0x7ff7f4044020 con 0x7ff804101930 2024-03-19T01:40:18.644 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.642+0000 7ff7f37fe640 1 --2- 172.21.15.99:0/90418591 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff7ec0a2800 0x7ff7ec0a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:18.644 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.642+0000 7ff80259c640 1 --2- 172.21.15.99:0/90418591 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff7ec0a2800 0x7ff7ec0a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:18.644 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.642+0000 7ff7f37fe640 1 -- 172.21.15.99:0/90418591 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7ff7f40f0420 con 0x7ff804101930 2024-03-19T01:40:18.644 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.642+0000 7ff80259c640 1 --2- 172.21.15.99:0/90418591 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff7ec0a2800 0x7ff7ec0a4cc0 secure :-1 s=READY pgs=217 cs=0 l=1 rev1=1 crypto rx=0x7ff804101790 tx=0x7ff7f8005f90 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:18.646 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.646+0000 7ff7f37fe640 1 -- 172.21.15.99:0/90418591 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7ff7f404b020 con 0x7ff804101930 2024-03-19T01:40:18.866 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.866+0000 7ff808bb3640 1 -- 172.21.15.99:0/90418591 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7ff804062420 con 0x7ff804101930 2024-03-19T01:40:18.866 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.866+0000 7ff7f37fe640 1 -- 172.21.15.99:0/90418591 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 20 v20) ==== 94+0+11878 (secure 0 0 0) 0x7ff7f40bd480 con 0x7ff804101930 2024-03-19T01:40:18.867 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:40:18.867 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":20,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14562,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6835/1631815621","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6834","nonce":1631815621},{"type":"v1","addr":"172.21.15.99:6835","nonce":1631815621}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":4},{"gid":14646,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6839/2236145144","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6838","nonce":2236145144},{"type":"v1","addr":"172.21.15.99:6839","nonce":2236145144}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11},{"gid":14658,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6841/1660990159","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6840","nonce":1660990159},{"type":"v1","addr":"172.21.15.99:6841","nonce":1660990159}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":12},{"gid":24383,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.112:6833/2969810977","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6832","nonce":2969810977},{"type":"v1","addr":"172.21.15.112:6833","nonce":2969810977}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3},{"gid":24404,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6837/2038634879","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6836","nonce":2038634879},{"type":"v1","addr":"172.21.15.99:6837","nonce":2038634879}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":24415,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.115:6833/2219320481","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6832","nonce":2219320481},{"type":"v1","addr":"172.21.15.115:6833","nonce":2219320481}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":20,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-03-19T01:38:56.124529+0000","modified":"2024-03-19T01:39:00.323326+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24487,"mds_1":24479,"mds_2":24472},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24416":{"gid":24416,"name":"e","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.112:6835/3690587528","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6834","nonce":3690587528},{"type":"v1","addr":"172.21.15.112:6835","nonce":3690587528}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24440":{"gid":24440,"name":"h","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.112:6837/2339823159","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6836","nonce":2339823159},{"type":"v1","addr":"172.21.15.112:6837","nonce":2339823159}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24460":{"gid":24460,"name":"f","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.115:6835/591078413","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6834","nonce":591078413},{"type":"v1","addr":"172.21.15.115:6835","nonce":591078413}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24472":{"gid":24472,"name":"c","rank":2,"incarnation":19,"state":"up:active","state_seq":7,"addr":"172.21.15.115:6837/1368988191","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6836","nonce":1368988191},{"type":"v1","addr":"172.21.15.115:6837","nonce":1368988191}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24479":{"gid":24479,"name":"b","rank":1,"incarnation":17,"state":"up:active","state_seq":4,"addr":"172.21.15.112:6839/986249680","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6838","nonce":986249680},{"type":"v1","addr":"172.21.15.112:6839","nonce":986249680}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24487":{"gid":24487,"name":"i","rank":0,"incarnation":14,"state":"up:active","state_seq":5,"addr":"172.21.15.115:6839/2743761968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6838","nonce":2743761968},{"type":"v1","addr":"172.21.15.115:6839","nonce":2743761968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24487,"qdb_cluster":[24487,24479,24472]},"id":1}]} 2024-03-19T01:40:18.867 INFO:teuthology.orchestra.run.smithi099.stderr:dumped fsmap epoch 20 2024-03-19T01:40:18.869 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.868+0000 7ff808bb3640 1 -- 172.21.15.99:0/90418591 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff7ec0a2800 msgr2=0x7ff7ec0a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:18.869 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.868+0000 7ff808bb3640 1 --2- 172.21.15.99:0/90418591 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff7ec0a2800 0x7ff7ec0a4cc0 secure :-1 s=READY pgs=217 cs=0 l=1 rev1=1 crypto rx=0x7ff804101790 tx=0x7ff7f8005f90 comp rx=0 tx=0).stop 2024-03-19T01:40:18.869 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.868+0000 7ff808bb3640 1 -- 172.21.15.99:0/90418591 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff804101930 msgr2=0x7ff80419c8c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:18.869 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.868+0000 7ff808bb3640 1 --2- 172.21.15.99:0/90418591 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff804101930 0x7ff80419c8c0 secure :-1 s=READY pgs=289 cs=0 l=1 rev1=1 crypto rx=0x7ff7f4005940 tx=0x7ff7f4032470 comp rx=0 tx=0).stop 2024-03-19T01:40:18.869 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.868+0000 7ff808bb3640 1 -- 172.21.15.99:0/90418591 shutdown_connections 2024-03-19T01:40:18.869 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.868+0000 7ff808bb3640 1 --2- 172.21.15.99:0/90418591 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff7ec0a2800 0x7ff7ec0a4cc0 unknown :-1 s=CLOSED pgs=217 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:18.869 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.868+0000 7ff808bb3640 1 --2- 172.21.15.99:0/90418591 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff8041022f0 0x7ff80419ef00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:18.869 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.868+0000 7ff808bb3640 1 --2- 172.21.15.99:0/90418591 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff804101930 0x7ff80419c8c0 unknown :-1 s=CLOSED pgs=289 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:18.869 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.868+0000 7ff808bb3640 1 --2- 172.21.15.99:0/90418591 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff804100740 0x7ff804069510 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:18.869 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.868+0000 7ff808bb3640 1 -- 172.21.15.99:0/90418591 >> 172.21.15.99:0/90418591 conn(0x7ff8040fbcb0 msgr2=0x7ff804104c80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:18.869 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.868+0000 7ff808bb3640 1 -- 172.21.15.99:0/90418591 shutdown_connections 2024-03-19T01:40:18.869 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.868+0000 7ff808bb3640 1 -- 172.21.15.99:0/90418591 wait complete. 2024-03-19T01:40:18.883 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-19T01:40:18.986 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.984+0000 7febbe190640 1 -- 172.21.15.99:0/2684181351 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7febb81019b0 msgr2=0x7febb8101e30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:18.987 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.984+0000 7febbe190640 1 --2- 172.21.15.99:0/2684181351 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7febb81019b0 0x7febb8101e30 secure :-1 s=READY pgs=290 cs=0 l=1 rev1=1 crypto rx=0x7feba0009900 tx=0x7feba002bc30 comp rx=0 tx=0).stop 2024-03-19T01:40:18.987 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.984+0000 7febbe190640 1 -- 172.21.15.99:0/2684181351 shutdown_connections 2024-03-19T01:40:18.987 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.984+0000 7febbe190640 1 --2- 172.21.15.99:0/2684181351 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7febb8102370 0x7febb8108c00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:18.987 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.984+0000 7febbe190640 1 --2- 172.21.15.99:0/2684181351 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7febb81019b0 0x7febb8101e30 unknown :-1 s=CLOSED pgs=290 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:18.987 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.984+0000 7febbe190640 1 --2- 172.21.15.99:0/2684181351 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7febb81007c0 0x7febb8100be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:18.987 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.984+0000 7febbe190640 1 -- 172.21.15.99:0/2684181351 >> 172.21.15.99:0/2684181351 conn(0x7febb80fbd50 msgr2=0x7febb80fe170 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:18.987 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.984+0000 7febbe190640 1 -- 172.21.15.99:0/2684181351 shutdown_connections 2024-03-19T01:40:18.987 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.984+0000 7febbe190640 1 -- 172.21.15.99:0/2684181351 wait complete. 2024-03-19T01:40:18.987 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.985+0000 7febbe190640 1 Processor -- start 2024-03-19T01:40:18.987 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.985+0000 7febbe190640 1 -- start start 2024-03-19T01:40:18.987 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.985+0000 7febbe190640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7febb81007c0 0x7febb8197b20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:18.987 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.985+0000 7febbe190640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7febb81019b0 0x7febb8198060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:18.987 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.985+0000 7febbe190640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7febb8102370 0x7febb819f0d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:18.988 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.985+0000 7febbe190640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7febb810d4f0 con 0x7febb8102370 2024-03-19T01:40:18.988 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.985+0000 7febbe190640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7febb810d370 con 0x7febb81019b0 2024-03-19T01:40:18.988 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.985+0000 7febbe190640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7febb810d670 con 0x7febb81007c0 2024-03-19T01:40:18.988 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.985+0000 7febb7fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7febb8102370 0x7febb819f0d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:18.988 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.985+0000 7febb7fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7febb8102370 0x7febb819f0d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:52926/0 (socket says 172.21.15.99:52926) 2024-03-19T01:40:18.988 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.985+0000 7febb7fff640 1 -- 172.21.15.99:0/4238660414 learned_addr learned my addr 172.21.15.99:0/4238660414 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:18.988 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.985+0000 7febaeffd640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7febb81019b0 0x7febb8198060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:18.988 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.985+0000 7febb77fe640 1 --2- 172.21.15.99:0/4238660414 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7febb81007c0 0x7febb8197b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:18.988 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.985+0000 7febb7fff640 1 -- 172.21.15.99:0/4238660414 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7febb81007c0 msgr2=0x7febb8197b20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:18.988 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.985+0000 7febb7fff640 1 --2- 172.21.15.99:0/4238660414 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7febb81007c0 0x7febb8197b20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:18.988 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.985+0000 7febb7fff640 1 -- 172.21.15.99:0/4238660414 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7febb81019b0 msgr2=0x7febb8198060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:18.988 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.985+0000 7febb7fff640 1 --2- 172.21.15.99:0/4238660414 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7febb81019b0 0x7febb8198060 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:18.988 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.985+0000 7febb7fff640 1 -- 172.21.15.99:0/4238660414 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7febb819f740 con 0x7febb8102370 2024-03-19T01:40:18.988 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.986+0000 7febb7fff640 1 --2- 172.21.15.99:0/4238660414 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7febb8102370 0x7febb819f0d0 secure :-1 s=READY pgs=291 cs=0 l=1 rev1=1 crypto rx=0x7febb000eb10 tx=0x7febb000c2d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:18.989 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.986+0000 7febb57fa640 1 -- 172.21.15.99:0/4238660414 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7febb0004b20 con 0x7febb8102370 2024-03-19T01:40:18.989 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.986+0000 7febb57fa640 1 -- 172.21.15.99:0/4238660414 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7febb0004cc0 con 0x7febb8102370 2024-03-19T01:40:18.989 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.986+0000 7febbe190640 1 -- 172.21.15.99:0/4238660414 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7febb819fa30 con 0x7febb8102370 2024-03-19T01:40:18.989 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.986+0000 7febb57fa640 1 -- 172.21.15.99:0/4238660414 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7febb0010760 con 0x7febb8102370 2024-03-19T01:40:18.989 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.986+0000 7febbe190640 1 -- 172.21.15.99:0/4238660414 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7febb819ffa0 con 0x7febb8102370 2024-03-19T01:40:18.990 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.987+0000 7febbe190640 1 -- 172.21.15.99:0/4238660414 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7febb8107b80 con 0x7febb8102370 2024-03-19T01:40:18.990 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.988+0000 7febb57fa640 1 -- 172.21.15.99:0/4238660414 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 18) ==== 137290+0+0 (secure 0 0 0) 0x7febb0005020 con 0x7febb8102370 2024-03-19T01:40:18.990 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.988+0000 7febb57fa640 1 --2- 172.21.15.99:0/4238660414 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7feba40a2880 0x7feba40a4d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:18.990 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.988+0000 7febb77fe640 1 --2- 172.21.15.99:0/4238660414 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7feba40a2880 0x7feba40a4d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:18.990 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.988+0000 7febb57fa640 1 -- 172.21.15.99:0/4238660414 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7febb00cf0f0 con 0x7febb8102370 2024-03-19T01:40:18.990 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.989+0000 7febb77fe640 1 --2- 172.21.15.99:0/4238660414 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7feba40a2880 0x7feba40a4d40 secure :-1 s=READY pgs=218 cs=0 l=1 rev1=1 crypto rx=0x7feba80095f0 tx=0x7feba8005f90 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:18.993 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:18.993+0000 7febb57fa640 1 -- 172.21.15.99:0/4238660414 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7febb009c150 con 0x7febb8102370 2024-03-19T01:40:19.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:18 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/859528443' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-03-19T01:40:19.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:18 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/1686719112' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-03-19T01:40:19.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:18 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/964676406' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-03-19T01:40:19.168 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.167+0000 7febbe190640 1 -- 172.21.15.99:0/4238660414 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7febb802d1f0 con 0x7febb8102370 2024-03-19T01:40:19.169 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.169+0000 7febb57fa640 1 -- 172.21.15.99:0/4238660414 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v91) ==== 74+0+22804 (secure 0 0 0) 0x7febb009ba50 con 0x7febb8102370 2024-03-19T01:40:19.170 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:40:19.170 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":91,"fsid":"438fb69c-e590-11ee-95c9-87774f69a715","created":"2024-03-19T01:31:10.165073+0000","modified":"2024-03-19T01:40:08.821469+0000","last_up_change":"2024-03-19T01:38:09.082688+0000","last_in_change":"2024-03-19T01:37:49.115793+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-19T01:34:31.001309+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-19T01:38:54.018623+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"86","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-19T01:38:55.319036+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"87","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"0c407a0e-a40e-43d7-ad9a-bb8f3cd4a9ba","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6803","nonce":2935260112}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6805","nonce":2935260112}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6808","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6809","nonce":2935260112}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6807","nonce":2935260112}]},"public_addr":"172.21.15.99:6803/2935260112","cluster_addr":"172.21.15.99:6805/2935260112","heartbeat_back_addr":"172.21.15.99:6809/2935260112","heartbeat_front_addr":"172.21.15.99:6807/2935260112","state":["exists","up"]},{"osd":1,"uuid":"a48f9173-4a06-49f5-bca2-2c9047b8f666","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6800","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6801","nonce":1468854289}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6802","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6803","nonce":1468854289}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6806","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6807","nonce":1468854289}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6804","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6805","nonce":1468854289}]},"public_addr":"172.21.15.112:6801/1468854289","cluster_addr":"172.21.15.112:6803/1468854289","heartbeat_back_addr":"172.21.15.112:6807/1468854289","heartbeat_front_addr":"172.21.15.112:6805/1468854289","state":["exists","up"]},{"osd":2,"uuid":"b7715fdd-b6c9-4348-bd16-5eb61b427eef","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6800","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6801","nonce":3766690472}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6802","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6803","nonce":3766690472}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6806","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6807","nonce":3766690472}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6804","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6805","nonce":3766690472}]},"public_addr":"172.21.15.115:6801/3766690472","cluster_addr":"172.21.15.115:6803/3766690472","heartbeat_back_addr":"172.21.15.115:6807/3766690472","heartbeat_front_addr":"172.21.15.115:6805/3766690472","state":["exists","up"]},{"osd":3,"uuid":"6697ed1b-96bf-42de-8440-d17e9785b359","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6810","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6811","nonce":3756732895}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6812","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6813","nonce":3756732895}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6816","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6817","nonce":3756732895}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6814","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6815","nonce":3756732895}]},"public_addr":"172.21.15.99:6811/3756732895","cluster_addr":"172.21.15.99:6813/3756732895","heartbeat_back_addr":"172.21.15.99:6817/3756732895","heartbeat_front_addr":"172.21.15.99:6815/3756732895","state":["exists","up"]},{"osd":4,"uuid":"d958d6b9-67f6-43d0-b704-aa55dc9df83a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":34,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6808","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6809","nonce":3717986391}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6810","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6811","nonce":3717986391}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6814","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6815","nonce":3717986391}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6812","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6813","nonce":3717986391}]},"public_addr":"172.21.15.112:6809/3717986391","cluster_addr":"172.21.15.112:6811/3717986391","heartbeat_back_addr":"172.21.15.112:6815/3717986391","heartbeat_front_addr":"172.21.15.112:6813/3717986391","state":["exists","up"]},{"osd":5,"uuid":"a73eb49a-7925-49ba-bc4f-515e17f8b8ae","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6808","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6809","nonce":1971806408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6810","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6811","nonce":1971806408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6814","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6815","nonce":1971806408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6812","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6813","nonce":1971806408}]},"public_addr":"172.21.15.115:6809/1971806408","cluster_addr":"172.21.15.115:6811/1971806408","heartbeat_back_addr":"172.21.15.115:6815/1971806408","heartbeat_front_addr":"172.21.15.115:6813/1971806408","state":["exists","up"]},{"osd":6,"uuid":"caa718cd-e08b-48de-832a-29546d71e508","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6818","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6819","nonce":1130832772}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6820","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6821","nonce":1130832772}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6824","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6825","nonce":1130832772}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6822","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6823","nonce":1130832772}]},"public_addr":"172.21.15.99:6819/1130832772","cluster_addr":"172.21.15.99:6821/1130832772","heartbeat_back_addr":"172.21.15.99:6825/1130832772","heartbeat_front_addr":"172.21.15.99:6823/1130832772","state":["exists","up"]},{"osd":7,"uuid":"dbee19a1-2315-40d3-88e7-876fcde28903","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":53,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6816","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6817","nonce":3610356576}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6818","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6819","nonce":3610356576}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6822","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6823","nonce":3610356576}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6820","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6821","nonce":3610356576}]},"public_addr":"172.21.15.112:6817/3610356576","cluster_addr":"172.21.15.112:6819/3610356576","heartbeat_back_addr":"172.21.15.112:6823/3610356576","heartbeat_front_addr":"172.21.15.112:6821/3610356576","state":["exists","up"]},{"osd":8,"uuid":"14a866dc-5ce3-42fc-8bcf-9088b68726cb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":60,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6816","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6817","nonce":4255803111}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6818","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6819","nonce":4255803111}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6822","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6823","nonce":4255803111}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6820","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6821","nonce":4255803111}]},"public_addr":"172.21.15.115:6817/4255803111","cluster_addr":"172.21.15.115:6819/4255803111","heartbeat_back_addr":"172.21.15.115:6823/4255803111","heartbeat_front_addr":"172.21.15.115:6821/4255803111","state":["exists","up"]},{"osd":9,"uuid":"1db93652-6035-4906-91ae-d64d5a909502","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":68,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6826","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6827","nonce":1410921523}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6828","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6829","nonce":1410921523}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6832","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6833","nonce":1410921523}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6830","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6831","nonce":1410921523}]},"public_addr":"172.21.15.99:6827/1410921523","cluster_addr":"172.21.15.99:6829/1410921523","heartbeat_back_addr":"172.21.15.99:6833/1410921523","heartbeat_front_addr":"172.21.15.99:6831/1410921523","state":["exists","up"]},{"osd":10,"uuid":"6c50a8a7-3345-4f4a-af0e-5606c7aca453","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":74,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6824","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6825","nonce":891805922}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6826","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6827","nonce":891805922}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6830","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6831","nonce":891805922}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6828","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6829","nonce":891805922}]},"public_addr":"172.21.15.112:6825/891805922","cluster_addr":"172.21.15.112:6827/891805922","heartbeat_back_addr":"172.21.15.112:6831/891805922","heartbeat_front_addr":"172.21.15.112:6829/891805922","state":["exists","up"]},{"osd":11,"uuid":"fd63b7fd-0db5-46d5-82fa-9707bbd0b495","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":82,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6824","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6825","nonce":938436244}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6826","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6827","nonce":938436244}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6830","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6831","nonce":938436244}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6828","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6829","nonce":938436244}]},"public_addr":"172.21.15.115:6825/938436244","cluster_addr":"172.21.15.115:6827/938436244","heartbeat_back_addr":"172.21.15.115:6831/938436244","heartbeat_front_addr":"172.21.15.115:6829/938436244","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:33:39.036822+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:34:03.339566+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:34:27.149273+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:34:51.315561+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:35:15.396031+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:35:38.957168+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:36:03.213914+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:36:27.763699+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:36:51.688670+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:37:17.280165+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:37:42.554633+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:38:07.438039+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.0","mappings":[{"from":8,"to":1}]},{"pgid":"2.8","mappings":[{"from":2,"to":11}]},{"pgid":"2.b","mappings":[{"from":8,"to":11}]},{"pgid":"2.e","mappings":[{"from":2,"to":4}]},{"pgid":"3.0","mappings":[{"from":5,"to":9}]},{"pgid":"3.2","mappings":[{"from":3,"to":9}]},{"pgid":"3.14","mappings":[{"from":1,"to":9}]},{"pgid":"3.17","mappings":[{"from":4,"to":8}]},{"pgid":"3.31","mappings":[{"from":4,"to":8}]},{"pgid":"3.37","mappings":[{"from":5,"to":7}]},{"pgid":"3.3a","mappings":[{"from":4,"to":6}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.99:6801/1328245934":"2024-03-20T01:32:06.661002+0000","172.21.15.99:0/1618965837":"2024-03-20T01:32:06.661002+0000","172.21.15.99:0/2491429851":"2024-03-20T01:32:06.661002+0000","172.21.15.99:6800/1328245934":"2024-03-20T01:32:06.661002+0000","172.21.15.99:0/2112231766":"2024-03-20T01:31:40.161751+0000","172.21.15.99:0/3341836545":"2024-03-20T01:31:40.161751+0000","172.21.15.99:6800/4223048293":"2024-03-20T01:31:40.161751+0000","172.21.15.99:6801/4223048293":"2024-03-20T01:31:40.161751+0000","172.21.15.99:0/2377621863":"2024-03-20T01:31:40.161751+0000","172.21.15.99:0/1904400935":"2024-03-20T01:32:06.661002+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-19T01:40:19.171 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.171+0000 7febbe190640 1 -- 172.21.15.99:0/4238660414 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7feba40a2880 msgr2=0x7feba40a4d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:19.171 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.171+0000 7febbe190640 1 --2- 172.21.15.99:0/4238660414 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7feba40a2880 0x7feba40a4d40 secure :-1 s=READY pgs=218 cs=0 l=1 rev1=1 crypto rx=0x7feba80095f0 tx=0x7feba8005f90 comp rx=0 tx=0).stop 2024-03-19T01:40:19.172 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.171+0000 7febbe190640 1 -- 172.21.15.99:0/4238660414 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7febb8102370 msgr2=0x7febb819f0d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:19.172 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.171+0000 7febbe190640 1 --2- 172.21.15.99:0/4238660414 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7febb8102370 0x7febb819f0d0 secure :-1 s=READY pgs=291 cs=0 l=1 rev1=1 crypto rx=0x7febb000eb10 tx=0x7febb000c2d0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.172 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.171+0000 7febbe190640 1 -- 172.21.15.99:0/4238660414 shutdown_connections 2024-03-19T01:40:19.172 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.171+0000 7febbe190640 1 --2- 172.21.15.99:0/4238660414 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7feba40a2880 0x7feba40a4d40 unknown :-1 s=CLOSED pgs=218 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.172 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.171+0000 7febbe190640 1 --2- 172.21.15.99:0/4238660414 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7febb81019b0 0x7febb8198060 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.172 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.171+0000 7febbe190640 1 --2- 172.21.15.99:0/4238660414 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7febb8102370 0x7febb819f0d0 unknown :-1 s=CLOSED pgs=291 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.172 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.171+0000 7febbe190640 1 --2- 172.21.15.99:0/4238660414 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7febb81007c0 0x7febb8197b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.172 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.171+0000 7febbe190640 1 -- 172.21.15.99:0/4238660414 >> 172.21.15.99:0/4238660414 conn(0x7febb80fbd50 msgr2=0x7febb8104d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:19.172 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.171+0000 7febbe190640 1 -- 172.21.15.99:0/4238660414 shutdown_connections 2024-03-19T01:40:19.172 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.171+0000 7febbe190640 1 -- 172.21.15.99:0/4238660414 wait complete. 2024-03-19T01:40:19.181 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:18 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/859528443' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-03-19T01:40:19.181 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:18 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/1686719112' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-03-19T01:40:19.182 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:18 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/964676406' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-03-19T01:40:19.183 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-19T01:40:19.289 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.284+0000 7f688d99d640 1 -- 172.21.15.99:0/2484745885 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6888101760 msgr2=0x7f6888101be0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:19.289 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.284+0000 7f688d99d640 1 --2- 172.21.15.99:0/2484745885 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6888101760 0x7f6888101be0 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 crypto rx=0x7f6874009860 tx=0x7f687402bb70 comp rx=0 tx=0).stop 2024-03-19T01:40:19.289 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.284+0000 7f688d99d640 1 -- 172.21.15.99:0/2484745885 shutdown_connections 2024-03-19T01:40:19.290 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.284+0000 7f688d99d640 1 --2- 172.21.15.99:0/2484745885 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6888101760 0x7f6888101be0 unknown :-1 s=CLOSED pgs=67 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.290 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.284+0000 7f688d99d640 1 --2- 172.21.15.99:0/2484745885 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6888102120 0x7f68881089b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.290 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.284+0000 7f688d99d640 1 --2- 172.21.15.99:0/2484745885 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f6888100570 0x7f6888100990 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.290 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.284+0000 7f688d99d640 1 -- 172.21.15.99:0/2484745885 >> 172.21.15.99:0/2484745885 conn(0x7f68880fbb00 msgr2=0x7f68880fdf20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:19.290 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.284+0000 7f688d99d640 1 -- 172.21.15.99:0/2484745885 shutdown_connections 2024-03-19T01:40:19.291 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.284+0000 7f688d99d640 1 -- 172.21.15.99:0/2484745885 wait complete. 2024-03-19T01:40:19.291 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.285+0000 7f688d99d640 1 Processor -- start 2024-03-19T01:40:19.291 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.285+0000 7f688d99d640 1 -- start start 2024-03-19T01:40:19.291 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.285+0000 7f688d99d640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6888100570 0x7f6888196880 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.291 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.285+0000 7f688d99d640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f6888101760 0x7f6888196dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.291 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.285+0000 7f688d99d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6888102120 0x7f688819de30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.292 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.285+0000 7f688d99d640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f688810c200 con 0x7f6888102120 2024-03-19T01:40:19.292 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.285+0000 7f688d99d640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f688810c080 con 0x7f6888100570 2024-03-19T01:40:19.292 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.285+0000 7f688d99d640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f688810c380 con 0x7f6888101760 2024-03-19T01:40:19.292 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.285+0000 7f688c99b640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6888100570 0x7f6888196880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.292 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.285+0000 7f688d19c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6888102120 0x7f688819de30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.292 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.285+0000 7f688c99b640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6888100570 0x7f6888196880 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:45166/0 (socket says 172.21.15.99:45166) 2024-03-19T01:40:19.292 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.285+0000 7f688c99b640 1 -- 172.21.15.99:0/2020662108 learned_addr learned my addr 172.21.15.99:0/2020662108 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:19.293 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.285+0000 7f6887fff640 1 --2- 172.21.15.99:0/2020662108 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f6888101760 0x7f6888196dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.293 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.285+0000 7f688c99b640 1 -- 172.21.15.99:0/2020662108 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f6888101760 msgr2=0x7f6888196dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:19.293 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.285+0000 7f688c99b640 1 --2- 172.21.15.99:0/2020662108 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f6888101760 0x7f6888196dc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.293 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.285+0000 7f688c99b640 1 -- 172.21.15.99:0/2020662108 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6888102120 msgr2=0x7f688819de30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:19.293 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.285+0000 7f688c99b640 1 --2- 172.21.15.99:0/2020662108 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6888102120 0x7f688819de30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.293 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.285+0000 7f688c99b640 1 -- 172.21.15.99:0/2020662108 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f688819e4a0 con 0x7f6888100570 2024-03-19T01:40:19.294 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.285+0000 7f688c99b640 1 --2- 172.21.15.99:0/2020662108 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6888100570 0x7f6888196880 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7f687c00e9c0 tx=0x7f687c00ee80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:19.294 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.285+0000 7f6885ffb640 1 -- 172.21.15.99:0/2020662108 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f687c00cd80 con 0x7f6888100570 2024-03-19T01:40:19.294 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.285+0000 7f6885ffb640 1 -- 172.21.15.99:0/2020662108 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f687c0035f0 con 0x7f6888100570 2024-03-19T01:40:19.294 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.285+0000 7f688d99d640 1 -- 172.21.15.99:0/2020662108 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f688819e790 con 0x7f6888100570 2024-03-19T01:40:19.294 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.285+0000 7f6885ffb640 1 -- 172.21.15.99:0/2020662108 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f687c010690 con 0x7f6888100570 2024-03-19T01:40:19.294 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.286+0000 7f688d99d640 1 -- 172.21.15.99:0/2020662108 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f688819ec60 con 0x7f6888100570 2024-03-19T01:40:19.295 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.287+0000 7f688d99d640 1 -- 172.21.15.99:0/2020662108 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f68881078e0 con 0x7f6888100570 2024-03-19T01:40:19.295 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.287+0000 7f6885ffb640 1 -- 172.21.15.99:0/2020662108 <== mon.2 v2:172.21.15.112:3300/0 4 ==== mgrmap(e 18) ==== 137290+0+0 (secure 0 0 0) 0x7f687c0037a0 con 0x7f6888100570 2024-03-19T01:40:19.295 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.288+0000 7f6885ffb640 1 --2- 172.21.15.99:0/2020662108 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f68780a2800 0x7f68780a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.295 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.288+0000 7f6887fff640 1 --2- 172.21.15.99:0/2020662108 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f68780a2800 0x7f68780a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.295 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.288+0000 7f6885ffb640 1 -- 172.21.15.99:0/2020662108 <== mon.2 v2:172.21.15.112:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f687c0cee00 con 0x7f6888100570 2024-03-19T01:40:19.295 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.288+0000 7f6887fff640 1 --2- 172.21.15.99:0/2020662108 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f68780a2800 0x7f68780a4cc0 secure :-1 s=READY pgs=219 cs=0 l=1 rev1=1 crypto rx=0x7f687402c180 tx=0x7f6874034ad0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:19.296 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.292+0000 7f6885ffb640 1 -- 172.21.15.99:0/2020662108 <== mon.2 v2:172.21.15.112:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f687c09be60 con 0x7f6888100570 2024-03-19T01:40:19.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:18 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/859528443' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-03-19T01:40:19.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:18 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/1686719112' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-03-19T01:40:19.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:18 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/964676406' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-03-19T01:40:19.505 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.505+0000 7f688d99d640 1 -- 172.21.15.99:0/2020662108 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f688802d1f0 con 0x7f6888100570 2024-03-19T01:40:19.506 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.506+0000 7f6885ffb640 1 -- 172.21.15.99:0/2020662108 <== mon.2 v2:172.21.15.112:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 20 v20) ==== 94+0+11878 (secure 0 0 0) 0x7f688802d1f0 con 0x7f6888100570 2024-03-19T01:40:19.507 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:40:19.507 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":20,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14562,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6835/1631815621","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6834","nonce":1631815621},{"type":"v1","addr":"172.21.15.99:6835","nonce":1631815621}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":4},{"gid":14646,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6839/2236145144","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6838","nonce":2236145144},{"type":"v1","addr":"172.21.15.99:6839","nonce":2236145144}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11},{"gid":14658,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6841/1660990159","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6840","nonce":1660990159},{"type":"v1","addr":"172.21.15.99:6841","nonce":1660990159}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":12},{"gid":24383,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.112:6833/2969810977","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6832","nonce":2969810977},{"type":"v1","addr":"172.21.15.112:6833","nonce":2969810977}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3},{"gid":24404,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6837/2038634879","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6836","nonce":2038634879},{"type":"v1","addr":"172.21.15.99:6837","nonce":2038634879}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":24415,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.115:6833/2219320481","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6832","nonce":2219320481},{"type":"v1","addr":"172.21.15.115:6833","nonce":2219320481}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":20,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-03-19T01:38:56.124529+0000","modified":"2024-03-19T01:39:00.323326+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24487,"mds_1":24479,"mds_2":24472},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24416":{"gid":24416,"name":"e","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.112:6835/3690587528","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6834","nonce":3690587528},{"type":"v1","addr":"172.21.15.112:6835","nonce":3690587528}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24440":{"gid":24440,"name":"h","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.112:6837/2339823159","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6836","nonce":2339823159},{"type":"v1","addr":"172.21.15.112:6837","nonce":2339823159}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24460":{"gid":24460,"name":"f","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.115:6835/591078413","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6834","nonce":591078413},{"type":"v1","addr":"172.21.15.115:6835","nonce":591078413}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24472":{"gid":24472,"name":"c","rank":2,"incarnation":19,"state":"up:active","state_seq":7,"addr":"172.21.15.115:6837/1368988191","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6836","nonce":1368988191},{"type":"v1","addr":"172.21.15.115:6837","nonce":1368988191}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24479":{"gid":24479,"name":"b","rank":1,"incarnation":17,"state":"up:active","state_seq":4,"addr":"172.21.15.112:6839/986249680","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6838","nonce":986249680},{"type":"v1","addr":"172.21.15.112:6839","nonce":986249680}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24487":{"gid":24487,"name":"i","rank":0,"incarnation":14,"state":"up:active","state_seq":5,"addr":"172.21.15.115:6839/2743761968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6838","nonce":2743761968},{"type":"v1","addr":"172.21.15.115:6839","nonce":2743761968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24487,"qdb_cluster":[24487,24479,24472]},"id":1}]} 2024-03-19T01:40:19.507 INFO:teuthology.orchestra.run.smithi099.stderr:dumped fsmap epoch 20 2024-03-19T01:40:19.509 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.508+0000 7f688d99d640 1 -- 172.21.15.99:0/2020662108 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f68780a2800 msgr2=0x7f68780a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:19.509 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.508+0000 7f688d99d640 1 --2- 172.21.15.99:0/2020662108 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f68780a2800 0x7f68780a4cc0 secure :-1 s=READY pgs=219 cs=0 l=1 rev1=1 crypto rx=0x7f687402c180 tx=0x7f6874034ad0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.509 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.508+0000 7f688d99d640 1 -- 172.21.15.99:0/2020662108 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6888100570 msgr2=0x7f6888196880 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:19.509 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.508+0000 7f688d99d640 1 --2- 172.21.15.99:0/2020662108 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6888100570 0x7f6888196880 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7f687c00e9c0 tx=0x7f687c00ee80 comp rx=0 tx=0).stop 2024-03-19T01:40:19.509 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.508+0000 7f688d99d640 1 -- 172.21.15.99:0/2020662108 shutdown_connections 2024-03-19T01:40:19.510 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.508+0000 7f688d99d640 1 --2- 172.21.15.99:0/2020662108 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6888102120 0x7f688819de30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.510 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.508+0000 7f688d99d640 1 --2- 172.21.15.99:0/2020662108 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f6888101760 0x7f6888196dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.510 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.508+0000 7f688d99d640 1 --2- 172.21.15.99:0/2020662108 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f68780a2800 0x7f68780a4cc0 unknown :-1 s=CLOSED pgs=219 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.510 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.508+0000 7f688d99d640 1 --2- 172.21.15.99:0/2020662108 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6888100570 0x7f6888196880 unknown :-1 s=CLOSED pgs=68 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.510 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.508+0000 7f688d99d640 1 -- 172.21.15.99:0/2020662108 >> 172.21.15.99:0/2020662108 conn(0x7f68880fbb00 msgr2=0x7f6888104ab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:19.510 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.509+0000 7f688d99d640 1 -- 172.21.15.99:0/2020662108 shutdown_connections 2024-03-19T01:40:19.510 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.509+0000 7f688d99d640 1 -- 172.21.15.99:0/2020662108 wait complete. 2024-03-19T01:40:19.520 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 20, 'flags': 50, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': True, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 32, 'explicitly_allowed_features': 32, 'created': '2024-03-19T01:38:56.124529+0000', 'modified': '2024-03-19T01:39:00.323326+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 300, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}, 'max_mds': 3, 'in': [0, 1, 2], 'up': {'mds_0': 24487, 'mds_1': 24479, 'mds_2': 24472}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_24416': {'gid': 24416, 'name': 'e', 'rank': 2, 'incarnation': 0, 'state': 'up:standby-replay', 'state_seq': 1, 'addr': '172.21.15.112:6835/3690587528', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.112:6834', 'nonce': 3690587528}, {'type': 'v1', 'addr': '172.21.15.112:6835', 'nonce': 3690587528}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}}, 'gid_24440': {'gid': 24440, 'name': 'h', 'rank': 1, 'incarnation': 0, 'state': 'up:standby-replay', 'state_seq': 1, 'addr': '172.21.15.112:6837/2339823159', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.112:6836', 'nonce': 2339823159}, {'type': 'v1', 'addr': '172.21.15.112:6837', 'nonce': 2339823159}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}}, 'gid_24460': {'gid': 24460, 'name': 'f', 'rank': 0, 'incarnation': 0, 'state': 'up:standby-replay', 'state_seq': 1, 'addr': '172.21.15.115:6835/591078413', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.115:6834', 'nonce': 591078413}, {'type': 'v1', 'addr': '172.21.15.115:6835', 'nonce': 591078413}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}}, 'gid_24472': {'gid': 24472, 'name': 'c', 'rank': 2, 'incarnation': 19, 'state': 'up:active', 'state_seq': 7, 'addr': '172.21.15.115:6837/1368988191', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.115:6836', 'nonce': 1368988191}, {'type': 'v1', 'addr': '172.21.15.115:6837', 'nonce': 1368988191}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}}, 'gid_24479': {'gid': 24479, 'name': 'b', 'rank': 1, 'incarnation': 17, 'state': 'up:active', 'state_seq': 4, 'addr': '172.21.15.112:6839/986249680', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.112:6838', 'nonce': 986249680}, {'type': 'v1', 'addr': '172.21.15.112:6839', 'nonce': 986249680}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}}, 'gid_24487': {'gid': 24487, 'name': 'i', 'rank': 0, 'incarnation': 14, 'state': 'up:active', 'state_seq': 5, 'addr': '172.21.15.115:6839/2743761968', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.115:6838', 'nonce': 2743761968}, {'type': 'v1', 'addr': '172.21.15.115:6839', 'nonce': 2743761968}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}}}, 'data_pools': [3], 'metadata_pool': 2, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 24487, 'qdb_cluster': [24487, 24479, 24472]} 2024-03-19T01:40:19.520 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: 3/3 2024-03-19T01:40:19.520 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.c status 2024-03-19T01:40:19.626 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.624+0000 7f1af5737640 1 -- 172.21.15.99:0/882487402 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1af00ffe80 msgr2=0x7f1af01002a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:19.626 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.624+0000 7f1af5737640 1 --2- 172.21.15.99:0/882487402 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1af00ffe80 0x7f1af01002a0 secure :-1 s=READY pgs=292 cs=0 l=1 rev1=1 crypto rx=0x7f1ae4009900 tx=0x7f1ae402bc30 comp rx=0 tx=0).stop 2024-03-19T01:40:19.626 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.624+0000 7f1af5737640 1 -- 172.21.15.99:0/882487402 shutdown_connections 2024-03-19T01:40:19.626 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.624+0000 7f1af5737640 1 --2- 172.21.15.99:0/882487402 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1af0101a20 0x7f1af0108340 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.626 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.624+0000 7f1af5737640 1 --2- 172.21.15.99:0/882487402 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1af0101060 0x7f1af01014e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.626 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.624+0000 7f1af5737640 1 --2- 172.21.15.99:0/882487402 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1af00ffe80 0x7f1af01002a0 unknown :-1 s=CLOSED pgs=292 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.626 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.624+0000 7f1af5737640 1 -- 172.21.15.99:0/882487402 >> 172.21.15.99:0/882487402 conn(0x7f1af00fb420 msgr2=0x7f1af00fd880 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:19.627 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.624+0000 7f1af5737640 1 -- 172.21.15.99:0/882487402 shutdown_connections 2024-03-19T01:40:19.627 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.624+0000 7f1af5737640 1 -- 172.21.15.99:0/882487402 wait complete. 2024-03-19T01:40:19.627 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.624+0000 7f1af5737640 1 Processor -- start 2024-03-19T01:40:19.627 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.624+0000 7f1af5737640 1 -- start start 2024-03-19T01:40:19.627 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.625+0000 7f1af5737640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1af00ffe80 0x7f1af0197bc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.627 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.625+0000 7f1af5737640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1af0101060 0x7f1af0198100 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.627 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.625+0000 7f1af5737640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1af0101a20 0x7f1af019f170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.627 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.625+0000 7f1af5737640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f1af010b2f0 con 0x7f1af0101060 2024-03-19T01:40:19.627 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.625+0000 7f1af5737640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f1af010b170 con 0x7f1af0101a20 2024-03-19T01:40:19.627 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.625+0000 7f1af5737640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f1af010b470 con 0x7f1af00ffe80 2024-03-19T01:40:19.627 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.625+0000 7f1aee7fc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1af0101060 0x7f1af0198100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.627 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.625+0000 7f1aee7fc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1af0101060 0x7f1af0198100 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:52952/0 (socket says 172.21.15.99:52952) 2024-03-19T01:40:19.627 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.625+0000 7f1aee7fc640 1 -- 172.21.15.99:0/2550428639 learned_addr learned my addr 172.21.15.99:0/2550428639 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:19.627 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.625+0000 7f1aeeffd640 1 --2- 172.21.15.99:0/2550428639 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1af00ffe80 0x7f1af0197bc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.627 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.625+0000 7f1aef7fe640 1 --2- 172.21.15.99:0/2550428639 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1af0101a20 0x7f1af019f170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.628 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.625+0000 7f1aee7fc640 1 -- 172.21.15.99:0/2550428639 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1af00ffe80 msgr2=0x7f1af0197bc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:19.628 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.625+0000 7f1aee7fc640 1 --2- 172.21.15.99:0/2550428639 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1af00ffe80 0x7f1af0197bc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.628 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.625+0000 7f1aee7fc640 1 -- 172.21.15.99:0/2550428639 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1af0101a20 msgr2=0x7f1af019f170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:19.628 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.625+0000 7f1aee7fc640 1 --2- 172.21.15.99:0/2550428639 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1af0101a20 0x7f1af019f170 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.628 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.625+0000 7f1aee7fc640 1 -- 172.21.15.99:0/2550428639 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f1af019f7e0 con 0x7f1af0101060 2024-03-19T01:40:19.628 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.625+0000 7f1aee7fc640 1 --2- 172.21.15.99:0/2550428639 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1af0101060 0x7f1af0198100 secure :-1 s=READY pgs=293 cs=0 l=1 rev1=1 crypto rx=0x7f1ae000e9f0 tx=0x7f1ae000edf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:19.628 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.625+0000 7f1ad7fff640 1 -- 172.21.15.99:0/2550428639 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1ae000cd80 con 0x7f1af0101060 2024-03-19T01:40:19.628 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.625+0000 7f1af5737640 1 -- 172.21.15.99:0/2550428639 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f1af019fad0 con 0x7f1af0101060 2024-03-19T01:40:19.628 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.625+0000 7f1ad7fff640 1 -- 172.21.15.99:0/2550428639 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f1ae0005500 con 0x7f1af0101060 2024-03-19T01:40:19.628 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.625+0000 7f1ad7fff640 1 -- 172.21.15.99:0/2550428639 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1ae0010690 con 0x7f1af0101060 2024-03-19T01:40:19.628 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.625+0000 7f1af5737640 1 -- 172.21.15.99:0/2550428639 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f1af019ffd0 con 0x7f1af0101060 2024-03-19T01:40:19.635 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.627+0000 7f1ad7fff640 1 -- 172.21.15.99:0/2550428639 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 18) ==== 137290+0+0 (secure 0 0 0) 0x7f1ae00046c0 con 0x7f1af0101060 2024-03-19T01:40:19.635 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.628+0000 7f1af83fb740 1 Processor -- start 2024-03-19T01:40:19.635 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.628+0000 7f1af83fb740 1 -- start start 2024-03-19T01:40:19.635 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.628+0000 7f1af83fb740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55bcc484aff0 0x55bcc484b410 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.635 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.628+0000 7f1af83fb740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55bcc484b9e0 0x55bcc48504e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.635 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.628+0000 7f1af83fb740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55bcc4850b50 0x55bcc4852fe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.636 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.628+0000 7f1af83fb740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55bcc47afde0 con 0x55bcc484b9e0 2024-03-19T01:40:19.636 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.628+0000 7f1af83fb740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55bcc4758c80 con 0x55bcc484aff0 2024-03-19T01:40:19.636 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.628+0000 7f1af83fb740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55bcc47abe80 con 0x55bcc4850b50 2024-03-19T01:40:19.636 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.628+0000 7f1aeeffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55bcc484b9e0 0x55bcc48504e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.636 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.628+0000 7f1aeeffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55bcc484b9e0 0x55bcc48504e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:52956/0 (socket says 172.21.15.99:52956) 2024-03-19T01:40:19.636 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.628+0000 7f1aeeffd640 1 -- 172.21.15.99:0/1894131353 learned_addr learned my addr 172.21.15.99:0/1894131353 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:19.636 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.628+0000 7f1aee7fc640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55bcc4850b50 0x55bcc4852fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.636 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.628+0000 7f1aef7fe640 1 --2- 172.21.15.99:0/1894131353 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55bcc484aff0 0x55bcc484b410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.636 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.628+0000 7f1ad7fff640 1 --2- 172.21.15.99:0/2550428639 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f1ad80a2800 0x7f1ad80a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.636 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.628+0000 7f1aef7fe640 1 --2- 172.21.15.99:0/2550428639 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f1ad80a2800 0x7f1ad80a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.636 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.628+0000 7f1aeeffd640 1 -- 172.21.15.99:0/1894131353 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55bcc4850b50 msgr2=0x55bcc4852fe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:40:19.636 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.628+0000 7f1aeeffd640 1 --2- 172.21.15.99:0/1894131353 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55bcc4850b50 0x55bcc4852fe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.636 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.628+0000 7f1aeeffd640 1 -- 172.21.15.99:0/1894131353 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55bcc484aff0 msgr2=0x55bcc484b410 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:40:19.636 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.628+0000 7f1aeeffd640 1 --2- 172.21.15.99:0/1894131353 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55bcc484aff0 0x55bcc484b410 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.637 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.628+0000 7f1aeeffd640 1 -- 172.21.15.99:0/1894131353 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55bcc476afb0 con 0x55bcc484b9e0 2024-03-19T01:40:19.637 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.628+0000 7f1aef7fe640 1 --2- 172.21.15.99:0/1894131353 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55bcc484aff0 0x55bcc484b410 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:40:19.637 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.628+0000 7f1ad7fff640 1 -- 172.21.15.99:0/2550428639 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f1ae00cee80 con 0x7f1af0101060 2024-03-19T01:40:19.637 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.628+0000 7f1aeeffd640 1 --2- 172.21.15.99:0/1894131353 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55bcc484b9e0 0x55bcc48504e0 secure :-1 s=READY pgs=294 cs=0 l=1 rev1=1 crypto rx=0x7f1ae4009900 tx=0x7f1ae4009560 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1ddb3bfdd112b1b3 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:19.637 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.629+0000 7f1aef7fe640 1 --2- 172.21.15.99:0/2550428639 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f1ad80a2800 0x7f1ad80a4cc0 secure :-1 s=READY pgs=220 cs=0 l=1 rev1=1 crypto rx=0x55bcc484c240 tx=0x7f1ae8004ab0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:19.637 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.629+0000 7f1ad57fa640 1 -- 172.21.15.99:0/1894131353 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1ae402cb80 con 0x55bcc484b9e0 2024-03-19T01:40:19.637 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.629+0000 7f1ad57fa640 1 -- 172.21.15.99:0/1894131353 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f1ae4005040 con 0x55bcc484b9e0 2024-03-19T01:40:19.637 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.629+0000 7f1ad57fa640 1 -- 172.21.15.99:0/1894131353 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1ae4004920 con 0x55bcc484b9e0 2024-03-19T01:40:19.637 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.629+0000 7f1af83fb740 1 -- 172.21.15.99:0/1894131353 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55bcc484b9e0 msgr2=0x55bcc48504e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:19.637 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.629+0000 7f1af83fb740 1 --2- 172.21.15.99:0/1894131353 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55bcc484b9e0 0x55bcc48504e0 secure :-1 s=READY pgs=294 cs=0 l=1 rev1=1 crypto rx=0x7f1ae4009900 tx=0x7f1ae4009560 comp rx=0 tx=0).stop 2024-03-19T01:40:19.637 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.629+0000 7f1af83fb740 1 -- 172.21.15.99:0/1894131353 shutdown_connections 2024-03-19T01:40:19.637 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.629+0000 7f1af83fb740 1 --2- 172.21.15.99:0/1894131353 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55bcc4850b50 0x55bcc4852fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.637 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.629+0000 7f1af83fb740 1 --2- 172.21.15.99:0/1894131353 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55bcc484b9e0 0x55bcc48504e0 unknown :-1 s=CLOSED pgs=294 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.637 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.629+0000 7f1af83fb740 1 --2- 172.21.15.99:0/1894131353 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55bcc484aff0 0x55bcc484b410 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.637 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.629+0000 7f1af83fb740 1 -- 172.21.15.99:0/1894131353 >> 172.21.15.99:0/1894131353 conn(0x55bcc4846560 msgr2=0x55bcc4846960 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:19.638 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.629+0000 7f1af83fb740 1 -- 172.21.15.99:0/1894131353 shutdown_connections 2024-03-19T01:40:19.638 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.629+0000 7f1af83fb740 1 -- 172.21.15.99:0/1894131353 wait complete. 2024-03-19T01:40:19.638 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.630+0000 7f1af83fb740 1 -- start start 2024-03-19T01:40:19.638 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.630+0000 7f1af83fb740 1 Processor -- start 2024-03-19T01:40:19.638 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.630+0000 7f1af83fb740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55bcc484aff0 0x55bcc4859900 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.638 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.630+0000 7f1af83fb740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55bcc484b9e0 0x55bcc4859e40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.638 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.630+0000 7f1af83fb740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55bcc4850b50 0x55bcc4860e80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.638 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.630+0000 7f1af83fb740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55bcc4853aa0 con 0x55bcc484b9e0 2024-03-19T01:40:19.638 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.630+0000 7f1af83fb740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55bcc476bfc0 con 0x55bcc484aff0 2024-03-19T01:40:19.638 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.630+0000 7f1af83fb740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55bcc4853c20 con 0x55bcc4850b50 2024-03-19T01:40:19.638 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.630+0000 7f1af83fb740 10 client.0 _ll_register_callbacks cb 0x55bcc47a7a20 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:40:19.638 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.630+0000 7f1aee7fc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55bcc484b9e0 0x55bcc4859e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.638 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.630+0000 7f1aee7fc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55bcc484b9e0 0x55bcc4859e40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:52966/0 (socket says 172.21.15.99:52966) 2024-03-19T01:40:19.639 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.630+0000 7f1aee7fc640 1 -- 172.21.15.99:0/1995440492 learned_addr learned my addr 172.21.15.99:0/1995440492 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:19.639 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.630+0000 7f1aeeffd640 1 --2- 172.21.15.99:0/1995440492 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55bcc484aff0 0x55bcc4859900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.639 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.630+0000 7f1aef7fe640 1 --2- 172.21.15.99:0/1995440492 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55bcc4850b50 0x55bcc4860e80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.639 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.630+0000 7f1aee7fc640 1 -- 172.21.15.99:0/1995440492 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55bcc4850b50 msgr2=0x55bcc4860e80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:40:19.639 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.630+0000 7f1aee7fc640 1 --2- 172.21.15.99:0/1995440492 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55bcc4850b50 0x55bcc4860e80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.639 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.630+0000 7f1aee7fc640 1 -- 172.21.15.99:0/1995440492 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55bcc484aff0 msgr2=0x55bcc4859900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:40:19.639 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.630+0000 7f1aee7fc640 1 --2- 172.21.15.99:0/1995440492 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55bcc484aff0 0x55bcc4859900 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.639 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.630+0000 7f1aee7fc640 1 -- 172.21.15.99:0/1995440492 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55bcc44c42d0 con 0x55bcc484b9e0 2024-03-19T01:40:19.639 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.630+0000 7f1aeeffd640 1 --2- 172.21.15.99:0/1995440492 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55bcc484aff0 0x55bcc4859900 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:40:19.639 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.630+0000 7f1aee7fc640 1 --2- 172.21.15.99:0/1995440492 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55bcc484b9e0 0x55bcc4859e40 secure :-1 s=READY pgs=295 cs=0 l=1 rev1=1 crypto rx=0x7f1ae00cccb0 tx=0x7f1ae0009310 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e5bd5d734f929a08 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:19.639 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.630+0000 7f1ad2ff5640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:40:19.639 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.631+0000 7f1ad2ff5640 1 -- 172.21.15.99:0/1995440492 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1ae009b320 con 0x55bcc484b9e0 2024-03-19T01:40:19.639 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.631+0000 7f1ad2ff5640 1 -- 172.21.15.99:0/1995440492 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f1ae00860b0 con 0x55bcc484b9e0 2024-03-19T01:40:19.639 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.631+0000 7f1af83fb740 1 -- 172.21.15.99:0/1995440492 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f1ae4005040 con 0x55bcc484b9e0 2024-03-19T01:40:19.639 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.631+0000 7f1ad2ff5640 1 -- 172.21.15.99:0/1995440492 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=20) ==== 24+0+0 (secure 0 0 0) 0x7f1ae009b140 con 0x55bcc484b9e0 2024-03-19T01:40:19.639 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.631+0000 7f1af83fb740 10 client.15084 fetch_fsmap learned FSMap version 20 2024-03-19T01:40:19.640 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.631+0000 7f1af83fb740 1 -- 172.21.15.99:0/1995440492 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=20,monmap=4+}) -- 0x55bcc458e6b0 con 0x55bcc484b9e0 2024-03-19T01:40:19.640 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.631+0000 7f1ad2ff5640 1 -- 172.21.15.99:0/1995440492 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f1ae4005040 con 0x55bcc484b9e0 2024-03-19T01:40:19.640 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.631+0000 7f1ad2ff5640 1 -- 172.21.15.99:0/1995440492 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 20) ==== 8338+0+0 (secure 0 0 0) 0x7f1ae009ac30 con 0x55bcc484b9e0 2024-03-19T01:40:19.640 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.631+0000 7f1af83fb740 10 client.15084 fetch_fsmap finished waiting for FSMap version 20 2024-03-19T01:40:19.640 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.631+0000 7f1af83fb740 10 client.15084 resolve_mds: resolved name 'c' to daemon mds.c 2024-03-19T01:40:19.640 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.631+0000 7f1af83fb740 20 client.15084 populate_metadata read hostname 'smithi099' 2024-03-19T01:40:19.640 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.631+0000 7f1af83fb740 1 --2- 172.21.15.99:0/1995440492 >> [v2:172.21.15.115:6836/1368988191,v1:172.21.15.115:6837/1368988191] conn(0x55bcc4866780 0x55bcc4867bf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.640 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.631+0000 7f1af83fb740 4 client.15084 mds_command: new command op to 24472 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:40:19.640 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.631+0000 7f1af83fb740 1 -- 172.21.15.99:0/1995440492 --> [v2:172.21.15.115:6836/1368988191,v1:172.21.15.115:6837/1368988191] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55bcc47bd320 con 0x55bcc4866780 2024-03-19T01:40:19.640 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.631+0000 7f1aeeffd640 1 --2- 172.21.15.99:0/1995440492 >> [v2:172.21.15.115:6836/1368988191,v1:172.21.15.115:6837/1368988191] conn(0x55bcc4866780 0x55bcc4867bf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.640 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.632+0000 7f1ad2ff5640 10 client.15084 ms_handle_connect on v2:172.21.15.115:6836/1368988191 2024-03-19T01:40:19.640 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.632+0000 7f1aeeffd640 1 --2- 172.21.15.99:0/1995440492 >> [v2:172.21.15.115:6836/1368988191,v1:172.21.15.115:6837/1368988191] conn(0x55bcc4866780 0x55bcc4867bf0 crc :-1 s=READY pgs=15 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=8de26423dcb1fd3 server_cookie=6176dc4eff19c80e in_seq=0 out_seq=0 2024-03-19T01:40:19.641 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.632+0000 7f1ad2ff5640 1 -- 172.21.15.99:0/1995440492 <== mds.2 v2:172.21.15.115:6836/1368988191 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x7f1ae40509c0 con 0x55bcc4866780 2024-03-19T01:40:19.641 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.632+0000 7f1ad2ff5640 10 client.15084 handle_command_reply: tid=0 2024-03-19T01:40:19.641 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.632+0000 7f1af83fb740 1 client.15084 shutdown 2024-03-19T01:40:19.641 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.632+0000 7f1af83fb740 1 -- 172.21.15.99:0/1995440492 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55bcc484b9e0 msgr2=0x55bcc4859e40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:19.641 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.632+0000 7f1af83fb740 1 --2- 172.21.15.99:0/1995440492 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55bcc484b9e0 0x55bcc4859e40 secure :-1 s=READY pgs=295 cs=0 l=1 rev1=1 crypto rx=0x7f1ae00cccb0 tx=0x7f1ae0009310 comp rx=0 tx=0).stop 2024-03-19T01:40:19.641 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.632+0000 7f1af83fb740 1 -- 172.21.15.99:0/1995440492 shutdown_connections 2024-03-19T01:40:19.641 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.632+0000 7f1af83fb740 1 --2- 172.21.15.99:0/1995440492 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55bcc4850b50 0x55bcc4860e80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.641 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.632+0000 7f1af83fb740 1 --2- 172.21.15.99:0/1995440492 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55bcc484b9e0 0x55bcc4859e40 unknown :-1 s=CLOSED pgs=295 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.641 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.632+0000 7f1af83fb740 1 --2- 172.21.15.99:0/1995440492 >> [v2:172.21.15.115:6836/1368988191,v1:172.21.15.115:6837/1368988191] conn(0x55bcc4866780 0x55bcc4867bf0 crc :-1 s=READY pgs=15 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.641 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.633+0000 7f1af83fb740 1 --2- 172.21.15.99:0/1995440492 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55bcc484aff0 0x55bcc4859900 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.641 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.633+0000 7f1ad2ff5640 0 client.15084 ms_handle_reset on v2:172.21.15.115:6836/1368988191 2024-03-19T01:40:19.642 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.633+0000 7f1af83fb740 1 -- 172.21.15.99:0/1995440492 >> 172.21.15.99:0/1995440492 conn(0x55bcc4846560 msgr2=0x55bcc48457e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:19.642 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.633+0000 7f1af83fb740 1 -- 172.21.15.99:0/1995440492 shutdown_connections 2024-03-19T01:40:19.642 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.633+0000 7f1af83fb740 1 -- 172.21.15.99:0/1995440492 wait complete. 2024-03-19T01:40:19.642 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.633+0000 7f1af83fb740 20 client.15084 trim_cache size 0 max 16384 2024-03-19T01:40:19.658 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.650+0000 7f1af83fb740 1 Processor -- start 2024-03-19T01:40:19.658 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.650+0000 7f1af83fb740 1 -- start start 2024-03-19T01:40:19.658 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.650+0000 7f1af83fb740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55bcc484aff0 0x55bcc4841670 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.658 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.650+0000 7f1af83fb740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55bcc484b9e0 0x55bcc4860e40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.658 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.650+0000 7f1af83fb740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55bcc4850b50 0x55bcc4859da0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.658 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.651+0000 7f1af83fb740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55bcc4865f30 con 0x55bcc484b9e0 2024-03-19T01:40:19.658 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.651+0000 7f1af83fb740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55bcc4866600 con 0x55bcc4850b50 2024-03-19T01:40:19.658 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.651+0000 7f1af83fb740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55bcc4866cd0 con 0x55bcc484aff0 2024-03-19T01:40:19.658 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.651+0000 7f1aee7fc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55bcc484b9e0 0x55bcc4860e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.658 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.651+0000 7f1aee7fc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55bcc484b9e0 0x55bcc4860e40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:52982/0 (socket says 172.21.15.99:52982) 2024-03-19T01:40:19.658 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.651+0000 7f1aee7fc640 1 -- 172.21.15.99:0/3365143559 learned_addr learned my addr 172.21.15.99:0/3365143559 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:19.659 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.651+0000 7f1aef7fe640 1 --2- 172.21.15.99:0/3365143559 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55bcc4850b50 0x55bcc4859da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.659 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.651+0000 7f1aeeffd640 1 --2- 172.21.15.99:0/3365143559 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55bcc484aff0 0x55bcc4841670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.659 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.651+0000 7f1aee7fc640 1 -- 172.21.15.99:0/3365143559 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55bcc484aff0 msgr2=0x55bcc4841670 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:40:19.659 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.651+0000 7f1aee7fc640 1 --2- 172.21.15.99:0/3365143559 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55bcc484aff0 0x55bcc4841670 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.659 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.651+0000 7f1aee7fc640 1 -- 172.21.15.99:0/3365143559 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55bcc4850b50 msgr2=0x55bcc4859da0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:40:19.659 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.651+0000 7f1aee7fc640 1 --2- 172.21.15.99:0/3365143559 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55bcc4850b50 0x55bcc4859da0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.660 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.651+0000 7f1aee7fc640 1 -- 172.21.15.99:0/3365143559 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55bcc4853de0 con 0x55bcc484b9e0 2024-03-19T01:40:19.660 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.651+0000 7f1aee7fc640 1 --2- 172.21.15.99:0/3365143559 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55bcc484b9e0 0x55bcc4860e40 secure :-1 s=READY pgs=296 cs=0 l=1 rev1=1 crypto rx=0x7f1ae0099a90 tx=0x7f1ae0099cf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4deccb9332a06805 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:19.660 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.651+0000 7f1ad4ff9640 1 -- 172.21.15.99:0/3365143559 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1ae009b320 con 0x55bcc484b9e0 2024-03-19T01:40:19.660 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.651+0000 7f1ad4ff9640 1 -- 172.21.15.99:0/3365143559 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f1ae00a3af0 con 0x55bcc484b9e0 2024-03-19T01:40:19.660 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.651+0000 7f1ad4ff9640 1 -- 172.21.15.99:0/3365143559 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1ae009b7e0 con 0x55bcc484b9e0 2024-03-19T01:40:19.660 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.652+0000 7f1af83fb740 1 -- 172.21.15.99:0/3365143559 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55bcc484b9e0 msgr2=0x55bcc4860e40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:19.660 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.652+0000 7f1af83fb740 1 --2- 172.21.15.99:0/3365143559 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55bcc484b9e0 0x55bcc4860e40 secure :-1 s=READY pgs=296 cs=0 l=1 rev1=1 crypto rx=0x7f1ae0099a90 tx=0x7f1ae0099cf0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.660 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.652+0000 7f1af83fb740 1 -- 172.21.15.99:0/3365143559 shutdown_connections 2024-03-19T01:40:19.660 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.652+0000 7f1af83fb740 1 --2- 172.21.15.99:0/3365143559 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55bcc4850b50 0x55bcc4859da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.660 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.652+0000 7f1af83fb740 1 --2- 172.21.15.99:0/3365143559 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55bcc484b9e0 0x55bcc4860e40 unknown :-1 s=CLOSED pgs=296 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.660 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.652+0000 7f1af83fb740 1 --2- 172.21.15.99:0/3365143559 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55bcc484aff0 0x55bcc4841670 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.660 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.652+0000 7f1af83fb740 1 -- 172.21.15.99:0/3365143559 >> 172.21.15.99:0/3365143559 conn(0x55bcc4846560 msgr2=0x55bcc48497c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:19.660 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.652+0000 7f1af83fb740 1 -- 172.21.15.99:0/3365143559 shutdown_connections 2024-03-19T01:40:19.661 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.652+0000 7f1af83fb740 1 -- 172.21.15.99:0/3365143559 wait complete. 2024-03-19T01:40:19.661 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.652+0000 7f1af83fb740 1 -- start start 2024-03-19T01:40:19.661 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.652+0000 7f1af83fb740 1 Processor -- start 2024-03-19T01:40:19.661 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.652+0000 7f1af83fb740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55bcc484aff0 0x55bcc485cae0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.661 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.652+0000 7f1af83fb740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55bcc484b9e0 0x55bcc485d020 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.661 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.652+0000 7f1af83fb740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55bcc4850b50 0x55bcc485ff80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.661 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.652+0000 7f1af83fb740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55bcc4867a70 con 0x55bcc484b9e0 2024-03-19T01:40:19.661 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.652+0000 7f1af83fb740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55bcc48673a0 con 0x55bcc484aff0 2024-03-19T01:40:19.661 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.652+0000 7f1af83fb740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55bcc4868140 con 0x55bcc4850b50 2024-03-19T01:40:19.661 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.652+0000 7f1af83fb740 10 client.0 _ll_register_callbacks cb 0x7f1ae0098050 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:40:19.661 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.652+0000 7f1aee7fc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55bcc484b9e0 0x55bcc485d020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.661 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.652+0000 7f1aee7fc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55bcc484b9e0 0x55bcc485d020 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:52998/0 (socket says 172.21.15.99:52998) 2024-03-19T01:40:19.662 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.652+0000 7f1aee7fc640 1 -- 172.21.15.99:0/184454319 learned_addr learned my addr 172.21.15.99:0/184454319 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:19.662 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.652+0000 7f1aeeffd640 1 --2- 172.21.15.99:0/184454319 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55bcc484aff0 0x55bcc485cae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.662 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.652+0000 7f1aef7fe640 1 --2- 172.21.15.99:0/184454319 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55bcc4850b50 0x55bcc485ff80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.662 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.653+0000 7f1aee7fc640 1 -- 172.21.15.99:0/184454319 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55bcc4850b50 msgr2=0x55bcc485ff80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:40:19.662 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.653+0000 7f1aee7fc640 1 --2- 172.21.15.99:0/184454319 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55bcc4850b50 0x55bcc485ff80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.662 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.653+0000 7f1aee7fc640 1 -- 172.21.15.99:0/184454319 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55bcc484aff0 msgr2=0x55bcc485cae0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:40:19.662 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.653+0000 7f1aee7fc640 1 --2- 172.21.15.99:0/184454319 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55bcc484aff0 0x55bcc485cae0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.662 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.653+0000 7f1aee7fc640 1 -- 172.21.15.99:0/184454319 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55bcc4860570 con 0x55bcc484b9e0 2024-03-19T01:40:19.662 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.653+0000 7f1aeeffd640 1 --2- 172.21.15.99:0/184454319 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55bcc484aff0 0x55bcc485cae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:40:19.662 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.653+0000 7f1aee7fc640 1 --2- 172.21.15.99:0/184454319 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55bcc484b9e0 0x55bcc485d020 secure :-1 s=READY pgs=297 cs=0 l=1 rev1=1 crypto rx=0x7f1ae00a6040 tx=0x7f1ae0086360 comp rx=0 tx=0).ready entity=mon.0 client_cookie=68e9e73804f0320c server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:19.662 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.653+0000 7f1ad2ff5640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:40:19.662 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.653+0000 7f1ad2ff5640 1 -- 172.21.15.99:0/184454319 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1ae00ae030 con 0x55bcc484b9e0 2024-03-19T01:40:19.663 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.653+0000 7f1ad2ff5640 1 -- 172.21.15.99:0/184454319 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f1ae00cce00 con 0x55bcc484b9e0 2024-03-19T01:40:19.663 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.653+0000 7f1af83fb740 1 -- 172.21.15.99:0/184454319 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f1ae00a3af0 con 0x55bcc484b9e0 2024-03-19T01:40:19.663 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.653+0000 7f1ad2ff5640 1 -- 172.21.15.99:0/184454319 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=20) ==== 24+0+0 (secure 0 0 0) 0x7f1ae009b320 con 0x55bcc484b9e0 2024-03-19T01:40:19.663 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.653+0000 7f1af83fb740 10 client.15090 fetch_fsmap learned FSMap version 20 2024-03-19T01:40:19.663 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.653+0000 7f1af83fb740 1 -- 172.21.15.99:0/184454319 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=20,monmap=4+}) -- 0x55bcc48cc9e0 con 0x55bcc484b9e0 2024-03-19T01:40:19.663 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.653+0000 7f1ad2ff5640 1 -- 172.21.15.99:0/184454319 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f1ae00a3af0 con 0x55bcc484b9e0 2024-03-19T01:40:19.663 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.653+0000 7f1ad2ff5640 1 -- 172.21.15.99:0/184454319 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 20) ==== 8338+0+0 (secure 0 0 0) 0x7f1ae009a6a0 con 0x55bcc484b9e0 2024-03-19T01:40:19.663 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.653+0000 7f1af83fb740 10 client.15090 fetch_fsmap finished waiting for FSMap version 20 2024-03-19T01:40:19.663 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.653+0000 7f1af83fb740 10 client.15090 resolve_mds: resolved name 'c' to daemon mds.c 2024-03-19T01:40:19.663 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.653+0000 7f1af83fb740 20 client.15090 populate_metadata read hostname 'smithi099' 2024-03-19T01:40:19.663 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.653+0000 7f1af83fb740 1 --2- 172.21.15.99:0/184454319 >> [v2:172.21.15.115:6836/1368988191,v1:172.21.15.115:6837/1368988191] conn(0x55bcc48d11b0 0x55bcc48cdcc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.663 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.653+0000 7f1af83fb740 4 client.15090 mds_command: new command op to 24472 tid=0 multi_id=0 [{"prefix": "status"}] 2024-03-19T01:40:19.663 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.653+0000 7f1af83fb740 1 -- 172.21.15.99:0/184454319 --> [v2:172.21.15.115:6836/1368988191,v1:172.21.15.115:6837/1368988191] -- command(tid 0: {"prefix": "status"}) -- 0x55bcc47b7670 con 0x55bcc48d11b0 2024-03-19T01:40:19.663 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.653+0000 7f1aeeffd640 1 --2- 172.21.15.99:0/184454319 >> [v2:172.21.15.115:6836/1368988191,v1:172.21.15.115:6837/1368988191] conn(0x55bcc48d11b0 0x55bcc48cdcc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.664 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.654+0000 7f1aeeffd640 1 --2- 172.21.15.99:0/184454319 >> [v2:172.21.15.115:6836/1368988191,v1:172.21.15.115:6837/1368988191] conn(0x55bcc48d11b0 0x55bcc48cdcc0 crc :-1 s=READY pgs=16 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=c94fd3988a5d9291 server_cookie=25ac677f9cccd34c in_seq=0 out_seq=0 2024-03-19T01:40:19.664 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.654+0000 7f1ad2ff5640 10 client.15090 ms_handle_connect on v2:172.21.15.115:6836/1368988191 2024-03-19T01:40:19.664 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.654+0000 7f1ad2ff5640 1 -- 172.21.15.99:0/184454319 <== mds.2 v2:172.21.15.115:6836/1368988191 1 ==== command_reply(tid 0: 0 ) ==== 8+0+327 (crc 0 0 0) 0x55bcc47b7670 con 0x55bcc48d11b0 2024-03-19T01:40:19.664 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.654+0000 7f1ad2ff5640 10 client.15090 handle_command_reply: tid=0 2024-03-19T01:40:19.664 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.654+0000 7f1af83fb740 1 client.15090 shutdown 2024-03-19T01:40:19.664 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.654+0000 7f1af83fb740 1 -- 172.21.15.99:0/184454319 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55bcc484b9e0 msgr2=0x55bcc485d020 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:19.664 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.654+0000 7f1af83fb740 1 --2- 172.21.15.99:0/184454319 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55bcc484b9e0 0x55bcc485d020 secure :-1 s=READY pgs=297 cs=0 l=1 rev1=1 crypto rx=0x7f1ae00a6040 tx=0x7f1ae0086360 comp rx=0 tx=0).stop 2024-03-19T01:40:19.664 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.654+0000 7f1af83fb740 1 -- 172.21.15.99:0/184454319 shutdown_connections 2024-03-19T01:40:19.664 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.654+0000 7f1af83fb740 1 --2- 172.21.15.99:0/184454319 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55bcc4850b50 0x55bcc485ff80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.664 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.654+0000 7f1af83fb740 1 --2- 172.21.15.99:0/184454319 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55bcc484b9e0 0x55bcc485d020 unknown :-1 s=CLOSED pgs=297 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.664 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.654+0000 7f1af83fb740 1 --2- 172.21.15.99:0/184454319 >> [v2:172.21.15.115:6836/1368988191,v1:172.21.15.115:6837/1368988191] conn(0x55bcc48d11b0 0x55bcc48cdcc0 crc :-1 s=READY pgs=16 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.664 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.654+0000 7f1af83fb740 1 --2- 172.21.15.99:0/184454319 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55bcc484aff0 0x55bcc485cae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.665 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.654+0000 7f1ad2ff5640 0 client.15090 ms_handle_reset on v2:172.21.15.115:6836/1368988191 2024-03-19T01:40:19.665 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.654+0000 7f1af83fb740 1 -- 172.21.15.99:0/184454319 >> 172.21.15.99:0/184454319 conn(0x55bcc4846560 msgr2=0x55bcc484a050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:19.665 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.654+0000 7f1af83fb740 1 -- 172.21.15.99:0/184454319 shutdown_connections 2024-03-19T01:40:19.665 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.654+0000 7f1af83fb740 1 -- 172.21.15.99:0/184454319 wait complete. 2024-03-19T01:40:19.665 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.654+0000 7f1af83fb740 20 client.15090 trim_cache size 0 max 16384 2024-03-19T01:40:19.665 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2024-03-19T01:40:19.665 INFO:teuthology.orchestra.run.smithi099.stdout: "cluster_fsid": "438fb69c-e590-11ee-95c9-87774f69a715", 2024-03-19T01:40:19.665 INFO:teuthology.orchestra.run.smithi099.stdout: "whoami": 2, 2024-03-19T01:40:19.665 INFO:teuthology.orchestra.run.smithi099.stdout: "id": 24472, 2024-03-19T01:40:19.666 INFO:teuthology.orchestra.run.smithi099.stdout: "want_state": "up:active", 2024-03-19T01:40:19.666 INFO:teuthology.orchestra.run.smithi099.stdout: "state": "up:active", 2024-03-19T01:40:19.666 INFO:teuthology.orchestra.run.smithi099.stdout: "fs_name": "cephfs", 2024-03-19T01:40:19.666 INFO:teuthology.orchestra.run.smithi099.stdout: "rank_uptime": 79.846065483999993, 2024-03-19T01:40:19.666 INFO:teuthology.orchestra.run.smithi099.stdout: "mdsmap_epoch": 20, 2024-03-19T01:40:19.666 INFO:teuthology.orchestra.run.smithi099.stdout: "osdmap_epoch": 91, 2024-03-19T01:40:19.666 INFO:teuthology.orchestra.run.smithi099.stdout: "osdmap_epoch_barrier": 87, 2024-03-19T01:40:19.666 INFO:teuthology.orchestra.run.smithi099.stdout: "uptime": 102.982756358 2024-03-19T01:40:19.666 INFO:teuthology.orchestra.run.smithi099.stdout:} 2024-03-19T01:40:19.667 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.655+0000 7f1af5737640 1 -- 172.21.15.99:0/2550428639 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f1ad80a2800 msgr2=0x7f1ad80a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:19.667 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.655+0000 7f1af5737640 1 --2- 172.21.15.99:0/2550428639 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f1ad80a2800 0x7f1ad80a4cc0 secure :-1 s=READY pgs=220 cs=0 l=1 rev1=1 crypto rx=0x55bcc484c240 tx=0x7f1ae8004ab0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.668 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.655+0000 7f1af5737640 1 -- 172.21.15.99:0/2550428639 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1af0101060 msgr2=0x7f1af0198100 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:19.668 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.655+0000 7f1af5737640 1 --2- 172.21.15.99:0/2550428639 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1af0101060 0x7f1af0198100 secure :-1 s=READY pgs=293 cs=0 l=1 rev1=1 crypto rx=0x7f1ae000e9f0 tx=0x7f1ae000edf0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.668 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.655+0000 7f1af5737640 1 -- 172.21.15.99:0/2550428639 shutdown_connections 2024-03-19T01:40:19.668 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.655+0000 7f1af5737640 1 --2- 172.21.15.99:0/2550428639 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f1ad80a2800 0x7f1ad80a4cc0 unknown :-1 s=CLOSED pgs=220 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.668 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.655+0000 7f1af5737640 1 --2- 172.21.15.99:0/2550428639 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1af0101a20 0x7f1af019f170 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.668 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.655+0000 7f1af5737640 1 --2- 172.21.15.99:0/2550428639 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1af0101060 0x7f1af0198100 unknown :-1 s=CLOSED pgs=293 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.668 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.655+0000 7f1af5737640 1 --2- 172.21.15.99:0/2550428639 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1af00ffe80 0x7f1af0197bc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.668 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.655+0000 7f1af5737640 1 -- 172.21.15.99:0/2550428639 >> 172.21.15.99:0/2550428639 conn(0x7f1af00fb420 msgr2=0x7f1af0104490 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:19.668 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.655+0000 7f1af5737640 1 -- 172.21.15.99:0/2550428639 shutdown_connections 2024-03-19T01:40:19.668 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.655+0000 7f1af5737640 1 -- 172.21.15.99:0/2550428639 wait complete. 2024-03-19T01:40:19.669 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.b status 2024-03-19T01:40:19.773 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.770+0000 7f4cf4a27640 1 -- 172.21.15.99:0/3253859270 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4cf01022e0 msgr2=0x7f4cf0108b70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:19.773 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.770+0000 7f4cf4a27640 1 --2- 172.21.15.99:0/3253859270 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4cf01022e0 0x7f4cf0108b70 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x7f4ce8009be0 tx=0x7f4ce802bef0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.773 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.770+0000 7f4cf4a27640 1 -- 172.21.15.99:0/3253859270 shutdown_connections 2024-03-19T01:40:19.773 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.770+0000 7f4cf4a27640 1 --2- 172.21.15.99:0/3253859270 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4cf01022e0 0x7f4cf0108b70 unknown :-1 s=CLOSED pgs=69 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.773 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.770+0000 7f4cf4a27640 1 --2- 172.21.15.99:0/3253859270 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4cf0101920 0x7f4cf0101da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.773 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.770+0000 7f4cf4a27640 1 --2- 172.21.15.99:0/3253859270 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4cf0100730 0x7f4cf0100b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.773 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.770+0000 7f4cf4a27640 1 -- 172.21.15.99:0/3253859270 >> 172.21.15.99:0/3253859270 conn(0x7f4cf00fbcc0 msgr2=0x7f4cf00fe0e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:19.773 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.770+0000 7f4cf4a27640 1 -- 172.21.15.99:0/3253859270 shutdown_connections 2024-03-19T01:40:19.773 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.770+0000 7f4cf4a27640 1 -- 172.21.15.99:0/3253859270 wait complete. 2024-03-19T01:40:19.773 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.771+0000 7f4cf4a27640 1 Processor -- start 2024-03-19T01:40:19.773 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.771+0000 7f4cf4a27640 1 -- start start 2024-03-19T01:40:19.774 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.771+0000 7f4cf4a27640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4cf0100730 0x7f4cf010b0b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.774 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.771+0000 7f4cf4a27640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4cf0101920 0x7f4cf01096f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.774 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.771+0000 7f4cf4a27640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4cf01022e0 0x7f4cf0109c30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.774 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.771+0000 7f4cf4a27640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f4cf010d4d0 con 0x7f4cf0101920 2024-03-19T01:40:19.774 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.771+0000 7f4cf4a27640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f4cf010d350 con 0x7f4cf01022e0 2024-03-19T01:40:19.774 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.771+0000 7f4cf4a27640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f4cf010d650 con 0x7f4cf0100730 2024-03-19T01:40:19.774 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.771+0000 7f4cedd9b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4cf0101920 0x7f4cf01096f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.774 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.771+0000 7f4cedd9b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4cf0101920 0x7f4cf01096f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:53008/0 (socket says 172.21.15.99:53008) 2024-03-19T01:40:19.774 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.771+0000 7f4cedd9b640 1 -- 172.21.15.99:0/39474516 learned_addr learned my addr 172.21.15.99:0/39474516 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:19.774 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.772+0000 7f4ceed9d640 1 --2- 172.21.15.99:0/39474516 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4cf01022e0 0x7f4cf0109c30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.774 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.772+0000 7f4cee59c640 1 --2- 172.21.15.99:0/39474516 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4cf0100730 0x7f4cf010b0b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.774 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.772+0000 7f4cedd9b640 1 -- 172.21.15.99:0/39474516 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4cf0100730 msgr2=0x7f4cf010b0b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:19.774 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.772+0000 7f4cedd9b640 1 --2- 172.21.15.99:0/39474516 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4cf0100730 0x7f4cf010b0b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.774 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.772+0000 7f4cedd9b640 1 -- 172.21.15.99:0/39474516 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4cf01022e0 msgr2=0x7f4cf0109c30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:19.775 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.772+0000 7f4cedd9b640 1 --2- 172.21.15.99:0/39474516 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4cf01022e0 0x7f4cf0109c30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.775 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.772+0000 7f4cedd9b640 1 -- 172.21.15.99:0/39474516 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f4cf010a4c0 con 0x7f4cf0101920 2024-03-19T01:40:19.775 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.772+0000 7f4cedd9b640 1 --2- 172.21.15.99:0/39474516 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4cf0101920 0x7f4cf01096f0 secure :-1 s=READY pgs=298 cs=0 l=1 rev1=1 crypto rx=0x7f4ce400e9f0 tx=0x7f4ce400edf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:19.775 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.772+0000 7f4cdf7fe640 1 -- 172.21.15.99:0/39474516 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4ce400ccc0 con 0x7f4cf0101920 2024-03-19T01:40:19.775 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.772+0000 7f4cf4a27640 1 -- 172.21.15.99:0/39474516 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f4cf01a1c20 con 0x7f4cf0101920 2024-03-19T01:40:19.775 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.772+0000 7f4cdf7fe640 1 -- 172.21.15.99:0/39474516 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f4ce4010910 con 0x7f4cf0101920 2024-03-19T01:40:19.775 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.772+0000 7f4cdf7fe640 1 -- 172.21.15.99:0/39474516 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4ce40188a0 con 0x7f4cf0101920 2024-03-19T01:40:19.775 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.772+0000 7f4cf4a27640 1 -- 172.21.15.99:0/39474516 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f4cf01a21c0 con 0x7f4cf0101920 2024-03-19T01:40:19.781 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.774+0000 7f4cdf7fe640 1 -- 172.21.15.99:0/39474516 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 18) ==== 137290+0+0 (secure 0 0 0) 0x7f4ce4010ac0 con 0x7f4cf0101920 2024-03-19T01:40:19.781 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.774+0000 7f4cf7615740 1 Processor -- start 2024-03-19T01:40:19.781 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.775+0000 7f4cf7615740 1 -- start start 2024-03-19T01:40:19.781 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.775+0000 7f4cf7615740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ddc7c21ff0 0x55ddc7c22410 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.781 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.775+0000 7f4cf7615740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ddc7c229e0 0x55ddc7c274e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.782 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.775+0000 7f4cf7615740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ddc7c27b50 0x55ddc7c29fe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.782 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.775+0000 7f4cf7615740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55ddc7b86de0 con 0x55ddc7c21ff0 2024-03-19T01:40:19.782 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.775+0000 7f4cf7615740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55ddc7b2fc80 con 0x55ddc7c229e0 2024-03-19T01:40:19.782 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.775+0000 7f4cdf7fe640 1 --2- 172.21.15.99:0/39474516 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f4cd80a2800 0x7f4cd80a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.782 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.775+0000 7f4cf7615740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55ddc7b82e80 con 0x55ddc7c27b50 2024-03-19T01:40:19.782 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.775+0000 7f4ceed9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ddc7c21ff0 0x55ddc7c22410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.782 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.775+0000 7f4cee59c640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ddc7c229e0 0x55ddc7c274e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.782 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.775+0000 7f4cdf7fe640 1 -- 172.21.15.99:0/39474516 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f4ce40ceee0 con 0x7f4cf0101920 2024-03-19T01:40:19.782 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.775+0000 7f4cee59c640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ddc7c229e0 0x55ddc7c274e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:45242/0 (socket says 172.21.15.99:45242) 2024-03-19T01:40:19.782 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.775+0000 7f4ceed9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ddc7c21ff0 0x55ddc7c22410 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:53020/0 (socket says 172.21.15.99:53020) 2024-03-19T01:40:19.782 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.775+0000 7f4cee59c640 1 -- 172.21.15.99:0/3982112239 learned_addr learned my addr 172.21.15.99:0/3982112239 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:19.782 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.775+0000 7f4cedd9b640 1 --2- 172.21.15.99:0/3982112239 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ddc7c27b50 0x55ddc7c29fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.782 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.775+0000 7f4ceed9d640 1 --2- 172.21.15.99:0/39474516 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f4cd80a2800 0x7f4cd80a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.783 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.775+0000 7f4cee59c640 1 -- 172.21.15.99:0/3982112239 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ddc7c27b50 msgr2=0x55ddc7c29fe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:40:19.783 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.775+0000 7f4cee59c640 1 --2- 172.21.15.99:0/3982112239 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ddc7c27b50 0x55ddc7c29fe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.783 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.775+0000 7f4cee59c640 1 -- 172.21.15.99:0/3982112239 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ddc7c21ff0 msgr2=0x55ddc7c22410 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:40:19.783 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.775+0000 7f4cee59c640 1 --2- 172.21.15.99:0/3982112239 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ddc7c21ff0 0x55ddc7c22410 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.783 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.775+0000 7f4cee59c640 1 -- 172.21.15.99:0/3982112239 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55ddc7b41fb0 con 0x55ddc7c229e0 2024-03-19T01:40:19.783 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.775+0000 7f4ceed9d640 1 --2- 172.21.15.99:0/39474516 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f4cd80a2800 0x7f4cd80a4cc0 secure :-1 s=READY pgs=221 cs=0 l=1 rev1=1 crypto rx=0x7f4ce8004ab0 tx=0x7f4ce8004a40 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:19.783 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.775+0000 7f4cee59c640 1 --2- 172.21.15.99:0/3982112239 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ddc7c229e0 0x55ddc7c274e0 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7f4ce0009ca0 tx=0x7f4ce0011810 comp rx=0 tx=0).ready entity=mon.2 client_cookie=5e1727188bb47f57 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:19.783 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.776+0000 7f4cdcff9640 1 -- 172.21.15.99:0/3982112239 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4ce0004230 con 0x55ddc7c229e0 2024-03-19T01:40:19.783 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.776+0000 7f4cdcff9640 1 -- 172.21.15.99:0/3982112239 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f4ce00043d0 con 0x55ddc7c229e0 2024-03-19T01:40:19.783 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.776+0000 7f4cdcff9640 1 -- 172.21.15.99:0/3982112239 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4ce0004e50 con 0x55ddc7c229e0 2024-03-19T01:40:19.783 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.776+0000 7f4cf7615740 1 -- 172.21.15.99:0/3982112239 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ddc7c229e0 msgr2=0x55ddc7c274e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:19.783 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.776+0000 7f4cf7615740 1 --2- 172.21.15.99:0/3982112239 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ddc7c229e0 0x55ddc7c274e0 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7f4ce0009ca0 tx=0x7f4ce0011810 comp rx=0 tx=0).stop 2024-03-19T01:40:19.783 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.776+0000 7f4cf7615740 1 -- 172.21.15.99:0/3982112239 shutdown_connections 2024-03-19T01:40:19.783 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.776+0000 7f4cf7615740 1 --2- 172.21.15.99:0/3982112239 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ddc7c229e0 0x55ddc7c274e0 unknown :-1 s=CLOSED pgs=70 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.784 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.776+0000 7f4cf7615740 1 --2- 172.21.15.99:0/3982112239 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ddc7c27b50 0x55ddc7c29fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.784 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.776+0000 7f4cf7615740 1 --2- 172.21.15.99:0/3982112239 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ddc7c21ff0 0x55ddc7c22410 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.784 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.776+0000 7f4cf7615740 1 -- 172.21.15.99:0/3982112239 >> 172.21.15.99:0/3982112239 conn(0x55ddc7c1d560 msgr2=0x55ddc7c1d960 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:19.784 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.776+0000 7f4cf7615740 1 -- 172.21.15.99:0/3982112239 shutdown_connections 2024-03-19T01:40:19.784 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.776+0000 7f4cf7615740 1 -- 172.21.15.99:0/3982112239 wait complete. 2024-03-19T01:40:19.784 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.777+0000 7f4cf7615740 1 -- start start 2024-03-19T01:40:19.784 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.777+0000 7f4cf7615740 1 Processor -- start 2024-03-19T01:40:19.784 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.777+0000 7f4cf7615740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ddc7c21ff0 0x55ddc7c2c4a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.784 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.777+0000 7f4cf7615740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ddc7c229e0 0x55ddc7c2c9e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.784 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.777+0000 7f4cf7615740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ddc7c27b50 0x55ddc7c33a20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.784 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.777+0000 7f4cf7615740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55ddc7c2ab00 con 0x55ddc7c27b50 2024-03-19T01:40:19.784 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.777+0000 7f4cf7615740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55ddc7b42fc0 con 0x55ddc7c21ff0 2024-03-19T01:40:19.784 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.777+0000 7f4cf7615740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55ddc7c2ac80 con 0x55ddc7c229e0 2024-03-19T01:40:19.784 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.777+0000 7f4cf7615740 10 client.0 _ll_register_callbacks cb 0x55ddc7b7ea20 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:40:19.785 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.777+0000 7f4cee59c640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ddc7c21ff0 0x55ddc7c2c4a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.785 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.777+0000 7f4ceed9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ddc7c27b50 0x55ddc7c33a20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.785 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.777+0000 7f4cee59c640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ddc7c21ff0 0x55ddc7c2c4a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:45248/0 (socket says 172.21.15.99:45248) 2024-03-19T01:40:19.785 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.777+0000 7f4cee59c640 1 -- 172.21.15.99:0/2837980357 learned_addr learned my addr 172.21.15.99:0/2837980357 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:19.785 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.777+0000 7f4cedd9b640 1 --2- 172.21.15.99:0/2837980357 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ddc7c229e0 0x55ddc7c2c9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.785 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.777+0000 7f4cee59c640 1 -- 172.21.15.99:0/2837980357 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ddc7c229e0 msgr2=0x55ddc7c2c9e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:40:19.785 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.777+0000 7f4cee59c640 1 --2- 172.21.15.99:0/2837980357 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ddc7c229e0 0x55ddc7c2c9e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.785 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.777+0000 7f4cee59c640 1 -- 172.21.15.99:0/2837980357 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ddc7c27b50 msgr2=0x55ddc7c33a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:40:19.785 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.777+0000 7f4cee59c640 1 --2- 172.21.15.99:0/2837980357 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ddc7c27b50 0x55ddc7c33a20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.785 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.777+0000 7f4cee59c640 1 -- 172.21.15.99:0/2837980357 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55ddc789b2d0 con 0x55ddc7c21ff0 2024-03-19T01:40:19.785 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.777+0000 7f4ceed9d640 1 --2- 172.21.15.99:0/2837980357 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ddc7c27b50 0x55ddc7c33a20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:40:19.785 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.778+0000 7f4cee59c640 1 --2- 172.21.15.99:0/2837980357 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ddc7c21ff0 0x55ddc7c2c4a0 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7f4ce0011d10 tx=0x7f4ce0019b30 comp rx=0 tx=0).ready entity=mon.2 client_cookie=5bd1464b2178591b server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:19.785 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.778+0000 7f4cd5ffb640 10 client.0 ms_handle_connect on v2:172.21.15.112:3300/0 2024-03-19T01:40:19.785 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.778+0000 7f4cd5ffb640 1 -- 172.21.15.99:0/2837980357 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4ce0028020 con 0x55ddc7c21ff0 2024-03-19T01:40:19.785 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.778+0000 7f4cd5ffb640 1 -- 172.21.15.99:0/2837980357 <== mon.2 v2:172.21.15.112:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f4ce0019d20 con 0x55ddc7c21ff0 2024-03-19T01:40:19.786 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.778+0000 7f4cf7615740 1 -- 172.21.15.99:0/2837980357 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f4ce00043d0 con 0x55ddc7c21ff0 2024-03-19T01:40:19.786 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.778+0000 7f4cd5ffb640 1 -- 172.21.15.99:0/2837980357 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_get_version_reply(handle=1 version=20) ==== 24+0+0 (secure 0 0 0) 0x7f4ce0033050 con 0x55ddc7c21ff0 2024-03-19T01:40:19.786 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.778+0000 7f4cf7615740 10 client.24878 fetch_fsmap learned FSMap version 20 2024-03-19T01:40:19.786 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.778+0000 7f4cf7615740 1 -- 172.21.15.99:0/2837980357 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,fsmap=20,monmap=4+}) -- 0x55ddc79656b0 con 0x55ddc7c21ff0 2024-03-19T01:40:19.786 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.778+0000 7f4cd5ffb640 1 -- 172.21.15.99:0/2837980357 <== mon.2 v2:172.21.15.112:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f4ce00043d0 con 0x55ddc7c21ff0 2024-03-19T01:40:19.786 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.778+0000 7f4cd5ffb640 1 -- 172.21.15.99:0/2837980357 <== mon.2 v2:172.21.15.112:3300/0 5 ==== fsmap(e 20) ==== 8338+0+0 (secure 0 0 0) 0x7f4ce0004b50 con 0x55ddc7c21ff0 2024-03-19T01:40:19.786 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.778+0000 7f4cf7615740 10 client.24878 fetch_fsmap finished waiting for FSMap version 20 2024-03-19T01:40:19.786 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.778+0000 7f4cf7615740 10 client.24878 resolve_mds: resolved name 'b' to daemon mds.b 2024-03-19T01:40:19.786 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.778+0000 7f4cf7615740 20 client.24878 populate_metadata read hostname 'smithi099' 2024-03-19T01:40:19.786 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.778+0000 7f4cf7615740 1 --2- 172.21.15.99:0/2837980357 >> [v2:172.21.15.112:6838/986249680,v1:172.21.15.112:6839/986249680] conn(0x55ddc7c39060 0x55ddc7c3a4d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.786 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.778+0000 7f4cf7615740 4 client.24878 mds_command: new command op to 24479 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:40:19.786 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.778+0000 7f4cf7615740 1 -- 172.21.15.99:0/2837980357 --> [v2:172.21.15.112:6838/986249680,v1:172.21.15.112:6839/986249680] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55ddc7b94320 con 0x55ddc7c39060 2024-03-19T01:40:19.786 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.778+0000 7f4cedd9b640 1 --2- 172.21.15.99:0/2837980357 >> [v2:172.21.15.112:6838/986249680,v1:172.21.15.112:6839/986249680] conn(0x55ddc7c39060 0x55ddc7c3a4d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.787 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.779+0000 7f4cd5ffb640 10 client.24878 ms_handle_connect on v2:172.21.15.112:6838/986249680 2024-03-19T01:40:19.787 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.779+0000 7f4cedd9b640 1 --2- 172.21.15.99:0/2837980357 >> [v2:172.21.15.112:6838/986249680,v1:172.21.15.112:6839/986249680] conn(0x55ddc7c39060 0x55ddc7c3a4d0 crc :-1 s=READY pgs=16 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=124ea17c075d7646 server_cookie=5a1c29f3759d1dea in_seq=0 out_seq=0 2024-03-19T01:40:19.787 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.779+0000 7f4cd5ffb640 1 -- 172.21.15.99:0/2837980357 <== mds.1 v2:172.21.15.112:6838/986249680 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x55ddc7b94320 con 0x55ddc7c39060 2024-03-19T01:40:19.787 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.779+0000 7f4cd5ffb640 10 client.24878 handle_command_reply: tid=0 2024-03-19T01:40:19.787 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.779+0000 7f4cf7615740 1 client.24878 shutdown 2024-03-19T01:40:19.787 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.780+0000 7f4cf7615740 1 -- 172.21.15.99:0/2837980357 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ddc7c21ff0 msgr2=0x55ddc7c2c4a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:19.787 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.780+0000 7f4cf7615740 1 --2- 172.21.15.99:0/2837980357 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ddc7c21ff0 0x55ddc7c2c4a0 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7f4ce0011d10 tx=0x7f4ce0019b30 comp rx=0 tx=0).stop 2024-03-19T01:40:19.787 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.780+0000 7f4cf7615740 1 -- 172.21.15.99:0/2837980357 shutdown_connections 2024-03-19T01:40:19.787 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.780+0000 7f4cf7615740 1 --2- 172.21.15.99:0/2837980357 >> [v2:172.21.15.112:6838/986249680,v1:172.21.15.112:6839/986249680] conn(0x55ddc7c39060 0x55ddc7c3a4d0 crc :-1 s=READY pgs=16 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.787 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.780+0000 7f4cf7615740 1 --2- 172.21.15.99:0/2837980357 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ddc7c27b50 0x55ddc7c33a20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.787 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.780+0000 7f4cd5ffb640 0 client.24878 ms_handle_reset on v2:172.21.15.112:6838/986249680 2024-03-19T01:40:19.787 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.780+0000 7f4cf7615740 1 --2- 172.21.15.99:0/2837980357 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ddc7c229e0 0x55ddc7c2c9e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.787 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.780+0000 7f4cf7615740 1 --2- 172.21.15.99:0/2837980357 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ddc7c21ff0 0x55ddc7c2c4a0 unknown :-1 s=CLOSED pgs=71 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.787 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.780+0000 7f4cf7615740 1 -- 172.21.15.99:0/2837980357 >> 172.21.15.99:0/2837980357 conn(0x55ddc7c1d560 msgr2=0x55ddc7c1c7e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:19.787 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.780+0000 7f4cf7615740 1 -- 172.21.15.99:0/2837980357 shutdown_connections 2024-03-19T01:40:19.788 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.780+0000 7f4cf7615740 1 -- 172.21.15.99:0/2837980357 wait complete. 2024-03-19T01:40:19.788 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.780+0000 7f4cf7615740 20 client.24878 trim_cache size 0 max 16384 2024-03-19T01:40:19.803 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.798+0000 7f4cf7615740 1 Processor -- start 2024-03-19T01:40:19.803 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.798+0000 7f4cf7615740 1 -- start start 2024-03-19T01:40:19.803 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.798+0000 7f4cf7615740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ddc7c21ff0 0x55ddc7c27470 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.803 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.798+0000 7f4cf7615740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ddc7c229e0 0x55ddc7c33940 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.803 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.798+0000 7f4cf7615740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ddc7c27b50 0x55ddc7c2b080 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.803 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.798+0000 7f4cf7615740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55ddc7c38810 con 0x55ddc7c21ff0 2024-03-19T01:40:19.803 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.798+0000 7f4cf7615740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55ddc7c38ee0 con 0x55ddc7c229e0 2024-03-19T01:40:19.803 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.798+0000 7f4cf7615740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55ddc7c395b0 con 0x55ddc7c27b50 2024-03-19T01:40:19.803 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.798+0000 7f4cee59c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ddc7c21ff0 0x55ddc7c27470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.803 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.798+0000 7f4cee59c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ddc7c21ff0 0x55ddc7c27470 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:53032/0 (socket says 172.21.15.99:53032) 2024-03-19T01:40:19.804 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.798+0000 7f4cee59c640 1 -- 172.21.15.99:0/4223878196 learned_addr learned my addr 172.21.15.99:0/4223878196 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:19.804 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.798+0000 7f4ceed9d640 1 --2- 172.21.15.99:0/4223878196 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ddc7c27b50 0x55ddc7c2b080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.804 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.798+0000 7f4cedd9b640 1 --2- 172.21.15.99:0/4223878196 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ddc7c229e0 0x55ddc7c33940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.804 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.798+0000 7f4cee59c640 1 -- 172.21.15.99:0/4223878196 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ddc7c27b50 msgr2=0x55ddc7c2b080 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:40:19.804 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.798+0000 7f4cee59c640 1 --2- 172.21.15.99:0/4223878196 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ddc7c27b50 0x55ddc7c2b080 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.804 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.798+0000 7f4cee59c640 1 -- 172.21.15.99:0/4223878196 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ddc7c229e0 msgr2=0x55ddc7c33940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:40:19.804 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.798+0000 7f4cee59c640 1 --2- 172.21.15.99:0/4223878196 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ddc7c229e0 0x55ddc7c33940 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.804 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.798+0000 7f4cee59c640 1 -- 172.21.15.99:0/4223878196 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55ddc7c18670 con 0x55ddc7c21ff0 2024-03-19T01:40:19.804 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.798+0000 7f4ceed9d640 1 --2- 172.21.15.99:0/4223878196 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ddc7c27b50 0x55ddc7c2b080 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-19T01:40:19.804 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.798+0000 7f4cee59c640 1 --2- 172.21.15.99:0/4223878196 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ddc7c21ff0 0x55ddc7c27470 secure :-1 s=READY pgs=299 cs=0 l=1 rev1=1 crypto rx=0x7f4ce0009ca0 tx=0x7f4ce0011810 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1fc4e1336e26d5a0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:19.804 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.798+0000 7f4cd7fff640 1 -- 172.21.15.99:0/4223878196 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4ce0028070 con 0x55ddc7c21ff0 2024-03-19T01:40:19.804 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.798+0000 7f4cd7fff640 1 -- 172.21.15.99:0/4223878196 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f4ce0009e50 con 0x55ddc7c21ff0 2024-03-19T01:40:19.804 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.798+0000 7f4cd7fff640 1 -- 172.21.15.99:0/4223878196 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4ce001e040 con 0x55ddc7c21ff0 2024-03-19T01:40:19.804 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.799+0000 7f4cf7615740 1 -- 172.21.15.99:0/4223878196 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ddc7c21ff0 msgr2=0x55ddc7c27470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:19.805 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.799+0000 7f4cf7615740 1 --2- 172.21.15.99:0/4223878196 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ddc7c21ff0 0x55ddc7c27470 secure :-1 s=READY pgs=299 cs=0 l=1 rev1=1 crypto rx=0x7f4ce0009ca0 tx=0x7f4ce0011810 comp rx=0 tx=0).stop 2024-03-19T01:40:19.805 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.799+0000 7f4cf7615740 1 -- 172.21.15.99:0/4223878196 shutdown_connections 2024-03-19T01:40:19.805 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.799+0000 7f4cf7615740 1 --2- 172.21.15.99:0/4223878196 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ddc7c229e0 0x55ddc7c33940 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.805 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.799+0000 7f4cf7615740 1 --2- 172.21.15.99:0/4223878196 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ddc7c27b50 0x55ddc7c2b080 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.805 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.799+0000 7f4cf7615740 1 --2- 172.21.15.99:0/4223878196 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ddc7c21ff0 0x55ddc7c27470 unknown :-1 s=CLOSED pgs=299 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.805 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.799+0000 7f4cf7615740 1 -- 172.21.15.99:0/4223878196 >> 172.21.15.99:0/4223878196 conn(0x55ddc7c1d560 msgr2=0x55ddc7c21050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:19.805 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.799+0000 7f4cf7615740 1 -- 172.21.15.99:0/4223878196 shutdown_connections 2024-03-19T01:40:19.805 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.799+0000 7f4cf7615740 1 -- 172.21.15.99:0/4223878196 wait complete. 2024-03-19T01:40:19.805 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.799+0000 7f4cf7615740 1 -- start start 2024-03-19T01:40:19.805 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.799+0000 7f4cf7615740 1 Processor -- start 2024-03-19T01:40:19.805 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.799+0000 7f4cf7615740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ddc7c21ff0 0x55ddc7c2f160 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.805 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.799+0000 7f4cf7615740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ddc7c229e0 0x55ddc7c2f6a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.805 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.799+0000 7f4cf7615740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ddc7c27b50 0x55ddc7c32670 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.805 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.800+0000 7f4cf7615740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55ddc7c3a350 con 0x55ddc7c27b50 2024-03-19T01:40:19.805 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.800+0000 7f4cf7615740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55ddc7c39c80 con 0x55ddc7c21ff0 2024-03-19T01:40:19.805 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.800+0000 7f4cf7615740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55ddc7c3aa20 con 0x55ddc7c229e0 2024-03-19T01:40:19.806 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.800+0000 7f4cf7615740 10 client.0 _ll_register_callbacks cb 0x7f4ce4005e30 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:40:19.806 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.800+0000 7f4ceed9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ddc7c27b50 0x55ddc7c32670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.806 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.800+0000 7f4ceed9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ddc7c27b50 0x55ddc7c32670 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:53044/0 (socket says 172.21.15.99:53044) 2024-03-19T01:40:19.806 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.800+0000 7f4ceed9d640 1 -- 172.21.15.99:0/829528131 learned_addr learned my addr 172.21.15.99:0/829528131 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:19.806 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.800+0000 7f4cee59c640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ddc7c21ff0 0x55ddc7c2f160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.806 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.800+0000 7f4cedd9b640 1 --2- 172.21.15.99:0/829528131 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ddc7c229e0 0x55ddc7c2f6a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.806 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.800+0000 7f4ceed9d640 1 -- 172.21.15.99:0/829528131 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ddc7c229e0 msgr2=0x55ddc7c2f6a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:40:19.806 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.800+0000 7f4ceed9d640 1 --2- 172.21.15.99:0/829528131 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ddc7c229e0 0x55ddc7c2f6a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.806 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.800+0000 7f4ceed9d640 1 -- 172.21.15.99:0/829528131 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ddc7c21ff0 msgr2=0x55ddc7c2f160 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:40:19.806 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.800+0000 7f4ceed9d640 1 --2- 172.21.15.99:0/829528131 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ddc7c21ff0 0x55ddc7c2f160 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.806 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.800+0000 7f4ceed9d640 1 -- 172.21.15.99:0/829528131 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55ddc7c32c60 con 0x55ddc7c27b50 2024-03-19T01:40:19.806 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.800+0000 7f4cee59c640 1 --2- 172.21.15.99:0/829528131 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ddc7c21ff0 0x55ddc7c2f160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:40:19.806 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.800+0000 7f4ceed9d640 1 --2- 172.21.15.99:0/829528131 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ddc7c27b50 0x55ddc7c32670 secure :-1 s=READY pgs=300 cs=0 l=1 rev1=1 crypto rx=0x7f4ce8009be0 tx=0x7f4ce8009500 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6e9e6663083143a server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:19.807 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.800+0000 7f4cd5ffb640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:40:19.807 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.800+0000 7f4cd5ffb640 1 -- 172.21.15.99:0/829528131 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4ce8055030 con 0x55ddc7c27b50 2024-03-19T01:40:19.807 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.800+0000 7f4cd5ffb640 1 -- 172.21.15.99:0/829528131 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f4ce803a040 con 0x55ddc7c27b50 2024-03-19T01:40:19.807 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.800+0000 7f4cf7615740 1 -- 172.21.15.99:0/829528131 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f4ce0009e50 con 0x55ddc7c27b50 2024-03-19T01:40:19.807 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.800+0000 7f4cd5ffb640 1 -- 172.21.15.99:0/829528131 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=20) ==== 24+0+0 (secure 0 0 0) 0x7f4ce8004040 con 0x55ddc7c27b50 2024-03-19T01:40:19.807 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.800+0000 7f4cf7615740 10 client.15108 fetch_fsmap learned FSMap version 20 2024-03-19T01:40:19.807 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.800+0000 7f4cf7615740 1 -- 172.21.15.99:0/829528131 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=20,monmap=4+}) -- 0x55ddc7c9ef00 con 0x55ddc7c27b50 2024-03-19T01:40:19.807 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.800+0000 7f4cd5ffb640 1 -- 172.21.15.99:0/829528131 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f4ce0009e50 con 0x55ddc7c27b50 2024-03-19T01:40:19.807 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.801+0000 7f4cd5ffb640 1 -- 172.21.15.99:0/829528131 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 20) ==== 8338+0+0 (secure 0 0 0) 0x7f4ce8034730 con 0x55ddc7c27b50 2024-03-19T01:40:19.807 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.801+0000 7f4cf7615740 10 client.15108 fetch_fsmap finished waiting for FSMap version 20 2024-03-19T01:40:19.807 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.801+0000 7f4cf7615740 10 client.15108 resolve_mds: resolved name 'b' to daemon mds.b 2024-03-19T01:40:19.807 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.801+0000 7f4cf7615740 20 client.15108 populate_metadata read hostname 'smithi099' 2024-03-19T01:40:19.807 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.801+0000 7f4cf7615740 1 --2- 172.21.15.99:0/829528131 >> [v2:172.21.15.112:6838/986249680,v1:172.21.15.112:6839/986249680] conn(0x55ddc7ca3ba0 0x55ddc7c9ffe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.807 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.801+0000 7f4cf7615740 4 client.15108 mds_command: new command op to 24479 tid=0 multi_id=0 [{"prefix": "status"}] 2024-03-19T01:40:19.807 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.801+0000 7f4cf7615740 1 -- 172.21.15.99:0/829528131 --> [v2:172.21.15.112:6838/986249680,v1:172.21.15.112:6839/986249680] -- command(tid 0: {"prefix": "status"}) -- 0x55ddc7b8e670 con 0x55ddc7ca3ba0 2024-03-19T01:40:19.808 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.801+0000 7f4cee59c640 1 --2- 172.21.15.99:0/829528131 >> [v2:172.21.15.112:6838/986249680,v1:172.21.15.112:6839/986249680] conn(0x55ddc7ca3ba0 0x55ddc7c9ffe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.808 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.801+0000 7f4cee59c640 1 --2- 172.21.15.99:0/829528131 >> [v2:172.21.15.112:6838/986249680,v1:172.21.15.112:6839/986249680] conn(0x55ddc7ca3ba0 0x55ddc7c9ffe0 crc :-1 s=READY pgs=17 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=b7c2fd65660af986 server_cookie=e0a5707d98667558 in_seq=0 out_seq=0 2024-03-19T01:40:19.808 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.801+0000 7f4cd5ffb640 10 client.15108 ms_handle_connect on v2:172.21.15.112:6838/986249680 2024-03-19T01:40:19.808 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.801+0000 7f4cd5ffb640 1 -- 172.21.15.99:0/829528131 <== mds.1 v2:172.21.15.112:6838/986249680 1 ==== command_reply(tid 0: 0 ) ==== 8+0+332 (crc 0 0 0) 0x55ddc7b8e670 con 0x55ddc7ca3ba0 2024-03-19T01:40:19.808 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.801+0000 7f4cd5ffb640 10 client.15108 handle_command_reply: tid=0 2024-03-19T01:40:19.808 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.801+0000 7f4cf7615740 1 client.15108 shutdown 2024-03-19T01:40:19.808 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.801+0000 7f4cf7615740 1 -- 172.21.15.99:0/829528131 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ddc7c27b50 msgr2=0x55ddc7c32670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:19.808 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.801+0000 7f4cf7615740 1 --2- 172.21.15.99:0/829528131 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ddc7c27b50 0x55ddc7c32670 secure :-1 s=READY pgs=300 cs=0 l=1 rev1=1 crypto rx=0x7f4ce8009be0 tx=0x7f4ce8009500 comp rx=0 tx=0).stop 2024-03-19T01:40:19.808 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.802+0000 7f4cf7615740 1 -- 172.21.15.99:0/829528131 shutdown_connections 2024-03-19T01:40:19.808 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.802+0000 7f4cf7615740 1 --2- 172.21.15.99:0/829528131 >> [v2:172.21.15.112:6838/986249680,v1:172.21.15.112:6839/986249680] conn(0x55ddc7ca3ba0 0x55ddc7c9ffe0 crc :-1 s=READY pgs=17 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.808 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.802+0000 7f4cf7615740 1 --2- 172.21.15.99:0/829528131 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ddc7c27b50 0x55ddc7c32670 unknown :-1 s=CLOSED pgs=300 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.808 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.802+0000 7f4cf7615740 1 --2- 172.21.15.99:0/829528131 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ddc7c229e0 0x55ddc7c2f6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.808 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.802+0000 7f4cd5ffb640 0 client.15108 ms_handle_reset on v2:172.21.15.112:6838/986249680 2024-03-19T01:40:19.809 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.802+0000 7f4cf7615740 1 --2- 172.21.15.99:0/829528131 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ddc7c21ff0 0x55ddc7c2f160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.809 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.802+0000 7f4cf7615740 1 -- 172.21.15.99:0/829528131 >> 172.21.15.99:0/829528131 conn(0x55ddc7c1d560 msgr2=0x55ddc7c21050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:19.809 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.802+0000 7f4cf7615740 1 -- 172.21.15.99:0/829528131 shutdown_connections 2024-03-19T01:40:19.809 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.802+0000 7f4cf7615740 1 -- 172.21.15.99:0/829528131 wait complete. 2024-03-19T01:40:19.809 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.802+0000 7f4cf7615740 20 client.15108 trim_cache size 0 max 16384 2024-03-19T01:40:19.809 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2024-03-19T01:40:19.809 INFO:teuthology.orchestra.run.smithi099.stdout: "cluster_fsid": "438fb69c-e590-11ee-95c9-87774f69a715", 2024-03-19T01:40:19.809 INFO:teuthology.orchestra.run.smithi099.stdout: "whoami": 1, 2024-03-19T01:40:19.809 INFO:teuthology.orchestra.run.smithi099.stdout: "id": 24479, 2024-03-19T01:40:19.809 INFO:teuthology.orchestra.run.smithi099.stdout: "want_state": "up:active", 2024-03-19T01:40:19.810 INFO:teuthology.orchestra.run.smithi099.stdout: "state": "up:active", 2024-03-19T01:40:19.810 INFO:teuthology.orchestra.run.smithi099.stdout: "fs_name": "cephfs", 2024-03-19T01:40:19.810 INFO:teuthology.orchestra.run.smithi099.stdout: "rank_uptime": 81.000190383000003, 2024-03-19T01:40:19.810 INFO:teuthology.orchestra.run.smithi099.stdout: "mdsmap_epoch": 20, 2024-03-19T01:40:19.810 INFO:teuthology.orchestra.run.smithi099.stdout: "osdmap_epoch": 91, 2024-03-19T01:40:19.810 INFO:teuthology.orchestra.run.smithi099.stdout: "osdmap_epoch_barrier": 87, 2024-03-19T01:40:19.810 INFO:teuthology.orchestra.run.smithi099.stdout: "uptime": 89.793001912999998 2024-03-19T01:40:19.810 INFO:teuthology.orchestra.run.smithi099.stdout:} 2024-03-19T01:40:19.811 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.802+0000 7f4cf4a27640 1 -- 172.21.15.99:0/39474516 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f4cd80a2800 msgr2=0x7f4cd80a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:19.811 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.802+0000 7f4cf4a27640 1 --2- 172.21.15.99:0/39474516 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f4cd80a2800 0x7f4cd80a4cc0 secure :-1 s=READY pgs=221 cs=0 l=1 rev1=1 crypto rx=0x7f4ce8004ab0 tx=0x7f4ce8004a40 comp rx=0 tx=0).stop 2024-03-19T01:40:19.811 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.802+0000 7f4cf4a27640 1 -- 172.21.15.99:0/39474516 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4cf0101920 msgr2=0x7f4cf01096f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:19.811 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.802+0000 7f4cf4a27640 1 --2- 172.21.15.99:0/39474516 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4cf0101920 0x7f4cf01096f0 secure :-1 s=READY pgs=298 cs=0 l=1 rev1=1 crypto rx=0x7f4ce400e9f0 tx=0x7f4ce400edf0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.811 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.802+0000 7f4cf4a27640 1 -- 172.21.15.99:0/39474516 shutdown_connections 2024-03-19T01:40:19.811 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.802+0000 7f4cf4a27640 1 --2- 172.21.15.99:0/39474516 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f4cd80a2800 0x7f4cd80a4cc0 unknown :-1 s=CLOSED pgs=221 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.811 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.802+0000 7f4cf4a27640 1 --2- 172.21.15.99:0/39474516 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4cf01022e0 0x7f4cf0109c30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.811 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.802+0000 7f4cf4a27640 1 --2- 172.21.15.99:0/39474516 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4cf0101920 0x7f4cf01096f0 unknown :-1 s=CLOSED pgs=298 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.811 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.802+0000 7f4cf4a27640 1 --2- 172.21.15.99:0/39474516 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4cf0100730 0x7f4cf010b0b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.812 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.802+0000 7f4cf4a27640 1 -- 172.21.15.99:0/39474516 >> 172.21.15.99:0/39474516 conn(0x7f4cf00fbcc0 msgr2=0x7f4cf0104c40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:19.812 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.802+0000 7f4cf4a27640 1 -- 172.21.15.99:0/39474516 shutdown_connections 2024-03-19T01:40:19.812 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.802+0000 7f4cf4a27640 1 -- 172.21.15.99:0/39474516 wait complete. 2024-03-19T01:40:19.813 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.i status 2024-03-19T01:40:19.918 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.916+0000 7f72243cd640 1 -- 172.21.15.99:0/3405083146 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f721c101850 msgr2=0x7f721c101c70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:19.918 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.916+0000 7f72243cd640 1 --2- 172.21.15.99:0/3405083146 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f721c101850 0x7f721c101c70 secure :-1 s=READY pgs=301 cs=0 l=1 rev1=1 crypto rx=0x7f7218009900 tx=0x7f721802bc30 comp rx=0 tx=0).stop 2024-03-19T01:40:19.918 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.916+0000 7f72243cd640 1 -- 172.21.15.99:0/3405083146 shutdown_connections 2024-03-19T01:40:19.918 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.916+0000 7f72243cd640 1 --2- 172.21.15.99:0/3405083146 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f721c102a40 0x7f721c102ec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.919 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.916+0000 7f72243cd640 1 --2- 172.21.15.99:0/3405083146 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f721c103400 0x7f721c109c90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.919 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.916+0000 7f72243cd640 1 --2- 172.21.15.99:0/3405083146 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f721c101850 0x7f721c101c70 unknown :-1 s=CLOSED pgs=301 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.919 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.916+0000 7f72243cd640 1 -- 172.21.15.99:0/3405083146 >> 172.21.15.99:0/3405083146 conn(0x7f721c0fcdc0 msgr2=0x7f721c0ff200 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:19.919 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.916+0000 7f72243cd640 1 -- 172.21.15.99:0/3405083146 shutdown_connections 2024-03-19T01:40:19.919 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.916+0000 7f72243cd640 1 -- 172.21.15.99:0/3405083146 wait complete. 2024-03-19T01:40:19.919 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.917+0000 7f72243cd640 1 Processor -- start 2024-03-19T01:40:19.919 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.917+0000 7f72243cd640 1 -- start start 2024-03-19T01:40:19.919 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.917+0000 7f72243cd640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f721c101850 0x7f721c197b80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.919 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.917+0000 7f72243cd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f721c102a40 0x7f721c1980c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.919 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.917+0000 7f72243cd640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f721c103400 0x7f721c19f130 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.919 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.917+0000 7f72243cd640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f721c10d4d0 con 0x7f721c102a40 2024-03-19T01:40:19.919 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.917+0000 7f72243cd640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f721c10d350 con 0x7f721c101850 2024-03-19T01:40:19.919 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.917+0000 7f72243cd640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f721c10d650 con 0x7f721c103400 2024-03-19T01:40:19.920 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.917+0000 7f7221968640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f721c102a40 0x7f721c1980c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.920 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.917+0000 7f7221968640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f721c102a40 0x7f721c1980c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:53054/0 (socket says 172.21.15.99:53054) 2024-03-19T01:40:19.920 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.917+0000 7f7222169640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f721c101850 0x7f721c197b80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.920 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.917+0000 7f7221968640 1 -- 172.21.15.99:0/3316836195 learned_addr learned my addr 172.21.15.99:0/3316836195 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:19.920 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.917+0000 7f722296a640 1 --2- 172.21.15.99:0/3316836195 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f721c103400 0x7f721c19f130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.920 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.917+0000 7f7221968640 1 -- 172.21.15.99:0/3316836195 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f721c103400 msgr2=0x7f721c19f130 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:19.920 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.917+0000 7f7221968640 1 --2- 172.21.15.99:0/3316836195 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f721c103400 0x7f721c19f130 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.920 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.917+0000 7f7221968640 1 -- 172.21.15.99:0/3316836195 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f721c101850 msgr2=0x7f721c197b80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:19.920 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.917+0000 7f7221968640 1 --2- 172.21.15.99:0/3316836195 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f721c101850 0x7f721c197b80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.920 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.917+0000 7f7221968640 1 -- 172.21.15.99:0/3316836195 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f721c19f7a0 con 0x7f721c102a40 2024-03-19T01:40:19.920 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.917+0000 7f722296a640 1 --2- 172.21.15.99:0/3316836195 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f721c103400 0x7f721c19f130 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:40:19.920 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.917+0000 7f7222169640 1 --2- 172.21.15.99:0/3316836195 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f721c101850 0x7f721c197b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:40:19.920 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.917+0000 7f7221968640 1 --2- 172.21.15.99:0/3316836195 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f721c102a40 0x7f721c1980c0 secure :-1 s=READY pgs=302 cs=0 l=1 rev1=1 crypto rx=0x7f720c00ea60 tx=0x7f720c00ef20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:19.921 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.917+0000 7f72137fe640 1 -- 172.21.15.99:0/3316836195 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f720c00cde0 con 0x7f721c102a40 2024-03-19T01:40:19.921 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.917+0000 7f72243cd640 1 -- 172.21.15.99:0/3316836195 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f721c19fa90 con 0x7f721c102a40 2024-03-19T01:40:19.921 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.917+0000 7f72137fe640 1 -- 172.21.15.99:0/3316836195 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f720c0035f0 con 0x7f721c102a40 2024-03-19T01:40:19.921 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.917+0000 7f72137fe640 1 -- 172.21.15.99:0/3316836195 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f720c010670 con 0x7f721c102a40 2024-03-19T01:40:19.921 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.918+0000 7f72243cd640 1 -- 172.21.15.99:0/3316836195 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f721c19ffe0 con 0x7f721c102a40 2024-03-19T01:40:19.927 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.920+0000 7f7226fb8740 1 Processor -- start 2024-03-19T01:40:19.927 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.920+0000 7f72137fe640 1 -- 172.21.15.99:0/3316836195 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 18) ==== 137290+0+0 (secure 0 0 0) 0x7f720c0037a0 con 0x7f721c102a40 2024-03-19T01:40:19.927 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.920+0000 7f7226fb8740 1 -- start start 2024-03-19T01:40:19.928 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.920+0000 7f7226fb8740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55b46b8d9ff0 0x55b46b8da410 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.928 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.920+0000 7f7226fb8740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55b46b8da9e0 0x55b46b8df4e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.928 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.921+0000 7f7226fb8740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55b46b8dfb50 0x55b46b8e1fe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.928 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.921+0000 7f7226fb8740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55b46b83ede0 con 0x55b46b8da9e0 2024-03-19T01:40:19.928 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.921+0000 7f7226fb8740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55b46b7e7c80 con 0x55b46b8dfb50 2024-03-19T01:40:19.928 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.921+0000 7f7226fb8740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55b46b83ae80 con 0x55b46b8d9ff0 2024-03-19T01:40:19.928 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.921+0000 7f7222169640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55b46b8da9e0 0x55b46b8df4e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.928 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.921+0000 7f7222169640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55b46b8da9e0 0x55b46b8df4e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:53068/0 (socket says 172.21.15.99:53068) 2024-03-19T01:40:19.928 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.921+0000 7f7222169640 1 -- 172.21.15.99:0/1046847151 learned_addr learned my addr 172.21.15.99:0/1046847151 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:19.928 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.921+0000 7f722296a640 1 --2- 172.21.15.99:0/1046847151 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55b46b8d9ff0 0x55b46b8da410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.928 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.921+0000 7f7221968640 1 --2- 172.21.15.99:0/1046847151 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55b46b8dfb50 0x55b46b8e1fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.928 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.921+0000 7f72137fe640 1 --2- 172.21.15.99:0/3316836195 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f72080a2800 0x7f72080a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.928 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.921+0000 7f722296a640 1 -- 172.21.15.99:0/1046847151 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55b46b8dfb50 msgr2=0x55b46b8e1fe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:40:19.928 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.921+0000 7f722296a640 1 --2- 172.21.15.99:0/1046847151 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55b46b8dfb50 0x55b46b8e1fe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.929 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.921+0000 7f722296a640 1 -- 172.21.15.99:0/1046847151 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55b46b8da9e0 msgr2=0x55b46b8df4e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:40:19.929 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.921+0000 7f722296a640 1 --2- 172.21.15.99:0/1046847151 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55b46b8da9e0 0x55b46b8df4e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.929 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.921+0000 7f722296a640 1 -- 172.21.15.99:0/1046847151 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55b46b7f9fb0 con 0x55b46b8d9ff0 2024-03-19T01:40:19.929 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.921+0000 7f72137fe640 1 -- 172.21.15.99:0/3316836195 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f720c0ced90 con 0x7f721c102a40 2024-03-19T01:40:19.929 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.921+0000 7f722296a640 1 --2- 172.21.15.99:0/3316836195 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f72080a2800 0x7f72080a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.929 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.921+0000 7f722296a640 1 --2- 172.21.15.99:0/1046847151 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55b46b8d9ff0 0x55b46b8da410 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7f7214005fd0 tx=0x7f7214011bf0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=b017adfa04e1fa15 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:19.929 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.921+0000 7f7210bfe640 1 -- 172.21.15.99:0/1046847151 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f721400f9a0 con 0x55b46b8d9ff0 2024-03-19T01:40:19.929 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.922+0000 7f7210bfe640 1 -- 172.21.15.99:0/1046847151 <== mon.1 v2:172.21.15.115:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f721400fc80 con 0x55b46b8d9ff0 2024-03-19T01:40:19.929 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.922+0000 7f7210bfe640 1 -- 172.21.15.99:0/1046847151 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f721401cbc0 con 0x55b46b8d9ff0 2024-03-19T01:40:19.929 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.922+0000 7f7226fb8740 1 -- 172.21.15.99:0/1046847151 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55b46b8d9ff0 msgr2=0x55b46b8da410 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:19.929 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.922+0000 7f7226fb8740 1 --2- 172.21.15.99:0/1046847151 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55b46b8d9ff0 0x55b46b8da410 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7f7214005fd0 tx=0x7f7214011bf0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.929 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.922+0000 7f722296a640 1 --2- 172.21.15.99:0/3316836195 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f72080a2800 0x7f72080a4cc0 secure :-1 s=READY pgs=222 cs=0 l=1 rev1=1 crypto rx=0x7f7214020000 tx=0x7f7214021000 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:19.929 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.922+0000 7f7226fb8740 1 -- 172.21.15.99:0/1046847151 shutdown_connections 2024-03-19T01:40:19.930 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.922+0000 7f7226fb8740 1 --2- 172.21.15.99:0/1046847151 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55b46b8dfb50 0x55b46b8e1fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.930 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.922+0000 7f7226fb8740 1 --2- 172.21.15.99:0/1046847151 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55b46b8da9e0 0x55b46b8df4e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.930 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.922+0000 7f7226fb8740 1 --2- 172.21.15.99:0/1046847151 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55b46b8d9ff0 0x55b46b8da410 unknown :-1 s=CLOSED pgs=70 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.930 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.922+0000 7f7226fb8740 1 -- 172.21.15.99:0/1046847151 >> 172.21.15.99:0/1046847151 conn(0x55b46b8d5560 msgr2=0x55b46b8d5960 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:19.930 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.922+0000 7f7226fb8740 1 -- 172.21.15.99:0/1046847151 shutdown_connections 2024-03-19T01:40:19.930 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.922+0000 7f7226fb8740 1 -- 172.21.15.99:0/1046847151 wait complete. 2024-03-19T01:40:19.930 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.923+0000 7f7226fb8740 1 -- start start 2024-03-19T01:40:19.930 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.923+0000 7f7226fb8740 1 Processor -- start 2024-03-19T01:40:19.930 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.923+0000 7f7226fb8740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55b46b8d9ff0 0x55b46b8e4510 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.930 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.923+0000 7f7226fb8740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55b46b8da9e0 0x55b46b8e4a50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.930 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.923+0000 7f7226fb8740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55b46b8dfb50 0x55b46b8eba90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.930 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.923+0000 7f7226fb8740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55b46b8e2b00 con 0x55b46b8dfb50 2024-03-19T01:40:19.930 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.923+0000 7f7226fb8740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55b46b7fafc0 con 0x55b46b8d9ff0 2024-03-19T01:40:19.931 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.923+0000 7f7226fb8740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55b46b8e2c80 con 0x55b46b8da9e0 2024-03-19T01:40:19.931 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.923+0000 7f7226fb8740 10 client.0 _ll_register_callbacks cb 0x55b46b836a20 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:40:19.931 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.923+0000 7f722296a640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55b46b8dfb50 0x55b46b8eba90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.931 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.923+0000 7f7222169640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55b46b8d9ff0 0x55b46b8e4510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.931 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.923+0000 7f7221968640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55b46b8da9e0 0x55b46b8e4a50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.931 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.923+0000 7f722296a640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55b46b8dfb50 0x55b46b8eba90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:53072/0 (socket says 172.21.15.99:53072) 2024-03-19T01:40:19.931 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.923+0000 7f722296a640 1 -- 172.21.15.99:0/2763114961 learned_addr learned my addr 172.21.15.99:0/2763114961 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:19.931 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.923+0000 7f722296a640 1 -- 172.21.15.99:0/2763114961 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55b46b8da9e0 msgr2=0x55b46b8e4a50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:40:19.931 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.923+0000 7f722296a640 1 --2- 172.21.15.99:0/2763114961 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55b46b8da9e0 0x55b46b8e4a50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.931 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.923+0000 7f722296a640 1 -- 172.21.15.99:0/2763114961 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55b46b8d9ff0 msgr2=0x55b46b8e4510 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:40:19.931 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.923+0000 7f722296a640 1 --2- 172.21.15.99:0/2763114961 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55b46b8d9ff0 0x55b46b8e4510 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.931 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.923+0000 7f722296a640 1 -- 172.21.15.99:0/2763114961 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55b46b5532d0 con 0x55b46b8dfb50 2024-03-19T01:40:19.931 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.923+0000 7f7221968640 1 --2- 172.21.15.99:0/2763114961 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55b46b8da9e0 0x55b46b8e4a50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:40:19.932 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.923+0000 7f7222169640 1 --2- 172.21.15.99:0/2763114961 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55b46b8d9ff0 0x55b46b8e4510 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:40:19.932 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.924+0000 7f722296a640 1 --2- 172.21.15.99:0/2763114961 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55b46b8dfb50 0x55b46b8eba90 secure :-1 s=READY pgs=303 cs=0 l=1 rev1=1 crypto rx=0x7f7214004690 tx=0x7f72140044f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7c5d91045443009d server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:19.932 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.924+0000 7f7205ffb640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:40:19.932 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.924+0000 7f7205ffb640 1 -- 172.21.15.99:0/2763114961 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f721400fe30 con 0x55b46b8dfb50 2024-03-19T01:40:19.932 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.924+0000 7f7205ffb640 1 -- 172.21.15.99:0/2763114961 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f721402e070 con 0x55b46b8dfb50 2024-03-19T01:40:19.932 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.924+0000 7f7226fb8740 1 -- 172.21.15.99:0/2763114961 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f721400fc80 con 0x55b46b8dfb50 2024-03-19T01:40:19.932 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.924+0000 7f7205ffb640 1 -- 172.21.15.99:0/2763114961 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=20) ==== 24+0+0 (secure 0 0 0) 0x7f721401c400 con 0x55b46b8dfb50 2024-03-19T01:40:19.932 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.924+0000 7f7226fb8740 10 client.15120 fetch_fsmap learned FSMap version 20 2024-03-19T01:40:19.932 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.924+0000 7f7226fb8740 1 -- 172.21.15.99:0/2763114961 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=20,monmap=4+}) -- 0x55b46b61d6b0 con 0x55b46b8dfb50 2024-03-19T01:40:19.932 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.924+0000 7f7205ffb640 1 -- 172.21.15.99:0/2763114961 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f721400fc80 con 0x55b46b8dfb50 2024-03-19T01:40:19.932 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.924+0000 7f7205ffb640 1 -- 172.21.15.99:0/2763114961 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 20) ==== 8338+0+0 (secure 0 0 0) 0x7f7214005050 con 0x55b46b8dfb50 2024-03-19T01:40:19.932 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.924+0000 7f7226fb8740 10 client.15120 fetch_fsmap finished waiting for FSMap version 20 2024-03-19T01:40:19.932 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.924+0000 7f7226fb8740 10 client.15120 resolve_mds: resolved name 'i' to daemon mds.i 2024-03-19T01:40:19.932 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.924+0000 7f7226fb8740 20 client.15120 populate_metadata read hostname 'smithi099' 2024-03-19T01:40:19.933 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.925+0000 7f7226fb8740 1 --2- 172.21.15.99:0/2763114961 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55b46b8f1150 0x55b46b8f25c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.933 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.925+0000 7f7226fb8740 4 client.15120 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:40:19.933 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.925+0000 7f7226fb8740 1 -- 172.21.15.99:0/2763114961 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55b46b84c320 con 0x55b46b8f1150 2024-03-19T01:40:19.933 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.925+0000 7f7222169640 1 --2- 172.21.15.99:0/2763114961 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55b46b8f1150 0x55b46b8f25c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.933 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.925+0000 7f7222169640 1 --2- 172.21.15.99:0/2763114961 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55b46b8f1150 0x55b46b8f25c0 crc :-1 s=READY pgs=21 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=3b66422fff6e721a server_cookie=9d2fecfbf621e19f in_seq=0 out_seq=0 2024-03-19T01:40:19.933 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.925+0000 7f7205ffb640 10 client.15120 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:40:19.933 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.926+0000 7f7205ffb640 1 -- 172.21.15.99:0/2763114961 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x55b46b84c320 con 0x55b46b8f1150 2024-03-19T01:40:19.933 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.926+0000 7f7205ffb640 10 client.15120 handle_command_reply: tid=0 2024-03-19T01:40:19.933 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.926+0000 7f7226fb8740 1 client.15120 shutdown 2024-03-19T01:40:19.933 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.926+0000 7f7226fb8740 1 -- 172.21.15.99:0/2763114961 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55b46b8dfb50 msgr2=0x55b46b8eba90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:19.933 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.926+0000 7f7226fb8740 1 --2- 172.21.15.99:0/2763114961 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55b46b8dfb50 0x55b46b8eba90 secure :-1 s=READY pgs=303 cs=0 l=1 rev1=1 crypto rx=0x7f7214004690 tx=0x7f72140044f0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.933 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.926+0000 7f7226fb8740 1 -- 172.21.15.99:0/2763114961 shutdown_connections 2024-03-19T01:40:19.933 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.926+0000 7f7226fb8740 1 --2- 172.21.15.99:0/2763114961 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55b46b8f1150 0x55b46b8f25c0 crc :-1 s=READY pgs=21 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.933 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.926+0000 7f7226fb8740 1 --2- 172.21.15.99:0/2763114961 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55b46b8dfb50 0x55b46b8eba90 unknown :-1 s=CLOSED pgs=303 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.934 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.926+0000 7f7205ffb640 0 client.15120 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:40:19.934 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.926+0000 7f7226fb8740 1 --2- 172.21.15.99:0/2763114961 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55b46b8da9e0 0x55b46b8e4a50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.934 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.926+0000 7f7226fb8740 1 --2- 172.21.15.99:0/2763114961 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55b46b8d9ff0 0x55b46b8e4510 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.934 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.926+0000 7f7226fb8740 1 -- 172.21.15.99:0/2763114961 >> 172.21.15.99:0/2763114961 conn(0x55b46b8d5560 msgr2=0x55b46b8d47e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:19.934 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.926+0000 7f7226fb8740 1 -- 172.21.15.99:0/2763114961 shutdown_connections 2024-03-19T01:40:19.934 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.926+0000 7f7226fb8740 1 -- 172.21.15.99:0/2763114961 wait complete. 2024-03-19T01:40:19.934 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.926+0000 7f7226fb8740 20 client.15120 trim_cache size 0 max 16384 2024-03-19T01:40:19.950 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.944+0000 7f7226fb8740 1 Processor -- start 2024-03-19T01:40:19.950 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.945+0000 7f7226fb8740 1 -- start start 2024-03-19T01:40:19.950 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.945+0000 7f7226fb8740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55b46b8d9ff0 0x55b46b8e1f40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.950 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.945+0000 7f7226fb8740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55b46b8da9e0 0x55b46b8eb9b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.950 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.945+0000 7f7226fb8740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55b46b8dfb50 0x55b46b8d5940 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.951 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.945+0000 7f7226fb8740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55b46b8f0900 con 0x55b46b8dfb50 2024-03-19T01:40:19.951 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.945+0000 7f7226fb8740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55b46b8f0fd0 con 0x55b46b8d9ff0 2024-03-19T01:40:19.951 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.945+0000 7f7226fb8740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55b46b8f16a0 con 0x55b46b8da9e0 2024-03-19T01:40:19.951 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.945+0000 7f722296a640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55b46b8dfb50 0x55b46b8d5940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.951 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.945+0000 7f7222169640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55b46b8d9ff0 0x55b46b8e1f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.951 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.945+0000 7f7221968640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55b46b8da9e0 0x55b46b8eb9b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.951 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.945+0000 7f7222169640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55b46b8d9ff0 0x55b46b8e1f40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:45326/0 (socket says 172.21.15.99:45326) 2024-03-19T01:40:19.951 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.945+0000 7f7222169640 1 -- 172.21.15.99:0/1492142582 learned_addr learned my addr 172.21.15.99:0/1492142582 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:19.951 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.945+0000 7f7221968640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55b46b8da9e0 0x55b46b8eb9b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:40374/0 (socket says 172.21.15.99:40374) 2024-03-19T01:40:19.951 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.945+0000 7f722296a640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55b46b8dfb50 0x55b46b8d5940 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:53084/0 (socket says 172.21.15.99:53084) 2024-03-19T01:40:19.951 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.945+0000 7f7222169640 1 -- 172.21.15.99:0/1492142582 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55b46b8da9e0 msgr2=0x55b46b8eb9b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:40:19.951 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.945+0000 7f7222169640 1 --2- 172.21.15.99:0/1492142582 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55b46b8da9e0 0x55b46b8eb9b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.951 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.945+0000 7f7222169640 1 -- 172.21.15.99:0/1492142582 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55b46b8dfb50 msgr2=0x55b46b8d5940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:40:19.951 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.945+0000 7f7222169640 1 --2- 172.21.15.99:0/1492142582 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55b46b8dfb50 0x55b46b8d5940 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.952 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.945+0000 7f7222169640 1 -- 172.21.15.99:0/1492142582 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55b46b8d0670 con 0x55b46b8d9ff0 2024-03-19T01:40:19.952 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.945+0000 7f7221968640 1 --2- 172.21.15.99:0/1492142582 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55b46b8da9e0 0x55b46b8eb9b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:40:19.952 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.945+0000 7f7222169640 1 --2- 172.21.15.99:0/1492142582 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55b46b8d9ff0 0x55b46b8e1f40 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7f7218036040 tx=0x7f7218005fd0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=7268fd114bdad929 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:19.952 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.946+0000 7f7207fff640 1 -- 172.21.15.99:0/1492142582 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7218046b10 con 0x55b46b8d9ff0 2024-03-19T01:40:19.952 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.946+0000 7f7207fff640 1 -- 172.21.15.99:0/1492142582 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7218046cb0 con 0x55b46b8d9ff0 2024-03-19T01:40:19.952 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.946+0000 7f7207fff640 1 -- 172.21.15.99:0/1492142582 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7218003970 con 0x55b46b8d9ff0 2024-03-19T01:40:19.952 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.946+0000 7f7226fb8740 1 -- 172.21.15.99:0/1492142582 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55b46b8d9ff0 msgr2=0x55b46b8e1f40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:19.952 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.946+0000 7f7226fb8740 1 --2- 172.21.15.99:0/1492142582 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55b46b8d9ff0 0x55b46b8e1f40 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7f7218036040 tx=0x7f7218005fd0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.952 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.946+0000 7f7226fb8740 1 -- 172.21.15.99:0/1492142582 shutdown_connections 2024-03-19T01:40:19.952 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.946+0000 7f7226fb8740 1 --2- 172.21.15.99:0/1492142582 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55b46b8dfb50 0x55b46b8d5940 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.952 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.946+0000 7f7226fb8740 1 --2- 172.21.15.99:0/1492142582 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55b46b8da9e0 0x55b46b8eb9b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.952 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.946+0000 7f7226fb8740 1 --2- 172.21.15.99:0/1492142582 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55b46b8d9ff0 0x55b46b8e1f40 unknown :-1 s=CLOSED pgs=72 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.952 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.946+0000 7f7226fb8740 1 -- 172.21.15.99:0/1492142582 >> 172.21.15.99:0/1492142582 conn(0x55b46b8d5560 msgr2=0x55b46b8db480 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:19.952 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.946+0000 7f7226fb8740 1 -- 172.21.15.99:0/1492142582 shutdown_connections 2024-03-19T01:40:19.953 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.946+0000 7f7226fb8740 1 -- 172.21.15.99:0/1492142582 wait complete. 2024-03-19T01:40:19.953 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.947+0000 7f7226fb8740 1 -- start start 2024-03-19T01:40:19.953 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.947+0000 7f7226fb8740 1 Processor -- start 2024-03-19T01:40:19.953 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.947+0000 7f7226fb8740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55b46b8d9ff0 0x55b46b8de8a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.953 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.947+0000 7f7226fb8740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55b46b8da9e0 0x55b46b8eb9b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.953 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.947+0000 7f7226fb8740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55b46b8dfb50 0x55b46b954eb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.953 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.947+0000 7f7226fb8740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55b46b8f2440 con 0x55b46b8da9e0 2024-03-19T01:40:19.953 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.947+0000 7f7226fb8740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55b46b8f1d70 con 0x55b46b8d9ff0 2024-03-19T01:40:19.953 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.947+0000 7f7226fb8740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55b46b8f2b10 con 0x55b46b8dfb50 2024-03-19T01:40:19.953 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.947+0000 7f7226fb8740 10 client.0 _ll_register_callbacks cb 0x7f721401f020 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:40:19.953 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.947+0000 7f7221968640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55b46b8da9e0 0x55b46b8eb9b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.953 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.947+0000 7f7221968640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55b46b8da9e0 0x55b46b8eb9b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:53092/0 (socket says 172.21.15.99:53092) 2024-03-19T01:40:19.954 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.947+0000 7f7221968640 1 -- 172.21.15.99:0/4127094516 learned_addr learned my addr 172.21.15.99:0/4127094516 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:19.954 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.947+0000 7f7222169640 1 --2- 172.21.15.99:0/4127094516 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55b46b8d9ff0 0x55b46b8de8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.954 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.947+0000 7f722296a640 1 --2- 172.21.15.99:0/4127094516 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55b46b8dfb50 0x55b46b954eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.954 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.947+0000 7f7221968640 1 -- 172.21.15.99:0/4127094516 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55b46b8dfb50 msgr2=0x55b46b954eb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:40:19.954 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.947+0000 7f7221968640 1 --2- 172.21.15.99:0/4127094516 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55b46b8dfb50 0x55b46b954eb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.954 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.947+0000 7f7221968640 1 -- 172.21.15.99:0/4127094516 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55b46b8d9ff0 msgr2=0x55b46b8de8a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:40:19.954 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.947+0000 7f7221968640 1 --2- 172.21.15.99:0/4127094516 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55b46b8d9ff0 0x55b46b8de8a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.954 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.947+0000 7f7221968640 1 -- 172.21.15.99:0/4127094516 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55b46b9555a0 con 0x55b46b8da9e0 2024-03-19T01:40:19.954 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.947+0000 7f7222169640 1 --2- 172.21.15.99:0/4127094516 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55b46b8d9ff0 0x55b46b8de8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:40:19.954 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.947+0000 7f7221968640 1 --2- 172.21.15.99:0/4127094516 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55b46b8da9e0 0x55b46b8eb9b0 secure :-1 s=READY pgs=304 cs=0 l=1 rev1=1 crypto rx=0x7f720c097b50 tx=0x7f720c0cef90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1c12dc9cc4587ad5 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:19.954 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.947+0000 7f7205ffb640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:40:19.954 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.948+0000 7f7205ffb640 1 -- 172.21.15.99:0/4127094516 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f720c09b230 con 0x55b46b8da9e0 2024-03-19T01:40:19.954 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.948+0000 7f7205ffb640 1 -- 172.21.15.99:0/4127094516 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f720c003b30 con 0x55b46b8da9e0 2024-03-19T01:40:19.954 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.948+0000 7f7226fb8740 1 -- 172.21.15.99:0/4127094516 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f7218046cb0 con 0x55b46b8da9e0 2024-03-19T01:40:19.955 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.948+0000 7f7205ffb640 1 -- 172.21.15.99:0/4127094516 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=20) ==== 24+0+0 (secure 0 0 0) 0x7f720c09b050 con 0x55b46b8da9e0 2024-03-19T01:40:19.955 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.948+0000 7f7226fb8740 10 client.15126 fetch_fsmap learned FSMap version 20 2024-03-19T01:40:19.955 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.948+0000 7f7226fb8740 1 -- 172.21.15.99:0/4127094516 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=20,monmap=4+}) -- 0x55b46b9569c0 con 0x55b46b8da9e0 2024-03-19T01:40:19.955 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.948+0000 7f7205ffb640 1 -- 172.21.15.99:0/4127094516 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7218046cb0 con 0x55b46b8da9e0 2024-03-19T01:40:19.955 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.948+0000 7f7205ffb640 1 -- 172.21.15.99:0/4127094516 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 20) ==== 8338+0+0 (secure 0 0 0) 0x7f720c09ac60 con 0x55b46b8da9e0 2024-03-19T01:40:19.955 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.948+0000 7f7226fb8740 10 client.15126 fetch_fsmap finished waiting for FSMap version 20 2024-03-19T01:40:19.955 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.948+0000 7f7226fb8740 10 client.15126 resolve_mds: resolved name 'i' to daemon mds.i 2024-03-19T01:40:19.955 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.948+0000 7f7226fb8740 20 client.15126 populate_metadata read hostname 'smithi099' 2024-03-19T01:40:19.955 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.948+0000 7f7226fb8740 1 --2- 172.21.15.99:0/4127094516 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55b46b95b510 0x55b46b95a220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:19.955 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.948+0000 7f7226fb8740 4 client.15126 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "status"}] 2024-03-19T01:40:19.955 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.948+0000 7f7226fb8740 1 -- 172.21.15.99:0/4127094516 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "status"}) -- 0x55b46b846670 con 0x55b46b95b510 2024-03-19T01:40:19.955 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.948+0000 7f7222169640 1 --2- 172.21.15.99:0/4127094516 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55b46b95b510 0x55b46b95a220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:19.955 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.949+0000 7f7222169640 1 --2- 172.21.15.99:0/4127094516 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55b46b95b510 0x55b46b95a220 crc :-1 s=READY pgs=22 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=810dd771e341ab3a server_cookie=22458f1a8056939d in_seq=0 out_seq=0 2024-03-19T01:40:19.955 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.949+0000 7f7205ffb640 10 client.15126 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:40:19.956 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.949+0000 7f7205ffb640 1 -- 172.21.15.99:0/4127094516 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+332 (crc 0 0 0) 0x55b46b846670 con 0x55b46b95b510 2024-03-19T01:40:19.956 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.949+0000 7f7205ffb640 10 client.15126 handle_command_reply: tid=0 2024-03-19T01:40:19.956 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.949+0000 7f7226fb8740 1 client.15126 shutdown 2024-03-19T01:40:19.956 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.949+0000 7f7226fb8740 1 -- 172.21.15.99:0/4127094516 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55b46b8da9e0 msgr2=0x55b46b8eb9b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:19.956 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.949+0000 7f7226fb8740 1 --2- 172.21.15.99:0/4127094516 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55b46b8da9e0 0x55b46b8eb9b0 secure :-1 s=READY pgs=304 cs=0 l=1 rev1=1 crypto rx=0x7f720c097b50 tx=0x7f720c0cef90 comp rx=0 tx=0).stop 2024-03-19T01:40:19.956 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.949+0000 7f7226fb8740 1 -- 172.21.15.99:0/4127094516 shutdown_connections 2024-03-19T01:40:19.956 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.949+0000 7f7226fb8740 1 --2- 172.21.15.99:0/4127094516 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55b46b95b510 0x55b46b95a220 crc :-1 s=READY pgs=22 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.956 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.949+0000 7f7226fb8740 1 --2- 172.21.15.99:0/4127094516 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55b46b8dfb50 0x55b46b954eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.956 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.949+0000 7f7226fb8740 1 --2- 172.21.15.99:0/4127094516 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55b46b8da9e0 0x55b46b8eb9b0 unknown :-1 s=CLOSED pgs=304 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.956 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.949+0000 7f7205ffb640 0 client.15126 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:40:19.956 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.949+0000 7f7226fb8740 1 --2- 172.21.15.99:0/4127094516 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55b46b8d9ff0 0x55b46b8de8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.956 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.949+0000 7f7226fb8740 1 -- 172.21.15.99:0/4127094516 >> 172.21.15.99:0/4127094516 conn(0x55b46b8d5560 msgr2=0x55b46b8d5940 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:19.956 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.949+0000 7f7226fb8740 1 -- 172.21.15.99:0/4127094516 shutdown_connections 2024-03-19T01:40:19.956 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.949+0000 7f7226fb8740 1 -- 172.21.15.99:0/4127094516 wait complete. 2024-03-19T01:40:19.957 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.949+0000 7f7226fb8740 20 client.15126 trim_cache size 0 max 16384 2024-03-19T01:40:19.957 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2024-03-19T01:40:19.957 INFO:teuthology.orchestra.run.smithi099.stdout: "cluster_fsid": "438fb69c-e590-11ee-95c9-87774f69a715", 2024-03-19T01:40:19.957 INFO:teuthology.orchestra.run.smithi099.stdout: "whoami": 0, 2024-03-19T01:40:19.957 INFO:teuthology.orchestra.run.smithi099.stdout: "id": 24487, 2024-03-19T01:40:19.957 INFO:teuthology.orchestra.run.smithi099.stdout: "want_state": "up:active", 2024-03-19T01:40:19.957 INFO:teuthology.orchestra.run.smithi099.stdout: "state": "up:active", 2024-03-19T01:40:19.957 INFO:teuthology.orchestra.run.smithi099.stdout: "fs_name": "cephfs", 2024-03-19T01:40:19.957 INFO:teuthology.orchestra.run.smithi099.stdout: "rank_uptime": 83.158791231999999, 2024-03-19T01:40:19.957 INFO:teuthology.orchestra.run.smithi099.stdout: "mdsmap_epoch": 20, 2024-03-19T01:40:19.957 INFO:teuthology.orchestra.run.smithi099.stdout: "osdmap_epoch": 91, 2024-03-19T01:40:19.958 INFO:teuthology.orchestra.run.smithi099.stdout: "osdmap_epoch_barrier": 87, 2024-03-19T01:40:19.958 INFO:teuthology.orchestra.run.smithi099.stdout: "uptime": 98.266486561999997 2024-03-19T01:40:19.958 INFO:teuthology.orchestra.run.smithi099.stdout:} 2024-03-19T01:40:19.959 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.950+0000 7f72243cd640 1 -- 172.21.15.99:0/3316836195 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f72080a2800 msgr2=0x7f72080a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:19.959 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.950+0000 7f72243cd640 1 --2- 172.21.15.99:0/3316836195 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f72080a2800 0x7f72080a4cc0 secure :-1 s=READY pgs=222 cs=0 l=1 rev1=1 crypto rx=0x7f7214020000 tx=0x7f7214021000 comp rx=0 tx=0).stop 2024-03-19T01:40:19.959 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.950+0000 7f72243cd640 1 -- 172.21.15.99:0/3316836195 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f721c102a40 msgr2=0x7f721c1980c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:19.959 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.950+0000 7f72243cd640 1 --2- 172.21.15.99:0/3316836195 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f721c102a40 0x7f721c1980c0 secure :-1 s=READY pgs=302 cs=0 l=1 rev1=1 crypto rx=0x7f720c00ea60 tx=0x7f720c00ef20 comp rx=0 tx=0).stop 2024-03-19T01:40:19.959 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.950+0000 7f72243cd640 1 -- 172.21.15.99:0/3316836195 shutdown_connections 2024-03-19T01:40:19.959 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.950+0000 7f72243cd640 1 --2- 172.21.15.99:0/3316836195 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f721c103400 0x7f721c19f130 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.959 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.950+0000 7f72243cd640 1 --2- 172.21.15.99:0/3316836195 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f721c102a40 0x7f721c1980c0 unknown :-1 s=CLOSED pgs=302 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.959 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.950+0000 7f72243cd640 1 --2- 172.21.15.99:0/3316836195 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f72080a2800 0x7f72080a4cc0 unknown :-1 s=CLOSED pgs=222 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.959 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.950+0000 7f72243cd640 1 --2- 172.21.15.99:0/3316836195 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f721c101850 0x7f721c197b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:19.959 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.950+0000 7f72243cd640 1 -- 172.21.15.99:0/3316836195 >> 172.21.15.99:0/3316836195 conn(0x7f721c0fcdc0 msgr2=0x7f721c105d40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:19.959 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.950+0000 7f72243cd640 1 -- 172.21.15.99:0/3316836195 shutdown_connections 2024-03-19T01:40:19.959 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:19.950+0000 7f72243cd640 1 -- 172.21.15.99:0/3316836195 wait complete. 2024-03-19T01:40:19.961 INFO:tasks.cephadm:Setup complete, yielding 2024-03-19T01:40:19.962 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-03-19T01:40:19.972 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi099.front.sepia.ceph.com 2024-03-19T01:40:19.972 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- bash -c 'ceph orch status' 2024-03-19T01:40:20.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:19 smithi099 ceph-mon[62951]: pgmap v327: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 409 B/s wr, 6 op/s 2024-03-19T01:40:20.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:19 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/90418591' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:40:20.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:19 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/4238660414' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-19T01:40:20.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:19 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/2020662108' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:40:20.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:19 smithi112 ceph-mon[66819]: pgmap v327: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 409 B/s wr, 6 op/s 2024-03-19T01:40:20.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:19 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/90418591' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:40:20.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:19 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/4238660414' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-19T01:40:20.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:19 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/2020662108' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:40:20.211 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:40:20.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:19 smithi115 ceph-mon[67332]: pgmap v327: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 409 B/s wr, 6 op/s 2024-03-19T01:40:20.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:19 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/90418591' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:40:20.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:19 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/4238660414' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-19T01:40:20.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:19 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/2020662108' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:40:21.844 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:21.842+0000 7f06ce96c700 1 -- 172.21.15.99:0/3618266060 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f06c8102dd0 msgr2=0x7f06c81051c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:21.844 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:21.842+0000 7f06ce96c700 1 --2- 172.21.15.99:0/3618266060 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f06c8102dd0 0x7f06c81051c0 secure :-1 s=READY pgs=305 cs=0 l=1 rev1=1 crypto rx=0x7f06bc009990 tx=0x7f06bc009ca0 comp rx=0 tx=0).stop 2024-03-19T01:40:21.844 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:21.842+0000 7f06ce96c700 1 -- 172.21.15.99:0/3618266060 shutdown_connections 2024-03-19T01:40:21.844 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:21.842+0000 7f06ce96c700 1 --2- 172.21.15.99:0/3618266060 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f06c8108030 0x7f06c810a4a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:21.844 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:21.842+0000 7f06ce96c700 1 --2- 172.21.15.99:0/3618266060 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f06c8102dd0 0x7f06c81051c0 unknown :-1 s=CLOSED pgs=305 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:21.845 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:21.842+0000 7f06ce96c700 1 --2- 172.21.15.99:0/3618266060 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f06c8105700 0x7f06c8107af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:21.845 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:21.842+0000 7f06ce96c700 1 -- 172.21.15.99:0/3618266060 >> 172.21.15.99:0/3618266060 conn(0x7f06c80fc800 msgr2=0x7f06c80fec20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:21.845 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:21.842+0000 7f06ce96c700 1 -- 172.21.15.99:0/3618266060 shutdown_connections 2024-03-19T01:40:21.845 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:21.842+0000 7f06ce96c700 1 -- 172.21.15.99:0/3618266060 wait complete. 2024-03-19T01:40:21.845 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:21.842+0000 7f06ce96c700 1 Processor -- start 2024-03-19T01:40:21.845 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:21.842+0000 7f06ce96c700 1 -- start start 2024-03-19T01:40:21.845 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:21.843+0000 7f06ce96c700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f06c8102dd0 0x7f06c8198570 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:21.845 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:21.843+0000 7f06ce96c700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f06c8105700 0x7f06c8198ab0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:21.845 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:21.843+0000 7f06ce96c700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f06c8108030 0x7f06c819fb30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:21.845 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:21.843+0000 7f06ce96c700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f06c810df70 con 0x7f06c8105700 2024-03-19T01:40:21.845 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:21.843+0000 7f06ce96c700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f06c810ddf0 con 0x7f06c8102dd0 2024-03-19T01:40:21.846 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:21.843+0000 7f06ce96c700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f06c810e0f0 con 0x7f06c8108030 2024-03-19T01:40:21.846 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:21.843+0000 7f06c77fe700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f06c8105700 0x7f06c8198ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:21.846 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:21.843+0000 7f06c7fff700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f06c8102dd0 0x7f06c8198570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:21.846 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:21.843+0000 7f06c77fe700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f06c8105700 0x7f06c8198ab0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:53104/0 (socket says 172.21.15.99:53104) 2024-03-19T01:40:21.846 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:21.843+0000 7f06ccf09700 1 --2- 172.21.15.99:0/4217589757 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f06c8108030 0x7f06c819fb30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:21.846 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:21.843+0000 7f06c77fe700 1 -- 172.21.15.99:0/4217589757 learned_addr learned my addr 172.21.15.99:0/4217589757 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:21.846 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:21.843+0000 7f06c7fff700 1 -- 172.21.15.99:0/4217589757 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f06c8108030 msgr2=0x7f06c819fb30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:21.846 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:21.843+0000 7f06c7fff700 1 --2- 172.21.15.99:0/4217589757 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f06c8108030 0x7f06c819fb30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:21.846 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:21.843+0000 7f06c7fff700 1 -- 172.21.15.99:0/4217589757 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f06c8105700 msgr2=0x7f06c8198ab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:21.846 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:21.843+0000 7f06c7fff700 1 --2- 172.21.15.99:0/4217589757 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f06c8105700 0x7f06c8198ab0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:21.846 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:21.843+0000 7f06c7fff700 1 -- 172.21.15.99:0/4217589757 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f06c81a01d0 con 0x7f06c8102dd0 2024-03-19T01:40:21.847 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:21.843+0000 7f06ccf09700 1 --2- 172.21.15.99:0/4217589757 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f06c8108030 0x7f06c819fb30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:40:21.847 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:21.843+0000 7f06c7fff700 1 --2- 172.21.15.99:0/4217589757 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f06c8102dd0 0x7f06c8198570 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7f06bc006010 tx=0x7f06bc00bb90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:21.847 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:21.843+0000 7f06c57fa700 1 -- 172.21.15.99:0/4217589757 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f06bc026070 con 0x7f06c8102dd0 2024-03-19T01:40:21.847 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:21.843+0000 7f06ce96c700 1 -- 172.21.15.99:0/4217589757 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f06c81a0460 con 0x7f06c8102dd0 2024-03-19T01:40:21.847 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:21.843+0000 7f06c57fa700 1 -- 172.21.15.99:0/4217589757 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f06bc003d70 con 0x7f06c8102dd0 2024-03-19T01:40:21.847 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:21.843+0000 7f06c57fa700 1 -- 172.21.15.99:0/4217589757 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f06bc021410 con 0x7f06c8102dd0 2024-03-19T01:40:21.847 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:21.844+0000 7f06ce96c700 1 -- 172.21.15.99:0/4217589757 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f06c81a0910 con 0x7f06c8102dd0 2024-03-19T01:40:21.848 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:21.844+0000 7f06ce96c700 1 -- 172.21.15.99:0/4217589757 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f06c8068fd0 con 0x7f06c8102dd0 2024-03-19T01:40:21.848 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:21.846+0000 7f06c57fa700 1 -- 172.21.15.99:0/4217589757 <== mon.2 v2:172.21.15.112:3300/0 4 ==== mgrmap(e 18) ==== 137290+0+0 (secure 0 0 0) 0x7f06bc029070 con 0x7f06c8102dd0 2024-03-19T01:40:21.848 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:21.846+0000 7f06c57fa700 1 --2- 172.21.15.99:0/4217589757 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f06b00a2890 0x7f06b00a4d50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:21.848 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:21.846+0000 7f06c77fe700 1 --2- 172.21.15.99:0/4217589757 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f06b00a2890 0x7f06b00a4d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:21.848 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:21.846+0000 7f06c57fa700 1 -- 172.21.15.99:0/4217589757 <== mon.2 v2:172.21.15.112:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f06bc0cfe00 con 0x7f06c8102dd0 2024-03-19T01:40:21.848 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:21.847+0000 7f06c77fe700 1 --2- 172.21.15.99:0/4217589757 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f06b00a2890 0x7f06b00a4d50 secure :-1 s=READY pgs=223 cs=0 l=1 rev1=1 crypto rx=0x7f06b8009d60 tx=0x7f06b8009260 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:21.853 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:21.853+0000 7f06c57fa700 1 -- 172.21.15.99:0/4217589757 <== mon.2 v2:172.21.15.112:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f06bc030080 con 0x7f06c8102dd0 2024-03-19T01:40:22.067 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:22.067+0000 7f06ce96c700 1 -- 172.21.15.99:0/4217589757 --> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] -- mgr_command(tid 0: {"prefix": "orch status", "target": ["mon-mgr", ""]}) -- 0x7f06c8062750 con 0x7f06b00a2890 2024-03-19T01:40:22.068 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:22.068+0000 7f06c57fa700 1 -- 172.21.15.99:0/4217589757 <== mgr.14150 v2:172.21.15.99:6800/4272410295 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+43 (secure 0 0 0) 0x7f06c8062750 con 0x7f06b00a2890 2024-03-19T01:40:22.068 INFO:teuthology.orchestra.run.smithi099.stdout:Backend: cephadm 2024-03-19T01:40:22.068 INFO:teuthology.orchestra.run.smithi099.stdout:Available: Yes 2024-03-19T01:40:22.068 INFO:teuthology.orchestra.run.smithi099.stdout:Paused: No 2024-03-19T01:40:22.070 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:22.070+0000 7f06ce96c700 1 -- 172.21.15.99:0/4217589757 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f06b00a2890 msgr2=0x7f06b00a4d50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:22.070 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:22.070+0000 7f06ce96c700 1 --2- 172.21.15.99:0/4217589757 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f06b00a2890 0x7f06b00a4d50 secure :-1 s=READY pgs=223 cs=0 l=1 rev1=1 crypto rx=0x7f06b8009d60 tx=0x7f06b8009260 comp rx=0 tx=0).stop 2024-03-19T01:40:22.070 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:22.070+0000 7f06ce96c700 1 -- 172.21.15.99:0/4217589757 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f06c8102dd0 msgr2=0x7f06c8198570 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:22.070 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:22.070+0000 7f06ce96c700 1 --2- 172.21.15.99:0/4217589757 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f06c8102dd0 0x7f06c8198570 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7f06bc006010 tx=0x7f06bc00bb90 comp rx=0 tx=0).stop 2024-03-19T01:40:22.071 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:22.070+0000 7f06ce96c700 1 -- 172.21.15.99:0/4217589757 shutdown_connections 2024-03-19T01:40:22.071 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:22.070+0000 7f06ce96c700 1 --2- 172.21.15.99:0/4217589757 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f06b00a2890 0x7f06b00a4d50 unknown :-1 s=CLOSED pgs=223 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:22.071 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:22.070+0000 7f06ce96c700 1 --2- 172.21.15.99:0/4217589757 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f06c8108030 0x7f06c819fb30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:22.071 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:22.070+0000 7f06ce96c700 1 --2- 172.21.15.99:0/4217589757 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f06c8102dd0 0x7f06c8198570 unknown :-1 s=CLOSED pgs=73 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:22.071 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:22.070+0000 7f06ce96c700 1 --2- 172.21.15.99:0/4217589757 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f06c8105700 0x7f06c8198ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:22.071 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:22.070+0000 7f06ce96c700 1 -- 172.21.15.99:0/4217589757 >> 172.21.15.99:0/4217589757 conn(0x7f06c80fc800 msgr2=0x7f06c8103850 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:22.071 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:22.070+0000 7f06ce96c700 1 -- 172.21.15.99:0/4217589757 shutdown_connections 2024-03-19T01:40:22.071 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:22.070+0000 7f06ce96c700 1 -- 172.21.15.99:0/4217589757 wait complete. 2024-03-19T01:40:22.104 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:21 smithi099 ceph-mon[62951]: pgmap v328: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.6 KiB/s rd, 343 B/s wr, 7 op/s 2024-03-19T01:40:22.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:21 smithi112 ceph-mon[66819]: pgmap v328: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.6 KiB/s rd, 343 B/s wr, 7 op/s 2024-03-19T01:40:22.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:21 smithi115 ceph-mon[67332]: pgmap v328: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.6 KiB/s rd, 343 B/s wr, 7 op/s 2024-03-19T01:40:22.738 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- bash -c 'ceph orch ps' 2024-03-19T01:40:22.987 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:40:24.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:23 smithi099 ceph-mon[62951]: from='client.24908 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:40:24.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:23 smithi099 ceph-mon[62951]: pgmap v329: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.1 KiB/s rd, 0 B/s wr, 6 op/s 2024-03-19T01:40:24.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:23 smithi112 ceph-mon[66819]: from='client.24908 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:40:24.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:23 smithi112 ceph-mon[66819]: pgmap v329: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.1 KiB/s rd, 0 B/s wr, 6 op/s 2024-03-19T01:40:24.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:23 smithi115 ceph-mon[67332]: from='client.24908 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:40:24.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:23 smithi115 ceph-mon[67332]: pgmap v329: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.1 KiB/s rd, 0 B/s wr, 6 op/s 2024-03-19T01:40:24.641 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:24.639+0000 7fef51f92700 1 -- 172.21.15.99:0/3102281307 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fef4c100600 msgr2=0x7fef4c100a20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:24.641 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:24.639+0000 7fef51f92700 1 --2- 172.21.15.99:0/3102281307 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fef4c100600 0x7fef4c100a20 secure :-1 s=READY pgs=306 cs=0 l=1 rev1=1 crypto rx=0x7fef4400b0a0 tx=0x7fef4400b3b0 comp rx=0 tx=0).stop 2024-03-19T01:40:24.641 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:24.639+0000 7fef51f92700 1 -- 172.21.15.99:0/3102281307 shutdown_connections 2024-03-19T01:40:24.641 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:24.639+0000 7fef51f92700 1 --2- 172.21.15.99:0/3102281307 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fef4c103f80 0x7fef4c10a830 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:24.641 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:24.639+0000 7fef51f92700 1 --2- 172.21.15.99:0/3102281307 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fef4c100600 0x7fef4c100a20 unknown :-1 s=CLOSED pgs=306 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:24.641 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:24.639+0000 7fef51f92700 1 --2- 172.21.15.99:0/3102281307 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fef4c1035e0 0x7fef4c103a40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:24.642 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:24.639+0000 7fef51f92700 1 -- 172.21.15.99:0/3102281307 >> 172.21.15.99:0/3102281307 conn(0x7fef4c0fbf80 msgr2=0x7fef4c0fe3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:24.642 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:24.639+0000 7fef51f92700 1 -- 172.21.15.99:0/3102281307 shutdown_connections 2024-03-19T01:40:24.642 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:24.639+0000 7fef51f92700 1 -- 172.21.15.99:0/3102281307 wait complete. 2024-03-19T01:40:24.642 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:24.639+0000 7fef51f92700 1 Processor -- start 2024-03-19T01:40:24.642 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:24.640+0000 7fef51f92700 1 -- start start 2024-03-19T01:40:24.642 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:24.640+0000 7fef51f92700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fef4c100600 0x7fef4c102f40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:24.642 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:24.640+0000 7fef51f92700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fef4c1035e0 0x7fef4c101590 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:24.642 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:24.640+0000 7fef51f92700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fef4c103f80 0x7fef4c101ad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:24.642 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:24.640+0000 7fef51f92700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fef4c10df20 con 0x7fef4c1035e0 2024-03-19T01:40:24.642 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:24.640+0000 7fef51f92700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fef4c10dda0 con 0x7fef4c100600 2024-03-19T01:40:24.642 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:24.640+0000 7fef51f92700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fef4c10e0a0 con 0x7fef4c103f80 2024-03-19T01:40:24.642 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:24.640+0000 7fef4b7fe700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fef4c100600 0x7fef4c102f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:24.642 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:24.640+0000 7fef4affd700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fef4c1035e0 0x7fef4c101590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:24.643 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:24.640+0000 7fef4b7fe700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fef4c100600 0x7fef4c102f40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:46890/0 (socket says 172.21.15.99:46890) 2024-03-19T01:40:24.643 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:24.640+0000 7fef4bfff700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fef4c103f80 0x7fef4c101ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:24.643 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:24.640+0000 7fef4b7fe700 1 -- 172.21.15.99:0/485603042 learned_addr learned my addr 172.21.15.99:0/485603042 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:24.643 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:24.640+0000 7fef4b7fe700 1 -- 172.21.15.99:0/485603042 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fef4c103f80 msgr2=0x7fef4c101ad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:24.643 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:24.640+0000 7fef4b7fe700 1 --2- 172.21.15.99:0/485603042 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fef4c103f80 0x7fef4c101ad0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:24.643 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:24.640+0000 7fef4b7fe700 1 -- 172.21.15.99:0/485603042 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fef4c1035e0 msgr2=0x7fef4c101590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:24.643 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:24.640+0000 7fef4b7fe700 1 --2- 172.21.15.99:0/485603042 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fef4c1035e0 0x7fef4c101590 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:24.643 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:24.640+0000 7fef4b7fe700 1 -- 172.21.15.99:0/485603042 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fef4c102380 con 0x7fef4c100600 2024-03-19T01:40:24.643 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:24.640+0000 7fef4bfff700 1 --2- 172.21.15.99:0/485603042 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fef4c103f80 0x7fef4c101ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:40:24.643 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:24.641+0000 7fef4b7fe700 1 --2- 172.21.15.99:0/485603042 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fef4c100600 0x7fef4c102f40 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x7fef44007fa0 tx=0x7fef44004d80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:24.643 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:24.641+0000 7fef48ff9700 1 -- 172.21.15.99:0/485603042 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fef44021070 con 0x7fef4c100600 2024-03-19T01:40:24.643 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:24.641+0000 7fef48ff9700 1 -- 172.21.15.99:0/485603042 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fef44007b80 con 0x7fef4c100600 2024-03-19T01:40:24.643 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:24.641+0000 7fef51f92700 1 -- 172.21.15.99:0/485603042 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fef4c1a2660 con 0x7fef4c100600 2024-03-19T01:40:24.644 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:24.641+0000 7fef48ff9700 1 -- 172.21.15.99:0/485603042 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fef440052a0 con 0x7fef4c100600 2024-03-19T01:40:24.644 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:24.641+0000 7fef51f92700 1 -- 172.21.15.99:0/485603042 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fef4c1a2b70 con 0x7fef4c100600 2024-03-19T01:40:24.645 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:24.642+0000 7fef51f92700 1 -- 172.21.15.99:0/485603042 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fef4c068fd0 con 0x7fef4c100600 2024-03-19T01:40:24.645 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:24.643+0000 7fef48ff9700 1 -- 172.21.15.99:0/485603042 <== mon.2 v2:172.21.15.112:3300/0 4 ==== mgrmap(e 18) ==== 137290+0+0 (secure 0 0 0) 0x7fef44024070 con 0x7fef4c100600 2024-03-19T01:40:24.645 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:24.643+0000 7fef48ff9700 1 --2- 172.21.15.99:0/485603042 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fef340a2890 0x7fef340a4d50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:24.645 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:24.643+0000 7fef4affd700 1 --2- 172.21.15.99:0/485603042 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fef340a2890 0x7fef340a4d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:24.645 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:24.643+0000 7fef48ff9700 1 -- 172.21.15.99:0/485603042 <== mon.2 v2:172.21.15.112:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7fef4400f020 con 0x7fef4c100600 2024-03-19T01:40:24.645 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:24.644+0000 7fef4affd700 1 --2- 172.21.15.99:0/485603042 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fef340a2890 0x7fef340a4d50 secure :-1 s=READY pgs=224 cs=0 l=1 rev1=1 crypto rx=0x7fef40009c60 tx=0x7fef40009260 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:24.646 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:24.647+0000 7fef48ff9700 1 -- 172.21.15.99:0/485603042 <== mon.2 v2:172.21.15.112:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7fef4402b020 con 0x7fef4c100600 2024-03-19T01:40:24.855 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:24.855+0000 7fef51f92700 1 -- 172.21.15.99:0/485603042 --> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] -- mgr_command(tid 0: {"prefix": "orch ps", "target": ["mon-mgr", ""]}) -- 0x7fef4c062750 con 0x7fef340a2890 2024-03-19T01:40:24.868 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:24.868+0000 7fef48ff9700 1 -- 172.21.15.99:0/485603042 <== mgr.14150 v2:172.21.15.99:6800/4272410295 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+4185 (secure 0 0 0) 0x7fef4c062750 con 0x7fef340a2890 2024-03-19T01:40:24.868 INFO:teuthology.orchestra.run.smithi099.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-19T01:40:24.868 INFO:teuthology.orchestra.run.smithi099.stdout:mds.a smithi099 running (97s) 16s ago 97s 13.7M - 19.0.0-2206-gb6ae2491 2312c8e42d00 4b3805d58da2 2024-03-19T01:40:24.868 INFO:teuthology.orchestra.run.smithi099.stdout:mds.b smithi112 running (95s) 28s ago 94s 17.9M - 19.0.0-2206-gb6ae2491 2312c8e42d00 de00165d1e05 2024-03-19T01:40:24.869 INFO:teuthology.orchestra.run.smithi099.stdout:mds.c smithi115 running (108s) 28s ago 108s 16.5M - 19.0.0-2206-gb6ae2491 2312c8e42d00 ff3f5e67fed9 2024-03-19T01:40:24.869 INFO:teuthology.orchestra.run.smithi099.stdout:mds.d smithi099 running (116s) 16s ago 115s 12.9M - 19.0.0-2206-gb6ae2491 2312c8e42d00 3051f64272da 2024-03-19T01:40:24.869 INFO:teuthology.orchestra.run.smithi099.stdout:mds.e smithi112 running (113s) 28s ago 113s 16.8M - 19.0.0-2206-gb6ae2491 2312c8e42d00 3378ef045bc8 2024-03-19T01:40:24.869 INFO:teuthology.orchestra.run.smithi099.stdout:mds.f smithi115 running (111s) 28s ago 110s 19.3M - 19.0.0-2206-gb6ae2491 2312c8e42d00 8b886dd7bfad 2024-03-19T01:40:24.869 INFO:teuthology.orchestra.run.smithi099.stdout:mds.g smithi099 running (100s) 16s ago 100s 15.2M - 19.0.0-2206-gb6ae2491 2312c8e42d00 f397a20a1088 2024-03-19T01:40:24.869 INFO:teuthology.orchestra.run.smithi099.stdout:mds.h smithi112 running (106s) 28s ago 105s 16.8M - 19.0.0-2206-gb6ae2491 2312c8e42d00 1b8f536d04b2 2024-03-19T01:40:24.869 INFO:teuthology.orchestra.run.smithi099.stdout:mds.i smithi115 running (103s) 28s ago 103s 28.2M - 19.0.0-2206-gb6ae2491 2312c8e42d00 03bb72380244 2024-03-19T01:40:24.869 INFO:teuthology.orchestra.run.smithi099.stdout:mds.j smithi099 running (119s) 16s ago 118s 16.3M - 19.0.0-2206-gb6ae2491 2312c8e42d00 ca9adb08a003 2024-03-19T01:40:24.869 INFO:teuthology.orchestra.run.smithi099.stdout:mds.k smithi112 running (2m) 28s ago 2m 13.7M - 19.0.0-2206-gb6ae2491 2312c8e42d00 20eddeea6ac3 2024-03-19T01:40:24.869 INFO:teuthology.orchestra.run.smithi099.stdout:mds.l smithi115 running (2m) 28s ago 2m 12.9M - 19.0.0-2206-gb6ae2491 2312c8e42d00 5547b1cc5bf1 2024-03-19T01:40:24.869 INFO:teuthology.orchestra.run.smithi099.stdout:mgr.x smithi099 *:9283,8765 running (9m) 16s ago 9m 513M - 19.0.0-2206-gb6ae2491 2312c8e42d00 565767728d8f 2024-03-19T01:40:24.870 INFO:teuthology.orchestra.run.smithi099.stdout:mgr.y smithi112 *:8443,8765 running (7m) 28s ago 7m 451M - 19.0.0-2206-gb6ae2491 2312c8e42d00 c0916e7a53e9 2024-03-19T01:40:24.870 INFO:teuthology.orchestra.run.smithi099.stdout:mgr.z smithi115 *:8443,8765 running (7m) 28s ago 7m 452M - 19.0.0-2206-gb6ae2491 2312c8e42d00 b2c0b74c9136 2024-03-19T01:40:24.870 INFO:teuthology.orchestra.run.smithi099.stdout:mon.a smithi099 running (9m) 16s ago 9m 50.7M 2048M 19.0.0-2206-gb6ae2491 2312c8e42d00 e320e9f5ecea 2024-03-19T01:40:24.870 INFO:teuthology.orchestra.run.smithi099.stdout:mon.b smithi112 running (7m) 28s ago 7m 41.7M 2048M 19.0.0-2206-gb6ae2491 2312c8e42d00 d293a73943f8 2024-03-19T01:40:24.870 INFO:teuthology.orchestra.run.smithi099.stdout:mon.c smithi115 running (7m) 28s ago 7m 37.8M 2048M 19.0.0-2206-gb6ae2491 2312c8e42d00 30f5aa3b44b1 2024-03-19T01:40:24.870 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0 smithi099 running (6m) 16s ago 6m 57.0M 4272M 19.0.0-2206-gb6ae2491 2312c8e42d00 81c9f16c5f7c 2024-03-19T01:40:24.870 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1 smithi112 running (6m) 28s ago 6m 51.7M 4270M 19.0.0-2206-gb6ae2491 2312c8e42d00 787465b96569 2024-03-19T01:40:24.870 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2 smithi115 running (6m) 28s ago 6m 49.9M 4270M 19.0.0-2206-gb6ae2491 2312c8e42d00 eda57b9f9f78 2024-03-19T01:40:24.870 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3 smithi099 running (5m) 16s ago 5m 55.2M 4272M 19.0.0-2206-gb6ae2491 2312c8e42d00 3ca45436f39e 2024-03-19T01:40:24.870 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4 smithi112 running (5m) 28s ago 5m 47.5M 4270M 19.0.0-2206-gb6ae2491 2312c8e42d00 f51be7fcdd3d 2024-03-19T01:40:24.870 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5 smithi115 running (4m) 28s ago 4m 54.5M 4270M 19.0.0-2206-gb6ae2491 2312c8e42d00 d9abad9bc879 2024-03-19T01:40:24.870 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6 smithi099 running (4m) 16s ago 4m 52.1M 4272M 19.0.0-2206-gb6ae2491 2312c8e42d00 5bc2c043b8ab 2024-03-19T01:40:24.870 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7 smithi112 running (4m) 28s ago 4m 53.9M 4270M 19.0.0-2206-gb6ae2491 2312c8e42d00 e99f85ee12ab 2024-03-19T01:40:24.871 INFO:teuthology.orchestra.run.smithi099.stdout:osd.8 smithi115 running (3m) 28s ago 3m 45.6M 4270M 19.0.0-2206-gb6ae2491 2312c8e42d00 edb84f05142c 2024-03-19T01:40:24.871 INFO:teuthology.orchestra.run.smithi099.stdout:osd.9 smithi099 running (3m) 16s ago 3m 52.0M 4272M 19.0.0-2206-gb6ae2491 2312c8e42d00 36df48bbbcfd 2024-03-19T01:40:24.871 INFO:teuthology.orchestra.run.smithi099.stdout:osd.10 smithi112 running (2m) 28s ago 2m 54.4M 4270M 19.0.0-2206-gb6ae2491 2312c8e42d00 141d2a5e8f28 2024-03-19T01:40:24.871 INFO:teuthology.orchestra.run.smithi099.stdout:osd.11 smithi115 running (2m) 28s ago 2m 46.7M 4270M 19.0.0-2206-gb6ae2491 2312c8e42d00 bdb2160f4732 2024-03-19T01:40:24.871 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:24.870+0000 7fef51f92700 1 -- 172.21.15.99:0/485603042 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fef340a2890 msgr2=0x7fef340a4d50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:24.871 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:24.870+0000 7fef51f92700 1 --2- 172.21.15.99:0/485603042 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fef340a2890 0x7fef340a4d50 secure :-1 s=READY pgs=224 cs=0 l=1 rev1=1 crypto rx=0x7fef40009c60 tx=0x7fef40009260 comp rx=0 tx=0).stop 2024-03-19T01:40:24.872 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:24.870+0000 7fef51f92700 1 -- 172.21.15.99:0/485603042 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fef4c100600 msgr2=0x7fef4c102f40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:24.872 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:24.870+0000 7fef51f92700 1 --2- 172.21.15.99:0/485603042 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fef4c100600 0x7fef4c102f40 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x7fef44007fa0 tx=0x7fef44004d80 comp rx=0 tx=0).stop 2024-03-19T01:40:24.872 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:24.870+0000 7fef51f92700 1 -- 172.21.15.99:0/485603042 shutdown_connections 2024-03-19T01:40:24.872 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:24.870+0000 7fef51f92700 1 --2- 172.21.15.99:0/485603042 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fef340a2890 0x7fef340a4d50 unknown :-1 s=CLOSED pgs=224 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:24.872 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:24.870+0000 7fef51f92700 1 --2- 172.21.15.99:0/485603042 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fef4c103f80 0x7fef4c101ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:24.872 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:24.870+0000 7fef51f92700 1 --2- 172.21.15.99:0/485603042 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fef4c100600 0x7fef4c102f40 unknown :-1 s=CLOSED pgs=74 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:24.872 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:24.870+0000 7fef51f92700 1 --2- 172.21.15.99:0/485603042 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fef4c1035e0 0x7fef4c101590 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:24.872 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:24.870+0000 7fef51f92700 1 -- 172.21.15.99:0/485603042 >> 172.21.15.99:0/485603042 conn(0x7fef4c0fbf80 msgr2=0x7fef4c1068e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:24.872 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:24.870+0000 7fef51f92700 1 -- 172.21.15.99:0/485603042 shutdown_connections 2024-03-19T01:40:24.872 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:24.870+0000 7fef51f92700 1 -- 172.21.15.99:0/485603042 wait complete. 2024-03-19T01:40:25.425 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- bash -c 'ceph orch ls' 2024-03-19T01:40:25.665 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:40:26.053 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:25 smithi099 ceph-mon[62951]: pgmap v330: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-03-19T01:40:26.053 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:25 smithi099 ceph-mon[62951]: from='client.24914 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:40:26.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:25 smithi112 ceph-mon[66819]: pgmap v330: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-03-19T01:40:26.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:25 smithi112 ceph-mon[66819]: from='client.24914 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:40:26.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:25 smithi115 ceph-mon[67332]: pgmap v330: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-03-19T01:40:26.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:25 smithi115 ceph-mon[67332]: from='client.24914 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:40:27.344 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:27.335+0000 7f8a54a33700 1 -- 172.21.15.99:0/1112375851 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8a50102de0 msgr2=0x7f8a50109690 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:27.344 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:27.335+0000 7f8a54a33700 1 --2- 172.21.15.99:0/1112375851 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8a50102de0 0x7f8a50109690 secure :-1 s=READY pgs=307 cs=0 l=1 rev1=1 crypto rx=0x7f8a480099c0 tx=0x7f8a48009cd0 comp rx=0 tx=0).stop 2024-03-19T01:40:27.344 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:27.335+0000 7f8a54a33700 1 -- 172.21.15.99:0/1112375851 shutdown_connections 2024-03-19T01:40:27.344 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:27.335+0000 7f8a54a33700 1 --2- 172.21.15.99:0/1112375851 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8a50102de0 0x7f8a50109690 unknown :-1 s=CLOSED pgs=307 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:27.345 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:27.335+0000 7f8a54a33700 1 --2- 172.21.15.99:0/1112375851 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8a50101240 0x7f8a50101660 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:27.345 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:27.335+0000 7f8a54a33700 1 --2- 172.21.15.99:0/1112375851 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8a50102440 0x7f8a501028a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:27.345 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:27.335+0000 7f8a54a33700 1 -- 172.21.15.99:0/1112375851 >> 172.21.15.99:0/1112375851 conn(0x7f8a500fc7e0 msgr2=0x7f8a500fec20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:27.345 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:27.335+0000 7f8a54a33700 1 -- 172.21.15.99:0/1112375851 shutdown_connections 2024-03-19T01:40:27.345 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:27.335+0000 7f8a54a33700 1 -- 172.21.15.99:0/1112375851 wait complete. 2024-03-19T01:40:27.345 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:27.335+0000 7f8a54a33700 1 Processor -- start 2024-03-19T01:40:27.345 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:27.336+0000 7f8a54a33700 1 -- start start 2024-03-19T01:40:27.345 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:27.336+0000 7f8a54a33700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8a50101240 0x7f8a5006ab40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:27.345 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:27.336+0000 7f8a54a33700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8a50102440 0x7f8a5019d250 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:27.345 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:27.336+0000 7f8a54a33700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8a50102de0 0x7f8a5019f8a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:27.345 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:27.336+0000 7f8a54a33700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f8a5010ce00 con 0x7f8a50102440 2024-03-19T01:40:27.345 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:27.336+0000 7f8a54a33700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f8a5010cc80 con 0x7f8a50102de0 2024-03-19T01:40:27.345 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:27.336+0000 7f8a54a33700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f8a5010cf80 con 0x7f8a50101240 2024-03-19T01:40:27.346 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:27.336+0000 7f8a3dd9b700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8a50102440 0x7f8a5019d250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:27.346 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:27.336+0000 7f8a3dd9b700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8a50102440 0x7f8a5019d250 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:46416/0 (socket says 172.21.15.99:46416) 2024-03-19T01:40:27.346 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:27.336+0000 7f8a3dd9b700 1 -- 172.21.15.99:0/1687103173 learned_addr learned my addr 172.21.15.99:0/1687103173 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:27.346 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:27.336+0000 7f8a4e59c700 1 --2- 172.21.15.99:0/1687103173 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8a50101240 0x7f8a5006ab40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:27.346 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:27.336+0000 7f8a4ed9d700 1 --2- 172.21.15.99:0/1687103173 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8a50102de0 0x7f8a5019f8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:27.346 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:27.336+0000 7f8a3dd9b700 1 -- 172.21.15.99:0/1687103173 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8a50101240 msgr2=0x7f8a5006ab40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:27.346 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:27.336+0000 7f8a3dd9b700 1 --2- 172.21.15.99:0/1687103173 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8a50101240 0x7f8a5006ab40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:27.346 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:27.336+0000 7f8a3dd9b700 1 -- 172.21.15.99:0/1687103173 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8a50102de0 msgr2=0x7f8a5019f8a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:27.346 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:27.336+0000 7f8a3dd9b700 1 --2- 172.21.15.99:0/1687103173 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8a50102de0 0x7f8a5019f8a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:27.346 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:27.336+0000 7f8a3dd9b700 1 -- 172.21.15.99:0/1687103173 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f8a501a0170 con 0x7f8a50102440 2024-03-19T01:40:27.346 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:27.336+0000 7f8a3dd9b700 1 --2- 172.21.15.99:0/1687103173 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8a50102440 0x7f8a5019d250 secure :-1 s=READY pgs=308 cs=0 l=1 rev1=1 crypto rx=0x7f8a3800e9a0 tx=0x7f8a3800ecb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:27.346 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:27.336+0000 7f8a47fff700 1 -- 172.21.15.99:0/1687103173 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8a3800ca10 con 0x7f8a50102440 2024-03-19T01:40:27.346 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:27.337+0000 7f8a47fff700 1 -- 172.21.15.99:0/1687103173 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f8a3800cbb0 con 0x7f8a50102440 2024-03-19T01:40:27.346 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:27.337+0000 7f8a54a33700 1 -- 172.21.15.99:0/1687103173 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f8a501a0460 con 0x7f8a50102440 2024-03-19T01:40:27.347 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:27.337+0000 7f8a47fff700 1 -- 172.21.15.99:0/1687103173 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8a38018680 con 0x7f8a50102440 2024-03-19T01:40:27.347 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:27.337+0000 7f8a54a33700 1 -- 172.21.15.99:0/1687103173 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f8a501a09a0 con 0x7f8a50102440 2024-03-19T01:40:27.347 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:27.338+0000 7f8a54a33700 1 -- 172.21.15.99:0/1687103173 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f8a50068fd0 con 0x7f8a50102440 2024-03-19T01:40:27.347 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:27.339+0000 7f8a47fff700 1 -- 172.21.15.99:0/1687103173 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 18) ==== 137290+0+0 (secure 0 0 0) 0x7f8a38010430 con 0x7f8a50102440 2024-03-19T01:40:27.347 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:27.339+0000 7f8a47fff700 1 --2- 172.21.15.99:0/1687103173 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f8a300a2890 0x7f8a300a4d50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:27.347 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:27.339+0000 7f8a4e59c700 1 --2- 172.21.15.99:0/1687103173 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f8a300a2890 0x7f8a300a4d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:27.347 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:27.339+0000 7f8a47fff700 1 -- 172.21.15.99:0/1687103173 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f8a380cece0 con 0x7f8a50102440 2024-03-19T01:40:27.347 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:27.340+0000 7f8a4e59c700 1 --2- 172.21.15.99:0/1687103173 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f8a300a2890 0x7f8a300a4d50 secure :-1 s=READY pgs=225 cs=0 l=1 rev1=1 crypto rx=0x7f8a501022a0 tx=0x7f8a4000b040 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:27.347 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:27.344+0000 7f8a47fff700 1 -- 172.21.15.99:0/1687103173 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f8a3809bd40 con 0x7f8a50102440 2024-03-19T01:40:27.551 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:27.551+0000 7f8a54a33700 1 -- 172.21.15.99:0/1687103173 --> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] -- mgr_command(tid 0: {"prefix": "orch ls", "target": ["mon-mgr", ""]}) -- 0x7f8a50062810 con 0x7f8a300a2890 2024-03-19T01:40:27.557 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:27.557+0000 7f8a47fff700 1 -- 172.21.15.99:0/1687103173 <== mgr.14150 v2:172.21.15.99:6800/4272410295 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+980 (secure 0 0 0) 0x7f8a50062810 con 0x7f8a300a2890 2024-03-19T01:40:27.557 INFO:teuthology.orchestra.run.smithi099.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-03-19T01:40:27.557 INFO:teuthology.orchestra.run.smithi099.stdout:mds.all 12/12 31s ago 2m smithi099=a;smithi099=d;smithi099=g;smithi099=j;smithi112=b;smithi112=e;smithi112=h;smithi112=k;smithi115=c;smithi115=f;smithi115=i;smithi115=l;count:12 2024-03-19T01:40:27.557 INFO:teuthology.orchestra.run.smithi099.stdout:mgr 3/3 31s ago 7m smithi099=x;smithi112=y;smithi115=z;count:3 2024-03-19T01:40:27.557 INFO:teuthology.orchestra.run.smithi099.stdout:mon 3/3 31s ago 7m smithi099:172.21.15.99=a;smithi112:172.21.15.112=b;smithi115:172.21.15.115=c;count:3 2024-03-19T01:40:27.557 INFO:teuthology.orchestra.run.smithi099.stdout:osd 12 31s ago - 2024-03-19T01:40:27.559 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:27.559+0000 7f8a54a33700 1 -- 172.21.15.99:0/1687103173 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f8a300a2890 msgr2=0x7f8a300a4d50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:27.560 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:27.559+0000 7f8a54a33700 1 --2- 172.21.15.99:0/1687103173 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f8a300a2890 0x7f8a300a4d50 secure :-1 s=READY pgs=225 cs=0 l=1 rev1=1 crypto rx=0x7f8a501022a0 tx=0x7f8a4000b040 comp rx=0 tx=0).stop 2024-03-19T01:40:27.560 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:27.559+0000 7f8a54a33700 1 -- 172.21.15.99:0/1687103173 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8a50102440 msgr2=0x7f8a5019d250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:27.560 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:27.559+0000 7f8a54a33700 1 --2- 172.21.15.99:0/1687103173 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8a50102440 0x7f8a5019d250 secure :-1 s=READY pgs=308 cs=0 l=1 rev1=1 crypto rx=0x7f8a3800e9a0 tx=0x7f8a3800ecb0 comp rx=0 tx=0).stop 2024-03-19T01:40:27.560 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:27.559+0000 7f8a54a33700 1 -- 172.21.15.99:0/1687103173 shutdown_connections 2024-03-19T01:40:27.560 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:27.559+0000 7f8a54a33700 1 --2- 172.21.15.99:0/1687103173 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f8a300a2890 0x7f8a300a4d50 unknown :-1 s=CLOSED pgs=225 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:27.560 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:27.559+0000 7f8a54a33700 1 --2- 172.21.15.99:0/1687103173 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8a50101240 0x7f8a5006ab40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:27.560 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:27.559+0000 7f8a54a33700 1 --2- 172.21.15.99:0/1687103173 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8a50102de0 0x7f8a5019f8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:27.560 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:27.559+0000 7f8a54a33700 1 --2- 172.21.15.99:0/1687103173 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8a50102440 0x7f8a5019d250 unknown :-1 s=CLOSED pgs=308 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:27.560 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:27.559+0000 7f8a54a33700 1 -- 172.21.15.99:0/1687103173 >> 172.21.15.99:0/1687103173 conn(0x7f8a500fc7e0 msgr2=0x7f8a50105740 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:27.560 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:27.559+0000 7f8a54a33700 1 -- 172.21.15.99:0/1687103173 shutdown_connections 2024-03-19T01:40:27.560 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:27.559+0000 7f8a54a33700 1 -- 172.21.15.99:0/1687103173 wait complete. 2024-03-19T01:40:28.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:27 smithi112 ceph-mon[66819]: pgmap v331: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 6 op/s 2024-03-19T01:40:28.156 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:27 smithi099 ceph-mon[62951]: pgmap v331: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 6 op/s 2024-03-19T01:40:28.182 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- bash -c 'ceph orch host ls' 2024-03-19T01:40:28.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:27 smithi115 ceph-mon[67332]: pgmap v331: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 6 op/s 2024-03-19T01:40:28.422 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:40:29.205 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:28 smithi099 ceph-mon[62951]: from='client.15144 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:40:29.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:28 smithi115 ceph-mon[67332]: from='client.15144 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:40:29.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:28 smithi112 ceph-mon[66819]: from='client.15144 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:40:30.017 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:29 smithi099 ceph-mon[62951]: pgmap v332: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 6 op/s 2024-03-19T01:40:30.019 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:30.019+0000 7fe2c3c77700 1 -- 172.21.15.99:0/2818521622 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe2bc104080 msgr2=0x7fe2bc10a930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:30.019 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:30.019+0000 7fe2c3c77700 1 --2- 172.21.15.99:0/2818521622 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe2bc104080 0x7fe2bc10a930 secure :-1 s=READY pgs=309 cs=0 l=1 rev1=1 crypto rx=0x7fe2b400b0d0 tx=0x7fe2b400b3e0 comp rx=0 tx=0).stop 2024-03-19T01:40:30.019 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:30.019+0000 7fe2c3c77700 1 -- 172.21.15.99:0/2818521622 shutdown_connections 2024-03-19T01:40:30.019 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:30.019+0000 7fe2c3c77700 1 --2- 172.21.15.99:0/2818521622 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fe2bc100660 0x7fe2bc100a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:30.019 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:30.019+0000 7fe2c3c77700 1 --2- 172.21.15.99:0/2818521622 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe2bc104080 0x7fe2bc10a930 unknown :-1 s=CLOSED pgs=309 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:30.019 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:30.019+0000 7fe2c3c77700 1 --2- 172.21.15.99:0/2818521622 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fe2bc1036e0 0x7fe2bc103b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:30.019 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:30.019+0000 7fe2c3c77700 1 -- 172.21.15.99:0/2818521622 >> 172.21.15.99:0/2818521622 conn(0x7fe2bc0fbfa0 msgr2=0x7fe2bc0fe3e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:30.020 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:30.019+0000 7fe2c3c77700 1 -- 172.21.15.99:0/2818521622 shutdown_connections 2024-03-19T01:40:30.020 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:30.019+0000 7fe2c3c77700 1 -- 172.21.15.99:0/2818521622 wait complete. 2024-03-19T01:40:30.021 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:30.020+0000 7fe2c3c77700 1 Processor -- start 2024-03-19T01:40:30.021 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:30.020+0000 7fe2c3c77700 1 -- start start 2024-03-19T01:40:30.022 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:30.020+0000 7fe2c3c77700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fe2bc100660 0x7fe2bc103040 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:30.022 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:30.020+0000 7fe2c3c77700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fe2bc1036e0 0x7fe2bc101690 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:30.022 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:30.020+0000 7fe2c3c77700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe2bc104080 0x7fe2bc101bd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:30.022 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:30.020+0000 7fe2c3c77700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fe2bc10df70 con 0x7fe2bc104080 2024-03-19T01:40:30.022 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:30.020+0000 7fe2c3c77700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fe2bc10ddf0 con 0x7fe2bc100660 2024-03-19T01:40:30.022 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:30.020+0000 7fe2c3c77700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fe2bc10e0f0 con 0x7fe2bc1036e0 2024-03-19T01:40:30.022 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:30.020+0000 7fe2c2214700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe2bc104080 0x7fe2bc101bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:30.022 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:30.020+0000 7fe2c2214700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe2bc104080 0x7fe2bc101bd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:46440/0 (socket says 172.21.15.99:46440) 2024-03-19T01:40:30.023 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:30.020+0000 7fe2c1a13700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fe2bc100660 0x7fe2bc103040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:30.023 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:30.020+0000 7fe2c2214700 1 -- 172.21.15.99:0/876028003 learned_addr learned my addr 172.21.15.99:0/876028003 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:30.023 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:30.020+0000 7fe2c1212700 1 --2- 172.21.15.99:0/876028003 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fe2bc1036e0 0x7fe2bc101690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:30.023 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:30.020+0000 7fe2c2214700 1 -- 172.21.15.99:0/876028003 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fe2bc1036e0 msgr2=0x7fe2bc101690 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:30.023 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:30.020+0000 7fe2c2214700 1 --2- 172.21.15.99:0/876028003 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fe2bc1036e0 0x7fe2bc101690 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:30.023 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:30.020+0000 7fe2c2214700 1 -- 172.21.15.99:0/876028003 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fe2bc100660 msgr2=0x7fe2bc103040 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:30.023 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:30.020+0000 7fe2c2214700 1 --2- 172.21.15.99:0/876028003 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fe2bc100660 0x7fe2bc103040 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:30.023 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:30.020+0000 7fe2c2214700 1 -- 172.21.15.99:0/876028003 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fe2bc102480 con 0x7fe2bc104080 2024-03-19T01:40:30.023 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:30.020+0000 7fe2c2214700 1 --2- 172.21.15.99:0/876028003 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe2bc104080 0x7fe2bc101bd0 secure :-1 s=READY pgs=310 cs=0 l=1 rev1=1 crypto rx=0x7fe2b400d010 tx=0x7fe2b4007a00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:30.023 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:30.020+0000 7fe2b2ffd700 1 -- 172.21.15.99:0/876028003 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe2b4003c00 con 0x7fe2bc104080 2024-03-19T01:40:30.023 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:30.021+0000 7fe2c3c77700 1 -- 172.21.15.99:0/876028003 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fe2bc1a25f0 con 0x7fe2bc104080 2024-03-19T01:40:30.023 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:30.021+0000 7fe2b2ffd700 1 -- 172.21.15.99:0/876028003 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe2b4022070 con 0x7fe2bc104080 2024-03-19T01:40:30.023 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:30.021+0000 7fe2b2ffd700 1 -- 172.21.15.99:0/876028003 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe2b401b790 con 0x7fe2bc104080 2024-03-19T01:40:30.023 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:30.021+0000 7fe2c3c77700 1 -- 172.21.15.99:0/876028003 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fe2bc1a29f0 con 0x7fe2bc104080 2024-03-19T01:40:30.024 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:30.022+0000 7fe2c3c77700 1 -- 172.21.15.99:0/876028003 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fe2bc068fd0 con 0x7fe2bc104080 2024-03-19T01:40:30.024 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:30.023+0000 7fe2b2ffd700 1 -- 172.21.15.99:0/876028003 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 18) ==== 137290+0+0 (secure 0 0 0) 0x7fe2b401d9e0 con 0x7fe2bc104080 2024-03-19T01:40:30.024 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:30.023+0000 7fe2b2ffd700 1 --2- 172.21.15.99:0/876028003 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fe2a80a2890 0x7fe2a80a4d50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:30.024 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:30.024+0000 7fe2c1a13700 1 --2- 172.21.15.99:0/876028003 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fe2a80a2890 0x7fe2a80a4d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:30.025 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:30.024+0000 7fe2b2ffd700 1 -- 172.21.15.99:0/876028003 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7fe2b40cfb00 con 0x7fe2bc104080 2024-03-19T01:40:30.025 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:30.024+0000 7fe2c1a13700 1 --2- 172.21.15.99:0/876028003 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fe2a80a2890 0x7fe2a80a4d50 secure :-1 s=READY pgs=226 cs=0 l=1 rev1=1 crypto rx=0x7fe2bc06a790 tx=0x7fe2ac009260 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:30.028 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:30.028+0000 7fe2b2ffd700 1 -- 172.21.15.99:0/876028003 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7fe2b409cbf0 con 0x7fe2bc104080 2024-03-19T01:40:30.233 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:30.232+0000 7fe2c3c77700 1 -- 172.21.15.99:0/876028003 --> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] -- mgr_command(tid 0: {"prefix": "orch host ls", "target": ["mon-mgr", ""]}) -- 0x7fe2bc062750 con 0x7fe2a80a2890 2024-03-19T01:40:30.234 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:30.234+0000 7fe2b2ffd700 1 -- 172.21.15.99:0/876028003 <== mgr.14150 v2:172.21.15.99:6800/4272410295 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+191 (secure 0 0 0) 0x7fe2bc062750 con 0x7fe2a80a2890 2024-03-19T01:40:30.234 INFO:teuthology.orchestra.run.smithi099.stdout:HOST ADDR LABELS STATUS 2024-03-19T01:40:30.234 INFO:teuthology.orchestra.run.smithi099.stdout:smithi099 172.21.15.99 2024-03-19T01:40:30.234 INFO:teuthology.orchestra.run.smithi099.stdout:smithi112 172.21.15.112 2024-03-19T01:40:30.234 INFO:teuthology.orchestra.run.smithi099.stdout:smithi115 172.21.15.115 2024-03-19T01:40:30.234 INFO:teuthology.orchestra.run.smithi099.stdout:3 hosts in cluster 2024-03-19T01:40:30.236 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:30.236+0000 7fe2c3c77700 1 -- 172.21.15.99:0/876028003 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fe2a80a2890 msgr2=0x7fe2a80a4d50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:30.236 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:30.236+0000 7fe2c3c77700 1 --2- 172.21.15.99:0/876028003 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fe2a80a2890 0x7fe2a80a4d50 secure :-1 s=READY pgs=226 cs=0 l=1 rev1=1 crypto rx=0x7fe2bc06a790 tx=0x7fe2ac009260 comp rx=0 tx=0).stop 2024-03-19T01:40:30.236 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:30.236+0000 7fe2c3c77700 1 -- 172.21.15.99:0/876028003 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe2bc104080 msgr2=0x7fe2bc101bd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:30.236 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:30.236+0000 7fe2c3c77700 1 --2- 172.21.15.99:0/876028003 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe2bc104080 0x7fe2bc101bd0 secure :-1 s=READY pgs=310 cs=0 l=1 rev1=1 crypto rx=0x7fe2b400d010 tx=0x7fe2b4007a00 comp rx=0 tx=0).stop 2024-03-19T01:40:30.236 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:30.236+0000 7fe2c3c77700 1 -- 172.21.15.99:0/876028003 shutdown_connections 2024-03-19T01:40:30.236 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:30.236+0000 7fe2c3c77700 1 --2- 172.21.15.99:0/876028003 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fe2a80a2890 0x7fe2a80a4d50 unknown :-1 s=CLOSED pgs=226 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:30.237 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:30.236+0000 7fe2c3c77700 1 --2- 172.21.15.99:0/876028003 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe2bc104080 0x7fe2bc101bd0 unknown :-1 s=CLOSED pgs=310 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:30.237 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:30.236+0000 7fe2c3c77700 1 --2- 172.21.15.99:0/876028003 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fe2bc100660 0x7fe2bc103040 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:30.237 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:30.236+0000 7fe2c3c77700 1 --2- 172.21.15.99:0/876028003 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fe2bc1036e0 0x7fe2bc101690 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:30.237 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:30.236+0000 7fe2c3c77700 1 -- 172.21.15.99:0/876028003 >> 172.21.15.99:0/876028003 conn(0x7fe2bc0fbfa0 msgr2=0x7fe2bc106910 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:30.237 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:30.236+0000 7fe2c3c77700 1 -- 172.21.15.99:0/876028003 shutdown_connections 2024-03-19T01:40:30.237 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:30.236+0000 7fe2c3c77700 1 -- 172.21.15.99:0/876028003 wait complete. 2024-03-19T01:40:30.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:29 smithi115 ceph-mon[67332]: pgmap v332: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 6 op/s 2024-03-19T01:40:30.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:29 smithi112 ceph-mon[66819]: pgmap v332: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 6 op/s 2024-03-19T01:40:30.889 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- bash -c 'ceph orch device ls' 2024-03-19T01:40:31.130 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:40:32.258 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:31 smithi099 ceph-mon[62951]: from='client.15150 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:40:32.259 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:31 smithi099 ceph-mon[62951]: pgmap v333: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 6 op/s 2024-03-19T01:40:32.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:31 smithi115 ceph-mon[67332]: from='client.15150 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:40:32.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:31 smithi115 ceph-mon[67332]: pgmap v333: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 6 op/s 2024-03-19T01:40:32.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:31 smithi112 ceph-mon[66819]: from='client.15150 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:40:32.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:31 smithi112 ceph-mon[66819]: pgmap v333: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 6 op/s 2024-03-19T01:40:32.841 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:32.839+0000 7fb58b054700 1 -- 172.21.15.99:0/221100157 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb584108030 msgr2=0x7fb58410a4a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:32.842 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:32.839+0000 7fb58b054700 1 --2- 172.21.15.99:0/221100157 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb584108030 0x7fb58410a4a0 secure :-1 s=READY pgs=311 cs=0 l=1 rev1=1 crypto rx=0x7fb57c009990 tx=0x7fb57c009ca0 comp rx=0 tx=0).stop 2024-03-19T01:40:32.842 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:32.839+0000 7fb58b054700 1 -- 172.21.15.99:0/221100157 shutdown_connections 2024-03-19T01:40:32.842 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:32.839+0000 7fb58b054700 1 --2- 172.21.15.99:0/221100157 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb584108030 0x7fb58410a4a0 unknown :-1 s=CLOSED pgs=311 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:32.842 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:32.839+0000 7fb58b054700 1 --2- 172.21.15.99:0/221100157 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb584102dd0 0x7fb5841051c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:32.842 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:32.839+0000 7fb58b054700 1 --2- 172.21.15.99:0/221100157 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb584105700 0x7fb584107af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:32.842 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:32.839+0000 7fb58b054700 1 -- 172.21.15.99:0/221100157 >> 172.21.15.99:0/221100157 conn(0x7fb5840fc7e0 msgr2=0x7fb5840fec20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:32.842 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:32.839+0000 7fb58b054700 1 -- 172.21.15.99:0/221100157 shutdown_connections 2024-03-19T01:40:32.842 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:32.839+0000 7fb58b054700 1 -- 172.21.15.99:0/221100157 wait complete. 2024-03-19T01:40:32.842 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:32.840+0000 7fb58b054700 1 Processor -- start 2024-03-19T01:40:32.842 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:32.840+0000 7fb58b054700 1 -- start start 2024-03-19T01:40:32.842 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:32.840+0000 7fb58b054700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb584102dd0 0x7fb584198530 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:32.842 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:32.840+0000 7fb58b054700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb584105700 0x7fb584198a70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:32.843 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:32.840+0000 7fb58b054700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb584108030 0x7fb58419faf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:32.843 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:32.840+0000 7fb58b054700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fb58410df10 con 0x7fb584108030 2024-03-19T01:40:32.843 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:32.840+0000 7fb58b054700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fb58410dd90 con 0x7fb584105700 2024-03-19T01:40:32.843 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:32.840+0000 7fb58b054700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fb58410e090 con 0x7fb584102dd0 2024-03-19T01:40:32.843 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:32.840+0000 7fb5895f1700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb584108030 0x7fb58419faf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:32.843 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:32.840+0000 7fb583fff700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb584105700 0x7fb584198a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:32.843 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:32.840+0000 7fb588df0700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb584102dd0 0x7fb584198530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:32.843 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:32.840+0000 7fb583fff700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb584105700 0x7fb584198a70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:36598/0 (socket says 172.21.15.99:36598) 2024-03-19T01:40:32.843 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:32.840+0000 7fb583fff700 1 -- 172.21.15.99:0/1373709788 learned_addr learned my addr 172.21.15.99:0/1373709788 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:32.843 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:32.841+0000 7fb583fff700 1 -- 172.21.15.99:0/1373709788 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb584102dd0 msgr2=0x7fb584198530 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:32.843 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:32.841+0000 7fb583fff700 1 --2- 172.21.15.99:0/1373709788 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb584102dd0 0x7fb584198530 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:32.843 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:32.841+0000 7fb583fff700 1 -- 172.21.15.99:0/1373709788 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb584108030 msgr2=0x7fb58419faf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:32.843 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:32.841+0000 7fb583fff700 1 --2- 172.21.15.99:0/1373709788 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb584108030 0x7fb58419faf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:32.843 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:32.841+0000 7fb583fff700 1 -- 172.21.15.99:0/1373709788 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb5841a0190 con 0x7fb584105700 2024-03-19T01:40:32.844 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:32.841+0000 7fb588df0700 1 --2- 172.21.15.99:0/1373709788 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb584102dd0 0x7fb584198530 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:40:32.844 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:32.841+0000 7fb5895f1700 1 --2- 172.21.15.99:0/1373709788 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb584108030 0x7fb58419faf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:40:32.844 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:32.841+0000 7fb583fff700 1 --2- 172.21.15.99:0/1373709788 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb584105700 0x7fb584198a70 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x7fb57400ef30 tx=0x7fb57400c3a0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:32.844 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:32.841+0000 7fb581ffb700 1 -- 172.21.15.99:0/1373709788 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb5740092e0 con 0x7fb584105700 2024-03-19T01:40:32.844 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:32.841+0000 7fb58b054700 1 -- 172.21.15.99:0/1373709788 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb584100bc0 con 0x7fb584105700 2024-03-19T01:40:32.844 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:32.841+0000 7fb581ffb700 1 -- 172.21.15.99:0/1373709788 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fb574004bb0 con 0x7fb584105700 2024-03-19T01:40:32.844 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:32.841+0000 7fb581ffb700 1 -- 172.21.15.99:0/1373709788 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb574004280 con 0x7fb584105700 2024-03-19T01:40:32.844 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:32.841+0000 7fb58b054700 1 -- 172.21.15.99:0/1373709788 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fb584101100 con 0x7fb584105700 2024-03-19T01:40:32.845 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:32.842+0000 7fb58b054700 1 -- 172.21.15.99:0/1373709788 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fb584068fd0 con 0x7fb584105700 2024-03-19T01:40:32.845 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:32.843+0000 7fb581ffb700 1 -- 172.21.15.99:0/1373709788 <== mon.2 v2:172.21.15.112:3300/0 4 ==== mgrmap(e 18) ==== 137290+0+0 (secure 0 0 0) 0x7fb57401b030 con 0x7fb584105700 2024-03-19T01:40:32.845 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:32.844+0000 7fb581ffb700 1 --2- 172.21.15.99:0/1373709788 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb5700a2890 0x7fb5700a4d50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:32.845 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:32.844+0000 7fb588df0700 1 --2- 172.21.15.99:0/1373709788 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb5700a2890 0x7fb5700a4d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:32.845 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:32.844+0000 7fb581ffb700 1 -- 172.21.15.99:0/1373709788 <== mon.2 v2:172.21.15.112:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7fb5740ce720 con 0x7fb584105700 2024-03-19T01:40:32.845 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:32.844+0000 7fb588df0700 1 --2- 172.21.15.99:0/1373709788 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb5700a2890 0x7fb5700a4d50 secure :-1 s=READY pgs=227 cs=0 l=1 rev1=1 crypto rx=0x7fb56c011490 tx=0x7fb56c009260 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:32.852 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:32.852+0000 7fb581ffb700 1 -- 172.21.15.99:0/1373709788 <== mon.2 v2:172.21.15.112:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7fb574022020 con 0x7fb584105700 2024-03-19T01:40:33.060 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:33.060+0000 7fb58b054700 1 -- 172.21.15.99:0/1373709788 --> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] -- mgr_command(tid 0: {"prefix": "orch device ls", "target": ["mon-mgr", ""]}) -- 0x7fb584062810 con 0x7fb5700a2890 2024-03-19T01:40:33.062 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:33.062+0000 7fb581ffb700 1 -- 172.21.15.99:0/1373709788 <== mgr.14150 v2:172.21.15.99:6800/4272410295 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+532 (secure 0 0 0) 0x7fb584062810 con 0x7fb5700a2890 2024-03-19T01:40:33.062 INFO:teuthology.orchestra.run.smithi099.stdout:HOST PATH TYPE DEVICE ID SIZE AVAILABLE REFRESHED REJECT REASONS 2024-03-19T01:40:33.062 INFO:teuthology.orchestra.run.smithi099.stdout:smithi099 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_PHFT62040101400BGN 372G No 3m ago Has a FileSystem, LVM detected 2024-03-19T01:40:33.063 INFO:teuthology.orchestra.run.smithi099.stdout:smithi112 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_PHFT620400NV400BGN 372G No 2m ago Has a FileSystem, LVM detected 2024-03-19T01:40:33.063 INFO:teuthology.orchestra.run.smithi099.stdout:smithi115 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_PHFT620400UU400BGN 372G No 2m ago Has a FileSystem, LVM detected 2024-03-19T01:40:33.064 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:33.064+0000 7fb58b054700 1 -- 172.21.15.99:0/1373709788 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb5700a2890 msgr2=0x7fb5700a4d50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:33.065 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:33.064+0000 7fb58b054700 1 --2- 172.21.15.99:0/1373709788 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb5700a2890 0x7fb5700a4d50 secure :-1 s=READY pgs=227 cs=0 l=1 rev1=1 crypto rx=0x7fb56c011490 tx=0x7fb56c009260 comp rx=0 tx=0).stop 2024-03-19T01:40:33.065 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:33.064+0000 7fb58b054700 1 -- 172.21.15.99:0/1373709788 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb584105700 msgr2=0x7fb584198a70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:33.065 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:33.064+0000 7fb58b054700 1 --2- 172.21.15.99:0/1373709788 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb584105700 0x7fb584198a70 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x7fb57400ef30 tx=0x7fb57400c3a0 comp rx=0 tx=0).stop 2024-03-19T01:40:33.065 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:33.064+0000 7fb58b054700 1 -- 172.21.15.99:0/1373709788 shutdown_connections 2024-03-19T01:40:33.065 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:33.064+0000 7fb58b054700 1 --2- 172.21.15.99:0/1373709788 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb5700a2890 0x7fb5700a4d50 unknown :-1 s=CLOSED pgs=227 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:33.065 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:33.064+0000 7fb58b054700 1 --2- 172.21.15.99:0/1373709788 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb584102dd0 0x7fb584198530 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:33.065 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:33.064+0000 7fb58b054700 1 --2- 172.21.15.99:0/1373709788 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb584108030 0x7fb58419faf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:33.065 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:33.064+0000 7fb58b054700 1 --2- 172.21.15.99:0/1373709788 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb584105700 0x7fb584198a70 unknown :-1 s=CLOSED pgs=75 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:33.065 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:33.064+0000 7fb58b054700 1 -- 172.21.15.99:0/1373709788 >> 172.21.15.99:0/1373709788 conn(0x7fb5840fc7e0 msgr2=0x7fb584103710 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:33.065 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:33.064+0000 7fb58b054700 1 -- 172.21.15.99:0/1373709788 shutdown_connections 2024-03-19T01:40:33.065 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:33.064+0000 7fb58b054700 1 -- 172.21.15.99:0/1373709788 wait complete. 2024-03-19T01:40:33.713 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-03-19T01:40:33.723 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi099.front.sepia.ceph.com 2024-03-19T01:40:33.724 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- bash -c 'ceph fs dump' 2024-03-19T01:40:33.962 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:40:34.084 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:33 smithi099 ceph-mon[62951]: pgmap v334: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 6 op/s 2024-03-19T01:40:34.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:33 smithi115 ceph-mon[67332]: pgmap v334: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 6 op/s 2024-03-19T01:40:34.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:33 smithi112 ceph-mon[66819]: pgmap v334: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 6 op/s 2024-03-19T01:40:35.078 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:34 smithi099 ceph-mon[62951]: from='client.24932 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:40:35.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:34 smithi115 ceph-mon[67332]: from='client.24932 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:40:35.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:34 smithi112 ceph-mon[66819]: from='client.24932 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:40:35.617 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:35.616+0000 7f52bbbf8700 1 -- 172.21.15.99:0/2807170875 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f52b4103ee0 msgr2=0x7f52b410a790 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:35.617 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:35.616+0000 7f52bbbf8700 1 --2- 172.21.15.99:0/2807170875 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f52b4103ee0 0x7f52b410a790 secure :-1 s=READY pgs=312 cs=0 l=1 rev1=1 crypto rx=0x7f52ac00b0d0 tx=0x7f52ac00b3e0 comp rx=0 tx=0).stop 2024-03-19T01:40:35.617 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:35.616+0000 7f52bbbf8700 1 -- 172.21.15.99:0/2807170875 shutdown_connections 2024-03-19T01:40:35.617 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:35.616+0000 7f52bbbf8700 1 --2- 172.21.15.99:0/2807170875 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f52b4103ee0 0x7f52b410a790 unknown :-1 s=CLOSED pgs=312 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:35.617 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:35.616+0000 7f52bbbf8700 1 --2- 172.21.15.99:0/2807170875 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f52b4102340 0x7f52b4102760 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:35.617 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:35.616+0000 7f52bbbf8700 1 --2- 172.21.15.99:0/2807170875 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f52b4103540 0x7f52b41039a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:35.617 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:35.616+0000 7f52bbbf8700 1 -- 172.21.15.99:0/2807170875 >> 172.21.15.99:0/2807170875 conn(0x7f52b40fd920 msgr2=0x7f52b40ffd40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:35.617 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:35.616+0000 7f52bbbf8700 1 -- 172.21.15.99:0/2807170875 shutdown_connections 2024-03-19T01:40:35.617 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:35.617+0000 7f52bbbf8700 1 -- 172.21.15.99:0/2807170875 wait complete. 2024-03-19T01:40:35.619 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:35.617+0000 7f52bbbf8700 1 Processor -- start 2024-03-19T01:40:35.619 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:35.617+0000 7f52bbbf8700 1 -- start start 2024-03-19T01:40:35.619 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:35.617+0000 7f52bbbf8700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f52b4102340 0x7f52b41984d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:35.619 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:35.617+0000 7f52bbbf8700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f52b4103540 0x7f52b4198a10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:35.619 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:35.617+0000 7f52bbbf8700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f52b4103ee0 0x7f52b419fa90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:35.619 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:35.617+0000 7f52bbbf8700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f52b410df10 con 0x7f52b4103ee0 2024-03-19T01:40:35.619 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:35.617+0000 7f52bbbf8700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f52b410dd90 con 0x7f52b4102340 2024-03-19T01:40:35.620 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:35.617+0000 7f52bbbf8700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f52b410e090 con 0x7f52b4103540 2024-03-19T01:40:35.620 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:35.617+0000 7f52b9994700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f52b4102340 0x7f52b41984d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:35.620 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:35.617+0000 7f52ba195700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f52b4103ee0 0x7f52b419fa90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:35.620 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:35.617+0000 7f52b9994700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f52b4102340 0x7f52b41984d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:36626/0 (socket says 172.21.15.99:36626) 2024-03-19T01:40:35.620 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:35.617+0000 7f52b9193700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f52b4103540 0x7f52b4198a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:35.620 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:35.617+0000 7f52b9994700 1 -- 172.21.15.99:0/2039417347 learned_addr learned my addr 172.21.15.99:0/2039417347 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:35.620 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:35.618+0000 7f52b9994700 1 -- 172.21.15.99:0/2039417347 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f52b4103540 msgr2=0x7f52b4198a10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:35.620 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:35.618+0000 7f52b9994700 1 --2- 172.21.15.99:0/2039417347 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f52b4103540 0x7f52b4198a10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:35.620 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:35.618+0000 7f52b9994700 1 -- 172.21.15.99:0/2039417347 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f52b4103ee0 msgr2=0x7f52b419fa90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:35.620 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:35.618+0000 7f52b9994700 1 --2- 172.21.15.99:0/2039417347 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f52b4103ee0 0x7f52b419fa90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:35.620 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:35.618+0000 7f52b9994700 1 -- 172.21.15.99:0/2039417347 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f52b41a0130 con 0x7f52b4102340 2024-03-19T01:40:35.620 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:35.618+0000 7f52ba195700 1 --2- 172.21.15.99:0/2039417347 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f52b4103ee0 0x7f52b419fa90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:40:35.620 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:35.618+0000 7f52b9994700 1 --2- 172.21.15.99:0/2039417347 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f52b4102340 0x7f52b41984d0 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7f52b000ea50 tx=0x7f52b000ed60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:35.620 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:35.618+0000 7f52aaffd700 1 -- 172.21.15.99:0/2039417347 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f52b000cbd0 con 0x7f52b4102340 2024-03-19T01:40:35.620 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:35.618+0000 7f52bbbf8700 1 -- 172.21.15.99:0/2039417347 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f52b41a0420 con 0x7f52b4102340 2024-03-19T01:40:35.621 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:35.618+0000 7f52aaffd700 1 -- 172.21.15.99:0/2039417347 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f52b000cd70 con 0x7f52b4102340 2024-03-19T01:40:35.621 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:35.618+0000 7f52bbbf8700 1 -- 172.21.15.99:0/2039417347 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f52b41a09b0 con 0x7f52b4102340 2024-03-19T01:40:35.621 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:35.618+0000 7f52aaffd700 1 -- 172.21.15.99:0/2039417347 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f52b00187a0 con 0x7f52b4102340 2024-03-19T01:40:35.621 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:35.619+0000 7f52bbbf8700 1 -- 172.21.15.99:0/2039417347 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f52b410e6d0 con 0x7f52b4102340 2024-03-19T01:40:35.622 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:35.620+0000 7f52aaffd700 1 -- 172.21.15.99:0/2039417347 <== mon.2 v2:172.21.15.112:3300/0 4 ==== mgrmap(e 18) ==== 137290+0+0 (secure 0 0 0) 0x7f52b001f020 con 0x7f52b4102340 2024-03-19T01:40:35.622 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:35.621+0000 7f52aaffd700 1 --2- 172.21.15.99:0/2039417347 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f52a00a2890 0x7f52a00a4d50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:35.622 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:35.621+0000 7f52b9193700 1 --2- 172.21.15.99:0/2039417347 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f52a00a2890 0x7f52a00a4d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:35.622 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:35.621+0000 7f52aaffd700 1 -- 172.21.15.99:0/2039417347 <== mon.2 v2:172.21.15.112:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f52b00cf220 con 0x7f52b4102340 2024-03-19T01:40:35.622 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:35.621+0000 7f52b9193700 1 --2- 172.21.15.99:0/2039417347 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f52a00a2890 0x7f52a00a4d50 secure :-1 s=READY pgs=228 cs=0 l=1 rev1=1 crypto rx=0x7f52a4009c60 tx=0x7f52a4009260 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:35.624 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:35.624+0000 7f52aaffd700 1 -- 172.21.15.99:0/2039417347 <== mon.2 v2:172.21.15.112:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f52b0026030 con 0x7f52b4102340 2024-03-19T01:40:35.868 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:35.868+0000 7f52bbbf8700 1 -- 172.21.15.99:0/2039417347 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_command({"prefix": "fs dump"} v 0) -- 0x7f52b4062f20 con 0x7f52b4102340 2024-03-19T01:40:35.868 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:35.868+0000 7f52aaffd700 1 -- 172.21.15.99:0/2039417347 <== mon.2 v2:172.21.15.112:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump"}]=0 dumped fsmap epoch 20 v20) ==== 76+0+3054 (secure 0 0 0) 0x7f52b009c280 con 0x7f52b4102340 2024-03-19T01:40:35.869 INFO:teuthology.orchestra.run.smithi099.stdout:e20 2024-03-19T01:40:35.869 INFO:teuthology.orchestra.run.smithi099.stdout:enable_multiple, ever_enabled_multiple: 1,1 2024-03-19T01:40:35.869 INFO:teuthology.orchestra.run.smithi099.stdout:default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2,11=minor log segments,12=quiesce subvolumes} 2024-03-19T01:40:35.869 INFO:teuthology.orchestra.run.smithi099.stdout:legacy client fscid: 1 2024-03-19T01:40:35.869 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:40:35.869 INFO:teuthology.orchestra.run.smithi099.stdout:Filesystem 'cephfs' (1) 2024-03-19T01:40:35.869 INFO:teuthology.orchestra.run.smithi099.stdout:fs_name cephfs 2024-03-19T01:40:35.869 INFO:teuthology.orchestra.run.smithi099.stdout:epoch 20 2024-03-19T01:40:35.869 INFO:teuthology.orchestra.run.smithi099.stdout:flags 32 joinable allow_snaps allow_multimds_snaps allow_standby_replay 2024-03-19T01:40:35.869 INFO:teuthology.orchestra.run.smithi099.stdout:created 2024-03-19T01:38:56.124529+0000 2024-03-19T01:40:35.869 INFO:teuthology.orchestra.run.smithi099.stdout:modified 2024-03-19T01:39:00.323326+0000 2024-03-19T01:40:35.870 INFO:teuthology.orchestra.run.smithi099.stdout:tableserver 0 2024-03-19T01:40:35.870 INFO:teuthology.orchestra.run.smithi099.stdout:root 0 2024-03-19T01:40:35.870 INFO:teuthology.orchestra.run.smithi099.stdout:session_timeout 300 2024-03-19T01:40:35.870 INFO:teuthology.orchestra.run.smithi099.stdout:session_autoclose 300 2024-03-19T01:40:35.870 INFO:teuthology.orchestra.run.smithi099.stdout:max_file_size 1099511627776 2024-03-19T01:40:35.870 INFO:teuthology.orchestra.run.smithi099.stdout:max_xattr_size 65536 2024-03-19T01:40:35.870 INFO:teuthology.orchestra.run.smithi099.stdout:required_client_features {} 2024-03-19T01:40:35.870 INFO:teuthology.orchestra.run.smithi099.stdout:last_failure 0 2024-03-19T01:40:35.870 INFO:teuthology.orchestra.run.smithi099.stdout:last_failure_osd_epoch 0 2024-03-19T01:40:35.870 INFO:teuthology.orchestra.run.smithi099.stdout:compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2,11=minor log segments,12=quiesce subvolumes} 2024-03-19T01:40:35.870 INFO:teuthology.orchestra.run.smithi099.stdout:max_mds 3 2024-03-19T01:40:35.870 INFO:teuthology.orchestra.run.smithi099.stdout:in 0,1,2 2024-03-19T01:40:35.870 INFO:teuthology.orchestra.run.smithi099.stdout:up {0=24487,1=24479,2=24472} 2024-03-19T01:40:35.870 INFO:teuthology.orchestra.run.smithi099.stdout:failed 2024-03-19T01:40:35.871 INFO:teuthology.orchestra.run.smithi099.stdout:damaged 2024-03-19T01:40:35.871 INFO:teuthology.orchestra.run.smithi099.stdout:stopped 2024-03-19T01:40:35.871 INFO:teuthology.orchestra.run.smithi099.stdout:data_pools [3] 2024-03-19T01:40:35.871 INFO:teuthology.orchestra.run.smithi099.stdout:metadata_pool 2 2024-03-19T01:40:35.871 INFO:teuthology.orchestra.run.smithi099.stdout:inline_data disabled 2024-03-19T01:40:35.871 INFO:teuthology.orchestra.run.smithi099.stdout:balancer 2024-03-19T01:40:35.871 INFO:teuthology.orchestra.run.smithi099.stdout:bal_rank_mask -1 2024-03-19T01:40:35.871 INFO:teuthology.orchestra.run.smithi099.stdout:standby_count_wanted 0 2024-03-19T01:40:35.871 INFO:teuthology.orchestra.run.smithi099.stdout:qdb_cluster leader: 24487 members: 24487,24479,24472 2024-03-19T01:40:35.871 INFO:teuthology.orchestra.run.smithi099.stdout:[mds.i{0:24487} state up:active seq 5 addr [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] compat {c=[1],r=[1],i=[1fff]}] 2024-03-19T01:40:35.871 INFO:teuthology.orchestra.run.smithi099.stdout:[mds.f{0:24460} state up:standby-replay seq 1 addr [v2:172.21.15.115:6834/591078413,v1:172.21.15.115:6835/591078413] compat {c=[1],r=[1],i=[1fff]}] 2024-03-19T01:40:35.871 INFO:teuthology.orchestra.run.smithi099.stdout:[mds.b{1:24479} state up:active seq 4 addr [v2:172.21.15.112:6838/986249680,v1:172.21.15.112:6839/986249680] compat {c=[1],r=[1],i=[1fff]}] 2024-03-19T01:40:35.871 INFO:teuthology.orchestra.run.smithi099.stdout:[mds.h{1:24440} state up:standby-replay seq 1 addr [v2:172.21.15.112:6836/2339823159,v1:172.21.15.112:6837/2339823159] compat {c=[1],r=[1],i=[1fff]}] 2024-03-19T01:40:35.871 INFO:teuthology.orchestra.run.smithi099.stdout:[mds.c{2:24472} state up:active seq 7 addr [v2:172.21.15.115:6836/1368988191,v1:172.21.15.115:6837/1368988191] compat {c=[1],r=[1],i=[1fff]}] 2024-03-19T01:40:35.872 INFO:teuthology.orchestra.run.smithi099.stdout:[mds.e{2:24416} state up:standby-replay seq 1 addr [v2:172.21.15.112:6834/3690587528,v1:172.21.15.112:6835/3690587528] compat {c=[1],r=[1],i=[1fff]}] 2024-03-19T01:40:35.872 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:40:35.872 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:40:35.872 INFO:teuthology.orchestra.run.smithi099.stdout:Standby daemons: 2024-03-19T01:40:35.872 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:40:35.872 INFO:teuthology.orchestra.run.smithi099.stdout:[mds.j{-1:14562} state up:standby seq 1 addr [v2:172.21.15.99:6834/1631815621,v1:172.21.15.99:6835/1631815621] compat {c=[1],r=[1],i=[1fff]}] 2024-03-19T01:40:35.872 INFO:teuthology.orchestra.run.smithi099.stdout:[mds.g{-1:14646} state up:standby seq 1 addr [v2:172.21.15.99:6838/2236145144,v1:172.21.15.99:6839/2236145144] compat {c=[1],r=[1],i=[1fff]}] 2024-03-19T01:40:35.872 INFO:teuthology.orchestra.run.smithi099.stdout:[mds.a{-1:14658} state up:standby seq 1 addr [v2:172.21.15.99:6840/1660990159,v1:172.21.15.99:6841/1660990159] compat {c=[1],r=[1],i=[1fff]}] 2024-03-19T01:40:35.872 INFO:teuthology.orchestra.run.smithi099.stdout:[mds.k{-1:24383} state up:standby seq 1 addr [v2:172.21.15.112:6832/2969810977,v1:172.21.15.112:6833/2969810977] compat {c=[1],r=[1],i=[1fff]}] 2024-03-19T01:40:35.872 INFO:teuthology.orchestra.run.smithi099.stdout:[mds.d{-1:24404} state up:standby seq 1 addr [v2:172.21.15.99:6836/2038634879,v1:172.21.15.99:6837/2038634879] compat {c=[1],r=[1],i=[1fff]}] 2024-03-19T01:40:35.872 INFO:teuthology.orchestra.run.smithi099.stdout:[mds.l{-1:24415} state up:standby seq 1 addr [v2:172.21.15.115:6832/2219320481,v1:172.21.15.115:6833/2219320481] compat {c=[1],r=[1],i=[1fff]}] 2024-03-19T01:40:35.873 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:35.871+0000 7f52bbbf8700 1 -- 172.21.15.99:0/2039417347 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f52a00a2890 msgr2=0x7f52a00a4d50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:35.873 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:35.871+0000 7f52bbbf8700 1 --2- 172.21.15.99:0/2039417347 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f52a00a2890 0x7f52a00a4d50 secure :-1 s=READY pgs=228 cs=0 l=1 rev1=1 crypto rx=0x7f52a4009c60 tx=0x7f52a4009260 comp rx=0 tx=0).stop 2024-03-19T01:40:35.873 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:35.871+0000 7f52bbbf8700 1 -- 172.21.15.99:0/2039417347 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f52b4102340 msgr2=0x7f52b41984d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:35.873 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:35.871+0000 7f52bbbf8700 1 --2- 172.21.15.99:0/2039417347 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f52b4102340 0x7f52b41984d0 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7f52b000ea50 tx=0x7f52b000ed60 comp rx=0 tx=0).stop 2024-03-19T01:40:35.873 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:35.871+0000 7f52bbbf8700 1 -- 172.21.15.99:0/2039417347 shutdown_connections 2024-03-19T01:40:35.873 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:35.871+0000 7f52bbbf8700 1 --2- 172.21.15.99:0/2039417347 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f52a00a2890 0x7f52a00a4d50 unknown :-1 s=CLOSED pgs=228 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:35.873 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:35.871+0000 7f52bbbf8700 1 --2- 172.21.15.99:0/2039417347 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f52b4103ee0 0x7f52b419fa90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:35.873 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:35.871+0000 7f52bbbf8700 1 --2- 172.21.15.99:0/2039417347 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f52b4102340 0x7f52b41984d0 unknown :-1 s=CLOSED pgs=76 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:35.873 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:35.871+0000 7f52bbbf8700 1 --2- 172.21.15.99:0/2039417347 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f52b4103540 0x7f52b4198a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:35.874 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:35.871+0000 7f52bbbf8700 1 -- 172.21.15.99:0/2039417347 >> 172.21.15.99:0/2039417347 conn(0x7f52b40fd920 msgr2=0x7f52b4106840 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:35.874 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:35.871+0000 7f52bbbf8700 1 -- 172.21.15.99:0/2039417347 shutdown_connections 2024-03-19T01:40:35.874 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:35.871+0000 7f52bbbf8700 1 -- 172.21.15.99:0/2039417347 wait complete. 2024-03-19T01:40:35.874 INFO:teuthology.orchestra.run.smithi099.stderr:dumped fsmap epoch 20 2024-03-19T01:40:36.160 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:35 smithi099 ceph-mon[62951]: pgmap v335: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-03-19T01:40:36.160 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:35 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/2039417347' entity='client.admin' cmd=[{"prefix": "fs dump"}]: dispatch 2024-03-19T01:40:36.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:35 smithi115 ceph-mon[67332]: pgmap v335: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-03-19T01:40:36.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:35 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/2039417347' entity='client.admin' cmd=[{"prefix": "fs dump"}]: dispatch 2024-03-19T01:40:36.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:35 smithi112 ceph-mon[66819]: pgmap v335: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-03-19T01:40:36.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:35 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/2039417347' entity='client.admin' cmd=[{"prefix": "fs dump"}]: dispatch 2024-03-19T01:40:36.455 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- bash -c 'ceph osd dump' 2024-03-19T01:40:36.695 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:40:37.980 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:37 smithi099 ceph-mon[62951]: pgmap v336: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 6 op/s 2024-03-19T01:40:38.197 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:38.196+0000 7f0c302cf700 1 -- 172.21.15.99:0/2982659068 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0c28101240 msgr2=0x7f0c28101660 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:38.197 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:38.196+0000 7f0c302cf700 1 --2- 172.21.15.99:0/2982659068 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0c28101240 0x7f0c28101660 secure :-1 s=READY pgs=313 cs=0 l=1 rev1=1 crypto rx=0x7f0c180099c0 tx=0x7f0c18009cd0 comp rx=0 tx=0).stop 2024-03-19T01:40:38.197 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:38.196+0000 7f0c302cf700 1 -- 172.21.15.99:0/2982659068 shutdown_connections 2024-03-19T01:40:38.197 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:38.196+0000 7f0c302cf700 1 --2- 172.21.15.99:0/2982659068 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f0c28102de0 0x7f0c28109690 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:38.197 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:38.196+0000 7f0c302cf700 1 --2- 172.21.15.99:0/2982659068 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0c28101240 0x7f0c28101660 unknown :-1 s=CLOSED pgs=313 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:38.197 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:38.196+0000 7f0c302cf700 1 --2- 172.21.15.99:0/2982659068 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f0c28102440 0x7f0c281028a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:38.198 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:38.196+0000 7f0c302cf700 1 -- 172.21.15.99:0/2982659068 >> 172.21.15.99:0/2982659068 conn(0x7f0c280fc7e0 msgr2=0x7f0c280fec20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:38.198 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:38.196+0000 7f0c302cf700 1 -- 172.21.15.99:0/2982659068 shutdown_connections 2024-03-19T01:40:38.198 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:38.196+0000 7f0c302cf700 1 -- 172.21.15.99:0/2982659068 wait complete. 2024-03-19T01:40:38.198 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:38.197+0000 7f0c302cf700 1 Processor -- start 2024-03-19T01:40:38.198 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:38.197+0000 7f0c302cf700 1 -- start start 2024-03-19T01:40:38.198 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:38.197+0000 7f0c302cf700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f0c28101240 0x7f0c281974b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:38.198 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:38.197+0000 7f0c302cf700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f0c28102440 0x7f0c281979f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:38.198 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:38.197+0000 7f0c302cf700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0c28102de0 0x7f0c2819ea70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:38.198 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:38.197+0000 7f0c302cf700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f0c2810ce30 con 0x7f0c28102de0 2024-03-19T01:40:38.198 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:38.197+0000 7f0c302cf700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f0c2810ccb0 con 0x7f0c28102440 2024-03-19T01:40:38.198 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:38.197+0000 7f0c302cf700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f0c2810cfb0 con 0x7f0c28101240 2024-03-19T01:40:38.198 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:38.197+0000 7f0c2e86c700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0c28102de0 0x7f0c2819ea70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:38.198 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:38.197+0000 7f0c2d86a700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f0c28102440 0x7f0c281979f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:38.199 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:38.197+0000 7f0c2e86c700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0c28102de0 0x7f0c2819ea70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:55180/0 (socket says 172.21.15.99:55180) 2024-03-19T01:40:38.199 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:38.197+0000 7f0c2e86c700 1 -- 172.21.15.99:0/2113966520 learned_addr learned my addr 172.21.15.99:0/2113966520 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:38.199 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:38.197+0000 7f0c2e06b700 1 --2- 172.21.15.99:0/2113966520 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f0c28101240 0x7f0c281974b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:38.200 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:38.197+0000 7f0c2d86a700 1 -- 172.21.15.99:0/2113966520 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f0c28101240 msgr2=0x7f0c281974b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:38.200 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:38.197+0000 7f0c2d86a700 1 --2- 172.21.15.99:0/2113966520 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f0c28101240 0x7f0c281974b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:38.200 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:38.197+0000 7f0c2d86a700 1 -- 172.21.15.99:0/2113966520 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0c28102de0 msgr2=0x7f0c2819ea70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:38.201 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:38.197+0000 7f0c2d86a700 1 --2- 172.21.15.99:0/2113966520 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0c28102de0 0x7f0c2819ea70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:38.201 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:38.197+0000 7f0c2d86a700 1 -- 172.21.15.99:0/2113966520 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f0c2819f110 con 0x7f0c28102440 2024-03-19T01:40:38.201 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:38.197+0000 7f0c2e86c700 1 --2- 172.21.15.99:0/2113966520 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0c28102de0 0x7f0c2819ea70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:40:38.201 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:38.198+0000 7f0c2d86a700 1 --2- 172.21.15.99:0/2113966520 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f0c28102440 0x7f0c281979f0 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 crypto rx=0x7f0c2400e9f0 tx=0x7f0c2400ed00 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:38.201 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:38.198+0000 7f0c1f7fe700 1 -- 172.21.15.99:0/2113966520 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0c2400c9c0 con 0x7f0c28102440 2024-03-19T01:40:38.201 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:38.198+0000 7f0c1f7fe700 1 -- 172.21.15.99:0/2113966520 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f0c2400cb60 con 0x7f0c28102440 2024-03-19T01:40:38.201 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:38.198+0000 7f0c302cf700 1 -- 172.21.15.99:0/2113966520 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f0c2819f400 con 0x7f0c28102440 2024-03-19T01:40:38.201 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:38.198+0000 7f0c1f7fe700 1 -- 172.21.15.99:0/2113966520 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0c24018680 con 0x7f0c28102440 2024-03-19T01:40:38.201 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:38.198+0000 7f0c302cf700 1 -- 172.21.15.99:0/2113966520 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f0c2819f990 con 0x7f0c28102440 2024-03-19T01:40:38.201 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:38.199+0000 7f0c302cf700 1 -- 172.21.15.99:0/2113966520 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f0c28068fd0 con 0x7f0c28102440 2024-03-19T01:40:38.201 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:38.200+0000 7f0c1f7fe700 1 -- 172.21.15.99:0/2113966520 <== mon.2 v2:172.21.15.112:3300/0 4 ==== mgrmap(e 18) ==== 137290+0+0 (secure 0 0 0) 0x7f0c24010430 con 0x7f0c28102440 2024-03-19T01:40:38.201 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:38.200+0000 7f0c1f7fe700 1 --2- 172.21.15.99:0/2113966520 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f0c140a2910 0x7f0c140a4dd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:38.201 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:38.200+0000 7f0c2e06b700 1 --2- 172.21.15.99:0/2113966520 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f0c140a2910 0x7f0c140a4dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:38.201 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:38.200+0000 7f0c1f7fe700 1 -- 172.21.15.99:0/2113966520 <== mon.2 v2:172.21.15.112:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f0c240ceb10 con 0x7f0c28102440 2024-03-19T01:40:38.202 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:38.201+0000 7f0c2e06b700 1 --2- 172.21.15.99:0/2113966520 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f0c140a2910 0x7f0c140a4dd0 secure :-1 s=READY pgs=229 cs=0 l=1 rev1=1 crypto rx=0x7f0c18000c00 tx=0x7f0c18005fb0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:38.204 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:38.205+0000 7f0c1f7fe700 1 -- 172.21.15.99:0/2113966520 <== mon.2 v2:172.21.15.112:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f0c2409bb70 con 0x7f0c28102440 2024-03-19T01:40:38.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:37 smithi115 ceph-mon[67332]: pgmap v336: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 6 op/s 2024-03-19T01:40:38.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:37 smithi112 ceph-mon[66819]: pgmap v336: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 6 op/s 2024-03-19T01:40:38.399 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:38.399+0000 7f0c302cf700 1 -- 172.21.15.99:0/2113966520 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_command({"prefix": "osd dump"} v 0) -- 0x7f0c28062f20 con 0x7f0c28102440 2024-03-19T01:40:38.399 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:38.400+0000 7f0c1f7fe700 1 -- 172.21.15.99:0/2113966520 <== mon.2 v2:172.21.15.112:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump"}]=0 v91) ==== 56+0+5364 (secure 0 0 0) 0x7f0c2409b470 con 0x7f0c28102440 2024-03-19T01:40:38.400 INFO:teuthology.orchestra.run.smithi099.stdout:epoch 91 2024-03-19T01:40:38.400 INFO:teuthology.orchestra.run.smithi099.stdout:fsid 438fb69c-e590-11ee-95c9-87774f69a715 2024-03-19T01:40:38.400 INFO:teuthology.orchestra.run.smithi099.stdout:created 2024-03-19T01:31:10.165073+0000 2024-03-19T01:40:38.400 INFO:teuthology.orchestra.run.smithi099.stdout:modified 2024-03-19T01:40:08.821469+0000 2024-03-19T01:40:38.400 INFO:teuthology.orchestra.run.smithi099.stdout:flags sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit 2024-03-19T01:40:38.400 INFO:teuthology.orchestra.run.smithi099.stdout:crush_version 26 2024-03-19T01:40:38.400 INFO:teuthology.orchestra.run.smithi099.stdout:full_ratio 0.95 2024-03-19T01:40:38.401 INFO:teuthology.orchestra.run.smithi099.stdout:backfillfull_ratio 0.9 2024-03-19T01:40:38.401 INFO:teuthology.orchestra.run.smithi099.stdout:nearfull_ratio 0.85 2024-03-19T01:40:38.401 INFO:teuthology.orchestra.run.smithi099.stdout:require_min_compat_client luminous 2024-03-19T01:40:38.401 INFO:teuthology.orchestra.run.smithi099.stdout:min_compat_client luminous 2024-03-19T01:40:38.401 INFO:teuthology.orchestra.run.smithi099.stdout:require_osd_release squid 2024-03-19T01:40:38.401 INFO:teuthology.orchestra.run.smithi099.stdout:stretch_mode_enabled false 2024-03-19T01:40:38.401 INFO:teuthology.orchestra.run.smithi099.stdout:pool 1 '.mgr' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode off last_change 22 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr read_balance_score 12.00 2024-03-19T01:40:38.401 INFO:teuthology.orchestra.run.smithi099.stdout:pool 2 'cephfs_metadata' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 86 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 16 recovery_priority 5 application cephfs read_balance_score 1.88 2024-03-19T01:40:38.401 INFO:teuthology.orchestra.run.smithi099.stdout:pool 3 'cephfs_data' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 64 pgp_num 64 autoscale_mode off last_change 87 flags hashpspool stripe_width 0 pg_num_min 64 target_size_ratio 0.9 application cephfs read_balance_score 1.50 2024-03-19T01:40:38.401 INFO:teuthology.orchestra.run.smithi099.stdout:max_osd 12 2024-03-19T01:40:38.401 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0 up in weight 1 up_from 8 up_thru 88 down_at 0 last_clean_interval [0,0) [v2:172.21.15.99:6802/2935260112,v1:172.21.15.99:6803/2935260112] [v2:172.21.15.99:6804/2935260112,v1:172.21.15.99:6805/2935260112] exists,up 0c407a0e-a40e-43d7-ad9a-bb8f3cd4a9ba 2024-03-19T01:40:38.401 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1 up in weight 1 up_from 13 up_thru 88 down_at 0 last_clean_interval [0,0) [v2:172.21.15.112:6800/1468854289,v1:172.21.15.112:6801/1468854289] [v2:172.21.15.112:6802/1468854289,v1:172.21.15.112:6803/1468854289] exists,up a48f9173-4a06-49f5-bca2-2c9047b8f666 2024-03-19T01:40:38.402 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2 up in weight 1 up_from 18 up_thru 85 down_at 0 last_clean_interval [0,0) [v2:172.21.15.115:6800/3766690472,v1:172.21.15.115:6801/3766690472] [v2:172.21.15.115:6802/3766690472,v1:172.21.15.115:6803/3766690472] exists,up b7715fdd-b6c9-4348-bd16-5eb61b427eef 2024-03-19T01:40:38.402 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3 up in weight 1 up_from 27 up_thru 85 down_at 0 last_clean_interval [0,0) [v2:172.21.15.99:6810/3756732895,v1:172.21.15.99:6811/3756732895] [v2:172.21.15.99:6812/3756732895,v1:172.21.15.99:6813/3756732895] exists,up 6697ed1b-96bf-42de-8440-d17e9785b359 2024-03-19T01:40:38.402 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4 up in weight 1 up_from 34 up_thru 85 down_at 0 last_clean_interval [0,0) [v2:172.21.15.112:6808/3717986391,v1:172.21.15.112:6809/3717986391] [v2:172.21.15.112:6810/3717986391,v1:172.21.15.112:6811/3717986391] exists,up d958d6b9-67f6-43d0-b704-aa55dc9df83a 2024-03-19T01:40:38.402 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5 up in weight 1 up_from 40 up_thru 85 down_at 0 last_clean_interval [0,0) [v2:172.21.15.115:6808/1971806408,v1:172.21.15.115:6809/1971806408] [v2:172.21.15.115:6810/1971806408,v1:172.21.15.115:6811/1971806408] exists,up a73eb49a-7925-49ba-bc4f-515e17f8b8ae 2024-03-19T01:40:38.402 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6 up in weight 1 up_from 47 up_thru 88 down_at 0 last_clean_interval [0,0) [v2:172.21.15.99:6818/1130832772,v1:172.21.15.99:6819/1130832772] [v2:172.21.15.99:6820/1130832772,v1:172.21.15.99:6821/1130832772] exists,up caa718cd-e08b-48de-832a-29546d71e508 2024-03-19T01:40:38.402 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7 up in weight 1 up_from 53 up_thru 88 down_at 0 last_clean_interval [0,0) [v2:172.21.15.112:6816/3610356576,v1:172.21.15.112:6817/3610356576] [v2:172.21.15.112:6818/3610356576,v1:172.21.15.112:6819/3610356576] exists,up dbee19a1-2315-40d3-88e7-876fcde28903 2024-03-19T01:40:38.402 INFO:teuthology.orchestra.run.smithi099.stdout:osd.8 up in weight 1 up_from 60 up_thru 85 down_at 0 last_clean_interval [0,0) [v2:172.21.15.115:6816/4255803111,v1:172.21.15.115:6817/4255803111] [v2:172.21.15.115:6818/4255803111,v1:172.21.15.115:6819/4255803111] exists,up 14a866dc-5ce3-42fc-8bcf-9088b68726cb 2024-03-19T01:40:38.402 INFO:teuthology.orchestra.run.smithi099.stdout:osd.9 up in weight 1 up_from 68 up_thru 90 down_at 0 last_clean_interval [0,0) [v2:172.21.15.99:6826/1410921523,v1:172.21.15.99:6827/1410921523] [v2:172.21.15.99:6828/1410921523,v1:172.21.15.99:6829/1410921523] exists,up 1db93652-6035-4906-91ae-d64d5a909502 2024-03-19T01:40:38.402 INFO:teuthology.orchestra.run.smithi099.stdout:osd.10 up in weight 1 up_from 74 up_thru 88 down_at 0 last_clean_interval [0,0) [v2:172.21.15.112:6824/891805922,v1:172.21.15.112:6825/891805922] [v2:172.21.15.112:6826/891805922,v1:172.21.15.112:6827/891805922] exists,up 6c50a8a7-3345-4f4a-af0e-5606c7aca453 2024-03-19T01:40:38.402 INFO:teuthology.orchestra.run.smithi099.stdout:osd.11 up in weight 1 up_from 82 up_thru 88 down_at 0 last_clean_interval [0,0) [v2:172.21.15.115:6824/938436244,v1:172.21.15.115:6825/938436244] [v2:172.21.15.115:6826/938436244,v1:172.21.15.115:6827/938436244] exists,up fd63b7fd-0db5-46d5-82fa-9707bbd0b495 2024-03-19T01:40:38.402 INFO:teuthology.orchestra.run.smithi099.stdout:pg_upmap_items 2.0 [8,1] 2024-03-19T01:40:38.402 INFO:teuthology.orchestra.run.smithi099.stdout:pg_upmap_items 2.8 [2,11] 2024-03-19T01:40:38.402 INFO:teuthology.orchestra.run.smithi099.stdout:pg_upmap_items 2.b [8,11] 2024-03-19T01:40:38.403 INFO:teuthology.orchestra.run.smithi099.stdout:pg_upmap_items 2.e [2,4] 2024-03-19T01:40:38.403 INFO:teuthology.orchestra.run.smithi099.stdout:pg_upmap_items 3.0 [5,9] 2024-03-19T01:40:38.403 INFO:teuthology.orchestra.run.smithi099.stdout:pg_upmap_items 3.2 [3,9] 2024-03-19T01:40:38.403 INFO:teuthology.orchestra.run.smithi099.stdout:pg_upmap_items 3.14 [1,9] 2024-03-19T01:40:38.403 INFO:teuthology.orchestra.run.smithi099.stdout:pg_upmap_items 3.17 [4,8] 2024-03-19T01:40:38.403 INFO:teuthology.orchestra.run.smithi099.stdout:pg_upmap_items 3.31 [4,8] 2024-03-19T01:40:38.403 INFO:teuthology.orchestra.run.smithi099.stdout:pg_upmap_items 3.37 [5,7] 2024-03-19T01:40:38.403 INFO:teuthology.orchestra.run.smithi099.stdout:pg_upmap_items 3.3a [4,6] 2024-03-19T01:40:38.403 INFO:teuthology.orchestra.run.smithi099.stdout:blocklist 172.21.15.99:6801/1328245934 expires 2024-03-20T01:32:06.661002+0000 2024-03-19T01:40:38.403 INFO:teuthology.orchestra.run.smithi099.stdout:blocklist 172.21.15.99:0/1618965837 expires 2024-03-20T01:32:06.661002+0000 2024-03-19T01:40:38.403 INFO:teuthology.orchestra.run.smithi099.stdout:blocklist 172.21.15.99:0/2491429851 expires 2024-03-20T01:32:06.661002+0000 2024-03-19T01:40:38.403 INFO:teuthology.orchestra.run.smithi099.stdout:blocklist 172.21.15.99:6800/1328245934 expires 2024-03-20T01:32:06.661002+0000 2024-03-19T01:40:38.403 INFO:teuthology.orchestra.run.smithi099.stdout:blocklist 172.21.15.99:0/2112231766 expires 2024-03-20T01:31:40.161751+0000 2024-03-19T01:40:38.404 INFO:teuthology.orchestra.run.smithi099.stdout:blocklist 172.21.15.99:0/3341836545 expires 2024-03-20T01:31:40.161751+0000 2024-03-19T01:40:38.404 INFO:teuthology.orchestra.run.smithi099.stdout:blocklist 172.21.15.99:6800/4223048293 expires 2024-03-20T01:31:40.161751+0000 2024-03-19T01:40:38.404 INFO:teuthology.orchestra.run.smithi099.stdout:blocklist 172.21.15.99:6801/4223048293 expires 2024-03-20T01:31:40.161751+0000 2024-03-19T01:40:38.404 INFO:teuthology.orchestra.run.smithi099.stdout:blocklist 172.21.15.99:0/2377621863 expires 2024-03-20T01:31:40.161751+0000 2024-03-19T01:40:38.404 INFO:teuthology.orchestra.run.smithi099.stdout:blocklist 172.21.15.99:0/1904400935 expires 2024-03-20T01:32:06.661002+0000 2024-03-19T01:40:38.405 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:38.402+0000 7f0c302cf700 1 -- 172.21.15.99:0/2113966520 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f0c140a2910 msgr2=0x7f0c140a4dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:38.405 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:38.402+0000 7f0c302cf700 1 --2- 172.21.15.99:0/2113966520 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f0c140a2910 0x7f0c140a4dd0 secure :-1 s=READY pgs=229 cs=0 l=1 rev1=1 crypto rx=0x7f0c18000c00 tx=0x7f0c18005fb0 comp rx=0 tx=0).stop 2024-03-19T01:40:38.405 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:38.402+0000 7f0c302cf700 1 -- 172.21.15.99:0/2113966520 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f0c28102440 msgr2=0x7f0c281979f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:38.405 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:38.402+0000 7f0c302cf700 1 --2- 172.21.15.99:0/2113966520 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f0c28102440 0x7f0c281979f0 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 crypto rx=0x7f0c2400e9f0 tx=0x7f0c2400ed00 comp rx=0 tx=0).stop 2024-03-19T01:40:38.405 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:38.402+0000 7f0c302cf700 1 -- 172.21.15.99:0/2113966520 shutdown_connections 2024-03-19T01:40:38.405 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:38.402+0000 7f0c302cf700 1 --2- 172.21.15.99:0/2113966520 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f0c140a2910 0x7f0c140a4dd0 unknown :-1 s=CLOSED pgs=229 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:38.406 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:38.402+0000 7f0c302cf700 1 --2- 172.21.15.99:0/2113966520 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f0c28101240 0x7f0c281974b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:38.406 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:38.402+0000 7f0c302cf700 1 --2- 172.21.15.99:0/2113966520 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0c28102de0 0x7f0c2819ea70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:38.406 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:38.402+0000 7f0c302cf700 1 --2- 172.21.15.99:0/2113966520 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f0c28102440 0x7f0c281979f0 unknown :-1 s=CLOSED pgs=77 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:38.406 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:38.402+0000 7f0c302cf700 1 -- 172.21.15.99:0/2113966520 >> 172.21.15.99:0/2113966520 conn(0x7f0c280fc7e0 msgr2=0x7f0c28105800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:38.406 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:38.402+0000 7f0c302cf700 1 -- 172.21.15.99:0/2113966520 shutdown_connections 2024-03-19T01:40:38.406 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:38.402+0000 7f0c302cf700 1 -- 172.21.15.99:0/2113966520 wait complete. 2024-03-19T01:40:39.110 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:38 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/2113966520' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-03-19T01:40:39.140 INFO:teuthology.run_tasks:Running task fs.ready... 2024-03-19T01:40:39.152 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-19T01:40:39.261 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.259+0000 7f417b9ab640 1 -- 172.21.15.99:0/2949680909 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4174100740 msgr2=0x7f4174100b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:39.261 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.259+0000 7f417b9ab640 1 --2- 172.21.15.99:0/2949680909 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4174100740 0x7f4174100b60 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7f4170009ca0 tx=0x7f417002c0f0 comp rx=0 tx=0).stop 2024-03-19T01:40:39.261 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.259+0000 7f417b9ab640 1 -- 172.21.15.99:0/2949680909 shutdown_connections 2024-03-19T01:40:39.261 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.259+0000 7f417b9ab640 1 --2- 172.21.15.99:0/2949680909 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f41741022f0 0x7f4174108b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:39.261 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.259+0000 7f417b9ab640 1 --2- 172.21.15.99:0/2949680909 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4174101930 0x7f4174101db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:39.261 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.259+0000 7f417b9ab640 1 --2- 172.21.15.99:0/2949680909 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4174100740 0x7f4174100b60 unknown :-1 s=CLOSED pgs=78 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:39.261 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.259+0000 7f417b9ab640 1 -- 172.21.15.99:0/2949680909 >> 172.21.15.99:0/2949680909 conn(0x7f41740fbcb0 msgr2=0x7f41740fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:39.262 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.259+0000 7f417b9ab640 1 -- 172.21.15.99:0/2949680909 shutdown_connections 2024-03-19T01:40:39.262 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.259+0000 7f417b9ab640 1 -- 172.21.15.99:0/2949680909 wait complete. 2024-03-19T01:40:39.262 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.259+0000 7f417b9ab640 1 Processor -- start 2024-03-19T01:40:39.262 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.260+0000 7f417b9ab640 1 -- start start 2024-03-19T01:40:39.262 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.260+0000 7f417b9ab640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4174100740 0x7f41740694f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:39.262 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.260+0000 7f417b9ab640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4174101930 0x7f417419ecc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:39.262 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.260+0000 7f417b9ab640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f41741022f0 0x7f41741a1210 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:39.262 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.260+0000 7f417b9ab640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f417410c3b0 con 0x7f4174101930 2024-03-19T01:40:39.262 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.260+0000 7f417b9ab640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f417410c230 con 0x7f41741022f0 2024-03-19T01:40:39.262 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.260+0000 7f417b9ab640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f417410c530 con 0x7f4174100740 2024-03-19T01:40:39.262 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.260+0000 7f4178f46640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4174101930 0x7f417419ecc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:39.262 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.260+0000 7f4179747640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4174100740 0x7f41740694f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:39.262 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.260+0000 7f4178f46640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4174101930 0x7f417419ecc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:55190/0 (socket says 172.21.15.99:55190) 2024-03-19T01:40:39.262 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.260+0000 7f4178f46640 1 -- 172.21.15.99:0/656754060 learned_addr learned my addr 172.21.15.99:0/656754060 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:39.263 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.260+0000 7f4179f48640 1 --2- 172.21.15.99:0/656754060 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f41741022f0 0x7f41741a1210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:39.263 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.260+0000 7f4178f46640 1 -- 172.21.15.99:0/656754060 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4174100740 msgr2=0x7f41740694f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:39.263 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.260+0000 7f4178f46640 1 --2- 172.21.15.99:0/656754060 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4174100740 0x7f41740694f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:39.263 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.260+0000 7f4178f46640 1 -- 172.21.15.99:0/656754060 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f41741022f0 msgr2=0x7f41741a1210 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:39.263 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.260+0000 7f4178f46640 1 --2- 172.21.15.99:0/656754060 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f41741022f0 0x7f41741a1210 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:39.263 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.260+0000 7f4178f46640 1 -- 172.21.15.99:0/656754060 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f41741a1a00 con 0x7f4174101930 2024-03-19T01:40:39.263 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.260+0000 7f4178f46640 1 --2- 172.21.15.99:0/656754060 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4174101930 0x7f417419ecc0 secure :-1 s=READY pgs=314 cs=0 l=1 rev1=1 crypto rx=0x7f416400e9f0 tx=0x7f416400edf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:39.263 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.260+0000 7f416a7fc640 1 -- 172.21.15.99:0/656754060 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f416400cd80 con 0x7f4174101930 2024-03-19T01:40:39.263 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.260+0000 7f416a7fc640 1 -- 172.21.15.99:0/656754060 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f4164005500 con 0x7f4174101930 2024-03-19T01:40:39.263 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.260+0000 7f417b9ab640 1 -- 172.21.15.99:0/656754060 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f41741a1c90 con 0x7f4174101930 2024-03-19T01:40:39.263 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.260+0000 7f416a7fc640 1 -- 172.21.15.99:0/656754060 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4164010690 con 0x7f4174101930 2024-03-19T01:40:39.263 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.260+0000 7f417b9ab640 1 -- 172.21.15.99:0/656754060 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f41741a2190 con 0x7f4174101930 2024-03-19T01:40:39.264 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.261+0000 7f417b9ab640 1 -- 172.21.15.99:0/656754060 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f4174107a60 con 0x7f4174101930 2024-03-19T01:40:39.264 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.262+0000 7f416a7fc640 1 -- 172.21.15.99:0/656754060 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 18) ==== 137290+0+0 (secure 0 0 0) 0x7f41640046c0 con 0x7f4174101930 2024-03-19T01:40:39.264 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.263+0000 7f416a7fc640 1 --2- 172.21.15.99:0/656754060 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f41600a2830 0x7f41600a4cf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:39.265 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.263+0000 7f4179747640 1 --2- 172.21.15.99:0/656754060 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f41600a2830 0x7f41600a4cf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:39.265 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.263+0000 7f416a7fc640 1 -- 172.21.15.99:0/656754060 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f41640cee70 con 0x7f4174101930 2024-03-19T01:40:39.265 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.263+0000 7f4179747640 1 --2- 172.21.15.99:0/656754060 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f41600a2830 0x7f41600a4cf0 secure :-1 s=READY pgs=230 cs=0 l=1 rev1=1 crypto rx=0x7f417002c700 tx=0x7f4170032590 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:39.267 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.267+0000 7f416a7fc640 1 -- 172.21.15.99:0/656754060 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f416409bed0 con 0x7f4174101930 2024-03-19T01:40:39.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:38 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/2113966520' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-03-19T01:40:39.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:38 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/2113966520' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-03-19T01:40:39.445 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.445+0000 7f417b9ab640 1 -- 172.21.15.99:0/656754060 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f4174062420 con 0x7f4174101930 2024-03-19T01:40:39.446 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.447+0000 7f416a7fc640 1 -- 172.21.15.99:0/656754060 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v91) ==== 74+0+22804 (secure 0 0 0) 0x7f416409b7d0 con 0x7f4174101930 2024-03-19T01:40:39.447 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:40:39.447 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":91,"fsid":"438fb69c-e590-11ee-95c9-87774f69a715","created":"2024-03-19T01:31:10.165073+0000","modified":"2024-03-19T01:40:08.821469+0000","last_up_change":"2024-03-19T01:38:09.082688+0000","last_in_change":"2024-03-19T01:37:49.115793+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-19T01:34:31.001309+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-19T01:38:54.018623+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"86","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-19T01:38:55.319036+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"87","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"0c407a0e-a40e-43d7-ad9a-bb8f3cd4a9ba","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6803","nonce":2935260112}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6805","nonce":2935260112}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6808","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6809","nonce":2935260112}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6807","nonce":2935260112}]},"public_addr":"172.21.15.99:6803/2935260112","cluster_addr":"172.21.15.99:6805/2935260112","heartbeat_back_addr":"172.21.15.99:6809/2935260112","heartbeat_front_addr":"172.21.15.99:6807/2935260112","state":["exists","up"]},{"osd":1,"uuid":"a48f9173-4a06-49f5-bca2-2c9047b8f666","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6800","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6801","nonce":1468854289}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6802","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6803","nonce":1468854289}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6806","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6807","nonce":1468854289}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6804","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6805","nonce":1468854289}]},"public_addr":"172.21.15.112:6801/1468854289","cluster_addr":"172.21.15.112:6803/1468854289","heartbeat_back_addr":"172.21.15.112:6807/1468854289","heartbeat_front_addr":"172.21.15.112:6805/1468854289","state":["exists","up"]},{"osd":2,"uuid":"b7715fdd-b6c9-4348-bd16-5eb61b427eef","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6800","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6801","nonce":3766690472}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6802","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6803","nonce":3766690472}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6806","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6807","nonce":3766690472}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6804","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6805","nonce":3766690472}]},"public_addr":"172.21.15.115:6801/3766690472","cluster_addr":"172.21.15.115:6803/3766690472","heartbeat_back_addr":"172.21.15.115:6807/3766690472","heartbeat_front_addr":"172.21.15.115:6805/3766690472","state":["exists","up"]},{"osd":3,"uuid":"6697ed1b-96bf-42de-8440-d17e9785b359","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6810","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6811","nonce":3756732895}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6812","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6813","nonce":3756732895}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6816","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6817","nonce":3756732895}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6814","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6815","nonce":3756732895}]},"public_addr":"172.21.15.99:6811/3756732895","cluster_addr":"172.21.15.99:6813/3756732895","heartbeat_back_addr":"172.21.15.99:6817/3756732895","heartbeat_front_addr":"172.21.15.99:6815/3756732895","state":["exists","up"]},{"osd":4,"uuid":"d958d6b9-67f6-43d0-b704-aa55dc9df83a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":34,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6808","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6809","nonce":3717986391}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6810","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6811","nonce":3717986391}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6814","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6815","nonce":3717986391}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6812","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6813","nonce":3717986391}]},"public_addr":"172.21.15.112:6809/3717986391","cluster_addr":"172.21.15.112:6811/3717986391","heartbeat_back_addr":"172.21.15.112:6815/3717986391","heartbeat_front_addr":"172.21.15.112:6813/3717986391","state":["exists","up"]},{"osd":5,"uuid":"a73eb49a-7925-49ba-bc4f-515e17f8b8ae","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6808","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6809","nonce":1971806408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6810","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6811","nonce":1971806408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6814","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6815","nonce":1971806408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6812","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6813","nonce":1971806408}]},"public_addr":"172.21.15.115:6809/1971806408","cluster_addr":"172.21.15.115:6811/1971806408","heartbeat_back_addr":"172.21.15.115:6815/1971806408","heartbeat_front_addr":"172.21.15.115:6813/1971806408","state":["exists","up"]},{"osd":6,"uuid":"caa718cd-e08b-48de-832a-29546d71e508","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6818","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6819","nonce":1130832772}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6820","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6821","nonce":1130832772}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6824","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6825","nonce":1130832772}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6822","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6823","nonce":1130832772}]},"public_addr":"172.21.15.99:6819/1130832772","cluster_addr":"172.21.15.99:6821/1130832772","heartbeat_back_addr":"172.21.15.99:6825/1130832772","heartbeat_front_addr":"172.21.15.99:6823/1130832772","state":["exists","up"]},{"osd":7,"uuid":"dbee19a1-2315-40d3-88e7-876fcde28903","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":53,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6816","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6817","nonce":3610356576}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6818","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6819","nonce":3610356576}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6822","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6823","nonce":3610356576}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6820","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6821","nonce":3610356576}]},"public_addr":"172.21.15.112:6817/3610356576","cluster_addr":"172.21.15.112:6819/3610356576","heartbeat_back_addr":"172.21.15.112:6823/3610356576","heartbeat_front_addr":"172.21.15.112:6821/3610356576","state":["exists","up"]},{"osd":8,"uuid":"14a866dc-5ce3-42fc-8bcf-9088b68726cb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":60,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6816","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6817","nonce":4255803111}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6818","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6819","nonce":4255803111}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6822","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6823","nonce":4255803111}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6820","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6821","nonce":4255803111}]},"public_addr":"172.21.15.115:6817/4255803111","cluster_addr":"172.21.15.115:6819/4255803111","heartbeat_back_addr":"172.21.15.115:6823/4255803111","heartbeat_front_addr":"172.21.15.115:6821/4255803111","state":["exists","up"]},{"osd":9,"uuid":"1db93652-6035-4906-91ae-d64d5a909502","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":68,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6826","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6827","nonce":1410921523}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6828","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6829","nonce":1410921523}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6832","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6833","nonce":1410921523}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6830","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6831","nonce":1410921523}]},"public_addr":"172.21.15.99:6827/1410921523","cluster_addr":"172.21.15.99:6829/1410921523","heartbeat_back_addr":"172.21.15.99:6833/1410921523","heartbeat_front_addr":"172.21.15.99:6831/1410921523","state":["exists","up"]},{"osd":10,"uuid":"6c50a8a7-3345-4f4a-af0e-5606c7aca453","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":74,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6824","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6825","nonce":891805922}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6826","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6827","nonce":891805922}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6830","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6831","nonce":891805922}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6828","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6829","nonce":891805922}]},"public_addr":"172.21.15.112:6825/891805922","cluster_addr":"172.21.15.112:6827/891805922","heartbeat_back_addr":"172.21.15.112:6831/891805922","heartbeat_front_addr":"172.21.15.112:6829/891805922","state":["exists","up"]},{"osd":11,"uuid":"fd63b7fd-0db5-46d5-82fa-9707bbd0b495","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":82,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6824","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6825","nonce":938436244}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6826","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6827","nonce":938436244}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6830","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6831","nonce":938436244}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6828","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6829","nonce":938436244}]},"public_addr":"172.21.15.115:6825/938436244","cluster_addr":"172.21.15.115:6827/938436244","heartbeat_back_addr":"172.21.15.115:6831/938436244","heartbeat_front_addr":"172.21.15.115:6829/938436244","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:33:39.036822+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:34:03.339566+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:34:27.149273+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:34:51.315561+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:35:15.396031+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:35:38.957168+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:36:03.213914+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:36:27.763699+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:36:51.688670+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:37:17.280165+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:37:42.554633+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:38:07.438039+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.0","mappings":[{"from":8,"to":1}]},{"pgid":"2.8","mappings":[{"from":2,"to":11}]},{"pgid":"2.b","mappings":[{"from":8,"to":11}]},{"pgid":"2.e","mappings":[{"from":2,"to":4}]},{"pgid":"3.0","mappings":[{"from":5,"to":9}]},{"pgid":"3.2","mappings":[{"from":3,"to":9}]},{"pgid":"3.14","mappings":[{"from":1,"to":9}]},{"pgid":"3.17","mappings":[{"from":4,"to":8}]},{"pgid":"3.31","mappings":[{"from":4,"to":8}]},{"pgid":"3.37","mappings":[{"from":5,"to":7}]},{"pgid":"3.3a","mappings":[{"from":4,"to":6}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.99:6801/1328245934":"2024-03-20T01:32:06.661002+0000","172.21.15.99:0/1618965837":"2024-03-20T01:32:06.661002+0000","172.21.15.99:0/2491429851":"2024-03-20T01:32:06.661002+0000","172.21.15.99:6800/1328245934":"2024-03-20T01:32:06.661002+0000","172.21.15.99:0/2112231766":"2024-03-20T01:31:40.161751+0000","172.21.15.99:0/3341836545":"2024-03-20T01:31:40.161751+0000","172.21.15.99:6800/4223048293":"2024-03-20T01:31:40.161751+0000","172.21.15.99:6801/4223048293":"2024-03-20T01:31:40.161751+0000","172.21.15.99:0/2377621863":"2024-03-20T01:31:40.161751+0000","172.21.15.99:0/1904400935":"2024-03-20T01:32:06.661002+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-19T01:40:39.449 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.449+0000 7f417b9ab640 1 -- 172.21.15.99:0/656754060 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f41600a2830 msgr2=0x7f41600a4cf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:39.449 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.449+0000 7f417b9ab640 1 --2- 172.21.15.99:0/656754060 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f41600a2830 0x7f41600a4cf0 secure :-1 s=READY pgs=230 cs=0 l=1 rev1=1 crypto rx=0x7f417002c700 tx=0x7f4170032590 comp rx=0 tx=0).stop 2024-03-19T01:40:39.449 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.449+0000 7f417b9ab640 1 -- 172.21.15.99:0/656754060 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4174101930 msgr2=0x7f417419ecc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:39.449 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.449+0000 7f417b9ab640 1 --2- 172.21.15.99:0/656754060 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4174101930 0x7f417419ecc0 secure :-1 s=READY pgs=314 cs=0 l=1 rev1=1 crypto rx=0x7f416400e9f0 tx=0x7f416400edf0 comp rx=0 tx=0).stop 2024-03-19T01:40:39.449 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.449+0000 7f417b9ab640 1 -- 172.21.15.99:0/656754060 shutdown_connections 2024-03-19T01:40:39.449 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.449+0000 7f417b9ab640 1 --2- 172.21.15.99:0/656754060 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f41600a2830 0x7f41600a4cf0 unknown :-1 s=CLOSED pgs=230 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:39.449 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.449+0000 7f417b9ab640 1 --2- 172.21.15.99:0/656754060 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f41741022f0 0x7f41741a1210 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:39.449 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.449+0000 7f417b9ab640 1 --2- 172.21.15.99:0/656754060 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4174101930 0x7f417419ecc0 unknown :-1 s=CLOSED pgs=314 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:39.450 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.449+0000 7f417b9ab640 1 --2- 172.21.15.99:0/656754060 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4174100740 0x7f41740694f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:39.450 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.449+0000 7f417b9ab640 1 -- 172.21.15.99:0/656754060 >> 172.21.15.99:0/656754060 conn(0x7f41740fbcb0 msgr2=0x7f4174104c30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:39.450 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.449+0000 7f417b9ab640 1 -- 172.21.15.99:0/656754060 shutdown_connections 2024-03-19T01:40:39.450 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.449+0000 7f417b9ab640 1 -- 172.21.15.99:0/656754060 wait complete. 2024-03-19T01:40:39.460 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-19T01:34:31.001309+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 12, 'score_stable': 12, 'optimal_score': 0.25, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-19T01:38:54.018623+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '86', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.8799999952316284, 'score_stable': 1.8799999952316284, 'optimal_score': 1, 'raw_score_acting': 1.8799999952316284, 'raw_score_stable': 1.8799999952316284, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-03-19T01:38:55.319036+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '87', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-19T01:40:39.461 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-19T01:40:39.566 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.564+0000 7ffa399f3640 1 -- 172.21.15.99:0/2573455128 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ffa341022f0 msgr2=0x7ffa34108b80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:39.566 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.564+0000 7ffa399f3640 1 --2- 172.21.15.99:0/2573455128 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ffa341022f0 0x7ffa34108b80 secure :-1 s=READY pgs=315 cs=0 l=1 rev1=1 crypto rx=0x7ffa2c009ca0 tx=0x7ffa2c02c0f0 comp rx=0 tx=0).stop 2024-03-19T01:40:39.566 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.564+0000 7ffa399f3640 1 -- 172.21.15.99:0/2573455128 shutdown_connections 2024-03-19T01:40:39.566 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.564+0000 7ffa399f3640 1 --2- 172.21.15.99:0/2573455128 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ffa34101930 0x7ffa34101db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:39.567 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.564+0000 7ffa399f3640 1 --2- 172.21.15.99:0/2573455128 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ffa341022f0 0x7ffa34108b80 unknown :-1 s=CLOSED pgs=315 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:39.567 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.564+0000 7ffa399f3640 1 --2- 172.21.15.99:0/2573455128 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ffa34100740 0x7ffa34100b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:39.567 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.564+0000 7ffa399f3640 1 -- 172.21.15.99:0/2573455128 >> 172.21.15.99:0/2573455128 conn(0x7ffa340fbcd0 msgr2=0x7ffa340fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:39.567 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.564+0000 7ffa399f3640 1 -- 172.21.15.99:0/2573455128 shutdown_connections 2024-03-19T01:40:39.567 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.564+0000 7ffa399f3640 1 -- 172.21.15.99:0/2573455128 wait complete. 2024-03-19T01:40:39.567 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.565+0000 7ffa399f3640 1 Processor -- start 2024-03-19T01:40:39.567 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.565+0000 7ffa399f3640 1 -- start start 2024-03-19T01:40:39.567 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.565+0000 7ffa399f3640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ffa34100740 0x7ffa34196a10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:39.567 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.565+0000 7ffa399f3640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ffa34101930 0x7ffa34196f50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:39.567 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.565+0000 7ffa399f3640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ffa341022f0 0x7ffa3419dfc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:39.567 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.565+0000 7ffa399f3640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7ffa3410c3c0 con 0x7ffa341022f0 2024-03-19T01:40:39.567 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.565+0000 7ffa399f3640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7ffa3410c240 con 0x7ffa34101930 2024-03-19T01:40:39.567 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.565+0000 7ffa399f3640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7ffa3410c540 con 0x7ffa34100740 2024-03-19T01:40:39.567 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.565+0000 7ffa337fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ffa341022f0 0x7ffa3419dfc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:39.568 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.565+0000 7ffa32ffd640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ffa34100740 0x7ffa34196a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:39.568 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.565+0000 7ffa337fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ffa341022f0 0x7ffa3419dfc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:55218/0 (socket says 172.21.15.99:55218) 2024-03-19T01:40:39.568 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.565+0000 7ffa337fe640 1 -- 172.21.15.99:0/3048073166 learned_addr learned my addr 172.21.15.99:0/3048073166 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:39.568 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.565+0000 7ffa32ffd640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ffa34100740 0x7ffa34196a10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:40848/0 (socket says 172.21.15.99:40848) 2024-03-19T01:40:39.568 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.565+0000 7ffa2bfff640 1 --2- 172.21.15.99:0/3048073166 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ffa34101930 0x7ffa34196f50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:39.568 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.565+0000 7ffa337fe640 1 -- 172.21.15.99:0/3048073166 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ffa34100740 msgr2=0x7ffa34196a10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:39.568 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.565+0000 7ffa337fe640 1 --2- 172.21.15.99:0/3048073166 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ffa34100740 0x7ffa34196a10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:39.568 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.565+0000 7ffa337fe640 1 -- 172.21.15.99:0/3048073166 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ffa34101930 msgr2=0x7ffa34196f50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:39.568 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.565+0000 7ffa337fe640 1 --2- 172.21.15.99:0/3048073166 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ffa34101930 0x7ffa34196f50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:39.568 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.565+0000 7ffa337fe640 1 -- 172.21.15.99:0/3048073166 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ffa3419e630 con 0x7ffa341022f0 2024-03-19T01:40:39.568 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.565+0000 7ffa337fe640 1 --2- 172.21.15.99:0/3048073166 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ffa341022f0 0x7ffa3419dfc0 secure :-1 s=READY pgs=316 cs=0 l=1 rev1=1 crypto rx=0x7ffa2c037040 tx=0x7ffa2c032f40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:39.568 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.565+0000 7ffa389f1640 1 -- 172.21.15.99:0/3048073166 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffa2c041020 con 0x7ffa341022f0 2024-03-19T01:40:39.568 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.565+0000 7ffa389f1640 1 -- 172.21.15.99:0/3048073166 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7ffa2c034cd0 con 0x7ffa341022f0 2024-03-19T01:40:39.568 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.565+0000 7ffa399f3640 1 -- 172.21.15.99:0/3048073166 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ffa34069e30 con 0x7ffa341022f0 2024-03-19T01:40:39.568 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.565+0000 7ffa389f1640 1 -- 172.21.15.99:0/3048073166 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffa2c039040 con 0x7ffa341022f0 2024-03-19T01:40:39.569 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.566+0000 7ffa399f3640 1 -- 172.21.15.99:0/3048073166 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7ffa3406a320 con 0x7ffa341022f0 2024-03-19T01:40:39.569 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.566+0000 7ffa399f3640 1 -- 172.21.15.99:0/3048073166 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7ffa34068a60 con 0x7ffa341022f0 2024-03-19T01:40:39.569 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.567+0000 7ffa389f1640 1 -- 172.21.15.99:0/3048073166 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 18) ==== 137290+0+0 (secure 0 0 0) 0x7ffa2c044020 con 0x7ffa341022f0 2024-03-19T01:40:39.569 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.568+0000 7ffa389f1640 1 --2- 172.21.15.99:0/3048073166 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ffa200a2800 0x7ffa200a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:39.569 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.568+0000 7ffa32ffd640 1 --2- 172.21.15.99:0/3048073166 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ffa200a2800 0x7ffa200a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:39.569 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.568+0000 7ffa389f1640 1 -- 172.21.15.99:0/3048073166 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7ffa2c0f0940 con 0x7ffa341022f0 2024-03-19T01:40:39.570 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.568+0000 7ffa32ffd640 1 --2- 172.21.15.99:0/3048073166 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ffa200a2800 0x7ffa200a4cc0 secure :-1 s=READY pgs=231 cs=0 l=1 rev1=1 crypto rx=0x7ffa34101790 tx=0x7ffa24005f90 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:39.572 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.572+0000 7ffa389f1640 1 -- 172.21.15.99:0/3048073166 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7ffa2c04b020 con 0x7ffa341022f0 2024-03-19T01:40:39.746 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.746+0000 7ffa399f3640 1 -- 172.21.15.99:0/3048073166 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) -- 0x7ffa3419e8c0 con 0x7ffa341022f0 2024-03-19T01:40:39.747 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.747+0000 7ffa389f1640 1 -- 172.21.15.99:0/3048073166 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v91) ==== 93+0+10 (secure 0 0 0) 0x7ffa2c0bda30 con 0x7ffa341022f0 2024-03-19T01:40:39.747 INFO:teuthology.orchestra.run.smithi099.stdout:pg_num: 1 2024-03-19T01:40:39.749 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.749+0000 7ffa399f3640 1 -- 172.21.15.99:0/3048073166 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ffa200a2800 msgr2=0x7ffa200a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:39.749 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.749+0000 7ffa399f3640 1 --2- 172.21.15.99:0/3048073166 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ffa200a2800 0x7ffa200a4cc0 secure :-1 s=READY pgs=231 cs=0 l=1 rev1=1 crypto rx=0x7ffa34101790 tx=0x7ffa24005f90 comp rx=0 tx=0).stop 2024-03-19T01:40:39.749 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.749+0000 7ffa399f3640 1 -- 172.21.15.99:0/3048073166 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ffa341022f0 msgr2=0x7ffa3419dfc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:39.749 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.749+0000 7ffa399f3640 1 --2- 172.21.15.99:0/3048073166 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ffa341022f0 0x7ffa3419dfc0 secure :-1 s=READY pgs=316 cs=0 l=1 rev1=1 crypto rx=0x7ffa2c037040 tx=0x7ffa2c032f40 comp rx=0 tx=0).stop 2024-03-19T01:40:39.749 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.749+0000 7ffa399f3640 1 -- 172.21.15.99:0/3048073166 shutdown_connections 2024-03-19T01:40:39.749 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.749+0000 7ffa399f3640 1 --2- 172.21.15.99:0/3048073166 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ffa200a2800 0x7ffa200a4cc0 unknown :-1 s=CLOSED pgs=231 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:39.749 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.749+0000 7ffa399f3640 1 --2- 172.21.15.99:0/3048073166 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ffa34101930 0x7ffa34196f50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:39.750 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.749+0000 7ffa399f3640 1 --2- 172.21.15.99:0/3048073166 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ffa341022f0 0x7ffa3419dfc0 unknown :-1 s=CLOSED pgs=316 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:39.750 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.749+0000 7ffa399f3640 1 --2- 172.21.15.99:0/3048073166 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ffa34100740 0x7ffa34196a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:39.750 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.749+0000 7ffa399f3640 1 -- 172.21.15.99:0/3048073166 >> 172.21.15.99:0/3048073166 conn(0x7ffa340fbcd0 msgr2=0x7ffa34104c30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:39.750 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.749+0000 7ffa399f3640 1 -- 172.21.15.99:0/3048073166 shutdown_connections 2024-03-19T01:40:39.750 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.749+0000 7ffa399f3640 1 -- 172.21.15.99:0/3048073166 wait complete. 2024-03-19T01:40:39.760 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-19T01:40:39.868 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.866+0000 7fe00dc80640 1 -- 172.21.15.99:0/96452208 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe008102a20 msgr2=0x7fe008102ea0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:39.868 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.866+0000 7fe00dc80640 1 --2- 172.21.15.99:0/96452208 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe008102a20 0x7fe008102ea0 secure :-1 s=READY pgs=317 cs=0 l=1 rev1=1 crypto rx=0x7fdffc009900 tx=0x7fdffc02bc30 comp rx=0 tx=0).stop 2024-03-19T01:40:39.868 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.866+0000 7fe00dc80640 1 -- 172.21.15.99:0/96452208 shutdown_connections 2024-03-19T01:40:39.868 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.866+0000 7fe00dc80640 1 --2- 172.21.15.99:0/96452208 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fe0081033e0 0x7fe008109c70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:39.868 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.866+0000 7fe00dc80640 1 --2- 172.21.15.99:0/96452208 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe008102a20 0x7fe008102ea0 unknown :-1 s=CLOSED pgs=317 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:39.868 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.866+0000 7fe00dc80640 1 --2- 172.21.15.99:0/96452208 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fe008101830 0x7fe008101c50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:39.868 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.866+0000 7fe00dc80640 1 -- 172.21.15.99:0/96452208 >> 172.21.15.99:0/96452208 conn(0x7fe0080fcdc0 msgr2=0x7fe0080ff1e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:39.868 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.866+0000 7fe00dc80640 1 -- 172.21.15.99:0/96452208 shutdown_connections 2024-03-19T01:40:39.868 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.866+0000 7fe00dc80640 1 -- 172.21.15.99:0/96452208 wait complete. 2024-03-19T01:40:39.868 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.866+0000 7fe00dc80640 1 Processor -- start 2024-03-19T01:40:39.869 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.866+0000 7fe00dc80640 1 -- start start 2024-03-19T01:40:39.869 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.867+0000 7fe00dc80640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fe008101830 0x7fe008197b60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:39.869 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.867+0000 7fe00dc80640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fe008102a20 0x7fe0081980a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:39.869 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.867+0000 7fe00dc80640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe0081033e0 0x7fe00819f110 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:39.869 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.867+0000 7fe00dc80640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fe00810d4b0 con 0x7fe0081033e0 2024-03-19T01:40:39.869 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.867+0000 7fe00dc80640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fe00810d330 con 0x7fe008101830 2024-03-19T01:40:39.869 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.867+0000 7fe00dc80640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fe00810d630 con 0x7fe008102a20 2024-03-19T01:40:39.869 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.867+0000 7fe007fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe0081033e0 0x7fe00819f110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:39.869 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.867+0000 7fe007fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe0081033e0 0x7fe00819f110 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:55236/0 (socket says 172.21.15.99:55236) 2024-03-19T01:40:39.869 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.867+0000 7fe007fff640 1 -- 172.21.15.99:0/2461465397 learned_addr learned my addr 172.21.15.99:0/2461465397 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:39.869 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.867+0000 7fe0077fe640 1 --2- 172.21.15.99:0/2461465397 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fe008101830 0x7fe008197b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:39.869 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.867+0000 7fe006ffd640 1 --2- 172.21.15.99:0/2461465397 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fe008102a20 0x7fe0081980a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:39.869 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.867+0000 7fe007fff640 1 -- 172.21.15.99:0/2461465397 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fe008102a20 msgr2=0x7fe0081980a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:39.869 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.867+0000 7fe007fff640 1 --2- 172.21.15.99:0/2461465397 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fe008102a20 0x7fe0081980a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:39.870 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.867+0000 7fe007fff640 1 -- 172.21.15.99:0/2461465397 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fe008101830 msgr2=0x7fe008197b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:39.870 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.867+0000 7fe007fff640 1 --2- 172.21.15.99:0/2461465397 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fe008101830 0x7fe008197b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:39.870 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.867+0000 7fe007fff640 1 -- 172.21.15.99:0/2461465397 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fe00819f780 con 0x7fe0081033e0 2024-03-19T01:40:39.870 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.867+0000 7fe007fff640 1 --2- 172.21.15.99:0/2461465397 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe0081033e0 0x7fe00819f110 secure :-1 s=READY pgs=318 cs=0 l=1 rev1=1 crypto rx=0x7fe00000efd0 tx=0x7fe00000c6a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:39.870 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.867+0000 7fe004ff9640 1 -- 172.21.15.99:0/2461465397 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe000003b70 con 0x7fe0081033e0 2024-03-19T01:40:39.870 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.867+0000 7fe004ff9640 1 -- 172.21.15.99:0/2461465397 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe000003d10 con 0x7fe0081033e0 2024-03-19T01:40:39.870 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.867+0000 7fe00dc80640 1 -- 172.21.15.99:0/2461465397 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fe00819fa70 con 0x7fe0081033e0 2024-03-19T01:40:39.870 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.867+0000 7fe004ff9640 1 -- 172.21.15.99:0/2461465397 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe0000109f0 con 0x7fe0081033e0 2024-03-19T01:40:39.870 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.867+0000 7fe00dc80640 1 -- 172.21.15.99:0/2461465397 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fe00819ff40 con 0x7fe0081033e0 2024-03-19T01:40:39.871 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.868+0000 7fe00dc80640 1 -- 172.21.15.99:0/2461465397 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fe008108c10 con 0x7fe0081033e0 2024-03-19T01:40:39.871 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.869+0000 7fe004ff9640 1 -- 172.21.15.99:0/2461465397 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 18) ==== 137290+0+0 (secure 0 0 0) 0x7fe000004020 con 0x7fe0081033e0 2024-03-19T01:40:39.871 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.870+0000 7fe004ff9640 1 --2- 172.21.15.99:0/2461465397 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fdff00a2800 0x7fdff00a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:39.871 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.870+0000 7fe0077fe640 1 --2- 172.21.15.99:0/2461465397 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fdff00a2800 0x7fdff00a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:39.871 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.870+0000 7fe004ff9640 1 -- 172.21.15.99:0/2461465397 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7fe0000ceea0 con 0x7fe0081033e0 2024-03-19T01:40:39.871 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.870+0000 7fe0077fe640 1 --2- 172.21.15.99:0/2461465397 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fdff00a2800 0x7fdff00a4cc0 secure :-1 s=READY pgs=232 cs=0 l=1 rev1=1 crypto rx=0x7fe008102880 tx=0x7fdff8005f90 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:39.874 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:39.874+0000 7fe004ff9640 1 -- 172.21.15.99:0/2461465397 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7fe00009bf00 con 0x7fe0081033e0 2024-03-19T01:40:40.052 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.052+0000 7fe00dc80640 1 -- 172.21.15.99:0/2461465397 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) -- 0x7fe008062420 con 0x7fe0081033e0 2024-03-19T01:40:40.052 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.052+0000 7fe004ff9640 1 -- 172.21.15.99:0/2461465397 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v91) ==== 104+0+11 (secure 0 0 0) 0x7fe00009b800 con 0x7fe0081033e0 2024-03-19T01:40:40.053 INFO:teuthology.orchestra.run.smithi099.stdout:pg_num: 32 2024-03-19T01:40:40.055 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.054+0000 7fe00dc80640 1 -- 172.21.15.99:0/2461465397 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fdff00a2800 msgr2=0x7fdff00a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:40.055 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.054+0000 7fe00dc80640 1 --2- 172.21.15.99:0/2461465397 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fdff00a2800 0x7fdff00a4cc0 secure :-1 s=READY pgs=232 cs=0 l=1 rev1=1 crypto rx=0x7fe008102880 tx=0x7fdff8005f90 comp rx=0 tx=0).stop 2024-03-19T01:40:40.055 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.054+0000 7fe00dc80640 1 -- 172.21.15.99:0/2461465397 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe0081033e0 msgr2=0x7fe00819f110 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:40.055 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.054+0000 7fe00dc80640 1 --2- 172.21.15.99:0/2461465397 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe0081033e0 0x7fe00819f110 secure :-1 s=READY pgs=318 cs=0 l=1 rev1=1 crypto rx=0x7fe00000efd0 tx=0x7fe00000c6a0 comp rx=0 tx=0).stop 2024-03-19T01:40:40.055 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.054+0000 7fe00dc80640 1 -- 172.21.15.99:0/2461465397 shutdown_connections 2024-03-19T01:40:40.055 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.054+0000 7fe00dc80640 1 --2- 172.21.15.99:0/2461465397 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe0081033e0 0x7fe00819f110 unknown :-1 s=CLOSED pgs=318 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:40.055 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.054+0000 7fe00dc80640 1 --2- 172.21.15.99:0/2461465397 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fe008102a20 0x7fe0081980a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:40.055 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.054+0000 7fe00dc80640 1 --2- 172.21.15.99:0/2461465397 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fdff00a2800 0x7fdff00a4cc0 unknown :-1 s=CLOSED pgs=232 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:40.055 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.054+0000 7fe00dc80640 1 --2- 172.21.15.99:0/2461465397 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fe008101830 0x7fe008197b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:40.055 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.054+0000 7fe00dc80640 1 -- 172.21.15.99:0/2461465397 >> 172.21.15.99:0/2461465397 conn(0x7fe0080fcdc0 msgr2=0x7fe008105de0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:40.055 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.054+0000 7fe00dc80640 1 -- 172.21.15.99:0/2461465397 shutdown_connections 2024-03-19T01:40:40.055 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.054+0000 7fe00dc80640 1 -- 172.21.15.99:0/2461465397 wait complete. 2024-03-19T01:40:40.065 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:39 smithi099 ceph-mon[62951]: pgmap v337: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 6 op/s 2024-03-19T01:40:40.065 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:39 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/656754060' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-19T01:40:40.065 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:39 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/3048073166' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-03-19T01:40:40.066 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-19T01:40:40.175 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.173+0000 7f2588652640 1 -- 172.21.15.99:0/3576862447 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f2580102320 msgr2=0x7f2580104710 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:40.175 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.173+0000 7f2588652640 1 --2- 172.21.15.99:0/3576862447 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f2580102320 0x7f2580104710 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7f257c009900 tx=0x7f257c02bc30 comp rx=0 tx=0).stop 2024-03-19T01:40:40.175 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.173+0000 7f2588652640 1 -- 172.21.15.99:0/3576862447 shutdown_connections 2024-03-19T01:40:40.176 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.173+0000 7f2588652640 1 --2- 172.21.15.99:0/3576862447 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f2580104c50 0x7f2580107040 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:40.176 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.173+0000 7f2588652640 1 --2- 172.21.15.99:0/3576862447 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2580107580 0x7f2580109970 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:40.176 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.173+0000 7f2588652640 1 --2- 172.21.15.99:0/3576862447 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f2580102320 0x7f2580104710 unknown :-1 s=CLOSED pgs=71 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:40.176 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.173+0000 7f2588652640 1 -- 172.21.15.99:0/3576862447 >> 172.21.15.99:0/3576862447 conn(0x7f25800fbcb0 msgr2=0x7f25800fe0d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:40.176 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.173+0000 7f2588652640 1 -- 172.21.15.99:0/3576862447 shutdown_connections 2024-03-19T01:40:40.176 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.173+0000 7f2588652640 1 -- 172.21.15.99:0/3576862447 wait complete. 2024-03-19T01:40:40.176 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.174+0000 7f2588652640 1 Processor -- start 2024-03-19T01:40:40.176 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.174+0000 7f2588652640 1 -- start start 2024-03-19T01:40:40.176 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.174+0000 7f2588652640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2580102320 0x7f2580197b90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:40.176 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.174+0000 7f2588652640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f2580104c50 0x7f25801980d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:40.176 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.174+0000 7f2588652640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f2580107580 0x7f258019f140 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:40.176 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.174+0000 7f2588652640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f258010d4c0 con 0x7f2580102320 2024-03-19T01:40:40.176 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.174+0000 7f2588652640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f258010d340 con 0x7f2580104c50 2024-03-19T01:40:40.177 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.174+0000 7f2588652640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f258010d640 con 0x7f2580107580 2024-03-19T01:40:40.177 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.174+0000 7f25863ee640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2580102320 0x7f2580197b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:40.177 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.174+0000 7f25863ee640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2580102320 0x7f2580197b90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:55258/0 (socket says 172.21.15.99:55258) 2024-03-19T01:40:40.177 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.174+0000 7f25863ee640 1 -- 172.21.15.99:0/3008820910 learned_addr learned my addr 172.21.15.99:0/3008820910 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:40.177 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.174+0000 7f2586bef640 1 --2- 172.21.15.99:0/3008820910 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f2580107580 0x7f258019f140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:40.177 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.174+0000 7f2585bed640 1 --2- 172.21.15.99:0/3008820910 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f2580104c50 0x7f25801980d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:40.177 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.174+0000 7f25863ee640 1 -- 172.21.15.99:0/3008820910 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f2580107580 msgr2=0x7f258019f140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:40.177 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.174+0000 7f25863ee640 1 --2- 172.21.15.99:0/3008820910 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f2580107580 0x7f258019f140 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:40.177 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.174+0000 7f25863ee640 1 -- 172.21.15.99:0/3008820910 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f2580104c50 msgr2=0x7f25801980d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:40.177 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.174+0000 7f25863ee640 1 --2- 172.21.15.99:0/3008820910 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f2580104c50 0x7f25801980d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:40.177 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.174+0000 7f25863ee640 1 -- 172.21.15.99:0/3008820910 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f258019f7b0 con 0x7f2580102320 2024-03-19T01:40:40.177 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.174+0000 7f25863ee640 1 --2- 172.21.15.99:0/3008820910 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2580102320 0x7f2580197b90 secure :-1 s=READY pgs=319 cs=0 l=1 rev1=1 crypto rx=0x7f257c009230 tx=0x7f257c02c650 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:40.177 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.175+0000 7f25777fe640 1 -- 172.21.15.99:0/3008820910 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f257c041070 con 0x7f2580102320 2024-03-19T01:40:40.177 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.175+0000 7f25777fe640 1 -- 172.21.15.99:0/3008820910 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f257c004450 con 0x7f2580102320 2024-03-19T01:40:40.177 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.175+0000 7f2588652640 1 -- 172.21.15.99:0/3008820910 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f258019fa40 con 0x7f2580102320 2024-03-19T01:40:40.178 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.175+0000 7f25777fe640 1 -- 172.21.15.99:0/3008820910 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f257c003e20 con 0x7f2580102320 2024-03-19T01:40:40.178 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.175+0000 7f2588652640 1 -- 172.21.15.99:0/3008820910 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f258019ff30 con 0x7f2580102320 2024-03-19T01:40:40.179 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.176+0000 7f2588652640 1 -- 172.21.15.99:0/3008820910 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f25800fd7a0 con 0x7f2580102320 2024-03-19T01:40:40.179 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.178+0000 7f25777fe640 1 -- 172.21.15.99:0/3008820910 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 18) ==== 137290+0+0 (secure 0 0 0) 0x7f257c044070 con 0x7f2580102320 2024-03-19T01:40:40.179 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.178+0000 7f25777fe640 1 --2- 172.21.15.99:0/3008820910 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f256c0a2800 0x7f256c0a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:40.179 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.178+0000 7f2585bed640 1 --2- 172.21.15.99:0/3008820910 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f256c0a2800 0x7f256c0a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:40.179 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.178+0000 7f25777fe640 1 -- 172.21.15.99:0/3008820910 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f257c0f09a0 con 0x7f2580102320 2024-03-19T01:40:40.179 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.179+0000 7f2585bed640 1 --2- 172.21.15.99:0/3008820910 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f256c0a2800 0x7f256c0a4cc0 secure :-1 s=READY pgs=233 cs=0 l=1 rev1=1 crypto rx=0x7f25801990a0 tx=0x7f2570005f90 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:40.181 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.182+0000 7f25777fe640 1 -- 172.21.15.99:0/3008820910 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f257c0bd3b0 con 0x7f2580102320 2024-03-19T01:40:40.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:39 smithi115 ceph-mon[67332]: pgmap v337: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 6 op/s 2024-03-19T01:40:40.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:39 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/656754060' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-19T01:40:40.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:39 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/3048073166' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-03-19T01:40:40.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:39 smithi112 ceph-mon[66819]: pgmap v337: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 6 op/s 2024-03-19T01:40:40.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:39 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/656754060' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-19T01:40:40.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:39 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/3048073166' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-03-19T01:40:40.362 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.362+0000 7f2588652640 1 -- 172.21.15.99:0/3008820910 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) -- 0x7f258002d380 con 0x7f2580102320 2024-03-19T01:40:40.362 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.362+0000 7f25777fe640 1 -- 172.21.15.99:0/3008820910 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v91) ==== 100+0+11 (secure 0 0 0) 0x7f257c0c1450 con 0x7f2580102320 2024-03-19T01:40:40.362 INFO:teuthology.orchestra.run.smithi099.stdout:pg_num: 64 2024-03-19T01:40:40.364 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.364+0000 7f2588652640 1 -- 172.21.15.99:0/3008820910 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f256c0a2800 msgr2=0x7f256c0a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:40.365 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.364+0000 7f2588652640 1 --2- 172.21.15.99:0/3008820910 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f256c0a2800 0x7f256c0a4cc0 secure :-1 s=READY pgs=233 cs=0 l=1 rev1=1 crypto rx=0x7f25801990a0 tx=0x7f2570005f90 comp rx=0 tx=0).stop 2024-03-19T01:40:40.365 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.364+0000 7f2588652640 1 -- 172.21.15.99:0/3008820910 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2580102320 msgr2=0x7f2580197b90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:40.365 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.364+0000 7f2588652640 1 --2- 172.21.15.99:0/3008820910 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2580102320 0x7f2580197b90 secure :-1 s=READY pgs=319 cs=0 l=1 rev1=1 crypto rx=0x7f257c009230 tx=0x7f257c02c650 comp rx=0 tx=0).stop 2024-03-19T01:40:40.365 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.364+0000 7f2588652640 1 -- 172.21.15.99:0/3008820910 shutdown_connections 2024-03-19T01:40:40.365 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.364+0000 7f2588652640 1 --2- 172.21.15.99:0/3008820910 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f256c0a2800 0x7f256c0a4cc0 unknown :-1 s=CLOSED pgs=233 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:40.365 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.364+0000 7f2588652640 1 --2- 172.21.15.99:0/3008820910 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f2580104c50 0x7f25801980d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:40.365 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.364+0000 7f2588652640 1 --2- 172.21.15.99:0/3008820910 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f2580107580 0x7f258019f140 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:40.365 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.364+0000 7f2588652640 1 --2- 172.21.15.99:0/3008820910 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2580102320 0x7f2580197b90 unknown :-1 s=CLOSED pgs=319 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:40.365 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.364+0000 7f2588652640 1 -- 172.21.15.99:0/3008820910 >> 172.21.15.99:0/3008820910 conn(0x7f25800fbcb0 msgr2=0x7f2580102d40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:40.365 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.364+0000 7f2588652640 1 -- 172.21.15.99:0/3008820910 shutdown_connections 2024-03-19T01:40:40.365 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.364+0000 7f2588652640 1 -- 172.21.15.99:0/3008820910 wait complete. 2024-03-19T01:40:40.375 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-19T01:40:40.481 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.481+0000 7f38f62a1640 1 -- 172.21.15.99:0/2269786222 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f38f01022c0 msgr2=0x7f38f01046b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:40.481 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.481+0000 7f38f62a1640 1 --2- 172.21.15.99:0/2269786222 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f38f01022c0 0x7f38f01046b0 secure :-1 s=READY pgs=320 cs=0 l=1 rev1=1 crypto rx=0x7f38e4009900 tx=0x7f38e402bc30 comp rx=0 tx=0).stop 2024-03-19T01:40:40.481 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.481+0000 7f38f62a1640 1 -- 172.21.15.99:0/2269786222 shutdown_connections 2024-03-19T01:40:40.481 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.481+0000 7f38f62a1640 1 --2- 172.21.15.99:0/2269786222 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f38f0104bf0 0x7f38f0106fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:40.481 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.481+0000 7f38f62a1640 1 --2- 172.21.15.99:0/2269786222 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f38f0107520 0x7f38f0109970 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:40.481 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.481+0000 7f38f62a1640 1 --2- 172.21.15.99:0/2269786222 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f38f01022c0 0x7f38f01046b0 unknown :-1 s=CLOSED pgs=320 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:40.482 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.481+0000 7f38f62a1640 1 -- 172.21.15.99:0/2269786222 >> 172.21.15.99:0/2269786222 conn(0x7f38f00fbcb0 msgr2=0x7f38f00fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:40.482 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.481+0000 7f38f62a1640 1 -- 172.21.15.99:0/2269786222 shutdown_connections 2024-03-19T01:40:40.482 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.481+0000 7f38f62a1640 1 -- 172.21.15.99:0/2269786222 wait complete. 2024-03-19T01:40:40.484 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.482+0000 7f38f62a1640 1 Processor -- start 2024-03-19T01:40:40.484 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.482+0000 7f38f62a1640 1 -- start start 2024-03-19T01:40:40.484 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.482+0000 7f38f62a1640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f38f01022c0 0x7f38f0197b60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:40.484 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.482+0000 7f38f62a1640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f38f0104bf0 0x7f38f01980a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:40.484 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.482+0000 7f38f62a1640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f38f0107520 0x7f38f019f110 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:40.484 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.482+0000 7f38f62a1640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f38f010d4c0 con 0x7f38f0107520 2024-03-19T01:40:40.485 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.482+0000 7f38f62a1640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f38f010d340 con 0x7f38f0104bf0 2024-03-19T01:40:40.485 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.482+0000 7f38f62a1640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f38f010d640 con 0x7f38f01022c0 2024-03-19T01:40:40.485 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.482+0000 7f38f483e640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f38f0107520 0x7f38f019f110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:40.485 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.482+0000 7f38f483e640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f38f0107520 0x7f38f019f110 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:55270/0 (socket says 172.21.15.99:55270) 2024-03-19T01:40:40.485 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.482+0000 7f38f483e640 1 -- 172.21.15.99:0/631031391 learned_addr learned my addr 172.21.15.99:0/631031391 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:40.485 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.482+0000 7f38ef7fe640 1 --2- 172.21.15.99:0/631031391 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f38f0104bf0 0x7f38f01980a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:40.485 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.482+0000 7f38effff640 1 --2- 172.21.15.99:0/631031391 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f38f01022c0 0x7f38f0197b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:40.485 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.482+0000 7f38f483e640 1 -- 172.21.15.99:0/631031391 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f38f01022c0 msgr2=0x7f38f0197b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:40.485 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.482+0000 7f38f483e640 1 --2- 172.21.15.99:0/631031391 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f38f01022c0 0x7f38f0197b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:40.485 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.482+0000 7f38f483e640 1 -- 172.21.15.99:0/631031391 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f38f0104bf0 msgr2=0x7f38f01980a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:40.485 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.482+0000 7f38f483e640 1 --2- 172.21.15.99:0/631031391 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f38f0104bf0 0x7f38f01980a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:40.485 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.482+0000 7f38f483e640 1 -- 172.21.15.99:0/631031391 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f38f019f780 con 0x7f38f0107520 2024-03-19T01:40:40.485 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.482+0000 7f38ef7fe640 1 --2- 172.21.15.99:0/631031391 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f38f0104bf0 0x7f38f01980a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:40:40.485 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.483+0000 7f38f483e640 1 --2- 172.21.15.99:0/631031391 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f38f0107520 0x7f38f019f110 secure :-1 s=READY pgs=321 cs=0 l=1 rev1=1 crypto rx=0x7f38e800eb10 tx=0x7f38e800c2d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:40.485 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.483+0000 7f38ed7fa640 1 -- 172.21.15.99:0/631031391 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f38e8004b20 con 0x7f38f0107520 2024-03-19T01:40:40.485 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.483+0000 7f38ed7fa640 1 -- 172.21.15.99:0/631031391 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f38e8004cc0 con 0x7f38f0107520 2024-03-19T01:40:40.486 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.483+0000 7f38f62a1640 1 -- 172.21.15.99:0/631031391 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f38f019fa70 con 0x7f38f0107520 2024-03-19T01:40:40.486 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.483+0000 7f38ed7fa640 1 -- 172.21.15.99:0/631031391 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f38e8010760 con 0x7f38f0107520 2024-03-19T01:40:40.486 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.483+0000 7f38f62a1640 1 -- 172.21.15.99:0/631031391 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f38f019ff70 con 0x7f38f0107520 2024-03-19T01:40:40.486 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.484+0000 7f38f62a1640 1 -- 172.21.15.99:0/631031391 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f38f00fd7a0 con 0x7f38f0107520 2024-03-19T01:40:40.486 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.485+0000 7f38ed7fa640 1 -- 172.21.15.99:0/631031391 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 18) ==== 137290+0+0 (secure 0 0 0) 0x7f38e8005020 con 0x7f38f0107520 2024-03-19T01:40:40.487 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.485+0000 7f38ed7fa640 1 --2- 172.21.15.99:0/631031391 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f38d80a2800 0x7f38d80a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:40.487 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.485+0000 7f38effff640 1 --2- 172.21.15.99:0/631031391 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f38d80a2800 0x7f38d80a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:40.487 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.486+0000 7f38ed7fa640 1 -- 172.21.15.99:0/631031391 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f38e80cf010 con 0x7f38f0107520 2024-03-19T01:40:40.487 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.486+0000 7f38effff640 1 --2- 172.21.15.99:0/631031391 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f38d80a2800 0x7f38d80a4cc0 secure :-1 s=READY pgs=234 cs=0 l=1 rev1=1 crypto rx=0x7f38e402c7b0 tx=0x7f38e4005fd0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:40.490 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.490+0000 7f38ed7fa640 1 -- 172.21.15.99:0/631031391 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f38e809c070 con 0x7f38f0107520 2024-03-19T01:40:40.710 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.709+0000 7f38f62a1640 1 -- 172.21.15.99:0/631031391 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f38f002d1f0 con 0x7f38f0107520 2024-03-19T01:40:40.711 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.710+0000 7f38ed7fa640 1 -- 172.21.15.99:0/631031391 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 20 v20) ==== 94+0+11878 (secure 0 0 0) 0x7f38e809b970 con 0x7f38f0107520 2024-03-19T01:40:40.711 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:40:40.711 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":20,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14562,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6835/1631815621","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6834","nonce":1631815621},{"type":"v1","addr":"172.21.15.99:6835","nonce":1631815621}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":4},{"gid":14646,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6839/2236145144","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6838","nonce":2236145144},{"type":"v1","addr":"172.21.15.99:6839","nonce":2236145144}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11},{"gid":14658,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6841/1660990159","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6840","nonce":1660990159},{"type":"v1","addr":"172.21.15.99:6841","nonce":1660990159}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":12},{"gid":24383,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.112:6833/2969810977","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6832","nonce":2969810977},{"type":"v1","addr":"172.21.15.112:6833","nonce":2969810977}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3},{"gid":24404,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6837/2038634879","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6836","nonce":2038634879},{"type":"v1","addr":"172.21.15.99:6837","nonce":2038634879}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":24415,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.115:6833/2219320481","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6832","nonce":2219320481},{"type":"v1","addr":"172.21.15.115:6833","nonce":2219320481}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":20,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-03-19T01:38:56.124529+0000","modified":"2024-03-19T01:39:00.323326+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24487,"mds_1":24479,"mds_2":24472},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24416":{"gid":24416,"name":"e","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.112:6835/3690587528","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6834","nonce":3690587528},{"type":"v1","addr":"172.21.15.112:6835","nonce":3690587528}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24440":{"gid":24440,"name":"h","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.112:6837/2339823159","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6836","nonce":2339823159},{"type":"v1","addr":"172.21.15.112:6837","nonce":2339823159}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24460":{"gid":24460,"name":"f","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.115:6835/591078413","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6834","nonce":591078413},{"type":"v1","addr":"172.21.15.115:6835","nonce":591078413}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24472":{"gid":24472,"name":"c","rank":2,"incarnation":19,"state":"up:active","state_seq":7,"addr":"172.21.15.115:6837/1368988191","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6836","nonce":1368988191},{"type":"v1","addr":"172.21.15.115:6837","nonce":1368988191}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24479":{"gid":24479,"name":"b","rank":1,"incarnation":17,"state":"up:active","state_seq":4,"addr":"172.21.15.112:6839/986249680","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6838","nonce":986249680},{"type":"v1","addr":"172.21.15.112:6839","nonce":986249680}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24487":{"gid":24487,"name":"i","rank":0,"incarnation":14,"state":"up:active","state_seq":5,"addr":"172.21.15.115:6839/2743761968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6838","nonce":2743761968},{"type":"v1","addr":"172.21.15.115:6839","nonce":2743761968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24487,"qdb_cluster":[24487,24479,24472]},"id":1}]} 2024-03-19T01:40:40.712 INFO:teuthology.orchestra.run.smithi099.stderr:dumped fsmap epoch 20 2024-03-19T01:40:40.713 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.712+0000 7f38f62a1640 1 -- 172.21.15.99:0/631031391 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f38d80a2800 msgr2=0x7f38d80a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:40.713 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.712+0000 7f38f62a1640 1 --2- 172.21.15.99:0/631031391 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f38d80a2800 0x7f38d80a4cc0 secure :-1 s=READY pgs=234 cs=0 l=1 rev1=1 crypto rx=0x7f38e402c7b0 tx=0x7f38e4005fd0 comp rx=0 tx=0).stop 2024-03-19T01:40:40.714 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.712+0000 7f38f62a1640 1 -- 172.21.15.99:0/631031391 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f38f0107520 msgr2=0x7f38f019f110 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:40.714 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.712+0000 7f38f62a1640 1 --2- 172.21.15.99:0/631031391 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f38f0107520 0x7f38f019f110 secure :-1 s=READY pgs=321 cs=0 l=1 rev1=1 crypto rx=0x7f38e800eb10 tx=0x7f38e800c2d0 comp rx=0 tx=0).stop 2024-03-19T01:40:40.714 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.712+0000 7f38f62a1640 1 -- 172.21.15.99:0/631031391 shutdown_connections 2024-03-19T01:40:40.714 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.712+0000 7f38f62a1640 1 --2- 172.21.15.99:0/631031391 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f38d80a2800 0x7f38d80a4cc0 unknown :-1 s=CLOSED pgs=234 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:40.714 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.712+0000 7f38f62a1640 1 --2- 172.21.15.99:0/631031391 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f38f0104bf0 0x7f38f01980a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:40.714 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.712+0000 7f38f62a1640 1 --2- 172.21.15.99:0/631031391 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f38f0107520 0x7f38f019f110 unknown :-1 s=CLOSED pgs=321 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:40.714 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.712+0000 7f38f62a1640 1 --2- 172.21.15.99:0/631031391 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f38f01022c0 0x7f38f0197b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:40.714 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.712+0000 7f38f62a1640 1 -- 172.21.15.99:0/631031391 >> 172.21.15.99:0/631031391 conn(0x7f38f00fbcb0 msgr2=0x7f38f0102d00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:40.714 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.712+0000 7f38f62a1640 1 -- 172.21.15.99:0/631031391 shutdown_connections 2024-03-19T01:40:40.714 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.712+0000 7f38f62a1640 1 -- 172.21.15.99:0/631031391 wait complete. 2024-03-19T01:40:40.725 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-19T01:40:40.833 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.830+0000 7f02dd987640 1 -- 172.21.15.99:0/3936582865 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f02d8104c10 msgr2=0x7f02d8107000 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:40.833 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.830+0000 7f02dd987640 1 --2- 172.21.15.99:0/3936582865 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f02d8104c10 0x7f02d8107000 secure :-1 s=READY pgs=322 cs=0 l=1 rev1=1 crypto rx=0x7f02c0009900 tx=0x7f02c002bc30 comp rx=0 tx=0).stop 2024-03-19T01:40:40.833 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.830+0000 7f02dd987640 1 -- 172.21.15.99:0/3936582865 shutdown_connections 2024-03-19T01:40:40.833 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.830+0000 7f02dd987640 1 --2- 172.21.15.99:0/3936582865 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f02d8107540 0x7f02d8109970 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:40.833 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.830+0000 7f02dd987640 1 --2- 172.21.15.99:0/3936582865 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f02d8104c10 0x7f02d8107000 unknown :-1 s=CLOSED pgs=322 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:40.833 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.830+0000 7f02dd987640 1 --2- 172.21.15.99:0/3936582865 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f02d81022e0 0x7f02d81046d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:40.834 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.830+0000 7f02dd987640 1 -- 172.21.15.99:0/3936582865 >> 172.21.15.99:0/3936582865 conn(0x7f02d80fbcb0 msgr2=0x7f02d80fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:40.834 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.830+0000 7f02dd987640 1 -- 172.21.15.99:0/3936582865 shutdown_connections 2024-03-19T01:40:40.834 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.830+0000 7f02dd987640 1 -- 172.21.15.99:0/3936582865 wait complete. 2024-03-19T01:40:40.834 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.831+0000 7f02dd987640 1 Processor -- start 2024-03-19T01:40:40.834 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.831+0000 7f02dd987640 1 -- start start 2024-03-19T01:40:40.834 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.831+0000 7f02dd987640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f02d81022e0 0x7f02d8197b40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:40.834 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.831+0000 7f02dd987640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f02d8104c10 0x7f02d8198080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:40.834 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.831+0000 7f02dd987640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f02d8107540 0x7f02d819f0f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:40.834 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.831+0000 7f02dd987640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f02d810d4d0 con 0x7f02d8104c10 2024-03-19T01:40:40.834 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.831+0000 7f02dd987640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f02d810d350 con 0x7f02d81022e0 2024-03-19T01:40:40.834 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.831+0000 7f02dd987640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f02d810d650 con 0x7f02d8107540 2024-03-19T01:40:40.834 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.831+0000 7f02cffff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f02d8104c10 0x7f02d8198080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:40.834 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.831+0000 7f02cffff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f02d8104c10 0x7f02d8198080 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:55294/0 (socket says 172.21.15.99:55294) 2024-03-19T01:40:40.834 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.831+0000 7f02cffff640 1 -- 172.21.15.99:0/3697305763 learned_addr learned my addr 172.21.15.99:0/3697305763 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:40.834 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.831+0000 7f02d77fe640 1 --2- 172.21.15.99:0/3697305763 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f02d8107540 0x7f02d819f0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:40.835 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.831+0000 7f02d6ffd640 1 --2- 172.21.15.99:0/3697305763 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f02d81022e0 0x7f02d8197b40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:40.835 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.831+0000 7f02cffff640 1 -- 172.21.15.99:0/3697305763 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f02d8107540 msgr2=0x7f02d819f0f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:40.835 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.831+0000 7f02cffff640 1 --2- 172.21.15.99:0/3697305763 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f02d8107540 0x7f02d819f0f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:40.835 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.831+0000 7f02cffff640 1 -- 172.21.15.99:0/3697305763 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f02d81022e0 msgr2=0x7f02d8197b40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:40.835 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.831+0000 7f02cffff640 1 --2- 172.21.15.99:0/3697305763 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f02d81022e0 0x7f02d8197b40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:40.835 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.831+0000 7f02cffff640 1 -- 172.21.15.99:0/3697305763 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f02d819f760 con 0x7f02d8104c10 2024-03-19T01:40:40.835 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.832+0000 7f02cffff640 1 --2- 172.21.15.99:0/3697305763 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f02d8104c10 0x7f02d8198080 secure :-1 s=READY pgs=323 cs=0 l=1 rev1=1 crypto rx=0x7f02c0000c00 tx=0x7f02c002c640 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:40.835 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.832+0000 7f02dc985640 1 -- 172.21.15.99:0/3697305763 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f02c0041070 con 0x7f02d8104c10 2024-03-19T01:40:40.835 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.832+0000 7f02dc985640 1 -- 172.21.15.99:0/3697305763 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f02c0004450 con 0x7f02d8104c10 2024-03-19T01:40:40.835 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.832+0000 7f02dc985640 1 -- 172.21.15.99:0/3697305763 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f02c0034440 con 0x7f02d8104c10 2024-03-19T01:40:40.835 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.832+0000 7f02dd987640 1 -- 172.21.15.99:0/3697305763 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f02d819f9f0 con 0x7f02d8104c10 2024-03-19T01:40:40.835 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.832+0000 7f02dd987640 1 -- 172.21.15.99:0/3697305763 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f02d819fee0 con 0x7f02d8104c10 2024-03-19T01:40:40.836 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.833+0000 7f02dd987640 1 -- 172.21.15.99:0/3697305763 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f02d80fd7a0 con 0x7f02d8104c10 2024-03-19T01:40:40.836 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.834+0000 7f02dc985640 1 -- 172.21.15.99:0/3697305763 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 18) ==== 137290+0+0 (secure 0 0 0) 0x7f02c0044070 con 0x7f02d8104c10 2024-03-19T01:40:40.836 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.834+0000 7f02dc985640 1 --2- 172.21.15.99:0/3697305763 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f02c40a2800 0x7f02c40a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:40.836 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.835+0000 7f02d6ffd640 1 --2- 172.21.15.99:0/3697305763 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f02c40a2800 0x7f02c40a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:40.836 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.835+0000 7f02dc985640 1 -- 172.21.15.99:0/3697305763 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f02c00f0560 con 0x7f02d8104c10 2024-03-19T01:40:40.836 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.835+0000 7f02d6ffd640 1 --2- 172.21.15.99:0/3697305763 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f02c40a2800 0x7f02c40a4cc0 secure :-1 s=READY pgs=235 cs=0 l=1 rev1=1 crypto rx=0x7f02c80095f0 tx=0x7f02c8005f90 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:40.839 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:40.839+0000 7f02dc985640 1 -- 172.21.15.99:0/3697305763 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f02c00bd000 con 0x7f02d8104c10 2024-03-19T01:40:41.036 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.035+0000 7f02dd987640 1 -- 172.21.15.99:0/3697305763 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f02d8062420 con 0x7f02d8104c10 2024-03-19T01:40:41.038 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.037+0000 7f02dc985640 1 -- 172.21.15.99:0/3697305763 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v91) ==== 74+0+22804 (secure 0 0 0) 0x7f02c00c10a0 con 0x7f02d8104c10 2024-03-19T01:40:41.038 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:40:41.038 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":91,"fsid":"438fb69c-e590-11ee-95c9-87774f69a715","created":"2024-03-19T01:31:10.165073+0000","modified":"2024-03-19T01:40:08.821469+0000","last_up_change":"2024-03-19T01:38:09.082688+0000","last_in_change":"2024-03-19T01:37:49.115793+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-19T01:34:31.001309+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-19T01:38:54.018623+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"86","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-19T01:38:55.319036+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"87","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"0c407a0e-a40e-43d7-ad9a-bb8f3cd4a9ba","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6803","nonce":2935260112}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6805","nonce":2935260112}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6808","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6809","nonce":2935260112}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6807","nonce":2935260112}]},"public_addr":"172.21.15.99:6803/2935260112","cluster_addr":"172.21.15.99:6805/2935260112","heartbeat_back_addr":"172.21.15.99:6809/2935260112","heartbeat_front_addr":"172.21.15.99:6807/2935260112","state":["exists","up"]},{"osd":1,"uuid":"a48f9173-4a06-49f5-bca2-2c9047b8f666","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6800","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6801","nonce":1468854289}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6802","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6803","nonce":1468854289}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6806","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6807","nonce":1468854289}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6804","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6805","nonce":1468854289}]},"public_addr":"172.21.15.112:6801/1468854289","cluster_addr":"172.21.15.112:6803/1468854289","heartbeat_back_addr":"172.21.15.112:6807/1468854289","heartbeat_front_addr":"172.21.15.112:6805/1468854289","state":["exists","up"]},{"osd":2,"uuid":"b7715fdd-b6c9-4348-bd16-5eb61b427eef","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6800","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6801","nonce":3766690472}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6802","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6803","nonce":3766690472}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6806","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6807","nonce":3766690472}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6804","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6805","nonce":3766690472}]},"public_addr":"172.21.15.115:6801/3766690472","cluster_addr":"172.21.15.115:6803/3766690472","heartbeat_back_addr":"172.21.15.115:6807/3766690472","heartbeat_front_addr":"172.21.15.115:6805/3766690472","state":["exists","up"]},{"osd":3,"uuid":"6697ed1b-96bf-42de-8440-d17e9785b359","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6810","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6811","nonce":3756732895}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6812","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6813","nonce":3756732895}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6816","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6817","nonce":3756732895}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6814","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6815","nonce":3756732895}]},"public_addr":"172.21.15.99:6811/3756732895","cluster_addr":"172.21.15.99:6813/3756732895","heartbeat_back_addr":"172.21.15.99:6817/3756732895","heartbeat_front_addr":"172.21.15.99:6815/3756732895","state":["exists","up"]},{"osd":4,"uuid":"d958d6b9-67f6-43d0-b704-aa55dc9df83a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":34,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6808","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6809","nonce":3717986391}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6810","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6811","nonce":3717986391}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6814","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6815","nonce":3717986391}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6812","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6813","nonce":3717986391}]},"public_addr":"172.21.15.112:6809/3717986391","cluster_addr":"172.21.15.112:6811/3717986391","heartbeat_back_addr":"172.21.15.112:6815/3717986391","heartbeat_front_addr":"172.21.15.112:6813/3717986391","state":["exists","up"]},{"osd":5,"uuid":"a73eb49a-7925-49ba-bc4f-515e17f8b8ae","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6808","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6809","nonce":1971806408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6810","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6811","nonce":1971806408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6814","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6815","nonce":1971806408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6812","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6813","nonce":1971806408}]},"public_addr":"172.21.15.115:6809/1971806408","cluster_addr":"172.21.15.115:6811/1971806408","heartbeat_back_addr":"172.21.15.115:6815/1971806408","heartbeat_front_addr":"172.21.15.115:6813/1971806408","state":["exists","up"]},{"osd":6,"uuid":"caa718cd-e08b-48de-832a-29546d71e508","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6818","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6819","nonce":1130832772}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6820","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6821","nonce":1130832772}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6824","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6825","nonce":1130832772}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6822","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6823","nonce":1130832772}]},"public_addr":"172.21.15.99:6819/1130832772","cluster_addr":"172.21.15.99:6821/1130832772","heartbeat_back_addr":"172.21.15.99:6825/1130832772","heartbeat_front_addr":"172.21.15.99:6823/1130832772","state":["exists","up"]},{"osd":7,"uuid":"dbee19a1-2315-40d3-88e7-876fcde28903","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":53,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6816","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6817","nonce":3610356576}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6818","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6819","nonce":3610356576}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6822","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6823","nonce":3610356576}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6820","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6821","nonce":3610356576}]},"public_addr":"172.21.15.112:6817/3610356576","cluster_addr":"172.21.15.112:6819/3610356576","heartbeat_back_addr":"172.21.15.112:6823/3610356576","heartbeat_front_addr":"172.21.15.112:6821/3610356576","state":["exists","up"]},{"osd":8,"uuid":"14a866dc-5ce3-42fc-8bcf-9088b68726cb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":60,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6816","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6817","nonce":4255803111}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6818","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6819","nonce":4255803111}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6822","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6823","nonce":4255803111}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6820","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6821","nonce":4255803111}]},"public_addr":"172.21.15.115:6817/4255803111","cluster_addr":"172.21.15.115:6819/4255803111","heartbeat_back_addr":"172.21.15.115:6823/4255803111","heartbeat_front_addr":"172.21.15.115:6821/4255803111","state":["exists","up"]},{"osd":9,"uuid":"1db93652-6035-4906-91ae-d64d5a909502","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":68,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6826","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6827","nonce":1410921523}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6828","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6829","nonce":1410921523}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6832","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6833","nonce":1410921523}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6830","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6831","nonce":1410921523}]},"public_addr":"172.21.15.99:6827/1410921523","cluster_addr":"172.21.15.99:6829/1410921523","heartbeat_back_addr":"172.21.15.99:6833/1410921523","heartbeat_front_addr":"172.21.15.99:6831/1410921523","state":["exists","up"]},{"osd":10,"uuid":"6c50a8a7-3345-4f4a-af0e-5606c7aca453","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":74,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6824","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6825","nonce":891805922}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6826","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6827","nonce":891805922}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6830","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6831","nonce":891805922}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6828","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6829","nonce":891805922}]},"public_addr":"172.21.15.112:6825/891805922","cluster_addr":"172.21.15.112:6827/891805922","heartbeat_back_addr":"172.21.15.112:6831/891805922","heartbeat_front_addr":"172.21.15.112:6829/891805922","state":["exists","up"]},{"osd":11,"uuid":"fd63b7fd-0db5-46d5-82fa-9707bbd0b495","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":82,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6824","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6825","nonce":938436244}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6826","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6827","nonce":938436244}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6830","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6831","nonce":938436244}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6828","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6829","nonce":938436244}]},"public_addr":"172.21.15.115:6825/938436244","cluster_addr":"172.21.15.115:6827/938436244","heartbeat_back_addr":"172.21.15.115:6831/938436244","heartbeat_front_addr":"172.21.15.115:6829/938436244","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:33:39.036822+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:34:03.339566+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:34:27.149273+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:34:51.315561+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:35:15.396031+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:35:38.957168+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:36:03.213914+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:36:27.763699+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:36:51.688670+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:37:17.280165+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:37:42.554633+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:38:07.438039+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.0","mappings":[{"from":8,"to":1}]},{"pgid":"2.8","mappings":[{"from":2,"to":11}]},{"pgid":"2.b","mappings":[{"from":8,"to":11}]},{"pgid":"2.e","mappings":[{"from":2,"to":4}]},{"pgid":"3.0","mappings":[{"from":5,"to":9}]},{"pgid":"3.2","mappings":[{"from":3,"to":9}]},{"pgid":"3.14","mappings":[{"from":1,"to":9}]},{"pgid":"3.17","mappings":[{"from":4,"to":8}]},{"pgid":"3.31","mappings":[{"from":4,"to":8}]},{"pgid":"3.37","mappings":[{"from":5,"to":7}]},{"pgid":"3.3a","mappings":[{"from":4,"to":6}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.99:6801/1328245934":"2024-03-20T01:32:06.661002+0000","172.21.15.99:0/1618965837":"2024-03-20T01:32:06.661002+0000","172.21.15.99:0/2491429851":"2024-03-20T01:32:06.661002+0000","172.21.15.99:6800/1328245934":"2024-03-20T01:32:06.661002+0000","172.21.15.99:0/2112231766":"2024-03-20T01:31:40.161751+0000","172.21.15.99:0/3341836545":"2024-03-20T01:31:40.161751+0000","172.21.15.99:6800/4223048293":"2024-03-20T01:31:40.161751+0000","172.21.15.99:6801/4223048293":"2024-03-20T01:31:40.161751+0000","172.21.15.99:0/2377621863":"2024-03-20T01:31:40.161751+0000","172.21.15.99:0/1904400935":"2024-03-20T01:32:06.661002+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-19T01:40:41.040 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.039+0000 7f02dd987640 1 -- 172.21.15.99:0/3697305763 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f02c40a2800 msgr2=0x7f02c40a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:41.040 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.039+0000 7f02dd987640 1 --2- 172.21.15.99:0/3697305763 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f02c40a2800 0x7f02c40a4cc0 secure :-1 s=READY pgs=235 cs=0 l=1 rev1=1 crypto rx=0x7f02c80095f0 tx=0x7f02c8005f90 comp rx=0 tx=0).stop 2024-03-19T01:40:41.040 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.039+0000 7f02dd987640 1 -- 172.21.15.99:0/3697305763 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f02d8104c10 msgr2=0x7f02d8198080 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:41.040 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.039+0000 7f02dd987640 1 --2- 172.21.15.99:0/3697305763 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f02d8104c10 0x7f02d8198080 secure :-1 s=READY pgs=323 cs=0 l=1 rev1=1 crypto rx=0x7f02c0000c00 tx=0x7f02c002c640 comp rx=0 tx=0).stop 2024-03-19T01:40:41.040 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.039+0000 7f02dd987640 1 -- 172.21.15.99:0/3697305763 shutdown_connections 2024-03-19T01:40:41.040 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.039+0000 7f02dd987640 1 --2- 172.21.15.99:0/3697305763 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f02d8107540 0x7f02d819f0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:41.041 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.039+0000 7f02dd987640 1 --2- 172.21.15.99:0/3697305763 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f02d8104c10 0x7f02d8198080 unknown :-1 s=CLOSED pgs=323 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:41.041 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.039+0000 7f02dd987640 1 --2- 172.21.15.99:0/3697305763 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f02c40a2800 0x7f02c40a4cc0 unknown :-1 s=CLOSED pgs=235 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:41.041 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.039+0000 7f02dd987640 1 --2- 172.21.15.99:0/3697305763 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f02d81022e0 0x7f02d8197b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:41.041 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.039+0000 7f02dd987640 1 -- 172.21.15.99:0/3697305763 >> 172.21.15.99:0/3697305763 conn(0x7f02d80fbcb0 msgr2=0x7f02d8102ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:41.041 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.039+0000 7f02dd987640 1 -- 172.21.15.99:0/3697305763 shutdown_connections 2024-03-19T01:40:41.041 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.039+0000 7f02dd987640 1 -- 172.21.15.99:0/3697305763 wait complete. 2024-03-19T01:40:41.050 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:40 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/2461465397' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-03-19T01:40:41.050 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:40 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/3008820910' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-03-19T01:40:41.050 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:40 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/631031391' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:40:41.052 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-19T01:34:31.001309+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 12, 'score_stable': 12, 'optimal_score': 0.25, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-19T01:38:54.018623+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '86', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.8799999952316284, 'score_stable': 1.8799999952316284, 'optimal_score': 1, 'raw_score_acting': 1.8799999952316284, 'raw_score_stable': 1.8799999952316284, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-03-19T01:38:55.319036+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '87', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-19T01:40:41.052 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-19T01:40:41.160 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.157+0000 7f987c759640 1 -- 172.21.15.99:0/3146872337 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9874101850 msgr2=0x7f9874101c70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:41.160 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.157+0000 7f987c759640 1 --2- 172.21.15.99:0/3146872337 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9874101850 0x7f9874101c70 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7f9864009900 tx=0x7f986402bc30 comp rx=0 tx=0).stop 2024-03-19T01:40:41.160 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.157+0000 7f987c759640 1 -- 172.21.15.99:0/3146872337 shutdown_connections 2024-03-19T01:40:41.161 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.157+0000 7f987c759640 1 --2- 172.21.15.99:0/3146872337 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9874102a40 0x7f9874102ec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:41.161 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.157+0000 7f987c759640 1 --2- 172.21.15.99:0/3146872337 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9874103400 0x7f9874109c90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:41.161 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.157+0000 7f987c759640 1 --2- 172.21.15.99:0/3146872337 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9874101850 0x7f9874101c70 unknown :-1 s=CLOSED pgs=72 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:41.161 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.157+0000 7f987c759640 1 -- 172.21.15.99:0/3146872337 >> 172.21.15.99:0/3146872337 conn(0x7f98740fcdc0 msgr2=0x7f98740ff200 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:41.161 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.157+0000 7f987c759640 1 -- 172.21.15.99:0/3146872337 shutdown_connections 2024-03-19T01:40:41.161 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.157+0000 7f987c759640 1 -- 172.21.15.99:0/3146872337 wait complete. 2024-03-19T01:40:41.161 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.158+0000 7f987c759640 1 Processor -- start 2024-03-19T01:40:41.161 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.158+0000 7f987c759640 1 -- start start 2024-03-19T01:40:41.161 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.158+0000 7f987c759640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9874101850 0x7f9874197b80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:41.161 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.158+0000 7f987c759640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9874102a40 0x7f98741980c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:41.161 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.158+0000 7f987c759640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9874103400 0x7f987419f130 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:41.161 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.158+0000 7f987c759640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f987410d4a0 con 0x7f9874103400 2024-03-19T01:40:41.161 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.158+0000 7f987c759640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f987410d320 con 0x7f9874102a40 2024-03-19T01:40:41.161 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.158+0000 7f987c759640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f987410d620 con 0x7f9874101850 2024-03-19T01:40:41.162 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.158+0000 7f987acf6640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9874103400 0x7f987419f130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:41.162 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.158+0000 7f987a4f5640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9874101850 0x7f9874197b80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:41.162 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.158+0000 7f9879cf4640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9874102a40 0x7f98741980c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:41.162 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.158+0000 7f987acf6640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9874103400 0x7f987419f130 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:55310/0 (socket says 172.21.15.99:55310) 2024-03-19T01:40:41.162 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.158+0000 7f987acf6640 1 -- 172.21.15.99:0/1656659839 learned_addr learned my addr 172.21.15.99:0/1656659839 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:41.162 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.158+0000 7f9879cf4640 1 -- 172.21.15.99:0/1656659839 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9874101850 msgr2=0x7f9874197b80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:41.162 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.158+0000 7f9879cf4640 1 --2- 172.21.15.99:0/1656659839 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9874101850 0x7f9874197b80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:41.162 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.158+0000 7f9879cf4640 1 -- 172.21.15.99:0/1656659839 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9874103400 msgr2=0x7f987419f130 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:41.162 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.158+0000 7f9879cf4640 1 --2- 172.21.15.99:0/1656659839 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9874103400 0x7f987419f130 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:41.162 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.158+0000 7f9879cf4640 1 -- 172.21.15.99:0/1656659839 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f987419f7a0 con 0x7f9874102a40 2024-03-19T01:40:41.162 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.159+0000 7f9879cf4640 1 --2- 172.21.15.99:0/1656659839 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9874102a40 0x7f98741980c0 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7f987000e9f0 tx=0x7f987000eeb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:41.162 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.159+0000 7f986b7fe640 1 -- 172.21.15.99:0/1656659839 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f987000ccc0 con 0x7f9874102a40 2024-03-19T01:40:41.162 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.159+0000 7f987c759640 1 -- 172.21.15.99:0/1656659839 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f987419fa90 con 0x7f9874102a40 2024-03-19T01:40:41.163 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.159+0000 7f986b7fe640 1 -- 172.21.15.99:0/1656659839 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f9870010910 con 0x7f9874102a40 2024-03-19T01:40:41.163 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.159+0000 7f986b7fe640 1 -- 172.21.15.99:0/1656659839 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9870018960 con 0x7f9874102a40 2024-03-19T01:40:41.163 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.159+0000 7f987c759640 1 -- 172.21.15.99:0/1656659839 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f987419ffe0 con 0x7f9874102a40 2024-03-19T01:40:41.164 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.160+0000 7f987c759640 1 -- 172.21.15.99:0/1656659839 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f9874108bf0 con 0x7f9874102a40 2024-03-19T01:40:41.164 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.161+0000 7f986b7fe640 1 -- 172.21.15.99:0/1656659839 <== mon.2 v2:172.21.15.112:3300/0 4 ==== mgrmap(e 18) ==== 137290+0+0 (secure 0 0 0) 0x7f9870010ac0 con 0x7f9874102a40 2024-03-19T01:40:41.164 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.162+0000 7f986b7fe640 1 --2- 172.21.15.99:0/1656659839 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f98600a2800 0x7f98600a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:41.164 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.162+0000 7f987a4f5640 1 --2- 172.21.15.99:0/1656659839 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f98600a2800 0x7f98600a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:41.164 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.162+0000 7f986b7fe640 1 -- 172.21.15.99:0/1656659839 <== mon.2 v2:172.21.15.112:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f98700cee50 con 0x7f9874102a40 2024-03-19T01:40:41.164 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.162+0000 7f987a4f5640 1 --2- 172.21.15.99:0/1656659839 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f98600a2800 0x7f98600a4cc0 secure :-1 s=READY pgs=236 cs=0 l=1 rev1=1 crypto rx=0x7f9864000c00 tx=0x7f9864005fd0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:41.167 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.167+0000 7f986b7fe640 1 -- 172.21.15.99:0/1656659839 <== mon.2 v2:172.21.15.112:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f987009b860 con 0x7f9874102a40 2024-03-19T01:40:41.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:40 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/2461465397' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-03-19T01:40:41.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:40 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/3008820910' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-03-19T01:40:41.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:40 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/631031391' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:40:41.350 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.349+0000 7f987c759640 1 -- 172.21.15.99:0/1656659839 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) -- 0x7f9874062420 con 0x7f9874102a40 2024-03-19T01:40:41.351 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.349+0000 7f986b7fe640 1 -- 172.21.15.99:0/1656659839 <== mon.2 v2:172.21.15.112:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v91) ==== 93+0+10 (secure 0 0 0) 0x7f987009f900 con 0x7f9874102a40 2024-03-19T01:40:41.351 INFO:teuthology.orchestra.run.smithi099.stdout:pg_num: 1 2024-03-19T01:40:41.353 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.351+0000 7f987c759640 1 -- 172.21.15.99:0/1656659839 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f98600a2800 msgr2=0x7f98600a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:41.353 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.351+0000 7f987c759640 1 --2- 172.21.15.99:0/1656659839 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f98600a2800 0x7f98600a4cc0 secure :-1 s=READY pgs=236 cs=0 l=1 rev1=1 crypto rx=0x7f9864000c00 tx=0x7f9864005fd0 comp rx=0 tx=0).stop 2024-03-19T01:40:41.353 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.351+0000 7f987c759640 1 -- 172.21.15.99:0/1656659839 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9874102a40 msgr2=0x7f98741980c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:41.353 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.351+0000 7f987c759640 1 --2- 172.21.15.99:0/1656659839 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9874102a40 0x7f98741980c0 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7f987000e9f0 tx=0x7f987000eeb0 comp rx=0 tx=0).stop 2024-03-19T01:40:41.353 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.352+0000 7f987c759640 1 -- 172.21.15.99:0/1656659839 shutdown_connections 2024-03-19T01:40:41.353 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.352+0000 7f987c759640 1 --2- 172.21.15.99:0/1656659839 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f98600a2800 0x7f98600a4cc0 unknown :-1 s=CLOSED pgs=236 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:41.353 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.352+0000 7f987c759640 1 --2- 172.21.15.99:0/1656659839 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9874102a40 0x7f98741980c0 unknown :-1 s=CLOSED pgs=79 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:41.353 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.352+0000 7f987c759640 1 --2- 172.21.15.99:0/1656659839 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9874103400 0x7f987419f130 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:41.353 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.352+0000 7f987c759640 1 --2- 172.21.15.99:0/1656659839 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9874101850 0x7f9874197b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:41.354 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.352+0000 7f987c759640 1 -- 172.21.15.99:0/1656659839 >> 172.21.15.99:0/1656659839 conn(0x7f98740fcdc0 msgr2=0x7f9874105dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:41.354 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.352+0000 7f987c759640 1 -- 172.21.15.99:0/1656659839 shutdown_connections 2024-03-19T01:40:41.354 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.352+0000 7f987c759640 1 -- 172.21.15.99:0/1656659839 wait complete. 2024-03-19T01:40:41.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:40 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/2461465397' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-03-19T01:40:41.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:40 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/3008820910' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-03-19T01:40:41.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:40 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/631031391' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:40:41.367 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-19T01:40:41.472 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.470+0000 7f83f57e6640 1 -- 172.21.15.99:0/1358136803 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f83f0101850 msgr2=0x7f83f0101c70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:41.472 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.470+0000 7f83f57e6640 1 --2- 172.21.15.99:0/1358136803 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f83f0101850 0x7f83f0101c70 secure :-1 s=READY pgs=324 cs=0 l=1 rev1=1 crypto rx=0x7f83e0009900 tx=0x7f83e002bc30 comp rx=0 tx=0).stop 2024-03-19T01:40:41.472 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.470+0000 7f83f57e6640 1 -- 172.21.15.99:0/1358136803 shutdown_connections 2024-03-19T01:40:41.472 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.470+0000 7f83f57e6640 1 --2- 172.21.15.99:0/1358136803 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f83f0103400 0x7f83f0109c90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:41.472 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.470+0000 7f83f57e6640 1 --2- 172.21.15.99:0/1358136803 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f83f0102a40 0x7f83f0102ec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:41.472 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.470+0000 7f83f57e6640 1 --2- 172.21.15.99:0/1358136803 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f83f0101850 0x7f83f0101c70 unknown :-1 s=CLOSED pgs=324 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:41.472 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.470+0000 7f83f57e6640 1 -- 172.21.15.99:0/1358136803 >> 172.21.15.99:0/1358136803 conn(0x7f83f00fcdc0 msgr2=0x7f83f00ff200 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:41.472 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.470+0000 7f83f57e6640 1 -- 172.21.15.99:0/1358136803 shutdown_connections 2024-03-19T01:40:41.472 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.470+0000 7f83f57e6640 1 -- 172.21.15.99:0/1358136803 wait complete. 2024-03-19T01:40:41.472 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.471+0000 7f83f57e6640 1 Processor -- start 2024-03-19T01:40:41.475 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.471+0000 7f83f57e6640 1 -- start start 2024-03-19T01:40:41.475 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.471+0000 7f83f57e6640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f83f0101850 0x7f83f0197ba0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:41.475 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.471+0000 7f83f57e6640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f83f0102a40 0x7f83f01980e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:41.475 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.471+0000 7f83f57e6640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f83f0103400 0x7f83f019f150 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:41.475 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.471+0000 7f83f57e6640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f83f010d4c0 con 0x7f83f0102a40 2024-03-19T01:40:41.475 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.471+0000 7f83f57e6640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f83f010d340 con 0x7f83f0101850 2024-03-19T01:40:41.475 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.471+0000 7f83f57e6640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f83f010d640 con 0x7f83f0103400 2024-03-19T01:40:41.475 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.471+0000 7f83ee7fc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f83f0102a40 0x7f83f01980e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:41.475 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.471+0000 7f83ee7fc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f83f0102a40 0x7f83f01980e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:55328/0 (socket says 172.21.15.99:55328) 2024-03-19T01:40:41.475 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.471+0000 7f83ee7fc640 1 -- 172.21.15.99:0/3946475281 learned_addr learned my addr 172.21.15.99:0/3946475281 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:41.475 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.472+0000 7f83eeffd640 1 --2- 172.21.15.99:0/3946475281 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f83f0101850 0x7f83f0197ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:41.475 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.472+0000 7f83ef7fe640 1 --2- 172.21.15.99:0/3946475281 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f83f0103400 0x7f83f019f150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:41.476 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.472+0000 7f83ee7fc640 1 -- 172.21.15.99:0/3946475281 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f83f0103400 msgr2=0x7f83f019f150 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:41.476 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.472+0000 7f83ee7fc640 1 --2- 172.21.15.99:0/3946475281 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f83f0103400 0x7f83f019f150 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:41.476 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.472+0000 7f83ee7fc640 1 -- 172.21.15.99:0/3946475281 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f83f0101850 msgr2=0x7f83f0197ba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:41.476 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.472+0000 7f83ee7fc640 1 --2- 172.21.15.99:0/3946475281 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f83f0101850 0x7f83f0197ba0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:41.476 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.472+0000 7f83ee7fc640 1 -- 172.21.15.99:0/3946475281 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f83f019f7c0 con 0x7f83f0102a40 2024-03-19T01:40:41.476 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.472+0000 7f83ee7fc640 1 --2- 172.21.15.99:0/3946475281 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f83f0102a40 0x7f83f01980e0 secure :-1 s=READY pgs=325 cs=0 l=1 rev1=1 crypto rx=0x7f83e400e9f0 tx=0x7f83e400edf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:41.476 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.472+0000 7f83d7fff640 1 -- 172.21.15.99:0/3946475281 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f83e400cd80 con 0x7f83f0102a40 2024-03-19T01:40:41.476 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.472+0000 7f83d7fff640 1 -- 172.21.15.99:0/3946475281 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f83e4005500 con 0x7f83f0102a40 2024-03-19T01:40:41.476 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.472+0000 7f83f57e6640 1 -- 172.21.15.99:0/3946475281 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f83f019fab0 con 0x7f83f0102a40 2024-03-19T01:40:41.476 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.472+0000 7f83d7fff640 1 -- 172.21.15.99:0/3946475281 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f83e4010690 con 0x7f83f0102a40 2024-03-19T01:40:41.476 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.472+0000 7f83f57e6640 1 -- 172.21.15.99:0/3946475281 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f83f019ffb0 con 0x7f83f0102a40 2024-03-19T01:40:41.476 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.473+0000 7f83f57e6640 1 -- 172.21.15.99:0/3946475281 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f83f0108bf0 con 0x7f83f0102a40 2024-03-19T01:40:41.476 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.474+0000 7f83d7fff640 1 -- 172.21.15.99:0/3946475281 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 18) ==== 137290+0+0 (secure 0 0 0) 0x7f83e40046c0 con 0x7f83f0102a40 2024-03-19T01:40:41.477 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.474+0000 7f83d7fff640 1 --2- 172.21.15.99:0/3946475281 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f83d80a2800 0x7f83d80a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:41.477 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.475+0000 7f83eeffd640 1 --2- 172.21.15.99:0/3946475281 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f83d80a2800 0x7f83d80a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:41.477 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.475+0000 7f83d7fff640 1 -- 172.21.15.99:0/3946475281 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f83e40cee90 con 0x7f83f0102a40 2024-03-19T01:40:41.477 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.475+0000 7f83eeffd640 1 --2- 172.21.15.99:0/3946475281 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f83d80a2800 0x7f83d80a4cc0 secure :-1 s=READY pgs=237 cs=0 l=1 rev1=1 crypto rx=0x7f83e0000c00 tx=0x7f83e0005fd0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:41.480 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.479+0000 7f83d7fff640 1 -- 172.21.15.99:0/3946475281 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f83e409bef0 con 0x7f83f0102a40 2024-03-19T01:40:41.654 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.653+0000 7f83f57e6640 1 -- 172.21.15.99:0/3946475281 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) -- 0x7f83f0062420 con 0x7f83f0102a40 2024-03-19T01:40:41.654 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.653+0000 7f83d7fff640 1 -- 172.21.15.99:0/3946475281 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v91) ==== 104+0+11 (secure 0 0 0) 0x7f83e409b7f0 con 0x7f83f0102a40 2024-03-19T01:40:41.654 INFO:teuthology.orchestra.run.smithi099.stdout:pg_num: 32 2024-03-19T01:40:41.656 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.655+0000 7f83f57e6640 1 -- 172.21.15.99:0/3946475281 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f83d80a2800 msgr2=0x7f83d80a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:41.656 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.655+0000 7f83f57e6640 1 --2- 172.21.15.99:0/3946475281 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f83d80a2800 0x7f83d80a4cc0 secure :-1 s=READY pgs=237 cs=0 l=1 rev1=1 crypto rx=0x7f83e0000c00 tx=0x7f83e0005fd0 comp rx=0 tx=0).stop 2024-03-19T01:40:41.656 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.655+0000 7f83f57e6640 1 -- 172.21.15.99:0/3946475281 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f83f0102a40 msgr2=0x7f83f01980e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:41.657 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.655+0000 7f83f57e6640 1 --2- 172.21.15.99:0/3946475281 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f83f0102a40 0x7f83f01980e0 secure :-1 s=READY pgs=325 cs=0 l=1 rev1=1 crypto rx=0x7f83e400e9f0 tx=0x7f83e400edf0 comp rx=0 tx=0).stop 2024-03-19T01:40:41.657 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.655+0000 7f83f57e6640 1 -- 172.21.15.99:0/3946475281 shutdown_connections 2024-03-19T01:40:41.657 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.655+0000 7f83f57e6640 1 --2- 172.21.15.99:0/3946475281 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f83f0103400 0x7f83f019f150 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:41.657 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.655+0000 7f83f57e6640 1 --2- 172.21.15.99:0/3946475281 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f83f0102a40 0x7f83f01980e0 unknown :-1 s=CLOSED pgs=325 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:41.657 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.655+0000 7f83f57e6640 1 --2- 172.21.15.99:0/3946475281 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f83d80a2800 0x7f83d80a4cc0 unknown :-1 s=CLOSED pgs=237 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:41.657 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.655+0000 7f83f57e6640 1 --2- 172.21.15.99:0/3946475281 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f83f0101850 0x7f83f0197ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:41.657 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.655+0000 7f83f57e6640 1 -- 172.21.15.99:0/3946475281 >> 172.21.15.99:0/3946475281 conn(0x7f83f00fcdc0 msgr2=0x7f83f0105dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:41.657 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.655+0000 7f83f57e6640 1 -- 172.21.15.99:0/3946475281 shutdown_connections 2024-03-19T01:40:41.657 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.655+0000 7f83f57e6640 1 -- 172.21.15.99:0/3946475281 wait complete. 2024-03-19T01:40:41.667 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-19T01:40:41.774 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.771+0000 7f03cea52640 1 -- 172.21.15.99:0/2365337531 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f03c8100720 msgr2=0x7f03c8100b40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:41.774 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.771+0000 7f03cea52640 1 --2- 172.21.15.99:0/2365337531 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f03c8100720 0x7f03c8100b40 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7f03b8009860 tx=0x7f03b802bb70 comp rx=0 tx=0).stop 2024-03-19T01:40:41.774 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.771+0000 7f03cea52640 1 -- 172.21.15.99:0/2365337531 shutdown_connections 2024-03-19T01:40:41.775 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.771+0000 7f03cea52640 1 --2- 172.21.15.99:0/2365337531 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f03c81022d0 0x7f03c8108b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:41.775 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.771+0000 7f03cea52640 1 --2- 172.21.15.99:0/2365337531 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f03c8101910 0x7f03c8101d90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:41.775 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.771+0000 7f03cea52640 1 --2- 172.21.15.99:0/2365337531 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f03c8100720 0x7f03c8100b40 unknown :-1 s=CLOSED pgs=80 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:41.775 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.771+0000 7f03cea52640 1 -- 172.21.15.99:0/2365337531 >> 172.21.15.99:0/2365337531 conn(0x7f03c80fbcb0 msgr2=0x7f03c80fe0d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:41.775 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.771+0000 7f03cea52640 1 -- 172.21.15.99:0/2365337531 shutdown_connections 2024-03-19T01:40:41.775 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.771+0000 7f03cea52640 1 -- 172.21.15.99:0/2365337531 wait complete. 2024-03-19T01:40:41.775 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.771+0000 7f03cea52640 1 Processor -- start 2024-03-19T01:40:41.775 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.771+0000 7f03cea52640 1 -- start start 2024-03-19T01:40:41.775 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.772+0000 7f03cea52640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f03c8100720 0x7f03c8197ba0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:41.775 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.772+0000 7f03cea52640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f03c8101910 0x7f03c81980e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:41.775 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.772+0000 7f03cea52640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f03c81022d0 0x7f03c819f150 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:41.775 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.772+0000 7f03cea52640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f03c810d4c0 con 0x7f03c81022d0 2024-03-19T01:40:41.775 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.772+0000 7f03cea52640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f03c810d340 con 0x7f03c8100720 2024-03-19T01:40:41.775 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.772+0000 7f03cea52640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f03c810d640 con 0x7f03c8101910 2024-03-19T01:40:41.775 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.772+0000 7f03ccfef640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f03c81022d0 0x7f03c819f150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:41.776 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.772+0000 7f03c77fe640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f03c8101910 0x7f03c81980e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:41.776 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.772+0000 7f03ccfef640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f03c81022d0 0x7f03c819f150 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:55350/0 (socket says 172.21.15.99:55350) 2024-03-19T01:40:41.776 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.772+0000 7f03c7fff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f03c8100720 0x7f03c8197ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:41.776 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.772+0000 7f03ccfef640 1 -- 172.21.15.99:0/261078693 learned_addr learned my addr 172.21.15.99:0/261078693 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:41.776 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.772+0000 7f03ccfef640 1 -- 172.21.15.99:0/261078693 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f03c8101910 msgr2=0x7f03c81980e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:41.776 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.772+0000 7f03ccfef640 1 --2- 172.21.15.99:0/261078693 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f03c8101910 0x7f03c81980e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:41.776 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.772+0000 7f03ccfef640 1 -- 172.21.15.99:0/261078693 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f03c8100720 msgr2=0x7f03c8197ba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:41.776 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.772+0000 7f03ccfef640 1 --2- 172.21.15.99:0/261078693 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f03c8100720 0x7f03c8197ba0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:41.776 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.772+0000 7f03ccfef640 1 -- 172.21.15.99:0/261078693 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f03c819f7c0 con 0x7f03c81022d0 2024-03-19T01:40:41.776 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.772+0000 7f03c77fe640 1 --2- 172.21.15.99:0/261078693 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f03c8101910 0x7f03c81980e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:40:41.776 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.772+0000 7f03c7fff640 1 --2- 172.21.15.99:0/261078693 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f03c8100720 0x7f03c8197ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:40:41.776 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.772+0000 7f03ccfef640 1 --2- 172.21.15.99:0/261078693 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f03c81022d0 0x7f03c819f150 secure :-1 s=READY pgs=326 cs=0 l=1 rev1=1 crypto rx=0x7f03c000eb10 tx=0x7f03c000c2d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:41.777 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.772+0000 7f03c57fa640 1 -- 172.21.15.99:0/261078693 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f03c0004b20 con 0x7f03c81022d0 2024-03-19T01:40:41.777 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.772+0000 7f03cea52640 1 -- 172.21.15.99:0/261078693 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f03c819fab0 con 0x7f03c81022d0 2024-03-19T01:40:41.777 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.772+0000 7f03c57fa640 1 -- 172.21.15.99:0/261078693 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f03c0004cc0 con 0x7f03c81022d0 2024-03-19T01:40:41.777 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.772+0000 7f03c57fa640 1 -- 172.21.15.99:0/261078693 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f03c0010760 con 0x7f03c81022d0 2024-03-19T01:40:41.777 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.772+0000 7f03cea52640 1 -- 172.21.15.99:0/261078693 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f03c81a0050 con 0x7f03c81022d0 2024-03-19T01:40:41.777 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.773+0000 7f03cea52640 1 -- 172.21.15.99:0/261078693 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f03c8107b10 con 0x7f03c81022d0 2024-03-19T01:40:41.778 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.774+0000 7f03c57fa640 1 -- 172.21.15.99:0/261078693 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 18) ==== 137290+0+0 (secure 0 0 0) 0x7f03c0005000 con 0x7f03c81022d0 2024-03-19T01:40:41.778 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.775+0000 7f03c57fa640 1 --2- 172.21.15.99:0/261078693 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f03b00a2800 0x7f03b00a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:41.778 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.775+0000 7f03c7fff640 1 --2- 172.21.15.99:0/261078693 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f03b00a2800 0x7f03b00a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:41.778 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.775+0000 7f03c57fa640 1 -- 172.21.15.99:0/261078693 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f03c00cf4d0 con 0x7f03c81022d0 2024-03-19T01:40:41.778 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.776+0000 7f03c7fff640 1 --2- 172.21.15.99:0/261078693 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f03b00a2800 0x7f03b00a4cc0 secure :-1 s=READY pgs=238 cs=0 l=1 rev1=1 crypto rx=0x7f03b8009830 tx=0x7f03b802e040 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:41.779 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.779+0000 7f03c57fa640 1 -- 172.21.15.99:0/261078693 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f03c009c530 con 0x7f03c81022d0 2024-03-19T01:40:41.956 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.955+0000 7f03cea52640 1 -- 172.21.15.99:0/261078693 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) -- 0x7f03c802d1f0 con 0x7f03c81022d0 2024-03-19T01:40:41.956 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.955+0000 7f03c57fa640 1 -- 172.21.15.99:0/261078693 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v91) ==== 100+0+11 (secure 0 0 0) 0x7f03c009be30 con 0x7f03c81022d0 2024-03-19T01:40:41.956 INFO:teuthology.orchestra.run.smithi099.stdout:pg_num: 64 2024-03-19T01:40:41.958 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.957+0000 7f03cea52640 1 -- 172.21.15.99:0/261078693 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f03b00a2800 msgr2=0x7f03b00a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:41.958 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.957+0000 7f03cea52640 1 --2- 172.21.15.99:0/261078693 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f03b00a2800 0x7f03b00a4cc0 secure :-1 s=READY pgs=238 cs=0 l=1 rev1=1 crypto rx=0x7f03b8009830 tx=0x7f03b802e040 comp rx=0 tx=0).stop 2024-03-19T01:40:41.959 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.957+0000 7f03cea52640 1 -- 172.21.15.99:0/261078693 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f03c81022d0 msgr2=0x7f03c819f150 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:41.959 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.957+0000 7f03cea52640 1 --2- 172.21.15.99:0/261078693 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f03c81022d0 0x7f03c819f150 secure :-1 s=READY pgs=326 cs=0 l=1 rev1=1 crypto rx=0x7f03c000eb10 tx=0x7f03c000c2d0 comp rx=0 tx=0).stop 2024-03-19T01:40:41.959 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.957+0000 7f03cea52640 1 -- 172.21.15.99:0/261078693 shutdown_connections 2024-03-19T01:40:41.959 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.957+0000 7f03cea52640 1 --2- 172.21.15.99:0/261078693 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f03c81022d0 0x7f03c819f150 unknown :-1 s=CLOSED pgs=326 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:41.959 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.957+0000 7f03cea52640 1 --2- 172.21.15.99:0/261078693 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f03c8101910 0x7f03c81980e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:41.959 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.957+0000 7f03cea52640 1 --2- 172.21.15.99:0/261078693 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f03b00a2800 0x7f03b00a4cc0 unknown :-1 s=CLOSED pgs=238 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:41.959 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.957+0000 7f03cea52640 1 --2- 172.21.15.99:0/261078693 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f03c8100720 0x7f03c8197ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:41.959 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.957+0000 7f03cea52640 1 -- 172.21.15.99:0/261078693 >> 172.21.15.99:0/261078693 conn(0x7f03c80fbcb0 msgr2=0x7f03c8104ce0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:41.959 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.957+0000 7f03cea52640 1 -- 172.21.15.99:0/261078693 shutdown_connections 2024-03-19T01:40:41.959 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:41.957+0000 7f03cea52640 1 -- 172.21.15.99:0/261078693 wait complete. 2024-03-19T01:40:41.968 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:41 smithi099 ceph-mon[62951]: pgmap v338: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 6 op/s 2024-03-19T01:40:41.968 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:41 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/3697305763' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-19T01:40:41.968 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:41 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/1656659839' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-03-19T01:40:41.968 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:41 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/3946475281' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-03-19T01:40:41.969 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-19T01:40:42.071 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.069+0000 7f37f6159640 1 -- 172.21.15.99:0/1235406607 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f37f01022e0 msgr2=0x7f37f01046d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:42.071 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.069+0000 7f37f6159640 1 --2- 172.21.15.99:0/1235406607 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f37f01022e0 0x7f37f01046d0 secure :-1 s=READY pgs=327 cs=0 l=1 rev1=1 crypto rx=0x7f37e0009900 tx=0x7f37e002bc30 comp rx=0 tx=0).stop 2024-03-19T01:40:42.071 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.070+0000 7f37f6159640 1 -- 172.21.15.99:0/1235406607 shutdown_connections 2024-03-19T01:40:42.071 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.070+0000 7f37f6159640 1 --2- 172.21.15.99:0/1235406607 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f37f0107540 0x7f37f0109970 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.071 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.070+0000 7f37f6159640 1 --2- 172.21.15.99:0/1235406607 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f37f0104c10 0x7f37f0107000 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.071 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.070+0000 7f37f6159640 1 --2- 172.21.15.99:0/1235406607 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f37f01022e0 0x7f37f01046d0 unknown :-1 s=CLOSED pgs=327 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.071 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.070+0000 7f37f6159640 1 -- 172.21.15.99:0/1235406607 >> 172.21.15.99:0/1235406607 conn(0x7f37f00fbcb0 msgr2=0x7f37f00fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:42.071 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.070+0000 7f37f6159640 1 -- 172.21.15.99:0/1235406607 shutdown_connections 2024-03-19T01:40:42.071 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.070+0000 7f37f6159640 1 -- 172.21.15.99:0/1235406607 wait complete. 2024-03-19T01:40:42.072 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.070+0000 7f37f6159640 1 Processor -- start 2024-03-19T01:40:42.072 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.070+0000 7f37f6159640 1 -- start start 2024-03-19T01:40:42.073 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.070+0000 7f37f6159640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f37f01022e0 0x7f37f0197bc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:42.073 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.070+0000 7f37f6159640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f37f0104c10 0x7f37f0198100 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:42.074 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.070+0000 7f37f6159640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f37f0107540 0x7f37f019f170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:42.074 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.070+0000 7f37f6159640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f37f010d4f0 con 0x7f37f0104c10 2024-03-19T01:40:42.074 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.070+0000 7f37f6159640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f37f010d370 con 0x7f37f0107540 2024-03-19T01:40:42.074 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.070+0000 7f37f6159640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f37f010d670 con 0x7f37f01022e0 2024-03-19T01:40:42.074 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.071+0000 7f37e6ffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f37f0104c10 0x7f37f0198100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:42.074 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.071+0000 7f37e6ffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f37f0104c10 0x7f37f0198100 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:55370/0 (socket says 172.21.15.99:55370) 2024-03-19T01:40:42.074 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.071+0000 7f37e6ffd640 1 -- 172.21.15.99:0/3223885361 learned_addr learned my addr 172.21.15.99:0/3223885361 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:42.074 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.071+0000 7f37ef7fe640 1 --2- 172.21.15.99:0/3223885361 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f37f01022e0 0x7f37f0197bc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:42.074 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.071+0000 7f37effff640 1 --2- 172.21.15.99:0/3223885361 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f37f0107540 0x7f37f019f170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:42.074 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.071+0000 7f37e6ffd640 1 -- 172.21.15.99:0/3223885361 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f37f01022e0 msgr2=0x7f37f0197bc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:42.075 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.071+0000 7f37e6ffd640 1 --2- 172.21.15.99:0/3223885361 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f37f01022e0 0x7f37f0197bc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.075 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.071+0000 7f37e6ffd640 1 -- 172.21.15.99:0/3223885361 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f37f0107540 msgr2=0x7f37f019f170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:42.075 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.071+0000 7f37e6ffd640 1 --2- 172.21.15.99:0/3223885361 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f37f0107540 0x7f37f019f170 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.075 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.071+0000 7f37e6ffd640 1 -- 172.21.15.99:0/3223885361 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f37f019f7e0 con 0x7f37f0104c10 2024-03-19T01:40:42.075 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.071+0000 7f37e6ffd640 1 --2- 172.21.15.99:0/3223885361 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f37f0104c10 0x7f37f0198100 secure :-1 s=READY pgs=328 cs=0 l=1 rev1=1 crypto rx=0x7f37d800e9f0 tx=0x7f37d800eeb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:42.075 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.071+0000 7f37ed7fa640 1 -- 172.21.15.99:0/3223885361 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f37d800ccc0 con 0x7f37f0104c10 2024-03-19T01:40:42.075 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.071+0000 7f37ed7fa640 1 -- 172.21.15.99:0/3223885361 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f37d8005500 con 0x7f37f0104c10 2024-03-19T01:40:42.075 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.071+0000 7f37f6159640 1 -- 172.21.15.99:0/3223885361 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f37f019fad0 con 0x7f37f0104c10 2024-03-19T01:40:42.075 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.071+0000 7f37ed7fa640 1 -- 172.21.15.99:0/3223885361 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f37d8010690 con 0x7f37f0104c10 2024-03-19T01:40:42.075 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.071+0000 7f37f6159640 1 -- 172.21.15.99:0/3223885361 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f37f01a0020 con 0x7f37f0104c10 2024-03-19T01:40:42.075 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.072+0000 7f37f6159640 1 -- 172.21.15.99:0/3223885361 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f37f00fd7a0 con 0x7f37f0104c10 2024-03-19T01:40:42.076 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.073+0000 7f37ed7fa640 1 -- 172.21.15.99:0/3223885361 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 18) ==== 137290+0+0 (secure 0 0 0) 0x7f37d80046c0 con 0x7f37f0104c10 2024-03-19T01:40:42.076 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.074+0000 7f37ed7fa640 1 --2- 172.21.15.99:0/3223885361 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f37d00a2880 0x7f37d00a4d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:42.076 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.074+0000 7f37ef7fe640 1 --2- 172.21.15.99:0/3223885361 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f37d00a2880 0x7f37d00a4d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:42.076 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.074+0000 7f37ed7fa640 1 -- 172.21.15.99:0/3223885361 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f37d80ced70 con 0x7f37f0104c10 2024-03-19T01:40:42.076 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.074+0000 7f37ef7fe640 1 --2- 172.21.15.99:0/3223885361 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f37d00a2880 0x7f37d00a4d40 secure :-1 s=READY pgs=239 cs=0 l=1 rev1=1 crypto rx=0x7f37e002c7b0 tx=0x7f37e0005fd0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:42.079 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.078+0000 7f37ed7fa640 1 -- 172.21.15.99:0/3223885361 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f37d809bdd0 con 0x7f37f0104c10 2024-03-19T01:40:42.291 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.290+0000 7f37f6159640 1 -- 172.21.15.99:0/3223885361 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f37f002d1f0 con 0x7f37f0104c10 2024-03-19T01:40:42.292 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.291+0000 7f37ed7fa640 1 -- 172.21.15.99:0/3223885361 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 20 v20) ==== 94+0+11878 (secure 0 0 0) 0x7f37d809b6d0 con 0x7f37f0104c10 2024-03-19T01:40:42.292 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:40:42.293 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":20,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14562,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6835/1631815621","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6834","nonce":1631815621},{"type":"v1","addr":"172.21.15.99:6835","nonce":1631815621}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":4},{"gid":14646,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6839/2236145144","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6838","nonce":2236145144},{"type":"v1","addr":"172.21.15.99:6839","nonce":2236145144}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11},{"gid":14658,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6841/1660990159","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6840","nonce":1660990159},{"type":"v1","addr":"172.21.15.99:6841","nonce":1660990159}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":12},{"gid":24383,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.112:6833/2969810977","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6832","nonce":2969810977},{"type":"v1","addr":"172.21.15.112:6833","nonce":2969810977}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3},{"gid":24404,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6837/2038634879","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6836","nonce":2038634879},{"type":"v1","addr":"172.21.15.99:6837","nonce":2038634879}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":24415,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.115:6833/2219320481","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6832","nonce":2219320481},{"type":"v1","addr":"172.21.15.115:6833","nonce":2219320481}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":20,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-03-19T01:38:56.124529+0000","modified":"2024-03-19T01:39:00.323326+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24487,"mds_1":24479,"mds_2":24472},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24416":{"gid":24416,"name":"e","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.112:6835/3690587528","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6834","nonce":3690587528},{"type":"v1","addr":"172.21.15.112:6835","nonce":3690587528}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24440":{"gid":24440,"name":"h","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.112:6837/2339823159","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6836","nonce":2339823159},{"type":"v1","addr":"172.21.15.112:6837","nonce":2339823159}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24460":{"gid":24460,"name":"f","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.115:6835/591078413","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6834","nonce":591078413},{"type":"v1","addr":"172.21.15.115:6835","nonce":591078413}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24472":{"gid":24472,"name":"c","rank":2,"incarnation":19,"state":"up:active","state_seq":7,"addr":"172.21.15.115:6837/1368988191","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6836","nonce":1368988191},{"type":"v1","addr":"172.21.15.115:6837","nonce":1368988191}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24479":{"gid":24479,"name":"b","rank":1,"incarnation":17,"state":"up:active","state_seq":4,"addr":"172.21.15.112:6839/986249680","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6838","nonce":986249680},{"type":"v1","addr":"172.21.15.112:6839","nonce":986249680}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24487":{"gid":24487,"name":"i","rank":0,"incarnation":14,"state":"up:active","state_seq":5,"addr":"172.21.15.115:6839/2743761968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6838","nonce":2743761968},{"type":"v1","addr":"172.21.15.115:6839","nonce":2743761968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24487,"qdb_cluster":[24487,24479,24472]},"id":1}]} 2024-03-19T01:40:42.293 INFO:teuthology.orchestra.run.smithi099.stderr:dumped fsmap epoch 20 2024-03-19T01:40:42.294 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.293+0000 7f37f6159640 1 -- 172.21.15.99:0/3223885361 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f37d00a2880 msgr2=0x7f37d00a4d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:42.294 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.293+0000 7f37f6159640 1 --2- 172.21.15.99:0/3223885361 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f37d00a2880 0x7f37d00a4d40 secure :-1 s=READY pgs=239 cs=0 l=1 rev1=1 crypto rx=0x7f37e002c7b0 tx=0x7f37e0005fd0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.295 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.293+0000 7f37f6159640 1 -- 172.21.15.99:0/3223885361 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f37f0104c10 msgr2=0x7f37f0198100 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:42.295 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.293+0000 7f37f6159640 1 --2- 172.21.15.99:0/3223885361 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f37f0104c10 0x7f37f0198100 secure :-1 s=READY pgs=328 cs=0 l=1 rev1=1 crypto rx=0x7f37d800e9f0 tx=0x7f37d800eeb0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.295 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.293+0000 7f37f6159640 1 -- 172.21.15.99:0/3223885361 shutdown_connections 2024-03-19T01:40:42.295 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.293+0000 7f37f6159640 1 --2- 172.21.15.99:0/3223885361 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f37d00a2880 0x7f37d00a4d40 unknown :-1 s=CLOSED pgs=239 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.295 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.293+0000 7f37f6159640 1 --2- 172.21.15.99:0/3223885361 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f37f0107540 0x7f37f019f170 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.295 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.293+0000 7f37f6159640 1 --2- 172.21.15.99:0/3223885361 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f37f0104c10 0x7f37f0198100 unknown :-1 s=CLOSED pgs=328 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.295 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.293+0000 7f37f6159640 1 --2- 172.21.15.99:0/3223885361 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f37f01022e0 0x7f37f0197bc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.295 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.293+0000 7f37f6159640 1 -- 172.21.15.99:0/3223885361 >> 172.21.15.99:0/3223885361 conn(0x7f37f00fbcb0 msgr2=0x7f37f0102d20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:42.295 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.293+0000 7f37f6159640 1 -- 172.21.15.99:0/3223885361 shutdown_connections 2024-03-19T01:40:42.295 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.293+0000 7f37f6159640 1 -- 172.21.15.99:0/3223885361 wait complete. 2024-03-19T01:40:42.306 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-19T01:40:42.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:41 smithi115 ceph-mon[67332]: pgmap v338: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 6 op/s 2024-03-19T01:40:42.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:41 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/3697305763' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-19T01:40:42.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:41 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/1656659839' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-03-19T01:40:42.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:41 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/3946475281' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-03-19T01:40:42.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:41 smithi112 ceph-mon[66819]: pgmap v338: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 6 op/s 2024-03-19T01:40:42.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:41 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/3697305763' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-19T01:40:42.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:41 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/1656659839' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-03-19T01:40:42.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:41 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/3946475281' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-03-19T01:40:42.409 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.406+0000 7f110c4de640 1 -- 172.21.15.99:0/1078782102 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1104103400 msgr2=0x7f1104109c90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:42.409 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.406+0000 7f110c4de640 1 --2- 172.21.15.99:0/1078782102 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1104103400 0x7f1104109c90 secure :-1 s=READY pgs=329 cs=0 l=1 rev1=1 crypto rx=0x7f1100009ca0 tx=0x7f110002c0f0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.409 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.406+0000 7f110c4de640 1 -- 172.21.15.99:0/1078782102 shutdown_connections 2024-03-19T01:40:42.410 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.406+0000 7f110c4de640 1 --2- 172.21.15.99:0/1078782102 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1104103400 0x7f1104109c90 unknown :-1 s=CLOSED pgs=329 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.410 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.406+0000 7f110c4de640 1 --2- 172.21.15.99:0/1078782102 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1104102a40 0x7f1104102ec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.410 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.406+0000 7f110c4de640 1 --2- 172.21.15.99:0/1078782102 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1104101850 0x7f1104101c70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.410 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.406+0000 7f110c4de640 1 -- 172.21.15.99:0/1078782102 >> 172.21.15.99:0/1078782102 conn(0x7f11040fcdc0 msgr2=0x7f11040ff200 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:42.410 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.406+0000 7f110c4de640 1 -- 172.21.15.99:0/1078782102 shutdown_connections 2024-03-19T01:40:42.410 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.406+0000 7f110c4de640 1 -- 172.21.15.99:0/1078782102 wait complete. 2024-03-19T01:40:42.410 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.406+0000 7f110c4de640 1 Processor -- start 2024-03-19T01:40:42.410 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.406+0000 7f110c4de640 1 -- start start 2024-03-19T01:40:42.410 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.407+0000 7f110c4de640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1104101850 0x7f1104197b80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:42.410 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.407+0000 7f110c4de640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1104102a40 0x7f11041980c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:42.410 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.407+0000 7f110c4de640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1104103400 0x7f110419f130 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:42.410 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.407+0000 7f110c4de640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f110410d4d0 con 0x7f1104102a40 2024-03-19T01:40:42.410 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.407+0000 7f110c4de640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f110410d350 con 0x7f1104103400 2024-03-19T01:40:42.411 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.407+0000 7f110c4de640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f110410d650 con 0x7f1104101850 2024-03-19T01:40:42.411 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.407+0000 7f1109a79640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1104102a40 0x7f11041980c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:42.411 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.407+0000 7f110a27a640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1104101850 0x7f1104197b80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:42.411 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.407+0000 7f1109a79640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1104102a40 0x7f11041980c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:55390/0 (socket says 172.21.15.99:55390) 2024-03-19T01:40:42.411 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.407+0000 7f1109a79640 1 -- 172.21.15.99:0/1010837617 learned_addr learned my addr 172.21.15.99:0/1010837617 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:42.411 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.407+0000 7f110aa7b640 1 --2- 172.21.15.99:0/1010837617 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1104103400 0x7f110419f130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:42.411 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.407+0000 7f1109a79640 1 -- 172.21.15.99:0/1010837617 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1104101850 msgr2=0x7f1104197b80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:42.411 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.407+0000 7f1109a79640 1 --2- 172.21.15.99:0/1010837617 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1104101850 0x7f1104197b80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.411 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.407+0000 7f1109a79640 1 -- 172.21.15.99:0/1010837617 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1104103400 msgr2=0x7f110419f130 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:42.411 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.407+0000 7f1109a79640 1 --2- 172.21.15.99:0/1010837617 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1104103400 0x7f110419f130 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.411 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.407+0000 7f1109a79640 1 -- 172.21.15.99:0/1010837617 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f110419f7a0 con 0x7f1104102a40 2024-03-19T01:40:42.411 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.407+0000 7f110a27a640 1 --2- 172.21.15.99:0/1010837617 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1104101850 0x7f1104197b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:40:42.411 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.407+0000 7f1109a79640 1 --2- 172.21.15.99:0/1010837617 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1104102a40 0x7f11041980c0 secure :-1 s=READY pgs=330 cs=0 l=1 rev1=1 crypto rx=0x7f10f400e9f0 tx=0x7f10f400edf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:42.411 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.407+0000 7f10fb7fe640 1 -- 172.21.15.99:0/1010837617 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f10f400cd80 con 0x7f1104102a40 2024-03-19T01:40:42.412 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.407+0000 7f110c4de640 1 -- 172.21.15.99:0/1010837617 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f110419fa90 con 0x7f1104102a40 2024-03-19T01:40:42.412 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.407+0000 7f10fb7fe640 1 -- 172.21.15.99:0/1010837617 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f10f4005500 con 0x7f1104102a40 2024-03-19T01:40:42.412 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.407+0000 7f10fb7fe640 1 -- 172.21.15.99:0/1010837617 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f10f4010690 con 0x7f1104102a40 2024-03-19T01:40:42.412 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.407+0000 7f110c4de640 1 -- 172.21.15.99:0/1010837617 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f110419ffe0 con 0x7f1104102a40 2024-03-19T01:40:42.412 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.408+0000 7f110c4de640 1 -- 172.21.15.99:0/1010837617 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f1104108b10 con 0x7f1104102a40 2024-03-19T01:40:42.413 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.409+0000 7f10fb7fe640 1 -- 172.21.15.99:0/1010837617 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 18) ==== 137290+0+0 (secure 0 0 0) 0x7f10f40046c0 con 0x7f1104102a40 2024-03-19T01:40:42.413 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.410+0000 7f10fb7fe640 1 --2- 172.21.15.99:0/1010837617 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f10f00a2800 0x7f10f00a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:42.413 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.410+0000 7f110a27a640 1 --2- 172.21.15.99:0/1010837617 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f10f00a2800 0x7f10f00a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:42.413 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.410+0000 7f10fb7fe640 1 -- 172.21.15.99:0/1010837617 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f10f40cee90 con 0x7f1104102a40 2024-03-19T01:40:42.413 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.410+0000 7f110a27a640 1 --2- 172.21.15.99:0/1010837617 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f10f00a2800 0x7f10f00a4cc0 secure :-1 s=READY pgs=240 cs=0 l=1 rev1=1 crypto rx=0x7f11041028a0 tx=0x7f10fc005f90 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:42.415 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.414+0000 7f10fb7fe640 1 -- 172.21.15.99:0/1010837617 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f10f409bef0 con 0x7f1104102a40 2024-03-19T01:40:42.595 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.590+0000 7f110c4de640 1 -- 172.21.15.99:0/1010837617 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f1104062420 con 0x7f1104102a40 2024-03-19T01:40:42.596 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.592+0000 7f10fb7fe640 1 -- 172.21.15.99:0/1010837617 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v91) ==== 74+0+22804 (secure 0 0 0) 0x7f10f409b7f0 con 0x7f1104102a40 2024-03-19T01:40:42.596 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T01:40:42.596 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":91,"fsid":"438fb69c-e590-11ee-95c9-87774f69a715","created":"2024-03-19T01:31:10.165073+0000","modified":"2024-03-19T01:40:08.821469+0000","last_up_change":"2024-03-19T01:38:09.082688+0000","last_in_change":"2024-03-19T01:37:49.115793+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-19T01:34:31.001309+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-19T01:38:54.018623+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"86","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-19T01:38:55.319036+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"87","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"0c407a0e-a40e-43d7-ad9a-bb8f3cd4a9ba","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6803","nonce":2935260112}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6805","nonce":2935260112}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6808","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6809","nonce":2935260112}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6807","nonce":2935260112}]},"public_addr":"172.21.15.99:6803/2935260112","cluster_addr":"172.21.15.99:6805/2935260112","heartbeat_back_addr":"172.21.15.99:6809/2935260112","heartbeat_front_addr":"172.21.15.99:6807/2935260112","state":["exists","up"]},{"osd":1,"uuid":"a48f9173-4a06-49f5-bca2-2c9047b8f666","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6800","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6801","nonce":1468854289}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6802","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6803","nonce":1468854289}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6806","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6807","nonce":1468854289}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6804","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6805","nonce":1468854289}]},"public_addr":"172.21.15.112:6801/1468854289","cluster_addr":"172.21.15.112:6803/1468854289","heartbeat_back_addr":"172.21.15.112:6807/1468854289","heartbeat_front_addr":"172.21.15.112:6805/1468854289","state":["exists","up"]},{"osd":2,"uuid":"b7715fdd-b6c9-4348-bd16-5eb61b427eef","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6800","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6801","nonce":3766690472}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6802","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6803","nonce":3766690472}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6806","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6807","nonce":3766690472}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6804","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6805","nonce":3766690472}]},"public_addr":"172.21.15.115:6801/3766690472","cluster_addr":"172.21.15.115:6803/3766690472","heartbeat_back_addr":"172.21.15.115:6807/3766690472","heartbeat_front_addr":"172.21.15.115:6805/3766690472","state":["exists","up"]},{"osd":3,"uuid":"6697ed1b-96bf-42de-8440-d17e9785b359","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6810","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6811","nonce":3756732895}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6812","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6813","nonce":3756732895}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6816","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6817","nonce":3756732895}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6814","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6815","nonce":3756732895}]},"public_addr":"172.21.15.99:6811/3756732895","cluster_addr":"172.21.15.99:6813/3756732895","heartbeat_back_addr":"172.21.15.99:6817/3756732895","heartbeat_front_addr":"172.21.15.99:6815/3756732895","state":["exists","up"]},{"osd":4,"uuid":"d958d6b9-67f6-43d0-b704-aa55dc9df83a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":34,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6808","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6809","nonce":3717986391}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6810","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6811","nonce":3717986391}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6814","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6815","nonce":3717986391}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6812","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6813","nonce":3717986391}]},"public_addr":"172.21.15.112:6809/3717986391","cluster_addr":"172.21.15.112:6811/3717986391","heartbeat_back_addr":"172.21.15.112:6815/3717986391","heartbeat_front_addr":"172.21.15.112:6813/3717986391","state":["exists","up"]},{"osd":5,"uuid":"a73eb49a-7925-49ba-bc4f-515e17f8b8ae","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6808","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6809","nonce":1971806408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6810","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6811","nonce":1971806408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6814","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6815","nonce":1971806408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6812","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6813","nonce":1971806408}]},"public_addr":"172.21.15.115:6809/1971806408","cluster_addr":"172.21.15.115:6811/1971806408","heartbeat_back_addr":"172.21.15.115:6815/1971806408","heartbeat_front_addr":"172.21.15.115:6813/1971806408","state":["exists","up"]},{"osd":6,"uuid":"caa718cd-e08b-48de-832a-29546d71e508","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6818","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6819","nonce":1130832772}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6820","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6821","nonce":1130832772}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6824","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6825","nonce":1130832772}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6822","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6823","nonce":1130832772}]},"public_addr":"172.21.15.99:6819/1130832772","cluster_addr":"172.21.15.99:6821/1130832772","heartbeat_back_addr":"172.21.15.99:6825/1130832772","heartbeat_front_addr":"172.21.15.99:6823/1130832772","state":["exists","up"]},{"osd":7,"uuid":"dbee19a1-2315-40d3-88e7-876fcde28903","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":53,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6816","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6817","nonce":3610356576}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6818","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6819","nonce":3610356576}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6822","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6823","nonce":3610356576}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6820","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6821","nonce":3610356576}]},"public_addr":"172.21.15.112:6817/3610356576","cluster_addr":"172.21.15.112:6819/3610356576","heartbeat_back_addr":"172.21.15.112:6823/3610356576","heartbeat_front_addr":"172.21.15.112:6821/3610356576","state":["exists","up"]},{"osd":8,"uuid":"14a866dc-5ce3-42fc-8bcf-9088b68726cb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":60,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6816","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6817","nonce":4255803111}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6818","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6819","nonce":4255803111}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6822","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6823","nonce":4255803111}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6820","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6821","nonce":4255803111}]},"public_addr":"172.21.15.115:6817/4255803111","cluster_addr":"172.21.15.115:6819/4255803111","heartbeat_back_addr":"172.21.15.115:6823/4255803111","heartbeat_front_addr":"172.21.15.115:6821/4255803111","state":["exists","up"]},{"osd":9,"uuid":"1db93652-6035-4906-91ae-d64d5a909502","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":68,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6826","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6827","nonce":1410921523}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6828","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6829","nonce":1410921523}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6832","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6833","nonce":1410921523}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6830","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6831","nonce":1410921523}]},"public_addr":"172.21.15.99:6827/1410921523","cluster_addr":"172.21.15.99:6829/1410921523","heartbeat_back_addr":"172.21.15.99:6833/1410921523","heartbeat_front_addr":"172.21.15.99:6831/1410921523","state":["exists","up"]},{"osd":10,"uuid":"6c50a8a7-3345-4f4a-af0e-5606c7aca453","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":74,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6824","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6825","nonce":891805922}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6826","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6827","nonce":891805922}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6830","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6831","nonce":891805922}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6828","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6829","nonce":891805922}]},"public_addr":"172.21.15.112:6825/891805922","cluster_addr":"172.21.15.112:6827/891805922","heartbeat_back_addr":"172.21.15.112:6831/891805922","heartbeat_front_addr":"172.21.15.112:6829/891805922","state":["exists","up"]},{"osd":11,"uuid":"fd63b7fd-0db5-46d5-82fa-9707bbd0b495","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":82,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6824","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6825","nonce":938436244}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6826","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6827","nonce":938436244}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6830","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6831","nonce":938436244}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6828","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6829","nonce":938436244}]},"public_addr":"172.21.15.115:6825/938436244","cluster_addr":"172.21.15.115:6827/938436244","heartbeat_back_addr":"172.21.15.115:6831/938436244","heartbeat_front_addr":"172.21.15.115:6829/938436244","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:33:39.036822+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:34:03.339566+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:34:27.149273+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:34:51.315561+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:35:15.396031+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:35:38.957168+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:36:03.213914+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:36:27.763699+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:36:51.688670+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:37:17.280165+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:37:42.554633+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:38:07.438039+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.0","mappings":[{"from":8,"to":1}]},{"pgid":"2.8","mappings":[{"from":2,"to":11}]},{"pgid":"2.b","mappings":[{"from":8,"to":11}]},{"pgid":"2.e","mappings":[{"from":2,"to":4}]},{"pgid":"3.0","mappings":[{"from":5,"to":9}]},{"pgid":"3.2","mappings":[{"from":3,"to":9}]},{"pgid":"3.14","mappings":[{"from":1,"to":9}]},{"pgid":"3.17","mappings":[{"from":4,"to":8}]},{"pgid":"3.31","mappings":[{"from":4,"to":8}]},{"pgid":"3.37","mappings":[{"from":5,"to":7}]},{"pgid":"3.3a","mappings":[{"from":4,"to":6}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.99:6801/1328245934":"2024-03-20T01:32:06.661002+0000","172.21.15.99:0/1618965837":"2024-03-20T01:32:06.661002+0000","172.21.15.99:0/2491429851":"2024-03-20T01:32:06.661002+0000","172.21.15.99:6800/1328245934":"2024-03-20T01:32:06.661002+0000","172.21.15.99:0/2112231766":"2024-03-20T01:31:40.161751+0000","172.21.15.99:0/3341836545":"2024-03-20T01:31:40.161751+0000","172.21.15.99:6800/4223048293":"2024-03-20T01:31:40.161751+0000","172.21.15.99:6801/4223048293":"2024-03-20T01:31:40.161751+0000","172.21.15.99:0/2377621863":"2024-03-20T01:31:40.161751+0000","172.21.15.99:0/1904400935":"2024-03-20T01:32:06.661002+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-19T01:40:42.597 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.594+0000 7f110c4de640 1 -- 172.21.15.99:0/1010837617 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f10f00a2800 msgr2=0x7f10f00a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:42.597 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.594+0000 7f110c4de640 1 --2- 172.21.15.99:0/1010837617 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f10f00a2800 0x7f10f00a4cc0 secure :-1 s=READY pgs=240 cs=0 l=1 rev1=1 crypto rx=0x7f11041028a0 tx=0x7f10fc005f90 comp rx=0 tx=0).stop 2024-03-19T01:40:42.597 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.594+0000 7f110c4de640 1 -- 172.21.15.99:0/1010837617 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1104102a40 msgr2=0x7f11041980c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:42.597 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.594+0000 7f110c4de640 1 --2- 172.21.15.99:0/1010837617 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1104102a40 0x7f11041980c0 secure :-1 s=READY pgs=330 cs=0 l=1 rev1=1 crypto rx=0x7f10f400e9f0 tx=0x7f10f400edf0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.597 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.594+0000 7f110c4de640 1 -- 172.21.15.99:0/1010837617 shutdown_connections 2024-03-19T01:40:42.598 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.594+0000 7f110c4de640 1 --2- 172.21.15.99:0/1010837617 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f10f00a2800 0x7f10f00a4cc0 unknown :-1 s=CLOSED pgs=240 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.598 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.594+0000 7f110c4de640 1 --2- 172.21.15.99:0/1010837617 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1104103400 0x7f110419f130 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.598 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.594+0000 7f110c4de640 1 --2- 172.21.15.99:0/1010837617 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1104102a40 0x7f11041980c0 unknown :-1 s=CLOSED pgs=330 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.598 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.594+0000 7f110c4de640 1 --2- 172.21.15.99:0/1010837617 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1104101850 0x7f1104197b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.598 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.594+0000 7f110c4de640 1 -- 172.21.15.99:0/1010837617 >> 172.21.15.99:0/1010837617 conn(0x7f11040fcdc0 msgr2=0x7f1104105ce0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:42.598 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.594+0000 7f110c4de640 1 -- 172.21.15.99:0/1010837617 shutdown_connections 2024-03-19T01:40:42.598 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.594+0000 7f110c4de640 1 -- 172.21.15.99:0/1010837617 wait complete. 2024-03-19T01:40:42.606 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 20, 'flags': 50, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': True, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 32, 'explicitly_allowed_features': 32, 'created': '2024-03-19T01:38:56.124529+0000', 'modified': '2024-03-19T01:39:00.323326+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 300, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}, 'max_mds': 3, 'in': [0, 1, 2], 'up': {'mds_0': 24487, 'mds_1': 24479, 'mds_2': 24472}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_24416': {'gid': 24416, 'name': 'e', 'rank': 2, 'incarnation': 0, 'state': 'up:standby-replay', 'state_seq': 1, 'addr': '172.21.15.112:6835/3690587528', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.112:6834', 'nonce': 3690587528}, {'type': 'v1', 'addr': '172.21.15.112:6835', 'nonce': 3690587528}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}}, 'gid_24440': {'gid': 24440, 'name': 'h', 'rank': 1, 'incarnation': 0, 'state': 'up:standby-replay', 'state_seq': 1, 'addr': '172.21.15.112:6837/2339823159', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.112:6836', 'nonce': 2339823159}, {'type': 'v1', 'addr': '172.21.15.112:6837', 'nonce': 2339823159}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}}, 'gid_24460': {'gid': 24460, 'name': 'f', 'rank': 0, 'incarnation': 0, 'state': 'up:standby-replay', 'state_seq': 1, 'addr': '172.21.15.115:6835/591078413', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.115:6834', 'nonce': 591078413}, {'type': 'v1', 'addr': '172.21.15.115:6835', 'nonce': 591078413}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}}, 'gid_24472': {'gid': 24472, 'name': 'c', 'rank': 2, 'incarnation': 19, 'state': 'up:active', 'state_seq': 7, 'addr': '172.21.15.115:6837/1368988191', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.115:6836', 'nonce': 1368988191}, {'type': 'v1', 'addr': '172.21.15.115:6837', 'nonce': 1368988191}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}}, 'gid_24479': {'gid': 24479, 'name': 'b', 'rank': 1, 'incarnation': 17, 'state': 'up:active', 'state_seq': 4, 'addr': '172.21.15.112:6839/986249680', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.112:6838', 'nonce': 986249680}, {'type': 'v1', 'addr': '172.21.15.112:6839', 'nonce': 986249680}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}}, 'gid_24487': {'gid': 24487, 'name': 'i', 'rank': 0, 'incarnation': 14, 'state': 'up:active', 'state_seq': 5, 'addr': '172.21.15.115:6839/2743761968', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.115:6838', 'nonce': 2743761968}, {'type': 'v1', 'addr': '172.21.15.115:6839', 'nonce': 2743761968}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}}}, 'data_pools': [3], 'metadata_pool': 2, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 24487, 'qdb_cluster': [24487, 24479, 24472]} 2024-03-19T01:40:42.606 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: 3/3 2024-03-19T01:40:42.607 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.c status 2024-03-19T01:40:42.712 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.709+0000 7f1a89636640 1 -- 172.21.15.99:0/2078216040 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1a84101920 msgr2=0x7f1a84101da0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:42.712 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.709+0000 7f1a89636640 1 --2- 172.21.15.99:0/2078216040 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1a84101920 0x7f1a84101da0 secure :-1 s=READY pgs=331 cs=0 l=1 rev1=1 crypto rx=0x7f1a6c009900 tx=0x7f1a6c02bc30 comp rx=0 tx=0).stop 2024-03-19T01:40:42.712 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.709+0000 7f1a89636640 1 -- 172.21.15.99:0/2078216040 shutdown_connections 2024-03-19T01:40:42.712 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.709+0000 7f1a89636640 1 --2- 172.21.15.99:0/2078216040 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1a841022e0 0x7f1a84108b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.712 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.709+0000 7f1a89636640 1 --2- 172.21.15.99:0/2078216040 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1a84101920 0x7f1a84101da0 unknown :-1 s=CLOSED pgs=331 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.713 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.709+0000 7f1a89636640 1 --2- 172.21.15.99:0/2078216040 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1a84100730 0x7f1a84100b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.713 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.709+0000 7f1a89636640 1 -- 172.21.15.99:0/2078216040 >> 172.21.15.99:0/2078216040 conn(0x7f1a840fbcc0 msgr2=0x7f1a840fe0e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:42.713 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.709+0000 7f1a89636640 1 -- 172.21.15.99:0/2078216040 shutdown_connections 2024-03-19T01:40:42.713 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.709+0000 7f1a89636640 1 -- 172.21.15.99:0/2078216040 wait complete. 2024-03-19T01:40:42.713 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.709+0000 7f1a89636640 1 Processor -- start 2024-03-19T01:40:42.713 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.709+0000 7f1a89636640 1 -- start start 2024-03-19T01:40:42.713 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.709+0000 7f1a89636640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1a84100730 0x7f1a8406a900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:42.713 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.710+0000 7f1a89636640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1a84101920 0x7f1a8419c8b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:42.713 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.710+0000 7f1a89636640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1a841022e0 0x7f1a8419eed0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:42.713 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.710+0000 7f1a89636640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f1a8410c390 con 0x7f1a84101920 2024-03-19T01:40:42.713 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.710+0000 7f1a89636640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f1a8410c210 con 0x7f1a84100730 2024-03-19T01:40:42.713 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.710+0000 7f1a89636640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f1a8410c510 con 0x7f1a841022e0 2024-03-19T01:40:42.713 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.710+0000 7f1a827fc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1a84101920 0x7f1a8419c8b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:42.714 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.710+0000 7f1a827fc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1a84101920 0x7f1a8419c8b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58112/0 (socket says 172.21.15.99:58112) 2024-03-19T01:40:42.714 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.710+0000 7f1a827fc640 1 -- 172.21.15.99:0/826672986 learned_addr learned my addr 172.21.15.99:0/826672986 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:42.714 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.710+0000 7f1a82ffd640 1 --2- 172.21.15.99:0/826672986 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1a84100730 0x7f1a8406a900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:42.714 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.710+0000 7f1a837fe640 1 --2- 172.21.15.99:0/826672986 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1a841022e0 0x7f1a8419eed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:42.714 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.710+0000 7f1a827fc640 1 -- 172.21.15.99:0/826672986 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1a841022e0 msgr2=0x7f1a8419eed0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:42.714 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.710+0000 7f1a827fc640 1 --2- 172.21.15.99:0/826672986 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1a841022e0 0x7f1a8419eed0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.714 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.710+0000 7f1a827fc640 1 -- 172.21.15.99:0/826672986 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1a84100730 msgr2=0x7f1a8406a900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:42.714 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.710+0000 7f1a827fc640 1 --2- 172.21.15.99:0/826672986 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1a84100730 0x7f1a8406a900 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.714 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.710+0000 7f1a827fc640 1 -- 172.21.15.99:0/826672986 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f1a8419f7e0 con 0x7f1a84101920 2024-03-19T01:40:42.714 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.710+0000 7f1a82ffd640 1 --2- 172.21.15.99:0/826672986 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1a84100730 0x7f1a8406a900 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:40:42.714 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.710+0000 7f1a837fe640 1 --2- 172.21.15.99:0/826672986 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1a841022e0 0x7f1a8419eed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:40:42.714 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.710+0000 7f1a827fc640 1 --2- 172.21.15.99:0/826672986 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1a84101920 0x7f1a8419c8b0 secure :-1 s=READY pgs=332 cs=0 l=1 rev1=1 crypto rx=0x7f1a6c009230 tx=0x7f1a6c02c650 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:42.714 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.710+0000 7f1a7bfff640 1 -- 172.21.15.99:0/826672986 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1a6c041070 con 0x7f1a84101920 2024-03-19T01:40:42.715 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.710+0000 7f1a89636640 1 -- 172.21.15.99:0/826672986 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f1a8419fa10 con 0x7f1a84101920 2024-03-19T01:40:42.715 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.710+0000 7f1a7bfff640 1 -- 172.21.15.99:0/826672986 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f1a6c004450 con 0x7f1a84101920 2024-03-19T01:40:42.715 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.710+0000 7f1a7bfff640 1 -- 172.21.15.99:0/826672986 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1a6c003e20 con 0x7f1a84101920 2024-03-19T01:40:42.715 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.710+0000 7f1a89636640 1 -- 172.21.15.99:0/826672986 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f1a8419ff00 con 0x7f1a84101920 2024-03-19T01:40:42.722 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.713+0000 7f1a7bfff640 1 -- 172.21.15.99:0/826672986 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 18) ==== 137290+0+0 (secure 0 0 0) 0x7f1a6c044070 con 0x7f1a84101920 2024-03-19T01:40:42.722 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.713+0000 7f1a8c21d740 1 Processor -- start 2024-03-19T01:40:42.722 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.713+0000 7f1a8c21d740 1 -- start start 2024-03-19T01:40:42.722 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.713+0000 7f1a8c21d740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5590eaa95ff0 0x5590eaa96410 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:42.722 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.713+0000 7f1a8c21d740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5590eaa969e0 0x5590eaa9b4e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:42.722 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.713+0000 7f1a7bfff640 1 --2- 172.21.15.99:0/826672986 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f1a700a2800 0x7f1a700a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:42.722 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.713+0000 7f1a8c21d740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5590eaa9bb50 0x5590eaa9dfe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:42.722 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.714+0000 7f1a8c21d740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x5590ea9fade0 con 0x5590eaa969e0 2024-03-19T01:40:42.722 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.714+0000 7f1a8c21d740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x5590ea9a3c80 con 0x5590eaa95ff0 2024-03-19T01:40:42.723 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.714+0000 7f1a8c21d740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x5590ea9f6e80 con 0x5590eaa9bb50 2024-03-19T01:40:42.723 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.714+0000 7f1a82ffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5590eaa969e0 0x5590eaa9b4e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:42.723 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.714+0000 7f1a827fc640 1 --2- 172.21.15.99:0/826672986 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f1a700a2800 0x7f1a700a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:42.723 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.714+0000 7f1a82ffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5590eaa969e0 0x5590eaa9b4e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58124/0 (socket says 172.21.15.99:58124) 2024-03-19T01:40:42.723 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.714+0000 7f1a82ffd640 1 -- 172.21.15.99:0/1194191327 learned_addr learned my addr 172.21.15.99:0/1194191327 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:42.723 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.714+0000 7f1a837fe640 1 --2- 172.21.15.99:0/1194191327 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5590eaa95ff0 0x5590eaa96410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:42.723 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.714+0000 7f1a837fe640 1 --2- 172.21.15.99:0/1194191327 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5590eaa9bb50 0x5590eaa9dfe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:42.723 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.714+0000 7f1a7bfff640 1 -- 172.21.15.99:0/826672986 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f1a6c0f0620 con 0x7f1a84101920 2024-03-19T01:40:42.723 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.714+0000 7f1a82ffd640 1 -- 172.21.15.99:0/1194191327 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5590eaa9bb50 msgr2=0x5590eaa9dfe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:40:42.723 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.714+0000 7f1a82ffd640 1 --2- 172.21.15.99:0/1194191327 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5590eaa9bb50 0x5590eaa9dfe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.723 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.714+0000 7f1a82ffd640 1 -- 172.21.15.99:0/1194191327 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5590eaa95ff0 msgr2=0x5590eaa96410 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:40:42.723 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.714+0000 7f1a82ffd640 1 --2- 172.21.15.99:0/1194191327 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5590eaa95ff0 0x5590eaa96410 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.723 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.714+0000 7f1a82ffd640 1 -- 172.21.15.99:0/1194191327 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5590ea9b5fb0 con 0x5590eaa969e0 2024-03-19T01:40:42.724 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.714+0000 7f1a837fe640 1 --2- 172.21.15.99:0/1194191327 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5590eaa95ff0 0x5590eaa96410 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:40:42.724 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.714+0000 7f1a827fc640 1 --2- 172.21.15.99:0/826672986 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f1a700a2800 0x7f1a700a4cc0 secure :-1 s=READY pgs=241 cs=0 l=1 rev1=1 crypto rx=0x7f1a6c0b7ad0 tx=0x7f1a6c088290 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:42.724 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.714+0000 7f1a82ffd640 1 --2- 172.21.15.99:0/1194191327 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5590eaa969e0 0x5590eaa9b4e0 secure :-1 s=READY pgs=333 cs=0 l=1 rev1=1 crypto rx=0x7f1a7c0114f0 tx=0x7f1a7c0119b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a5a54f3d79581761 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:42.724 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.714+0000 7f1a797fa640 1 -- 172.21.15.99:0/1194191327 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1a7c01e070 con 0x5590eaa969e0 2024-03-19T01:40:42.724 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.714+0000 7f1a797fa640 1 -- 172.21.15.99:0/1194191327 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f1a7c011250 con 0x5590eaa969e0 2024-03-19T01:40:42.724 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.714+0000 7f1a797fa640 1 -- 172.21.15.99:0/1194191327 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1a7c0198b0 con 0x5590eaa969e0 2024-03-19T01:40:42.724 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.715+0000 7f1a8c21d740 1 -- 172.21.15.99:0/1194191327 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5590eaa969e0 msgr2=0x5590eaa9b4e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:42.724 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.715+0000 7f1a8c21d740 1 --2- 172.21.15.99:0/1194191327 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5590eaa969e0 0x5590eaa9b4e0 secure :-1 s=READY pgs=333 cs=0 l=1 rev1=1 crypto rx=0x7f1a7c0114f0 tx=0x7f1a7c0119b0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.724 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.715+0000 7f1a8c21d740 1 -- 172.21.15.99:0/1194191327 shutdown_connections 2024-03-19T01:40:42.724 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.715+0000 7f1a8c21d740 1 --2- 172.21.15.99:0/1194191327 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5590eaa9bb50 0x5590eaa9dfe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.724 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.715+0000 7f1a8c21d740 1 --2- 172.21.15.99:0/1194191327 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5590eaa969e0 0x5590eaa9b4e0 unknown :-1 s=CLOSED pgs=333 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.724 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.715+0000 7f1a8c21d740 1 --2- 172.21.15.99:0/1194191327 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5590eaa95ff0 0x5590eaa96410 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.724 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.715+0000 7f1a8c21d740 1 -- 172.21.15.99:0/1194191327 >> 172.21.15.99:0/1194191327 conn(0x5590eaa91560 msgr2=0x5590eaa91960 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:42.725 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.715+0000 7f1a8c21d740 1 -- 172.21.15.99:0/1194191327 shutdown_connections 2024-03-19T01:40:42.725 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.715+0000 7f1a8c21d740 1 -- 172.21.15.99:0/1194191327 wait complete. 2024-03-19T01:40:42.725 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.715+0000 7f1a8c21d740 1 -- start start 2024-03-19T01:40:42.725 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.715+0000 7f1a8c21d740 1 Processor -- start 2024-03-19T01:40:42.725 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.716+0000 7f1a8c21d740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5590eaa95ff0 0x5590eaaa0530 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:42.725 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.716+0000 7f1a8c21d740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5590eaa969e0 0x5590eaaa0a70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:42.725 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.716+0000 7f1a8c21d740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5590eaa9bb50 0x5590eaaa7ab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:42.725 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.716+0000 7f1a8c21d740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x5590eaa9eb00 con 0x5590eaa9bb50 2024-03-19T01:40:42.725 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.716+0000 7f1a8c21d740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x5590ea9b6fc0 con 0x5590eaa969e0 2024-03-19T01:40:42.725 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.716+0000 7f1a8c21d740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x5590eaa9ec80 con 0x5590eaa95ff0 2024-03-19T01:40:42.725 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.716+0000 7f1a8c21d740 10 client.0 _ll_register_callbacks cb 0x5590ea9f2a20 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:40:42.725 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.716+0000 7f1a837fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5590eaa9bb50 0x5590eaaa7ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:42.725 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.716+0000 7f1a837fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5590eaa9bb50 0x5590eaaa7ab0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58126/0 (socket says 172.21.15.99:58126) 2024-03-19T01:40:42.726 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.716+0000 7f1a837fe640 1 -- 172.21.15.99:0/2289796208 learned_addr learned my addr 172.21.15.99:0/2289796208 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:42.726 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.716+0000 7f1a82ffd640 1 --2- 172.21.15.99:0/2289796208 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5590eaa969e0 0x5590eaaa0a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:42.726 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.716+0000 7f1a837fe640 1 --2- 172.21.15.99:0/2289796208 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5590eaa95ff0 0x5590eaaa0530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:42.726 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.716+0000 7f1a837fe640 1 -- 172.21.15.99:0/2289796208 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5590eaa95ff0 msgr2=0x5590eaaa0530 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:40:42.726 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.716+0000 7f1a837fe640 1 --2- 172.21.15.99:0/2289796208 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5590eaa95ff0 0x5590eaaa0530 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.726 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.716+0000 7f1a837fe640 1 -- 172.21.15.99:0/2289796208 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5590eaa969e0 msgr2=0x5590eaaa0a70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:40:42.726 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.716+0000 7f1a837fe640 1 --2- 172.21.15.99:0/2289796208 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5590eaa969e0 0x5590eaaa0a70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.726 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.716+0000 7f1a837fe640 1 -- 172.21.15.99:0/2289796208 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5590ea70f2d0 con 0x5590eaa9bb50 2024-03-19T01:40:42.726 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.716+0000 7f1a82ffd640 1 --2- 172.21.15.99:0/2289796208 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5590eaa969e0 0x5590eaaa0a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:40:42.726 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.716+0000 7f1a837fe640 1 --2- 172.21.15.99:0/2289796208 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5590eaa9bb50 0x5590eaaa7ab0 secure :-1 s=READY pgs=334 cs=0 l=1 rev1=1 crypto rx=0x7f1a7400fb40 tx=0x7f1a74005bd0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=24163cb7cf801bdb server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:42.726 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.716+0000 7f1a6a7fc640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:40:42.726 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.716+0000 7f1a6a7fc640 1 -- 172.21.15.99:0/2289796208 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1a74003e40 con 0x5590eaa9bb50 2024-03-19T01:40:42.726 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.716+0000 7f1a6a7fc640 1 -- 172.21.15.99:0/2289796208 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f1a74004b20 con 0x5590eaa9bb50 2024-03-19T01:40:42.727 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.716+0000 7f1a8c21d740 1 -- 172.21.15.99:0/2289796208 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f1a7c011250 con 0x5590eaa9bb50 2024-03-19T01:40:42.727 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.716+0000 7f1a6a7fc640 1 -- 172.21.15.99:0/2289796208 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=20) ==== 24+0+0 (secure 0 0 0) 0x7f1a7400e810 con 0x5590eaa9bb50 2024-03-19T01:40:42.727 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.716+0000 7f1a8c21d740 10 client.15246 fetch_fsmap learned FSMap version 20 2024-03-19T01:40:42.727 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.716+0000 7f1a8c21d740 1 -- 172.21.15.99:0/2289796208 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=20,monmap=4+}) -- 0x5590ea7d96b0 con 0x5590eaa9bb50 2024-03-19T01:40:42.727 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.717+0000 7f1a6a7fc640 1 -- 172.21.15.99:0/2289796208 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f1a7c011250 con 0x5590eaa9bb50 2024-03-19T01:40:42.727 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.717+0000 7f1a6a7fc640 1 -- 172.21.15.99:0/2289796208 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 20) ==== 8338+0+0 (secure 0 0 0) 0x7f1a740046c0 con 0x5590eaa9bb50 2024-03-19T01:40:42.727 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.717+0000 7f1a8c21d740 10 client.15246 fetch_fsmap finished waiting for FSMap version 20 2024-03-19T01:40:42.727 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.717+0000 7f1a8c21d740 10 client.15246 resolve_mds: resolved name 'c' to daemon mds.c 2024-03-19T01:40:42.727 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.717+0000 7f1a8c21d740 20 client.15246 populate_metadata read hostname 'smithi099' 2024-03-19T01:40:42.727 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.717+0000 7f1a8c21d740 1 --2- 172.21.15.99:0/2289796208 >> [v2:172.21.15.115:6836/1368988191,v1:172.21.15.115:6837/1368988191] conn(0x5590eaaad350 0x5590eaaae7c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:42.727 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.717+0000 7f1a8c21d740 4 client.15246 mds_command: new command op to 24472 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:40:42.727 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.717+0000 7f1a8c21d740 1 -- 172.21.15.99:0/2289796208 --> [v2:172.21.15.115:6836/1368988191,v1:172.21.15.115:6837/1368988191] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5590eaa08320 con 0x5590eaaad350 2024-03-19T01:40:42.727 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.717+0000 7f1a82ffd640 1 --2- 172.21.15.99:0/2289796208 >> [v2:172.21.15.115:6836/1368988191,v1:172.21.15.115:6837/1368988191] conn(0x5590eaaad350 0x5590eaaae7c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:42.727 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.717+0000 7f1a6a7fc640 10 client.15246 ms_handle_connect on v2:172.21.15.115:6836/1368988191 2024-03-19T01:40:42.728 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.717+0000 7f1a82ffd640 1 --2- 172.21.15.99:0/2289796208 >> [v2:172.21.15.115:6836/1368988191,v1:172.21.15.115:6837/1368988191] conn(0x5590eaaad350 0x5590eaaae7c0 crc :-1 s=READY pgs=17 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=40d8e869bda2741c server_cookie=4e00eee606861718 in_seq=0 out_seq=0 2024-03-19T01:40:42.728 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.718+0000 7f1a6a7fc640 1 -- 172.21.15.99:0/2289796208 <== mds.2 v2:172.21.15.115:6836/1368988191 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x5590eaa08320 con 0x5590eaaad350 2024-03-19T01:40:42.728 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.718+0000 7f1a6a7fc640 10 client.15246 handle_command_reply: tid=0 2024-03-19T01:40:42.728 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.718+0000 7f1a8c21d740 1 client.15246 shutdown 2024-03-19T01:40:42.728 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.718+0000 7f1a8c21d740 1 -- 172.21.15.99:0/2289796208 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5590eaa9bb50 msgr2=0x5590eaaa7ab0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:42.728 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.718+0000 7f1a8c21d740 1 --2- 172.21.15.99:0/2289796208 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5590eaa9bb50 0x5590eaaa7ab0 secure :-1 s=READY pgs=334 cs=0 l=1 rev1=1 crypto rx=0x7f1a7400fb40 tx=0x7f1a74005bd0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.728 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.718+0000 7f1a8c21d740 1 -- 172.21.15.99:0/2289796208 shutdown_connections 2024-03-19T01:40:42.728 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.718+0000 7f1a8c21d740 1 --2- 172.21.15.99:0/2289796208 >> [v2:172.21.15.115:6836/1368988191,v1:172.21.15.115:6837/1368988191] conn(0x5590eaaad350 0x5590eaaae7c0 crc :-1 s=READY pgs=17 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.728 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.718+0000 7f1a8c21d740 1 --2- 172.21.15.99:0/2289796208 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5590eaa969e0 0x5590eaaa0a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.728 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.718+0000 7f1a8c21d740 1 --2- 172.21.15.99:0/2289796208 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5590eaa9bb50 0x5590eaaa7ab0 unknown :-1 s=CLOSED pgs=334 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.728 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.718+0000 7f1a6a7fc640 0 client.15246 ms_handle_reset on v2:172.21.15.115:6836/1368988191 2024-03-19T01:40:42.728 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.718+0000 7f1a8c21d740 1 --2- 172.21.15.99:0/2289796208 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5590eaa95ff0 0x5590eaaa0530 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.728 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.718+0000 7f1a8c21d740 1 -- 172.21.15.99:0/2289796208 >> 172.21.15.99:0/2289796208 conn(0x5590eaa91560 msgr2=0x5590eaa907e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:42.728 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.718+0000 7f1a8c21d740 1 -- 172.21.15.99:0/2289796208 shutdown_connections 2024-03-19T01:40:42.728 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.718+0000 7f1a8c21d740 1 -- 172.21.15.99:0/2289796208 wait complete. 2024-03-19T01:40:42.729 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.718+0000 7f1a8c21d740 20 client.15246 trim_cache size 0 max 16384 2024-03-19T01:40:42.737 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.736+0000 7f1a8c21d740 1 Processor -- start 2024-03-19T01:40:42.737 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.736+0000 7f1a8c21d740 1 -- start start 2024-03-19T01:40:42.743 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.737+0000 7f1a8c21d740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5590eaa95ff0 0x5590eaa9e9e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:42.743 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.737+0000 7f1a8c21d740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5590eaa969e0 0x5590eaaa79d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:42.743 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.737+0000 7f1a8c21d740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5590eaa9bb50 0x5590eaaa0140 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:42.743 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.737+0000 7f1a8c21d740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x5590eaaacb00 con 0x5590eaa969e0 2024-03-19T01:40:42.743 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.737+0000 7f1a8c21d740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x5590eaaad1d0 con 0x5590eaa9bb50 2024-03-19T01:40:42.744 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.737+0000 7f1a8c21d740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x5590eaaad8a0 con 0x5590eaa95ff0 2024-03-19T01:40:42.744 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.737+0000 7f1a82ffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5590eaa969e0 0x5590eaaa79d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:42.744 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.737+0000 7f1a82ffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5590eaa969e0 0x5590eaaa79d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58128/0 (socket says 172.21.15.99:58128) 2024-03-19T01:40:42.744 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.737+0000 7f1a82ffd640 1 -- 172.21.15.99:0/2138023300 learned_addr learned my addr 172.21.15.99:0/2138023300 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:42.744 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.737+0000 7f1a837fe640 1 --2- 172.21.15.99:0/2138023300 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5590eaa95ff0 0x5590eaa9e9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:42.744 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.737+0000 7f1a837fe640 1 --2- 172.21.15.99:0/2138023300 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5590eaa9bb50 0x5590eaaa0140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:42.744 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.737+0000 7f1a82ffd640 1 -- 172.21.15.99:0/2138023300 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5590eaa95ff0 msgr2=0x5590eaa9e9e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:40:42.744 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.737+0000 7f1a82ffd640 1 --2- 172.21.15.99:0/2138023300 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5590eaa95ff0 0x5590eaa9e9e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.744 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.737+0000 7f1a82ffd640 1 -- 172.21.15.99:0/2138023300 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5590eaa9bb50 msgr2=0x5590eaaa0140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:40:42.744 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.737+0000 7f1a82ffd640 1 --2- 172.21.15.99:0/2138023300 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5590eaa9bb50 0x5590eaaa0140 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.744 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.737+0000 7f1a82ffd640 1 -- 172.21.15.99:0/2138023300 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5590eaaa1520 con 0x5590eaa969e0 2024-03-19T01:40:42.744 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.737+0000 7f1a82ffd640 1 --2- 172.21.15.99:0/2138023300 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5590eaa969e0 0x5590eaaa79d0 secure :-1 s=READY pgs=335 cs=0 l=1 rev1=1 crypto rx=0x7f1a7c01c040 tx=0x7f1a7c005550 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a891a76c42c96351 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:42.744 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.737+0000 7f1a78ff9640 1 -- 172.21.15.99:0/2138023300 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1a7c02b070 con 0x5590eaa969e0 2024-03-19T01:40:42.744 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.737+0000 7f1a78ff9640 1 -- 172.21.15.99:0/2138023300 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f1a7c01f040 con 0x5590eaa969e0 2024-03-19T01:40:42.745 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.737+0000 7f1a78ff9640 1 -- 172.21.15.99:0/2138023300 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1a7c01e040 con 0x5590eaa969e0 2024-03-19T01:40:42.745 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.738+0000 7f1a8c21d740 1 -- 172.21.15.99:0/2138023300 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5590eaa969e0 msgr2=0x5590eaaa79d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:42.745 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.738+0000 7f1a8c21d740 1 --2- 172.21.15.99:0/2138023300 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5590eaa969e0 0x5590eaaa79d0 secure :-1 s=READY pgs=335 cs=0 l=1 rev1=1 crypto rx=0x7f1a7c01c040 tx=0x7f1a7c005550 comp rx=0 tx=0).stop 2024-03-19T01:40:42.745 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.738+0000 7f1a8c21d740 1 -- 172.21.15.99:0/2138023300 shutdown_connections 2024-03-19T01:40:42.745 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.738+0000 7f1a8c21d740 1 --2- 172.21.15.99:0/2138023300 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5590eaa9bb50 0x5590eaaa0140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.745 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.738+0000 7f1a8c21d740 1 --2- 172.21.15.99:0/2138023300 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5590eaa969e0 0x5590eaaa79d0 unknown :-1 s=CLOSED pgs=335 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.745 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.738+0000 7f1a8c21d740 1 --2- 172.21.15.99:0/2138023300 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5590eaa95ff0 0x5590eaa9e9e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.745 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.738+0000 7f1a8c21d740 1 -- 172.21.15.99:0/2138023300 >> 172.21.15.99:0/2138023300 conn(0x5590eaa91560 msgr2=0x5590eaa91940 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:42.745 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.738+0000 7f1a8c21d740 1 -- 172.21.15.99:0/2138023300 shutdown_connections 2024-03-19T01:40:42.745 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.738+0000 7f1a8c21d740 1 -- 172.21.15.99:0/2138023300 wait complete. 2024-03-19T01:40:42.745 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.738+0000 7f1a8c21d740 1 -- start start 2024-03-19T01:40:42.745 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.738+0000 7f1a8c21d740 1 Processor -- start 2024-03-19T01:40:42.745 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.739+0000 7f1a8c21d740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5590eaa95ff0 0x5590eaaa3a40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:42.746 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.739+0000 7f1a8c21d740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5590eaa969e0 0x5590eaaa3f80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:42.746 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.739+0000 7f1a8c21d740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5590eaa9bb50 0x5590eaaa6fa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:42.746 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.739+0000 7f1a8c21d740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x5590eaaae640 con 0x5590eaa969e0 2024-03-19T01:40:42.746 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.739+0000 7f1a8c21d740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x5590eaaadf70 con 0x5590eaa95ff0 2024-03-19T01:40:42.746 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.739+0000 7f1a8c21d740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x5590eaaaed10 con 0x5590eaa9bb50 2024-03-19T01:40:42.746 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.739+0000 7f1a8c21d740 10 client.0 _ll_register_callbacks cb 0x7f1a74013c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:40:42.746 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.739+0000 7f1a82ffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5590eaa969e0 0x5590eaaa3f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:42.746 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.739+0000 7f1a82ffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5590eaa969e0 0x5590eaaa3f80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58144/0 (socket says 172.21.15.99:58144) 2024-03-19T01:40:42.746 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.739+0000 7f1a82ffd640 1 -- 172.21.15.99:0/155350990 learned_addr learned my addr 172.21.15.99:0/155350990 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:42.746 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.739+0000 7f1a837fe640 1 --2- 172.21.15.99:0/155350990 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5590eaa95ff0 0x5590eaaa3a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:42.746 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.739+0000 7f1a837fe640 1 --2- 172.21.15.99:0/155350990 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5590eaa9bb50 0x5590eaaa6fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:42.746 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.739+0000 7f1a82ffd640 1 -- 172.21.15.99:0/155350990 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5590eaa9bb50 msgr2=0x5590eaaa6fa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:40:42.746 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.739+0000 7f1a82ffd640 1 --2- 172.21.15.99:0/155350990 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5590eaa9bb50 0x5590eaaa6fa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.747 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.739+0000 7f1a82ffd640 1 -- 172.21.15.99:0/155350990 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5590eaa95ff0 msgr2=0x5590eaaa3a40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:40:42.747 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.739+0000 7f1a82ffd640 1 --2- 172.21.15.99:0/155350990 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5590eaa95ff0 0x5590eaaa3a40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.747 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.739+0000 7f1a82ffd640 1 -- 172.21.15.99:0/155350990 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5590eaaa7590 con 0x5590eaa969e0 2024-03-19T01:40:42.747 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.739+0000 7f1a82ffd640 1 --2- 172.21.15.99:0/155350990 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5590eaa969e0 0x5590eaaa3f80 secure :-1 s=READY pgs=336 cs=0 l=1 rev1=1 crypto rx=0x7f1a7c009230 tx=0x7f1a7c009530 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f9068f56d742c5b9 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:42.747 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.739+0000 7f1a6a7fc640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:40:42.747 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.739+0000 7f1a6a7fc640 1 -- 172.21.15.99:0/155350990 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1a7c01e070 con 0x5590eaa969e0 2024-03-19T01:40:42.747 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.739+0000 7f1a6a7fc640 1 -- 172.21.15.99:0/155350990 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f1a7c02c460 con 0x5590eaa969e0 2024-03-19T01:40:42.747 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.739+0000 7f1a8c21d740 1 -- 172.21.15.99:0/155350990 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f1a7c01f040 con 0x5590eaa969e0 2024-03-19T01:40:42.747 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.739+0000 7f1a6a7fc640 1 -- 172.21.15.99:0/155350990 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=20) ==== 24+0+0 (secure 0 0 0) 0x7f1a7c02b040 con 0x5590eaa969e0 2024-03-19T01:40:42.747 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.739+0000 7f1a8c21d740 10 client.15252 fetch_fsmap learned FSMap version 20 2024-03-19T01:40:42.747 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.739+0000 7f1a8c21d740 1 -- 172.21.15.99:0/155350990 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=20,monmap=4+}) -- 0x5590eab138e0 con 0x5590eaa969e0 2024-03-19T01:40:42.747 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.740+0000 7f1a6a7fc640 1 -- 172.21.15.99:0/155350990 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f1a7c01f040 con 0x5590eaa969e0 2024-03-19T01:40:42.747 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.740+0000 7f1a6a7fc640 1 -- 172.21.15.99:0/155350990 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 20) ==== 8338+0+0 (secure 0 0 0) 0x7f1a7c00f5c0 con 0x5590eaa969e0 2024-03-19T01:40:42.748 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.740+0000 7f1a8c21d740 10 client.15252 fetch_fsmap finished waiting for FSMap version 20 2024-03-19T01:40:42.748 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.740+0000 7f1a8c21d740 10 client.15252 resolve_mds: resolved name 'c' to daemon mds.c 2024-03-19T01:40:42.748 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.740+0000 7f1a8c21d740 20 client.15252 populate_metadata read hostname 'smithi099' 2024-03-19T01:40:42.748 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.740+0000 7f1a8c21d740 1 --2- 172.21.15.99:0/155350990 >> [v2:172.21.15.115:6836/1368988191,v1:172.21.15.115:6837/1368988191] conn(0x5590eab17e80 0x5590eab16b90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:42.748 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.740+0000 7f1a8c21d740 4 client.15252 mds_command: new command op to 24472 tid=0 multi_id=0 [{"prefix": "status"}] 2024-03-19T01:40:42.748 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.740+0000 7f1a8c21d740 1 -- 172.21.15.99:0/155350990 --> [v2:172.21.15.115:6836/1368988191,v1:172.21.15.115:6837/1368988191] -- command(tid 0: {"prefix": "status"}) -- 0x5590eaa02670 con 0x5590eab17e80 2024-03-19T01:40:42.748 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.740+0000 7f1a837fe640 1 --2- 172.21.15.99:0/155350990 >> [v2:172.21.15.115:6836/1368988191,v1:172.21.15.115:6837/1368988191] conn(0x5590eab17e80 0x5590eab16b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:42.748 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.740+0000 7f1a837fe640 1 --2- 172.21.15.99:0/155350990 >> [v2:172.21.15.115:6836/1368988191,v1:172.21.15.115:6837/1368988191] conn(0x5590eab17e80 0x5590eab16b90 crc :-1 s=READY pgs=18 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=50c9feddb7d05dfe server_cookie=8217047d61f5941d in_seq=0 out_seq=0 2024-03-19T01:40:42.748 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.740+0000 7f1a6a7fc640 10 client.15252 ms_handle_connect on v2:172.21.15.115:6836/1368988191 2024-03-19T01:40:42.748 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.740+0000 7f1a6a7fc640 1 -- 172.21.15.99:0/155350990 <== mds.2 v2:172.21.15.115:6836/1368988191 1 ==== command_reply(tid 0: 0 ) ==== 8+0+327 (crc 0 0 0) 0x5590eaa02670 con 0x5590eab17e80 2024-03-19T01:40:42.748 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.740+0000 7f1a6a7fc640 10 client.15252 handle_command_reply: tid=0 2024-03-19T01:40:42.748 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.740+0000 7f1a8c21d740 1 client.15252 shutdown 2024-03-19T01:40:42.748 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.741+0000 7f1a8c21d740 1 -- 172.21.15.99:0/155350990 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5590eaa969e0 msgr2=0x5590eaaa3f80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:42.748 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.741+0000 7f1a8c21d740 1 --2- 172.21.15.99:0/155350990 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5590eaa969e0 0x5590eaaa3f80 secure :-1 s=READY pgs=336 cs=0 l=1 rev1=1 crypto rx=0x7f1a7c009230 tx=0x7f1a7c009530 comp rx=0 tx=0).stop 2024-03-19T01:40:42.749 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.741+0000 7f1a8c21d740 1 -- 172.21.15.99:0/155350990 shutdown_connections 2024-03-19T01:40:42.749 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.741+0000 7f1a8c21d740 1 --2- 172.21.15.99:0/155350990 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5590eaa9bb50 0x5590eaaa6fa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.749 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.741+0000 7f1a8c21d740 1 --2- 172.21.15.99:0/155350990 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5590eaa969e0 0x5590eaaa3f80 unknown :-1 s=CLOSED pgs=336 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.749 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.741+0000 7f1a8c21d740 1 --2- 172.21.15.99:0/155350990 >> [v2:172.21.15.115:6836/1368988191,v1:172.21.15.115:6837/1368988191] conn(0x5590eab17e80 0x5590eab16b90 crc :-1 s=READY pgs=18 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.749 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.741+0000 7f1a8c21d740 1 --2- 172.21.15.99:0/155350990 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5590eaa95ff0 0x5590eaaa3a40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.749 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.741+0000 7f1a6a7fc640 0 client.15252 ms_handle_reset on v2:172.21.15.115:6836/1368988191 2024-03-19T01:40:42.749 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.741+0000 7f1a8c21d740 1 -- 172.21.15.99:0/155350990 >> 172.21.15.99:0/155350990 conn(0x5590eaa91560 msgr2=0x5590eaaa0140 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:42.749 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.741+0000 7f1a8c21d740 1 -- 172.21.15.99:0/155350990 shutdown_connections 2024-03-19T01:40:42.749 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.741+0000 7f1a8c21d740 1 -- 172.21.15.99:0/155350990 wait complete. 2024-03-19T01:40:42.749 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.741+0000 7f1a8c21d740 20 client.15252 trim_cache size 0 max 16384 2024-03-19T01:40:42.750 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2024-03-19T01:40:42.750 INFO:teuthology.orchestra.run.smithi099.stdout: "cluster_fsid": "438fb69c-e590-11ee-95c9-87774f69a715", 2024-03-19T01:40:42.750 INFO:teuthology.orchestra.run.smithi099.stdout: "whoami": 2, 2024-03-19T01:40:42.750 INFO:teuthology.orchestra.run.smithi099.stdout: "id": 24472, 2024-03-19T01:40:42.750 INFO:teuthology.orchestra.run.smithi099.stdout: "want_state": "up:active", 2024-03-19T01:40:42.750 INFO:teuthology.orchestra.run.smithi099.stdout: "state": "up:active", 2024-03-19T01:40:42.750 INFO:teuthology.orchestra.run.smithi099.stdout: "fs_name": "cephfs", 2024-03-19T01:40:42.750 INFO:teuthology.orchestra.run.smithi099.stdout: "rank_uptime": 102.933583188, 2024-03-19T01:40:42.750 INFO:teuthology.orchestra.run.smithi099.stdout: "mdsmap_epoch": 20, 2024-03-19T01:40:42.750 INFO:teuthology.orchestra.run.smithi099.stdout: "osdmap_epoch": 91, 2024-03-19T01:40:42.750 INFO:teuthology.orchestra.run.smithi099.stdout: "osdmap_epoch_barrier": 87, 2024-03-19T01:40:42.750 INFO:teuthology.orchestra.run.smithi099.stdout: "uptime": 126.07027386199999 2024-03-19T01:40:42.750 INFO:teuthology.orchestra.run.smithi099.stdout:} 2024-03-19T01:40:42.751 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.741+0000 7f1a89636640 1 -- 172.21.15.99:0/826672986 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f1a700a2800 msgr2=0x7f1a700a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:42.751 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.741+0000 7f1a89636640 1 --2- 172.21.15.99:0/826672986 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f1a700a2800 0x7f1a700a4cc0 secure :-1 s=READY pgs=241 cs=0 l=1 rev1=1 crypto rx=0x7f1a6c0b7ad0 tx=0x7f1a6c088290 comp rx=0 tx=0).stop 2024-03-19T01:40:42.751 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.741+0000 7f1a89636640 1 -- 172.21.15.99:0/826672986 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1a84101920 msgr2=0x7f1a8419c8b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:42.751 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.741+0000 7f1a89636640 1 --2- 172.21.15.99:0/826672986 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1a84101920 0x7f1a8419c8b0 secure :-1 s=READY pgs=332 cs=0 l=1 rev1=1 crypto rx=0x7f1a6c009230 tx=0x7f1a6c02c650 comp rx=0 tx=0).stop 2024-03-19T01:40:42.752 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.741+0000 7f1a89636640 1 -- 172.21.15.99:0/826672986 shutdown_connections 2024-03-19T01:40:42.752 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.741+0000 7f1a89636640 1 --2- 172.21.15.99:0/826672986 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1a841022e0 0x7f1a8419eed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.752 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.741+0000 7f1a89636640 1 --2- 172.21.15.99:0/826672986 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1a84101920 0x7f1a8419c8b0 unknown :-1 s=CLOSED pgs=332 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.752 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.741+0000 7f1a89636640 1 --2- 172.21.15.99:0/826672986 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f1a700a2800 0x7f1a700a4cc0 unknown :-1 s=CLOSED pgs=241 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.752 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.741+0000 7f1a89636640 1 --2- 172.21.15.99:0/826672986 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1a84100730 0x7f1a8406a900 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.752 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.741+0000 7f1a89636640 1 -- 172.21.15.99:0/826672986 >> 172.21.15.99:0/826672986 conn(0x7f1a840fbcc0 msgr2=0x7f1a84104ca0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:42.752 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.741+0000 7f1a89636640 1 -- 172.21.15.99:0/826672986 shutdown_connections 2024-03-19T01:40:42.752 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.741+0000 7f1a89636640 1 -- 172.21.15.99:0/826672986 wait complete. 2024-03-19T01:40:42.753 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.b status 2024-03-19T01:40:42.857 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.854+0000 7fc36e987640 1 -- 172.21.15.99:0/1342076757 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc3681022a0 msgr2=0x7fc368104690 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:42.858 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.854+0000 7fc36e987640 1 --2- 172.21.15.99:0/1342076757 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc3681022a0 0x7fc368104690 secure :-1 s=READY pgs=337 cs=0 l=1 rev1=1 crypto rx=0x7fc358009860 tx=0x7fc35802bb70 comp rx=0 tx=0).stop 2024-03-19T01:40:42.858 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.854+0000 7fc36e987640 1 -- 172.21.15.99:0/1342076757 shutdown_connections 2024-03-19T01:40:42.858 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.854+0000 7fc36e987640 1 --2- 172.21.15.99:0/1342076757 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fc368104bd0 0x7fc368106fc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.858 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.854+0000 7fc36e987640 1 --2- 172.21.15.99:0/1342076757 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fc368107500 0x7fc368109950 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.858 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.854+0000 7fc36e987640 1 --2- 172.21.15.99:0/1342076757 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc3681022a0 0x7fc368104690 unknown :-1 s=CLOSED pgs=337 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.858 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.854+0000 7fc36e987640 1 -- 172.21.15.99:0/1342076757 >> 172.21.15.99:0/1342076757 conn(0x7fc3680fbcb0 msgr2=0x7fc3680fe0d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:42.858 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.854+0000 7fc36e987640 1 -- 172.21.15.99:0/1342076757 shutdown_connections 2024-03-19T01:40:42.858 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.854+0000 7fc36e987640 1 -- 172.21.15.99:0/1342076757 wait complete. 2024-03-19T01:40:42.858 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.855+0000 7fc36e987640 1 Processor -- start 2024-03-19T01:40:42.858 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.855+0000 7fc36e987640 1 -- start start 2024-03-19T01:40:42.859 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.855+0000 7fc36e987640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc3681022a0 0x7fc368197b90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:42.859 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.855+0000 7fc36e987640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fc368104bd0 0x7fc3681980d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:42.859 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.855+0000 7fc36e987640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fc368107500 0x7fc36819f140 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:42.859 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.855+0000 7fc36e987640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fc36810d4c0 con 0x7fc3681022a0 2024-03-19T01:40:42.859 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.855+0000 7fc36e987640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fc36810d340 con 0x7fc368107500 2024-03-19T01:40:42.859 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.855+0000 7fc36e987640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fc36810d640 con 0x7fc368104bd0 2024-03-19T01:40:42.859 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.855+0000 7fc367fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc3681022a0 0x7fc368197b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:42.859 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.855+0000 7fc367fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc3681022a0 0x7fc368197b90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58172/0 (socket says 172.21.15.99:58172) 2024-03-19T01:40:42.859 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.855+0000 7fc367fff640 1 -- 172.21.15.99:0/106022312 learned_addr learned my addr 172.21.15.99:0/106022312 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:42.859 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.855+0000 7fc3677fe640 1 --2- 172.21.15.99:0/106022312 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fc368104bd0 0x7fc3681980d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:42.859 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.855+0000 7fc36cf24640 1 --2- 172.21.15.99:0/106022312 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fc368107500 0x7fc36819f140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:42.859 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.856+0000 7fc367fff640 1 -- 172.21.15.99:0/106022312 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fc368104bd0 msgr2=0x7fc3681980d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:42.859 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.856+0000 7fc367fff640 1 --2- 172.21.15.99:0/106022312 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fc368104bd0 0x7fc3681980d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.860 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.856+0000 7fc367fff640 1 -- 172.21.15.99:0/106022312 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fc368107500 msgr2=0x7fc36819f140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:42.860 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.856+0000 7fc367fff640 1 --2- 172.21.15.99:0/106022312 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fc368107500 0x7fc36819f140 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.860 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.856+0000 7fc367fff640 1 -- 172.21.15.99:0/106022312 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fc36819f7b0 con 0x7fc3681022a0 2024-03-19T01:40:42.860 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.856+0000 7fc367fff640 1 --2- 172.21.15.99:0/106022312 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc3681022a0 0x7fc368197b90 secure :-1 s=READY pgs=338 cs=0 l=1 rev1=1 crypto rx=0x7fc358005940 tx=0x7fc358032470 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:42.860 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.856+0000 7fc3657fa640 1 -- 172.21.15.99:0/106022312 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc358041020 con 0x7fc3681022a0 2024-03-19T01:40:42.860 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.856+0000 7fc36e987640 1 -- 172.21.15.99:0/106022312 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fc36819fa40 con 0x7fc3681022a0 2024-03-19T01:40:42.860 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.856+0000 7fc3657fa640 1 -- 172.21.15.99:0/106022312 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fc35802c9f0 con 0x7fc3681022a0 2024-03-19T01:40:42.860 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.856+0000 7fc3657fa640 1 -- 172.21.15.99:0/106022312 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc358039040 con 0x7fc3681022a0 2024-03-19T01:40:42.860 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.856+0000 7fc36e987640 1 -- 172.21.15.99:0/106022312 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fc36819ff30 con 0x7fc3681022a0 2024-03-19T01:40:42.867 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.858+0000 7fc3657fa640 1 -- 172.21.15.99:0/106022312 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 18) ==== 137290+0+0 (secure 0 0 0) 0x7fc358044020 con 0x7fc3681022a0 2024-03-19T01:40:42.867 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.859+0000 7fc371571740 1 Processor -- start 2024-03-19T01:40:42.867 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.859+0000 7fc371571740 1 -- start start 2024-03-19T01:40:42.867 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.859+0000 7fc371571740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a3119bdff0 0x55a3119be410 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:42.867 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.859+0000 7fc371571740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a3119be9e0 0x55a3119c34e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:42.867 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.859+0000 7fc371571740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a3119c3b50 0x55a3119c5fe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:42.867 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.859+0000 7fc371571740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55a311922de0 con 0x55a3119bdff0 2024-03-19T01:40:42.868 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.859+0000 7fc371571740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55a3118cbc80 con 0x55a3119be9e0 2024-03-19T01:40:42.868 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.859+0000 7fc371571740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55a31191ee80 con 0x55a3119c3b50 2024-03-19T01:40:42.868 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.859+0000 7fc3657fa640 1 --2- 172.21.15.99:0/106022312 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fc3500a2800 0x7fc3500a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:42.868 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.859+0000 7fc36cf24640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a3119bdff0 0x55a3119be410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:42.868 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.859+0000 7fc36cf24640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a3119bdff0 0x55a3119be410 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58188/0 (socket says 172.21.15.99:58188) 2024-03-19T01:40:42.868 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.859+0000 7fc36cf24640 1 -- 172.21.15.99:0/2534717171 learned_addr learned my addr 172.21.15.99:0/2534717171 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:42.868 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.859+0000 7fc367fff640 1 --2- 172.21.15.99:0/2534717171 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a3119be9e0 0x55a3119c34e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:42.868 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.859+0000 7fc36cf24640 1 --2- 172.21.15.99:0/106022312 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fc3500a2800 0x7fc3500a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:42.868 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.859+0000 7fc3677fe640 1 --2- 172.21.15.99:0/2534717171 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a3119c3b50 0x55a3119c5fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:42.868 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.859+0000 7fc3657fa640 1 -- 172.21.15.99:0/106022312 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7fc3580f0450 con 0x7fc3681022a0 2024-03-19T01:40:42.868 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.859+0000 7fc36cf24640 1 -- 172.21.15.99:0/2534717171 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a3119c3b50 msgr2=0x55a3119c5fe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:40:42.868 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.859+0000 7fc36cf24640 1 --2- 172.21.15.99:0/2534717171 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a3119c3b50 0x55a3119c5fe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.868 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.859+0000 7fc36cf24640 1 -- 172.21.15.99:0/2534717171 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a3119be9e0 msgr2=0x55a3119c34e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:40:42.868 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.859+0000 7fc36cf24640 1 --2- 172.21.15.99:0/2534717171 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a3119be9e0 0x55a3119c34e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.869 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.859+0000 7fc36cf24640 1 -- 172.21.15.99:0/2534717171 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55a3118ddfb0 con 0x55a3119bdff0 2024-03-19T01:40:42.869 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.860+0000 7fc36cf24640 1 --2- 172.21.15.99:0/2534717171 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a3119bdff0 0x55a3119be410 secure :-1 s=READY pgs=339 cs=0 l=1 rev1=1 crypto rx=0x7fc360011e20 tx=0x7fc36000f790 comp rx=0 tx=0).ready entity=mon.0 client_cookie=fcc4779399ff2232 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:42.869 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.860+0000 7fc36cf24640 1 --2- 172.21.15.99:0/106022312 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fc3500a2800 0x7fc3500a4cc0 secure :-1 s=READY pgs=242 cs=0 l=1 rev1=1 crypto rx=0x7fc36000fc90 tx=0x7fc36001e000 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:42.869 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.860+0000 7fc34e7fc640 1 -- 172.21.15.99:0/2534717171 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc36001dcb0 con 0x55a3119bdff0 2024-03-19T01:40:42.869 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.860+0000 7fc34e7fc640 1 -- 172.21.15.99:0/2534717171 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fc36001de50 con 0x55a3119bdff0 2024-03-19T01:40:42.869 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.860+0000 7fc34e7fc640 1 -- 172.21.15.99:0/2534717171 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc360004280 con 0x55a3119bdff0 2024-03-19T01:40:42.869 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.860+0000 7fc371571740 1 -- 172.21.15.99:0/2534717171 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a3119bdff0 msgr2=0x55a3119be410 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:42.869 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.860+0000 7fc371571740 1 --2- 172.21.15.99:0/2534717171 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a3119bdff0 0x55a3119be410 secure :-1 s=READY pgs=339 cs=0 l=1 rev1=1 crypto rx=0x7fc360011e20 tx=0x7fc36000f790 comp rx=0 tx=0).stop 2024-03-19T01:40:42.869 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.860+0000 7fc371571740 1 -- 172.21.15.99:0/2534717171 shutdown_connections 2024-03-19T01:40:42.869 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.860+0000 7fc371571740 1 --2- 172.21.15.99:0/2534717171 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a3119be9e0 0x55a3119c34e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.869 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.860+0000 7fc371571740 1 --2- 172.21.15.99:0/2534717171 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a3119c3b50 0x55a3119c5fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.869 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.860+0000 7fc371571740 1 --2- 172.21.15.99:0/2534717171 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a3119bdff0 0x55a3119be410 unknown :-1 s=CLOSED pgs=339 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.869 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.860+0000 7fc371571740 1 -- 172.21.15.99:0/2534717171 >> 172.21.15.99:0/2534717171 conn(0x55a3119b9560 msgr2=0x55a3119b9960 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:42.869 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.860+0000 7fc371571740 1 -- 172.21.15.99:0/2534717171 shutdown_connections 2024-03-19T01:40:42.870 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.860+0000 7fc371571740 1 -- 172.21.15.99:0/2534717171 wait complete. 2024-03-19T01:40:42.870 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.861+0000 7fc371571740 1 -- start start 2024-03-19T01:40:42.870 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.861+0000 7fc371571740 1 Processor -- start 2024-03-19T01:40:42.870 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.861+0000 7fc371571740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a3119bdff0 0x55a3119c8560 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:42.870 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.861+0000 7fc371571740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a3119be9e0 0x55a3119c8aa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:42.870 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.861+0000 7fc371571740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a3119c3b50 0x55a3119cfae0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:42.870 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.861+0000 7fc371571740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55a3119c6a40 con 0x55a3119be9e0 2024-03-19T01:40:42.870 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.861+0000 7fc371571740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55a3118defc0 con 0x55a3119c3b50 2024-03-19T01:40:42.870 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.861+0000 7fc371571740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55a3119c6bc0 con 0x55a3119bdff0 2024-03-19T01:40:42.870 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.861+0000 7fc371571740 10 client.0 _ll_register_callbacks cb 0x55a31191aa20 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:40:42.870 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.861+0000 7fc3677fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a3119be9e0 0x55a3119c8aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:42.870 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.861+0000 7fc3677fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a3119be9e0 0x55a3119c8aa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58200/0 (socket says 172.21.15.99:58200) 2024-03-19T01:40:42.870 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.861+0000 7fc3677fe640 1 -- 172.21.15.99:0/2861861919 learned_addr learned my addr 172.21.15.99:0/2861861919 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:42.870 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.861+0000 7fc367fff640 1 --2- 172.21.15.99:0/2861861919 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a3119bdff0 0x55a3119c8560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:42.871 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.861+0000 7fc36cf24640 1 --2- 172.21.15.99:0/2861861919 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a3119c3b50 0x55a3119cfae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:42.871 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.861+0000 7fc3677fe640 1 -- 172.21.15.99:0/2861861919 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a3119bdff0 msgr2=0x55a3119c8560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:40:42.871 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.861+0000 7fc3677fe640 1 --2- 172.21.15.99:0/2861861919 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a3119bdff0 0x55a3119c8560 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.871 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.861+0000 7fc3677fe640 1 -- 172.21.15.99:0/2861861919 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a3119c3b50 msgr2=0x55a3119cfae0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:40:42.871 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.861+0000 7fc3677fe640 1 --2- 172.21.15.99:0/2861861919 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a3119c3b50 0x55a3119cfae0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.871 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.861+0000 7fc3677fe640 1 -- 172.21.15.99:0/2861861919 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55a3116372d0 con 0x55a3119be9e0 2024-03-19T01:40:42.871 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.862+0000 7fc3677fe640 1 --2- 172.21.15.99:0/2861861919 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a3119be9e0 0x55a3119c8aa0 secure :-1 s=READY pgs=340 cs=0 l=1 rev1=1 crypto rx=0x7fc35c0116a0 tx=0x7fc35c011b60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=68d49cd219b67e7a server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:42.871 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.862+0000 7fc3477fe640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:40:42.871 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.862+0000 7fc3477fe640 1 -- 172.21.15.99:0/2861861919 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc35c01e060 con 0x55a3119be9e0 2024-03-19T01:40:42.871 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.862+0000 7fc3477fe640 1 -- 172.21.15.99:0/2861861919 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7fc35c00fb90 con 0x55a3119be9e0 2024-03-19T01:40:42.871 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.862+0000 7fc371571740 1 -- 172.21.15.99:0/2861861919 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fc36001de50 con 0x55a3119be9e0 2024-03-19T01:40:42.871 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.862+0000 7fc3477fe640 1 -- 172.21.15.99:0/2861861919 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=20) ==== 24+0+0 (secure 0 0 0) 0x7fc35c00fdd0 con 0x55a3119be9e0 2024-03-19T01:40:42.871 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.862+0000 7fc371571740 10 client.15264 fetch_fsmap learned FSMap version 20 2024-03-19T01:40:42.871 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.862+0000 7fc371571740 1 -- 172.21.15.99:0/2861861919 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=20,monmap=4+}) -- 0x55a3117016b0 con 0x55a3119be9e0 2024-03-19T01:40:42.872 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.862+0000 7fc3477fe640 1 -- 172.21.15.99:0/2861861919 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fc36001de50 con 0x55a3119be9e0 2024-03-19T01:40:42.872 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.862+0000 7fc3477fe640 1 -- 172.21.15.99:0/2861861919 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 20) ==== 8338+0+0 (secure 0 0 0) 0x7fc35c019070 con 0x55a3119be9e0 2024-03-19T01:40:42.872 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.862+0000 7fc371571740 10 client.15264 fetch_fsmap finished waiting for FSMap version 20 2024-03-19T01:40:42.872 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.862+0000 7fc371571740 10 client.15264 resolve_mds: resolved name 'b' to daemon mds.b 2024-03-19T01:40:42.872 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.862+0000 7fc371571740 20 client.15264 populate_metadata read hostname 'smithi099' 2024-03-19T01:40:42.872 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.862+0000 7fc371571740 1 --2- 172.21.15.99:0/2861861919 >> [v2:172.21.15.112:6838/986249680,v1:172.21.15.112:6839/986249680] conn(0x55a3119d5440 0x55a3119d68b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:42.872 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.862+0000 7fc371571740 4 client.15264 mds_command: new command op to 24479 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:40:42.872 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.862+0000 7fc371571740 1 -- 172.21.15.99:0/2861861919 --> [v2:172.21.15.112:6838/986249680,v1:172.21.15.112:6839/986249680] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55a311930320 con 0x55a3119d5440 2024-03-19T01:40:42.872 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.863+0000 7fc367fff640 1 --2- 172.21.15.99:0/2861861919 >> [v2:172.21.15.112:6838/986249680,v1:172.21.15.112:6839/986249680] conn(0x55a3119d5440 0x55a3119d68b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:42.872 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.863+0000 7fc3477fe640 10 client.15264 ms_handle_connect on v2:172.21.15.112:6838/986249680 2024-03-19T01:40:42.872 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.863+0000 7fc367fff640 1 --2- 172.21.15.99:0/2861861919 >> [v2:172.21.15.112:6838/986249680,v1:172.21.15.112:6839/986249680] conn(0x55a3119d5440 0x55a3119d68b0 crc :-1 s=READY pgs=18 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=6c5eb4c049e036e7 server_cookie=4b37d34e4f45524c in_seq=0 out_seq=0 2024-03-19T01:40:42.872 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.863+0000 7fc3477fe640 1 -- 172.21.15.99:0/2861861919 <== mds.1 v2:172.21.15.112:6838/986249680 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x55a311930320 con 0x55a3119d5440 2024-03-19T01:40:42.872 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.863+0000 7fc3477fe640 10 client.15264 handle_command_reply: tid=0 2024-03-19T01:40:42.872 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.863+0000 7fc371571740 1 client.15264 shutdown 2024-03-19T01:40:42.872 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.864+0000 7fc371571740 1 -- 172.21.15.99:0/2861861919 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a3119be9e0 msgr2=0x55a3119c8aa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:42.873 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.864+0000 7fc371571740 1 --2- 172.21.15.99:0/2861861919 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a3119be9e0 0x55a3119c8aa0 secure :-1 s=READY pgs=340 cs=0 l=1 rev1=1 crypto rx=0x7fc35c0116a0 tx=0x7fc35c011b60 comp rx=0 tx=0).stop 2024-03-19T01:40:42.873 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.864+0000 7fc371571740 1 -- 172.21.15.99:0/2861861919 shutdown_connections 2024-03-19T01:40:42.873 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.864+0000 7fc371571740 1 --2- 172.21.15.99:0/2861861919 >> [v2:172.21.15.112:6838/986249680,v1:172.21.15.112:6839/986249680] conn(0x55a3119d5440 0x55a3119d68b0 crc :-1 s=READY pgs=18 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.873 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.864+0000 7fc371571740 1 --2- 172.21.15.99:0/2861861919 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a3119c3b50 0x55a3119cfae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.873 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.864+0000 7fc371571740 1 --2- 172.21.15.99:0/2861861919 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a3119be9e0 0x55a3119c8aa0 unknown :-1 s=CLOSED pgs=340 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.873 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.864+0000 7fc3477fe640 0 client.15264 ms_handle_reset on v2:172.21.15.112:6838/986249680 2024-03-19T01:40:42.873 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.864+0000 7fc371571740 1 --2- 172.21.15.99:0/2861861919 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a3119bdff0 0x55a3119c8560 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.873 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.864+0000 7fc371571740 1 -- 172.21.15.99:0/2861861919 >> 172.21.15.99:0/2861861919 conn(0x55a3119b9560 msgr2=0x55a3119b87e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:42.873 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.864+0000 7fc371571740 1 -- 172.21.15.99:0/2861861919 shutdown_connections 2024-03-19T01:40:42.873 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.864+0000 7fc371571740 1 -- 172.21.15.99:0/2861861919 wait complete. 2024-03-19T01:40:42.873 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.864+0000 7fc371571740 20 client.15264 trim_cache size 0 max 16384 2024-03-19T01:40:42.888 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.881+0000 7fc371571740 1 Processor -- start 2024-03-19T01:40:42.888 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.881+0000 7fc371571740 1 -- start start 2024-03-19T01:40:42.888 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.881+0000 7fc371571740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a3119bdff0 0x55a3119b9940 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:42.888 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.881+0000 7fc371571740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a3119be9e0 0x55a3119b7f50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:42.888 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.881+0000 7fc371571740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a3119c3b50 0x55a3119b8490 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:42.888 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.881+0000 7fc371571740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55a3119d4bf0 con 0x55a3119c3b50 2024-03-19T01:40:42.888 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.881+0000 7fc371571740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55a3119d52c0 con 0x55a3119bdff0 2024-03-19T01:40:42.888 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.881+0000 7fc371571740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55a3119d5990 con 0x55a3119be9e0 2024-03-19T01:40:42.888 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.881+0000 7fc36cf24640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a3119c3b50 0x55a3119b8490 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:42.889 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.881+0000 7fc36cf24640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a3119c3b50 0x55a3119b8490 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58214/0 (socket says 172.21.15.99:58214) 2024-03-19T01:40:42.889 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.882+0000 7fc36cf24640 1 -- 172.21.15.99:0/2254023008 learned_addr learned my addr 172.21.15.99:0/2254023008 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:42.889 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.882+0000 7fc3677fe640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a3119be9e0 0x55a3119b7f50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:42.889 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.882+0000 7fc367fff640 1 --2- 172.21.15.99:0/2254023008 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a3119bdff0 0x55a3119b9940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:42.889 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.882+0000 7fc36cf24640 1 -- 172.21.15.99:0/2254023008 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a3119be9e0 msgr2=0x55a3119b7f50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:40:42.889 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.882+0000 7fc36cf24640 1 --2- 172.21.15.99:0/2254023008 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a3119be9e0 0x55a3119b7f50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.889 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.882+0000 7fc36cf24640 1 -- 172.21.15.99:0/2254023008 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a3119bdff0 msgr2=0x55a3119b9940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:40:42.889 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.882+0000 7fc36cf24640 1 --2- 172.21.15.99:0/2254023008 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a3119bdff0 0x55a3119b9940 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.889 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.882+0000 7fc36cf24640 1 -- 172.21.15.99:0/2254023008 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55a3119c96e0 con 0x55a3119c3b50 2024-03-19T01:40:42.889 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.882+0000 7fc36cf24640 1 --2- 172.21.15.99:0/2254023008 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a3119c3b50 0x55a3119b8490 secure :-1 s=READY pgs=341 cs=0 l=1 rev1=1 crypto rx=0x55a3119bf240 tx=0x7fc360005bd0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=88d8a9f2ed9ad128 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:42.889 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.882+0000 7fc34dffb640 1 -- 172.21.15.99:0/2254023008 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc36002a040 con 0x55a3119c3b50 2024-03-19T01:40:42.889 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.882+0000 7fc34dffb640 1 -- 172.21.15.99:0/2254023008 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fc36002b040 con 0x55a3119c3b50 2024-03-19T01:40:42.889 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.882+0000 7fc34dffb640 1 -- 172.21.15.99:0/2254023008 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc360005040 con 0x55a3119c3b50 2024-03-19T01:40:42.889 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.883+0000 7fc371571740 1 -- 172.21.15.99:0/2254023008 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a3119c3b50 msgr2=0x55a3119b8490 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:42.890 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.883+0000 7fc371571740 1 --2- 172.21.15.99:0/2254023008 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a3119c3b50 0x55a3119b8490 secure :-1 s=READY pgs=341 cs=0 l=1 rev1=1 crypto rx=0x55a3119bf240 tx=0x7fc360005bd0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.890 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.883+0000 7fc371571740 1 -- 172.21.15.99:0/2254023008 shutdown_connections 2024-03-19T01:40:42.890 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.883+0000 7fc371571740 1 --2- 172.21.15.99:0/2254023008 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a3119c3b50 0x55a3119b8490 unknown :-1 s=CLOSED pgs=341 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.890 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.883+0000 7fc371571740 1 --2- 172.21.15.99:0/2254023008 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a3119be9e0 0x55a3119b7f50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.890 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.883+0000 7fc371571740 1 --2- 172.21.15.99:0/2254023008 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a3119bdff0 0x55a3119b9940 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.890 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.883+0000 7fc371571740 1 -- 172.21.15.99:0/2254023008 >> 172.21.15.99:0/2254023008 conn(0x55a3119b9560 msgr2=0x55a3119bfff0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:42.890 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.883+0000 7fc371571740 1 -- 172.21.15.99:0/2254023008 shutdown_connections 2024-03-19T01:40:42.890 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.883+0000 7fc371571740 1 -- 172.21.15.99:0/2254023008 wait complete. 2024-03-19T01:40:42.890 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.883+0000 7fc371571740 1 -- start start 2024-03-19T01:40:42.890 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.883+0000 7fc371571740 1 Processor -- start 2024-03-19T01:40:42.890 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.883+0000 7fc371571740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a3119bdff0 0x55a311a36690 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:42.891 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.883+0000 7fc371571740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a3119be9e0 0x55a311a36bd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:42.891 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.883+0000 7fc371571740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a3119c3b50 0x55a311a3db30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:42.891 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.883+0000 7fc371571740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55a3119d6730 con 0x55a3119be9e0 2024-03-19T01:40:42.891 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.883+0000 7fc371571740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55a3119d6060 con 0x55a3119bdff0 2024-03-19T01:40:42.891 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.883+0000 7fc371571740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55a3119d6e00 con 0x55a3119c3b50 2024-03-19T01:40:42.891 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.883+0000 7fc371571740 10 client.0 _ll_register_callbacks cb 0x7fc35c009870 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:40:42.891 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.883+0000 7fc3677fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a3119be9e0 0x55a311a36bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:42.891 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.883+0000 7fc3677fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a3119be9e0 0x55a311a36bd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58224/0 (socket says 172.21.15.99:58224) 2024-03-19T01:40:42.891 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.883+0000 7fc3677fe640 1 -- 172.21.15.99:0/4044139948 learned_addr learned my addr 172.21.15.99:0/4044139948 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:42.891 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.884+0000 7fc367fff640 1 --2- 172.21.15.99:0/4044139948 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a3119bdff0 0x55a311a36690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:42.891 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.884+0000 7fc36cf24640 1 --2- 172.21.15.99:0/4044139948 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a3119c3b50 0x55a311a3db30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:42.891 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.884+0000 7fc3677fe640 1 -- 172.21.15.99:0/4044139948 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a3119c3b50 msgr2=0x55a311a3db30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:40:42.891 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.884+0000 7fc3677fe640 1 --2- 172.21.15.99:0/4044139948 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a3119c3b50 0x55a311a3db30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.891 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.884+0000 7fc3677fe640 1 -- 172.21.15.99:0/4044139948 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a3119bdff0 msgr2=0x55a311a36690 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:40:42.892 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.884+0000 7fc3677fe640 1 --2- 172.21.15.99:0/4044139948 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a3119bdff0 0x55a311a36690 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.892 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.884+0000 7fc3677fe640 1 -- 172.21.15.99:0/4044139948 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55a311a3e120 con 0x55a3119be9e0 2024-03-19T01:40:42.892 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.884+0000 7fc3677fe640 1 --2- 172.21.15.99:0/4044139948 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a3119be9e0 0x55a311a36bd0 secure :-1 s=READY pgs=342 cs=0 l=1 rev1=1 crypto rx=0x55a3119c9a40 tx=0x7fc35c009e10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c05b92cd769aac0e server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:42.892 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.884+0000 7fc3477fe640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:40:42.892 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.884+0000 7fc3477fe640 1 -- 172.21.15.99:0/4044139948 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc35c032060 con 0x55a3119be9e0 2024-03-19T01:40:42.892 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.884+0000 7fc3477fe640 1 -- 172.21.15.99:0/4044139948 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7fc35c005250 con 0x55a3119be9e0 2024-03-19T01:40:42.892 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.884+0000 7fc371571740 1 -- 172.21.15.99:0/4044139948 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fc36002b040 con 0x55a3119be9e0 2024-03-19T01:40:42.892 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.884+0000 7fc3477fe640 1 -- 172.21.15.99:0/4044139948 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=20) ==== 24+0+0 (secure 0 0 0) 0x7fc35c009e80 con 0x55a3119be9e0 2024-03-19T01:40:42.892 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.884+0000 7fc371571740 10 client.15270 fetch_fsmap learned FSMap version 20 2024-03-19T01:40:42.892 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.884+0000 7fc371571740 1 -- 172.21.15.99:0/4044139948 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=20,monmap=4+}) -- 0x55a311a3f4c0 con 0x55a3119be9e0 2024-03-19T01:40:42.892 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.884+0000 7fc3477fe640 1 -- 172.21.15.99:0/4044139948 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fc36002b040 con 0x55a3119be9e0 2024-03-19T01:40:42.892 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.884+0000 7fc3477fe640 1 -- 172.21.15.99:0/4044139948 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 20) ==== 8338+0+0 (secure 0 0 0) 0x7fc35c019040 con 0x55a3119be9e0 2024-03-19T01:40:42.892 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.885+0000 7fc371571740 10 client.15270 fetch_fsmap finished waiting for FSMap version 20 2024-03-19T01:40:42.893 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.885+0000 7fc371571740 10 client.15270 resolve_mds: resolved name 'b' to daemon mds.b 2024-03-19T01:40:42.893 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.885+0000 7fc371571740 20 client.15270 populate_metadata read hostname 'smithi099' 2024-03-19T01:40:42.893 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.885+0000 7fc371571740 1 --2- 172.21.15.99:0/4044139948 >> [v2:172.21.15.112:6838/986249680,v1:172.21.15.112:6839/986249680] conn(0x55a311a440d0 0x55a311a42de0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:42.893 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.885+0000 7fc371571740 4 client.15270 mds_command: new command op to 24479 tid=0 multi_id=0 [{"prefix": "status"}] 2024-03-19T01:40:42.893 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.885+0000 7fc371571740 1 -- 172.21.15.99:0/4044139948 --> [v2:172.21.15.112:6838/986249680,v1:172.21.15.112:6839/986249680] -- command(tid 0: {"prefix": "status"}) -- 0x55a31192a670 con 0x55a311a440d0 2024-03-19T01:40:42.893 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.885+0000 7fc367fff640 1 --2- 172.21.15.99:0/4044139948 >> [v2:172.21.15.112:6838/986249680,v1:172.21.15.112:6839/986249680] conn(0x55a311a440d0 0x55a311a42de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:42.893 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.885+0000 7fc3477fe640 10 client.15270 ms_handle_connect on v2:172.21.15.112:6838/986249680 2024-03-19T01:40:42.893 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.885+0000 7fc367fff640 1 --2- 172.21.15.99:0/4044139948 >> [v2:172.21.15.112:6838/986249680,v1:172.21.15.112:6839/986249680] conn(0x55a311a440d0 0x55a311a42de0 crc :-1 s=READY pgs=19 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=c0553220b14a74ab server_cookie=8b51acaa90f9833c in_seq=0 out_seq=0 2024-03-19T01:40:42.893 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.885+0000 7fc3477fe640 1 -- 172.21.15.99:0/4044139948 <== mds.1 v2:172.21.15.112:6838/986249680 1 ==== command_reply(tid 0: 0 ) ==== 8+0+322 (crc 0 0 0) 0x55a31192a670 con 0x55a311a440d0 2024-03-19T01:40:42.893 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.885+0000 7fc3477fe640 10 client.15270 handle_command_reply: tid=0 2024-03-19T01:40:42.893 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.885+0000 7fc371571740 1 client.15270 shutdown 2024-03-19T01:40:42.893 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.885+0000 7fc371571740 1 -- 172.21.15.99:0/4044139948 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a3119be9e0 msgr2=0x55a311a36bd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:42.893 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.885+0000 7fc371571740 1 --2- 172.21.15.99:0/4044139948 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a3119be9e0 0x55a311a36bd0 secure :-1 s=READY pgs=342 cs=0 l=1 rev1=1 crypto rx=0x55a3119c9a40 tx=0x7fc35c009e10 comp rx=0 tx=0).stop 2024-03-19T01:40:42.893 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.885+0000 7fc371571740 1 -- 172.21.15.99:0/4044139948 shutdown_connections 2024-03-19T01:40:42.894 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.885+0000 7fc371571740 1 --2- 172.21.15.99:0/4044139948 >> [v2:172.21.15.112:6838/986249680,v1:172.21.15.112:6839/986249680] conn(0x55a311a440d0 0x55a311a42de0 crc :-1 s=READY pgs=19 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.894 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.885+0000 7fc371571740 1 --2- 172.21.15.99:0/4044139948 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a3119c3b50 0x55a311a3db30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.894 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.885+0000 7fc371571740 1 --2- 172.21.15.99:0/4044139948 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a3119be9e0 0x55a311a36bd0 unknown :-1 s=CLOSED pgs=342 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.894 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.885+0000 7fc3477fe640 0 client.15270 ms_handle_reset on v2:172.21.15.112:6838/986249680 2024-03-19T01:40:42.894 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.885+0000 7fc371571740 1 --2- 172.21.15.99:0/4044139948 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a3119bdff0 0x55a311a36690 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.894 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.885+0000 7fc371571740 1 -- 172.21.15.99:0/4044139948 >> 172.21.15.99:0/4044139948 conn(0x55a3119b9560 msgr2=0x55a3119bc7c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:42.894 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.886+0000 7fc371571740 1 -- 172.21.15.99:0/4044139948 shutdown_connections 2024-03-19T01:40:42.894 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.886+0000 7fc371571740 1 -- 172.21.15.99:0/4044139948 wait complete. 2024-03-19T01:40:42.894 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.886+0000 7fc371571740 20 client.15270 trim_cache size 0 max 16384 2024-03-19T01:40:42.894 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2024-03-19T01:40:42.895 INFO:teuthology.orchestra.run.smithi099.stdout: "cluster_fsid": "438fb69c-e590-11ee-95c9-87774f69a715", 2024-03-19T01:40:42.895 INFO:teuthology.orchestra.run.smithi099.stdout: "whoami": 1, 2024-03-19T01:40:42.895 INFO:teuthology.orchestra.run.smithi099.stdout: "id": 24479, 2024-03-19T01:40:42.895 INFO:teuthology.orchestra.run.smithi099.stdout: "want_state": "up:active", 2024-03-19T01:40:42.895 INFO:teuthology.orchestra.run.smithi099.stdout: "state": "up:active", 2024-03-19T01:40:42.895 INFO:teuthology.orchestra.run.smithi099.stdout: "fs_name": "cephfs", 2024-03-19T01:40:42.895 INFO:teuthology.orchestra.run.smithi099.stdout: "rank_uptime": 104.085002708, 2024-03-19T01:40:42.895 INFO:teuthology.orchestra.run.smithi099.stdout: "mdsmap_epoch": 20, 2024-03-19T01:40:42.895 INFO:teuthology.orchestra.run.smithi099.stdout: "osdmap_epoch": 91, 2024-03-19T01:40:42.895 INFO:teuthology.orchestra.run.smithi099.stdout: "osdmap_epoch_barrier": 87, 2024-03-19T01:40:42.895 INFO:teuthology.orchestra.run.smithi099.stdout: "uptime": 112.877817689 2024-03-19T01:40:42.895 INFO:teuthology.orchestra.run.smithi099.stdout:} 2024-03-19T01:40:42.896 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.886+0000 7fc36e987640 1 -- 172.21.15.99:0/106022312 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fc3500a2800 msgr2=0x7fc3500a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:42.896 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.886+0000 7fc36e987640 1 --2- 172.21.15.99:0/106022312 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fc3500a2800 0x7fc3500a4cc0 secure :-1 s=READY pgs=242 cs=0 l=1 rev1=1 crypto rx=0x7fc36000fc90 tx=0x7fc36001e000 comp rx=0 tx=0).stop 2024-03-19T01:40:42.896 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.886+0000 7fc36e987640 1 -- 172.21.15.99:0/106022312 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc3681022a0 msgr2=0x7fc368197b90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:42.896 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.886+0000 7fc36e987640 1 --2- 172.21.15.99:0/106022312 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc3681022a0 0x7fc368197b90 secure :-1 s=READY pgs=338 cs=0 l=1 rev1=1 crypto rx=0x7fc358005940 tx=0x7fc358032470 comp rx=0 tx=0).stop 2024-03-19T01:40:42.896 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.886+0000 7fc36e987640 1 -- 172.21.15.99:0/106022312 shutdown_connections 2024-03-19T01:40:42.896 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.886+0000 7fc36e987640 1 --2- 172.21.15.99:0/106022312 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fc3500a2800 0x7fc3500a4cc0 unknown :-1 s=CLOSED pgs=242 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.897 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.886+0000 7fc36e987640 1 --2- 172.21.15.99:0/106022312 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fc368107500 0x7fc36819f140 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.897 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.886+0000 7fc36e987640 1 --2- 172.21.15.99:0/106022312 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fc368104bd0 0x7fc3681980d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.897 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.886+0000 7fc36e987640 1 --2- 172.21.15.99:0/106022312 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc3681022a0 0x7fc368197b90 unknown :-1 s=CLOSED pgs=338 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:42.897 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.886+0000 7fc36e987640 1 -- 172.21.15.99:0/106022312 >> 172.21.15.99:0/106022312 conn(0x7fc3680fbcb0 msgr2=0x7fc368102d60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:42.897 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.886+0000 7fc36e987640 1 -- 172.21.15.99:0/106022312 shutdown_connections 2024-03-19T01:40:42.897 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:42.886+0000 7fc36e987640 1 -- 172.21.15.99:0/106022312 wait complete. 2024-03-19T01:40:42.899 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.i status 2024-03-19T01:40:43.013 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.000+0000 7fc6a8dcd640 1 -- 172.21.15.99:0/3420989774 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc6a4102a00 msgr2=0x7fc6a4104e70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:43.013 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.000+0000 7fc6a8dcd640 1 --2- 172.21.15.99:0/3420989774 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc6a4102a00 0x7fc6a4104e70 secure :-1 s=READY pgs=343 cs=0 l=1 rev1=1 crypto rx=0x7fc698009900 tx=0x7fc69802bc30 comp rx=0 tx=0).stop 2024-03-19T01:40:43.013 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.001+0000 7fc6a8dcd640 1 -- 172.21.15.99:0/3420989774 shutdown_connections 2024-03-19T01:40:43.013 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.001+0000 7fc6a8dcd640 1 --2- 172.21.15.99:0/3420989774 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fc6a41053b0 0x7fc6a4105830 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:43.014 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.001+0000 7fc6a8dcd640 1 --2- 172.21.15.99:0/3420989774 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc6a4102a00 0x7fc6a4104e70 unknown :-1 s=CLOSED pgs=343 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:43.014 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.001+0000 7fc6a8dcd640 1 --2- 172.21.15.99:0/3420989774 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fc6a4100090 0x7fc6a41024c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:43.014 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.001+0000 7fc6a8dcd640 1 -- 172.21.15.99:0/3420989774 >> 172.21.15.99:0/3420989774 conn(0x7fc6a40fbcb0 msgr2=0x7fc6a40fe0d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:43.014 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.001+0000 7fc6a8dcd640 1 -- 172.21.15.99:0/3420989774 shutdown_connections 2024-03-19T01:40:43.014 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.001+0000 7fc6a8dcd640 1 -- 172.21.15.99:0/3420989774 wait complete. 2024-03-19T01:40:43.014 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.001+0000 7fc6a8dcd640 1 Processor -- start 2024-03-19T01:40:43.014 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.001+0000 7fc6a8dcd640 1 -- start start 2024-03-19T01:40:43.014 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.001+0000 7fc6a8dcd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc6a4100090 0x7fc6a419e160 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:43.014 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.001+0000 7fc6a8dcd640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fc6a4102a00 0x7fc6a419e6a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:43.014 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.001+0000 7fc6a8dcd640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fc6a41053b0 0x7fc6a41a5710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:43.014 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.001+0000 7fc6a8dcd640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fc6a4113a90 con 0x7fc6a4100090 2024-03-19T01:40:43.015 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.001+0000 7fc6a8dcd640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fc6a4113910 con 0x7fc6a41053b0 2024-03-19T01:40:43.015 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.002+0000 7fc6a8dcd640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fc6a4113c10 con 0x7fc6a4102a00 2024-03-19T01:40:43.015 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.002+0000 7fc6a259c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc6a4100090 0x7fc6a419e160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:43.015 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.002+0000 7fc6a259c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc6a4100090 0x7fc6a419e160 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58238/0 (socket says 172.21.15.99:58238) 2024-03-19T01:40:43.015 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.002+0000 7fc6a259c640 1 -- 172.21.15.99:0/4027717202 learned_addr learned my addr 172.21.15.99:0/4027717202 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:43.015 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.002+0000 7fc6a2d9d640 1 --2- 172.21.15.99:0/4027717202 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fc6a41053b0 0x7fc6a41a5710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:43.015 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.002+0000 7fc6a1d9b640 1 --2- 172.21.15.99:0/4027717202 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fc6a4102a00 0x7fc6a419e6a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:43.015 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.002+0000 7fc6a259c640 1 -- 172.21.15.99:0/4027717202 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fc6a4102a00 msgr2=0x7fc6a419e6a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:43.015 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.002+0000 7fc6a259c640 1 --2- 172.21.15.99:0/4027717202 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fc6a4102a00 0x7fc6a419e6a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:43.015 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.002+0000 7fc6a259c640 1 -- 172.21.15.99:0/4027717202 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fc6a41053b0 msgr2=0x7fc6a41a5710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:43.015 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.002+0000 7fc6a259c640 1 --2- 172.21.15.99:0/4027717202 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fc6a41053b0 0x7fc6a41a5710 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:43.015 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.002+0000 7fc6a259c640 1 -- 172.21.15.99:0/4027717202 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fc6a41a5d80 con 0x7fc6a4100090 2024-03-19T01:40:43.015 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.002+0000 7fc6a259c640 1 --2- 172.21.15.99:0/4027717202 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc6a4100090 0x7fc6a419e160 secure :-1 s=READY pgs=344 cs=0 l=1 rev1=1 crypto rx=0x7fc69400e9f0 tx=0x7fc69400edf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:43.015 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.002+0000 7fc6937fe640 1 -- 172.21.15.99:0/4027717202 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc69400ccc0 con 0x7fc6a4100090 2024-03-19T01:40:43.016 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.002+0000 7fc6937fe640 1 -- 172.21.15.99:0/4027717202 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fc69400ce60 con 0x7fc6a4100090 2024-03-19T01:40:43.016 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.002+0000 7fc6a8dcd640 1 -- 172.21.15.99:0/4027717202 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fc6a41a6070 con 0x7fc6a4100090 2024-03-19T01:40:43.016 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.002+0000 7fc6937fe640 1 -- 172.21.15.99:0/4027717202 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc694010690 con 0x7fc6a4100090 2024-03-19T01:40:43.016 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.002+0000 7fc6a8dcd640 1 -- 172.21.15.99:0/4027717202 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fc6a41a6590 con 0x7fc6a4100090 2024-03-19T01:40:43.016 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.004+0000 7fc6937fe640 1 -- 172.21.15.99:0/4027717202 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 18) ==== 137290+0+0 (secure 0 0 0) 0x7fc6940046c0 con 0x7fc6a4100090 2024-03-19T01:40:43.016 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.004+0000 7fc6ab9bc740 1 Processor -- start 2024-03-19T01:40:43.016 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.005+0000 7fc6ab9bc740 1 -- start start 2024-03-19T01:40:43.016 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.005+0000 7fc6ab9bc740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564fee43eff0 0x564fee43f410 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:43.016 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.005+0000 7fc6ab9bc740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564fee43f9e0 0x564fee4444e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:43.016 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.005+0000 7fc6ab9bc740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564fee444b50 0x564fee446fe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:43.016 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.005+0000 7fc6ab9bc740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x564fee3a3de0 con 0x564fee43f9e0 2024-03-19T01:40:43.017 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.005+0000 7fc6ab9bc740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x564fee34cc80 con 0x564fee444b50 2024-03-19T01:40:43.017 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.005+0000 7fc6ab9bc740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x564fee39fe80 con 0x564fee43eff0 2024-03-19T01:40:43.017 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.005+0000 7fc6937fe640 1 --2- 172.21.15.99:0/4027717202 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fc68c0a2880 0x7fc68c0a4d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:43.017 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.005+0000 7fc6a259c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564fee43f9e0 0x564fee4444e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:43.017 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.005+0000 7fc6a2d9d640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564fee43eff0 0x564fee43f410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:43.017 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.005+0000 7fc6a259c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564fee43f9e0 0x564fee4444e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58250/0 (socket says 172.21.15.99:58250) 2024-03-19T01:40:43.017 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.005+0000 7fc6a259c640 1 -- 172.21.15.99:0/935094029 learned_addr learned my addr 172.21.15.99:0/935094029 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:43.017 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.005+0000 7fc6a1d9b640 1 --2- 172.21.15.99:0/935094029 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564fee444b50 0x564fee446fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:43.017 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.005+0000 7fc6a2d9d640 1 --2- 172.21.15.99:0/4027717202 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fc68c0a2880 0x7fc68c0a4d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:43.017 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.005+0000 7fc6937fe640 1 -- 172.21.15.99:0/4027717202 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7fc6940cedf0 con 0x7fc6a4100090 2024-03-19T01:40:43.017 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.005+0000 7fc6a1d9b640 1 -- 172.21.15.99:0/935094029 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564fee43eff0 msgr2=0x564fee43f410 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:40:43.017 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.005+0000 7fc6a1d9b640 1 --2- 172.21.15.99:0/935094029 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564fee43eff0 0x564fee43f410 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:43.017 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.005+0000 7fc6a1d9b640 1 -- 172.21.15.99:0/935094029 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564fee43f9e0 msgr2=0x564fee4444e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:40:43.017 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.005+0000 7fc6a1d9b640 1 --2- 172.21.15.99:0/935094029 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564fee43f9e0 0x564fee4444e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:43.017 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.005+0000 7fc6a1d9b640 1 -- 172.21.15.99:0/935094029 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x564fee35efb0 con 0x564fee444b50 2024-03-19T01:40:43.018 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.006+0000 7fc6a2d9d640 1 --2- 172.21.15.99:0/4027717202 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fc68c0a2880 0x7fc68c0a4d40 secure :-1 s=READY pgs=243 cs=0 l=1 rev1=1 crypto rx=0x564fee440240 tx=0x7fc69c009500 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:43.018 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.006+0000 7fc6a1d9b640 1 --2- 172.21.15.99:0/935094029 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564fee444b50 0x564fee446fe0 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 crypto rx=0x7fc698009230 tx=0x7fc698009560 comp rx=0 tx=0).ready entity=mon.2 client_cookie=25a2aaf89e833f3c server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:43.018 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.006+0000 7fc690ff9640 1 -- 172.21.15.99:0/935094029 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc69802cb80 con 0x564fee444b50 2024-03-19T01:40:43.018 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.006+0000 7fc690ff9640 1 -- 172.21.15.99:0/935094029 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fc698005040 con 0x564fee444b50 2024-03-19T01:40:43.018 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.006+0000 7fc690ff9640 1 -- 172.21.15.99:0/935094029 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc698004920 con 0x564fee444b50 2024-03-19T01:40:43.018 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.006+0000 7fc6ab9bc740 1 -- 172.21.15.99:0/935094029 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564fee444b50 msgr2=0x564fee446fe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:43.018 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.006+0000 7fc6ab9bc740 1 --2- 172.21.15.99:0/935094029 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564fee444b50 0x564fee446fe0 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 crypto rx=0x7fc698009230 tx=0x7fc698009560 comp rx=0 tx=0).stop 2024-03-19T01:40:43.018 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.006+0000 7fc6ab9bc740 1 -- 172.21.15.99:0/935094029 shutdown_connections 2024-03-19T01:40:43.018 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.006+0000 7fc6ab9bc740 1 --2- 172.21.15.99:0/935094029 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564fee444b50 0x564fee446fe0 unknown :-1 s=CLOSED pgs=81 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:43.018 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.006+0000 7fc6ab9bc740 1 --2- 172.21.15.99:0/935094029 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564fee43f9e0 0x564fee4444e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:43.018 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.006+0000 7fc6ab9bc740 1 --2- 172.21.15.99:0/935094029 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564fee43eff0 0x564fee43f410 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:43.018 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.006+0000 7fc6ab9bc740 1 -- 172.21.15.99:0/935094029 >> 172.21.15.99:0/935094029 conn(0x564fee43a560 msgr2=0x564fee43a960 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:43.018 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.007+0000 7fc6ab9bc740 1 -- 172.21.15.99:0/935094029 shutdown_connections 2024-03-19T01:40:43.019 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.007+0000 7fc6ab9bc740 1 -- 172.21.15.99:0/935094029 wait complete. 2024-03-19T01:40:43.019 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.007+0000 7fc6ab9bc740 1 -- start start 2024-03-19T01:40:43.019 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.007+0000 7fc6ab9bc740 1 Processor -- start 2024-03-19T01:40:43.019 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.007+0000 7fc6ab9bc740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564fee43eff0 0x564fee4494d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:43.019 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.007+0000 7fc6ab9bc740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564fee43f9e0 0x564fee449a10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:43.019 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.007+0000 7fc6ab9bc740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564fee444b50 0x564fee450a50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:43.019 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.007+0000 7fc6ab9bc740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x564fee447aa0 con 0x564fee43f9e0 2024-03-19T01:40:43.019 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.007+0000 7fc6ab9bc740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x564fee35ffc0 con 0x564fee444b50 2024-03-19T01:40:43.019 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.007+0000 7fc6ab9bc740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x564fee447c20 con 0x564fee43eff0 2024-03-19T01:40:43.019 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.007+0000 7fc6ab9bc740 10 client.0 _ll_register_callbacks cb 0x564fee39ba20 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:40:43.019 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.007+0000 7fc6a1d9b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564fee43f9e0 0x564fee449a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:43.019 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.007+0000 7fc6a1d9b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564fee43f9e0 0x564fee449a10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58266/0 (socket says 172.21.15.99:58266) 2024-03-19T01:40:43.019 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.007+0000 7fc6a1d9b640 1 -- 172.21.15.99:0/664842518 learned_addr learned my addr 172.21.15.99:0/664842518 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:43.020 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.007+0000 7fc6a259c640 1 --2- 172.21.15.99:0/664842518 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564fee43eff0 0x564fee4494d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:43.020 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.007+0000 7fc6a2d9d640 1 --2- 172.21.15.99:0/664842518 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564fee444b50 0x564fee450a50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:43.020 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.007+0000 7fc6a1d9b640 1 -- 172.21.15.99:0/664842518 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564fee43eff0 msgr2=0x564fee4494d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:40:43.020 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.007+0000 7fc6a1d9b640 1 --2- 172.21.15.99:0/664842518 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564fee43eff0 0x564fee4494d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:43.020 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.007+0000 7fc6a1d9b640 1 -- 172.21.15.99:0/664842518 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564fee444b50 msgr2=0x564fee450a50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:40:43.020 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.007+0000 7fc6a1d9b640 1 --2- 172.21.15.99:0/664842518 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564fee444b50 0x564fee450a50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:43.020 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.007+0000 7fc6a1d9b640 1 -- 172.21.15.99:0/664842518 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x564fee0b82d0 con 0x564fee43f9e0 2024-03-19T01:40:43.020 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.008+0000 7fc6a1d9b640 1 --2- 172.21.15.99:0/664842518 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564fee43f9e0 0x564fee449a10 secure :-1 s=READY pgs=345 cs=0 l=1 rev1=1 crypto rx=0x7fc69802c170 tx=0x7fc698004260 comp rx=0 tx=0).ready entity=mon.0 client_cookie=eb248bdbac258348 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:43.020 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.008+0000 7fc689ffb640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:40:43.020 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.008+0000 7fc689ffb640 1 -- 172.21.15.99:0/664842518 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc698005600 con 0x564fee43f9e0 2024-03-19T01:40:43.020 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.008+0000 7fc689ffb640 1 -- 172.21.15.99:0/664842518 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7fc6980046c0 con 0x564fee43f9e0 2024-03-19T01:40:43.020 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.008+0000 7fc6ab9bc740 1 -- 172.21.15.99:0/664842518 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fc698005040 con 0x564fee43f9e0 2024-03-19T01:40:43.020 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.008+0000 7fc689ffb640 1 -- 172.21.15.99:0/664842518 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=20) ==== 24+0+0 (secure 0 0 0) 0x7fc698004900 con 0x564fee43f9e0 2024-03-19T01:40:43.021 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.008+0000 7fc6ab9bc740 10 client.15282 fetch_fsmap learned FSMap version 20 2024-03-19T01:40:43.021 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.008+0000 7fc6ab9bc740 1 -- 172.21.15.99:0/664842518 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=20,monmap=4+}) -- 0x564fee1826b0 con 0x564fee43f9e0 2024-03-19T01:40:43.021 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.008+0000 7fc689ffb640 1 -- 172.21.15.99:0/664842518 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fc698005040 con 0x564fee43f9e0 2024-03-19T01:40:43.021 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.008+0000 7fc689ffb640 1 -- 172.21.15.99:0/664842518 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 20) ==== 8338+0+0 (secure 0 0 0) 0x7fc698034440 con 0x564fee43f9e0 2024-03-19T01:40:43.021 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.008+0000 7fc6ab9bc740 10 client.15282 fetch_fsmap finished waiting for FSMap version 20 2024-03-19T01:40:43.021 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.008+0000 7fc6ab9bc740 10 client.15282 resolve_mds: resolved name 'i' to daemon mds.i 2024-03-19T01:40:43.021 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.008+0000 7fc6ab9bc740 20 client.15282 populate_metadata read hostname 'smithi099' 2024-03-19T01:40:43.021 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.008+0000 7fc6ab9bc740 1 --2- 172.21.15.99:0/664842518 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x564fee455df0 0x564fee457260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:43.021 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.008+0000 7fc6ab9bc740 4 client.15282 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:40:43.021 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.008+0000 7fc6ab9bc740 1 -- 172.21.15.99:0/664842518 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x564fee3b1320 con 0x564fee455df0 2024-03-19T01:40:43.021 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.008+0000 7fc6a259c640 1 --2- 172.21.15.99:0/664842518 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x564fee455df0 0x564fee457260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:43.021 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.009+0000 7fc689ffb640 10 client.15282 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:40:43.021 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.009+0000 7fc6a259c640 1 --2- 172.21.15.99:0/664842518 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x564fee455df0 0x564fee457260 crc :-1 s=READY pgs=23 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=6be6f06e5dcbc25e server_cookie=bfaaa158b7e60085 in_seq=0 out_seq=0 2024-03-19T01:40:43.021 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.009+0000 7fc689ffb640 1 -- 172.21.15.99:0/664842518 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x564fee3b1320 con 0x564fee455df0 2024-03-19T01:40:43.022 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.009+0000 7fc689ffb640 10 client.15282 handle_command_reply: tid=0 2024-03-19T01:40:43.022 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.009+0000 7fc6ab9bc740 1 client.15282 shutdown 2024-03-19T01:40:43.022 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.010+0000 7fc6ab9bc740 1 -- 172.21.15.99:0/664842518 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564fee43f9e0 msgr2=0x564fee449a10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:43.022 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.010+0000 7fc6ab9bc740 1 --2- 172.21.15.99:0/664842518 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564fee43f9e0 0x564fee449a10 secure :-1 s=READY pgs=345 cs=0 l=1 rev1=1 crypto rx=0x7fc69802c170 tx=0x7fc698004260 comp rx=0 tx=0).stop 2024-03-19T01:40:43.022 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.010+0000 7fc6ab9bc740 1 -- 172.21.15.99:0/664842518 shutdown_connections 2024-03-19T01:40:43.022 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.010+0000 7fc6ab9bc740 1 --2- 172.21.15.99:0/664842518 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x564fee455df0 0x564fee457260 crc :-1 s=READY pgs=23 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:43.022 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.010+0000 7fc6ab9bc740 1 --2- 172.21.15.99:0/664842518 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564fee444b50 0x564fee450a50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:43.022 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.010+0000 7fc6ab9bc740 1 --2- 172.21.15.99:0/664842518 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564fee43f9e0 0x564fee449a10 unknown :-1 s=CLOSED pgs=345 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:43.022 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.010+0000 7fc689ffb640 0 client.15282 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:40:43.022 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.010+0000 7fc6ab9bc740 1 --2- 172.21.15.99:0/664842518 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564fee43eff0 0x564fee4494d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:43.022 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.010+0000 7fc6ab9bc740 1 -- 172.21.15.99:0/664842518 >> 172.21.15.99:0/664842518 conn(0x564fee43a560 msgr2=0x564fee4397e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:43.022 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.010+0000 7fc6ab9bc740 1 -- 172.21.15.99:0/664842518 shutdown_connections 2024-03-19T01:40:43.023 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.010+0000 7fc6ab9bc740 1 -- 172.21.15.99:0/664842518 wait complete. 2024-03-19T01:40:43.023 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.010+0000 7fc6ab9bc740 20 client.15282 trim_cache size 0 max 16384 2024-03-19T01:40:43.035 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.027+0000 7fc6ab9bc740 1 Processor -- start 2024-03-19T01:40:43.035 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.027+0000 7fc6ab9bc740 1 -- start start 2024-03-19T01:40:43.035 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.027+0000 7fc6ab9bc740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564fee43eff0 0x564fee454f40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:43.035 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.027+0000 7fc6ab9bc740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564fee43f9e0 0x564fee450a10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:43.035 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.027+0000 7fc6ab9bc740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564fee444b50 0x564fee43a940 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:43.035 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.027+0000 7fc6ab9bc740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x564fee4555a0 con 0x564fee43f9e0 2024-03-19T01:40:43.035 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.027+0000 7fc6ab9bc740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x564fee455c70 con 0x564fee43eff0 2024-03-19T01:40:43.036 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.027+0000 7fc6ab9bc740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x564fee456340 con 0x564fee444b50 2024-03-19T01:40:43.036 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.027+0000 7fc6a1d9b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564fee43f9e0 0x564fee450a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:43.036 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.027+0000 7fc6a1d9b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564fee43f9e0 0x564fee450a10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58278/0 (socket says 172.21.15.99:58278) 2024-03-19T01:40:43.036 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.027+0000 7fc6a1d9b640 1 -- 172.21.15.99:0/1097657353 learned_addr learned my addr 172.21.15.99:0/1097657353 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:43.036 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.028+0000 7fc6a259c640 1 --2- 172.21.15.99:0/1097657353 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564fee43eff0 0x564fee454f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:43.036 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.028+0000 7fc6a2d9d640 1 --2- 172.21.15.99:0/1097657353 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564fee444b50 0x564fee43a940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:43.036 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.028+0000 7fc6a1d9b640 1 -- 172.21.15.99:0/1097657353 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564fee444b50 msgr2=0x564fee43a940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:40:43.036 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.028+0000 7fc6a1d9b640 1 --2- 172.21.15.99:0/1097657353 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564fee444b50 0x564fee43a940 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:43.036 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.028+0000 7fc6a1d9b640 1 -- 172.21.15.99:0/1097657353 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564fee43eff0 msgr2=0x564fee454f40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:40:43.036 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.028+0000 7fc6a1d9b640 1 --2- 172.21.15.99:0/1097657353 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564fee43eff0 0x564fee454f40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:43.036 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.028+0000 7fc6a1d9b640 1 -- 172.21.15.99:0/1097657353 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x564fee43fdc0 con 0x564fee43f9e0 2024-03-19T01:40:43.036 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.028+0000 7fc6a2d9d640 1 --2- 172.21.15.99:0/1097657353 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564fee444b50 0x564fee43a940 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-19T01:40:43.037 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.028+0000 7fc6a1d9b640 1 --2- 172.21.15.99:0/1097657353 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564fee43f9e0 0x564fee450a10 secure :-1 s=READY pgs=346 cs=0 l=1 rev1=1 crypto rx=0x7fc698006010 tx=0x7fc698005e50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6a6d77fec80da07a server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:43.037 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.028+0000 7fc68bfff640 1 -- 172.21.15.99:0/1097657353 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc698034440 con 0x564fee43f9e0 2024-03-19T01:40:43.037 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.028+0000 7fc68bfff640 1 -- 172.21.15.99:0/1097657353 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fc6980345e0 con 0x564fee43f9e0 2024-03-19T01:40:43.037 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.028+0000 7fc68bfff640 1 -- 172.21.15.99:0/1097657353 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc698004d90 con 0x564fee43f9e0 2024-03-19T01:40:43.037 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.028+0000 7fc6ab9bc740 1 -- 172.21.15.99:0/1097657353 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564fee43f9e0 msgr2=0x564fee450a10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:43.037 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.028+0000 7fc6ab9bc740 1 --2- 172.21.15.99:0/1097657353 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564fee43f9e0 0x564fee450a10 secure :-1 s=READY pgs=346 cs=0 l=1 rev1=1 crypto rx=0x7fc698006010 tx=0x7fc698005e50 comp rx=0 tx=0).stop 2024-03-19T01:40:43.037 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.029+0000 7fc6ab9bc740 1 -- 172.21.15.99:0/1097657353 shutdown_connections 2024-03-19T01:40:43.037 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.029+0000 7fc6ab9bc740 1 --2- 172.21.15.99:0/1097657353 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564fee444b50 0x564fee43a940 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:43.037 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.029+0000 7fc6ab9bc740 1 --2- 172.21.15.99:0/1097657353 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564fee43f9e0 0x564fee450a10 unknown :-1 s=CLOSED pgs=346 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:43.037 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.029+0000 7fc6ab9bc740 1 --2- 172.21.15.99:0/1097657353 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564fee43eff0 0x564fee454f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:43.037 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.029+0000 7fc6ab9bc740 1 -- 172.21.15.99:0/1097657353 >> 172.21.15.99:0/1097657353 conn(0x564fee43a560 msgr2=0x564fee447fc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:43.037 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.029+0000 7fc6ab9bc740 1 -- 172.21.15.99:0/1097657353 shutdown_connections 2024-03-19T01:40:43.037 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.029+0000 7fc6ab9bc740 1 -- 172.21.15.99:0/1097657353 wait complete. 2024-03-19T01:40:43.038 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.029+0000 7fc6ab9bc740 1 -- start start 2024-03-19T01:40:43.038 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.029+0000 7fc6ab9bc740 1 Processor -- start 2024-03-19T01:40:43.038 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.029+0000 7fc6ab9bc740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564fee43eff0 0x564fee44c260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:43.038 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.029+0000 7fc6ab9bc740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564fee43f9e0 0x564fee44c7a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:43.038 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.029+0000 7fc6ab9bc740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564fee444b50 0x564fee44f660 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:43.038 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.029+0000 7fc6ab9bc740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x564fee4570e0 con 0x564fee43eff0 2024-03-19T01:40:43.038 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.029+0000 7fc6ab9bc740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x564fee456a10 con 0x564fee444b50 2024-03-19T01:40:43.038 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.029+0000 7fc6ab9bc740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x564fee4577b0 con 0x564fee43f9e0 2024-03-19T01:40:43.038 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.029+0000 7fc6ab9bc740 10 client.0 _ll_register_callbacks cb 0x7fc6940973b0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:40:43.038 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.029+0000 7fc6a259c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564fee43eff0 0x564fee44c260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:43.038 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.029+0000 7fc6a259c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564fee43eff0 0x564fee44c260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58286/0 (socket says 172.21.15.99:58286) 2024-03-19T01:40:43.038 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.029+0000 7fc6a259c640 1 -- 172.21.15.99:0/2876941738 learned_addr learned my addr 172.21.15.99:0/2876941738 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:43.038 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.029+0000 7fc6a2d9d640 1 --2- 172.21.15.99:0/2876941738 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564fee444b50 0x564fee44f660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:43.039 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.029+0000 7fc6a1d9b640 1 --2- 172.21.15.99:0/2876941738 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564fee43f9e0 0x564fee44c7a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:43.039 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.029+0000 7fc6a259c640 1 -- 172.21.15.99:0/2876941738 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564fee43f9e0 msgr2=0x564fee44c7a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:40:43.039 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.029+0000 7fc6a259c640 1 --2- 172.21.15.99:0/2876941738 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564fee43f9e0 0x564fee44c7a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:43.039 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.029+0000 7fc6a259c640 1 -- 172.21.15.99:0/2876941738 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564fee444b50 msgr2=0x564fee44f660 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:40:43.039 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.029+0000 7fc6a259c640 1 --2- 172.21.15.99:0/2876941738 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564fee444b50 0x564fee44f660 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:43.039 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.029+0000 7fc6a259c640 1 -- 172.21.15.99:0/2876941738 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x564fee44fc50 con 0x564fee43eff0 2024-03-19T01:40:43.039 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.030+0000 7fc6a259c640 1 --2- 172.21.15.99:0/2876941738 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564fee43eff0 0x564fee44c260 secure :-1 s=READY pgs=347 cs=0 l=1 rev1=1 crypto rx=0x7fc694098120 tx=0x7fc694086040 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1e24d662105ac08c server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:43.039 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.030+0000 7fc689ffb640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:40:43.039 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.030+0000 7fc689ffb640 1 -- 172.21.15.99:0/2876941738 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc6940b8070 con 0x564fee43eff0 2024-03-19T01:40:43.039 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.030+0000 7fc689ffb640 1 -- 172.21.15.99:0/2876941738 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7fc6940860b0 con 0x564fee43eff0 2024-03-19T01:40:43.039 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.030+0000 7fc6ab9bc740 1 -- 172.21.15.99:0/2876941738 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fc6980345e0 con 0x564fee43eff0 2024-03-19T01:40:43.039 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.030+0000 7fc689ffb640 1 -- 172.21.15.99:0/2876941738 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=20) ==== 24+0+0 (secure 0 0 0) 0x7fc69409b750 con 0x564fee43eff0 2024-03-19T01:40:43.039 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.030+0000 7fc6ab9bc740 10 client.15288 fetch_fsmap learned FSMap version 20 2024-03-19T01:40:43.040 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.030+0000 7fc6ab9bc740 1 -- 172.21.15.99:0/2876941738 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=20,monmap=4+}) -- 0x564fee4bbc30 con 0x564fee43eff0 2024-03-19T01:40:43.040 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.030+0000 7fc689ffb640 1 -- 172.21.15.99:0/2876941738 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fc6980345e0 con 0x564fee43eff0 2024-03-19T01:40:43.040 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.030+0000 7fc689ffb640 1 -- 172.21.15.99:0/2876941738 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 20) ==== 8338+0+0 (secure 0 0 0) 0x7fc694084ce0 con 0x564fee43eff0 2024-03-19T01:40:43.040 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.030+0000 7fc6ab9bc740 10 client.15288 fetch_fsmap finished waiting for FSMap version 20 2024-03-19T01:40:43.040 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.030+0000 7fc6ab9bc740 10 client.15288 resolve_mds: resolved name 'i' to daemon mds.i 2024-03-19T01:40:43.040 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.030+0000 7fc6ab9bc740 20 client.15288 populate_metadata read hostname 'smithi099' 2024-03-19T01:40:43.040 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.030+0000 7fc6ab9bc740 1 --2- 172.21.15.99:0/2876941738 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x564fee4c0840 0x564fee4bf550 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:43.040 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.030+0000 7fc6ab9bc740 4 client.15288 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "status"}] 2024-03-19T01:40:43.040 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.030+0000 7fc6ab9bc740 1 -- 172.21.15.99:0/2876941738 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "status"}) -- 0x564fee3ab670 con 0x564fee4c0840 2024-03-19T01:40:43.040 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.030+0000 7fc6a1d9b640 1 --2- 172.21.15.99:0/2876941738 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x564fee4c0840 0x564fee4bf550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:43.040 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.031+0000 7fc6a1d9b640 1 --2- 172.21.15.99:0/2876941738 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x564fee4c0840 0x564fee4bf550 crc :-1 s=READY pgs=24 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=210a4011cbc01ebf server_cookie=a65b459bcb036947 in_seq=0 out_seq=0 2024-03-19T01:40:43.040 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.031+0000 7fc689ffb640 10 client.15288 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:40:43.040 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.031+0000 7fc689ffb640 1 -- 172.21.15.99:0/2876941738 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+327 (crc 0 0 0) 0x564fee3ab670 con 0x564fee4c0840 2024-03-19T01:40:43.040 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.031+0000 7fc689ffb640 10 client.15288 handle_command_reply: tid=0 2024-03-19T01:40:43.040 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.031+0000 7fc6ab9bc740 1 client.15288 shutdown 2024-03-19T01:40:43.040 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.031+0000 7fc6ab9bc740 1 -- 172.21.15.99:0/2876941738 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564fee43eff0 msgr2=0x564fee44c260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:43.041 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.031+0000 7fc6ab9bc740 1 --2- 172.21.15.99:0/2876941738 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564fee43eff0 0x564fee44c260 secure :-1 s=READY pgs=347 cs=0 l=1 rev1=1 crypto rx=0x7fc694098120 tx=0x7fc694086040 comp rx=0 tx=0).stop 2024-03-19T01:40:43.041 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.031+0000 7fc6ab9bc740 1 -- 172.21.15.99:0/2876941738 shutdown_connections 2024-03-19T01:40:43.041 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.031+0000 7fc6ab9bc740 1 --2- 172.21.15.99:0/2876941738 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x564fee4c0840 0x564fee4bf550 crc :-1 s=READY pgs=24 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:43.041 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.031+0000 7fc6ab9bc740 1 --2- 172.21.15.99:0/2876941738 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564fee444b50 0x564fee44f660 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:43.041 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.031+0000 7fc689ffb640 0 client.15288 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:40:43.041 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.031+0000 7fc6ab9bc740 1 --2- 172.21.15.99:0/2876941738 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564fee43f9e0 0x564fee44c7a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:43.041 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.031+0000 7fc6ab9bc740 1 --2- 172.21.15.99:0/2876941738 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564fee43eff0 0x564fee44c260 unknown :-1 s=CLOSED pgs=347 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:43.041 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.031+0000 7fc6ab9bc740 1 -- 172.21.15.99:0/2876941738 >> 172.21.15.99:0/2876941738 conn(0x564fee43a560 msgr2=0x564fee43a940 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:43.041 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.031+0000 7fc6ab9bc740 1 -- 172.21.15.99:0/2876941738 shutdown_connections 2024-03-19T01:40:43.041 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.031+0000 7fc6ab9bc740 1 -- 172.21.15.99:0/2876941738 wait complete. 2024-03-19T01:40:43.041 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.031+0000 7fc6ab9bc740 20 client.15288 trim_cache size 0 max 16384 2024-03-19T01:40:43.042 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2024-03-19T01:40:43.042 INFO:teuthology.orchestra.run.smithi099.stdout: "cluster_fsid": "438fb69c-e590-11ee-95c9-87774f69a715", 2024-03-19T01:40:43.042 INFO:teuthology.orchestra.run.smithi099.stdout: "whoami": 0, 2024-03-19T01:40:43.042 INFO:teuthology.orchestra.run.smithi099.stdout: "id": 24487, 2024-03-19T01:40:43.042 INFO:teuthology.orchestra.run.smithi099.stdout: "want_state": "up:active", 2024-03-19T01:40:43.042 INFO:teuthology.orchestra.run.smithi099.stdout: "state": "up:active", 2024-03-19T01:40:43.042 INFO:teuthology.orchestra.run.smithi099.stdout: "fs_name": "cephfs", 2024-03-19T01:40:43.042 INFO:teuthology.orchestra.run.smithi099.stdout: "rank_uptime": 106.24177888299999, 2024-03-19T01:40:43.042 INFO:teuthology.orchestra.run.smithi099.stdout: "mdsmap_epoch": 20, 2024-03-19T01:40:43.042 INFO:teuthology.orchestra.run.smithi099.stdout: "osdmap_epoch": 91, 2024-03-19T01:40:43.042 INFO:teuthology.orchestra.run.smithi099.stdout: "osdmap_epoch_barrier": 87, 2024-03-19T01:40:43.042 INFO:teuthology.orchestra.run.smithi099.stdout: "uptime": 121.349478034 2024-03-19T01:40:43.042 INFO:teuthology.orchestra.run.smithi099.stdout:} 2024-03-19T01:40:43.044 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.032+0000 7fc6a8dcd640 1 -- 172.21.15.99:0/4027717202 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fc68c0a2880 msgr2=0x7fc68c0a4d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:43.044 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.032+0000 7fc6a8dcd640 1 --2- 172.21.15.99:0/4027717202 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fc68c0a2880 0x7fc68c0a4d40 secure :-1 s=READY pgs=243 cs=0 l=1 rev1=1 crypto rx=0x564fee440240 tx=0x7fc69c009500 comp rx=0 tx=0).stop 2024-03-19T01:40:43.044 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.032+0000 7fc6a8dcd640 1 -- 172.21.15.99:0/4027717202 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc6a4100090 msgr2=0x7fc6a419e160 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:43.044 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.032+0000 7fc6a8dcd640 1 --2- 172.21.15.99:0/4027717202 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc6a4100090 0x7fc6a419e160 secure :-1 s=READY pgs=344 cs=0 l=1 rev1=1 crypto rx=0x7fc69400e9f0 tx=0x7fc69400edf0 comp rx=0 tx=0).stop 2024-03-19T01:40:43.044 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.032+0000 7fc6a8dcd640 1 -- 172.21.15.99:0/4027717202 shutdown_connections 2024-03-19T01:40:43.044 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.032+0000 7fc6a8dcd640 1 --2- 172.21.15.99:0/4027717202 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fc68c0a2880 0x7fc68c0a4d40 unknown :-1 s=CLOSED pgs=243 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:43.044 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.032+0000 7fc6a8dcd640 1 --2- 172.21.15.99:0/4027717202 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fc6a41053b0 0x7fc6a41a5710 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:43.044 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.032+0000 7fc6a8dcd640 1 --2- 172.21.15.99:0/4027717202 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fc6a4102a00 0x7fc6a419e6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:43.044 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.032+0000 7fc6a8dcd640 1 --2- 172.21.15.99:0/4027717202 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc6a4100090 0x7fc6a419e160 unknown :-1 s=CLOSED pgs=344 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:43.044 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.032+0000 7fc6a8dcd640 1 -- 172.21.15.99:0/4027717202 >> 172.21.15.99:0/4027717202 conn(0x7fc6a40fbcb0 msgr2=0x7fc6a4100a30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:43.045 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.032+0000 7fc6a8dcd640 1 -- 172.21.15.99:0/4027717202 shutdown_connections 2024-03-19T01:40:43.045 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:43.032+0000 7fc6a8dcd640 1 -- 172.21.15.99:0/4027717202 wait complete. 2024-03-19T01:40:43.045 INFO:teuthology.run_tasks:Running task kclient... 2024-03-19T01:40:43.069 INFO:tasks.kclient:Mounting kernel clients... 2024-03-19T01:40:43.069 INFO:tasks.kclient:config is {'client.0': {}} 2024-03-19T01:40:43.069 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-19T01:40:43.069 DEBUG:teuthology.orchestra.run.smithi099:> ip netns list 2024-03-19T01:40:43.085 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-19T01:40:43.085 DEBUG:teuthology.orchestra.run.smithi099:> sudo ip link delete ceph-brx 2024-03-19T01:40:43.148 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:42 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/261078693' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-03-19T01:40:43.148 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:42 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/3223885361' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:40:43.148 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:42 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/1010837617' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-19T01:40:43.154 INFO:teuthology.orchestra.run.smithi099.stderr:Cannot find device "ceph-brx" 2024-03-19T01:40:43.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-19T01:40:43.156 INFO:tasks.kclient:client.0 config is {'mntopts': ['ms_mode=crc', 'nowsync'], 'syntax': 'v1', 'mount_subvol_num': 1} 2024-03-19T01:40:43.156 INFO:tasks.cephfs.mount:cephfs_mntpt = None 2024-03-19T01:40:43.156 INFO:tasks.cephfs.mount:hostfs_mntpt = /home/ubuntu/cephtest/mnt.0 2024-03-19T01:40:43.156 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph fs ls 2024-03-19T01:40:43.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:42 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/261078693' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-03-19T01:40:43.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:42 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/3223885361' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:40:43.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:42 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/1010837617' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-19T01:40:43.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:42 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/261078693' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-03-19T01:40:43.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:42 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/3223885361' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T01:40:43.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:42 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/1010837617' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-19T01:40:43.434 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T01:40:44.217 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:43 smithi099 ceph-mon[62951]: pgmap v339: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 6 op/s 2024-03-19T01:40:44.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:43 smithi115 ceph-mon[67332]: pgmap v339: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 6 op/s 2024-03-19T01:40:44.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:43 smithi112 ceph-mon[66819]: pgmap v339: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 6 op/s 2024-03-19T01:40:45.107 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:45.104+0000 7f519ed42700 1 -- 172.21.15.99:0/1771889287 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f51981014a0 msgr2=0x7f5198101880 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:45.107 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:45.104+0000 7f519ed42700 1 --2- 172.21.15.99:0/1771889287 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f51981014a0 0x7f5198101880 secure :-1 s=READY pgs=348 cs=0 l=1 rev1=1 crypto rx=0x7f519000b0d0 tx=0x7f519000b3e0 comp rx=0 tx=0).stop 2024-03-19T01:40:45.107 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:45.104+0000 7f519ed42700 1 -- 172.21.15.99:0/1771889287 shutdown_connections 2024-03-19T01:40:45.107 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:45.104+0000 7f519ed42700 1 --2- 172.21.15.99:0/1771889287 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f51981064f0 0x7f519810e0c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:45.107 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:45.104+0000 7f519ed42700 1 --2- 172.21.15.99:0/1771889287 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f51981014a0 0x7f5198101880 unknown :-1 s=CLOSED pgs=348 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:45.108 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:45.104+0000 7f519ed42700 1 --2- 172.21.15.99:0/1771889287 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5198101e50 0x7f5198105e10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:45.108 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:45.104+0000 7f519ed42700 1 -- 172.21.15.99:0/1771889287 >> 172.21.15.99:0/1771889287 conn(0x7f5198077260 msgr2=0x7f5198077670 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:45.108 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:45.104+0000 7f519ed42700 1 -- 172.21.15.99:0/1771889287 shutdown_connections 2024-03-19T01:40:45.108 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:45.104+0000 7f519ed42700 1 -- 172.21.15.99:0/1771889287 wait complete. 2024-03-19T01:40:45.108 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:45.105+0000 7f519ed42700 1 Processor -- start 2024-03-19T01:40:45.108 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:45.105+0000 7f519ed42700 1 -- start start 2024-03-19T01:40:45.108 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:45.105+0000 7f519ed42700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f51981014a0 0x7f51981a08e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:45.108 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:45.105+0000 7f519ed42700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5198101e50 0x7f51981a0e20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:45.108 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:45.105+0000 7f519ed42700 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f51981064f0 0x7f519819aaf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:45.108 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:45.105+0000 7f519ed42700 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f5198110440 con 0x7f5198101e50 2024-03-19T01:40:45.108 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:45.105+0000 7f519ed42700 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f51981102c0 con 0x7f51981014a0 2024-03-19T01:40:45.108 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:45.105+0000 7f519ed42700 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f51981105c0 con 0x7f51981064f0 2024-03-19T01:40:45.108 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:45.105+0000 7f5197fff700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5198101e50 0x7f51981a0e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:45.109 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:45.105+0000 7f519cade700 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f51981014a0 0x7f51981a08e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:45.109 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:45.105+0000 7f5197fff700 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5198101e50 0x7f51981a0e20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58296/0 (socket says 172.21.15.99:58296) 2024-03-19T01:40:45.109 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:45.105+0000 7f5197fff700 1 -- 172.21.15.99:0/2236624579 learned_addr learned my addr 172.21.15.99:0/2236624579 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:45.109 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:45.105+0000 7f519d2df700 1 --2- 172.21.15.99:0/2236624579 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f51981064f0 0x7f519819aaf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:45.109 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:45.105+0000 7f519cade700 1 -- 172.21.15.99:0/2236624579 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f51981064f0 msgr2=0x7f519819aaf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:45.109 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:45.105+0000 7f519cade700 1 --2- 172.21.15.99:0/2236624579 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f51981064f0 0x7f519819aaf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:45.109 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:45.105+0000 7f519cade700 1 -- 172.21.15.99:0/2236624579 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5198101e50 msgr2=0x7f51981a0e20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:45.109 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:45.105+0000 7f519cade700 1 --2- 172.21.15.99:0/2236624579 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5198101e50 0x7f51981a0e20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:45.109 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:45.105+0000 7f519cade700 1 -- 172.21.15.99:0/2236624579 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f519819b250 con 0x7f51981014a0 2024-03-19T01:40:45.109 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:45.105+0000 7f5197fff700 1 --2- 172.21.15.99:0/2236624579 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5198101e50 0x7f51981a0e20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:40:45.109 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:45.106+0000 7f519cade700 1 --2- 172.21.15.99:0/2236624579 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f51981014a0 0x7f51981a08e0 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7f519000d010 tx=0x7f5190007a20 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:45.109 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:45.106+0000 7f5195ffb700 1 -- 172.21.15.99:0/2236624579 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5190003c00 con 0x7f51981014a0 2024-03-19T01:40:45.110 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:45.106+0000 7f5195ffb700 1 -- 172.21.15.99:0/2236624579 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f5190022070 con 0x7f51981014a0 2024-03-19T01:40:45.110 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:45.106+0000 7f519ed42700 1 -- 172.21.15.99:0/2236624579 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f519819b4e0 con 0x7f51981014a0 2024-03-19T01:40:45.110 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:45.106+0000 7f5195ffb700 1 -- 172.21.15.99:0/2236624579 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f519001b6c0 con 0x7f51981014a0 2024-03-19T01:40:45.110 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:45.106+0000 7f519ed42700 1 -- 172.21.15.99:0/2236624579 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f51981a77b0 con 0x7f51981014a0 2024-03-19T01:40:45.111 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:45.107+0000 7f519ed42700 1 -- 172.21.15.99:0/2236624579 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f51981028f0 con 0x7f51981014a0 2024-03-19T01:40:45.111 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:45.108+0000 7f5195ffb700 1 -- 172.21.15.99:0/2236624579 <== mon.2 v2:172.21.15.112:3300/0 4 ==== mgrmap(e 18) ==== 137290+0+0 (secure 0 0 0) 0x7f519001d470 con 0x7f51981014a0 2024-03-19T01:40:45.111 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:45.108+0000 7f5195ffb700 1 --2- 172.21.15.99:0/2236624579 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f51840a2890 0x7f51840a4d50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:45.111 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:45.108+0000 7f5197fff700 1 --2- 172.21.15.99:0/2236624579 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f51840a2890 0x7f51840a4d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:45.111 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:45.109+0000 7f5195ffb700 1 -- 172.21.15.99:0/2236624579 <== mon.2 v2:172.21.15.112:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f519000f020 con 0x7f51981014a0 2024-03-19T01:40:45.111 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:45.109+0000 7f5197fff700 1 --2- 172.21.15.99:0/2236624579 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f51840a2890 0x7f51840a4d50 secure :-1 s=READY pgs=244 cs=0 l=1 rev1=1 crypto rx=0x7f51980ffcc0 tx=0x7f5188009260 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:45.114 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:45.114+0000 7f5195ffb700 1 -- 172.21.15.99:0/2236624579 <== mon.2 v2:172.21.15.112:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f519009c880 con 0x7f51981014a0 2024-03-19T01:40:45.354 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:45.353+0000 7f519ed42700 1 -- 172.21.15.99:0/2236624579 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_command({"prefix": "fs ls"} v 0) -- 0x7f519819bfd0 con 0x7f51981014a0 2024-03-19T01:40:45.354 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:45.353+0000 7f5195ffb700 1 -- 172.21.15.99:0/2236624579 <== mon.2 v2:172.21.15.112:3300/0 7 ==== mon_command_ack([{"prefix": "fs ls"}]=0 v20) ==== 53+0+73 (secure 0 0 0) 0x7f519009c180 con 0x7f51981014a0 2024-03-19T01:40:45.354 INFO:teuthology.orchestra.run.smithi099.stdout:name: cephfs, metadata pool: cephfs_metadata, data pools: [cephfs_data ] 2024-03-19T01:40:45.356 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:45.355+0000 7f519ed42700 1 -- 172.21.15.99:0/2236624579 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f51840a2890 msgr2=0x7f51840a4d50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:45.356 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:45.355+0000 7f519ed42700 1 --2- 172.21.15.99:0/2236624579 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f51840a2890 0x7f51840a4d50 secure :-1 s=READY pgs=244 cs=0 l=1 rev1=1 crypto rx=0x7f51980ffcc0 tx=0x7f5188009260 comp rx=0 tx=0).stop 2024-03-19T01:40:45.356 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:45.355+0000 7f519ed42700 1 -- 172.21.15.99:0/2236624579 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f51981014a0 msgr2=0x7f51981a08e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:45.356 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:45.355+0000 7f519ed42700 1 --2- 172.21.15.99:0/2236624579 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f51981014a0 0x7f51981a08e0 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7f519000d010 tx=0x7f5190007a20 comp rx=0 tx=0).stop 2024-03-19T01:40:45.356 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:45.355+0000 7f519ed42700 1 -- 172.21.15.99:0/2236624579 shutdown_connections 2024-03-19T01:40:45.356 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:45.355+0000 7f519ed42700 1 --2- 172.21.15.99:0/2236624579 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f51840a2890 0x7f51840a4d50 unknown :-1 s=CLOSED pgs=244 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:45.357 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:45.355+0000 7f519ed42700 1 --2- 172.21.15.99:0/2236624579 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f51981064f0 0x7f519819aaf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:45.357 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:45.355+0000 7f519ed42700 1 --2- 172.21.15.99:0/2236624579 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f51981014a0 0x7f51981a08e0 unknown :-1 s=CLOSED pgs=82 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:45.357 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:45.355+0000 7f519ed42700 1 --2- 172.21.15.99:0/2236624579 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5198101e50 0x7f51981a0e20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:45.357 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:45.355+0000 7f519ed42700 1 -- 172.21.15.99:0/2236624579 >> 172.21.15.99:0/2236624579 conn(0x7f5198077260 msgr2=0x7f51981009d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:45.357 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:45.355+0000 7f519ed42700 1 -- 172.21.15.99:0/2236624579 shutdown_connections 2024-03-19T01:40:45.357 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:45.355+0000 7f519ed42700 1 -- 172.21.15.99:0/2236624579 wait complete. 2024-03-19T01:40:46.007 INFO:tasks.cephfs.mount:Mounting default Ceph FS; just confirmed its presence on cluster 2024-03-19T01:40:46.007 INFO:tasks.cephfs.mount:Mounting Ceph FS. Following are details of mount; remember "None" represents Python type None - 2024-03-19T01:40:46.007 INFO:tasks.cephfs.mount:self.client_remote.hostname = smithi099.front.sepia.ceph.com 2024-03-19T01:40:46.007 INFO:tasks.cephfs.mount:self.client.name = client.0 2024-03-19T01:40:46.007 INFO:tasks.cephfs.mount:self.hostfs_mntpt = /home/ubuntu/cephtest/mnt.0 2024-03-19T01:40:46.008 INFO:tasks.cephfs.mount:self.cephfs_name = None 2024-03-19T01:40:46.008 INFO:tasks.cephfs.mount:self.cephfs_mntpt = None 2024-03-19T01:40:46.008 INFO:tasks.cephfs.mount:self.client_keyring_path = None 2024-03-19T01:40:46.008 INFO:tasks.cephfs.mount:Setting the 'None' netns for '/home/ubuntu/cephtest/mnt.0' 2024-03-19T01:40:46.008 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-19T01:40:46.008 DEBUG:teuthology.orchestra.run.smithi099:> ip addr 2024-03-19T01:40:46.025 INFO:teuthology.orchestra.run.smithi099.stdout:1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 2024-03-19T01:40:46.025 INFO:teuthology.orchestra.run.smithi099.stdout: link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2024-03-19T01:40:46.026 INFO:teuthology.orchestra.run.smithi099.stdout: inet 127.0.0.1/8 scope host lo 2024-03-19T01:40:46.026 INFO:teuthology.orchestra.run.smithi099.stdout: valid_lft forever preferred_lft forever 2024-03-19T01:40:46.026 INFO:teuthology.orchestra.run.smithi099.stdout: inet6 ::1/128 scope host 2024-03-19T01:40:46.026 INFO:teuthology.orchestra.run.smithi099.stdout: valid_lft forever preferred_lft forever 2024-03-19T01:40:46.026 INFO:teuthology.orchestra.run.smithi099.stdout:2: eno1: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-19T01:40:46.026 INFO:teuthology.orchestra.run.smithi099.stdout: link/ether 0c:c4:7a:d8:a2:10 brd ff:ff:ff:ff:ff:ff 2024-03-19T01:40:46.026 INFO:teuthology.orchestra.run.smithi099.stdout: altname enp7s0f0 2024-03-19T01:40:46.026 INFO:teuthology.orchestra.run.smithi099.stdout:3: eno2: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-19T01:40:46.026 INFO:teuthology.orchestra.run.smithi099.stdout: link/ether 0c:c4:7a:d8:a2:11 brd ff:ff:ff:ff:ff:ff 2024-03-19T01:40:46.026 INFO:teuthology.orchestra.run.smithi099.stdout: altname enp7s0f1 2024-03-19T01:40:46.026 INFO:teuthology.orchestra.run.smithi099.stdout:4: enp3s0f0: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-03-19T01:40:46.026 INFO:teuthology.orchestra.run.smithi099.stdout: link/ether 0c:c4:7a:8f:d8:02 brd ff:ff:ff:ff:ff:ff 2024-03-19T01:40:46.026 INFO:teuthology.orchestra.run.smithi099.stdout:5: enp3s0f1: mtu 1500 qdisc mq state UP group default qlen 1000 2024-03-19T01:40:46.026 INFO:teuthology.orchestra.run.smithi099.stdout: link/ether 0c:c4:7a:8f:d8:03 brd ff:ff:ff:ff:ff:ff 2024-03-19T01:40:46.027 INFO:teuthology.orchestra.run.smithi099.stdout: inet 172.21.15.99/20 brd 172.21.15.255 scope global dynamic noprefixroute enp3s0f1 2024-03-19T01:40:46.027 INFO:teuthology.orchestra.run.smithi099.stdout: valid_lft 40626sec preferred_lft 40626sec 2024-03-19T01:40:46.027 INFO:teuthology.orchestra.run.smithi099.stdout: inet6 fe80::ec4:7aff:fe8f:d803/64 scope link 2024-03-19T01:40:46.027 INFO:teuthology.orchestra.run.smithi099.stdout: valid_lft forever preferred_lft forever 2024-03-19T01:40:46.027 INFO:tasks.cephfs.mount:Setuping the 'ceph-brx' with 192.168.255.254/16 2024-03-19T01:40:46.027 DEBUG:teuthology.orchestra.run.smithi099:> (cd / && exec stdin-killer --timeout=300 -- bash -c ' 2024-03-19T01:40:46.027 DEBUG:teuthology.orchestra.run.smithi099:> set -e 2024-03-19T01:40:46.027 DEBUG:teuthology.orchestra.run.smithi099:> sudo ip link add name ceph-brx type bridge 2024-03-19T01:40:46.028 DEBUG:teuthology.orchestra.run.smithi099:> sudo ip addr flush dev ceph-brx 2024-03-19T01:40:46.028 DEBUG:teuthology.orchestra.run.smithi099:> sudo ip link set ceph-brx up 2024-03-19T01:40:46.028 DEBUG:teuthology.orchestra.run.smithi099:> sudo ip addr add 192.168.255.254/16 brd 192.168.255.255 dev ceph-brx 2024-03-19T01:40:46.028 DEBUG:teuthology.orchestra.run.smithi099:> ') 2024-03-19T01:40:46.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:45 smithi099 ceph-mon[62951]: pgmap v340: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-03-19T01:40:46.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:45 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/2236624579' entity='client.admin' cmd=[{"prefix": "fs ls"}]: dispatch 2024-03-19T01:40:46.116 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:46 stdin-killer INFO: expiration expected; waiting 300 seconds for command to complete 2024-03-19T01:40:46.221 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:46 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-03-19T01:40:46.227 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-19T01:40:46.227 DEBUG:teuthology.orchestra.run.smithi099:> echo 1 | sudo tee /proc/sys/net/ipv4/ip_forward 2024-03-19T01:40:46.294 INFO:teuthology.orchestra.run.smithi099.stdout:1 2024-03-19T01:40:46.295 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-19T01:40:46.295 DEBUG:teuthology.orchestra.run.smithi099:> route 2024-03-19T01:40:46.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:45 smithi115 ceph-mon[67332]: pgmap v340: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-03-19T01:40:46.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:45 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/2236624579' entity='client.admin' cmd=[{"prefix": "fs ls"}]: dispatch 2024-03-19T01:40:46.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:45 smithi112 ceph-mon[66819]: pgmap v340: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s 2024-03-19T01:40:46.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:45 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/2236624579' entity='client.admin' cmd=[{"prefix": "fs ls"}]: dispatch 2024-03-19T01:40:46.368 INFO:teuthology.orchestra.run.smithi099.stdout:Kernel IP routing table 2024-03-19T01:40:46.368 INFO:teuthology.orchestra.run.smithi099.stdout:Destination Gateway Genmask Flags Metric Ref Use Iface 2024-03-19T01:40:46.368 INFO:teuthology.orchestra.run.smithi099.stdout:default _gateway 0.0.0.0 UG 100 0 0 enp3s0f1 2024-03-19T01:40:46.368 INFO:teuthology.orchestra.run.smithi099.stdout:172.21.0.0 0.0.0.0 255.255.240.0 U 100 0 0 enp3s0f1 2024-03-19T01:40:46.368 INFO:teuthology.orchestra.run.smithi099.stdout:192.168.0.0 0.0.0.0 255.255.0.0 U 0 0 0 ceph-brx 2024-03-19T01:40:46.369 DEBUG:teuthology.orchestra.run.smithi099:> (cd / && exec stdin-killer --timeout=300 -- bash -c ' 2024-03-19T01:40:46.369 DEBUG:teuthology.orchestra.run.smithi099:> set -e 2024-03-19T01:40:46.369 DEBUG:teuthology.orchestra.run.smithi099:> sudo iptables -A FORWARD -o enp3s0f1 -i ceph-brx -j ACCEPT 2024-03-19T01:40:46.369 DEBUG:teuthology.orchestra.run.smithi099:> sudo iptables -A FORWARD -i enp3s0f1 -o ceph-brx -j ACCEPT 2024-03-19T01:40:46.369 DEBUG:teuthology.orchestra.run.smithi099:> sudo iptables -t nat -A POSTROUTING -s 192.168.255.254/16 -o enp3s0f1 -j MASQUERADE 2024-03-19T01:40:46.369 DEBUG:teuthology.orchestra.run.smithi099:> ') 2024-03-19T01:40:46.457 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:46 stdin-killer INFO: expiration expected; waiting 300 seconds for command to complete 2024-03-19T01:40:46.682 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:46 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-03-19T01:40:46.685 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-19T01:40:46.685 DEBUG:teuthology.orchestra.run.smithi099:> ip netns list 2024-03-19T01:40:46.700 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-19T01:40:46.700 DEBUG:teuthology.orchestra.run.smithi099:> ip netns list-id 2024-03-19T01:40:46.755 DEBUG:teuthology.orchestra.run.smithi099:> (cd / && exec stdin-killer --timeout=300 -- bash -c ' 2024-03-19T01:40:46.755 DEBUG:teuthology.orchestra.run.smithi099:> set -e 2024-03-19T01:40:46.755 DEBUG:teuthology.orchestra.run.smithi099:> sudo ip netns add ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-19T01:40:46.755 DEBUG:teuthology.orchestra.run.smithi099:> sudo ip netns set ceph-ns--home-ubuntu-cephtest-mnt.0 0 2024-03-19T01:40:46.755 DEBUG:teuthology.orchestra.run.smithi099:> ') 2024-03-19T01:40:46.843 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:46 stdin-killer INFO: expiration expected; waiting 300 seconds for command to complete 2024-03-19T01:40:46.871 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:46 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-03-19T01:40:46.878 INFO:tasks.cephfs.mount:Setuping the netns 'ceph-ns--home-ubuntu-cephtest-mnt.0' with 192.168.0.1/16 2024-03-19T01:40:46.878 DEBUG:teuthology.orchestra.run.smithi099:> (cd / && exec stdin-killer --timeout=300 -- bash -c ' 2024-03-19T01:40:46.878 DEBUG:teuthology.orchestra.run.smithi099:> set -e 2024-03-19T01:40:46.878 DEBUG:teuthology.orchestra.run.smithi099:> sudo ip link add veth0 netns ceph-ns--home-ubuntu-cephtest-mnt.0 type veth peer name brx.0 2024-03-19T01:40:46.878 DEBUG:teuthology.orchestra.run.smithi099:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.0 ip addr add 192.168.0.1/16 brd 192.168.255.255 dev veth0 2024-03-19T01:40:46.878 DEBUG:teuthology.orchestra.run.smithi099:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.0 ip link set veth0 up 2024-03-19T01:40:46.879 DEBUG:teuthology.orchestra.run.smithi099:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.0 ip link set lo up 2024-03-19T01:40:46.879 DEBUG:teuthology.orchestra.run.smithi099:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.0 ip route add default via 192.168.255.254 2024-03-19T01:40:46.879 DEBUG:teuthology.orchestra.run.smithi099:> ') 2024-03-19T01:40:46.928 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:46 stdin-killer INFO: expiration expected; waiting 300 seconds for command to complete 2024-03-19T01:40:47.009 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:47 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-03-19T01:40:47.016 DEBUG:teuthology.orchestra.run.smithi099:> (cd / && exec stdin-killer --timeout=300 -- bash -c ' 2024-03-19T01:40:47.016 DEBUG:teuthology.orchestra.run.smithi099:> set -e 2024-03-19T01:40:47.016 DEBUG:teuthology.orchestra.run.smithi099:> sudo ip link set brx.0 up 2024-03-19T01:40:47.016 DEBUG:teuthology.orchestra.run.smithi099:> sudo ip link set dev brx.0 master ceph-brx 2024-03-19T01:40:47.016 DEBUG:teuthology.orchestra.run.smithi099:> ') 2024-03-19T01:40:47.064 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:47 stdin-killer INFO: expiration expected; waiting 300 seconds for command to complete 2024-03-19T01:40:47.092 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:47 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-03-19T01:40:47.098 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-19T01:40:47.098 DEBUG:teuthology.orchestra.run.smithi099:> mkdir -p -v /home/ubuntu/cephtest/mnt.0 2024-03-19T01:40:47.112 INFO:teuthology.orchestra.run.smithi099.stdout:mkdir: created directory '/home/ubuntu/cephtest/mnt.0' 2024-03-19T01:40:47.112 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-03-19T01:40:47.113 DEBUG:teuthology.orchestra.run.smithi099:> chmod 0000 /home/ubuntu/cephtest/mnt.0 2024-03-19T01:40:47.167 WARNING:tasks.cephfs.mount:You cannot specify both: cephfs_mntpt and mount_subvol_num 2024-03-19T01:40:47.167 INFO:tasks.cephfs.mount:Mounting subvol 1 for now 2024-03-19T01:40:47.168 INFO:tasks.cephfs.kernel_mount:mounting using device: :/volumes/qa/sv_1/6b9b1809-7324-48f8-a03e-5dbb285d942e 2024-03-19T01:40:47.168 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-19T01:40:47.168 DEBUG:teuthology.orchestra.run.smithi099:> sudo nsenter --net=/var/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage /bin/mount -t ceph :/volumes/qa/sv_1/6b9b1809-7324-48f8-a03e-5dbb285d942e /home/ubuntu/cephtest/mnt.0 -v -o norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,name=0,mds_namespace=cephfs,ms_mode=crc,nowsync 2024-03-19T01:40:47.252 INFO:teuthology.orchestra.run.smithi099.stdout:parsing options: rw,norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,name=0,mds_namespace=cephfs,ms_mode=crc,nowsync 2024-03-19T01:40:47.252 INFO:teuthology.orchestra.run.smithi099.stdout:mount.ceph: options "norequire_active_mds,norbytes,name=0,mds_namespace=cephfs,ms_mode=crc,nowsync". 2024-03-19T01:40:47.252 INFO:teuthology.orchestra.run.smithi099.stdout:invalid new device string format 2024-03-19T01:40:47.389 INFO:teuthology.orchestra.run.smithi099.stdout:parsing options: rw,norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,name=0,mds_namespace=cephfs,ms_mode=crc,nowsync 2024-03-19T01:40:47.389 INFO:teuthology.orchestra.run.smithi099.stdout:mount.ceph: options "norequire_active_mds,norbytes,name=0,mds_namespace=cephfs,ms_mode=crc,nowsync". 2024-03-19T01:40:47.389 INFO:teuthology.orchestra.run.smithi099.stdout:invalid new device string format 2024-03-19T01:40:47.389 INFO:teuthology.orchestra.run.smithi099.stdout:mount.ceph: resolved to: "172.21.15.99:3300,172.21.15.112:3300,172.21.15.115:3300" 2024-03-19T01:40:47.389 INFO:teuthology.orchestra.run.smithi099.stdout:mount.ceph: trying mount with old device syntax: 172.21.15.99:3300,172.21.15.112:3300,172.21.15.115:3300:/volumes/qa/sv_1/6b9b1809-7324-48f8-a03e-5dbb285d942e 2024-03-19T01:40:47.389 INFO:teuthology.orchestra.run.smithi099.stdout:mount.ceph: options "norequire_active_mds,norbytes,name=0,mds_namespace=cephfs,ms_mode=crc,nowsync,key=0,fsid=438fb69c-e590-11ee-95c9-87774f69a715" will pass to kernel 2024-03-19T01:40:47.390 INFO:teuthology.orchestra.run.smithi099.stdout:mount: /home/ubuntu/cephtest/mnt.0 does not contain SELinux labels. 2024-03-19T01:40:47.390 INFO:teuthology.orchestra.run.smithi099.stdout: You just mounted a file system that supports labels which does not 2024-03-19T01:40:47.390 INFO:teuthology.orchestra.run.smithi099.stdout: contain labels, onto an SELinux box. It is likely that confined 2024-03-19T01:40:47.390 INFO:teuthology.orchestra.run.smithi099.stdout: applications will generate AVC messages and not be allowed access to 2024-03-19T01:40:47.390 INFO:teuthology.orchestra.run.smithi099.stdout: this file system. For more details see restorecon(8) and mount(8). 2024-03-19T01:40:47.392 INFO:tasks.cephfs.kernel_mount:mount command passed 2024-03-19T01:40:47.392 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-19T01:40:47.392 DEBUG:teuthology.orchestra.run.smithi099:> sudo chmod 1777 /home/ubuntu/cephtest/mnt.0 2024-03-19T01:40:47.460 DEBUG:teuthology.orchestra.run.smithi099:> (cd /home/ubuntu/cephtest/mnt.0 && exec stdin-killer --timeout=900 -- bash -c 'getfattr --only-values -n ceph.client_id .') 2024-03-19T01:40:47.548 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:47 stdin-killer INFO: expiration expected; waiting 900 seconds for command to complete 2024-03-19T01:40:47.549 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:47 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-03-19T01:40:47.555 INFO:teuthology.orchestra.run.smithi099.stdout:client25120 2024-03-19T01:40:47.555 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2024-03-19T01:40:47.555 DEBUG:teuthology.orchestra.run.smithi099:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-19T01:40:47.572 DEBUG:teuthology.orchestra.run.smithi099:> (cd /home/ubuntu/cephtest/mnt.0 && exec stdin-killer --timeout=900 -- bash -c 'getfattr --only-values -n ceph.client_id .') 2024-03-19T01:40:47.660 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:47 stdin-killer INFO: expiration expected; waiting 900 seconds for command to complete 2024-03-19T01:40:47.661 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:47 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-03-19T01:40:47.668 INFO:teuthology.orchestra.run.smithi099.stdout:client25120 2024-03-19T01:40:47.668 DEBUG:teuthology.orchestra.run.smithi099:> (cd /home/ubuntu/cephtest/mnt.0 && exec stdin-killer --timeout=900 -- bash -c 'getfattr --only-values -n ceph.client_id .') 2024-03-19T01:40:47.716 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:47 stdin-killer INFO: expiration expected; waiting 900 seconds for command to complete 2024-03-19T01:40:47.718 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:47 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-03-19T01:40:47.723 INFO:teuthology.orchestra.run.smithi099.stdout:client25120 2024-03-19T01:40:47.723 DEBUG:teuthology.orchestra.run.smithi099:> (cd /home/ubuntu/cephtest/mnt.0 && exec stdin-killer --timeout=300 -- bash -c 'sudo dd if=/sys/kernel/debug/ceph/438fb69c-e590-11ee-95c9-87774f69a715.client25120/status') 2024-03-19T01:40:47.771 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:47 stdin-killer INFO: expiration expected; waiting 300 seconds for command to complete 2024-03-19T01:40:47.784 INFO:teuthology.orchestra.run.smithi099.stdout:instance: client.25120 (3)172.21.15.99:0/2815080355 2024-03-19T01:40:47.784 INFO:teuthology.orchestra.run.smithi099.stdout:blocklisted: false 2024-03-19T01:40:47.785 INFO:teuthology.orchestra.run.smithi099.stderr:0+1 records in 2024-03-19T01:40:47.785 INFO:teuthology.orchestra.run.smithi099.stderr:0+1 records out 2024-03-19T01:40:47.785 INFO:teuthology.orchestra.run.smithi099.stderr:71 bytes copied, 2.054e-05 s, 3.5 MB/s 2024-03-19T01:40:47.785 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:47 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-03-19T01:40:47.792 INFO:teuthology.run_tasks:Running task exec... 2024-03-19T01:40:47.814 INFO:teuthology.task.exec:Executing custom commands... 2024-03-19T01:40:47.815 INFO:teuthology.task.exec:Running commands on role mon.a host ubuntu@smithi099.front.sepia.ceph.com 2024-03-19T01:40:47.815 DEBUG:teuthology.orchestra.run.smithi099:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph fs set cephfs balance_automate false' 2024-03-19T01:40:47.926 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:47.920+0000 7fb7bbeea640 1 -- 172.21.15.99:0/1172511915 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb7b4101970 msgr2=0x7fb7b4101df0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:47.926 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:47.920+0000 7fb7bbeea640 1 --2- 172.21.15.99:0/1172511915 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb7b4101970 0x7fb7b4101df0 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x7fb7a4009860 tx=0x7fb7a402bb70 comp rx=0 tx=0).stop 2024-03-19T01:40:47.926 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:47.920+0000 7fb7bbeea640 1 -- 172.21.15.99:0/1172511915 shutdown_connections 2024-03-19T01:40:47.927 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:47.920+0000 7fb7bbeea640 1 --2- 172.21.15.99:0/1172511915 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb7b4102330 0x7fb7b4108bc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:47.927 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:47.920+0000 7fb7bbeea640 1 --2- 172.21.15.99:0/1172511915 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb7b4101970 0x7fb7b4101df0 unknown :-1 s=CLOSED pgs=74 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:47.927 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:47.920+0000 7fb7bbeea640 1 --2- 172.21.15.99:0/1172511915 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb7b4100780 0x7fb7b4100ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:47.927 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:47.920+0000 7fb7bbeea640 1 -- 172.21.15.99:0/1172511915 >> 172.21.15.99:0/1172511915 conn(0x7fb7b40fbcf0 msgr2=0x7fb7b40fe130 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:47.927 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:47.920+0000 7fb7bbeea640 1 -- 172.21.15.99:0/1172511915 shutdown_connections 2024-03-19T01:40:47.928 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:47.920+0000 7fb7bbeea640 1 -- 172.21.15.99:0/1172511915 wait complete. 2024-03-19T01:40:47.928 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:47.920+0000 7fb7bbeea640 1 Processor -- start 2024-03-19T01:40:47.928 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:47.920+0000 7fb7bbeea640 1 -- start start 2024-03-19T01:40:47.928 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:47.920+0000 7fb7bbeea640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb7b4100780 0x7fb7b4197b10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:47.928 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:47.920+0000 7fb7bbeea640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb7b4101970 0x7fb7b4198050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:47.928 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:47.920+0000 7fb7bbeea640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb7b4102330 0x7fb7b419f070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:47.929 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:47.920+0000 7fb7bbeea640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fb7b410c400 con 0x7fb7b4101970 2024-03-19T01:40:47.929 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:47.920+0000 7fb7bbeea640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fb7b410c280 con 0x7fb7b4102330 2024-03-19T01:40:47.929 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:47.920+0000 7fb7bbeea640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fb7b410c580 con 0x7fb7b4100780 2024-03-19T01:40:47.929 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:47.921+0000 7fb7b9485640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb7b4101970 0x7fb7b4198050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:47.929 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:47.921+0000 7fb7b9485640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb7b4101970 0x7fb7b4198050 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58304/0 (socket says 172.21.15.99:58304) 2024-03-19T01:40:47.930 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:47.921+0000 7fb7b9485640 1 -- 172.21.15.99:0/3720764319 learned_addr learned my addr 172.21.15.99:0/3720764319 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:47.930 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:47.921+0000 7fb7ba487640 1 --2- 172.21.15.99:0/3720764319 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb7b4102330 0x7fb7b419f070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:47.930 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:47.921+0000 7fb7b9c86640 1 --2- 172.21.15.99:0/3720764319 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb7b4100780 0x7fb7b4197b10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:47.930 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:47.921+0000 7fb7b9485640 1 -- 172.21.15.99:0/3720764319 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb7b4100780 msgr2=0x7fb7b4197b10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:47.930 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:47.921+0000 7fb7b9485640 1 --2- 172.21.15.99:0/3720764319 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb7b4100780 0x7fb7b4197b10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:47.930 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:47.921+0000 7fb7b9485640 1 -- 172.21.15.99:0/3720764319 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb7b4102330 msgr2=0x7fb7b419f070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:47.931 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:47.921+0000 7fb7b9485640 1 --2- 172.21.15.99:0/3720764319 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb7b4102330 0x7fb7b419f070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:47.931 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:47.921+0000 7fb7b9485640 1 -- 172.21.15.99:0/3720764319 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb7b419f6e0 con 0x7fb7b4101970 2024-03-19T01:40:47.931 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:47.921+0000 7fb7ba487640 1 --2- 172.21.15.99:0/3720764319 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb7b4102330 0x7fb7b419f070 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:40:47.931 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:47.921+0000 7fb7b9485640 1 --2- 172.21.15.99:0/3720764319 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb7b4101970 0x7fb7b4198050 secure :-1 s=READY pgs=349 cs=0 l=1 rev1=1 crypto rx=0x7fb7a4005940 tx=0x7fb7a4032470 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:47.931 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:47.921+0000 7fb7aaffd640 1 -- 172.21.15.99:0/3720764319 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb7a4041020 con 0x7fb7b4101970 2024-03-19T01:40:47.931 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:47.921+0000 7fb7bbeea640 1 -- 172.21.15.99:0/3720764319 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb7b419f970 con 0x7fb7b4101970 2024-03-19T01:40:47.932 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:47.921+0000 7fb7aaffd640 1 -- 172.21.15.99:0/3720764319 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fb7a402c9f0 con 0x7fb7b4101970 2024-03-19T01:40:47.932 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:47.921+0000 7fb7aaffd640 1 -- 172.21.15.99:0/3720764319 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb7a4039040 con 0x7fb7b4101970 2024-03-19T01:40:47.932 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:47.921+0000 7fb7bbeea640 1 -- 172.21.15.99:0/3720764319 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fb7b419fe60 con 0x7fb7b4101970 2024-03-19T01:40:47.932 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:47.922+0000 7fb7bbeea640 1 -- 172.21.15.99:0/3720764319 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fb7b41a0090 con 0x7fb7b4101970 2024-03-19T01:40:47.932 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:47.923+0000 7fb7aaffd640 1 -- 172.21.15.99:0/3720764319 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 18) ==== 137290+0+0 (secure 0 0 0) 0x7fb7a4044020 con 0x7fb7b4101970 2024-03-19T01:40:47.932 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:47.924+0000 7fb7aaffd640 1 --2- 172.21.15.99:0/3720764319 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb7a00a28a0 0x7fb7a00a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:47.933 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:47.924+0000 7fb7b9c86640 1 --2- 172.21.15.99:0/3720764319 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb7a00a28a0 0x7fb7a00a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:47.933 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:47.924+0000 7fb7aaffd640 1 -- 172.21.15.99:0/3720764319 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7fb7a40f0450 con 0x7fb7b4101970 2024-03-19T01:40:47.933 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:47.924+0000 7fb7b9c86640 1 --2- 172.21.15.99:0/3720764319 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb7a00a28a0 0x7fb7a00a4d60 secure :-1 s=READY pgs=245 cs=0 l=1 rev1=1 crypto rx=0x7fb7b41017d0 tx=0x7fb7b0009500 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:47.934 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:47.928+0000 7fb7aaffd640 1 -- 172.21.15.99:0/3720764319 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7fb7a404b020 con 0x7fb7b4101970 2024-03-19T01:40:48.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:47 smithi099 ceph-mon[62951]: pgmap v341: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 6 op/s 2024-03-19T01:40:48.141 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:48.140+0000 7fb7bbeea640 1 -- 172.21.15.99:0/3720764319 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "balance_automate", "val": "false"} v 0) -- 0x7fb7b41a0260 con 0x7fb7b4101970 2024-03-19T01:40:48.163 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:47 smithi112 ceph-mon[66819]: pgmap v341: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 6 op/s 2024-03-19T01:40:48.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:47 smithi115 ceph-mon[67332]: pgmap v341: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 6 op/s 2024-03-19T01:40:48.908 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:48.908+0000 7fb7aaffd640 1 -- 172.21.15.99:0/3720764319 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "balance_automate", "val": "false"}]=0 v21) ==== 118+0+0 (secure 0 0 0) 0x7fb7a40bd4b0 con 0x7fb7b4101970 2024-03-19T01:40:48.912 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:48.911+0000 7fb7bbeea640 1 -- 172.21.15.99:0/3720764319 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb7a00a28a0 msgr2=0x7fb7a00a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:48.912 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:48.911+0000 7fb7bbeea640 1 --2- 172.21.15.99:0/3720764319 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb7a00a28a0 0x7fb7a00a4d60 secure :-1 s=READY pgs=245 cs=0 l=1 rev1=1 crypto rx=0x7fb7b41017d0 tx=0x7fb7b0009500 comp rx=0 tx=0).stop 2024-03-19T01:40:48.912 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:48.911+0000 7fb7bbeea640 1 -- 172.21.15.99:0/3720764319 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb7b4101970 msgr2=0x7fb7b4198050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:48.912 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:48.911+0000 7fb7bbeea640 1 --2- 172.21.15.99:0/3720764319 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb7b4101970 0x7fb7b4198050 secure :-1 s=READY pgs=349 cs=0 l=1 rev1=1 crypto rx=0x7fb7a4005940 tx=0x7fb7a4032470 comp rx=0 tx=0).stop 2024-03-19T01:40:48.912 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:48.911+0000 7fb7bbeea640 1 -- 172.21.15.99:0/3720764319 shutdown_connections 2024-03-19T01:40:48.912 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:48.911+0000 7fb7bbeea640 1 --2- 172.21.15.99:0/3720764319 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb7a00a28a0 0x7fb7a00a4d60 unknown :-1 s=CLOSED pgs=245 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:48.912 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:48.911+0000 7fb7bbeea640 1 --2- 172.21.15.99:0/3720764319 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb7b4102330 0x7fb7b419f070 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:48.912 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:48.911+0000 7fb7bbeea640 1 --2- 172.21.15.99:0/3720764319 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb7b4101970 0x7fb7b4198050 unknown :-1 s=CLOSED pgs=349 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:48.912 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:48.911+0000 7fb7bbeea640 1 --2- 172.21.15.99:0/3720764319 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb7b4100780 0x7fb7b4197b10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:48.912 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:48.911+0000 7fb7bbeea640 1 -- 172.21.15.99:0/3720764319 >> 172.21.15.99:0/3720764319 conn(0x7fb7b40fbcf0 msgr2=0x7fb7b406a000 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:48.912 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:48.912+0000 7fb7bbeea640 1 -- 172.21.15.99:0/3720764319 shutdown_connections 2024-03-19T01:40:48.912 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:48.912+0000 7fb7bbeea640 1 -- 172.21.15.99:0/3720764319 wait complete. 2024-03-19T01:40:48.925 DEBUG:teuthology.orchestra.run.smithi099:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph fs subvolumegroup pin cephfs qa random 0.10' 2024-03-19T01:40:49.037 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:49.034+0000 7f7088710640 1 -- 172.21.15.99:0/3574629727 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f7080101930 msgr2=0x7f7080101db0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:49.037 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:49.034+0000 7f7088710640 1 --2- 172.21.15.99:0/3574629727 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f7080101930 0x7f7080101db0 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x7f7070009860 tx=0x7f707002bb70 comp rx=0 tx=0).stop 2024-03-19T01:40:49.037 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:49.034+0000 7f7088710640 1 -- 172.21.15.99:0/3574629727 shutdown_connections 2024-03-19T01:40:49.037 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:49.034+0000 7f7088710640 1 --2- 172.21.15.99:0/3574629727 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f70801022f0 0x7f7080108b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:49.037 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:49.034+0000 7f7088710640 1 --2- 172.21.15.99:0/3574629727 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f7080101930 0x7f7080101db0 unknown :-1 s=CLOSED pgs=75 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:49.037 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:49.034+0000 7f7088710640 1 --2- 172.21.15.99:0/3574629727 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f7080100740 0x7f7080100b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:49.037 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:49.034+0000 7f7088710640 1 -- 172.21.15.99:0/3574629727 >> 172.21.15.99:0/3574629727 conn(0x7f70800fbcb0 msgr2=0x7f70800fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:49.037 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:49.034+0000 7f7088710640 1 -- 172.21.15.99:0/3574629727 shutdown_connections 2024-03-19T01:40:49.037 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:49.034+0000 7f7088710640 1 -- 172.21.15.99:0/3574629727 wait complete. 2024-03-19T01:40:49.038 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:49.035+0000 7f7088710640 1 Processor -- start 2024-03-19T01:40:49.038 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:49.035+0000 7f7088710640 1 -- start start 2024-03-19T01:40:49.038 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:49.035+0000 7f7088710640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7080100740 0x7f70801969d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:49.038 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:49.035+0000 7f7088710640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f7080101930 0x7f7080196f10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:49.038 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:49.035+0000 7f7088710640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f70801022f0 0x7f708019df80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:49.038 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:49.035+0000 7f7088710640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f708010c3c0 con 0x7f7080100740 2024-03-19T01:40:49.038 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:49.035+0000 7f7088710640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f708010c240 con 0x7f70801022f0 2024-03-19T01:40:49.038 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:49.035+0000 7f7088710640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f708010c540 con 0x7f7080101930 2024-03-19T01:40:49.038 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:49.035+0000 7f70864ac640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7080100740 0x7f70801969d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:49.038 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:49.035+0000 7f70864ac640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7080100740 0x7f70801969d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58324/0 (socket says 172.21.15.99:58324) 2024-03-19T01:40:49.038 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:49.035+0000 7f70864ac640 1 -- 172.21.15.99:0/566288944 learned_addr learned my addr 172.21.15.99:0/566288944 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:40:49.038 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:49.035+0000 7f7085cab640 1 --2- 172.21.15.99:0/566288944 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f7080101930 0x7f7080196f10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:49.038 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:49.035+0000 7f7086cad640 1 --2- 172.21.15.99:0/566288944 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f70801022f0 0x7f708019df80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:49.039 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:49.035+0000 7f70864ac640 1 -- 172.21.15.99:0/566288944 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f7080101930 msgr2=0x7f7080196f10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:49.039 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:49.035+0000 7f70864ac640 1 --2- 172.21.15.99:0/566288944 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f7080101930 0x7f7080196f10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:49.039 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:49.035+0000 7f70864ac640 1 -- 172.21.15.99:0/566288944 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f70801022f0 msgr2=0x7f708019df80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:49.039 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:49.035+0000 7f70864ac640 1 --2- 172.21.15.99:0/566288944 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f70801022f0 0x7f708019df80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:49.039 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:49.036+0000 7f70864ac640 1 -- 172.21.15.99:0/566288944 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f708019e5f0 con 0x7f7080100740 2024-03-19T01:40:49.039 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:49.036+0000 7f7085cab640 1 --2- 172.21.15.99:0/566288944 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f7080101930 0x7f7080196f10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:40:49.039 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:49.036+0000 7f70864ac640 1 --2- 172.21.15.99:0/566288944 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7080100740 0x7f70801969d0 secure :-1 s=READY pgs=350 cs=0 l=1 rev1=1 crypto rx=0x7f707c00e9c0 tx=0x7f707c00ee80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:49.039 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:49.036+0000 7f70777fe640 1 -- 172.21.15.99:0/566288944 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f707c00cd80 con 0x7f7080100740 2024-03-19T01:40:49.039 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:49.036+0000 7f70777fe640 1 -- 172.21.15.99:0/566288944 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f707c0035f0 con 0x7f7080100740 2024-03-19T01:40:49.039 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:49.036+0000 7f7088710640 1 -- 172.21.15.99:0/566288944 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f708006a0c0 con 0x7f7080100740 2024-03-19T01:40:49.039 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:49.036+0000 7f70777fe640 1 -- 172.21.15.99:0/566288944 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f707c010690 con 0x7f7080100740 2024-03-19T01:40:49.039 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:49.036+0000 7f7088710640 1 -- 172.21.15.99:0/566288944 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f708006a590 con 0x7f7080100740 2024-03-19T01:40:49.039 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:49.037+0000 7f7088710640 1 -- 172.21.15.99:0/566288944 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f708004f580 con 0x7f7080100740 2024-03-19T01:40:49.040 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:49.038+0000 7f70777fe640 1 -- 172.21.15.99:0/566288944 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 18) ==== 137290+0+0 (secure 0 0 0) 0x7f707c0037a0 con 0x7f7080100740 2024-03-19T01:40:49.040 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:49.038+0000 7f70777fe640 1 --2- 172.21.15.99:0/566288944 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f706c0a28a0 0x7f706c0a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:40:49.040 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:49.039+0000 7f7085cab640 1 --2- 172.21.15.99:0/566288944 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f706c0a28a0 0x7f706c0a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:40:49.040 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:49.039+0000 7f70777fe640 1 -- 172.21.15.99:0/566288944 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f707c0cee00 con 0x7f7080100740 2024-03-19T01:40:49.040 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:49.039+0000 7f7085cab640 1 --2- 172.21.15.99:0/566288944 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f706c0a28a0 0x7f706c0a4d60 secure :-1 s=READY pgs=246 cs=0 l=1 rev1=1 crypto rx=0x7f7070009830 tx=0x7f7070034ab0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:40:49.043 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:49.043+0000 7f70777fe640 1 -- 172.21.15.99:0/566288944 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f707c09be60 con 0x7f7080100740 2024-03-19T01:40:49.245 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:49.245+0000 7f7088710640 1 -- 172.21.15.99:0/566288944 --> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] -- mgr_command(tid 0: {"prefix": "fs subvolumegroup pin", "vol_name": "cephfs", "group_name": "qa", "pin_type": "random", "pin_setting": "0.10", "target": ["mon-mgr", ""]}) -- 0x7f708006a8f0 con 0x7f706c0a28a0 2024-03-19T01:40:49.250 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:48 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/3720764319' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "balance_automate", "val": "false"}]: dispatch 2024-03-19T01:40:49.260 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:49.260+0000 7f70777fe640 1 -- 172.21.15.99:0/566288944 <== mgr.14150 v2:172.21.15.99:6800/4272410295 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+3 (secure 0 0 0) 0x7f708006a8f0 con 0x7f706c0a28a0 2024-03-19T01:40:49.260 INFO:teuthology.orchestra.run.smithi099.stdout:{} 2024-03-19T01:40:49.262 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:49.262+0000 7f7088710640 1 -- 172.21.15.99:0/566288944 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f706c0a28a0 msgr2=0x7f706c0a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:49.262 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:49.262+0000 7f7088710640 1 --2- 172.21.15.99:0/566288944 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f706c0a28a0 0x7f706c0a4d60 secure :-1 s=READY pgs=246 cs=0 l=1 rev1=1 crypto rx=0x7f7070009830 tx=0x7f7070034ab0 comp rx=0 tx=0).stop 2024-03-19T01:40:49.263 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:49.262+0000 7f7088710640 1 -- 172.21.15.99:0/566288944 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7080100740 msgr2=0x7f70801969d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:40:49.263 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:49.262+0000 7f7088710640 1 --2- 172.21.15.99:0/566288944 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7080100740 0x7f70801969d0 secure :-1 s=READY pgs=350 cs=0 l=1 rev1=1 crypto rx=0x7f707c00e9c0 tx=0x7f707c00ee80 comp rx=0 tx=0).stop 2024-03-19T01:40:49.263 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:49.262+0000 7f7088710640 1 -- 172.21.15.99:0/566288944 shutdown_connections 2024-03-19T01:40:49.263 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:49.262+0000 7f7088710640 1 --2- 172.21.15.99:0/566288944 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f706c0a28a0 0x7f706c0a4d60 unknown :-1 s=CLOSED pgs=246 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:49.263 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:49.262+0000 7f7088710640 1 --2- 172.21.15.99:0/566288944 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f70801022f0 0x7f708019df80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:49.263 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:49.262+0000 7f7088710640 1 --2- 172.21.15.99:0/566288944 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f7080101930 0x7f7080196f10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:49.263 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:49.262+0000 7f7088710640 1 --2- 172.21.15.99:0/566288944 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7080100740 0x7f70801969d0 unknown :-1 s=CLOSED pgs=350 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:40:49.263 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:49.262+0000 7f7088710640 1 -- 172.21.15.99:0/566288944 >> 172.21.15.99:0/566288944 conn(0x7f70800fbcb0 msgr2=0x7f7080104bd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:40:49.263 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:49.262+0000 7f7088710640 1 -- 172.21.15.99:0/566288944 shutdown_connections 2024-03-19T01:40:49.263 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T01:40:49.262+0000 7f7088710640 1 -- 172.21.15.99:0/566288944 wait complete. 2024-03-19T01:40:49.273 INFO:teuthology.run_tasks:Running task check-counter... 2024-03-19T01:40:49.285 INFO:teuthology.run_tasks:Running task background_exec... 2024-03-19T01:40:49.322 INFO:teuthology.task.background_exec:Running background command on role mon.a host ubuntu@smithi099.front.sepia.ceph.com 2024-03-19T01:40:49.323 DEBUG:teuthology.orchestra.run.smithi099:> sudo TESTDIR=/home/ubuntu/cephtest daemon-helper kill --kill-group bash -c 'while sleep 13; do ceph tell mds.cephfs:0 flush journal; done' 2024-03-19T01:40:49.323 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:48 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/3720764319' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "balance_automate", "val": "false"}]: dispatch 2024-03-19T01:40:49.325 INFO:teuthology.run_tasks:Running task workunit... 2024-03-19T01:40:49.359 INFO:tasks.workunit:Pulling workunits from ref e48b1f6c0608b2615e1890cc46286c3877d6e910 2024-03-19T01:40:49.359 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-03-19T01:40:49.359 INFO:tasks.workunit:timeout=3h 2024-03-19T01:40:49.359 INFO:tasks.workunit:cleanup=True 2024-03-19T01:40:49.360 DEBUG:teuthology.orchestra.run.smithi099:> stat -- /home/ubuntu/cephtest/mnt.0 2024-03-19T01:40:49.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:48 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/3720764319' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "balance_automate", "val": "false"}]: dispatch 2024-03-19T01:40:49.375 INFO:teuthology.orchestra.run.smithi099.stdout: File: /home/ubuntu/cephtest/mnt.0 2024-03-19T01:40:49.375 INFO:teuthology.orchestra.run.smithi099.stdout: Size: 0 Blocks: 0 IO Block: 65536 directory 2024-03-19T01:40:49.375 INFO:teuthology.orchestra.run.smithi099.stdout:Device: 77h/119d Inode: 1099511628283 Links: 2 2024-03-19T01:40:49.375 INFO:teuthology.orchestra.run.smithi099.stdout:Access: (1777/drwxrwxrwt) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-19T01:40:49.375 INFO:teuthology.orchestra.run.smithi099.stdout:Context: system_u:object_r:unlabeled_t:s0 2024-03-19T01:40:49.375 INFO:teuthology.orchestra.run.smithi099.stdout:Access: 2024-03-19 01:39:01.706658749 +0000 2024-03-19T01:40:49.375 INFO:teuthology.orchestra.run.smithi099.stdout:Modify: 2024-03-19 01:39:01.706658749 +0000 2024-03-19T01:40:49.375 INFO:teuthology.orchestra.run.smithi099.stdout:Change: 2024-03-19 01:40:47.457858131 +0000 2024-03-19T01:40:49.375 INFO:teuthology.orchestra.run.smithi099.stdout: Birth: 2024-03-19 01:39:01.706658749 +0000 2024-03-19T01:40:49.375 INFO:tasks.workunit:Did not need to create dir /home/ubuntu/cephtest/mnt.0 2024-03-19T01:40:49.376 DEBUG:teuthology.orchestra.run.smithi099:> cd -- /home/ubuntu/cephtest/mnt.0 && sudo install -d -m 0755 --owner=ubuntu -- client.0 2024-03-19T01:40:49.448 DEBUG:teuthology.orchestra.run.smithi099:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone https://github.com/batrick/ceph /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 && git checkout e48b1f6c0608b2615e1890cc46286c3877d6e910 2024-03-19T01:40:49.504 INFO:tasks.workunit.client.0.smithi099.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2024-03-19T01:40:49.591 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:40:49 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:40:49.249+0000 7f67c3a45700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-19T01:40:49.591 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:40:49 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:40:49.249+0000 7f67c3a45700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-19T01:40:49.591 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:40:49 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:40:49.249+0000 7f67c3a45700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-19T01:40:49.591 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:40:49 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:40:49.249+0000 7f67c3a45700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-19T01:40:49.591 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 01:40:49 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T01:40:49.249+0000 7f67c3a45700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-19T01:40:50.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:49 smithi115 ceph-mon[67332]: pgmap v342: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 6 op/s 2024-03-19T01:40:50.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:49 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/3720764319' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "balance_automate", "val": "false"}]': finished 2024-03-19T01:40:50.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:49 smithi115 ceph-mon[67332]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T01:40:50.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:49 smithi099 ceph-mon[62951]: pgmap v342: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 6 op/s 2024-03-19T01:40:50.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:49 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/3720764319' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "balance_automate", "val": "false"}]': finished 2024-03-19T01:40:50.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:49 smithi099 ceph-mon[62951]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T01:40:50.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:49 smithi112 ceph-mon[66819]: pgmap v342: 97 pgs: 97 active+clean; 640 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 6 op/s 2024-03-19T01:40:50.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:49 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/3720764319' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "balance_automate", "val": "false"}]': finished 2024-03-19T01:40:50.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:49 smithi112 ceph-mon[66819]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T01:40:51.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:50 smithi099 ceph-mon[62951]: from='client.15306 -' entity='client.admin' cmd=[{"prefix": "fs subvolumegroup pin", "vol_name": "cephfs", "group_name": "qa", "pin_type": "random", "pin_setting": "0.10", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:40:51.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:50 smithi099 ceph-mon[62951]: mgrmap e19: x(active, since 8m), standbys: z, y 2024-03-19T01:40:51.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:50 smithi115 ceph-mon[67332]: from='client.15306 -' entity='client.admin' cmd=[{"prefix": "fs subvolumegroup pin", "vol_name": "cephfs", "group_name": "qa", "pin_type": "random", "pin_setting": "0.10", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:40:51.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:50 smithi115 ceph-mon[67332]: mgrmap e19: x(active, since 8m), standbys: z, y 2024-03-19T01:40:51.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:50 smithi112 ceph-mon[66819]: from='client.15306 -' entity='client.admin' cmd=[{"prefix": "fs subvolumegroup pin", "vol_name": "cephfs", "group_name": "qa", "pin_type": "random", "pin_setting": "0.10", "target": ["mon-mgr", ""]}]: dispatch 2024-03-19T01:40:51.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:50 smithi112 ceph-mon[66819]: mgrmap e19: x(active, since 8m), standbys: z, y 2024-03-19T01:40:52.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:51 smithi115 ceph-mon[67332]: pgmap v343: 97 pgs: 97 active+clean; 653 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 4.5 KiB/s rd, 1.4 KiB/s wr, 6 op/s 2024-03-19T01:40:52.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:51 smithi099 ceph-mon[62951]: pgmap v343: 97 pgs: 97 active+clean; 653 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 4.5 KiB/s rd, 1.4 KiB/s wr, 6 op/s 2024-03-19T01:40:52.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:51 smithi112 ceph-mon[66819]: pgmap v343: 97 pgs: 97 active+clean; 653 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 4.5 KiB/s rd, 1.4 KiB/s wr, 6 op/s 2024-03-19T01:40:53.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:52 smithi115 ceph-mon[67332]: pgmap v344: 97 pgs: 97 active+clean; 654 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 1.9 KiB/s wr, 7 op/s 2024-03-19T01:40:53.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:52 smithi099 ceph-mon[62951]: pgmap v344: 97 pgs: 97 active+clean; 654 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 1.9 KiB/s wr, 7 op/s 2024-03-19T01:40:53.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:52 smithi112 ceph-mon[66819]: pgmap v344: 97 pgs: 97 active+clean; 654 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 1.9 KiB/s wr, 7 op/s 2024-03-19T01:40:56.095 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:55 smithi099 ceph-mon[62951]: pgmap v345: 97 pgs: 97 active+clean; 654 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.8 KiB/s rd, 1.9 KiB/s wr, 6 op/s 2024-03-19T01:40:56.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:55 smithi112 ceph-mon[66819]: pgmap v345: 97 pgs: 97 active+clean; 654 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.8 KiB/s rd, 1.9 KiB/s wr, 6 op/s 2024-03-19T01:40:56.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:55 smithi115 ceph-mon[67332]: pgmap v345: 97 pgs: 97 active+clean; 654 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.8 KiB/s rd, 1.9 KiB/s wr, 6 op/s 2024-03-19T01:40:58.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:57 smithi099 ceph-mon[62951]: pgmap v346: 97 pgs: 97 active+clean; 654 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 4.7 KiB/s rd, 1.9 KiB/s wr, 7 op/s 2024-03-19T01:40:58.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:57 smithi112 ceph-mon[66819]: pgmap v346: 97 pgs: 97 active+clean; 654 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 4.7 KiB/s rd, 1.9 KiB/s wr, 7 op/s 2024-03-19T01:40:58.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:57 smithi115 ceph-mon[67332]: pgmap v346: 97 pgs: 97 active+clean; 654 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 4.7 KiB/s rd, 1.9 KiB/s wr, 7 op/s 2024-03-19T01:41:00.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:40:59 smithi099 ceph-mon[62951]: pgmap v347: 97 pgs: 97 active+clean; 654 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 1.9 KiB/s wr, 7 op/s 2024-03-19T01:41:00.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:40:59 smithi112 ceph-mon[66819]: pgmap v347: 97 pgs: 97 active+clean; 654 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 1.9 KiB/s wr, 7 op/s 2024-03-19T01:41:00.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:40:59 smithi115 ceph-mon[67332]: pgmap v347: 97 pgs: 97 active+clean; 654 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 1.9 KiB/s wr, 7 op/s 2024-03-19T01:41:01.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:41:01 smithi112 ceph-mon[66819]: pgmap v348: 97 pgs: 97 active+clean; 656 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 4.8 KiB/s rd, 2.1 KiB/s wr, 7 op/s 2024-03-19T01:41:01.806 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:41:01 smithi115 ceph-mon[67332]: pgmap v348: 97 pgs: 97 active+clean; 656 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 4.8 KiB/s rd, 2.1 KiB/s wr, 7 op/s 2024-03-19T01:41:01.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:41:01 smithi099 ceph-mon[62951]: pgmap v348: 97 pgs: 97 active+clean; 656 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 4.8 KiB/s rd, 2.1 KiB/s wr, 7 op/s 2024-03-19T01:41:02.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.463+0000 7fbb976f8640 1 -- 172.21.15.99:0/776069835 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbb90102a70 msgr2=0x7fbb90102ef0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:41:02.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.463+0000 7fbb976f8640 1 --2- 172.21.15.99:0/776069835 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbb90102a70 0x7fbb90102ef0 secure :-1 s=READY pgs=353 cs=0 l=1 rev1=1 crypto rx=0x7fbb80009900 tx=0x7fbb8002bc30 comp rx=0 tx=0).stop 2024-03-19T01:41:02.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.463+0000 7fbb976f8640 1 -- 172.21.15.99:0/776069835 shutdown_connections 2024-03-19T01:41:02.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.463+0000 7fbb976f8640 1 --2- 172.21.15.99:0/776069835 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fbb90103430 0x7fbb90109cc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:02.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.463+0000 7fbb976f8640 1 --2- 172.21.15.99:0/776069835 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbb90102a70 0x7fbb90102ef0 unknown :-1 s=CLOSED pgs=353 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:02.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.463+0000 7fbb976f8640 1 --2- 172.21.15.99:0/776069835 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fbb90101880 0x7fbb90101ca0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:02.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.463+0000 7fbb976f8640 1 -- 172.21.15.99:0/776069835 >> 172.21.15.99:0/776069835 conn(0x7fbb900fcdf0 msgr2=0x7fbb900ff230 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:41:02.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.463+0000 7fbb976f8640 1 -- 172.21.15.99:0/776069835 shutdown_connections 2024-03-19T01:41:02.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.463+0000 7fbb976f8640 1 -- 172.21.15.99:0/776069835 wait complete. 2024-03-19T01:41:02.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.463+0000 7fbb976f8640 1 Processor -- start 2024-03-19T01:41:02.466 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.463+0000 7fbb976f8640 1 -- start start 2024-03-19T01:41:02.466 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.463+0000 7fbb976f8640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fbb90101880 0x7fbb90199c80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:02.466 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.463+0000 7fbb976f8640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbb90102a70 0x7fbb9019a1c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:02.466 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.463+0000 7fbb976f8640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fbb90103430 0x7fbb901a1220 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:02.466 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.463+0000 7fbb976f8640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fbb9010d4e0 con 0x7fbb90102a70 2024-03-19T01:41:02.466 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.463+0000 7fbb976f8640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fbb9010d360 con 0x7fbb90103430 2024-03-19T01:41:02.466 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.463+0000 7fbb976f8640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fbb9010d660 con 0x7fbb90101880 2024-03-19T01:41:02.466 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.464+0000 7fbb94c93640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbb90102a70 0x7fbb9019a1c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:02.466 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.464+0000 7fbb95494640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fbb90101880 0x7fbb90199c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:02.466 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.464+0000 7fbb94c93640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbb90102a70 0x7fbb9019a1c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:44078/0 (socket says 172.21.15.99:44078) 2024-03-19T01:41:02.466 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.464+0000 7fbb94c93640 1 -- 172.21.15.99:0/2202725081 learned_addr learned my addr 172.21.15.99:0/2202725081 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:41:02.466 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.464+0000 7fbb95c95640 1 --2- 172.21.15.99:0/2202725081 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fbb90103430 0x7fbb901a1220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:02.467 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.464+0000 7fbb94c93640 1 -- 172.21.15.99:0/2202725081 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fbb90101880 msgr2=0x7fbb90199c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:41:02.467 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.464+0000 7fbb94c93640 1 --2- 172.21.15.99:0/2202725081 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fbb90101880 0x7fbb90199c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:02.467 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.464+0000 7fbb94c93640 1 -- 172.21.15.99:0/2202725081 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fbb90103430 msgr2=0x7fbb901a1220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:41:02.467 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.464+0000 7fbb94c93640 1 --2- 172.21.15.99:0/2202725081 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fbb90103430 0x7fbb901a1220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:02.467 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.464+0000 7fbb94c93640 1 -- 172.21.15.99:0/2202725081 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fbb901a18c0 con 0x7fbb90102a70 2024-03-19T01:41:02.467 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.464+0000 7fbb95494640 1 --2- 172.21.15.99:0/2202725081 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fbb90101880 0x7fbb90199c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:41:02.467 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.464+0000 7fbb94c93640 1 --2- 172.21.15.99:0/2202725081 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbb90102a70 0x7fbb9019a1c0 secure :-1 s=READY pgs=354 cs=0 l=1 rev1=1 crypto rx=0x7fbb80009230 tx=0x7fbb8002c650 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:41:02.467 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.464+0000 7fbb867fc640 1 -- 172.21.15.99:0/2202725081 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbb80041070 con 0x7fbb90102a70 2024-03-19T01:41:02.467 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.464+0000 7fbb867fc640 1 -- 172.21.15.99:0/2202725081 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fbb80004450 con 0x7fbb90102a70 2024-03-19T01:41:02.467 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.464+0000 7fbb867fc640 1 -- 172.21.15.99:0/2202725081 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbb80003e20 con 0x7fbb90102a70 2024-03-19T01:41:02.467 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.464+0000 7fbb976f8640 1 -- 172.21.15.99:0/2202725081 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fbb901a1b50 con 0x7fbb90102a70 2024-03-19T01:41:02.467 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.464+0000 7fbb976f8640 1 -- 172.21.15.99:0/2202725081 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fbb901a2040 con 0x7fbb90102a70 2024-03-19T01:41:02.474 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.467+0000 7fbb9a3be740 1 Processor -- start 2024-03-19T01:41:02.474 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.467+0000 7fbb9a3be740 1 -- start start 2024-03-19T01:41:02.474 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.467+0000 7fbb9a3be740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a755448100 0x55a755448520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:02.474 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.467+0000 7fbb9a3be740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a755448af0 0x55a75544d5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:02.474 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.467+0000 7fbb9a3be740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a75544dc60 0x55a7554500f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:02.474 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.467+0000 7fbb9a3be740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55a7553acbe0 con 0x55a755448af0 2024-03-19T01:41:02.474 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.467+0000 7fbb9a3be740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55a755355960 con 0x55a755448100 2024-03-19T01:41:02.474 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.467+0000 7fbb9a3be740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55a7553a8c80 con 0x55a75544dc60 2024-03-19T01:41:02.474 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.467+0000 7fbb867fc640 1 -- 172.21.15.99:0/2202725081 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 19) ==== 137341+0+0 (secure 0 0 0) 0x7fbb80044070 con 0x7fbb90102a70 2024-03-19T01:41:02.474 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.468+0000 7fbb95c95640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a755448100 0x55a755448520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:02.475 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.468+0000 7fbb95494640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a755448af0 0x55a75544d5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:02.475 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.468+0000 7fbb95c95640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a755448100 0x55a755448520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:42268/0 (socket says 172.21.15.99:42268) 2024-03-19T01:41:02.475 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.468+0000 7fbb95c95640 1 -- 172.21.15.99:0/2561333274 learned_addr learned my addr 172.21.15.99:0/2561333274 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:41:02.475 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.468+0000 7fbb94c93640 1 --2- 172.21.15.99:0/2561333274 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a75544dc60 0x55a7554500f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:02.475 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.468+0000 7fbb95c95640 1 -- 172.21.15.99:0/2561333274 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a75544dc60 msgr2=0x55a7554500f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:41:02.475 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.468+0000 7fbb95c95640 1 --2- 172.21.15.99:0/2561333274 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a75544dc60 0x55a7554500f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:02.475 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.468+0000 7fbb95c95640 1 -- 172.21.15.99:0/2561333274 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a755448af0 msgr2=0x55a75544d5f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:41:02.475 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.468+0000 7fbb95c95640 1 --2- 172.21.15.99:0/2561333274 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a755448af0 0x55a75544d5f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:02.475 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.468+0000 7fbb95c95640 1 -- 172.21.15.99:0/2561333274 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55a75534cd20 con 0x55a755448100 2024-03-19T01:41:02.475 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.468+0000 7fbb95c95640 1 --2- 172.21.15.99:0/2561333274 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a755448100 0x55a755448520 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 crypto rx=0x7fbb88009fd0 tx=0x7fbb880118d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=71ae8dc82f0410aa server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:41:02.475 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.468+0000 7fbb867fc640 1 --2- 172.21.15.99:0/2202725081 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fbb7c0a2940 0x7fbb7c0a4e00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:02.475 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.468+0000 7fbb7b7fe640 1 -- 172.21.15.99:0/2561333274 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbb88004d20 con 0x55a755448100 2024-03-19T01:41:02.475 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.468+0000 7fbb7b7fe640 1 -- 172.21.15.99:0/2561333274 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fbb88004050 con 0x55a755448100 2024-03-19T01:41:02.475 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.468+0000 7fbb7b7fe640 1 -- 172.21.15.99:0/2561333274 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbb88005220 con 0x55a755448100 2024-03-19T01:41:02.476 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.468+0000 7fbb95494640 1 --2- 172.21.15.99:0/2202725081 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fbb7c0a2940 0x7fbb7c0a4e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:02.476 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.468+0000 7fbb867fc640 1 -- 172.21.15.99:0/2202725081 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7fbb800f0700 con 0x7fbb90102a70 2024-03-19T01:41:02.476 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.469+0000 7fbb9a3be740 1 -- 172.21.15.99:0/2561333274 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a755448100 msgr2=0x55a755448520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:41:02.476 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.469+0000 7fbb9a3be740 1 --2- 172.21.15.99:0/2561333274 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a755448100 0x55a755448520 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 crypto rx=0x7fbb88009fd0 tx=0x7fbb880118d0 comp rx=0 tx=0).stop 2024-03-19T01:41:02.476 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.469+0000 7fbb95494640 1 --2- 172.21.15.99:0/2202725081 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fbb7c0a2940 0x7fbb7c0a4e00 secure :-1 s=READY pgs=247 cs=0 l=1 rev1=1 crypto rx=0x55a755448ef0 tx=0x7fbb8c005f90 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:41:02.476 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.469+0000 7fbb9a3be740 1 -- 172.21.15.99:0/2561333274 shutdown_connections 2024-03-19T01:41:02.476 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.469+0000 7fbb9a3be740 1 --2- 172.21.15.99:0/2561333274 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a75544dc60 0x55a7554500f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:02.476 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.469+0000 7fbb9a3be740 1 --2- 172.21.15.99:0/2561333274 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a755448af0 0x55a75544d5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:02.476 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.469+0000 7fbb9a3be740 1 --2- 172.21.15.99:0/2561333274 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a755448100 0x55a755448520 unknown :-1 s=CLOSED pgs=83 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:02.476 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.469+0000 7fbb9a3be740 1 -- 172.21.15.99:0/2561333274 >> 172.21.15.99:0/2561333274 conn(0x55a755443670 msgr2=0x55a755443a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:41:02.476 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.469+0000 7fbb9a3be740 1 -- 172.21.15.99:0/2561333274 shutdown_connections 2024-03-19T01:41:02.476 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.469+0000 7fbb9a3be740 1 -- 172.21.15.99:0/2561333274 wait complete. 2024-03-19T01:41:02.476 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.469+0000 7fbb9a3be740 1 -- start start 2024-03-19T01:41:02.476 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.469+0000 7fbb9a3be740 1 Processor -- start 2024-03-19T01:41:02.477 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.470+0000 7fbb9a3be740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a755448100 0x55a755452100 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:02.477 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.470+0000 7fbb9a3be740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a755448af0 0x55a755452640 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:02.477 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.470+0000 7fbb9a3be740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a75544dc60 0x55a755459680 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:02.477 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.470+0000 7fbb9a3be740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55a755450c10 con 0x55a755448af0 2024-03-19T01:41:02.477 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.470+0000 7fbb9a3be740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55a755368db0 con 0x55a75544dc60 2024-03-19T01:41:02.477 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.470+0000 7fbb9a3be740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55a755450d90 con 0x55a755448100 2024-03-19T01:41:02.477 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.470+0000 7fbb9a3be740 10 client.0 _ll_register_callbacks cb 0x55a75536de70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:41:02.477 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.470+0000 7fbb94c93640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a755448af0 0x55a755452640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:02.477 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.470+0000 7fbb95c95640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a75544dc60 0x55a755459680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:02.477 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.470+0000 7fbb95494640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a755448100 0x55a755452100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:02.477 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.470+0000 7fbb95c95640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a75544dc60 0x55a755459680 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:42274/0 (socket says 172.21.15.99:42274) 2024-03-19T01:41:02.477 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.470+0000 7fbb95494640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a755448100 0x55a755452100 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:50586/0 (socket says 172.21.15.99:50586) 2024-03-19T01:41:02.478 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.470+0000 7fbb95c95640 1 -- 172.21.15.99:0/415701519 learned_addr learned my addr 172.21.15.99:0/415701519 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:41:02.478 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.470+0000 7fbb94c93640 1 -- 172.21.15.99:0/415701519 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a755448100 msgr2=0x55a755452100 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:41:02.478 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.470+0000 7fbb94c93640 1 --2- 172.21.15.99:0/415701519 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a755448100 0x55a755452100 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:02.478 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.470+0000 7fbb94c93640 1 -- 172.21.15.99:0/415701519 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a75544dc60 msgr2=0x55a755459680 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:41:02.478 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.470+0000 7fbb94c93640 1 --2- 172.21.15.99:0/415701519 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a75544dc60 0x55a755459680 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:02.478 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.470+0000 7fbb94c93640 1 -- 172.21.15.99:0/415701519 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55a755367c60 con 0x55a755448af0 2024-03-19T01:41:02.478 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.470+0000 7fbb95c95640 1 --2- 172.21.15.99:0/415701519 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a75544dc60 0x55a755459680 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:41:02.478 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.470+0000 7fbb94c93640 1 --2- 172.21.15.99:0/415701519 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a755448af0 0x55a755452640 secure :-1 s=READY pgs=355 cs=0 l=1 rev1=1 crypto rx=0x7fbb800ee120 tx=0x7fbb80009560 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6a17717a29d9f055 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:41:02.478 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.470+0000 7fbb78ff9640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:41:02.478 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.471+0000 7fbb78ff9640 1 -- 172.21.15.99:0/415701519 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbb8004b030 con 0x55a755448af0 2024-03-19T01:41:02.478 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.471+0000 7fbb78ff9640 1 -- 172.21.15.99:0/415701519 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7fbb800b7bd0 con 0x55a755448af0 2024-03-19T01:41:02.478 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.471+0000 7fbb9a3be740 1 -- 172.21.15.99:0/415701519 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fbb88004050 con 0x55a755448af0 2024-03-19T01:41:02.478 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.471+0000 7fbb78ff9640 1 -- 172.21.15.99:0/415701519 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=21) ==== 24+0+0 (secure 0 0 0) 0x7fbb800bcba0 con 0x55a755448af0 2024-03-19T01:41:02.478 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.471+0000 7fbb9a3be740 10 client.15324 fetch_fsmap learned FSMap version 21 2024-03-19T01:41:02.479 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.471+0000 7fbb9a3be740 1 -- 172.21.15.99:0/415701519 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=21,monmap=4+}) -- 0x55a7550c1210 con 0x55a755448af0 2024-03-19T01:41:02.479 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.471+0000 7fbb78ff9640 1 -- 172.21.15.99:0/415701519 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fbb88004050 con 0x55a755448af0 2024-03-19T01:41:02.479 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.471+0000 7fbb78ff9640 1 -- 172.21.15.99:0/415701519 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 21) ==== 8338+0+0 (secure 0 0 0) 0x7fbb800f3ce0 con 0x55a755448af0 2024-03-19T01:41:02.479 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.471+0000 7fbb9a3be740 10 client.15324 fetch_fsmap finished waiting for FSMap version 21 2024-03-19T01:41:02.479 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.471+0000 7fbb9a3be740 10 client.15324 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:41:02.479 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.471+0000 7fbb9a3be740 20 client.15324 populate_metadata read hostname 'smithi099' 2024-03-19T01:41:02.479 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.471+0000 7fbb9a3be740 1 --2- 172.21.15.99:0/415701519 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55a75545df80 0x55a7554603a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:02.479 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.471+0000 7fbb9a3be740 4 client.15324 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:41:02.479 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.471+0000 7fbb9a3be740 1 -- 172.21.15.99:0/415701519 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55a7553ba120 con 0x55a75545df80 2024-03-19T01:41:02.479 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.471+0000 7fbb95494640 1 --2- 172.21.15.99:0/415701519 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55a75545df80 0x55a7554603a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:02.479 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.472+0000 7fbb78ff9640 10 client.15324 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:41:02.479 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.472+0000 7fbb95494640 1 --2- 172.21.15.99:0/415701519 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55a75545df80 0x55a7554603a0 crc :-1 s=READY pgs=27 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=d685492736f9c874 server_cookie=aa1c71cdba7b1f1 in_seq=0 out_seq=0 2024-03-19T01:41:02.479 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.473+0000 7fbb78ff9640 1 -- 172.21.15.99:0/415701519 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x55a7553ba120 con 0x55a75545df80 2024-03-19T01:41:02.479 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.473+0000 7fbb78ff9640 10 client.15324 handle_command_reply: tid=0 2024-03-19T01:41:02.480 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.473+0000 7fbb9a3be740 1 client.15324 shutdown 2024-03-19T01:41:02.480 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.473+0000 7fbb9a3be740 1 -- 172.21.15.99:0/415701519 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a755448af0 msgr2=0x55a755452640 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:41:02.480 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.473+0000 7fbb9a3be740 1 --2- 172.21.15.99:0/415701519 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a755448af0 0x55a755452640 secure :-1 s=READY pgs=355 cs=0 l=1 rev1=1 crypto rx=0x7fbb800ee120 tx=0x7fbb80009560 comp rx=0 tx=0).stop 2024-03-19T01:41:02.480 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.473+0000 7fbb9a3be740 1 -- 172.21.15.99:0/415701519 shutdown_connections 2024-03-19T01:41:02.480 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.473+0000 7fbb9a3be740 1 --2- 172.21.15.99:0/415701519 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55a75545df80 0x55a7554603a0 crc :-1 s=READY pgs=27 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:02.480 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.473+0000 7fbb9a3be740 1 --2- 172.21.15.99:0/415701519 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a75544dc60 0x55a755459680 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:02.480 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.473+0000 7fbb9a3be740 1 --2- 172.21.15.99:0/415701519 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a755448af0 0x55a755452640 unknown :-1 s=CLOSED pgs=355 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:02.480 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.473+0000 7fbb9a3be740 1 --2- 172.21.15.99:0/415701519 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a755448100 0x55a755452100 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:02.480 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.473+0000 7fbb9a3be740 1 -- 172.21.15.99:0/415701519 >> 172.21.15.99:0/415701519 conn(0x55a755443670 msgr2=0x55a7554448c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:41:02.480 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.473+0000 7fbb78ff9640 0 client.15324 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:41:02.480 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.473+0000 7fbb9a3be740 1 -- 172.21.15.99:0/415701519 shutdown_connections 2024-03-19T01:41:02.480 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.473+0000 7fbb9a3be740 1 -- 172.21.15.99:0/415701519 wait complete. 2024-03-19T01:41:02.480 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.473+0000 7fbb9a3be740 20 client.15324 trim_cache size 0 max 16384 2024-03-19T01:41:02.494 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.492+0000 7fbb9a3be740 1 Processor -- start 2024-03-19T01:41:02.494 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.492+0000 7fbb9a3be740 1 -- start start 2024-03-19T01:41:02.495 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.493+0000 7fbb9a3be740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a755443670 0x55a755442ab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:02.495 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.493+0000 7fbb9a3be740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a755448100 0x55a755459640 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:02.495 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.493+0000 7fbb9a3be740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a755448af0 0x55a755446fe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:02.495 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.493+0000 7fbb9a3be740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fbb901ad770 con 0x55a755448af0 2024-03-19T01:41:02.495 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.493+0000 7fbb9a3be740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fbb901ab040 con 0x55a755443670 2024-03-19T01:41:02.495 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.493+0000 7fbb9a3be740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fbb901a8e90 con 0x55a755448100 2024-03-19T01:41:02.495 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.493+0000 7fbb95c95640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a755448af0 0x55a755446fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:02.495 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.493+0000 7fbb95494640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a755443670 0x55a755442ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:02.495 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.493+0000 7fbb94c93640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a755448100 0x55a755459640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:02.495 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.493+0000 7fbb95c95640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a755448af0 0x55a755446fe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:44098/0 (socket says 172.21.15.99:44098) 2024-03-19T01:41:02.495 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.493+0000 7fbb95c95640 1 -- 172.21.15.99:0/3007576253 learned_addr learned my addr 172.21.15.99:0/3007576253 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:41:02.495 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.493+0000 7fbb95c95640 1 -- 172.21.15.99:0/3007576253 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a755448100 msgr2=0x55a755459640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:41:02.496 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.493+0000 7fbb95c95640 1 --2- 172.21.15.99:0/3007576253 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a755448100 0x55a755459640 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:02.496 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.493+0000 7fbb95c95640 1 -- 172.21.15.99:0/3007576253 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a755443670 msgr2=0x55a755442ab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:41:02.496 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.493+0000 7fbb95c95640 1 --2- 172.21.15.99:0/3007576253 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a755443670 0x55a755442ab0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:02.496 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.493+0000 7fbb95c95640 1 -- 172.21.15.99:0/3007576253 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55a75518d620 con 0x55a755448af0 2024-03-19T01:41:02.496 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.493+0000 7fbb94c93640 1 --2- 172.21.15.99:0/3007576253 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a755448100 0x55a755459640 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:41:02.496 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.493+0000 7fbb95494640 1 --2- 172.21.15.99:0/3007576253 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a755443670 0x55a755442ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:41:02.496 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.493+0000 7fbb95c95640 1 --2- 172.21.15.99:0/3007576253 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a755448af0 0x55a755446fe0 secure :-1 s=READY pgs=356 cs=0 l=1 rev1=1 crypto rx=0x7fbb88009f70 tx=0x7fbb88003ef0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=78280c1b39c755f2 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:41:02.496 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.493+0000 7fbb7affd640 1 -- 172.21.15.99:0/3007576253 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbb88028070 con 0x55a755448af0 2024-03-19T01:41:02.496 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.494+0000 7fbb7affd640 1 -- 172.21.15.99:0/3007576253 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fbb88004050 con 0x55a755448af0 2024-03-19T01:41:02.496 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.494+0000 7fbb7affd640 1 -- 172.21.15.99:0/3007576253 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbb88005310 con 0x55a755448af0 2024-03-19T01:41:02.496 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.494+0000 7fbb9a3be740 1 -- 172.21.15.99:0/3007576253 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a755448af0 msgr2=0x55a755446fe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:41:02.496 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.494+0000 7fbb9a3be740 1 --2- 172.21.15.99:0/3007576253 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a755448af0 0x55a755446fe0 secure :-1 s=READY pgs=356 cs=0 l=1 rev1=1 crypto rx=0x7fbb88009f70 tx=0x7fbb88003ef0 comp rx=0 tx=0).stop 2024-03-19T01:41:02.496 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.494+0000 7fbb9a3be740 1 -- 172.21.15.99:0/3007576253 shutdown_connections 2024-03-19T01:41:02.496 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.494+0000 7fbb9a3be740 1 --2- 172.21.15.99:0/3007576253 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a755448af0 0x55a755446fe0 unknown :-1 s=CLOSED pgs=356 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:02.497 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.494+0000 7fbb9a3be740 1 --2- 172.21.15.99:0/3007576253 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a755448100 0x55a755459640 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:02.497 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.494+0000 7fbb9a3be740 1 --2- 172.21.15.99:0/3007576253 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a755443670 0x55a755442ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:02.497 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.494+0000 7fbb9a3be740 1 -- 172.21.15.99:0/3007576253 >> 172.21.15.99:0/3007576253 conn(0x55a755451cc0 msgr2=0x55a7554468d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:41:02.497 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.494+0000 7fbb9a3be740 1 -- 172.21.15.99:0/3007576253 shutdown_connections 2024-03-19T01:41:02.497 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.494+0000 7fbb9a3be740 1 -- 172.21.15.99:0/3007576253 wait complete. 2024-03-19T01:41:02.497 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.495+0000 7fbb9a3be740 1 -- start start 2024-03-19T01:41:02.497 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.495+0000 7fbb9a3be740 1 Processor -- start 2024-03-19T01:41:02.499 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.495+0000 7fbb9a3be740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a755443670 0x55a755456f40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:02.500 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.495+0000 7fbb9a3be740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a755448100 0x55a755457480 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:02.500 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.495+0000 7fbb9a3be740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a755448af0 0x55a755458440 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:02.500 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.495+0000 7fbb9a3be740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fbb901a9500 con 0x55a755443670 2024-03-19T01:41:02.500 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.495+0000 7fbb9a3be740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fbb901aa970 con 0x55a755448af0 2024-03-19T01:41:02.500 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.495+0000 7fbb9a3be740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fbb901a9bd0 con 0x55a755448100 2024-03-19T01:41:02.500 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.495+0000 7fbb9a3be740 10 client.0 _ll_register_callbacks cb 0x7fbb80005770 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:41:02.500 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.495+0000 7fbb95494640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a755443670 0x55a755456f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:02.500 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.495+0000 7fbb95494640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a755443670 0x55a755456f40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:44100/0 (socket says 172.21.15.99:44100) 2024-03-19T01:41:02.500 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.495+0000 7fbb95494640 1 -- 172.21.15.99:0/1906713497 learned_addr learned my addr 172.21.15.99:0/1906713497 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:41:02.500 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.495+0000 7fbb95c95640 1 --2- 172.21.15.99:0/1906713497 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a755448af0 0x55a755458440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:02.500 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.495+0000 7fbb94c93640 1 --2- 172.21.15.99:0/1906713497 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a755448100 0x55a755457480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:02.500 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.495+0000 7fbb95494640 1 -- 172.21.15.99:0/1906713497 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a755448100 msgr2=0x55a755457480 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:41:02.500 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.495+0000 7fbb95494640 1 --2- 172.21.15.99:0/1906713497 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a755448100 0x55a755457480 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:02.501 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.495+0000 7fbb95494640 1 -- 172.21.15.99:0/1906713497 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a755448af0 msgr2=0x55a755458440 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:41:02.501 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.495+0000 7fbb95494640 1 --2- 172.21.15.99:0/1906713497 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a755448af0 0x55a755458440 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:02.501 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.495+0000 7fbb95494640 1 -- 172.21.15.99:0/1906713497 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55a755458a30 con 0x55a755443670 2024-03-19T01:41:02.501 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.496+0000 7fbb95494640 1 --2- 172.21.15.99:0/1906713497 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a755443670 0x55a755456f40 secure :-1 s=READY pgs=357 cs=0 l=1 rev1=1 crypto rx=0x7fbb9019ab50 tx=0x7fbb8c0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=647601d9ed1084a9 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:41:02.501 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.496+0000 7fbb78ff9640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:41:02.501 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.496+0000 7fbb78ff9640 1 -- 172.21.15.99:0/1906713497 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbb8c0041e0 con 0x55a755443670 2024-03-19T01:41:02.501 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.496+0000 7fbb78ff9640 1 -- 172.21.15.99:0/1906713497 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7fbb8c02bb90 con 0x55a755443670 2024-03-19T01:41:02.501 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.496+0000 7fbb9a3be740 1 -- 172.21.15.99:0/1906713497 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fbb88004050 con 0x55a755443670 2024-03-19T01:41:02.501 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.496+0000 7fbb78ff9640 1 -- 172.21.15.99:0/1906713497 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=21) ==== 24+0+0 (secure 0 0 0) 0x7fbb8c02bd90 con 0x55a755443670 2024-03-19T01:41:02.501 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.496+0000 7fbb9a3be740 10 client.15330 fetch_fsmap learned FSMap version 21 2024-03-19T01:41:02.501 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.496+0000 7fbb9a3be740 1 -- 172.21.15.99:0/1906713497 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=21,monmap=4+}) -- 0x55a7554c44f0 con 0x55a755443670 2024-03-19T01:41:02.501 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.496+0000 7fbb78ff9640 1 -- 172.21.15.99:0/1906713497 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fbb88004050 con 0x55a755443670 2024-03-19T01:41:02.501 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.496+0000 7fbb78ff9640 1 -- 172.21.15.99:0/1906713497 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 21) ==== 8338+0+0 (secure 0 0 0) 0x7fbb8c00fb60 con 0x55a755443670 2024-03-19T01:41:02.502 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.496+0000 7fbb9a3be740 10 client.15330 fetch_fsmap finished waiting for FSMap version 21 2024-03-19T01:41:02.502 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.496+0000 7fbb9a3be740 10 client.15330 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:41:02.502 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.496+0000 7fbb9a3be740 20 client.15330 populate_metadata read hostname 'smithi099' 2024-03-19T01:41:02.502 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.496+0000 7fbb9a3be740 1 --2- 172.21.15.99:0/1906713497 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55a75544dc60 0x55a7554c6f10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:02.502 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.496+0000 7fbb9a3be740 4 client.15330 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:41:02.502 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.496+0000 7fbb9a3be740 1 -- 172.21.15.99:0/1906713497 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55a7553b4470 con 0x55a75544dc60 2024-03-19T01:41:02.502 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.497+0000 7fbb94c93640 1 --2- 172.21.15.99:0/1906713497 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55a75544dc60 0x55a7554c6f10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:02.502 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.497+0000 7fbb94c93640 1 --2- 172.21.15.99:0/1906713497 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55a75544dc60 0x55a7554c6f10 crc :-1 s=READY pgs=28 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=a2fdf78f131d014 server_cookie=43ccc66ce2bc2d01 in_seq=0 out_seq=0 2024-03-19T01:41:02.502 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.497+0000 7fbb78ff9640 10 client.15330 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:41:02.508 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.507+0000 7fbb78ff9640 1 -- 172.21.15.99:0/1906713497 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x55a7553b4470 con 0x55a75544dc60 2024-03-19T01:41:02.508 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.507+0000 7fbb78ff9640 10 client.15330 handle_command_reply: tid=0 2024-03-19T01:41:02.508 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.507+0000 7fbb9a3be740 1 client.15330 shutdown 2024-03-19T01:41:02.508 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.507+0000 7fbb9a3be740 1 -- 172.21.15.99:0/1906713497 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a755443670 msgr2=0x55a755456f40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:41:02.508 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.507+0000 7fbb9a3be740 1 --2- 172.21.15.99:0/1906713497 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a755443670 0x55a755456f40 secure :-1 s=READY pgs=357 cs=0 l=1 rev1=1 crypto rx=0x7fbb9019ab50 tx=0x7fbb8c0046c0 comp rx=0 tx=0).stop 2024-03-19T01:41:02.508 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.507+0000 7fbb9a3be740 1 -- 172.21.15.99:0/1906713497 shutdown_connections 2024-03-19T01:41:02.508 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.507+0000 7fbb9a3be740 1 --2- 172.21.15.99:0/1906713497 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55a75544dc60 0x55a7554c6f10 crc :-1 s=READY pgs=28 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:02.508 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.507+0000 7fbb9a3be740 1 --2- 172.21.15.99:0/1906713497 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a755448af0 0x55a755458440 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:02.508 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.507+0000 7fbb78ff9640 0 client.15330 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:41:02.508 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.507+0000 7fbb9a3be740 1 --2- 172.21.15.99:0/1906713497 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a755448100 0x55a755457480 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:02.508 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.507+0000 7fbb9a3be740 1 --2- 172.21.15.99:0/1906713497 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a755443670 0x55a755456f40 unknown :-1 s=CLOSED pgs=357 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:02.508 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.507+0000 7fbb9a3be740 1 -- 172.21.15.99:0/1906713497 >> 172.21.15.99:0/1906713497 conn(0x55a755451cc0 msgr2=0x55a755447160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:41:02.508 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.507+0000 7fbb9a3be740 1 -- 172.21.15.99:0/1906713497 shutdown_connections 2024-03-19T01:41:02.509 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.507+0000 7fbb9a3be740 1 -- 172.21.15.99:0/1906713497 wait complete. 2024-03-19T01:41:02.509 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.508+0000 7fbb9a3be740 20 client.15330 trim_cache size 0 max 16384 2024-03-19T01:41:02.509 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:41:02.509 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:41:02.509 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:41:02.509 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:41:02.510 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.508+0000 7fbb976f8640 1 -- 172.21.15.99:0/2202725081 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fbb7c0a2940 msgr2=0x7fbb7c0a4e00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:41:02.510 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.508+0000 7fbb976f8640 1 --2- 172.21.15.99:0/2202725081 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fbb7c0a2940 0x7fbb7c0a4e00 secure :-1 s=READY pgs=247 cs=0 l=1 rev1=1 crypto rx=0x55a755448ef0 tx=0x7fbb8c005f90 comp rx=0 tx=0).stop 2024-03-19T01:41:02.510 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.508+0000 7fbb976f8640 1 -- 172.21.15.99:0/2202725081 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbb90102a70 msgr2=0x7fbb9019a1c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:41:02.510 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.508+0000 7fbb976f8640 1 --2- 172.21.15.99:0/2202725081 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbb90102a70 0x7fbb9019a1c0 secure :-1 s=READY pgs=354 cs=0 l=1 rev1=1 crypto rx=0x7fbb80009230 tx=0x7fbb8002c650 comp rx=0 tx=0).stop 2024-03-19T01:41:02.510 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.508+0000 7fbb976f8640 1 -- 172.21.15.99:0/2202725081 shutdown_connections 2024-03-19T01:41:02.510 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.508+0000 7fbb976f8640 1 --2- 172.21.15.99:0/2202725081 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fbb7c0a2940 0x7fbb7c0a4e00 unknown :-1 s=CLOSED pgs=247 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:02.510 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.508+0000 7fbb976f8640 1 --2- 172.21.15.99:0/2202725081 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fbb90103430 0x7fbb901a1220 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:02.510 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.508+0000 7fbb976f8640 1 --2- 172.21.15.99:0/2202725081 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbb90102a70 0x7fbb9019a1c0 unknown :-1 s=CLOSED pgs=354 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:02.510 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.508+0000 7fbb976f8640 1 --2- 172.21.15.99:0/2202725081 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fbb90101880 0x7fbb90199c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:02.510 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.508+0000 7fbb976f8640 1 -- 172.21.15.99:0/2202725081 >> 172.21.15.99:0/2202725081 conn(0x7fbb900fcdf0 msgr2=0x7fbb90073a50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:41:02.511 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.508+0000 7fbb976f8640 1 -- 172.21.15.99:0/2202725081 shutdown_connections 2024-03-19T01:41:02.511 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:02.508+0000 7fbb976f8640 1 -- 172.21.15.99:0/2202725081 wait complete. 2024-03-19T01:41:04.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:41:03 smithi099 ceph-mon[62951]: pgmap v349: 97 pgs: 97 active+clean; 656 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.2 KiB/s rd, 682 B/s wr, 7 op/s 2024-03-19T01:41:04.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:41:03 smithi112 ceph-mon[66819]: pgmap v349: 97 pgs: 97 active+clean; 656 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.2 KiB/s rd, 682 B/s wr, 7 op/s 2024-03-19T01:41:04.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:41:03 smithi115 ceph-mon[67332]: pgmap v349: 97 pgs: 97 active+clean; 656 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.2 KiB/s rd, 682 B/s wr, 7 op/s 2024-03-19T01:41:06.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:41:05 smithi099 ceph-mon[62951]: pgmap v350: 97 pgs: 97 active+clean; 656 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 KiB/s rd, 341 B/s wr, 5 op/s 2024-03-19T01:41:06.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:41:05 smithi112 ceph-mon[66819]: pgmap v350: 97 pgs: 97 active+clean; 656 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 KiB/s rd, 341 B/s wr, 5 op/s 2024-03-19T01:41:06.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:41:05 smithi115 ceph-mon[67332]: pgmap v350: 97 pgs: 97 active+clean; 656 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 KiB/s rd, 341 B/s wr, 5 op/s 2024-03-19T01:41:08.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:41:07 smithi099 ceph-mon[62951]: pgmap v351: 97 pgs: 97 active+clean; 659 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.8 KiB/s rd, 1.6 KiB/s wr, 9 op/s 2024-03-19T01:41:08.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:41:07 smithi112 ceph-mon[66819]: pgmap v351: 97 pgs: 97 active+clean; 659 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.8 KiB/s rd, 1.6 KiB/s wr, 9 op/s 2024-03-19T01:41:08.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:41:07 smithi115 ceph-mon[67332]: pgmap v351: 97 pgs: 97 active+clean; 659 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.8 KiB/s rd, 1.6 KiB/s wr, 9 op/s 2024-03-19T01:41:10.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:41:09 smithi099 ceph-mon[62951]: pgmap v352: 97 pgs: 97 active+clean; 659 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.4 KiB/s rd, 1.9 KiB/s wr, 10 op/s 2024-03-19T01:41:10.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:41:09 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:41:10.104 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:41:09 smithi115 ceph-mon[67332]: pgmap v352: 97 pgs: 97 active+clean; 659 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.4 KiB/s rd, 1.9 KiB/s wr, 10 op/s 2024-03-19T01:41:10.104 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:41:09 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:41:10.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:41:09 smithi112 ceph-mon[66819]: pgmap v352: 97 pgs: 97 active+clean; 659 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.4 KiB/s rd, 1.9 KiB/s wr, 10 op/s 2024-03-19T01:41:10.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:41:09 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:41:12.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:41:11 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:41:12.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:41:11 smithi115 ceph-mon[67332]: pgmap v353: 97 pgs: 97 active+clean; 659 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.8 KiB/s rd, 1.9 KiB/s wr, 10 op/s 2024-03-19T01:41:12.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:41:11 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:41:12.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:41:11 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:41:12.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:41:11 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:41:12.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:41:11 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:41:12.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:41:11 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:41:12.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:41:11 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:41:12.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:41:11 smithi099 ceph-mon[62951]: pgmap v353: 97 pgs: 97 active+clean; 659 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.8 KiB/s rd, 1.9 KiB/s wr, 10 op/s 2024-03-19T01:41:12.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:41:11 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:41:12.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:41:11 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:41:12.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:41:11 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:41:12.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:41:11 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:41:12.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:41:11 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:41:12.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:41:11 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:41:12.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:41:11 smithi112 ceph-mon[66819]: pgmap v353: 97 pgs: 97 active+clean; 659 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.8 KiB/s rd, 1.9 KiB/s wr, 10 op/s 2024-03-19T01:41:12.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:41:11 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:41:12.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:41:11 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:41:12.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:41:11 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:41:12.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:41:11 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:41:12.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:41:11 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:41:12.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:41:12 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:41:12.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:41:12 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:41:12.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:41:12 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:41:13.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:41:12 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:41:13.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:41:12 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:41:13.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:41:12 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:41:13.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:41:12 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:41:13.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:41:12 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:41:13.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:41:12 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:41:14.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:41:13 smithi099 ceph-mon[62951]: pgmap v354: 97 pgs: 97 active+clean; 659 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.2 KiB/s rd, 1.7 KiB/s wr, 10 op/s 2024-03-19T01:41:14.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:41:13 smithi112 ceph-mon[66819]: pgmap v354: 97 pgs: 97 active+clean; 659 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.2 KiB/s rd, 1.7 KiB/s wr, 10 op/s 2024-03-19T01:41:14.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:41:13 smithi115 ceph-mon[67332]: pgmap v354: 97 pgs: 97 active+clean; 659 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.2 KiB/s rd, 1.7 KiB/s wr, 10 op/s 2024-03-19T01:41:15.645 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.641+0000 7f476e419640 1 -- 172.21.15.99:0/2048652385 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f47681022f0 msgr2=0x7f4768108b80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:41:15.646 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.641+0000 7f476e419640 1 --2- 172.21.15.99:0/2048652385 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f47681022f0 0x7f4768108b80 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 crypto rx=0x7f4760009be0 tx=0x7f476002bef0 comp rx=0 tx=0).stop 2024-03-19T01:41:15.646 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.641+0000 7f476e419640 1 -- 172.21.15.99:0/2048652385 shutdown_connections 2024-03-19T01:41:15.646 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.641+0000 7f476e419640 1 --2- 172.21.15.99:0/2048652385 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f47681022f0 0x7f4768108b80 unknown :-1 s=CLOSED pgs=84 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:15.646 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.641+0000 7f476e419640 1 --2- 172.21.15.99:0/2048652385 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4768101930 0x7f4768101db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:15.646 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.641+0000 7f476e419640 1 --2- 172.21.15.99:0/2048652385 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4768100740 0x7f4768100b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:15.646 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.641+0000 7f476e419640 1 -- 172.21.15.99:0/2048652385 >> 172.21.15.99:0/2048652385 conn(0x7f47680fbcb0 msgr2=0x7f47680fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:41:15.647 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.641+0000 7f476e419640 1 -- 172.21.15.99:0/2048652385 shutdown_connections 2024-03-19T01:41:15.647 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.641+0000 7f476e419640 1 -- 172.21.15.99:0/2048652385 wait complete. 2024-03-19T01:41:15.647 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.642+0000 7f476e419640 1 Processor -- start 2024-03-19T01:41:15.647 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.642+0000 7f476e419640 1 -- start start 2024-03-19T01:41:15.647 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.642+0000 7f476e419640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4768100740 0x7f4768069a30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:15.647 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.642+0000 7f476e419640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4768101930 0x7f476819c7c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:15.647 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.642+0000 7f476e419640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f47681022f0 0x7f476819ee00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:15.648 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.642+0000 7f476e419640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f476810c410 con 0x7f4768100740 2024-03-19T01:41:15.648 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.642+0000 7f476e419640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f476810c290 con 0x7f47681022f0 2024-03-19T01:41:15.648 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.642+0000 7f476e419640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f476810c590 con 0x7f4768101930 2024-03-19T01:41:15.648 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.643+0000 7f4767fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4768100740 0x7f4768069a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:15.648 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.643+0000 7f47677fe640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4768101930 0x7f476819c7c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:15.648 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.643+0000 7f47677fe640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4768101930 0x7f476819c7c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:47470/0 (socket says 172.21.15.99:47470) 2024-03-19T01:41:15.648 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.643+0000 7f47677fe640 1 -- 172.21.15.99:0/1485470062 learned_addr learned my addr 172.21.15.99:0/1485470062 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:41:15.648 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.643+0000 7f476c9b6640 1 --2- 172.21.15.99:0/1485470062 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f47681022f0 0x7f476819ee00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:15.649 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.643+0000 7f4767fff640 1 -- 172.21.15.99:0/1485470062 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4768101930 msgr2=0x7f476819c7c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:41:15.649 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.643+0000 7f4767fff640 1 --2- 172.21.15.99:0/1485470062 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4768101930 0x7f476819c7c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:15.649 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.643+0000 7f4767fff640 1 -- 172.21.15.99:0/1485470062 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f47681022f0 msgr2=0x7f476819ee00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:41:15.649 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.643+0000 7f4767fff640 1 --2- 172.21.15.99:0/1485470062 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f47681022f0 0x7f476819ee00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:15.649 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.643+0000 7f4767fff640 1 -- 172.21.15.99:0/1485470062 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f476819f6c0 con 0x7f4768100740 2024-03-19T01:41:15.649 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.643+0000 7f47677fe640 1 --2- 172.21.15.99:0/1485470062 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4768101930 0x7f476819c7c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:41:15.649 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.643+0000 7f4767fff640 1 --2- 172.21.15.99:0/1485470062 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4768100740 0x7f4768069a30 secure :-1 s=READY pgs=358 cs=0 l=1 rev1=1 crypto rx=0x7f4758005fd0 tx=0x7f475800ec20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:41:15.650 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.643+0000 7f47657fa640 1 -- 172.21.15.99:0/1485470062 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f475800ccc0 con 0x7f4768100740 2024-03-19T01:41:15.650 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.643+0000 7f476e419640 1 -- 172.21.15.99:0/1485470062 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f476819f9b0 con 0x7f4768100740 2024-03-19T01:41:15.650 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.643+0000 7f47657fa640 1 -- 172.21.15.99:0/1485470062 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f475800ee70 con 0x7f4768100740 2024-03-19T01:41:15.650 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.643+0000 7f476e419640 1 -- 172.21.15.99:0/1485470062 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f476819ff50 con 0x7f4768100740 2024-03-19T01:41:15.650 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.643+0000 7f47657fa640 1 -- 172.21.15.99:0/1485470062 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4758010430 con 0x7f4768100740 2024-03-19T01:41:15.662 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.646+0000 7f4771009740 1 Processor -- start 2024-03-19T01:41:15.662 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.646+0000 7f4771009740 1 -- start start 2024-03-19T01:41:15.662 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.646+0000 7f4771009740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c7e21cd510 0x55c7e21cd930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:15.663 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.646+0000 7f4771009740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c7e21cdf00 0x55c7e21d2a00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:15.663 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.646+0000 7f4771009740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c7e21d3070 0x55c7e21d5500 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:15.663 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.646+0000 7f4771009740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55c7e212dbe0 con 0x55c7e21cd510 2024-03-19T01:41:15.663 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.646+0000 7f4771009740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55c7e20d6960 con 0x55c7e21cdf00 2024-03-19T01:41:15.663 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.646+0000 7f4771009740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55c7e2129c80 con 0x55c7e21d3070 2024-03-19T01:41:15.663 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.646+0000 7f476c9b6640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c7e21cd510 0x55c7e21cd930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:15.663 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.647+0000 7f476c9b6640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c7e21cd510 0x55c7e21cd930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:60166/0 (socket says 172.21.15.99:60166) 2024-03-19T01:41:15.664 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.647+0000 7f476c9b6640 1 -- 172.21.15.99:0/1601207609 learned_addr learned my addr 172.21.15.99:0/1601207609 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:41:15.664 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.647+0000 7f47677fe640 1 --2- 172.21.15.99:0/1601207609 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c7e21d3070 0x55c7e21d5500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:15.664 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.647+0000 7f47657fa640 1 -- 172.21.15.99:0/1485470062 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 19) ==== 137341+0+0 (secure 0 0 0) 0x7f47580105d0 con 0x7f4768100740 2024-03-19T01:41:15.664 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.647+0000 7f47657fa640 1 --2- 172.21.15.99:0/1485470062 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f47500a2a40 0x7f47500a4f00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:15.664 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.648+0000 7f476c9b6640 1 -- 172.21.15.99:0/1601207609 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c7e21d3070 msgr2=0x55c7e21d5500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:41:15.664 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.648+0000 7f476c9b6640 1 --2- 172.21.15.99:0/1601207609 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c7e21d3070 0x55c7e21d5500 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:15.664 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.648+0000 7f476c9b6640 1 -- 172.21.15.99:0/1601207609 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c7e21cdf00 msgr2=0x55c7e21d2a00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:41:15.664 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.648+0000 7f47657fa640 1 -- 172.21.15.99:0/1485470062 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f47580cee00 con 0x7f4768100740 2024-03-19T01:41:15.664 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.648+0000 7f4767fff640 1 --2- 172.21.15.99:0/1601207609 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c7e21cdf00 0x55c7e21d2a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:15.664 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.648+0000 7f476c9b6640 1 --2- 172.21.15.99:0/1601207609 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c7e21cdf00 0x55c7e21d2a00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:15.664 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.648+0000 7f476c9b6640 1 -- 172.21.15.99:0/1601207609 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55c7e20cdd20 con 0x55c7e21cd510 2024-03-19T01:41:15.664 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.648+0000 7f476c9b6640 1 --2- 172.21.15.99:0/1485470062 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f47500a2a40 0x7f47500a4f00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:15.664 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.648+0000 7f476c9b6640 1 --2- 172.21.15.99:0/1601207609 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c7e21cd510 0x55c7e21cd930 secure :-1 s=READY pgs=359 cs=0 l=1 rev1=1 crypto rx=0x7f476002c500 tx=0x7f4760009880 comp rx=0 tx=0).ready entity=mon.0 client_cookie=632b2b59c1ab0256 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:41:15.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.648+0000 7f474e7fc640 1 -- 172.21.15.99:0/1601207609 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f476002c890 con 0x55c7e21cd510 2024-03-19T01:41:15.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.648+0000 7f476c9b6640 1 --2- 172.21.15.99:0/1485470062 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f47500a2a40 0x7f47500a4f00 secure :-1 s=READY pgs=248 cs=0 l=1 rev1=1 crypto rx=0x7f476004d000 tx=0x7f476004a000 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:41:15.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.649+0000 7f474e7fc640 1 -- 172.21.15.99:0/1601207609 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f4760004040 con 0x55c7e21cd510 2024-03-19T01:41:15.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.649+0000 7f474e7fc640 1 -- 172.21.15.99:0/1601207609 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4760034670 con 0x55c7e21cd510 2024-03-19T01:41:15.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.649+0000 7f4771009740 1 -- 172.21.15.99:0/1601207609 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c7e21cd510 msgr2=0x55c7e21cd930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:41:15.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.649+0000 7f4771009740 1 --2- 172.21.15.99:0/1601207609 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c7e21cd510 0x55c7e21cd930 secure :-1 s=READY pgs=359 cs=0 l=1 rev1=1 crypto rx=0x7f476002c500 tx=0x7f4760009880 comp rx=0 tx=0).stop 2024-03-19T01:41:15.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.649+0000 7f4771009740 1 -- 172.21.15.99:0/1601207609 shutdown_connections 2024-03-19T01:41:15.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.649+0000 7f4771009740 1 --2- 172.21.15.99:0/1601207609 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c7e21cdf00 0x55c7e21d2a00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:15.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.649+0000 7f4771009740 1 --2- 172.21.15.99:0/1601207609 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c7e21d3070 0x55c7e21d5500 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:15.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.649+0000 7f4771009740 1 --2- 172.21.15.99:0/1601207609 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c7e21cd510 0x55c7e21cd930 unknown :-1 s=CLOSED pgs=359 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:15.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.649+0000 7f4771009740 1 -- 172.21.15.99:0/1601207609 >> 172.21.15.99:0/1601207609 conn(0x55c7e21c4670 msgr2=0x55c7e21c4a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:41:15.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.649+0000 7f4771009740 1 -- 172.21.15.99:0/1601207609 shutdown_connections 2024-03-19T01:41:15.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.649+0000 7f4771009740 1 -- 172.21.15.99:0/1601207609 wait complete. 2024-03-19T01:41:15.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.650+0000 7f4771009740 1 -- start start 2024-03-19T01:41:15.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.650+0000 7f4771009740 1 Processor -- start 2024-03-19T01:41:15.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.650+0000 7f4771009740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c7e21cd510 0x55c7e21d75a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:15.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.650+0000 7f4771009740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c7e21cdf00 0x55c7e21d7ae0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:15.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.650+0000 7f4771009740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c7e21d3070 0x55c7e21deab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:15.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.650+0000 7f4771009740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55c7e21d6020 con 0x55c7e21cdf00 2024-03-19T01:41:15.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.650+0000 7f4771009740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55c7e20e9db0 con 0x55c7e21d3070 2024-03-19T01:41:15.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.650+0000 7f4771009740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55c7e21d61a0 con 0x55c7e21cd510 2024-03-19T01:41:15.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.650+0000 7f4767fff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c7e21cd510 0x55c7e21d75a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:15.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.650+0000 7f4767fff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c7e21cd510 0x55c7e21d75a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:47482/0 (socket says 172.21.15.99:47482) 2024-03-19T01:41:15.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.650+0000 7f4767fff640 1 -- 172.21.15.99:0/3840502090 learned_addr learned my addr 172.21.15.99:0/3840502090 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:41:15.667 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.650+0000 7f4771009740 10 client.0 _ll_register_callbacks cb 0x55c7e20eee70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:41:15.667 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.650+0000 7f47677fe640 1 --2- 172.21.15.99:0/3840502090 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c7e21cdf00 0x55c7e21d7ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:15.667 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.651+0000 7f476c9b6640 1 --2- 172.21.15.99:0/3840502090 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c7e21d3070 0x55c7e21deab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:15.667 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.651+0000 7f4767fff640 1 -- 172.21.15.99:0/3840502090 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c7e21d3070 msgr2=0x55c7e21deab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:41:15.667 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.651+0000 7f4767fff640 1 --2- 172.21.15.99:0/3840502090 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c7e21d3070 0x55c7e21deab0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:15.667 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.651+0000 7f4767fff640 1 -- 172.21.15.99:0/3840502090 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c7e21cdf00 msgr2=0x55c7e21d7ae0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:41:15.667 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.651+0000 7f4767fff640 1 --2- 172.21.15.99:0/3840502090 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c7e21cdf00 0x55c7e21d7ae0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:15.667 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.651+0000 7f4767fff640 1 -- 172.21.15.99:0/3840502090 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55c7e20e8c60 con 0x55c7e21cd510 2024-03-19T01:41:15.667 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.651+0000 7f4767fff640 1 --2- 172.21.15.99:0/3840502090 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c7e21cd510 0x55c7e21d75a0 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7f4758009230 tx=0x7f475809af00 comp rx=0 tx=0).ready entity=mon.1 client_cookie=2d061f7025dc17db server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:41:15.667 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.651+0000 7f47477fe640 10 client.0 ms_handle_connect on v2:172.21.15.115:3300/0 2024-03-19T01:41:15.667 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.651+0000 7f47477fe640 1 -- 172.21.15.99:0/3840502090 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f47580b3020 con 0x55c7e21cd510 2024-03-19T01:41:15.667 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.651+0000 7f47477fe640 1 -- 172.21.15.99:0/3840502090 <== mon.1 v2:172.21.15.115:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f47580a38b0 con 0x55c7e21cd510 2024-03-19T01:41:15.667 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.651+0000 7f4771009740 1 -- 172.21.15.99:0/3840502090 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f4760004040 con 0x55c7e21cd510 2024-03-19T01:41:15.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.651+0000 7f47477fe640 1 -- 172.21.15.99:0/3840502090 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_get_version_reply(handle=1 version=21) ==== 24+0+0 (secure 0 0 0) 0x7f47580b4450 con 0x55c7e21cd510 2024-03-19T01:41:15.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.651+0000 7f4771009740 10 client.25168 fetch_fsmap learned FSMap version 21 2024-03-19T01:41:15.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.651+0000 7f4771009740 1 -- 172.21.15.99:0/3840502090 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,fsmap=21,monmap=4+}) -- 0x55c7e1e42210 con 0x55c7e21cd510 2024-03-19T01:41:15.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.651+0000 7f47477fe640 1 -- 172.21.15.99:0/3840502090 <== mon.1 v2:172.21.15.115:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f4760004040 con 0x55c7e21cd510 2024-03-19T01:41:15.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.652+0000 7f47477fe640 1 -- 172.21.15.99:0/3840502090 <== mon.1 v2:172.21.15.115:3300/0 5 ==== fsmap(e 21) ==== 8338+0+0 (secure 0 0 0) 0x7f4758084b30 con 0x55c7e21cd510 2024-03-19T01:41:15.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.652+0000 7f4771009740 10 client.25168 fetch_fsmap finished waiting for FSMap version 21 2024-03-19T01:41:15.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.652+0000 7f4771009740 10 client.25168 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:41:15.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.652+0000 7f4771009740 20 client.25168 populate_metadata read hostname 'smithi099' 2024-03-19T01:41:15.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.652+0000 7f4771009740 1 --2- 172.21.15.99:0/3840502090 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55c7e21e3390 0x55c7e21e5820 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:15.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.652+0000 7f4771009740 4 client.25168 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:41:15.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.652+0000 7f4771009740 1 -- 172.21.15.99:0/3840502090 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55c7e213b120 con 0x55c7e21e3390 2024-03-19T01:41:15.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.652+0000 7f47677fe640 1 --2- 172.21.15.99:0/3840502090 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55c7e21e3390 0x55c7e21e5820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:15.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.652+0000 7f47477fe640 10 client.25168 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:41:15.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.652+0000 7f47677fe640 1 --2- 172.21.15.99:0/3840502090 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55c7e21e3390 0x55c7e21e5820 crc :-1 s=READY pgs=29 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=6890156b749de713 server_cookie=a1ed0cf31c294de in_seq=0 out_seq=0 2024-03-19T01:41:15.669 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.653+0000 7f47477fe640 1 -- 172.21.15.99:0/3840502090 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x55c7e213b120 con 0x55c7e21e3390 2024-03-19T01:41:15.669 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.653+0000 7f47477fe640 10 client.25168 handle_command_reply: tid=0 2024-03-19T01:41:15.669 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.653+0000 7f4771009740 1 client.25168 shutdown 2024-03-19T01:41:15.669 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.653+0000 7f4771009740 1 -- 172.21.15.99:0/3840502090 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c7e21cd510 msgr2=0x55c7e21d75a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:41:15.669 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.653+0000 7f4771009740 1 --2- 172.21.15.99:0/3840502090 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c7e21cd510 0x55c7e21d75a0 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7f4758009230 tx=0x7f475809af00 comp rx=0 tx=0).stop 2024-03-19T01:41:15.669 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.653+0000 7f4771009740 1 -- 172.21.15.99:0/3840502090 shutdown_connections 2024-03-19T01:41:15.669 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.653+0000 7f4771009740 1 --2- 172.21.15.99:0/3840502090 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55c7e21e3390 0x55c7e21e5820 crc :-1 s=READY pgs=29 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:15.669 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.653+0000 7f4771009740 1 --2- 172.21.15.99:0/3840502090 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c7e21d3070 0x55c7e21deab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:15.669 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.653+0000 7f47477fe640 0 client.25168 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:41:15.669 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.653+0000 7f4771009740 1 --2- 172.21.15.99:0/3840502090 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c7e21cdf00 0x55c7e21d7ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:15.669 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.653+0000 7f4771009740 1 --2- 172.21.15.99:0/3840502090 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c7e21cd510 0x55c7e21d75a0 unknown :-1 s=CLOSED pgs=76 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:15.669 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.653+0000 7f4771009740 1 -- 172.21.15.99:0/3840502090 >> 172.21.15.99:0/3840502090 conn(0x55c7e21c4670 msgr2=0x55c7e21c58c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:41:15.669 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.653+0000 7f4771009740 1 -- 172.21.15.99:0/3840502090 shutdown_connections 2024-03-19T01:41:15.670 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.653+0000 7f4771009740 1 -- 172.21.15.99:0/3840502090 wait complete. 2024-03-19T01:41:15.670 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.653+0000 7f4771009740 20 client.25168 trim_cache size 0 max 16384 2024-03-19T01:41:15.675 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.668+0000 7f4771009740 1 Processor -- start 2024-03-19T01:41:15.675 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.668+0000 7f4771009740 1 -- start start 2024-03-19T01:41:15.675 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.668+0000 7f4771009740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c7e21cd510 0x55c7e21d78c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:15.675 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.668+0000 7f4771009740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c7e21cdf00 0x55c7e21ce790 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:15.675 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.668+0000 7f4771009740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c7e21d3070 0x55c7e21cecd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:15.675 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.668+0000 7f4771009740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f4754010e70 con 0x55c7e21cdf00 2024-03-19T01:41:15.675 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.668+0000 7f4771009740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f475400e740 con 0x55c7e21cd510 2024-03-19T01:41:15.675 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.668+0000 7f4771009740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f475400c590 con 0x55c7e21d3070 2024-03-19T01:41:15.675 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.668+0000 7f47677fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c7e21cdf00 0x55c7e21ce790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:15.675 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.669+0000 7f476c9b6640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c7e21d3070 0x55c7e21cecd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:15.675 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.669+0000 7f47677fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c7e21cdf00 0x55c7e21ce790 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:60186/0 (socket says 172.21.15.99:60186) 2024-03-19T01:41:15.676 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.669+0000 7f47677fe640 1 -- 172.21.15.99:0/1655857356 learned_addr learned my addr 172.21.15.99:0/1655857356 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:41:15.676 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.669+0000 7f4767fff640 1 --2- 172.21.15.99:0/1655857356 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c7e21cd510 0x55c7e21d78c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:15.676 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.669+0000 7f4767fff640 1 -- 172.21.15.99:0/1655857356 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c7e21d3070 msgr2=0x55c7e21cecd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:41:15.676 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.669+0000 7f4767fff640 1 --2- 172.21.15.99:0/1655857356 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c7e21d3070 0x55c7e21cecd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:15.676 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.669+0000 7f4767fff640 1 -- 172.21.15.99:0/1655857356 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c7e21cdf00 msgr2=0x55c7e21ce790 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:41:15.676 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.669+0000 7f4767fff640 1 --2- 172.21.15.99:0/1655857356 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c7e21cdf00 0x55c7e21ce790 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:15.676 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.669+0000 7f4767fff640 1 -- 172.21.15.99:0/1655857356 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55c7e1f0e620 con 0x55c7e21cd510 2024-03-19T01:41:15.676 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.669+0000 7f476c9b6640 1 --2- 172.21.15.99:0/1655857356 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c7e21d3070 0x55c7e21cecd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:41:15.676 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.669+0000 7f47677fe640 1 --2- 172.21.15.99:0/1655857356 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c7e21cdf00 0x55c7e21ce790 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:41:15.676 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.669+0000 7f4767fff640 1 --2- 172.21.15.99:0/1655857356 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c7e21cd510 0x55c7e21d78c0 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7f4758084f50 tx=0x7f47580ccb70 comp rx=0 tx=0).ready entity=mon.2 client_cookie=86108a62b79abd08 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:41:15.676 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.669+0000 7f474dffb640 1 -- 172.21.15.99:0/1655857356 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f475800ce60 con 0x55c7e21cd510 2024-03-19T01:41:15.676 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.669+0000 7f474dffb640 1 -- 172.21.15.99:0/1655857356 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f47580a3d40 con 0x55c7e21cd510 2024-03-19T01:41:15.676 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.669+0000 7f474dffb640 1 -- 172.21.15.99:0/1655857356 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f475809a9f0 con 0x55c7e21cd510 2024-03-19T01:41:15.676 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.670+0000 7f4771009740 1 -- 172.21.15.99:0/1655857356 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c7e21cd510 msgr2=0x55c7e21d78c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:41:15.677 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.670+0000 7f4771009740 1 --2- 172.21.15.99:0/1655857356 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c7e21cd510 0x55c7e21d78c0 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7f4758084f50 tx=0x7f47580ccb70 comp rx=0 tx=0).stop 2024-03-19T01:41:15.677 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.670+0000 7f4771009740 1 -- 172.21.15.99:0/1655857356 shutdown_connections 2024-03-19T01:41:15.677 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.670+0000 7f4771009740 1 --2- 172.21.15.99:0/1655857356 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c7e21d3070 0x55c7e21cecd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:15.677 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.670+0000 7f4771009740 1 --2- 172.21.15.99:0/1655857356 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c7e21cdf00 0x55c7e21ce790 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:15.677 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.670+0000 7f4771009740 1 --2- 172.21.15.99:0/1655857356 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c7e21cd510 0x55c7e21d78c0 unknown :-1 s=CLOSED pgs=85 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:15.677 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.670+0000 7f4771009740 1 -- 172.21.15.99:0/1655857356 >> 172.21.15.99:0/1655857356 conn(0x55c7e21c4670 msgr2=0x55c7e21d71b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:41:15.677 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.670+0000 7f4771009740 1 -- 172.21.15.99:0/1655857356 shutdown_connections 2024-03-19T01:41:15.677 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.670+0000 7f4771009740 1 -- 172.21.15.99:0/1655857356 wait complete. 2024-03-19T01:41:15.677 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.670+0000 7f4771009740 1 -- start start 2024-03-19T01:41:15.677 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.671+0000 7f4771009740 1 Processor -- start 2024-03-19T01:41:15.677 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.671+0000 7f4771009740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c7e21cd510 0x55c7e21dc3c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:15.677 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.671+0000 7f4771009740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c7e21cdf00 0x55c7e21dc900 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:15.677 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.671+0000 7f4771009740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c7e21d3070 0x55c7e21dd910 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:15.677 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.671+0000 7f4771009740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f475400cc00 con 0x55c7e21cdf00 2024-03-19T01:41:15.677 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.671+0000 7f4771009740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f475400e070 con 0x55c7e21d3070 2024-03-19T01:41:15.678 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.671+0000 7f4771009740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f475400d2d0 con 0x55c7e21cd510 2024-03-19T01:41:15.678 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.671+0000 7f4771009740 10 client.0 _ll_register_callbacks cb 0x7f4758009600 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:41:15.678 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.671+0000 7f4767fff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c7e21cd510 0x55c7e21dc3c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:15.678 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.671+0000 7f47677fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c7e21cdf00 0x55c7e21dc900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:15.678 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.671+0000 7f4767fff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c7e21cd510 0x55c7e21dc3c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:47496/0 (socket says 172.21.15.99:47496) 2024-03-19T01:41:15.678 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.671+0000 7f4767fff640 1 -- 172.21.15.99:0/748414307 learned_addr learned my addr 172.21.15.99:0/748414307 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:41:15.678 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.671+0000 7f476c9b6640 1 --2- 172.21.15.99:0/748414307 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c7e21d3070 0x55c7e21dd910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:15.678 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.671+0000 7f4767fff640 1 -- 172.21.15.99:0/748414307 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c7e21d3070 msgr2=0x55c7e21dd910 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:41:15.678 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.671+0000 7f4767fff640 1 --2- 172.21.15.99:0/748414307 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c7e21d3070 0x55c7e21dd910 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:15.678 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.671+0000 7f4767fff640 1 -- 172.21.15.99:0/748414307 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c7e21cdf00 msgr2=0x55c7e21dc900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:41:15.678 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.671+0000 7f4767fff640 1 --2- 172.21.15.99:0/748414307 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c7e21cdf00 0x55c7e21dc900 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:15.678 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.671+0000 7f4767fff640 1 -- 172.21.15.99:0/748414307 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55c7e21ddf00 con 0x55c7e21cd510 2024-03-19T01:41:15.679 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.671+0000 7f47677fe640 1 --2- 172.21.15.99:0/748414307 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c7e21cdf00 0x55c7e21dc900 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:41:15.679 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.671+0000 7f4767fff640 1 --2- 172.21.15.99:0/748414307 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c7e21cd510 0x55c7e21dc3c0 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 crypto rx=0x7f475801b040 tx=0x7f475809b160 comp rx=0 tx=0).ready entity=mon.1 client_cookie=1685c7e1f7f4c615 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:41:15.679 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.671+0000 7f47477fe640 10 client.0 ms_handle_connect on v2:172.21.15.115:3300/0 2024-03-19T01:41:15.679 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.672+0000 7f47477fe640 1 -- 172.21.15.99:0/748414307 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f47580b8070 con 0x55c7e21cd510 2024-03-19T01:41:15.679 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.672+0000 7f47477fe640 1 -- 172.21.15.99:0/748414307 <== mon.1 v2:172.21.15.115:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f47580972f0 con 0x55c7e21cd510 2024-03-19T01:41:15.679 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.672+0000 7f4771009740 1 -- 172.21.15.99:0/748414307 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f47580a3d40 con 0x55c7e21cd510 2024-03-19T01:41:15.679 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.672+0000 7f47477fe640 1 -- 172.21.15.99:0/748414307 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_get_version_reply(handle=1 version=21) ==== 24+0+0 (secure 0 0 0) 0x7f475800ce60 con 0x55c7e21cd510 2024-03-19T01:41:15.679 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.672+0000 7f4771009740 10 client.25174 fetch_fsmap learned FSMap version 21 2024-03-19T01:41:15.679 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.672+0000 7f4771009740 1 -- 172.21.15.99:0/748414307 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,fsmap=21,monmap=4+}) -- 0x55c7e2249ca0 con 0x55c7e21cd510 2024-03-19T01:41:15.679 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.672+0000 7f47477fe640 1 -- 172.21.15.99:0/748414307 <== mon.1 v2:172.21.15.115:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f47580a3d40 con 0x55c7e21cd510 2024-03-19T01:41:15.679 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.672+0000 7f47477fe640 1 -- 172.21.15.99:0/748414307 <== mon.1 v2:172.21.15.115:3300/0 5 ==== fsmap(e 21) ==== 8338+0+0 (secure 0 0 0) 0x7f475809a620 con 0x55c7e21cd510 2024-03-19T01:41:15.679 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.672+0000 7f4771009740 10 client.25174 fetch_fsmap finished waiting for FSMap version 21 2024-03-19T01:41:15.679 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.672+0000 7f4771009740 10 client.25174 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:41:15.679 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.672+0000 7f4771009740 20 client.25174 populate_metadata read hostname 'smithi099' 2024-03-19T01:41:15.680 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.672+0000 7f4771009740 1 --2- 172.21.15.99:0/748414307 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55c7e224a420 0x55c7e224c840 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:15.680 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.672+0000 7f4771009740 4 client.25174 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:41:15.680 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.672+0000 7f4771009740 1 -- 172.21.15.99:0/748414307 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55c7e2135470 con 0x55c7e224a420 2024-03-19T01:41:15.680 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.672+0000 7f47677fe640 1 --2- 172.21.15.99:0/748414307 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55c7e224a420 0x55c7e224c840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:15.680 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.673+0000 7f47477fe640 10 client.25174 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:41:15.680 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.673+0000 7f47677fe640 1 --2- 172.21.15.99:0/748414307 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55c7e224a420 0x55c7e224c840 crc :-1 s=READY pgs=30 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=efd3a6e8219abf33 server_cookie=24648c36f8d15587 in_seq=0 out_seq=0 2024-03-19T01:41:15.681 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.678+0000 7f47477fe640 1 -- 172.21.15.99:0/748414307 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x55c7e2135470 con 0x55c7e224a420 2024-03-19T01:41:15.681 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.678+0000 7f47477fe640 10 client.25174 handle_command_reply: tid=0 2024-03-19T01:41:15.681 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.678+0000 7f4771009740 1 client.25174 shutdown 2024-03-19T01:41:15.681 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.678+0000 7f4771009740 1 -- 172.21.15.99:0/748414307 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c7e21cd510 msgr2=0x55c7e21dc3c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:41:15.681 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.678+0000 7f4771009740 1 --2- 172.21.15.99:0/748414307 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c7e21cd510 0x55c7e21dc3c0 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 crypto rx=0x7f475801b040 tx=0x7f475809b160 comp rx=0 tx=0).stop 2024-03-19T01:41:15.682 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.678+0000 7f4771009740 1 -- 172.21.15.99:0/748414307 shutdown_connections 2024-03-19T01:41:15.682 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.678+0000 7f4771009740 1 --2- 172.21.15.99:0/748414307 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55c7e224a420 0x55c7e224c840 crc :-1 s=READY pgs=30 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:15.682 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.678+0000 7f4771009740 1 --2- 172.21.15.99:0/748414307 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c7e21d3070 0x55c7e21dd910 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:15.682 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.678+0000 7f4771009740 1 --2- 172.21.15.99:0/748414307 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c7e21cdf00 0x55c7e21dc900 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:15.682 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.678+0000 7f4771009740 1 --2- 172.21.15.99:0/748414307 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c7e21cd510 0x55c7e21dc3c0 unknown :-1 s=CLOSED pgs=77 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:15.682 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.678+0000 7f4771009740 1 -- 172.21.15.99:0/748414307 >> 172.21.15.99:0/748414307 conn(0x55c7e21c4670 msgr2=0x55c7e21c78d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:41:15.682 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.679+0000 7f47477fe640 0 client.25174 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:41:15.682 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.679+0000 7f4771009740 1 -- 172.21.15.99:0/748414307 shutdown_connections 2024-03-19T01:41:15.682 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.679+0000 7f4771009740 1 -- 172.21.15.99:0/748414307 wait complete. 2024-03-19T01:41:15.682 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.679+0000 7f4771009740 20 client.25174 trim_cache size 0 max 16384 2024-03-19T01:41:15.683 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:41:15.683 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:41:15.683 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:41:15.683 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:41:15.684 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.679+0000 7f476e419640 1 -- 172.21.15.99:0/1485470062 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f47500a2a40 msgr2=0x7f47500a4f00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:41:15.684 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.679+0000 7f476e419640 1 --2- 172.21.15.99:0/1485470062 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f47500a2a40 0x7f47500a4f00 secure :-1 s=READY pgs=248 cs=0 l=1 rev1=1 crypto rx=0x7f476004d000 tx=0x7f476004a000 comp rx=0 tx=0).stop 2024-03-19T01:41:15.684 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.679+0000 7f476e419640 1 -- 172.21.15.99:0/1485470062 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4768100740 msgr2=0x7f4768069a30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:41:15.684 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.679+0000 7f476e419640 1 --2- 172.21.15.99:0/1485470062 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4768100740 0x7f4768069a30 secure :-1 s=READY pgs=358 cs=0 l=1 rev1=1 crypto rx=0x7f4758005fd0 tx=0x7f475800ec20 comp rx=0 tx=0).stop 2024-03-19T01:41:15.684 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.679+0000 7f476e419640 1 -- 172.21.15.99:0/1485470062 shutdown_connections 2024-03-19T01:41:15.684 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.679+0000 7f476e419640 1 --2- 172.21.15.99:0/1485470062 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f47500a2a40 0x7f47500a4f00 unknown :-1 s=CLOSED pgs=248 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:15.684 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.679+0000 7f476e419640 1 --2- 172.21.15.99:0/1485470062 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f47681022f0 0x7f476819ee00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:15.684 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.679+0000 7f476e419640 1 --2- 172.21.15.99:0/1485470062 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4768101930 0x7f476819c7c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:15.684 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.679+0000 7f476e419640 1 --2- 172.21.15.99:0/1485470062 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4768100740 0x7f4768069a30 unknown :-1 s=CLOSED pgs=358 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:15.684 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.679+0000 7f476e419640 1 -- 172.21.15.99:0/1485470062 >> 172.21.15.99:0/1485470062 conn(0x7f47680fbcb0 msgr2=0x7f4768104b60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:41:15.684 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.680+0000 7f476e419640 1 -- 172.21.15.99:0/1485470062 shutdown_connections 2024-03-19T01:41:15.685 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:15.680+0000 7f476e419640 1 -- 172.21.15.99:0/1485470062 wait complete. 2024-03-19T01:41:16.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:41:15 smithi099 ceph-mon[62951]: pgmap v355: 97 pgs: 97 active+clean; 659 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.8 KiB/s rd, 1.7 KiB/s wr, 9 op/s 2024-03-19T01:41:16.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:41:15 smithi112 ceph-mon[66819]: pgmap v355: 97 pgs: 97 active+clean; 659 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.8 KiB/s rd, 1.7 KiB/s wr, 9 op/s 2024-03-19T01:41:16.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:41:15 smithi115 ceph-mon[67332]: pgmap v355: 97 pgs: 97 active+clean; 659 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.8 KiB/s rd, 1.7 KiB/s wr, 9 op/s 2024-03-19T01:41:18.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:41:17 smithi099 ceph-mon[62951]: pgmap v356: 97 pgs: 97 active+clean; 659 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 1.7 KiB/s wr, 10 op/s 2024-03-19T01:41:18.108 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:41:17 smithi115 ceph-mon[67332]: pgmap v356: 97 pgs: 97 active+clean; 659 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 1.7 KiB/s wr, 10 op/s 2024-03-19T01:41:18.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:41:17 smithi112 ceph-mon[66819]: pgmap v356: 97 pgs: 97 active+clean; 659 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 1.7 KiB/s wr, 10 op/s 2024-03-19T01:41:20.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:41:19 smithi099 ceph-mon[62951]: pgmap v357: 97 pgs: 97 active+clean; 659 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 511 B/s wr, 7 op/s 2024-03-19T01:41:20.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:41:19 smithi112 ceph-mon[66819]: pgmap v357: 97 pgs: 97 active+clean; 659 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 511 B/s wr, 7 op/s 2024-03-19T01:41:20.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:41:19 smithi115 ceph-mon[67332]: pgmap v357: 97 pgs: 97 active+clean; 659 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 511 B/s wr, 7 op/s 2024-03-19T01:41:22.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:41:21 smithi099 ceph-mon[62951]: pgmap v358: 97 pgs: 97 active+clean; 660 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.4 KiB/s rd, 511 B/s wr, 7 op/s 2024-03-19T01:41:22.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:41:21 smithi112 ceph-mon[66819]: pgmap v358: 97 pgs: 97 active+clean; 660 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.4 KiB/s rd, 511 B/s wr, 7 op/s 2024-03-19T01:41:22.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:41:21 smithi115 ceph-mon[67332]: pgmap v358: 97 pgs: 97 active+clean; 660 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.4 KiB/s rd, 511 B/s wr, 7 op/s 2024-03-19T01:41:24.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:41:23 smithi099 ceph-mon[62951]: pgmap v359: 97 pgs: 97 active+clean; 660 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.0 KiB/s rd, 511 B/s wr, 7 op/s 2024-03-19T01:41:24.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:41:23 smithi112 ceph-mon[66819]: pgmap v359: 97 pgs: 97 active+clean; 660 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.0 KiB/s rd, 511 B/s wr, 7 op/s 2024-03-19T01:41:24.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:41:23 smithi115 ceph-mon[67332]: pgmap v359: 97 pgs: 97 active+clean; 660 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.0 KiB/s rd, 511 B/s wr, 7 op/s 2024-03-19T01:41:26.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:41:25 smithi099 ceph-mon[62951]: pgmap v360: 97 pgs: 97 active+clean; 660 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.6 KiB/s rd, 511 B/s wr, 6 op/s 2024-03-19T01:41:26.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:41:25 smithi112 ceph-mon[66819]: pgmap v360: 97 pgs: 97 active+clean; 660 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.6 KiB/s rd, 511 B/s wr, 6 op/s 2024-03-19T01:41:26.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:41:25 smithi115 ceph-mon[67332]: pgmap v360: 97 pgs: 97 active+clean; 660 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 2.6 KiB/s rd, 511 B/s wr, 6 op/s 2024-03-19T01:41:28.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:41:27 smithi099 ceph-mon[62951]: pgmap v361: 97 pgs: 97 active+clean; 670 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 1.4 KiB/s wr, 7 op/s 2024-03-19T01:41:28.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:41:27 smithi112 ceph-mon[66819]: pgmap v361: 97 pgs: 97 active+clean; 670 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 1.4 KiB/s wr, 7 op/s 2024-03-19T01:41:28.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:41:27 smithi115 ceph-mon[67332]: pgmap v361: 97 pgs: 97 active+clean; 670 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 1.4 KiB/s wr, 7 op/s 2024-03-19T01:41:28.770 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.767+0000 7f587480b640 1 -- 172.21.15.99:0/4035002902 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f58701022f0 msgr2=0x7f5870108b80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:41:28.770 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.767+0000 7f587480b640 1 --2- 172.21.15.99:0/4035002902 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f58701022f0 0x7f5870108b80 secure :-1 s=READY pgs=360 cs=0 l=1 rev1=1 crypto rx=0x7f5868009ca0 tx=0x7f586802c0f0 comp rx=0 tx=0).stop 2024-03-19T01:41:28.770 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.767+0000 7f587480b640 1 -- 172.21.15.99:0/4035002902 shutdown_connections 2024-03-19T01:41:28.770 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.767+0000 7f587480b640 1 --2- 172.21.15.99:0/4035002902 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5870101930 0x7f5870101db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:28.771 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.767+0000 7f587480b640 1 --2- 172.21.15.99:0/4035002902 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f58701022f0 0x7f5870108b80 unknown :-1 s=CLOSED pgs=360 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:28.771 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.767+0000 7f587480b640 1 --2- 172.21.15.99:0/4035002902 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5870100740 0x7f5870100b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:28.771 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.767+0000 7f587480b640 1 -- 172.21.15.99:0/4035002902 >> 172.21.15.99:0/4035002902 conn(0x7f58700fbcb0 msgr2=0x7f58700fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:41:28.771 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.767+0000 7f587480b640 1 -- 172.21.15.99:0/4035002902 shutdown_connections 2024-03-19T01:41:28.771 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.767+0000 7f587480b640 1 -- 172.21.15.99:0/4035002902 wait complete. 2024-03-19T01:41:28.771 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.768+0000 7f587480b640 1 Processor -- start 2024-03-19T01:41:28.771 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.768+0000 7f587480b640 1 -- start start 2024-03-19T01:41:28.771 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.768+0000 7f587480b640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5870100740 0x7f5870197a40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:28.771 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.768+0000 7f587480b640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5870101930 0x7f5870197f80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:28.771 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.768+0000 7f587480b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f58701022f0 0x7f587019eff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:28.771 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.768+0000 7f587480b640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f587010d4e0 con 0x7f58701022f0 2024-03-19T01:41:28.771 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.768+0000 7f587480b640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f587010d360 con 0x7f5870100740 2024-03-19T01:41:28.771 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.768+0000 7f587480b640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f587010d660 con 0x7f5870101930 2024-03-19T01:41:28.771 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.768+0000 7f586ed9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f58701022f0 0x7f587019eff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:28.771 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.768+0000 7f586ed9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f58701022f0 0x7f587019eff0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:52642/0 (socket says 172.21.15.99:52642) 2024-03-19T01:41:28.771 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.768+0000 7f586ed9d640 1 -- 172.21.15.99:0/1720526961 learned_addr learned my addr 172.21.15.99:0/1720526961 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:41:28.772 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.768+0000 7f586e59c640 1 --2- 172.21.15.99:0/1720526961 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5870100740 0x7f5870197a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:28.772 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.768+0000 7f586dd9b640 1 --2- 172.21.15.99:0/1720526961 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5870101930 0x7f5870197f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:28.772 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.768+0000 7f586ed9d640 1 -- 172.21.15.99:0/1720526961 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5870101930 msgr2=0x7f5870197f80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:41:28.772 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.768+0000 7f586ed9d640 1 --2- 172.21.15.99:0/1720526961 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5870101930 0x7f5870197f80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:28.772 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.768+0000 7f586ed9d640 1 -- 172.21.15.99:0/1720526961 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5870100740 msgr2=0x7f5870197a40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:41:28.772 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.768+0000 7f586ed9d640 1 --2- 172.21.15.99:0/1720526961 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5870100740 0x7f5870197a40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:28.772 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.768+0000 7f586ed9d640 1 -- 172.21.15.99:0/1720526961 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f587019f660 con 0x7f58701022f0 2024-03-19T01:41:28.772 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.769+0000 7f586ed9d640 1 --2- 172.21.15.99:0/1720526961 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f58701022f0 0x7f587019eff0 secure :-1 s=READY pgs=361 cs=0 l=1 rev1=1 crypto rx=0x7f5868037040 tx=0x7f5868032f40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:41:28.772 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.769+0000 7f585f7fe640 1 -- 172.21.15.99:0/1720526961 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5868041020 con 0x7f58701022f0 2024-03-19T01:41:28.772 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.769+0000 7f585f7fe640 1 -- 172.21.15.99:0/1720526961 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f5868034cd0 con 0x7f58701022f0 2024-03-19T01:41:28.772 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.769+0000 7f587480b640 1 -- 172.21.15.99:0/1720526961 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f587019f8f0 con 0x7f58701022f0 2024-03-19T01:41:28.772 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.769+0000 7f585f7fe640 1 -- 172.21.15.99:0/1720526961 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5868039040 con 0x7f58701022f0 2024-03-19T01:41:28.772 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.769+0000 7f587480b640 1 -- 172.21.15.99:0/1720526961 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f587019fde0 con 0x7f58701022f0 2024-03-19T01:41:28.779 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.771+0000 7f585f7fe640 1 -- 172.21.15.99:0/1720526961 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 19) ==== 137341+0+0 (secure 0 0 0) 0x7f5868044020 con 0x7f58701022f0 2024-03-19T01:41:28.779 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.771+0000 7f58773f8740 1 Processor -- start 2024-03-19T01:41:28.779 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.772+0000 7f58773f8740 1 -- start start 2024-03-19T01:41:28.779 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.772+0000 7f58773f8740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5627571c0100 0x5627571c0520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:28.779 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.772+0000 7f585f7fe640 1 --2- 172.21.15.99:0/1720526961 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f58580a2940 0x7f58580a4e00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:28.779 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.772+0000 7f58773f8740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5627571c0af0 0x5627571c55f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:28.779 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.772+0000 7f58773f8740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5627571c5c60 0x5627571c80f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:28.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.772+0000 7f58773f8740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x562757124be0 con 0x5627571c0100 2024-03-19T01:41:28.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.772+0000 7f58773f8740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x5627570cd960 con 0x5627571c5c60 2024-03-19T01:41:28.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.772+0000 7f58773f8740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x562757120c80 con 0x5627571c0af0 2024-03-19T01:41:28.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.772+0000 7f586e59c640 1 --2- 172.21.15.99:0/1720526961 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f58580a2940 0x7f58580a4e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:28.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.772+0000 7f586dd9b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5627571c0100 0x5627571c0520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:28.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.772+0000 7f586ed9d640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5627571c5c60 0x5627571c80f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:28.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.772+0000 7f586dd9b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5627571c0100 0x5627571c0520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:52654/0 (socket says 172.21.15.99:52654) 2024-03-19T01:41:28.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.772+0000 7f586dd9b640 1 -- 172.21.15.99:0/685476688 learned_addr learned my addr 172.21.15.99:0/685476688 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:41:28.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.772+0000 7f585f7fe640 1 -- 172.21.15.99:0/1720526961 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f58680f0a10 con 0x7f58701022f0 2024-03-19T01:41:28.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.772+0000 7f586dd9b640 1 --2- 172.21.15.99:0/685476688 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5627571c0af0 0x5627571c55f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:28.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.772+0000 7f586ed9d640 1 -- 172.21.15.99:0/685476688 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5627571c0af0 msgr2=0x5627571c55f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:41:28.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.772+0000 7f586ed9d640 1 --2- 172.21.15.99:0/685476688 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5627571c0af0 0x5627571c55f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:28.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.772+0000 7f586ed9d640 1 -- 172.21.15.99:0/685476688 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5627571c0100 msgr2=0x5627571c0520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:41:28.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.772+0000 7f586ed9d640 1 --2- 172.21.15.99:0/685476688 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5627571c0100 0x5627571c0520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:28.781 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.772+0000 7f586ed9d640 1 -- 172.21.15.99:0/685476688 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5627570c4d20 con 0x5627571c5c60 2024-03-19T01:41:28.781 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.772+0000 7f586dd9b640 1 --2- 172.21.15.99:0/685476688 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5627571c0af0 0x5627571c55f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:41:28.781 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.772+0000 7f586ed9d640 1 --2- 172.21.15.99:0/685476688 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5627571c5c60 0x5627571c80f0 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 crypto rx=0x7f58680ee490 tx=0x7f586803a8e0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=8a13afca7ca7c936 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:41:28.781 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.772+0000 7f586e59c640 1 --2- 172.21.15.99:0/1720526961 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f58580a2940 0x7f58580a4e00 secure :-1 s=READY pgs=249 cs=0 l=1 rev1=1 crypto rx=0x7f5870101790 tx=0x7f5860005f90 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:41:28.781 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.773+0000 7f585cff9640 1 -- 172.21.15.99:0/685476688 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f58680bcef0 con 0x5627571c5c60 2024-03-19T01:41:28.781 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.773+0000 7f585cff9640 1 -- 172.21.15.99:0/685476688 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f58680c6980 con 0x5627571c5c60 2024-03-19T01:41:28.781 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.773+0000 7f585cff9640 1 -- 172.21.15.99:0/685476688 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f58680ca6c0 con 0x5627571c5c60 2024-03-19T01:41:28.781 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.773+0000 7f58773f8740 1 -- 172.21.15.99:0/685476688 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5627571c5c60 msgr2=0x5627571c80f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:41:28.781 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.773+0000 7f58773f8740 1 --2- 172.21.15.99:0/685476688 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5627571c5c60 0x5627571c80f0 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 crypto rx=0x7f58680ee490 tx=0x7f586803a8e0 comp rx=0 tx=0).stop 2024-03-19T01:41:28.781 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.773+0000 7f58773f8740 1 -- 172.21.15.99:0/685476688 shutdown_connections 2024-03-19T01:41:28.781 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.773+0000 7f58773f8740 1 --2- 172.21.15.99:0/685476688 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5627571c5c60 0x5627571c80f0 unknown :-1 s=CLOSED pgs=86 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:28.781 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.773+0000 7f58773f8740 1 --2- 172.21.15.99:0/685476688 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5627571c0af0 0x5627571c55f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:28.781 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.773+0000 7f58773f8740 1 --2- 172.21.15.99:0/685476688 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5627571c0100 0x5627571c0520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:28.781 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.773+0000 7f58773f8740 1 -- 172.21.15.99:0/685476688 >> 172.21.15.99:0/685476688 conn(0x5627571bb670 msgr2=0x5627571bba70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:41:28.781 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.773+0000 7f58773f8740 1 -- 172.21.15.99:0/685476688 shutdown_connections 2024-03-19T01:41:28.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.774+0000 7f58773f8740 1 -- 172.21.15.99:0/685476688 wait complete. 2024-03-19T01:41:28.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.774+0000 7f58773f8740 1 -- start start 2024-03-19T01:41:28.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.774+0000 7f58773f8740 1 Processor -- start 2024-03-19T01:41:28.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.774+0000 7f58773f8740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5627571c0100 0x5627571c9d90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:28.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.774+0000 7f58773f8740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5627571c0af0 0x5627571ca2d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:28.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.774+0000 7f58773f8740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5627571c5c60 0x5627571d1310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:28.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.774+0000 7f58773f8740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x5627571c89d0 con 0x5627571c0100 2024-03-19T01:41:28.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.774+0000 7f58773f8740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x5627570e0db0 con 0x5627571c5c60 2024-03-19T01:41:28.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.774+0000 7f58773f8740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x5627571c8b50 con 0x5627571c0af0 2024-03-19T01:41:28.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.774+0000 7f58773f8740 10 client.0 _ll_register_callbacks cb 0x5627570e5e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:41:28.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.774+0000 7f586e59c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5627571c0100 0x5627571c9d90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:28.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.774+0000 7f586ed9d640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5627571c5c60 0x5627571d1310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:28.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.774+0000 7f586e59c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5627571c0100 0x5627571c9d90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:52660/0 (socket says 172.21.15.99:52660) 2024-03-19T01:41:28.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.774+0000 7f586e59c640 1 -- 172.21.15.99:0/1965801498 learned_addr learned my addr 172.21.15.99:0/1965801498 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:41:28.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.774+0000 7f586dd9b640 1 --2- 172.21.15.99:0/1965801498 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5627571c0af0 0x5627571ca2d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:28.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.774+0000 7f586e59c640 1 -- 172.21.15.99:0/1965801498 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5627571c0af0 msgr2=0x5627571ca2d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:41:28.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.774+0000 7f586e59c640 1 --2- 172.21.15.99:0/1965801498 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5627571c0af0 0x5627571ca2d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:28.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.774+0000 7f586e59c640 1 -- 172.21.15.99:0/1965801498 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5627571c5c60 msgr2=0x5627571d1310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:41:28.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.774+0000 7f586e59c640 1 --2- 172.21.15.99:0/1965801498 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5627571c5c60 0x5627571d1310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:28.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.774+0000 7f586e59c640 1 -- 172.21.15.99:0/1965801498 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5627570dfc60 con 0x5627571c0100 2024-03-19T01:41:28.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.774+0000 7f586ed9d640 1 --2- 172.21.15.99:0/1965801498 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5627571c5c60 0x5627571d1310 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:41:28.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.775+0000 7f586e59c640 1 --2- 172.21.15.99:0/1965801498 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5627571c0100 0x5627571c9d90 secure :-1 s=READY pgs=362 cs=0 l=1 rev1=1 crypto rx=0x7f586000ec90 tx=0x7f5860004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=80f2f98a1bf00fb4 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:41:28.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.775+0000 7f5855ffb640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:41:28.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.775+0000 7f5855ffb640 1 -- 172.21.15.99:0/1965801498 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f586001ccb0 con 0x5627571c0100 2024-03-19T01:41:28.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.775+0000 7f5855ffb640 1 -- 172.21.15.99:0/1965801498 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f5860004020 con 0x5627571c0100 2024-03-19T01:41:28.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.775+0000 7f58773f8740 1 -- 172.21.15.99:0/1965801498 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f58680c6980 con 0x5627571c0100 2024-03-19T01:41:28.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.775+0000 7f5855ffb640 1 -- 172.21.15.99:0/1965801498 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=21) ==== 24+0+0 (secure 0 0 0) 0x7f586001ce50 con 0x5627571c0100 2024-03-19T01:41:28.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.775+0000 7f58773f8740 10 client.15360 fetch_fsmap learned FSMap version 21 2024-03-19T01:41:28.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.775+0000 7f58773f8740 1 -- 172.21.15.99:0/1965801498 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=21,monmap=4+}) -- 0x562756e39210 con 0x5627571c0100 2024-03-19T01:41:28.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.775+0000 7f5855ffb640 1 -- 172.21.15.99:0/1965801498 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f58680c6980 con 0x5627571c0100 2024-03-19T01:41:28.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.775+0000 7f5855ffb640 1 -- 172.21.15.99:0/1965801498 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 21) ==== 8338+0+0 (secure 0 0 0) 0x7f5860027430 con 0x5627571c0100 2024-03-19T01:41:28.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.775+0000 7f58773f8740 10 client.15360 fetch_fsmap finished waiting for FSMap version 21 2024-03-19T01:41:28.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.775+0000 7f58773f8740 10 client.15360 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:41:28.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.775+0000 7f58773f8740 20 client.15360 populate_metadata read hostname 'smithi099' 2024-03-19T01:41:28.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.775+0000 7f58773f8740 1 --2- 172.21.15.99:0/1965801498 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5627571d5c20 0x5627571d80c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:28.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.775+0000 7f58773f8740 4 client.15360 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:41:28.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.775+0000 7f58773f8740 1 -- 172.21.15.99:0/1965801498 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x562757132120 con 0x5627571d5c20 2024-03-19T01:41:28.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.776+0000 7f586dd9b640 1 --2- 172.21.15.99:0/1965801498 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5627571d5c20 0x5627571d80c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:28.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.776+0000 7f5855ffb640 10 client.15360 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:41:28.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.776+0000 7f586dd9b640 1 --2- 172.21.15.99:0/1965801498 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5627571d5c20 0x5627571d80c0 crc :-1 s=READY pgs=31 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=5701f3a3263e779c server_cookie=3ef81d85aad4dea0 in_seq=0 out_seq=0 2024-03-19T01:41:28.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.777+0000 7f5855ffb640 1 -- 172.21.15.99:0/1965801498 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x562757132120 con 0x5627571d5c20 2024-03-19T01:41:28.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.777+0000 7f5855ffb640 10 client.15360 handle_command_reply: tid=0 2024-03-19T01:41:28.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.777+0000 7f58773f8740 1 client.15360 shutdown 2024-03-19T01:41:28.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.777+0000 7f58773f8740 1 -- 172.21.15.99:0/1965801498 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5627571c0100 msgr2=0x5627571c9d90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:41:28.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.777+0000 7f58773f8740 1 --2- 172.21.15.99:0/1965801498 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5627571c0100 0x5627571c9d90 secure :-1 s=READY pgs=362 cs=0 l=1 rev1=1 crypto rx=0x7f586000ec90 tx=0x7f5860004730 comp rx=0 tx=0).stop 2024-03-19T01:41:28.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.777+0000 7f58773f8740 1 -- 172.21.15.99:0/1965801498 shutdown_connections 2024-03-19T01:41:28.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.777+0000 7f58773f8740 1 --2- 172.21.15.99:0/1965801498 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5627571d5c20 0x5627571d80c0 crc :-1 s=READY pgs=31 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:28.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.777+0000 7f58773f8740 1 --2- 172.21.15.99:0/1965801498 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5627571c5c60 0x5627571d1310 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:28.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.777+0000 7f5855ffb640 0 client.15360 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:41:28.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.777+0000 7f58773f8740 1 --2- 172.21.15.99:0/1965801498 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5627571c0af0 0x5627571ca2d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:28.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.777+0000 7f58773f8740 1 --2- 172.21.15.99:0/1965801498 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5627571c0100 0x5627571c9d90 unknown :-1 s=CLOSED pgs=362 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:28.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.777+0000 7f58773f8740 1 -- 172.21.15.99:0/1965801498 >> 172.21.15.99:0/1965801498 conn(0x5627571bb670 msgr2=0x5627571bc8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:41:28.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.777+0000 7f58773f8740 1 -- 172.21.15.99:0/1965801498 shutdown_connections 2024-03-19T01:41:28.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.777+0000 7f58773f8740 1 -- 172.21.15.99:0/1965801498 wait complete. 2024-03-19T01:41:28.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.777+0000 7f58773f8740 20 client.15360 trim_cache size 0 max 16384 2024-03-19T01:41:28.794 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.792+0000 7f58773f8740 1 Processor -- start 2024-03-19T01:41:28.794 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.792+0000 7f58773f8740 1 -- start start 2024-03-19T01:41:28.794 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.792+0000 7f58773f8740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5627571c0100 0x5627571ca230 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:28.794 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.792+0000 7f58773f8740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5627571c0af0 0x5627571c1ca0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:28.794 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.792+0000 7f58773f8740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5627571c5c60 0x5627571dc9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:28.794 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.792+0000 7f58773f8740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f58701ab650 con 0x5627571c0af0 2024-03-19T01:41:28.795 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.792+0000 7f58773f8740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f58701a8f20 con 0x5627571c5c60 2024-03-19T01:41:28.795 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.792+0000 7f58773f8740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f58701a6d70 con 0x5627571c0100 2024-03-19T01:41:28.795 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.792+0000 7f586dd9b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5627571c0af0 0x5627571c1ca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:28.795 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.792+0000 7f586e59c640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5627571c0100 0x5627571ca230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:28.795 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.792+0000 7f586ed9d640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5627571c5c60 0x5627571dc9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:28.795 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.792+0000 7f586e59c640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5627571c0100 0x5627571ca230 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:43944/0 (socket says 172.21.15.99:43944) 2024-03-19T01:41:28.795 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.792+0000 7f586dd9b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5627571c0af0 0x5627571c1ca0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:52670/0 (socket says 172.21.15.99:52670) 2024-03-19T01:41:28.795 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.792+0000 7f586e59c640 1 -- 172.21.15.99:0/1624696625 learned_addr learned my addr 172.21.15.99:0/1624696625 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:41:28.795 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.792+0000 7f586e59c640 1 -- 172.21.15.99:0/1624696625 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5627571c5c60 msgr2=0x5627571dc9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:41:28.795 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.792+0000 7f586e59c640 1 --2- 172.21.15.99:0/1624696625 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5627571c5c60 0x5627571dc9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:28.795 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.792+0000 7f586e59c640 1 -- 172.21.15.99:0/1624696625 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5627571c0af0 msgr2=0x5627571c1ca0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:41:28.795 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.792+0000 7f586e59c640 1 --2- 172.21.15.99:0/1624696625 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5627571c0af0 0x5627571c1ca0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:28.795 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.792+0000 7f586e59c640 1 -- 172.21.15.99:0/1624696625 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x562756f05620 con 0x5627571c0100 2024-03-19T01:41:28.796 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.792+0000 7f586ed9d640 1 --2- 172.21.15.99:0/1624696625 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5627571c5c60 0x5627571dc9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:41:28.796 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.793+0000 7f586e59c640 1 --2- 172.21.15.99:0/1624696625 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5627571c0100 0x5627571ca230 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7f5860016040 tx=0x7f5860004680 comp rx=0 tx=0).ready entity=mon.1 client_cookie=17f710535ac5ccdf server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:41:28.796 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.793+0000 7f5857fff640 1 -- 172.21.15.99:0/1624696625 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5860005b90 con 0x5627571c0100 2024-03-19T01:41:28.796 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.793+0000 7f5857fff640 1 -- 172.21.15.99:0/1624696625 <== mon.1 v2:172.21.15.115:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f586002d030 con 0x5627571c0100 2024-03-19T01:41:28.796 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.793+0000 7f5857fff640 1 -- 172.21.15.99:0/1624696625 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5860027630 con 0x5627571c0100 2024-03-19T01:41:28.799 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.794+0000 7f58773f8740 1 -- 172.21.15.99:0/1624696625 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5627571c0100 msgr2=0x5627571ca230 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:41:28.799 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.794+0000 7f58773f8740 1 --2- 172.21.15.99:0/1624696625 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5627571c0100 0x5627571ca230 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7f5860016040 tx=0x7f5860004680 comp rx=0 tx=0).stop 2024-03-19T01:41:28.799 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.794+0000 7f58773f8740 1 -- 172.21.15.99:0/1624696625 shutdown_connections 2024-03-19T01:41:28.799 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.794+0000 7f58773f8740 1 --2- 172.21.15.99:0/1624696625 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5627571c5c60 0x5627571dc9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:28.799 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.794+0000 7f58773f8740 1 --2- 172.21.15.99:0/1624696625 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5627571c0af0 0x5627571c1ca0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:28.799 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.794+0000 7f58773f8740 1 --2- 172.21.15.99:0/1624696625 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5627571c0100 0x5627571ca230 unknown :-1 s=CLOSED pgs=78 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:28.799 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.794+0000 7f58773f8740 1 -- 172.21.15.99:0/1624696625 >> 172.21.15.99:0/1624696625 conn(0x5627571bb670 msgr2=0x5627571c1590 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:41:28.799 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.794+0000 7f58773f8740 1 -- 172.21.15.99:0/1624696625 shutdown_connections 2024-03-19T01:41:28.799 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.794+0000 7f58773f8740 1 -- 172.21.15.99:0/1624696625 wait complete. 2024-03-19T01:41:28.799 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.794+0000 7f58773f8740 1 -- start start 2024-03-19T01:41:28.800 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.794+0000 7f58773f8740 1 Processor -- start 2024-03-19T01:41:28.800 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.794+0000 7f58773f8740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5627571c0100 0x562757237510 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:28.800 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.794+0000 7f58773f8740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5627571c0af0 0x562757237a50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:28.800 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.794+0000 7f58773f8740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5627571c5c60 0x56275723ea90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:28.800 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.794+0000 7f58773f8740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f58701a73e0 con 0x5627571c0100 2024-03-19T01:41:28.800 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.794+0000 7f58773f8740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f58701a8850 con 0x5627571c5c60 2024-03-19T01:41:28.800 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.794+0000 7f58773f8740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f58701a7ab0 con 0x5627571c0af0 2024-03-19T01:41:28.800 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.794+0000 7f58773f8740 10 client.0 _ll_register_callbacks cb 0x7f586000e940 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:41:28.800 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.794+0000 7f586e59c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5627571c0100 0x562757237510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:28.800 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.794+0000 7f586ed9d640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5627571c5c60 0x56275723ea90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:28.800 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.794+0000 7f586ed9d640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5627571c5c60 0x56275723ea90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:56578/0 (socket says 172.21.15.99:56578) 2024-03-19T01:41:28.800 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.794+0000 7f586e59c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5627571c0100 0x562757237510 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:52676/0 (socket says 172.21.15.99:52676) 2024-03-19T01:41:28.800 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.794+0000 7f586ed9d640 1 -- 172.21.15.99:0/100293402 learned_addr learned my addr 172.21.15.99:0/100293402 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:41:28.801 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.794+0000 7f586dd9b640 1 --2- 172.21.15.99:0/100293402 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5627571c0af0 0x562757237a50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:28.801 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.795+0000 7f586ed9d640 1 -- 172.21.15.99:0/100293402 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5627571c0af0 msgr2=0x562757237a50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:41:28.801 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.795+0000 7f586ed9d640 1 --2- 172.21.15.99:0/100293402 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5627571c0af0 0x562757237a50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:28.801 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.795+0000 7f586ed9d640 1 -- 172.21.15.99:0/100293402 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5627571c0100 msgr2=0x562757237510 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:41:28.801 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.795+0000 7f586ed9d640 1 --2- 172.21.15.99:0/100293402 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5627571c0100 0x562757237510 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:28.801 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.795+0000 7f586ed9d640 1 -- 172.21.15.99:0/100293402 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x56275723f080 con 0x5627571c5c60 2024-03-19T01:41:28.801 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.795+0000 7f586ed9d640 1 --2- 172.21.15.99:0/100293402 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5627571c5c60 0x56275723ea90 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7f58680b9d80 tx=0x7f58680dd400 comp rx=0 tx=0).ready entity=mon.2 client_cookie=c13dd22d7e6f605c server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:41:28.801 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.795+0000 7f5855ffb640 10 client.0 ms_handle_connect on v2:172.21.15.112:3300/0 2024-03-19T01:41:28.801 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.795+0000 7f5855ffb640 1 -- 172.21.15.99:0/100293402 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f58680bdab0 con 0x5627571c5c60 2024-03-19T01:41:28.801 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.795+0000 7f5855ffb640 1 -- 172.21.15.99:0/100293402 <== mon.2 v2:172.21.15.112:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f58680c1120 con 0x5627571c5c60 2024-03-19T01:41:28.801 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.795+0000 7f58773f8740 1 -- 172.21.15.99:0/100293402 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f586002d030 con 0x5627571c5c60 2024-03-19T01:41:28.801 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.795+0000 7f5855ffb640 1 -- 172.21.15.99:0/100293402 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_get_version_reply(handle=1 version=21) ==== 24+0+0 (secure 0 0 0) 0x7f58680bcd10 con 0x5627571c5c60 2024-03-19T01:41:28.801 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.795+0000 7f58773f8740 10 client.25139 fetch_fsmap learned FSMap version 21 2024-03-19T01:41:28.801 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.796+0000 7f58773f8740 1 -- 172.21.15.99:0/100293402 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,fsmap=21,monmap=4+}) -- 0x562757240400 con 0x5627571c5c60 2024-03-19T01:41:28.802 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.796+0000 7f5855ffb640 1 -- 172.21.15.99:0/100293402 <== mon.2 v2:172.21.15.112:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f586002d030 con 0x5627571c5c60 2024-03-19T01:41:28.802 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.796+0000 7f5855ffb640 1 -- 172.21.15.99:0/100293402 <== mon.2 v2:172.21.15.112:3300/0 5 ==== fsmap(e 21) ==== 8338+0+0 (secure 0 0 0) 0x7f586803c030 con 0x5627571c5c60 2024-03-19T01:41:28.802 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.796+0000 7f58773f8740 10 client.25139 fetch_fsmap finished waiting for FSMap version 21 2024-03-19T01:41:28.802 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.796+0000 7f58773f8740 10 client.25139 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:41:28.802 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.796+0000 7f58773f8740 20 client.25139 populate_metadata read hostname 'smithi099' 2024-03-19T01:41:28.802 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.796+0000 7f58773f8740 1 --2- 172.21.15.99:0/100293402 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x562757240b60 0x562757242f80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:28.802 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.796+0000 7f58773f8740 4 client.25139 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:41:28.802 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.796+0000 7f58773f8740 1 -- 172.21.15.99:0/100293402 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x56275712c470 con 0x562757240b60 2024-03-19T01:41:28.802 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.796+0000 7f586e59c640 1 --2- 172.21.15.99:0/100293402 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x562757240b60 0x562757242f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:28.802 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.797+0000 7f5855ffb640 10 client.25139 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:41:28.802 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.797+0000 7f586e59c640 1 --2- 172.21.15.99:0/100293402 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x562757240b60 0x562757242f80 crc :-1 s=READY pgs=32 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=34239dbbcc038520 server_cookie=bf3d2d5115e29f6a in_seq=0 out_seq=0 2024-03-19T01:41:28.805 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.803+0000 7f5855ffb640 1 -- 172.21.15.99:0/100293402 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x56275712c470 con 0x562757240b60 2024-03-19T01:41:28.805 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.803+0000 7f5855ffb640 10 client.25139 handle_command_reply: tid=0 2024-03-19T01:41:28.805 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.803+0000 7f58773f8740 1 client.25139 shutdown 2024-03-19T01:41:28.805 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.803+0000 7f58773f8740 1 -- 172.21.15.99:0/100293402 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5627571c5c60 msgr2=0x56275723ea90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:41:28.805 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.803+0000 7f58773f8740 1 --2- 172.21.15.99:0/100293402 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5627571c5c60 0x56275723ea90 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7f58680b9d80 tx=0x7f58680dd400 comp rx=0 tx=0).stop 2024-03-19T01:41:28.805 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.803+0000 7f58773f8740 1 -- 172.21.15.99:0/100293402 shutdown_connections 2024-03-19T01:41:28.805 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.803+0000 7f58773f8740 1 --2- 172.21.15.99:0/100293402 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x562757240b60 0x562757242f80 crc :-1 s=READY pgs=32 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:28.805 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.803+0000 7f58773f8740 1 --2- 172.21.15.99:0/100293402 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5627571c5c60 0x56275723ea90 unknown :-1 s=CLOSED pgs=87 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:28.805 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.803+0000 7f58773f8740 1 --2- 172.21.15.99:0/100293402 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5627571c0af0 0x562757237a50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:28.806 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.803+0000 7f5855ffb640 0 client.25139 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:41:28.806 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.803+0000 7f58773f8740 1 --2- 172.21.15.99:0/100293402 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5627571c0100 0x562757237510 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:28.806 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.803+0000 7f58773f8740 1 -- 172.21.15.99:0/100293402 >> 172.21.15.99:0/100293402 conn(0x5627571bb670 msgr2=0x5627571c1590 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:41:28.806 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.804+0000 7f58773f8740 1 -- 172.21.15.99:0/100293402 shutdown_connections 2024-03-19T01:41:28.806 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.804+0000 7f58773f8740 1 -- 172.21.15.99:0/100293402 wait complete. 2024-03-19T01:41:28.806 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.804+0000 7f58773f8740 20 client.25139 trim_cache size 0 max 16384 2024-03-19T01:41:28.806 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:41:28.806 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:41:28.806 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:41:28.807 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:41:28.807 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.804+0000 7f587480b640 1 -- 172.21.15.99:0/1720526961 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f58580a2940 msgr2=0x7f58580a4e00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:41:28.807 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.804+0000 7f587480b640 1 --2- 172.21.15.99:0/1720526961 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f58580a2940 0x7f58580a4e00 secure :-1 s=READY pgs=249 cs=0 l=1 rev1=1 crypto rx=0x7f5870101790 tx=0x7f5860005f90 comp rx=0 tx=0).stop 2024-03-19T01:41:28.807 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.804+0000 7f587480b640 1 -- 172.21.15.99:0/1720526961 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f58701022f0 msgr2=0x7f587019eff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:41:28.808 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.804+0000 7f587480b640 1 --2- 172.21.15.99:0/1720526961 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f58701022f0 0x7f587019eff0 secure :-1 s=READY pgs=361 cs=0 l=1 rev1=1 crypto rx=0x7f5868037040 tx=0x7f5868032f40 comp rx=0 tx=0).stop 2024-03-19T01:41:28.808 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.804+0000 7f587480b640 1 -- 172.21.15.99:0/1720526961 shutdown_connections 2024-03-19T01:41:28.808 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.804+0000 7f587480b640 1 --2- 172.21.15.99:0/1720526961 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f58701022f0 0x7f587019eff0 unknown :-1 s=CLOSED pgs=361 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:28.808 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.804+0000 7f587480b640 1 --2- 172.21.15.99:0/1720526961 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5870101930 0x7f5870197f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:28.808 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.804+0000 7f587480b640 1 --2- 172.21.15.99:0/1720526961 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f58580a2940 0x7f58580a4e00 unknown :-1 s=CLOSED pgs=249 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:28.808 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.804+0000 7f587480b640 1 --2- 172.21.15.99:0/1720526961 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5870100740 0x7f5870197a40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:28.808 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.804+0000 7f587480b640 1 -- 172.21.15.99:0/1720526961 >> 172.21.15.99:0/1720526961 conn(0x7f58700fbcb0 msgr2=0x7f5870104bd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:41:28.808 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.804+0000 7f587480b640 1 -- 172.21.15.99:0/1720526961 shutdown_connections 2024-03-19T01:41:28.808 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:28.804+0000 7f587480b640 1 -- 172.21.15.99:0/1720526961 wait complete. 2024-03-19T01:41:29.438 INFO:tasks.workunit.client.0.smithi099.stderr:Note: switching to 'e48b1f6c0608b2615e1890cc46286c3877d6e910'. 2024-03-19T01:41:29.438 INFO:tasks.workunit.client.0.smithi099.stderr: 2024-03-19T01:41:29.438 INFO:tasks.workunit.client.0.smithi099.stderr:You are in 'detached HEAD' state. You can look around, make experimental 2024-03-19T01:41:29.438 INFO:tasks.workunit.client.0.smithi099.stderr:changes and commit them, and you can discard any commits you make in this 2024-03-19T01:41:29.438 INFO:tasks.workunit.client.0.smithi099.stderr:state without impacting any branches by switching back to a branch. 2024-03-19T01:41:29.438 INFO:tasks.workunit.client.0.smithi099.stderr: 2024-03-19T01:41:29.438 INFO:tasks.workunit.client.0.smithi099.stderr:If you want to create a new branch to retain commits you create, you may 2024-03-19T01:41:29.438 INFO:tasks.workunit.client.0.smithi099.stderr:do so (now or later) by using -c with the switch command. Example: 2024-03-19T01:41:29.439 INFO:tasks.workunit.client.0.smithi099.stderr: 2024-03-19T01:41:29.439 INFO:tasks.workunit.client.0.smithi099.stderr: git switch -c 2024-03-19T01:41:29.439 INFO:tasks.workunit.client.0.smithi099.stderr: 2024-03-19T01:41:29.439 INFO:tasks.workunit.client.0.smithi099.stderr:Or undo this operation with: 2024-03-19T01:41:29.439 INFO:tasks.workunit.client.0.smithi099.stderr: 2024-03-19T01:41:29.439 INFO:tasks.workunit.client.0.smithi099.stderr: git switch - 2024-03-19T01:41:29.439 INFO:tasks.workunit.client.0.smithi099.stderr: 2024-03-19T01:41:29.439 INFO:tasks.workunit.client.0.smithi099.stderr:Turn off this advice by setting config variable advice.detachedHead to false 2024-03-19T01:41:29.439 INFO:tasks.workunit.client.0.smithi099.stderr: 2024-03-19T01:41:29.439 INFO:tasks.workunit.client.0.smithi099.stderr:HEAD is now at e48b1f6c060 testfix 2024-03-19T01:41:29.443 DEBUG:teuthology.orchestra.run.smithi099:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2024-03-19T01:41:29.508 INFO:tasks.workunit.client.0.smithi099.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-03-19T01:41:29.510 INFO:tasks.workunit.client.0.smithi099.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-03-19T01:41:29.510 INFO:tasks.workunit.client.0.smithi099.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-03-19T01:41:30.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:41:29 smithi099 ceph-mon[62951]: pgmap v362: 97 pgs: 97 active+clean; 670 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.8 KiB/s rd, 1.3 KiB/s wr, 7 op/s 2024-03-19T01:41:30.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:41:29 smithi112 ceph-mon[66819]: pgmap v362: 97 pgs: 97 active+clean; 670 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.8 KiB/s rd, 1.3 KiB/s wr, 7 op/s 2024-03-19T01:41:30.277 INFO:tasks.workunit.client.0.smithi099.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-03-19T01:41:30.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:41:29 smithi115 ceph-mon[67332]: pgmap v362: 97 pgs: 97 active+clean; 670 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 3.8 KiB/s rd, 1.3 KiB/s wr, 7 op/s 2024-03-19T01:41:30.350 INFO:tasks.workunit.client.0.smithi099.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-03-19T01:41:30.391 INFO:tasks.workunit.client.0.smithi099.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-03-19T01:41:30.393 INFO:tasks.workunit.client.0.smithi099.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-03-19T01:41:30.393 INFO:tasks.workunit.client.0.smithi099.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-03-19T01:41:30.432 INFO:tasks.workunit.client.0.smithi099.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-03-19T01:41:30.435 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2024-03-19T01:41:30.435 DEBUG:teuthology.orchestra.run.smithi099:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2024-03-19T01:41:30.491 INFO:tasks.workunit:Running workunits matching suites/dbench.sh on client.0... 2024-03-19T01:41:30.492 INFO:tasks.workunit:Running workunit suites/dbench.sh... 2024-03-19T01:41:30.493 DEBUG:teuthology.orchestra.run.smithi099:workunit test suites/dbench.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=e48b1f6c0608b2615e1890cc46286c3877d6e910 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/dbench.sh 2024-03-19T01:41:30.552 INFO:tasks.workunit.client.0.smithi099.stderr:+ dbench 1 2024-03-19T01:41:30.568 INFO:tasks.workunit.client.0.smithi099.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-03-19T01:41:30.568 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-03-19T01:41:30.568 INFO:tasks.workunit.client.0.smithi099.stdout:Running for 600 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 120 secs 2024-03-19T01:41:30.572 INFO:tasks.workunit.client.0.smithi099.stdout:0 of 1 processes prepared for launch 0 sec 2024-03-19T01:41:30.672 INFO:tasks.workunit.client.0.smithi099.stdout:1 of 1 processes prepared for launch 0 sec 2024-03-19T01:41:30.673 INFO:tasks.workunit.client.0.smithi099.stdout:releasing clients 2024-03-19T01:41:31.673 INFO:tasks.workunit.client.0.smithi099.stdout: 1 2917 33.47 MB/sec warmup 1 sec latency 10.246 ms 2024-03-19T01:41:32.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:41:31 smithi099 ceph-mon[62951]: pgmap v363: 97 pgs: 97 active+clean; 671 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 4.3 KiB/s rd, 1.7 KiB/s wr, 7 op/s 2024-03-19T01:41:32.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:41:31 smithi112 ceph-mon[66819]: pgmap v363: 97 pgs: 97 active+clean; 671 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 4.3 KiB/s rd, 1.7 KiB/s wr, 7 op/s 2024-03-19T01:41:32.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:41:31 smithi115 ceph-mon[67332]: pgmap v363: 97 pgs: 97 active+clean; 671 KiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 4.3 KiB/s rd, 1.7 KiB/s wr, 7 op/s 2024-03-19T01:41:32.672 INFO:tasks.workunit.client.0.smithi099.stdout: 1 4855 21.19 MB/sec warmup 2 sec latency 34.834 ms 2024-03-19T01:41:33.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:41:32 smithi099 ceph-mon[62951]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T01:41:33.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:41:32 smithi112 ceph-mon[66819]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T01:41:33.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:41:32 smithi115 ceph-mon[67332]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T01:41:33.673 INFO:tasks.workunit.client.0.smithi099.stdout: 1 7572 19.73 MB/sec warmup 3 sec latency 13.277 ms 2024-03-19T01:41:34.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:41:33 smithi099 ceph-mon[62951]: pgmap v364: 97 pgs: 97 active+clean; 1.2 MiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 4.3 KiB/s rd, 50 KiB/s wr, 13 op/s 2024-03-19T01:41:34.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:41:33 smithi112 ceph-mon[66819]: pgmap v364: 97 pgs: 97 active+clean; 1.2 MiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 4.3 KiB/s rd, 50 KiB/s wr, 13 op/s 2024-03-19T01:41:34.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:41:33 smithi115 ceph-mon[67332]: pgmap v364: 97 pgs: 97 active+clean; 1.2 MiB data, 367 MiB used, 1.0 TiB / 1.0 TiB avail; 4.3 KiB/s rd, 50 KiB/s wr, 13 op/s 2024-03-19T01:41:34.673 INFO:tasks.workunit.client.0.smithi099.stdout: 1 10450 18.41 MB/sec warmup 4 sec latency 25.437 ms 2024-03-19T01:41:35.673 INFO:tasks.workunit.client.0.smithi099.stdout: 1 12461 16.74 MB/sec warmup 5 sec latency 26.045 ms 2024-03-19T01:41:36.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:41:35 smithi112 ceph-mon[66819]: pgmap v365: 97 pgs: 97 active+clean; 1.5 MiB data, 368 MiB used, 1.0 TiB / 1.0 TiB avail; 3.9 KiB/s rd, 88 KiB/s wr, 14 op/s 2024-03-19T01:41:36.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:41:35 smithi115 ceph-mon[67332]: pgmap v365: 97 pgs: 97 active+clean; 1.5 MiB data, 368 MiB used, 1.0 TiB / 1.0 TiB avail; 3.9 KiB/s rd, 88 KiB/s wr, 14 op/s 2024-03-19T01:41:36.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:41:35 smithi099 ceph-mon[62951]: pgmap v365: 97 pgs: 97 active+clean; 1.5 MiB data, 368 MiB used, 1.0 TiB / 1.0 TiB avail; 3.9 KiB/s rd, 88 KiB/s wr, 14 op/s 2024-03-19T01:41:36.673 INFO:tasks.workunit.client.0.smithi099.stdout: 1 14788 16.24 MB/sec warmup 6 sec latency 30.220 ms 2024-03-19T01:41:37.673 INFO:tasks.workunit.client.0.smithi099.stdout: 1 17704 16.41 MB/sec warmup 7 sec latency 38.151 ms 2024-03-19T01:41:38.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:41:37 smithi112 ceph-mon[66819]: pgmap v366: 97 pgs: 97 active+clean; 4.6 MiB data, 389 MiB used, 1.0 TiB / 1.0 TiB avail; 91 KiB/s rd, 400 KiB/s wr, 49 op/s 2024-03-19T01:41:38.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:41:37 smithi112 ceph-mon[66819]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T01:41:38.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:41:37 smithi115 ceph-mon[67332]: pgmap v366: 97 pgs: 97 active+clean; 4.6 MiB data, 389 MiB used, 1.0 TiB / 1.0 TiB avail; 91 KiB/s rd, 400 KiB/s wr, 49 op/s 2024-03-19T01:41:38.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:41:37 smithi115 ceph-mon[67332]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T01:41:38.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:41:37 smithi099 ceph-mon[62951]: pgmap v366: 97 pgs: 97 active+clean; 4.6 MiB data, 389 MiB used, 1.0 TiB / 1.0 TiB avail; 91 KiB/s rd, 400 KiB/s wr, 49 op/s 2024-03-19T01:41:38.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:41:37 smithi099 ceph-mon[62951]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T01:41:38.673 INFO:tasks.workunit.client.0.smithi099.stdout: 1 19449 15.23 MB/sec warmup 8 sec latency 30.284 ms 2024-03-19T01:41:39.673 INFO:tasks.workunit.client.0.smithi099.stdout: 1 22010 15.10 MB/sec warmup 9 sec latency 29.998 ms 2024-03-19T01:41:40.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:41:39 smithi112 ceph-mon[66819]: pgmap v367: 97 pgs: 97 active+clean; 8.2 MiB data, 399 MiB used, 1.0 TiB / 1.0 TiB avail; 340 KiB/s rd, 758 KiB/s wr, 107 op/s 2024-03-19T01:41:40.142 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:41:39 smithi099 ceph-mon[62951]: pgmap v367: 97 pgs: 97 active+clean; 8.2 MiB data, 399 MiB used, 1.0 TiB / 1.0 TiB avail; 340 KiB/s rd, 758 KiB/s wr, 107 op/s 2024-03-19T01:41:40.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:41:39 smithi115 ceph-mon[67332]: pgmap v367: 97 pgs: 97 active+clean; 8.2 MiB data, 399 MiB used, 1.0 TiB / 1.0 TiB avail; 340 KiB/s rd, 758 KiB/s wr, 107 op/s 2024-03-19T01:41:40.673 INFO:tasks.workunit.client.0.smithi099.stdout: 1 24955 15.34 MB/sec warmup 10 sec latency 16.729 ms 2024-03-19T01:41:41.673 INFO:tasks.workunit.client.0.smithi099.stdout: 1 26790 14.58 MB/sec warmup 11 sec latency 39.874 ms 2024-03-19T01:41:41.900 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.898+0000 7f768d33b640 1 -- 172.21.15.99:0/1395452962 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f76881022c0 msgr2=0x7f7688108b50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:41:41.901 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.898+0000 7f768d33b640 1 --2- 172.21.15.99:0/1395452962 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f76881022c0 0x7f7688108b50 secure :-1 s=READY pgs=363 cs=0 l=1 rev1=1 crypto rx=0x7f7680009ca0 tx=0x7f768002c0f0 comp rx=0 tx=0).stop 2024-03-19T01:41:41.901 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.898+0000 7f768d33b640 1 -- 172.21.15.99:0/1395452962 shutdown_connections 2024-03-19T01:41:41.901 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.898+0000 7f768d33b640 1 --2- 172.21.15.99:0/1395452962 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f7688101900 0x7f7688101d80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:41.901 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.898+0000 7f768d33b640 1 --2- 172.21.15.99:0/1395452962 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f76881022c0 0x7f7688108b50 unknown :-1 s=CLOSED pgs=363 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:41.901 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.898+0000 7f768d33b640 1 --2- 172.21.15.99:0/1395452962 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f7688100710 0x7f7688100b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:41.901 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.898+0000 7f768d33b640 1 -- 172.21.15.99:0/1395452962 >> 172.21.15.99:0/1395452962 conn(0x7f76880fbc80 msgr2=0x7f76880fe0c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:41:41.901 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.898+0000 7f768d33b640 1 -- 172.21.15.99:0/1395452962 shutdown_connections 2024-03-19T01:41:41.901 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.898+0000 7f768d33b640 1 -- 172.21.15.99:0/1395452962 wait complete. 2024-03-19T01:41:41.901 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.899+0000 7f768d33b640 1 Processor -- start 2024-03-19T01:41:41.901 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.899+0000 7f768d33b640 1 -- start start 2024-03-19T01:41:41.901 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.899+0000 7f768d33b640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f7688100710 0x7f76881979e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:41.901 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.899+0000 7f768d33b640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f7688101900 0x7f7688197f20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:41.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.899+0000 7f768d33b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f76881022c0 0x7f768819ef90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:41.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.899+0000 7f768d33b640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f768810c390 con 0x7f76881022c0 2024-03-19T01:41:41.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.899+0000 7f768d33b640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f768810c210 con 0x7f7688100710 2024-03-19T01:41:41.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.899+0000 7f768d33b640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f768810c510 con 0x7f7688101900 2024-03-19T01:41:41.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.899+0000 7f76877fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f76881022c0 0x7f768819ef90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:41.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.899+0000 7f7686ffd640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f7688100710 0x7f76881979e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:41.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.899+0000 7f7686ffd640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f7688100710 0x7f76881979e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:34608/0 (socket says 172.21.15.99:34608) 2024-03-19T01:41:41.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.899+0000 7f767ffff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f7688101900 0x7f7688197f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:41.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.899+0000 7f7686ffd640 1 -- 172.21.15.99:0/2586468259 learned_addr learned my addr 172.21.15.99:0/2586468259 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:41:41.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.900+0000 7f76877fe640 1 -- 172.21.15.99:0/2586468259 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f7688101900 msgr2=0x7f7688197f20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:41:41.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.900+0000 7f76877fe640 1 --2- 172.21.15.99:0/2586468259 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f7688101900 0x7f7688197f20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:41.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.900+0000 7f76877fe640 1 -- 172.21.15.99:0/2586468259 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f7688100710 msgr2=0x7f76881979e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:41:41.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.900+0000 7f76877fe640 1 --2- 172.21.15.99:0/2586468259 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f7688100710 0x7f76881979e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:41.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.900+0000 7f76877fe640 1 -- 172.21.15.99:0/2586468259 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f768819f600 con 0x7f76881022c0 2024-03-19T01:41:41.903 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.900+0000 7f76877fe640 1 --2- 172.21.15.99:0/2586468259 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f76881022c0 0x7f768819ef90 secure :-1 s=READY pgs=364 cs=0 l=1 rev1=1 crypto rx=0x7f768002c630 tx=0x7f7680032f20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:41:41.903 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.900+0000 7f7684ff9640 1 -- 172.21.15.99:0/2586468259 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7680041020 con 0x7f76881022c0 2024-03-19T01:41:41.903 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.900+0000 7f768d33b640 1 -- 172.21.15.99:0/2586468259 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f768819f890 con 0x7f76881022c0 2024-03-19T01:41:41.903 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.900+0000 7f7684ff9640 1 -- 172.21.15.99:0/2586468259 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7680034cd0 con 0x7f76881022c0 2024-03-19T01:41:41.903 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.900+0000 7f7684ff9640 1 -- 172.21.15.99:0/2586468259 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7680039040 con 0x7f76881022c0 2024-03-19T01:41:41.903 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.900+0000 7f768d33b640 1 -- 172.21.15.99:0/2586468259 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f768819fd50 con 0x7f76881022c0 2024-03-19T01:41:41.909 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.902+0000 7f7684ff9640 1 -- 172.21.15.99:0/2586468259 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 19) ==== 137341+0+0 (secure 0 0 0) 0x7f7680044020 con 0x7f76881022c0 2024-03-19T01:41:41.909 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.902+0000 7f7690000740 1 Processor -- start 2024-03-19T01:41:41.909 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.902+0000 7f7690000740 1 -- start start 2024-03-19T01:41:41.909 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.903+0000 7f7690000740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557e26da5100 0x557e26da5520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:41.909 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.903+0000 7f7690000740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557e26da5af0 0x557e26daa5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:41.909 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.903+0000 7f7690000740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557e26daac60 0x557e26dad0f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:41.910 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.903+0000 7f7690000740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x557e26d09be0 con 0x557e26daac60 2024-03-19T01:41:41.910 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.903+0000 7f7690000740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x557e26cb2960 con 0x557e26da5af0 2024-03-19T01:41:41.910 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.903+0000 7f7690000740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x557e26d05c80 con 0x557e26da5100 2024-03-19T01:41:41.910 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.903+0000 7f7684ff9640 1 --2- 172.21.15.99:0/2586468259 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f76740a2940 0x7f76740a4e00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:41.910 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.903+0000 7f767ffff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557e26daac60 0x557e26dad0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:41.910 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.903+0000 7f7686ffd640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557e26da5af0 0x557e26daa5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:41.910 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.903+0000 7f7686ffd640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557e26da5af0 0x557e26daa5f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:34616/0 (socket says 172.21.15.99:34616) 2024-03-19T01:41:41.910 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.903+0000 7f767ffff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557e26daac60 0x557e26dad0f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:33894/0 (socket says 172.21.15.99:33894) 2024-03-19T01:41:41.910 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.903+0000 7f7686ffd640 1 -- 172.21.15.99:0/1474042605 learned_addr learned my addr 172.21.15.99:0/1474042605 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:41:41.910 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.903+0000 7f767ffff640 1 --2- 172.21.15.99:0/1474042605 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557e26da5100 0x557e26da5520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:41.910 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.903+0000 7f76877fe640 1 --2- 172.21.15.99:0/2586468259 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f76740a2940 0x7f76740a4e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:41.911 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.903+0000 7f7684ff9640 1 -- 172.21.15.99:0/2586468259 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f76800f0910 con 0x7f76881022c0 2024-03-19T01:41:41.911 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.903+0000 7f7686ffd640 1 -- 172.21.15.99:0/1474042605 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557e26da5100 msgr2=0x557e26da5520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:41:41.911 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.903+0000 7f7686ffd640 1 --2- 172.21.15.99:0/1474042605 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557e26da5100 0x557e26da5520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:41.911 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.903+0000 7f7686ffd640 1 -- 172.21.15.99:0/1474042605 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557e26daac60 msgr2=0x557e26dad0f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:41:41.911 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.903+0000 7f7686ffd640 1 --2- 172.21.15.99:0/1474042605 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557e26daac60 0x557e26dad0f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:41.911 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.903+0000 7f7686ffd640 1 -- 172.21.15.99:0/1474042605 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x557e26ca9d20 con 0x557e26da5af0 2024-03-19T01:41:41.911 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.904+0000 7f7686ffd640 1 --2- 172.21.15.99:0/1474042605 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557e26da5af0 0x557e26daa5f0 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 crypto rx=0x7f7678009d70 tx=0x7f7678011810 comp rx=0 tx=0).ready entity=mon.2 client_cookie=65eff404413c45f6 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:41:41.911 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.904+0000 7f76877fe640 1 --2- 172.21.15.99:0/2586468259 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f76740a2940 0x7f76740a4e00 secure :-1 s=READY pgs=250 cs=0 l=1 rev1=1 crypto rx=0x7f768004c000 tx=0x7f768008ef30 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:41:41.911 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.904+0000 7f767d7fa640 1 -- 172.21.15.99:0/1474042605 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7678009e60 con 0x557e26da5af0 2024-03-19T01:41:41.911 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.904+0000 7f767d7fa640 1 -- 172.21.15.99:0/1474042605 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f76780035f0 con 0x557e26da5af0 2024-03-19T01:41:41.911 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.904+0000 7f767d7fa640 1 -- 172.21.15.99:0/1474042605 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7678019670 con 0x557e26da5af0 2024-03-19T01:41:41.911 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.904+0000 7f7690000740 1 -- 172.21.15.99:0/1474042605 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557e26da5af0 msgr2=0x557e26daa5f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:41:41.911 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.904+0000 7f7690000740 1 --2- 172.21.15.99:0/1474042605 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557e26da5af0 0x557e26daa5f0 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 crypto rx=0x7f7678009d70 tx=0x7f7678011810 comp rx=0 tx=0).stop 2024-03-19T01:41:41.911 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.905+0000 7f7690000740 1 -- 172.21.15.99:0/1474042605 shutdown_connections 2024-03-19T01:41:41.912 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.905+0000 7f7690000740 1 --2- 172.21.15.99:0/1474042605 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557e26da5af0 0x557e26daa5f0 unknown :-1 s=CLOSED pgs=88 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:41.912 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.905+0000 7f7690000740 1 --2- 172.21.15.99:0/1474042605 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557e26daac60 0x557e26dad0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:41.912 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.905+0000 7f7690000740 1 --2- 172.21.15.99:0/1474042605 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557e26da5100 0x557e26da5520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:41.912 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.905+0000 7f7690000740 1 -- 172.21.15.99:0/1474042605 >> 172.21.15.99:0/1474042605 conn(0x557e26da0670 msgr2=0x557e26da0a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:41:41.912 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.905+0000 7f7690000740 1 -- 172.21.15.99:0/1474042605 shutdown_connections 2024-03-19T01:41:41.912 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.905+0000 7f7690000740 1 -- 172.21.15.99:0/1474042605 wait complete. 2024-03-19T01:41:41.912 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.905+0000 7f7690000740 1 -- start start 2024-03-19T01:41:41.912 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.905+0000 7f7690000740 1 Processor -- start 2024-03-19T01:41:41.912 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.905+0000 7f7690000740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557e26da5100 0x557e26daf010 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:41.912 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.905+0000 7f7690000740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557e26da5af0 0x557e26daf550 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:41.912 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.905+0000 7f7690000740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557e26daac60 0x557e26db6590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:41.912 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.905+0000 7f7690000740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x557e26dadc10 con 0x557e26da5100 2024-03-19T01:41:41.913 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.905+0000 7f7690000740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x557e26cc5db0 con 0x557e26daac60 2024-03-19T01:41:41.913 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.905+0000 7f7690000740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x557e26dadd90 con 0x557e26da5af0 2024-03-19T01:41:41.913 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.905+0000 7f7690000740 10 client.0 _ll_register_callbacks cb 0x557e26ccae70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:41:41.913 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.905+0000 7f767ffff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557e26da5100 0x557e26daf010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:41.913 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.905+0000 7f767ffff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557e26da5100 0x557e26daf010 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:33902/0 (socket says 172.21.15.99:33902) 2024-03-19T01:41:41.913 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.905+0000 7f7686ffd640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557e26da5af0 0x557e26daf550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:41.913 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.905+0000 7f767ffff640 1 -- 172.21.15.99:0/75046082 learned_addr learned my addr 172.21.15.99:0/75046082 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:41:41.913 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.905+0000 7f767ffff640 1 --2- 172.21.15.99:0/75046082 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557e26daac60 0x557e26db6590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:41.913 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.905+0000 7f767ffff640 1 -- 172.21.15.99:0/75046082 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557e26da5af0 msgr2=0x557e26daf550 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:41:41.913 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.905+0000 7f767ffff640 1 --2- 172.21.15.99:0/75046082 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557e26da5af0 0x557e26daf550 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:41.913 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.905+0000 7f767ffff640 1 -- 172.21.15.99:0/75046082 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557e26daac60 msgr2=0x557e26db6590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:41:41.913 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.905+0000 7f767ffff640 1 --2- 172.21.15.99:0/75046082 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557e26daac60 0x557e26db6590 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:41.913 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.905+0000 7f767ffff640 1 -- 172.21.15.99:0/75046082 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x557e26cc4c60 con 0x557e26da5100 2024-03-19T01:41:41.913 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.906+0000 7f767ffff640 1 --2- 172.21.15.99:0/75046082 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557e26da5100 0x557e26daf010 secure :-1 s=READY pgs=365 cs=0 l=1 rev1=1 crypto rx=0x7f767001ea70 tx=0x7f7670004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d0bf1df6a397335d server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:41:41.914 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.906+0000 7f766e7fc640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:41:41.914 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.906+0000 7f766e7fc640 1 -- 172.21.15.99:0/75046082 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7670003dc0 con 0x557e26da5100 2024-03-19T01:41:41.914 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.906+0000 7f766e7fc640 1 -- 172.21.15.99:0/75046082 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f76700047a0 con 0x557e26da5100 2024-03-19T01:41:41.914 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.906+0000 7f7690000740 1 -- 172.21.15.99:0/75046082 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f76780035f0 con 0x557e26da5100 2024-03-19T01:41:41.914 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.906+0000 7f766e7fc640 1 -- 172.21.15.99:0/75046082 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=23) ==== 24+0+0 (secure 0 0 0) 0x7f767002f050 con 0x557e26da5100 2024-03-19T01:41:41.914 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.906+0000 7f7690000740 10 client.15378 fetch_fsmap learned FSMap version 23 2024-03-19T01:41:41.914 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.906+0000 7f7690000740 1 -- 172.21.15.99:0/75046082 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=23,monmap=4+}) -- 0x557e26a1e210 con 0x557e26da5100 2024-03-19T01:41:41.914 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.906+0000 7f766e7fc640 1 -- 172.21.15.99:0/75046082 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f76780035f0 con 0x557e26da5100 2024-03-19T01:41:41.914 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.906+0000 7f766e7fc640 1 -- 172.21.15.99:0/75046082 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 23) ==== 8346+0+0 (secure 0 0 0) 0x7f767000e620 con 0x557e26da5100 2024-03-19T01:41:41.914 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.906+0000 7f7690000740 10 client.15378 fetch_fsmap finished waiting for FSMap version 23 2024-03-19T01:41:41.914 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.906+0000 7f7690000740 10 client.15378 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:41:41.914 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.906+0000 7f7690000740 20 client.15378 populate_metadata read hostname 'smithi099' 2024-03-19T01:41:41.914 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.906+0000 7f7690000740 1 --2- 172.21.15.99:0/75046082 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x557e26dbae90 0x557e26dbd300 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:41.914 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.906+0000 7f7690000740 4 client.15378 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:41:41.915 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.906+0000 7f7690000740 1 -- 172.21.15.99:0/75046082 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x557e26d17120 con 0x557e26dbae90 2024-03-19T01:41:41.915 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.907+0000 7f7686ffd640 1 --2- 172.21.15.99:0/75046082 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x557e26dbae90 0x557e26dbd300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:41.915 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.907+0000 7f766e7fc640 10 client.15378 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:41:41.915 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.907+0000 7f7686ffd640 1 --2- 172.21.15.99:0/75046082 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x557e26dbae90 0x557e26dbd300 crc :-1 s=READY pgs=33 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=fdbd20e92086d668 server_cookie=21b96b9721a8048a in_seq=0 out_seq=0 2024-03-19T01:41:41.915 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.908+0000 7f766e7fc640 1 -- 172.21.15.99:0/75046082 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x557e26d17120 con 0x557e26dbae90 2024-03-19T01:41:41.915 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.908+0000 7f766e7fc640 10 client.15378 handle_command_reply: tid=0 2024-03-19T01:41:41.915 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.908+0000 7f7690000740 1 client.15378 shutdown 2024-03-19T01:41:41.915 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.908+0000 7f7690000740 1 -- 172.21.15.99:0/75046082 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557e26da5100 msgr2=0x557e26daf010 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:41:41.915 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.908+0000 7f7690000740 1 --2- 172.21.15.99:0/75046082 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557e26da5100 0x557e26daf010 secure :-1 s=READY pgs=365 cs=0 l=1 rev1=1 crypto rx=0x7f767001ea70 tx=0x7f7670004730 comp rx=0 tx=0).stop 2024-03-19T01:41:41.915 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.908+0000 7f7690000740 1 -- 172.21.15.99:0/75046082 shutdown_connections 2024-03-19T01:41:41.915 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.908+0000 7f7690000740 1 --2- 172.21.15.99:0/75046082 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x557e26dbae90 0x557e26dbd300 crc :-1 s=READY pgs=33 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:41.915 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.908+0000 7f7690000740 1 --2- 172.21.15.99:0/75046082 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557e26daac60 0x557e26db6590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:41.915 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.908+0000 7f766e7fc640 0 client.15378 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:41:41.916 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.908+0000 7f7690000740 1 --2- 172.21.15.99:0/75046082 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557e26da5af0 0x557e26daf550 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:41.916 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.908+0000 7f7690000740 1 --2- 172.21.15.99:0/75046082 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557e26da5100 0x557e26daf010 unknown :-1 s=CLOSED pgs=365 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:41.916 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.908+0000 7f7690000740 1 -- 172.21.15.99:0/75046082 >> 172.21.15.99:0/75046082 conn(0x557e26da0670 msgr2=0x557e26da18c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:41:41.916 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.908+0000 7f7690000740 1 -- 172.21.15.99:0/75046082 shutdown_connections 2024-03-19T01:41:41.916 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.908+0000 7f7690000740 1 -- 172.21.15.99:0/75046082 wait complete. 2024-03-19T01:41:41.916 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.908+0000 7f7690000740 20 client.15378 trim_cache size 0 max 16384 2024-03-19T01:41:41.929 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.923+0000 7f7690000740 1 Processor -- start 2024-03-19T01:41:41.929 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.923+0000 7f7690000740 1 -- start start 2024-03-19T01:41:41.929 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.923+0000 7f7690000740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557e26da5100 0x557e26daf4b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:41.930 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.923+0000 7f7690000740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557e26da5af0 0x557e26db6550 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:41.930 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.923+0000 7f7690000740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557e26daac60 0x557e26da6ca0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:41.930 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.923+0000 7f7690000740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f76881a9380 con 0x557e26daac60 2024-03-19T01:41:41.930 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.923+0000 7f7690000740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f76881a6c50 con 0x557e26da5af0 2024-03-19T01:41:41.930 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.923+0000 7f7690000740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f76881a4aa0 con 0x557e26da5100 2024-03-19T01:41:41.930 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.924+0000 7f767ffff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557e26daac60 0x557e26da6ca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:41.930 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.924+0000 7f767ffff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557e26da5100 0x557e26daf4b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:41.930 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.924+0000 7f7686ffd640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557e26da5af0 0x557e26db6550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:41.930 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.924+0000 7f767ffff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557e26da5100 0x557e26daf4b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:59676/0 (socket says 172.21.15.99:59676) 2024-03-19T01:41:41.930 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.924+0000 7f767ffff640 1 -- 172.21.15.99:0/2557778169 learned_addr learned my addr 172.21.15.99:0/2557778169 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:41:41.930 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.924+0000 7f7686ffd640 1 -- 172.21.15.99:0/2557778169 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557e26da5100 msgr2=0x557e26daf4b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:41:41.930 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.924+0000 7f7686ffd640 1 --2- 172.21.15.99:0/2557778169 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557e26da5100 0x557e26daf4b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:41.930 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.924+0000 7f7686ffd640 1 -- 172.21.15.99:0/2557778169 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557e26daac60 msgr2=0x557e26da6ca0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:41:41.931 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.924+0000 7f7686ffd640 1 --2- 172.21.15.99:0/2557778169 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557e26daac60 0x557e26da6ca0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:41.931 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.924+0000 7f7686ffd640 1 -- 172.21.15.99:0/2557778169 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x557e26aea620 con 0x557e26da5af0 2024-03-19T01:41:41.931 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.924+0000 7f7686ffd640 1 --2- 172.21.15.99:0/2557778169 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557e26da5af0 0x557e26db6550 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x557e26da15e0 tx=0x7f7678005d30 comp rx=0 tx=0).ready entity=mon.2 client_cookie=aaf63677690da199 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:41:41.931 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.924+0000 7f767cff9640 1 -- 172.21.15.99:0/2557778169 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7678030070 con 0x557e26da5af0 2024-03-19T01:41:41.931 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.924+0000 7f767cff9640 1 -- 172.21.15.99:0/2557778169 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f767802a030 con 0x557e26da5af0 2024-03-19T01:41:41.931 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.924+0000 7f767cff9640 1 -- 172.21.15.99:0/2557778169 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7678003e60 con 0x557e26da5af0 2024-03-19T01:41:41.931 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.925+0000 7f7690000740 1 -- 172.21.15.99:0/2557778169 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557e26da5af0 msgr2=0x557e26db6550 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:41:41.931 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.925+0000 7f7690000740 1 --2- 172.21.15.99:0/2557778169 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557e26da5af0 0x557e26db6550 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x557e26da15e0 tx=0x7f7678005d30 comp rx=0 tx=0).stop 2024-03-19T01:41:41.931 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.925+0000 7f7690000740 1 -- 172.21.15.99:0/2557778169 shutdown_connections 2024-03-19T01:41:41.931 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.925+0000 7f7690000740 1 --2- 172.21.15.99:0/2557778169 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557e26da5af0 0x557e26db6550 unknown :-1 s=CLOSED pgs=89 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:41.931 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.925+0000 7f7690000740 1 --2- 172.21.15.99:0/2557778169 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557e26daac60 0x557e26da6ca0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:41.931 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.925+0000 7f7690000740 1 --2- 172.21.15.99:0/2557778169 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557e26da5100 0x557e26daf4b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:41.931 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.925+0000 7f7690000740 1 -- 172.21.15.99:0/2557778169 >> 172.21.15.99:0/2557778169 conn(0x557e26da0670 msgr2=0x557e26da6590 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:41:41.931 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.925+0000 7f7690000740 1 -- 172.21.15.99:0/2557778169 shutdown_connections 2024-03-19T01:41:41.931 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.925+0000 7f7690000740 1 -- 172.21.15.99:0/2557778169 wait complete. 2024-03-19T01:41:41.931 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.925+0000 7f7690000740 1 -- start start 2024-03-19T01:41:41.932 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.925+0000 7f7690000740 1 Processor -- start 2024-03-19T01:41:41.932 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.926+0000 7f7690000740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557e26da5100 0x557e26db3870 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:41.932 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.926+0000 7f7690000740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557e26da5af0 0x557e26db3db0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:41.932 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.926+0000 7f7690000740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557e26daac60 0x557e26db4dc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:41.932 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.926+0000 7f7690000740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f76881a5110 con 0x557e26da5100 2024-03-19T01:41:41.932 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.926+0000 7f7690000740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f76881a6580 con 0x557e26da5af0 2024-03-19T01:41:41.932 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.926+0000 7f7690000740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f76881a57e0 con 0x557e26daac60 2024-03-19T01:41:41.932 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.926+0000 7f7690000740 10 client.0 _ll_register_callbacks cb 0x7f767001e680 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:41:41.932 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.926+0000 7f767ffff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557e26da5100 0x557e26db3870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:41.932 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.926+0000 7f7686ffd640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557e26da5af0 0x557e26db3db0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:41.932 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.926+0000 7f767ffff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557e26da5100 0x557e26db3870 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:33928/0 (socket says 172.21.15.99:33928) 2024-03-19T01:41:41.932 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.926+0000 7f767ffff640 1 -- 172.21.15.99:0/4202701125 learned_addr learned my addr 172.21.15.99:0/4202701125 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:41:41.932 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.926+0000 7f767ffff640 1 --2- 172.21.15.99:0/4202701125 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557e26daac60 0x557e26db4dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:41.932 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.926+0000 7f767ffff640 1 -- 172.21.15.99:0/4202701125 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557e26daac60 msgr2=0x557e26db4dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:41:41.932 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.926+0000 7f767ffff640 1 --2- 172.21.15.99:0/4202701125 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557e26daac60 0x557e26db4dc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:41.933 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.926+0000 7f767ffff640 1 -- 172.21.15.99:0/4202701125 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557e26da5af0 msgr2=0x557e26db3db0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:41:41.933 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.926+0000 7f767ffff640 1 --2- 172.21.15.99:0/4202701125 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557e26da5af0 0x557e26db3db0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:41.933 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.926+0000 7f767ffff640 1 -- 172.21.15.99:0/4202701125 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x557e26db53b0 con 0x557e26da5100 2024-03-19T01:41:41.933 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.926+0000 7f7686ffd640 1 --2- 172.21.15.99:0/4202701125 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557e26da5af0 0x557e26db3db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:41:41.933 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.926+0000 7f767ffff640 1 --2- 172.21.15.99:0/4202701125 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557e26da5100 0x557e26db3870 secure :-1 s=READY pgs=366 cs=0 l=1 rev1=1 crypto rx=0x7f76700097c0 tx=0x7f7670005f90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=62b01f4ca6eedf16 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:41:41.933 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.926+0000 7f766e7fc640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:41:41.933 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.926+0000 7f766e7fc640 1 -- 172.21.15.99:0/4202701125 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f767000e480 con 0x557e26da5100 2024-03-19T01:41:41.933 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.926+0000 7f766e7fc640 1 -- 172.21.15.99:0/4202701125 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f767000e620 con 0x557e26da5100 2024-03-19T01:41:41.933 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.926+0000 7f7690000740 1 -- 172.21.15.99:0/4202701125 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f767802a030 con 0x557e26da5100 2024-03-19T01:41:41.933 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.926+0000 7f766e7fc640 1 -- 172.21.15.99:0/4202701125 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=23) ==== 24+0+0 (secure 0 0 0) 0x7f767000e820 con 0x557e26da5100 2024-03-19T01:41:41.933 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.926+0000 7f7690000740 10 client.15384 fetch_fsmap learned FSMap version 23 2024-03-19T01:41:41.933 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.926+0000 7f7690000740 1 -- 172.21.15.99:0/4202701125 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=23,monmap=4+}) -- 0x557e26db66c0 con 0x557e26da5100 2024-03-19T01:41:41.933 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.927+0000 7f766e7fc640 1 -- 172.21.15.99:0/4202701125 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f767802a030 con 0x557e26da5100 2024-03-19T01:41:41.933 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.927+0000 7f766e7fc640 1 -- 172.21.15.99:0/4202701125 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 23) ==== 8346+0+0 (secure 0 0 0) 0x7f76700055a0 con 0x557e26da5100 2024-03-19T01:41:41.933 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.927+0000 7f7690000740 10 client.15384 fetch_fsmap finished waiting for FSMap version 23 2024-03-19T01:41:41.934 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.927+0000 7f7690000740 10 client.15384 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:41:41.934 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.927+0000 7f7690000740 20 client.15384 populate_metadata read hostname 'smithi099' 2024-03-19T01:41:41.934 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.927+0000 7f7690000740 1 --2- 172.21.15.99:0/4202701125 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x557e26e217f0 0x557e26e23c10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:41.934 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.927+0000 7f7690000740 4 client.15384 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:41:41.934 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.927+0000 7f7690000740 1 -- 172.21.15.99:0/4202701125 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x557e26d11470 con 0x557e26e217f0 2024-03-19T01:41:41.934 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.927+0000 7f7686ffd640 1 --2- 172.21.15.99:0/4202701125 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x557e26e217f0 0x557e26e23c10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:41.934 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.927+0000 7f7686ffd640 1 --2- 172.21.15.99:0/4202701125 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x557e26e217f0 0x557e26e23c10 crc :-1 s=READY pgs=34 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=e913bf20fa86e3c1 server_cookie=3621e5abfe54228b in_seq=0 out_seq=0 2024-03-19T01:41:41.934 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.927+0000 7f766e7fc640 10 client.15384 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:41:41.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.978+0000 7f766e7fc640 1 -- 172.21.15.99:0/4202701125 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x557e26d11470 con 0x557e26e217f0 2024-03-19T01:41:41.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.978+0000 7f766e7fc640 10 client.15384 handle_command_reply: tid=0 2024-03-19T01:41:41.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.978+0000 7f7690000740 1 client.15384 shutdown 2024-03-19T01:41:41.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.978+0000 7f7690000740 1 -- 172.21.15.99:0/4202701125 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557e26da5100 msgr2=0x557e26db3870 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:41:41.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.978+0000 7f7690000740 1 --2- 172.21.15.99:0/4202701125 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557e26da5100 0x557e26db3870 secure :-1 s=READY pgs=366 cs=0 l=1 rev1=1 crypto rx=0x7f76700097c0 tx=0x7f7670005f90 comp rx=0 tx=0).stop 2024-03-19T01:41:41.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.978+0000 7f7690000740 1 -- 172.21.15.99:0/4202701125 shutdown_connections 2024-03-19T01:41:41.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.978+0000 7f7690000740 1 --2- 172.21.15.99:0/4202701125 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x557e26e217f0 0x557e26e23c10 crc :-1 s=READY pgs=34 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:41.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.978+0000 7f7690000740 1 --2- 172.21.15.99:0/4202701125 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557e26da5af0 0x557e26db3db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:41.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.978+0000 7f7690000740 1 --2- 172.21.15.99:0/4202701125 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557e26daac60 0x557e26db4dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:41.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.978+0000 7f7690000740 1 --2- 172.21.15.99:0/4202701125 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557e26da5100 0x557e26db3870 unknown :-1 s=CLOSED pgs=366 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:41.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.978+0000 7f766e7fc640 0 client.15384 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:41:41.980 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.978+0000 7f7690000740 1 -- 172.21.15.99:0/4202701125 >> 172.21.15.99:0/4202701125 conn(0x557e26da0670 msgr2=0x557e26daf310 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:41:41.980 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.978+0000 7f7690000740 1 -- 172.21.15.99:0/4202701125 shutdown_connections 2024-03-19T01:41:41.980 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.978+0000 7f7690000740 1 -- 172.21.15.99:0/4202701125 wait complete. 2024-03-19T01:41:41.980 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.979+0000 7f7690000740 20 client.15384 trim_cache size 0 max 16384 2024-03-19T01:41:41.980 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:41:41.980 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:41:41.980 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:41:41.980 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:41:41.981 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.979+0000 7f768d33b640 1 -- 172.21.15.99:0/2586468259 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f76740a2940 msgr2=0x7f76740a4e00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:41:41.981 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.979+0000 7f768d33b640 1 --2- 172.21.15.99:0/2586468259 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f76740a2940 0x7f76740a4e00 secure :-1 s=READY pgs=250 cs=0 l=1 rev1=1 crypto rx=0x7f768004c000 tx=0x7f768008ef30 comp rx=0 tx=0).stop 2024-03-19T01:41:41.981 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.979+0000 7f768d33b640 1 -- 172.21.15.99:0/2586468259 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f76881022c0 msgr2=0x7f768819ef90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:41:41.981 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.979+0000 7f768d33b640 1 --2- 172.21.15.99:0/2586468259 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f76881022c0 0x7f768819ef90 secure :-1 s=READY pgs=364 cs=0 l=1 rev1=1 crypto rx=0x7f768002c630 tx=0x7f7680032f20 comp rx=0 tx=0).stop 2024-03-19T01:41:41.981 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.979+0000 7f768d33b640 1 -- 172.21.15.99:0/2586468259 shutdown_connections 2024-03-19T01:41:41.981 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.979+0000 7f768d33b640 1 --2- 172.21.15.99:0/2586468259 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f76881022c0 0x7f768819ef90 unknown :-1 s=CLOSED pgs=364 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:41.982 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.979+0000 7f768d33b640 1 --2- 172.21.15.99:0/2586468259 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f7688101900 0x7f7688197f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:41.982 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.979+0000 7f768d33b640 1 --2- 172.21.15.99:0/2586468259 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f76740a2940 0x7f76740a4e00 unknown :-1 s=CLOSED pgs=250 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:41.982 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.979+0000 7f768d33b640 1 --2- 172.21.15.99:0/2586468259 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f7688100710 0x7f76881979e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:41.982 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.979+0000 7f768d33b640 1 -- 172.21.15.99:0/2586468259 >> 172.21.15.99:0/2586468259 conn(0x7f76880fbc80 msgr2=0x7f76880fd620 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:41:41.982 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.979+0000 7f768d33b640 1 -- 172.21.15.99:0/2586468259 shutdown_connections 2024-03-19T01:41:41.982 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:41.979+0000 7f768d33b640 1 -- 172.21.15.99:0/2586468259 wait complete. 2024-03-19T01:41:42.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:41:41 smithi112 ceph-mon[66819]: pgmap v368: 97 pgs: 97 active+clean; 10 MiB data, 416 MiB used, 1.0 TiB / 1.0 TiB avail; 446 KiB/s rd, 963 KiB/s wr, 140 op/s 2024-03-19T01:41:42.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:41:41 smithi115 ceph-mon[67332]: pgmap v368: 97 pgs: 97 active+clean; 10 MiB data, 416 MiB used, 1.0 TiB / 1.0 TiB avail; 446 KiB/s rd, 963 KiB/s wr, 140 op/s 2024-03-19T01:41:42.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:41:41 smithi099 ceph-mon[62951]: pgmap v368: 97 pgs: 97 active+clean; 10 MiB data, 416 MiB used, 1.0 TiB / 1.0 TiB avail; 446 KiB/s rd, 963 KiB/s wr, 140 op/s 2024-03-19T01:41:42.673 INFO:tasks.workunit.client.0.smithi099.stdout: 1 29102 14.52 MB/sec warmup 12 sec latency 19.298 ms 2024-03-19T01:41:43.674 INFO:tasks.workunit.client.0.smithi099.stdout: 1 32078 14.75 MB/sec warmup 13 sec latency 12.522 ms 2024-03-19T01:41:44.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:41:43 smithi112 ceph-mon[66819]: pgmap v369: 97 pgs: 97 active+clean; 12 MiB data, 435 MiB used, 1.0 TiB / 1.0 TiB avail; 537 KiB/s rd, 1.2 MiB/s wr, 163 op/s 2024-03-19T01:41:44.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:41:43 smithi115 ceph-mon[67332]: pgmap v369: 97 pgs: 97 active+clean; 12 MiB data, 435 MiB used, 1.0 TiB / 1.0 TiB avail; 537 KiB/s rd, 1.2 MiB/s wr, 163 op/s 2024-03-19T01:41:44.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:41:43 smithi099 ceph-mon[62951]: pgmap v369: 97 pgs: 97 active+clean; 12 MiB data, 435 MiB used, 1.0 TiB / 1.0 TiB avail; 537 KiB/s rd, 1.2 MiB/s wr, 163 op/s 2024-03-19T01:41:44.673 INFO:tasks.workunit.client.0.smithi099.stdout: 1 33931 14.18 MB/sec warmup 14 sec latency 33.631 ms 2024-03-19T01:41:45.673 INFO:tasks.workunit.client.0.smithi099.stdout: 1 36230 14.16 MB/sec warmup 15 sec latency 27.009 ms 2024-03-19T01:41:46.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:41:45 smithi112 ceph-mon[66819]: pgmap v370: 97 pgs: 97 active+clean; 12 MiB data, 438 MiB used, 1.0 TiB / 1.0 TiB avail; 536 KiB/s rd, 1.2 MiB/s wr, 165 op/s 2024-03-19T01:41:46.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:41:45 smithi115 ceph-mon[67332]: pgmap v370: 97 pgs: 97 active+clean; 12 MiB data, 438 MiB used, 1.0 TiB / 1.0 TiB avail; 536 KiB/s rd, 1.2 MiB/s wr, 165 op/s 2024-03-19T01:41:46.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:41:45 smithi099 ceph-mon[62951]: pgmap v370: 97 pgs: 97 active+clean; 12 MiB data, 438 MiB used, 1.0 TiB / 1.0 TiB avail; 536 KiB/s rd, 1.2 MiB/s wr, 165 op/s 2024-03-19T01:41:46.673 INFO:tasks.workunit.client.0.smithi099.stdout: 1 39167 14.37 MB/sec warmup 16 sec latency 17.995 ms 2024-03-19T01:41:47.673 INFO:tasks.workunit.client.0.smithi099.stdout: 1 41024 13.93 MB/sec warmup 17 sec latency 34.903 ms 2024-03-19T01:41:48.107 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:41:47 smithi115 ceph-mon[67332]: pgmap v371: 97 pgs: 97 active+clean; 17 MiB data, 450 MiB used, 1.0 TiB / 1.0 TiB avail; 855 KiB/s rd, 1.7 MiB/s wr, 265 op/s 2024-03-19T01:41:48.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:41:47 smithi112 ceph-mon[66819]: pgmap v371: 97 pgs: 97 active+clean; 17 MiB data, 450 MiB used, 1.0 TiB / 1.0 TiB avail; 855 KiB/s rd, 1.7 MiB/s wr, 265 op/s 2024-03-19T01:41:48.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:41:47 smithi099 ceph-mon[62951]: pgmap v371: 97 pgs: 97 active+clean; 17 MiB data, 450 MiB used, 1.0 TiB / 1.0 TiB avail; 855 KiB/s rd, 1.7 MiB/s wr, 265 op/s 2024-03-19T01:41:48.673 INFO:tasks.workunit.client.0.smithi099.stdout: 1 43280 13.93 MB/sec warmup 18 sec latency 36.110 ms 2024-03-19T01:41:49.673 INFO:tasks.workunit.client.0.smithi099.stdout: 1 45962 13.94 MB/sec warmup 19 sec latency 19.241 ms 2024-03-19T01:41:50.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:41:49 smithi115 ceph-mon[67332]: pgmap v372: 97 pgs: 97 active+clean; 18 MiB data, 466 MiB used, 1.0 TiB / 1.0 TiB avail; 768 KiB/s rd, 1.4 MiB/s wr, 239 op/s 2024-03-19T01:41:50.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:41:49 smithi112 ceph-mon[66819]: pgmap v372: 97 pgs: 97 active+clean; 18 MiB data, 466 MiB used, 1.0 TiB / 1.0 TiB avail; 768 KiB/s rd, 1.4 MiB/s wr, 239 op/s 2024-03-19T01:41:50.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:41:49 smithi099 ceph-mon[62951]: pgmap v372: 97 pgs: 97 active+clean; 18 MiB data, 466 MiB used, 1.0 TiB / 1.0 TiB avail; 768 KiB/s rd, 1.4 MiB/s wr, 239 op/s 2024-03-19T01:41:50.673 INFO:tasks.workunit.client.0.smithi099.stdout: 1 47851 13.74 MB/sec warmup 20 sec latency 37.882 ms 2024-03-19T01:41:51.673 INFO:tasks.workunit.client.0.smithi099.stdout: 1 50282 13.75 MB/sec warmup 21 sec latency 22.182 ms 2024-03-19T01:41:52.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:41:51 smithi112 ceph-mon[66819]: pgmap v373: 97 pgs: 97 active+clean; 19 MiB data, 473 MiB used, 1.0 TiB / 1.0 TiB avail; 568 KiB/s rd, 1.2 MiB/s wr, 200 op/s 2024-03-19T01:41:52.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:41:51 smithi115 ceph-mon[67332]: pgmap v373: 97 pgs: 97 active+clean; 19 MiB data, 473 MiB used, 1.0 TiB / 1.0 TiB avail; 568 KiB/s rd, 1.2 MiB/s wr, 200 op/s 2024-03-19T01:41:52.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:41:51 smithi099 ceph-mon[62951]: pgmap v373: 97 pgs: 97 active+clean; 19 MiB data, 473 MiB used, 1.0 TiB / 1.0 TiB avail; 568 KiB/s rd, 1.2 MiB/s wr, 200 op/s 2024-03-19T01:41:52.673 INFO:tasks.workunit.client.0.smithi099.stdout: 1 53202 13.91 MB/sec warmup 22 sec latency 28.295 ms 2024-03-19T01:41:53.673 INFO:tasks.workunit.client.0.smithi099.stdout: 1 55047 13.62 MB/sec warmup 23 sec latency 38.660 ms 2024-03-19T01:41:54.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:41:53 smithi115 ceph-mon[67332]: pgmap v374: 97 pgs: 97 active+clean; 23 MiB data, 489 MiB used, 1.0 TiB / 1.0 TiB avail; 706 KiB/s rd, 1.4 MiB/s wr, 227 op/s 2024-03-19T01:41:54.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:41:53 smithi099 ceph-mon[62951]: pgmap v374: 97 pgs: 97 active+clean; 23 MiB data, 489 MiB used, 1.0 TiB / 1.0 TiB avail; 706 KiB/s rd, 1.4 MiB/s wr, 227 op/s 2024-03-19T01:41:54.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:41:53 smithi112 ceph-mon[66819]: pgmap v374: 97 pgs: 97 active+clean; 23 MiB data, 489 MiB used, 1.0 TiB / 1.0 TiB avail; 706 KiB/s rd, 1.4 MiB/s wr, 227 op/s 2024-03-19T01:41:54.673 INFO:tasks.workunit.client.0.smithi099.stdout: 1 57376 13.62 MB/sec warmup 24 sec latency 26.982 ms 2024-03-19T01:41:55.070 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.068+0000 7f923dc14640 1 -- 172.21.15.99:0/3980677114 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9238101960 msgr2=0x7f9238101de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:41:55.071 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.068+0000 7f923dc14640 1 --2- 172.21.15.99:0/3980677114 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9238101960 0x7f9238101de0 secure :-1 s=READY pgs=367 cs=0 l=1 rev1=1 crypto rx=0x7f922c009900 tx=0x7f922c02bc30 comp rx=0 tx=0).stop 2024-03-19T01:41:55.071 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.068+0000 7f923dc14640 1 -- 172.21.15.99:0/3980677114 shutdown_connections 2024-03-19T01:41:55.071 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.068+0000 7f923dc14640 1 --2- 172.21.15.99:0/3980677114 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9238102320 0x7f9238108bb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:55.071 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.068+0000 7f923dc14640 1 --2- 172.21.15.99:0/3980677114 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9238101960 0x7f9238101de0 unknown :-1 s=CLOSED pgs=367 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:55.071 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.068+0000 7f923dc14640 1 --2- 172.21.15.99:0/3980677114 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9238100770 0x7f9238100b90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:55.071 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.068+0000 7f923dc14640 1 -- 172.21.15.99:0/3980677114 >> 172.21.15.99:0/3980677114 conn(0x7f92380fbcc0 msgr2=0x7f92380fe120 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:41:55.071 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.068+0000 7f923dc14640 1 -- 172.21.15.99:0/3980677114 shutdown_connections 2024-03-19T01:41:55.071 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.068+0000 7f923dc14640 1 -- 172.21.15.99:0/3980677114 wait complete. 2024-03-19T01:41:55.071 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.069+0000 7f923dc14640 1 Processor -- start 2024-03-19T01:41:55.071 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.069+0000 7f923dc14640 1 -- start start 2024-03-19T01:41:55.071 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.069+0000 7f923dc14640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9238100770 0x7f9238197aa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:55.072 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.069+0000 7f923dc14640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9238101960 0x7f9238197fe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:55.072 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.069+0000 7f923dc14640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9238102320 0x7f923819f050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:55.072 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.069+0000 7f923dc14640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f923810d510 con 0x7f9238100770 2024-03-19T01:41:55.072 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.069+0000 7f923dc14640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f923810d390 con 0x7f9238101960 2024-03-19T01:41:55.072 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.069+0000 7f923dc14640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f923810d690 con 0x7f9238102320 2024-03-19T01:41:55.072 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.069+0000 7f92377fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9238100770 0x7f9238197aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:55.072 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.069+0000 7f92377fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9238100770 0x7f9238197aa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:51666/0 (socket says 172.21.15.99:51666) 2024-03-19T01:41:55.072 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.069+0000 7f92377fe640 1 -- 172.21.15.99:0/4175320642 learned_addr learned my addr 172.21.15.99:0/4175320642 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:41:55.072 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.069+0000 7f9236ffd640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9238101960 0x7f9238197fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:55.072 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.069+0000 7f9237fff640 1 --2- 172.21.15.99:0/4175320642 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9238102320 0x7f923819f050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:55.072 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.069+0000 7f92377fe640 1 -- 172.21.15.99:0/4175320642 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9238102320 msgr2=0x7f923819f050 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:41:55.072 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.069+0000 7f92377fe640 1 --2- 172.21.15.99:0/4175320642 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9238102320 0x7f923819f050 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:55.072 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.069+0000 7f92377fe640 1 -- 172.21.15.99:0/4175320642 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9238101960 msgr2=0x7f9238197fe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:41:55.072 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.069+0000 7f92377fe640 1 --2- 172.21.15.99:0/4175320642 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9238101960 0x7f9238197fe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:55.072 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.069+0000 7f92377fe640 1 -- 172.21.15.99:0/4175320642 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f923819f6c0 con 0x7f9238100770 2024-03-19T01:41:55.073 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.069+0000 7f9236ffd640 1 --2- 172.21.15.99:0/4175320642 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9238101960 0x7f9238197fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:41:55.073 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.069+0000 7f92377fe640 1 --2- 172.21.15.99:0/4175320642 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9238100770 0x7f9238197aa0 secure :-1 s=READY pgs=368 cs=0 l=1 rev1=1 crypto rx=0x7f922800e9c0 tx=0x7f922800ee80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:41:55.073 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.070+0000 7f9234ff9640 1 -- 172.21.15.99:0/4175320642 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f922800cd80 con 0x7f9238100770 2024-03-19T01:41:55.073 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.070+0000 7f9234ff9640 1 -- 172.21.15.99:0/4175320642 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f92280035f0 con 0x7f9238100770 2024-03-19T01:41:55.073 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.070+0000 7f923dc14640 1 -- 172.21.15.99:0/4175320642 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f923819f9b0 con 0x7f9238100770 2024-03-19T01:41:55.073 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.070+0000 7f9234ff9640 1 -- 172.21.15.99:0/4175320642 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9228010690 con 0x7f9238100770 2024-03-19T01:41:55.073 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.070+0000 7f923dc14640 1 -- 172.21.15.99:0/4175320642 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f923819fed0 con 0x7f9238100770 2024-03-19T01:41:55.079 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.072+0000 7f9234ff9640 1 -- 172.21.15.99:0/4175320642 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 19) ==== 137341+0+0 (secure 0 0 0) 0x7f92280037a0 con 0x7f9238100770 2024-03-19T01:41:55.080 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.072+0000 7f92407fd740 1 Processor -- start 2024-03-19T01:41:55.080 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.072+0000 7f92407fd740 1 -- start start 2024-03-19T01:41:55.080 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.072+0000 7f92407fd740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c27f062100 0x55c27f062520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:55.080 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.072+0000 7f92407fd740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c27f062af0 0x55c27f0675f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:55.080 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.072+0000 7f9234ff9640 1 --2- 172.21.15.99:0/4175320642 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f92200a29c0 0x7f92200a4e80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:55.080 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.072+0000 7f92407fd740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c27f067c60 0x55c27f06a0f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:55.080 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.072+0000 7f92407fd740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55c27efc6be0 con 0x55c27f062af0 2024-03-19T01:41:55.080 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.072+0000 7f92407fd740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55c27ef6f960 con 0x55c27f067c60 2024-03-19T01:41:55.080 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.072+0000 7f92407fd740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55c27efc2c80 con 0x55c27f062100 2024-03-19T01:41:55.080 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.072+0000 7f92377fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c27f062af0 0x55c27f0675f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:55.080 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.072+0000 7f9237fff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c27f062100 0x55c27f062520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:55.080 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.072+0000 7f9236ffd640 1 --2- 172.21.15.99:0/4175320642 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f92200a29c0 0x7f92200a4e80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:55.080 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.072+0000 7f92377fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c27f062af0 0x55c27f0675f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:51674/0 (socket says 172.21.15.99:51674) 2024-03-19T01:41:55.080 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.072+0000 7f92377fe640 1 -- 172.21.15.99:0/1713746503 learned_addr learned my addr 172.21.15.99:0/1713746503 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:41:55.081 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.073+0000 7f9237fff640 1 --2- 172.21.15.99:0/1713746503 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c27f067c60 0x55c27f06a0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:55.081 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.073+0000 7f9234ff9640 1 -- 172.21.15.99:0/4175320642 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f92280cef00 con 0x7f9238100770 2024-03-19T01:41:55.081 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.073+0000 7f9237fff640 1 -- 172.21.15.99:0/1713746503 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c27f067c60 msgr2=0x55c27f06a0f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:41:55.081 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.073+0000 7f9237fff640 1 --2- 172.21.15.99:0/1713746503 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c27f067c60 0x55c27f06a0f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:55.081 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.073+0000 7f9237fff640 1 -- 172.21.15.99:0/1713746503 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c27f062af0 msgr2=0x55c27f0675f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:41:55.081 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.073+0000 7f9237fff640 1 --2- 172.21.15.99:0/1713746503 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c27f062af0 0x55c27f0675f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:55.081 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.073+0000 7f9237fff640 1 -- 172.21.15.99:0/1713746503 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55c27ef66d20 con 0x55c27f062100 2024-03-19T01:41:55.081 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.073+0000 7f92377fe640 1 --2- 172.21.15.99:0/1713746503 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c27f062af0 0x55c27f0675f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:41:55.081 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.073+0000 7f9236ffd640 1 --2- 172.21.15.99:0/4175320642 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f92200a29c0 0x7f92200a4e80 secure :-1 s=READY pgs=251 cs=0 l=1 rev1=1 crypto rx=0x7f922c02c7b0 tx=0x7f922c005fd0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:41:55.081 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.073+0000 7f9237fff640 1 --2- 172.21.15.99:0/1713746503 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c27f062100 0x55c27f062520 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7f9230011a70 tx=0x7f9230004ab0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=ca3942edc1d3134 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:41:55.081 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.073+0000 7f921dffb640 1 -- 172.21.15.99:0/1713746503 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9230004b20 con 0x55c27f062100 2024-03-19T01:41:55.081 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.073+0000 7f921dffb640 1 -- 172.21.15.99:0/1713746503 <== mon.1 v2:172.21.15.115:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f9230004cc0 con 0x55c27f062100 2024-03-19T01:41:55.081 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.073+0000 7f921dffb640 1 -- 172.21.15.99:0/1713746503 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9230005660 con 0x55c27f062100 2024-03-19T01:41:55.081 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.074+0000 7f92407fd740 1 -- 172.21.15.99:0/1713746503 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c27f062100 msgr2=0x55c27f062520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:41:55.081 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.074+0000 7f92407fd740 1 --2- 172.21.15.99:0/1713746503 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c27f062100 0x55c27f062520 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7f9230011a70 tx=0x7f9230004ab0 comp rx=0 tx=0).stop 2024-03-19T01:41:55.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.074+0000 7f92407fd740 1 -- 172.21.15.99:0/1713746503 shutdown_connections 2024-03-19T01:41:55.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.074+0000 7f92407fd740 1 --2- 172.21.15.99:0/1713746503 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c27f067c60 0x55c27f06a0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:55.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.074+0000 7f92407fd740 1 --2- 172.21.15.99:0/1713746503 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c27f062af0 0x55c27f0675f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:55.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.074+0000 7f92407fd740 1 --2- 172.21.15.99:0/1713746503 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c27f062100 0x55c27f062520 unknown :-1 s=CLOSED pgs=79 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:55.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.074+0000 7f92407fd740 1 -- 172.21.15.99:0/1713746503 >> 172.21.15.99:0/1713746503 conn(0x55c27f05d670 msgr2=0x55c27f05da70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:41:55.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.074+0000 7f92407fd740 1 -- 172.21.15.99:0/1713746503 shutdown_connections 2024-03-19T01:41:55.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.074+0000 7f92407fd740 1 -- 172.21.15.99:0/1713746503 wait complete. 2024-03-19T01:41:55.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.074+0000 7f92407fd740 1 -- start start 2024-03-19T01:41:55.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.074+0000 7f92407fd740 1 Processor -- start 2024-03-19T01:41:55.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.075+0000 7f92407fd740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c27f062100 0x55c27f06c160 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:55.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.075+0000 7f92407fd740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c27f062af0 0x55c27f06c6a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:55.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.075+0000 7f92407fd740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c27f067c60 0x55c27f0736e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:55.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.075+0000 7f92407fd740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55c27f06ac10 con 0x55c27f062100 2024-03-19T01:41:55.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.075+0000 7f92407fd740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55c27ef82db0 con 0x55c27f062af0 2024-03-19T01:41:55.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.075+0000 7f92407fd740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55c27f06ad90 con 0x55c27f067c60 2024-03-19T01:41:55.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.075+0000 7f92407fd740 10 client.0 _ll_register_callbacks cb 0x55c27ef87e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:41:55.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.075+0000 7f92377fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c27f062100 0x55c27f06c160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:55.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.075+0000 7f92377fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c27f062100 0x55c27f06c160 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:51686/0 (socket says 172.21.15.99:51686) 2024-03-19T01:41:55.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.075+0000 7f9236ffd640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c27f062af0 0x55c27f06c6a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:55.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.075+0000 7f9237fff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c27f067c60 0x55c27f0736e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:55.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.075+0000 7f92377fe640 1 -- 172.21.15.99:0/2252100111 learned_addr learned my addr 172.21.15.99:0/2252100111 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:41:55.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.075+0000 7f92377fe640 1 -- 172.21.15.99:0/2252100111 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c27f067c60 msgr2=0x55c27f0736e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:41:55.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.075+0000 7f92377fe640 1 --2- 172.21.15.99:0/2252100111 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c27f067c60 0x55c27f0736e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:55.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.075+0000 7f92377fe640 1 -- 172.21.15.99:0/2252100111 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c27f062af0 msgr2=0x55c27f06c6a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:41:55.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.075+0000 7f92377fe640 1 --2- 172.21.15.99:0/2252100111 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c27f062af0 0x55c27f06c6a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:55.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.075+0000 7f92377fe640 1 -- 172.21.15.99:0/2252100111 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55c27ef81c60 con 0x55c27f062100 2024-03-19T01:41:55.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.075+0000 7f9236ffd640 1 --2- 172.21.15.99:0/2252100111 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c27f062af0 0x55c27f06c6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:41:55.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.075+0000 7f92377fe640 1 --2- 172.21.15.99:0/2252100111 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c27f062100 0x55c27f06c160 secure :-1 s=READY pgs=369 cs=0 l=1 rev1=1 crypto rx=0x7f922809b400 tx=0x7f922806a560 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c39bffdd5f096604 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:41:55.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.075+0000 7f9216ffd640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:41:55.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.075+0000 7f9216ffd640 1 -- 172.21.15.99:0/2252100111 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9228012eb0 con 0x55c27f062100 2024-03-19T01:41:55.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.075+0000 7f9216ffd640 1 -- 172.21.15.99:0/2252100111 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f9228003b10 con 0x55c27f062100 2024-03-19T01:41:55.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.075+0000 7f92407fd740 1 -- 172.21.15.99:0/2252100111 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f9230004cc0 con 0x55c27f062100 2024-03-19T01:41:55.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.075+0000 7f9216ffd640 1 -- 172.21.15.99:0/2252100111 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=23) ==== 24+0+0 (secure 0 0 0) 0x7f922809be00 con 0x55c27f062100 2024-03-19T01:41:55.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.075+0000 7f92407fd740 10 client.15396 fetch_fsmap learned FSMap version 23 2024-03-19T01:41:55.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.075+0000 7f92407fd740 1 -- 172.21.15.99:0/2252100111 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=23,monmap=4+}) -- 0x55c27ecdb210 con 0x55c27f062100 2024-03-19T01:41:55.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.076+0000 7f9216ffd640 1 -- 172.21.15.99:0/2252100111 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f9230004cc0 con 0x55c27f062100 2024-03-19T01:41:55.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.076+0000 7f9216ffd640 1 -- 172.21.15.99:0/2252100111 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 23) ==== 8346+0+0 (secure 0 0 0) 0x7f922806d060 con 0x55c27f062100 2024-03-19T01:41:55.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.076+0000 7f92407fd740 10 client.15396 fetch_fsmap finished waiting for FSMap version 23 2024-03-19T01:41:55.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.076+0000 7f92407fd740 10 client.15396 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:41:55.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.076+0000 7f92407fd740 20 client.15396 populate_metadata read hostname 'smithi099' 2024-03-19T01:41:55.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.076+0000 7f92407fd740 1 --2- 172.21.15.99:0/2252100111 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55c27f077fd0 0x55c27f07a3e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:55.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.076+0000 7f92407fd740 4 client.15396 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:41:55.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.076+0000 7f92407fd740 1 -- 172.21.15.99:0/2252100111 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55c27efd4120 con 0x55c27f077fd0 2024-03-19T01:41:55.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.076+0000 7f9236ffd640 1 --2- 172.21.15.99:0/2252100111 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55c27f077fd0 0x55c27f07a3e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:55.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.076+0000 7f9216ffd640 10 client.15396 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:41:55.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.076+0000 7f9236ffd640 1 --2- 172.21.15.99:0/2252100111 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55c27f077fd0 0x55c27f07a3e0 crc :-1 s=READY pgs=35 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=b8d8d54c63d69869 server_cookie=ff59556b868ccd48 in_seq=0 out_seq=0 2024-03-19T01:41:55.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.077+0000 7f9216ffd640 1 -- 172.21.15.99:0/2252100111 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x55c27efd4120 con 0x55c27f077fd0 2024-03-19T01:41:55.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.077+0000 7f9216ffd640 10 client.15396 handle_command_reply: tid=0 2024-03-19T01:41:55.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.077+0000 7f92407fd740 1 client.15396 shutdown 2024-03-19T01:41:55.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.078+0000 7f92407fd740 1 -- 172.21.15.99:0/2252100111 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c27f062100 msgr2=0x55c27f06c160 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:41:55.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.078+0000 7f92407fd740 1 --2- 172.21.15.99:0/2252100111 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c27f062100 0x55c27f06c160 secure :-1 s=READY pgs=369 cs=0 l=1 rev1=1 crypto rx=0x7f922809b400 tx=0x7f922806a560 comp rx=0 tx=0).stop 2024-03-19T01:41:55.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.078+0000 7f92407fd740 1 -- 172.21.15.99:0/2252100111 shutdown_connections 2024-03-19T01:41:55.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.078+0000 7f92407fd740 1 --2- 172.21.15.99:0/2252100111 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55c27f077fd0 0x55c27f07a3e0 crc :-1 s=READY pgs=35 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:55.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.078+0000 7f92407fd740 1 --2- 172.21.15.99:0/2252100111 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c27f062af0 0x55c27f06c6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:55.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.078+0000 7f92407fd740 1 --2- 172.21.15.99:0/2252100111 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c27f067c60 0x55c27f0736e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:55.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.078+0000 7f9216ffd640 0 client.15396 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:41:55.086 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.078+0000 7f92407fd740 1 --2- 172.21.15.99:0/2252100111 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c27f062100 0x55c27f06c160 unknown :-1 s=CLOSED pgs=369 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:55.086 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.078+0000 7f92407fd740 1 -- 172.21.15.99:0/2252100111 >> 172.21.15.99:0/2252100111 conn(0x55c27f05d670 msgr2=0x55c27f05e8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:41:55.086 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.078+0000 7f92407fd740 1 -- 172.21.15.99:0/2252100111 shutdown_connections 2024-03-19T01:41:55.086 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.078+0000 7f92407fd740 1 -- 172.21.15.99:0/2252100111 wait complete. 2024-03-19T01:41:55.086 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.078+0000 7f92407fd740 20 client.15396 trim_cache size 0 max 16384 2024-03-19T01:41:55.097 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.093+0000 7f92407fd740 1 Processor -- start 2024-03-19T01:41:55.097 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.093+0000 7f92407fd740 1 -- start start 2024-03-19T01:41:55.097 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.093+0000 7f92407fd740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c27f062100 0x55c27f06bd70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:55.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.093+0000 7f92407fd740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c27f062af0 0x55c27f073670 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:55.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.093+0000 7f92407fd740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c27f067c60 0x55c27f06c2b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:55.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.093+0000 7f92407fd740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f92381ab670 con 0x55c27f062100 2024-03-19T01:41:55.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.093+0000 7f92407fd740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f92381a8f40 con 0x55c27f062af0 2024-03-19T01:41:55.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.093+0000 7f92407fd740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f92381a6d90 con 0x55c27f067c60 2024-03-19T01:41:55.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.093+0000 7f92377fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c27f062100 0x55c27f06bd70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:55.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.093+0000 7f92377fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c27f062100 0x55c27f06bd70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:51702/0 (socket says 172.21.15.99:51702) 2024-03-19T01:41:55.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.093+0000 7f92377fe640 1 -- 172.21.15.99:0/3074277193 learned_addr learned my addr 172.21.15.99:0/3074277193 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:41:55.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.093+0000 7f9236ffd640 1 --2- 172.21.15.99:0/3074277193 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c27f062af0 0x55c27f073670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:55.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.093+0000 7f9237fff640 1 --2- 172.21.15.99:0/3074277193 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c27f067c60 0x55c27f06c2b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:55.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.093+0000 7f92377fe640 1 -- 172.21.15.99:0/3074277193 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c27f067c60 msgr2=0x55c27f06c2b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:41:55.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.093+0000 7f92377fe640 1 --2- 172.21.15.99:0/3074277193 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c27f067c60 0x55c27f06c2b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:55.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.093+0000 7f92377fe640 1 -- 172.21.15.99:0/3074277193 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c27f062af0 msgr2=0x55c27f073670 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:41:55.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.093+0000 7f92377fe640 1 --2- 172.21.15.99:0/3074277193 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c27f062af0 0x55c27f073670 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:55.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.093+0000 7f92377fe640 1 -- 172.21.15.99:0/3074277193 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55c27eda7620 con 0x55c27f062100 2024-03-19T01:41:55.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.093+0000 7f92377fe640 1 --2- 172.21.15.99:0/3074277193 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c27f062100 0x55c27f06bd70 secure :-1 s=READY pgs=370 cs=0 l=1 rev1=1 crypto rx=0x7f9228066710 tx=0x7f9228086050 comp rx=0 tx=0).ready entity=mon.0 client_cookie=be6f630a0be4961 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:41:55.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.093+0000 7f921d7fa640 1 -- 172.21.15.99:0/3074277193 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f922809b060 con 0x55c27f062100 2024-03-19T01:41:55.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.093+0000 7f921d7fa640 1 -- 172.21.15.99:0/3074277193 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f92280a3a60 con 0x55c27f062100 2024-03-19T01:41:55.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.093+0000 7f921d7fa640 1 -- 172.21.15.99:0/3074277193 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f922809adb0 con 0x55c27f062100 2024-03-19T01:41:55.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.094+0000 7f92407fd740 1 -- 172.21.15.99:0/3074277193 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c27f062100 msgr2=0x55c27f06bd70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:41:55.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.094+0000 7f92407fd740 1 --2- 172.21.15.99:0/3074277193 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c27f062100 0x55c27f06bd70 secure :-1 s=READY pgs=370 cs=0 l=1 rev1=1 crypto rx=0x7f9228066710 tx=0x7f9228086050 comp rx=0 tx=0).stop 2024-03-19T01:41:55.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.094+0000 7f92407fd740 1 -- 172.21.15.99:0/3074277193 shutdown_connections 2024-03-19T01:41:55.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.094+0000 7f92407fd740 1 --2- 172.21.15.99:0/3074277193 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c27f062af0 0x55c27f073670 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:55.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.094+0000 7f92407fd740 1 --2- 172.21.15.99:0/3074277193 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c27f067c60 0x55c27f06c2b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:55.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.094+0000 7f92407fd740 1 --2- 172.21.15.99:0/3074277193 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c27f062100 0x55c27f06bd70 unknown :-1 s=CLOSED pgs=370 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:55.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.094+0000 7f92407fd740 1 -- 172.21.15.99:0/3074277193 >> 172.21.15.99:0/3074277193 conn(0x55c27f05d670 msgr2=0x55c27f05da50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:41:55.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.094+0000 7f92407fd740 1 -- 172.21.15.99:0/3074277193 shutdown_connections 2024-03-19T01:41:55.100 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.094+0000 7f92407fd740 1 -- 172.21.15.99:0/3074277193 wait complete. 2024-03-19T01:41:55.100 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.094+0000 7f92407fd740 1 -- start start 2024-03-19T01:41:55.100 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.095+0000 7f92407fd740 1 Processor -- start 2024-03-19T01:41:55.100 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.095+0000 7f92407fd740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c27f062100 0x55c27f070920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:55.100 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.095+0000 7f92407fd740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c27f062af0 0x55c27f070e60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:55.100 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.095+0000 7f92407fd740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c27f067c60 0x55c27f071e70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:55.100 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.095+0000 7f92407fd740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f92381a7400 con 0x55c27f062100 2024-03-19T01:41:55.100 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.095+0000 7f92407fd740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f92381a8870 con 0x55c27f062af0 2024-03-19T01:41:55.100 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.095+0000 7f92407fd740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f92381a7ad0 con 0x55c27f067c60 2024-03-19T01:41:55.100 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.095+0000 7f92407fd740 10 client.0 _ll_register_callbacks cb 0x7f9228009770 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:41:55.100 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.095+0000 7f92377fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c27f062100 0x55c27f070920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:55.100 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.095+0000 7f92377fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c27f062100 0x55c27f070920 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:51708/0 (socket says 172.21.15.99:51708) 2024-03-19T01:41:55.100 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.095+0000 7f92377fe640 1 -- 172.21.15.99:0/3531257887 learned_addr learned my addr 172.21.15.99:0/3531257887 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:41:55.100 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.095+0000 7f9236ffd640 1 --2- 172.21.15.99:0/3531257887 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c27f062af0 0x55c27f070e60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:55.101 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.095+0000 7f9237fff640 1 --2- 172.21.15.99:0/3531257887 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c27f067c60 0x55c27f071e70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:55.101 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.095+0000 7f92377fe640 1 -- 172.21.15.99:0/3531257887 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c27f067c60 msgr2=0x55c27f071e70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:41:55.101 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.095+0000 7f92377fe640 1 --2- 172.21.15.99:0/3531257887 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c27f067c60 0x55c27f071e70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:55.101 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.095+0000 7f92377fe640 1 -- 172.21.15.99:0/3531257887 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c27f062af0 msgr2=0x55c27f070e60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:41:55.101 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.095+0000 7f92377fe640 1 --2- 172.21.15.99:0/3531257887 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c27f062af0 0x55c27f070e60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:55.101 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.095+0000 7f92377fe640 1 -- 172.21.15.99:0/3531257887 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55c27f072460 con 0x55c27f062100 2024-03-19T01:41:55.101 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.095+0000 7f9237fff640 1 --2- 172.21.15.99:0/3531257887 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c27f067c60 0x55c27f071e70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-19T01:41:55.101 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.095+0000 7f92377fe640 1 --2- 172.21.15.99:0/3531257887 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c27f062100 0x55c27f070920 secure :-1 s=READY pgs=371 cs=0 l=1 rev1=1 crypto rx=0x7f92280980d0 tx=0x7f922806d380 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5dd7394b4289992e server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:41:55.101 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.095+0000 7f9216ffd640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:41:55.101 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.095+0000 7f9216ffd640 1 -- 172.21.15.99:0/3531257887 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f92280b4490 con 0x55c27f062100 2024-03-19T01:41:55.101 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.096+0000 7f9216ffd640 1 -- 172.21.15.99:0/3531257887 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f922809ab50 con 0x55c27f062100 2024-03-19T01:41:55.101 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.096+0000 7f92407fd740 1 -- 172.21.15.99:0/3531257887 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f92280a3a60 con 0x55c27f062100 2024-03-19T01:41:55.101 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.096+0000 7f9216ffd640 1 -- 172.21.15.99:0/3531257887 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=23) ==== 24+0+0 (secure 0 0 0) 0x7f922809b700 con 0x55c27f062100 2024-03-19T01:41:55.102 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.096+0000 7f92407fd740 10 client.15402 fetch_fsmap learned FSMap version 23 2024-03-19T01:41:55.102 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.096+0000 7f92407fd740 1 -- 172.21.15.99:0/3531257887 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=23,monmap=4+}) -- 0x55c27f073770 con 0x55c27f062100 2024-03-19T01:41:55.102 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.096+0000 7f9216ffd640 1 -- 172.21.15.99:0/3531257887 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f92280a3a60 con 0x55c27f062100 2024-03-19T01:41:55.102 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.096+0000 7f9216ffd640 1 -- 172.21.15.99:0/3531257887 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 23) ==== 8346+0+0 (secure 0 0 0) 0x7f9228097450 con 0x55c27f062100 2024-03-19T01:41:55.102 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.096+0000 7f92407fd740 10 client.15402 fetch_fsmap finished waiting for FSMap version 23 2024-03-19T01:41:55.102 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.096+0000 7f92407fd740 10 client.15402 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:41:55.102 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.096+0000 7f92407fd740 20 client.15402 populate_metadata read hostname 'smithi099' 2024-03-19T01:41:55.102 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.096+0000 7f92407fd740 1 --2- 172.21.15.99:0/3531257887 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55c27f077fd0 0x55c27f0e0650 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:41:55.102 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.096+0000 7f92407fd740 4 client.15402 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:41:55.102 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.096+0000 7f92407fd740 1 -- 172.21.15.99:0/3531257887 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55c27efce470 con 0x55c27f077fd0 2024-03-19T01:41:55.102 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.096+0000 7f9236ffd640 1 --2- 172.21.15.99:0/3531257887 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55c27f077fd0 0x55c27f0e0650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:41:55.102 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.097+0000 7f9216ffd640 10 client.15402 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:41:55.102 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.097+0000 7f9236ffd640 1 --2- 172.21.15.99:0/3531257887 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55c27f077fd0 0x55c27f0e0650 crc :-1 s=READY pgs=36 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=826cc9fd5ed12b75 server_cookie=1d3599d45f36948a in_seq=0 out_seq=0 2024-03-19T01:41:55.153 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.152+0000 7f9216ffd640 1 -- 172.21.15.99:0/3531257887 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x55c27efce470 con 0x55c27f077fd0 2024-03-19T01:41:55.153 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.152+0000 7f9216ffd640 10 client.15402 handle_command_reply: tid=0 2024-03-19T01:41:55.153 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.152+0000 7f92407fd740 1 client.15402 shutdown 2024-03-19T01:41:55.153 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.153+0000 7f92407fd740 1 -- 172.21.15.99:0/3531257887 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c27f062100 msgr2=0x55c27f070920 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:41:55.153 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.153+0000 7f92407fd740 1 --2- 172.21.15.99:0/3531257887 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c27f062100 0x55c27f070920 secure :-1 s=READY pgs=371 cs=0 l=1 rev1=1 crypto rx=0x7f92280980d0 tx=0x7f922806d380 comp rx=0 tx=0).stop 2024-03-19T01:41:55.153 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.153+0000 7f92407fd740 1 -- 172.21.15.99:0/3531257887 shutdown_connections 2024-03-19T01:41:55.154 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.153+0000 7f92407fd740 1 --2- 172.21.15.99:0/3531257887 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55c27f077fd0 0x55c27f0e0650 crc :-1 s=READY pgs=36 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:55.154 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.153+0000 7f92407fd740 1 --2- 172.21.15.99:0/3531257887 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c27f062af0 0x55c27f070e60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:55.154 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.153+0000 7f92407fd740 1 --2- 172.21.15.99:0/3531257887 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c27f067c60 0x55c27f071e70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:55.154 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.153+0000 7f9216ffd640 0 client.15402 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:41:55.154 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.153+0000 7f92407fd740 1 --2- 172.21.15.99:0/3531257887 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c27f062100 0x55c27f070920 unknown :-1 s=CLOSED pgs=371 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:55.154 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.153+0000 7f92407fd740 1 -- 172.21.15.99:0/3531257887 >> 172.21.15.99:0/3531257887 conn(0x55c27f05d670 msgr2=0x55c27f06bd50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:41:55.154 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.153+0000 7f92407fd740 1 -- 172.21.15.99:0/3531257887 shutdown_connections 2024-03-19T01:41:55.154 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.153+0000 7f92407fd740 1 -- 172.21.15.99:0/3531257887 wait complete. 2024-03-19T01:41:55.154 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.153+0000 7f92407fd740 20 client.15402 trim_cache size 0 max 16384 2024-03-19T01:41:55.154 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:41:55.154 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:41:55.155 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:41:55.155 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:41:55.155 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.153+0000 7f923dc14640 1 -- 172.21.15.99:0/4175320642 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f92200a29c0 msgr2=0x7f92200a4e80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:41:55.156 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.153+0000 7f923dc14640 1 --2- 172.21.15.99:0/4175320642 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f92200a29c0 0x7f92200a4e80 secure :-1 s=READY pgs=251 cs=0 l=1 rev1=1 crypto rx=0x7f922c02c7b0 tx=0x7f922c005fd0 comp rx=0 tx=0).stop 2024-03-19T01:41:55.156 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.153+0000 7f923dc14640 1 -- 172.21.15.99:0/4175320642 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9238100770 msgr2=0x7f9238197aa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:41:55.156 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.153+0000 7f923dc14640 1 --2- 172.21.15.99:0/4175320642 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9238100770 0x7f9238197aa0 secure :-1 s=READY pgs=368 cs=0 l=1 rev1=1 crypto rx=0x7f922800e9c0 tx=0x7f922800ee80 comp rx=0 tx=0).stop 2024-03-19T01:41:55.156 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.153+0000 7f923dc14640 1 -- 172.21.15.99:0/4175320642 shutdown_connections 2024-03-19T01:41:55.156 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.153+0000 7f923dc14640 1 --2- 172.21.15.99:0/4175320642 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f92200a29c0 0x7f92200a4e80 unknown :-1 s=CLOSED pgs=251 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:55.156 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.153+0000 7f923dc14640 1 --2- 172.21.15.99:0/4175320642 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9238101960 0x7f9238197fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:55.156 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.153+0000 7f923dc14640 1 --2- 172.21.15.99:0/4175320642 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9238102320 0x7f923819f050 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:55.156 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.153+0000 7f923dc14640 1 --2- 172.21.15.99:0/4175320642 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9238100770 0x7f9238197aa0 unknown :-1 s=CLOSED pgs=368 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:41:55.156 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.153+0000 7f923dc14640 1 -- 172.21.15.99:0/4175320642 >> 172.21.15.99:0/4175320642 conn(0x7f92380fbcc0 msgr2=0x7f9238104c00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:41:55.156 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.154+0000 7f923dc14640 1 -- 172.21.15.99:0/4175320642 shutdown_connections 2024-03-19T01:41:55.156 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:41:55.154+0000 7f923dc14640 1 -- 172.21.15.99:0/4175320642 wait complete. 2024-03-19T01:41:55.673 INFO:tasks.workunit.client.0.smithi099.stdout: 1 60148 13.66 MB/sec warmup 25 sec latency 32.232 ms 2024-03-19T01:41:56.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:41:55 smithi115 ceph-mon[67332]: pgmap v375: 97 pgs: 97 active+clean; 24 MiB data, 490 MiB used, 1.0 TiB / 1.0 TiB avail; 667 KiB/s rd, 1.3 MiB/s wr, 216 op/s 2024-03-19T01:41:56.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:41:55 smithi099 ceph-mon[62951]: pgmap v375: 97 pgs: 97 active+clean; 24 MiB data, 490 MiB used, 1.0 TiB / 1.0 TiB avail; 667 KiB/s rd, 1.3 MiB/s wr, 216 op/s 2024-03-19T01:41:56.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:41:55 smithi112 ceph-mon[66819]: pgmap v375: 97 pgs: 97 active+clean; 24 MiB data, 490 MiB used, 1.0 TiB / 1.0 TiB avail; 667 KiB/s rd, 1.3 MiB/s wr, 216 op/s 2024-03-19T01:41:56.674 INFO:tasks.workunit.client.0.smithi099.stdout: 1 61252 13.34 MB/sec warmup 26 sec latency 306.784 ms 2024-03-19T01:41:57.674 INFO:tasks.workunit.client.0.smithi099.stdout: 1 62783 13.08 MB/sec warmup 27 sec latency 316.381 ms 2024-03-19T01:41:58.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:41:57 smithi115 ceph-mon[67332]: pgmap v376: 97 pgs: 97 active+clean; 26 MiB data, 501 MiB used, 1.0 TiB / 1.0 TiB avail; 757 KiB/s rd, 1.5 MiB/s wr, 251 op/s 2024-03-19T01:41:58.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:41:57 smithi099 ceph-mon[62951]: pgmap v376: 97 pgs: 97 active+clean; 26 MiB data, 501 MiB used, 1.0 TiB / 1.0 TiB avail; 757 KiB/s rd, 1.5 MiB/s wr, 251 op/s 2024-03-19T01:41:58.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:41:57 smithi112 ceph-mon[66819]: pgmap v376: 97 pgs: 97 active+clean; 26 MiB data, 501 MiB used, 1.0 TiB / 1.0 TiB avail; 757 KiB/s rd, 1.5 MiB/s wr, 251 op/s 2024-03-19T01:41:58.674 INFO:tasks.workunit.client.0.smithi099.stdout: 1 64843 13.07 MB/sec warmup 28 sec latency 29.510 ms 2024-03-19T01:41:59.674 INFO:tasks.workunit.client.0.smithi099.stdout: 1 67576 13.21 MB/sec warmup 29 sec latency 19.592 ms 2024-03-19T01:42:00.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:41:59 smithi115 ceph-mon[67332]: pgmap v377: 97 pgs: 97 active+clean; 43 MiB data, 592 MiB used, 1.0 TiB / 1.0 TiB avail; 731 KiB/s rd, 3.3 MiB/s wr, 243 op/s 2024-03-19T01:42:00.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:41:59 smithi099 ceph-mon[62951]: pgmap v377: 97 pgs: 97 active+clean; 43 MiB data, 592 MiB used, 1.0 TiB / 1.0 TiB avail; 731 KiB/s rd, 3.3 MiB/s wr, 243 op/s 2024-03-19T01:42:00.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:41:59 smithi112 ceph-mon[66819]: pgmap v377: 97 pgs: 97 active+clean; 43 MiB data, 592 MiB used, 1.0 TiB / 1.0 TiB avail; 731 KiB/s rd, 3.3 MiB/s wr, 243 op/s 2024-03-19T01:42:00.674 INFO:tasks.workunit.client.0.smithi099.stdout: 1 69398 12.99 MB/sec warmup 30 sec latency 31.968 ms 2024-03-19T01:42:01.674 INFO:tasks.workunit.client.0.smithi099.stdout: 1 71629 13.02 MB/sec warmup 31 sec latency 27.872 ms 2024-03-19T01:42:02.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:42:01 smithi115 ceph-mon[67332]: pgmap v378: 97 pgs: 97 active+clean; 50 MiB data, 621 MiB used, 1.0 TiB / 1.0 TiB avail; 913 KiB/s rd, 5.4 MiB/s wr, 308 op/s 2024-03-19T01:42:02.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:42:01 smithi099 ceph-mon[62951]: pgmap v378: 97 pgs: 97 active+clean; 50 MiB data, 621 MiB used, 1.0 TiB / 1.0 TiB avail; 913 KiB/s rd, 5.4 MiB/s wr, 308 op/s 2024-03-19T01:42:02.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:42:01 smithi112 ceph-mon[66819]: pgmap v378: 97 pgs: 97 active+clean; 50 MiB data, 621 MiB used, 1.0 TiB / 1.0 TiB avail; 913 KiB/s rd, 5.4 MiB/s wr, 308 op/s 2024-03-19T01:42:02.674 INFO:tasks.workunit.client.0.smithi099.stdout: 1 74358 13.06 MB/sec warmup 32 sec latency 31.642 ms 2024-03-19T01:42:03.674 INFO:tasks.workunit.client.0.smithi099.stdout: 1 76235 12.97 MB/sec warmup 33 sec latency 27.464 ms 2024-03-19T01:42:04.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:42:03 smithi115 ceph-mon[67332]: pgmap v379: 97 pgs: 97 active+clean; 44 MiB data, 579 MiB used, 1.0 TiB / 1.0 TiB avail; 1.2 MiB/s rd, 7.1 MiB/s wr, 387 op/s 2024-03-19T01:42:04.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:42:03 smithi099 ceph-mon[62951]: pgmap v379: 97 pgs: 97 active+clean; 44 MiB data, 579 MiB used, 1.0 TiB / 1.0 TiB avail; 1.2 MiB/s rd, 7.1 MiB/s wr, 387 op/s 2024-03-19T01:42:04.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:42:03 smithi112 ceph-mon[66819]: pgmap v379: 97 pgs: 97 active+clean; 44 MiB data, 579 MiB used, 1.0 TiB / 1.0 TiB avail; 1.2 MiB/s rd, 7.1 MiB/s wr, 387 op/s 2024-03-19T01:42:04.674 INFO:tasks.workunit.client.0.smithi099.stdout: 1 78552 12.99 MB/sec warmup 34 sec latency 14.407 ms 2024-03-19T01:42:05.674 INFO:tasks.workunit.client.0.smithi099.stdout: 1 80962 12.94 MB/sec warmup 35 sec latency 25.361 ms 2024-03-19T01:42:06.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:42:05 smithi115 ceph-mon[67332]: pgmap v380: 97 pgs: 97 active+clean; 44 MiB data, 563 MiB used, 1.0 TiB / 1.0 TiB avail; 1.2 MiB/s rd, 6.8 MiB/s wr, 330 op/s 2024-03-19T01:42:06.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:42:05 smithi099 ceph-mon[62951]: pgmap v380: 97 pgs: 97 active+clean; 44 MiB data, 563 MiB used, 1.0 TiB / 1.0 TiB avail; 1.2 MiB/s rd, 6.8 MiB/s wr, 330 op/s 2024-03-19T01:42:06.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:42:05 smithi112 ceph-mon[66819]: pgmap v380: 97 pgs: 97 active+clean; 44 MiB data, 563 MiB used, 1.0 TiB / 1.0 TiB avail; 1.2 MiB/s rd, 6.8 MiB/s wr, 330 op/s 2024-03-19T01:42:06.674 INFO:tasks.workunit.client.0.smithi099.stdout: 1 82856 12.86 MB/sec warmup 36 sec latency 38.003 ms 2024-03-19T01:42:07.674 INFO:tasks.workunit.client.0.smithi099.stdout: 1 85499 12.96 MB/sec warmup 37 sec latency 12.156 ms 2024-03-19T01:42:08.245 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.244+0000 7f7daec12640 1 -- 172.21.15.99:0/3023343888 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7da81022f0 msgr2=0x7f7da8108b80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:42:08.246 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.244+0000 7f7daec12640 1 --2- 172.21.15.99:0/3023343888 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7da81022f0 0x7f7da8108b80 secure :-1 s=READY pgs=372 cs=0 l=1 rev1=1 crypto rx=0x7f7da0009ca0 tx=0x7f7da002c0f0 comp rx=0 tx=0).stop 2024-03-19T01:42:08.246 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.244+0000 7f7daec12640 1 -- 172.21.15.99:0/3023343888 shutdown_connections 2024-03-19T01:42:08.246 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.244+0000 7f7daec12640 1 --2- 172.21.15.99:0/3023343888 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7da81022f0 0x7f7da8108b80 unknown :-1 s=CLOSED pgs=372 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:08.246 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.244+0000 7f7daec12640 1 --2- 172.21.15.99:0/3023343888 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f7da8101930 0x7f7da8101db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:08.246 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.244+0000 7f7daec12640 1 --2- 172.21.15.99:0/3023343888 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f7da8100740 0x7f7da8100b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:08.246 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.244+0000 7f7daec12640 1 -- 172.21.15.99:0/3023343888 >> 172.21.15.99:0/3023343888 conn(0x7f7da80fbcb0 msgr2=0x7f7da80fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:42:08.246 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.244+0000 7f7daec12640 1 -- 172.21.15.99:0/3023343888 shutdown_connections 2024-03-19T01:42:08.246 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.244+0000 7f7daec12640 1 -- 172.21.15.99:0/3023343888 wait complete. 2024-03-19T01:42:08.246 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.244+0000 7f7daec12640 1 Processor -- start 2024-03-19T01:42:08.246 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.244+0000 7f7daec12640 1 -- start start 2024-03-19T01:42:08.246 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.245+0000 7f7daec12640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f7da8100740 0x7f7da8196920 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:08.246 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.245+0000 7f7daec12640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f7da8101930 0x7f7da8196e60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:08.247 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.245+0000 7f7daec12640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7da81022f0 0x7f7da819ded0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:08.247 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.245+0000 7f7daec12640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f7da810c3a0 con 0x7f7da81022f0 2024-03-19T01:42:08.247 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.245+0000 7f7daec12640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f7da810c220 con 0x7f7da8100740 2024-03-19T01:42:08.247 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.245+0000 7f7daec12640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f7da810c520 con 0x7f7da8101930 2024-03-19T01:42:08.247 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.245+0000 7f7dad1af640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7da81022f0 0x7f7da819ded0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:08.247 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.245+0000 7f7da7fff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f7da8101930 0x7f7da8196e60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:08.247 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.245+0000 7f7dad1af640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7da81022f0 0x7f7da819ded0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:50274/0 (socket says 172.21.15.99:50274) 2024-03-19T01:42:08.247 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.245+0000 7f7dad1af640 1 -- 172.21.15.99:0/2860036957 learned_addr learned my addr 172.21.15.99:0/2860036957 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:42:08.247 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.245+0000 7f7dac9ae640 1 --2- 172.21.15.99:0/2860036957 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f7da8100740 0x7f7da8196920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:08.247 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.245+0000 7f7dad1af640 1 -- 172.21.15.99:0/2860036957 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f7da8101930 msgr2=0x7f7da8196e60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:42:08.247 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.245+0000 7f7dad1af640 1 --2- 172.21.15.99:0/2860036957 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f7da8101930 0x7f7da8196e60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:08.247 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.245+0000 7f7dad1af640 1 -- 172.21.15.99:0/2860036957 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f7da8100740 msgr2=0x7f7da8196920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:42:08.247 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.245+0000 7f7dad1af640 1 --2- 172.21.15.99:0/2860036957 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f7da8100740 0x7f7da8196920 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:08.247 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.245+0000 7f7dad1af640 1 -- 172.21.15.99:0/2860036957 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7da819e540 con 0x7f7da81022f0 2024-03-19T01:42:08.248 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.245+0000 7f7dad1af640 1 --2- 172.21.15.99:0/2860036957 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7da81022f0 0x7f7da819ded0 secure :-1 s=READY pgs=373 cs=0 l=1 rev1=1 crypto rx=0x7f7da0037040 tx=0x7f7da0032f40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:42:08.248 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.245+0000 7f7da5ffb640 1 -- 172.21.15.99:0/2860036957 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7da0041020 con 0x7f7da81022f0 2024-03-19T01:42:08.248 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.245+0000 7f7da5ffb640 1 -- 172.21.15.99:0/2860036957 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7da0034cd0 con 0x7f7da81022f0 2024-03-19T01:42:08.248 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.245+0000 7f7daec12640 1 -- 172.21.15.99:0/2860036957 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f7da819e7d0 con 0x7f7da81022f0 2024-03-19T01:42:08.248 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.245+0000 7f7da5ffb640 1 -- 172.21.15.99:0/2860036957 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7da0039040 con 0x7f7da81022f0 2024-03-19T01:42:08.248 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.246+0000 7f7daec12640 1 -- 172.21.15.99:0/2860036957 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f7da819ecc0 con 0x7f7da81022f0 2024-03-19T01:42:08.254 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.248+0000 7f7da5ffb640 1 -- 172.21.15.99:0/2860036957 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 19) ==== 137341+0+0 (secure 0 0 0) 0x7f7da0044020 con 0x7f7da81022f0 2024-03-19T01:42:08.254 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.248+0000 7f7db1801740 1 Processor -- start 2024-03-19T01:42:08.254 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.248+0000 7f7db1801740 1 -- start start 2024-03-19T01:42:08.254 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.248+0000 7f7db1801740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5649cab6a310 0x5649cab6a730 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:08.255 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.248+0000 7f7db1801740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5649cab6ad00 0x5649cab6f800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:08.255 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.248+0000 7f7db1801740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5649cab6fe70 0x5649cab72300 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:08.255 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.248+0000 7f7da5ffb640 1 --2- 172.21.15.99:0/2860036957 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f7d980a2940 0x7f7d980a4e00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:08.255 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.248+0000 7f7db1801740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x5649caaccbe0 con 0x5649cab6fe70 2024-03-19T01:42:08.255 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.248+0000 7f7db1801740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x5649caa75960 con 0x5649cab6ad00 2024-03-19T01:42:08.255 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.248+0000 7f7db1801740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x5649caac8c80 con 0x5649cab6a310 2024-03-19T01:42:08.255 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.248+0000 7f7dac9ae640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5649cab6ad00 0x5649cab6f800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:08.255 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.248+0000 7f7dad1af640 1 --2- 172.21.15.99:0/2860036957 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f7d980a2940 0x7f7d980a4e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:08.255 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.248+0000 7f7da7fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5649cab6fe70 0x5649cab72300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:08.255 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.248+0000 7f7dac9ae640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5649cab6ad00 0x5649cab6f800 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:39716/0 (socket says 172.21.15.99:39716) 2024-03-19T01:42:08.255 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.248+0000 7f7dac9ae640 1 -- 172.21.15.99:0/939773987 learned_addr learned my addr 172.21.15.99:0/939773987 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:42:08.255 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.249+0000 7f7da7fff640 1 --2- 172.21.15.99:0/939773987 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5649cab6a310 0x5649cab6a730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:08.255 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.249+0000 7f7da5ffb640 1 -- 172.21.15.99:0/2860036957 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f7da00f09c0 con 0x7f7da81022f0 2024-03-19T01:42:08.256 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.249+0000 7f7dac9ae640 1 -- 172.21.15.99:0/939773987 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5649cab6a310 msgr2=0x5649cab6a730 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:42:08.256 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.249+0000 7f7dac9ae640 1 --2- 172.21.15.99:0/939773987 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5649cab6a310 0x5649cab6a730 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:08.256 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.249+0000 7f7dac9ae640 1 -- 172.21.15.99:0/939773987 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5649cab6fe70 msgr2=0x5649cab72300 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:42:08.256 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.249+0000 7f7dac9ae640 1 --2- 172.21.15.99:0/939773987 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5649cab6fe70 0x5649cab72300 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:08.256 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.249+0000 7f7dac9ae640 1 -- 172.21.15.99:0/939773987 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5649caa6cd20 con 0x5649cab6ad00 2024-03-19T01:42:08.256 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.249+0000 7f7da7fff640 1 --2- 172.21.15.99:0/939773987 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5649cab6a310 0x5649cab6a730 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-19T01:42:08.256 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.249+0000 7f7dac9ae640 1 --2- 172.21.15.99:0/939773987 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5649cab6ad00 0x5649cab6f800 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 crypto rx=0x7f7d9c009ca0 tx=0x7f7d9c011810 comp rx=0 tx=0).ready entity=mon.2 client_cookie=425eb4302595f08d server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:42:08.256 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.249+0000 7f7dad1af640 1 --2- 172.21.15.99:0/2860036957 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f7d980a2940 0x7f7d980a4e00 secure :-1 s=READY pgs=252 cs=0 l=1 rev1=1 crypto rx=0x7f7da00880f0 tx=0x7f7da00e7d20 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:42:08.256 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.249+0000 7f7d92ffd640 1 -- 172.21.15.99:0/939773987 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7d9c004220 con 0x5649cab6ad00 2024-03-19T01:42:08.256 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.249+0000 7f7d92ffd640 1 -- 172.21.15.99:0/939773987 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7d9c0043c0 con 0x5649cab6ad00 2024-03-19T01:42:08.256 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.249+0000 7f7d92ffd640 1 -- 172.21.15.99:0/939773987 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7d9c0198d0 con 0x5649cab6ad00 2024-03-19T01:42:08.256 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.250+0000 7f7db1801740 1 -- 172.21.15.99:0/939773987 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5649cab6ad00 msgr2=0x5649cab6f800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:42:08.256 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.250+0000 7f7db1801740 1 --2- 172.21.15.99:0/939773987 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5649cab6ad00 0x5649cab6f800 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 crypto rx=0x7f7d9c009ca0 tx=0x7f7d9c011810 comp rx=0 tx=0).stop 2024-03-19T01:42:08.256 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.250+0000 7f7db1801740 1 -- 172.21.15.99:0/939773987 shutdown_connections 2024-03-19T01:42:08.257 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.250+0000 7f7db1801740 1 --2- 172.21.15.99:0/939773987 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5649cab6ad00 0x5649cab6f800 unknown :-1 s=CLOSED pgs=90 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:08.257 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.250+0000 7f7db1801740 1 --2- 172.21.15.99:0/939773987 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5649cab6fe70 0x5649cab72300 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:08.257 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.250+0000 7f7db1801740 1 --2- 172.21.15.99:0/939773987 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5649cab6a310 0x5649cab6a730 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:08.257 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.250+0000 7f7db1801740 1 -- 172.21.15.99:0/939773987 >> 172.21.15.99:0/939773987 conn(0x5649cab63670 msgr2=0x5649cab63a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:42:08.257 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.250+0000 7f7db1801740 1 -- 172.21.15.99:0/939773987 shutdown_connections 2024-03-19T01:42:08.257 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.250+0000 7f7db1801740 1 -- 172.21.15.99:0/939773987 wait complete. 2024-03-19T01:42:08.257 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.250+0000 7f7db1801740 1 -- start start 2024-03-19T01:42:08.257 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.250+0000 7f7db1801740 1 Processor -- start 2024-03-19T01:42:08.257 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.251+0000 7f7db1801740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5649cab6a310 0x5649cab74bf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:08.257 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.251+0000 7f7db1801740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5649cab6ad00 0x5649cab77140 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:08.257 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.251+0000 7f7db1801740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5649cab6fe70 0x5649cab79690 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:08.257 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.251+0000 7f7db1801740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x5649cab67fe0 con 0x5649cab6a310 2024-03-19T01:42:08.257 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.251+0000 7f7db1801740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x5649caa88db0 con 0x5649cab6fe70 2024-03-19T01:42:08.257 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.251+0000 7f7db1801740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x5649cab68160 con 0x5649cab6ad00 2024-03-19T01:42:08.258 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.251+0000 7f7db1801740 10 client.0 _ll_register_callbacks cb 0x5649caa8de70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:42:08.258 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.251+0000 7f7da7fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5649cab6a310 0x5649cab74bf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:08.258 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.251+0000 7f7dac9ae640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5649cab6ad00 0x5649cab77140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:08.258 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.251+0000 7f7dac9ae640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5649cab6ad00 0x5649cab77140 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:46298/0 (socket says 172.21.15.99:46298) 2024-03-19T01:42:08.258 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.251+0000 7f7dac9ae640 1 -- 172.21.15.99:0/1511858632 learned_addr learned my addr 172.21.15.99:0/1511858632 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:42:08.258 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.251+0000 7f7da7fff640 1 --2- 172.21.15.99:0/1511858632 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5649cab6fe70 0x5649cab79690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:08.258 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.251+0000 7f7dac9ae640 1 -- 172.21.15.99:0/1511858632 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5649cab6fe70 msgr2=0x5649cab79690 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:42:08.258 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.251+0000 7f7dac9ae640 1 --2- 172.21.15.99:0/1511858632 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5649cab6fe70 0x5649cab79690 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:08.258 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.251+0000 7f7dac9ae640 1 -- 172.21.15.99:0/1511858632 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5649cab6a310 msgr2=0x5649cab74bf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:42:08.258 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.251+0000 7f7dac9ae640 1 --2- 172.21.15.99:0/1511858632 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5649cab6a310 0x5649cab74bf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:08.258 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.251+0000 7f7dac9ae640 1 -- 172.21.15.99:0/1511858632 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5649caa87c60 con 0x5649cab6ad00 2024-03-19T01:42:08.258 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.251+0000 7f7da7fff640 1 --2- 172.21.15.99:0/1511858632 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5649cab6fe70 0x5649cab79690 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-19T01:42:08.258 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.251+0000 7f7dac9ae640 1 --2- 172.21.15.99:0/1511858632 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5649cab6ad00 0x5649cab77140 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7f7d9c011d10 tx=0x7f7d9c0038a0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=1d5e1221a079039b server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:42:08.259 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.251+0000 7f7d87fff640 10 client.0 ms_handle_connect on v2:172.21.15.115:3300/0 2024-03-19T01:42:08.259 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.251+0000 7f7d87fff640 1 -- 172.21.15.99:0/1511858632 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7d9c028070 con 0x5649cab6ad00 2024-03-19T01:42:08.259 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.251+0000 7f7db1801740 1 -- 172.21.15.99:0/1511858632 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f7d9c0043c0 con 0x5649cab6ad00 2024-03-19T01:42:08.259 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.251+0000 7f7d87fff640 1 -- 172.21.15.99:0/1511858632 <== mon.1 v2:172.21.15.115:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f7d9c005040 con 0x5649cab6ad00 2024-03-19T01:42:08.259 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.252+0000 7f7d87fff640 1 -- 172.21.15.99:0/1511858632 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_get_version_reply(handle=1 version=23) ==== 24+0+0 (secure 0 0 0) 0x7f7d9c034050 con 0x5649cab6ad00 2024-03-19T01:42:08.259 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.252+0000 7f7db1801740 10 client.25240 fetch_fsmap learned FSMap version 23 2024-03-19T01:42:08.259 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.252+0000 7f7db1801740 1 -- 172.21.15.99:0/1511858632 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,fsmap=23,monmap=4+}) -- 0x5649ca7e1210 con 0x5649cab6ad00 2024-03-19T01:42:08.259 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.252+0000 7f7d87fff640 1 -- 172.21.15.99:0/1511858632 <== mon.1 v2:172.21.15.115:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7d9c0043c0 con 0x5649cab6ad00 2024-03-19T01:42:08.259 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.252+0000 7f7d87fff640 1 -- 172.21.15.99:0/1511858632 <== mon.1 v2:172.21.15.115:3300/0 5 ==== fsmap(e 23) ==== 8346+0+0 (secure 0 0 0) 0x7f7d9c004a70 con 0x5649cab6ad00 2024-03-19T01:42:08.259 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.252+0000 7f7db1801740 10 client.25240 fetch_fsmap finished waiting for FSMap version 23 2024-03-19T01:42:08.259 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.252+0000 7f7db1801740 10 client.25240 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:42:08.259 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.252+0000 7f7db1801740 20 client.25240 populate_metadata read hostname 'smithi099' 2024-03-19T01:42:08.259 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.252+0000 7f7db1801740 1 --2- 172.21.15.99:0/1511858632 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5649cab7dcf0 0x5649cab80170 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:08.259 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.252+0000 7f7db1801740 4 client.25240 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:42:08.260 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.252+0000 7f7db1801740 1 -- 172.21.15.99:0/1511858632 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5649caada120 con 0x5649cab7dcf0 2024-03-19T01:42:08.260 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.252+0000 7f7da7fff640 1 --2- 172.21.15.99:0/1511858632 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5649cab7dcf0 0x5649cab80170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:08.260 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.253+0000 7f7d87fff640 10 client.25240 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:42:08.260 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.253+0000 7f7da7fff640 1 --2- 172.21.15.99:0/1511858632 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5649cab7dcf0 0x5649cab80170 crc :-1 s=READY pgs=37 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=fdeefbe131fd0a4b server_cookie=681362b6011f5d9c in_seq=0 out_seq=0 2024-03-19T01:42:08.260 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.253+0000 7f7d87fff640 1 -- 172.21.15.99:0/1511858632 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x5649caada120 con 0x5649cab7dcf0 2024-03-19T01:42:08.260 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.253+0000 7f7d87fff640 10 client.25240 handle_command_reply: tid=0 2024-03-19T01:42:08.260 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.253+0000 7f7db1801740 1 client.25240 shutdown 2024-03-19T01:42:08.260 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.254+0000 7f7db1801740 1 -- 172.21.15.99:0/1511858632 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5649cab6ad00 msgr2=0x5649cab77140 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:42:08.260 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.254+0000 7f7db1801740 1 --2- 172.21.15.99:0/1511858632 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5649cab6ad00 0x5649cab77140 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7f7d9c011d10 tx=0x7f7d9c0038a0 comp rx=0 tx=0).stop 2024-03-19T01:42:08.260 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.254+0000 7f7db1801740 1 -- 172.21.15.99:0/1511858632 shutdown_connections 2024-03-19T01:42:08.260 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.254+0000 7f7db1801740 1 --2- 172.21.15.99:0/1511858632 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5649cab7dcf0 0x5649cab80170 crc :-1 s=READY pgs=37 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:08.260 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.254+0000 7f7db1801740 1 --2- 172.21.15.99:0/1511858632 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5649cab6fe70 0x5649cab79690 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:08.260 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.254+0000 7f7db1801740 1 --2- 172.21.15.99:0/1511858632 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5649cab6ad00 0x5649cab77140 unknown :-1 s=CLOSED pgs=80 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:08.261 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.254+0000 7f7db1801740 1 --2- 172.21.15.99:0/1511858632 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5649cab6a310 0x5649cab74bf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:08.261 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.254+0000 7f7d87fff640 0 client.25240 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:42:08.261 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.254+0000 7f7db1801740 1 -- 172.21.15.99:0/1511858632 >> 172.21.15.99:0/1511858632 conn(0x5649cab63670 msgr2=0x5649cab685b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:42:08.261 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.254+0000 7f7db1801740 1 -- 172.21.15.99:0/1511858632 shutdown_connections 2024-03-19T01:42:08.261 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.254+0000 7f7db1801740 1 -- 172.21.15.99:0/1511858632 wait complete. 2024-03-19T01:42:08.261 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.254+0000 7f7db1801740 20 client.25240 trim_cache size 0 max 16384 2024-03-19T01:42:08.273 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.269+0000 7f7db1801740 1 Processor -- start 2024-03-19T01:42:08.273 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.269+0000 7f7db1801740 1 -- start start 2024-03-19T01:42:08.273 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.269+0000 7f7db1801740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5649cab6a310 0x5649cab62ab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:08.273 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.269+0000 7f7db1801740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5649cab6ad00 0x5649cab85a80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:08.274 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.269+0000 7f7db1801740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5649cab6fe70 0x5649cab78c10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:08.274 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.269+0000 7f7db1801740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f7da81a9430 con 0x5649cab6a310 2024-03-19T01:42:08.274 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.269+0000 7f7db1801740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f7da81a6d00 con 0x5649cab6ad00 2024-03-19T01:42:08.274 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.269+0000 7f7db1801740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f7da81a4b50 con 0x5649cab6fe70 2024-03-19T01:42:08.274 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.269+0000 7f7da7fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5649cab6a310 0x5649cab62ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:08.274 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.269+0000 7f7da7fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5649cab6a310 0x5649cab62ab0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:50298/0 (socket says 172.21.15.99:50298) 2024-03-19T01:42:08.274 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.269+0000 7f7da7fff640 1 -- 172.21.15.99:0/455417383 learned_addr learned my addr 172.21.15.99:0/455417383 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:42:08.274 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.269+0000 7f7dac9ae640 1 --2- 172.21.15.99:0/455417383 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5649cab6ad00 0x5649cab85a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:08.274 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.269+0000 7f7da7fff640 1 --2- 172.21.15.99:0/455417383 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5649cab6fe70 0x5649cab78c10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:08.274 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.269+0000 7f7da7fff640 1 -- 172.21.15.99:0/455417383 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5649cab6fe70 msgr2=0x5649cab78c10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:42:08.274 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.269+0000 7f7da7fff640 1 --2- 172.21.15.99:0/455417383 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5649cab6fe70 0x5649cab78c10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:08.274 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.269+0000 7f7da7fff640 1 -- 172.21.15.99:0/455417383 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5649cab6ad00 msgr2=0x5649cab85a80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:42:08.274 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.269+0000 7f7da7fff640 1 --2- 172.21.15.99:0/455417383 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5649cab6ad00 0x5649cab85a80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:08.275 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.269+0000 7f7da7fff640 1 -- 172.21.15.99:0/455417383 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5649ca8ad620 con 0x5649cab6a310 2024-03-19T01:42:08.275 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.270+0000 7f7da7fff640 1 --2- 172.21.15.99:0/455417383 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5649cab6a310 0x5649cab62ab0 secure :-1 s=READY pgs=374 cs=0 l=1 rev1=1 crypto rx=0x5649cab80a20 tx=0x7f7d94004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=883b62ca44ba43ac server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:42:08.275 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.270+0000 7f7d927fc640 1 -- 172.21.15.99:0/455417383 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7d94026d50 con 0x5649cab6a310 2024-03-19T01:42:08.275 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.270+0000 7f7d927fc640 1 -- 172.21.15.99:0/455417383 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7d94026310 con 0x5649cab6a310 2024-03-19T01:42:08.275 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.270+0000 7f7d927fc640 1 -- 172.21.15.99:0/455417383 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7d94005210 con 0x5649cab6a310 2024-03-19T01:42:08.275 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.270+0000 7f7db1801740 1 -- 172.21.15.99:0/455417383 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5649cab6a310 msgr2=0x5649cab62ab0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:42:08.275 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.270+0000 7f7db1801740 1 --2- 172.21.15.99:0/455417383 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5649cab6a310 0x5649cab62ab0 secure :-1 s=READY pgs=374 cs=0 l=1 rev1=1 crypto rx=0x5649cab80a20 tx=0x7f7d94004730 comp rx=0 tx=0).stop 2024-03-19T01:42:08.275 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.270+0000 7f7db1801740 1 -- 172.21.15.99:0/455417383 shutdown_connections 2024-03-19T01:42:08.275 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.270+0000 7f7db1801740 1 --2- 172.21.15.99:0/455417383 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5649cab6ad00 0x5649cab85a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:08.275 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.270+0000 7f7db1801740 1 --2- 172.21.15.99:0/455417383 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5649cab6fe70 0x5649cab78c10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:08.275 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.270+0000 7f7db1801740 1 --2- 172.21.15.99:0/455417383 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5649cab6a310 0x5649cab62ab0 unknown :-1 s=CLOSED pgs=374 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:08.275 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.270+0000 7f7db1801740 1 -- 172.21.15.99:0/455417383 >> 172.21.15.99:0/455417383 conn(0x5649cab63670 msgr2=0x5649cab63a50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:42:08.275 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.270+0000 7f7db1801740 1 -- 172.21.15.99:0/455417383 shutdown_connections 2024-03-19T01:42:08.275 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.270+0000 7f7db1801740 1 -- 172.21.15.99:0/455417383 wait complete. 2024-03-19T01:42:08.275 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.271+0000 7f7db1801740 1 -- start start 2024-03-19T01:42:08.276 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.271+0000 7f7db1801740 1 Processor -- start 2024-03-19T01:42:08.276 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.271+0000 7f7db1801740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5649cab6a310 0x5649cab77120 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:08.276 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.271+0000 7f7db1801740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5649cab6ad00 0x5649cab77660 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:08.276 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.271+0000 7f7db1801740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5649cab6fe70 0x5649cab78670 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:08.276 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.271+0000 7f7db1801740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f7da81a51c0 con 0x5649cab6a310 2024-03-19T01:42:08.276 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.271+0000 7f7db1801740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f7da81a6630 con 0x5649cab6ad00 2024-03-19T01:42:08.276 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.271+0000 7f7db1801740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f7da81a5890 con 0x5649cab6fe70 2024-03-19T01:42:08.276 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.271+0000 7f7db1801740 10 client.0 _ll_register_callbacks cb 0x7f7d94009a10 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:42:08.276 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.271+0000 7f7da7fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5649cab6a310 0x5649cab77120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:08.276 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.271+0000 7f7da7fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5649cab6a310 0x5649cab77120 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:50310/0 (socket says 172.21.15.99:50310) 2024-03-19T01:42:08.276 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.271+0000 7f7da7fff640 1 -- 172.21.15.99:0/3418932837 learned_addr learned my addr 172.21.15.99:0/3418932837 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:42:08.276 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.271+0000 7f7dac9ae640 1 --2- 172.21.15.99:0/3418932837 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5649cab6ad00 0x5649cab77660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:08.276 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.271+0000 7f7da7fff640 1 --2- 172.21.15.99:0/3418932837 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5649cab6fe70 0x5649cab78670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:08.276 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.271+0000 7f7da7fff640 1 -- 172.21.15.99:0/3418932837 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5649cab6fe70 msgr2=0x5649cab78670 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:42:08.277 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.271+0000 7f7da7fff640 1 --2- 172.21.15.99:0/3418932837 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5649cab6fe70 0x5649cab78670 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:08.277 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.271+0000 7f7da7fff640 1 -- 172.21.15.99:0/3418932837 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5649cab6ad00 msgr2=0x5649cab77660 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:42:08.277 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.271+0000 7f7da7fff640 1 --2- 172.21.15.99:0/3418932837 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5649cab6ad00 0x5649cab77660 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:08.277 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.271+0000 7f7da7fff640 1 -- 172.21.15.99:0/3418932837 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5649cab78c60 con 0x5649cab6a310 2024-03-19T01:42:08.277 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.271+0000 7f7dac9ae640 1 --2- 172.21.15.99:0/3418932837 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5649cab6ad00 0x5649cab77660 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:42:08.277 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.272+0000 7f7da7fff640 1 --2- 172.21.15.99:0/3418932837 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5649cab6a310 0x5649cab77120 secure :-1 s=READY pgs=375 cs=0 l=1 rev1=1 crypto rx=0x7f7d9401e040 tx=0x7f7d94009de0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c8b46ca2d7c1ecd5 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:42:08.277 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.272+0000 7f7d87fff640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:42:08.277 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.272+0000 7f7d87fff640 1 -- 172.21.15.99:0/3418932837 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7d94038060 con 0x5649cab6a310 2024-03-19T01:42:08.277 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.272+0000 7f7db1801740 1 -- 172.21.15.99:0/3418932837 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f7d94026310 con 0x5649cab6a310 2024-03-19T01:42:08.277 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.272+0000 7f7d87fff640 1 -- 172.21.15.99:0/3418932837 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f7d94025040 con 0x5649cab6a310 2024-03-19T01:42:08.277 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.272+0000 7f7d87fff640 1 -- 172.21.15.99:0/3418932837 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=23) ==== 24+0+0 (secure 0 0 0) 0x7f7d9401ba30 con 0x5649cab6a310 2024-03-19T01:42:08.277 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.272+0000 7f7db1801740 10 client.15420 fetch_fsmap learned FSMap version 23 2024-03-19T01:42:08.277 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.272+0000 7f7db1801740 1 -- 172.21.15.99:0/3418932837 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=23,monmap=4+}) -- 0x5649cabe43d0 con 0x5649cab6a310 2024-03-19T01:42:08.277 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.272+0000 7f7d87fff640 1 -- 172.21.15.99:0/3418932837 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7d94026310 con 0x5649cab6a310 2024-03-19T01:42:08.278 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.272+0000 7f7d87fff640 1 -- 172.21.15.99:0/3418932837 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 23) ==== 8346+0+0 (secure 0 0 0) 0x7f7d94004960 con 0x5649cab6a310 2024-03-19T01:42:08.278 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.272+0000 7f7db1801740 10 client.15420 fetch_fsmap finished waiting for FSMap version 23 2024-03-19T01:42:08.278 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.272+0000 7f7db1801740 10 client.15420 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:42:08.278 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.272+0000 7f7db1801740 20 client.15420 populate_metadata read hostname 'smithi099' 2024-03-19T01:42:08.278 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.272+0000 7f7db1801740 1 --2- 172.21.15.99:0/3418932837 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5649cabe4d50 0x5649cabe7170 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:08.278 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.272+0000 7f7db1801740 4 client.15420 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:42:08.278 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.272+0000 7f7db1801740 1 -- 172.21.15.99:0/3418932837 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x5649caad4470 con 0x5649cabe4d50 2024-03-19T01:42:08.278 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.272+0000 7f7dac9ae640 1 --2- 172.21.15.99:0/3418932837 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5649cabe4d50 0x5649cabe7170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:08.278 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.273+0000 7f7d87fff640 10 client.15420 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:42:08.278 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.273+0000 7f7dac9ae640 1 --2- 172.21.15.99:0/3418932837 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5649cabe4d50 0x5649cabe7170 crc :-1 s=READY pgs=38 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=56a5942246abd8f3 server_cookie=142d3e43fbf12201 in_seq=0 out_seq=0 2024-03-19T01:42:08.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:42:07 smithi115 ceph-mon[67332]: pgmap v381: 97 pgs: 97 active+clean; 48 MiB data, 582 MiB used, 1.0 TiB / 1.0 TiB avail; 1.8 MiB/s rd, 7.1 MiB/s wr, 376 op/s 2024-03-19T01:42:08.321 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.320+0000 7f7d87fff640 1 -- 172.21.15.99:0/3418932837 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x5649caad4470 con 0x5649cabe4d50 2024-03-19T01:42:08.321 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.320+0000 7f7d87fff640 10 client.15420 handle_command_reply: tid=0 2024-03-19T01:42:08.321 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.320+0000 7f7db1801740 1 client.15420 shutdown 2024-03-19T01:42:08.321 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.321+0000 7f7db1801740 1 -- 172.21.15.99:0/3418932837 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5649cab6a310 msgr2=0x5649cab77120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:42:08.321 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.321+0000 7f7db1801740 1 --2- 172.21.15.99:0/3418932837 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5649cab6a310 0x5649cab77120 secure :-1 s=READY pgs=375 cs=0 l=1 rev1=1 crypto rx=0x7f7d9401e040 tx=0x7f7d94009de0 comp rx=0 tx=0).stop 2024-03-19T01:42:08.321 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.321+0000 7f7db1801740 1 -- 172.21.15.99:0/3418932837 shutdown_connections 2024-03-19T01:42:08.322 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.321+0000 7f7db1801740 1 --2- 172.21.15.99:0/3418932837 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5649cabe4d50 0x5649cabe7170 crc :-1 s=READY pgs=38 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:08.322 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.321+0000 7f7db1801740 1 --2- 172.21.15.99:0/3418932837 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5649cab6ad00 0x5649cab77660 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:08.322 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.321+0000 7f7d87fff640 0 client.15420 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:42:08.322 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.321+0000 7f7db1801740 1 --2- 172.21.15.99:0/3418932837 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5649cab6fe70 0x5649cab78670 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:08.322 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.321+0000 7f7db1801740 1 --2- 172.21.15.99:0/3418932837 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5649cab6a310 0x5649cab77120 unknown :-1 s=CLOSED pgs=375 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:08.322 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.321+0000 7f7db1801740 1 -- 172.21.15.99:0/3418932837 >> 172.21.15.99:0/3418932837 conn(0x5649cab63670 msgr2=0x5649cab84a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:42:08.322 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.321+0000 7f7db1801740 1 -- 172.21.15.99:0/3418932837 shutdown_connections 2024-03-19T01:42:08.322 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.321+0000 7f7db1801740 1 -- 172.21.15.99:0/3418932837 wait complete. 2024-03-19T01:42:08.322 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.321+0000 7f7db1801740 20 client.15420 trim_cache size 0 max 16384 2024-03-19T01:42:08.322 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:42:08.323 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:42:08.323 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:42:08.323 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:42:08.323 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.321+0000 7f7daec12640 1 -- 172.21.15.99:0/2860036957 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f7d980a2940 msgr2=0x7f7d980a4e00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:42:08.323 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.321+0000 7f7daec12640 1 --2- 172.21.15.99:0/2860036957 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f7d980a2940 0x7f7d980a4e00 secure :-1 s=READY pgs=252 cs=0 l=1 rev1=1 crypto rx=0x7f7da00880f0 tx=0x7f7da00e7d20 comp rx=0 tx=0).stop 2024-03-19T01:42:08.323 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.321+0000 7f7daec12640 1 -- 172.21.15.99:0/2860036957 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7da81022f0 msgr2=0x7f7da819ded0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:42:08.324 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.321+0000 7f7daec12640 1 --2- 172.21.15.99:0/2860036957 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7da81022f0 0x7f7da819ded0 secure :-1 s=READY pgs=373 cs=0 l=1 rev1=1 crypto rx=0x7f7da0037040 tx=0x7f7da0032f40 comp rx=0 tx=0).stop 2024-03-19T01:42:08.324 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.321+0000 7f7daec12640 1 -- 172.21.15.99:0/2860036957 shutdown_connections 2024-03-19T01:42:08.324 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.321+0000 7f7daec12640 1 --2- 172.21.15.99:0/2860036957 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7da81022f0 0x7f7da819ded0 unknown :-1 s=CLOSED pgs=373 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:08.324 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.322+0000 7f7daec12640 1 --2- 172.21.15.99:0/2860036957 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f7da8101930 0x7f7da8196e60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:08.324 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.322+0000 7f7daec12640 1 --2- 172.21.15.99:0/2860036957 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f7d980a2940 0x7f7d980a4e00 unknown :-1 s=CLOSED pgs=252 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:08.324 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.322+0000 7f7daec12640 1 --2- 172.21.15.99:0/2860036957 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f7da8100740 0x7f7da8196920 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:08.324 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.322+0000 7f7daec12640 1 -- 172.21.15.99:0/2860036957 >> 172.21.15.99:0/2860036957 conn(0x7f7da80fbcb0 msgr2=0x7f7da8104bd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:42:08.324 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.322+0000 7f7daec12640 1 -- 172.21.15.99:0/2860036957 shutdown_connections 2024-03-19T01:42:08.324 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:08.322+0000 7f7daec12640 1 -- 172.21.15.99:0/2860036957 wait complete. 2024-03-19T01:42:08.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:42:07 smithi099 ceph-mon[62951]: pgmap v381: 97 pgs: 97 active+clean; 48 MiB data, 582 MiB used, 1.0 TiB / 1.0 TiB avail; 1.8 MiB/s rd, 7.1 MiB/s wr, 376 op/s 2024-03-19T01:42:08.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:42:07 smithi112 ceph-mon[66819]: pgmap v381: 97 pgs: 97 active+clean; 48 MiB data, 582 MiB used, 1.0 TiB / 1.0 TiB avail; 1.8 MiB/s rd, 7.1 MiB/s wr, 376 op/s 2024-03-19T01:42:08.674 INFO:tasks.workunit.client.0.smithi099.stdout: 1 87561 12.81 MB/sec warmup 38 sec latency 32.767 ms 2024-03-19T01:42:09.674 INFO:tasks.workunit.client.0.smithi099.stdout: 1 89480 12.81 MB/sec warmup 39 sec latency 38.694 ms 2024-03-19T01:42:10.142 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:42:09 smithi099 ceph-mon[62951]: pgmap v382: 97 pgs: 97 active+clean; 51 MiB data, 595 MiB used, 1.0 TiB / 1.0 TiB avail; 2.4 MiB/s rd, 7.1 MiB/s wr, 380 op/s 2024-03-19T01:42:10.142 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:42:09 smithi099 ceph-mon[62951]: mgrmap e20: x(active, since 10m), standbys: z, y 2024-03-19T01:42:10.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:42:09 smithi115 ceph-mon[67332]: pgmap v382: 97 pgs: 97 active+clean; 51 MiB data, 595 MiB used, 1.0 TiB / 1.0 TiB avail; 2.4 MiB/s rd, 7.1 MiB/s wr, 380 op/s 2024-03-19T01:42:10.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:42:09 smithi115 ceph-mon[67332]: mgrmap e20: x(active, since 10m), standbys: z, y 2024-03-19T01:42:10.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:42:09 smithi112 ceph-mon[66819]: pgmap v382: 97 pgs: 97 active+clean; 51 MiB data, 595 MiB used, 1.0 TiB / 1.0 TiB avail; 2.4 MiB/s rd, 7.1 MiB/s wr, 380 op/s 2024-03-19T01:42:10.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:42:09 smithi112 ceph-mon[66819]: mgrmap e20: x(active, since 10m), standbys: z, y 2024-03-19T01:42:10.674 INFO:tasks.workunit.client.0.smithi099.stdout: 1 92063 12.84 MB/sec warmup 40 sec latency 28.148 ms 2024-03-19T01:42:11.674 INFO:tasks.workunit.client.0.smithi099.stdout: 1 93788 12.76 MB/sec warmup 41 sec latency 39.512 ms 2024-03-19T01:42:12.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:42:11 smithi115 ceph-mon[67332]: pgmap v383: 97 pgs: 97 active+clean; 48 MiB data, 596 MiB used, 1.0 TiB / 1.0 TiB avail; 2.4 MiB/s rd, 4.9 MiB/s wr, 303 op/s 2024-03-19T01:42:12.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:42:11 smithi099 ceph-mon[62951]: pgmap v383: 97 pgs: 97 active+clean; 48 MiB data, 596 MiB used, 1.0 TiB / 1.0 TiB avail; 2.4 MiB/s rd, 4.9 MiB/s wr, 303 op/s 2024-03-19T01:42:12.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:42:11 smithi112 ceph-mon[66819]: pgmap v383: 97 pgs: 97 active+clean; 48 MiB data, 596 MiB used, 1.0 TiB / 1.0 TiB avail; 2.4 MiB/s rd, 4.9 MiB/s wr, 303 op/s 2024-03-19T01:42:12.674 INFO:tasks.workunit.client.0.smithi099.stdout: 1 96197 12.79 MB/sec warmup 42 sec latency 49.601 ms 2024-03-19T01:42:13.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:42:12 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:42:13.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:42:12 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:42:13.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:42:12 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:42:13.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:42:12 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:42:13.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:42:12 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:42:13.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:42:12 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:42:13.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:42:12 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:42:13.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:42:12 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:42:13.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:42:12 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:42:13.674 INFO:tasks.workunit.client.0.smithi099.stdout: 1 98267 12.66 MB/sec warmup 43 sec latency 28.474 ms 2024-03-19T01:42:14.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:42:13 smithi115 ceph-mon[67332]: pgmap v384: 97 pgs: 97 active+clean; 49 MiB data, 586 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 MiB/s rd, 2.8 MiB/s wr, 263 op/s 2024-03-19T01:42:14.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:42:13 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:42:14.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:42:13 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:42:14.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:42:13 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:42:14.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:42:13 smithi099 ceph-mon[62951]: pgmap v384: 97 pgs: 97 active+clean; 49 MiB data, 586 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 MiB/s rd, 2.8 MiB/s wr, 263 op/s 2024-03-19T01:42:14.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:42:13 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:42:14.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:42:13 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:42:14.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:42:13 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:42:14.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:42:13 smithi112 ceph-mon[66819]: pgmap v384: 97 pgs: 97 active+clean; 49 MiB data, 586 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 MiB/s rd, 2.8 MiB/s wr, 263 op/s 2024-03-19T01:42:14.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:42:13 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:42:14.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:42:13 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:42:14.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:42:13 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:42:14.674 INFO:tasks.workunit.client.0.smithi099.stdout: 1 100274 12.66 MB/sec warmup 44 sec latency 25.279 ms 2024-03-19T01:42:15.674 INFO:tasks.workunit.client.0.smithi099.stdout: 1 103029 12.76 MB/sec warmup 45 sec latency 15.831 ms 2024-03-19T01:42:16.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:42:15 smithi115 ceph-mon[67332]: pgmap v385: 97 pgs: 97 active+clean; 51 MiB data, 590 MiB used, 1.0 TiB / 1.0 TiB avail; 2.8 MiB/s rd, 1.2 MiB/s wr, 211 op/s 2024-03-19T01:42:16.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:42:15 smithi099 ceph-mon[62951]: pgmap v385: 97 pgs: 97 active+clean; 51 MiB data, 590 MiB used, 1.0 TiB / 1.0 TiB avail; 2.8 MiB/s rd, 1.2 MiB/s wr, 211 op/s 2024-03-19T01:42:16.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:42:15 smithi112 ceph-mon[66819]: pgmap v385: 97 pgs: 97 active+clean; 51 MiB data, 590 MiB used, 1.0 TiB / 1.0 TiB avail; 2.8 MiB/s rd, 1.2 MiB/s wr, 211 op/s 2024-03-19T01:42:16.674 INFO:tasks.workunit.client.0.smithi099.stdout: 1 104756 12.63 MB/sec warmup 46 sec latency 51.230 ms 2024-03-19T01:42:17.674 INFO:tasks.workunit.client.0.smithi099.stdout: 1 107006 12.65 MB/sec warmup 47 sec latency 53.646 ms 2024-03-19T01:42:18.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:42:17 smithi115 ceph-mon[67332]: pgmap v386: 97 pgs: 97 active+clean; 52 MiB data, 608 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 MiB/s rd, 1.3 MiB/s wr, 219 op/s 2024-03-19T01:42:18.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:42:17 smithi099 ceph-mon[62951]: pgmap v386: 97 pgs: 97 active+clean; 52 MiB data, 608 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 MiB/s rd, 1.3 MiB/s wr, 219 op/s 2024-03-19T01:42:18.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:42:17 smithi112 ceph-mon[66819]: pgmap v386: 97 pgs: 97 active+clean; 52 MiB data, 608 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 MiB/s rd, 1.3 MiB/s wr, 219 op/s 2024-03-19T01:42:18.674 INFO:tasks.workunit.client.0.smithi099.stdout: 1 109376 12.65 MB/sec warmup 48 sec latency 28.190 ms 2024-03-19T01:42:19.674 INFO:tasks.workunit.client.0.smithi099.stdout: 1 111238 12.57 MB/sec warmup 49 sec latency 28.367 ms 2024-03-19T01:42:20.163 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:42:19 smithi112 ceph-mon[66819]: pgmap v387: 97 pgs: 97 active+clean; 56 MiB data, 614 MiB used, 1.0 TiB / 1.0 TiB avail; 3.0 MiB/s rd, 1.3 MiB/s wr, 226 op/s 2024-03-19T01:42:20.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:42:19 smithi115 ceph-mon[67332]: pgmap v387: 97 pgs: 97 active+clean; 56 MiB data, 614 MiB used, 1.0 TiB / 1.0 TiB avail; 3.0 MiB/s rd, 1.3 MiB/s wr, 226 op/s 2024-03-19T01:42:20.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:42:19 smithi099 ceph-mon[62951]: pgmap v387: 97 pgs: 97 active+clean; 56 MiB data, 614 MiB used, 1.0 TiB / 1.0 TiB avail; 3.0 MiB/s rd, 1.3 MiB/s wr, 226 op/s 2024-03-19T01:42:20.674 INFO:tasks.workunit.client.0.smithi099.stdout: 1 113904 12.66 MB/sec warmup 50 sec latency 14.499 ms 2024-03-19T01:42:21.416 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.413+0000 7f2476a4b640 1 -- 172.21.15.99:0/4251032984 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2470100740 msgr2=0x7f2470100b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:42:21.416 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.413+0000 7f2476a4b640 1 --2- 172.21.15.99:0/4251032984 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2470100740 0x7f2470100b60 secure :-1 s=READY pgs=376 cs=0 l=1 rev1=1 crypto rx=0x7f2460009900 tx=0x7f246002bc30 comp rx=0 tx=0).stop 2024-03-19T01:42:21.416 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.414+0000 7f2476a4b640 1 -- 172.21.15.99:0/4251032984 shutdown_connections 2024-03-19T01:42:21.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.414+0000 7f2476a4b640 1 --2- 172.21.15.99:0/4251032984 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f24701022f0 0x7f2470108b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:21.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.414+0000 7f2476a4b640 1 --2- 172.21.15.99:0/4251032984 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f2470101930 0x7f2470101db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:21.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.414+0000 7f2476a4b640 1 --2- 172.21.15.99:0/4251032984 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2470100740 0x7f2470100b60 unknown :-1 s=CLOSED pgs=376 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:21.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.414+0000 7f2476a4b640 1 -- 172.21.15.99:0/4251032984 >> 172.21.15.99:0/4251032984 conn(0x7f24700fbcb0 msgr2=0x7f24700fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:42:21.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.414+0000 7f2476a4b640 1 -- 172.21.15.99:0/4251032984 shutdown_connections 2024-03-19T01:42:21.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.414+0000 7f2476a4b640 1 -- 172.21.15.99:0/4251032984 wait complete. 2024-03-19T01:42:21.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.414+0000 7f2476a4b640 1 Processor -- start 2024-03-19T01:42:21.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.414+0000 7f2476a4b640 1 -- start start 2024-03-19T01:42:21.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.414+0000 7f2476a4b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2470100740 0x7f2470197b00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:21.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.414+0000 7f2476a4b640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f2470101930 0x7f2470198040 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:21.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.414+0000 7f2476a4b640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f24701022f0 0x7f247019f0b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:21.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.414+0000 7f2476a4b640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f247010c390 con 0x7f2470100740 2024-03-19T01:42:21.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.414+0000 7f2476a4b640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f247010c210 con 0x7f2470101930 2024-03-19T01:42:21.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.414+0000 7f2476a4b640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f247010c510 con 0x7f24701022f0 2024-03-19T01:42:21.418 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.415+0000 7f246ffff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2470100740 0x7f2470197b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:21.418 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.415+0000 7f246ffff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2470100740 0x7f2470197b00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:35468/0 (socket says 172.21.15.99:35468) 2024-03-19T01:42:21.418 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.415+0000 7f246ffff640 1 -- 172.21.15.99:0/960673325 learned_addr learned my addr 172.21.15.99:0/960673325 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:42:21.418 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.415+0000 7f2474fe8640 1 --2- 172.21.15.99:0/960673325 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f24701022f0 0x7f247019f0b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:21.418 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.415+0000 7f246f7fe640 1 --2- 172.21.15.99:0/960673325 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f2470101930 0x7f2470198040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:21.418 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.415+0000 7f246ffff640 1 -- 172.21.15.99:0/960673325 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f24701022f0 msgr2=0x7f247019f0b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:42:21.418 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.415+0000 7f246ffff640 1 --2- 172.21.15.99:0/960673325 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f24701022f0 0x7f247019f0b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:21.418 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.415+0000 7f246ffff640 1 -- 172.21.15.99:0/960673325 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f2470101930 msgr2=0x7f2470198040 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:42:21.418 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.415+0000 7f246ffff640 1 --2- 172.21.15.99:0/960673325 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f2470101930 0x7f2470198040 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:21.418 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.415+0000 7f246ffff640 1 -- 172.21.15.99:0/960673325 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f247019f720 con 0x7f2470100740 2024-03-19T01:42:21.419 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.415+0000 7f246ffff640 1 --2- 172.21.15.99:0/960673325 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2470100740 0x7f2470197b00 secure :-1 s=READY pgs=377 cs=0 l=1 rev1=1 crypto rx=0x7f2460000c00 tx=0x7f246002c640 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:42:21.419 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.415+0000 7f246d7fa640 1 -- 172.21.15.99:0/960673325 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2460041070 con 0x7f2470100740 2024-03-19T01:42:21.419 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.415+0000 7f246d7fa640 1 -- 172.21.15.99:0/960673325 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f2460004450 con 0x7f2470100740 2024-03-19T01:42:21.419 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.415+0000 7f2476a4b640 1 -- 172.21.15.99:0/960673325 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f247019f9b0 con 0x7f2470100740 2024-03-19T01:42:21.419 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.415+0000 7f246d7fa640 1 -- 172.21.15.99:0/960673325 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2460034440 con 0x7f2470100740 2024-03-19T01:42:21.419 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.415+0000 7f2476a4b640 1 -- 172.21.15.99:0/960673325 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f247019fea0 con 0x7f2470100740 2024-03-19T01:42:21.427 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.418+0000 7f2479636740 1 Processor -- start 2024-03-19T01:42:21.427 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.418+0000 7f246d7fa640 1 -- 172.21.15.99:0/960673325 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f2460044070 con 0x7f2470100740 2024-03-19T01:42:21.427 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.418+0000 7f2479636740 1 -- start start 2024-03-19T01:42:21.427 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.418+0000 7f2479636740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557c57e93100 0x557c57e93520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:21.427 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.418+0000 7f2479636740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557c57e93af0 0x557c57e985f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:21.428 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.418+0000 7f2479636740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557c57e98c60 0x557c57e9b0f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:21.428 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.418+0000 7f2479636740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x557c57df7be0 con 0x557c57e93100 2024-03-19T01:42:21.428 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.418+0000 7f2479636740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x557c57da0960 con 0x557c57e93af0 2024-03-19T01:42:21.428 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.418+0000 7f2479636740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x557c57df3c80 con 0x557c57e98c60 2024-03-19T01:42:21.428 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.418+0000 7f2474fe8640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557c57e93100 0x557c57e93520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:21.428 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.418+0000 7f2474fe8640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557c57e93100 0x557c57e93520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:35474/0 (socket says 172.21.15.99:35474) 2024-03-19T01:42:21.428 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.418+0000 7f2474fe8640 1 -- 172.21.15.99:0/243435739 learned_addr learned my addr 172.21.15.99:0/243435739 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:42:21.428 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.418+0000 7f246ffff640 1 --2- 172.21.15.99:0/243435739 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557c57e93af0 0x557c57e985f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:21.428 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.418+0000 7f246f7fe640 1 --2- 172.21.15.99:0/243435739 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557c57e98c60 0x557c57e9b0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:21.428 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.419+0000 7f246f7fe640 1 -- 172.21.15.99:0/243435739 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557c57e93af0 msgr2=0x557c57e985f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:42:21.428 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.419+0000 7f246f7fe640 1 --2- 172.21.15.99:0/243435739 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557c57e93af0 0x557c57e985f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:21.428 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.419+0000 7f246f7fe640 1 -- 172.21.15.99:0/243435739 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557c57e93100 msgr2=0x557c57e93520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:42:21.429 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.419+0000 7f246f7fe640 1 --2- 172.21.15.99:0/243435739 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557c57e93100 0x557c57e93520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:21.429 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.419+0000 7f246f7fe640 1 -- 172.21.15.99:0/243435739 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x557c57d97d20 con 0x557c57e98c60 2024-03-19T01:42:21.429 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.419+0000 7f2474fe8640 1 --2- 172.21.15.99:0/243435739 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557c57e93100 0x557c57e93520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:42:21.429 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.419+0000 7f246d7fa640 1 --2- 172.21.15.99:0/960673325 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f24580a28a0 0x7f24580a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:21.429 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.419+0000 7f246f7fe640 1 --2- 172.21.15.99:0/243435739 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557c57e98c60 0x557c57e9b0f0 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 crypto rx=0x7f2464009bf0 tx=0x7f2464011810 comp rx=0 tx=0).ready entity=mon.1 client_cookie=1dec5230145ef3de server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:42:21.429 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.419+0000 7f2474fe8640 1 --2- 172.21.15.99:0/960673325 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f24580a28a0 0x7f24580a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:21.429 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.419+0000 7f24567fc640 1 -- 172.21.15.99:0/243435739 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2464004050 con 0x557c57e98c60 2024-03-19T01:42:21.429 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.419+0000 7f24567fc640 1 -- 172.21.15.99:0/243435739 <== mon.1 v2:172.21.15.115:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f2464005500 con 0x557c57e98c60 2024-03-19T01:42:21.429 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.419+0000 7f24567fc640 1 -- 172.21.15.99:0/243435739 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2464004e40 con 0x557c57e98c60 2024-03-19T01:42:21.429 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.419+0000 7f246d7fa640 1 -- 172.21.15.99:0/960673325 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f24600f0560 con 0x7f2470100740 2024-03-19T01:42:21.429 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.420+0000 7f2479636740 1 -- 172.21.15.99:0/243435739 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557c57e98c60 msgr2=0x557c57e9b0f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:42:21.429 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.420+0000 7f2479636740 1 --2- 172.21.15.99:0/243435739 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557c57e98c60 0x557c57e9b0f0 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 crypto rx=0x7f2464009bf0 tx=0x7f2464011810 comp rx=0 tx=0).stop 2024-03-19T01:42:21.430 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.420+0000 7f2474fe8640 1 --2- 172.21.15.99:0/960673325 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f24580a28a0 0x7f24580a4d60 secure :-1 s=READY pgs=253 cs=0 l=1 rev1=1 crypto rx=0x557c57e94350 tx=0x7f2468009500 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:42:21.430 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.420+0000 7f2479636740 1 -- 172.21.15.99:0/243435739 shutdown_connections 2024-03-19T01:42:21.430 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.420+0000 7f2479636740 1 --2- 172.21.15.99:0/243435739 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557c57e93af0 0x557c57e985f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:21.430 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.420+0000 7f2479636740 1 --2- 172.21.15.99:0/243435739 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557c57e98c60 0x557c57e9b0f0 unknown :-1 s=CLOSED pgs=81 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:21.430 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.420+0000 7f2479636740 1 --2- 172.21.15.99:0/243435739 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557c57e93100 0x557c57e93520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:21.430 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.420+0000 7f2479636740 1 -- 172.21.15.99:0/243435739 >> 172.21.15.99:0/243435739 conn(0x557c57e8e670 msgr2=0x557c57e8ea70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:42:21.430 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.420+0000 7f2479636740 1 -- 172.21.15.99:0/243435739 shutdown_connections 2024-03-19T01:42:21.430 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.420+0000 7f2479636740 1 -- 172.21.15.99:0/243435739 wait complete. 2024-03-19T01:42:21.430 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.421+0000 7f2479636740 1 -- start start 2024-03-19T01:42:21.430 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.421+0000 7f2479636740 1 Processor -- start 2024-03-19T01:42:21.430 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.421+0000 7f2479636740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557c57e93100 0x557c57e9d0b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:21.430 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.421+0000 7f2479636740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557c57e93af0 0x557c57e9d5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:21.431 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.421+0000 7f2479636740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557c57e98c60 0x557c57ea4630 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:21.431 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.421+0000 7f2479636740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x557c57e9bc10 con 0x557c57e93af0 2024-03-19T01:42:21.431 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.421+0000 7f2479636740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x557c57db3db0 con 0x557c57e93100 2024-03-19T01:42:21.431 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.421+0000 7f2479636740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x557c57e9bd90 con 0x557c57e98c60 2024-03-19T01:42:21.431 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.421+0000 7f2479636740 10 client.0 _ll_register_callbacks cb 0x557c57db8e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:42:21.431 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.421+0000 7f246f7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557c57e93af0 0x557c57e9d5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:21.431 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.421+0000 7f246ffff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557c57e93100 0x557c57e9d0b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:21.431 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.421+0000 7f2474fe8640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557c57e98c60 0x557c57ea4630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:21.431 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.421+0000 7f246ffff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557c57e93100 0x557c57e9d0b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:37186/0 (socket says 172.21.15.99:37186) 2024-03-19T01:42:21.431 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.421+0000 7f246ffff640 1 -- 172.21.15.99:0/2674209203 learned_addr learned my addr 172.21.15.99:0/2674209203 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:42:21.431 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.421+0000 7f246f7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557c57e93af0 0x557c57e9d5f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:35482/0 (socket says 172.21.15.99:35482) 2024-03-19T01:42:21.431 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.422+0000 7f246ffff640 1 -- 172.21.15.99:0/2674209203 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557c57e98c60 msgr2=0x557c57ea4630 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:42:21.431 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.422+0000 7f246ffff640 1 --2- 172.21.15.99:0/2674209203 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557c57e98c60 0x557c57ea4630 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:21.432 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.422+0000 7f246ffff640 1 -- 172.21.15.99:0/2674209203 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557c57e93af0 msgr2=0x557c57e9d5f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:42:21.432 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.422+0000 7f246ffff640 1 --2- 172.21.15.99:0/2674209203 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557c57e93af0 0x557c57e9d5f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:21.432 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.422+0000 7f246ffff640 1 -- 172.21.15.99:0/2674209203 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x557c57db2c60 con 0x557c57e93100 2024-03-19T01:42:21.432 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.422+0000 7f246ffff640 1 --2- 172.21.15.99:0/2674209203 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557c57e93100 0x557c57e9d0b0 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7f2460005ab0 tx=0x7f2460003ec0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=a4cf95965d8fb59a server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:42:21.432 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.422+0000 7f244f7fe640 10 client.0 ms_handle_connect on v2:172.21.15.112:3300/0 2024-03-19T01:42:21.432 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.422+0000 7f244f7fe640 1 -- 172.21.15.99:0/2674209203 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f246004b070 con 0x557c57e93100 2024-03-19T01:42:21.432 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.422+0000 7f244f7fe640 1 -- 172.21.15.99:0/2674209203 <== mon.2 v2:172.21.15.112:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f24600c51d0 con 0x557c57e93100 2024-03-19T01:42:21.432 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.422+0000 7f2479636740 1 -- 172.21.15.99:0/2674209203 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f2464005500 con 0x557c57e93100 2024-03-19T01:42:21.432 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.422+0000 7f244f7fe640 1 -- 172.21.15.99:0/2674209203 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_get_version_reply(handle=1 version=23) ==== 24+0+0 (secure 0 0 0) 0x7f24600b92a0 con 0x557c57e93100 2024-03-19T01:42:21.432 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.422+0000 7f2479636740 10 client.25205 fetch_fsmap learned FSMap version 23 2024-03-19T01:42:21.432 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.422+0000 7f2479636740 1 -- 172.21.15.99:0/2674209203 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,fsmap=23,monmap=4+}) -- 0x557c57b0c210 con 0x557c57e93100 2024-03-19T01:42:21.432 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.422+0000 7f244f7fe640 1 -- 172.21.15.99:0/2674209203 <== mon.2 v2:172.21.15.112:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f2464005500 con 0x557c57e93100 2024-03-19T01:42:21.432 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.423+0000 7f244f7fe640 1 -- 172.21.15.99:0/2674209203 <== mon.2 v2:172.21.15.112:3300/0 5 ==== fsmap(e 23) ==== 8346+0+0 (secure 0 0 0) 0x7f24600c3070 con 0x557c57e93100 2024-03-19T01:42:21.433 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.423+0000 7f2479636740 10 client.25205 fetch_fsmap finished waiting for FSMap version 23 2024-03-19T01:42:21.433 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.423+0000 7f2479636740 10 client.25205 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:42:21.433 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.423+0000 7f2479636740 20 client.25205 populate_metadata read hostname 'smithi099' 2024-03-19T01:42:21.433 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.423+0000 7f2479636740 1 --2- 172.21.15.99:0/2674209203 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x557c57ea8f80 0x557c57eab440 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:21.433 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.423+0000 7f2479636740 4 client.25205 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:42:21.433 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.423+0000 7f2479636740 1 -- 172.21.15.99:0/2674209203 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x557c57e05120 con 0x557c57ea8f80 2024-03-19T01:42:21.433 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.423+0000 7f246f7fe640 1 --2- 172.21.15.99:0/2674209203 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x557c57ea8f80 0x557c57eab440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:21.433 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.423+0000 7f244f7fe640 10 client.25205 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:42:21.433 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.423+0000 7f246f7fe640 1 --2- 172.21.15.99:0/2674209203 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x557c57ea8f80 0x557c57eab440 crc :-1 s=READY pgs=39 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=4f8297f54f83e59f server_cookie=426630af7c752bf0 in_seq=0 out_seq=0 2024-03-19T01:42:21.433 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.424+0000 7f244f7fe640 1 -- 172.21.15.99:0/2674209203 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x557c57e05120 con 0x557c57ea8f80 2024-03-19T01:42:21.433 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.424+0000 7f244f7fe640 10 client.25205 handle_command_reply: tid=0 2024-03-19T01:42:21.433 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.424+0000 7f2479636740 1 client.25205 shutdown 2024-03-19T01:42:21.433 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.424+0000 7f2479636740 1 -- 172.21.15.99:0/2674209203 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557c57e93100 msgr2=0x557c57e9d0b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:42:21.433 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.424+0000 7f2479636740 1 --2- 172.21.15.99:0/2674209203 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557c57e93100 0x557c57e9d0b0 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7f2460005ab0 tx=0x7f2460003ec0 comp rx=0 tx=0).stop 2024-03-19T01:42:21.434 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.424+0000 7f2479636740 1 -- 172.21.15.99:0/2674209203 shutdown_connections 2024-03-19T01:42:21.434 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.424+0000 7f2479636740 1 --2- 172.21.15.99:0/2674209203 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x557c57ea8f80 0x557c57eab440 crc :-1 s=READY pgs=39 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:21.434 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.424+0000 7f2479636740 1 --2- 172.21.15.99:0/2674209203 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557c57e98c60 0x557c57ea4630 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:21.434 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.424+0000 7f244f7fe640 0 client.25205 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:42:21.434 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.424+0000 7f2479636740 1 --2- 172.21.15.99:0/2674209203 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557c57e93af0 0x557c57e9d5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:21.434 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.424+0000 7f2479636740 1 --2- 172.21.15.99:0/2674209203 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557c57e93100 0x557c57e9d0b0 unknown :-1 s=CLOSED pgs=91 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:21.434 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.424+0000 7f2479636740 1 -- 172.21.15.99:0/2674209203 >> 172.21.15.99:0/2674209203 conn(0x557c57e8e670 msgr2=0x557c57e8f8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:42:21.434 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.424+0000 7f2479636740 1 -- 172.21.15.99:0/2674209203 shutdown_connections 2024-03-19T01:42:21.434 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.424+0000 7f2479636740 1 -- 172.21.15.99:0/2674209203 wait complete. 2024-03-19T01:42:21.434 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.424+0000 7f2479636740 20 client.25205 trim_cache size 0 max 16384 2024-03-19T01:42:21.441 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.440+0000 7f2479636740 1 Processor -- start 2024-03-19T01:42:21.441 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.440+0000 7f2479636740 1 -- start start 2024-03-19T01:42:21.441 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.440+0000 7f2479636740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557c57e93100 0x557c57e9d330 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:21.441 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.440+0000 7f2479636740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557c57e93af0 0x557c57ea45c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:21.441 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.440+0000 7f2479636740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557c57e98c60 0x557c57eafd00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:21.441 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.440+0000 7f2479636740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f24701ab6f0 con 0x557c57e98c60 2024-03-19T01:42:21.441 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.440+0000 7f2479636740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f24701a8fc0 con 0x557c57e93af0 2024-03-19T01:42:21.441 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.440+0000 7f2479636740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f24701a6e10 con 0x557c57e93100 2024-03-19T01:42:21.441 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.441+0000 7f2474fe8640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557c57e98c60 0x557c57eafd00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:21.441 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.441+0000 7f246ffff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557c57e93100 0x557c57e9d330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:21.441 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.441+0000 7f2474fe8640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557c57e98c60 0x557c57eafd00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:35496/0 (socket says 172.21.15.99:35496) 2024-03-19T01:42:21.442 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.441+0000 7f2474fe8640 1 -- 172.21.15.99:0/758062327 learned_addr learned my addr 172.21.15.99:0/758062327 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:42:21.442 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.441+0000 7f246f7fe640 1 --2- 172.21.15.99:0/758062327 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557c57e93af0 0x557c57ea45c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:21.446 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.441+0000 7f2474fe8640 1 -- 172.21.15.99:0/758062327 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557c57e93100 msgr2=0x557c57e9d330 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:42:21.446 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.441+0000 7f2474fe8640 1 --2- 172.21.15.99:0/758062327 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557c57e93100 0x557c57e9d330 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:21.446 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.441+0000 7f2474fe8640 1 -- 172.21.15.99:0/758062327 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557c57e93af0 msgr2=0x557c57ea45c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:42:21.446 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.441+0000 7f2474fe8640 1 --2- 172.21.15.99:0/758062327 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557c57e93af0 0x557c57ea45c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:21.446 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.441+0000 7f2474fe8640 1 -- 172.21.15.99:0/758062327 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x557c57bd8620 con 0x557c57e98c60 2024-03-19T01:42:21.446 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.441+0000 7f246f7fe640 1 --2- 172.21.15.99:0/758062327 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557c57e93af0 0x557c57ea45c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:42:21.447 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.441+0000 7f2474fe8640 1 --2- 172.21.15.99:0/758062327 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557c57e98c60 0x557c57eafd00 secure :-1 s=READY pgs=378 cs=0 l=1 rev1=1 crypto rx=0x7f2468003aa0 tx=0x7f246801a000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=51c674f8313d260 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:42:21.447 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.441+0000 7f2455ffb640 1 -- 172.21.15.99:0/758062327 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2468004100 con 0x557c57e98c60 2024-03-19T01:42:21.447 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.441+0000 7f2455ffb640 1 -- 172.21.15.99:0/758062327 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f2468004780 con 0x557c57e98c60 2024-03-19T01:42:21.447 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.441+0000 7f2455ffb640 1 -- 172.21.15.99:0/758062327 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f246801c400 con 0x557c57e98c60 2024-03-19T01:42:21.447 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.442+0000 7f2479636740 1 -- 172.21.15.99:0/758062327 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557c57e98c60 msgr2=0x557c57eafd00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:42:21.447 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.442+0000 7f2479636740 1 --2- 172.21.15.99:0/758062327 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557c57e98c60 0x557c57eafd00 secure :-1 s=READY pgs=378 cs=0 l=1 rev1=1 crypto rx=0x7f2468003aa0 tx=0x7f246801a000 comp rx=0 tx=0).stop 2024-03-19T01:42:21.447 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.442+0000 7f2479636740 1 -- 172.21.15.99:0/758062327 shutdown_connections 2024-03-19T01:42:21.447 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.442+0000 7f2479636740 1 --2- 172.21.15.99:0/758062327 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557c57e93af0 0x557c57ea45c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:21.447 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.442+0000 7f2479636740 1 --2- 172.21.15.99:0/758062327 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557c57e98c60 0x557c57eafd00 unknown :-1 s=CLOSED pgs=378 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:21.447 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.442+0000 7f2479636740 1 --2- 172.21.15.99:0/758062327 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557c57e93100 0x557c57e9d330 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:21.447 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.442+0000 7f2479636740 1 -- 172.21.15.99:0/758062327 >> 172.21.15.99:0/758062327 conn(0x557c57e8e670 msgr2=0x557c57e92160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:42:21.447 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.442+0000 7f2479636740 1 -- 172.21.15.99:0/758062327 shutdown_connections 2024-03-19T01:42:21.448 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.442+0000 7f2479636740 1 -- 172.21.15.99:0/758062327 wait complete. 2024-03-19T01:42:21.448 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.442+0000 7f2479636740 1 -- start start 2024-03-19T01:42:21.448 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.443+0000 7f2479636740 1 Processor -- start 2024-03-19T01:42:21.448 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.443+0000 7f2479636740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557c57e93100 0x557c57ea42d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:21.448 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.443+0000 7f2479636740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557c57e93af0 0x557c57ea2950 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:21.448 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.443+0000 7f2479636740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557c57e98c60 0x557c57ea2e90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:21.448 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.443+0000 7f2479636740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f24701a7480 con 0x557c57e98c60 2024-03-19T01:42:21.448 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.443+0000 7f2479636740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f24701a88f0 con 0x557c57e93100 2024-03-19T01:42:21.448 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.443+0000 7f2479636740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f24701a7b50 con 0x557c57e93af0 2024-03-19T01:42:21.448 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.443+0000 7f2479636740 10 client.0 _ll_register_callbacks cb 0x7f2460009290 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:42:21.448 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.443+0000 7f2474fe8640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557c57e98c60 0x557c57ea2e90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:21.448 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.443+0000 7f246f7fe640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557c57e93af0 0x557c57ea2950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:21.448 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.443+0000 7f2474fe8640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557c57e98c60 0x557c57ea2e90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:35510/0 (socket says 172.21.15.99:35510) 2024-03-19T01:42:21.448 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.443+0000 7f246ffff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557c57e93100 0x557c57ea42d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:21.449 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.443+0000 7f246f7fe640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557c57e93af0 0x557c57ea2950 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:47566/0 (socket says 172.21.15.99:47566) 2024-03-19T01:42:21.449 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.443+0000 7f2474fe8640 1 -- 172.21.15.99:0/2702129557 learned_addr learned my addr 172.21.15.99:0/2702129557 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:42:21.449 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.443+0000 7f2474fe8640 1 -- 172.21.15.99:0/2702129557 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557c57e93af0 msgr2=0x557c57ea2950 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:42:21.449 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.443+0000 7f2474fe8640 1 --2- 172.21.15.99:0/2702129557 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557c57e93af0 0x557c57ea2950 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:21.449 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.443+0000 7f2474fe8640 1 -- 172.21.15.99:0/2702129557 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557c57e93100 msgr2=0x557c57ea42d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:42:21.449 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.443+0000 7f2474fe8640 1 --2- 172.21.15.99:0/2702129557 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557c57e93100 0x557c57ea42d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:21.449 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.443+0000 7f2474fe8640 1 -- 172.21.15.99:0/2702129557 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x557c57ea34d0 con 0x557c57e98c60 2024-03-19T01:42:21.449 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.443+0000 7f246ffff640 1 --2- 172.21.15.99:0/2702129557 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557c57e93100 0x557c57ea42d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:42:21.449 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.444+0000 7f2474fe8640 1 --2- 172.21.15.99:0/2702129557 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557c57e98c60 0x557c57ea2e90 secure :-1 s=READY pgs=379 cs=0 l=1 rev1=1 crypto rx=0x7f246800fd10 tx=0x7f2468009780 comp rx=0 tx=0).ready entity=mon.0 client_cookie=db295323ff80e5b3 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:42:21.449 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.444+0000 7f244f7fe640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:42:21.450 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.444+0000 7f244f7fe640 1 -- 172.21.15.99:0/2702129557 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2468004a40 con 0x557c57e98c60 2024-03-19T01:42:21.450 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.444+0000 7f244f7fe640 1 -- 172.21.15.99:0/2702129557 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f2468004be0 con 0x557c57e98c60 2024-03-19T01:42:21.450 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.444+0000 7f2479636740 1 -- 172.21.15.99:0/2702129557 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f2468004780 con 0x557c57e98c60 2024-03-19T01:42:21.450 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.444+0000 7f244f7fe640 1 -- 172.21.15.99:0/2702129557 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=23) ==== 24+0+0 (secure 0 0 0) 0x7f2468004de0 con 0x557c57e98c60 2024-03-19T01:42:21.450 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.444+0000 7f2479636740 10 client.15438 fetch_fsmap learned FSMap version 23 2024-03-19T01:42:21.450 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.444+0000 7f2479636740 1 -- 172.21.15.99:0/2702129557 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=23,monmap=4+}) -- 0x557c57f101c0 con 0x557c57e98c60 2024-03-19T01:42:21.450 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.444+0000 7f244f7fe640 1 -- 172.21.15.99:0/2702129557 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f2468004780 con 0x557c57e98c60 2024-03-19T01:42:21.450 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.444+0000 7f244f7fe640 1 -- 172.21.15.99:0/2702129557 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 23) ==== 8346+0+0 (secure 0 0 0) 0x7f2468030070 con 0x557c57e98c60 2024-03-19T01:42:21.450 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.444+0000 7f2479636740 10 client.15438 fetch_fsmap finished waiting for FSMap version 23 2024-03-19T01:42:21.450 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.444+0000 7f2479636740 10 client.15438 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:42:21.450 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.444+0000 7f2479636740 20 client.15438 populate_metadata read hostname 'smithi099' 2024-03-19T01:42:21.450 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.444+0000 7f2479636740 1 --2- 172.21.15.99:0/2702129557 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x557c57f10b50 0x557c57f12f70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:21.451 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.444+0000 7f2479636740 4 client.15438 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:42:21.451 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.444+0000 7f2479636740 1 -- 172.21.15.99:0/2702129557 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x557c57dff470 con 0x557c57f10b50 2024-03-19T01:42:21.451 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.444+0000 7f246ffff640 1 --2- 172.21.15.99:0/2702129557 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x557c57f10b50 0x557c57f12f70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:21.451 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.445+0000 7f246ffff640 1 --2- 172.21.15.99:0/2702129557 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x557c57f10b50 0x557c57f12f70 crc :-1 s=READY pgs=40 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=9ff062cbbe0d6292 server_cookie=e00d30ca83cb71fe in_seq=0 out_seq=0 2024-03-19T01:42:21.451 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.445+0000 7f244f7fe640 10 client.15438 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:42:21.493 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.493+0000 7f244f7fe640 1 -- 172.21.15.99:0/2702129557 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x557c57dff470 con 0x557c57f10b50 2024-03-19T01:42:21.493 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.493+0000 7f244f7fe640 10 client.15438 handle_command_reply: tid=0 2024-03-19T01:42:21.494 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.493+0000 7f2479636740 1 client.15438 shutdown 2024-03-19T01:42:21.494 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.493+0000 7f2479636740 1 -- 172.21.15.99:0/2702129557 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557c57e98c60 msgr2=0x557c57ea2e90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:42:21.494 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.493+0000 7f2479636740 1 --2- 172.21.15.99:0/2702129557 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557c57e98c60 0x557c57ea2e90 secure :-1 s=READY pgs=379 cs=0 l=1 rev1=1 crypto rx=0x7f246800fd10 tx=0x7f2468009780 comp rx=0 tx=0).stop 2024-03-19T01:42:21.494 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.493+0000 7f2479636740 1 -- 172.21.15.99:0/2702129557 shutdown_connections 2024-03-19T01:42:21.494 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.493+0000 7f2479636740 1 --2- 172.21.15.99:0/2702129557 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x557c57f10b50 0x557c57f12f70 crc :-1 s=READY pgs=40 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:21.494 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.493+0000 7f2479636740 1 --2- 172.21.15.99:0/2702129557 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557c57e98c60 0x557c57ea2e90 unknown :-1 s=CLOSED pgs=379 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:21.494 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.493+0000 7f2479636740 1 --2- 172.21.15.99:0/2702129557 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557c57e93af0 0x557c57ea2950 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:21.494 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.493+0000 7f244f7fe640 0 client.15438 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:42:21.494 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.493+0000 7f2479636740 1 --2- 172.21.15.99:0/2702129557 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557c57e93100 0x557c57ea42d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:21.494 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.493+0000 7f2479636740 1 -- 172.21.15.99:0/2702129557 >> 172.21.15.99:0/2702129557 conn(0x557c57e8e670 msgr2=0x557c57e92160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:42:21.494 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.493+0000 7f2479636740 1 -- 172.21.15.99:0/2702129557 shutdown_connections 2024-03-19T01:42:21.494 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.493+0000 7f2479636740 1 -- 172.21.15.99:0/2702129557 wait complete. 2024-03-19T01:42:21.494 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.493+0000 7f2479636740 20 client.15438 trim_cache size 0 max 16384 2024-03-19T01:42:21.495 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:42:21.495 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:42:21.495 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:42:21.495 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:42:21.496 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.494+0000 7f2476a4b640 1 -- 172.21.15.99:0/960673325 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f24580a28a0 msgr2=0x7f24580a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:42:21.496 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.494+0000 7f2476a4b640 1 --2- 172.21.15.99:0/960673325 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f24580a28a0 0x7f24580a4d60 secure :-1 s=READY pgs=253 cs=0 l=1 rev1=1 crypto rx=0x557c57e94350 tx=0x7f2468009500 comp rx=0 tx=0).stop 2024-03-19T01:42:21.496 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.494+0000 7f2476a4b640 1 -- 172.21.15.99:0/960673325 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2470100740 msgr2=0x7f2470197b00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:42:21.496 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.494+0000 7f2476a4b640 1 --2- 172.21.15.99:0/960673325 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2470100740 0x7f2470197b00 secure :-1 s=READY pgs=377 cs=0 l=1 rev1=1 crypto rx=0x7f2460000c00 tx=0x7f246002c640 comp rx=0 tx=0).stop 2024-03-19T01:42:21.496 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.494+0000 7f2476a4b640 1 -- 172.21.15.99:0/960673325 shutdown_connections 2024-03-19T01:42:21.496 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.494+0000 7f2476a4b640 1 --2- 172.21.15.99:0/960673325 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f24580a28a0 0x7f24580a4d60 unknown :-1 s=CLOSED pgs=253 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:21.496 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.494+0000 7f2476a4b640 1 --2- 172.21.15.99:0/960673325 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f2470101930 0x7f2470198040 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:21.496 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.494+0000 7f2476a4b640 1 --2- 172.21.15.99:0/960673325 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f24701022f0 0x7f247019f0b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:21.497 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.494+0000 7f2476a4b640 1 --2- 172.21.15.99:0/960673325 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2470100740 0x7f2470197b00 unknown :-1 s=CLOSED pgs=377 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:21.497 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.494+0000 7f2476a4b640 1 -- 172.21.15.99:0/960673325 >> 172.21.15.99:0/960673325 conn(0x7f24700fbcb0 msgr2=0x7f24700fd710 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:42:21.497 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.494+0000 7f2476a4b640 1 -- 172.21.15.99:0/960673325 shutdown_connections 2024-03-19T01:42:21.497 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:21.494+0000 7f2476a4b640 1 -- 172.21.15.99:0/960673325 wait complete. 2024-03-19T01:42:21.674 INFO:tasks.workunit.client.0.smithi099.stdout: 1 115986 12.55 MB/sec warmup 51 sec latency 27.866 ms 2024-03-19T01:42:22.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:42:21 smithi115 ceph-mon[67332]: pgmap v388: 97 pgs: 97 active+clean; 56 MiB data, 662 MiB used, 1.0 TiB / 1.0 TiB avail; 2.6 MiB/s rd, 1.1 MiB/s wr, 185 op/s 2024-03-19T01:42:22.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:42:21 smithi099 ceph-mon[62951]: pgmap v388: 97 pgs: 97 active+clean; 56 MiB data, 662 MiB used, 1.0 TiB / 1.0 TiB avail; 2.6 MiB/s rd, 1.1 MiB/s wr, 185 op/s 2024-03-19T01:42:22.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:42:21 smithi112 ceph-mon[66819]: pgmap v388: 97 pgs: 97 active+clean; 56 MiB data, 662 MiB used, 1.0 TiB / 1.0 TiB avail; 2.6 MiB/s rd, 1.1 MiB/s wr, 185 op/s 2024-03-19T01:42:22.675 INFO:tasks.workunit.client.0.smithi099.stdout: 1 118092 12.56 MB/sec warmup 52 sec latency 27.727 ms 2024-03-19T01:42:23.677 INFO:tasks.workunit.client.0.smithi099.stdout: 1 120869 12.66 MB/sec warmup 53 sec latency 17.829 ms 2024-03-19T01:42:24.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:42:23 smithi115 ceph-mon[67332]: pgmap v389: 97 pgs: 97 active+clean; 60 MiB data, 679 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 MiB/s rd, 1.4 MiB/s wr, 240 op/s 2024-03-19T01:42:24.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:42:23 smithi099 ceph-mon[62951]: pgmap v389: 97 pgs: 97 active+clean; 60 MiB data, 679 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 MiB/s rd, 1.4 MiB/s wr, 240 op/s 2024-03-19T01:42:24.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:42:23 smithi112 ceph-mon[66819]: pgmap v389: 97 pgs: 97 active+clean; 60 MiB data, 679 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 MiB/s rd, 1.4 MiB/s wr, 240 op/s 2024-03-19T01:42:24.675 INFO:tasks.workunit.client.0.smithi099.stdout: 1 122833 12.54 MB/sec warmup 54 sec latency 35.321 ms 2024-03-19T01:42:25.675 INFO:tasks.workunit.client.0.smithi099.stdout: 1 124973 12.56 MB/sec warmup 55 sec latency 26.902 ms 2024-03-19T01:42:26.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:42:25 smithi115 ceph-mon[67332]: pgmap v390: 97 pgs: 97 active+clean; 61 MiB data, 685 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 MiB/s rd, 1.3 MiB/s wr, 218 op/s 2024-03-19T01:42:26.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:42:25 smithi099 ceph-mon[62951]: pgmap v390: 97 pgs: 97 active+clean; 61 MiB data, 685 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 MiB/s rd, 1.3 MiB/s wr, 218 op/s 2024-03-19T01:42:26.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:42:25 smithi112 ceph-mon[66819]: pgmap v390: 97 pgs: 97 active+clean; 61 MiB data, 685 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 MiB/s rd, 1.3 MiB/s wr, 218 op/s 2024-03-19T01:42:26.675 INFO:tasks.workunit.client.0.smithi099.stdout: 1 127726 12.64 MB/sec warmup 56 sec latency 16.310 ms 2024-03-19T01:42:27.675 INFO:tasks.workunit.client.0.smithi099.stdout: 1 129308 12.54 MB/sec warmup 57 sec latency 76.003 ms 2024-03-19T01:42:28.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:42:27 smithi115 ceph-mon[67332]: pgmap v391: 97 pgs: 97 active+clean; 62 MiB data, 676 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 MiB/s rd, 1.3 MiB/s wr, 202 op/s 2024-03-19T01:42:28.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:42:27 smithi099 ceph-mon[62951]: pgmap v391: 97 pgs: 97 active+clean; 62 MiB data, 676 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 MiB/s rd, 1.3 MiB/s wr, 202 op/s 2024-03-19T01:42:28.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:42:27 smithi112 ceph-mon[66819]: pgmap v391: 97 pgs: 97 active+clean; 62 MiB data, 676 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 MiB/s rd, 1.3 MiB/s wr, 202 op/s 2024-03-19T01:42:28.675 INFO:tasks.workunit.client.0.smithi099.stdout: 1 131751 12.56 MB/sec warmup 58 sec latency 16.298 ms 2024-03-19T01:42:29.675 INFO:tasks.workunit.client.0.smithi099.stdout: 1 134170 12.55 MB/sec warmup 59 sec latency 31.307 ms 2024-03-19T01:42:30.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:42:29 smithi115 ceph-mon[67332]: pgmap v392: 97 pgs: 97 active+clean; 67 MiB data, 695 MiB used, 1.0 TiB / 1.0 TiB avail; 3.1 MiB/s rd, 1.6 MiB/s wr, 275 op/s 2024-03-19T01:42:30.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:42:29 smithi099 ceph-mon[62951]: pgmap v392: 97 pgs: 97 active+clean; 67 MiB data, 695 MiB used, 1.0 TiB / 1.0 TiB avail; 3.1 MiB/s rd, 1.6 MiB/s wr, 275 op/s 2024-03-19T01:42:30.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:42:29 smithi112 ceph-mon[66819]: pgmap v392: 97 pgs: 97 active+clean; 67 MiB data, 695 MiB used, 1.0 TiB / 1.0 TiB avail; 3.1 MiB/s rd, 1.6 MiB/s wr, 275 op/s 2024-03-19T01:42:30.675 INFO:tasks.workunit.client.0.smithi099.stdout: 1 136038 12.50 MB/sec warmup 60 sec latency 41.492 ms 2024-03-19T01:42:31.675 INFO:tasks.workunit.client.0.smithi099.stdout: 1 138692 12.57 MB/sec warmup 61 sec latency 13.773 ms 2024-03-19T01:42:32.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:42:31 smithi115 ceph-mon[67332]: pgmap v393: 97 pgs: 97 active+clean; 67 MiB data, 702 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 MiB/s rd, 1.3 MiB/s wr, 214 op/s 2024-03-19T01:42:32.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:42:31 smithi099 ceph-mon[62951]: pgmap v393: 97 pgs: 97 active+clean; 67 MiB data, 702 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 MiB/s rd, 1.3 MiB/s wr, 214 op/s 2024-03-19T01:42:32.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:42:31 smithi112 ceph-mon[66819]: pgmap v393: 97 pgs: 97 active+clean; 67 MiB data, 702 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 MiB/s rd, 1.3 MiB/s wr, 214 op/s 2024-03-19T01:42:32.675 INFO:tasks.workunit.client.0.smithi099.stdout: 1 140507 12.46 MB/sec warmup 62 sec latency 75.598 ms 2024-03-19T01:42:33.675 INFO:tasks.workunit.client.0.smithi099.stdout: 1 142503 12.48 MB/sec warmup 63 sec latency 24.148 ms 2024-03-19T01:42:34.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:42:33 smithi115 ceph-mon[67332]: pgmap v394: 97 pgs: 97 active+clean; 69 MiB data, 706 MiB used, 1.0 TiB / 1.0 TiB avail; 2.8 MiB/s rd, 1.3 MiB/s wr, 229 op/s 2024-03-19T01:42:34.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:42:33 smithi099 ceph-mon[62951]: pgmap v394: 97 pgs: 97 active+clean; 69 MiB data, 706 MiB used, 1.0 TiB / 1.0 TiB avail; 2.8 MiB/s rd, 1.3 MiB/s wr, 229 op/s 2024-03-19T01:42:34.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:42:33 smithi112 ceph-mon[66819]: pgmap v394: 97 pgs: 97 active+clean; 69 MiB data, 706 MiB used, 1.0 TiB / 1.0 TiB avail; 2.8 MiB/s rd, 1.3 MiB/s wr, 229 op/s 2024-03-19T01:42:34.584 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.582+0000 7f183f974640 1 -- 172.21.15.99:0/332668357 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1838100740 msgr2=0x7f1838100b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:42:34.584 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.582+0000 7f183f974640 1 --2- 172.21.15.99:0/332668357 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1838100740 0x7f1838100b60 secure :-1 s=READY pgs=380 cs=0 l=1 rev1=1 crypto rx=0x7f1834009900 tx=0x7f183402bc30 comp rx=0 tx=0).stop 2024-03-19T01:42:34.584 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.582+0000 7f183f974640 1 -- 172.21.15.99:0/332668357 shutdown_connections 2024-03-19T01:42:34.584 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.582+0000 7f183f974640 1 --2- 172.21.15.99:0/332668357 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1838101930 0x7f1838101db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:34.584 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.582+0000 7f183f974640 1 --2- 172.21.15.99:0/332668357 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f18381022f0 0x7f1838108b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:34.584 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.582+0000 7f183f974640 1 --2- 172.21.15.99:0/332668357 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1838100740 0x7f1838100b60 unknown :-1 s=CLOSED pgs=380 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:34.585 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.582+0000 7f183f974640 1 -- 172.21.15.99:0/332668357 >> 172.21.15.99:0/332668357 conn(0x7f18380fbcb0 msgr2=0x7f18380fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:42:34.585 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.582+0000 7f183f974640 1 -- 172.21.15.99:0/332668357 shutdown_connections 2024-03-19T01:42:34.585 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.582+0000 7f183f974640 1 -- 172.21.15.99:0/332668357 wait complete. 2024-03-19T01:42:34.585 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.583+0000 7f183f974640 1 Processor -- start 2024-03-19T01:42:34.585 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.583+0000 7f183f974640 1 -- start start 2024-03-19T01:42:34.585 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.583+0000 7f183f974640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1838100740 0x7f1838196a40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:34.585 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.583+0000 7f183f974640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1838101930 0x7f1838196f80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:34.585 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.583+0000 7f183f974640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f18381022f0 0x7f183819dff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:34.585 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.583+0000 7f183f974640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f183810c3c0 con 0x7f1838101930 2024-03-19T01:42:34.585 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.583+0000 7f183f974640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f183810c240 con 0x7f18381022f0 2024-03-19T01:42:34.585 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.583+0000 7f183f974640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f183810c540 con 0x7f1838100740 2024-03-19T01:42:34.585 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.583+0000 7f183cf0f640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1838101930 0x7f1838196f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:34.585 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.583+0000 7f183d710640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1838100740 0x7f1838196a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:34.585 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.583+0000 7f183cf0f640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1838101930 0x7f1838196f80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:48692/0 (socket says 172.21.15.99:48692) 2024-03-19T01:42:34.586 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.583+0000 7f183cf0f640 1 -- 172.21.15.99:0/4014718713 learned_addr learned my addr 172.21.15.99:0/4014718713 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:42:34.586 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.583+0000 7f183df11640 1 --2- 172.21.15.99:0/4014718713 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f18381022f0 0x7f183819dff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:34.586 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.583+0000 7f183cf0f640 1 -- 172.21.15.99:0/4014718713 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1838100740 msgr2=0x7f1838196a40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:42:34.586 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.583+0000 7f183cf0f640 1 --2- 172.21.15.99:0/4014718713 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1838100740 0x7f1838196a40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:34.586 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.583+0000 7f183cf0f640 1 -- 172.21.15.99:0/4014718713 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f18381022f0 msgr2=0x7f183819dff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:42:34.586 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.583+0000 7f183cf0f640 1 --2- 172.21.15.99:0/4014718713 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f18381022f0 0x7f183819dff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:34.586 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.583+0000 7f183cf0f640 1 -- 172.21.15.99:0/4014718713 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f183819e660 con 0x7f1838101930 2024-03-19T01:42:34.586 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.583+0000 7f183cf0f640 1 --2- 172.21.15.99:0/4014718713 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1838101930 0x7f1838196f80 secure :-1 s=READY pgs=381 cs=0 l=1 rev1=1 crypto rx=0x7f182800e9f0 tx=0x7f182800eeb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:42:34.586 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.583+0000 7f182e7fc640 1 -- 172.21.15.99:0/4014718713 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f182800cd80 con 0x7f1838101930 2024-03-19T01:42:34.586 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.584+0000 7f183f974640 1 -- 172.21.15.99:0/4014718713 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f183819e950 con 0x7f1838101930 2024-03-19T01:42:34.586 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.584+0000 7f182e7fc640 1 -- 172.21.15.99:0/4014718713 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f1828010910 con 0x7f1838101930 2024-03-19T01:42:34.586 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.584+0000 7f182e7fc640 1 -- 172.21.15.99:0/4014718713 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f18280188f0 con 0x7f1838101930 2024-03-19T01:42:34.586 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.584+0000 7f183f974640 1 -- 172.21.15.99:0/4014718713 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f183819eea0 con 0x7f1838101930 2024-03-19T01:42:34.593 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.586+0000 7f182e7fc640 1 -- 172.21.15.99:0/4014718713 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f1828010ac0 con 0x7f1838101930 2024-03-19T01:42:34.594 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.586+0000 7f1842636740 1 Processor -- start 2024-03-19T01:42:34.594 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.587+0000 7f1842636740 1 -- start start 2024-03-19T01:42:34.594 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.587+0000 7f1842636740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563dd5711100 0x563dd5711520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:34.594 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.587+0000 7f1842636740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563dd5711af0 0x563dd57165f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:34.594 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.587+0000 7f182e7fc640 1 --2- 172.21.15.99:0/4014718713 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f18240a28a0 0x7f18240a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:34.594 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.587+0000 7f1842636740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563dd5716c60 0x563dd57190f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:34.594 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.587+0000 7f1842636740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x563dd5675be0 con 0x563dd5711100 2024-03-19T01:42:34.594 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.587+0000 7f1842636740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x563dd561e960 con 0x563dd5711af0 2024-03-19T01:42:34.594 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.587+0000 7f1842636740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x563dd5671c80 con 0x563dd5716c60 2024-03-19T01:42:34.594 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.587+0000 7f183df11640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563dd5711100 0x563dd5711520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:34.594 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.587+0000 7f183df11640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563dd5711100 0x563dd5711520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:48708/0 (socket says 172.21.15.99:48708) 2024-03-19T01:42:34.594 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.587+0000 7f183df11640 1 -- 172.21.15.99:0/303336126 learned_addr learned my addr 172.21.15.99:0/303336126 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:42:34.595 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.587+0000 7f183cf0f640 1 --2- 172.21.15.99:0/4014718713 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f18240a28a0 0x7f18240a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:34.595 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.587+0000 7f183d710640 1 --2- 172.21.15.99:0/303336126 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563dd5711af0 0x563dd57165f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:34.595 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.587+0000 7f183df11640 1 --2- 172.21.15.99:0/303336126 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563dd5716c60 0x563dd57190f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:34.595 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.587+0000 7f182e7fc640 1 -- 172.21.15.99:0/4014718713 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f18280cee60 con 0x7f1838101930 2024-03-19T01:42:34.595 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.587+0000 7f183df11640 1 -- 172.21.15.99:0/303336126 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563dd5716c60 msgr2=0x563dd57190f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:42:34.595 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.587+0000 7f183df11640 1 --2- 172.21.15.99:0/303336126 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563dd5716c60 0x563dd57190f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:34.595 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.587+0000 7f183df11640 1 -- 172.21.15.99:0/303336126 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563dd5711af0 msgr2=0x563dd57165f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:42:34.595 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.587+0000 7f183df11640 1 --2- 172.21.15.99:0/303336126 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563dd5711af0 0x563dd57165f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:34.595 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.587+0000 7f183df11640 1 -- 172.21.15.99:0/303336126 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x563dd5615d20 con 0x563dd5711100 2024-03-19T01:42:34.595 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.587+0000 7f183d710640 1 --2- 172.21.15.99:0/303336126 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563dd5711af0 0x563dd57165f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:42:34.595 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.587+0000 7f183cf0f640 1 --2- 172.21.15.99:0/4014718713 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f18240a28a0 0x7f18240a4d60 secure :-1 s=READY pgs=254 cs=0 l=1 rev1=1 crypto rx=0x7f1828097cb0 tx=0x7f1828066b70 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:42:34.595 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.587+0000 7f183df11640 1 --2- 172.21.15.99:0/303336126 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563dd5711100 0x563dd5711520 secure :-1 s=READY pgs=382 cs=0 l=1 rev1=1 crypto rx=0x7f1830009fd0 tx=0x7f1830005bd0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ad8a6519fbc046b7 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:42:34.595 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.588+0000 7f18237fe640 1 -- 172.21.15.99:0/303336126 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1830015030 con 0x563dd5711100 2024-03-19T01:42:34.595 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.588+0000 7f18237fe640 1 -- 172.21.15.99:0/303336126 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f1830018070 con 0x563dd5711100 2024-03-19T01:42:34.596 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.588+0000 7f18237fe640 1 -- 172.21.15.99:0/303336126 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f183001a030 con 0x563dd5711100 2024-03-19T01:42:34.596 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.588+0000 7f1842636740 1 -- 172.21.15.99:0/303336126 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563dd5711100 msgr2=0x563dd5711520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:42:34.596 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.588+0000 7f1842636740 1 --2- 172.21.15.99:0/303336126 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563dd5711100 0x563dd5711520 secure :-1 s=READY pgs=382 cs=0 l=1 rev1=1 crypto rx=0x7f1830009fd0 tx=0x7f1830005bd0 comp rx=0 tx=0).stop 2024-03-19T01:42:34.596 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.588+0000 7f1842636740 1 -- 172.21.15.99:0/303336126 shutdown_connections 2024-03-19T01:42:34.596 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.588+0000 7f1842636740 1 --2- 172.21.15.99:0/303336126 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563dd5711af0 0x563dd57165f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:34.596 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.588+0000 7f1842636740 1 --2- 172.21.15.99:0/303336126 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563dd5716c60 0x563dd57190f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:34.596 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.588+0000 7f1842636740 1 --2- 172.21.15.99:0/303336126 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563dd5711100 0x563dd5711520 unknown :-1 s=CLOSED pgs=382 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:34.596 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.588+0000 7f1842636740 1 -- 172.21.15.99:0/303336126 >> 172.21.15.99:0/303336126 conn(0x563dd570c670 msgr2=0x563dd570ca70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:42:34.596 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.589+0000 7f1842636740 1 -- 172.21.15.99:0/303336126 shutdown_connections 2024-03-19T01:42:34.596 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.589+0000 7f1842636740 1 -- 172.21.15.99:0/303336126 wait complete. 2024-03-19T01:42:34.596 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.589+0000 7f1842636740 1 -- start start 2024-03-19T01:42:34.596 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.589+0000 7f1842636740 1 Processor -- start 2024-03-19T01:42:34.596 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.589+0000 7f1842636740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563dd5711100 0x563dd571b010 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:34.596 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.589+0000 7f1842636740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563dd5711af0 0x563dd571b550 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:34.597 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.589+0000 7f1842636740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563dd5716c60 0x563dd5722590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:34.597 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.589+0000 7f1842636740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x563dd5719bb0 con 0x563dd5711af0 2024-03-19T01:42:34.597 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.589+0000 7f1842636740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x563dd5631db0 con 0x563dd5711100 2024-03-19T01:42:34.597 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.589+0000 7f1842636740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x563dd5719d30 con 0x563dd5716c60 2024-03-19T01:42:34.597 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.589+0000 7f183d710640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563dd5711af0 0x563dd571b550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:34.597 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.589+0000 7f1842636740 10 client.0 _ll_register_callbacks cb 0x563dd5636e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:42:34.597 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.589+0000 7f183df11640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563dd5711100 0x563dd571b010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:34.597 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.589+0000 7f183d710640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563dd5711af0 0x563dd571b550 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:48712/0 (socket says 172.21.15.99:48712) 2024-03-19T01:42:34.597 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.589+0000 7f183d710640 1 -- 172.21.15.99:0/3047655744 learned_addr learned my addr 172.21.15.99:0/3047655744 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:42:34.597 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.590+0000 7f183df11640 1 --2- 172.21.15.99:0/3047655744 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563dd5716c60 0x563dd5722590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:34.597 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.590+0000 7f183d710640 1 -- 172.21.15.99:0/3047655744 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563dd5716c60 msgr2=0x563dd5722590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:42:34.597 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.590+0000 7f183d710640 1 --2- 172.21.15.99:0/3047655744 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563dd5716c60 0x563dd5722590 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:34.597 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.590+0000 7f183d710640 1 -- 172.21.15.99:0/3047655744 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563dd5711100 msgr2=0x563dd571b010 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:42:34.597 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.590+0000 7f183d710640 1 --2- 172.21.15.99:0/3047655744 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563dd5711100 0x563dd571b010 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:34.598 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.590+0000 7f183d710640 1 -- 172.21.15.99:0/3047655744 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x563dd5630c60 con 0x563dd5711af0 2024-03-19T01:42:34.598 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.590+0000 7f183d710640 1 --2- 172.21.15.99:0/3047655744 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563dd5711af0 0x563dd571b550 secure :-1 s=READY pgs=383 cs=0 l=1 rev1=1 crypto rx=0x7f183402cfd0 tx=0x7f1834004250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f9d0fa6aa690f651 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:42:34.598 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.590+0000 7f1820ff9640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:42:34.598 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.590+0000 7f1820ff9640 1 -- 172.21.15.99:0/3047655744 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1834005d50 con 0x563dd5711af0 2024-03-19T01:42:34.598 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.590+0000 7f1820ff9640 1 -- 172.21.15.99:0/3047655744 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f1834005230 con 0x563dd5711af0 2024-03-19T01:42:34.598 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.590+0000 7f1842636740 1 -- 172.21.15.99:0/3047655744 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f1830018070 con 0x563dd5711af0 2024-03-19T01:42:34.598 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.590+0000 7f1820ff9640 1 -- 172.21.15.99:0/3047655744 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=23) ==== 24+0+0 (secure 0 0 0) 0x7f1834005430 con 0x563dd5711af0 2024-03-19T01:42:34.598 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.590+0000 7f1842636740 10 client.15450 fetch_fsmap learned FSMap version 23 2024-03-19T01:42:34.598 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.590+0000 7f1842636740 1 -- 172.21.15.99:0/3047655744 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=23,monmap=4+}) -- 0x563dd538a210 con 0x563dd5711af0 2024-03-19T01:42:34.598 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.590+0000 7f1820ff9640 1 -- 172.21.15.99:0/3047655744 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f1830018070 con 0x563dd5711af0 2024-03-19T01:42:34.598 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.590+0000 7f1820ff9640 1 -- 172.21.15.99:0/3047655744 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 23) ==== 8346+0+0 (secure 0 0 0) 0x7f18340037b0 con 0x563dd5711af0 2024-03-19T01:42:34.598 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.590+0000 7f1842636740 10 client.15450 fetch_fsmap finished waiting for FSMap version 23 2024-03-19T01:42:34.598 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.590+0000 7f1842636740 10 client.15450 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:42:34.598 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.591+0000 7f1842636740 20 client.15450 populate_metadata read hostname 'smithi099' 2024-03-19T01:42:34.599 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.591+0000 7f1842636740 1 --2- 172.21.15.99:0/3047655744 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x563dd5726dc0 0x563dd5729240 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:34.599 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.591+0000 7f1842636740 4 client.15450 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:42:34.599 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.591+0000 7f1842636740 1 -- 172.21.15.99:0/3047655744 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x563dd5683120 con 0x563dd5726dc0 2024-03-19T01:42:34.599 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.591+0000 7f183df11640 1 --2- 172.21.15.99:0/3047655744 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x563dd5726dc0 0x563dd5729240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:34.599 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.591+0000 7f1820ff9640 10 client.15450 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:42:34.599 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.591+0000 7f183df11640 1 --2- 172.21.15.99:0/3047655744 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x563dd5726dc0 0x563dd5729240 crc :-1 s=READY pgs=41 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=9ae633d41312e1b6 server_cookie=55f58653fcb72f0d in_seq=0 out_seq=0 2024-03-19T01:42:34.599 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.592+0000 7f1820ff9640 1 -- 172.21.15.99:0/3047655744 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x563dd5683120 con 0x563dd5726dc0 2024-03-19T01:42:34.599 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.592+0000 7f1820ff9640 10 client.15450 handle_command_reply: tid=0 2024-03-19T01:42:34.599 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.592+0000 7f1842636740 1 client.15450 shutdown 2024-03-19T01:42:34.599 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.592+0000 7f1842636740 1 -- 172.21.15.99:0/3047655744 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563dd5711af0 msgr2=0x563dd571b550 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:42:34.599 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.592+0000 7f1842636740 1 --2- 172.21.15.99:0/3047655744 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563dd5711af0 0x563dd571b550 secure :-1 s=READY pgs=383 cs=0 l=1 rev1=1 crypto rx=0x7f183402cfd0 tx=0x7f1834004250 comp rx=0 tx=0).stop 2024-03-19T01:42:34.599 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.592+0000 7f1842636740 1 -- 172.21.15.99:0/3047655744 shutdown_connections 2024-03-19T01:42:34.599 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.592+0000 7f1842636740 1 --2- 172.21.15.99:0/3047655744 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x563dd5726dc0 0x563dd5729240 crc :-1 s=READY pgs=41 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:34.599 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.592+0000 7f1842636740 1 --2- 172.21.15.99:0/3047655744 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563dd5716c60 0x563dd5722590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:34.600 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.592+0000 7f1842636740 1 --2- 172.21.15.99:0/3047655744 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563dd5711af0 0x563dd571b550 unknown :-1 s=CLOSED pgs=383 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:34.600 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.592+0000 7f1820ff9640 0 client.15450 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:42:34.600 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.592+0000 7f1842636740 1 --2- 172.21.15.99:0/3047655744 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563dd5711100 0x563dd571b010 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:34.600 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.592+0000 7f1842636740 1 -- 172.21.15.99:0/3047655744 >> 172.21.15.99:0/3047655744 conn(0x563dd570c670 msgr2=0x563dd570d8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:42:34.600 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.592+0000 7f1842636740 1 -- 172.21.15.99:0/3047655744 shutdown_connections 2024-03-19T01:42:34.600 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.592+0000 7f1842636740 1 -- 172.21.15.99:0/3047655744 wait complete. 2024-03-19T01:42:34.600 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.592+0000 7f1842636740 20 client.15450 trim_cache size 0 max 16384 2024-03-19T01:42:34.612 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.607+0000 7f1842636740 1 Processor -- start 2024-03-19T01:42:34.612 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.607+0000 7f1842636740 1 -- start start 2024-03-19T01:42:34.612 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.607+0000 7f1842636740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563dd5711100 0x563dd570bab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:34.612 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.607+0000 7f1842636740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563dd5711af0 0x563dd5722550 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:34.612 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.607+0000 7f1842636740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563dd5716c60 0x563dd570ffe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:34.613 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.607+0000 7f1842636740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f18381a94c0 con 0x563dd5711100 2024-03-19T01:42:34.613 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.607+0000 7f1842636740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f18381a6d90 con 0x563dd5716c60 2024-03-19T01:42:34.613 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.607+0000 7f1842636740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f18381a4be0 con 0x563dd5711af0 2024-03-19T01:42:34.613 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.607+0000 7f183df11640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563dd5711100 0x563dd570bab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:34.613 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.607+0000 7f183df11640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563dd5711100 0x563dd570bab0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:48728/0 (socket says 172.21.15.99:48728) 2024-03-19T01:42:34.613 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.607+0000 7f183df11640 1 -- 172.21.15.99:0/2925393318 learned_addr learned my addr 172.21.15.99:0/2925393318 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:42:34.613 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.607+0000 7f183d710640 1 --2- 172.21.15.99:0/2925393318 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563dd5711af0 0x563dd5722550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:34.613 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.607+0000 7f183df11640 1 --2- 172.21.15.99:0/2925393318 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563dd5716c60 0x563dd570ffe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:34.613 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.607+0000 7f183df11640 1 -- 172.21.15.99:0/2925393318 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563dd5711af0 msgr2=0x563dd5722550 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:42:34.613 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.607+0000 7f183df11640 1 --2- 172.21.15.99:0/2925393318 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563dd5711af0 0x563dd5722550 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:34.613 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.607+0000 7f183df11640 1 -- 172.21.15.99:0/2925393318 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563dd5716c60 msgr2=0x563dd570ffe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:42:34.613 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.607+0000 7f183df11640 1 --2- 172.21.15.99:0/2925393318 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563dd5716c60 0x563dd570ffe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:34.613 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.607+0000 7f183df11640 1 -- 172.21.15.99:0/2925393318 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x563dd5456620 con 0x563dd5711100 2024-03-19T01:42:34.614 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.607+0000 7f183df11640 1 --2- 172.21.15.99:0/2925393318 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563dd5711100 0x563dd570bab0 secure :-1 s=READY pgs=384 cs=0 l=1 rev1=1 crypto rx=0x7f1830015000 tx=0x7f1830022040 comp rx=0 tx=0).ready entity=mon.0 client_cookie=63637628a0388f66 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:42:34.614 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.608+0000 7f1822ffd640 1 -- 172.21.15.99:0/2925393318 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f183001a040 con 0x563dd5711100 2024-03-19T01:42:34.614 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.608+0000 7f1822ffd640 1 -- 172.21.15.99:0/2925393318 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f1830018040 con 0x563dd5711100 2024-03-19T01:42:34.614 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.608+0000 7f1822ffd640 1 -- 172.21.15.99:0/2925393318 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f183002a040 con 0x563dd5711100 2024-03-19T01:42:34.614 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.608+0000 7f1842636740 1 -- 172.21.15.99:0/2925393318 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563dd5711100 msgr2=0x563dd570bab0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:42:34.614 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.608+0000 7f1842636740 1 --2- 172.21.15.99:0/2925393318 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563dd5711100 0x563dd570bab0 secure :-1 s=READY pgs=384 cs=0 l=1 rev1=1 crypto rx=0x7f1830015000 tx=0x7f1830022040 comp rx=0 tx=0).stop 2024-03-19T01:42:34.614 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.608+0000 7f1842636740 1 -- 172.21.15.99:0/2925393318 shutdown_connections 2024-03-19T01:42:34.614 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.608+0000 7f1842636740 1 --2- 172.21.15.99:0/2925393318 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563dd5716c60 0x563dd570ffe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:34.614 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.608+0000 7f1842636740 1 --2- 172.21.15.99:0/2925393318 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563dd5711af0 0x563dd5722550 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:34.614 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.608+0000 7f1842636740 1 --2- 172.21.15.99:0/2925393318 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563dd5711100 0x563dd570bab0 unknown :-1 s=CLOSED pgs=384 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:34.614 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.608+0000 7f1842636740 1 -- 172.21.15.99:0/2925393318 >> 172.21.15.99:0/2925393318 conn(0x563dd570c670 msgr2=0x563dd570f8d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:42:34.614 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.608+0000 7f1842636740 1 -- 172.21.15.99:0/2925393318 shutdown_connections 2024-03-19T01:42:34.614 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.608+0000 7f1842636740 1 -- 172.21.15.99:0/2925393318 wait complete. 2024-03-19T01:42:34.614 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.609+0000 7f1842636740 1 -- start start 2024-03-19T01:42:34.615 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.609+0000 7f1842636740 1 Processor -- start 2024-03-19T01:42:34.615 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.609+0000 7f1842636740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563dd5711100 0x563dd571fb00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:34.615 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.609+0000 7f1842636740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563dd5711af0 0x563dd5720040 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:34.615 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.609+0000 7f1842636740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563dd5716c60 0x563dd5720f90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:34.615 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.609+0000 7f1842636740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f18381a5250 con 0x563dd5711af0 2024-03-19T01:42:34.615 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.609+0000 7f1842636740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f18381a66c0 con 0x563dd5711100 2024-03-19T01:42:34.615 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.609+0000 7f1842636740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f18381a5920 con 0x563dd5716c60 2024-03-19T01:42:34.615 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.609+0000 7f183d710640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563dd5711af0 0x563dd5720040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:34.615 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.609+0000 7f1842636740 10 client.0 _ll_register_callbacks cb 0x7f1834009780 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:42:34.615 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.609+0000 7f183df11640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563dd5711100 0x563dd571fb00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:34.615 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.609+0000 7f183d710640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563dd5711af0 0x563dd5720040 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:48740/0 (socket says 172.21.15.99:48740) 2024-03-19T01:42:34.615 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.609+0000 7f183d710640 1 -- 172.21.15.99:0/2230234950 learned_addr learned my addr 172.21.15.99:0/2230234950 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:42:34.615 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.609+0000 7f183df11640 1 --2- 172.21.15.99:0/2230234950 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563dd5716c60 0x563dd5720f90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:34.616 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.610+0000 7f183d710640 1 -- 172.21.15.99:0/2230234950 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563dd5716c60 msgr2=0x563dd5720f90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:42:34.616 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.610+0000 7f183d710640 1 --2- 172.21.15.99:0/2230234950 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563dd5716c60 0x563dd5720f90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:34.616 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.610+0000 7f183d710640 1 -- 172.21.15.99:0/2230234950 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563dd5711100 msgr2=0x563dd571fb00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:42:34.616 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.610+0000 7f183d710640 1 --2- 172.21.15.99:0/2230234950 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563dd5711100 0x563dd571fb00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:34.616 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.610+0000 7f183d710640 1 -- 172.21.15.99:0/2230234950 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x563dd5721580 con 0x563dd5711af0 2024-03-19T01:42:34.616 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.610+0000 7f183d710640 1 --2- 172.21.15.99:0/2230234950 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563dd5711af0 0x563dd5720040 secure :-1 s=READY pgs=385 cs=0 l=1 rev1=1 crypto rx=0x7f1834009850 tx=0x7f18340092d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ed28ba27e50c0cf7 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:42:34.616 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.610+0000 7f1820ff9640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:42:34.616 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.610+0000 7f1820ff9640 1 -- 172.21.15.99:0/2230234950 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1834005d50 con 0x563dd5711af0 2024-03-19T01:42:34.616 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.610+0000 7f1820ff9640 1 -- 172.21.15.99:0/2230234950 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f18340035f0 con 0x563dd5711af0 2024-03-19T01:42:34.616 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.610+0000 7f1842636740 1 -- 172.21.15.99:0/2230234950 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f1830018040 con 0x563dd5711af0 2024-03-19T01:42:34.616 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.610+0000 7f1820ff9640 1 -- 172.21.15.99:0/2230234950 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=23) ==== 24+0+0 (secure 0 0 0) 0x7f18340037f0 con 0x563dd5711af0 2024-03-19T01:42:34.616 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.610+0000 7f1842636740 10 client.15456 fetch_fsmap learned FSMap version 23 2024-03-19T01:42:34.616 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.610+0000 7f1842636740 1 -- 172.21.15.99:0/2230234950 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=23,monmap=4+}) -- 0x563dd5722900 con 0x563dd5711af0 2024-03-19T01:42:34.616 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.610+0000 7f1820ff9640 1 -- 172.21.15.99:0/2230234950 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f1830018040 con 0x563dd5711af0 2024-03-19T01:42:34.616 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.610+0000 7f1820ff9640 1 -- 172.21.15.99:0/2230234950 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 23) ==== 8346+0+0 (secure 0 0 0) 0x7f18340344d0 con 0x563dd5711af0 2024-03-19T01:42:34.617 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.611+0000 7f1842636740 10 client.15456 fetch_fsmap finished waiting for FSMap version 23 2024-03-19T01:42:34.617 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.611+0000 7f1842636740 10 client.15456 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:42:34.617 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.611+0000 7f1842636740 20 client.15456 populate_metadata read hostname 'smithi099' 2024-03-19T01:42:34.617 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.611+0000 7f1842636740 1 --2- 172.21.15.99:0/2230234950 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x563dd5726dc0 0x563dd578f900 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:34.617 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.611+0000 7f1842636740 4 client.15456 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:42:34.617 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.611+0000 7f1842636740 1 -- 172.21.15.99:0/2230234950 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x563dd567d470 con 0x563dd5726dc0 2024-03-19T01:42:34.617 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.611+0000 7f183df11640 1 --2- 172.21.15.99:0/2230234950 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x563dd5726dc0 0x563dd578f900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:34.617 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.611+0000 7f183df11640 1 --2- 172.21.15.99:0/2230234950 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x563dd5726dc0 0x563dd578f900 crc :-1 s=READY pgs=42 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=260e7929e8302e7a server_cookie=1bce24c006653dac in_seq=0 out_seq=0 2024-03-19T01:42:34.617 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.611+0000 7f1820ff9640 10 client.15456 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:42:34.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.664+0000 7f1820ff9640 1 -- 172.21.15.99:0/2230234950 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x563dd567d470 con 0x563dd5726dc0 2024-03-19T01:42:34.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.664+0000 7f1820ff9640 10 client.15456 handle_command_reply: tid=0 2024-03-19T01:42:34.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.664+0000 7f1842636740 1 client.15456 shutdown 2024-03-19T01:42:34.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.664+0000 7f1842636740 1 -- 172.21.15.99:0/2230234950 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563dd5711af0 msgr2=0x563dd5720040 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:42:34.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.664+0000 7f1842636740 1 --2- 172.21.15.99:0/2230234950 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563dd5711af0 0x563dd5720040 secure :-1 s=READY pgs=385 cs=0 l=1 rev1=1 crypto rx=0x7f1834009850 tx=0x7f18340092d0 comp rx=0 tx=0).stop 2024-03-19T01:42:34.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.665+0000 7f1842636740 1 -- 172.21.15.99:0/2230234950 shutdown_connections 2024-03-19T01:42:34.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.665+0000 7f1842636740 1 --2- 172.21.15.99:0/2230234950 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x563dd5726dc0 0x563dd578f900 crc :-1 s=READY pgs=42 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:34.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.665+0000 7f1842636740 1 --2- 172.21.15.99:0/2230234950 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563dd5716c60 0x563dd5720f90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:34.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.665+0000 7f1842636740 1 --2- 172.21.15.99:0/2230234950 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563dd5711af0 0x563dd5720040 unknown :-1 s=CLOSED pgs=385 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:34.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.665+0000 7f1842636740 1 --2- 172.21.15.99:0/2230234950 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563dd5711100 0x563dd571fb00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:34.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.665+0000 7f1820ff9640 0 client.15456 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:42:34.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.665+0000 7f1842636740 1 -- 172.21.15.99:0/2230234950 >> 172.21.15.99:0/2230234950 conn(0x563dd570c670 msgr2=0x563dd5710160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:42:34.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.665+0000 7f1842636740 1 -- 172.21.15.99:0/2230234950 shutdown_connections 2024-03-19T01:42:34.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.665+0000 7f1842636740 1 -- 172.21.15.99:0/2230234950 wait complete. 2024-03-19T01:42:34.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.665+0000 7f1842636740 20 client.15456 trim_cache size 0 max 16384 2024-03-19T01:42:34.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:42:34.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:42:34.667 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:42:34.667 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:42:34.667 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.665+0000 7f183f974640 1 -- 172.21.15.99:0/4014718713 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f18240a28a0 msgr2=0x7f18240a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:42:34.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.665+0000 7f183f974640 1 --2- 172.21.15.99:0/4014718713 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f18240a28a0 0x7f18240a4d60 secure :-1 s=READY pgs=254 cs=0 l=1 rev1=1 crypto rx=0x7f1828097cb0 tx=0x7f1828066b70 comp rx=0 tx=0).stop 2024-03-19T01:42:34.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.665+0000 7f183f974640 1 -- 172.21.15.99:0/4014718713 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1838101930 msgr2=0x7f1838196f80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:42:34.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.665+0000 7f183f974640 1 --2- 172.21.15.99:0/4014718713 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1838101930 0x7f1838196f80 secure :-1 s=READY pgs=381 cs=0 l=1 rev1=1 crypto rx=0x7f182800e9f0 tx=0x7f182800eeb0 comp rx=0 tx=0).stop 2024-03-19T01:42:34.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.665+0000 7f183f974640 1 -- 172.21.15.99:0/4014718713 shutdown_connections 2024-03-19T01:42:34.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.665+0000 7f183f974640 1 --2- 172.21.15.99:0/4014718713 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f18240a28a0 0x7f18240a4d60 unknown :-1 s=CLOSED pgs=254 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:34.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.665+0000 7f183f974640 1 --2- 172.21.15.99:0/4014718713 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f18381022f0 0x7f183819dff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:34.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.665+0000 7f183f974640 1 --2- 172.21.15.99:0/4014718713 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1838101930 0x7f1838196f80 unknown :-1 s=CLOSED pgs=381 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:34.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.665+0000 7f183f974640 1 --2- 172.21.15.99:0/4014718713 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1838100740 0x7f1838196a40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:34.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.665+0000 7f183f974640 1 -- 172.21.15.99:0/4014718713 >> 172.21.15.99:0/4014718713 conn(0x7f18380fbcb0 msgr2=0x7f1838104b60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:42:34.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.665+0000 7f183f974640 1 -- 172.21.15.99:0/4014718713 shutdown_connections 2024-03-19T01:42:34.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:34.666+0000 7f183f974640 1 -- 172.21.15.99:0/4014718713 wait complete. 2024-03-19T01:42:34.675 INFO:tasks.workunit.client.0.smithi099.stdout: 1 144787 12.46 MB/sec warmup 64 sec latency 34.994 ms 2024-03-19T01:42:35.675 INFO:tasks.workunit.client.0.smithi099.stdout: 1 146717 12.42 MB/sec warmup 65 sec latency 19.090 ms 2024-03-19T01:42:36.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:42:35 smithi115 ceph-mon[67332]: pgmap v395: 97 pgs: 97 active+clean; 75 MiB data, 717 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 MiB/s rd, 1.5 MiB/s wr, 212 op/s 2024-03-19T01:42:36.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:42:35 smithi099 ceph-mon[62951]: pgmap v395: 97 pgs: 97 active+clean; 75 MiB data, 717 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 MiB/s rd, 1.5 MiB/s wr, 212 op/s 2024-03-19T01:42:36.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:42:35 smithi112 ceph-mon[66819]: pgmap v395: 97 pgs: 97 active+clean; 75 MiB data, 717 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 MiB/s rd, 1.5 MiB/s wr, 212 op/s 2024-03-19T01:42:36.675 INFO:tasks.workunit.client.0.smithi099.stdout: 1 149360 12.49 MB/sec warmup 66 sec latency 15.412 ms 2024-03-19T01:42:37.675 INFO:tasks.workunit.client.0.smithi099.stdout: 1 151064 12.39 MB/sec warmup 67 sec latency 93.324 ms 2024-03-19T01:42:38.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:42:37 smithi115 ceph-mon[67332]: pgmap v396: 97 pgs: 97 active+clean; 75 MiB data, 710 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 MiB/s rd, 2.0 MiB/s wr, 248 op/s 2024-03-19T01:42:38.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:42:37 smithi099 ceph-mon[62951]: pgmap v396: 97 pgs: 97 active+clean; 75 MiB data, 710 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 MiB/s rd, 2.0 MiB/s wr, 248 op/s 2024-03-19T01:42:38.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:42:37 smithi112 ceph-mon[66819]: pgmap v396: 97 pgs: 97 active+clean; 75 MiB data, 710 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 MiB/s rd, 2.0 MiB/s wr, 248 op/s 2024-03-19T01:42:38.675 INFO:tasks.workunit.client.0.smithi099.stdout: 1 153146 12.40 MB/sec warmup 68 sec latency 28.463 ms 2024-03-19T01:42:39.675 INFO:tasks.workunit.client.0.smithi099.stdout: 1 155910 12.43 MB/sec warmup 69 sec latency 13.754 ms 2024-03-19T01:42:40.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:42:39 smithi115 ceph-mon[67332]: pgmap v397: 97 pgs: 97 active+clean; 76 MiB data, 744 MiB used, 1.0 TiB / 1.0 TiB avail; 3.2 MiB/s rd, 2.4 MiB/s wr, 294 op/s 2024-03-19T01:42:40.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:42:39 smithi099 ceph-mon[62951]: pgmap v397: 97 pgs: 97 active+clean; 76 MiB data, 744 MiB used, 1.0 TiB / 1.0 TiB avail; 3.2 MiB/s rd, 2.4 MiB/s wr, 294 op/s 2024-03-19T01:42:40.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:42:39 smithi112 ceph-mon[66819]: pgmap v397: 97 pgs: 97 active+clean; 76 MiB data, 744 MiB used, 1.0 TiB / 1.0 TiB avail; 3.2 MiB/s rd, 2.4 MiB/s wr, 294 op/s 2024-03-19T01:42:40.675 INFO:tasks.workunit.client.0.smithi099.stdout: 1 157612 12.39 MB/sec warmup 70 sec latency 44.227 ms 2024-03-19T01:42:41.675 INFO:tasks.workunit.client.0.smithi099.stdout: 1 160095 12.42 MB/sec warmup 71 sec latency 19.103 ms 2024-03-19T01:42:42.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:42:41 smithi115 ceph-mon[67332]: pgmap v398: 97 pgs: 97 active+clean; 75 MiB data, 773 MiB used, 1.0 TiB / 1.0 TiB avail; 2.6 MiB/s rd, 2.3 MiB/s wr, 226 op/s 2024-03-19T01:42:42.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:42:41 smithi099 ceph-mon[62951]: pgmap v398: 97 pgs: 97 active+clean; 75 MiB data, 773 MiB used, 1.0 TiB / 1.0 TiB avail; 2.6 MiB/s rd, 2.3 MiB/s wr, 226 op/s 2024-03-19T01:42:42.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:42:41 smithi112 ceph-mon[66819]: pgmap v398: 97 pgs: 97 active+clean; 75 MiB data, 773 MiB used, 1.0 TiB / 1.0 TiB avail; 2.6 MiB/s rd, 2.3 MiB/s wr, 226 op/s 2024-03-19T01:42:42.675 INFO:tasks.workunit.client.0.smithi099.stdout: 1 162138 12.35 MB/sec warmup 72 sec latency 63.957 ms 2024-03-19T01:42:43.676 INFO:tasks.workunit.client.0.smithi099.stdout: 1 164164 12.35 MB/sec warmup 73 sec latency 38.197 ms 2024-03-19T01:42:44.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:42:43 smithi115 ceph-mon[67332]: pgmap v399: 97 pgs: 97 active+clean; 79 MiB data, 810 MiB used, 1.0 TiB / 1.0 TiB avail; 3.1 MiB/s rd, 3.1 MiB/s wr, 305 op/s 2024-03-19T01:42:44.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:42:43 smithi099 ceph-mon[62951]: pgmap v399: 97 pgs: 97 active+clean; 79 MiB data, 810 MiB used, 1.0 TiB / 1.0 TiB avail; 3.1 MiB/s rd, 3.1 MiB/s wr, 305 op/s 2024-03-19T01:42:44.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:42:43 smithi112 ceph-mon[66819]: pgmap v399: 97 pgs: 97 active+clean; 79 MiB data, 810 MiB used, 1.0 TiB / 1.0 TiB avail; 3.1 MiB/s rd, 3.1 MiB/s wr, 305 op/s 2024-03-19T01:42:44.675 INFO:tasks.workunit.client.0.smithi099.stdout: 1 166963 12.42 MB/sec warmup 74 sec latency 14.086 ms 2024-03-19T01:42:45.675 INFO:tasks.workunit.client.0.smithi099.stdout: 1 168796 12.34 MB/sec warmup 75 sec latency 41.298 ms 2024-03-19T01:42:46.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:42:45 smithi115 ceph-mon[67332]: pgmap v400: 97 pgs: 97 active+clean; 79 MiB data, 806 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 MiB/s rd, 3.1 MiB/s wr, 294 op/s 2024-03-19T01:42:46.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:42:45 smithi099 ceph-mon[62951]: pgmap v400: 97 pgs: 97 active+clean; 79 MiB data, 806 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 MiB/s rd, 3.1 MiB/s wr, 294 op/s 2024-03-19T01:42:46.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:42:45 smithi112 ceph-mon[66819]: pgmap v400: 97 pgs: 97 active+clean; 79 MiB data, 806 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 MiB/s rd, 3.1 MiB/s wr, 294 op/s 2024-03-19T01:42:46.675 INFO:tasks.workunit.client.0.smithi099.stdout: 1 170958 12.36 MB/sec warmup 76 sec latency 27.336 ms 2024-03-19T01:42:47.676 INFO:tasks.workunit.client.0.smithi099.stdout: 1 173284 12.36 MB/sec warmup 77 sec latency 52.947 ms 2024-03-19T01:42:47.759 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.757+0000 7f1aa3968640 1 -- 172.21.15.99:0/676090467 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1a9c100740 msgr2=0x7f1a9c100b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:42:47.759 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.757+0000 7f1aa3968640 1 --2- 172.21.15.99:0/676090467 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1a9c100740 0x7f1a9c100b60 secure :-1 s=READY pgs=386 cs=0 l=1 rev1=1 crypto rx=0x7f1a98009900 tx=0x7f1a9802bc30 comp rx=0 tx=0).stop 2024-03-19T01:42:47.759 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.757+0000 7f1aa3968640 1 -- 172.21.15.99:0/676090467 shutdown_connections 2024-03-19T01:42:47.759 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.757+0000 7f1aa3968640 1 --2- 172.21.15.99:0/676090467 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1a9c101930 0x7f1a9c101db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:47.759 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.757+0000 7f1aa3968640 1 --2- 172.21.15.99:0/676090467 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1a9c1022f0 0x7f1a9c108b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:47.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.757+0000 7f1aa3968640 1 --2- 172.21.15.99:0/676090467 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1a9c100740 0x7f1a9c100b60 unknown :-1 s=CLOSED pgs=386 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:47.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.757+0000 7f1aa3968640 1 -- 172.21.15.99:0/676090467 >> 172.21.15.99:0/676090467 conn(0x7f1a9c0fbcb0 msgr2=0x7f1a9c0fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:42:47.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.757+0000 7f1aa3968640 1 -- 172.21.15.99:0/676090467 shutdown_connections 2024-03-19T01:42:47.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.757+0000 7f1aa3968640 1 -- 172.21.15.99:0/676090467 wait complete. 2024-03-19T01:42:47.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.758+0000 7f1aa3968640 1 Processor -- start 2024-03-19T01:42:47.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.758+0000 7f1aa3968640 1 -- start start 2024-03-19T01:42:47.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.758+0000 7f1aa3968640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1a9c100740 0x7f1a9c069a30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:47.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.758+0000 7f1aa3968640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1a9c101930 0x7f1a9c19c830 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:47.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.758+0000 7f1aa3968640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1a9c1022f0 0x7f1a9c19ee70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:47.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.758+0000 7f1aa3968640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f1a9c10c390 con 0x7f1a9c100740 2024-03-19T01:42:47.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.758+0000 7f1aa3968640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f1a9c10c210 con 0x7f1a9c101930 2024-03-19T01:42:47.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.758+0000 7f1aa3968640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f1a9c10c510 con 0x7f1a9c1022f0 2024-03-19T01:42:47.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.758+0000 7f1aa1704640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1a9c100740 0x7f1a9c069a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:47.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.758+0000 7f1aa1704640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1a9c100740 0x7f1a9c069a30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:48820/0 (socket says 172.21.15.99:48820) 2024-03-19T01:42:47.761 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.758+0000 7f1aa1704640 1 -- 172.21.15.99:0/1372918157 learned_addr learned my addr 172.21.15.99:0/1372918157 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:42:47.761 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.758+0000 7f1aa1f05640 1 --2- 172.21.15.99:0/1372918157 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1a9c1022f0 0x7f1a9c19ee70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:47.761 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.758+0000 7f1aa0f03640 1 --2- 172.21.15.99:0/1372918157 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1a9c101930 0x7f1a9c19c830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:47.761 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.758+0000 7f1aa1704640 1 -- 172.21.15.99:0/1372918157 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1a9c1022f0 msgr2=0x7f1a9c19ee70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:42:47.761 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.758+0000 7f1aa1704640 1 --2- 172.21.15.99:0/1372918157 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1a9c1022f0 0x7f1a9c19ee70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:47.761 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.758+0000 7f1aa1704640 1 -- 172.21.15.99:0/1372918157 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1a9c101930 msgr2=0x7f1a9c19c830 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:42:47.761 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.758+0000 7f1aa1704640 1 --2- 172.21.15.99:0/1372918157 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1a9c101930 0x7f1a9c19c830 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:47.761 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.758+0000 7f1aa1704640 1 -- 172.21.15.99:0/1372918157 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f1a9c19f730 con 0x7f1a9c100740 2024-03-19T01:42:47.761 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.758+0000 7f1aa1704640 1 --2- 172.21.15.99:0/1372918157 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1a9c100740 0x7f1a9c069a30 secure :-1 s=READY pgs=387 cs=0 l=1 rev1=1 crypto rx=0x7f1a98000c00 tx=0x7f1a9802c640 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:42:47.761 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.759+0000 7f1a927fc640 1 -- 172.21.15.99:0/1372918157 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1a98041070 con 0x7f1a9c100740 2024-03-19T01:42:47.761 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.759+0000 7f1aa3968640 1 -- 172.21.15.99:0/1372918157 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f1a9c19f9c0 con 0x7f1a9c100740 2024-03-19T01:42:47.761 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.759+0000 7f1a927fc640 1 -- 172.21.15.99:0/1372918157 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f1a98004450 con 0x7f1a9c100740 2024-03-19T01:42:47.761 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.759+0000 7f1a927fc640 1 -- 172.21.15.99:0/1372918157 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1a98034440 con 0x7f1a9c100740 2024-03-19T01:42:47.761 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.759+0000 7f1aa3968640 1 -- 172.21.15.99:0/1372918157 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f1a9c19feb0 con 0x7f1a9c100740 2024-03-19T01:42:47.768 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.761+0000 7f1a927fc640 1 -- 172.21.15.99:0/1372918157 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f1a98044070 con 0x7f1a9c100740 2024-03-19T01:42:47.768 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.762+0000 7f1aa662a740 1 Processor -- start 2024-03-19T01:42:47.768 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.762+0000 7f1aa662a740 1 -- start start 2024-03-19T01:42:47.768 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.762+0000 7f1aa662a740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56345a373100 0x56345a373520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:47.768 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.762+0000 7f1aa662a740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56345a373af0 0x56345a3785f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:47.768 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.762+0000 7f1aa662a740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56345a378c60 0x56345a37b0f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:47.768 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.762+0000 7f1aa662a740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x56345a2d7be0 con 0x56345a373af0 2024-03-19T01:42:47.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.762+0000 7f1aa662a740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x56345a280960 con 0x56345a378c60 2024-03-19T01:42:47.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.762+0000 7f1aa662a740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x56345a2d3c80 con 0x56345a373100 2024-03-19T01:42:47.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.762+0000 7f1a927fc640 1 --2- 172.21.15.99:0/1372918157 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f1a880a28a0 0x7f1a880a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:47.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.762+0000 7f1aa1704640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56345a373af0 0x56345a3785f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:47.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.762+0000 7f1aa1704640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56345a373af0 0x56345a3785f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:48824/0 (socket says 172.21.15.99:48824) 2024-03-19T01:42:47.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.762+0000 7f1aa1704640 1 -- 172.21.15.99:0/1354457789 learned_addr learned my addr 172.21.15.99:0/1354457789 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:42:47.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.762+0000 7f1aa0f03640 1 --2- 172.21.15.99:0/1354457789 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56345a378c60 0x56345a37b0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:47.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.762+0000 7f1aa1f05640 1 --2- 172.21.15.99:0/1354457789 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56345a373100 0x56345a373520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:47.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.762+0000 7f1aa1f05640 1 --2- 172.21.15.99:0/1372918157 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f1a880a28a0 0x7f1a880a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:47.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.762+0000 7f1a927fc640 1 -- 172.21.15.99:0/1372918157 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f1a980f05c0 con 0x7f1a9c100740 2024-03-19T01:42:47.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.762+0000 7f1aa1704640 1 -- 172.21.15.99:0/1354457789 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56345a373100 msgr2=0x56345a373520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:42:47.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.762+0000 7f1aa1704640 1 --2- 172.21.15.99:0/1354457789 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56345a373100 0x56345a373520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:47.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.762+0000 7f1aa1704640 1 -- 172.21.15.99:0/1354457789 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56345a378c60 msgr2=0x56345a37b0f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:42:47.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.762+0000 7f1aa1704640 1 --2- 172.21.15.99:0/1354457789 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56345a378c60 0x56345a37b0f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:47.770 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.762+0000 7f1aa1704640 1 -- 172.21.15.99:0/1354457789 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x56345a277d20 con 0x56345a373af0 2024-03-19T01:42:47.770 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.762+0000 7f1aa0f03640 1 --2- 172.21.15.99:0/1354457789 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56345a378c60 0x56345a37b0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:42:47.770 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.763+0000 7f1aa1f05640 1 --2- 172.21.15.99:0/1354457789 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56345a373100 0x56345a373520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:42:47.770 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.763+0000 7f1aa1704640 1 --2- 172.21.15.99:0/1354457789 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56345a373af0 0x56345a3785f0 secure :-1 s=READY pgs=388 cs=0 l=1 rev1=1 crypto rx=0x7f1a980ee020 tx=0x7f1a980bcf00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c80caf705175ee0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:42:47.770 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.763+0000 7f1aa1f05640 1 --2- 172.21.15.99:0/1372918157 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f1a880a28a0 0x7f1a880a4d60 secure :-1 s=READY pgs=255 cs=0 l=1 rev1=1 crypto rx=0x56345a374350 tx=0x7f1a94009500 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:42:47.770 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.763+0000 7f1a877fe640 1 -- 172.21.15.99:0/1354457789 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1a9804b070 con 0x56345a373af0 2024-03-19T01:42:47.770 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.763+0000 7f1a877fe640 1 -- 172.21.15.99:0/1354457789 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f1a98004c80 con 0x56345a373af0 2024-03-19T01:42:47.770 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.763+0000 7f1a877fe640 1 -- 172.21.15.99:0/1354457789 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1a980b9670 con 0x56345a373af0 2024-03-19T01:42:47.770 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.764+0000 7f1aa662a740 1 -- 172.21.15.99:0/1354457789 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56345a373af0 msgr2=0x56345a3785f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:42:47.770 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.764+0000 7f1aa662a740 1 --2- 172.21.15.99:0/1354457789 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56345a373af0 0x56345a3785f0 secure :-1 s=READY pgs=388 cs=0 l=1 rev1=1 crypto rx=0x7f1a980ee020 tx=0x7f1a980bcf00 comp rx=0 tx=0).stop 2024-03-19T01:42:47.770 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.764+0000 7f1aa662a740 1 -- 172.21.15.99:0/1354457789 shutdown_connections 2024-03-19T01:42:47.770 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.764+0000 7f1aa662a740 1 --2- 172.21.15.99:0/1354457789 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56345a378c60 0x56345a37b0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:47.770 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.764+0000 7f1aa662a740 1 --2- 172.21.15.99:0/1354457789 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56345a373af0 0x56345a3785f0 unknown :-1 s=CLOSED pgs=388 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:47.770 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.764+0000 7f1aa662a740 1 --2- 172.21.15.99:0/1354457789 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56345a373100 0x56345a373520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:47.771 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.764+0000 7f1aa662a740 1 -- 172.21.15.99:0/1354457789 >> 172.21.15.99:0/1354457789 conn(0x56345a36e670 msgr2=0x56345a36ea70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:42:47.771 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.764+0000 7f1aa662a740 1 -- 172.21.15.99:0/1354457789 shutdown_connections 2024-03-19T01:42:47.771 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.764+0000 7f1aa662a740 1 -- 172.21.15.99:0/1354457789 wait complete. 2024-03-19T01:42:47.771 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.764+0000 7f1aa662a740 1 -- start start 2024-03-19T01:42:47.771 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.764+0000 7f1aa662a740 1 Processor -- start 2024-03-19T01:42:47.771 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.765+0000 7f1aa662a740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56345a373100 0x56345a37ce30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:47.771 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.765+0000 7f1aa662a740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56345a373af0 0x56345a37d370 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:47.771 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.765+0000 7f1aa662a740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56345a378c60 0x56345a3843b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:47.771 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.765+0000 7f1aa662a740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x56345a37b9d0 con 0x56345a373af0 2024-03-19T01:42:47.771 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.765+0000 7f1aa662a740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x56345a293db0 con 0x56345a378c60 2024-03-19T01:42:47.771 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.765+0000 7f1aa662a740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x56345a37bb50 con 0x56345a373100 2024-03-19T01:42:47.771 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.765+0000 7f1aa662a740 10 client.0 _ll_register_callbacks cb 0x56345a298e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:42:47.771 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.765+0000 7f1aa0f03640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56345a373af0 0x56345a37d370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:47.771 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.765+0000 7f1aa0f03640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56345a373af0 0x56345a37d370 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:48832/0 (socket says 172.21.15.99:48832) 2024-03-19T01:42:47.772 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.765+0000 7f1aa0f03640 1 -- 172.21.15.99:0/1696729764 learned_addr learned my addr 172.21.15.99:0/1696729764 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:42:47.772 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.765+0000 7f1aa1704640 1 --2- 172.21.15.99:0/1696729764 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56345a373100 0x56345a37ce30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:47.772 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.765+0000 7f1aa1f05640 1 --2- 172.21.15.99:0/1696729764 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56345a378c60 0x56345a3843b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:47.772 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.765+0000 7f1aa0f03640 1 -- 172.21.15.99:0/1696729764 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56345a373100 msgr2=0x56345a37ce30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:42:47.772 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.765+0000 7f1aa0f03640 1 --2- 172.21.15.99:0/1696729764 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56345a373100 0x56345a37ce30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:47.772 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.765+0000 7f1aa0f03640 1 -- 172.21.15.99:0/1696729764 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56345a378c60 msgr2=0x56345a3843b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:42:47.772 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.765+0000 7f1aa0f03640 1 --2- 172.21.15.99:0/1696729764 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56345a378c60 0x56345a3843b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:47.772 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.765+0000 7f1aa0f03640 1 -- 172.21.15.99:0/1696729764 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x56345a292c60 con 0x56345a373af0 2024-03-19T01:42:47.772 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.765+0000 7f1aa0f03640 1 --2- 172.21.15.99:0/1696729764 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56345a373af0 0x56345a37d370 secure :-1 s=READY pgs=389 cs=0 l=1 rev1=1 crypto rx=0x7f1a8c012720 tx=0x7f1a8c012b20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6b6db3c330b475ce server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:42:47.772 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.765+0000 7f1a84ff9640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:42:47.772 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.765+0000 7f1a84ff9640 1 -- 172.21.15.99:0/1696729764 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1a8c004030 con 0x56345a373af0 2024-03-19T01:42:47.772 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.765+0000 7f1a84ff9640 1 -- 172.21.15.99:0/1696729764 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f1a8c0041d0 con 0x56345a373af0 2024-03-19T01:42:47.772 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.765+0000 7f1aa662a740 1 -- 172.21.15.99:0/1696729764 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f1a98004c80 con 0x56345a373af0 2024-03-19T01:42:47.772 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.765+0000 7f1a84ff9640 1 -- 172.21.15.99:0/1696729764 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=23) ==== 24+0+0 (secure 0 0 0) 0x7f1a8c004410 con 0x56345a373af0 2024-03-19T01:42:47.773 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.765+0000 7f1aa662a740 10 client.15468 fetch_fsmap learned FSMap version 23 2024-03-19T01:42:47.773 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.765+0000 7f1aa662a740 1 -- 172.21.15.99:0/1696729764 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=23,monmap=4+}) -- 0x563459fec210 con 0x56345a373af0 2024-03-19T01:42:47.773 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.766+0000 7f1a84ff9640 1 -- 172.21.15.99:0/1696729764 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f1a98004c80 con 0x56345a373af0 2024-03-19T01:42:47.773 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.766+0000 7f1a84ff9640 1 -- 172.21.15.99:0/1696729764 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 23) ==== 8346+0+0 (secure 0 0 0) 0x7f1a8c003bc0 con 0x56345a373af0 2024-03-19T01:42:47.773 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.766+0000 7f1aa662a740 10 client.15468 fetch_fsmap finished waiting for FSMap version 23 2024-03-19T01:42:47.773 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.766+0000 7f1aa662a740 10 client.15468 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:42:47.773 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.766+0000 7f1aa662a740 20 client.15468 populate_metadata read hostname 'smithi099' 2024-03-19T01:42:47.773 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.766+0000 7f1aa662a740 1 --2- 172.21.15.99:0/1696729764 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56345a388d60 0x56345a38b1c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:47.773 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.766+0000 7f1aa662a740 4 client.15468 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:42:47.773 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.766+0000 7f1aa662a740 1 -- 172.21.15.99:0/1696729764 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x56345a2e5120 con 0x56345a388d60 2024-03-19T01:42:47.773 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.766+0000 7f1aa1704640 1 --2- 172.21.15.99:0/1696729764 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56345a388d60 0x56345a38b1c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:47.773 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.766+0000 7f1aa1704640 1 --2- 172.21.15.99:0/1696729764 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56345a388d60 0x56345a38b1c0 crc :-1 s=READY pgs=43 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=7ff2c29e35c8bff server_cookie=9d0c7efcca9a98c5 in_seq=0 out_seq=0 2024-03-19T01:42:47.773 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.766+0000 7f1a84ff9640 10 client.15468 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:42:47.773 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.767+0000 7f1a84ff9640 1 -- 172.21.15.99:0/1696729764 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x56345a2e5120 con 0x56345a388d60 2024-03-19T01:42:47.774 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.767+0000 7f1a84ff9640 10 client.15468 handle_command_reply: tid=0 2024-03-19T01:42:47.774 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.767+0000 7f1aa662a740 1 client.15468 shutdown 2024-03-19T01:42:47.774 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.767+0000 7f1aa662a740 1 -- 172.21.15.99:0/1696729764 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56345a373af0 msgr2=0x56345a37d370 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:42:47.774 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.767+0000 7f1aa662a740 1 --2- 172.21.15.99:0/1696729764 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56345a373af0 0x56345a37d370 secure :-1 s=READY pgs=389 cs=0 l=1 rev1=1 crypto rx=0x7f1a8c012720 tx=0x7f1a8c012b20 comp rx=0 tx=0).stop 2024-03-19T01:42:47.774 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.767+0000 7f1aa662a740 1 -- 172.21.15.99:0/1696729764 shutdown_connections 2024-03-19T01:42:47.774 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.767+0000 7f1aa662a740 1 --2- 172.21.15.99:0/1696729764 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56345a388d60 0x56345a38b1c0 crc :-1 s=READY pgs=43 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:47.774 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.767+0000 7f1aa662a740 1 --2- 172.21.15.99:0/1696729764 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56345a378c60 0x56345a3843b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:47.774 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.767+0000 7f1aa662a740 1 --2- 172.21.15.99:0/1696729764 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56345a373af0 0x56345a37d370 unknown :-1 s=CLOSED pgs=389 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:47.774 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.767+0000 7f1aa662a740 1 --2- 172.21.15.99:0/1696729764 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56345a373100 0x56345a37ce30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:47.774 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.767+0000 7f1a84ff9640 0 client.15468 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:42:47.774 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.767+0000 7f1aa662a740 1 -- 172.21.15.99:0/1696729764 >> 172.21.15.99:0/1696729764 conn(0x56345a36e670 msgr2=0x56345a36f8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:42:47.774 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.767+0000 7f1aa662a740 1 -- 172.21.15.99:0/1696729764 shutdown_connections 2024-03-19T01:42:47.774 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.767+0000 7f1aa662a740 1 -- 172.21.15.99:0/1696729764 wait complete. 2024-03-19T01:42:47.774 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.767+0000 7f1aa662a740 20 client.15468 trim_cache size 0 max 16384 2024-03-19T01:42:47.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.783+0000 7f1aa662a740 1 Processor -- start 2024-03-19T01:42:47.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.783+0000 7f1aa662a740 1 -- start start 2024-03-19T01:42:47.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.783+0000 7f1aa662a740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56345a373100 0x56345a36dab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:47.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.783+0000 7f1aa662a740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56345a373af0 0x56345a384370 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:47.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.783+0000 7f1aa662a740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56345a378c60 0x56345a374ca0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:47.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.783+0000 7f1aa662a740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f1a9c1ab6d0 con 0x56345a373af0 2024-03-19T01:42:47.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.783+0000 7f1aa662a740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f1a9c1a8fa0 con 0x56345a378c60 2024-03-19T01:42:47.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.783+0000 7f1aa662a740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f1a9c1a6df0 con 0x56345a373100 2024-03-19T01:42:47.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.783+0000 7f1aa0f03640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56345a373af0 0x56345a384370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:47.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.783+0000 7f1aa1704640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56345a373100 0x56345a36dab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:47.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.783+0000 7f1aa1704640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56345a373100 0x56345a36dab0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:46778/0 (socket says 172.21.15.99:46778) 2024-03-19T01:42:47.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.783+0000 7f1aa1f05640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56345a378c60 0x56345a374ca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:47.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.783+0000 7f1aa1704640 1 -- 172.21.15.99:0/3732985268 learned_addr learned my addr 172.21.15.99:0/3732985268 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:42:47.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.783+0000 7f1aa0f03640 1 -- 172.21.15.99:0/3732985268 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56345a373100 msgr2=0x56345a36dab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:42:47.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.783+0000 7f1aa0f03640 1 --2- 172.21.15.99:0/3732985268 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56345a373100 0x56345a36dab0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:47.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.783+0000 7f1aa0f03640 1 -- 172.21.15.99:0/3732985268 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56345a378c60 msgr2=0x56345a374ca0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:42:47.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.783+0000 7f1aa0f03640 1 --2- 172.21.15.99:0/3732985268 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56345a378c60 0x56345a374ca0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:47.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.783+0000 7f1aa0f03640 1 -- 172.21.15.99:0/3732985268 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x56345a0b8620 con 0x56345a373af0 2024-03-19T01:42:47.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.783+0000 7f1aa1f05640 1 --2- 172.21.15.99:0/3732985268 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56345a378c60 0x56345a374ca0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:42:47.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.784+0000 7f1aa0f03640 1 --2- 172.21.15.99:0/3732985268 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56345a373af0 0x56345a384370 secure :-1 s=READY pgs=390 cs=0 l=1 rev1=1 crypto rx=0x7f1a8c0126f0 tx=0x7f1a8c005870 comp rx=0 tx=0).ready entity=mon.0 client_cookie=40f161604a9b8ef6 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:42:47.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.784+0000 7f1a86ffd640 1 -- 172.21.15.99:0/3732985268 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1a8c0106f0 con 0x56345a373af0 2024-03-19T01:42:47.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.784+0000 7f1a86ffd640 1 -- 172.21.15.99:0/3732985268 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f1a8c010890 con 0x56345a373af0 2024-03-19T01:42:47.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.784+0000 7f1a86ffd640 1 -- 172.21.15.99:0/3732985268 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1a8c01c680 con 0x56345a373af0 2024-03-19T01:42:47.789 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.785+0000 7f1aa662a740 1 -- 172.21.15.99:0/3732985268 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56345a373af0 msgr2=0x56345a384370 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:42:47.789 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.785+0000 7f1aa662a740 1 --2- 172.21.15.99:0/3732985268 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56345a373af0 0x56345a384370 secure :-1 s=READY pgs=390 cs=0 l=1 rev1=1 crypto rx=0x7f1a8c0126f0 tx=0x7f1a8c005870 comp rx=0 tx=0).stop 2024-03-19T01:42:47.789 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.785+0000 7f1aa662a740 1 -- 172.21.15.99:0/3732985268 shutdown_connections 2024-03-19T01:42:47.789 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.785+0000 7f1aa662a740 1 --2- 172.21.15.99:0/3732985268 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56345a378c60 0x56345a374ca0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:47.789 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.785+0000 7f1aa662a740 1 --2- 172.21.15.99:0/3732985268 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56345a373af0 0x56345a384370 unknown :-1 s=CLOSED pgs=390 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:47.789 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.785+0000 7f1aa662a740 1 --2- 172.21.15.99:0/3732985268 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56345a373100 0x56345a36dab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:47.789 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.785+0000 7f1aa662a740 1 -- 172.21.15.99:0/3732985268 >> 172.21.15.99:0/3732985268 conn(0x56345a36e670 msgr2=0x56345a374590 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:42:47.789 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.785+0000 7f1aa662a740 1 -- 172.21.15.99:0/3732985268 shutdown_connections 2024-03-19T01:42:47.789 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.785+0000 7f1aa662a740 1 -- 172.21.15.99:0/3732985268 wait complete. 2024-03-19T01:42:47.789 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.785+0000 7f1aa662a740 1 -- start start 2024-03-19T01:42:47.790 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.785+0000 7f1aa662a740 1 Processor -- start 2024-03-19T01:42:47.790 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.785+0000 7f1aa662a740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56345a373100 0x56345a381d20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:47.790 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.785+0000 7f1aa662a740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56345a373af0 0x56345a382260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:47.790 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.785+0000 7f1aa662a740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56345a378c60 0x56345a383270 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:47.790 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.785+0000 7f1aa662a740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f1a9c1a7460 con 0x56345a373af0 2024-03-19T01:42:47.790 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.785+0000 7f1aa662a740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f1a9c1a88d0 con 0x56345a378c60 2024-03-19T01:42:47.790 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.785+0000 7f1aa662a740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f1a9c1a7b30 con 0x56345a373100 2024-03-19T01:42:47.790 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.785+0000 7f1aa662a740 10 client.0 _ll_register_callbacks cb 0x7f1a8c009930 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:42:47.790 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.785+0000 7f1aa0f03640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56345a373af0 0x56345a382260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:47.790 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.786+0000 7f1aa1704640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56345a373100 0x56345a381d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:47.790 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.786+0000 7f1aa0f03640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56345a373af0 0x56345a382260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:48850/0 (socket says 172.21.15.99:48850) 2024-03-19T01:42:47.790 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.786+0000 7f1aa0f03640 1 -- 172.21.15.99:0/1905280150 learned_addr learned my addr 172.21.15.99:0/1905280150 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:42:47.790 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.786+0000 7f1aa1f05640 1 --2- 172.21.15.99:0/1905280150 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56345a378c60 0x56345a383270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:47.791 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.786+0000 7f1aa0f03640 1 -- 172.21.15.99:0/1905280150 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56345a373100 msgr2=0x56345a381d20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:42:47.791 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.786+0000 7f1aa0f03640 1 --2- 172.21.15.99:0/1905280150 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56345a373100 0x56345a381d20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:47.791 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.786+0000 7f1aa0f03640 1 -- 172.21.15.99:0/1905280150 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56345a378c60 msgr2=0x56345a383270 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:42:47.791 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.786+0000 7f1aa0f03640 1 --2- 172.21.15.99:0/1905280150 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56345a378c60 0x56345a383270 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:47.791 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.786+0000 7f1aa0f03640 1 -- 172.21.15.99:0/1905280150 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x56345a383860 con 0x56345a373af0 2024-03-19T01:42:47.791 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.786+0000 7f1aa1704640 1 --2- 172.21.15.99:0/1905280150 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56345a373100 0x56345a381d20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:42:47.791 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.786+0000 7f1aa1f05640 1 --2- 172.21.15.99:0/1905280150 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56345a378c60 0x56345a383270 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:42:47.791 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.786+0000 7f1aa0f03640 1 --2- 172.21.15.99:0/1905280150 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56345a373af0 0x56345a382260 secure :-1 s=READY pgs=391 cs=0 l=1 rev1=1 crypto rx=0x7f1a8c009000 tx=0x7f1a8c010c70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5805d02beb36119b server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:42:47.791 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.786+0000 7f1a84ff9640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:42:47.791 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.786+0000 7f1a84ff9640 1 -- 172.21.15.99:0/1905280150 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1a8c0106f0 con 0x56345a373af0 2024-03-19T01:42:47.791 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.786+0000 7f1a84ff9640 1 -- 172.21.15.99:0/1905280150 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f1a8c005250 con 0x56345a373af0 2024-03-19T01:42:47.791 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.786+0000 7f1aa662a740 1 -- 172.21.15.99:0/1905280150 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f1a8c010890 con 0x56345a373af0 2024-03-19T01:42:47.791 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.786+0000 7f1a84ff9640 1 -- 172.21.15.99:0/1905280150 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=23) ==== 24+0+0 (secure 0 0 0) 0x7f1a8c00fea0 con 0x56345a373af0 2024-03-19T01:42:47.791 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.786+0000 7f1aa662a740 10 client.15474 fetch_fsmap learned FSMap version 23 2024-03-19T01:42:47.792 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.786+0000 7f1aa662a740 1 -- 172.21.15.99:0/1905280150 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=23,monmap=4+}) -- 0x56345a3ef390 con 0x56345a373af0 2024-03-19T01:42:47.792 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.786+0000 7f1a84ff9640 1 -- 172.21.15.99:0/1905280150 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f1a8c010890 con 0x56345a373af0 2024-03-19T01:42:47.792 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.787+0000 7f1a84ff9640 1 -- 172.21.15.99:0/1905280150 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 23) ==== 8346+0+0 (secure 0 0 0) 0x7f1a8c004c70 con 0x56345a373af0 2024-03-19T01:42:47.792 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.787+0000 7f1aa662a740 10 client.15474 fetch_fsmap finished waiting for FSMap version 23 2024-03-19T01:42:47.792 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.787+0000 7f1aa662a740 10 client.15474 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:42:47.792 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.787+0000 7f1aa662a740 20 client.15474 populate_metadata read hostname 'smithi099' 2024-03-19T01:42:47.792 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.787+0000 7f1aa662a740 1 --2- 172.21.15.99:0/1905280150 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56345a3efd10 0x56345a3f2130 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:42:47.792 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.787+0000 7f1aa662a740 4 client.15474 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:42:47.792 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.787+0000 7f1aa662a740 1 -- 172.21.15.99:0/1905280150 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x56345a2df470 con 0x56345a3efd10 2024-03-19T01:42:47.792 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.787+0000 7f1aa1704640 1 --2- 172.21.15.99:0/1905280150 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56345a3efd10 0x56345a3f2130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:42:47.792 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.787+0000 7f1a84ff9640 10 client.15474 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:42:47.792 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.787+0000 7f1aa1704640 1 --2- 172.21.15.99:0/1905280150 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56345a3efd10 0x56345a3f2130 crc :-1 s=READY pgs=44 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=7bbcc55c5d912968 server_cookie=33e5fd2831079fbe in_seq=0 out_seq=0 2024-03-19T01:42:47.850 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.849+0000 7f1a84ff9640 1 -- 172.21.15.99:0/1905280150 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x56345a2df470 con 0x56345a3efd10 2024-03-19T01:42:47.850 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.849+0000 7f1a84ff9640 10 client.15474 handle_command_reply: tid=0 2024-03-19T01:42:47.850 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.849+0000 7f1aa662a740 1 client.15474 shutdown 2024-03-19T01:42:47.850 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.849+0000 7f1aa662a740 1 -- 172.21.15.99:0/1905280150 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56345a373af0 msgr2=0x56345a382260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:42:47.850 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.849+0000 7f1aa662a740 1 --2- 172.21.15.99:0/1905280150 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56345a373af0 0x56345a382260 secure :-1 s=READY pgs=391 cs=0 l=1 rev1=1 crypto rx=0x7f1a8c009000 tx=0x7f1a8c010c70 comp rx=0 tx=0).stop 2024-03-19T01:42:47.850 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.849+0000 7f1aa662a740 1 -- 172.21.15.99:0/1905280150 shutdown_connections 2024-03-19T01:42:47.850 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.849+0000 7f1aa662a740 1 --2- 172.21.15.99:0/1905280150 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56345a3efd10 0x56345a3f2130 crc :-1 s=READY pgs=44 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:47.850 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.849+0000 7f1aa662a740 1 --2- 172.21.15.99:0/1905280150 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56345a378c60 0x56345a383270 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:47.850 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.849+0000 7f1aa662a740 1 --2- 172.21.15.99:0/1905280150 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56345a373af0 0x56345a382260 unknown :-1 s=CLOSED pgs=391 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:47.850 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.849+0000 7f1a84ff9640 0 client.15474 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:42:47.851 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.849+0000 7f1aa662a740 1 --2- 172.21.15.99:0/1905280150 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56345a373100 0x56345a381d20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:47.851 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.849+0000 7f1aa662a740 1 -- 172.21.15.99:0/1905280150 >> 172.21.15.99:0/1905280150 conn(0x56345a36e670 msgr2=0x56345a374590 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:42:47.851 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.849+0000 7f1aa662a740 1 -- 172.21.15.99:0/1905280150 shutdown_connections 2024-03-19T01:42:47.851 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.849+0000 7f1aa662a740 1 -- 172.21.15.99:0/1905280150 wait complete. 2024-03-19T01:42:47.851 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.849+0000 7f1aa662a740 20 client.15474 trim_cache size 0 max 16384 2024-03-19T01:42:47.851 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:42:47.851 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:42:47.851 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:42:47.851 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:42:47.852 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.850+0000 7f1aa3968640 1 -- 172.21.15.99:0/1372918157 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f1a880a28a0 msgr2=0x7f1a880a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:42:47.852 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.850+0000 7f1aa3968640 1 --2- 172.21.15.99:0/1372918157 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f1a880a28a0 0x7f1a880a4d60 secure :-1 s=READY pgs=255 cs=0 l=1 rev1=1 crypto rx=0x56345a374350 tx=0x7f1a94009500 comp rx=0 tx=0).stop 2024-03-19T01:42:47.852 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.850+0000 7f1aa3968640 1 -- 172.21.15.99:0/1372918157 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1a9c100740 msgr2=0x7f1a9c069a30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:42:47.852 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.850+0000 7f1aa3968640 1 --2- 172.21.15.99:0/1372918157 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1a9c100740 0x7f1a9c069a30 secure :-1 s=READY pgs=387 cs=0 l=1 rev1=1 crypto rx=0x7f1a98000c00 tx=0x7f1a9802c640 comp rx=0 tx=0).stop 2024-03-19T01:42:47.853 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.850+0000 7f1aa3968640 1 -- 172.21.15.99:0/1372918157 shutdown_connections 2024-03-19T01:42:47.853 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.850+0000 7f1aa3968640 1 --2- 172.21.15.99:0/1372918157 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f1a880a28a0 0x7f1a880a4d60 unknown :-1 s=CLOSED pgs=255 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:47.853 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.850+0000 7f1aa3968640 1 --2- 172.21.15.99:0/1372918157 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1a9c101930 0x7f1a9c19c830 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:47.853 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.850+0000 7f1aa3968640 1 --2- 172.21.15.99:0/1372918157 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1a9c1022f0 0x7f1a9c19ee70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:47.853 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.850+0000 7f1aa3968640 1 --2- 172.21.15.99:0/1372918157 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1a9c100740 0x7f1a9c069a30 unknown :-1 s=CLOSED pgs=387 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:42:47.853 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.850+0000 7f1aa3968640 1 -- 172.21.15.99:0/1372918157 >> 172.21.15.99:0/1372918157 conn(0x7f1a9c0fbcb0 msgr2=0x7f1a9c104c30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:42:47.853 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.850+0000 7f1aa3968640 1 -- 172.21.15.99:0/1372918157 shutdown_connections 2024-03-19T01:42:47.853 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:42:47.850+0000 7f1aa3968640 1 -- 172.21.15.99:0/1372918157 wait complete. 2024-03-19T01:42:48.309 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:42:47 smithi115 ceph-mon[67332]: pgmap v401: 97 pgs: 97 active+clean; 78 MiB data, 818 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 MiB/s rd, 3.1 MiB/s wr, 265 op/s 2024-03-19T01:42:48.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:42:47 smithi099 ceph-mon[62951]: pgmap v401: 97 pgs: 97 active+clean; 78 MiB data, 818 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 MiB/s rd, 3.1 MiB/s wr, 265 op/s 2024-03-19T01:42:48.363 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:42:47 smithi112 ceph-mon[66819]: pgmap v401: 97 pgs: 97 active+clean; 78 MiB data, 818 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 MiB/s rd, 3.1 MiB/s wr, 265 op/s 2024-03-19T01:42:48.676 INFO:tasks.workunit.client.0.smithi099.stdout: 1 175073 12.31 MB/sec warmup 78 sec latency 18.740 ms 2024-03-19T01:42:49.676 INFO:tasks.workunit.client.0.smithi099.stdout: 1 177751 12.37 MB/sec warmup 79 sec latency 18.262 ms 2024-03-19T01:42:50.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:42:49 smithi115 ceph-mon[67332]: pgmap v402: 97 pgs: 97 active+clean; 83 MiB data, 836 MiB used, 1.0 TiB / 1.0 TiB avail; 3.1 MiB/s rd, 3.3 MiB/s wr, 310 op/s 2024-03-19T01:42:50.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:42:49 smithi099 ceph-mon[62951]: pgmap v402: 97 pgs: 97 active+clean; 83 MiB data, 836 MiB used, 1.0 TiB / 1.0 TiB avail; 3.1 MiB/s rd, 3.3 MiB/s wr, 310 op/s 2024-03-19T01:42:50.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:42:49 smithi112 ceph-mon[66819]: pgmap v402: 97 pgs: 97 active+clean; 83 MiB data, 836 MiB used, 1.0 TiB / 1.0 TiB avail; 3.1 MiB/s rd, 3.3 MiB/s wr, 310 op/s 2024-03-19T01:42:50.676 INFO:tasks.workunit.client.0.smithi099.stdout: 1 179967 12.31 MB/sec warmup 80 sec latency 36.111 ms 2024-03-19T01:42:51.676 INFO:tasks.workunit.client.0.smithi099.stdout: 1 182096 12.32 MB/sec warmup 81 sec latency 50.194 ms 2024-03-19T01:42:52.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:42:51 smithi115 ceph-mon[67332]: pgmap v403: 97 pgs: 97 active+clean; 84 MiB data, 839 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 MiB/s rd, 3.2 MiB/s wr, 272 op/s 2024-03-19T01:42:52.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:42:51 smithi099 ceph-mon[62951]: pgmap v403: 97 pgs: 97 active+clean; 84 MiB data, 839 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 MiB/s rd, 3.2 MiB/s wr, 272 op/s 2024-03-19T01:42:52.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:42:51 smithi112 ceph-mon[66819]: pgmap v403: 97 pgs: 97 active+clean; 84 MiB data, 839 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 MiB/s rd, 3.2 MiB/s wr, 272 op/s 2024-03-19T01:42:52.676 INFO:tasks.workunit.client.0.smithi099.stdout: 1 184719 12.38 MB/sec warmup 82 sec latency 31.106 ms 2024-03-19T01:42:53.676 INFO:tasks.workunit.client.0.smithi099.stdout: 1 186142 12.30 MB/sec warmup 83 sec latency 52.382 ms 2024-03-19T01:42:54.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:42:53 smithi115 ceph-mon[67332]: pgmap v404: 97 pgs: 97 active+clean; 81 MiB data, 826 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 MiB/s rd, 3.3 MiB/s wr, 293 op/s 2024-03-19T01:42:54.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:42:53 smithi099 ceph-mon[62951]: pgmap v404: 97 pgs: 97 active+clean; 81 MiB data, 826 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 MiB/s rd, 3.3 MiB/s wr, 293 op/s 2024-03-19T01:42:54.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:42:53 smithi112 ceph-mon[66819]: pgmap v404: 97 pgs: 97 active+clean; 81 MiB data, 826 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 MiB/s rd, 3.3 MiB/s wr, 293 op/s 2024-03-19T01:42:54.676 INFO:tasks.workunit.client.0.smithi099.stdout: 1 188556 12.32 MB/sec warmup 84 sec latency 9.616 ms 2024-03-19T01:42:55.676 INFO:tasks.workunit.client.0.smithi099.stdout: 1 190934 12.32 MB/sec warmup 85 sec latency 40.142 ms 2024-03-19T01:42:56.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:42:55 smithi115 ceph-mon[67332]: pgmap v405: 97 pgs: 97 active+clean; 83 MiB data, 835 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 MiB/s rd, 2.8 MiB/s wr, 266 op/s 2024-03-19T01:42:56.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:42:55 smithi099 ceph-mon[62951]: pgmap v405: 97 pgs: 97 active+clean; 83 MiB data, 835 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 MiB/s rd, 2.8 MiB/s wr, 266 op/s 2024-03-19T01:42:56.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:42:55 smithi112 ceph-mon[66819]: pgmap v405: 97 pgs: 97 active+clean; 83 MiB data, 835 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 MiB/s rd, 2.8 MiB/s wr, 266 op/s 2024-03-19T01:42:56.676 INFO:tasks.workunit.client.0.smithi099.stdout: 1 192949 12.31 MB/sec warmup 86 sec latency 28.518 ms 2024-03-19T01:42:57.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:42:56 smithi115 ceph-mon[67332]: pgmap v406: 97 pgs: 97 active+clean; 83 MiB data, 847 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 MiB/s rd, 3.3 MiB/s wr, 293 op/s 2024-03-19T01:42:57.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:42:56 smithi099 ceph-mon[62951]: pgmap v406: 97 pgs: 97 active+clean; 83 MiB data, 847 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 MiB/s rd, 3.3 MiB/s wr, 293 op/s 2024-03-19T01:42:57.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:42:56 smithi112 ceph-mon[66819]: pgmap v406: 97 pgs: 97 active+clean; 83 MiB data, 847 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 MiB/s rd, 3.3 MiB/s wr, 293 op/s 2024-03-19T01:42:57.676 INFO:tasks.workunit.client.0.smithi099.stdout: 1 195401 12.33 MB/sec warmup 87 sec latency 14.094 ms 2024-03-19T01:42:58.676 INFO:tasks.workunit.client.0.smithi099.stdout: 1 196872 12.25 MB/sec warmup 88 sec latency 75.695 ms 2024-03-19T01:42:59.676 INFO:tasks.workunit.client.0.smithi099.stdout: 1 199234 12.27 MB/sec warmup 89 sec latency 27.114 ms 2024-03-19T01:43:00.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:42:59 smithi112 ceph-mon[66819]: pgmap v407: 97 pgs: 97 active+clean; 86 MiB data, 873 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 MiB/s rd, 3.3 MiB/s wr, 334 op/s 2024-03-19T01:43:00.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:42:59 smithi115 ceph-mon[67332]: pgmap v407: 97 pgs: 97 active+clean; 86 MiB data, 873 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 MiB/s rd, 3.3 MiB/s wr, 334 op/s 2024-03-19T01:43:00.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:42:59 smithi099 ceph-mon[62951]: pgmap v407: 97 pgs: 97 active+clean; 86 MiB data, 873 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 MiB/s rd, 3.3 MiB/s wr, 334 op/s 2024-03-19T01:43:00.676 INFO:tasks.workunit.client.0.smithi099.stdout: 1 201825 12.28 MB/sec warmup 90 sec latency 18.250 ms 2024-03-19T01:43:00.943 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.940+0000 7f5bacee8640 1 -- 172.21.15.99:0/279914934 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5ba8102320 msgr2=0x7f5ba8104710 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:43:00.943 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.940+0000 7f5bacee8640 1 --2- 172.21.15.99:0/279914934 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5ba8102320 0x7f5ba8104710 secure :-1 s=READY pgs=392 cs=0 l=1 rev1=1 crypto rx=0x7f5b98009900 tx=0x7f5b9802bc30 comp rx=0 tx=0).stop 2024-03-19T01:43:00.943 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.941+0000 7f5bacee8640 1 -- 172.21.15.99:0/279914934 shutdown_connections 2024-03-19T01:43:00.943 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.941+0000 7f5bacee8640 1 --2- 172.21.15.99:0/279914934 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5ba8104c50 0x7f5ba8107040 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:00.943 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.941+0000 7f5bacee8640 1 --2- 172.21.15.99:0/279914934 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5ba8107580 0x7f5ba8109970 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:00.943 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.941+0000 7f5bacee8640 1 --2- 172.21.15.99:0/279914934 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5ba8102320 0x7f5ba8104710 unknown :-1 s=CLOSED pgs=392 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:00.943 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.941+0000 7f5bacee8640 1 -- 172.21.15.99:0/279914934 >> 172.21.15.99:0/279914934 conn(0x7f5ba80fbcb0 msgr2=0x7f5ba80fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:43:00.943 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.941+0000 7f5bacee8640 1 -- 172.21.15.99:0/279914934 shutdown_connections 2024-03-19T01:43:00.943 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.941+0000 7f5bacee8640 1 -- 172.21.15.99:0/279914934 wait complete. 2024-03-19T01:43:00.943 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.941+0000 7f5bacee8640 1 Processor -- start 2024-03-19T01:43:00.943 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.941+0000 7f5bacee8640 1 -- start start 2024-03-19T01:43:00.943 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.941+0000 7f5bacee8640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5ba8102320 0x7f5ba8197ad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:00.944 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.941+0000 7f5bacee8640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5ba8104c50 0x7f5ba8198010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:00.944 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.942+0000 7f5bacee8640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5ba8107580 0x7f5ba819f080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:00.944 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.942+0000 7f5bacee8640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f5ba810d4e0 con 0x7f5ba8102320 2024-03-19T01:43:00.944 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.942+0000 7f5bacee8640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f5ba810d360 con 0x7f5ba8107580 2024-03-19T01:43:00.944 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.942+0000 7f5bacee8640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f5ba810d660 con 0x7f5ba8104c50 2024-03-19T01:43:00.944 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.942+0000 7f5ba659c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5ba8102320 0x7f5ba8197ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:00.944 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.942+0000 7f5ba5d9b640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5ba8104c50 0x7f5ba8198010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:00.944 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.942+0000 7f5ba659c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5ba8102320 0x7f5ba8197ad0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:40490/0 (socket says 172.21.15.99:40490) 2024-03-19T01:43:00.944 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.942+0000 7f5ba659c640 1 -- 172.21.15.99:0/2821357472 learned_addr learned my addr 172.21.15.99:0/2821357472 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:43:00.944 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.942+0000 7f5ba6d9d640 1 --2- 172.21.15.99:0/2821357472 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5ba8107580 0x7f5ba819f080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:00.944 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.942+0000 7f5ba6d9d640 1 -- 172.21.15.99:0/2821357472 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5ba8104c50 msgr2=0x7f5ba8198010 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:43:00.944 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.942+0000 7f5ba6d9d640 1 --2- 172.21.15.99:0/2821357472 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5ba8104c50 0x7f5ba8198010 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:00.944 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.942+0000 7f5ba6d9d640 1 -- 172.21.15.99:0/2821357472 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5ba8102320 msgr2=0x7f5ba8197ad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:43:00.944 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.942+0000 7f5ba6d9d640 1 --2- 172.21.15.99:0/2821357472 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5ba8102320 0x7f5ba8197ad0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:00.945 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.942+0000 7f5ba6d9d640 1 -- 172.21.15.99:0/2821357472 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f5ba819f6f0 con 0x7f5ba8107580 2024-03-19T01:43:00.945 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.942+0000 7f5ba659c640 1 --2- 172.21.15.99:0/2821357472 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5ba8102320 0x7f5ba8197ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:43:00.945 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.942+0000 7f5ba5d9b640 1 --2- 172.21.15.99:0/2821357472 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5ba8104c50 0x7f5ba8198010 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:43:00.945 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.942+0000 7f5ba6d9d640 1 --2- 172.21.15.99:0/2821357472 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5ba8107580 0x7f5ba819f080 secure :-1 s=READY pgs=92 cs=0 l=1 rev1=1 crypto rx=0x7f5ba0009fd0 tx=0x7f5ba000ef10 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:43:00.945 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.942+0000 7f5b977fe640 1 -- 172.21.15.99:0/2821357472 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5ba000ce20 con 0x7f5ba8107580 2024-03-19T01:43:00.945 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.942+0000 7f5bacee8640 1 -- 172.21.15.99:0/2821357472 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f5ba819f9e0 con 0x7f5ba8107580 2024-03-19T01:43:00.945 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.942+0000 7f5b977fe640 1 -- 172.21.15.99:0/2821357472 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f5ba0005020 con 0x7f5ba8107580 2024-03-19T01:43:00.945 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.942+0000 7f5bacee8640 1 -- 172.21.15.99:0/2821357472 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f5ba819ff80 con 0x7f5ba8107580 2024-03-19T01:43:00.945 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.942+0000 7f5b977fe640 1 -- 172.21.15.99:0/2821357472 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5ba0010720 con 0x7f5ba8107580 2024-03-19T01:43:00.950 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.945+0000 7f5b977fe640 1 -- 172.21.15.99:0/2821357472 <== mon.2 v2:172.21.15.112:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f5ba0005540 con 0x7f5ba8107580 2024-03-19T01:43:00.950 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.945+0000 7f5bafbaf740 1 Processor -- start 2024-03-19T01:43:00.950 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.945+0000 7f5bafbaf740 1 -- start start 2024-03-19T01:43:00.950 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.945+0000 7f5bafbaf740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e67aba4100 0x55e67aba4520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:00.950 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.945+0000 7f5b977fe640 1 --2- 172.21.15.99:0/2821357472 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f5b900a2920 0x7f5b900a4de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:00.950 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.945+0000 7f5bafbaf740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e67aba4af0 0x55e67aba95f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:00.950 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.945+0000 7f5bafbaf740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e67abae070 0x55e67abb0500 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:00.950 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.945+0000 7f5bafbaf740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55e67ab08be0 con 0x55e67aba4af0 2024-03-19T01:43:00.950 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.945+0000 7f5bafbaf740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55e67aab1960 con 0x55e67aba4100 2024-03-19T01:43:00.950 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.945+0000 7f5bafbaf740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55e67ab04c80 con 0x55e67abae070 2024-03-19T01:43:00.950 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.945+0000 7f5ba659c640 1 --2- 172.21.15.99:0/2821357472 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f5b900a2920 0x7f5b900a4de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:00.950 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.945+0000 7f5ba5d9b640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e67aba4100 0x55e67aba4520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:00.951 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.945+0000 7f5ba5d9b640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e67aba4100 0x55e67aba4520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:32946/0 (socket says 172.21.15.99:32946) 2024-03-19T01:43:00.951 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.945+0000 7f5ba5d9b640 1 -- 172.21.15.99:0/2333210510 learned_addr learned my addr 172.21.15.99:0/2333210510 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:43:00.951 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.945+0000 7f5ba6d9d640 1 --2- 172.21.15.99:0/2333210510 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e67abae070 0x55e67abb0500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:00.951 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.946+0000 7f5b977fe640 1 -- 172.21.15.99:0/2821357472 <== mon.2 v2:172.21.15.112:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f5ba00cf030 con 0x7f5ba8107580 2024-03-19T01:43:00.951 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.946+0000 7f5ba5d9b640 1 --2- 172.21.15.99:0/2333210510 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e67aba4af0 0x55e67aba95f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:00.951 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.946+0000 7f5ba5d9b640 1 -- 172.21.15.99:0/2333210510 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e67abae070 msgr2=0x55e67abb0500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:43:00.951 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.946+0000 7f5ba5d9b640 1 --2- 172.21.15.99:0/2333210510 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e67abae070 0x55e67abb0500 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:00.951 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.946+0000 7f5ba5d9b640 1 -- 172.21.15.99:0/2333210510 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e67aba4af0 msgr2=0x55e67aba95f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:43:00.951 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.946+0000 7f5ba5d9b640 1 --2- 172.21.15.99:0/2333210510 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e67aba4af0 0x55e67aba95f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:00.951 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.946+0000 7f5ba5d9b640 1 -- 172.21.15.99:0/2333210510 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55e67aaa8d20 con 0x55e67aba4100 2024-03-19T01:43:00.951 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.946+0000 7f5ba659c640 1 --2- 172.21.15.99:0/2821357472 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f5b900a2920 0x7f5b900a4de0 secure :-1 s=READY pgs=256 cs=0 l=1 rev1=1 crypto rx=0x7f5b98000c00 tx=0x7f5b9802c880 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:43:00.951 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.946+0000 7f5ba5d9b640 1 --2- 172.21.15.99:0/2333210510 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e67aba4100 0x55e67aba4520 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7f5b9c011b80 tx=0x7f5b9c0046c0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=a9b63675cc8c99bc server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:43:00.951 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.946+0000 7f5b94ff9640 1 -- 172.21.15.99:0/2333210510 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5b9c00fc50 con 0x55e67aba4100 2024-03-19T01:43:00.951 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.946+0000 7f5b94ff9640 1 -- 172.21.15.99:0/2333210510 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f5b9c00fdf0 con 0x55e67aba4100 2024-03-19T01:43:00.952 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.946+0000 7f5b94ff9640 1 -- 172.21.15.99:0/2333210510 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5b9c003ab0 con 0x55e67aba4100 2024-03-19T01:43:00.952 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.947+0000 7f5bafbaf740 1 -- 172.21.15.99:0/2333210510 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e67aba4100 msgr2=0x55e67aba4520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:43:00.952 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.947+0000 7f5bafbaf740 1 --2- 172.21.15.99:0/2333210510 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e67aba4100 0x55e67aba4520 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7f5b9c011b80 tx=0x7f5b9c0046c0 comp rx=0 tx=0).stop 2024-03-19T01:43:00.952 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.947+0000 7f5bafbaf740 1 -- 172.21.15.99:0/2333210510 shutdown_connections 2024-03-19T01:43:00.952 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.947+0000 7f5bafbaf740 1 --2- 172.21.15.99:0/2333210510 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e67abae070 0x55e67abb0500 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:00.952 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.947+0000 7f5bafbaf740 1 --2- 172.21.15.99:0/2333210510 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e67aba4af0 0x55e67aba95f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:00.952 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.947+0000 7f5bafbaf740 1 --2- 172.21.15.99:0/2333210510 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e67aba4100 0x55e67aba4520 unknown :-1 s=CLOSED pgs=93 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:00.952 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.947+0000 7f5bafbaf740 1 -- 172.21.15.99:0/2333210510 >> 172.21.15.99:0/2333210510 conn(0x55e67ab9f670 msgr2=0x55e67ab9fa70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:43:00.952 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.947+0000 7f5bafbaf740 1 -- 172.21.15.99:0/2333210510 shutdown_connections 2024-03-19T01:43:00.952 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.947+0000 7f5bafbaf740 1 -- 172.21.15.99:0/2333210510 wait complete. 2024-03-19T01:43:00.952 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.947+0000 7f5bafbaf740 1 -- start start 2024-03-19T01:43:00.952 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.947+0000 7f5bafbaf740 1 Processor -- start 2024-03-19T01:43:00.952 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.948+0000 7f5bafbaf740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e67aba4100 0x55e67abb24e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:00.953 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.948+0000 7f5bafbaf740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e67aba4af0 0x55e67abb2a20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:00.953 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.948+0000 7f5bafbaf740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e67abae070 0x55e67abb9a60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:00.953 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.948+0000 7f5bafbaf740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55e67abb1020 con 0x55e67abae070 2024-03-19T01:43:00.953 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.948+0000 7f5bafbaf740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55e67aac4db0 con 0x55e67aba4100 2024-03-19T01:43:00.953 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.948+0000 7f5bafbaf740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55e67abb11a0 con 0x55e67aba4af0 2024-03-19T01:43:00.953 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.948+0000 7f5bafbaf740 10 client.0 _ll_register_callbacks cb 0x55e67aac9e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:43:00.953 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.948+0000 7f5ba6d9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e67abae070 0x55e67abb9a60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:00.953 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.948+0000 7f5ba6d9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e67abae070 0x55e67abb9a60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:40502/0 (socket says 172.21.15.99:40502) 2024-03-19T01:43:00.953 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.948+0000 7f5ba5d9b640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e67aba4af0 0x55e67abb2a20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:00.953 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.948+0000 7f5ba6d9d640 1 -- 172.21.15.99:0/3963673387 learned_addr learned my addr 172.21.15.99:0/3963673387 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:43:00.953 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.948+0000 7f5ba659c640 1 --2- 172.21.15.99:0/3963673387 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e67aba4100 0x55e67abb24e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:00.953 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.948+0000 7f5ba6d9d640 1 -- 172.21.15.99:0/3963673387 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e67aba4af0 msgr2=0x55e67abb2a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:43:00.954 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.948+0000 7f5ba6d9d640 1 --2- 172.21.15.99:0/3963673387 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e67aba4af0 0x55e67abb2a20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:00.954 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.948+0000 7f5ba6d9d640 1 -- 172.21.15.99:0/3963673387 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e67aba4100 msgr2=0x55e67abb24e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:43:00.954 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.948+0000 7f5ba6d9d640 1 --2- 172.21.15.99:0/3963673387 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e67aba4100 0x55e67abb24e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:00.954 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.948+0000 7f5ba6d9d640 1 -- 172.21.15.99:0/3963673387 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55e67aac3c60 con 0x55e67abae070 2024-03-19T01:43:00.954 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.948+0000 7f5ba6d9d640 1 --2- 172.21.15.99:0/3963673387 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e67abae070 0x55e67abb9a60 secure :-1 s=READY pgs=393 cs=0 l=1 rev1=1 crypto rx=0x7f5ba009b690 tx=0x7f5ba009c1d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f10cee674bbfa9d0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:43:00.954 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.948+0000 7f5b8dffb640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:43:00.954 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.948+0000 7f5b8dffb640 1 -- 172.21.15.99:0/3963673387 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5ba00a4130 con 0x55e67abae070 2024-03-19T01:43:00.954 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.948+0000 7f5b8dffb640 1 -- 172.21.15.99:0/3963673387 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f5ba00051d0 con 0x55e67abae070 2024-03-19T01:43:00.954 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.948+0000 7f5bafbaf740 1 -- 172.21.15.99:0/3963673387 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f5b9c00fdf0 con 0x55e67abae070 2024-03-19T01:43:00.954 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.948+0000 7f5b8dffb640 1 -- 172.21.15.99:0/3963673387 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=23) ==== 24+0+0 (secure 0 0 0) 0x7f5ba00b7170 con 0x55e67abae070 2024-03-19T01:43:00.954 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.948+0000 7f5bafbaf740 10 client.15486 fetch_fsmap learned FSMap version 23 2024-03-19T01:43:00.954 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.948+0000 7f5bafbaf740 1 -- 172.21.15.99:0/3963673387 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=23,monmap=4+}) -- 0x55e67a81d210 con 0x55e67abae070 2024-03-19T01:43:00.954 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.949+0000 7f5b8dffb640 1 -- 172.21.15.99:0/3963673387 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f5b9c00fdf0 con 0x55e67abae070 2024-03-19T01:43:00.954 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.949+0000 7f5b8dffb640 1 -- 172.21.15.99:0/3963673387 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 23) ==== 8346+0+0 (secure 0 0 0) 0x7f5ba00a3ca0 con 0x55e67abae070 2024-03-19T01:43:00.955 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.949+0000 7f5bafbaf740 10 client.15486 fetch_fsmap finished waiting for FSMap version 23 2024-03-19T01:43:00.955 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.949+0000 7f5bafbaf740 10 client.15486 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:43:00.955 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.949+0000 7f5bafbaf740 20 client.15486 populate_metadata read hostname 'smithi099' 2024-03-19T01:43:00.955 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.949+0000 7f5bafbaf740 1 --2- 172.21.15.99:0/3963673387 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e67abbe3c0 0x55e67abc07d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:00.955 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.949+0000 7f5bafbaf740 4 client.15486 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:43:00.955 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.949+0000 7f5bafbaf740 1 -- 172.21.15.99:0/3963673387 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55e67ab16120 con 0x55e67abbe3c0 2024-03-19T01:43:00.955 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.949+0000 7f5ba659c640 1 --2- 172.21.15.99:0/3963673387 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e67abbe3c0 0x55e67abc07d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:00.955 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.949+0000 7f5ba659c640 1 --2- 172.21.15.99:0/3963673387 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e67abbe3c0 0x55e67abc07d0 crc :-1 s=READY pgs=45 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=cf4f4e5b8208de16 server_cookie=195b1082edc62a87 in_seq=0 out_seq=0 2024-03-19T01:43:00.955 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.949+0000 7f5b8dffb640 10 client.15486 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:43:00.956 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.950+0000 7f5b8dffb640 1 -- 172.21.15.99:0/3963673387 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x55e67ab16120 con 0x55e67abbe3c0 2024-03-19T01:43:00.956 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.950+0000 7f5b8dffb640 10 client.15486 handle_command_reply: tid=0 2024-03-19T01:43:00.956 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.950+0000 7f5bafbaf740 1 client.15486 shutdown 2024-03-19T01:43:00.956 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.951+0000 7f5bafbaf740 1 -- 172.21.15.99:0/3963673387 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e67abae070 msgr2=0x55e67abb9a60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:43:00.956 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.951+0000 7f5bafbaf740 1 --2- 172.21.15.99:0/3963673387 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e67abae070 0x55e67abb9a60 secure :-1 s=READY pgs=393 cs=0 l=1 rev1=1 crypto rx=0x7f5ba009b690 tx=0x7f5ba009c1d0 comp rx=0 tx=0).stop 2024-03-19T01:43:00.956 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.951+0000 7f5bafbaf740 1 -- 172.21.15.99:0/3963673387 shutdown_connections 2024-03-19T01:43:00.956 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.951+0000 7f5bafbaf740 1 --2- 172.21.15.99:0/3963673387 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e67abbe3c0 0x55e67abc07d0 crc :-1 s=READY pgs=45 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:00.956 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.951+0000 7f5bafbaf740 1 --2- 172.21.15.99:0/3963673387 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e67abae070 0x55e67abb9a60 unknown :-1 s=CLOSED pgs=393 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:00.956 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.951+0000 7f5bafbaf740 1 --2- 172.21.15.99:0/3963673387 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e67aba4af0 0x55e67abb2a20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:00.957 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.951+0000 7f5bafbaf740 1 --2- 172.21.15.99:0/3963673387 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e67aba4100 0x55e67abb24e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:00.957 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.951+0000 7f5b8dffb640 0 client.15486 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:43:00.957 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.951+0000 7f5bafbaf740 1 -- 172.21.15.99:0/3963673387 >> 172.21.15.99:0/3963673387 conn(0x55e67ab9f670 msgr2=0x55e67aba08c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:43:00.957 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.951+0000 7f5bafbaf740 1 -- 172.21.15.99:0/3963673387 shutdown_connections 2024-03-19T01:43:00.957 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.951+0000 7f5bafbaf740 1 -- 172.21.15.99:0/3963673387 wait complete. 2024-03-19T01:43:00.957 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.951+0000 7f5bafbaf740 20 client.15486 trim_cache size 0 max 16384 2024-03-19T01:43:00.967 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.966+0000 7f5bafbaf740 1 Processor -- start 2024-03-19T01:43:00.967 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.966+0000 7f5bafbaf740 1 -- start start 2024-03-19T01:43:00.967 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.966+0000 7f5bafbaf740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e67aba4100 0x55e67ab9e220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:00.967 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.966+0000 7f5bafbaf740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e67aba4af0 0x55e67abb9a20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:00.967 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.966+0000 7f5bafbaf740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e67abae070 0x55e67ab9e760 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:00.967 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.966+0000 7f5bafbaf740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f5ba81a9440 con 0x55e67aba4100 2024-03-19T01:43:00.968 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.966+0000 7f5bafbaf740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f5ba81a6d10 con 0x55e67abae070 2024-03-19T01:43:00.968 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.966+0000 7f5bafbaf740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f5ba81a4b60 con 0x55e67aba4af0 2024-03-19T01:43:00.968 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.966+0000 7f5ba659c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e67aba4100 0x55e67ab9e220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:00.968 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.966+0000 7f5ba5d9b640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e67aba4af0 0x55e67abb9a20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:00.968 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.966+0000 7f5ba659c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e67aba4100 0x55e67ab9e220 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:40510/0 (socket says 172.21.15.99:40510) 2024-03-19T01:43:00.968 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.966+0000 7f5ba659c640 1 -- 172.21.15.99:0/844385557 learned_addr learned my addr 172.21.15.99:0/844385557 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:43:00.968 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.966+0000 7f5ba6d9d640 1 --2- 172.21.15.99:0/844385557 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e67abae070 0x55e67ab9e760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:00.968 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.967+0000 7f5ba6d9d640 1 -- 172.21.15.99:0/844385557 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e67aba4af0 msgr2=0x55e67abb9a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:43:00.968 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.967+0000 7f5ba6d9d640 1 --2- 172.21.15.99:0/844385557 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e67aba4af0 0x55e67abb9a20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:00.968 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.967+0000 7f5ba6d9d640 1 -- 172.21.15.99:0/844385557 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e67aba4100 msgr2=0x55e67ab9e220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:43:00.968 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.967+0000 7f5ba6d9d640 1 --2- 172.21.15.99:0/844385557 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e67aba4100 0x55e67ab9e220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:00.968 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.967+0000 7f5ba6d9d640 1 -- 172.21.15.99:0/844385557 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55e67a8e9620 con 0x55e67abae070 2024-03-19T01:43:00.968 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.967+0000 7f5ba659c640 1 --2- 172.21.15.99:0/844385557 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e67aba4100 0x55e67ab9e220 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:43:00.968 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.967+0000 7f5ba5d9b640 1 --2- 172.21.15.99:0/844385557 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e67aba4af0 0x55e67abb9a20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:43:00.968 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.967+0000 7f5ba6d9d640 1 --2- 172.21.15.99:0/844385557 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e67abae070 0x55e67ab9e760 secure :-1 s=READY pgs=94 cs=0 l=1 rev1=1 crypto rx=0x7f5ba009b920 tx=0x7f5ba009b890 comp rx=0 tx=0).ready entity=mon.2 client_cookie=685b6948f14e7d64 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:43:00.969 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.967+0000 7f5b8ffff640 1 -- 172.21.15.99:0/844385557 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5ba0099c90 con 0x55e67abae070 2024-03-19T01:43:00.969 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.967+0000 7f5b8ffff640 1 -- 172.21.15.99:0/844385557 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f5ba00a3ca0 con 0x55e67abae070 2024-03-19T01:43:00.969 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.967+0000 7f5b8ffff640 1 -- 172.21.15.99:0/844385557 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5ba00860b0 con 0x55e67abae070 2024-03-19T01:43:00.972 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.968+0000 7f5bafbaf740 1 -- 172.21.15.99:0/844385557 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e67abae070 msgr2=0x55e67ab9e760 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:43:00.972 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.968+0000 7f5bafbaf740 1 --2- 172.21.15.99:0/844385557 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e67abae070 0x55e67ab9e760 secure :-1 s=READY pgs=94 cs=0 l=1 rev1=1 crypto rx=0x7f5ba009b920 tx=0x7f5ba009b890 comp rx=0 tx=0).stop 2024-03-19T01:43:00.972 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.968+0000 7f5bafbaf740 1 -- 172.21.15.99:0/844385557 shutdown_connections 2024-03-19T01:43:00.973 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.968+0000 7f5bafbaf740 1 --2- 172.21.15.99:0/844385557 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e67abae070 0x55e67ab9e760 unknown :-1 s=CLOSED pgs=94 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:00.973 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.968+0000 7f5bafbaf740 1 --2- 172.21.15.99:0/844385557 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e67aba4af0 0x55e67abb9a20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:00.973 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.968+0000 7f5bafbaf740 1 --2- 172.21.15.99:0/844385557 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e67aba4100 0x55e67ab9e220 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:00.973 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.968+0000 7f5bafbaf740 1 -- 172.21.15.99:0/844385557 >> 172.21.15.99:0/844385557 conn(0x55e67ab9f670 msgr2=0x55e67ab9a940 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:43:00.973 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.968+0000 7f5bafbaf740 1 -- 172.21.15.99:0/844385557 shutdown_connections 2024-03-19T01:43:00.973 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.968+0000 7f5bafbaf740 1 -- 172.21.15.99:0/844385557 wait complete. 2024-03-19T01:43:00.973 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.968+0000 7f5bafbaf740 1 -- start start 2024-03-19T01:43:00.973 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.968+0000 7f5bafbaf740 1 Processor -- start 2024-03-19T01:43:00.973 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.968+0000 7f5bafbaf740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e67aba4100 0x55e67abb73e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:00.973 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.968+0000 7f5bafbaf740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e67aba4af0 0x55e67abb7920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:00.973 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.968+0000 7f5bafbaf740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e67abae070 0x55e67abb8930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:00.973 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.968+0000 7f5bafbaf740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f5ba81a51d0 con 0x55e67aba4100 2024-03-19T01:43:00.973 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.968+0000 7f5bafbaf740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f5ba81a6640 con 0x55e67abae070 2024-03-19T01:43:00.973 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.968+0000 7f5bafbaf740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f5ba81a58a0 con 0x55e67aba4af0 2024-03-19T01:43:00.973 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.969+0000 7f5bafbaf740 10 client.0 _ll_register_callbacks cb 0x7f5ba009b760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:43:00.974 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.969+0000 7f5ba659c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e67aba4100 0x55e67abb73e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:00.974 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.969+0000 7f5ba659c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e67aba4100 0x55e67abb73e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:40520/0 (socket says 172.21.15.99:40520) 2024-03-19T01:43:00.974 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.969+0000 7f5ba659c640 1 -- 172.21.15.99:0/2213944853 learned_addr learned my addr 172.21.15.99:0/2213944853 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:43:00.974 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.969+0000 7f5ba5d9b640 1 --2- 172.21.15.99:0/2213944853 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e67aba4af0 0x55e67abb7920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:00.974 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.969+0000 7f5ba6d9d640 1 --2- 172.21.15.99:0/2213944853 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e67abae070 0x55e67abb8930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:00.974 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.969+0000 7f5ba659c640 1 -- 172.21.15.99:0/2213944853 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e67aba4af0 msgr2=0x55e67abb7920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:43:00.974 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.969+0000 7f5ba659c640 1 --2- 172.21.15.99:0/2213944853 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e67aba4af0 0x55e67abb7920 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:00.974 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.969+0000 7f5ba659c640 1 -- 172.21.15.99:0/2213944853 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e67abae070 msgr2=0x55e67abb8930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:43:00.974 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.969+0000 7f5ba659c640 1 --2- 172.21.15.99:0/2213944853 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e67abae070 0x55e67abb8930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:00.974 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.969+0000 7f5ba659c640 1 -- 172.21.15.99:0/2213944853 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55e67abb8f20 con 0x55e67aba4100 2024-03-19T01:43:00.974 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.969+0000 7f5ba659c640 1 --2- 172.21.15.99:0/2213944853 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e67aba4100 0x55e67abb73e0 secure :-1 s=READY pgs=394 cs=0 l=1 rev1=1 crypto rx=0x7f5b98009430 tx=0x7f5b9802e050 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6e91e0829c0f66e2 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:43:00.974 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.969+0000 7f5b8dffb640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:43:00.975 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.969+0000 7f5b8dffb640 1 -- 172.21.15.99:0/2213944853 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5b98057070 con 0x55e67aba4100 2024-03-19T01:43:00.975 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.969+0000 7f5b8dffb640 1 -- 172.21.15.99:0/2213944853 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f5b98053c40 con 0x55e67aba4100 2024-03-19T01:43:00.975 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.969+0000 7f5bafbaf740 1 -- 172.21.15.99:0/2213944853 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f5ba00a3ca0 con 0x55e67aba4100 2024-03-19T01:43:00.975 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.969+0000 7f5b8dffb640 1 -- 172.21.15.99:0/2213944853 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=23) ==== 24+0+0 (secure 0 0 0) 0x7f5b98004d10 con 0x55e67aba4100 2024-03-19T01:43:00.975 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.969+0000 7f5bafbaf740 10 client.15492 fetch_fsmap learned FSMap version 23 2024-03-19T01:43:00.975 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.969+0000 7f5bafbaf740 1 -- 172.21.15.99:0/2213944853 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=23,monmap=4+}) -- 0x55e67ac24a50 con 0x55e67aba4100 2024-03-19T01:43:00.975 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.970+0000 7f5b8dffb640 1 -- 172.21.15.99:0/2213944853 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f5ba00a3ca0 con 0x55e67aba4100 2024-03-19T01:43:00.975 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.970+0000 7f5b8dffb640 1 -- 172.21.15.99:0/2213944853 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 23) ==== 8346+0+0 (secure 0 0 0) 0x7f5b980054b0 con 0x55e67aba4100 2024-03-19T01:43:00.975 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.970+0000 7f5bafbaf740 10 client.15492 fetch_fsmap finished waiting for FSMap version 23 2024-03-19T01:43:00.975 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.970+0000 7f5bafbaf740 10 client.15492 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:43:00.975 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.970+0000 7f5bafbaf740 20 client.15492 populate_metadata read hostname 'smithi099' 2024-03-19T01:43:00.975 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.970+0000 7f5bafbaf740 1 --2- 172.21.15.99:0/2213944853 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e67ac25490 0x55e67ac278b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:00.975 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.970+0000 7f5bafbaf740 4 client.15492 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:43:00.975 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.970+0000 7f5bafbaf740 1 -- 172.21.15.99:0/2213944853 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55e67ab10470 con 0x55e67ac25490 2024-03-19T01:43:00.975 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.970+0000 7f5ba5d9b640 1 --2- 172.21.15.99:0/2213944853 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e67ac25490 0x55e67ac278b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:00.975 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.970+0000 7f5ba5d9b640 1 --2- 172.21.15.99:0/2213944853 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e67ac25490 0x55e67ac278b0 crc :-1 s=READY pgs=46 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=fcee2dccfe4cf9d6 server_cookie=1e72884868d8df57 in_seq=0 out_seq=0 2024-03-19T01:43:00.976 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:00.970+0000 7f5b8dffb640 10 client.15492 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:43:01.022 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:01.021+0000 7f5b8dffb640 1 -- 172.21.15.99:0/2213944853 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x55e67ab10470 con 0x55e67ac25490 2024-03-19T01:43:01.022 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:01.021+0000 7f5b8dffb640 10 client.15492 handle_command_reply: tid=0 2024-03-19T01:43:01.022 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:01.022+0000 7f5bafbaf740 1 client.15492 shutdown 2024-03-19T01:43:01.022 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:01.022+0000 7f5bafbaf740 1 -- 172.21.15.99:0/2213944853 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e67aba4100 msgr2=0x55e67abb73e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:43:01.022 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:01.022+0000 7f5bafbaf740 1 --2- 172.21.15.99:0/2213944853 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e67aba4100 0x55e67abb73e0 secure :-1 s=READY pgs=394 cs=0 l=1 rev1=1 crypto rx=0x7f5b98009430 tx=0x7f5b9802e050 comp rx=0 tx=0).stop 2024-03-19T01:43:01.022 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:01.022+0000 7f5bafbaf740 1 -- 172.21.15.99:0/2213944853 shutdown_connections 2024-03-19T01:43:01.022 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:01.022+0000 7f5bafbaf740 1 --2- 172.21.15.99:0/2213944853 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e67ac25490 0x55e67ac278b0 crc :-1 s=READY pgs=46 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:01.023 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:01.022+0000 7f5bafbaf740 1 --2- 172.21.15.99:0/2213944853 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e67abae070 0x55e67abb8930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:01.023 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:01.022+0000 7f5bafbaf740 1 --2- 172.21.15.99:0/2213944853 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e67aba4af0 0x55e67abb7920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:01.023 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:01.022+0000 7f5bafbaf740 1 --2- 172.21.15.99:0/2213944853 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e67aba4100 0x55e67abb73e0 unknown :-1 s=CLOSED pgs=394 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:01.023 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:01.022+0000 7f5b8dffb640 0 client.15492 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:43:01.023 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:01.022+0000 7f5bafbaf740 1 -- 172.21.15.99:0/2213944853 >> 172.21.15.99:0/2213944853 conn(0x55e67ab9f670 msgr2=0x55e67ab9e220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:43:01.023 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:01.022+0000 7f5bafbaf740 1 -- 172.21.15.99:0/2213944853 shutdown_connections 2024-03-19T01:43:01.023 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:01.022+0000 7f5bafbaf740 1 -- 172.21.15.99:0/2213944853 wait complete. 2024-03-19T01:43:01.023 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:01.022+0000 7f5bafbaf740 20 client.15492 trim_cache size 0 max 16384 2024-03-19T01:43:01.023 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:43:01.023 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:43:01.024 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:43:01.024 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:43:01.024 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:01.023+0000 7f5bacee8640 1 -- 172.21.15.99:0/2821357472 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f5b900a2920 msgr2=0x7f5b900a4de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:43:01.024 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:01.023+0000 7f5bacee8640 1 --2- 172.21.15.99:0/2821357472 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f5b900a2920 0x7f5b900a4de0 secure :-1 s=READY pgs=256 cs=0 l=1 rev1=1 crypto rx=0x7f5b98000c00 tx=0x7f5b9802c880 comp rx=0 tx=0).stop 2024-03-19T01:43:01.024 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:01.023+0000 7f5bacee8640 1 -- 172.21.15.99:0/2821357472 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5ba8107580 msgr2=0x7f5ba819f080 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:43:01.025 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:01.023+0000 7f5bacee8640 1 --2- 172.21.15.99:0/2821357472 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5ba8107580 0x7f5ba819f080 secure :-1 s=READY pgs=92 cs=0 l=1 rev1=1 crypto rx=0x7f5ba0009fd0 tx=0x7f5ba000ef10 comp rx=0 tx=0).stop 2024-03-19T01:43:01.025 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:01.023+0000 7f5bacee8640 1 -- 172.21.15.99:0/2821357472 shutdown_connections 2024-03-19T01:43:01.025 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:01.023+0000 7f5bacee8640 1 --2- 172.21.15.99:0/2821357472 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f5b900a2920 0x7f5b900a4de0 unknown :-1 s=CLOSED pgs=256 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:01.025 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:01.023+0000 7f5bacee8640 1 --2- 172.21.15.99:0/2821357472 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5ba8107580 0x7f5ba819f080 unknown :-1 s=CLOSED pgs=92 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:01.025 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:01.023+0000 7f5bacee8640 1 --2- 172.21.15.99:0/2821357472 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5ba8104c50 0x7f5ba8198010 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:01.025 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:01.023+0000 7f5bacee8640 1 --2- 172.21.15.99:0/2821357472 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5ba8102320 0x7f5ba8197ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:01.025 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:01.023+0000 7f5bacee8640 1 -- 172.21.15.99:0/2821357472 >> 172.21.15.99:0/2821357472 conn(0x7f5ba80fbcb0 msgr2=0x7f5ba8102c60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:43:01.025 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:01.023+0000 7f5bacee8640 1 -- 172.21.15.99:0/2821357472 shutdown_connections 2024-03-19T01:43:01.025 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:01.023+0000 7f5bacee8640 1 -- 172.21.15.99:0/2821357472 wait complete. 2024-03-19T01:43:01.676 INFO:tasks.workunit.client.0.smithi099.stdout: 1 203588 12.26 MB/sec warmup 91 sec latency 21.910 ms 2024-03-19T01:43:02.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:43:01 smithi112 ceph-mon[66819]: pgmap v408: 97 pgs: 97 active+clean; 87 MiB data, 892 MiB used, 1.0 TiB / 1.0 TiB avail; 2.8 MiB/s rd, 3.0 MiB/s wr, 271 op/s 2024-03-19T01:43:02.185 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:43:01 smithi115 ceph-mon[67332]: pgmap v408: 97 pgs: 97 active+clean; 87 MiB data, 892 MiB used, 1.0 TiB / 1.0 TiB avail; 2.8 MiB/s rd, 3.0 MiB/s wr, 271 op/s 2024-03-19T01:43:02.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:43:01 smithi099 ceph-mon[62951]: pgmap v408: 97 pgs: 97 active+clean; 87 MiB data, 892 MiB used, 1.0 TiB / 1.0 TiB avail; 2.8 MiB/s rd, 3.0 MiB/s wr, 271 op/s 2024-03-19T01:43:02.676 INFO:tasks.workunit.client.0.smithi099.stdout: 1 206058 12.28 MB/sec warmup 92 sec latency 11.670 ms 2024-03-19T01:43:03.676 INFO:tasks.workunit.client.0.smithi099.stdout: 1 207575 12.21 MB/sec warmup 93 sec latency 58.670 ms 2024-03-19T01:43:04.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:43:03 smithi112 ceph-mon[66819]: pgmap v409: 97 pgs: 97 active+clean; 87 MiB data, 918 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 MiB/s rd, 3.0 MiB/s wr, 270 op/s 2024-03-19T01:43:04.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:43:03 smithi115 ceph-mon[67332]: pgmap v409: 97 pgs: 97 active+clean; 87 MiB data, 918 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 MiB/s rd, 3.0 MiB/s wr, 270 op/s 2024-03-19T01:43:04.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:43:03 smithi099 ceph-mon[62951]: pgmap v409: 97 pgs: 97 active+clean; 87 MiB data, 918 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 MiB/s rd, 3.0 MiB/s wr, 270 op/s 2024-03-19T01:43:04.676 INFO:tasks.workunit.client.0.smithi099.stdout: 1 209872 12.23 MB/sec warmup 94 sec latency 18.008 ms 2024-03-19T01:43:05.676 INFO:tasks.workunit.client.0.smithi099.stdout: 1 212298 12.23 MB/sec warmup 95 sec latency 43.374 ms 2024-03-19T01:43:06.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:43:05 smithi112 ceph-mon[66819]: pgmap v410: 97 pgs: 97 active+clean; 88 MiB data, 932 MiB used, 1.0 TiB / 1.0 TiB avail; 2.8 MiB/s rd, 2.8 MiB/s wr, 272 op/s 2024-03-19T01:43:06.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:43:05 smithi115 ceph-mon[67332]: pgmap v410: 97 pgs: 97 active+clean; 88 MiB data, 932 MiB used, 1.0 TiB / 1.0 TiB avail; 2.8 MiB/s rd, 2.8 MiB/s wr, 272 op/s 2024-03-19T01:43:06.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:43:05 smithi099 ceph-mon[62951]: pgmap v410: 97 pgs: 97 active+clean; 88 MiB data, 932 MiB used, 1.0 TiB / 1.0 TiB avail; 2.8 MiB/s rd, 2.8 MiB/s wr, 272 op/s 2024-03-19T01:43:06.676 INFO:tasks.workunit.client.0.smithi099.stdout: 1 214230 12.22 MB/sec warmup 96 sec latency 28.726 ms 2024-03-19T01:43:07.676 INFO:tasks.workunit.client.0.smithi099.stdout: 1 216698 12.24 MB/sec warmup 97 sec latency 55.148 ms 2024-03-19T01:43:08.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:43:07 smithi112 ceph-mon[66819]: pgmap v411: 97 pgs: 97 active+clean; 90 MiB data, 960 MiB used, 1.0 TiB / 1.0 TiB avail; 3.0 MiB/s rd, 3.3 MiB/s wr, 313 op/s 2024-03-19T01:43:08.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:43:07 smithi115 ceph-mon[67332]: pgmap v411: 97 pgs: 97 active+clean; 90 MiB data, 960 MiB used, 1.0 TiB / 1.0 TiB avail; 3.0 MiB/s rd, 3.3 MiB/s wr, 313 op/s 2024-03-19T01:43:08.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:43:07 smithi099 ceph-mon[62951]: pgmap v411: 97 pgs: 97 active+clean; 90 MiB data, 960 MiB used, 1.0 TiB / 1.0 TiB avail; 3.0 MiB/s rd, 3.3 MiB/s wr, 313 op/s 2024-03-19T01:43:08.676 INFO:tasks.workunit.client.0.smithi099.stdout: 1 218198 12.17 MB/sec warmup 98 sec latency 94.739 ms 2024-03-19T01:43:09.676 INFO:tasks.workunit.client.0.smithi099.stdout: 1 220512 12.19 MB/sec warmup 99 sec latency 54.759 ms 2024-03-19T01:43:10.143 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:43:09 smithi099 ceph-mon[62951]: pgmap v412: 97 pgs: 97 active+clean; 91 MiB data, 971 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 MiB/s rd, 3.1 MiB/s wr, 295 op/s 2024-03-19T01:43:10.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:43:09 smithi115 ceph-mon[67332]: pgmap v412: 97 pgs: 97 active+clean; 91 MiB data, 971 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 MiB/s rd, 3.1 MiB/s wr, 295 op/s 2024-03-19T01:43:10.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:43:09 smithi112 ceph-mon[66819]: pgmap v412: 97 pgs: 97 active+clean; 91 MiB data, 971 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 MiB/s rd, 3.1 MiB/s wr, 295 op/s 2024-03-19T01:43:10.676 INFO:tasks.workunit.client.0.smithi099.stdout: 1 222724 12.18 MB/sec warmup 100 sec latency 41.838 ms 2024-03-19T01:43:11.676 INFO:tasks.workunit.client.0.smithi099.stdout: 1 224606 12.16 MB/sec warmup 101 sec latency 27.522 ms 2024-03-19T01:43:11.806 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:43:11 smithi115 ceph-mon[67332]: pgmap v413: 97 pgs: 97 active+clean; 92 MiB data, 985 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 MiB/s rd, 2.9 MiB/s wr, 272 op/s 2024-03-19T01:43:11.840 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:43:11 smithi099 ceph-mon[62951]: pgmap v413: 97 pgs: 97 active+clean; 92 MiB data, 985 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 MiB/s rd, 2.9 MiB/s wr, 272 op/s 2024-03-19T01:43:11.861 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:43:11 smithi112 ceph-mon[66819]: pgmap v413: 97 pgs: 97 active+clean; 92 MiB data, 985 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 MiB/s rd, 2.9 MiB/s wr, 272 op/s 2024-03-19T01:43:12.677 INFO:tasks.workunit.client.0.smithi099.stdout: 1 227339 12.20 MB/sec warmup 102 sec latency 11.731 ms 2024-03-19T01:43:13.677 INFO:tasks.workunit.client.0.smithi099.stdout: 1 229352 12.15 MB/sec warmup 103 sec latency 52.374 ms 2024-03-19T01:43:14.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:43:13 smithi112 ceph-mon[66819]: pgmap v414: 97 pgs: 97 active+clean; 93 MiB data, 990 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 MiB/s rd, 3.1 MiB/s wr, 281 op/s 2024-03-19T01:43:14.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:43:13 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:43:14.113 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.112+0000 7fe2b6787640 1 -- 172.21.15.99:0/278775755 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe2b0101980 msgr2=0x7fe2b0101e00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:43:14.113 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.112+0000 7fe2b6787640 1 --2- 172.21.15.99:0/278775755 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe2b0101980 0x7fe2b0101e00 secure :-1 s=READY pgs=395 cs=0 l=1 rev1=1 crypto rx=0x7fe2a4009900 tx=0x7fe2a402bc30 comp rx=0 tx=0).stop 2024-03-19T01:43:14.113 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.112+0000 7fe2b6787640 1 -- 172.21.15.99:0/278775755 shutdown_connections 2024-03-19T01:43:14.113 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.112+0000 7fe2b6787640 1 --2- 172.21.15.99:0/278775755 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fe2b0102340 0x7fe2b0108bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:14.113 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.112+0000 7fe2b6787640 1 --2- 172.21.15.99:0/278775755 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe2b0101980 0x7fe2b0101e00 unknown :-1 s=CLOSED pgs=395 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:14.113 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.112+0000 7fe2b6787640 1 --2- 172.21.15.99:0/278775755 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fe2b0100790 0x7fe2b0100bb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:14.113 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.112+0000 7fe2b6787640 1 -- 172.21.15.99:0/278775755 >> 172.21.15.99:0/278775755 conn(0x7fe2b00fbd00 msgr2=0x7fe2b00fe140 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:43:14.113 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.112+0000 7fe2b6787640 1 -- 172.21.15.99:0/278775755 shutdown_connections 2024-03-19T01:43:14.113 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.112+0000 7fe2b6787640 1 -- 172.21.15.99:0/278775755 wait complete. 2024-03-19T01:43:14.116 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.113+0000 7fe2b6787640 1 Processor -- start 2024-03-19T01:43:14.116 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.113+0000 7fe2b6787640 1 -- start start 2024-03-19T01:43:14.116 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.113+0000 7fe2b6787640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fe2b0100790 0x7fe2b01969c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:14.116 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.113+0000 7fe2b6787640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe2b0101980 0x7fe2b0196f00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:14.116 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.113+0000 7fe2b6787640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fe2b0102340 0x7fe2b019df70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:14.116 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.113+0000 7fe2b6787640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fe2b010c410 con 0x7fe2b0101980 2024-03-19T01:43:14.116 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.113+0000 7fe2b6787640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fe2b010c290 con 0x7fe2b0102340 2024-03-19T01:43:14.116 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.113+0000 7fe2b6787640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fe2b010c590 con 0x7fe2b0100790 2024-03-19T01:43:14.116 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.113+0000 7fe2af7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe2b0101980 0x7fe2b0196f00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:14.116 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.113+0000 7fe2af7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe2b0101980 0x7fe2b0196f00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:56168/0 (socket says 172.21.15.99:56168) 2024-03-19T01:43:14.116 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.113+0000 7fe2af7fe640 1 -- 172.21.15.99:0/760358370 learned_addr learned my addr 172.21.15.99:0/760358370 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:43:14.116 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.113+0000 7fe2affff640 1 --2- 172.21.15.99:0/760358370 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fe2b0100790 0x7fe2b01969c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:14.117 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.113+0000 7fe2b4d24640 1 --2- 172.21.15.99:0/760358370 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fe2b0102340 0x7fe2b019df70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:14.117 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.114+0000 7fe2af7fe640 1 -- 172.21.15.99:0/760358370 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fe2b0100790 msgr2=0x7fe2b01969c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:43:14.117 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.114+0000 7fe2af7fe640 1 --2- 172.21.15.99:0/760358370 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fe2b0100790 0x7fe2b01969c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:14.117 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.114+0000 7fe2af7fe640 1 -- 172.21.15.99:0/760358370 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fe2b0102340 msgr2=0x7fe2b019df70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:43:14.117 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.114+0000 7fe2af7fe640 1 --2- 172.21.15.99:0/760358370 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fe2b0102340 0x7fe2b019df70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:14.117 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.114+0000 7fe2af7fe640 1 -- 172.21.15.99:0/760358370 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fe2b019e5e0 con 0x7fe2b0101980 2024-03-19T01:43:14.117 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.114+0000 7fe2b4d24640 1 --2- 172.21.15.99:0/760358370 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fe2b0102340 0x7fe2b019df70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:43:14.117 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.114+0000 7fe2af7fe640 1 --2- 172.21.15.99:0/760358370 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe2b0101980 0x7fe2b0196f00 secure :-1 s=READY pgs=396 cs=0 l=1 rev1=1 crypto rx=0x7fe2a4009230 tx=0x7fe2a402c650 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:43:14.117 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.114+0000 7fe2ad7fa640 1 -- 172.21.15.99:0/760358370 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe2a4041070 con 0x7fe2b0101980 2024-03-19T01:43:14.117 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.114+0000 7fe2b6787640 1 -- 172.21.15.99:0/760358370 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fe2b019e870 con 0x7fe2b0101980 2024-03-19T01:43:14.117 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.114+0000 7fe2b6787640 1 -- 172.21.15.99:0/760358370 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fe2b019edb0 con 0x7fe2b0101980 2024-03-19T01:43:14.117 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.114+0000 7fe2ad7fa640 1 -- 172.21.15.99:0/760358370 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe2a4004450 con 0x7fe2b0101980 2024-03-19T01:43:14.117 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.114+0000 7fe2ad7fa640 1 -- 172.21.15.99:0/760358370 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe2a404b060 con 0x7fe2b0101980 2024-03-19T01:43:14.120 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.116+0000 7fe2ad7fa640 1 -- 172.21.15.99:0/760358370 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7fe2a4044070 con 0x7fe2b0101980 2024-03-19T01:43:14.121 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.116+0000 7fe2ad7fa640 1 --2- 172.21.15.99:0/760358370 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fe2980a28a0 0x7fe2980a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:14.127 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.117+0000 7fe2affff640 1 --2- 172.21.15.99:0/760358370 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fe2980a28a0 0x7fe2980a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:14.127 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.117+0000 7fe2b9375740 1 Processor -- start 2024-03-19T01:43:14.128 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.117+0000 7fe2ad7fa640 1 -- 172.21.15.99:0/760358370 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7fe2a40f02c0 con 0x7fe2b0101980 2024-03-19T01:43:14.128 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.117+0000 7fe2b9375740 1 -- start start 2024-03-19T01:43:14.128 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.117+0000 7fe2b9375740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5602afcc0780 0x5602afcbeed0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:14.128 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.117+0000 7fe2b9375740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5602afcbf410 0x5602afcbf890 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:14.128 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.117+0000 7fe2b9375740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5602afcc9420 0x5602afccb810 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:14.128 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.117+0000 7fe2b9375740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x5602afc23be0 con 0x5602afcbf410 2024-03-19T01:43:14.128 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.117+0000 7fe2b9375740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x5602afbcc960 con 0x5602afcc0780 2024-03-19T01:43:14.128 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.117+0000 7fe2b9375740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x5602afc1fc80 con 0x5602afcc9420 2024-03-19T01:43:14.128 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.117+0000 7fe2af7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5602afcbf410 0x5602afcbf890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:14.128 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.117+0000 7fe2af7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5602afcbf410 0x5602afcbf890 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:56182/0 (socket says 172.21.15.99:56182) 2024-03-19T01:43:14.128 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.117+0000 7fe2af7fe640 1 -- 172.21.15.99:0/3633070386 learned_addr learned my addr 172.21.15.99:0/3633070386 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:43:14.128 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.117+0000 7fe2affff640 1 --2- 172.21.15.99:0/760358370 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fe2980a28a0 0x7fe2980a4d60 secure :-1 s=READY pgs=257 cs=0 l=1 rev1=1 crypto rx=0x7fe2b01017e0 tx=0x7fe2a0005f90 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:43:14.128 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.117+0000 7fe2b4d24640 1 --2- 172.21.15.99:0/3633070386 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5602afcc9420 0x5602afccb810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:14.129 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.117+0000 7fe2affff640 1 --2- 172.21.15.99:0/3633070386 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5602afcc0780 0x5602afcbeed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:14.129 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.117+0000 7fe2af7fe640 1 -- 172.21.15.99:0/3633070386 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5602afcc9420 msgr2=0x5602afccb810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:43:14.129 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.117+0000 7fe2af7fe640 1 --2- 172.21.15.99:0/3633070386 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5602afcc9420 0x5602afccb810 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:14.129 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.117+0000 7fe2af7fe640 1 -- 172.21.15.99:0/3633070386 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5602afcc0780 msgr2=0x5602afcbeed0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:43:14.129 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.117+0000 7fe2af7fe640 1 --2- 172.21.15.99:0/3633070386 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5602afcc0780 0x5602afcbeed0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:14.129 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.117+0000 7fe2af7fe640 1 -- 172.21.15.99:0/3633070386 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5602afbc3d20 con 0x5602afcbf410 2024-03-19T01:43:14.129 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.117+0000 7fe2affff640 1 --2- 172.21.15.99:0/3633070386 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5602afcc0780 0x5602afcbeed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-19T01:43:14.129 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.117+0000 7fe2af7fe640 1 --2- 172.21.15.99:0/3633070386 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5602afcbf410 0x5602afcbf890 secure :-1 s=READY pgs=397 cs=0 l=1 rev1=1 crypto rx=0x7fe2a402c7b0 tx=0x7fe2a4005e50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=aa88bf64c57a7caa server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:43:14.129 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.118+0000 7fe2967fc640 1 -- 172.21.15.99:0/3633070386 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe2a40c0d70 con 0x5602afcbf410 2024-03-19T01:43:14.129 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.118+0000 7fe2967fc640 1 -- 172.21.15.99:0/3633070386 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe2a40c53c0 con 0x5602afcbf410 2024-03-19T01:43:14.129 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.118+0000 7fe2967fc640 1 -- 172.21.15.99:0/3633070386 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe2a40c5130 con 0x5602afcbf410 2024-03-19T01:43:14.129 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.118+0000 7fe2b9375740 1 -- 172.21.15.99:0/3633070386 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5602afcbf410 msgr2=0x5602afcbf890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:43:14.130 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.118+0000 7fe2b9375740 1 --2- 172.21.15.99:0/3633070386 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5602afcbf410 0x5602afcbf890 secure :-1 s=READY pgs=397 cs=0 l=1 rev1=1 crypto rx=0x7fe2a402c7b0 tx=0x7fe2a4005e50 comp rx=0 tx=0).stop 2024-03-19T01:43:14.130 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.118+0000 7fe2b9375740 1 -- 172.21.15.99:0/3633070386 shutdown_connections 2024-03-19T01:43:14.130 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.118+0000 7fe2b9375740 1 --2- 172.21.15.99:0/3633070386 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5602afcc9420 0x5602afccb810 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:14.130 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.118+0000 7fe2b9375740 1 --2- 172.21.15.99:0/3633070386 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5602afcbf410 0x5602afcbf890 unknown :-1 s=CLOSED pgs=397 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:14.130 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.118+0000 7fe2b9375740 1 --2- 172.21.15.99:0/3633070386 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5602afcc0780 0x5602afcbeed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:14.130 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.118+0000 7fe2b9375740 1 -- 172.21.15.99:0/3633070386 >> 172.21.15.99:0/3633070386 conn(0x5602afcba670 msgr2=0x5602afcbaa70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:43:14.130 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.118+0000 7fe2b9375740 1 -- 172.21.15.99:0/3633070386 shutdown_connections 2024-03-19T01:43:14.130 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.118+0000 7fe2b9375740 1 -- 172.21.15.99:0/3633070386 wait complete. 2024-03-19T01:43:14.130 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.119+0000 7fe2b9375740 1 -- start start 2024-03-19T01:43:14.130 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.119+0000 7fe2b9375740 1 Processor -- start 2024-03-19T01:43:14.131 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.119+0000 7fe2b9375740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5602afcbf410 0x5602afccd200 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:14.131 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.119+0000 7fe2b9375740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5602afcc0780 0x5602afccd740 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:14.131 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.119+0000 7fe2b9375740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5602afcc9420 0x5602afcd4780 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:14.131 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.119+0000 7fe2b9375740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x5602afccbe10 con 0x5602afcc9420 2024-03-19T01:43:14.131 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.119+0000 7fe2b9375740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x5602afbdfdb0 con 0x5602afcbf410 2024-03-19T01:43:14.131 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.119+0000 7fe2b9375740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x5602afccbf90 con 0x5602afcc0780 2024-03-19T01:43:14.131 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.119+0000 7fe2b9375740 10 client.0 _ll_register_callbacks cb 0x5602afbe4e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:43:14.131 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.119+0000 7fe2affff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5602afcbf410 0x5602afccd200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:14.131 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.119+0000 7fe2b4d24640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5602afcc9420 0x5602afcd4780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:14.131 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.119+0000 7fe2affff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5602afcbf410 0x5602afccd200 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:56540/0 (socket says 172.21.15.99:56540) 2024-03-19T01:43:14.132 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.119+0000 7fe2affff640 1 -- 172.21.15.99:0/749273107 learned_addr learned my addr 172.21.15.99:0/749273107 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:43:14.132 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.119+0000 7fe2af7fe640 1 --2- 172.21.15.99:0/749273107 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5602afcc0780 0x5602afccd740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:14.132 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.120+0000 7fe2affff640 1 -- 172.21.15.99:0/749273107 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5602afcc0780 msgr2=0x5602afccd740 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:43:14.132 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.120+0000 7fe2affff640 1 --2- 172.21.15.99:0/749273107 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5602afcc0780 0x5602afccd740 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:14.132 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.120+0000 7fe2affff640 1 -- 172.21.15.99:0/749273107 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5602afcc9420 msgr2=0x5602afcd4780 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:43:14.132 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.120+0000 7fe2affff640 1 --2- 172.21.15.99:0/749273107 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5602afcc9420 0x5602afcd4780 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:14.132 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.120+0000 7fe2affff640 1 -- 172.21.15.99:0/749273107 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5602afbdec60 con 0x5602afcbf410 2024-03-19T01:43:14.132 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.120+0000 7fe2b4d24640 1 --2- 172.21.15.99:0/749273107 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5602afcc9420 0x5602afcd4780 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:43:14.132 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.120+0000 7fe2affff640 1 --2- 172.21.15.99:0/749273107 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5602afcbf410 0x5602afccd200 secure :-1 s=READY pgs=95 cs=0 l=1 rev1=1 crypto rx=0x7fe2a001ed30 tx=0x7fe2a0004730 comp rx=0 tx=0).ready entity=mon.2 client_cookie=6b556c02bcf83690 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:43:14.132 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.120+0000 7fe28f7fe640 10 client.0 ms_handle_connect on v2:172.21.15.112:3300/0 2024-03-19T01:43:14.132 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.120+0000 7fe28f7fe640 1 -- 172.21.15.99:0/749273107 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe2a001e320 con 0x5602afcbf410 2024-03-19T01:43:14.132 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.120+0000 7fe28f7fe640 1 -- 172.21.15.99:0/749273107 <== mon.2 v2:172.21.15.112:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7fe2a000e930 con 0x5602afcbf410 2024-03-19T01:43:14.132 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.120+0000 7fe2b9375740 1 -- 172.21.15.99:0/749273107 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fe2a40c53c0 con 0x5602afcbf410 2024-03-19T01:43:14.133 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.120+0000 7fe28f7fe640 1 -- 172.21.15.99:0/749273107 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_get_version_reply(handle=1 version=23) ==== 24+0+0 (secure 0 0 0) 0x7fe2a000eb30 con 0x5602afcbf410 2024-03-19T01:43:14.133 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.120+0000 7fe2b9375740 10 client.25277 fetch_fsmap learned FSMap version 23 2024-03-19T01:43:14.133 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.120+0000 7fe2b9375740 1 -- 172.21.15.99:0/749273107 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,fsmap=23,monmap=4+}) -- 0x5602af938210 con 0x5602afcbf410 2024-03-19T01:43:14.133 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.120+0000 7fe28f7fe640 1 -- 172.21.15.99:0/749273107 <== mon.2 v2:172.21.15.112:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe2a40c53c0 con 0x5602afcbf410 2024-03-19T01:43:14.133 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.120+0000 7fe28f7fe640 1 -- 172.21.15.99:0/749273107 <== mon.2 v2:172.21.15.112:3300/0 5 ==== fsmap(e 23) ==== 8346+0+0 (secure 0 0 0) 0x7fe2a0025020 con 0x5602afcbf410 2024-03-19T01:43:14.133 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.121+0000 7fe2b9375740 10 client.25277 fetch_fsmap finished waiting for FSMap version 23 2024-03-19T01:43:14.133 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.121+0000 7fe2b9375740 10 client.25277 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:43:14.133 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.121+0000 7fe2b9375740 20 client.25277 populate_metadata read hostname 'smithi099' 2024-03-19T01:43:14.133 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.121+0000 7fe2b9375740 1 --2- 172.21.15.99:0/749273107 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5602afcd9080 0x5602afcdb520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:14.133 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.121+0000 7fe2b9375740 4 client.25277 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:43:14.133 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.121+0000 7fe2b9375740 1 -- 172.21.15.99:0/749273107 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5602afc31120 con 0x5602afcd9080 2024-03-19T01:43:14.133 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.121+0000 7fe2af7fe640 1 --2- 172.21.15.99:0/749273107 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5602afcd9080 0x5602afcdb520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:14.133 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.121+0000 7fe28f7fe640 10 client.25277 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:43:14.133 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.121+0000 7fe2af7fe640 1 --2- 172.21.15.99:0/749273107 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5602afcd9080 0x5602afcdb520 crc :-1 s=READY pgs=47 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=bd9e9261f4c14605 server_cookie=a5d728ef0a26d963 in_seq=0 out_seq=0 2024-03-19T01:43:14.134 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.122+0000 7fe28f7fe640 1 -- 172.21.15.99:0/749273107 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x5602afc31120 con 0x5602afcd9080 2024-03-19T01:43:14.134 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.122+0000 7fe28f7fe640 10 client.25277 handle_command_reply: tid=0 2024-03-19T01:43:14.134 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.122+0000 7fe2b9375740 1 client.25277 shutdown 2024-03-19T01:43:14.134 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.122+0000 7fe2b9375740 1 -- 172.21.15.99:0/749273107 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5602afcbf410 msgr2=0x5602afccd200 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:43:14.134 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.122+0000 7fe2b9375740 1 --2- 172.21.15.99:0/749273107 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5602afcbf410 0x5602afccd200 secure :-1 s=READY pgs=95 cs=0 l=1 rev1=1 crypto rx=0x7fe2a001ed30 tx=0x7fe2a0004730 comp rx=0 tx=0).stop 2024-03-19T01:43:14.134 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.122+0000 7fe2b9375740 1 -- 172.21.15.99:0/749273107 shutdown_connections 2024-03-19T01:43:14.134 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.122+0000 7fe2b9375740 1 --2- 172.21.15.99:0/749273107 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5602afcd9080 0x5602afcdb520 crc :-1 s=READY pgs=47 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:14.134 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.122+0000 7fe2b9375740 1 --2- 172.21.15.99:0/749273107 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5602afcc9420 0x5602afcd4780 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:14.134 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.122+0000 7fe2b9375740 1 --2- 172.21.15.99:0/749273107 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5602afcc0780 0x5602afccd740 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:14.134 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.122+0000 7fe28f7fe640 0 client.25277 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:43:14.134 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.122+0000 7fe2b9375740 1 --2- 172.21.15.99:0/749273107 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5602afcbf410 0x5602afccd200 unknown :-1 s=CLOSED pgs=95 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:14.134 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.122+0000 7fe2b9375740 1 -- 172.21.15.99:0/749273107 >> 172.21.15.99:0/749273107 conn(0x5602afcba670 msgr2=0x5602afcbb8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:43:14.134 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.122+0000 7fe2b9375740 1 -- 172.21.15.99:0/749273107 shutdown_connections 2024-03-19T01:43:14.134 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.122+0000 7fe2b9375740 1 -- 172.21.15.99:0/749273107 wait complete. 2024-03-19T01:43:14.134 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.122+0000 7fe2b9375740 20 client.25277 trim_cache size 0 max 16384 2024-03-19T01:43:14.137 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.137+0000 7fe2b9375740 1 Processor -- start 2024-03-19T01:43:14.137 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.137+0000 7fe2b9375740 1 -- start start 2024-03-19T01:43:14.147 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.137+0000 7fe2b9375740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5602afcbf410 0x5602afccce10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:14.147 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.137+0000 7fe2b9375740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5602afcc0780 0x5602afcd4740 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:14.147 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.137+0000 7fe2b9375740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5602afcc9420 0x5602afccd350 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:14.148 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.137+0000 7fe2b9375740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fe2b01a9480 con 0x5602afcc0780 2024-03-19T01:43:14.148 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.138+0000 7fe2b9375740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fe2b01a6d50 con 0x5602afcc9420 2024-03-19T01:43:14.148 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.138+0000 7fe2b9375740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fe2b01a4ba0 con 0x5602afcbf410 2024-03-19T01:43:14.148 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.138+0000 7fe2af7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5602afcc0780 0x5602afcd4740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:14.148 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.138+0000 7fe2af7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5602afcc0780 0x5602afcd4740 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:56194/0 (socket says 172.21.15.99:56194) 2024-03-19T01:43:14.148 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.138+0000 7fe2af7fe640 1 -- 172.21.15.99:0/2346617178 learned_addr learned my addr 172.21.15.99:0/2346617178 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:43:14.148 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.138+0000 7fe2affff640 1 --2- 172.21.15.99:0/2346617178 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5602afcbf410 0x5602afccce10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:14.148 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.138+0000 7fe2b4d24640 1 --2- 172.21.15.99:0/2346617178 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5602afcc9420 0x5602afccd350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:14.148 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.138+0000 7fe2af7fe640 1 -- 172.21.15.99:0/2346617178 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5602afcbf410 msgr2=0x5602afccce10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:43:14.148 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.138+0000 7fe2af7fe640 1 --2- 172.21.15.99:0/2346617178 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5602afcbf410 0x5602afccce10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:14.148 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.138+0000 7fe2af7fe640 1 -- 172.21.15.99:0/2346617178 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5602afcc9420 msgr2=0x5602afccd350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:43:14.148 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.138+0000 7fe2af7fe640 1 --2- 172.21.15.99:0/2346617178 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5602afcc9420 0x5602afccd350 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:14.148 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.138+0000 7fe2af7fe640 1 -- 172.21.15.99:0/2346617178 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5602afa04620 con 0x5602afcc0780 2024-03-19T01:43:14.148 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.138+0000 7fe2af7fe640 1 --2- 172.21.15.99:0/2346617178 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5602afcc0780 0x5602afcd4740 secure :-1 s=READY pgs=398 cs=0 l=1 rev1=1 crypto rx=0x7fe2a4009430 tx=0x7fe2a4003920 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ec761338c5fb3c84 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:43:14.149 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.138+0000 7fe295ffb640 1 -- 172.21.15.99:0/2346617178 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe2a409f070 con 0x5602afcc0780 2024-03-19T01:43:14.149 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.138+0000 7fe295ffb640 1 -- 172.21.15.99:0/2346617178 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe2a40b94b0 con 0x5602afcc0780 2024-03-19T01:43:14.149 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.138+0000 7fe295ffb640 1 -- 172.21.15.99:0/2346617178 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe2a40bb0b0 con 0x5602afcc0780 2024-03-19T01:43:14.149 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.139+0000 7fe2b9375740 1 -- 172.21.15.99:0/2346617178 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5602afcc0780 msgr2=0x5602afcd4740 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:43:14.149 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.139+0000 7fe2b9375740 1 --2- 172.21.15.99:0/2346617178 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5602afcc0780 0x5602afcd4740 secure :-1 s=READY pgs=398 cs=0 l=1 rev1=1 crypto rx=0x7fe2a4009430 tx=0x7fe2a4003920 comp rx=0 tx=0).stop 2024-03-19T01:43:14.149 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.139+0000 7fe2b9375740 1 -- 172.21.15.99:0/2346617178 shutdown_connections 2024-03-19T01:43:14.149 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.139+0000 7fe2b9375740 1 --2- 172.21.15.99:0/2346617178 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5602afcc9420 0x5602afccd350 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:14.149 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.139+0000 7fe2b9375740 1 --2- 172.21.15.99:0/2346617178 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5602afcc0780 0x5602afcd4740 unknown :-1 s=CLOSED pgs=398 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:14.149 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.139+0000 7fe2b9375740 1 --2- 172.21.15.99:0/2346617178 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5602afcbf410 0x5602afccce10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:14.149 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.139+0000 7fe2b9375740 1 -- 172.21.15.99:0/2346617178 >> 172.21.15.99:0/2346617178 conn(0x5602afcba670 msgr2=0x5602afcbaa50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:43:14.149 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.139+0000 7fe2b9375740 1 -- 172.21.15.99:0/2346617178 shutdown_connections 2024-03-19T01:43:14.149 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.139+0000 7fe2b9375740 1 -- 172.21.15.99:0/2346617178 wait complete. 2024-03-19T01:43:14.149 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.139+0000 7fe2b9375740 1 -- start start 2024-03-19T01:43:14.149 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.139+0000 7fe2b9375740 1 Processor -- start 2024-03-19T01:43:14.150 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.140+0000 7fe2b9375740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5602afcbf410 0x5602afcd1a50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:14.150 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.140+0000 7fe2b9375740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5602afcc0780 0x5602afcd1f90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:14.150 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.140+0000 7fe2b9375740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5602afcc9420 0x5602afcd2f50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:14.150 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.140+0000 7fe2b9375740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fe2b01a5210 con 0x5602afcbf410 2024-03-19T01:43:14.150 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.140+0000 7fe2b9375740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fe2b01a6680 con 0x5602afcc9420 2024-03-19T01:43:14.150 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.140+0000 7fe2b9375740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fe2b01a58e0 con 0x5602afcc0780 2024-03-19T01:43:14.150 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.140+0000 7fe2b9375740 10 client.0 _ll_register_callbacks cb 0x7fe2a00099a0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:43:14.150 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.140+0000 7fe2affff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5602afcbf410 0x5602afcd1a50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:14.150 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.140+0000 7fe2affff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5602afcbf410 0x5602afcd1a50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:56206/0 (socket says 172.21.15.99:56206) 2024-03-19T01:43:14.150 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.140+0000 7fe2affff640 1 -- 172.21.15.99:0/3354052455 learned_addr learned my addr 172.21.15.99:0/3354052455 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:43:14.150 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.140+0000 7fe2af7fe640 1 --2- 172.21.15.99:0/3354052455 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5602afcc0780 0x5602afcd1f90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:14.150 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.140+0000 7fe2b4d24640 1 --2- 172.21.15.99:0/3354052455 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5602afcc9420 0x5602afcd2f50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:14.150 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.140+0000 7fe2affff640 1 -- 172.21.15.99:0/3354052455 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5602afcc0780 msgr2=0x5602afcd1f90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:43:14.151 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.140+0000 7fe2affff640 1 --2- 172.21.15.99:0/3354052455 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5602afcc0780 0x5602afcd1f90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:14.151 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.140+0000 7fe2affff640 1 -- 172.21.15.99:0/3354052455 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5602afcc9420 msgr2=0x5602afcd2f50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:43:14.151 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.140+0000 7fe2affff640 1 --2- 172.21.15.99:0/3354052455 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5602afcc9420 0x5602afcd2f50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:14.151 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.140+0000 7fe2affff640 1 -- 172.21.15.99:0/3354052455 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5602afcd3540 con 0x5602afcbf410 2024-03-19T01:43:14.151 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.140+0000 7fe2affff640 1 --2- 172.21.15.99:0/3354052455 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5602afcbf410 0x5602afcd1a50 secure :-1 s=READY pgs=399 cs=0 l=1 rev1=1 crypto rx=0x7fe2a001c4c0 tx=0x7fe2a001e420 comp rx=0 tx=0).ready entity=mon.0 client_cookie=45d880acfa6f0740 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:43:14.151 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.140+0000 7fe28f7fe640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:43:14.151 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.140+0000 7fe28f7fe640 1 -- 172.21.15.99:0/3354052455 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe2a001c6b0 con 0x5602afcbf410 2024-03-19T01:43:14.151 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.141+0000 7fe2b9375740 1 -- 172.21.15.99:0/3354052455 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fe2a40b94b0 con 0x5602afcbf410 2024-03-19T01:43:14.151 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.141+0000 7fe28f7fe640 1 -- 172.21.15.99:0/3354052455 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7fe2a001c850 con 0x5602afcbf410 2024-03-19T01:43:14.151 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.141+0000 7fe28f7fe640 1 -- 172.21.15.99:0/3354052455 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=23) ==== 24+0+0 (secure 0 0 0) 0x7fe2a001cab0 con 0x5602afcbf410 2024-03-19T01:43:14.151 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.141+0000 7fe2b9375740 10 client.15510 fetch_fsmap learned FSMap version 23 2024-03-19T01:43:14.151 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.141+0000 7fe2b9375740 1 -- 172.21.15.99:0/3354052455 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=23,monmap=4+}) -- 0x5602afcd48c0 con 0x5602afcbf410 2024-03-19T01:43:14.151 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.141+0000 7fe28f7fe640 1 -- 172.21.15.99:0/3354052455 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe2a40b94b0 con 0x5602afcbf410 2024-03-19T01:43:14.152 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.141+0000 7fe28f7fe640 1 -- 172.21.15.99:0/3354052455 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 23) ==== 8346+0+0 (secure 0 0 0) 0x7fe2a0025040 con 0x5602afcbf410 2024-03-19T01:43:14.152 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.141+0000 7fe2b9375740 10 client.15510 fetch_fsmap finished waiting for FSMap version 23 2024-03-19T01:43:14.152 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.141+0000 7fe2b9375740 10 client.15510 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:43:14.152 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.141+0000 7fe2b9375740 20 client.15510 populate_metadata read hostname 'smithi099' 2024-03-19T01:43:14.152 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.141+0000 7fe2b9375740 1 --2- 172.21.15.99:0/3354052455 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5602afd3fb20 0x5602afd41f40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:14.152 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.141+0000 7fe2b9375740 4 client.15510 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:43:14.152 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.141+0000 7fe2b9375740 1 -- 172.21.15.99:0/3354052455 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x5602afc2b470 con 0x5602afd3fb20 2024-03-19T01:43:14.152 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.141+0000 7fe2af7fe640 1 --2- 172.21.15.99:0/3354052455 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5602afd3fb20 0x5602afd41f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:14.152 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.142+0000 7fe28f7fe640 10 client.15510 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:43:14.152 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.142+0000 7fe2af7fe640 1 --2- 172.21.15.99:0/3354052455 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5602afd3fb20 0x5602afd41f40 crc :-1 s=READY pgs=48 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=b7dc5253e3d39211 server_cookie=85e9523c9a890fe4 in_seq=0 out_seq=0 2024-03-19T01:43:14.217 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.208+0000 7fe28f7fe640 1 -- 172.21.15.99:0/3354052455 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x5602afc2b470 con 0x5602afd3fb20 2024-03-19T01:43:14.217 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.208+0000 7fe28f7fe640 10 client.15510 handle_command_reply: tid=0 2024-03-19T01:43:14.218 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.209+0000 7fe2b9375740 1 client.15510 shutdown 2024-03-19T01:43:14.218 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.209+0000 7fe2b9375740 1 -- 172.21.15.99:0/3354052455 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5602afcbf410 msgr2=0x5602afcd1a50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:43:14.218 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.209+0000 7fe2b9375740 1 --2- 172.21.15.99:0/3354052455 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5602afcbf410 0x5602afcd1a50 secure :-1 s=READY pgs=399 cs=0 l=1 rev1=1 crypto rx=0x7fe2a001c4c0 tx=0x7fe2a001e420 comp rx=0 tx=0).stop 2024-03-19T01:43:14.218 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.209+0000 7fe2b9375740 1 -- 172.21.15.99:0/3354052455 shutdown_connections 2024-03-19T01:43:14.218 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.209+0000 7fe2b9375740 1 --2- 172.21.15.99:0/3354052455 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5602afd3fb20 0x5602afd41f40 crc :-1 s=READY pgs=48 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:14.218 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.209+0000 7fe2b9375740 1 --2- 172.21.15.99:0/3354052455 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5602afcc9420 0x5602afcd2f50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:14.218 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.209+0000 7fe28f7fe640 0 client.15510 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:43:14.218 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.209+0000 7fe2b9375740 1 --2- 172.21.15.99:0/3354052455 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5602afcc0780 0x5602afcd1f90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:14.218 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.209+0000 7fe2b9375740 1 --2- 172.21.15.99:0/3354052455 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5602afcbf410 0x5602afcd1a50 unknown :-1 s=CLOSED pgs=399 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:14.218 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.209+0000 7fe2b9375740 1 -- 172.21.15.99:0/3354052455 >> 172.21.15.99:0/3354052455 conn(0x5602afcba670 msgr2=0x5602afccce10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:43:14.218 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.209+0000 7fe2b9375740 1 -- 172.21.15.99:0/3354052455 shutdown_connections 2024-03-19T01:43:14.218 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.209+0000 7fe2b9375740 1 -- 172.21.15.99:0/3354052455 wait complete. 2024-03-19T01:43:14.218 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.209+0000 7fe2b9375740 20 client.15510 trim_cache size 0 max 16384 2024-03-19T01:43:14.219 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:43:14.219 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:43:14.219 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:43:14.219 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:43:14.220 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.209+0000 7fe2b6787640 1 -- 172.21.15.99:0/760358370 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fe2980a28a0 msgr2=0x7fe2980a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:43:14.220 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.209+0000 7fe2b6787640 1 --2- 172.21.15.99:0/760358370 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fe2980a28a0 0x7fe2980a4d60 secure :-1 s=READY pgs=257 cs=0 l=1 rev1=1 crypto rx=0x7fe2b01017e0 tx=0x7fe2a0005f90 comp rx=0 tx=0).stop 2024-03-19T01:43:14.220 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.209+0000 7fe2b6787640 1 -- 172.21.15.99:0/760358370 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe2b0101980 msgr2=0x7fe2b0196f00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:43:14.220 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.209+0000 7fe2b6787640 1 --2- 172.21.15.99:0/760358370 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe2b0101980 0x7fe2b0196f00 secure :-1 s=READY pgs=396 cs=0 l=1 rev1=1 crypto rx=0x7fe2a4009230 tx=0x7fe2a402c650 comp rx=0 tx=0).stop 2024-03-19T01:43:14.220 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.210+0000 7fe2b6787640 1 -- 172.21.15.99:0/760358370 shutdown_connections 2024-03-19T01:43:14.220 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.210+0000 7fe2b6787640 1 --2- 172.21.15.99:0/760358370 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fe2980a28a0 0x7fe2980a4d60 unknown :-1 s=CLOSED pgs=257 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:14.220 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.210+0000 7fe2b6787640 1 --2- 172.21.15.99:0/760358370 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fe2b0102340 0x7fe2b019df70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:14.220 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.210+0000 7fe2b6787640 1 --2- 172.21.15.99:0/760358370 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe2b0101980 0x7fe2b0196f00 unknown :-1 s=CLOSED pgs=396 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:14.220 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.210+0000 7fe2b6787640 1 --2- 172.21.15.99:0/760358370 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fe2b0100790 0x7fe2b01969c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:14.221 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.210+0000 7fe2b6787640 1 -- 172.21.15.99:0/760358370 >> 172.21.15.99:0/760358370 conn(0x7fe2b00fbd00 msgr2=0x7fe2b0104bb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:43:14.221 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.210+0000 7fe2b6787640 1 -- 172.21.15.99:0/760358370 shutdown_connections 2024-03-19T01:43:14.221 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:14.210+0000 7fe2b6787640 1 -- 172.21.15.99:0/760358370 wait complete. 2024-03-19T01:43:14.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:43:13 smithi115 ceph-mon[67332]: pgmap v414: 97 pgs: 97 active+clean; 93 MiB data, 990 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 MiB/s rd, 3.1 MiB/s wr, 281 op/s 2024-03-19T01:43:14.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:43:13 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:43:14.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:43:13 smithi099 ceph-mon[62951]: pgmap v414: 97 pgs: 97 active+clean; 93 MiB data, 990 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 MiB/s rd, 3.1 MiB/s wr, 281 op/s 2024-03-19T01:43:14.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:43:13 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:43:14.677 INFO:tasks.workunit.client.0.smithi099.stdout: 1 231078 12.15 MB/sec warmup 104 sec latency 24.966 ms 2024-03-19T01:43:15.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:43:14 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:43:15.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:43:14 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:43:15.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:43:14 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:43:15.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:43:14 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:43:15.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:43:14 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:43:15.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:43:14 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:43:15.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:43:14 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:43:15.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:43:14 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:43:15.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:43:14 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:43:15.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:43:14 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:43:15.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:43:14 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:43:15.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:43:14 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:43:15.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:43:14 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:43:15.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:43:14 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:43:15.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:43:14 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:43:15.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:43:14 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:43:15.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:43:14 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:43:15.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:43:14 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:43:15.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:43:14 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:43:15.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:43:14 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:43:15.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:43:14 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:43:15.677 INFO:tasks.workunit.client.0.smithi099.stdout: 1 233367 12.14 MB/sec warmup 105 sec latency 14.284 ms 2024-03-19T01:43:16.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:43:15 smithi115 ceph-mon[67332]: pgmap v415: 97 pgs: 97 active+clean; 94 MiB data, 977 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 MiB/s rd, 2.8 MiB/s wr, 251 op/s 2024-03-19T01:43:16.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:43:15 smithi099 ceph-mon[62951]: pgmap v415: 97 pgs: 97 active+clean; 94 MiB data, 977 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 MiB/s rd, 2.8 MiB/s wr, 251 op/s 2024-03-19T01:43:16.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:43:15 smithi112 ceph-mon[66819]: pgmap v415: 97 pgs: 97 active+clean; 94 MiB data, 977 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 MiB/s rd, 2.8 MiB/s wr, 251 op/s 2024-03-19T01:43:16.677 INFO:tasks.workunit.client.0.smithi099.stdout: 1 235251 12.12 MB/sec warmup 106 sec latency 28.811 ms 2024-03-19T01:43:17.677 INFO:tasks.workunit.client.0.smithi099.stdout: 1 237949 12.17 MB/sec warmup 107 sec latency 28.674 ms 2024-03-19T01:43:18.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:43:17 smithi115 ceph-mon[67332]: pgmap v416: 97 pgs: 97 active+clean; 97 MiB data, 1001 MiB used, 1.0 TiB / 1.0 TiB avail; 2.6 MiB/s rd, 3.4 MiB/s wr, 308 op/s 2024-03-19T01:43:18.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:43:17 smithi099 ceph-mon[62951]: pgmap v416: 97 pgs: 97 active+clean; 97 MiB data, 1001 MiB used, 1.0 TiB / 1.0 TiB avail; 2.6 MiB/s rd, 3.4 MiB/s wr, 308 op/s 2024-03-19T01:43:18.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:43:17 smithi112 ceph-mon[66819]: pgmap v416: 97 pgs: 97 active+clean; 97 MiB data, 1001 MiB used, 1.0 TiB / 1.0 TiB avail; 2.6 MiB/s rd, 3.4 MiB/s wr, 308 op/s 2024-03-19T01:43:18.677 INFO:tasks.workunit.client.0.smithi099.stdout: 1 239674 12.11 MB/sec warmup 108 sec latency 103.924 ms 2024-03-19T01:43:19.677 INFO:tasks.workunit.client.0.smithi099.stdout: 1 241821 12.12 MB/sec warmup 109 sec latency 18.679 ms 2024-03-19T01:43:20.163 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:43:19 smithi112 ceph-mon[66819]: pgmap v417: 97 pgs: 97 active+clean; 98 MiB data, 1017 MiB used, 1.0 TiB / 1.0 TiB avail; 2.0 MiB/s rd, 3.0 MiB/s wr, 258 op/s 2024-03-19T01:43:20.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:43:19 smithi115 ceph-mon[67332]: pgmap v417: 97 pgs: 97 active+clean; 98 MiB data, 1017 MiB used, 1.0 TiB / 1.0 TiB avail; 2.0 MiB/s rd, 3.0 MiB/s wr, 258 op/s 2024-03-19T01:43:20.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:43:19 smithi099 ceph-mon[62951]: pgmap v417: 97 pgs: 97 active+clean; 98 MiB data, 1017 MiB used, 1.0 TiB / 1.0 TiB avail; 2.0 MiB/s rd, 3.0 MiB/s wr, 258 op/s 2024-03-19T01:43:20.677 INFO:tasks.workunit.client.0.smithi099.stdout: 1 244138 12.12 MB/sec warmup 110 sec latency 29.982 ms 2024-03-19T01:43:21.677 INFO:tasks.workunit.client.0.smithi099.stdout: 1 246038 12.10 MB/sec warmup 111 sec latency 28.632 ms 2024-03-19T01:43:22.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:43:21 smithi115 ceph-mon[67332]: pgmap v418: 97 pgs: 97 active+clean; 99 MiB data, 1.0 GiB used, 1.0 TiB / 1.0 TiB avail; 1.4 MiB/s rd, 3.0 MiB/s wr, 261 op/s 2024-03-19T01:43:22.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:43:21 smithi099 ceph-mon[62951]: pgmap v418: 97 pgs: 97 active+clean; 99 MiB data, 1.0 GiB used, 1.0 TiB / 1.0 TiB avail; 1.4 MiB/s rd, 3.0 MiB/s wr, 261 op/s 2024-03-19T01:43:22.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:43:21 smithi112 ceph-mon[66819]: pgmap v418: 97 pgs: 97 active+clean; 99 MiB data, 1.0 GiB used, 1.0 TiB / 1.0 TiB avail; 1.4 MiB/s rd, 3.0 MiB/s wr, 261 op/s 2024-03-19T01:43:22.677 INFO:tasks.workunit.client.0.smithi099.stdout: 1 248593 12.14 MB/sec warmup 112 sec latency 22.882 ms 2024-03-19T01:43:23.677 INFO:tasks.workunit.client.0.smithi099.stdout: 1 250247 12.08 MB/sec warmup 113 sec latency 70.508 ms 2024-03-19T01:43:24.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:43:23 smithi115 ceph-mon[67332]: pgmap v419: 97 pgs: 97 active+clean; 102 MiB data, 1017 MiB used, 1.0 TiB / 1.0 TiB avail; 1.3 MiB/s rd, 3.2 MiB/s wr, 284 op/s 2024-03-19T01:43:24.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:43:23 smithi099 ceph-mon[62951]: pgmap v419: 97 pgs: 97 active+clean; 102 MiB data, 1017 MiB used, 1.0 TiB / 1.0 TiB avail; 1.3 MiB/s rd, 3.2 MiB/s wr, 284 op/s 2024-03-19T01:43:24.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:43:23 smithi112 ceph-mon[66819]: pgmap v419: 97 pgs: 97 active+clean; 102 MiB data, 1017 MiB used, 1.0 TiB / 1.0 TiB avail; 1.3 MiB/s rd, 3.2 MiB/s wr, 284 op/s 2024-03-19T01:43:24.677 INFO:tasks.workunit.client.0.smithi099.stdout: 1 252449 12.09 MB/sec warmup 114 sec latency 17.420 ms 2024-03-19T01:43:25.677 INFO:tasks.workunit.client.0.smithi099.stdout: 1 254486 12.06 MB/sec warmup 115 sec latency 41.385 ms 2024-03-19T01:43:26.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:43:25 smithi115 ceph-mon[67332]: pgmap v420: 97 pgs: 97 active+clean; 102 MiB data, 1019 MiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 2.8 MiB/s wr, 255 op/s 2024-03-19T01:43:26.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:43:25 smithi099 ceph-mon[62951]: pgmap v420: 97 pgs: 97 active+clean; 102 MiB data, 1019 MiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 2.8 MiB/s wr, 255 op/s 2024-03-19T01:43:26.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:43:25 smithi112 ceph-mon[66819]: pgmap v420: 97 pgs: 97 active+clean; 102 MiB data, 1019 MiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 2.8 MiB/s wr, 255 op/s 2024-03-19T01:43:26.677 INFO:tasks.workunit.client.0.smithi099.stdout: 1 256479 12.07 MB/sec warmup 116 sec latency 39.870 ms 2024-03-19T01:43:27.312 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.309+0000 7f274746e640 1 -- 172.21.15.99:0/1849459235 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2740100740 msgr2=0x7f2740100b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:43:27.312 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.309+0000 7f274746e640 1 --2- 172.21.15.99:0/1849459235 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2740100740 0x7f2740100b60 secure :-1 s=READY pgs=400 cs=0 l=1 rev1=1 crypto rx=0x7f273c009900 tx=0x7f273c02bc30 comp rx=0 tx=0).stop 2024-03-19T01:43:27.312 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.309+0000 7f274746e640 1 -- 172.21.15.99:0/1849459235 shutdown_connections 2024-03-19T01:43:27.312 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.309+0000 7f274746e640 1 --2- 172.21.15.99:0/1849459235 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f2740101930 0x7f2740101db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:27.312 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.309+0000 7f274746e640 1 --2- 172.21.15.99:0/1849459235 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f27401022f0 0x7f2740108b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:27.312 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.309+0000 7f274746e640 1 --2- 172.21.15.99:0/1849459235 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2740100740 0x7f2740100b60 unknown :-1 s=CLOSED pgs=400 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:27.312 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.309+0000 7f274746e640 1 -- 172.21.15.99:0/1849459235 >> 172.21.15.99:0/1849459235 conn(0x7f27400fbcb0 msgr2=0x7f27400fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:43:27.312 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.309+0000 7f274746e640 1 -- 172.21.15.99:0/1849459235 shutdown_connections 2024-03-19T01:43:27.312 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.309+0000 7f274746e640 1 -- 172.21.15.99:0/1849459235 wait complete. 2024-03-19T01:43:27.313 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.310+0000 7f274746e640 1 Processor -- start 2024-03-19T01:43:27.313 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.310+0000 7f274746e640 1 -- start start 2024-03-19T01:43:27.313 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.310+0000 7f274746e640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f2740100740 0x7f2740197ad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:27.313 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.310+0000 7f274746e640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f2740101930 0x7f2740198010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:27.313 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.310+0000 7f274746e640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f27401022f0 0x7f274019f080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:27.313 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.310+0000 7f274746e640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f274010c3c0 con 0x7f27401022f0 2024-03-19T01:43:27.313 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.310+0000 7f274746e640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f274010c240 con 0x7f2740100740 2024-03-19T01:43:27.313 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.310+0000 7f274746e640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f274010c540 con 0x7f2740101930 2024-03-19T01:43:27.313 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.310+0000 7f2745a0b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f27401022f0 0x7f274019f080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:27.313 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.310+0000 7f2744a09640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f2740101930 0x7f2740198010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:27.313 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.310+0000 7f2745a0b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f27401022f0 0x7f274019f080 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:52402/0 (socket says 172.21.15.99:52402) 2024-03-19T01:43:27.313 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.310+0000 7f274520a640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f2740100740 0x7f2740197ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:27.313 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.310+0000 7f2745a0b640 1 -- 172.21.15.99:0/3000939464 learned_addr learned my addr 172.21.15.99:0/3000939464 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:43:27.314 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.310+0000 7f2745a0b640 1 -- 172.21.15.99:0/3000939464 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f2740101930 msgr2=0x7f2740198010 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:43:27.314 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.310+0000 7f2745a0b640 1 --2- 172.21.15.99:0/3000939464 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f2740101930 0x7f2740198010 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:27.314 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.310+0000 7f2745a0b640 1 -- 172.21.15.99:0/3000939464 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f2740100740 msgr2=0x7f2740197ad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:43:27.314 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.310+0000 7f2745a0b640 1 --2- 172.21.15.99:0/3000939464 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f2740100740 0x7f2740197ad0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:27.314 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.310+0000 7f2745a0b640 1 -- 172.21.15.99:0/3000939464 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f274019f6f0 con 0x7f27401022f0 2024-03-19T01:43:27.314 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.310+0000 7f274520a640 1 --2- 172.21.15.99:0/3000939464 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f2740100740 0x7f2740197ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:43:27.314 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.311+0000 7f2745a0b640 1 --2- 172.21.15.99:0/3000939464 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f27401022f0 0x7f274019f080 secure :-1 s=READY pgs=401 cs=0 l=1 rev1=1 crypto rx=0x7f273800eb10 tx=0x7f273800c2d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:43:27.314 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.311+0000 7f27367fc640 1 -- 172.21.15.99:0/3000939464 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2738004b20 con 0x7f27401022f0 2024-03-19T01:43:27.314 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.311+0000 7f27367fc640 1 -- 172.21.15.99:0/3000939464 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f2738004cc0 con 0x7f27401022f0 2024-03-19T01:43:27.314 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.311+0000 7f274746e640 1 -- 172.21.15.99:0/3000939464 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f274019f9e0 con 0x7f27401022f0 2024-03-19T01:43:27.314 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.311+0000 7f27367fc640 1 -- 172.21.15.99:0/3000939464 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2738010760 con 0x7f27401022f0 2024-03-19T01:43:27.314 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.311+0000 7f274746e640 1 -- 172.21.15.99:0/3000939464 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f274019ff30 con 0x7f27401022f0 2024-03-19T01:43:27.321 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.313+0000 7f27367fc640 1 -- 172.21.15.99:0/3000939464 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f2738005020 con 0x7f27401022f0 2024-03-19T01:43:27.321 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.314+0000 7f274a057740 1 Processor -- start 2024-03-19T01:43:27.321 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.314+0000 7f274a057740 1 -- start start 2024-03-19T01:43:27.321 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.314+0000 7f274a057740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55febdfba100 0x55febdfba520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:27.321 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.314+0000 7f274a057740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55febdfbaaf0 0x55febdfbf5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:27.321 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.314+0000 7f274a057740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55febdfbfc60 0x55febdfc20f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:27.322 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.314+0000 7f274a057740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55febdf1ebe0 con 0x55febdfbaaf0 2024-03-19T01:43:27.322 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.314+0000 7f274a057740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55febdec7960 con 0x55febdfba100 2024-03-19T01:43:27.322 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.314+0000 7f274a057740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55febdf1ac80 con 0x55febdfbfc60 2024-03-19T01:43:27.322 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.314+0000 7f27367fc640 1 --2- 172.21.15.99:0/3000939464 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f272c0a28a0 0x7f272c0a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:27.322 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.314+0000 7f274520a640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55febdfbaaf0 0x55febdfbf5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:27.322 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.314+0000 7f2744a09640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55febdfba100 0x55febdfba520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:27.322 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.314+0000 7f274520a640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55febdfbaaf0 0x55febdfbf5f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:52404/0 (socket says 172.21.15.99:52404) 2024-03-19T01:43:27.322 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.314+0000 7f274520a640 1 -- 172.21.15.99:0/756054600 learned_addr learned my addr 172.21.15.99:0/756054600 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:43:27.322 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.314+0000 7f2744a09640 1 --2- 172.21.15.99:0/756054600 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55febdfbfc60 0x55febdfc20f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:27.322 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.314+0000 7f2745a0b640 1 --2- 172.21.15.99:0/3000939464 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f272c0a28a0 0x7f272c0a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:27.322 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.314+0000 7f27367fc640 1 -- 172.21.15.99:0/3000939464 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f27380cf180 con 0x7f27401022f0 2024-03-19T01:43:27.322 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.314+0000 7f2744a09640 1 -- 172.21.15.99:0/756054600 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55febdfbfc60 msgr2=0x55febdfc20f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:43:27.323 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.314+0000 7f2744a09640 1 --2- 172.21.15.99:0/756054600 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55febdfbfc60 0x55febdfc20f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:27.323 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.314+0000 7f2744a09640 1 -- 172.21.15.99:0/756054600 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55febdfbaaf0 msgr2=0x55febdfbf5f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:43:27.323 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.314+0000 7f2744a09640 1 --2- 172.21.15.99:0/756054600 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55febdfbaaf0 0x55febdfbf5f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:27.323 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.314+0000 7f2744a09640 1 -- 172.21.15.99:0/756054600 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55febdebed20 con 0x55febdfba100 2024-03-19T01:43:27.323 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.315+0000 7f274520a640 1 --2- 172.21.15.99:0/756054600 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55febdfbaaf0 0x55febdfbf5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:43:27.323 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.315+0000 7f2745a0b640 1 --2- 172.21.15.99:0/3000939464 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f272c0a28a0 0x7f272c0a4d60 secure :-1 s=READY pgs=258 cs=0 l=1 rev1=1 crypto rx=0x7f2738009200 tx=0x7f27380668a0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:43:27.323 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.315+0000 7f2744a09640 1 --2- 172.21.15.99:0/756054600 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55febdfba100 0x55febdfba520 secure :-1 s=READY pgs=96 cs=0 l=1 rev1=1 crypto rx=0x7f2730009730 tx=0x7f27300096c0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=cd6c7de2e5caefd2 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:43:27.323 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.315+0000 7f272b7fe640 1 -- 172.21.15.99:0/756054600 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2730004050 con 0x55febdfba100 2024-03-19T01:43:27.323 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.315+0000 7f272b7fe640 1 -- 172.21.15.99:0/756054600 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f2730005500 con 0x55febdfba100 2024-03-19T01:43:27.323 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.315+0000 7f272b7fe640 1 -- 172.21.15.99:0/756054600 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2730004d00 con 0x55febdfba100 2024-03-19T01:43:27.323 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.316+0000 7f274a057740 1 -- 172.21.15.99:0/756054600 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55febdfba100 msgr2=0x55febdfba520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:43:27.323 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.316+0000 7f274a057740 1 --2- 172.21.15.99:0/756054600 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55febdfba100 0x55febdfba520 secure :-1 s=READY pgs=96 cs=0 l=1 rev1=1 crypto rx=0x7f2730009730 tx=0x7f27300096c0 comp rx=0 tx=0).stop 2024-03-19T01:43:27.323 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.316+0000 7f274a057740 1 -- 172.21.15.99:0/756054600 shutdown_connections 2024-03-19T01:43:27.323 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.316+0000 7f274a057740 1 --2- 172.21.15.99:0/756054600 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55febdfbfc60 0x55febdfc20f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:27.324 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.316+0000 7f274a057740 1 --2- 172.21.15.99:0/756054600 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55febdfbaaf0 0x55febdfbf5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:27.324 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.316+0000 7f274a057740 1 --2- 172.21.15.99:0/756054600 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55febdfba100 0x55febdfba520 unknown :-1 s=CLOSED pgs=96 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:27.324 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.316+0000 7f274a057740 1 -- 172.21.15.99:0/756054600 >> 172.21.15.99:0/756054600 conn(0x55febdfb5670 msgr2=0x55febdfb5a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:43:27.324 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.316+0000 7f274a057740 1 -- 172.21.15.99:0/756054600 shutdown_connections 2024-03-19T01:43:27.324 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.316+0000 7f274a057740 1 -- 172.21.15.99:0/756054600 wait complete. 2024-03-19T01:43:27.324 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.316+0000 7f274a057740 1 -- start start 2024-03-19T01:43:27.324 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.316+0000 7f274a057740 1 Processor -- start 2024-03-19T01:43:27.324 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.316+0000 7f274a057740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55febdfba100 0x55febdfc40e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:27.324 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.316+0000 7f274a057740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55febdfbaaf0 0x55febdfc4620 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:27.324 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.316+0000 7f274a057740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55febdfbfc60 0x55febdfcb660 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:27.324 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.316+0000 7f274a057740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55febdfc2c10 con 0x55febdfbaaf0 2024-03-19T01:43:27.324 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.317+0000 7f274a057740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55febdedadb0 con 0x55febdfbfc60 2024-03-19T01:43:27.324 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.317+0000 7f274a057740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55febdfc2d90 con 0x55febdfba100 2024-03-19T01:43:27.324 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.317+0000 7f274a057740 10 client.0 _ll_register_callbacks cb 0x55febdedfe70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:43:27.324 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.317+0000 7f274520a640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55febdfbaaf0 0x55febdfc4620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:27.325 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.317+0000 7f274520a640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55febdfbaaf0 0x55febdfc4620 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:52412/0 (socket says 172.21.15.99:52412) 2024-03-19T01:43:27.325 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.317+0000 7f274520a640 1 -- 172.21.15.99:0/1877827672 learned_addr learned my addr 172.21.15.99:0/1877827672 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:43:27.325 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.317+0000 7f2744a09640 1 --2- 172.21.15.99:0/1877827672 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55febdfba100 0x55febdfc40e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:27.325 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.317+0000 7f2744a09640 1 --2- 172.21.15.99:0/1877827672 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55febdfbfc60 0x55febdfcb660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:27.325 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.317+0000 7f274520a640 1 -- 172.21.15.99:0/1877827672 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55febdfba100 msgr2=0x55febdfc40e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:43:27.325 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.317+0000 7f274520a640 1 --2- 172.21.15.99:0/1877827672 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55febdfba100 0x55febdfc40e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:27.325 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.317+0000 7f274520a640 1 -- 172.21.15.99:0/1877827672 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55febdfbfc60 msgr2=0x55febdfcb660 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:43:27.325 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.317+0000 7f274520a640 1 --2- 172.21.15.99:0/1877827672 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55febdfbfc60 0x55febdfcb660 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:27.325 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.317+0000 7f274520a640 1 -- 172.21.15.99:0/1877827672 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55febded9c60 con 0x55febdfbaaf0 2024-03-19T01:43:27.325 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.317+0000 7f2744a09640 1 --2- 172.21.15.99:0/1877827672 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55febdfbfc60 0x55febdfcb660 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-19T01:43:27.325 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.317+0000 7f274520a640 1 --2- 172.21.15.99:0/1877827672 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55febdfbaaf0 0x55febdfc4620 secure :-1 s=READY pgs=402 cs=0 l=1 rev1=1 crypto rx=0x7f273c036040 tx=0x7f273c005270 comp rx=0 tx=0).ready entity=mon.0 client_cookie=90c2876e638b1179 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:43:27.325 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.317+0000 7f2728ff9640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:43:27.325 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.317+0000 7f2728ff9640 1 -- 172.21.15.99:0/1877827672 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f273c005d50 con 0x55febdfbaaf0 2024-03-19T01:43:27.325 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.317+0000 7f2728ff9640 1 -- 172.21.15.99:0/1877827672 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f273c0048c0 con 0x55febdfbaaf0 2024-03-19T01:43:27.326 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.317+0000 7f274a057740 1 -- 172.21.15.99:0/1877827672 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f2730005500 con 0x55febdfbaaf0 2024-03-19T01:43:27.326 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.317+0000 7f2728ff9640 1 -- 172.21.15.99:0/1877827672 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=23) ==== 24+0+0 (secure 0 0 0) 0x7f273c004ac0 con 0x55febdfbaaf0 2024-03-19T01:43:27.326 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.317+0000 7f274a057740 10 client.15522 fetch_fsmap learned FSMap version 23 2024-03-19T01:43:27.326 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.317+0000 7f274a057740 1 -- 172.21.15.99:0/1877827672 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=23,monmap=4+}) -- 0x55febdc33210 con 0x55febdfbaaf0 2024-03-19T01:43:27.326 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.318+0000 7f2728ff9640 1 -- 172.21.15.99:0/1877827672 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f2730005500 con 0x55febdfbaaf0 2024-03-19T01:43:27.326 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.318+0000 7f2728ff9640 1 -- 172.21.15.99:0/1877827672 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 23) ==== 8346+0+0 (secure 0 0 0) 0x7f273c003bf0 con 0x55febdfbaaf0 2024-03-19T01:43:27.326 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.318+0000 7f274a057740 10 client.15522 fetch_fsmap finished waiting for FSMap version 23 2024-03-19T01:43:27.326 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.318+0000 7f274a057740 10 client.15522 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:43:27.326 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.318+0000 7f274a057740 20 client.15522 populate_metadata read hostname 'smithi099' 2024-03-19T01:43:27.326 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.318+0000 7f274a057740 1 --2- 172.21.15.99:0/1877827672 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55febdfcff90 0x55febdfd23d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:27.326 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.318+0000 7f274a057740 4 client.15522 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:43:27.326 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.318+0000 7f274a057740 1 -- 172.21.15.99:0/1877827672 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55febdf2c120 con 0x55febdfcff90 2024-03-19T01:43:27.326 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.318+0000 7f2744a09640 1 --2- 172.21.15.99:0/1877827672 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55febdfcff90 0x55febdfd23d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:27.327 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.318+0000 7f2728ff9640 10 client.15522 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:43:27.327 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.318+0000 7f2744a09640 1 --2- 172.21.15.99:0/1877827672 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55febdfcff90 0x55febdfd23d0 crc :-1 s=READY pgs=49 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=2064a8fddf93364d server_cookie=36f8de573db7c77b in_seq=0 out_seq=0 2024-03-19T01:43:27.327 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.319+0000 7f2728ff9640 1 -- 172.21.15.99:0/1877827672 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x55febdf2c120 con 0x55febdfcff90 2024-03-19T01:43:27.327 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.319+0000 7f2728ff9640 10 client.15522 handle_command_reply: tid=0 2024-03-19T01:43:27.327 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.319+0000 7f274a057740 1 client.15522 shutdown 2024-03-19T01:43:27.327 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.319+0000 7f274a057740 1 -- 172.21.15.99:0/1877827672 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55febdfbaaf0 msgr2=0x55febdfc4620 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:43:27.327 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.319+0000 7f274a057740 1 --2- 172.21.15.99:0/1877827672 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55febdfbaaf0 0x55febdfc4620 secure :-1 s=READY pgs=402 cs=0 l=1 rev1=1 crypto rx=0x7f273c036040 tx=0x7f273c005270 comp rx=0 tx=0).stop 2024-03-19T01:43:27.327 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.319+0000 7f274a057740 1 -- 172.21.15.99:0/1877827672 shutdown_connections 2024-03-19T01:43:27.327 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.319+0000 7f274a057740 1 --2- 172.21.15.99:0/1877827672 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55febdfcff90 0x55febdfd23d0 crc :-1 s=READY pgs=49 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:27.327 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.319+0000 7f274a057740 1 --2- 172.21.15.99:0/1877827672 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55febdfbfc60 0x55febdfcb660 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:27.327 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.319+0000 7f274a057740 1 --2- 172.21.15.99:0/1877827672 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55febdfbaaf0 0x55febdfc4620 unknown :-1 s=CLOSED pgs=402 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:27.327 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.319+0000 7f2728ff9640 0 client.15522 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:43:27.327 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.319+0000 7f274a057740 1 --2- 172.21.15.99:0/1877827672 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55febdfba100 0x55febdfc40e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:27.327 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.319+0000 7f274a057740 1 -- 172.21.15.99:0/1877827672 >> 172.21.15.99:0/1877827672 conn(0x55febdfb5670 msgr2=0x55febdfb68c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:43:27.328 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.319+0000 7f274a057740 1 -- 172.21.15.99:0/1877827672 shutdown_connections 2024-03-19T01:43:27.328 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.319+0000 7f274a057740 1 -- 172.21.15.99:0/1877827672 wait complete. 2024-03-19T01:43:27.328 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.319+0000 7f274a057740 20 client.15522 trim_cache size 0 max 16384 2024-03-19T01:43:27.338 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.334+0000 7f274a057740 1 Processor -- start 2024-03-19T01:43:27.339 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.334+0000 7f274a057740 1 -- start start 2024-03-19T01:43:27.339 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.334+0000 7f274a057740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55febdfba100 0x55febdfb4220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:27.339 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.334+0000 7f274a057740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55febdfbaaf0 0x55febdfcb620 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:27.339 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.334+0000 7f274a057740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55febdfbfc60 0x55febdfb4760 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:27.339 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.334+0000 7f274a057740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f27401ab680 con 0x55febdfba100 2024-03-19T01:43:27.339 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.334+0000 7f274a057740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f27401a8f50 con 0x55febdfbaaf0 2024-03-19T01:43:27.339 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.334+0000 7f274a057740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f27401a6da0 con 0x55febdfbfc60 2024-03-19T01:43:27.339 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.334+0000 7f2744a09640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55febdfba100 0x55febdfb4220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:27.339 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.334+0000 7f2744a09640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55febdfba100 0x55febdfb4220 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:52416/0 (socket says 172.21.15.99:52416) 2024-03-19T01:43:27.339 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.334+0000 7f2744a09640 1 -- 172.21.15.99:0/1753743884 learned_addr learned my addr 172.21.15.99:0/1753743884 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:43:27.339 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.334+0000 7f274520a640 1 --2- 172.21.15.99:0/1753743884 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55febdfbaaf0 0x55febdfcb620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:27.339 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.334+0000 7f2744a09640 1 --2- 172.21.15.99:0/1753743884 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55febdfbfc60 0x55febdfb4760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:27.339 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.335+0000 7f2744a09640 1 -- 172.21.15.99:0/1753743884 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55febdfbfc60 msgr2=0x55febdfb4760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:43:27.340 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.335+0000 7f2744a09640 1 --2- 172.21.15.99:0/1753743884 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55febdfbfc60 0x55febdfb4760 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:27.340 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.335+0000 7f2744a09640 1 -- 172.21.15.99:0/1753743884 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55febdfbaaf0 msgr2=0x55febdfcb620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:43:27.340 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.335+0000 7f2744a09640 1 --2- 172.21.15.99:0/1753743884 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55febdfbaaf0 0x55febdfcb620 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:27.340 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.335+0000 7f2744a09640 1 -- 172.21.15.99:0/1753743884 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55febdcff620 con 0x55febdfba100 2024-03-19T01:43:27.340 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.335+0000 7f2744a09640 1 --2- 172.21.15.99:0/1753743884 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55febdfba100 0x55febdfb4220 secure :-1 s=READY pgs=403 cs=0 l=1 rev1=1 crypto rx=0x55febdfbb350 tx=0x7f27300096c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=22211a6b4e170a3c server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:43:27.340 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.335+0000 7f272affd640 1 -- 172.21.15.99:0/1753743884 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2730035070 con 0x55febdfba100 2024-03-19T01:43:27.340 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.335+0000 7f272affd640 1 -- 172.21.15.99:0/1753743884 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f27300047e0 con 0x55febdfba100 2024-03-19T01:43:27.340 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.335+0000 7f272affd640 1 -- 172.21.15.99:0/1753743884 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2730004fd0 con 0x55febdfba100 2024-03-19T01:43:27.340 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.335+0000 7f274a057740 1 -- 172.21.15.99:0/1753743884 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55febdfba100 msgr2=0x55febdfb4220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:43:27.340 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.335+0000 7f274a057740 1 --2- 172.21.15.99:0/1753743884 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55febdfba100 0x55febdfb4220 secure :-1 s=READY pgs=403 cs=0 l=1 rev1=1 crypto rx=0x55febdfbb350 tx=0x7f27300096c0 comp rx=0 tx=0).stop 2024-03-19T01:43:27.340 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.335+0000 7f274a057740 1 -- 172.21.15.99:0/1753743884 shutdown_connections 2024-03-19T01:43:27.340 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.335+0000 7f274a057740 1 --2- 172.21.15.99:0/1753743884 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55febdfbaaf0 0x55febdfcb620 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:27.340 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.335+0000 7f274a057740 1 --2- 172.21.15.99:0/1753743884 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55febdfbfc60 0x55febdfb4760 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:27.340 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.335+0000 7f274a057740 1 --2- 172.21.15.99:0/1753743884 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55febdfba100 0x55febdfb4220 unknown :-1 s=CLOSED pgs=403 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:27.341 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.335+0000 7f274a057740 1 -- 172.21.15.99:0/1753743884 >> 172.21.15.99:0/1753743884 conn(0x55febdfb5670 msgr2=0x55febdfc40e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:43:27.341 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.336+0000 7f274a057740 1 -- 172.21.15.99:0/1753743884 shutdown_connections 2024-03-19T01:43:27.341 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.336+0000 7f274a057740 1 -- 172.21.15.99:0/1753743884 wait complete. 2024-03-19T01:43:27.341 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.336+0000 7f274a057740 1 -- start start 2024-03-19T01:43:27.341 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.336+0000 7f274a057740 1 Processor -- start 2024-03-19T01:43:27.341 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.336+0000 7f274a057740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55febdfba100 0x55febdfc8fb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:27.341 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.336+0000 7f274a057740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55febdfbaaf0 0x55febdfc94f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:27.341 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.336+0000 7f274a057740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55febdfbfc60 0x55febdfca500 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:27.341 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.336+0000 7f274a057740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f27401a7410 con 0x55febdfbaaf0 2024-03-19T01:43:27.341 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.336+0000 7f274a057740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f27401a8880 con 0x55febdfbfc60 2024-03-19T01:43:27.341 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.336+0000 7f274a057740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f27401a7ae0 con 0x55febdfba100 2024-03-19T01:43:27.341 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.336+0000 7f274a057740 10 client.0 _ll_register_callbacks cb 0x7f273c009780 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:43:27.341 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.336+0000 7f274520a640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55febdfbaaf0 0x55febdfc94f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:27.342 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.336+0000 7f274520a640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55febdfbaaf0 0x55febdfc94f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:52424/0 (socket says 172.21.15.99:52424) 2024-03-19T01:43:27.342 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.336+0000 7f274520a640 1 -- 172.21.15.99:0/3687102890 learned_addr learned my addr 172.21.15.99:0/3687102890 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:43:27.342 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.336+0000 7f2744a09640 1 --2- 172.21.15.99:0/3687102890 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55febdfba100 0x55febdfc8fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:27.342 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.336+0000 7f2744a09640 1 --2- 172.21.15.99:0/3687102890 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55febdfbfc60 0x55febdfca500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:27.342 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.337+0000 7f274520a640 1 -- 172.21.15.99:0/3687102890 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55febdfba100 msgr2=0x55febdfc8fb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:43:27.342 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.337+0000 7f274520a640 1 --2- 172.21.15.99:0/3687102890 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55febdfba100 0x55febdfc8fb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:27.342 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.337+0000 7f274520a640 1 -- 172.21.15.99:0/3687102890 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55febdfbfc60 msgr2=0x55febdfca500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:43:27.342 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.337+0000 7f274520a640 1 --2- 172.21.15.99:0/3687102890 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55febdfbfc60 0x55febdfca500 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:27.342 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.337+0000 7f274520a640 1 -- 172.21.15.99:0/3687102890 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55febdfcaaf0 con 0x55febdfbaaf0 2024-03-19T01:43:27.342 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.337+0000 7f2744a09640 1 --2- 172.21.15.99:0/3687102890 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55febdfbfc60 0x55febdfca500 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-19T01:43:27.342 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.337+0000 7f274520a640 1 --2- 172.21.15.99:0/3687102890 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55febdfbaaf0 0x55febdfc94f0 secure :-1 s=READY pgs=404 cs=0 l=1 rev1=1 crypto rx=0x7f273c02e000 tx=0x7f273c009830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=59531a94d8a1504b server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:43:27.342 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.337+0000 7f2728ff9640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:43:27.342 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.337+0000 7f2728ff9640 1 -- 172.21.15.99:0/3687102890 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f273c005d50 con 0x55febdfbaaf0 2024-03-19T01:43:27.343 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.337+0000 7f2728ff9640 1 -- 172.21.15.99:0/3687102890 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f273c0048c0 con 0x55febdfbaaf0 2024-03-19T01:43:27.343 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.337+0000 7f274a057740 1 -- 172.21.15.99:0/3687102890 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f27300047e0 con 0x55febdfbaaf0 2024-03-19T01:43:27.343 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.337+0000 7f2728ff9640 1 -- 172.21.15.99:0/3687102890 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=23) ==== 24+0+0 (secure 0 0 0) 0x7f273c005670 con 0x55febdfbaaf0 2024-03-19T01:43:27.343 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.337+0000 7f274a057740 10 client.15528 fetch_fsmap learned FSMap version 23 2024-03-19T01:43:27.343 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.337+0000 7f274a057740 1 -- 172.21.15.99:0/3687102890 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=23,monmap=4+}) -- 0x55febe0364f0 con 0x55febdfbaaf0 2024-03-19T01:43:27.343 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.337+0000 7f2728ff9640 1 -- 172.21.15.99:0/3687102890 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f27300047e0 con 0x55febdfbaaf0 2024-03-19T01:43:27.343 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.337+0000 7f2728ff9640 1 -- 172.21.15.99:0/3687102890 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 23) ==== 8346+0+0 (secure 0 0 0) 0x7f273c058030 con 0x55febdfbaaf0 2024-03-19T01:43:27.343 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.337+0000 7f274a057740 10 client.15528 fetch_fsmap finished waiting for FSMap version 23 2024-03-19T01:43:27.343 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.337+0000 7f274a057740 10 client.15528 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:43:27.343 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.337+0000 7f274a057740 20 client.15528 populate_metadata read hostname 'smithi099' 2024-03-19T01:43:27.343 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.337+0000 7f274a057740 1 --2- 172.21.15.99:0/3687102890 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55febe036f20 0x55febe039340 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:27.343 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.337+0000 7f274a057740 4 client.15528 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:43:27.343 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.337+0000 7f274a057740 1 -- 172.21.15.99:0/3687102890 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55febdf26470 con 0x55febe036f20 2024-03-19T01:43:27.343 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.338+0000 7f2744a09640 1 --2- 172.21.15.99:0/3687102890 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55febe036f20 0x55febe039340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:27.343 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.338+0000 7f2728ff9640 10 client.15528 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:43:27.344 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.338+0000 7f2744a09640 1 --2- 172.21.15.99:0/3687102890 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55febe036f20 0x55febe039340 crc :-1 s=READY pgs=50 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=cd39479d22376d61 server_cookie=5924f7cdaf7f6d96 in_seq=0 out_seq=0 2024-03-19T01:43:27.385 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.384+0000 7f2728ff9640 1 -- 172.21.15.99:0/3687102890 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x55febdf26470 con 0x55febe036f20 2024-03-19T01:43:27.386 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.384+0000 7f2728ff9640 10 client.15528 handle_command_reply: tid=0 2024-03-19T01:43:27.386 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.385+0000 7f274a057740 1 client.15528 shutdown 2024-03-19T01:43:27.386 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.385+0000 7f274a057740 1 -- 172.21.15.99:0/3687102890 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55febdfbaaf0 msgr2=0x55febdfc94f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:43:27.386 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.385+0000 7f274a057740 1 --2- 172.21.15.99:0/3687102890 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55febdfbaaf0 0x55febdfc94f0 secure :-1 s=READY pgs=404 cs=0 l=1 rev1=1 crypto rx=0x7f273c02e000 tx=0x7f273c009830 comp rx=0 tx=0).stop 2024-03-19T01:43:27.386 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.385+0000 7f274a057740 1 -- 172.21.15.99:0/3687102890 shutdown_connections 2024-03-19T01:43:27.386 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.385+0000 7f274a057740 1 --2- 172.21.15.99:0/3687102890 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55febe036f20 0x55febe039340 crc :-1 s=READY pgs=50 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:27.386 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.385+0000 7f274a057740 1 --2- 172.21.15.99:0/3687102890 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55febdfbfc60 0x55febdfca500 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:27.386 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.385+0000 7f274a057740 1 --2- 172.21.15.99:0/3687102890 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55febdfbaaf0 0x55febdfc94f0 unknown :-1 s=CLOSED pgs=404 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:27.386 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.385+0000 7f2728ff9640 0 client.15528 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:43:27.386 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.385+0000 7f274a057740 1 --2- 172.21.15.99:0/3687102890 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55febdfba100 0x55febdfc8fb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:27.386 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.385+0000 7f274a057740 1 -- 172.21.15.99:0/3687102890 >> 172.21.15.99:0/3687102890 conn(0x55febdfb5670 msgr2=0x55febdfc3fb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:43:27.386 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.385+0000 7f274a057740 1 -- 172.21.15.99:0/3687102890 shutdown_connections 2024-03-19T01:43:27.386 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.385+0000 7f274a057740 1 -- 172.21.15.99:0/3687102890 wait complete. 2024-03-19T01:43:27.387 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.385+0000 7f274a057740 20 client.15528 trim_cache size 0 max 16384 2024-03-19T01:43:27.387 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:43:27.387 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:43:27.387 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:43:27.387 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:43:27.388 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.385+0000 7f274746e640 1 -- 172.21.15.99:0/3000939464 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f272c0a28a0 msgr2=0x7f272c0a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:43:27.388 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.385+0000 7f274746e640 1 --2- 172.21.15.99:0/3000939464 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f272c0a28a0 0x7f272c0a4d60 secure :-1 s=READY pgs=258 cs=0 l=1 rev1=1 crypto rx=0x7f2738009200 tx=0x7f27380668a0 comp rx=0 tx=0).stop 2024-03-19T01:43:27.388 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.385+0000 7f274746e640 1 -- 172.21.15.99:0/3000939464 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f27401022f0 msgr2=0x7f274019f080 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:43:27.388 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.385+0000 7f274746e640 1 --2- 172.21.15.99:0/3000939464 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f27401022f0 0x7f274019f080 secure :-1 s=READY pgs=401 cs=0 l=1 rev1=1 crypto rx=0x7f273800eb10 tx=0x7f273800c2d0 comp rx=0 tx=0).stop 2024-03-19T01:43:27.388 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.386+0000 7f274746e640 1 -- 172.21.15.99:0/3000939464 shutdown_connections 2024-03-19T01:43:27.388 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.386+0000 7f274746e640 1 --2- 172.21.15.99:0/3000939464 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f27401022f0 0x7f274019f080 unknown :-1 s=CLOSED pgs=401 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:27.388 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.386+0000 7f274746e640 1 --2- 172.21.15.99:0/3000939464 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f2740101930 0x7f2740198010 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:27.388 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.386+0000 7f274746e640 1 --2- 172.21.15.99:0/3000939464 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f272c0a28a0 0x7f272c0a4d60 unknown :-1 s=CLOSED pgs=258 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:27.388 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.386+0000 7f274746e640 1 --2- 172.21.15.99:0/3000939464 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f2740100740 0x7f2740197ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:27.388 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.386+0000 7f274746e640 1 -- 172.21.15.99:0/3000939464 >> 172.21.15.99:0/3000939464 conn(0x7f27400fbcb0 msgr2=0x7f27400fd6e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:43:27.389 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.386+0000 7f274746e640 1 -- 172.21.15.99:0/3000939464 shutdown_connections 2024-03-19T01:43:27.389 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:27.386+0000 7f274746e640 1 -- 172.21.15.99:0/3000939464 wait complete. 2024-03-19T01:43:27.677 INFO:tasks.workunit.client.0.smithi099.stdout: 1 258783 12.07 MB/sec warmup 117 sec latency 44.049 ms 2024-03-19T01:43:28.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:43:27 smithi115 ceph-mon[67332]: pgmap v421: 97 pgs: 97 active+clean; 102 MiB data, 1.0 GiB used, 1.0 TiB / 1.0 TiB avail; 1.2 MiB/s rd, 3.2 MiB/s wr, 286 op/s 2024-03-19T01:43:28.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:43:27 smithi099 ceph-mon[62951]: pgmap v421: 97 pgs: 97 active+clean; 102 MiB data, 1.0 GiB used, 1.0 TiB / 1.0 TiB avail; 1.2 MiB/s rd, 3.2 MiB/s wr, 286 op/s 2024-03-19T01:43:28.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:43:27 smithi112 ceph-mon[66819]: pgmap v421: 97 pgs: 97 active+clean; 102 MiB data, 1.0 GiB used, 1.0 TiB / 1.0 TiB avail; 1.2 MiB/s rd, 3.2 MiB/s wr, 286 op/s 2024-03-19T01:43:28.677 INFO:tasks.workunit.client.0.smithi099.stdout: 1 260201 12.03 MB/sec warmup 118 sec latency 67.646 ms 2024-03-19T01:43:29.677 INFO:tasks.workunit.client.0.smithi099.stdout: 1 262835 12.06 MB/sec warmup 119 sec latency 19.068 ms 2024-03-19T01:43:30.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:43:29 smithi115 ceph-mon[67332]: pgmap v422: 97 pgs: 97 active+clean; 104 MiB data, 1.0 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 3.0 MiB/s wr, 256 op/s 2024-03-19T01:43:30.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:43:29 smithi099 ceph-mon[62951]: pgmap v422: 97 pgs: 97 active+clean; 104 MiB data, 1.0 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 3.0 MiB/s wr, 256 op/s 2024-03-19T01:43:30.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:43:29 smithi112 ceph-mon[66819]: pgmap v422: 97 pgs: 97 active+clean; 104 MiB data, 1.0 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 3.0 MiB/s wr, 256 op/s 2024-03-19T01:43:31.677 INFO:tasks.workunit.client.0.smithi099.stdout: 1 266866 12.62 MB/sec execute 1 sec latency 42.899 ms 2024-03-19T01:43:32.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:43:31 smithi115 ceph-mon[67332]: pgmap v423: 97 pgs: 97 active+clean; 100 MiB data, 1.0 GiB used, 1.0 TiB / 1.0 TiB avail; 994 KiB/s rd, 2.7 MiB/s wr, 231 op/s 2024-03-19T01:43:32.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:43:31 smithi099 ceph-mon[62951]: pgmap v423: 97 pgs: 97 active+clean; 100 MiB data, 1.0 GiB used, 1.0 TiB / 1.0 TiB avail; 994 KiB/s rd, 2.7 MiB/s wr, 231 op/s 2024-03-19T01:43:32.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:43:31 smithi112 ceph-mon[66819]: pgmap v423: 97 pgs: 97 active+clean; 100 MiB data, 1.0 GiB used, 1.0 TiB / 1.0 TiB avail; 994 KiB/s rd, 2.7 MiB/s wr, 231 op/s 2024-03-19T01:43:32.677 INFO:tasks.workunit.client.0.smithi099.stdout: 1 269639 14.79 MB/sec execute 2 sec latency 28.622 ms 2024-03-19T01:43:33.677 INFO:tasks.workunit.client.0.smithi099.stdout: 1 271251 12.16 MB/sec execute 3 sec latency 39.846 ms 2024-03-19T01:43:34.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:43:33 smithi115 ceph-mon[67332]: pgmap v424: 97 pgs: 97 active+clean; 103 MiB data, 1.0 GiB used, 1.0 TiB / 1.0 TiB avail; 1.2 MiB/s rd, 3.1 MiB/s wr, 292 op/s 2024-03-19T01:43:34.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:43:33 smithi099 ceph-mon[62951]: pgmap v424: 97 pgs: 97 active+clean; 103 MiB data, 1.0 GiB used, 1.0 TiB / 1.0 TiB avail; 1.2 MiB/s rd, 3.1 MiB/s wr, 292 op/s 2024-03-19T01:43:34.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:43:33 smithi112 ceph-mon[66819]: pgmap v424: 97 pgs: 97 active+clean; 103 MiB data, 1.0 GiB used, 1.0 TiB / 1.0 TiB avail; 1.2 MiB/s rd, 3.1 MiB/s wr, 292 op/s 2024-03-19T01:43:34.677 INFO:tasks.workunit.client.0.smithi099.stdout: 1 273479 12.51 MB/sec execute 4 sec latency 100.947 ms 2024-03-19T01:43:35.678 INFO:tasks.workunit.client.0.smithi099.stdout: 1 275541 11.47 MB/sec execute 5 sec latency 29.318 ms 2024-03-19T01:43:36.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:43:35 smithi115 ceph-mon[67332]: pgmap v425: 97 pgs: 97 active+clean; 103 MiB data, 1.0 GiB used, 1.0 TiB / 1.0 TiB avail; 961 KiB/s rd, 2.6 MiB/s wr, 232 op/s 2024-03-19T01:43:36.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:43:35 smithi099 ceph-mon[62951]: pgmap v425: 97 pgs: 97 active+clean; 103 MiB data, 1.0 GiB used, 1.0 TiB / 1.0 TiB avail; 961 KiB/s rd, 2.6 MiB/s wr, 232 op/s 2024-03-19T01:43:36.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:43:35 smithi112 ceph-mon[66819]: pgmap v425: 97 pgs: 97 active+clean; 103 MiB data, 1.0 GiB used, 1.0 TiB / 1.0 TiB avail; 961 KiB/s rd, 2.6 MiB/s wr, 232 op/s 2024-03-19T01:43:36.677 INFO:tasks.workunit.client.0.smithi099.stdout: 1 277484 11.65 MB/sec execute 6 sec latency 29.213 ms 2024-03-19T01:43:37.677 INFO:tasks.workunit.client.0.smithi099.stdout: 1 280056 11.99 MB/sec execute 7 sec latency 24.078 ms 2024-03-19T01:43:38.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:43:37 smithi115 ceph-mon[67332]: pgmap v426: 97 pgs: 97 active+clean; 107 MiB data, 1.1 GiB used, 1.0 TiB / 1.0 TiB avail; 1.4 MiB/s rd, 3.4 MiB/s wr, 308 op/s 2024-03-19T01:43:38.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:43:37 smithi099 ceph-mon[62951]: pgmap v426: 97 pgs: 97 active+clean; 107 MiB data, 1.1 GiB used, 1.0 TiB / 1.0 TiB avail; 1.4 MiB/s rd, 3.4 MiB/s wr, 308 op/s 2024-03-19T01:43:38.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:43:37 smithi112 ceph-mon[66819]: pgmap v426: 97 pgs: 97 active+clean; 107 MiB data, 1.1 GiB used, 1.0 TiB / 1.0 TiB avail; 1.4 MiB/s rd, 3.4 MiB/s wr, 308 op/s 2024-03-19T01:43:38.678 INFO:tasks.workunit.client.0.smithi099.stdout: 1 281861 11.72 MB/sec execute 8 sec latency 30.240 ms 2024-03-19T01:43:39.677 INFO:tasks.workunit.client.0.smithi099.stdout: 1 284087 11.93 MB/sec execute 9 sec latency 52.638 ms 2024-03-19T01:43:40.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:43:39 smithi115 ceph-mon[67332]: pgmap v427: 97 pgs: 97 active+clean; 107 MiB data, 1.1 GiB used, 1.0 TiB / 1.0 TiB avail; 1.3 MiB/s rd, 3.1 MiB/s wr, 287 op/s 2024-03-19T01:43:40.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:43:39 smithi099 ceph-mon[62951]: pgmap v427: 97 pgs: 97 active+clean; 107 MiB data, 1.1 GiB used, 1.0 TiB / 1.0 TiB avail; 1.3 MiB/s rd, 3.1 MiB/s wr, 287 op/s 2024-03-19T01:43:40.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:43:39 smithi112 ceph-mon[66819]: pgmap v427: 97 pgs: 97 active+clean; 107 MiB data, 1.1 GiB used, 1.0 TiB / 1.0 TiB avail; 1.3 MiB/s rd, 3.1 MiB/s wr, 287 op/s 2024-03-19T01:43:40.478 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.476+0000 7f8d5548c640 1 -- 172.21.15.99:0/938948680 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8d50100740 msgr2=0x7f8d50100b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:43:40.478 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.476+0000 7f8d5548c640 1 --2- 172.21.15.99:0/938948680 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8d50100740 0x7f8d50100b60 secure :-1 s=READY pgs=405 cs=0 l=1 rev1=1 crypto rx=0x7f8d40009900 tx=0x7f8d4002bc30 comp rx=0 tx=0).stop 2024-03-19T01:43:40.478 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.476+0000 7f8d5548c640 1 -- 172.21.15.99:0/938948680 shutdown_connections 2024-03-19T01:43:40.478 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.476+0000 7f8d5548c640 1 --2- 172.21.15.99:0/938948680 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8d501022f0 0x7f8d50108b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:40.478 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.476+0000 7f8d5548c640 1 --2- 172.21.15.99:0/938948680 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8d50101930 0x7f8d50101db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:40.478 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.476+0000 7f8d5548c640 1 --2- 172.21.15.99:0/938948680 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8d50100740 0x7f8d50100b60 unknown :-1 s=CLOSED pgs=405 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:40.478 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.476+0000 7f8d5548c640 1 -- 172.21.15.99:0/938948680 >> 172.21.15.99:0/938948680 conn(0x7f8d500fbcb0 msgr2=0x7f8d500fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:43:40.479 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.476+0000 7f8d5548c640 1 -- 172.21.15.99:0/938948680 shutdown_connections 2024-03-19T01:43:40.479 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.476+0000 7f8d5548c640 1 -- 172.21.15.99:0/938948680 wait complete. 2024-03-19T01:43:40.479 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.476+0000 7f8d5548c640 1 Processor -- start 2024-03-19T01:43:40.479 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.476+0000 7f8d5548c640 1 -- start start 2024-03-19T01:43:40.479 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.477+0000 7f8d5548c640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8d50100740 0x7f8d50196a00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:40.479 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.477+0000 7f8d5548c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8d50101930 0x7f8d50196f40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:40.479 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.477+0000 7f8d5548c640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8d501022f0 0x7f8d5019dfb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:40.479 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.477+0000 7f8d5548c640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f8d5010c390 con 0x7f8d50101930 2024-03-19T01:43:40.479 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.477+0000 7f8d5548c640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f8d5010c210 con 0x7f8d501022f0 2024-03-19T01:43:40.479 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.477+0000 7f8d5548c640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f8d5010c510 con 0x7f8d50100740 2024-03-19T01:43:40.479 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.477+0000 7f8d47fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8d50101930 0x7f8d50196f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:40.479 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.477+0000 7f8d47fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8d50101930 0x7f8d50196f40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:48892/0 (socket says 172.21.15.99:48892) 2024-03-19T01:43:40.479 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.477+0000 7f8d47fff640 1 -- 172.21.15.99:0/1839666741 learned_addr learned my addr 172.21.15.99:0/1839666741 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:43:40.479 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.477+0000 7f8d4effd640 1 --2- 172.21.15.99:0/1839666741 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8d50100740 0x7f8d50196a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:40.480 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.477+0000 7f8d4f7fe640 1 --2- 172.21.15.99:0/1839666741 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8d501022f0 0x7f8d5019dfb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:40.481 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.477+0000 7f8d47fff640 1 -- 172.21.15.99:0/1839666741 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8d50100740 msgr2=0x7f8d50196a00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:43:40.481 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.477+0000 7f8d47fff640 1 --2- 172.21.15.99:0/1839666741 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8d50100740 0x7f8d50196a00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:40.481 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.477+0000 7f8d47fff640 1 -- 172.21.15.99:0/1839666741 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8d501022f0 msgr2=0x7f8d5019dfb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:43:40.481 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.477+0000 7f8d47fff640 1 --2- 172.21.15.99:0/1839666741 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8d501022f0 0x7f8d5019dfb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:40.481 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.477+0000 7f8d47fff640 1 -- 172.21.15.99:0/1839666741 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f8d5019e620 con 0x7f8d50101930 2024-03-19T01:43:40.481 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.477+0000 7f8d4f7fe640 1 --2- 172.21.15.99:0/1839666741 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8d501022f0 0x7f8d5019dfb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:43:40.481 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.477+0000 7f8d47fff640 1 --2- 172.21.15.99:0/1839666741 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8d50101930 0x7f8d50196f40 secure :-1 s=READY pgs=406 cs=0 l=1 rev1=1 crypto rx=0x7f8d3800e9f0 tx=0x7f8d3800eeb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:43:40.482 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.477+0000 7f8d4cff9640 1 -- 172.21.15.99:0/1839666741 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8d3800cd80 con 0x7f8d50101930 2024-03-19T01:43:40.482 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.477+0000 7f8d4cff9640 1 -- 172.21.15.99:0/1839666741 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f8d38010910 con 0x7f8d50101930 2024-03-19T01:43:40.482 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.477+0000 7f8d5548c640 1 -- 172.21.15.99:0/1839666741 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f8d5019e910 con 0x7f8d50101930 2024-03-19T01:43:40.482 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.477+0000 7f8d4cff9640 1 -- 172.21.15.99:0/1839666741 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8d38018990 con 0x7f8d50101930 2024-03-19T01:43:40.482 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.477+0000 7f8d5548c640 1 -- 172.21.15.99:0/1839666741 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f8d5019eeb0 con 0x7f8d50101930 2024-03-19T01:43:40.488 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.480+0000 7f8d4cff9640 1 -- 172.21.15.99:0/1839666741 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f8d38010ac0 con 0x7f8d50101930 2024-03-19T01:43:40.488 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.480+0000 7f8d5807e740 1 Processor -- start 2024-03-19T01:43:40.488 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.480+0000 7f8d5807e740 1 -- start start 2024-03-19T01:43:40.488 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.480+0000 7f8d5807e740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561d62291100 0x561d62291520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:40.488 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.480+0000 7f8d5807e740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561d62291af0 0x561d622965f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:40.489 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.480+0000 7f8d5807e740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561d62296c60 0x561d622990f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:40.489 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.480+0000 7f8d5807e740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x561d621f5be0 con 0x561d62296c60 2024-03-19T01:43:40.489 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.480+0000 7f8d5807e740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x561d6219e960 con 0x561d62291af0 2024-03-19T01:43:40.489 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.480+0000 7f8d5807e740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x561d621f1c80 con 0x561d62291100 2024-03-19T01:43:40.489 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.481+0000 7f8d4cff9640 1 --2- 172.21.15.99:0/1839666741 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f8d3c0a2920 0x7f8d3c0a4de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:40.489 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.481+0000 7f8d47fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561d62296c60 0x561d622990f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:40.489 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.481+0000 7f8d4effd640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561d62291af0 0x561d622965f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:40.489 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.481+0000 7f8d4effd640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561d62291af0 0x561d622965f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:51534/0 (socket says 172.21.15.99:51534) 2024-03-19T01:43:40.489 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.481+0000 7f8d4effd640 1 -- 172.21.15.99:0/585760033 learned_addr learned my addr 172.21.15.99:0/585760033 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:43:40.489 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.481+0000 7f8d4f7fe640 1 --2- 172.21.15.99:0/585760033 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561d62291100 0x561d62291520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:40.489 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.481+0000 7f8d4f7fe640 1 --2- 172.21.15.99:0/1839666741 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f8d3c0a2920 0x7f8d3c0a4de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:40.489 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.481+0000 7f8d4cff9640 1 -- 172.21.15.99:0/1839666741 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f8d380ced30 con 0x7f8d50101930 2024-03-19T01:43:40.489 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.481+0000 7f8d4effd640 1 -- 172.21.15.99:0/585760033 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561d62291100 msgr2=0x561d62291520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:43:40.490 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.481+0000 7f8d4effd640 1 --2- 172.21.15.99:0/585760033 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561d62291100 0x561d62291520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:40.490 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.481+0000 7f8d4effd640 1 -- 172.21.15.99:0/585760033 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561d62296c60 msgr2=0x561d622990f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:43:40.490 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.481+0000 7f8d4effd640 1 --2- 172.21.15.99:0/585760033 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561d62296c60 0x561d622990f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:40.490 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.481+0000 7f8d4effd640 1 -- 172.21.15.99:0/585760033 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x561d62195d20 con 0x561d62291af0 2024-03-19T01:43:40.490 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.481+0000 7f8d4effd640 1 --2- 172.21.15.99:0/585760033 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561d62291af0 0x561d622965f0 secure :-1 s=READY pgs=97 cs=0 l=1 rev1=1 crypto rx=0x7f8d40009900 tx=0x7f8d40009560 comp rx=0 tx=0).ready entity=mon.2 client_cookie=45ed895cc3f61d9 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:43:40.490 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.481+0000 7f8d4f7fe640 1 --2- 172.21.15.99:0/1839666741 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f8d3c0a2920 0x7f8d3c0a4de0 secure :-1 s=READY pgs=259 cs=0 l=1 rev1=1 crypto rx=0x7f8d480099e0 tx=0x7f8d48009500 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:43:40.490 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.481+0000 7f8d457fa640 1 -- 172.21.15.99:0/585760033 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8d4002cb80 con 0x561d62291af0 2024-03-19T01:43:40.490 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.481+0000 7f8d457fa640 1 -- 172.21.15.99:0/585760033 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f8d40005040 con 0x561d62291af0 2024-03-19T01:43:40.490 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.481+0000 7f8d457fa640 1 -- 172.21.15.99:0/585760033 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8d400048c0 con 0x561d62291af0 2024-03-19T01:43:40.490 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.482+0000 7f8d5807e740 1 -- 172.21.15.99:0/585760033 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561d62291af0 msgr2=0x561d622965f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:43:40.490 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.482+0000 7f8d5807e740 1 --2- 172.21.15.99:0/585760033 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561d62291af0 0x561d622965f0 secure :-1 s=READY pgs=97 cs=0 l=1 rev1=1 crypto rx=0x7f8d40009900 tx=0x7f8d40009560 comp rx=0 tx=0).stop 2024-03-19T01:43:40.490 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.482+0000 7f8d5807e740 1 -- 172.21.15.99:0/585760033 shutdown_connections 2024-03-19T01:43:40.490 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.482+0000 7f8d5807e740 1 --2- 172.21.15.99:0/585760033 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561d62291af0 0x561d622965f0 unknown :-1 s=CLOSED pgs=97 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:40.490 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.482+0000 7f8d5807e740 1 --2- 172.21.15.99:0/585760033 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561d62296c60 0x561d622990f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:40.491 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.482+0000 7f8d5807e740 1 --2- 172.21.15.99:0/585760033 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561d62291100 0x561d62291520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:40.491 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.482+0000 7f8d5807e740 1 -- 172.21.15.99:0/585760033 >> 172.21.15.99:0/585760033 conn(0x561d6228c670 msgr2=0x561d6228ca70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:43:40.491 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.482+0000 7f8d5807e740 1 -- 172.21.15.99:0/585760033 shutdown_connections 2024-03-19T01:43:40.491 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.482+0000 7f8d5807e740 1 -- 172.21.15.99:0/585760033 wait complete. 2024-03-19T01:43:40.491 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.483+0000 7f8d5807e740 1 -- start start 2024-03-19T01:43:40.491 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.483+0000 7f8d5807e740 1 Processor -- start 2024-03-19T01:43:40.491 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.483+0000 7f8d5807e740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561d62291100 0x561d6229b0b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:40.491 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.483+0000 7f8d5807e740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561d62291af0 0x561d6229b5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:40.491 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.483+0000 7f8d5807e740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561d62296c60 0x561d622a2630 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:40.491 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.483+0000 7f8d5807e740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x561d62299bb0 con 0x561d62291100 2024-03-19T01:43:40.491 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.483+0000 7f8d5807e740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x561d621b1db0 con 0x561d62296c60 2024-03-19T01:43:40.491 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.483+0000 7f8d5807e740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x561d62299d30 con 0x561d62291af0 2024-03-19T01:43:40.491 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.483+0000 7f8d5807e740 10 client.0 _ll_register_callbacks cb 0x561d621b6e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:43:40.492 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.483+0000 7f8d4effd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561d62291100 0x561d6229b0b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:40.492 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.483+0000 7f8d4effd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561d62291100 0x561d6229b0b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:48914/0 (socket says 172.21.15.99:48914) 2024-03-19T01:43:40.492 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.483+0000 7f8d4effd640 1 -- 172.21.15.99:0/1376685434 learned_addr learned my addr 172.21.15.99:0/1376685434 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:43:40.492 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.483+0000 7f8d4f7fe640 1 --2- 172.21.15.99:0/1376685434 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561d62296c60 0x561d622a2630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:40.492 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.483+0000 7f8d47fff640 1 --2- 172.21.15.99:0/1376685434 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561d62291af0 0x561d6229b5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:40.492 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.483+0000 7f8d4effd640 1 -- 172.21.15.99:0/1376685434 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561d62291af0 msgr2=0x561d6229b5f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:43:40.492 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.483+0000 7f8d4effd640 1 --2- 172.21.15.99:0/1376685434 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561d62291af0 0x561d6229b5f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:40.492 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.483+0000 7f8d4effd640 1 -- 172.21.15.99:0/1376685434 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561d62296c60 msgr2=0x561d622a2630 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:43:40.492 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.483+0000 7f8d4effd640 1 --2- 172.21.15.99:0/1376685434 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561d62296c60 0x561d622a2630 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:40.492 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.483+0000 7f8d4effd640 1 -- 172.21.15.99:0/1376685434 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x561d621b0c60 con 0x561d62291100 2024-03-19T01:43:40.492 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.483+0000 7f8d4effd640 1 --2- 172.21.15.99:0/1376685434 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561d62291100 0x561d6229b0b0 secure :-1 s=READY pgs=407 cs=0 l=1 rev1=1 crypto rx=0x7f8d4002c170 tx=0x7f8d400054a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6b01ec77ea9178b2 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:43:40.492 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.483+0000 7f8d367fc640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:43:40.492 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.484+0000 7f8d367fc640 1 -- 172.21.15.99:0/1376685434 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8d40004ad0 con 0x561d62291100 2024-03-19T01:43:40.492 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.484+0000 7f8d367fc640 1 -- 172.21.15.99:0/1376685434 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f8d40004c70 con 0x561d62291100 2024-03-19T01:43:40.493 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.484+0000 7f8d5807e740 1 -- 172.21.15.99:0/1376685434 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f8d40005040 con 0x561d62291100 2024-03-19T01:43:40.493 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.484+0000 7f8d367fc640 1 -- 172.21.15.99:0/1376685434 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=23) ==== 24+0+0 (secure 0 0 0) 0x7f8d40056050 con 0x561d62291100 2024-03-19T01:43:40.493 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.484+0000 7f8d5807e740 10 client.15540 fetch_fsmap learned FSMap version 23 2024-03-19T01:43:40.493 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.484+0000 7f8d5807e740 1 -- 172.21.15.99:0/1376685434 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=23,monmap=4+}) -- 0x561d61f0a210 con 0x561d62291100 2024-03-19T01:43:40.493 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.484+0000 7f8d367fc640 1 -- 172.21.15.99:0/1376685434 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f8d40005040 con 0x561d62291100 2024-03-19T01:43:40.493 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.484+0000 7f8d367fc640 1 -- 172.21.15.99:0/1376685434 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 23) ==== 8346+0+0 (secure 0 0 0) 0x7f8d4004d070 con 0x561d62291100 2024-03-19T01:43:40.493 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.484+0000 7f8d5807e740 10 client.15540 fetch_fsmap finished waiting for FSMap version 23 2024-03-19T01:43:40.493 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.484+0000 7f8d5807e740 10 client.15540 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:43:40.493 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.484+0000 7f8d5807e740 20 client.15540 populate_metadata read hostname 'smithi099' 2024-03-19T01:43:40.493 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.484+0000 7f8d5807e740 1 --2- 172.21.15.99:0/1376685434 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x561d622a6b20 0x561d622a8f10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:40.493 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.484+0000 7f8d5807e740 4 client.15540 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:43:40.493 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.484+0000 7f8d5807e740 1 -- 172.21.15.99:0/1376685434 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x561d62203120 con 0x561d622a6b20 2024-03-19T01:43:40.493 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.484+0000 7f8d47fff640 1 --2- 172.21.15.99:0/1376685434 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x561d622a6b20 0x561d622a8f10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:40.493 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.485+0000 7f8d367fc640 10 client.15540 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:43:40.493 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.485+0000 7f8d47fff640 1 --2- 172.21.15.99:0/1376685434 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x561d622a6b20 0x561d622a8f10 crc :-1 s=READY pgs=51 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=86af7f9fbd0bd84d server_cookie=9039cb3643b400bd in_seq=0 out_seq=0 2024-03-19T01:43:40.494 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.486+0000 7f8d367fc640 1 -- 172.21.15.99:0/1376685434 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x561d62203120 con 0x561d622a6b20 2024-03-19T01:43:40.494 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.486+0000 7f8d367fc640 10 client.15540 handle_command_reply: tid=0 2024-03-19T01:43:40.494 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.486+0000 7f8d5807e740 1 client.15540 shutdown 2024-03-19T01:43:40.494 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.486+0000 7f8d5807e740 1 -- 172.21.15.99:0/1376685434 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561d62291100 msgr2=0x561d6229b0b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:43:40.494 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.486+0000 7f8d5807e740 1 --2- 172.21.15.99:0/1376685434 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561d62291100 0x561d6229b0b0 secure :-1 s=READY pgs=407 cs=0 l=1 rev1=1 crypto rx=0x7f8d4002c170 tx=0x7f8d400054a0 comp rx=0 tx=0).stop 2024-03-19T01:43:40.494 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.486+0000 7f8d5807e740 1 -- 172.21.15.99:0/1376685434 shutdown_connections 2024-03-19T01:43:40.494 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.486+0000 7f8d5807e740 1 --2- 172.21.15.99:0/1376685434 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x561d622a6b20 0x561d622a8f10 crc :-1 s=READY pgs=51 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:40.494 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.486+0000 7f8d5807e740 1 --2- 172.21.15.99:0/1376685434 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561d62296c60 0x561d622a2630 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:40.494 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.486+0000 7f8d367fc640 0 client.15540 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:43:40.494 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.486+0000 7f8d5807e740 1 --2- 172.21.15.99:0/1376685434 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561d62291af0 0x561d6229b5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:40.494 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.486+0000 7f8d5807e740 1 --2- 172.21.15.99:0/1376685434 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561d62291100 0x561d6229b0b0 unknown :-1 s=CLOSED pgs=407 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:40.494 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.486+0000 7f8d5807e740 1 -- 172.21.15.99:0/1376685434 >> 172.21.15.99:0/1376685434 conn(0x561d6228c670 msgr2=0x561d6228d8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:43:40.494 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.486+0000 7f8d5807e740 1 -- 172.21.15.99:0/1376685434 shutdown_connections 2024-03-19T01:43:40.495 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.486+0000 7f8d5807e740 1 -- 172.21.15.99:0/1376685434 wait complete. 2024-03-19T01:43:40.495 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.486+0000 7f8d5807e740 20 client.15540 trim_cache size 0 max 16384 2024-03-19T01:43:40.502 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.501+0000 7f8d5807e740 1 Processor -- start 2024-03-19T01:43:40.502 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.501+0000 7f8d5807e740 1 -- start start 2024-03-19T01:43:40.503 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.502+0000 7f8d5807e740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561d62291100 0x561d6228ffe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:40.503 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.502+0000 7f8d5807e740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561d62291af0 0x561d6229a6d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:40.503 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.502+0000 7f8d5807e740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561d62296c60 0x561d6229ac10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:40.503 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.502+0000 7f8d5807e740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f8d501aa570 con 0x561d62291af0 2024-03-19T01:43:40.503 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.502+0000 7f8d5807e740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f8d501a7e40 con 0x561d62291100 2024-03-19T01:43:40.503 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.502+0000 7f8d5807e740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f8d501a5c90 con 0x561d62296c60 2024-03-19T01:43:40.503 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.502+0000 7f8d47fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561d62291af0 0x561d6229a6d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:40.503 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.502+0000 7f8d4effd640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561d62291100 0x561d6228ffe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:40.503 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.502+0000 7f8d47fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561d62291af0 0x561d6229a6d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:48928/0 (socket says 172.21.15.99:48928) 2024-03-19T01:43:40.503 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.502+0000 7f8d47fff640 1 -- 172.21.15.99:0/2426830143 learned_addr learned my addr 172.21.15.99:0/2426830143 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:43:40.503 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.502+0000 7f8d4f7fe640 1 --2- 172.21.15.99:0/2426830143 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561d62296c60 0x561d6229ac10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:40.503 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.502+0000 7f8d4effd640 1 -- 172.21.15.99:0/2426830143 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561d62296c60 msgr2=0x561d6229ac10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:43:40.503 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.502+0000 7f8d4effd640 1 --2- 172.21.15.99:0/2426830143 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561d62296c60 0x561d6229ac10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:40.504 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.502+0000 7f8d4effd640 1 -- 172.21.15.99:0/2426830143 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561d62291af0 msgr2=0x561d6229a6d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:43:40.504 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.502+0000 7f8d4effd640 1 --2- 172.21.15.99:0/2426830143 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561d62291af0 0x561d6229a6d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:40.504 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.502+0000 7f8d4effd640 1 -- 172.21.15.99:0/2426830143 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x561d61fd6620 con 0x561d62291100 2024-03-19T01:43:40.504 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.502+0000 7f8d47fff640 1 --2- 172.21.15.99:0/2426830143 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561d62291af0 0x561d6229a6d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:43:40.504 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.502+0000 7f8d4effd640 1 --2- 172.21.15.99:0/2426830143 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561d62291100 0x561d6228ffe0 secure :-1 s=READY pgs=98 cs=0 l=1 rev1=1 crypto rx=0x7f8d4002e000 tx=0x7f8d4002ce00 comp rx=0 tx=0).ready entity=mon.2 client_cookie=b67234b3adfe205e server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:43:40.504 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.502+0000 7f8d44ff9640 1 -- 172.21.15.99:0/2426830143 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8d40005710 con 0x561d62291100 2024-03-19T01:43:40.504 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.502+0000 7f8d44ff9640 1 -- 172.21.15.99:0/2426830143 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f8d40005040 con 0x561d62291100 2024-03-19T01:43:40.504 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.502+0000 7f8d44ff9640 1 -- 172.21.15.99:0/2426830143 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8d40045bb0 con 0x561d62291100 2024-03-19T01:43:40.508 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.503+0000 7f8d5807e740 1 -- 172.21.15.99:0/2426830143 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561d62291100 msgr2=0x561d6228ffe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:43:40.508 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.503+0000 7f8d5807e740 1 --2- 172.21.15.99:0/2426830143 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561d62291100 0x561d6228ffe0 secure :-1 s=READY pgs=98 cs=0 l=1 rev1=1 crypto rx=0x7f8d4002e000 tx=0x7f8d4002ce00 comp rx=0 tx=0).stop 2024-03-19T01:43:40.508 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.503+0000 7f8d5807e740 1 -- 172.21.15.99:0/2426830143 shutdown_connections 2024-03-19T01:43:40.508 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.503+0000 7f8d5807e740 1 --2- 172.21.15.99:0/2426830143 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561d62296c60 0x561d6229ac10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:40.508 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.503+0000 7f8d5807e740 1 --2- 172.21.15.99:0/2426830143 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561d62291af0 0x561d6229a6d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:40.508 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.503+0000 7f8d5807e740 1 --2- 172.21.15.99:0/2426830143 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561d62291100 0x561d6228ffe0 unknown :-1 s=CLOSED pgs=98 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:40.508 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.503+0000 7f8d5807e740 1 -- 172.21.15.99:0/2426830143 >> 172.21.15.99:0/2426830143 conn(0x561d6228c670 msgr2=0x561d6228f8d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:43:40.509 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.503+0000 7f8d5807e740 1 -- 172.21.15.99:0/2426830143 shutdown_connections 2024-03-19T01:43:40.509 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.503+0000 7f8d5807e740 1 -- 172.21.15.99:0/2426830143 wait complete. 2024-03-19T01:43:40.509 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.503+0000 7f8d5807e740 1 -- start start 2024-03-19T01:43:40.509 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.504+0000 7f8d5807e740 1 Processor -- start 2024-03-19T01:43:40.509 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.504+0000 7f8d5807e740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561d62291100 0x561d62307070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:40.509 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.504+0000 7f8d5807e740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561d62291af0 0x561d623075d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:40.509 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.504+0000 7f8d5807e740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561d62296c60 0x561d6230def0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:40.509 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.504+0000 7f8d5807e740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f8d501a6300 con 0x561d62291af0 2024-03-19T01:43:40.509 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.504+0000 7f8d5807e740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f8d501a7770 con 0x561d62296c60 2024-03-19T01:43:40.509 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.504+0000 7f8d5807e740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f8d501a69d0 con 0x561d62291100 2024-03-19T01:43:40.509 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.504+0000 7f8d5807e740 10 client.0 _ll_register_callbacks cb 0x7f8d380973a0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:43:40.509 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.504+0000 7f8d47fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561d62291af0 0x561d623075d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:40.509 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.504+0000 7f8d47fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561d62291af0 0x561d623075d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:48944/0 (socket says 172.21.15.99:48944) 2024-03-19T01:43:40.510 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.504+0000 7f8d47fff640 1 -- 172.21.15.99:0/886725610 learned_addr learned my addr 172.21.15.99:0/886725610 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:43:40.510 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.504+0000 7f8d4effd640 1 --2- 172.21.15.99:0/886725610 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561d62291100 0x561d62307070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:40.510 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.504+0000 7f8d4f7fe640 1 --2- 172.21.15.99:0/886725610 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561d62296c60 0x561d6230def0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:40.510 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.504+0000 7f8d47fff640 1 -- 172.21.15.99:0/886725610 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561d62291100 msgr2=0x561d62307070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:43:40.510 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.504+0000 7f8d47fff640 1 --2- 172.21.15.99:0/886725610 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561d62291100 0x561d62307070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:40.510 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.504+0000 7f8d47fff640 1 -- 172.21.15.99:0/886725610 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561d62296c60 msgr2=0x561d6230def0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:43:40.510 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.504+0000 7f8d47fff640 1 --2- 172.21.15.99:0/886725610 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561d62296c60 0x561d6230def0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:40.510 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.504+0000 7f8d47fff640 1 -- 172.21.15.99:0/886725610 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x561d6230e530 con 0x561d62291af0 2024-03-19T01:43:40.510 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.504+0000 7f8d47fff640 1 --2- 172.21.15.99:0/886725610 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561d62291af0 0x561d623075d0 secure :-1 s=READY pgs=408 cs=0 l=1 rev1=1 crypto rx=0x7f8d380cc840 tx=0x7f8d380cc6c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7d94214e99ded6e5 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:43:40.510 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.504+0000 7f8d367fc640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:43:40.510 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.504+0000 7f8d367fc640 1 -- 172.21.15.99:0/886725610 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8d380b4060 con 0x561d62291af0 2024-03-19T01:43:40.510 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.504+0000 7f8d367fc640 1 -- 172.21.15.99:0/886725610 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f8d38099a40 con 0x561d62291af0 2024-03-19T01:43:40.510 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.504+0000 7f8d5807e740 1 -- 172.21.15.99:0/886725610 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f8d40005040 con 0x561d62291af0 2024-03-19T01:43:40.511 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.504+0000 7f8d367fc640 1 -- 172.21.15.99:0/886725610 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=23) ==== 24+0+0 (secure 0 0 0) 0x7f8d38026040 con 0x561d62291af0 2024-03-19T01:43:40.511 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.504+0000 7f8d5807e740 10 client.15546 fetch_fsmap learned FSMap version 23 2024-03-19T01:43:40.511 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.504+0000 7f8d5807e740 1 -- 172.21.15.99:0/886725610 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=23,monmap=4+}) -- 0x561d6230f890 con 0x561d62291af0 2024-03-19T01:43:40.511 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.505+0000 7f8d367fc640 1 -- 172.21.15.99:0/886725610 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f8d40005040 con 0x561d62291af0 2024-03-19T01:43:40.511 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.505+0000 7f8d367fc640 1 -- 172.21.15.99:0/886725610 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 23) ==== 8346+0+0 (secure 0 0 0) 0x7f8d380ca070 con 0x561d62291af0 2024-03-19T01:43:40.511 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.505+0000 7f8d5807e740 10 client.15546 fetch_fsmap finished waiting for FSMap version 23 2024-03-19T01:43:40.511 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.505+0000 7f8d5807e740 10 client.15546 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:43:40.511 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.505+0000 7f8d5807e740 20 client.15546 populate_metadata read hostname 'smithi099' 2024-03-19T01:43:40.511 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.505+0000 7f8d5807e740 1 --2- 172.21.15.99:0/886725610 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x561d62310250 0x561d62312670 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:40.511 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.505+0000 7f8d5807e740 4 client.15546 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:43:40.511 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.505+0000 7f8d5807e740 1 -- 172.21.15.99:0/886725610 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x561d621fd470 con 0x561d62310250 2024-03-19T01:43:40.511 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.505+0000 7f8d4effd640 1 --2- 172.21.15.99:0/886725610 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x561d62310250 0x561d62312670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:40.511 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.505+0000 7f8d4effd640 1 --2- 172.21.15.99:0/886725610 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x561d62310250 0x561d62312670 crc :-1 s=READY pgs=52 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=8e60a0761a0d21b2 server_cookie=9a689da72aecb2d9 in_seq=0 out_seq=0 2024-03-19T01:43:40.512 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.505+0000 7f8d367fc640 10 client.15546 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:43:40.554 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.554+0000 7f8d367fc640 1 -- 172.21.15.99:0/886725610 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x561d621fd470 con 0x561d62310250 2024-03-19T01:43:40.555 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.554+0000 7f8d367fc640 10 client.15546 handle_command_reply: tid=0 2024-03-19T01:43:40.555 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.554+0000 7f8d5807e740 1 client.15546 shutdown 2024-03-19T01:43:40.555 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.554+0000 7f8d5807e740 1 -- 172.21.15.99:0/886725610 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561d62291af0 msgr2=0x561d623075d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:43:40.555 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.554+0000 7f8d5807e740 1 --2- 172.21.15.99:0/886725610 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561d62291af0 0x561d623075d0 secure :-1 s=READY pgs=408 cs=0 l=1 rev1=1 crypto rx=0x7f8d380cc840 tx=0x7f8d380cc6c0 comp rx=0 tx=0).stop 2024-03-19T01:43:40.555 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.554+0000 7f8d5807e740 1 -- 172.21.15.99:0/886725610 shutdown_connections 2024-03-19T01:43:40.555 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.554+0000 7f8d5807e740 1 --2- 172.21.15.99:0/886725610 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x561d62310250 0x561d62312670 crc :-1 s=READY pgs=52 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:40.555 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.554+0000 7f8d5807e740 1 --2- 172.21.15.99:0/886725610 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561d62296c60 0x561d6230def0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:40.555 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.554+0000 7f8d5807e740 1 --2- 172.21.15.99:0/886725610 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561d62291af0 0x561d623075d0 unknown :-1 s=CLOSED pgs=408 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:40.555 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.554+0000 7f8d367fc640 0 client.15546 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:43:40.555 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.554+0000 7f8d5807e740 1 --2- 172.21.15.99:0/886725610 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561d62291100 0x561d62307070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:40.555 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.554+0000 7f8d5807e740 1 -- 172.21.15.99:0/886725610 >> 172.21.15.99:0/886725610 conn(0x561d6228c670 msgr2=0x561d62290160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:43:40.555 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.554+0000 7f8d5807e740 1 -- 172.21.15.99:0/886725610 shutdown_connections 2024-03-19T01:43:40.556 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.554+0000 7f8d5807e740 1 -- 172.21.15.99:0/886725610 wait complete. 2024-03-19T01:43:40.556 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.554+0000 7f8d5807e740 20 client.15546 trim_cache size 0 max 16384 2024-03-19T01:43:40.556 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:43:40.556 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:43:40.556 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:43:40.556 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:43:40.557 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.555+0000 7f8d5548c640 1 -- 172.21.15.99:0/1839666741 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f8d3c0a2920 msgr2=0x7f8d3c0a4de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:43:40.557 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.555+0000 7f8d5548c640 1 --2- 172.21.15.99:0/1839666741 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f8d3c0a2920 0x7f8d3c0a4de0 secure :-1 s=READY pgs=259 cs=0 l=1 rev1=1 crypto rx=0x7f8d480099e0 tx=0x7f8d48009500 comp rx=0 tx=0).stop 2024-03-19T01:43:40.557 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.555+0000 7f8d5548c640 1 -- 172.21.15.99:0/1839666741 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8d50101930 msgr2=0x7f8d50196f40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:43:40.557 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.555+0000 7f8d5548c640 1 --2- 172.21.15.99:0/1839666741 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8d50101930 0x7f8d50196f40 secure :-1 s=READY pgs=406 cs=0 l=1 rev1=1 crypto rx=0x7f8d3800e9f0 tx=0x7f8d3800eeb0 comp rx=0 tx=0).stop 2024-03-19T01:43:40.557 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.555+0000 7f8d5548c640 1 -- 172.21.15.99:0/1839666741 shutdown_connections 2024-03-19T01:43:40.557 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.555+0000 7f8d5548c640 1 --2- 172.21.15.99:0/1839666741 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f8d3c0a2920 0x7f8d3c0a4de0 unknown :-1 s=CLOSED pgs=259 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:40.557 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.555+0000 7f8d5548c640 1 --2- 172.21.15.99:0/1839666741 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8d501022f0 0x7f8d5019dfb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:40.558 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.555+0000 7f8d5548c640 1 --2- 172.21.15.99:0/1839666741 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8d50101930 0x7f8d50196f40 unknown :-1 s=CLOSED pgs=406 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:40.558 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.555+0000 7f8d5548c640 1 --2- 172.21.15.99:0/1839666741 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8d50100740 0x7f8d50196a00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:40.558 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.555+0000 7f8d5548c640 1 -- 172.21.15.99:0/1839666741 >> 172.21.15.99:0/1839666741 conn(0x7f8d500fbcb0 msgr2=0x7f8d50104c30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:43:40.558 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.555+0000 7f8d5548c640 1 -- 172.21.15.99:0/1839666741 shutdown_connections 2024-03-19T01:43:40.558 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:40.555+0000 7f8d5548c640 1 -- 172.21.15.99:0/1839666741 wait complete. 2024-03-19T01:43:40.678 INFO:tasks.workunit.client.0.smithi099.stdout: 1 286046 11.35 MB/sec execute 10 sec latency 39.525 ms 2024-03-19T01:43:41.678 INFO:tasks.workunit.client.0.smithi099.stdout: 1 288011 11.55 MB/sec execute 11 sec latency 35.150 ms 2024-03-19T01:43:42.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:43:41 smithi115 ceph-mon[67332]: pgmap v428: 97 pgs: 97 active+clean; 107 MiB data, 1.1 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 2.8 MiB/s wr, 235 op/s 2024-03-19T01:43:42.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:43:41 smithi099 ceph-mon[62951]: pgmap v428: 97 pgs: 97 active+clean; 107 MiB data, 1.1 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 2.8 MiB/s wr, 235 op/s 2024-03-19T01:43:42.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:43:41 smithi112 ceph-mon[66819]: pgmap v428: 97 pgs: 97 active+clean; 107 MiB data, 1.1 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 2.8 MiB/s wr, 235 op/s 2024-03-19T01:43:42.678 INFO:tasks.workunit.client.0.smithi099.stdout: 1 290502 11.67 MB/sec execute 12 sec latency 30.765 ms 2024-03-19T01:43:43.678 INFO:tasks.workunit.client.0.smithi099.stdout: 1 292328 11.62 MB/sec execute 13 sec latency 33.517 ms 2024-03-19T01:43:44.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:43:43 smithi115 ceph-mon[67332]: pgmap v429: 97 pgs: 97 active+clean; 110 MiB data, 1.1 GiB used, 1.0 TiB / 1.0 TiB avail; 1.4 MiB/s rd, 3.3 MiB/s wr, 298 op/s 2024-03-19T01:43:44.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:43:43 smithi099 ceph-mon[62951]: pgmap v429: 97 pgs: 97 active+clean; 110 MiB data, 1.1 GiB used, 1.0 TiB / 1.0 TiB avail; 1.4 MiB/s rd, 3.3 MiB/s wr, 298 op/s 2024-03-19T01:43:44.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:43:43 smithi112 ceph-mon[66819]: pgmap v429: 97 pgs: 97 active+clean; 110 MiB data, 1.1 GiB used, 1.0 TiB / 1.0 TiB avail; 1.4 MiB/s rd, 3.3 MiB/s wr, 298 op/s 2024-03-19T01:43:44.678 INFO:tasks.workunit.client.0.smithi099.stdout: 1 294643 11.77 MB/sec execute 14 sec latency 59.725 ms 2024-03-19T01:43:45.678 INFO:tasks.workunit.client.0.smithi099.stdout: 1 296299 11.36 MB/sec execute 15 sec latency 29.041 ms 2024-03-19T01:43:46.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:43:45 smithi115 ceph-mon[67332]: pgmap v430: 97 pgs: 97 active+clean; 110 MiB data, 1.1 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 2.7 MiB/s wr, 231 op/s 2024-03-19T01:43:46.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:43:45 smithi099 ceph-mon[62951]: pgmap v430: 97 pgs: 97 active+clean; 110 MiB data, 1.1 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 2.7 MiB/s wr, 231 op/s 2024-03-19T01:43:46.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:43:45 smithi112 ceph-mon[66819]: pgmap v430: 97 pgs: 97 active+clean; 110 MiB data, 1.1 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 2.7 MiB/s wr, 231 op/s 2024-03-19T01:43:46.678 INFO:tasks.workunit.client.0.smithi099.stdout: 1 298566 11.51 MB/sec execute 16 sec latency 31.284 ms 2024-03-19T01:43:47.678 INFO:tasks.workunit.client.0.smithi099.stdout: 1 300874 11.53 MB/sec execute 17 sec latency 39.309 ms 2024-03-19T01:43:48.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:43:47 smithi115 ceph-mon[67332]: pgmap v431: 97 pgs: 97 active+clean; 113 MiB data, 1.1 GiB used, 1.0 TiB / 1.0 TiB avail; 1.4 MiB/s rd, 3.4 MiB/s wr, 299 op/s 2024-03-19T01:43:48.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:43:47 smithi099 ceph-mon[62951]: pgmap v431: 97 pgs: 97 active+clean; 113 MiB data, 1.1 GiB used, 1.0 TiB / 1.0 TiB avail; 1.4 MiB/s rd, 3.4 MiB/s wr, 299 op/s 2024-03-19T01:43:48.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:43:47 smithi112 ceph-mon[66819]: pgmap v431: 97 pgs: 97 active+clean; 113 MiB data, 1.1 GiB used, 1.0 TiB / 1.0 TiB avail; 1.4 MiB/s rd, 3.4 MiB/s wr, 299 op/s 2024-03-19T01:43:48.678 INFO:tasks.workunit.client.0.smithi099.stdout: 1 302780 11.41 MB/sec execute 18 sec latency 25.405 ms 2024-03-19T01:43:49.678 INFO:tasks.workunit.client.0.smithi099.stdout: 1 305031 11.62 MB/sec execute 19 sec latency 93.382 ms 2024-03-19T01:43:50.163 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:43:49 smithi112 ceph-mon[66819]: pgmap v432: 97 pgs: 97 active+clean; 113 MiB data, 1.1 GiB used, 1.0 TiB / 1.0 TiB avail; 1007 KiB/s rd, 2.7 MiB/s wr, 233 op/s 2024-03-19T01:43:50.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:43:49 smithi115 ceph-mon[67332]: pgmap v432: 97 pgs: 97 active+clean; 113 MiB data, 1.1 GiB used, 1.0 TiB / 1.0 TiB avail; 1007 KiB/s rd, 2.7 MiB/s wr, 233 op/s 2024-03-19T01:43:50.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:43:49 smithi099 ceph-mon[62951]: pgmap v432: 97 pgs: 97 active+clean; 113 MiB data, 1.1 GiB used, 1.0 TiB / 1.0 TiB avail; 1007 KiB/s rd, 2.7 MiB/s wr, 233 op/s 2024-03-19T01:43:50.678 INFO:tasks.workunit.client.0.smithi099.stdout: 1 306724 11.38 MB/sec execute 20 sec latency 42.724 ms 2024-03-19T01:43:51.678 INFO:tasks.workunit.client.0.smithi099.stdout: 1 309114 11.50 MB/sec execute 21 sec latency 10.083 ms 2024-03-19T01:43:52.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:43:51 smithi115 ceph-mon[67332]: pgmap v433: 97 pgs: 97 active+clean; 115 MiB data, 1.2 GiB used, 1.0 TiB / 1.0 TiB avail; 1.2 MiB/s rd, 2.9 MiB/s wr, 276 op/s 2024-03-19T01:43:52.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:43:51 smithi099 ceph-mon[62951]: pgmap v433: 97 pgs: 97 active+clean; 115 MiB data, 1.2 GiB used, 1.0 TiB / 1.0 TiB avail; 1.2 MiB/s rd, 2.9 MiB/s wr, 276 op/s 2024-03-19T01:43:52.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:43:51 smithi112 ceph-mon[66819]: pgmap v433: 97 pgs: 97 active+clean; 115 MiB data, 1.2 GiB used, 1.0 TiB / 1.0 TiB avail; 1.2 MiB/s rd, 2.9 MiB/s wr, 276 op/s 2024-03-19T01:43:52.678 INFO:tasks.workunit.client.0.smithi099.stdout: 1 311363 11.48 MB/sec execute 22 sec latency 21.785 ms 2024-03-19T01:43:53.650 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.647+0000 7f5944cf8640 1 -- 172.21.15.99:0/1249533719 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5940107580 msgr2=0x7f5940109970 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:43:53.650 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.647+0000 7f5944cf8640 1 --2- 172.21.15.99:0/1249533719 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5940107580 0x7f5940109970 secure :-1 s=READY pgs=409 cs=0 l=1 rev1=1 crypto rx=0x7f5938009ca0 tx=0x7f593802c0f0 comp rx=0 tx=0).stop 2024-03-19T01:43:53.650 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.647+0000 7f5944cf8640 1 -- 172.21.15.99:0/1249533719 shutdown_connections 2024-03-19T01:43:53.650 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.647+0000 7f5944cf8640 1 --2- 172.21.15.99:0/1249533719 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5940107580 0x7f5940109970 unknown :-1 s=CLOSED pgs=409 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:53.650 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.647+0000 7f5944cf8640 1 --2- 172.21.15.99:0/1249533719 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5940104c50 0x7f5940107040 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:53.650 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.647+0000 7f5944cf8640 1 --2- 172.21.15.99:0/1249533719 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5940102320 0x7f5940104710 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:53.650 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.647+0000 7f5944cf8640 1 -- 172.21.15.99:0/1249533719 >> 172.21.15.99:0/1249533719 conn(0x7f59400fbcb0 msgr2=0x7f59400fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:43:53.650 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.648+0000 7f5944cf8640 1 -- 172.21.15.99:0/1249533719 shutdown_connections 2024-03-19T01:43:53.651 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.648+0000 7f5944cf8640 1 -- 172.21.15.99:0/1249533719 wait complete. 2024-03-19T01:43:53.651 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.648+0000 7f5944cf8640 1 Processor -- start 2024-03-19T01:43:53.651 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.648+0000 7f5944cf8640 1 -- start start 2024-03-19T01:43:53.651 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.648+0000 7f5944cf8640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5940102320 0x7f5940197ad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:53.651 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.648+0000 7f5944cf8640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5940104c50 0x7f5940198010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:53.651 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.648+0000 7f5944cf8640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5940107580 0x7f594019f080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:53.651 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.648+0000 7f5944cf8640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f594010d4e0 con 0x7f5940102320 2024-03-19T01:43:53.651 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.648+0000 7f5944cf8640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f594010d360 con 0x7f5940107580 2024-03-19T01:43:53.651 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.648+0000 7f5944cf8640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f594010d660 con 0x7f5940104c50 2024-03-19T01:43:53.651 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.649+0000 7f593e59c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5940102320 0x7f5940197ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:53.651 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.649+0000 7f5937fff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5940104c50 0x7f5940198010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:53.651 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.649+0000 7f593e59c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5940102320 0x7f5940197ad0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:43408/0 (socket says 172.21.15.99:43408) 2024-03-19T01:43:53.651 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.649+0000 7f593e59c640 1 -- 172.21.15.99:0/2253388653 learned_addr learned my addr 172.21.15.99:0/2253388653 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:43:53.651 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.649+0000 7f593ed9d640 1 --2- 172.21.15.99:0/2253388653 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5940107580 0x7f594019f080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:53.651 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.649+0000 7f593e59c640 1 -- 172.21.15.99:0/2253388653 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5940104c50 msgr2=0x7f5940198010 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:43:53.651 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.649+0000 7f593e59c640 1 --2- 172.21.15.99:0/2253388653 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5940104c50 0x7f5940198010 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:53.652 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.649+0000 7f593e59c640 1 -- 172.21.15.99:0/2253388653 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5940107580 msgr2=0x7f594019f080 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:43:53.652 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.649+0000 7f593e59c640 1 --2- 172.21.15.99:0/2253388653 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5940107580 0x7f594019f080 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:53.652 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.649+0000 7f593e59c640 1 -- 172.21.15.99:0/2253388653 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f594019f6f0 con 0x7f5940102320 2024-03-19T01:43:53.652 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.649+0000 7f593ed9d640 1 --2- 172.21.15.99:0/2253388653 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5940107580 0x7f594019f080 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:43:53.652 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.649+0000 7f593e59c640 1 --2- 172.21.15.99:0/2253388653 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5940102320 0x7f5940197ad0 secure :-1 s=READY pgs=410 cs=0 l=1 rev1=1 crypto rx=0x7f593000e9f0 tx=0x7f593000eeb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:43:53.652 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.649+0000 7f59377fe640 1 -- 172.21.15.99:0/2253388653 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f593000cd80 con 0x7f5940102320 2024-03-19T01:43:53.652 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.649+0000 7f59377fe640 1 -- 172.21.15.99:0/2253388653 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f5930010910 con 0x7f5940102320 2024-03-19T01:43:53.652 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.649+0000 7f5944cf8640 1 -- 172.21.15.99:0/2253388653 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f594019f9e0 con 0x7f5940102320 2024-03-19T01:43:53.652 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.649+0000 7f59377fe640 1 -- 172.21.15.99:0/2253388653 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5930018990 con 0x7f5940102320 2024-03-19T01:43:53.652 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.649+0000 7f5944cf8640 1 -- 172.21.15.99:0/2253388653 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f594019ff30 con 0x7f5940102320 2024-03-19T01:43:53.659 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.652+0000 7f59377fe640 1 -- 172.21.15.99:0/2253388653 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f5930010ac0 con 0x7f5940102320 2024-03-19T01:43:53.659 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.652+0000 7f59479c3740 1 Processor -- start 2024-03-19T01:43:53.659 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.652+0000 7f59479c3740 1 -- start start 2024-03-19T01:43:53.659 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.652+0000 7f59479c3740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56217e3f7510 0x56217e3f7930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:53.659 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.652+0000 7f59479c3740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56217e3f7f00 0x56217e3fca00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:53.659 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.652+0000 7f59479c3740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56217e3fd070 0x56217e3ff500 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:53.659 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.652+0000 7f59479c3740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x56217e357be0 con 0x56217e3f7f00 2024-03-19T01:43:53.659 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.652+0000 7f59479c3740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x56217e300960 con 0x56217e3f7510 2024-03-19T01:43:53.660 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.652+0000 7f59479c3740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x56217e353c80 con 0x56217e3fd070 2024-03-19T01:43:53.660 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.652+0000 7f59377fe640 1 --2- 172.21.15.99:0/2253388653 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f592c0a28a0 0x7f592c0a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:53.660 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.653+0000 7f593e59c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56217e3f7f00 0x56217e3fca00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:53.660 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.653+0000 7f5937fff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56217e3fd070 0x56217e3ff500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:53.660 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.653+0000 7f593ed9d640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56217e3f7510 0x56217e3f7930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:53.660 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.653+0000 7f5937fff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56217e3fd070 0x56217e3ff500 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:53110/0 (socket says 172.21.15.99:53110) 2024-03-19T01:43:53.660 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.653+0000 7f5937fff640 1 -- 172.21.15.99:0/2659998316 learned_addr learned my addr 172.21.15.99:0/2659998316 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:43:53.660 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.653+0000 7f593ed9d640 1 --2- 172.21.15.99:0/2253388653 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f592c0a28a0 0x7f592c0a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:53.660 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.653+0000 7f59377fe640 1 -- 172.21.15.99:0/2253388653 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f59300cedf0 con 0x7f5940102320 2024-03-19T01:43:53.660 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.653+0000 7f5937fff640 1 -- 172.21.15.99:0/2659998316 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56217e3f7510 msgr2=0x56217e3f7930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:43:53.660 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.653+0000 7f5937fff640 1 --2- 172.21.15.99:0/2659998316 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56217e3f7510 0x56217e3f7930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:53.661 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.653+0000 7f5937fff640 1 -- 172.21.15.99:0/2659998316 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56217e3f7f00 msgr2=0x56217e3fca00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:43:53.661 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.653+0000 7f5937fff640 1 --2- 172.21.15.99:0/2659998316 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56217e3f7f00 0x56217e3fca00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:53.661 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.653+0000 7f5937fff640 1 -- 172.21.15.99:0/2659998316 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x56217e2f7d20 con 0x56217e3fd070 2024-03-19T01:43:53.661 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.653+0000 7f593ed9d640 1 --2- 172.21.15.99:0/2253388653 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f592c0a28a0 0x7f592c0a4d60 secure :-1 s=READY pgs=260 cs=0 l=1 rev1=1 crypto rx=0x7f5938005e30 tx=0x7f5938032590 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:43:53.661 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.653+0000 7f5937fff640 1 --2- 172.21.15.99:0/2659998316 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56217e3fd070 0x56217e3ff500 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7f5928009c60 tx=0x7f5928011810 comp rx=0 tx=0).ready entity=mon.1 client_cookie=887d6bd2226ff840 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:43:53.661 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.653+0000 7f5934ff9640 1 -- 172.21.15.99:0/2659998316 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5928004230 con 0x56217e3fd070 2024-03-19T01:43:53.661 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.653+0000 7f5934ff9640 1 -- 172.21.15.99:0/2659998316 <== mon.1 v2:172.21.15.115:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f59280043d0 con 0x56217e3fd070 2024-03-19T01:43:53.661 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.653+0000 7f5934ff9640 1 -- 172.21.15.99:0/2659998316 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5928019840 con 0x56217e3fd070 2024-03-19T01:43:53.661 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.654+0000 7f59479c3740 1 -- 172.21.15.99:0/2659998316 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56217e3fd070 msgr2=0x56217e3ff500 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:43:53.661 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.654+0000 7f59479c3740 1 --2- 172.21.15.99:0/2659998316 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56217e3fd070 0x56217e3ff500 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7f5928009c60 tx=0x7f5928011810 comp rx=0 tx=0).stop 2024-03-19T01:43:53.661 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.654+0000 7f59479c3740 1 -- 172.21.15.99:0/2659998316 shutdown_connections 2024-03-19T01:43:53.661 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.654+0000 7f59479c3740 1 --2- 172.21.15.99:0/2659998316 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56217e3fd070 0x56217e3ff500 unknown :-1 s=CLOSED pgs=82 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:53.661 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.654+0000 7f59479c3740 1 --2- 172.21.15.99:0/2659998316 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56217e3f7f00 0x56217e3fca00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:53.661 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.654+0000 7f59479c3740 1 --2- 172.21.15.99:0/2659998316 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56217e3f7510 0x56217e3f7930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:53.661 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.654+0000 7f59479c3740 1 -- 172.21.15.99:0/2659998316 >> 172.21.15.99:0/2659998316 conn(0x56217e3ee670 msgr2=0x56217e3eea70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:43:53.662 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.654+0000 7f59479c3740 1 -- 172.21.15.99:0/2659998316 shutdown_connections 2024-03-19T01:43:53.662 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.654+0000 7f59479c3740 1 -- 172.21.15.99:0/2659998316 wait complete. 2024-03-19T01:43:53.662 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.655+0000 7f59479c3740 1 -- start start 2024-03-19T01:43:53.662 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.655+0000 7f59479c3740 1 Processor -- start 2024-03-19T01:43:53.662 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.655+0000 7f59479c3740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56217e3f7510 0x56217e401540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:53.662 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.655+0000 7f59479c3740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56217e3f7f00 0x56217e401a80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:53.662 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.655+0000 7f59479c3740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56217e3fd070 0x56217e408ac0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:53.662 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.655+0000 7f59479c3740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x56217e400020 con 0x56217e3f7f00 2024-03-19T01:43:53.662 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.655+0000 7f59479c3740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x56217e313db0 con 0x56217e3f7510 2024-03-19T01:43:53.662 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.655+0000 7f59479c3740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x56217e4001a0 con 0x56217e3fd070 2024-03-19T01:43:53.662 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.655+0000 7f59479c3740 10 client.0 _ll_register_callbacks cb 0x56217e318e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:43:53.662 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.655+0000 7f5937fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56217e3f7f00 0x56217e401a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:53.662 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.655+0000 7f5937fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56217e3f7f00 0x56217e401a80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:43426/0 (socket says 172.21.15.99:43426) 2024-03-19T01:43:53.662 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.655+0000 7f5937fff640 1 -- 172.21.15.99:0/569826155 learned_addr learned my addr 172.21.15.99:0/569826155 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:43:53.663 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.655+0000 7f593ed9d640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56217e3fd070 0x56217e408ac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:53.663 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.655+0000 7f593e59c640 1 --2- 172.21.15.99:0/569826155 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56217e3f7510 0x56217e401540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:53.663 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.656+0000 7f593e59c640 1 -- 172.21.15.99:0/569826155 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56217e3fd070 msgr2=0x56217e408ac0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:43:53.663 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.656+0000 7f593e59c640 1 --2- 172.21.15.99:0/569826155 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56217e3fd070 0x56217e408ac0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:53.663 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.656+0000 7f593e59c640 1 -- 172.21.15.99:0/569826155 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56217e3f7f00 msgr2=0x56217e401a80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:43:53.663 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.656+0000 7f593e59c640 1 --2- 172.21.15.99:0/569826155 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56217e3f7f00 0x56217e401a80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:53.663 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.656+0000 7f593e59c640 1 -- 172.21.15.99:0/569826155 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x56217e312c60 con 0x56217e3f7510 2024-03-19T01:43:53.663 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.656+0000 7f5937fff640 1 --2- 172.21.15.99:0/569826155 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56217e3f7f00 0x56217e401a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:43:53.663 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.656+0000 7f593e59c640 1 --2- 172.21.15.99:0/569826155 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56217e3f7510 0x56217e401540 secure :-1 s=READY pgs=99 cs=0 l=1 rev1=1 crypto rx=0x7f59300cc900 tx=0x7f593006b3e0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=9ea02ff552daebb8 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:43:53.663 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.656+0000 7f5925ffb640 10 client.0 ms_handle_connect on v2:172.21.15.112:3300/0 2024-03-19T01:43:53.663 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.656+0000 7f5925ffb640 1 -- 172.21.15.99:0/569826155 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f593009f8a0 con 0x56217e3f7510 2024-03-19T01:43:53.663 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.656+0000 7f5925ffb640 1 -- 172.21.15.99:0/569826155 <== mon.2 v2:172.21.15.112:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f593006d110 con 0x56217e3f7510 2024-03-19T01:43:53.663 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.656+0000 7f59479c3740 1 -- 172.21.15.99:0/569826155 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f59280043d0 con 0x56217e3f7510 2024-03-19T01:43:53.663 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.656+0000 7f5925ffb640 1 -- 172.21.15.99:0/569826155 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_get_version_reply(handle=1 version=23) ==== 24+0+0 (secure 0 0 0) 0x7f59300a3ef0 con 0x56217e3f7510 2024-03-19T01:43:53.664 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.656+0000 7f59479c3740 10 client.25331 fetch_fsmap learned FSMap version 23 2024-03-19T01:43:53.664 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.656+0000 7f59479c3740 1 -- 172.21.15.99:0/569826155 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,fsmap=23,monmap=4+}) -- 0x56217e06c210 con 0x56217e3f7510 2024-03-19T01:43:53.664 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.656+0000 7f5925ffb640 1 -- 172.21.15.99:0/569826155 <== mon.2 v2:172.21.15.112:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f59280043d0 con 0x56217e3f7510 2024-03-19T01:43:53.664 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.656+0000 7f5925ffb640 1 -- 172.21.15.99:0/569826155 <== mon.2 v2:172.21.15.112:3300/0 5 ==== fsmap(e 23) ==== 8346+0+0 (secure 0 0 0) 0x7f59300afb30 con 0x56217e3f7510 2024-03-19T01:43:53.664 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.656+0000 7f59479c3740 10 client.25331 fetch_fsmap finished waiting for FSMap version 23 2024-03-19T01:43:53.664 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.657+0000 7f59479c3740 10 client.25331 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:43:53.664 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.657+0000 7f59479c3740 20 client.25331 populate_metadata read hostname 'smithi099' 2024-03-19T01:43:53.664 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.657+0000 7f59479c3740 1 --2- 172.21.15.99:0/569826155 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56217e40d360 0x56217e40f7f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:53.664 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.657+0000 7f59479c3740 4 client.25331 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:43:53.664 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.657+0000 7f59479c3740 1 -- 172.21.15.99:0/569826155 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x56217e365120 con 0x56217e40d360 2024-03-19T01:43:53.664 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.657+0000 7f5937fff640 1 --2- 172.21.15.99:0/569826155 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56217e40d360 0x56217e40f7f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:53.664 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.657+0000 7f5937fff640 1 --2- 172.21.15.99:0/569826155 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56217e40d360 0x56217e40f7f0 crc :-1 s=READY pgs=53 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=a0df7ecede810565 server_cookie=b68592429c771722 in_seq=0 out_seq=0 2024-03-19T01:43:53.664 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.657+0000 7f5925ffb640 10 client.25331 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:43:53.664 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.658+0000 7f5925ffb640 1 -- 172.21.15.99:0/569826155 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x56217e365120 con 0x56217e40d360 2024-03-19T01:43:53.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.658+0000 7f5925ffb640 10 client.25331 handle_command_reply: tid=0 2024-03-19T01:43:53.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.658+0000 7f59479c3740 1 client.25331 shutdown 2024-03-19T01:43:53.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.658+0000 7f59479c3740 1 -- 172.21.15.99:0/569826155 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56217e3f7510 msgr2=0x56217e401540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:43:53.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.658+0000 7f59479c3740 1 --2- 172.21.15.99:0/569826155 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56217e3f7510 0x56217e401540 secure :-1 s=READY pgs=99 cs=0 l=1 rev1=1 crypto rx=0x7f59300cc900 tx=0x7f593006b3e0 comp rx=0 tx=0).stop 2024-03-19T01:43:53.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.658+0000 7f59479c3740 1 -- 172.21.15.99:0/569826155 shutdown_connections 2024-03-19T01:43:53.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.658+0000 7f59479c3740 1 --2- 172.21.15.99:0/569826155 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56217e40d360 0x56217e40f7f0 crc :-1 s=READY pgs=53 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:53.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.658+0000 7f59479c3740 1 --2- 172.21.15.99:0/569826155 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56217e3fd070 0x56217e408ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:53.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.658+0000 7f59479c3740 1 --2- 172.21.15.99:0/569826155 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56217e3f7f00 0x56217e401a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:53.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.658+0000 7f5925ffb640 0 client.25331 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:43:53.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.658+0000 7f59479c3740 1 --2- 172.21.15.99:0/569826155 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56217e3f7510 0x56217e401540 unknown :-1 s=CLOSED pgs=99 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:53.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.658+0000 7f59479c3740 1 -- 172.21.15.99:0/569826155 >> 172.21.15.99:0/569826155 conn(0x56217e3ee670 msgr2=0x56217e3ef8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:43:53.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.658+0000 7f59479c3740 1 -- 172.21.15.99:0/569826155 shutdown_connections 2024-03-19T01:43:53.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.658+0000 7f59479c3740 1 -- 172.21.15.99:0/569826155 wait complete. 2024-03-19T01:43:53.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.658+0000 7f59479c3740 20 client.25331 trim_cache size 0 max 16384 2024-03-19T01:43:53.674 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.674+0000 7f59479c3740 1 Processor -- start 2024-03-19T01:43:53.675 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.674+0000 7f59479c3740 1 -- start start 2024-03-19T01:43:53.675 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.674+0000 7f59479c3740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56217e3f7510 0x56217e3ed890 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:53.675 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.674+0000 7f59479c3740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56217e3f7f00 0x56217e400b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:53.675 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.674+0000 7f59479c3740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56217e3fd070 0x56217e4010a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:53.675 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.674+0000 7f59479c3740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f59401a9440 con 0x56217e3fd070 2024-03-19T01:43:53.675 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.674+0000 7f59479c3740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f59401a6d10 con 0x56217e3f7f00 2024-03-19T01:43:53.675 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.674+0000 7f59479c3740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f59401a4b60 con 0x56217e3f7510 2024-03-19T01:43:53.675 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.674+0000 7f5937fff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56217e3f7f00 0x56217e400b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:53.675 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.674+0000 7f593ed9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56217e3fd070 0x56217e4010a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:53.675 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.674+0000 7f5937fff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56217e3f7f00 0x56217e400b60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:50882/0 (socket says 172.21.15.99:50882) 2024-03-19T01:43:53.675 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.674+0000 7f593e59c640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56217e3f7510 0x56217e3ed890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:53.675 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.674+0000 7f5937fff640 1 -- 172.21.15.99:0/1333722113 learned_addr learned my addr 172.21.15.99:0/1333722113 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:43:53.676 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.674+0000 7f5937fff640 1 -- 172.21.15.99:0/1333722113 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56217e3f7510 msgr2=0x56217e3ed890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:43:53.676 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.674+0000 7f5937fff640 1 --2- 172.21.15.99:0/1333722113 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56217e3f7510 0x56217e3ed890 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:53.676 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.674+0000 7f5937fff640 1 -- 172.21.15.99:0/1333722113 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56217e3fd070 msgr2=0x56217e4010a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:43:53.676 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.674+0000 7f5937fff640 1 --2- 172.21.15.99:0/1333722113 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56217e3fd070 0x56217e4010a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:53.676 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.674+0000 7f5937fff640 1 -- 172.21.15.99:0/1333722113 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x56217e138620 con 0x56217e3f7f00 2024-03-19T01:43:53.676 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.674+0000 7f5937fff640 1 --2- 172.21.15.99:0/1333722113 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56217e3f7f00 0x56217e400b60 secure :-1 s=READY pgs=100 cs=0 l=1 rev1=1 crypto rx=0x7f5928005120 tx=0x7f5928009b60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=8fb7c0f492dd4323 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:43:53.676 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.674+0000 7f5927fff640 1 -- 172.21.15.99:0/1333722113 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5928031070 con 0x56217e3f7f00 2024-03-19T01:43:53.676 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.675+0000 7f5927fff640 1 -- 172.21.15.99:0/1333722113 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f59280043d0 con 0x56217e3f7f00 2024-03-19T01:43:53.676 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.675+0000 7f5927fff640 1 -- 172.21.15.99:0/1333722113 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f592801f040 con 0x56217e3f7f00 2024-03-19T01:43:53.679 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.675+0000 7f59479c3740 1 -- 172.21.15.99:0/1333722113 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56217e3f7f00 msgr2=0x56217e400b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:43:53.680 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.675+0000 7f59479c3740 1 --2- 172.21.15.99:0/1333722113 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56217e3f7f00 0x56217e400b60 secure :-1 s=READY pgs=100 cs=0 l=1 rev1=1 crypto rx=0x7f5928005120 tx=0x7f5928009b60 comp rx=0 tx=0).stop 2024-03-19T01:43:53.680 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.675+0000 7f59479c3740 1 -- 172.21.15.99:0/1333722113 shutdown_connections 2024-03-19T01:43:53.680 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.675+0000 7f59479c3740 1 --2- 172.21.15.99:0/1333722113 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56217e3f7f00 0x56217e400b60 unknown :-1 s=CLOSED pgs=100 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:53.680 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.675+0000 7f59479c3740 1 --2- 172.21.15.99:0/1333722113 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56217e3fd070 0x56217e4010a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:53.680 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.675+0000 7f59479c3740 1 --2- 172.21.15.99:0/1333722113 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56217e3f7510 0x56217e3ed890 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:53.680 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.675+0000 7f59479c3740 1 -- 172.21.15.99:0/1333722113 >> 172.21.15.99:0/1333722113 conn(0x56217e3ee670 msgr2=0x56217e3f2160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:43:53.680 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.676+0000 7f59479c3740 1 -- 172.21.15.99:0/1333722113 shutdown_connections 2024-03-19T01:43:53.680 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.676+0000 7f59479c3740 1 -- 172.21.15.99:0/1333722113 wait complete. 2024-03-19T01:43:53.680 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.676+0000 7f59479c3740 1 -- start start 2024-03-19T01:43:53.680 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.676+0000 7f59479c3740 1 Processor -- start 2024-03-19T01:43:53.680 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.676+0000 7f59479c3740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56217e3f7510 0x56217e4060d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:53.680 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.676+0000 7f59479c3740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56217e3f7f00 0x56217e406610 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:53.680 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.676+0000 7f59479c3740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56217e3fd070 0x56217e4075d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:53.680 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.676+0000 7f59479c3740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f59401a51d0 con 0x56217e3fd070 2024-03-19T01:43:53.680 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.676+0000 7f59479c3740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f59401a6640 con 0x56217e3f7510 2024-03-19T01:43:53.681 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.676+0000 7f59479c3740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f59401a58a0 con 0x56217e3f7f00 2024-03-19T01:43:53.681 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.676+0000 7f59479c3740 10 client.0 _ll_register_callbacks cb 0x7f593000cf20 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:43:53.681 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.676+0000 7f593ed9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56217e3fd070 0x56217e4075d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:53.681 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.676+0000 7f593ed9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56217e3fd070 0x56217e4075d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:43444/0 (socket says 172.21.15.99:43444) 2024-03-19T01:43:53.681 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.676+0000 7f593e59c640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56217e3f7510 0x56217e4060d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:53.681 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.676+0000 7f5937fff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56217e3f7f00 0x56217e406610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:53.681 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.676+0000 7f593ed9d640 1 -- 172.21.15.99:0/860700431 learned_addr learned my addr 172.21.15.99:0/860700431 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:43:53.681 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.677+0000 7f593ed9d640 1 -- 172.21.15.99:0/860700431 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56217e3f7f00 msgr2=0x56217e406610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:43:53.681 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.677+0000 7f593ed9d640 1 --2- 172.21.15.99:0/860700431 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56217e3f7f00 0x56217e406610 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:53.681 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.677+0000 7f593ed9d640 1 -- 172.21.15.99:0/860700431 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56217e3f7510 msgr2=0x56217e4060d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:43:53.681 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.677+0000 7f593ed9d640 1 --2- 172.21.15.99:0/860700431 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56217e3f7510 0x56217e4060d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:53.681 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.677+0000 7f593ed9d640 1 -- 172.21.15.99:0/860700431 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x56217e407bc0 con 0x56217e3fd070 2024-03-19T01:43:53.681 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.677+0000 7f593e59c640 1 --2- 172.21.15.99:0/860700431 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56217e3f7510 0x56217e4060d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:43:53.681 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.677+0000 7f593ed9d640 1 --2- 172.21.15.99:0/860700431 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56217e3fd070 0x56217e4075d0 secure :-1 s=READY pgs=411 cs=0 l=1 rev1=1 crypto rx=0x7f5938009ca0 tx=0x7f593802c0f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=bf16c582e3a9f211 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:43:53.682 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.677+0000 7f5925ffb640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:43:53.682 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.677+0000 7f5925ffb640 1 -- 172.21.15.99:0/860700431 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5938039040 con 0x56217e3fd070 2024-03-19T01:43:53.682 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.677+0000 7f5925ffb640 1 -- 172.21.15.99:0/860700431 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f5938034c00 con 0x56217e3fd070 2024-03-19T01:43:53.682 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.677+0000 7f59479c3740 1 -- 172.21.15.99:0/860700431 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f59280043d0 con 0x56217e3fd070 2024-03-19T01:43:53.682 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.677+0000 7f5925ffb640 1 -- 172.21.15.99:0/860700431 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=23) ==== 24+0+0 (secure 0 0 0) 0x7f5938004b40 con 0x56217e3fd070 2024-03-19T01:43:53.682 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.677+0000 7f59479c3740 10 client.15564 fetch_fsmap learned FSMap version 23 2024-03-19T01:43:53.682 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.677+0000 7f59479c3740 1 -- 172.21.15.99:0/860700431 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=23,monmap=4+}) -- 0x56217e473980 con 0x56217e3fd070 2024-03-19T01:43:53.682 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.677+0000 7f5925ffb640 1 -- 172.21.15.99:0/860700431 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f59280043d0 con 0x56217e3fd070 2024-03-19T01:43:53.682 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.678+0000 7f5925ffb640 1 -- 172.21.15.99:0/860700431 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 23) ==== 8346+0+0 (secure 0 0 0) 0x7f5938005140 con 0x56217e3fd070 2024-03-19T01:43:53.682 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.678+0000 7f59479c3740 10 client.15564 fetch_fsmap finished waiting for FSMap version 23 2024-03-19T01:43:53.682 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.678+0000 7f59479c3740 10 client.15564 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:43:53.682 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.678+0000 7f59479c3740 20 client.15564 populate_metadata read hostname 'smithi099' 2024-03-19T01:43:53.682 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.678+0000 7f59479c3740 1 --2- 172.21.15.99:0/860700431 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56217e474490 0x56217e4768b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:43:53.683 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.678+0000 7f59479c3740 4 client.15564 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:43:53.683 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.678+0000 7f59479c3740 1 -- 172.21.15.99:0/860700431 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x56217e35f470 con 0x56217e474490 2024-03-19T01:43:53.683 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.678+0000 7f593e59c640 1 --2- 172.21.15.99:0/860700431 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56217e474490 0x56217e4768b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:43:53.683 INFO:tasks.workunit.client.0.smithi099.stdout: 1 313145 11.37 MB/sec execute 23 sec latency 41.096 ms 2024-03-19T01:43:53.683 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.678+0000 7f5925ffb640 10 client.15564 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:43:53.683 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.678+0000 7f593e59c640 1 --2- 172.21.15.99:0/860700431 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56217e474490 0x56217e4768b0 crc :-1 s=READY pgs=54 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=7c30e94e6d8863d3 server_cookie=bfcff2e2db69308f in_seq=0 out_seq=0 2024-03-19T01:43:53.725 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.724+0000 7f5925ffb640 1 -- 172.21.15.99:0/860700431 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x56217e35f470 con 0x56217e474490 2024-03-19T01:43:53.725 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.724+0000 7f5925ffb640 10 client.15564 handle_command_reply: tid=0 2024-03-19T01:43:53.725 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.724+0000 7f59479c3740 1 client.15564 shutdown 2024-03-19T01:43:53.725 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.724+0000 7f59479c3740 1 -- 172.21.15.99:0/860700431 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56217e3fd070 msgr2=0x56217e4075d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:43:53.725 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.724+0000 7f59479c3740 1 --2- 172.21.15.99:0/860700431 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56217e3fd070 0x56217e4075d0 secure :-1 s=READY pgs=411 cs=0 l=1 rev1=1 crypto rx=0x7f5938009ca0 tx=0x7f593802c0f0 comp rx=0 tx=0).stop 2024-03-19T01:43:53.725 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.724+0000 7f59479c3740 1 -- 172.21.15.99:0/860700431 shutdown_connections 2024-03-19T01:43:53.725 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.724+0000 7f59479c3740 1 --2- 172.21.15.99:0/860700431 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56217e474490 0x56217e4768b0 crc :-1 s=READY pgs=54 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:53.725 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.724+0000 7f59479c3740 1 --2- 172.21.15.99:0/860700431 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56217e3fd070 0x56217e4075d0 unknown :-1 s=CLOSED pgs=411 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:53.725 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.724+0000 7f5925ffb640 0 client.15564 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:43:53.725 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.724+0000 7f59479c3740 1 --2- 172.21.15.99:0/860700431 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56217e3f7f00 0x56217e406610 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:53.725 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.724+0000 7f59479c3740 1 --2- 172.21.15.99:0/860700431 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56217e3f7510 0x56217e4060d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:53.726 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.724+0000 7f59479c3740 1 -- 172.21.15.99:0/860700431 >> 172.21.15.99:0/860700431 conn(0x56217e3ee670 msgr2=0x56217e3f8950 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:43:53.726 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.724+0000 7f59479c3740 1 -- 172.21.15.99:0/860700431 shutdown_connections 2024-03-19T01:43:53.726 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.724+0000 7f59479c3740 1 -- 172.21.15.99:0/860700431 wait complete. 2024-03-19T01:43:53.726 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.725+0000 7f59479c3740 20 client.15564 trim_cache size 0 max 16384 2024-03-19T01:43:53.726 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:43:53.726 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:43:53.726 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:43:53.726 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:43:53.727 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.725+0000 7f5944cf8640 1 -- 172.21.15.99:0/2253388653 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f592c0a28a0 msgr2=0x7f592c0a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:43:53.727 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.725+0000 7f5944cf8640 1 --2- 172.21.15.99:0/2253388653 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f592c0a28a0 0x7f592c0a4d60 secure :-1 s=READY pgs=260 cs=0 l=1 rev1=1 crypto rx=0x7f5938005e30 tx=0x7f5938032590 comp rx=0 tx=0).stop 2024-03-19T01:43:53.727 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.725+0000 7f5944cf8640 1 -- 172.21.15.99:0/2253388653 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5940102320 msgr2=0x7f5940197ad0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:43:53.727 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.725+0000 7f5944cf8640 1 --2- 172.21.15.99:0/2253388653 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5940102320 0x7f5940197ad0 secure :-1 s=READY pgs=410 cs=0 l=1 rev1=1 crypto rx=0x7f593000e9f0 tx=0x7f593000eeb0 comp rx=0 tx=0).stop 2024-03-19T01:43:53.727 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.725+0000 7f5944cf8640 1 -- 172.21.15.99:0/2253388653 shutdown_connections 2024-03-19T01:43:53.727 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.725+0000 7f5944cf8640 1 --2- 172.21.15.99:0/2253388653 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f592c0a28a0 0x7f592c0a4d60 unknown :-1 s=CLOSED pgs=260 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:53.727 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.725+0000 7f5944cf8640 1 --2- 172.21.15.99:0/2253388653 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5940107580 0x7f594019f080 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:53.727 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.725+0000 7f5944cf8640 1 --2- 172.21.15.99:0/2253388653 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5940104c50 0x7f5940198010 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:53.728 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.725+0000 7f5944cf8640 1 --2- 172.21.15.99:0/2253388653 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5940102320 0x7f5940197ad0 unknown :-1 s=CLOSED pgs=410 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:43:53.728 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.725+0000 7f5944cf8640 1 -- 172.21.15.99:0/2253388653 >> 172.21.15.99:0/2253388653 conn(0x7f59400fbcb0 msgr2=0x7f5940102c60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:43:53.728 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.725+0000 7f5944cf8640 1 -- 172.21.15.99:0/2253388653 shutdown_connections 2024-03-19T01:43:53.728 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:43:53.725+0000 7f5944cf8640 1 -- 172.21.15.99:0/2253388653 wait complete. 2024-03-19T01:43:54.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:43:53 smithi115 ceph-mon[67332]: pgmap v434: 97 pgs: 97 active+clean; 116 MiB data, 1.2 GiB used, 1.0 TiB / 1.0 TiB avail; 1.2 MiB/s rd, 3.3 MiB/s wr, 291 op/s 2024-03-19T01:43:54.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:43:53 smithi099 ceph-mon[62951]: pgmap v434: 97 pgs: 97 active+clean; 116 MiB data, 1.2 GiB used, 1.0 TiB / 1.0 TiB avail; 1.2 MiB/s rd, 3.3 MiB/s wr, 291 op/s 2024-03-19T01:43:54.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:43:53 smithi112 ceph-mon[66819]: pgmap v434: 97 pgs: 97 active+clean; 116 MiB data, 1.2 GiB used, 1.0 TiB / 1.0 TiB avail; 1.2 MiB/s rd, 3.3 MiB/s wr, 291 op/s 2024-03-19T01:43:54.678 INFO:tasks.workunit.client.0.smithi099.stdout: 1 315350 11.42 MB/sec execute 24 sec latency 61.991 ms 2024-03-19T01:43:55.678 INFO:tasks.workunit.client.0.smithi099.stdout: 1 317223 11.40 MB/sec execute 25 sec latency 29.239 ms 2024-03-19T01:43:56.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:43:55 smithi115 ceph-mon[67332]: pgmap v435: 97 pgs: 97 active+clean; 115 MiB data, 1.2 GiB used, 1.0 TiB / 1.0 TiB avail; 983 KiB/s rd, 2.6 MiB/s wr, 210 op/s 2024-03-19T01:43:56.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:43:55 smithi099 ceph-mon[62951]: pgmap v435: 97 pgs: 97 active+clean; 115 MiB data, 1.2 GiB used, 1.0 TiB / 1.0 TiB avail; 983 KiB/s rd, 2.6 MiB/s wr, 210 op/s 2024-03-19T01:43:56.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:43:55 smithi112 ceph-mon[66819]: pgmap v435: 97 pgs: 97 active+clean; 115 MiB data, 1.2 GiB used, 1.0 TiB / 1.0 TiB avail; 983 KiB/s rd, 2.6 MiB/s wr, 210 op/s 2024-03-19T01:43:56.678 INFO:tasks.workunit.client.0.smithi099.stdout: 1 319653 11.50 MB/sec execute 26 sec latency 14.742 ms 2024-03-19T01:43:57.678 INFO:tasks.workunit.client.0.smithi099.stdout: 1 321654 11.33 MB/sec execute 27 sec latency 16.524 ms 2024-03-19T01:43:58.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:43:57 smithi115 ceph-mon[67332]: pgmap v436: 97 pgs: 97 active+clean; 119 MiB data, 1.2 GiB used, 1.0 TiB / 1.0 TiB avail; 1.3 MiB/s rd, 3.4 MiB/s wr, 294 op/s 2024-03-19T01:43:58.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:43:57 smithi115 ceph-mon[67332]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T01:43:58.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:43:57 smithi099 ceph-mon[62951]: pgmap v436: 97 pgs: 97 active+clean; 119 MiB data, 1.2 GiB used, 1.0 TiB / 1.0 TiB avail; 1.3 MiB/s rd, 3.4 MiB/s wr, 294 op/s 2024-03-19T01:43:58.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:43:57 smithi099 ceph-mon[62951]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T01:43:58.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:43:57 smithi112 ceph-mon[66819]: pgmap v436: 97 pgs: 97 active+clean; 119 MiB data, 1.2 GiB used, 1.0 TiB / 1.0 TiB avail; 1.3 MiB/s rd, 3.4 MiB/s wr, 294 op/s 2024-03-19T01:43:58.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:43:57 smithi112 ceph-mon[66819]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T01:43:58.678 INFO:tasks.workunit.client.0.smithi099.stdout: 1 323516 11.37 MB/sec execute 28 sec latency 31.950 ms 2024-03-19T01:43:59.678 INFO:tasks.workunit.client.0.smithi099.stdout: 1 325709 11.40 MB/sec execute 29 sec latency 53.454 ms 2024-03-19T01:44:00.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:43:59 smithi115 ceph-mon[67332]: pgmap v437: 97 pgs: 97 active+clean; 119 MiB data, 1.2 GiB used, 1.0 TiB / 1.0 TiB avail; 972 KiB/s rd, 2.9 MiB/s wr, 244 op/s 2024-03-19T01:44:00.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:43:59 smithi099 ceph-mon[62951]: pgmap v437: 97 pgs: 97 active+clean; 119 MiB data, 1.2 GiB used, 1.0 TiB / 1.0 TiB avail; 972 KiB/s rd, 2.9 MiB/s wr, 244 op/s 2024-03-19T01:44:00.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:43:59 smithi112 ceph-mon[66819]: pgmap v437: 97 pgs: 97 active+clean; 119 MiB data, 1.2 GiB used, 1.0 TiB / 1.0 TiB avail; 972 KiB/s rd, 2.9 MiB/s wr, 244 op/s 2024-03-19T01:44:00.678 INFO:tasks.workunit.client.0.smithi099.stdout: 1 327544 11.31 MB/sec execute 30 sec latency 44.757 ms 2024-03-19T01:44:01.678 INFO:tasks.workunit.client.0.smithi099.stdout: 1 330138 11.48 MB/sec execute 31 sec latency 18.913 ms 2024-03-19T01:44:02.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:44:01 smithi115 ceph-mon[67332]: pgmap v438: 97 pgs: 97 active+clean; 125 MiB data, 1.2 GiB used, 1.0 TiB / 1.0 TiB avail; 1.2 MiB/s rd, 3.2 MiB/s wr, 284 op/s 2024-03-19T01:44:02.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:44:01 smithi099 ceph-mon[62951]: pgmap v438: 97 pgs: 97 active+clean; 125 MiB data, 1.2 GiB used, 1.0 TiB / 1.0 TiB avail; 1.2 MiB/s rd, 3.2 MiB/s wr, 284 op/s 2024-03-19T01:44:02.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:44:01 smithi112 ceph-mon[66819]: pgmap v438: 97 pgs: 97 active+clean; 125 MiB data, 1.2 GiB used, 1.0 TiB / 1.0 TiB avail; 1.2 MiB/s rd, 3.2 MiB/s wr, 284 op/s 2024-03-19T01:44:02.678 INFO:tasks.workunit.client.0.smithi099.stdout: 1 331863 11.31 MB/sec execute 32 sec latency 30.997 ms 2024-03-19T01:44:03.678 INFO:tasks.workunit.client.0.smithi099.stdout: 1 334021 11.38 MB/sec execute 33 sec latency 12.973 ms 2024-03-19T01:44:04.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:44:03 smithi115 ceph-mon[67332]: pgmap v439: 97 pgs: 97 active+clean; 126 MiB data, 1.1 GiB used, 1.0 TiB / 1.0 TiB avail; 1.0 MiB/s rd, 3.1 MiB/s wr, 251 op/s 2024-03-19T01:44:04.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:44:03 smithi099 ceph-mon[62951]: pgmap v439: 97 pgs: 97 active+clean; 126 MiB data, 1.1 GiB used, 1.0 TiB / 1.0 TiB avail; 1.0 MiB/s rd, 3.1 MiB/s wr, 251 op/s 2024-03-19T01:44:04.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:44:03 smithi112 ceph-mon[66819]: pgmap v439: 97 pgs: 97 active+clean; 126 MiB data, 1.1 GiB used, 1.0 TiB / 1.0 TiB avail; 1.0 MiB/s rd, 3.1 MiB/s wr, 251 op/s 2024-03-19T01:44:04.679 INFO:tasks.workunit.client.0.smithi099.stdout: 1 335919 11.27 MB/sec execute 34 sec latency 53.751 ms 2024-03-19T01:44:05.679 INFO:tasks.workunit.client.0.smithi099.stdout: 1 337966 11.30 MB/sec execute 35 sec latency 39.333 ms 2024-03-19T01:44:06.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:44:05 smithi115 ceph-mon[67332]: pgmap v440: 97 pgs: 97 active+clean; 126 MiB data, 1.1 GiB used, 1.0 TiB / 1.0 TiB avail; 989 KiB/s rd, 2.6 MiB/s wr, 232 op/s 2024-03-19T01:44:06.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:44:05 smithi099 ceph-mon[62951]: pgmap v440: 97 pgs: 97 active+clean; 126 MiB data, 1.1 GiB used, 1.0 TiB / 1.0 TiB avail; 989 KiB/s rd, 2.6 MiB/s wr, 232 op/s 2024-03-19T01:44:06.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:44:05 smithi112 ceph-mon[66819]: pgmap v440: 97 pgs: 97 active+clean; 126 MiB data, 1.1 GiB used, 1.0 TiB / 1.0 TiB avail; 989 KiB/s rd, 2.6 MiB/s wr, 232 op/s 2024-03-19T01:44:06.678 INFO:tasks.workunit.client.0.smithi099.stdout: 1 340754 11.48 MB/sec execute 36 sec latency 13.217 ms 2024-03-19T01:44:06.819 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.817+0000 7f3d22d40640 1 -- 172.21.15.99:0/34702613 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3d1c0ffb40 msgr2=0x7f3d1c0fff60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:44:06.819 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.817+0000 7f3d22d40640 1 --2- 172.21.15.99:0/34702613 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3d1c0ffb40 0x7f3d1c0fff60 secure :-1 s=READY pgs=412 cs=0 l=1 rev1=1 crypto rx=0x7f3d10009930 tx=0x7f3d1002bc30 comp rx=0 tx=0).stop 2024-03-19T01:44:06.819 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.817+0000 7f3d22d40640 1 -- 172.21.15.99:0/34702613 shutdown_connections 2024-03-19T01:44:06.819 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.817+0000 7f3d22d40640 1 --2- 172.21.15.99:0/34702613 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3d1c1034d0 0x7f3d1c109d60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:06.819 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.817+0000 7f3d22d40640 1 --2- 172.21.15.99:0/34702613 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f3d1c102b10 0x7f3d1c102f90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:06.819 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.817+0000 7f3d22d40640 1 --2- 172.21.15.99:0/34702613 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3d1c0ffb40 0x7f3d1c0fff60 unknown :-1 s=CLOSED pgs=412 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:06.819 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.817+0000 7f3d22d40640 1 -- 172.21.15.99:0/34702613 >> 172.21.15.99:0/34702613 conn(0x7f3d1c0fb470 msgr2=0x7f3d1c0fd8b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:44:06.819 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.817+0000 7f3d22d40640 1 -- 172.21.15.99:0/34702613 shutdown_connections 2024-03-19T01:44:06.819 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.817+0000 7f3d22d40640 1 -- 172.21.15.99:0/34702613 wait complete. 2024-03-19T01:44:06.819 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.817+0000 7f3d22d40640 1 Processor -- start 2024-03-19T01:44:06.820 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.817+0000 7f3d22d40640 1 -- start start 2024-03-19T01:44:06.820 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.817+0000 7f3d22d40640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3d1c0ffb40 0x7f3d1c102480 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:06.820 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.817+0000 7f3d22d40640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f3d1c102b10 0x7f3d1c100ac0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:06.820 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.817+0000 7f3d22d40640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3d1c1034d0 0x7f3d1c101000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:06.820 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.817+0000 7f3d22d40640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f3d1c10d510 con 0x7f3d1c0ffb40 2024-03-19T01:44:06.820 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.818+0000 7f3d22d40640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f3d1c10d390 con 0x7f3d1c1034d0 2024-03-19T01:44:06.820 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.818+0000 7f3d22d40640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f3d1c10d690 con 0x7f3d1c102b10 2024-03-19T01:44:06.820 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.818+0000 7f3d20adc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3d1c0ffb40 0x7f3d1c102480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:06.820 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.818+0000 7f3d1bfff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f3d1c102b10 0x7f3d1c100ac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:06.820 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.818+0000 7f3d212dd640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3d1c1034d0 0x7f3d1c101000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:06.820 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.818+0000 7f3d20adc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3d1c0ffb40 0x7f3d1c102480 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:57782/0 (socket says 172.21.15.99:57782) 2024-03-19T01:44:06.820 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.818+0000 7f3d20adc640 1 -- 172.21.15.99:0/545472371 learned_addr learned my addr 172.21.15.99:0/545472371 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:44:06.820 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.818+0000 7f3d20adc640 1 -- 172.21.15.99:0/545472371 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f3d1c102b10 msgr2=0x7f3d1c100ac0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:44:06.821 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.818+0000 7f3d20adc640 1 --2- 172.21.15.99:0/545472371 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f3d1c102b10 0x7f3d1c100ac0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:06.821 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.818+0000 7f3d20adc640 1 -- 172.21.15.99:0/545472371 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3d1c1034d0 msgr2=0x7f3d1c101000 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:44:06.821 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.818+0000 7f3d20adc640 1 --2- 172.21.15.99:0/545472371 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3d1c1034d0 0x7f3d1c101000 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:06.821 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.818+0000 7f3d20adc640 1 -- 172.21.15.99:0/545472371 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f3d1c101890 con 0x7f3d1c0ffb40 2024-03-19T01:44:06.821 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.818+0000 7f3d212dd640 1 --2- 172.21.15.99:0/545472371 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3d1c1034d0 0x7f3d1c101000 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:44:06.821 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.818+0000 7f3d20adc640 1 --2- 172.21.15.99:0/545472371 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3d1c0ffb40 0x7f3d1c102480 secure :-1 s=READY pgs=413 cs=0 l=1 rev1=1 crypto rx=0x7f3d1002c7b0 tx=0x7f3d1002c650 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:44:06.821 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.818+0000 7f3d19ffb640 1 -- 172.21.15.99:0/545472371 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3d10041070 con 0x7f3d1c0ffb40 2024-03-19T01:44:06.821 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.818+0000 7f3d22d40640 1 -- 172.21.15.99:0/545472371 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f3d1c1a1bf0 con 0x7f3d1c0ffb40 2024-03-19T01:44:06.821 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.818+0000 7f3d19ffb640 1 -- 172.21.15.99:0/545472371 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f3d100054e0 con 0x7f3d1c0ffb40 2024-03-19T01:44:06.821 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.818+0000 7f3d19ffb640 1 -- 172.21.15.99:0/545472371 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3d10034440 con 0x7f3d1c0ffb40 2024-03-19T01:44:06.821 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.818+0000 7f3d22d40640 1 -- 172.21.15.99:0/545472371 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f3d1c1a2140 con 0x7f3d1c0ffb40 2024-03-19T01:44:06.829 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.821+0000 7f3d19ffb640 1 -- 172.21.15.99:0/545472371 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f3d10044070 con 0x7f3d1c0ffb40 2024-03-19T01:44:06.829 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.821+0000 7f3d259fd740 1 Processor -- start 2024-03-19T01:44:06.829 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.821+0000 7f3d259fd740 1 -- start start 2024-03-19T01:44:06.829 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.821+0000 7f3d19ffb640 1 --2- 172.21.15.99:0/545472371 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f3d0c0a28a0 0x7f3d0c0a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:06.829 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.821+0000 7f3d259fd740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5648d6920100 0x5648d6920520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:06.829 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.821+0000 7f3d259fd740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5648d6920af0 0x5648d69255f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:06.829 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.821+0000 7f3d259fd740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5648d6925c60 0x5648d69280f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:06.829 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.822+0000 7f3d259fd740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x5648d6884be0 con 0x5648d6925c60 2024-03-19T01:44:06.830 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.822+0000 7f3d259fd740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x5648d682d960 con 0x5648d6920af0 2024-03-19T01:44:06.830 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.822+0000 7f3d259fd740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x5648d6880c80 con 0x5648d6920100 2024-03-19T01:44:06.830 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.822+0000 7f3d19ffb640 1 -- 172.21.15.99:0/545472371 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f3d100f04c0 con 0x7f3d1c0ffb40 2024-03-19T01:44:06.830 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.822+0000 7f3d1bfff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5648d6920af0 0x5648d69255f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:06.830 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.822+0000 7f3d212dd640 1 --2- 172.21.15.99:0/545472371 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f3d0c0a28a0 0x7f3d0c0a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:06.830 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.822+0000 7f3d20adc640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5648d6920100 0x5648d6920520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:06.830 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.822+0000 7f3d1bfff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5648d6920af0 0x5648d69255f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:47674/0 (socket says 172.21.15.99:47674) 2024-03-19T01:44:06.830 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.822+0000 7f3d1bfff640 1 -- 172.21.15.99:0/707202347 learned_addr learned my addr 172.21.15.99:0/707202347 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:44:06.830 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.822+0000 7f3d212dd640 1 --2- 172.21.15.99:0/707202347 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5648d6925c60 0x5648d69280f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:06.830 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.822+0000 7f3d1bfff640 1 -- 172.21.15.99:0/707202347 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5648d6920100 msgr2=0x5648d6920520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:44:06.830 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.822+0000 7f3d1bfff640 1 --2- 172.21.15.99:0/707202347 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5648d6920100 0x5648d6920520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:06.831 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.822+0000 7f3d1bfff640 1 -- 172.21.15.99:0/707202347 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5648d6925c60 msgr2=0x5648d69280f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:44:06.831 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.822+0000 7f3d1bfff640 1 --2- 172.21.15.99:0/707202347 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5648d6925c60 0x5648d69280f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:06.831 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.822+0000 7f3d1bfff640 1 -- 172.21.15.99:0/707202347 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5648d6824d20 con 0x5648d6920af0 2024-03-19T01:44:06.831 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.822+0000 7f3d212dd640 1 --2- 172.21.15.99:0/545472371 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f3d0c0a28a0 0x7f3d0c0a4d60 secure :-1 s=READY pgs=261 cs=0 l=1 rev1=1 crypto rx=0x7f3d14005fd0 tx=0x7f3d14011720 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:44:06.831 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.822+0000 7f3d1bfff640 1 --2- 172.21.15.99:0/707202347 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5648d6920af0 0x5648d69255f0 secure :-1 s=READY pgs=101 cs=0 l=1 rev1=1 crypto rx=0x7f3d08009ca0 tx=0x7f3d08011810 comp rx=0 tx=0).ready entity=mon.2 client_cookie=f145ece48676450 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:44:06.831 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.822+0000 7f3d06ffd640 1 -- 172.21.15.99:0/707202347 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3d0801e070 con 0x5648d6920af0 2024-03-19T01:44:06.831 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.822+0000 7f3d06ffd640 1 -- 172.21.15.99:0/707202347 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f3d08009e50 con 0x5648d6920af0 2024-03-19T01:44:06.831 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.822+0000 7f3d06ffd640 1 -- 172.21.15.99:0/707202347 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3d0800e8e0 con 0x5648d6920af0 2024-03-19T01:44:06.831 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.823+0000 7f3d259fd740 1 -- 172.21.15.99:0/707202347 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5648d6920af0 msgr2=0x5648d69255f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:44:06.831 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.823+0000 7f3d259fd740 1 --2- 172.21.15.99:0/707202347 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5648d6920af0 0x5648d69255f0 secure :-1 s=READY pgs=101 cs=0 l=1 rev1=1 crypto rx=0x7f3d08009ca0 tx=0x7f3d08011810 comp rx=0 tx=0).stop 2024-03-19T01:44:06.831 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.823+0000 7f3d259fd740 1 -- 172.21.15.99:0/707202347 shutdown_connections 2024-03-19T01:44:06.831 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.823+0000 7f3d259fd740 1 --2- 172.21.15.99:0/707202347 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5648d6920af0 0x5648d69255f0 unknown :-1 s=CLOSED pgs=101 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:06.831 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.823+0000 7f3d259fd740 1 --2- 172.21.15.99:0/707202347 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5648d6925c60 0x5648d69280f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:06.831 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.823+0000 7f3d259fd740 1 --2- 172.21.15.99:0/707202347 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5648d6920100 0x5648d6920520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:06.832 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.823+0000 7f3d259fd740 1 -- 172.21.15.99:0/707202347 >> 172.21.15.99:0/707202347 conn(0x5648d691b670 msgr2=0x5648d691ba70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:44:06.832 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.823+0000 7f3d259fd740 1 -- 172.21.15.99:0/707202347 shutdown_connections 2024-03-19T01:44:06.832 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.823+0000 7f3d259fd740 1 -- 172.21.15.99:0/707202347 wait complete. 2024-03-19T01:44:06.832 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.823+0000 7f3d259fd740 1 -- start start 2024-03-19T01:44:06.832 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.824+0000 7f3d259fd740 1 Processor -- start 2024-03-19T01:44:06.832 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.824+0000 7f3d259fd740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5648d6920100 0x5648d692a100 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:06.832 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.824+0000 7f3d259fd740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5648d6920af0 0x5648d692a640 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:06.832 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.824+0000 7f3d259fd740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5648d6925c60 0x5648d6931680 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:06.832 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.824+0000 7f3d259fd740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x5648d6928c10 con 0x5648d6920100 2024-03-19T01:44:06.832 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.824+0000 7f3d259fd740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x5648d6840db0 con 0x5648d6920af0 2024-03-19T01:44:06.832 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.824+0000 7f3d259fd740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x5648d6928d90 con 0x5648d6925c60 2024-03-19T01:44:06.832 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.824+0000 7f3d259fd740 10 client.0 _ll_register_callbacks cb 0x5648d6845e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:44:06.832 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.824+0000 7f3d20adc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5648d6920100 0x5648d692a100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:06.832 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.824+0000 7f3d20adc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5648d6920100 0x5648d692a100 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:57802/0 (socket says 172.21.15.99:57802) 2024-03-19T01:44:06.833 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.824+0000 7f3d1bfff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5648d6920af0 0x5648d692a640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:06.833 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.824+0000 7f3d20adc640 1 -- 172.21.15.99:0/3615383600 learned_addr learned my addr 172.21.15.99:0/3615383600 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:44:06.833 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.824+0000 7f3d212dd640 1 --2- 172.21.15.99:0/3615383600 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5648d6925c60 0x5648d6931680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:06.833 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.824+0000 7f3d20adc640 1 -- 172.21.15.99:0/3615383600 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5648d6925c60 msgr2=0x5648d6931680 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:44:06.833 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.824+0000 7f3d20adc640 1 --2- 172.21.15.99:0/3615383600 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5648d6925c60 0x5648d6931680 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:06.833 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.824+0000 7f3d20adc640 1 -- 172.21.15.99:0/3615383600 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5648d6920af0 msgr2=0x5648d692a640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:44:06.833 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.824+0000 7f3d20adc640 1 --2- 172.21.15.99:0/3615383600 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5648d6920af0 0x5648d692a640 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:06.833 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.824+0000 7f3d20adc640 1 -- 172.21.15.99:0/3615383600 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5648d683fc60 con 0x5648d6920100 2024-03-19T01:44:06.833 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.824+0000 7f3d1bfff640 1 --2- 172.21.15.99:0/3615383600 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5648d6920af0 0x5648d692a640 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:44:06.833 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.824+0000 7f3d20adc640 1 --2- 172.21.15.99:0/3615383600 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5648d6920100 0x5648d692a100 secure :-1 s=READY pgs=414 cs=0 l=1 rev1=1 crypto rx=0x7f3d1002f020 tx=0x7f3d10009650 comp rx=0 tx=0).ready entity=mon.0 client_cookie=317d03d7c71b5a80 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:44:06.833 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.824+0000 7f3cfbfff640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:44:06.833 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.825+0000 7f3cfbfff640 1 -- 172.21.15.99:0/3615383600 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3d1004b040 con 0x5648d6920100 2024-03-19T01:44:06.833 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.825+0000 7f3d259fd740 1 -- 172.21.15.99:0/3615383600 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f3d08009e50 con 0x5648d6920100 2024-03-19T01:44:06.834 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.825+0000 7f3cfbfff640 1 -- 172.21.15.99:0/3615383600 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f3d100035f0 con 0x5648d6920100 2024-03-19T01:44:06.834 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.825+0000 7f3cfbfff640 1 -- 172.21.15.99:0/3615383600 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) ==== 24+0+0 (secure 0 0 0) 0x7f3d10004c80 con 0x5648d6920100 2024-03-19T01:44:06.834 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.825+0000 7f3d259fd740 10 client.15576 fetch_fsmap learned FSMap version 24 2024-03-19T01:44:06.834 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.825+0000 7f3d259fd740 1 -- 172.21.15.99:0/3615383600 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) -- 0x5648d6599210 con 0x5648d6920100 2024-03-19T01:44:06.834 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.825+0000 7f3cfbfff640 1 -- 172.21.15.99:0/3615383600 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f3d08009e50 con 0x5648d6920100 2024-03-19T01:44:06.834 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.825+0000 7f3cfbfff640 1 -- 172.21.15.99:0/3615383600 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 24) ==== 8342+0+0 (secure 0 0 0) 0x7f3d100f00e0 con 0x5648d6920100 2024-03-19T01:44:06.834 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.825+0000 7f3d259fd740 10 client.15576 fetch_fsmap finished waiting for FSMap version 24 2024-03-19T01:44:06.834 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.825+0000 7f3d259fd740 10 client.15576 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:44:06.834 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.825+0000 7f3d259fd740 20 client.15576 populate_metadata read hostname 'smithi099' 2024-03-19T01:44:06.834 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.825+0000 7f3d259fd740 1 --2- 172.21.15.99:0/3615383600 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5648d6936020 0x5648d69384b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:06.834 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.825+0000 7f3d259fd740 4 client.15576 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:44:06.834 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.825+0000 7f3d259fd740 1 -- 172.21.15.99:0/3615383600 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5648d6892120 con 0x5648d6936020 2024-03-19T01:44:06.834 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.825+0000 7f3d1bfff640 1 --2- 172.21.15.99:0/3615383600 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5648d6936020 0x5648d69384b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:06.834 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.826+0000 7f3d1bfff640 1 --2- 172.21.15.99:0/3615383600 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5648d6936020 0x5648d69384b0 crc :-1 s=READY pgs=55 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=80cd9b2bf1d86cbc server_cookie=22884e4e3b793344 in_seq=0 out_seq=0 2024-03-19T01:44:06.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.826+0000 7f3cfbfff640 10 client.15576 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:44:06.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.826+0000 7f3cfbfff640 1 -- 172.21.15.99:0/3615383600 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x5648d6892120 con 0x5648d6936020 2024-03-19T01:44:06.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.826+0000 7f3cfbfff640 10 client.15576 handle_command_reply: tid=0 2024-03-19T01:44:06.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.826+0000 7f3d259fd740 1 client.15576 shutdown 2024-03-19T01:44:06.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.826+0000 7f3d259fd740 1 -- 172.21.15.99:0/3615383600 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5648d6920100 msgr2=0x5648d692a100 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:44:06.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.826+0000 7f3d259fd740 1 --2- 172.21.15.99:0/3615383600 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5648d6920100 0x5648d692a100 secure :-1 s=READY pgs=414 cs=0 l=1 rev1=1 crypto rx=0x7f3d1002f020 tx=0x7f3d10009650 comp rx=0 tx=0).stop 2024-03-19T01:44:06.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.826+0000 7f3d259fd740 1 -- 172.21.15.99:0/3615383600 shutdown_connections 2024-03-19T01:44:06.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.826+0000 7f3d259fd740 1 --2- 172.21.15.99:0/3615383600 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5648d6936020 0x5648d69384b0 crc :-1 s=READY pgs=55 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:06.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.826+0000 7f3d259fd740 1 --2- 172.21.15.99:0/3615383600 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5648d6920af0 0x5648d692a640 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:06.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.826+0000 7f3d259fd740 1 --2- 172.21.15.99:0/3615383600 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5648d6925c60 0x5648d6931680 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:06.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.826+0000 7f3cfbfff640 0 client.15576 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:44:06.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.826+0000 7f3d259fd740 1 --2- 172.21.15.99:0/3615383600 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5648d6920100 0x5648d692a100 unknown :-1 s=CLOSED pgs=414 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:06.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.826+0000 7f3d259fd740 1 -- 172.21.15.99:0/3615383600 >> 172.21.15.99:0/3615383600 conn(0x5648d691b670 msgr2=0x5648d691c8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:44:06.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.827+0000 7f3d259fd740 1 -- 172.21.15.99:0/3615383600 shutdown_connections 2024-03-19T01:44:06.836 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.827+0000 7f3d259fd740 1 -- 172.21.15.99:0/3615383600 wait complete. 2024-03-19T01:44:06.836 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.827+0000 7f3d259fd740 20 client.15576 trim_cache size 0 max 16384 2024-03-19T01:44:06.844 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.843+0000 7f3d259fd740 1 Processor -- start 2024-03-19T01:44:06.845 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.843+0000 7f3d259fd740 1 -- start start 2024-03-19T01:44:06.845 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.843+0000 7f3d259fd740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5648d6920100 0x5648d69293d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:06.845 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.843+0000 7f3d259fd740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5648d6920af0 0x5648d6931640 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:06.845 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.843+0000 7f3d259fd740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5648d6925c60 0x5648d6929910 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:06.845 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.843+0000 7f3d259fd740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f3d1c1ad720 con 0x5648d6925c60 2024-03-19T01:44:06.845 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.843+0000 7f3d259fd740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f3d1c1aaff0 con 0x5648d6920100 2024-03-19T01:44:06.845 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.843+0000 7f3d259fd740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f3d1c1a8e40 con 0x5648d6920af0 2024-03-19T01:44:06.845 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.843+0000 7f3d212dd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5648d6925c60 0x5648d6929910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:06.845 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.843+0000 7f3d1bfff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5648d6920af0 0x5648d6931640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:06.845 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.843+0000 7f3d20adc640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5648d6920100 0x5648d69293d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:06.845 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.843+0000 7f3d1bfff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5648d6920af0 0x5648d6931640 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:49842/0 (socket says 172.21.15.99:49842) 2024-03-19T01:44:06.845 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.843+0000 7f3d20adc640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5648d6920100 0x5648d69293d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:47698/0 (socket says 172.21.15.99:47698) 2024-03-19T01:44:06.845 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.843+0000 7f3d1bfff640 1 -- 172.21.15.99:0/3012935017 learned_addr learned my addr 172.21.15.99:0/3012935017 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:44:06.846 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.843+0000 7f3d212dd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5648d6925c60 0x5648d6929910 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:57810/0 (socket says 172.21.15.99:57810) 2024-03-19T01:44:06.846 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.843+0000 7f3d20adc640 1 -- 172.21.15.99:0/3012935017 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5648d6920af0 msgr2=0x5648d6931640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:44:06.846 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.843+0000 7f3d20adc640 1 --2- 172.21.15.99:0/3012935017 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5648d6920af0 0x5648d6931640 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:06.846 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.843+0000 7f3d20adc640 1 -- 172.21.15.99:0/3012935017 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5648d6925c60 msgr2=0x5648d6929910 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:44:06.846 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.843+0000 7f3d20adc640 1 --2- 172.21.15.99:0/3012935017 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5648d6925c60 0x5648d6929910 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:06.846 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.843+0000 7f3d20adc640 1 -- 172.21.15.99:0/3012935017 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5648d6665620 con 0x5648d6920100 2024-03-19T01:44:06.846 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.843+0000 7f3d20adc640 1 --2- 172.21.15.99:0/3012935017 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5648d6920100 0x5648d69293d0 secure :-1 s=READY pgs=102 cs=0 l=1 rev1=1 crypto rx=0x7f3d100bcbd0 tx=0x7f3d10005fd0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=27ea072e4b534989 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:44:06.846 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.844+0000 7f3d067fc640 1 -- 172.21.15.99:0/3012935017 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3d100bca10 con 0x5648d6920100 2024-03-19T01:44:06.846 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.844+0000 7f3d067fc640 1 -- 172.21.15.99:0/3012935017 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f3d10004c80 con 0x5648d6920100 2024-03-19T01:44:06.846 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.844+0000 7f3d067fc640 1 -- 172.21.15.99:0/3012935017 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3d100035f0 con 0x5648d6920100 2024-03-19T01:44:06.846 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.844+0000 7f3d259fd740 1 -- 172.21.15.99:0/3012935017 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5648d6920100 msgr2=0x5648d69293d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:44:06.846 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.844+0000 7f3d259fd740 1 --2- 172.21.15.99:0/3012935017 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5648d6920100 0x5648d69293d0 secure :-1 s=READY pgs=102 cs=0 l=1 rev1=1 crypto rx=0x7f3d100bcbd0 tx=0x7f3d10005fd0 comp rx=0 tx=0).stop 2024-03-19T01:44:06.847 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.844+0000 7f3d259fd740 1 -- 172.21.15.99:0/3012935017 shutdown_connections 2024-03-19T01:44:06.847 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.844+0000 7f3d259fd740 1 --2- 172.21.15.99:0/3012935017 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5648d6925c60 0x5648d6929910 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:06.847 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.844+0000 7f3d259fd740 1 --2- 172.21.15.99:0/3012935017 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5648d6920af0 0x5648d6931640 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:06.847 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.844+0000 7f3d259fd740 1 --2- 172.21.15.99:0/3012935017 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5648d6920100 0x5648d69293d0 unknown :-1 s=CLOSED pgs=102 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:06.847 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.844+0000 7f3d259fd740 1 -- 172.21.15.99:0/3012935017 >> 172.21.15.99:0/3012935017 conn(0x5648d691b670 msgr2=0x5648d691f160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:44:06.847 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.844+0000 7f3d259fd740 1 -- 172.21.15.99:0/3012935017 shutdown_connections 2024-03-19T01:44:06.847 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.844+0000 7f3d259fd740 1 -- 172.21.15.99:0/3012935017 wait complete. 2024-03-19T01:44:06.847 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.845+0000 7f3d259fd740 1 -- start start 2024-03-19T01:44:06.847 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.845+0000 7f3d259fd740 1 Processor -- start 2024-03-19T01:44:06.850 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.845+0000 7f3d259fd740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5648d6920100 0x5648d692ee80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:06.850 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.845+0000 7f3d259fd740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5648d6920af0 0x5648d692f3c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:06.850 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.845+0000 7f3d259fd740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5648d6925c60 0x5648d69303d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:06.850 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.845+0000 7f3d259fd740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f3d1c1a94b0 con 0x5648d6920100 2024-03-19T01:44:06.850 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.845+0000 7f3d259fd740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f3d1c1aa920 con 0x5648d6920af0 2024-03-19T01:44:06.850 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.845+0000 7f3d259fd740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f3d1c1a9b80 con 0x5648d6925c60 2024-03-19T01:44:06.851 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.845+0000 7f3d259fd740 10 client.0 _ll_register_callbacks cb 0x7f3d10005b50 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:44:06.851 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.845+0000 7f3d20adc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5648d6920100 0x5648d692ee80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:06.851 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.845+0000 7f3d1bfff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5648d6920af0 0x5648d692f3c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:06.851 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.845+0000 7f3d20adc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5648d6920100 0x5648d692ee80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:57826/0 (socket says 172.21.15.99:57826) 2024-03-19T01:44:06.851 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.845+0000 7f3d20adc640 1 -- 172.21.15.99:0/13243428 learned_addr learned my addr 172.21.15.99:0/13243428 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:44:06.851 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.845+0000 7f3d212dd640 1 --2- 172.21.15.99:0/13243428 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5648d6925c60 0x5648d69303d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:06.851 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.846+0000 7f3d20adc640 1 -- 172.21.15.99:0/13243428 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5648d6925c60 msgr2=0x5648d69303d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:44:06.851 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.846+0000 7f3d20adc640 1 --2- 172.21.15.99:0/13243428 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5648d6925c60 0x5648d69303d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:06.851 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.846+0000 7f3d20adc640 1 -- 172.21.15.99:0/13243428 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5648d6920af0 msgr2=0x5648d692f3c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:44:06.851 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.846+0000 7f3d20adc640 1 --2- 172.21.15.99:0/13243428 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5648d6920af0 0x5648d692f3c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:06.851 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.846+0000 7f3d20adc640 1 -- 172.21.15.99:0/13243428 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5648d69309c0 con 0x5648d6920100 2024-03-19T01:44:06.851 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.846+0000 7f3d1bfff640 1 --2- 172.21.15.99:0/13243428 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5648d6920af0 0x5648d692f3c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:44:06.851 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.846+0000 7f3d20adc640 1 --2- 172.21.15.99:0/13243428 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5648d6920100 0x5648d692ee80 secure :-1 s=READY pgs=415 cs=0 l=1 rev1=1 crypto rx=0x7f3d100c1100 tx=0x7f3d10004610 comp rx=0 tx=0).ready entity=mon.0 client_cookie=10caf77b374fcd6e server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:44:06.851 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.846+0000 7f3cfbfff640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:44:06.851 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.846+0000 7f3cfbfff640 1 -- 172.21.15.99:0/13243428 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3d1009f070 con 0x5648d6920100 2024-03-19T01:44:06.852 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.846+0000 7f3cfbfff640 1 -- 172.21.15.99:0/13243428 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f3d100035f0 con 0x5648d6920100 2024-03-19T01:44:06.852 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.846+0000 7f3d259fd740 1 -- 172.21.15.99:0/13243428 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f3d10004c80 con 0x5648d6920100 2024-03-19T01:44:06.852 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.846+0000 7f3cfbfff640 1 -- 172.21.15.99:0/13243428 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) ==== 24+0+0 (secure 0 0 0) 0x7f3d100bca10 con 0x5648d6920100 2024-03-19T01:44:06.852 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.846+0000 7f3d259fd740 10 client.15582 fetch_fsmap learned FSMap version 24 2024-03-19T01:44:06.852 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.846+0000 7f3d259fd740 1 -- 172.21.15.99:0/13243428 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) -- 0x5648d69309c0 con 0x5648d6920100 2024-03-19T01:44:06.852 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.846+0000 7f3cfbfff640 1 -- 172.21.15.99:0/13243428 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f3d10004c80 con 0x5648d6920100 2024-03-19T01:44:06.852 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.847+0000 7f3cfbfff640 1 -- 172.21.15.99:0/13243428 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 24) ==== 8342+0+0 (secure 0 0 0) 0x7f3d100bc320 con 0x5648d6920100 2024-03-19T01:44:06.852 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.847+0000 7f3d259fd740 10 client.15582 fetch_fsmap finished waiting for FSMap version 24 2024-03-19T01:44:06.852 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.847+0000 7f3d259fd740 10 client.15582 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:44:06.852 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.847+0000 7f3d259fd740 20 client.15582 populate_metadata read hostname 'smithi099' 2024-03-19T01:44:06.852 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.847+0000 7f3d259fd740 1 --2- 172.21.15.99:0/13243428 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5648d699caa0 0x5648d699eec0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:06.852 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.847+0000 7f3d259fd740 4 client.15582 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:44:06.852 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.847+0000 7f3d259fd740 1 -- 172.21.15.99:0/13243428 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x5648d688c470 con 0x5648d699caa0 2024-03-19T01:44:06.852 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.847+0000 7f3d1bfff640 1 --2- 172.21.15.99:0/13243428 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5648d699caa0 0x5648d699eec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:06.853 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.847+0000 7f3cfbfff640 10 client.15582 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:44:06.853 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.847+0000 7f3d1bfff640 1 --2- 172.21.15.99:0/13243428 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5648d699caa0 0x5648d699eec0 crc :-1 s=READY pgs=56 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=d4a77e2d2dff2c46 server_cookie=4eb83d3188935c27 in_seq=0 out_seq=0 2024-03-19T01:44:06.894 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.893+0000 7f3cfbfff640 1 -- 172.21.15.99:0/13243428 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x5648d688c470 con 0x5648d699caa0 2024-03-19T01:44:06.894 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.893+0000 7f3cfbfff640 10 client.15582 handle_command_reply: tid=0 2024-03-19T01:44:06.894 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.893+0000 7f3d259fd740 1 client.15582 shutdown 2024-03-19T01:44:06.894 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.893+0000 7f3d259fd740 1 -- 172.21.15.99:0/13243428 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5648d6920100 msgr2=0x5648d692ee80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:44:06.894 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.893+0000 7f3d259fd740 1 --2- 172.21.15.99:0/13243428 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5648d6920100 0x5648d692ee80 secure :-1 s=READY pgs=415 cs=0 l=1 rev1=1 crypto rx=0x7f3d100c1100 tx=0x7f3d10004610 comp rx=0 tx=0).stop 2024-03-19T01:44:06.894 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.893+0000 7f3d259fd740 1 -- 172.21.15.99:0/13243428 shutdown_connections 2024-03-19T01:44:06.894 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.893+0000 7f3d259fd740 1 --2- 172.21.15.99:0/13243428 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5648d699caa0 0x5648d699eec0 crc :-1 s=READY pgs=56 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:06.895 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.893+0000 7f3d259fd740 1 --2- 172.21.15.99:0/13243428 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5648d6920af0 0x5648d692f3c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:06.895 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.893+0000 7f3cfbfff640 0 client.15582 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:44:06.895 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.893+0000 7f3d259fd740 1 --2- 172.21.15.99:0/13243428 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5648d6925c60 0x5648d69303d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:06.895 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.893+0000 7f3d259fd740 1 --2- 172.21.15.99:0/13243428 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5648d6920100 0x5648d692ee80 unknown :-1 s=CLOSED pgs=415 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:06.895 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.893+0000 7f3d259fd740 1 -- 172.21.15.99:0/13243428 >> 172.21.15.99:0/13243428 conn(0x5648d691b670 msgr2=0x5648d6921590 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:44:06.895 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.894+0000 7f3d259fd740 1 -- 172.21.15.99:0/13243428 shutdown_connections 2024-03-19T01:44:06.895 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.894+0000 7f3d259fd740 1 -- 172.21.15.99:0/13243428 wait complete. 2024-03-19T01:44:06.895 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.894+0000 7f3d259fd740 20 client.15582 trim_cache size 0 max 16384 2024-03-19T01:44:06.895 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:44:06.895 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:44:06.896 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:44:06.896 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:44:06.896 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.894+0000 7f3d22d40640 1 -- 172.21.15.99:0/545472371 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f3d0c0a28a0 msgr2=0x7f3d0c0a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:44:06.896 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.894+0000 7f3d22d40640 1 --2- 172.21.15.99:0/545472371 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f3d0c0a28a0 0x7f3d0c0a4d60 secure :-1 s=READY pgs=261 cs=0 l=1 rev1=1 crypto rx=0x7f3d14005fd0 tx=0x7f3d14011720 comp rx=0 tx=0).stop 2024-03-19T01:44:06.897 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.894+0000 7f3d22d40640 1 -- 172.21.15.99:0/545472371 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3d1c0ffb40 msgr2=0x7f3d1c102480 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:44:06.897 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.894+0000 7f3d22d40640 1 --2- 172.21.15.99:0/545472371 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3d1c0ffb40 0x7f3d1c102480 secure :-1 s=READY pgs=413 cs=0 l=1 rev1=1 crypto rx=0x7f3d1002c7b0 tx=0x7f3d1002c650 comp rx=0 tx=0).stop 2024-03-19T01:44:06.897 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.894+0000 7f3d22d40640 1 -- 172.21.15.99:0/545472371 shutdown_connections 2024-03-19T01:44:06.897 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.894+0000 7f3d22d40640 1 --2- 172.21.15.99:0/545472371 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f3d0c0a28a0 0x7f3d0c0a4d60 unknown :-1 s=CLOSED pgs=261 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:06.897 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.894+0000 7f3d22d40640 1 --2- 172.21.15.99:0/545472371 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3d1c1034d0 0x7f3d1c101000 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:06.897 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.894+0000 7f3d22d40640 1 --2- 172.21.15.99:0/545472371 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f3d1c102b10 0x7f3d1c100ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:06.897 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.894+0000 7f3d22d40640 1 --2- 172.21.15.99:0/545472371 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3d1c0ffb40 0x7f3d1c102480 unknown :-1 s=CLOSED pgs=413 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:06.897 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.894+0000 7f3d22d40640 1 -- 172.21.15.99:0/545472371 >> 172.21.15.99:0/545472371 conn(0x7f3d1c0fb470 msgr2=0x7f3d1c105e10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:44:06.897 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.894+0000 7f3d22d40640 1 -- 172.21.15.99:0/545472371 shutdown_connections 2024-03-19T01:44:06.897 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:06.894+0000 7f3d22d40640 1 -- 172.21.15.99:0/545472371 wait complete. 2024-03-19T01:44:07.679 INFO:tasks.workunit.client.0.smithi099.stdout: 1 342497 11.33 MB/sec execute 37 sec latency 16.855 ms 2024-03-19T01:44:08.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:44:07 smithi115 ceph-mon[67332]: pgmap v441: 97 pgs: 97 active+clean; 128 MiB data, 1.2 GiB used, 1.0 TiB / 1.0 TiB avail; 1.4 MiB/s rd, 3.4 MiB/s wr, 318 op/s 2024-03-19T01:44:08.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:44:07 smithi115 ceph-mon[67332]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T01:44:08.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:44:07 smithi099 ceph-mon[62951]: pgmap v441: 97 pgs: 97 active+clean; 128 MiB data, 1.2 GiB used, 1.0 TiB / 1.0 TiB avail; 1.4 MiB/s rd, 3.4 MiB/s wr, 318 op/s 2024-03-19T01:44:08.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:44:07 smithi099 ceph-mon[62951]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T01:44:08.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:44:07 smithi112 ceph-mon[66819]: pgmap v441: 97 pgs: 97 active+clean; 128 MiB data, 1.2 GiB used, 1.0 TiB / 1.0 TiB avail; 1.4 MiB/s rd, 3.4 MiB/s wr, 318 op/s 2024-03-19T01:44:08.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:44:07 smithi112 ceph-mon[66819]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T01:44:08.679 INFO:tasks.workunit.client.0.smithi099.stdout: 1 344677 11.39 MB/sec execute 38 sec latency 25.343 ms 2024-03-19T01:44:09.679 INFO:tasks.workunit.client.0.smithi099.stdout: 1 346826 11.38 MB/sec execute 39 sec latency 45.565 ms 2024-03-19T01:44:10.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:44:09 smithi115 ceph-mon[67332]: pgmap v442: 97 pgs: 97 active+clean; 128 MiB data, 1.2 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 3.0 MiB/s wr, 244 op/s 2024-03-19T01:44:10.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:44:09 smithi099 ceph-mon[62951]: pgmap v442: 97 pgs: 97 active+clean; 128 MiB data, 1.2 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 3.0 MiB/s wr, 244 op/s 2024-03-19T01:44:10.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:44:09 smithi112 ceph-mon[66819]: pgmap v442: 97 pgs: 97 active+clean; 128 MiB data, 1.2 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 3.0 MiB/s wr, 244 op/s 2024-03-19T01:44:10.679 INFO:tasks.workunit.client.0.smithi099.stdout: 1 348734 11.35 MB/sec execute 40 sec latency 50.707 ms 2024-03-19T01:44:11.679 INFO:tasks.workunit.client.0.smithi099.stdout: 1 351374 11.48 MB/sec execute 41 sec latency 15.539 ms 2024-03-19T01:44:12.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:44:11 smithi115 ceph-mon[67332]: pgmap v443: 97 pgs: 97 active+clean; 129 MiB data, 1.2 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 2.9 MiB/s wr, 246 op/s 2024-03-19T01:44:12.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:44:11 smithi099 ceph-mon[62951]: pgmap v443: 97 pgs: 97 active+clean; 129 MiB data, 1.2 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 2.9 MiB/s wr, 246 op/s 2024-03-19T01:44:12.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:44:11 smithi112 ceph-mon[66819]: pgmap v443: 97 pgs: 97 active+clean; 129 MiB data, 1.2 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 2.9 MiB/s wr, 246 op/s 2024-03-19T01:44:12.679 INFO:tasks.workunit.client.0.smithi099.stdout: 1 353073 11.34 MB/sec execute 42 sec latency 36.054 ms 2024-03-19T01:44:13.679 INFO:tasks.workunit.client.0.smithi099.stdout: 1 355285 11.40 MB/sec execute 43 sec latency 9.767 ms 2024-03-19T01:44:14.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:44:13 smithi115 ceph-mon[67332]: pgmap v444: 97 pgs: 97 active+clean; 132 MiB data, 1.2 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 2.9 MiB/s wr, 265 op/s 2024-03-19T01:44:14.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:44:13 smithi099 ceph-mon[62951]: pgmap v444: 97 pgs: 97 active+clean; 132 MiB data, 1.2 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 2.9 MiB/s wr, 265 op/s 2024-03-19T01:44:14.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:44:13 smithi112 ceph-mon[66819]: pgmap v444: 97 pgs: 97 active+clean; 132 MiB data, 1.2 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 2.9 MiB/s wr, 265 op/s 2024-03-19T01:44:14.679 INFO:tasks.workunit.client.0.smithi099.stdout: 1 357271 11.32 MB/sec execute 44 sec latency 67.384 ms 2024-03-19T01:44:15.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:44:14 smithi115 ceph-mon[67332]: pgmap v445: 97 pgs: 97 active+clean; 132 MiB data, 1.2 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 2.8 MiB/s wr, 246 op/s 2024-03-19T01:44:15.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:44:14 smithi099 ceph-mon[62951]: pgmap v445: 97 pgs: 97 active+clean; 132 MiB data, 1.2 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 2.8 MiB/s wr, 246 op/s 2024-03-19T01:44:15.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:44:14 smithi112 ceph-mon[66819]: pgmap v445: 97 pgs: 97 active+clean; 132 MiB data, 1.2 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 2.8 MiB/s wr, 246 op/s 2024-03-19T01:44:15.679 INFO:tasks.workunit.client.0.smithi099.stdout: 1 358911 11.32 MB/sec execute 45 sec latency 72.013 ms 2024-03-19T01:44:16.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:44:15 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:44:16.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:44:15 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:44:16.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:44:15 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:44:16.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:44:15 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:44:16.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:44:15 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:44:16.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:44:15 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:44:16.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:44:15 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:44:16.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:44:15 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:44:16.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:44:16 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:44:16.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:44:16 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:44:16.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:44:16 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:44:16.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:44:16 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:44:16.679 INFO:tasks.workunit.client.0.smithi099.stdout: 1 361417 11.36 MB/sec execute 46 sec latency 16.486 ms 2024-03-19T01:44:17.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:44:17 smithi115 ceph-mon[67332]: pgmap v446: 97 pgs: 97 active+clean; 136 MiB data, 1.2 GiB used, 1.0 TiB / 1.0 TiB avail; 1.4 MiB/s rd, 3.6 MiB/s wr, 319 op/s 2024-03-19T01:44:17.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:44:17 smithi099 ceph-mon[62951]: pgmap v446: 97 pgs: 97 active+clean; 136 MiB data, 1.2 GiB used, 1.0 TiB / 1.0 TiB avail; 1.4 MiB/s rd, 3.6 MiB/s wr, 319 op/s 2024-03-19T01:44:17.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:44:17 smithi112 ceph-mon[66819]: pgmap v446: 97 pgs: 97 active+clean; 136 MiB data, 1.2 GiB used, 1.0 TiB / 1.0 TiB avail; 1.4 MiB/s rd, 3.6 MiB/s wr, 319 op/s 2024-03-19T01:44:17.679 INFO:tasks.workunit.client.0.smithi099.stdout: 1 363012 11.29 MB/sec execute 47 sec latency 36.946 ms 2024-03-19T01:44:18.679 INFO:tasks.workunit.client.0.smithi099.stdout: 1 365633 11.40 MB/sec execute 48 sec latency 15.693 ms 2024-03-19T01:44:19.679 INFO:tasks.workunit.client.0.smithi099.stdout: 1 367757 11.32 MB/sec execute 49 sec latency 26.692 ms 2024-03-19T01:44:19.986 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.983+0000 7f3ce5697640 1 -- 172.21.15.99:0/3848277486 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3ce01075c0 msgr2=0x7f3ce01099b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:44:19.986 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.983+0000 7f3ce5697640 1 --2- 172.21.15.99:0/3848277486 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3ce01075c0 0x7f3ce01099b0 secure :-1 s=READY pgs=416 cs=0 l=1 rev1=1 crypto rx=0x7f3cd8009ca0 tx=0x7f3cd802c0f0 comp rx=0 tx=0).stop 2024-03-19T01:44:19.986 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.983+0000 7f3ce5697640 1 -- 172.21.15.99:0/3848277486 shutdown_connections 2024-03-19T01:44:19.986 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.983+0000 7f3ce5697640 1 --2- 172.21.15.99:0/3848277486 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3ce01075c0 0x7f3ce01099b0 unknown :-1 s=CLOSED pgs=416 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:19.986 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.983+0000 7f3ce5697640 1 --2- 172.21.15.99:0/3848277486 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f3ce0104c90 0x7f3ce0107080 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:19.986 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.983+0000 7f3ce5697640 1 --2- 172.21.15.99:0/3848277486 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3ce0102360 0x7f3ce0104750 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:19.986 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.983+0000 7f3ce5697640 1 -- 172.21.15.99:0/3848277486 >> 172.21.15.99:0/3848277486 conn(0x7f3ce00fbcb0 msgr2=0x7f3ce00fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:44:19.986 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.984+0000 7f3ce5697640 1 -- 172.21.15.99:0/3848277486 shutdown_connections 2024-03-19T01:44:19.987 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.984+0000 7f3ce5697640 1 -- 172.21.15.99:0/3848277486 wait complete. 2024-03-19T01:44:19.987 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.984+0000 7f3ce5697640 1 Processor -- start 2024-03-19T01:44:19.987 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.984+0000 7f3ce5697640 1 -- start start 2024-03-19T01:44:19.987 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.984+0000 7f3ce5697640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3ce0102360 0x7f3ce0197aa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:19.987 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.984+0000 7f3ce5697640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f3ce0104c90 0x7f3ce0197fe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:19.987 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.984+0000 7f3ce5697640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3ce01075c0 0x7f3ce019f050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:19.987 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.984+0000 7f3ce5697640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f3ce010d4e0 con 0x7f3ce01075c0 2024-03-19T01:44:19.987 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.984+0000 7f3ce5697640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f3ce010d360 con 0x7f3ce0102360 2024-03-19T01:44:19.987 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.984+0000 7f3ce5697640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f3ce010d660 con 0x7f3ce0104c90 2024-03-19T01:44:19.987 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.984+0000 7f3cdf7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3ce01075c0 0x7f3ce019f050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:19.987 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.984+0000 7f3cdeffd640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3ce0102360 0x7f3ce0197aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:19.987 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.984+0000 7f3cdeffd640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3ce0102360 0x7f3ce0197aa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:37890/0 (socket says 172.21.15.99:37890) 2024-03-19T01:44:19.987 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.984+0000 7f3cdf7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3ce01075c0 0x7f3ce019f050 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58526/0 (socket says 172.21.15.99:58526) 2024-03-19T01:44:19.988 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.984+0000 7f3cdeffd640 1 -- 172.21.15.99:0/1891071460 learned_addr learned my addr 172.21.15.99:0/1891071460 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:44:19.988 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.984+0000 7f3cd7fff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f3ce0104c90 0x7f3ce0197fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:19.988 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.985+0000 7f3cdf7fe640 1 -- 172.21.15.99:0/1891071460 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f3ce0104c90 msgr2=0x7f3ce0197fe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:44:19.988 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.985+0000 7f3cdf7fe640 1 --2- 172.21.15.99:0/1891071460 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f3ce0104c90 0x7f3ce0197fe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:19.988 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.985+0000 7f3cdf7fe640 1 -- 172.21.15.99:0/1891071460 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3ce0102360 msgr2=0x7f3ce0197aa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:44:19.988 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.985+0000 7f3cdf7fe640 1 --2- 172.21.15.99:0/1891071460 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3ce0102360 0x7f3ce0197aa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:19.988 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.985+0000 7f3cdf7fe640 1 -- 172.21.15.99:0/1891071460 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f3ce019f6c0 con 0x7f3ce01075c0 2024-03-19T01:44:19.988 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.985+0000 7f3cdeffd640 1 --2- 172.21.15.99:0/1891071460 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3ce0102360 0x7f3ce0197aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:44:19.988 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.985+0000 7f3cdf7fe640 1 --2- 172.21.15.99:0/1891071460 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3ce01075c0 0x7f3ce019f050 secure :-1 s=READY pgs=417 cs=0 l=1 rev1=1 crypto rx=0x7f3cd802c630 tx=0x7f3cd8032f20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:44:19.988 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.985+0000 7f3cdcff9640 1 -- 172.21.15.99:0/1891071460 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3cd8041020 con 0x7f3ce01075c0 2024-03-19T01:44:19.988 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.985+0000 7f3ce5697640 1 -- 172.21.15.99:0/1891071460 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f3ce019f950 con 0x7f3ce01075c0 2024-03-19T01:44:19.988 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.985+0000 7f3cdcff9640 1 -- 172.21.15.99:0/1891071460 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f3cd8034cd0 con 0x7f3ce01075c0 2024-03-19T01:44:19.988 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.985+0000 7f3cdcff9640 1 -- 172.21.15.99:0/1891071460 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3cd8039040 con 0x7f3ce01075c0 2024-03-19T01:44:19.988 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.985+0000 7f3ce5697640 1 -- 172.21.15.99:0/1891071460 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f3ce019fe10 con 0x7f3ce01075c0 2024-03-19T01:44:19.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.988+0000 7f3cdcff9640 1 -- 172.21.15.99:0/1891071460 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f3cd8044020 con 0x7f3ce01075c0 2024-03-19T01:44:19.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.988+0000 7f3ce8282740 1 Processor -- start 2024-03-19T01:44:19.996 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.988+0000 7f3ce8282740 1 -- start start 2024-03-19T01:44:19.996 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.988+0000 7f3ce8282740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564102f9c100 0x564102f9c520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:19.996 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.988+0000 7f3ce8282740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564102f9caf0 0x564102fa15f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:19.996 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.988+0000 7f3ce8282740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564102fa1c60 0x564102fa40f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:19.996 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.988+0000 7f3ce8282740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x564102f00be0 con 0x564102f9caf0 2024-03-19T01:44:19.996 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.988+0000 7f3ce8282740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x564102ea9960 con 0x564102f9c100 2024-03-19T01:44:19.996 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.988+0000 7f3ce8282740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x564102efcc80 con 0x564102fa1c60 2024-03-19T01:44:19.996 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.988+0000 7f3cd7fff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564102f9c100 0x564102f9c520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:19.996 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.988+0000 7f3cdeffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564102f9caf0 0x564102fa15f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:19.996 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.988+0000 7f3cd7fff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564102f9c100 0x564102f9c520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:37894/0 (socket says 172.21.15.99:37894) 2024-03-19T01:44:19.996 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.988+0000 7f3cd7fff640 1 -- 172.21.15.99:0/3403959201 learned_addr learned my addr 172.21.15.99:0/3403959201 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:44:19.996 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.988+0000 7f3cd7fff640 1 --2- 172.21.15.99:0/3403959201 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564102fa1c60 0x564102fa40f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:19.997 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.989+0000 7f3cdcff9640 1 --2- 172.21.15.99:0/1891071460 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f3ccc0a28a0 0x7f3ccc0a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:19.997 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.989+0000 7f3cd7fff640 1 -- 172.21.15.99:0/3403959201 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564102fa1c60 msgr2=0x564102fa40f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:44:19.997 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.989+0000 7f3cd7fff640 1 --2- 172.21.15.99:0/3403959201 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564102fa1c60 0x564102fa40f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:19.997 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.989+0000 7f3cd7fff640 1 -- 172.21.15.99:0/3403959201 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564102f9caf0 msgr2=0x564102fa15f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:44:19.997 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.989+0000 7f3cd7fff640 1 --2- 172.21.15.99:0/3403959201 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564102f9caf0 0x564102fa15f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:19.997 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.989+0000 7f3cd7fff640 1 -- 172.21.15.99:0/3403959201 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x564102ea0d20 con 0x564102f9c100 2024-03-19T01:44:19.997 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.989+0000 7f3cdeffd640 1 --2- 172.21.15.99:0/3403959201 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564102f9caf0 0x564102fa15f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:44:19.997 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.989+0000 7f3cdcff9640 1 -- 172.21.15.99:0/1891071460 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f3cd80f0840 con 0x7f3ce01075c0 2024-03-19T01:44:19.997 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.989+0000 7f3cdf7fe640 1 --2- 172.21.15.99:0/1891071460 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f3ccc0a28a0 0x7f3ccc0a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:19.997 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.989+0000 7f3cd7fff640 1 --2- 172.21.15.99:0/3403959201 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564102f9c100 0x564102f9c520 secure :-1 s=READY pgs=103 cs=0 l=1 rev1=1 crypto rx=0x7f3cc80058c0 tx=0x7f3cc8005850 comp rx=0 tx=0).ready entity=mon.2 client_cookie=35f75d60ae4a5bb2 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:44:19.997 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.989+0000 7f3cd57fa640 1 -- 172.21.15.99:0/3403959201 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3cc8004030 con 0x564102f9c100 2024-03-19T01:44:19.997 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.989+0000 7f3cdf7fe640 1 --2- 172.21.15.99:0/1891071460 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f3ccc0a28a0 0x7f3ccc0a4d60 secure :-1 s=READY pgs=262 cs=0 l=1 rev1=1 crypto rx=0x7f3cd80ee300 tx=0x7f3cd80323c0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:44:19.997 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.989+0000 7f3cd57fa640 1 -- 172.21.15.99:0/3403959201 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f3cc8005500 con 0x564102f9c100 2024-03-19T01:44:19.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.989+0000 7f3cd57fa640 1 -- 172.21.15.99:0/3403959201 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3cc8004c10 con 0x564102f9c100 2024-03-19T01:44:19.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.990+0000 7f3ce8282740 1 -- 172.21.15.99:0/3403959201 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564102f9c100 msgr2=0x564102f9c520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:44:19.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.990+0000 7f3ce8282740 1 --2- 172.21.15.99:0/3403959201 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564102f9c100 0x564102f9c520 secure :-1 s=READY pgs=103 cs=0 l=1 rev1=1 crypto rx=0x7f3cc80058c0 tx=0x7f3cc8005850 comp rx=0 tx=0).stop 2024-03-19T01:44:19.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.990+0000 7f3ce8282740 1 -- 172.21.15.99:0/3403959201 shutdown_connections 2024-03-19T01:44:19.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.990+0000 7f3ce8282740 1 --2- 172.21.15.99:0/3403959201 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564102fa1c60 0x564102fa40f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:19.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.990+0000 7f3ce8282740 1 --2- 172.21.15.99:0/3403959201 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564102f9caf0 0x564102fa15f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:19.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.990+0000 7f3ce8282740 1 --2- 172.21.15.99:0/3403959201 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564102f9c100 0x564102f9c520 unknown :-1 s=CLOSED pgs=103 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:19.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.990+0000 7f3ce8282740 1 -- 172.21.15.99:0/3403959201 >> 172.21.15.99:0/3403959201 conn(0x564102f97670 msgr2=0x564102f97a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:44:19.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.990+0000 7f3ce8282740 1 -- 172.21.15.99:0/3403959201 shutdown_connections 2024-03-19T01:44:19.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.990+0000 7f3ce8282740 1 -- 172.21.15.99:0/3403959201 wait complete. 2024-03-19T01:44:19.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.990+0000 7f3ce8282740 1 -- start start 2024-03-19T01:44:19.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.990+0000 7f3ce8282740 1 Processor -- start 2024-03-19T01:44:19.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.991+0000 7f3ce8282740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564102f9c100 0x564102fa6140 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:19.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.991+0000 7f3ce8282740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564102f9caf0 0x564102fa6680 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:19.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.991+0000 7f3ce8282740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564102fa1c60 0x564102fad6c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:19.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.991+0000 7f3ce8282740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x564102fa4c10 con 0x564102f9c100 2024-03-19T01:44:19.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.991+0000 7f3ce8282740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x564102ebcdb0 con 0x564102fa1c60 2024-03-19T01:44:19.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.991+0000 7f3ce8282740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x564102fa4d90 con 0x564102f9caf0 2024-03-19T01:44:19.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.991+0000 7f3ce8282740 10 client.0 _ll_register_callbacks cb 0x564102ec1e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:44:19.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.991+0000 7f3cd7fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564102f9c100 0x564102fa6140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:19.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.991+0000 7f3cd7fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564102f9c100 0x564102fa6140 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58534/0 (socket says 172.21.15.99:58534) 2024-03-19T01:44:19.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.991+0000 7f3cd7fff640 1 -- 172.21.15.99:0/1692070197 learned_addr learned my addr 172.21.15.99:0/1692070197 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:44:19.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.991+0000 7f3cdeffd640 1 --2- 172.21.15.99:0/1692070197 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564102f9caf0 0x564102fa6680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:19.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.991+0000 7f3cd7fff640 1 --2- 172.21.15.99:0/1692070197 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564102fa1c60 0x564102fad6c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:19.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.991+0000 7f3cd7fff640 1 -- 172.21.15.99:0/1692070197 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564102f9caf0 msgr2=0x564102fa6680 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:44:20.000 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.991+0000 7f3cd7fff640 1 --2- 172.21.15.99:0/1692070197 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564102f9caf0 0x564102fa6680 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:20.000 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.991+0000 7f3cd7fff640 1 -- 172.21.15.99:0/1692070197 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564102fa1c60 msgr2=0x564102fad6c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:44:20.000 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.991+0000 7f3cd7fff640 1 --2- 172.21.15.99:0/1692070197 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564102fa1c60 0x564102fad6c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:20.000 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.991+0000 7f3cd7fff640 1 -- 172.21.15.99:0/1692070197 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x564102ebbc60 con 0x564102f9c100 2024-03-19T01:44:20.000 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.991+0000 7f3cdeffd640 1 --2- 172.21.15.99:0/1692070197 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564102f9caf0 0x564102fa6680 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:44:20.000 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.991+0000 7f3cd7fff640 1 --2- 172.21.15.99:0/1692070197 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564102f9c100 0x564102fa6140 secure :-1 s=READY pgs=418 cs=0 l=1 rev1=1 crypto rx=0x7f3cc800a010 tx=0x7f3cc80044d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9703dcb92112deec server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:44:20.000 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.991+0000 7f3cc67fc640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:44:20.000 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.991+0000 7f3cc67fc640 1 -- 172.21.15.99:0/1692070197 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3cc802d060 con 0x564102f9c100 2024-03-19T01:44:20.000 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.991+0000 7f3cc67fc640 1 -- 172.21.15.99:0/1692070197 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f3cc8004990 con 0x564102f9c100 2024-03-19T01:44:20.000 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.991+0000 7f3ce8282740 1 -- 172.21.15.99:0/1692070197 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f3cc8005500 con 0x564102f9c100 2024-03-19T01:44:20.000 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.991+0000 7f3cc67fc640 1 -- 172.21.15.99:0/1692070197 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=25) ==== 24+0+0 (secure 0 0 0) 0x7f3cc8004b90 con 0x564102f9c100 2024-03-19T01:44:20.000 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.992+0000 7f3ce8282740 10 client.15594 fetch_fsmap learned FSMap version 25 2024-03-19T01:44:20.000 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.992+0000 7f3ce8282740 1 -- 172.21.15.99:0/1692070197 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=25,monmap=4+}) -- 0x564102c15210 con 0x564102f9c100 2024-03-19T01:44:20.000 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.992+0000 7f3cc67fc640 1 -- 172.21.15.99:0/1692070197 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f3cc8005500 con 0x564102f9c100 2024-03-19T01:44:20.001 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.992+0000 7f3cc67fc640 1 -- 172.21.15.99:0/1692070197 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 25) ==== 8338+0+0 (secure 0 0 0) 0x7f3cc801f5d0 con 0x564102f9c100 2024-03-19T01:44:20.001 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.992+0000 7f3ce8282740 10 client.15594 fetch_fsmap finished waiting for FSMap version 25 2024-03-19T01:44:20.001 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.992+0000 7f3ce8282740 10 client.15594 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:44:20.001 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.992+0000 7f3ce8282740 20 client.15594 populate_metadata read hostname 'smithi099' 2024-03-19T01:44:20.001 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.992+0000 7f3ce8282740 1 --2- 172.21.15.99:0/1692070197 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x564102fb1cd0 0x564102fb40c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:20.001 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.992+0000 7f3ce8282740 4 client.15594 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:44:20.001 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.992+0000 7f3ce8282740 1 -- 172.21.15.99:0/1692070197 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x564102f0e120 con 0x564102fb1cd0 2024-03-19T01:44:20.001 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.992+0000 7f3cdeffd640 1 --2- 172.21.15.99:0/1692070197 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x564102fb1cd0 0x564102fb40c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:20.001 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.992+0000 7f3cdeffd640 1 --2- 172.21.15.99:0/1692070197 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x564102fb1cd0 0x564102fb40c0 crc :-1 s=READY pgs=57 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=73f30017dd4c0e1e server_cookie=e961e4288e11756 in_seq=0 out_seq=0 2024-03-19T01:44:20.001 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.992+0000 7f3cc67fc640 10 client.15594 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:44:20.001 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.993+0000 7f3cc67fc640 1 -- 172.21.15.99:0/1692070197 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x564102f0e120 con 0x564102fb1cd0 2024-03-19T01:44:20.001 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.993+0000 7f3cc67fc640 10 client.15594 handle_command_reply: tid=0 2024-03-19T01:44:20.001 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.993+0000 7f3ce8282740 1 client.15594 shutdown 2024-03-19T01:44:20.002 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.993+0000 7f3ce8282740 1 -- 172.21.15.99:0/1692070197 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564102f9c100 msgr2=0x564102fa6140 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:44:20.002 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.993+0000 7f3ce8282740 1 --2- 172.21.15.99:0/1692070197 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564102f9c100 0x564102fa6140 secure :-1 s=READY pgs=418 cs=0 l=1 rev1=1 crypto rx=0x7f3cc800a010 tx=0x7f3cc80044d0 comp rx=0 tx=0).stop 2024-03-19T01:44:20.002 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.993+0000 7f3ce8282740 1 -- 172.21.15.99:0/1692070197 shutdown_connections 2024-03-19T01:44:20.002 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.993+0000 7f3ce8282740 1 --2- 172.21.15.99:0/1692070197 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x564102fb1cd0 0x564102fb40c0 crc :-1 s=READY pgs=57 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:20.002 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.993+0000 7f3ce8282740 1 --2- 172.21.15.99:0/1692070197 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564102fa1c60 0x564102fad6c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:20.002 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.993+0000 7f3ce8282740 1 --2- 172.21.15.99:0/1692070197 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564102f9caf0 0x564102fa6680 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:20.002 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.993+0000 7f3cc67fc640 0 client.15594 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:44:20.002 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.993+0000 7f3ce8282740 1 --2- 172.21.15.99:0/1692070197 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564102f9c100 0x564102fa6140 unknown :-1 s=CLOSED pgs=418 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:20.002 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.993+0000 7f3ce8282740 1 -- 172.21.15.99:0/1692070197 >> 172.21.15.99:0/1692070197 conn(0x564102f97670 msgr2=0x564102f988c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:44:20.002 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.993+0000 7f3ce8282740 1 -- 172.21.15.99:0/1692070197 shutdown_connections 2024-03-19T01:44:20.002 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.993+0000 7f3ce8282740 1 -- 172.21.15.99:0/1692070197 wait complete. 2024-03-19T01:44:20.002 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:19.993+0000 7f3ce8282740 20 client.15594 trim_cache size 0 max 16384 2024-03-19T01:44:20.013 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.008+0000 7f3ce8282740 1 Processor -- start 2024-03-19T01:44:20.013 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.008+0000 7f3ce8282740 1 -- start start 2024-03-19T01:44:20.014 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.009+0000 7f3ce8282740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564102f9c100 0x564102f96220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:20.014 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.009+0000 7f3ce8282740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564102f9caf0 0x564102fad650 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:20.014 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.009+0000 7f3ce8282740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564102fa1c60 0x564102f96760 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:20.014 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.009+0000 7f3ce8282740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f3ce01ab640 con 0x564102f9c100 2024-03-19T01:44:20.014 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.009+0000 7f3ce8282740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f3ce01a8f10 con 0x564102f9caf0 2024-03-19T01:44:20.014 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.009+0000 7f3ce8282740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f3ce01a6d60 con 0x564102fa1c60 2024-03-19T01:44:20.014 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.009+0000 7f3cd7fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564102f9c100 0x564102f96220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:20.014 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.009+0000 7f3cd7fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564102f9c100 0x564102f96220 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58548/0 (socket says 172.21.15.99:58548) 2024-03-19T01:44:20.014 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.009+0000 7f3cdeffd640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564102f9caf0 0x564102fad650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:20.014 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.009+0000 7f3cd7fff640 1 -- 172.21.15.99:0/148476602 learned_addr learned my addr 172.21.15.99:0/148476602 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:44:20.014 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.009+0000 7f3cd7fff640 1 --2- 172.21.15.99:0/148476602 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564102fa1c60 0x564102f96760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:20.014 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.009+0000 7f3cd7fff640 1 -- 172.21.15.99:0/148476602 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564102fa1c60 msgr2=0x564102f96760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:44:20.014 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.009+0000 7f3cd7fff640 1 --2- 172.21.15.99:0/148476602 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564102fa1c60 0x564102f96760 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:20.015 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.009+0000 7f3cd7fff640 1 -- 172.21.15.99:0/148476602 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564102f9caf0 msgr2=0x564102fad650 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:44:20.015 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.009+0000 7f3cd7fff640 1 --2- 172.21.15.99:0/148476602 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564102f9caf0 0x564102fad650 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:20.015 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.009+0000 7f3cd7fff640 1 -- 172.21.15.99:0/148476602 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x564102ce1620 con 0x564102f9c100 2024-03-19T01:44:20.015 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.009+0000 7f3cdeffd640 1 --2- 172.21.15.99:0/148476602 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564102f9caf0 0x564102fad650 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:44:20.015 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.009+0000 7f3cd7fff640 1 --2- 172.21.15.99:0/148476602 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564102f9c100 0x564102f96220 secure :-1 s=READY pgs=419 cs=0 l=1 rev1=1 crypto rx=0x7f3cc8000c00 tx=0x7f3cc8005970 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7078e6d6e1147ac3 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:44:20.015 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.009+0000 7f3cd4ff9640 1 -- 172.21.15.99:0/148476602 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3cc8037030 con 0x564102f9c100 2024-03-19T01:44:20.015 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.009+0000 7f3cd4ff9640 1 -- 172.21.15.99:0/148476602 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f3cc8005470 con 0x564102f9c100 2024-03-19T01:44:20.015 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.009+0000 7f3cd4ff9640 1 -- 172.21.15.99:0/148476602 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3cc802d040 con 0x564102f9c100 2024-03-19T01:44:20.015 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.010+0000 7f3ce8282740 1 -- 172.21.15.99:0/148476602 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564102f9c100 msgr2=0x564102f96220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:44:20.015 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.010+0000 7f3ce8282740 1 --2- 172.21.15.99:0/148476602 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564102f9c100 0x564102f96220 secure :-1 s=READY pgs=419 cs=0 l=1 rev1=1 crypto rx=0x7f3cc8000c00 tx=0x7f3cc8005970 comp rx=0 tx=0).stop 2024-03-19T01:44:20.015 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.010+0000 7f3ce8282740 1 -- 172.21.15.99:0/148476602 shutdown_connections 2024-03-19T01:44:20.015 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.010+0000 7f3ce8282740 1 --2- 172.21.15.99:0/148476602 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564102f9caf0 0x564102fad650 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:20.015 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.010+0000 7f3ce8282740 1 --2- 172.21.15.99:0/148476602 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564102fa1c60 0x564102f96760 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:20.015 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.010+0000 7f3ce8282740 1 --2- 172.21.15.99:0/148476602 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564102f9c100 0x564102f96220 unknown :-1 s=CLOSED pgs=419 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:20.016 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.010+0000 7f3ce8282740 1 -- 172.21.15.99:0/148476602 >> 172.21.15.99:0/148476602 conn(0x564102f97670 msgr2=0x564102f9b160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:44:20.016 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.010+0000 7f3ce8282740 1 -- 172.21.15.99:0/148476602 shutdown_connections 2024-03-19T01:44:20.016 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.010+0000 7f3ce8282740 1 -- 172.21.15.99:0/148476602 wait complete. 2024-03-19T01:44:20.016 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.010+0000 7f3ce8282740 1 -- start start 2024-03-19T01:44:20.016 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.011+0000 7f3ce8282740 1 Processor -- start 2024-03-19T01:44:20.016 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.011+0000 7f3ce8282740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564102f9c100 0x564102faacb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:20.016 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.011+0000 7f3ce8282740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564102f9caf0 0x564102fab1f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:20.016 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.011+0000 7f3ce8282740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564102fa1c60 0x564102fac1b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:20.016 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.011+0000 7f3ce8282740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f3ce01a73d0 con 0x564102f9c100 2024-03-19T01:44:20.016 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.011+0000 7f3ce8282740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f3ce01a8840 con 0x564102f9caf0 2024-03-19T01:44:20.016 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.011+0000 7f3ce8282740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f3ce01a7aa0 con 0x564102fa1c60 2024-03-19T01:44:20.016 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.011+0000 7f3ce8282740 10 client.0 _ll_register_callbacks cb 0x7f3cd0009530 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:44:20.016 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.011+0000 7f3cd7fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564102f9c100 0x564102faacb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:20.016 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.011+0000 7f3cd7fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564102f9c100 0x564102faacb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58558/0 (socket says 172.21.15.99:58558) 2024-03-19T01:44:20.017 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.011+0000 7f3cd7fff640 1 -- 172.21.15.99:0/293304380 learned_addr learned my addr 172.21.15.99:0/293304380 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:44:20.017 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.011+0000 7f3cdeffd640 1 --2- 172.21.15.99:0/293304380 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564102f9caf0 0x564102fab1f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:20.017 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.011+0000 7f3cd7fff640 1 -- 172.21.15.99:0/293304380 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564102fa1c60 msgr2=0x564102fac1b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:44:20.017 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.011+0000 7f3cd7fff640 1 --2- 172.21.15.99:0/293304380 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564102fa1c60 0x564102fac1b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:20.017 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.011+0000 7f3cd7fff640 1 -- 172.21.15.99:0/293304380 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564102f9caf0 msgr2=0x564102fab1f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:44:20.017 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.011+0000 7f3cd7fff640 1 --2- 172.21.15.99:0/293304380 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564102f9caf0 0x564102fab1f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:20.017 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.011+0000 7f3cd7fff640 1 -- 172.21.15.99:0/293304380 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x564102fac7a0 con 0x564102f9c100 2024-03-19T01:44:20.017 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.011+0000 7f3cd7fff640 1 --2- 172.21.15.99:0/293304380 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564102f9c100 0x564102faacb0 secure :-1 s=READY pgs=420 cs=0 l=1 rev1=1 crypto rx=0x7f3cc800a010 tx=0x7f3cc8009e90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=75fcad78e7d0d13f server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:44:20.017 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.011+0000 7f3cc67fc640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:44:20.017 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.011+0000 7f3cc67fc640 1 -- 172.21.15.99:0/293304380 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3cc8041030 con 0x564102f9c100 2024-03-19T01:44:20.017 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.011+0000 7f3cc67fc640 1 -- 172.21.15.99:0/293304380 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f3cc8004930 con 0x564102f9c100 2024-03-19T01:44:20.017 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.011+0000 7f3ce8282740 1 -- 172.21.15.99:0/293304380 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f3cc8005470 con 0x564102f9c100 2024-03-19T01:44:20.017 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.012+0000 7f3cc67fc640 1 -- 172.21.15.99:0/293304380 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=25) ==== 24+0+0 (secure 0 0 0) 0x7f3cc8037070 con 0x564102f9c100 2024-03-19T01:44:20.018 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.012+0000 7f3ce8282740 10 client.15600 fetch_fsmap learned FSMap version 25 2024-03-19T01:44:20.018 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.012+0000 7f3ce8282740 1 -- 172.21.15.99:0/293304380 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=25,monmap=4+}) -- 0x564103017fb0 con 0x564102f9c100 2024-03-19T01:44:20.018 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.012+0000 7f3cc67fc640 1 -- 172.21.15.99:0/293304380 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f3cc8005470 con 0x564102f9c100 2024-03-19T01:44:20.018 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.012+0000 7f3cc67fc640 1 -- 172.21.15.99:0/293304380 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 25) ==== 8338+0+0 (secure 0 0 0) 0x7f3cc803c040 con 0x564102f9c100 2024-03-19T01:44:20.018 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.012+0000 7f3ce8282740 10 client.15600 fetch_fsmap finished waiting for FSMap version 25 2024-03-19T01:44:20.018 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.012+0000 7f3ce8282740 10 client.15600 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:44:20.018 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.012+0000 7f3ce8282740 20 client.15600 populate_metadata read hostname 'smithi099' 2024-03-19T01:44:20.018 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.012+0000 7f3ce8282740 1 --2- 172.21.15.99:0/293304380 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x564103018700 0x56410301ab20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:20.018 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.012+0000 7f3ce8282740 4 client.15600 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:44:20.018 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.012+0000 7f3ce8282740 1 -- 172.21.15.99:0/293304380 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x564102f08470 con 0x564103018700 2024-03-19T01:44:20.018 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.012+0000 7f3cdeffd640 1 --2- 172.21.15.99:0/293304380 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x564103018700 0x56410301ab20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:20.018 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.012+0000 7f3cc67fc640 10 client.15600 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:44:20.018 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.012+0000 7f3cdeffd640 1 --2- 172.21.15.99:0/293304380 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x564103018700 0x56410301ab20 crc :-1 s=READY pgs=58 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=8488f29fe3c01418 server_cookie=1675ad79b5c9ebf5 in_seq=0 out_seq=0 2024-03-19T01:44:20.081 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.080+0000 7f3cc67fc640 1 -- 172.21.15.99:0/293304380 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x564102f08470 con 0x564103018700 2024-03-19T01:44:20.081 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.080+0000 7f3cc67fc640 10 client.15600 handle_command_reply: tid=0 2024-03-19T01:44:20.081 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.080+0000 7f3ce8282740 1 client.15600 shutdown 2024-03-19T01:44:20.081 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.080+0000 7f3ce8282740 1 -- 172.21.15.99:0/293304380 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564102f9c100 msgr2=0x564102faacb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:44:20.081 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.080+0000 7f3ce8282740 1 --2- 172.21.15.99:0/293304380 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564102f9c100 0x564102faacb0 secure :-1 s=READY pgs=420 cs=0 l=1 rev1=1 crypto rx=0x7f3cc800a010 tx=0x7f3cc8009e90 comp rx=0 tx=0).stop 2024-03-19T01:44:20.081 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.080+0000 7f3ce8282740 1 -- 172.21.15.99:0/293304380 shutdown_connections 2024-03-19T01:44:20.081 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.080+0000 7f3ce8282740 1 --2- 172.21.15.99:0/293304380 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x564103018700 0x56410301ab20 crc :-1 s=READY pgs=58 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:20.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.080+0000 7f3ce8282740 1 --2- 172.21.15.99:0/293304380 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564102f9caf0 0x564102fab1f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:20.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.080+0000 7f3ce8282740 1 --2- 172.21.15.99:0/293304380 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564102fa1c60 0x564102fac1b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:20.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.080+0000 7f3cc67fc640 0 client.15600 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:44:20.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.080+0000 7f3ce8282740 1 --2- 172.21.15.99:0/293304380 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564102f9c100 0x564102faacb0 unknown :-1 s=CLOSED pgs=420 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:20.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.080+0000 7f3ce8282740 1 -- 172.21.15.99:0/293304380 >> 172.21.15.99:0/293304380 conn(0x564102f97670 msgr2=0x564102f9d590 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:44:20.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.080+0000 7f3ce8282740 1 -- 172.21.15.99:0/293304380 shutdown_connections 2024-03-19T01:44:20.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.080+0000 7f3ce8282740 1 -- 172.21.15.99:0/293304380 wait complete. 2024-03-19T01:44:20.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.080+0000 7f3ce8282740 20 client.15600 trim_cache size 0 max 16384 2024-03-19T01:44:20.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:44:20.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:44:20.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:44:20.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:44:20.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.081+0000 7f3ce5697640 1 -- 172.21.15.99:0/1891071460 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f3ccc0a28a0 msgr2=0x7f3ccc0a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:44:20.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.081+0000 7f3ce5697640 1 --2- 172.21.15.99:0/1891071460 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f3ccc0a28a0 0x7f3ccc0a4d60 secure :-1 s=READY pgs=262 cs=0 l=1 rev1=1 crypto rx=0x7f3cd80ee300 tx=0x7f3cd80323c0 comp rx=0 tx=0).stop 2024-03-19T01:44:20.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.081+0000 7f3ce5697640 1 -- 172.21.15.99:0/1891071460 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3ce01075c0 msgr2=0x7f3ce019f050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:44:20.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.081+0000 7f3ce5697640 1 --2- 172.21.15.99:0/1891071460 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3ce01075c0 0x7f3ce019f050 secure :-1 s=READY pgs=417 cs=0 l=1 rev1=1 crypto rx=0x7f3cd802c630 tx=0x7f3cd8032f20 comp rx=0 tx=0).stop 2024-03-19T01:44:20.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.081+0000 7f3ce5697640 1 -- 172.21.15.99:0/1891071460 shutdown_connections 2024-03-19T01:44:20.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.081+0000 7f3ce5697640 1 --2- 172.21.15.99:0/1891071460 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3ce01075c0 0x7f3ce019f050 unknown :-1 s=CLOSED pgs=417 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:20.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.081+0000 7f3ce5697640 1 --2- 172.21.15.99:0/1891071460 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f3ce0104c90 0x7f3ce0197fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:20.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.081+0000 7f3ce5697640 1 --2- 172.21.15.99:0/1891071460 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f3ccc0a28a0 0x7f3ccc0a4d60 unknown :-1 s=CLOSED pgs=262 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:20.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.081+0000 7f3ce5697640 1 --2- 172.21.15.99:0/1891071460 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3ce0102360 0x7f3ce0197aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:20.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.081+0000 7f3ce5697640 1 -- 172.21.15.99:0/1891071460 >> 172.21.15.99:0/1891071460 conn(0x7f3ce00fbcb0 msgr2=0x7f3ce0102cc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:44:20.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.081+0000 7f3ce5697640 1 -- 172.21.15.99:0/1891071460 shutdown_connections 2024-03-19T01:44:20.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:20.081+0000 7f3ce5697640 1 -- 172.21.15.99:0/1891071460 wait complete. 2024-03-19T01:44:20.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:44:19 smithi115 ceph-mon[67332]: pgmap v447: 97 pgs: 97 active+clean; 136 MiB data, 1.2 GiB used, 1.0 TiB / 1.0 TiB avail; 1.0 MiB/s rd, 2.8 MiB/s wr, 239 op/s 2024-03-19T01:44:20.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:44:19 smithi099 ceph-mon[62951]: pgmap v447: 97 pgs: 97 active+clean; 136 MiB data, 1.2 GiB used, 1.0 TiB / 1.0 TiB avail; 1.0 MiB/s rd, 2.8 MiB/s wr, 239 op/s 2024-03-19T01:44:20.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:44:19 smithi112 ceph-mon[66819]: pgmap v447: 97 pgs: 97 active+clean; 136 MiB data, 1.2 GiB used, 1.0 TiB / 1.0 TiB avail; 1.0 MiB/s rd, 2.8 MiB/s wr, 239 op/s 2024-03-19T01:44:20.679 INFO:tasks.workunit.client.0.smithi099.stdout: 1 369453 11.33 MB/sec execute 50 sec latency 58.491 ms 2024-03-19T01:44:21.679 INFO:tasks.workunit.client.0.smithi099.stdout: 1 372090 11.37 MB/sec execute 51 sec latency 28.045 ms 2024-03-19T01:44:22.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:44:21 smithi115 ceph-mon[67332]: pgmap v448: 97 pgs: 97 active+clean; 137 MiB data, 1.3 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 2.8 MiB/s wr, 245 op/s 2024-03-19T01:44:22.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:44:21 smithi099 ceph-mon[62951]: pgmap v448: 97 pgs: 97 active+clean; 137 MiB data, 1.3 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 2.8 MiB/s wr, 245 op/s 2024-03-19T01:44:22.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:44:21 smithi112 ceph-mon[66819]: pgmap v448: 97 pgs: 97 active+clean; 137 MiB data, 1.3 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 2.8 MiB/s wr, 245 op/s 2024-03-19T01:44:22.679 INFO:tasks.workunit.client.0.smithi099.stdout: 1 373919 11.36 MB/sec execute 52 sec latency 33.227 ms 2024-03-19T01:44:23.679 INFO:tasks.workunit.client.0.smithi099.stdout: 1 376389 11.41 MB/sec execute 53 sec latency 10.628 ms 2024-03-19T01:44:24.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:44:23 smithi115 ceph-mon[67332]: pgmap v449: 97 pgs: 97 active+clean; 136 MiB data, 1.3 GiB used, 1.0 TiB / 1.0 TiB avail; 1.3 MiB/s rd, 3.2 MiB/s wr, 307 op/s 2024-03-19T01:44:24.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:44:23 smithi099 ceph-mon[62951]: pgmap v449: 97 pgs: 97 active+clean; 136 MiB data, 1.3 GiB used, 1.0 TiB / 1.0 TiB avail; 1.3 MiB/s rd, 3.2 MiB/s wr, 307 op/s 2024-03-19T01:44:24.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:44:23 smithi112 ceph-mon[66819]: pgmap v449: 97 pgs: 97 active+clean; 136 MiB data, 1.3 GiB used, 1.0 TiB / 1.0 TiB avail; 1.3 MiB/s rd, 3.2 MiB/s wr, 307 op/s 2024-03-19T01:44:24.679 INFO:tasks.workunit.client.0.smithi099.stdout: 1 378595 11.35 MB/sec execute 54 sec latency 37.679 ms 2024-03-19T01:44:25.679 INFO:tasks.workunit.client.0.smithi099.stdout: 1 380317 11.35 MB/sec execute 55 sec latency 63.669 ms 2024-03-19T01:44:26.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:44:25 smithi115 ceph-mon[67332]: pgmap v450: 97 pgs: 97 active+clean; 135 MiB data, 1.3 GiB used, 1.0 TiB / 1.0 TiB avail; 1.0 MiB/s rd, 2.7 MiB/s wr, 241 op/s 2024-03-19T01:44:26.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:44:25 smithi099 ceph-mon[62951]: pgmap v450: 97 pgs: 97 active+clean; 135 MiB data, 1.3 GiB used, 1.0 TiB / 1.0 TiB avail; 1.0 MiB/s rd, 2.7 MiB/s wr, 241 op/s 2024-03-19T01:44:26.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:44:25 smithi112 ceph-mon[66819]: pgmap v450: 97 pgs: 97 active+clean; 135 MiB data, 1.3 GiB used, 1.0 TiB / 1.0 TiB avail; 1.0 MiB/s rd, 2.7 MiB/s wr, 241 op/s 2024-03-19T01:44:26.679 INFO:tasks.workunit.client.0.smithi099.stdout: 1 383189 11.46 MB/sec execute 56 sec latency 15.009 ms 2024-03-19T01:44:27.680 INFO:tasks.workunit.client.0.smithi099.stdout: 1 384666 11.37 MB/sec execute 57 sec latency 31.257 ms 2024-03-19T01:44:28.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:44:27 smithi115 ceph-mon[67332]: pgmap v451: 97 pgs: 97 active+clean; 135 MiB data, 1.3 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 3.1 MiB/s wr, 272 op/s 2024-03-19T01:44:28.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:44:27 smithi099 ceph-mon[62951]: pgmap v451: 97 pgs: 97 active+clean; 135 MiB data, 1.3 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 3.1 MiB/s wr, 272 op/s 2024-03-19T01:44:28.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:44:27 smithi112 ceph-mon[66819]: pgmap v451: 97 pgs: 97 active+clean; 135 MiB data, 1.3 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 3.1 MiB/s wr, 272 op/s 2024-03-19T01:44:28.680 INFO:tasks.workunit.client.0.smithi099.stdout: 1 387093 11.42 MB/sec execute 58 sec latency 9.939 ms 2024-03-19T01:44:29.680 INFO:tasks.workunit.client.0.smithi099.stdout: 1 389229 11.36 MB/sec execute 59 sec latency 28.904 ms 2024-03-19T01:44:30.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:44:29 smithi115 ceph-mon[67332]: pgmap v452: 97 pgs: 97 active+clean; 139 MiB data, 1.3 GiB used, 1.0 TiB / 1.0 TiB avail; 1.0 MiB/s rd, 2.8 MiB/s wr, 252 op/s 2024-03-19T01:44:30.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:44:29 smithi099 ceph-mon[62951]: pgmap v452: 97 pgs: 97 active+clean; 139 MiB data, 1.3 GiB used, 1.0 TiB / 1.0 TiB avail; 1.0 MiB/s rd, 2.8 MiB/s wr, 252 op/s 2024-03-19T01:44:30.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:44:29 smithi112 ceph-mon[66819]: pgmap v452: 97 pgs: 97 active+clean; 139 MiB data, 1.3 GiB used, 1.0 TiB / 1.0 TiB avail; 1.0 MiB/s rd, 2.8 MiB/s wr, 252 op/s 2024-03-19T01:44:30.680 INFO:tasks.workunit.client.0.smithi099.stdout: 1 390942 11.36 MB/sec execute 60 sec latency 41.729 ms 2024-03-19T01:44:31.680 INFO:tasks.workunit.client.0.smithi099.stdout: 1 393719 11.46 MB/sec execute 61 sec latency 14.019 ms 2024-03-19T01:44:32.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:44:31 smithi115 ceph-mon[67332]: pgmap v453: 97 pgs: 97 active+clean; 137 MiB data, 1.3 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 2.7 MiB/s wr, 259 op/s 2024-03-19T01:44:32.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:44:31 smithi099 ceph-mon[62951]: pgmap v453: 97 pgs: 97 active+clean; 137 MiB data, 1.3 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 2.7 MiB/s wr, 259 op/s 2024-03-19T01:44:32.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:44:31 smithi112 ceph-mon[66819]: pgmap v453: 97 pgs: 97 active+clean; 137 MiB data, 1.3 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 2.7 MiB/s wr, 259 op/s 2024-03-19T01:44:32.680 INFO:tasks.workunit.client.0.smithi099.stdout: 1 395297 11.38 MB/sec execute 62 sec latency 32.252 ms 2024-03-19T01:44:33.175 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.173+0000 7f10fc2a1640 1 -- 172.21.15.99:0/1449141331 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f10f41022f0 msgr2=0x7f10f4108b80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:44:33.175 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.173+0000 7f10fc2a1640 1 --2- 172.21.15.99:0/1449141331 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f10f41022f0 0x7f10f4108b80 secure :-1 s=READY pgs=421 cs=0 l=1 rev1=1 crypto rx=0x7f10ec009ca0 tx=0x7f10ec02c0f0 comp rx=0 tx=0).stop 2024-03-19T01:44:33.175 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.173+0000 7f10fc2a1640 1 -- 172.21.15.99:0/1449141331 shutdown_connections 2024-03-19T01:44:33.175 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.173+0000 7f10fc2a1640 1 --2- 172.21.15.99:0/1449141331 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f10f41022f0 0x7f10f4108b80 unknown :-1 s=CLOSED pgs=421 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:33.175 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.173+0000 7f10fc2a1640 1 --2- 172.21.15.99:0/1449141331 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f10f4101930 0x7f10f4101db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:33.175 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.173+0000 7f10fc2a1640 1 --2- 172.21.15.99:0/1449141331 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f10f4100740 0x7f10f4100b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:33.176 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.173+0000 7f10fc2a1640 1 -- 172.21.15.99:0/1449141331 >> 172.21.15.99:0/1449141331 conn(0x7f10f40fbcb0 msgr2=0x7f10f40fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:44:33.176 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.173+0000 7f10fc2a1640 1 -- 172.21.15.99:0/1449141331 shutdown_connections 2024-03-19T01:44:33.176 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.173+0000 7f10fc2a1640 1 -- 172.21.15.99:0/1449141331 wait complete. 2024-03-19T01:44:33.176 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.174+0000 7f10fc2a1640 1 Processor -- start 2024-03-19T01:44:33.176 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.174+0000 7f10fc2a1640 1 -- start start 2024-03-19T01:44:33.176 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.174+0000 7f10fc2a1640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f10f4100740 0x7f10f4197a80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:33.176 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.174+0000 7f10fc2a1640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f10f4101930 0x7f10f4197fc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:33.176 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.174+0000 7f10fc2a1640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f10f41022f0 0x7f10f419f030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:33.176 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.174+0000 7f10fc2a1640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f10f410c3c0 con 0x7f10f41022f0 2024-03-19T01:44:33.176 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.174+0000 7f10fc2a1640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f10f410c240 con 0x7f10f4100740 2024-03-19T01:44:33.176 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.174+0000 7f10fc2a1640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f10f410c540 con 0x7f10f4101930 2024-03-19T01:44:33.176 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.174+0000 7f10fa83e640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f10f41022f0 0x7f10f419f030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:33.176 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.174+0000 7f10fa83e640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f10f41022f0 0x7f10f419f030 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:48588/0 (socket says 172.21.15.99:48588) 2024-03-19T01:44:33.177 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.174+0000 7f10fa03d640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f10f4100740 0x7f10f4197a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:33.177 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.174+0000 7f10fa83e640 1 -- 172.21.15.99:0/730274323 learned_addr learned my addr 172.21.15.99:0/730274323 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:44:33.177 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.174+0000 7f10f983c640 1 --2- 172.21.15.99:0/730274323 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f10f4101930 0x7f10f4197fc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:33.177 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.174+0000 7f10fa83e640 1 -- 172.21.15.99:0/730274323 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f10f4101930 msgr2=0x7f10f4197fc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:44:33.177 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.174+0000 7f10fa83e640 1 --2- 172.21.15.99:0/730274323 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f10f4101930 0x7f10f4197fc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:33.177 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.174+0000 7f10fa83e640 1 -- 172.21.15.99:0/730274323 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f10f4100740 msgr2=0x7f10f4197a80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:44:33.177 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.174+0000 7f10fa83e640 1 --2- 172.21.15.99:0/730274323 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f10f4100740 0x7f10f4197a80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:33.177 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.174+0000 7f10fa83e640 1 -- 172.21.15.99:0/730274323 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f10f419f6a0 con 0x7f10f41022f0 2024-03-19T01:44:33.177 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.174+0000 7f10fa03d640 1 --2- 172.21.15.99:0/730274323 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f10f4100740 0x7f10f4197a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:44:33.177 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.175+0000 7f10fa83e640 1 --2- 172.21.15.99:0/730274323 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f10f41022f0 0x7f10f419f030 secure :-1 s=READY pgs=422 cs=0 l=1 rev1=1 crypto rx=0x7f10ec037040 tx=0x7f10ec032f40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:44:33.177 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.175+0000 7f10eb7fe640 1 -- 172.21.15.99:0/730274323 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f10ec041020 con 0x7f10f41022f0 2024-03-19T01:44:33.177 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.175+0000 7f10eb7fe640 1 -- 172.21.15.99:0/730274323 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f10ec034cd0 con 0x7f10f41022f0 2024-03-19T01:44:33.178 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.175+0000 7f10fc2a1640 1 -- 172.21.15.99:0/730274323 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f10f419f930 con 0x7f10f41022f0 2024-03-19T01:44:33.178 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.175+0000 7f10eb7fe640 1 -- 172.21.15.99:0/730274323 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f10ec039040 con 0x7f10f41022f0 2024-03-19T01:44:33.178 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.175+0000 7f10fc2a1640 1 -- 172.21.15.99:0/730274323 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f10f419fe70 con 0x7f10f41022f0 2024-03-19T01:44:33.184 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.178+0000 7f10eb7fe640 1 -- 172.21.15.99:0/730274323 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f10ec044020 con 0x7f10f41022f0 2024-03-19T01:44:33.185 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.178+0000 7f10fef6b740 1 Processor -- start 2024-03-19T01:44:33.185 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.178+0000 7f10fef6b740 1 -- start start 2024-03-19T01:44:33.185 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.178+0000 7f10fef6b740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e428afe510 0x55e428afe930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:33.185 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.178+0000 7f10fef6b740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e428afef00 0x55e428b03a00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:33.185 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.178+0000 7f10fef6b740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e428b04070 0x55e428b06500 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:33.185 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.178+0000 7f10fef6b740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55e428a5ebe0 con 0x55e428afef00 2024-03-19T01:44:33.185 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.178+0000 7f10fef6b740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55e428a07960 con 0x55e428b04070 2024-03-19T01:44:33.185 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.178+0000 7f10fef6b740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55e428a5ac80 con 0x55e428afe510 2024-03-19T01:44:33.185 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.178+0000 7f10eb7fe640 1 --2- 172.21.15.99:0/730274323 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f10e00a2920 0x7f10e00a4de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:33.185 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.178+0000 7f10f983c640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e428afe510 0x55e428afe930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:33.185 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.178+0000 7f10fa03d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e428afef00 0x55e428b03a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:33.185 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.178+0000 7f10f983c640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e428afe510 0x55e428afe930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:43402/0 (socket says 172.21.15.99:43402) 2024-03-19T01:44:33.186 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.178+0000 7f10fa03d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e428afef00 0x55e428b03a00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:48594/0 (socket says 172.21.15.99:48594) 2024-03-19T01:44:33.186 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.178+0000 7f10f983c640 1 -- 172.21.15.99:0/2722070515 learned_addr learned my addr 172.21.15.99:0/2722070515 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:44:33.186 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.178+0000 7f10f983c640 1 --2- 172.21.15.99:0/2722070515 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e428b04070 0x55e428b06500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:33.186 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.178+0000 7f10fa83e640 1 --2- 172.21.15.99:0/730274323 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f10e00a2920 0x7f10e00a4de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:33.186 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.178+0000 7f10eb7fe640 1 -- 172.21.15.99:0/730274323 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f10ec0f08c0 con 0x7f10f41022f0 2024-03-19T01:44:33.186 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.178+0000 7f10f983c640 1 -- 172.21.15.99:0/2722070515 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e428b04070 msgr2=0x55e428b06500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:44:33.186 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.178+0000 7f10f983c640 1 --2- 172.21.15.99:0/2722070515 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e428b04070 0x55e428b06500 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:33.186 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.178+0000 7f10f983c640 1 -- 172.21.15.99:0/2722070515 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e428afef00 msgr2=0x55e428b03a00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:44:33.186 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.178+0000 7f10f983c640 1 --2- 172.21.15.99:0/2722070515 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e428afef00 0x55e428b03a00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:33.186 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.178+0000 7f10f983c640 1 -- 172.21.15.99:0/2722070515 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55e4289fed20 con 0x55e428afe510 2024-03-19T01:44:33.186 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.178+0000 7f10fa03d640 1 --2- 172.21.15.99:0/2722070515 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e428afef00 0x55e428b03a00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:44:33.186 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.178+0000 7f10f983c640 1 --2- 172.21.15.99:0/2722070515 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e428afe510 0x55e428afe930 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 crypto rx=0x7f10e4011810 tx=0x7f10e40097d0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=5935e944dff51f0d server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:44:33.187 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.178+0000 7f10fa83e640 1 --2- 172.21.15.99:0/730274323 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f10e00a2920 0x7f10e00a4de0 secure :-1 s=READY pgs=263 cs=0 l=1 rev1=1 crypto rx=0x7f10ec0bd240 tx=0x7f10ec088070 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:44:33.187 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.179+0000 7f10e8bfe640 1 -- 172.21.15.99:0/2722070515 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f10e4004030 con 0x55e428afe510 2024-03-19T01:44:33.187 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.179+0000 7f10e8bfe640 1 -- 172.21.15.99:0/2722070515 <== mon.1 v2:172.21.15.115:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f10e4011e50 con 0x55e428afe510 2024-03-19T01:44:33.187 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.179+0000 7f10e8bfe640 1 -- 172.21.15.99:0/2722070515 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f10e40037f0 con 0x55e428afe510 2024-03-19T01:44:33.187 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.179+0000 7f10fef6b740 1 -- 172.21.15.99:0/2722070515 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e428afe510 msgr2=0x55e428afe930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:44:33.187 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.179+0000 7f10fef6b740 1 --2- 172.21.15.99:0/2722070515 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e428afe510 0x55e428afe930 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 crypto rx=0x7f10e4011810 tx=0x7f10e40097d0 comp rx=0 tx=0).stop 2024-03-19T01:44:33.187 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.179+0000 7f10fef6b740 1 -- 172.21.15.99:0/2722070515 shutdown_connections 2024-03-19T01:44:33.187 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.179+0000 7f10fef6b740 1 --2- 172.21.15.99:0/2722070515 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e428b04070 0x55e428b06500 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:33.187 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.179+0000 7f10fef6b740 1 --2- 172.21.15.99:0/2722070515 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e428afef00 0x55e428b03a00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:33.187 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.179+0000 7f10fef6b740 1 --2- 172.21.15.99:0/2722070515 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e428afe510 0x55e428afe930 unknown :-1 s=CLOSED pgs=83 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:33.187 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.179+0000 7f10fef6b740 1 -- 172.21.15.99:0/2722070515 >> 172.21.15.99:0/2722070515 conn(0x55e428af5670 msgr2=0x55e428af5a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:44:33.187 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.179+0000 7f10fef6b740 1 -- 172.21.15.99:0/2722070515 shutdown_connections 2024-03-19T01:44:33.187 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.179+0000 7f10fef6b740 1 -- 172.21.15.99:0/2722070515 wait complete. 2024-03-19T01:44:33.188 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.180+0000 7f10fef6b740 1 -- start start 2024-03-19T01:44:33.188 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.180+0000 7f10fef6b740 1 Processor -- start 2024-03-19T01:44:33.188 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.180+0000 7f10fef6b740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e428afe510 0x55e428b08520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:33.188 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.180+0000 7f10fef6b740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e428afef00 0x55e428b08a60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:33.188 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.180+0000 7f10fef6b740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e428b04070 0x55e428b0faa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:33.188 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.180+0000 7f10fef6b740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55e428b07020 con 0x55e428afe510 2024-03-19T01:44:33.188 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.180+0000 7f10fef6b740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55e428a1adb0 con 0x55e428b04070 2024-03-19T01:44:33.188 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.180+0000 7f10fef6b740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55e428b071a0 con 0x55e428afef00 2024-03-19T01:44:33.188 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.180+0000 7f10fef6b740 10 client.0 _ll_register_callbacks cb 0x55e428a1fe70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:44:33.188 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.180+0000 7f10f983c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e428afe510 0x55e428b08520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:33.188 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.180+0000 7f10f983c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e428afe510 0x55e428b08520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:48598/0 (socket says 172.21.15.99:48598) 2024-03-19T01:44:33.188 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.180+0000 7f10f983c640 1 -- 172.21.15.99:0/27840810 learned_addr learned my addr 172.21.15.99:0/27840810 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:44:33.188 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.180+0000 7f10fa03d640 1 --2- 172.21.15.99:0/27840810 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e428afef00 0x55e428b08a60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:33.189 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.180+0000 7f10f983c640 1 --2- 172.21.15.99:0/27840810 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e428b04070 0x55e428b0faa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:33.189 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.180+0000 7f10f983c640 1 -- 172.21.15.99:0/27840810 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e428afef00 msgr2=0x55e428b08a60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:44:33.189 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.180+0000 7f10f983c640 1 --2- 172.21.15.99:0/27840810 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e428afef00 0x55e428b08a60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:33.189 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.180+0000 7f10f983c640 1 -- 172.21.15.99:0/27840810 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e428b04070 msgr2=0x55e428b0faa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:44:33.189 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.180+0000 7f10f983c640 1 --2- 172.21.15.99:0/27840810 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e428b04070 0x55e428b0faa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:33.189 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.180+0000 7f10f983c640 1 -- 172.21.15.99:0/27840810 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55e428a19c60 con 0x55e428afe510 2024-03-19T01:44:33.189 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.181+0000 7f10f983c640 1 --2- 172.21.15.99:0/27840810 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e428afe510 0x55e428b08520 secure :-1 s=READY pgs=423 cs=0 l=1 rev1=1 crypto rx=0x7f10e40117e0 tx=0x7f10e4005f90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=29f74780ab121afd server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:44:33.189 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.181+0000 7f10ddffb640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:44:33.189 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.181+0000 7f10ddffb640 1 -- 172.21.15.99:0/27840810 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f10e4005050 con 0x55e428afe510 2024-03-19T01:44:33.189 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.181+0000 7f10ddffb640 1 -- 172.21.15.99:0/27840810 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f10e40051f0 con 0x55e428afe510 2024-03-19T01:44:33.189 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.181+0000 7f10fef6b740 1 -- 172.21.15.99:0/27840810 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f10e4011e50 con 0x55e428afe510 2024-03-19T01:44:33.189 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.181+0000 7f10ddffb640 1 -- 172.21.15.99:0/27840810 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=25) ==== 24+0+0 (secure 0 0 0) 0x7f10e4034050 con 0x55e428afe510 2024-03-19T01:44:33.189 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.181+0000 7f10fef6b740 10 client.15612 fetch_fsmap learned FSMap version 25 2024-03-19T01:44:33.190 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.181+0000 7f10fef6b740 1 -- 172.21.15.99:0/27840810 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=25,monmap=4+}) -- 0x55e428773210 con 0x55e428afe510 2024-03-19T01:44:33.190 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.181+0000 7f10ddffb640 1 -- 172.21.15.99:0/27840810 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f10e4011e50 con 0x55e428afe510 2024-03-19T01:44:33.190 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.181+0000 7f10ddffb640 1 -- 172.21.15.99:0/27840810 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 25) ==== 8338+0+0 (secure 0 0 0) 0x7f10e401f3f0 con 0x55e428afe510 2024-03-19T01:44:33.190 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.181+0000 7f10fef6b740 10 client.15612 fetch_fsmap finished waiting for FSMap version 25 2024-03-19T01:44:33.190 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.181+0000 7f10fef6b740 10 client.15612 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:44:33.190 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.181+0000 7f10fef6b740 20 client.15612 populate_metadata read hostname 'smithi099' 2024-03-19T01:44:33.190 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.181+0000 7f10fef6b740 1 --2- 172.21.15.99:0/27840810 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e428b14130 0x55e428b16540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:33.190 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.181+0000 7f10fef6b740 4 client.15612 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:44:33.190 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.181+0000 7f10fef6b740 1 -- 172.21.15.99:0/27840810 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55e428a6c120 con 0x55e428b14130 2024-03-19T01:44:33.190 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.181+0000 7f10fa03d640 1 --2- 172.21.15.99:0/27840810 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e428b14130 0x55e428b16540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:33.190 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.182+0000 7f10fa03d640 1 --2- 172.21.15.99:0/27840810 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e428b14130 0x55e428b16540 crc :-1 s=READY pgs=59 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=cd6b6e3d5a7ab97a server_cookie=912e6719362df8b0 in_seq=0 out_seq=0 2024-03-19T01:44:33.190 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.182+0000 7f10ddffb640 10 client.15612 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:44:33.190 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.183+0000 7f10ddffb640 1 -- 172.21.15.99:0/27840810 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x55e428a6c120 con 0x55e428b14130 2024-03-19T01:44:33.190 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.183+0000 7f10ddffb640 10 client.15612 handle_command_reply: tid=0 2024-03-19T01:44:33.191 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.183+0000 7f10fef6b740 1 client.15612 shutdown 2024-03-19T01:44:33.191 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.183+0000 7f10fef6b740 1 -- 172.21.15.99:0/27840810 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e428afe510 msgr2=0x55e428b08520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:44:33.191 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.183+0000 7f10fef6b740 1 --2- 172.21.15.99:0/27840810 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e428afe510 0x55e428b08520 secure :-1 s=READY pgs=423 cs=0 l=1 rev1=1 crypto rx=0x7f10e40117e0 tx=0x7f10e4005f90 comp rx=0 tx=0).stop 2024-03-19T01:44:33.191 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.183+0000 7f10fef6b740 1 -- 172.21.15.99:0/27840810 shutdown_connections 2024-03-19T01:44:33.191 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.183+0000 7f10fef6b740 1 --2- 172.21.15.99:0/27840810 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e428b14130 0x55e428b16540 crc :-1 s=READY pgs=59 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:33.191 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.183+0000 7f10fef6b740 1 --2- 172.21.15.99:0/27840810 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e428b04070 0x55e428b0faa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:33.191 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.183+0000 7f10ddffb640 0 client.15612 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:44:33.191 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.183+0000 7f10fef6b740 1 --2- 172.21.15.99:0/27840810 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e428afef00 0x55e428b08a60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:33.191 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.183+0000 7f10fef6b740 1 --2- 172.21.15.99:0/27840810 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e428afe510 0x55e428b08520 unknown :-1 s=CLOSED pgs=423 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:33.191 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.183+0000 7f10fef6b740 1 -- 172.21.15.99:0/27840810 >> 172.21.15.99:0/27840810 conn(0x55e428af5670 msgr2=0x55e428af68c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:44:33.191 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.183+0000 7f10fef6b740 1 -- 172.21.15.99:0/27840810 shutdown_connections 2024-03-19T01:44:33.191 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.183+0000 7f10fef6b740 1 -- 172.21.15.99:0/27840810 wait complete. 2024-03-19T01:44:33.191 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.183+0000 7f10fef6b740 20 client.15612 trim_cache size 0 max 16384 2024-03-19T01:44:33.205 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.199+0000 7f10fef6b740 1 Processor -- start 2024-03-19T01:44:33.205 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.199+0000 7f10fef6b740 1 -- start start 2024-03-19T01:44:33.206 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.199+0000 7f10fef6b740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e428afe510 0x55e428b08840 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:33.206 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.199+0000 7f10fef6b740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e428afef00 0x55e428b0fa60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:33.206 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.199+0000 7f10fef6b740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e428b04070 0x55e428aff9a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:33.206 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.199+0000 7f10fef6b740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f10f41a9410 con 0x55e428b04070 2024-03-19T01:44:33.206 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.199+0000 7f10fef6b740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f10f41a6ce0 con 0x55e428afe510 2024-03-19T01:44:33.206 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.199+0000 7f10fef6b740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f10f41a4b30 con 0x55e428afef00 2024-03-19T01:44:33.206 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.199+0000 7f10f983c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e428b04070 0x55e428aff9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:33.206 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.199+0000 7f10f983c640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e428afe510 0x55e428b08840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:33.206 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.199+0000 7f10f983c640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e428afe510 0x55e428b08840 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:58536/0 (socket says 172.21.15.99:58536) 2024-03-19T01:44:33.206 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.199+0000 7f10f983c640 1 -- 172.21.15.99:0/1612984138 learned_addr learned my addr 172.21.15.99:0/1612984138 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:44:33.206 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.199+0000 7f10fa03d640 1 --2- 172.21.15.99:0/1612984138 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e428afef00 0x55e428b0fa60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:33.206 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.200+0000 7f10f983c640 1 -- 172.21.15.99:0/1612984138 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e428afef00 msgr2=0x55e428b0fa60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:44:33.206 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.200+0000 7f10f983c640 1 --2- 172.21.15.99:0/1612984138 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e428afef00 0x55e428b0fa60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:33.206 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.200+0000 7f10f983c640 1 -- 172.21.15.99:0/1612984138 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e428b04070 msgr2=0x55e428aff9a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:44:33.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.200+0000 7f10f983c640 1 --2- 172.21.15.99:0/1612984138 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e428b04070 0x55e428aff9a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:33.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.200+0000 7f10f983c640 1 -- 172.21.15.99:0/1612984138 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55e42883f620 con 0x55e428afe510 2024-03-19T01:44:33.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.200+0000 7f10f983c640 1 --2- 172.21.15.99:0/1612984138 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e428afe510 0x55e428b08840 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 crypto rx=0x7f10e401f7f0 tx=0x7f10e4009d20 comp rx=0 tx=0).ready entity=mon.2 client_cookie=bc006ac60ec4fd44 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:44:33.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.200+0000 7f10dffff640 1 -- 172.21.15.99:0/1612984138 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f10e4031020 con 0x55e428afe510 2024-03-19T01:44:33.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.200+0000 7f10dffff640 1 -- 172.21.15.99:0/1612984138 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f10e4011e50 con 0x55e428afe510 2024-03-19T01:44:33.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.200+0000 7f10dffff640 1 -- 172.21.15.99:0/1612984138 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f10e4023040 con 0x55e428afe510 2024-03-19T01:44:33.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.201+0000 7f10fef6b740 1 -- 172.21.15.99:0/1612984138 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e428afe510 msgr2=0x55e428b08840 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:44:33.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.201+0000 7f10fef6b740 1 --2- 172.21.15.99:0/1612984138 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e428afe510 0x55e428b08840 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 crypto rx=0x7f10e401f7f0 tx=0x7f10e4009d20 comp rx=0 tx=0).stop 2024-03-19T01:44:33.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.201+0000 7f10fef6b740 1 -- 172.21.15.99:0/1612984138 shutdown_connections 2024-03-19T01:44:33.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.201+0000 7f10fef6b740 1 --2- 172.21.15.99:0/1612984138 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e428b04070 0x55e428aff9a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:33.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.201+0000 7f10fef6b740 1 --2- 172.21.15.99:0/1612984138 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e428afef00 0x55e428b0fa60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:33.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.201+0000 7f10fef6b740 1 --2- 172.21.15.99:0/1612984138 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e428afe510 0x55e428b08840 unknown :-1 s=CLOSED pgs=104 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:33.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.201+0000 7f10fef6b740 1 -- 172.21.15.99:0/1612984138 >> 172.21.15.99:0/1612984138 conn(0x55e428af5670 msgr2=0x55e428b08130 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:44:33.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.201+0000 7f10fef6b740 1 -- 172.21.15.99:0/1612984138 shutdown_connections 2024-03-19T01:44:33.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.201+0000 7f10fef6b740 1 -- 172.21.15.99:0/1612984138 wait complete. 2024-03-19T01:44:33.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.201+0000 7f10fef6b740 1 -- start start 2024-03-19T01:44:33.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.201+0000 7f10fef6b740 1 Processor -- start 2024-03-19T01:44:33.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.201+0000 7f10fef6b740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e428afe510 0x55e428b0c9d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:33.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.201+0000 7f10fef6b740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e428afef00 0x55e428b0cf10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:33.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.201+0000 7f10fef6b740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e428b04070 0x55e428b78420 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:33.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.201+0000 7f10fef6b740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f10f41a51a0 con 0x55e428afef00 2024-03-19T01:44:33.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.201+0000 7f10fef6b740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f10f41a6610 con 0x55e428afe510 2024-03-19T01:44:33.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.201+0000 7f10fef6b740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f10f41a5870 con 0x55e428b04070 2024-03-19T01:44:33.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.201+0000 7f10fef6b740 10 client.0 _ll_register_callbacks cb 0x7f10f0009b50 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:44:33.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.201+0000 7f10fa03d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e428afef00 0x55e428b0cf10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:33.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.201+0000 7f10fa03d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e428afef00 0x55e428b0cf10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:48620/0 (socket says 172.21.15.99:48620) 2024-03-19T01:44:33.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.201+0000 7f10fa03d640 1 -- 172.21.15.99:0/2859309005 learned_addr learned my addr 172.21.15.99:0/2859309005 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:44:33.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.201+0000 7f10f983c640 1 --2- 172.21.15.99:0/2859309005 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e428afe510 0x55e428b0c9d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:33.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.202+0000 7f10f983c640 1 --2- 172.21.15.99:0/2859309005 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e428b04070 0x55e428b78420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:33.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.202+0000 7f10fa03d640 1 -- 172.21.15.99:0/2859309005 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e428b04070 msgr2=0x55e428b78420 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:44:33.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.202+0000 7f10fa03d640 1 --2- 172.21.15.99:0/2859309005 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e428b04070 0x55e428b78420 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:33.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.202+0000 7f10fa03d640 1 -- 172.21.15.99:0/2859309005 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e428afe510 msgr2=0x55e428b0c9d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:44:33.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.202+0000 7f10fa03d640 1 --2- 172.21.15.99:0/2859309005 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e428afe510 0x55e428b0c9d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:33.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.202+0000 7f10fa03d640 1 -- 172.21.15.99:0/2859309005 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55e428b78960 con 0x55e428afef00 2024-03-19T01:44:33.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.202+0000 7f10fa03d640 1 --2- 172.21.15.99:0/2859309005 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e428afef00 0x55e428b0cf10 secure :-1 s=READY pgs=424 cs=0 l=1 rev1=1 crypto rx=0x7f10f0004700 tx=0x7f10f0012b80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=880604c4e6308704 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:44:33.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.202+0000 7f10ddffb640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:44:33.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.202+0000 7f10ddffb640 1 -- 172.21.15.99:0/2859309005 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f10f00216f0 con 0x55e428afef00 2024-03-19T01:44:33.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.202+0000 7f10ddffb640 1 -- 172.21.15.99:0/2859309005 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f10f0004030 con 0x55e428afef00 2024-03-19T01:44:33.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.202+0000 7f10fef6b740 1 -- 172.21.15.99:0/2859309005 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f10e4011e50 con 0x55e428afef00 2024-03-19T01:44:33.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.202+0000 7f10ddffb640 1 -- 172.21.15.99:0/2859309005 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=25) ==== 24+0+0 (secure 0 0 0) 0x7f10f0004230 con 0x55e428afef00 2024-03-19T01:44:33.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.202+0000 7f10fef6b740 10 client.15618 fetch_fsmap learned FSMap version 25 2024-03-19T01:44:33.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.202+0000 7f10fef6b740 1 -- 172.21.15.99:0/2859309005 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=25,monmap=4+}) -- 0x55e428b79d10 con 0x55e428afef00 2024-03-19T01:44:33.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.202+0000 7f10ddffb640 1 -- 172.21.15.99:0/2859309005 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f10e4011e50 con 0x55e428afef00 2024-03-19T01:44:33.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.202+0000 7f10ddffb640 1 -- 172.21.15.99:0/2859309005 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 25) ==== 8338+0+0 (secure 0 0 0) 0x7f10f002a480 con 0x55e428afef00 2024-03-19T01:44:33.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.203+0000 7f10fef6b740 10 client.15618 fetch_fsmap finished waiting for FSMap version 25 2024-03-19T01:44:33.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.203+0000 7f10fef6b740 10 client.15618 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:44:33.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.203+0000 7f10fef6b740 20 client.15618 populate_metadata read hostname 'smithi099' 2024-03-19T01:44:33.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.203+0000 7f10fef6b740 1 --2- 172.21.15.99:0/2859309005 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e428b7a6a0 0x55e428b7cac0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:33.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.203+0000 7f10fef6b740 4 client.15618 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:44:33.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.203+0000 7f10fef6b740 1 -- 172.21.15.99:0/2859309005 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55e428a66470 con 0x55e428b7a6a0 2024-03-19T01:44:33.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.203+0000 7f10f983c640 1 --2- 172.21.15.99:0/2859309005 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e428b7a6a0 0x55e428b7cac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:33.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.203+0000 7f10ddffb640 10 client.15618 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:44:33.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.203+0000 7f10f983c640 1 --2- 172.21.15.99:0/2859309005 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e428b7a6a0 0x55e428b7cac0 crc :-1 s=READY pgs=60 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=c12a7ecd952f2688 server_cookie=811346a3acba65b4 in_seq=0 out_seq=0 2024-03-19T01:44:33.257 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.255+0000 7f10ddffb640 1 -- 172.21.15.99:0/2859309005 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x55e428a66470 con 0x55e428b7a6a0 2024-03-19T01:44:33.257 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.255+0000 7f10ddffb640 10 client.15618 handle_command_reply: tid=0 2024-03-19T01:44:33.257 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.255+0000 7f10fef6b740 1 client.15618 shutdown 2024-03-19T01:44:33.257 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.255+0000 7f10fef6b740 1 -- 172.21.15.99:0/2859309005 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e428afef00 msgr2=0x55e428b0cf10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:44:33.257 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.256+0000 7f10fef6b740 1 --2- 172.21.15.99:0/2859309005 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e428afef00 0x55e428b0cf10 secure :-1 s=READY pgs=424 cs=0 l=1 rev1=1 crypto rx=0x7f10f0004700 tx=0x7f10f0012b80 comp rx=0 tx=0).stop 2024-03-19T01:44:33.257 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.256+0000 7f10fef6b740 1 -- 172.21.15.99:0/2859309005 shutdown_connections 2024-03-19T01:44:33.257 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.256+0000 7f10fef6b740 1 --2- 172.21.15.99:0/2859309005 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e428b7a6a0 0x55e428b7cac0 crc :-1 s=READY pgs=60 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:33.257 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.256+0000 7f10fef6b740 1 --2- 172.21.15.99:0/2859309005 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e428b04070 0x55e428b78420 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:33.257 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.256+0000 7f10fef6b740 1 --2- 172.21.15.99:0/2859309005 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e428afef00 0x55e428b0cf10 unknown :-1 s=CLOSED pgs=424 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:33.257 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.256+0000 7f10fef6b740 1 --2- 172.21.15.99:0/2859309005 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e428afe510 0x55e428b0c9d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:33.258 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.256+0000 7f10fef6b740 1 -- 172.21.15.99:0/2859309005 >> 172.21.15.99:0/2859309005 conn(0x55e428af5670 msgr2=0x55e428aff9a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:44:33.258 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.256+0000 7f10ddffb640 0 client.15618 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:44:33.258 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.256+0000 7f10fef6b740 1 -- 172.21.15.99:0/2859309005 shutdown_connections 2024-03-19T01:44:33.258 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.256+0000 7f10fef6b740 1 -- 172.21.15.99:0/2859309005 wait complete. 2024-03-19T01:44:33.258 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.256+0000 7f10fef6b740 20 client.15618 trim_cache size 0 max 16384 2024-03-19T01:44:33.258 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:44:33.258 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:44:33.258 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:44:33.258 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:44:33.259 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.256+0000 7f10fc2a1640 1 -- 172.21.15.99:0/730274323 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f10e00a2920 msgr2=0x7f10e00a4de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:44:33.259 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.256+0000 7f10fc2a1640 1 --2- 172.21.15.99:0/730274323 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f10e00a2920 0x7f10e00a4de0 secure :-1 s=READY pgs=263 cs=0 l=1 rev1=1 crypto rx=0x7f10ec0bd240 tx=0x7f10ec088070 comp rx=0 tx=0).stop 2024-03-19T01:44:33.259 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.256+0000 7f10fc2a1640 1 -- 172.21.15.99:0/730274323 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f10f41022f0 msgr2=0x7f10f419f030 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:44:33.259 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.256+0000 7f10fc2a1640 1 --2- 172.21.15.99:0/730274323 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f10f41022f0 0x7f10f419f030 secure :-1 s=READY pgs=422 cs=0 l=1 rev1=1 crypto rx=0x7f10ec037040 tx=0x7f10ec032f40 comp rx=0 tx=0).stop 2024-03-19T01:44:33.260 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.256+0000 7f10fc2a1640 1 -- 172.21.15.99:0/730274323 shutdown_connections 2024-03-19T01:44:33.260 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.256+0000 7f10fc2a1640 1 --2- 172.21.15.99:0/730274323 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f10f41022f0 0x7f10f419f030 unknown :-1 s=CLOSED pgs=422 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:33.260 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.256+0000 7f10fc2a1640 1 --2- 172.21.15.99:0/730274323 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f10f4101930 0x7f10f4197fc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:33.260 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.256+0000 7f10fc2a1640 1 --2- 172.21.15.99:0/730274323 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f10e00a2920 0x7f10e00a4de0 unknown :-1 s=CLOSED pgs=263 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:33.260 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.256+0000 7f10fc2a1640 1 --2- 172.21.15.99:0/730274323 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f10f4100740 0x7f10f4197a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:33.260 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.256+0000 7f10fc2a1640 1 -- 172.21.15.99:0/730274323 >> 172.21.15.99:0/730274323 conn(0x7f10f40fbcb0 msgr2=0x7f10f40fd6d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:44:33.260 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.256+0000 7f10fc2a1640 1 -- 172.21.15.99:0/730274323 shutdown_connections 2024-03-19T01:44:33.260 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:33.256+0000 7f10fc2a1640 1 -- 172.21.15.99:0/730274323 wait complete. 2024-03-19T01:44:33.680 INFO:tasks.workunit.client.0.smithi099.stdout: 1 397692 11.42 MB/sec execute 63 sec latency 24.999 ms 2024-03-19T01:44:34.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:44:33 smithi115 ceph-mon[67332]: pgmap v454: 97 pgs: 97 active+clean; 140 MiB data, 1.3 GiB used, 1.0 TiB / 1.0 TiB avail; 1.3 MiB/s rd, 3.0 MiB/s wr, 298 op/s 2024-03-19T01:44:34.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:44:33 smithi099 ceph-mon[62951]: pgmap v454: 97 pgs: 97 active+clean; 140 MiB data, 1.3 GiB used, 1.0 TiB / 1.0 TiB avail; 1.3 MiB/s rd, 3.0 MiB/s wr, 298 op/s 2024-03-19T01:44:34.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:44:33 smithi112 ceph-mon[66819]: pgmap v454: 97 pgs: 97 active+clean; 140 MiB data, 1.3 GiB used, 1.0 TiB / 1.0 TiB avail; 1.3 MiB/s rd, 3.0 MiB/s wr, 298 op/s 2024-03-19T01:44:34.680 INFO:tasks.workunit.client.0.smithi099.stdout: 1 399699 11.35 MB/sec execute 64 sec latency 23.091 ms 2024-03-19T01:44:35.680 INFO:tasks.workunit.client.0.smithi099.stdout: 1 401416 11.37 MB/sec execute 65 sec latency 49.409 ms 2024-03-19T01:44:36.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:44:35 smithi115 ceph-mon[67332]: pgmap v455: 97 pgs: 97 active+clean; 140 MiB data, 1.3 GiB used, 1.0 TiB / 1.0 TiB avail; 1.0 MiB/s rd, 2.4 MiB/s wr, 220 op/s 2024-03-19T01:44:36.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:44:35 smithi099 ceph-mon[62951]: pgmap v455: 97 pgs: 97 active+clean; 140 MiB data, 1.3 GiB used, 1.0 TiB / 1.0 TiB avail; 1.0 MiB/s rd, 2.4 MiB/s wr, 220 op/s 2024-03-19T01:44:36.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:44:35 smithi112 ceph-mon[66819]: pgmap v455: 97 pgs: 97 active+clean; 140 MiB data, 1.3 GiB used, 1.0 TiB / 1.0 TiB avail; 1.0 MiB/s rd, 2.4 MiB/s wr, 220 op/s 2024-03-19T01:44:36.680 INFO:tasks.workunit.client.0.smithi099.stdout: 1 403830 11.38 MB/sec execute 66 sec latency 41.616 ms 2024-03-19T01:44:37.680 INFO:tasks.workunit.client.0.smithi099.stdout: 1 405651 11.34 MB/sec execute 67 sec latency 24.003 ms 2024-03-19T01:44:38.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:44:37 smithi115 ceph-mon[67332]: pgmap v456: 97 pgs: 97 active+clean; 143 MiB data, 1.3 GiB used, 1.0 TiB / 1.0 TiB avail; 1.3 MiB/s rd, 3.1 MiB/s wr, 285 op/s 2024-03-19T01:44:38.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:44:37 smithi099 ceph-mon[62951]: pgmap v456: 97 pgs: 97 active+clean; 143 MiB data, 1.3 GiB used, 1.0 TiB / 1.0 TiB avail; 1.3 MiB/s rd, 3.1 MiB/s wr, 285 op/s 2024-03-19T01:44:38.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:44:37 smithi112 ceph-mon[66819]: pgmap v456: 97 pgs: 97 active+clean; 143 MiB data, 1.3 GiB used, 1.0 TiB / 1.0 TiB avail; 1.3 MiB/s rd, 3.1 MiB/s wr, 285 op/s 2024-03-19T01:44:38.680 INFO:tasks.workunit.client.0.smithi099.stdout: 1 408181 11.42 MB/sec execute 68 sec latency 16.348 ms 2024-03-19T01:44:39.680 INFO:tasks.workunit.client.0.smithi099.stdout: 1 409758 11.34 MB/sec execute 69 sec latency 80.562 ms 2024-03-19T01:44:40.142 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:44:39 smithi099 ceph-mon[62951]: pgmap v457: 97 pgs: 97 active+clean; 143 MiB data, 1.4 GiB used, 1.0 TiB / 1.0 TiB avail; 1.2 MiB/s rd, 2.9 MiB/s wr, 271 op/s 2024-03-19T01:44:40.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:44:39 smithi115 ceph-mon[67332]: pgmap v457: 97 pgs: 97 active+clean; 143 MiB data, 1.4 GiB used, 1.0 TiB / 1.0 TiB avail; 1.2 MiB/s rd, 2.9 MiB/s wr, 271 op/s 2024-03-19T01:44:40.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:44:39 smithi112 ceph-mon[66819]: pgmap v457: 97 pgs: 97 active+clean; 143 MiB data, 1.4 GiB used, 1.0 TiB / 1.0 TiB avail; 1.2 MiB/s rd, 2.9 MiB/s wr, 271 op/s 2024-03-19T01:44:40.680 INFO:tasks.workunit.client.0.smithi099.stdout: 1 411916 11.37 MB/sec execute 70 sec latency 45.736 ms 2024-03-19T01:44:41.680 INFO:tasks.workunit.client.0.smithi099.stdout: 1 414369 11.37 MB/sec execute 71 sec latency 28.946 ms 2024-03-19T01:44:42.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:44:41 smithi115 ceph-mon[67332]: pgmap v458: 97 pgs: 97 active+clean; 144 MiB data, 1.4 GiB used, 1.0 TiB / 1.0 TiB avail; 1.3 MiB/s rd, 2.8 MiB/s wr, 267 op/s 2024-03-19T01:44:42.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:44:41 smithi099 ceph-mon[62951]: pgmap v458: 97 pgs: 97 active+clean; 144 MiB data, 1.4 GiB used, 1.0 TiB / 1.0 TiB avail; 1.3 MiB/s rd, 2.8 MiB/s wr, 267 op/s 2024-03-19T01:44:42.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:44:41 smithi112 ceph-mon[66819]: pgmap v458: 97 pgs: 97 active+clean; 144 MiB data, 1.4 GiB used, 1.0 TiB / 1.0 TiB avail; 1.3 MiB/s rd, 2.8 MiB/s wr, 267 op/s 2024-03-19T01:44:42.680 INFO:tasks.workunit.client.0.smithi099.stdout: 1 416227 11.35 MB/sec execute 72 sec latency 32.909 ms 2024-03-19T01:44:43.680 INFO:tasks.workunit.client.0.smithi099.stdout: 1 418807 11.43 MB/sec execute 73 sec latency 18.508 ms 2024-03-19T01:44:44.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:44:43 smithi115 ceph-mon[67332]: pgmap v459: 97 pgs: 97 active+clean; 145 MiB data, 1.4 GiB used, 1.0 TiB / 1.0 TiB avail; 1.2 MiB/s rd, 3.2 MiB/s wr, 266 op/s 2024-03-19T01:44:44.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:44:43 smithi099 ceph-mon[62951]: pgmap v459: 97 pgs: 97 active+clean; 145 MiB data, 1.4 GiB used, 1.0 TiB / 1.0 TiB avail; 1.2 MiB/s rd, 3.2 MiB/s wr, 266 op/s 2024-03-19T01:44:44.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:44:43 smithi112 ceph-mon[66819]: pgmap v459: 97 pgs: 97 active+clean; 145 MiB data, 1.4 GiB used, 1.0 TiB / 1.0 TiB avail; 1.2 MiB/s rd, 3.2 MiB/s wr, 266 op/s 2024-03-19T01:44:44.680 INFO:tasks.workunit.client.0.smithi099.stdout: 1 420592 11.35 MB/sec execute 74 sec latency 26.080 ms 2024-03-19T01:44:45.681 INFO:tasks.workunit.client.0.smithi099.stdout: 1 422599 11.38 MB/sec execute 75 sec latency 49.379 ms 2024-03-19T01:44:46.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:44:45 smithi115 ceph-mon[67332]: pgmap v460: 97 pgs: 97 active+clean; 147 MiB data, 1.4 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 2.8 MiB/s wr, 247 op/s 2024-03-19T01:44:46.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:44:45 smithi099 ceph-mon[62951]: pgmap v460: 97 pgs: 97 active+clean; 147 MiB data, 1.4 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 2.8 MiB/s wr, 247 op/s 2024-03-19T01:44:46.347 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.347+0000 7f6a83cd1640 1 -- 172.21.15.99:0/1071456074 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6a7c100740 msgr2=0x7f6a7c100b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:44:46.347 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.347+0000 7f6a83cd1640 1 --2- 172.21.15.99:0/1071456074 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6a7c100740 0x7f6a7c100b60 secure :-1 s=READY pgs=425 cs=0 l=1 rev1=1 crypto rx=0x7f6a78009900 tx=0x7f6a7802bc30 comp rx=0 tx=0).stop 2024-03-19T01:44:46.347 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.347+0000 7f6a83cd1640 1 -- 172.21.15.99:0/1071456074 shutdown_connections 2024-03-19T01:44:46.348 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.347+0000 7f6a83cd1640 1 --2- 172.21.15.99:0/1071456074 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6a7c1022f0 0x7f6a7c108b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:46.348 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.347+0000 7f6a83cd1640 1 --2- 172.21.15.99:0/1071456074 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f6a7c101930 0x7f6a7c101db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:46.348 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.347+0000 7f6a83cd1640 1 --2- 172.21.15.99:0/1071456074 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6a7c100740 0x7f6a7c100b60 unknown :-1 s=CLOSED pgs=425 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:46.348 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.347+0000 7f6a83cd1640 1 -- 172.21.15.99:0/1071456074 >> 172.21.15.99:0/1071456074 conn(0x7f6a7c0fbcb0 msgr2=0x7f6a7c0fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:44:46.348 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.347+0000 7f6a83cd1640 1 -- 172.21.15.99:0/1071456074 shutdown_connections 2024-03-19T01:44:46.348 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.347+0000 7f6a83cd1640 1 -- 172.21.15.99:0/1071456074 wait complete. 2024-03-19T01:44:46.350 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.348+0000 7f6a83cd1640 1 Processor -- start 2024-03-19T01:44:46.350 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.348+0000 7f6a83cd1640 1 -- start start 2024-03-19T01:44:46.350 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.348+0000 7f6a83cd1640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f6a7c100740 0x7f6a7c197b40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:46.350 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.348+0000 7f6a83cd1640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6a7c101930 0x7f6a7c198080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:46.350 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.348+0000 7f6a83cd1640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6a7c1022f0 0x7f6a7c19f0a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:46.350 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.348+0000 7f6a83cd1640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f6a7c10c3b0 con 0x7f6a7c101930 2024-03-19T01:44:46.350 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.348+0000 7f6a83cd1640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f6a7c10c230 con 0x7f6a7c1022f0 2024-03-19T01:44:46.351 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.348+0000 7f6a83cd1640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f6a7c10c530 con 0x7f6a7c100740 2024-03-19T01:44:46.351 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.348+0000 7f6a8126c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6a7c101930 0x7f6a7c198080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:46.351 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.348+0000 7f6a8126c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6a7c101930 0x7f6a7c198080 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:54556/0 (socket says 172.21.15.99:54556) 2024-03-19T01:44:46.351 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.348+0000 7f6a8126c640 1 -- 172.21.15.99:0/2651216223 learned_addr learned my addr 172.21.15.99:0/2651216223 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:44:46.351 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.348+0000 7f6a81a6d640 1 --2- 172.21.15.99:0/2651216223 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f6a7c100740 0x7f6a7c197b40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:46.351 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.348+0000 7f6a8226e640 1 --2- 172.21.15.99:0/2651216223 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6a7c1022f0 0x7f6a7c19f0a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:46.351 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.349+0000 7f6a8126c640 1 -- 172.21.15.99:0/2651216223 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f6a7c100740 msgr2=0x7f6a7c197b40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:44:46.351 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.349+0000 7f6a8126c640 1 --2- 172.21.15.99:0/2651216223 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f6a7c100740 0x7f6a7c197b40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:46.351 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.349+0000 7f6a8126c640 1 -- 172.21.15.99:0/2651216223 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6a7c1022f0 msgr2=0x7f6a7c19f0a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:44:46.351 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.349+0000 7f6a8126c640 1 --2- 172.21.15.99:0/2651216223 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6a7c1022f0 0x7f6a7c19f0a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:46.351 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.349+0000 7f6a8126c640 1 -- 172.21.15.99:0/2651216223 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f6a7c19f710 con 0x7f6a7c101930 2024-03-19T01:44:46.351 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.349+0000 7f6a8126c640 1 --2- 172.21.15.99:0/2651216223 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6a7c101930 0x7f6a7c198080 secure :-1 s=READY pgs=426 cs=0 l=1 rev1=1 crypto rx=0x7f6a6c00e9f0 tx=0x7f6a6c00edf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:44:46.352 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.349+0000 7f6a72ffd640 1 -- 172.21.15.99:0/2651216223 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6a6c00cd80 con 0x7f6a7c101930 2024-03-19T01:44:46.352 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.349+0000 7f6a83cd1640 1 -- 172.21.15.99:0/2651216223 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f6a7c19fa00 con 0x7f6a7c101930 2024-03-19T01:44:46.352 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.349+0000 7f6a72ffd640 1 -- 172.21.15.99:0/2651216223 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f6a6c005500 con 0x7f6a7c101930 2024-03-19T01:44:46.352 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.349+0000 7f6a72ffd640 1 -- 172.21.15.99:0/2651216223 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6a6c010690 con 0x7f6a7c101930 2024-03-19T01:44:46.352 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.349+0000 7f6a83cd1640 1 -- 172.21.15.99:0/2651216223 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f6a7c19ff00 con 0x7f6a7c101930 2024-03-19T01:44:46.358 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.351+0000 7f6a72ffd640 1 -- 172.21.15.99:0/2651216223 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f6a6c0046c0 con 0x7f6a7c101930 2024-03-19T01:44:46.358 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.352+0000 7f6a86997740 1 Processor -- start 2024-03-19T01:44:46.358 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.352+0000 7f6a86997740 1 -- start start 2024-03-19T01:44:46.358 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.352+0000 7f6a86997740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55940f676100 0x55940f676520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:46.359 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.352+0000 7f6a86997740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55940f676af0 0x55940f67b5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:46.359 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.352+0000 7f6a86997740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55940f67bc60 0x55940f67e0f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:46.359 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.352+0000 7f6a86997740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55940f5dabe0 con 0x55940f676100 2024-03-19T01:44:46.359 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.352+0000 7f6a86997740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55940f583960 con 0x55940f676af0 2024-03-19T01:44:46.359 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.352+0000 7f6a86997740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55940f5d6c80 con 0x55940f67bc60 2024-03-19T01:44:46.359 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.352+0000 7f6a72ffd640 1 --2- 172.21.15.99:0/2651216223 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f6a680a28a0 0x7f6a680a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:46.359 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.352+0000 7f6a81a6d640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55940f676af0 0x55940f67b5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:46.359 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.352+0000 7f6a81a6d640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55940f676af0 0x55940f67b5f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:52190/0 (socket says 172.21.15.99:52190) 2024-03-19T01:44:46.359 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.352+0000 7f6a81a6d640 1 -- 172.21.15.99:0/4190573492 learned_addr learned my addr 172.21.15.99:0/4190573492 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:44:46.359 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.352+0000 7f6a8126c640 1 --2- 172.21.15.99:0/4190573492 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55940f67bc60 0x55940f67e0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:46.359 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.352+0000 7f6a8226e640 1 --2- 172.21.15.99:0/4190573492 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55940f676100 0x55940f676520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:46.359 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.352+0000 7f6a8226e640 1 --2- 172.21.15.99:0/2651216223 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f6a680a28a0 0x7f6a680a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:46.359 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.352+0000 7f6a72ffd640 1 -- 172.21.15.99:0/2651216223 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f6a6c0cee80 con 0x7f6a7c101930 2024-03-19T01:44:46.359 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.353+0000 7f6a81a6d640 1 -- 172.21.15.99:0/4190573492 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55940f67bc60 msgr2=0x55940f67e0f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:44:46.360 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.353+0000 7f6a81a6d640 1 --2- 172.21.15.99:0/4190573492 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55940f67bc60 0x55940f67e0f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:46.360 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.353+0000 7f6a81a6d640 1 -- 172.21.15.99:0/4190573492 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55940f676100 msgr2=0x55940f676520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:44:46.360 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.353+0000 7f6a81a6d640 1 --2- 172.21.15.99:0/4190573492 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55940f676100 0x55940f676520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:46.360 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.353+0000 7f6a81a6d640 1 -- 172.21.15.99:0/4190573492 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55940f57ad20 con 0x55940f676af0 2024-03-19T01:44:46.360 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.353+0000 7f6a8226e640 1 --2- 172.21.15.99:0/4190573492 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55940f676100 0x55940f676520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-19T01:44:46.360 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.353+0000 7f6a81a6d640 1 --2- 172.21.15.99:0/4190573492 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55940f676af0 0x55940f67b5f0 secure :-1 s=READY pgs=105 cs=0 l=1 rev1=1 crypto rx=0x7f6a78009900 tx=0x7f6a78009560 comp rx=0 tx=0).ready entity=mon.2 client_cookie=8eca9fef766c27be server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:44:46.360 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.353+0000 7f6a67fff640 1 -- 172.21.15.99:0/4190573492 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6a7802cb80 con 0x55940f676af0 2024-03-19T01:44:46.360 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.353+0000 7f6a8226e640 1 --2- 172.21.15.99:0/2651216223 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f6a680a28a0 0x7f6a680a4d60 secure :-1 s=READY pgs=264 cs=0 l=1 rev1=1 crypto rx=0x7f6a74005c40 tx=0x7f6a74009500 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:44:46.360 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.353+0000 7f6a67fff640 1 -- 172.21.15.99:0/4190573492 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f6a78005040 con 0x55940f676af0 2024-03-19T01:44:46.360 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.353+0000 7f6a67fff640 1 -- 172.21.15.99:0/4190573492 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6a78004920 con 0x55940f676af0 2024-03-19T01:44:46.360 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.354+0000 7f6a86997740 1 -- 172.21.15.99:0/4190573492 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55940f676af0 msgr2=0x55940f67b5f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:44:46.360 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.354+0000 7f6a86997740 1 --2- 172.21.15.99:0/4190573492 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55940f676af0 0x55940f67b5f0 secure :-1 s=READY pgs=105 cs=0 l=1 rev1=1 crypto rx=0x7f6a78009900 tx=0x7f6a78009560 comp rx=0 tx=0).stop 2024-03-19T01:44:46.360 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.354+0000 7f6a86997740 1 -- 172.21.15.99:0/4190573492 shutdown_connections 2024-03-19T01:44:46.360 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.354+0000 7f6a86997740 1 --2- 172.21.15.99:0/4190573492 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55940f676af0 0x55940f67b5f0 unknown :-1 s=CLOSED pgs=105 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:46.361 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.354+0000 7f6a86997740 1 --2- 172.21.15.99:0/4190573492 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55940f67bc60 0x55940f67e0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:46.361 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.354+0000 7f6a86997740 1 --2- 172.21.15.99:0/4190573492 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55940f676100 0x55940f676520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:46.361 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.354+0000 7f6a86997740 1 -- 172.21.15.99:0/4190573492 >> 172.21.15.99:0/4190573492 conn(0x55940f671670 msgr2=0x55940f671a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:44:46.361 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.354+0000 7f6a86997740 1 -- 172.21.15.99:0/4190573492 shutdown_connections 2024-03-19T01:44:46.361 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.354+0000 7f6a86997740 1 -- 172.21.15.99:0/4190573492 wait complete. 2024-03-19T01:44:46.361 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.354+0000 7f6a86997740 1 -- start start 2024-03-19T01:44:46.361 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.354+0000 7f6a86997740 1 Processor -- start 2024-03-19T01:44:46.361 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.355+0000 7f6a86997740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55940f676100 0x55940f6800b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:46.361 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.355+0000 7f6a86997740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55940f676af0 0x55940f6805f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:46.361 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.355+0000 7f6a86997740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55940f67bc60 0x55940f687630 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:46.361 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.355+0000 7f6a86997740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55940f67ebb0 con 0x55940f676100 2024-03-19T01:44:46.361 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.355+0000 7f6a86997740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55940f596db0 con 0x55940f676af0 2024-03-19T01:44:46.361 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.355+0000 7f6a86997740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55940f67ed30 con 0x55940f67bc60 2024-03-19T01:44:46.361 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.355+0000 7f6a81a6d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55940f676100 0x55940f6800b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:46.362 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.355+0000 7f6a86997740 10 client.0 _ll_register_callbacks cb 0x55940f59be70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:44:46.362 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.355+0000 7f6a81a6d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55940f676100 0x55940f6800b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:54570/0 (socket says 172.21.15.99:54570) 2024-03-19T01:44:46.362 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.355+0000 7f6a81a6d640 1 -- 172.21.15.99:0/3349608320 learned_addr learned my addr 172.21.15.99:0/3349608320 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:44:46.362 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.355+0000 7f6a8126c640 1 --2- 172.21.15.99:0/3349608320 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55940f676af0 0x55940f6805f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:46.362 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.355+0000 7f6a8226e640 1 --2- 172.21.15.99:0/3349608320 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55940f67bc60 0x55940f687630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:46.362 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.355+0000 7f6a8126c640 1 -- 172.21.15.99:0/3349608320 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55940f67bc60 msgr2=0x55940f687630 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:44:46.362 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.355+0000 7f6a8126c640 1 --2- 172.21.15.99:0/3349608320 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55940f67bc60 0x55940f687630 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:46.362 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.355+0000 7f6a8126c640 1 -- 172.21.15.99:0/3349608320 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55940f676100 msgr2=0x55940f6800b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:44:46.362 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.355+0000 7f6a8126c640 1 --2- 172.21.15.99:0/3349608320 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55940f676100 0x55940f6800b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:46.362 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.355+0000 7f6a8126c640 1 -- 172.21.15.99:0/3349608320 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55940f595c60 con 0x55940f676af0 2024-03-19T01:44:46.362 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.355+0000 7f6a81a6d640 1 --2- 172.21.15.99:0/3349608320 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55940f676100 0x55940f6800b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:44:46.362 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.355+0000 7f6a8126c640 1 --2- 172.21.15.99:0/3349608320 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55940f676af0 0x55940f6805f0 secure :-1 s=READY pgs=106 cs=0 l=1 rev1=1 crypto rx=0x7f6a6c00e9c0 tx=0x7f6a6c06b3c0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=36f8dff93e90f88 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:44:46.362 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.355+0000 7f6a657fa640 10 client.0 ms_handle_connect on v2:172.21.15.112:3300/0 2024-03-19T01:44:46.363 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.355+0000 7f6a657fa640 1 -- 172.21.15.99:0/3349608320 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6a6c09b140 con 0x55940f676af0 2024-03-19T01:44:46.363 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.355+0000 7f6a86997740 1 -- 172.21.15.99:0/3349608320 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f6a78005040 con 0x55940f676af0 2024-03-19T01:44:46.363 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.355+0000 7f6a657fa640 1 -- 172.21.15.99:0/3349608320 <== mon.2 v2:172.21.15.112:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f6a6c0860b0 con 0x55940f676af0 2024-03-19T01:44:46.363 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.356+0000 7f6a657fa640 1 -- 172.21.15.99:0/3349608320 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_get_version_reply(handle=1 version=25) ==== 24+0+0 (secure 0 0 0) 0x7f6a6c09bee0 con 0x55940f676af0 2024-03-19T01:44:46.363 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.356+0000 7f6a86997740 10 client.25403 fetch_fsmap learned FSMap version 25 2024-03-19T01:44:46.363 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.356+0000 7f6a86997740 1 -- 172.21.15.99:0/3349608320 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,fsmap=25,monmap=4+}) -- 0x55940f2ef210 con 0x55940f676af0 2024-03-19T01:44:46.363 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.356+0000 7f6a657fa640 1 -- 172.21.15.99:0/3349608320 <== mon.2 v2:172.21.15.112:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f6a78005040 con 0x55940f676af0 2024-03-19T01:44:46.363 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.356+0000 7f6a657fa640 1 -- 172.21.15.99:0/3349608320 <== mon.2 v2:172.21.15.112:3300/0 5 ==== fsmap(e 25) ==== 8338+0+0 (secure 0 0 0) 0x7f6a6c0a3af0 con 0x55940f676af0 2024-03-19T01:44:46.363 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.356+0000 7f6a86997740 10 client.25403 fetch_fsmap finished waiting for FSMap version 25 2024-03-19T01:44:46.363 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.356+0000 7f6a86997740 10 client.25403 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:44:46.363 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.356+0000 7f6a86997740 20 client.25403 populate_metadata read hostname 'smithi099' 2024-03-19T01:44:46.363 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.356+0000 7f6a86997740 1 --2- 172.21.15.99:0/3349608320 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55940f68bec0 0x55940f68e350 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:46.363 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.356+0000 7f6a86997740 4 client.25403 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:44:46.363 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.356+0000 7f6a86997740 1 -- 172.21.15.99:0/3349608320 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55940f5e8120 con 0x55940f68bec0 2024-03-19T01:44:46.364 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.356+0000 7f6a81a6d640 1 --2- 172.21.15.99:0/3349608320 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55940f68bec0 0x55940f68e350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:46.364 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.357+0000 7f6a81a6d640 1 --2- 172.21.15.99:0/3349608320 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55940f68bec0 0x55940f68e350 crc :-1 s=READY pgs=61 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=4d77dc62ba6d7208 server_cookie=dfb7c461b4a0604c in_seq=0 out_seq=0 2024-03-19T01:44:46.364 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.357+0000 7f6a657fa640 10 client.25403 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:44:46.364 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.357+0000 7f6a657fa640 1 -- 172.21.15.99:0/3349608320 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x7f6a780509c0 con 0x55940f68bec0 2024-03-19T01:44:46.364 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.357+0000 7f6a657fa640 10 client.25403 handle_command_reply: tid=0 2024-03-19T01:44:46.364 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.358+0000 7f6a86997740 1 client.25403 shutdown 2024-03-19T01:44:46.364 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.358+0000 7f6a86997740 1 -- 172.21.15.99:0/3349608320 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55940f676af0 msgr2=0x55940f6805f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:44:46.364 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.358+0000 7f6a86997740 1 --2- 172.21.15.99:0/3349608320 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55940f676af0 0x55940f6805f0 secure :-1 s=READY pgs=106 cs=0 l=1 rev1=1 crypto rx=0x7f6a6c00e9c0 tx=0x7f6a6c06b3c0 comp rx=0 tx=0).stop 2024-03-19T01:44:46.364 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.358+0000 7f6a86997740 1 -- 172.21.15.99:0/3349608320 shutdown_connections 2024-03-19T01:44:46.364 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.358+0000 7f6a86997740 1 --2- 172.21.15.99:0/3349608320 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55940f68bec0 0x55940f68e350 crc :-1 s=READY pgs=61 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:46.364 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.358+0000 7f6a86997740 1 --2- 172.21.15.99:0/3349608320 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55940f676af0 0x55940f6805f0 unknown :-1 s=CLOSED pgs=106 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:46.364 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.358+0000 7f6a86997740 1 --2- 172.21.15.99:0/3349608320 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55940f67bc60 0x55940f687630 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:46.364 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.358+0000 7f6a86997740 1 --2- 172.21.15.99:0/3349608320 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55940f676100 0x55940f6800b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:46.364 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.358+0000 7f6a657fa640 0 client.25403 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:44:46.365 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.358+0000 7f6a86997740 1 -- 172.21.15.99:0/3349608320 >> 172.21.15.99:0/3349608320 conn(0x55940f671670 msgr2=0x55940f6728c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:44:46.365 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.358+0000 7f6a86997740 1 -- 172.21.15.99:0/3349608320 shutdown_connections 2024-03-19T01:44:46.365 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.358+0000 7f6a86997740 1 -- 172.21.15.99:0/3349608320 wait complete. 2024-03-19T01:44:46.365 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.358+0000 7f6a86997740 20 client.25403 trim_cache size 0 max 16384 2024-03-19T01:44:46.365 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:44:45 smithi112 ceph-mon[66819]: pgmap v460: 97 pgs: 97 active+clean; 147 MiB data, 1.4 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 2.8 MiB/s wr, 247 op/s 2024-03-19T01:44:46.377 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.373+0000 7f6a86997740 1 Processor -- start 2024-03-19T01:44:46.377 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.373+0000 7f6a86997740 1 -- start start 2024-03-19T01:44:46.378 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.373+0000 7f6a86997740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55940f676100 0x55940f674fe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:46.378 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.373+0000 7f6a86997740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55940f676af0 0x55940f6875f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:46.378 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.373+0000 7f6a86997740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55940f67bc60 0x55940f692c50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:46.378 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.373+0000 7f6a86997740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f6a7c1a93d0 con 0x55940f676100 2024-03-19T01:44:46.378 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.373+0000 7f6a86997740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f6a7c1a6ca0 con 0x55940f676af0 2024-03-19T01:44:46.378 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.373+0000 7f6a86997740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f6a7c1a4af0 con 0x55940f67bc60 2024-03-19T01:44:46.378 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.373+0000 7f6a81a6d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55940f676100 0x55940f674fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:46.378 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.373+0000 7f6a81a6d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55940f676100 0x55940f674fe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:54574/0 (socket says 172.21.15.99:54574) 2024-03-19T01:44:46.378 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.373+0000 7f6a81a6d640 1 -- 172.21.15.99:0/654361337 learned_addr learned my addr 172.21.15.99:0/654361337 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:44:46.378 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.373+0000 7f6a8226e640 1 --2- 172.21.15.99:0/654361337 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55940f67bc60 0x55940f692c50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:46.378 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.373+0000 7f6a8126c640 1 --2- 172.21.15.99:0/654361337 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55940f676af0 0x55940f6875f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:46.378 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.374+0000 7f6a81a6d640 1 -- 172.21.15.99:0/654361337 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55940f67bc60 msgr2=0x55940f692c50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:44:46.378 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.374+0000 7f6a81a6d640 1 --2- 172.21.15.99:0/654361337 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55940f67bc60 0x55940f692c50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:46.379 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.374+0000 7f6a81a6d640 1 -- 172.21.15.99:0/654361337 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55940f676af0 msgr2=0x55940f6875f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:44:46.379 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.374+0000 7f6a81a6d640 1 --2- 172.21.15.99:0/654361337 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55940f676af0 0x55940f6875f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:46.379 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.374+0000 7f6a81a6d640 1 -- 172.21.15.99:0/654361337 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55940f3bb620 con 0x55940f676100 2024-03-19T01:44:46.379 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.374+0000 7f6a8126c640 1 --2- 172.21.15.99:0/654361337 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55940f676af0 0x55940f6875f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:44:46.379 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.374+0000 7f6a81a6d640 1 --2- 172.21.15.99:0/654361337 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55940f676100 0x55940f674fe0 secure :-1 s=READY pgs=427 cs=0 l=1 rev1=1 crypto rx=0x7f6a7802e010 tx=0x7f6a78000bc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=50a321bec96dd786 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:44:46.379 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.374+0000 7f6a677fe640 1 -- 172.21.15.99:0/654361337 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6a78054070 con 0x55940f676100 2024-03-19T01:44:46.379 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.374+0000 7f6a677fe640 1 -- 172.21.15.99:0/654361337 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f6a7804b040 con 0x55940f676100 2024-03-19T01:44:46.379 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.374+0000 7f6a677fe640 1 -- 172.21.15.99:0/654361337 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6a78003a10 con 0x55940f676100 2024-03-19T01:44:46.379 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.375+0000 7f6a86997740 1 -- 172.21.15.99:0/654361337 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55940f676100 msgr2=0x55940f674fe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:44:46.379 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.375+0000 7f6a86997740 1 --2- 172.21.15.99:0/654361337 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55940f676100 0x55940f674fe0 secure :-1 s=READY pgs=427 cs=0 l=1 rev1=1 crypto rx=0x7f6a7802e010 tx=0x7f6a78000bc0 comp rx=0 tx=0).stop 2024-03-19T01:44:46.379 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.375+0000 7f6a86997740 1 -- 172.21.15.99:0/654361337 shutdown_connections 2024-03-19T01:44:46.379 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.375+0000 7f6a86997740 1 --2- 172.21.15.99:0/654361337 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55940f676af0 0x55940f6875f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:46.379 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.375+0000 7f6a86997740 1 --2- 172.21.15.99:0/654361337 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55940f67bc60 0x55940f692c50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:46.380 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.375+0000 7f6a86997740 1 --2- 172.21.15.99:0/654361337 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55940f676100 0x55940f674fe0 unknown :-1 s=CLOSED pgs=427 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:46.380 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.375+0000 7f6a86997740 1 -- 172.21.15.99:0/654361337 >> 172.21.15.99:0/654361337 conn(0x55940f671670 msgr2=0x55940f6748d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:44:46.380 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.375+0000 7f6a86997740 1 -- 172.21.15.99:0/654361337 shutdown_connections 2024-03-19T01:44:46.380 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.375+0000 7f6a86997740 1 -- 172.21.15.99:0/654361337 wait complete. 2024-03-19T01:44:46.380 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.375+0000 7f6a86997740 1 -- start start 2024-03-19T01:44:46.380 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.375+0000 7f6a86997740 1 Processor -- start 2024-03-19T01:44:46.380 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.375+0000 7f6a86997740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55940f676100 0x55940f687160 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:46.380 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.376+0000 7f6a86997740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55940f676af0 0x55940f685770 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:46.380 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.376+0000 7f6a86997740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55940f67bc60 0x55940f685cb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:46.380 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.376+0000 7f6a86997740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f6a7c1a5160 con 0x55940f67bc60 2024-03-19T01:44:46.380 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.376+0000 7f6a86997740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f6a7c1a65d0 con 0x55940f676100 2024-03-19T01:44:46.380 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.376+0000 7f6a86997740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f6a7c1a5830 con 0x55940f676af0 2024-03-19T01:44:46.380 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.376+0000 7f6a86997740 10 client.0 _ll_register_callbacks cb 0x7f6a6c009720 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:44:46.380 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.376+0000 7f6a81a6d640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55940f676100 0x55940f687160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:46.381 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.376+0000 7f6a8226e640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55940f67bc60 0x55940f685cb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:46.381 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.376+0000 7f6a8126c640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55940f676af0 0x55940f685770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:46.381 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.376+0000 7f6a81a6d640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55940f676100 0x55940f687160 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:52212/0 (socket says 172.21.15.99:52212) 2024-03-19T01:44:46.381 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.376+0000 7f6a81a6d640 1 -- 172.21.15.99:0/161941363 learned_addr learned my addr 172.21.15.99:0/161941363 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:44:46.381 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.376+0000 7f6a8126c640 1 -- 172.21.15.99:0/161941363 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55940f676100 msgr2=0x55940f687160 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:44:46.381 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.376+0000 7f6a8126c640 1 --2- 172.21.15.99:0/161941363 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55940f676100 0x55940f687160 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:46.381 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.376+0000 7f6a8126c640 1 -- 172.21.15.99:0/161941363 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55940f67bc60 msgr2=0x55940f685cb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:44:46.381 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.376+0000 7f6a8126c640 1 --2- 172.21.15.99:0/161941363 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55940f67bc60 0x55940f685cb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:46.381 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.376+0000 7f6a8126c640 1 -- 172.21.15.99:0/161941363 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55940f6861f0 con 0x55940f676af0 2024-03-19T01:44:46.381 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.376+0000 7f6a81a6d640 1 --2- 172.21.15.99:0/161941363 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55940f676100 0x55940f687160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:44:46.381 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.376+0000 7f6a8126c640 1 --2- 172.21.15.99:0/161941363 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55940f676af0 0x55940f685770 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 crypto rx=0x7f6a6c09b770 tx=0x7f6a6c004bb0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=3bdd9f9f36bf8350 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:44:46.381 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.376+0000 7f6a657fa640 10 client.0 ms_handle_connect on v2:172.21.15.115:3300/0 2024-03-19T01:44:46.381 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.376+0000 7f6a657fa640 1 -- 172.21.15.99:0/161941363 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6a6c09b140 con 0x55940f676af0 2024-03-19T01:44:46.382 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.376+0000 7f6a86997740 1 -- 172.21.15.99:0/161941363 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f6a7804b040 con 0x55940f676af0 2024-03-19T01:44:46.382 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.376+0000 7f6a657fa640 1 -- 172.21.15.99:0/161941363 <== mon.1 v2:172.21.15.115:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f6a6c0860b0 con 0x55940f676af0 2024-03-19T01:44:46.382 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.376+0000 7f6a657fa640 1 -- 172.21.15.99:0/161941363 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_get_version_reply(handle=1 version=25) ==== 24+0+0 (secure 0 0 0) 0x7f6a6c09b5b0 con 0x55940f676af0 2024-03-19T01:44:46.382 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.376+0000 7f6a86997740 10 client.25459 fetch_fsmap learned FSMap version 25 2024-03-19T01:44:46.382 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.377+0000 7f6a86997740 1 -- 172.21.15.99:0/161941363 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,fsmap=25,monmap=4+}) -- 0x55940f6f2fd0 con 0x55940f676af0 2024-03-19T01:44:46.382 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.377+0000 7f6a657fa640 1 -- 172.21.15.99:0/161941363 <== mon.1 v2:172.21.15.115:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f6a7804b040 con 0x55940f676af0 2024-03-19T01:44:46.382 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.377+0000 7f6a657fa640 1 -- 172.21.15.99:0/161941363 <== mon.1 v2:172.21.15.115:3300/0 5 ==== fsmap(e 25) ==== 8338+0+0 (secure 0 0 0) 0x7f6a6c0afce0 con 0x55940f676af0 2024-03-19T01:44:46.382 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.377+0000 7f6a86997740 10 client.25459 fetch_fsmap finished waiting for FSMap version 25 2024-03-19T01:44:46.382 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.377+0000 7f6a86997740 10 client.25459 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:44:46.382 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.377+0000 7f6a86997740 20 client.25459 populate_metadata read hostname 'smithi099' 2024-03-19T01:44:46.382 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.377+0000 7f6a86997740 1 --2- 172.21.15.99:0/161941363 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55940f6f39d0 0x55940f6f5df0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:46.382 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.377+0000 7f6a86997740 4 client.25459 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:44:46.382 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.377+0000 7f6a86997740 1 -- 172.21.15.99:0/161941363 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55940f5e2470 con 0x55940f6f39d0 2024-03-19T01:44:46.383 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.377+0000 7f6a81a6d640 1 --2- 172.21.15.99:0/161941363 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55940f6f39d0 0x55940f6f5df0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:46.383 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.377+0000 7f6a657fa640 10 client.25459 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:44:46.383 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.377+0000 7f6a81a6d640 1 --2- 172.21.15.99:0/161941363 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55940f6f39d0 0x55940f6f5df0 crc :-1 s=READY pgs=62 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=af9b2822a86602ca server_cookie=e39ee2b399f31e01 in_seq=0 out_seq=0 2024-03-19T01:44:46.422 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.421+0000 7f6a657fa640 1 -- 172.21.15.99:0/161941363 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x55940f5e2470 con 0x55940f6f39d0 2024-03-19T01:44:46.422 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.421+0000 7f6a657fa640 10 client.25459 handle_command_reply: tid=0 2024-03-19T01:44:46.422 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.421+0000 7f6a86997740 1 client.25459 shutdown 2024-03-19T01:44:46.422 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.422+0000 7f6a86997740 1 -- 172.21.15.99:0/161941363 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55940f676af0 msgr2=0x55940f685770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:44:46.423 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.422+0000 7f6a86997740 1 --2- 172.21.15.99:0/161941363 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55940f676af0 0x55940f685770 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 crypto rx=0x7f6a6c09b770 tx=0x7f6a6c004bb0 comp rx=0 tx=0).stop 2024-03-19T01:44:46.423 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.422+0000 7f6a86997740 1 -- 172.21.15.99:0/161941363 shutdown_connections 2024-03-19T01:44:46.423 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.422+0000 7f6a86997740 1 --2- 172.21.15.99:0/161941363 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55940f6f39d0 0x55940f6f5df0 crc :-1 s=READY pgs=62 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:46.423 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.422+0000 7f6a86997740 1 --2- 172.21.15.99:0/161941363 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55940f67bc60 0x55940f685cb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:46.423 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.422+0000 7f6a86997740 1 --2- 172.21.15.99:0/161941363 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55940f676af0 0x55940f685770 unknown :-1 s=CLOSED pgs=84 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:46.423 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.422+0000 7f6a86997740 1 --2- 172.21.15.99:0/161941363 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55940f676100 0x55940f687160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:46.423 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.422+0000 7f6a657fa640 0 client.25459 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:44:46.423 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.422+0000 7f6a86997740 1 -- 172.21.15.99:0/161941363 >> 172.21.15.99:0/161941363 conn(0x55940f671670 msgr2=0x55940f675160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:44:46.423 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.422+0000 7f6a86997740 1 -- 172.21.15.99:0/161941363 shutdown_connections 2024-03-19T01:44:46.423 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.422+0000 7f6a86997740 1 -- 172.21.15.99:0/161941363 wait complete. 2024-03-19T01:44:46.423 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.422+0000 7f6a86997740 20 client.25459 trim_cache size 0 max 16384 2024-03-19T01:44:46.424 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:44:46.424 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:44:46.424 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:44:46.424 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:44:46.425 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.422+0000 7f6a83cd1640 1 -- 172.21.15.99:0/2651216223 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f6a680a28a0 msgr2=0x7f6a680a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:44:46.425 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.422+0000 7f6a83cd1640 1 --2- 172.21.15.99:0/2651216223 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f6a680a28a0 0x7f6a680a4d60 secure :-1 s=READY pgs=264 cs=0 l=1 rev1=1 crypto rx=0x7f6a74005c40 tx=0x7f6a74009500 comp rx=0 tx=0).stop 2024-03-19T01:44:46.425 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.422+0000 7f6a83cd1640 1 -- 172.21.15.99:0/2651216223 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6a7c101930 msgr2=0x7f6a7c198080 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:44:46.425 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.422+0000 7f6a83cd1640 1 --2- 172.21.15.99:0/2651216223 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6a7c101930 0x7f6a7c198080 secure :-1 s=READY pgs=426 cs=0 l=1 rev1=1 crypto rx=0x7f6a6c00e9f0 tx=0x7f6a6c00edf0 comp rx=0 tx=0).stop 2024-03-19T01:44:46.425 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.423+0000 7f6a83cd1640 1 -- 172.21.15.99:0/2651216223 shutdown_connections 2024-03-19T01:44:46.425 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.423+0000 7f6a83cd1640 1 --2- 172.21.15.99:0/2651216223 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f6a680a28a0 0x7f6a680a4d60 unknown :-1 s=CLOSED pgs=264 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:46.425 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.423+0000 7f6a83cd1640 1 --2- 172.21.15.99:0/2651216223 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6a7c1022f0 0x7f6a7c19f0a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:46.425 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.423+0000 7f6a83cd1640 1 --2- 172.21.15.99:0/2651216223 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6a7c101930 0x7f6a7c198080 unknown :-1 s=CLOSED pgs=426 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:46.425 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.423+0000 7f6a83cd1640 1 --2- 172.21.15.99:0/2651216223 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f6a7c100740 0x7f6a7c197b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:46.425 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.423+0000 7f6a83cd1640 1 -- 172.21.15.99:0/2651216223 >> 172.21.15.99:0/2651216223 conn(0x7f6a7c0fbcb0 msgr2=0x7f6a7c0fd730 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:44:46.425 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.423+0000 7f6a83cd1640 1 -- 172.21.15.99:0/2651216223 shutdown_connections 2024-03-19T01:44:46.425 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:46.423+0000 7f6a83cd1640 1 -- 172.21.15.99:0/2651216223 wait complete. 2024-03-19T01:44:46.681 INFO:tasks.workunit.client.0.smithi099.stdout: 1 425060 11.39 MB/sec execute 76 sec latency 22.836 ms 2024-03-19T01:44:47.681 INFO:tasks.workunit.client.0.smithi099.stdout: 1 427058 11.40 MB/sec execute 77 sec latency 30.945 ms 2024-03-19T01:44:48.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:44:47 smithi115 ceph-mon[67332]: pgmap v461: 97 pgs: 97 active+clean; 147 MiB data, 1.4 GiB used, 1.0 TiB / 1.0 TiB avail; 1.2 MiB/s rd, 3.2 MiB/s wr, 271 op/s 2024-03-19T01:44:48.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:44:47 smithi099 ceph-mon[62951]: pgmap v461: 97 pgs: 97 active+clean; 147 MiB data, 1.4 GiB used, 1.0 TiB / 1.0 TiB avail; 1.2 MiB/s rd, 3.2 MiB/s wr, 271 op/s 2024-03-19T01:44:48.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:44:47 smithi112 ceph-mon[66819]: pgmap v461: 97 pgs: 97 active+clean; 147 MiB data, 1.4 GiB used, 1.0 TiB / 1.0 TiB avail; 1.2 MiB/s rd, 3.2 MiB/s wr, 271 op/s 2024-03-19T01:44:48.681 INFO:tasks.workunit.client.0.smithi099.stdout: 1 429495 11.43 MB/sec execute 78 sec latency 19.024 ms 2024-03-19T01:44:49.681 INFO:tasks.workunit.client.0.smithi099.stdout: 1 431576 11.38 MB/sec execute 79 sec latency 27.028 ms 2024-03-19T01:44:50.166 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:44:49 smithi112 ceph-mon[66819]: pgmap v462: 97 pgs: 97 active+clean; 149 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 3.0 MiB/s wr, 241 op/s 2024-03-19T01:44:50.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:44:49 smithi115 ceph-mon[67332]: pgmap v462: 97 pgs: 97 active+clean; 149 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 3.0 MiB/s wr, 241 op/s 2024-03-19T01:44:50.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:44:49 smithi099 ceph-mon[62951]: pgmap v462: 97 pgs: 97 active+clean; 149 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 3.0 MiB/s wr, 241 op/s 2024-03-19T01:44:50.681 INFO:tasks.workunit.client.0.smithi099.stdout: 1 433351 11.39 MB/sec execute 80 sec latency 55.561 ms 2024-03-19T01:44:51.681 INFO:tasks.workunit.client.0.smithi099.stdout: 1 435777 11.40 MB/sec execute 81 sec latency 36.302 ms 2024-03-19T01:44:52.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:44:51 smithi115 ceph-mon[67332]: pgmap v463: 97 pgs: 97 active+clean; 152 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 1.3 MiB/s rd, 3.1 MiB/s wr, 290 op/s 2024-03-19T01:44:52.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:44:51 smithi099 ceph-mon[62951]: pgmap v463: 97 pgs: 97 active+clean; 152 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 1.3 MiB/s rd, 3.1 MiB/s wr, 290 op/s 2024-03-19T01:44:52.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:44:51 smithi112 ceph-mon[66819]: pgmap v463: 97 pgs: 97 active+clean; 152 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 1.3 MiB/s rd, 3.1 MiB/s wr, 290 op/s 2024-03-19T01:44:52.681 INFO:tasks.workunit.client.0.smithi099.stdout: 1 437766 11.40 MB/sec execute 82 sec latency 23.992 ms 2024-03-19T01:44:53.681 INFO:tasks.workunit.client.0.smithi099.stdout: 1 440151 11.43 MB/sec execute 83 sec latency 15.485 ms 2024-03-19T01:44:54.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:44:53 smithi115 ceph-mon[67332]: pgmap v464: 97 pgs: 97 active+clean; 155 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 2.9 MiB/s wr, 256 op/s 2024-03-19T01:44:54.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:44:53 smithi099 ceph-mon[62951]: pgmap v464: 97 pgs: 97 active+clean; 155 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 2.9 MiB/s wr, 256 op/s 2024-03-19T01:44:54.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:44:53 smithi112 ceph-mon[66819]: pgmap v464: 97 pgs: 97 active+clean; 155 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 2.9 MiB/s wr, 256 op/s 2024-03-19T01:44:54.681 INFO:tasks.workunit.client.0.smithi099.stdout: 1 442152 11.38 MB/sec execute 84 sec latency 35.007 ms 2024-03-19T01:44:55.681 INFO:tasks.workunit.client.0.smithi099.stdout: 1 444071 11.39 MB/sec execute 85 sec latency 74.965 ms 2024-03-19T01:44:56.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:44:55 smithi115 ceph-mon[67332]: pgmap v465: 97 pgs: 97 active+clean; 155 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 2.5 MiB/s wr, 243 op/s 2024-03-19T01:44:56.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:44:55 smithi099 ceph-mon[62951]: pgmap v465: 97 pgs: 97 active+clean; 155 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 2.5 MiB/s wr, 243 op/s 2024-03-19T01:44:56.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:44:55 smithi112 ceph-mon[66819]: pgmap v465: 97 pgs: 97 active+clean; 155 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 2.5 MiB/s wr, 243 op/s 2024-03-19T01:44:56.681 INFO:tasks.workunit.client.0.smithi099.stdout: 1 446394 11.40 MB/sec execute 86 sec latency 36.436 ms 2024-03-19T01:44:57.681 INFO:tasks.workunit.client.0.smithi099.stdout: 1 448197 11.37 MB/sec execute 87 sec latency 25.278 ms 2024-03-19T01:44:58.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:44:57 smithi115 ceph-mon[67332]: pgmap v466: 97 pgs: 97 active+clean; 161 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 1.3 MiB/s rd, 3.3 MiB/s wr, 288 op/s 2024-03-19T01:44:58.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:44:57 smithi099 ceph-mon[62951]: pgmap v466: 97 pgs: 97 active+clean; 161 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 1.3 MiB/s rd, 3.3 MiB/s wr, 288 op/s 2024-03-19T01:44:58.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:44:57 smithi112 ceph-mon[66819]: pgmap v466: 97 pgs: 97 active+clean; 161 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 1.3 MiB/s rd, 3.3 MiB/s wr, 288 op/s 2024-03-19T01:44:58.681 INFO:tasks.workunit.client.0.smithi099.stdout: 1 450720 11.43 MB/sec execute 88 sec latency 66.428 ms 2024-03-19T01:44:59.513 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.513+0000 7ffa10df4640 1 -- 172.21.15.99:0/2232848878 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ffa0c1022f0 msgr2=0x7ffa0c108b80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:44:59.513 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.513+0000 7ffa10df4640 1 --2- 172.21.15.99:0/2232848878 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ffa0c1022f0 0x7ffa0c108b80 secure :-1 s=READY pgs=428 cs=0 l=1 rev1=1 crypto rx=0x7ff9fc009900 tx=0x7ff9fc02bc30 comp rx=0 tx=0).stop 2024-03-19T01:44:59.513 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.513+0000 7ffa10df4640 1 -- 172.21.15.99:0/2232848878 shutdown_connections 2024-03-19T01:44:59.513 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.513+0000 7ffa10df4640 1 --2- 172.21.15.99:0/2232848878 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ffa0c101930 0x7ffa0c101db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:59.513 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.513+0000 7ffa10df4640 1 --2- 172.21.15.99:0/2232848878 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ffa0c1022f0 0x7ffa0c108b80 unknown :-1 s=CLOSED pgs=428 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:59.514 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.513+0000 7ffa10df4640 1 --2- 172.21.15.99:0/2232848878 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ffa0c100740 0x7ffa0c100b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:59.514 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.513+0000 7ffa10df4640 1 -- 172.21.15.99:0/2232848878 >> 172.21.15.99:0/2232848878 conn(0x7ffa0c0fbcb0 msgr2=0x7ffa0c0fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:44:59.514 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.513+0000 7ffa10df4640 1 -- 172.21.15.99:0/2232848878 shutdown_connections 2024-03-19T01:44:59.514 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.513+0000 7ffa10df4640 1 -- 172.21.15.99:0/2232848878 wait complete. 2024-03-19T01:44:59.516 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.514+0000 7ffa10df4640 1 Processor -- start 2024-03-19T01:44:59.516 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.514+0000 7ffa10df4640 1 -- start start 2024-03-19T01:44:59.516 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.514+0000 7ffa10df4640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ffa0c100740 0x7ffa0c196950 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:59.516 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.514+0000 7ffa10df4640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ffa0c101930 0x7ffa0c196e90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:59.516 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.514+0000 7ffa10df4640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ffa0c1022f0 0x7ffa0c19df00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:59.516 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.514+0000 7ffa10df4640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7ffa0c10c3a0 con 0x7ffa0c1022f0 2024-03-19T01:44:59.516 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.514+0000 7ffa10df4640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7ffa0c10c220 con 0x7ffa0c101930 2024-03-19T01:44:59.517 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.514+0000 7ffa10df4640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7ffa0c10c520 con 0x7ffa0c100740 2024-03-19T01:44:59.517 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.514+0000 7ffa0ad9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ffa0c1022f0 0x7ffa0c19df00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:59.517 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.514+0000 7ffa0a59c640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ffa0c100740 0x7ffa0c196950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:59.517 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.514+0000 7ffa0ad9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ffa0c1022f0 0x7ffa0c19df00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:48328/0 (socket says 172.21.15.99:48328) 2024-03-19T01:44:59.517 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.514+0000 7ffa0ad9d640 1 -- 172.21.15.99:0/779102332 learned_addr learned my addr 172.21.15.99:0/779102332 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:44:59.517 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.514+0000 7ffa09d9b640 1 --2- 172.21.15.99:0/779102332 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ffa0c101930 0x7ffa0c196e90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:59.517 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.514+0000 7ffa09d9b640 1 -- 172.21.15.99:0/779102332 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ffa0c100740 msgr2=0x7ffa0c196950 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:44:59.517 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.514+0000 7ffa09d9b640 1 --2- 172.21.15.99:0/779102332 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ffa0c100740 0x7ffa0c196950 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:59.517 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.514+0000 7ffa09d9b640 1 -- 172.21.15.99:0/779102332 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ffa0c1022f0 msgr2=0x7ffa0c19df00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:44:59.517 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.514+0000 7ffa09d9b640 1 --2- 172.21.15.99:0/779102332 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ffa0c1022f0 0x7ffa0c19df00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:59.517 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.514+0000 7ffa09d9b640 1 -- 172.21.15.99:0/779102332 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ffa0c19e570 con 0x7ffa0c101930 2024-03-19T01:44:59.517 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.514+0000 7ffa0ad9d640 1 --2- 172.21.15.99:0/779102332 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ffa0c1022f0 0x7ffa0c19df00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:44:59.517 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.514+0000 7ffa0a59c640 1 --2- 172.21.15.99:0/779102332 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ffa0c100740 0x7ffa0c196950 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:44:59.517 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.514+0000 7ffa09d9b640 1 --2- 172.21.15.99:0/779102332 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ffa0c101930 0x7ffa0c196e90 secure :-1 s=READY pgs=107 cs=0 l=1 rev1=1 crypto rx=0x7ffa0000e9f0 tx=0x7ffa0000eeb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:44:59.518 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.515+0000 7ff9fb7fe640 1 -- 172.21.15.99:0/779102332 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffa0000ccc0 con 0x7ffa0c101930 2024-03-19T01:44:59.518 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.515+0000 7ff9fb7fe640 1 -- 172.21.15.99:0/779102332 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7ffa00010910 con 0x7ffa0c101930 2024-03-19T01:44:59.518 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.515+0000 7ffa10df4640 1 -- 172.21.15.99:0/779102332 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ffa0c19e860 con 0x7ffa0c101930 2024-03-19T01:44:59.518 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.515+0000 7ff9fb7fe640 1 -- 172.21.15.99:0/779102332 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffa00018960 con 0x7ffa0c101930 2024-03-19T01:44:59.518 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.515+0000 7ffa10df4640 1 -- 172.21.15.99:0/779102332 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7ffa0c19edb0 con 0x7ffa0c101930 2024-03-19T01:44:59.527 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.517+0000 7ff9fb7fe640 1 -- 172.21.15.99:0/779102332 <== mon.2 v2:172.21.15.112:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7ffa00010ac0 con 0x7ffa0c101930 2024-03-19T01:44:59.527 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.517+0000 7ffa139e6740 1 Processor -- start 2024-03-19T01:44:59.527 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.517+0000 7ffa139e6740 1 -- start start 2024-03-19T01:44:59.527 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.517+0000 7ffa139e6740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559ed45b0310 0x559ed45b0730 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:59.527 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.517+0000 7ffa139e6740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559ed45b0d00 0x559ed45b5800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:59.527 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.517+0000 7ffa139e6740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559ed45b5e70 0x559ed45b8300 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:59.527 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.517+0000 7ffa139e6740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x559ed4512be0 con 0x559ed45b0d00 2024-03-19T01:44:59.527 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.517+0000 7ffa139e6740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x559ed44bb960 con 0x559ed45b0310 2024-03-19T01:44:59.527 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.517+0000 7ffa139e6740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x559ed450ec80 con 0x559ed45b5e70 2024-03-19T01:44:59.527 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.518+0000 7ffa0a59c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559ed45b0d00 0x559ed45b5800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:59.527 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.518+0000 7ffa09d9b640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559ed45b5e70 0x559ed45b8300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:59.527 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.518+0000 7ffa0ad9d640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559ed45b0310 0x559ed45b0730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:59.528 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.518+0000 7ffa0ad9d640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559ed45b0310 0x559ed45b0730 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:54432/0 (socket says 172.21.15.99:54432) 2024-03-19T01:44:59.528 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.518+0000 7ffa0a59c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559ed45b0d00 0x559ed45b5800 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:48344/0 (socket says 172.21.15.99:48344) 2024-03-19T01:44:59.528 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.518+0000 7ffa0ad9d640 1 -- 172.21.15.99:0/2914067621 learned_addr learned my addr 172.21.15.99:0/2914067621 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:44:59.528 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.518+0000 7ffa09d9b640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559ed45b5e70 0x559ed45b8300 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:47030/0 (socket says 172.21.15.99:47030) 2024-03-19T01:44:59.528 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.518+0000 7ff9fb7fe640 1 --2- 172.21.15.99:0/779102332 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff9f40a28a0 0x7ff9f40a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:59.528 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.518+0000 7ffa0ad9d640 1 -- 172.21.15.99:0/2914067621 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559ed45b5e70 msgr2=0x559ed45b8300 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:44:59.528 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.518+0000 7ffa0ad9d640 1 --2- 172.21.15.99:0/2914067621 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559ed45b5e70 0x559ed45b8300 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:59.528 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.518+0000 7ffa0ad9d640 1 -- 172.21.15.99:0/2914067621 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559ed45b0d00 msgr2=0x559ed45b5800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:44:59.528 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.518+0000 7ffa0ad9d640 1 --2- 172.21.15.99:0/2914067621 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559ed45b0d00 0x559ed45b5800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:59.528 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.518+0000 7ffa0ad9d640 1 -- 172.21.15.99:0/2914067621 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x559ed44b2d20 con 0x559ed45b0310 2024-03-19T01:44:59.528 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.518+0000 7ffa09d9b640 1 --2- 172.21.15.99:0/2914067621 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559ed45b5e70 0x559ed45b8300 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:44:59.528 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.518+0000 7ffa0ad9d640 1 --2- 172.21.15.99:0/779102332 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff9f40a28a0 0x7ff9f40a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:59.528 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.518+0000 7ff9fb7fe640 1 -- 172.21.15.99:0/779102332 <== mon.2 v2:172.21.15.112:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7ffa000cee50 con 0x7ffa0c101930 2024-03-19T01:44:59.528 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.518+0000 7ffa0ad9d640 1 --2- 172.21.15.99:0/2914067621 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559ed45b0310 0x559ed45b0730 secure :-1 s=READY pgs=108 cs=0 l=1 rev1=1 crypto rx=0x7ff9fc005890 tx=0x7ff9fc009560 comp rx=0 tx=0).ready entity=mon.2 client_cookie=5c72beabf0be8725 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:44:59.529 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.518+0000 7ff9f8ff9640 1 -- 172.21.15.99:0/2914067621 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff9fc050070 con 0x559ed45b0310 2024-03-19T01:44:59.529 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.518+0000 7ff9f8ff9640 1 -- 172.21.15.99:0/2914067621 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7ff9fc005410 con 0x559ed45b0310 2024-03-19T01:44:59.529 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.518+0000 7ff9f8ff9640 1 -- 172.21.15.99:0/2914067621 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff9fc003af0 con 0x559ed45b0310 2024-03-19T01:44:59.529 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.519+0000 7ffa139e6740 1 -- 172.21.15.99:0/2914067621 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559ed45b0310 msgr2=0x559ed45b0730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:44:59.529 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.519+0000 7ffa0ad9d640 1 --2- 172.21.15.99:0/779102332 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff9f40a28a0 0x7ff9f40a4d60 secure :-1 s=READY pgs=265 cs=0 l=1 rev1=1 crypto rx=0x7ff9fc04d000 tx=0x7ff9fc04a000 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:44:59.529 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.519+0000 7ffa139e6740 1 --2- 172.21.15.99:0/2914067621 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559ed45b0310 0x559ed45b0730 secure :-1 s=READY pgs=108 cs=0 l=1 rev1=1 crypto rx=0x7ff9fc005890 tx=0x7ff9fc009560 comp rx=0 tx=0).stop 2024-03-19T01:44:59.529 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.519+0000 7ffa139e6740 1 -- 172.21.15.99:0/2914067621 shutdown_connections 2024-03-19T01:44:59.529 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.519+0000 7ffa139e6740 1 --2- 172.21.15.99:0/2914067621 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559ed45b5e70 0x559ed45b8300 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:59.529 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.519+0000 7ffa139e6740 1 --2- 172.21.15.99:0/2914067621 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559ed45b0d00 0x559ed45b5800 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:59.529 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.519+0000 7ffa139e6740 1 --2- 172.21.15.99:0/2914067621 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559ed45b0310 0x559ed45b0730 unknown :-1 s=CLOSED pgs=108 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:59.529 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.519+0000 7ffa139e6740 1 -- 172.21.15.99:0/2914067621 >> 172.21.15.99:0/2914067621 conn(0x559ed45a9670 msgr2=0x559ed45a9a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:44:59.529 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.519+0000 7ffa139e6740 1 -- 172.21.15.99:0/2914067621 shutdown_connections 2024-03-19T01:44:59.530 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.519+0000 7ffa139e6740 1 -- 172.21.15.99:0/2914067621 wait complete. 2024-03-19T01:44:59.530 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.519+0000 7ffa139e6740 1 -- start start 2024-03-19T01:44:59.530 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.519+0000 7ffa139e6740 1 Processor -- start 2024-03-19T01:44:59.530 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.520+0000 7ffa139e6740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559ed45b0310 0x559ed45ba340 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:59.530 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.520+0000 7ffa139e6740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559ed45b0d00 0x559ed45ba880 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:59.530 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.520+0000 7ffa139e6740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559ed45b5e70 0x559ed45c18c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:59.530 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.520+0000 7ffa139e6740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x559ed45b8e20 con 0x559ed45b5e70 2024-03-19T01:44:59.530 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.520+0000 7ffa139e6740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x559ed44cedb0 con 0x559ed45b0d00 2024-03-19T01:44:59.530 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.520+0000 7ffa139e6740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x559ed45b8fa0 con 0x559ed45b0310 2024-03-19T01:44:59.530 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.520+0000 7ffa139e6740 10 client.0 _ll_register_callbacks cb 0x559ed44d3e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:44:59.530 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.520+0000 7ffa0a59c640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559ed45b0310 0x559ed45ba340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:59.530 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.520+0000 7ffa0ad9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559ed45b5e70 0x559ed45c18c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:59.530 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.520+0000 7ffa0a59c640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559ed45b0310 0x559ed45ba340 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:47032/0 (socket says 172.21.15.99:47032) 2024-03-19T01:44:59.531 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.520+0000 7ffa0a59c640 1 -- 172.21.15.99:0/2951891774 learned_addr learned my addr 172.21.15.99:0/2951891774 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:44:59.531 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.520+0000 7ffa09d9b640 1 --2- 172.21.15.99:0/2951891774 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559ed45b0d00 0x559ed45ba880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:59.531 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.520+0000 7ffa0a59c640 1 -- 172.21.15.99:0/2951891774 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559ed45b0d00 msgr2=0x559ed45ba880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:44:59.531 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.520+0000 7ffa0a59c640 1 --2- 172.21.15.99:0/2951891774 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559ed45b0d00 0x559ed45ba880 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:59.531 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.520+0000 7ffa0a59c640 1 -- 172.21.15.99:0/2951891774 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559ed45b5e70 msgr2=0x559ed45c18c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:44:59.531 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.520+0000 7ffa0a59c640 1 --2- 172.21.15.99:0/2951891774 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559ed45b5e70 0x559ed45c18c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:59.531 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.520+0000 7ffa0a59c640 1 -- 172.21.15.99:0/2951891774 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({osdmap=0}) -- 0x559ed44cdc60 con 0x559ed45b0310 2024-03-19T01:44:59.531 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.520+0000 7ffa0ad9d640 1 --2- 172.21.15.99:0/2951891774 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559ed45b5e70 0x559ed45c18c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:44:59.531 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.520+0000 7ffa0a59c640 1 --2- 172.21.15.99:0/2951891774 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559ed45b0310 0x559ed45ba340 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7ffa04005fd0 tx=0x7ffa04011810 comp rx=0 tx=0).ready entity=mon.1 client_cookie=3e3a1f78bae8c71b server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:44:59.531 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.520+0000 7ff9f1ffb640 10 client.0 ms_handle_connect on v2:172.21.15.115:3300/0 2024-03-19T01:44:59.531 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.520+0000 7ff9f1ffb640 1 -- 172.21.15.99:0/2951891774 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffa04003c40 con 0x559ed45b0310 2024-03-19T01:44:59.531 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.521+0000 7ff9f1ffb640 1 -- 172.21.15.99:0/2951891774 <== mon.1 v2:172.21.15.115:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7ffa04003de0 con 0x559ed45b0310 2024-03-19T01:44:59.531 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.521+0000 7ffa139e6740 1 -- 172.21.15.99:0/2951891774 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7ff9fc005410 con 0x559ed45b0310 2024-03-19T01:44:59.531 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.521+0000 7ff9f1ffb640 1 -- 172.21.15.99:0/2951891774 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_get_version_reply(handle=1 version=25) ==== 24+0+0 (secure 0 0 0) 0x7ffa04004b20 con 0x559ed45b0310 2024-03-19T01:44:59.532 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.521+0000 7ffa139e6740 10 client.25471 fetch_fsmap learned FSMap version 25 2024-03-19T01:44:59.532 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.521+0000 7ffa139e6740 1 -- 172.21.15.99:0/2951891774 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,fsmap=25,monmap=4+}) -- 0x559ed4227210 con 0x559ed45b0310 2024-03-19T01:44:59.532 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.521+0000 7ff9f1ffb640 1 -- 172.21.15.99:0/2951891774 <== mon.1 v2:172.21.15.115:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7ff9fc005410 con 0x559ed45b0310 2024-03-19T01:44:59.532 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.521+0000 7ff9f1ffb640 1 -- 172.21.15.99:0/2951891774 <== mon.1 v2:172.21.15.115:3300/0 5 ==== fsmap(e 25) ==== 8338+0+0 (secure 0 0 0) 0x7ffa04019030 con 0x559ed45b0310 2024-03-19T01:44:59.532 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.521+0000 7ffa139e6740 10 client.25471 fetch_fsmap finished waiting for FSMap version 25 2024-03-19T01:44:59.532 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.521+0000 7ffa139e6740 10 client.25471 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:44:59.532 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.521+0000 7ffa139e6740 20 client.25471 populate_metadata read hostname 'smithi099' 2024-03-19T01:44:59.532 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.521+0000 7ffa139e6740 1 --2- 172.21.15.99:0/2951891774 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x559ed45c6270 0x559ed45c86a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:59.532 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.521+0000 7ffa139e6740 4 client.25471 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:44:59.532 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.521+0000 7ffa139e6740 1 -- 172.21.15.99:0/2951891774 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x559ed4520120 con 0x559ed45c6270 2024-03-19T01:44:59.532 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.521+0000 7ffa09d9b640 1 --2- 172.21.15.99:0/2951891774 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x559ed45c6270 0x559ed45c86a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:59.532 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.522+0000 7ffa09d9b640 1 --2- 172.21.15.99:0/2951891774 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x559ed45c6270 0x559ed45c86a0 crc :-1 s=READY pgs=63 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=d0edeaa7a3dcf1b7 server_cookie=ed6f328509026f21 in_seq=0 out_seq=0 2024-03-19T01:44:59.532 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.522+0000 7ff9f1ffb640 10 client.25471 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:44:59.533 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.523+0000 7ff9f1ffb640 1 -- 172.21.15.99:0/2951891774 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x559ed4520120 con 0x559ed45c6270 2024-03-19T01:44:59.533 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.523+0000 7ff9f1ffb640 10 client.25471 handle_command_reply: tid=0 2024-03-19T01:44:59.533 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.523+0000 7ffa139e6740 1 client.25471 shutdown 2024-03-19T01:44:59.533 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.523+0000 7ffa139e6740 1 -- 172.21.15.99:0/2951891774 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559ed45b0310 msgr2=0x559ed45ba340 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:44:59.533 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.523+0000 7ffa139e6740 1 --2- 172.21.15.99:0/2951891774 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559ed45b0310 0x559ed45ba340 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7ffa04005fd0 tx=0x7ffa04011810 comp rx=0 tx=0).stop 2024-03-19T01:44:59.533 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.523+0000 7ffa139e6740 1 -- 172.21.15.99:0/2951891774 shutdown_connections 2024-03-19T01:44:59.533 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.523+0000 7ffa139e6740 1 --2- 172.21.15.99:0/2951891774 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x559ed45c6270 0x559ed45c86a0 crc :-1 s=READY pgs=63 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:59.533 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.523+0000 7ffa139e6740 1 --2- 172.21.15.99:0/2951891774 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559ed45b0d00 0x559ed45ba880 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:59.533 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.523+0000 7ffa139e6740 1 --2- 172.21.15.99:0/2951891774 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559ed45b5e70 0x559ed45c18c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:59.533 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.523+0000 7ff9f1ffb640 0 client.25471 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:44:59.533 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.523+0000 7ffa139e6740 1 --2- 172.21.15.99:0/2951891774 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559ed45b0310 0x559ed45ba340 unknown :-1 s=CLOSED pgs=85 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:59.533 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.523+0000 7ffa139e6740 1 -- 172.21.15.99:0/2951891774 >> 172.21.15.99:0/2951891774 conn(0x559ed45a9670 msgr2=0x559ed45aa8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:44:59.533 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.523+0000 7ffa139e6740 1 -- 172.21.15.99:0/2951891774 shutdown_connections 2024-03-19T01:44:59.533 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.523+0000 7ffa139e6740 1 -- 172.21.15.99:0/2951891774 wait complete. 2024-03-19T01:44:59.534 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.523+0000 7ffa139e6740 20 client.25471 trim_cache size 0 max 16384 2024-03-19T01:44:59.538 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.538+0000 7ffa139e6740 1 Processor -- start 2024-03-19T01:44:59.538 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.538+0000 7ffa139e6740 1 -- start start 2024-03-19T01:44:59.538 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.538+0000 7ffa139e6740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559ed45b0310 0x559ed45b9960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:59.539 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.539+0000 7ffa139e6740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559ed45b0d00 0x559ed45c1880 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:59.539 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.539+0000 7ffa139e6740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559ed45b5e70 0x559ed45b9ea0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:59.539 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.539+0000 7ffa139e6740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7ffa0c1a9460 con 0x559ed45b0310 2024-03-19T01:44:59.539 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.539+0000 7ffa139e6740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7ffa0c1a6d30 con 0x559ed45b5e70 2024-03-19T01:44:59.539 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.539+0000 7ffa139e6740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7ffa0c1a4b80 con 0x559ed45b0d00 2024-03-19T01:44:59.540 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.539+0000 7ffa0a59c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559ed45b0310 0x559ed45b9960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:59.540 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.539+0000 7ffa0a59c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559ed45b0310 0x559ed45b9960 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:48356/0 (socket says 172.21.15.99:48356) 2024-03-19T01:44:59.540 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.539+0000 7ffa0ad9d640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559ed45b5e70 0x559ed45b9ea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:59.540 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.539+0000 7ffa0a59c640 1 -- 172.21.15.99:0/1431254179 learned_addr learned my addr 172.21.15.99:0/1431254179 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:44:59.540 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.539+0000 7ffa0a59c640 1 -- 172.21.15.99:0/1431254179 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559ed45b0d00 msgr2=0x559ed45c1880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:44:59.540 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.539+0000 7ffa0a59c640 1 --2- 172.21.15.99:0/1431254179 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559ed45b0d00 0x559ed45c1880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:59.540 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.539+0000 7ffa0a59c640 1 -- 172.21.15.99:0/1431254179 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559ed45b5e70 msgr2=0x559ed45b9ea0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:44:59.540 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.539+0000 7ffa0a59c640 1 --2- 172.21.15.99:0/1431254179 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559ed45b5e70 0x559ed45b9ea0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:59.540 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.539+0000 7ffa0a59c640 1 -- 172.21.15.99:0/1431254179 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x559ed42f3620 con 0x559ed45b0310 2024-03-19T01:44:59.541 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.540+0000 7ffa0a59c640 1 --2- 172.21.15.99:0/1431254179 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559ed45b0310 0x559ed45b9960 secure :-1 s=READY pgs=429 cs=0 l=1 rev1=1 crypto rx=0x7ffa0401aa50 tx=0x7ffa0400f3c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9c17695b19b0b29b server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:44:59.541 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.540+0000 7ff9f3fff640 1 -- 172.21.15.99:0/1431254179 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffa0401add0 con 0x559ed45b0310 2024-03-19T01:44:59.541 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.540+0000 7ff9f3fff640 1 -- 172.21.15.99:0/1431254179 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7ffa04030980 con 0x559ed45b0310 2024-03-19T01:44:59.544 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.541+0000 7ff9f3fff640 1 -- 172.21.15.99:0/1431254179 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffa040257d0 con 0x559ed45b0310 2024-03-19T01:44:59.544 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.541+0000 7ffa139e6740 1 -- 172.21.15.99:0/1431254179 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559ed45b0310 msgr2=0x559ed45b9960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:44:59.544 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.541+0000 7ffa139e6740 1 --2- 172.21.15.99:0/1431254179 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559ed45b0310 0x559ed45b9960 secure :-1 s=READY pgs=429 cs=0 l=1 rev1=1 crypto rx=0x7ffa0401aa50 tx=0x7ffa0400f3c0 comp rx=0 tx=0).stop 2024-03-19T01:44:59.544 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.541+0000 7ffa139e6740 1 -- 172.21.15.99:0/1431254179 shutdown_connections 2024-03-19T01:44:59.544 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.541+0000 7ffa139e6740 1 --2- 172.21.15.99:0/1431254179 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559ed45b5e70 0x559ed45b9ea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:59.544 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.541+0000 7ffa139e6740 1 --2- 172.21.15.99:0/1431254179 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559ed45b0d00 0x559ed45c1880 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:59.544 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.541+0000 7ffa139e6740 1 --2- 172.21.15.99:0/1431254179 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559ed45b0310 0x559ed45b9960 unknown :-1 s=CLOSED pgs=429 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:59.544 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.541+0000 7ffa139e6740 1 -- 172.21.15.99:0/1431254179 >> 172.21.15.99:0/1431254179 conn(0x559ed45a9670 msgr2=0x559ed45ad160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:44:59.544 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.541+0000 7ffa139e6740 1 -- 172.21.15.99:0/1431254179 shutdown_connections 2024-03-19T01:44:59.545 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.541+0000 7ffa139e6740 1 -- 172.21.15.99:0/1431254179 wait complete. 2024-03-19T01:44:59.545 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.542+0000 7ffa139e6740 1 -- start start 2024-03-19T01:44:59.545 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.542+0000 7ffa139e6740 1 Processor -- start 2024-03-19T01:44:59.545 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.542+0000 7ffa139e6740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559ed45b0310 0x559ed45bce20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:59.545 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.542+0000 7ffa139e6740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559ed45b0d00 0x559ed45bd360 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:59.545 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.542+0000 7ffa139e6740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559ed45b5e70 0x559ed45c0380 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:59.545 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.542+0000 7ffa139e6740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7ffa0c1a51f0 con 0x559ed45b5e70 2024-03-19T01:44:59.545 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.542+0000 7ffa139e6740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7ffa0c1a6660 con 0x559ed45b0d00 2024-03-19T01:44:59.545 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.542+0000 7ffa139e6740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7ffa0c1a58c0 con 0x559ed45b0310 2024-03-19T01:44:59.545 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.542+0000 7ffa139e6740 10 client.0 _ll_register_callbacks cb 0x7ffa0400f4a0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:44:59.545 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.542+0000 7ffa0ad9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559ed45b5e70 0x559ed45c0380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:59.545 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.542+0000 7ffa09d9b640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559ed45b0d00 0x559ed45bd360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:59.545 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.542+0000 7ffa09d9b640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559ed45b0d00 0x559ed45bd360 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:54460/0 (socket says 172.21.15.99:54460) 2024-03-19T01:44:59.546 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.542+0000 7ffa09d9b640 1 -- 172.21.15.99:0/3339491953 learned_addr learned my addr 172.21.15.99:0/3339491953 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:44:59.546 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.542+0000 7ffa0a59c640 1 --2- 172.21.15.99:0/3339491953 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559ed45b0310 0x559ed45bce20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:59.546 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.543+0000 7ffa09d9b640 1 -- 172.21.15.99:0/3339491953 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559ed45b0310 msgr2=0x559ed45bce20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:44:59.546 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.543+0000 7ffa09d9b640 1 --2- 172.21.15.99:0/3339491953 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559ed45b0310 0x559ed45bce20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:59.546 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.543+0000 7ffa09d9b640 1 -- 172.21.15.99:0/3339491953 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559ed45b5e70 msgr2=0x559ed45c0380 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:44:59.546 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.543+0000 7ffa09d9b640 1 --2- 172.21.15.99:0/3339491953 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559ed45b5e70 0x559ed45c0380 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:59.546 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.543+0000 7ffa09d9b640 1 -- 172.21.15.99:0/3339491953 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x559ed45c0970 con 0x559ed45b0d00 2024-03-19T01:44:59.546 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.543+0000 7ffa0ad9d640 1 --2- 172.21.15.99:0/3339491953 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559ed45b5e70 0x559ed45c0380 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:44:59.546 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.543+0000 7ffa09d9b640 1 --2- 172.21.15.99:0/3339491953 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559ed45b0d00 0x559ed45bd360 secure :-1 s=READY pgs=109 cs=0 l=1 rev1=1 crypto rx=0x7ffa00000c00 tx=0x7ffa000d2930 comp rx=0 tx=0).ready entity=mon.2 client_cookie=5f807759d836f693 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:44:59.546 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.543+0000 7ff9f1ffb640 10 client.0 ms_handle_connect on v2:172.21.15.112:3300/0 2024-03-19T01:44:59.546 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.543+0000 7ff9f1ffb640 1 -- 172.21.15.99:0/3339491953 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffa0009b1c0 con 0x559ed45b0d00 2024-03-19T01:44:59.546 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.543+0000 7ff9f1ffb640 1 -- 172.21.15.99:0/3339491953 <== mon.2 v2:172.21.15.112:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7ffa00099b60 con 0x559ed45b0d00 2024-03-19T01:44:59.546 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.543+0000 7ffa139e6740 1 -- 172.21.15.99:0/3339491953 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7ffa04030980 con 0x559ed45b0d00 2024-03-19T01:44:59.547 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.544+0000 7ff9f1ffb640 1 -- 172.21.15.99:0/3339491953 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_get_version_reply(handle=1 version=25) ==== 24+0+0 (secure 0 0 0) 0x7ffa0009b860 con 0x559ed45b0d00 2024-03-19T01:44:59.547 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.544+0000 7ffa139e6740 10 client.25427 fetch_fsmap learned FSMap version 25 2024-03-19T01:44:59.547 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.544+0000 7ffa139e6740 1 -- 172.21.15.99:0/3339491953 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,fsmap=25,monmap=4+}) -- 0x559ed463b5e0 con 0x559ed45b0d00 2024-03-19T01:44:59.547 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.544+0000 7ff9f1ffb640 1 -- 172.21.15.99:0/3339491953 <== mon.2 v2:172.21.15.112:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7ffa04030980 con 0x559ed45b0d00 2024-03-19T01:44:59.547 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.544+0000 7ff9f1ffb640 1 -- 172.21.15.99:0/3339491953 <== mon.2 v2:172.21.15.112:3300/0 5 ==== fsmap(e 25) ==== 8338+0+0 (secure 0 0 0) 0x7ffa0009acb0 con 0x559ed45b0d00 2024-03-19T01:44:59.548 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.544+0000 7ffa139e6740 10 client.25427 fetch_fsmap finished waiting for FSMap version 25 2024-03-19T01:44:59.548 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.544+0000 7ffa139e6740 10 client.25427 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:44:59.548 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.545+0000 7ffa139e6740 20 client.25427 populate_metadata read hostname 'smithi099' 2024-03-19T01:44:59.548 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.545+0000 7ffa139e6740 1 --2- 172.21.15.99:0/3339491953 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x559ed463bec0 0x559ed463e2e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:44:59.548 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.545+0000 7ffa139e6740 4 client.25427 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:44:59.548 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.545+0000 7ffa139e6740 1 -- 172.21.15.99:0/3339491953 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x559ed451a470 con 0x559ed463bec0 2024-03-19T01:44:59.548 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.545+0000 7ffa0a59c640 1 --2- 172.21.15.99:0/3339491953 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x559ed463bec0 0x559ed463e2e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:44:59.548 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.545+0000 7ffa0a59c640 1 --2- 172.21.15.99:0/3339491953 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x559ed463bec0 0x559ed463e2e0 crc :-1 s=READY pgs=64 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=7e0ec205fd8eafb server_cookie=b022f6754e3c2054 in_seq=0 out_seq=0 2024-03-19T01:44:59.548 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.545+0000 7ff9f1ffb640 10 client.25427 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:44:59.603 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.603+0000 7ff9f1ffb640 1 -- 172.21.15.99:0/3339491953 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x559ed451a470 con 0x559ed463bec0 2024-03-19T01:44:59.603 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.603+0000 7ff9f1ffb640 10 client.25427 handle_command_reply: tid=0 2024-03-19T01:44:59.603 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.603+0000 7ffa139e6740 1 client.25427 shutdown 2024-03-19T01:44:59.603 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.603+0000 7ffa139e6740 1 -- 172.21.15.99:0/3339491953 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559ed45b0d00 msgr2=0x559ed45bd360 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:44:59.603 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.603+0000 7ffa139e6740 1 --2- 172.21.15.99:0/3339491953 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559ed45b0d00 0x559ed45bd360 secure :-1 s=READY pgs=109 cs=0 l=1 rev1=1 crypto rx=0x7ffa00000c00 tx=0x7ffa000d2930 comp rx=0 tx=0).stop 2024-03-19T01:44:59.604 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.603+0000 7ffa139e6740 1 -- 172.21.15.99:0/3339491953 shutdown_connections 2024-03-19T01:44:59.604 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.603+0000 7ffa139e6740 1 --2- 172.21.15.99:0/3339491953 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x559ed463bec0 0x559ed463e2e0 crc :-1 s=READY pgs=64 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:59.604 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.603+0000 7ffa139e6740 1 --2- 172.21.15.99:0/3339491953 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559ed45b0d00 0x559ed45bd360 unknown :-1 s=CLOSED pgs=109 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:59.604 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.603+0000 7ffa139e6740 1 --2- 172.21.15.99:0/3339491953 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559ed45b5e70 0x559ed45c0380 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:59.604 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.603+0000 7ff9f1ffb640 0 client.25427 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:44:59.604 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.603+0000 7ffa139e6740 1 --2- 172.21.15.99:0/3339491953 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559ed45b0310 0x559ed45bce20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:59.604 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.603+0000 7ffa139e6740 1 -- 172.21.15.99:0/3339491953 >> 172.21.15.99:0/3339491953 conn(0x559ed45a9670 msgr2=0x559ed45b16a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:44:59.604 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.603+0000 7ffa139e6740 1 -- 172.21.15.99:0/3339491953 shutdown_connections 2024-03-19T01:44:59.604 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.603+0000 7ffa139e6740 1 -- 172.21.15.99:0/3339491953 wait complete. 2024-03-19T01:44:59.604 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.603+0000 7ffa139e6740 20 client.25427 trim_cache size 0 max 16384 2024-03-19T01:44:59.605 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:44:59.605 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:44:59.605 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:44:59.605 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:44:59.606 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.604+0000 7ffa10df4640 1 -- 172.21.15.99:0/779102332 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff9f40a28a0 msgr2=0x7ff9f40a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:44:59.606 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.604+0000 7ffa10df4640 1 --2- 172.21.15.99:0/779102332 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff9f40a28a0 0x7ff9f40a4d60 secure :-1 s=READY pgs=265 cs=0 l=1 rev1=1 crypto rx=0x7ff9fc04d000 tx=0x7ff9fc04a000 comp rx=0 tx=0).stop 2024-03-19T01:44:59.606 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.604+0000 7ffa10df4640 1 -- 172.21.15.99:0/779102332 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ffa0c101930 msgr2=0x7ffa0c196e90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:44:59.606 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.604+0000 7ffa10df4640 1 --2- 172.21.15.99:0/779102332 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ffa0c101930 0x7ffa0c196e90 secure :-1 s=READY pgs=107 cs=0 l=1 rev1=1 crypto rx=0x7ffa0000e9f0 tx=0x7ffa0000eeb0 comp rx=0 tx=0).stop 2024-03-19T01:44:59.606 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.604+0000 7ffa10df4640 1 -- 172.21.15.99:0/779102332 shutdown_connections 2024-03-19T01:44:59.606 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.604+0000 7ffa10df4640 1 --2- 172.21.15.99:0/779102332 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff9f40a28a0 0x7ff9f40a4d60 unknown :-1 s=CLOSED pgs=265 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:59.606 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.604+0000 7ffa10df4640 1 --2- 172.21.15.99:0/779102332 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ffa0c101930 0x7ffa0c196e90 unknown :-1 s=CLOSED pgs=107 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:59.606 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.604+0000 7ffa10df4640 1 --2- 172.21.15.99:0/779102332 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ffa0c1022f0 0x7ffa0c19df00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:59.606 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.604+0000 7ffa10df4640 1 --2- 172.21.15.99:0/779102332 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ffa0c100740 0x7ffa0c196950 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:44:59.606 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.604+0000 7ffa10df4640 1 -- 172.21.15.99:0/779102332 >> 172.21.15.99:0/779102332 conn(0x7ffa0c0fbcb0 msgr2=0x7ffa0c104bd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:44:59.606 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.604+0000 7ffa10df4640 1 -- 172.21.15.99:0/779102332 shutdown_connections 2024-03-19T01:44:59.607 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:44:59.604+0000 7ffa10df4640 1 -- 172.21.15.99:0/779102332 wait complete. 2024-03-19T01:44:59.681 INFO:tasks.workunit.client.0.smithi099.stdout: 1 452271 11.37 MB/sec execute 89 sec latency 42.047 ms 2024-03-19T01:45:00.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:44:59 smithi115 ceph-mon[67332]: pgmap v467: 97 pgs: 97 active+clean; 160 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 1.2 MiB/s rd, 3.0 MiB/s wr, 267 op/s 2024-03-19T01:45:00.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:44:59 smithi099 ceph-mon[62951]: pgmap v467: 97 pgs: 97 active+clean; 160 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 1.2 MiB/s rd, 3.0 MiB/s wr, 267 op/s 2024-03-19T01:45:00.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:44:59 smithi112 ceph-mon[66819]: pgmap v467: 97 pgs: 97 active+clean; 160 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 1.2 MiB/s rd, 3.0 MiB/s wr, 267 op/s 2024-03-19T01:45:00.681 INFO:tasks.workunit.client.0.smithi099.stdout: 1 454508 11.39 MB/sec execute 90 sec latency 58.874 ms 2024-03-19T01:45:01.681 INFO:tasks.workunit.client.0.smithi099.stdout: 1 456468 11.35 MB/sec execute 91 sec latency 57.412 ms 2024-03-19T01:45:02.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:45:01 smithi115 ceph-mon[67332]: pgmap v468: 97 pgs: 97 active+clean; 157 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 2.7 MiB/s wr, 253 op/s 2024-03-19T01:45:02.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:45:01 smithi099 ceph-mon[62951]: pgmap v468: 97 pgs: 97 active+clean; 157 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 2.7 MiB/s wr, 253 op/s 2024-03-19T01:45:02.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:45:01 smithi112 ceph-mon[66819]: pgmap v468: 97 pgs: 97 active+clean; 157 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 2.7 MiB/s wr, 253 op/s 2024-03-19T01:45:02.681 INFO:tasks.workunit.client.0.smithi099.stdout: 1 458344 11.36 MB/sec execute 92 sec latency 27.077 ms 2024-03-19T01:45:03.681 INFO:tasks.workunit.client.0.smithi099.stdout: 1 459860 11.52 MB/sec execute 93 sec latency 54.244 ms 2024-03-19T01:45:04.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:45:03 smithi115 ceph-mon[67332]: pgmap v469: 97 pgs: 97 active+clean; 159 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 3.0 MiB/s wr, 258 op/s 2024-03-19T01:45:04.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:45:03 smithi099 ceph-mon[62951]: pgmap v469: 97 pgs: 97 active+clean; 159 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 3.0 MiB/s wr, 258 op/s 2024-03-19T01:45:04.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:45:03 smithi112 ceph-mon[66819]: pgmap v469: 97 pgs: 97 active+clean; 159 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 3.0 MiB/s wr, 258 op/s 2024-03-19T01:45:04.681 INFO:tasks.workunit.client.0.smithi099.stdout: 1 462334 11.54 MB/sec execute 94 sec latency 11.220 ms 2024-03-19T01:45:05.682 INFO:tasks.workunit.client.0.smithi099.stdout: 1 464473 11.50 MB/sec execute 95 sec latency 23.402 ms 2024-03-19T01:45:06.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:45:05 smithi115 ceph-mon[67332]: pgmap v470: 97 pgs: 97 active+clean; 159 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 1.0 MiB/s rd, 2.9 MiB/s wr, 248 op/s 2024-03-19T01:45:06.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:45:05 smithi099 ceph-mon[62951]: pgmap v470: 97 pgs: 97 active+clean; 159 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 1.0 MiB/s rd, 2.9 MiB/s wr, 248 op/s 2024-03-19T01:45:06.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:45:05 smithi112 ceph-mon[66819]: pgmap v470: 97 pgs: 97 active+clean; 159 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 1.0 MiB/s rd, 2.9 MiB/s wr, 248 op/s 2024-03-19T01:45:06.682 INFO:tasks.workunit.client.0.smithi099.stdout: 1 466156 11.50 MB/sec execute 96 sec latency 63.091 ms 2024-03-19T01:45:07.682 INFO:tasks.workunit.client.0.smithi099.stdout: 1 466847 11.43 MB/sec execute 97 sec latency 697.801 ms 2024-03-19T01:45:08.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:45:07 smithi115 ceph-mon[67332]: pgmap v471: 97 pgs: 97 active+clean; 154 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 3.2 MiB/s wr, 272 op/s 2024-03-19T01:45:08.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:45:07 smithi115 ceph-mon[67332]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T01:45:08.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:45:07 smithi099 ceph-mon[62951]: pgmap v471: 97 pgs: 97 active+clean; 154 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 3.2 MiB/s wr, 272 op/s 2024-03-19T01:45:08.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:45:07 smithi099 ceph-mon[62951]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T01:45:08.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:45:07 smithi112 ceph-mon[66819]: pgmap v471: 97 pgs: 97 active+clean; 154 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 3.2 MiB/s wr, 272 op/s 2024-03-19T01:45:08.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:45:07 smithi112 ceph-mon[66819]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T01:45:08.682 INFO:tasks.workunit.client.0.smithi099.stdout: 1 466847 11.32 MB/sec execute 98 sec latency 1697.853 ms 2024-03-19T01:45:09.682 INFO:tasks.workunit.client.0.smithi099.stdout: 1 466847 11.20 MB/sec execute 99 sec latency 2697.947 ms 2024-03-19T01:45:10.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:45:09 smithi115 ceph-mon[67332]: pgmap v472: 97 pgs: 97 active+clean; 152 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 855 KiB/s rd, 2.7 MiB/s wr, 241 op/s 2024-03-19T01:45:10.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:45:09 smithi099 ceph-mon[62951]: pgmap v472: 97 pgs: 97 active+clean; 152 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 855 KiB/s rd, 2.7 MiB/s wr, 241 op/s 2024-03-19T01:45:10.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:45:09 smithi112 ceph-mon[66819]: pgmap v472: 97 pgs: 97 active+clean; 152 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 855 KiB/s rd, 2.7 MiB/s wr, 241 op/s 2024-03-19T01:45:10.682 INFO:tasks.workunit.client.0.smithi099.stdout: 1 466847 11.09 MB/sec execute 100 sec latency 3698.044 ms 2024-03-19T01:45:11.682 INFO:tasks.workunit.client.0.smithi099.stdout: 1 466847 10.98 MB/sec execute 101 sec latency 4698.128 ms 2024-03-19T01:45:12.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:45:11 smithi115 ceph-mon[67332]: pgmap v473: 97 pgs: 97 active+clean; 146 MiB data, 1.4 GiB used, 1.0 TiB / 1.0 TiB avail; 818 KiB/s rd, 2.8 MiB/s wr, 243 op/s 2024-03-19T01:45:12.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:45:11 smithi099 ceph-mon[62951]: pgmap v473: 97 pgs: 97 active+clean; 146 MiB data, 1.4 GiB used, 1.0 TiB / 1.0 TiB avail; 818 KiB/s rd, 2.8 MiB/s wr, 243 op/s 2024-03-19T01:45:12.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:45:11 smithi112 ceph-mon[66819]: pgmap v473: 97 pgs: 97 active+clean; 146 MiB data, 1.4 GiB used, 1.0 TiB / 1.0 TiB avail; 818 KiB/s rd, 2.8 MiB/s wr, 243 op/s 2024-03-19T01:45:12.682 INFO:tasks.workunit.client.0.smithi099.stdout: 1 469258 11.01 MB/sec execute 102 sec latency 4723.600 ms 2024-03-19T01:45:12.710 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.698+0000 7f71e0b40640 1 -- 172.21.15.99:0/2802129151 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f71dc101930 msgr2=0x7f71dc101db0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:45:12.710 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.698+0000 7f71e0b40640 1 --2- 172.21.15.99:0/2802129151 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f71dc101930 0x7f71dc101db0 secure :-1 s=READY pgs=430 cs=0 l=1 rev1=1 crypto rx=0x7f71c4009900 tx=0x7f71c402bc30 comp rx=0 tx=0).stop 2024-03-19T01:45:12.710 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.698+0000 7f71e0b40640 1 -- 172.21.15.99:0/2802129151 shutdown_connections 2024-03-19T01:45:12.710 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.698+0000 7f71e0b40640 1 --2- 172.21.15.99:0/2802129151 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f71dc1022f0 0x7f71dc108b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:12.710 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.698+0000 7f71e0b40640 1 --2- 172.21.15.99:0/2802129151 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f71dc101930 0x7f71dc101db0 unknown :-1 s=CLOSED pgs=430 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:12.711 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.698+0000 7f71e0b40640 1 --2- 172.21.15.99:0/2802129151 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f71dc100740 0x7f71dc100b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:12.711 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.698+0000 7f71e0b40640 1 -- 172.21.15.99:0/2802129151 >> 172.21.15.99:0/2802129151 conn(0x7f71dc0fbcb0 msgr2=0x7f71dc0fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:45:12.711 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.699+0000 7f71e0b40640 1 -- 172.21.15.99:0/2802129151 shutdown_connections 2024-03-19T01:45:12.711 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.699+0000 7f71e0b40640 1 -- 172.21.15.99:0/2802129151 wait complete. 2024-03-19T01:45:12.711 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.699+0000 7f71e0b40640 1 Processor -- start 2024-03-19T01:45:12.711 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.699+0000 7f71e0b40640 1 -- start start 2024-03-19T01:45:12.711 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.699+0000 7f71e0b40640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f71dc100740 0x7f71dc197aa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:12.711 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.699+0000 7f71e0b40640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f71dc101930 0x7f71dc197fe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:12.711 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.699+0000 7f71e0b40640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f71dc1022f0 0x7f71dc19f050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:12.711 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.699+0000 7f71e0b40640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f71dc10c3c0 con 0x7f71dc101930 2024-03-19T01:45:12.711 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.699+0000 7f71e0b40640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f71dc10c240 con 0x7f71dc100740 2024-03-19T01:45:12.711 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.699+0000 7f71e0b40640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f71dc10c540 con 0x7f71dc1022f0 2024-03-19T01:45:12.711 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.699+0000 7f71d3fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f71dc101930 0x7f71dc197fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:12.711 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.699+0000 7f71da59c640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f71dc100740 0x7f71dc197aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:12.712 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.700+0000 7f71d3fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f71dc101930 0x7f71dc197fe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:46068/0 (socket says 172.21.15.99:46068) 2024-03-19T01:45:12.712 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.700+0000 7f71da59c640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f71dc100740 0x7f71dc197aa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:52902/0 (socket says 172.21.15.99:52902) 2024-03-19T01:45:12.712 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.700+0000 7f71d3fff640 1 -- 172.21.15.99:0/3043836227 learned_addr learned my addr 172.21.15.99:0/3043836227 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:45:12.712 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.700+0000 7f71dad9d640 1 --2- 172.21.15.99:0/3043836227 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f71dc1022f0 0x7f71dc19f050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:12.712 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.700+0000 7f71d3fff640 1 -- 172.21.15.99:0/3043836227 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f71dc1022f0 msgr2=0x7f71dc19f050 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:45:12.712 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.700+0000 7f71d3fff640 1 --2- 172.21.15.99:0/3043836227 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f71dc1022f0 0x7f71dc19f050 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:12.712 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.700+0000 7f71d3fff640 1 -- 172.21.15.99:0/3043836227 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f71dc100740 msgr2=0x7f71dc197aa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:45:12.712 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.700+0000 7f71d3fff640 1 --2- 172.21.15.99:0/3043836227 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f71dc100740 0x7f71dc197aa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:12.712 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.700+0000 7f71d3fff640 1 -- 172.21.15.99:0/3043836227 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f71dc19f6c0 con 0x7f71dc101930 2024-03-19T01:45:12.712 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.700+0000 7f71da59c640 1 --2- 172.21.15.99:0/3043836227 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f71dc100740 0x7f71dc197aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:45:12.712 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.700+0000 7f71dad9d640 1 --2- 172.21.15.99:0/3043836227 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f71dc1022f0 0x7f71dc19f050 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:45:12.712 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.700+0000 7f71d3fff640 1 --2- 172.21.15.99:0/3043836227 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f71dc101930 0x7f71dc197fe0 secure :-1 s=READY pgs=431 cs=0 l=1 rev1=1 crypto rx=0x7f71c4009230 tx=0x7f71c402c650 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:45:12.712 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.700+0000 7f71d37fe640 1 -- 172.21.15.99:0/3043836227 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f71c4041070 con 0x7f71dc101930 2024-03-19T01:45:12.713 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.700+0000 7f71e0b40640 1 -- 172.21.15.99:0/3043836227 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f71dc19f950 con 0x7f71dc101930 2024-03-19T01:45:12.713 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.700+0000 7f71d37fe640 1 -- 172.21.15.99:0/3043836227 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f71c4004450 con 0x7f71dc101930 2024-03-19T01:45:12.713 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.700+0000 7f71d37fe640 1 -- 172.21.15.99:0/3043836227 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f71c4003e20 con 0x7f71dc101930 2024-03-19T01:45:12.713 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.700+0000 7f71e0b40640 1 -- 172.21.15.99:0/3043836227 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f71dc19fe40 con 0x7f71dc101930 2024-03-19T01:45:12.713 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.703+0000 7f71d37fe640 1 -- 172.21.15.99:0/3043836227 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f71c4044070 con 0x7f71dc101930 2024-03-19T01:45:12.713 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.703+0000 7f71e37fd740 1 Processor -- start 2024-03-19T01:45:12.713 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.703+0000 7f71e37fd740 1 -- start start 2024-03-19T01:45:12.713 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.703+0000 7f71e37fd740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556382bc5100 0x556382bc5520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:12.713 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.703+0000 7f71e37fd740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556382bc5af0 0x556382bca5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:12.713 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.703+0000 7f71e37fd740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556382bcac60 0x556382bcd0f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:12.713 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.703+0000 7f71e37fd740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x556382b29be0 con 0x556382bc5100 2024-03-19T01:45:12.713 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.703+0000 7f71e37fd740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x556382ad2960 con 0x556382bc5af0 2024-03-19T01:45:12.713 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.703+0000 7f71e37fd740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x556382b25c80 con 0x556382bcac60 2024-03-19T01:45:12.713 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.704+0000 7f71dad9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556382bc5100 0x556382bc5520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:12.714 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.704+0000 7f71dad9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556382bc5100 0x556382bc5520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:46078/0 (socket says 172.21.15.99:46078) 2024-03-19T01:45:12.714 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.704+0000 7f71d3fff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556382bcac60 0x556382bcd0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:12.714 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.704+0000 7f71dad9d640 1 -- 172.21.15.99:0/1000770840 learned_addr learned my addr 172.21.15.99:0/1000770840 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:45:12.714 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.704+0000 7f71da59c640 1 --2- 172.21.15.99:0/1000770840 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556382bc5af0 0x556382bca5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:12.714 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.704+0000 7f71d37fe640 1 --2- 172.21.15.99:0/3043836227 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f71c80a28a0 0x7f71c80a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:12.714 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.704+0000 7f71da59c640 1 -- 172.21.15.99:0/1000770840 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556382bcac60 msgr2=0x556382bcd0f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:45:12.714 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.704+0000 7f71da59c640 1 --2- 172.21.15.99:0/1000770840 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556382bcac60 0x556382bcd0f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:12.714 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.704+0000 7f71da59c640 1 -- 172.21.15.99:0/1000770840 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556382bc5100 msgr2=0x556382bc5520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:45:12.714 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.704+0000 7f71da59c640 1 --2- 172.21.15.99:0/1000770840 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556382bc5100 0x556382bc5520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:12.714 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.704+0000 7f71da59c640 1 -- 172.21.15.99:0/1000770840 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x556382ac9d20 con 0x556382bc5af0 2024-03-19T01:45:12.714 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.704+0000 7f71dad9d640 1 --2- 172.21.15.99:0/3043836227 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f71c80a28a0 0x7f71c80a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:12.714 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.704+0000 7f71da59c640 1 --2- 172.21.15.99:0/1000770840 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556382bc5af0 0x556382bca5f0 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 crypto rx=0x7f71cc009fa0 tx=0x7f71cc0118d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=43d97764e4f63cbe server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:45:12.714 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.704+0000 7f71d37fe640 1 -- 172.21.15.99:0/3043836227 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f71c40f05f0 con 0x7f71dc101930 2024-03-19T01:45:12.715 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.704+0000 7f71d0ff9640 1 -- 172.21.15.99:0/1000770840 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f71cc004310 con 0x556382bc5af0 2024-03-19T01:45:12.715 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.704+0000 7f71d0ff9640 1 -- 172.21.15.99:0/1000770840 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f71cc0044b0 con 0x556382bc5af0 2024-03-19T01:45:12.715 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.704+0000 7f71d0ff9640 1 -- 172.21.15.99:0/1000770840 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f71cc0035f0 con 0x556382bc5af0 2024-03-19T01:45:12.715 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.705+0000 7f71e37fd740 1 -- 172.21.15.99:0/1000770840 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556382bc5af0 msgr2=0x556382bca5f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:45:12.715 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.705+0000 7f71e37fd740 1 --2- 172.21.15.99:0/1000770840 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556382bc5af0 0x556382bca5f0 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 crypto rx=0x7f71cc009fa0 tx=0x7f71cc0118d0 comp rx=0 tx=0).stop 2024-03-19T01:45:12.715 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.705+0000 7f71dad9d640 1 --2- 172.21.15.99:0/3043836227 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f71c80a28a0 0x7f71c80a4d60 secure :-1 s=READY pgs=266 cs=0 l=1 rev1=1 crypto rx=0x556382bc6350 tx=0x7f71d4004a40 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:45:12.715 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.705+0000 7f71e37fd740 1 -- 172.21.15.99:0/1000770840 shutdown_connections 2024-03-19T01:45:12.715 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.705+0000 7f71e37fd740 1 --2- 172.21.15.99:0/1000770840 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556382bc5af0 0x556382bca5f0 unknown :-1 s=CLOSED pgs=110 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:12.715 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.705+0000 7f71e37fd740 1 --2- 172.21.15.99:0/1000770840 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556382bcac60 0x556382bcd0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:12.715 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.705+0000 7f71e37fd740 1 --2- 172.21.15.99:0/1000770840 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556382bc5100 0x556382bc5520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:12.715 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.705+0000 7f71e37fd740 1 -- 172.21.15.99:0/1000770840 >> 172.21.15.99:0/1000770840 conn(0x556382bc0670 msgr2=0x556382bc0a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:45:12.715 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.705+0000 7f71e37fd740 1 -- 172.21.15.99:0/1000770840 shutdown_connections 2024-03-19T01:45:12.715 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.705+0000 7f71e37fd740 1 -- 172.21.15.99:0/1000770840 wait complete. 2024-03-19T01:45:12.715 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.705+0000 7f71e37fd740 1 -- start start 2024-03-19T01:45:12.715 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.705+0000 7f71e37fd740 1 Processor -- start 2024-03-19T01:45:12.716 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.706+0000 7f71e37fd740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556382bc5100 0x556382bcf130 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:12.716 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.706+0000 7f71e37fd740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556382bc5af0 0x556382bcf670 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:12.716 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.706+0000 7f71e37fd740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556382bcac60 0x556382bd66b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:12.716 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.706+0000 7f71e37fd740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x556382bcdc10 con 0x556382bc5af0 2024-03-19T01:45:12.716 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.706+0000 7f71e37fd740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x556382ae5db0 con 0x556382bc5100 2024-03-19T01:45:12.716 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.706+0000 7f71e37fd740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x556382bcdd90 con 0x556382bcac60 2024-03-19T01:45:12.716 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.706+0000 7f71e37fd740 10 client.0 _ll_register_callbacks cb 0x556382aeae70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:45:12.716 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.706+0000 7f71d3fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556382bc5af0 0x556382bcf670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:12.716 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.706+0000 7f71da59c640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556382bc5100 0x556382bcf130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:12.716 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.706+0000 7f71dad9d640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556382bcac60 0x556382bd66b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:12.716 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.706+0000 7f71da59c640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556382bc5100 0x556382bcf130 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:52928/0 (socket says 172.21.15.99:52928) 2024-03-19T01:45:12.716 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.706+0000 7f71dad9d640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556382bcac60 0x556382bd66b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:42830/0 (socket says 172.21.15.99:42830) 2024-03-19T01:45:12.716 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.706+0000 7f71da59c640 1 -- 172.21.15.99:0/908642285 learned_addr learned my addr 172.21.15.99:0/908642285 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:45:12.716 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.706+0000 7f71da59c640 1 -- 172.21.15.99:0/908642285 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556382bcac60 msgr2=0x556382bd66b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:45:12.717 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.706+0000 7f71da59c640 1 --2- 172.21.15.99:0/908642285 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556382bcac60 0x556382bd66b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:12.717 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.706+0000 7f71da59c640 1 -- 172.21.15.99:0/908642285 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556382bc5af0 msgr2=0x556382bcf670 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:45:12.717 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.706+0000 7f71da59c640 1 --2- 172.21.15.99:0/908642285 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556382bc5af0 0x556382bcf670 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:12.717 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.706+0000 7f71da59c640 1 -- 172.21.15.99:0/908642285 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x556382ae4c60 con 0x556382bc5100 2024-03-19T01:45:12.717 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.706+0000 7f71da59c640 1 --2- 172.21.15.99:0/908642285 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556382bc5100 0x556382bcf130 secure :-1 s=READY pgs=111 cs=0 l=1 rev1=1 crypto rx=0x7f71cc01b040 tx=0x7f71cc009d00 comp rx=0 tx=0).ready entity=mon.2 client_cookie=82d4c558327f652e server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:45:12.717 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.706+0000 7f71c1ffb640 10 client.0 ms_handle_connect on v2:172.21.15.112:3300/0 2024-03-19T01:45:12.717 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.706+0000 7f71c1ffb640 1 -- 172.21.15.99:0/908642285 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f71cc028070 con 0x556382bc5100 2024-03-19T01:45:12.717 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.707+0000 7f71e37fd740 1 -- 172.21.15.99:0/908642285 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f71cc0044b0 con 0x556382bc5100 2024-03-19T01:45:12.717 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.707+0000 7f71c1ffb640 1 -- 172.21.15.99:0/908642285 <== mon.2 v2:172.21.15.112:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f71cc005000 con 0x556382bc5100 2024-03-19T01:45:12.717 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.707+0000 7f71c1ffb640 1 -- 172.21.15.99:0/908642285 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_get_version_reply(handle=1 version=26) ==== 24+0+0 (secure 0 0 0) 0x7f71cc034050 con 0x556382bc5100 2024-03-19T01:45:12.717 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.707+0000 7f71e37fd740 10 client.25439 fetch_fsmap learned FSMap version 26 2024-03-19T01:45:12.717 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.707+0000 7f71e37fd740 1 -- 172.21.15.99:0/908642285 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,fsmap=26,monmap=4+}) -- 0x55638283e210 con 0x556382bc5100 2024-03-19T01:45:12.717 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.707+0000 7f71c1ffb640 1 -- 172.21.15.99:0/908642285 <== mon.2 v2:172.21.15.112:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f71cc0044b0 con 0x556382bc5100 2024-03-19T01:45:12.717 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.707+0000 7f71c1ffb640 1 -- 172.21.15.99:0/908642285 <== mon.2 v2:172.21.15.112:3300/0 5 ==== fsmap(e 26) ==== 8342+0+0 (secure 0 0 0) 0x7f71cc003cf0 con 0x556382bc5100 2024-03-19T01:45:12.718 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.707+0000 7f71e37fd740 10 client.25439 fetch_fsmap finished waiting for FSMap version 26 2024-03-19T01:45:12.718 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.707+0000 7f71e37fd740 10 client.25439 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:45:12.718 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.707+0000 7f71e37fd740 20 client.25439 populate_metadata read hostname 'smithi099' 2024-03-19T01:45:12.718 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.707+0000 7f71e37fd740 1 --2- 172.21.15.99:0/908642285 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x556382bdad50 0x556382bdd180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:12.718 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.707+0000 7f71e37fd740 4 client.25439 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:45:12.718 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.707+0000 7f71e37fd740 1 -- 172.21.15.99:0/908642285 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x556382b37120 con 0x556382bdad50 2024-03-19T01:45:12.718 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.707+0000 7f71d3fff640 1 --2- 172.21.15.99:0/908642285 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x556382bdad50 0x556382bdd180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:12.718 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.708+0000 7f71c1ffb640 10 client.25439 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:45:12.718 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.708+0000 7f71d3fff640 1 --2- 172.21.15.99:0/908642285 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x556382bdad50 0x556382bdd180 crc :-1 s=READY pgs=65 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=b10f3edefb970c45 server_cookie=810a81072d1230ee in_seq=0 out_seq=0 2024-03-19T01:45:12.718 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.709+0000 7f71c1ffb640 1 -- 172.21.15.99:0/908642285 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x556382b37120 con 0x556382bdad50 2024-03-19T01:45:12.718 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.709+0000 7f71c1ffb640 10 client.25439 handle_command_reply: tid=0 2024-03-19T01:45:12.718 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.709+0000 7f71e37fd740 1 client.25439 shutdown 2024-03-19T01:45:12.718 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.709+0000 7f71e37fd740 1 -- 172.21.15.99:0/908642285 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556382bc5100 msgr2=0x556382bcf130 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:45:12.719 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.709+0000 7f71e37fd740 1 --2- 172.21.15.99:0/908642285 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556382bc5100 0x556382bcf130 secure :-1 s=READY pgs=111 cs=0 l=1 rev1=1 crypto rx=0x7f71cc01b040 tx=0x7f71cc009d00 comp rx=0 tx=0).stop 2024-03-19T01:45:12.719 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.709+0000 7f71e37fd740 1 -- 172.21.15.99:0/908642285 shutdown_connections 2024-03-19T01:45:12.719 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.709+0000 7f71e37fd740 1 --2- 172.21.15.99:0/908642285 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x556382bdad50 0x556382bdd180 crc :-1 s=READY pgs=65 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:12.719 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.709+0000 7f71e37fd740 1 --2- 172.21.15.99:0/908642285 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556382bcac60 0x556382bd66b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:12.719 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.709+0000 7f71e37fd740 1 --2- 172.21.15.99:0/908642285 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556382bc5af0 0x556382bcf670 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:12.719 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.709+0000 7f71c1ffb640 0 client.25439 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:45:12.719 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.709+0000 7f71e37fd740 1 --2- 172.21.15.99:0/908642285 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556382bc5100 0x556382bcf130 unknown :-1 s=CLOSED pgs=111 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:12.719 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.709+0000 7f71e37fd740 1 -- 172.21.15.99:0/908642285 >> 172.21.15.99:0/908642285 conn(0x556382bc0670 msgr2=0x556382bc18c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:45:12.719 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.709+0000 7f71e37fd740 1 -- 172.21.15.99:0/908642285 shutdown_connections 2024-03-19T01:45:12.719 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.709+0000 7f71e37fd740 1 -- 172.21.15.99:0/908642285 wait complete. 2024-03-19T01:45:12.719 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.709+0000 7f71e37fd740 20 client.25439 trim_cache size 0 max 16384 2024-03-19T01:45:12.728 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.724+0000 7f71e37fd740 1 Processor -- start 2024-03-19T01:45:12.728 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.724+0000 7f71e37fd740 1 -- start start 2024-03-19T01:45:12.728 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.724+0000 7f71e37fd740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556382bc5100 0x556382bce750 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:12.728 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.724+0000 7f71e37fd740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556382bc5af0 0x556382bd6670 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:12.728 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.724+0000 7f71e37fd740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556382bcac60 0x556382bcec90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:12.729 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.724+0000 7f71e37fd740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f71dc1ab6f0 con 0x556382bc5100 2024-03-19T01:45:12.729 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.724+0000 7f71e37fd740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f71dc1a8fc0 con 0x556382bc5af0 2024-03-19T01:45:12.729 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.724+0000 7f71e37fd740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f71dc1a6e10 con 0x556382bcac60 2024-03-19T01:45:12.729 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.724+0000 7f71da59c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556382bc5100 0x556382bce750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:12.729 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.724+0000 7f71da59c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556382bc5100 0x556382bce750 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:46106/0 (socket says 172.21.15.99:46106) 2024-03-19T01:45:12.729 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.724+0000 7f71da59c640 1 -- 172.21.15.99:0/1636062332 learned_addr learned my addr 172.21.15.99:0/1636062332 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:45:12.729 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.724+0000 7f71d3fff640 1 --2- 172.21.15.99:0/1636062332 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556382bc5af0 0x556382bd6670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:12.729 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.724+0000 7f71dad9d640 1 --2- 172.21.15.99:0/1636062332 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556382bcac60 0x556382bcec90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:12.729 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.724+0000 7f71da59c640 1 -- 172.21.15.99:0/1636062332 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556382bcac60 msgr2=0x556382bcec90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:45:12.729 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.724+0000 7f71da59c640 1 --2- 172.21.15.99:0/1636062332 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556382bcac60 0x556382bcec90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:12.729 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.724+0000 7f71da59c640 1 -- 172.21.15.99:0/1636062332 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556382bc5af0 msgr2=0x556382bd6670 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:45:12.729 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.724+0000 7f71da59c640 1 --2- 172.21.15.99:0/1636062332 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556382bc5af0 0x556382bd6670 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:12.729 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.724+0000 7f71da59c640 1 -- 172.21.15.99:0/1636062332 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55638290a620 con 0x556382bc5100 2024-03-19T01:45:12.729 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.724+0000 7f71da59c640 1 --2- 172.21.15.99:0/1636062332 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556382bc5100 0x556382bce750 secure :-1 s=READY pgs=432 cs=0 l=1 rev1=1 crypto rx=0x7f71cc0058c0 tx=0x7f71cc001e50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a514a73b498e3e93 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:45:12.730 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.725+0000 7f71c3fff640 1 -- 172.21.15.99:0/1636062332 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f71cc03d070 con 0x556382bc5100 2024-03-19T01:45:12.730 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.725+0000 7f71c3fff640 1 -- 172.21.15.99:0/1636062332 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f71cc00f770 con 0x556382bc5100 2024-03-19T01:45:12.730 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.725+0000 7f71c3fff640 1 -- 172.21.15.99:0/1636062332 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f71cc028070 con 0x556382bc5100 2024-03-19T01:45:12.730 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.725+0000 7f71e37fd740 1 -- 172.21.15.99:0/1636062332 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556382bc5100 msgr2=0x556382bce750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:45:12.730 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.725+0000 7f71e37fd740 1 --2- 172.21.15.99:0/1636062332 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556382bc5100 0x556382bce750 secure :-1 s=READY pgs=432 cs=0 l=1 rev1=1 crypto rx=0x7f71cc0058c0 tx=0x7f71cc001e50 comp rx=0 tx=0).stop 2024-03-19T01:45:12.730 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.725+0000 7f71e37fd740 1 -- 172.21.15.99:0/1636062332 shutdown_connections 2024-03-19T01:45:12.730 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.725+0000 7f71e37fd740 1 --2- 172.21.15.99:0/1636062332 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556382bc5af0 0x556382bd6670 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:12.730 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.725+0000 7f71e37fd740 1 --2- 172.21.15.99:0/1636062332 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556382bcac60 0x556382bcec90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:12.730 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.725+0000 7f71e37fd740 1 --2- 172.21.15.99:0/1636062332 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556382bc5100 0x556382bce750 unknown :-1 s=CLOSED pgs=432 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:12.730 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.725+0000 7f71e37fd740 1 -- 172.21.15.99:0/1636062332 >> 172.21.15.99:0/1636062332 conn(0x556382bc0670 msgr2=0x556382bc4160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:45:12.730 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.725+0000 7f71e37fd740 1 -- 172.21.15.99:0/1636062332 shutdown_connections 2024-03-19T01:45:12.730 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.725+0000 7f71e37fd740 1 -- 172.21.15.99:0/1636062332 wait complete. 2024-03-19T01:45:12.730 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.726+0000 7f71e37fd740 1 -- start start 2024-03-19T01:45:12.730 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.726+0000 7f71e37fd740 1 Processor -- start 2024-03-19T01:45:12.731 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.726+0000 7f71e37fd740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556382bc5100 0x556382bd3870 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:12.731 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.726+0000 7f71e37fd740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556382bc5af0 0x556382bd3db0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:12.731 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.726+0000 7f71e37fd740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556382bcac60 0x556382bd4dc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:12.731 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.726+0000 7f71e37fd740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f71dc1a7480 con 0x556382bcac60 2024-03-19T01:45:12.731 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.726+0000 7f71e37fd740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f71dc1a88f0 con 0x556382bc5af0 2024-03-19T01:45:12.731 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.726+0000 7f71e37fd740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f71dc1a7b50 con 0x556382bc5100 2024-03-19T01:45:12.731 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.726+0000 7f71e37fd740 10 client.0 _ll_register_callbacks cb 0x7f71d4009ee0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:45:12.731 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.726+0000 7f71dad9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556382bcac60 0x556382bd4dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:12.731 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.726+0000 7f71da59c640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556382bc5100 0x556382bd3870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:12.731 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.726+0000 7f71d3fff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556382bc5af0 0x556382bd3db0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:12.731 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.726+0000 7f71dad9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556382bcac60 0x556382bd4dc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:46108/0 (socket says 172.21.15.99:46108) 2024-03-19T01:45:12.731 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.726+0000 7f71dad9d640 1 -- 172.21.15.99:0/360643233 learned_addr learned my addr 172.21.15.99:0/360643233 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:45:12.731 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.726+0000 7f71d3fff640 1 -- 172.21.15.99:0/360643233 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556382bc5100 msgr2=0x556382bd3870 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:45:12.732 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.726+0000 7f71d3fff640 1 --2- 172.21.15.99:0/360643233 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556382bc5100 0x556382bd3870 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:12.732 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.726+0000 7f71d3fff640 1 -- 172.21.15.99:0/360643233 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556382bcac60 msgr2=0x556382bd4dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:45:12.732 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.726+0000 7f71d3fff640 1 --2- 172.21.15.99:0/360643233 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556382bcac60 0x556382bd4dc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:12.732 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.726+0000 7f71d3fff640 1 -- 172.21.15.99:0/360643233 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x556382bd53b0 con 0x556382bc5af0 2024-03-19T01:45:12.732 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.726+0000 7f71dad9d640 1 --2- 172.21.15.99:0/360643233 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556382bcac60 0x556382bd4dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:45:12.732 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.726+0000 7f71d3fff640 1 --2- 172.21.15.99:0/360643233 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556382bc5af0 0x556382bd3db0 secure :-1 s=READY pgs=112 cs=0 l=1 rev1=1 crypto rx=0x7f71c40ee100 tx=0x7f71c40dd040 comp rx=0 tx=0).ready entity=mon.2 client_cookie=20ec8bc20fe55126 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:45:12.732 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.726+0000 7f71c1ffb640 10 client.0 ms_handle_connect on v2:172.21.15.112:3300/0 2024-03-19T01:45:12.732 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.727+0000 7f71c1ffb640 1 -- 172.21.15.99:0/360643233 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f71c404b040 con 0x556382bc5af0 2024-03-19T01:45:12.732 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.727+0000 7f71e37fd740 1 -- 172.21.15.99:0/360643233 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f71cc00f770 con 0x556382bc5af0 2024-03-19T01:45:12.732 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.727+0000 7f71c1ffb640 1 -- 172.21.15.99:0/360643233 <== mon.2 v2:172.21.15.112:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f71c40035f0 con 0x556382bc5af0 2024-03-19T01:45:12.732 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.727+0000 7f71c1ffb640 1 -- 172.21.15.99:0/360643233 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_get_version_reply(handle=1 version=26) ==== 24+0+0 (secure 0 0 0) 0x7f71c40db030 con 0x556382bc5af0 2024-03-19T01:45:12.732 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.727+0000 7f71e37fd740 10 client.25445 fetch_fsmap learned FSMap version 26 2024-03-19T01:45:12.732 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.727+0000 7f71e37fd740 1 -- 172.21.15.99:0/360643233 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,fsmap=26,monmap=4+}) -- 0x556382bd6730 con 0x556382bc5af0 2024-03-19T01:45:12.732 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.727+0000 7f71c1ffb640 1 -- 172.21.15.99:0/360643233 <== mon.2 v2:172.21.15.112:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f71cc00f770 con 0x556382bc5af0 2024-03-19T01:45:12.733 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.727+0000 7f71c1ffb640 1 -- 172.21.15.99:0/360643233 <== mon.2 v2:172.21.15.112:3300/0 5 ==== fsmap(e 26) ==== 8342+0+0 (secure 0 0 0) 0x7f71c40b9670 con 0x556382bc5af0 2024-03-19T01:45:12.733 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.727+0000 7f71e37fd740 10 client.25445 fetch_fsmap finished waiting for FSMap version 26 2024-03-19T01:45:12.733 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.727+0000 7f71e37fd740 10 client.25445 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:45:12.733 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.727+0000 7f71e37fd740 20 client.25445 populate_metadata read hostname 'smithi099' 2024-03-19T01:45:12.733 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.727+0000 7f71e37fd740 1 --2- 172.21.15.99:0/360643233 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x556382c41540 0x556382c43960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:12.733 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.727+0000 7f71e37fd740 4 client.25445 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:45:12.733 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.727+0000 7f71e37fd740 1 -- 172.21.15.99:0/360643233 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x556382b31470 con 0x556382c41540 2024-03-19T01:45:12.733 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.727+0000 7f71da59c640 1 --2- 172.21.15.99:0/360643233 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x556382c41540 0x556382c43960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:12.733 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.728+0000 7f71da59c640 1 --2- 172.21.15.99:0/360643233 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x556382c41540 0x556382c43960 crc :-1 s=READY pgs=66 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=c22daae87fabb860 server_cookie=b563c67e7d6fb58 in_seq=0 out_seq=0 2024-03-19T01:45:12.733 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.728+0000 7f71c1ffb640 10 client.25445 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:45:12.777 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.776+0000 7f71c1ffb640 1 -- 172.21.15.99:0/360643233 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x556382b31470 con 0x556382c41540 2024-03-19T01:45:12.777 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.776+0000 7f71c1ffb640 10 client.25445 handle_command_reply: tid=0 2024-03-19T01:45:12.777 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.777+0000 7f71e37fd740 1 client.25445 shutdown 2024-03-19T01:45:12.778 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.777+0000 7f71e37fd740 1 -- 172.21.15.99:0/360643233 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556382bc5af0 msgr2=0x556382bd3db0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:45:12.778 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.777+0000 7f71e37fd740 1 --2- 172.21.15.99:0/360643233 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556382bc5af0 0x556382bd3db0 secure :-1 s=READY pgs=112 cs=0 l=1 rev1=1 crypto rx=0x7f71c40ee100 tx=0x7f71c40dd040 comp rx=0 tx=0).stop 2024-03-19T01:45:12.778 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.777+0000 7f71e37fd740 1 -- 172.21.15.99:0/360643233 shutdown_connections 2024-03-19T01:45:12.778 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.777+0000 7f71e37fd740 1 --2- 172.21.15.99:0/360643233 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x556382c41540 0x556382c43960 crc :-1 s=READY pgs=66 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:12.778 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.777+0000 7f71e37fd740 1 --2- 172.21.15.99:0/360643233 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556382bc5af0 0x556382bd3db0 unknown :-1 s=CLOSED pgs=112 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:12.778 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.777+0000 7f71e37fd740 1 --2- 172.21.15.99:0/360643233 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556382bcac60 0x556382bd4dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:12.778 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.777+0000 7f71c1ffb640 0 client.25445 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:45:12.778 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.777+0000 7f71e37fd740 1 --2- 172.21.15.99:0/360643233 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556382bc5100 0x556382bd3870 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:12.778 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.777+0000 7f71e37fd740 1 -- 172.21.15.99:0/360643233 >> 172.21.15.99:0/360643233 conn(0x556382bc0670 msgr2=0x556382bc6380 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:45:12.778 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.777+0000 7f71e37fd740 1 -- 172.21.15.99:0/360643233 shutdown_connections 2024-03-19T01:45:12.778 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.777+0000 7f71e37fd740 1 -- 172.21.15.99:0/360643233 wait complete. 2024-03-19T01:45:12.778 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.777+0000 7f71e37fd740 20 client.25445 trim_cache size 0 max 16384 2024-03-19T01:45:12.779 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:45:12.779 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:45:12.779 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:45:12.779 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:45:12.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.777+0000 7f71e0b40640 1 -- 172.21.15.99:0/3043836227 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f71c80a28a0 msgr2=0x7f71c80a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:45:12.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.777+0000 7f71e0b40640 1 --2- 172.21.15.99:0/3043836227 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f71c80a28a0 0x7f71c80a4d60 secure :-1 s=READY pgs=266 cs=0 l=1 rev1=1 crypto rx=0x556382bc6350 tx=0x7f71d4004a40 comp rx=0 tx=0).stop 2024-03-19T01:45:12.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.777+0000 7f71e0b40640 1 -- 172.21.15.99:0/3043836227 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f71dc101930 msgr2=0x7f71dc197fe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:45:12.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.777+0000 7f71e0b40640 1 --2- 172.21.15.99:0/3043836227 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f71dc101930 0x7f71dc197fe0 secure :-1 s=READY pgs=431 cs=0 l=1 rev1=1 crypto rx=0x7f71c4009230 tx=0x7f71c402c650 comp rx=0 tx=0).stop 2024-03-19T01:45:12.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.778+0000 7f71e0b40640 1 -- 172.21.15.99:0/3043836227 shutdown_connections 2024-03-19T01:45:12.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.778+0000 7f71e0b40640 1 --2- 172.21.15.99:0/3043836227 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f71dc1022f0 0x7f71dc19f050 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:12.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.778+0000 7f71e0b40640 1 --2- 172.21.15.99:0/3043836227 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f71dc101930 0x7f71dc197fe0 unknown :-1 s=CLOSED pgs=431 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:12.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.778+0000 7f71e0b40640 1 --2- 172.21.15.99:0/3043836227 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f71c80a28a0 0x7f71c80a4d60 unknown :-1 s=CLOSED pgs=266 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:12.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.778+0000 7f71e0b40640 1 --2- 172.21.15.99:0/3043836227 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f71dc100740 0x7f71dc197aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:12.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.778+0000 7f71e0b40640 1 -- 172.21.15.99:0/3043836227 >> 172.21.15.99:0/3043836227 conn(0x7f71dc0fbcb0 msgr2=0x7f71dc0fd6d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:45:12.781 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.778+0000 7f71e0b40640 1 -- 172.21.15.99:0/3043836227 shutdown_connections 2024-03-19T01:45:12.781 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:12.778+0000 7f71e0b40640 1 -- 172.21.15.99:0/3043836227 wait complete. 2024-03-19T01:45:13.682 INFO:tasks.workunit.client.0.smithi099.stdout: 1 471335 10.97 MB/sec execute 103 sec latency 19.640 ms 2024-03-19T01:45:14.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:45:13 smithi115 ceph-mon[67332]: pgmap v474: 97 pgs: 97 active+clean; 146 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 707 KiB/s rd, 2.8 MiB/s wr, 223 op/s 2024-03-19T01:45:14.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:45:13 smithi099 ceph-mon[62951]: pgmap v474: 97 pgs: 97 active+clean; 146 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 707 KiB/s rd, 2.8 MiB/s wr, 223 op/s 2024-03-19T01:45:14.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:45:13 smithi112 ceph-mon[66819]: pgmap v474: 97 pgs: 97 active+clean; 146 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 707 KiB/s rd, 2.8 MiB/s wr, 223 op/s 2024-03-19T01:45:14.682 INFO:tasks.workunit.client.0.smithi099.stdout: 1 473294 10.99 MB/sec execute 104 sec latency 30.806 ms 2024-03-19T01:45:15.682 INFO:tasks.workunit.client.0.smithi099.stdout: 1 476067 11.05 MB/sec execute 105 sec latency 14.510 ms 2024-03-19T01:45:16.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:45:15 smithi115 ceph-mon[67332]: pgmap v475: 97 pgs: 97 active+clean; 146 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 421 KiB/s rd, 2.2 MiB/s wr, 154 op/s 2024-03-19T01:45:16.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:45:15 smithi099 ceph-mon[62951]: pgmap v475: 97 pgs: 97 active+clean; 146 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 421 KiB/s rd, 2.2 MiB/s wr, 154 op/s 2024-03-19T01:45:16.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:45:15 smithi112 ceph-mon[66819]: pgmap v475: 97 pgs: 97 active+clean; 146 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 421 KiB/s rd, 2.2 MiB/s wr, 154 op/s 2024-03-19T01:45:16.682 INFO:tasks.workunit.client.0.smithi099.stdout: 1 477197 10.98 MB/sec execute 106 sec latency 57.020 ms 2024-03-19T01:45:17.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:45:16 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:45:17.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:45:16 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:45:17.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:45:16 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:45:17.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:45:16 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:45:17.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:45:16 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:45:17.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:45:16 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:45:17.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:45:16 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:45:17.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:45:16 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:45:17.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:45:16 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:45:17.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:45:16 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:45:17.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:45:16 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:45:17.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:45:16 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:45:17.682 INFO:tasks.workunit.client.0.smithi099.stdout: 1 479892 11.03 MB/sec execute 107 sec latency 14.354 ms 2024-03-19T01:45:18.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:45:17 smithi115 ceph-mon[67332]: pgmap v476: 97 pgs: 97 active+clean; 147 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 576 KiB/s rd, 2.7 MiB/s wr, 216 op/s 2024-03-19T01:45:18.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:45:17 smithi099 ceph-mon[62951]: pgmap v476: 97 pgs: 97 active+clean; 147 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 576 KiB/s rd, 2.7 MiB/s wr, 216 op/s 2024-03-19T01:45:18.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:45:17 smithi112 ceph-mon[66819]: pgmap v476: 97 pgs: 97 active+clean; 147 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 576 KiB/s rd, 2.7 MiB/s wr, 216 op/s 2024-03-19T01:45:18.682 INFO:tasks.workunit.client.0.smithi099.stdout: 1 480727 10.95 MB/sec execute 108 sec latency 352.879 ms 2024-03-19T01:45:19.682 INFO:tasks.workunit.client.0.smithi099.stdout: 1 483456 11.00 MB/sec execute 109 sec latency 12.875 ms 2024-03-19T01:45:20.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:45:19 smithi115 ceph-mon[67332]: pgmap v477: 97 pgs: 97 active+clean; 146 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 485 KiB/s rd, 2.5 MiB/s wr, 203 op/s 2024-03-19T01:45:20.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:45:19 smithi099 ceph-mon[62951]: pgmap v477: 97 pgs: 97 active+clean; 146 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 485 KiB/s rd, 2.5 MiB/s wr, 203 op/s 2024-03-19T01:45:20.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:45:19 smithi112 ceph-mon[66819]: pgmap v477: 97 pgs: 97 active+clean; 146 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 485 KiB/s rd, 2.5 MiB/s wr, 203 op/s 2024-03-19T01:45:20.682 INFO:tasks.workunit.client.0.smithi099.stdout: 1 485360 10.96 MB/sec execute 110 sec latency 36.399 ms 2024-03-19T01:45:21.683 INFO:tasks.workunit.client.0.smithi099.stdout: 1 487225 10.98 MB/sec execute 111 sec latency 63.912 ms 2024-03-19T01:45:22.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:45:21 smithi115 ceph-mon[67332]: pgmap v478: 97 pgs: 97 active+clean; 148 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 480 KiB/s rd, 2.4 MiB/s wr, 191 op/s 2024-03-19T01:45:22.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:45:21 smithi099 ceph-mon[62951]: pgmap v478: 97 pgs: 97 active+clean; 148 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 480 KiB/s rd, 2.4 MiB/s wr, 191 op/s 2024-03-19T01:45:22.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:45:21 smithi112 ceph-mon[66819]: pgmap v478: 97 pgs: 97 active+clean; 148 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 480 KiB/s rd, 2.4 MiB/s wr, 191 op/s 2024-03-19T01:45:22.683 INFO:tasks.workunit.client.0.smithi099.stdout: 1 489369 10.95 MB/sec execute 112 sec latency 36.688 ms 2024-03-19T01:45:23.683 INFO:tasks.workunit.client.0.smithi099.stdout: 1 491377 10.97 MB/sec execute 113 sec latency 29.484 ms 2024-03-19T01:45:24.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:45:23 smithi115 ceph-mon[67332]: pgmap v479: 97 pgs: 97 active+clean; 148 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 601 KiB/s rd, 2.6 MiB/s wr, 223 op/s 2024-03-19T01:45:24.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:45:23 smithi099 ceph-mon[62951]: pgmap v479: 97 pgs: 97 active+clean; 148 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 601 KiB/s rd, 2.6 MiB/s wr, 223 op/s 2024-03-19T01:45:24.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:45:23 smithi112 ceph-mon[66819]: pgmap v479: 97 pgs: 97 active+clean; 148 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 601 KiB/s rd, 2.6 MiB/s wr, 223 op/s 2024-03-19T01:45:24.683 INFO:tasks.workunit.client.0.smithi099.stdout: 1 493993 11.02 MB/sec execute 114 sec latency 22.042 ms 2024-03-19T01:45:25.683 INFO:tasks.workunit.client.0.smithi099.stdout: 1 495830 10.98 MB/sec execute 115 sec latency 27.946 ms 2024-03-19T01:45:25.868 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.868+0000 7f4965d1d640 1 -- 172.21.15.99:0/2913889425 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4960100740 msgr2=0x7f4960100b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:45:25.868 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.868+0000 7f4965d1d640 1 --2- 172.21.15.99:0/2913889425 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4960100740 0x7f4960100b60 secure :-1 s=READY pgs=433 cs=0 l=1 rev1=1 crypto rx=0x7f4950009900 tx=0x7f495002bc30 comp rx=0 tx=0).stop 2024-03-19T01:45:25.868 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.868+0000 7f4965d1d640 1 -- 172.21.15.99:0/2913889425 shutdown_connections 2024-03-19T01:45:25.869 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.868+0000 7f4965d1d640 1 --2- 172.21.15.99:0/2913889425 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4960101930 0x7f4960101db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:25.869 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.868+0000 7f4965d1d640 1 --2- 172.21.15.99:0/2913889425 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f49601022f0 0x7f4960108b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:25.869 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.868+0000 7f4965d1d640 1 --2- 172.21.15.99:0/2913889425 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4960100740 0x7f4960100b60 unknown :-1 s=CLOSED pgs=433 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:25.869 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.868+0000 7f4965d1d640 1 -- 172.21.15.99:0/2913889425 >> 172.21.15.99:0/2913889425 conn(0x7f49600fbcb0 msgr2=0x7f49600fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:45:25.869 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.868+0000 7f4965d1d640 1 -- 172.21.15.99:0/2913889425 shutdown_connections 2024-03-19T01:45:25.869 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.868+0000 7f4965d1d640 1 -- 172.21.15.99:0/2913889425 wait complete. 2024-03-19T01:45:25.869 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.868+0000 7f4965d1d640 1 Processor -- start 2024-03-19T01:45:25.869 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.869+0000 7f4965d1d640 1 -- start start 2024-03-19T01:45:25.871 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.869+0000 7f4965d1d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4960100740 0x7f4960197ad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:25.871 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.869+0000 7f4965d1d640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4960101930 0x7f4960198010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:25.871 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.869+0000 7f4965d1d640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f49601022f0 0x7f496019f080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:25.871 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.869+0000 7f4965d1d640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f496010c3c0 con 0x7f4960100740 2024-03-19T01:45:25.871 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.869+0000 7f4965d1d640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f496010c240 con 0x7f4960101930 2024-03-19T01:45:25.871 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.869+0000 7f4965d1d640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f496010c540 con 0x7f49601022f0 2024-03-19T01:45:25.871 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.869+0000 7f495f7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4960100740 0x7f4960197ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:25.872 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.869+0000 7f495f7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4960100740 0x7f4960197ad0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:57836/0 (socket says 172.21.15.99:57836) 2024-03-19T01:45:25.872 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.869+0000 7f495f7fe640 1 -- 172.21.15.99:0/2661485148 learned_addr learned my addr 172.21.15.99:0/2661485148 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:45:25.872 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.869+0000 7f495effd640 1 --2- 172.21.15.99:0/2661485148 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4960101930 0x7f4960198010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:25.872 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.869+0000 7f495ffff640 1 --2- 172.21.15.99:0/2661485148 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f49601022f0 0x7f496019f080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:25.872 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.869+0000 7f495f7fe640 1 -- 172.21.15.99:0/2661485148 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f49601022f0 msgr2=0x7f496019f080 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:45:25.872 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.869+0000 7f495f7fe640 1 --2- 172.21.15.99:0/2661485148 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f49601022f0 0x7f496019f080 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:25.872 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.869+0000 7f495f7fe640 1 -- 172.21.15.99:0/2661485148 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4960101930 msgr2=0x7f4960198010 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:45:25.872 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.869+0000 7f495f7fe640 1 --2- 172.21.15.99:0/2661485148 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4960101930 0x7f4960198010 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:25.872 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.869+0000 7f495f7fe640 1 -- 172.21.15.99:0/2661485148 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f496019f6f0 con 0x7f4960100740 2024-03-19T01:45:25.872 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.869+0000 7f495f7fe640 1 --2- 172.21.15.99:0/2661485148 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4960100740 0x7f4960197ad0 secure :-1 s=READY pgs=434 cs=0 l=1 rev1=1 crypto rx=0x7f4950000c00 tx=0x7f495002c640 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:45:25.872 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.870+0000 7f495cff9640 1 -- 172.21.15.99:0/2661485148 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4950041070 con 0x7f4960100740 2024-03-19T01:45:25.872 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.870+0000 7f495cff9640 1 -- 172.21.15.99:0/2661485148 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f4950004450 con 0x7f4960100740 2024-03-19T01:45:25.872 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.870+0000 7f4965d1d640 1 -- 172.21.15.99:0/2661485148 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f496019f980 con 0x7f4960100740 2024-03-19T01:45:25.872 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.870+0000 7f495cff9640 1 -- 172.21.15.99:0/2661485148 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4950034440 con 0x7f4960100740 2024-03-19T01:45:25.872 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.870+0000 7f4965d1d640 1 -- 172.21.15.99:0/2661485148 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f496019fec0 con 0x7f4960100740 2024-03-19T01:45:25.879 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.872+0000 7f495cff9640 1 -- 172.21.15.99:0/2661485148 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f4950044070 con 0x7f4960100740 2024-03-19T01:45:25.879 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.872+0000 7f49689e0740 1 Processor -- start 2024-03-19T01:45:25.879 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.872+0000 7f49689e0740 1 -- start start 2024-03-19T01:45:25.879 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.872+0000 7f495cff9640 1 --2- 172.21.15.99:0/2661485148 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f49480a2920 0x7f49480a4de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:25.879 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.872+0000 7f49689e0740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d569fee100 0x55d569fee520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:25.879 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.872+0000 7f49689e0740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d569feeaf0 0x55d569ff35f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:25.879 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.872+0000 7f49689e0740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d569ff3c60 0x55d569ff60f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:25.879 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.872+0000 7f49689e0740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55d569f52be0 con 0x55d569feeaf0 2024-03-19T01:45:25.879 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.872+0000 7f49689e0740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55d569efb960 con 0x55d569fee100 2024-03-19T01:45:25.880 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.872+0000 7f49689e0740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55d569f4ec80 con 0x55d569ff3c60 2024-03-19T01:45:25.880 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.873+0000 7f495f7fe640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d569fee100 0x55d569fee520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:25.880 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.873+0000 7f495ffff640 1 --2- 172.21.15.99:0/2661485148 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f49480a2920 0x7f49480a4de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:25.880 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.873+0000 7f495effd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d569feeaf0 0x55d569ff35f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:25.880 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.873+0000 7f495f7fe640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d569fee100 0x55d569fee520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:60756/0 (socket says 172.21.15.99:60756) 2024-03-19T01:45:25.880 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.873+0000 7f495effd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d569feeaf0 0x55d569ff35f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:57838/0 (socket says 172.21.15.99:57838) 2024-03-19T01:45:25.880 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.873+0000 7f495f7fe640 1 -- 172.21.15.99:0/2146339709 learned_addr learned my addr 172.21.15.99:0/2146339709 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:45:25.880 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.873+0000 7f495ffff640 1 --2- 172.21.15.99:0/2146339709 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d569ff3c60 0x55d569ff60f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:25.880 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.873+0000 7f495cff9640 1 -- 172.21.15.99:0/2661485148 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f49500f0500 con 0x7f4960100740 2024-03-19T01:45:25.880 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.873+0000 7f495effd640 1 -- 172.21.15.99:0/2146339709 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d569ff3c60 msgr2=0x55d569ff60f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:45:25.880 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.873+0000 7f495effd640 1 --2- 172.21.15.99:0/2146339709 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d569ff3c60 0x55d569ff60f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:25.880 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.873+0000 7f495effd640 1 -- 172.21.15.99:0/2146339709 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d569fee100 msgr2=0x55d569fee520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:45:25.880 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.873+0000 7f495effd640 1 --2- 172.21.15.99:0/2146339709 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d569fee100 0x55d569fee520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:25.880 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.873+0000 7f495effd640 1 -- 172.21.15.99:0/2146339709 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55d569ef2d20 con 0x55d569feeaf0 2024-03-19T01:45:25.881 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.873+0000 7f495f7fe640 1 --2- 172.21.15.99:0/2146339709 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d569fee100 0x55d569fee520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:45:25.881 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.873+0000 7f495ffff640 1 --2- 172.21.15.99:0/2661485148 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f49480a2920 0x7f49480a4de0 secure :-1 s=READY pgs=267 cs=0 l=1 rev1=1 crypto rx=0x7f4958005fd0 tx=0x7f4958009b80 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:45:25.881 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.873+0000 7f495effd640 1 --2- 172.21.15.99:0/2146339709 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d569feeaf0 0x55d569ff35f0 secure :-1 s=READY pgs=435 cs=0 l=1 rev1=1 crypto rx=0x7f4954009bf0 tx=0x7f4954011810 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9ee24fa63c2fa826 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:45:25.881 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.873+0000 7f4945ffb640 1 -- 172.21.15.99:0/2146339709 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4954004050 con 0x55d569feeaf0 2024-03-19T01:45:25.881 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.873+0000 7f4945ffb640 1 -- 172.21.15.99:0/2146339709 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f49540041f0 con 0x55d569feeaf0 2024-03-19T01:45:25.881 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.873+0000 7f4945ffb640 1 -- 172.21.15.99:0/2146339709 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f49540197c0 con 0x55d569feeaf0 2024-03-19T01:45:25.881 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.874+0000 7f49689e0740 1 -- 172.21.15.99:0/2146339709 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d569feeaf0 msgr2=0x55d569ff35f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:45:25.881 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.874+0000 7f49689e0740 1 --2- 172.21.15.99:0/2146339709 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d569feeaf0 0x55d569ff35f0 secure :-1 s=READY pgs=435 cs=0 l=1 rev1=1 crypto rx=0x7f4954009bf0 tx=0x7f4954011810 comp rx=0 tx=0).stop 2024-03-19T01:45:25.881 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.874+0000 7f49689e0740 1 -- 172.21.15.99:0/2146339709 shutdown_connections 2024-03-19T01:45:25.881 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.874+0000 7f49689e0740 1 --2- 172.21.15.99:0/2146339709 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d569ff3c60 0x55d569ff60f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:25.881 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.874+0000 7f49689e0740 1 --2- 172.21.15.99:0/2146339709 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d569feeaf0 0x55d569ff35f0 unknown :-1 s=CLOSED pgs=435 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:25.881 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.874+0000 7f49689e0740 1 --2- 172.21.15.99:0/2146339709 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d569fee100 0x55d569fee520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:25.881 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.874+0000 7f49689e0740 1 -- 172.21.15.99:0/2146339709 >> 172.21.15.99:0/2146339709 conn(0x55d569fe9670 msgr2=0x55d569fe9a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:45:25.881 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.874+0000 7f49689e0740 1 -- 172.21.15.99:0/2146339709 shutdown_connections 2024-03-19T01:45:25.882 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.874+0000 7f49689e0740 1 -- 172.21.15.99:0/2146339709 wait complete. 2024-03-19T01:45:25.882 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.874+0000 7f49689e0740 1 -- start start 2024-03-19T01:45:25.882 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.875+0000 7f49689e0740 1 Processor -- start 2024-03-19T01:45:25.882 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.875+0000 7f49689e0740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d569fee100 0x55d569ff8070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:25.882 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.875+0000 7f49689e0740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d569feeaf0 0x55d569ff85b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:25.882 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.875+0000 7f49689e0740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d569ff3c60 0x55d569fff5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:25.882 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.875+0000 7f49689e0740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55d569ff6c10 con 0x55d569feeaf0 2024-03-19T01:45:25.882 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.875+0000 7f49689e0740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55d569f0edb0 con 0x55d569fee100 2024-03-19T01:45:25.882 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.875+0000 7f49689e0740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55d569ff6d90 con 0x55d569ff3c60 2024-03-19T01:45:25.882 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.875+0000 7f49689e0740 10 client.0 _ll_register_callbacks cb 0x55d569f13e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:45:25.882 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.875+0000 7f495effd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d569feeaf0 0x55d569ff85b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:25.882 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.875+0000 7f495f7fe640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d569fee100 0x55d569ff8070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:25.882 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.875+0000 7f495effd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d569feeaf0 0x55d569ff85b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:57850/0 (socket says 172.21.15.99:57850) 2024-03-19T01:45:25.882 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.875+0000 7f495effd640 1 -- 172.21.15.99:0/3823518080 learned_addr learned my addr 172.21.15.99:0/3823518080 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:45:25.883 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.875+0000 7f495ffff640 1 --2- 172.21.15.99:0/3823518080 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d569ff3c60 0x55d569fff5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:25.883 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.875+0000 7f495f7fe640 1 -- 172.21.15.99:0/3823518080 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d569ff3c60 msgr2=0x55d569fff5f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:45:25.883 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.875+0000 7f495f7fe640 1 --2- 172.21.15.99:0/3823518080 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d569ff3c60 0x55d569fff5f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:25.883 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.875+0000 7f495f7fe640 1 -- 172.21.15.99:0/3823518080 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d569feeaf0 msgr2=0x55d569ff85b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:45:25.883 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.875+0000 7f495f7fe640 1 --2- 172.21.15.99:0/3823518080 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d569feeaf0 0x55d569ff85b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:25.883 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.875+0000 7f495f7fe640 1 -- 172.21.15.99:0/3823518080 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55d569f0dc60 con 0x55d569fee100 2024-03-19T01:45:25.883 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.875+0000 7f495effd640 1 --2- 172.21.15.99:0/3823518080 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d569feeaf0 0x55d569ff85b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:45:25.883 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.875+0000 7f495f7fe640 1 --2- 172.21.15.99:0/3823518080 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d569fee100 0x55d569ff8070 secure :-1 s=READY pgs=113 cs=0 l=1 rev1=1 crypto rx=0x7f49500b9400 tx=0x7f495004cc90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=20898634476e0bb4 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:45:25.883 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.875+0000 7f493effd640 10 client.0 ms_handle_connect on v2:172.21.15.112:3300/0 2024-03-19T01:45:25.883 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.876+0000 7f493effd640 1 -- 172.21.15.99:0/3823518080 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f495004b040 con 0x55d569fee100 2024-03-19T01:45:25.883 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.876+0000 7f493effd640 1 -- 172.21.15.99:0/3823518080 <== mon.2 v2:172.21.15.112:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f49500bb2a0 con 0x55d569fee100 2024-03-19T01:45:25.883 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.876+0000 7f49689e0740 1 -- 172.21.15.99:0/3823518080 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f49540041f0 con 0x55d569fee100 2024-03-19T01:45:25.883 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.876+0000 7f493effd640 1 -- 172.21.15.99:0/3823518080 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_get_version_reply(handle=1 version=26) ==== 24+0+0 (secure 0 0 0) 0x7f49500cf860 con 0x55d569fee100 2024-03-19T01:45:25.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.876+0000 7f49689e0740 10 client.25457 fetch_fsmap learned FSMap version 26 2024-03-19T01:45:25.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.876+0000 7f49689e0740 1 -- 172.21.15.99:0/3823518080 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,fsmap=26,monmap=4+}) -- 0x55d569c67210 con 0x55d569fee100 2024-03-19T01:45:25.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.876+0000 7f493effd640 1 -- 172.21.15.99:0/3823518080 <== mon.2 v2:172.21.15.112:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f49540041f0 con 0x55d569fee100 2024-03-19T01:45:25.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.876+0000 7f493effd640 1 -- 172.21.15.99:0/3823518080 <== mon.2 v2:172.21.15.112:3300/0 5 ==== fsmap(e 26) ==== 8342+0+0 (secure 0 0 0) 0x7f495004cd00 con 0x55d569fee100 2024-03-19T01:45:25.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.876+0000 7f49689e0740 10 client.25457 fetch_fsmap finished waiting for FSMap version 26 2024-03-19T01:45:25.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.876+0000 7f49689e0740 10 client.25457 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:45:25.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.876+0000 7f49689e0740 20 client.25457 populate_metadata read hostname 'smithi099' 2024-03-19T01:45:25.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.876+0000 7f49689e0740 1 --2- 172.21.15.99:0/3823518080 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55d56a003ec0 0x55d56a006350 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:25.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.876+0000 7f49689e0740 4 client.25457 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:45:25.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.876+0000 7f49689e0740 1 -- 172.21.15.99:0/3823518080 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55d569f60120 con 0x55d56a003ec0 2024-03-19T01:45:25.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.876+0000 7f495effd640 1 --2- 172.21.15.99:0/3823518080 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55d56a003ec0 0x55d56a006350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:25.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.877+0000 7f493effd640 10 client.25457 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:45:25.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.877+0000 7f495effd640 1 --2- 172.21.15.99:0/3823518080 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55d56a003ec0 0x55d56a006350 crc :-1 s=READY pgs=67 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=f16b04638c31f908 server_cookie=7f0417e8cf6ceee2 in_seq=0 out_seq=0 2024-03-19T01:45:25.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.878+0000 7f493effd640 1 -- 172.21.15.99:0/3823518080 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x55d569f60120 con 0x55d56a003ec0 2024-03-19T01:45:25.885 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.878+0000 7f493effd640 10 client.25457 handle_command_reply: tid=0 2024-03-19T01:45:25.885 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.878+0000 7f49689e0740 1 client.25457 shutdown 2024-03-19T01:45:25.885 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.878+0000 7f49689e0740 1 -- 172.21.15.99:0/3823518080 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d569fee100 msgr2=0x55d569ff8070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:45:25.885 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.878+0000 7f49689e0740 1 --2- 172.21.15.99:0/3823518080 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d569fee100 0x55d569ff8070 secure :-1 s=READY pgs=113 cs=0 l=1 rev1=1 crypto rx=0x7f49500b9400 tx=0x7f495004cc90 comp rx=0 tx=0).stop 2024-03-19T01:45:25.885 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.878+0000 7f49689e0740 1 -- 172.21.15.99:0/3823518080 shutdown_connections 2024-03-19T01:45:25.885 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.878+0000 7f49689e0740 1 --2- 172.21.15.99:0/3823518080 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55d56a003ec0 0x55d56a006350 crc :-1 s=READY pgs=67 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:25.885 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.878+0000 7f49689e0740 1 --2- 172.21.15.99:0/3823518080 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d569ff3c60 0x55d569fff5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:25.885 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.878+0000 7f493effd640 0 client.25457 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:45:25.885 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.878+0000 7f49689e0740 1 --2- 172.21.15.99:0/3823518080 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d569feeaf0 0x55d569ff85b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:25.885 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.878+0000 7f49689e0740 1 --2- 172.21.15.99:0/3823518080 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d569fee100 0x55d569ff8070 unknown :-1 s=CLOSED pgs=113 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:25.885 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.878+0000 7f49689e0740 1 -- 172.21.15.99:0/3823518080 >> 172.21.15.99:0/3823518080 conn(0x55d569fe9670 msgr2=0x55d569fea8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:45:25.885 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.878+0000 7f49689e0740 1 -- 172.21.15.99:0/3823518080 shutdown_connections 2024-03-19T01:45:25.885 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.878+0000 7f49689e0740 1 -- 172.21.15.99:0/3823518080 wait complete. 2024-03-19T01:45:25.885 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.878+0000 7f49689e0740 20 client.25457 trim_cache size 0 max 16384 2024-03-19T01:45:25.897 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.893+0000 7f49689e0740 1 Processor -- start 2024-03-19T01:45:25.897 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.893+0000 7f49689e0740 1 -- start start 2024-03-19T01:45:25.898 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.893+0000 7f49689e0740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d569fee100 0x55d569ff8510 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:25.898 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.893+0000 7f49689e0740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d569feeaf0 0x55d569fff580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:25.898 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.893+0000 7f49689e0740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d569ff3c60 0x55d569fecfe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:25.898 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.893+0000 7f49689e0740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f49601ab640 con 0x55d569feeaf0 2024-03-19T01:45:25.898 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.893+0000 7f49689e0740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f49601a8f10 con 0x55d569fee100 2024-03-19T01:45:25.898 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.893+0000 7f49689e0740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f49601a6d60 con 0x55d569ff3c60 2024-03-19T01:45:25.898 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.894+0000 7f495effd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d569feeaf0 0x55d569fff580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:25.898 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.894+0000 7f495effd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d569feeaf0 0x55d569fff580 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:57852/0 (socket says 172.21.15.99:57852) 2024-03-19T01:45:25.898 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.894+0000 7f495effd640 1 -- 172.21.15.99:0/2578873618 learned_addr learned my addr 172.21.15.99:0/2578873618 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:45:25.898 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.894+0000 7f495f7fe640 1 --2- 172.21.15.99:0/2578873618 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d569fee100 0x55d569ff8510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:25.898 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.894+0000 7f495ffff640 1 --2- 172.21.15.99:0/2578873618 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d569ff3c60 0x55d569fecfe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:25.898 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.894+0000 7f495effd640 1 -- 172.21.15.99:0/2578873618 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d569ff3c60 msgr2=0x55d569fecfe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:45:25.898 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.894+0000 7f495effd640 1 --2- 172.21.15.99:0/2578873618 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d569ff3c60 0x55d569fecfe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:25.899 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.894+0000 7f495effd640 1 -- 172.21.15.99:0/2578873618 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d569fee100 msgr2=0x55d569ff8510 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:45:25.899 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.894+0000 7f495effd640 1 --2- 172.21.15.99:0/2578873618 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d569fee100 0x55d569ff8510 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:25.899 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.894+0000 7f495effd640 1 -- 172.21.15.99:0/2578873618 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55d569d33620 con 0x55d569feeaf0 2024-03-19T01:45:25.899 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.894+0000 7f495effd640 1 --2- 172.21.15.99:0/2578873618 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d569feeaf0 0x55d569fff580 secure :-1 s=READY pgs=436 cs=0 l=1 rev1=1 crypto rx=0x7f495401b040 tx=0x7f4954003a90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=48af32a4a9902234 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:45:25.899 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.894+0000 7f49457fa640 1 -- 172.21.15.99:0/2578873618 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4954003cf0 con 0x55d569feeaf0 2024-03-19T01:45:25.899 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.894+0000 7f49457fa640 1 -- 172.21.15.99:0/2578873618 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f4954005520 con 0x55d569feeaf0 2024-03-19T01:45:25.899 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.894+0000 7f49457fa640 1 -- 172.21.15.99:0/2578873618 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4954019940 con 0x55d569feeaf0 2024-03-19T01:45:25.899 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.895+0000 7f49689e0740 1 -- 172.21.15.99:0/2578873618 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d569feeaf0 msgr2=0x55d569fff580 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:45:25.899 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.895+0000 7f49689e0740 1 --2- 172.21.15.99:0/2578873618 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d569feeaf0 0x55d569fff580 secure :-1 s=READY pgs=436 cs=0 l=1 rev1=1 crypto rx=0x7f495401b040 tx=0x7f4954003a90 comp rx=0 tx=0).stop 2024-03-19T01:45:25.899 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.895+0000 7f49689e0740 1 -- 172.21.15.99:0/2578873618 shutdown_connections 2024-03-19T01:45:25.899 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.895+0000 7f49689e0740 1 --2- 172.21.15.99:0/2578873618 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d569ff3c60 0x55d569fecfe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:25.899 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.895+0000 7f49689e0740 1 --2- 172.21.15.99:0/2578873618 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d569feeaf0 0x55d569fff580 unknown :-1 s=CLOSED pgs=436 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:25.899 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.895+0000 7f49689e0740 1 --2- 172.21.15.99:0/2578873618 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d569fee100 0x55d569ff8510 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:25.899 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.895+0000 7f49689e0740 1 -- 172.21.15.99:0/2578873618 >> 172.21.15.99:0/2578873618 conn(0x55d569fe9670 msgr2=0x55d569fec8d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:45:25.899 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.895+0000 7f49689e0740 1 -- 172.21.15.99:0/2578873618 shutdown_connections 2024-03-19T01:45:25.900 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.895+0000 7f49689e0740 1 -- 172.21.15.99:0/2578873618 wait complete. 2024-03-19T01:45:25.900 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.895+0000 7f49689e0740 1 -- start start 2024-03-19T01:45:25.900 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.895+0000 7f49689e0740 1 Processor -- start 2024-03-19T01:45:25.900 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.895+0000 7f49689e0740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d569fee100 0x55d569ffd340 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:25.900 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.895+0000 7f49689e0740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d569feeaf0 0x55d569ffd880 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:25.900 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.895+0000 7f49689e0740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d569ff3c60 0x55d569ffe890 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:25.900 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.895+0000 7f49689e0740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f49601a73d0 con 0x55d569feeaf0 2024-03-19T01:45:25.900 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.895+0000 7f49689e0740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f49601a8840 con 0x55d569ff3c60 2024-03-19T01:45:25.900 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.895+0000 7f49689e0740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f49601a7aa0 con 0x55d569fee100 2024-03-19T01:45:25.900 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.896+0000 7f49689e0740 10 client.0 _ll_register_callbacks cb 0x7f49500edf00 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:45:25.900 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.896+0000 7f495effd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d569feeaf0 0x55d569ffd880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:25.900 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.896+0000 7f495f7fe640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d569fee100 0x55d569ffd340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:25.900 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.896+0000 7f495effd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d569feeaf0 0x55d569ffd880 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:57854/0 (socket says 172.21.15.99:57854) 2024-03-19T01:45:25.900 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.896+0000 7f495effd640 1 -- 172.21.15.99:0/67541209 learned_addr learned my addr 172.21.15.99:0/67541209 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:45:25.901 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.896+0000 7f495ffff640 1 --2- 172.21.15.99:0/67541209 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d569ff3c60 0x55d569ffe890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:25.901 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.896+0000 7f495f7fe640 1 -- 172.21.15.99:0/67541209 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d569ff3c60 msgr2=0x55d569ffe890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:45:25.901 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.896+0000 7f495f7fe640 1 --2- 172.21.15.99:0/67541209 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d569ff3c60 0x55d569ffe890 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:25.901 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.896+0000 7f495f7fe640 1 -- 172.21.15.99:0/67541209 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d569feeaf0 msgr2=0x55d569ffd880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:45:25.901 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.896+0000 7f495f7fe640 1 --2- 172.21.15.99:0/67541209 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d569feeaf0 0x55d569ffd880 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:25.901 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.896+0000 7f495f7fe640 1 -- 172.21.15.99:0/67541209 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55d569ffee80 con 0x55d569fee100 2024-03-19T01:45:25.901 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.896+0000 7f495effd640 1 --2- 172.21.15.99:0/67541209 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d569feeaf0 0x55d569ffd880 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:45:25.901 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.896+0000 7f495f7fe640 1 --2- 172.21.15.99:0/67541209 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d569fee100 0x55d569ffd340 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 crypto rx=0x7f49500b9920 tx=0x7f495002bc30 comp rx=0 tx=0).ready entity=mon.1 client_cookie=4c0b1220a57b337e server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:45:25.901 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.896+0000 7f493effd640 10 client.0 ms_handle_connect on v2:172.21.15.115:3300/0 2024-03-19T01:45:25.901 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.896+0000 7f493effd640 1 -- 172.21.15.99:0/67541209 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f495004b040 con 0x55d569fee100 2024-03-19T01:45:25.901 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.896+0000 7f49689e0740 1 -- 172.21.15.99:0/67541209 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f4954005520 con 0x55d569fee100 2024-03-19T01:45:25.901 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.896+0000 7f493effd640 1 -- 172.21.15.99:0/67541209 <== mon.1 v2:172.21.15.115:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f49500bb2a0 con 0x55d569fee100 2024-03-19T01:45:25.901 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.896+0000 7f493effd640 1 -- 172.21.15.99:0/67541209 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_get_version_reply(handle=1 version=26) ==== 24+0+0 (secure 0 0 0) 0x7f49500d0fe0 con 0x55d569fee100 2024-03-19T01:45:25.901 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.896+0000 7f49689e0740 10 client.25510 fetch_fsmap learned FSMap version 26 2024-03-19T01:45:25.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.896+0000 7f49689e0740 1 -- 172.21.15.99:0/67541209 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,fsmap=26,monmap=4+}) -- 0x55d56a06a8b0 con 0x55d569fee100 2024-03-19T01:45:25.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.897+0000 7f493effd640 1 -- 172.21.15.99:0/67541209 <== mon.1 v2:172.21.15.115:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f4954005520 con 0x55d569fee100 2024-03-19T01:45:25.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.897+0000 7f493effd640 1 -- 172.21.15.99:0/67541209 <== mon.1 v2:172.21.15.115:3300/0 5 ==== fsmap(e 26) ==== 8342+0+0 (secure 0 0 0) 0x7f49500035f0 con 0x55d569fee100 2024-03-19T01:45:25.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.897+0000 7f49689e0740 10 client.25510 fetch_fsmap finished waiting for FSMap version 26 2024-03-19T01:45:25.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.897+0000 7f49689e0740 10 client.25510 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:45:25.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.897+0000 7f49689e0740 20 client.25510 populate_metadata read hostname 'smithi099' 2024-03-19T01:45:25.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.897+0000 7f49689e0740 1 --2- 172.21.15.99:0/67541209 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55d56a06b160 0x55d56a06d580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:25.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.897+0000 7f49689e0740 4 client.25510 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:45:25.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.897+0000 7f49689e0740 1 -- 172.21.15.99:0/67541209 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55d569f5a470 con 0x55d56a06b160 2024-03-19T01:45:25.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.897+0000 7f495effd640 1 --2- 172.21.15.99:0/67541209 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55d56a06b160 0x55d56a06d580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:25.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.898+0000 7f495effd640 1 --2- 172.21.15.99:0/67541209 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55d56a06b160 0x55d56a06d580 crc :-1 s=READY pgs=68 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=f5e460d0f192bdc6 server_cookie=33dc55ee3daa9af0 in_seq=0 out_seq=0 2024-03-19T01:45:25.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.898+0000 7f493effd640 10 client.25510 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:45:25.947 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.946+0000 7f493effd640 1 -- 172.21.15.99:0/67541209 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x55d569f5a470 con 0x55d56a06b160 2024-03-19T01:45:25.948 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.946+0000 7f493effd640 10 client.25510 handle_command_reply: tid=0 2024-03-19T01:45:25.948 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.946+0000 7f49689e0740 1 client.25510 shutdown 2024-03-19T01:45:25.948 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.947+0000 7f49689e0740 1 -- 172.21.15.99:0/67541209 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d569fee100 msgr2=0x55d569ffd340 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:45:25.948 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.947+0000 7f49689e0740 1 --2- 172.21.15.99:0/67541209 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d569fee100 0x55d569ffd340 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 crypto rx=0x7f49500b9920 tx=0x7f495002bc30 comp rx=0 tx=0).stop 2024-03-19T01:45:25.948 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.947+0000 7f49689e0740 1 -- 172.21.15.99:0/67541209 shutdown_connections 2024-03-19T01:45:25.948 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.947+0000 7f49689e0740 1 --2- 172.21.15.99:0/67541209 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55d56a06b160 0x55d56a06d580 crc :-1 s=READY pgs=68 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:25.948 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.947+0000 7f49689e0740 1 --2- 172.21.15.99:0/67541209 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d569ff3c60 0x55d569ffe890 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:25.948 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.947+0000 7f49689e0740 1 --2- 172.21.15.99:0/67541209 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d569feeaf0 0x55d569ffd880 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:25.948 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.947+0000 7f49689e0740 1 --2- 172.21.15.99:0/67541209 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d569fee100 0x55d569ffd340 unknown :-1 s=CLOSED pgs=86 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:25.948 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.947+0000 7f493effd640 0 client.25510 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:45:25.948 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.947+0000 7f49689e0740 1 -- 172.21.15.99:0/67541209 >> 172.21.15.99:0/67541209 conn(0x55d569fe9670 msgr2=0x55d569fed160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:45:25.948 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.947+0000 7f49689e0740 1 -- 172.21.15.99:0/67541209 shutdown_connections 2024-03-19T01:45:25.949 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.947+0000 7f49689e0740 1 -- 172.21.15.99:0/67541209 wait complete. 2024-03-19T01:45:25.949 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.947+0000 7f49689e0740 20 client.25510 trim_cache size 0 max 16384 2024-03-19T01:45:25.949 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:45:25.949 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:45:25.949 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:45:25.949 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:45:25.950 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.947+0000 7f4965d1d640 1 -- 172.21.15.99:0/2661485148 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f49480a2920 msgr2=0x7f49480a4de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:45:25.950 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.947+0000 7f4965d1d640 1 --2- 172.21.15.99:0/2661485148 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f49480a2920 0x7f49480a4de0 secure :-1 s=READY pgs=267 cs=0 l=1 rev1=1 crypto rx=0x7f4958005fd0 tx=0x7f4958009b80 comp rx=0 tx=0).stop 2024-03-19T01:45:25.950 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.947+0000 7f4965d1d640 1 -- 172.21.15.99:0/2661485148 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4960100740 msgr2=0x7f4960197ad0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:45:25.950 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.947+0000 7f4965d1d640 1 --2- 172.21.15.99:0/2661485148 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4960100740 0x7f4960197ad0 secure :-1 s=READY pgs=434 cs=0 l=1 rev1=1 crypto rx=0x7f4950000c00 tx=0x7f495002c640 comp rx=0 tx=0).stop 2024-03-19T01:45:25.950 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.947+0000 7f4965d1d640 1 -- 172.21.15.99:0/2661485148 shutdown_connections 2024-03-19T01:45:25.950 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.947+0000 7f4965d1d640 1 --2- 172.21.15.99:0/2661485148 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f49480a2920 0x7f49480a4de0 unknown :-1 s=CLOSED pgs=267 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:25.950 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.947+0000 7f4965d1d640 1 --2- 172.21.15.99:0/2661485148 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4960101930 0x7f4960198010 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:25.950 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.947+0000 7f4965d1d640 1 --2- 172.21.15.99:0/2661485148 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f49601022f0 0x7f496019f080 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:25.951 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.947+0000 7f4965d1d640 1 --2- 172.21.15.99:0/2661485148 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4960100740 0x7f4960197ad0 unknown :-1 s=CLOSED pgs=434 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:25.951 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.947+0000 7f4965d1d640 1 -- 172.21.15.99:0/2661485148 >> 172.21.15.99:0/2661485148 conn(0x7f49600fbcb0 msgr2=0x7f49600fd6e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:45:25.951 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.948+0000 7f4965d1d640 1 -- 172.21.15.99:0/2661485148 shutdown_connections 2024-03-19T01:45:25.951 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:25.948+0000 7f4965d1d640 1 -- 172.21.15.99:0/2661485148 wait complete. 2024-03-19T01:45:26.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:45:25 smithi115 ceph-mon[67332]: pgmap v480: 97 pgs: 97 active+clean; 148 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 564 KiB/s rd, 2.6 MiB/s wr, 220 op/s 2024-03-19T01:45:26.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:45:25 smithi099 ceph-mon[62951]: pgmap v480: 97 pgs: 97 active+clean; 148 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 564 KiB/s rd, 2.6 MiB/s wr, 220 op/s 2024-03-19T01:45:26.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:45:25 smithi112 ceph-mon[66819]: pgmap v480: 97 pgs: 97 active+clean; 148 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 564 KiB/s rd, 2.6 MiB/s wr, 220 op/s 2024-03-19T01:45:26.683 INFO:tasks.workunit.client.0.smithi099.stdout: 1 497787 11.00 MB/sec execute 116 sec latency 81.039 ms 2024-03-19T01:45:27.683 INFO:tasks.workunit.client.0.smithi099.stdout: 1 499959 10.98 MB/sec execute 117 sec latency 16.633 ms 2024-03-19T01:45:28.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:45:27 smithi115 ceph-mon[67332]: pgmap v481: 97 pgs: 97 active+clean; 147 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 607 KiB/s rd, 2.9 MiB/s wr, 251 op/s 2024-03-19T01:45:28.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:45:27 smithi099 ceph-mon[62951]: pgmap v481: 97 pgs: 97 active+clean; 147 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 607 KiB/s rd, 2.9 MiB/s wr, 251 op/s 2024-03-19T01:45:28.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:45:27 smithi112 ceph-mon[66819]: pgmap v481: 97 pgs: 97 active+clean; 147 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 607 KiB/s rd, 2.9 MiB/s wr, 251 op/s 2024-03-19T01:45:28.683 INFO:tasks.workunit.client.0.smithi099.stdout: 1 501920 10.98 MB/sec execute 118 sec latency 39.679 ms 2024-03-19T01:45:29.683 INFO:tasks.workunit.client.0.smithi099.stdout: 1 504572 11.03 MB/sec execute 119 sec latency 20.543 ms 2024-03-19T01:45:30.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:45:29 smithi115 ceph-mon[67332]: pgmap v482: 97 pgs: 97 active+clean; 149 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 614 KiB/s rd, 2.8 MiB/s wr, 243 op/s 2024-03-19T01:45:30.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:45:29 smithi099 ceph-mon[62951]: pgmap v482: 97 pgs: 97 active+clean; 149 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 614 KiB/s rd, 2.8 MiB/s wr, 243 op/s 2024-03-19T01:45:30.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:45:29 smithi112 ceph-mon[66819]: pgmap v482: 97 pgs: 97 active+clean; 149 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 614 KiB/s rd, 2.8 MiB/s wr, 243 op/s 2024-03-19T01:45:30.683 INFO:tasks.workunit.client.0.smithi099.stdout: 1 506184 11.00 MB/sec execute 120 sec latency 38.577 ms 2024-03-19T01:45:31.683 INFO:tasks.workunit.client.0.smithi099.stdout: 1 508395 11.02 MB/sec execute 121 sec latency 49.670 ms 2024-03-19T01:45:32.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:45:31 smithi115 ceph-mon[67332]: pgmap v483: 97 pgs: 97 active+clean; 148 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 658 KiB/s rd, 2.8 MiB/s wr, 253 op/s 2024-03-19T01:45:32.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:45:31 smithi099 ceph-mon[62951]: pgmap v483: 97 pgs: 97 active+clean; 148 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 658 KiB/s rd, 2.8 MiB/s wr, 253 op/s 2024-03-19T01:45:32.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:45:31 smithi112 ceph-mon[66819]: pgmap v483: 97 pgs: 97 active+clean; 148 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 658 KiB/s rd, 2.8 MiB/s wr, 253 op/s 2024-03-19T01:45:32.683 INFO:tasks.workunit.client.0.smithi099.stdout: 1 510419 10.99 MB/sec execute 122 sec latency 25.638 ms 2024-03-19T01:45:33.683 INFO:tasks.workunit.client.0.smithi099.stdout: 1 512386 11.00 MB/sec execute 123 sec latency 29.426 ms 2024-03-19T01:45:34.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:45:33 smithi115 ceph-mon[67332]: pgmap v484: 97 pgs: 97 active+clean; 150 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 668 KiB/s rd, 2.8 MiB/s wr, 269 op/s 2024-03-19T01:45:34.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:45:33 smithi099 ceph-mon[62951]: pgmap v484: 97 pgs: 97 active+clean; 150 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 668 KiB/s rd, 2.8 MiB/s wr, 269 op/s 2024-03-19T01:45:34.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:45:33 smithi112 ceph-mon[66819]: pgmap v484: 97 pgs: 97 active+clean; 150 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 668 KiB/s rd, 2.8 MiB/s wr, 269 op/s 2024-03-19T01:45:34.683 INFO:tasks.workunit.client.0.smithi099.stdout: 1 515271 11.05 MB/sec execute 124 sec latency 13.249 ms 2024-03-19T01:45:35.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:45:34 smithi115 ceph-mon[67332]: pgmap v485: 97 pgs: 97 active+clean; 150 MiB data, 1.6 GiB used, 1.0 TiB / 1.0 TiB avail; 520 KiB/s rd, 2.5 MiB/s wr, 229 op/s 2024-03-19T01:45:35.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:45:34 smithi099 ceph-mon[62951]: pgmap v485: 97 pgs: 97 active+clean; 150 MiB data, 1.6 GiB used, 1.0 TiB / 1.0 TiB avail; 520 KiB/s rd, 2.5 MiB/s wr, 229 op/s 2024-03-19T01:45:35.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:45:34 smithi112 ceph-mon[66819]: pgmap v485: 97 pgs: 97 active+clean; 150 MiB data, 1.6 GiB used, 1.0 TiB / 1.0 TiB avail; 520 KiB/s rd, 2.5 MiB/s wr, 229 op/s 2024-03-19T01:45:35.683 INFO:tasks.workunit.client.0.smithi099.stdout: 1 516900 11.02 MB/sec execute 125 sec latency 82.431 ms 2024-03-19T01:45:36.683 INFO:tasks.workunit.client.0.smithi099.stdout: 1 519091 11.04 MB/sec execute 126 sec latency 73.157 ms 2024-03-19T01:45:37.683 INFO:tasks.workunit.client.0.smithi099.stdout: 1 521450 11.04 MB/sec execute 127 sec latency 23.923 ms 2024-03-19T01:45:38.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:45:37 smithi115 ceph-mon[67332]: pgmap v486: 97 pgs: 97 active+clean; 152 MiB data, 1.6 GiB used, 1.0 TiB / 1.0 TiB avail; 667 KiB/s rd, 2.9 MiB/s wr, 265 op/s 2024-03-19T01:45:38.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:45:37 smithi099 ceph-mon[62951]: pgmap v486: 97 pgs: 97 active+clean; 152 MiB data, 1.6 GiB used, 1.0 TiB / 1.0 TiB avail; 667 KiB/s rd, 2.9 MiB/s wr, 265 op/s 2024-03-19T01:45:38.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:45:37 smithi112 ceph-mon[66819]: pgmap v486: 97 pgs: 97 active+clean; 152 MiB data, 1.6 GiB used, 1.0 TiB / 1.0 TiB avail; 667 KiB/s rd, 2.9 MiB/s wr, 265 op/s 2024-03-19T01:45:38.683 INFO:tasks.workunit.client.0.smithi099.stdout: 1 523598 11.06 MB/sec execute 128 sec latency 27.753 ms 2024-03-19T01:45:39.043 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.041+0000 7fc28cc14640 1 -- 172.21.15.99:0/2170780657 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fc2881075d0 msgr2=0x7fc2881099c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:45:39.044 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.041+0000 7fc28cc14640 1 --2- 172.21.15.99:0/2170780657 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fc2881075d0 0x7fc2881099c0 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7fc280009be0 tx=0x7fc28002bef0 comp rx=0 tx=0).stop 2024-03-19T01:45:39.044 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.041+0000 7fc28cc14640 1 -- 172.21.15.99:0/2170780657 shutdown_connections 2024-03-19T01:45:39.044 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.041+0000 7fc28cc14640 1 --2- 172.21.15.99:0/2170780657 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fc2881075d0 0x7fc2881099c0 unknown :-1 s=CLOSED pgs=87 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:39.044 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.041+0000 7fc28cc14640 1 --2- 172.21.15.99:0/2170780657 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc288104ca0 0x7fc288107090 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:39.044 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.041+0000 7fc28cc14640 1 --2- 172.21.15.99:0/2170780657 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fc288102370 0x7fc288104760 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:39.044 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.041+0000 7fc28cc14640 1 -- 172.21.15.99:0/2170780657 >> 172.21.15.99:0/2170780657 conn(0x7fc2880fbcc0 msgr2=0x7fc2880fe100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:45:39.044 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.042+0000 7fc28cc14640 1 -- 172.21.15.99:0/2170780657 shutdown_connections 2024-03-19T01:45:39.044 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.042+0000 7fc28cc14640 1 -- 172.21.15.99:0/2170780657 wait complete. 2024-03-19T01:45:39.044 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.042+0000 7fc28cc14640 1 Processor -- start 2024-03-19T01:45:39.044 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.042+0000 7fc28cc14640 1 -- start start 2024-03-19T01:45:39.044 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.042+0000 7fc28cc14640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc288102370 0x7fc288197a20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:39.045 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.042+0000 7fc28cc14640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fc288104ca0 0x7fc288197f60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:39.045 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.042+0000 7fc28cc14640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fc2881075d0 0x7fc28819efd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:39.045 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.042+0000 7fc28cc14640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fc28810d4d0 con 0x7fc288102370 2024-03-19T01:45:39.045 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.042+0000 7fc28cc14640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fc28810d350 con 0x7fc288104ca0 2024-03-19T01:45:39.045 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.042+0000 7fc28cc14640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fc28810d650 con 0x7fc2881075d0 2024-03-19T01:45:39.045 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.043+0000 7fc285d9b640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fc288104ca0 0x7fc288197f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:39.045 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.043+0000 7fc286d9d640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fc2881075d0 0x7fc28819efd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:39.045 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.043+0000 7fc28659c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc288102370 0x7fc288197a20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:39.045 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.043+0000 7fc285d9b640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fc288104ca0 0x7fc288197f60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:50828/0 (socket says 172.21.15.99:50828) 2024-03-19T01:45:39.045 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.043+0000 7fc28659c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc288102370 0x7fc288197a20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:44880/0 (socket says 172.21.15.99:44880) 2024-03-19T01:45:39.045 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.043+0000 7fc285d9b640 1 -- 172.21.15.99:0/935384540 learned_addr learned my addr 172.21.15.99:0/935384540 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:45:39.045 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.043+0000 7fc285d9b640 1 -- 172.21.15.99:0/935384540 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fc2881075d0 msgr2=0x7fc28819efd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:45:39.045 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.043+0000 7fc285d9b640 1 --2- 172.21.15.99:0/935384540 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fc2881075d0 0x7fc28819efd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:39.045 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.043+0000 7fc285d9b640 1 -- 172.21.15.99:0/935384540 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc288102370 msgr2=0x7fc288197a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:45:39.046 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.043+0000 7fc285d9b640 1 --2- 172.21.15.99:0/935384540 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc288102370 0x7fc288197a20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:39.046 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.043+0000 7fc285d9b640 1 -- 172.21.15.99:0/935384540 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fc28819f640 con 0x7fc288104ca0 2024-03-19T01:45:39.046 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.043+0000 7fc285d9b640 1 --2- 172.21.15.99:0/935384540 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fc288104ca0 0x7fc288197f60 secure :-1 s=READY pgs=114 cs=0 l=1 rev1=1 crypto rx=0x7fc27c00e9a0 tx=0x7fc27c00ee60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:45:39.046 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.043+0000 7fc2777fe640 1 -- 172.21.15.99:0/935384540 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc27c00cd80 con 0x7fc288104ca0 2024-03-19T01:45:39.046 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.043+0000 7fc28cc14640 1 -- 172.21.15.99:0/935384540 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fc28819f930 con 0x7fc288104ca0 2024-03-19T01:45:39.046 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.043+0000 7fc2777fe640 1 -- 172.21.15.99:0/935384540 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fc27c0035f0 con 0x7fc288104ca0 2024-03-19T01:45:39.046 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.043+0000 7fc2777fe640 1 -- 172.21.15.99:0/935384540 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc27c010670 con 0x7fc288104ca0 2024-03-19T01:45:39.046 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.043+0000 7fc28cc14640 1 -- 172.21.15.99:0/935384540 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fc28819fed0 con 0x7fc288104ca0 2024-03-19T01:45:39.053 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.045+0000 7fc2777fe640 1 -- 172.21.15.99:0/935384540 <== mon.2 v2:172.21.15.112:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7fc27c0037a0 con 0x7fc288104ca0 2024-03-19T01:45:39.053 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.046+0000 7fc28f7fd740 1 Processor -- start 2024-03-19T01:45:39.053 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.046+0000 7fc2777fe640 1 --2- 172.21.15.99:0/935384540 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fc2700a28a0 0x7fc2700a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:39.054 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.046+0000 7fc2777fe640 1 -- 172.21.15.99:0/935384540 <== mon.2 v2:172.21.15.112:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7fc27c0ced30 con 0x7fc288104ca0 2024-03-19T01:45:39.054 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.046+0000 7fc286d9d640 1 --2- 172.21.15.99:0/935384540 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fc2700a28a0 0x7fc2700a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:39.054 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.046+0000 7fc28f7fd740 1 -- start start 2024-03-19T01:45:39.054 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.046+0000 7fc28f7fd740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55da76552510 0x55da76552930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:39.054 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.046+0000 7fc28f7fd740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55da76552f00 0x55da76557a00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:39.054 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.046+0000 7fc28f7fd740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55da76558070 0x55da7655a500 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:39.054 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.046+0000 7fc28f7fd740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55da764b2be0 con 0x55da76558070 2024-03-19T01:45:39.054 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.046+0000 7fc28f7fd740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55da7645b960 con 0x55da76552510 2024-03-19T01:45:39.054 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.046+0000 7fc28f7fd740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55da764aec80 con 0x55da76552f00 2024-03-19T01:45:39.054 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.047+0000 7fc286d9d640 1 --2- 172.21.15.99:0/935384540 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fc2700a28a0 0x7fc2700a4d60 secure :-1 s=READY pgs=268 cs=0 l=1 rev1=1 crypto rx=0x7fc280004ab0 tx=0x7fc280004a40 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:45:39.054 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.047+0000 7fc285d9b640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55da76552f00 0x55da76557a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:39.054 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.047+0000 7fc28659c640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55da76552510 0x55da76552930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:39.054 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.047+0000 7fc285d9b640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55da76552f00 0x55da76557a00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:58984/0 (socket says 172.21.15.99:58984) 2024-03-19T01:45:39.055 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.047+0000 7fc285d9b640 1 -- 172.21.15.99:0/27870959 learned_addr learned my addr 172.21.15.99:0/27870959 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:45:39.055 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.047+0000 7fc286d9d640 1 --2- 172.21.15.99:0/27870959 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55da76558070 0x55da7655a500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:39.055 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.047+0000 7fc28659c640 1 -- 172.21.15.99:0/27870959 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55da76552f00 msgr2=0x55da76557a00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:45:39.055 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.047+0000 7fc28659c640 1 --2- 172.21.15.99:0/27870959 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55da76552f00 0x55da76557a00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:39.055 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.047+0000 7fc28659c640 1 -- 172.21.15.99:0/27870959 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55da76558070 msgr2=0x55da7655a500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:45:39.055 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.047+0000 7fc28659c640 1 --2- 172.21.15.99:0/27870959 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55da76558070 0x55da7655a500 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:39.055 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.047+0000 7fc28659c640 1 -- 172.21.15.99:0/27870959 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55da76452d20 con 0x55da76552510 2024-03-19T01:45:39.055 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.047+0000 7fc285d9b640 1 --2- 172.21.15.99:0/27870959 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55da76552f00 0x55da76557a00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:45:39.055 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.047+0000 7fc28659c640 1 --2- 172.21.15.99:0/27870959 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55da76552510 0x55da76552930 secure :-1 s=READY pgs=115 cs=0 l=1 rev1=1 crypto rx=0x7fc278009c20 tx=0x7fc278011810 comp rx=0 tx=0).ready entity=mon.2 client_cookie=7fb2317118d12a8a server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:45:39.055 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.047+0000 7fc274ff9640 1 -- 172.21.15.99:0/27870959 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc278004050 con 0x55da76552510 2024-03-19T01:45:39.055 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.047+0000 7fc274ff9640 1 -- 172.21.15.99:0/27870959 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fc278005500 con 0x55da76552510 2024-03-19T01:45:39.055 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.047+0000 7fc274ff9640 1 -- 172.21.15.99:0/27870959 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc278004d70 con 0x55da76552510 2024-03-19T01:45:39.055 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.048+0000 7fc28f7fd740 1 -- 172.21.15.99:0/27870959 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55da76552510 msgr2=0x55da76552930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:45:39.056 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.048+0000 7fc28f7fd740 1 --2- 172.21.15.99:0/27870959 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55da76552510 0x55da76552930 secure :-1 s=READY pgs=115 cs=0 l=1 rev1=1 crypto rx=0x7fc278009c20 tx=0x7fc278011810 comp rx=0 tx=0).stop 2024-03-19T01:45:39.056 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.048+0000 7fc28f7fd740 1 -- 172.21.15.99:0/27870959 shutdown_connections 2024-03-19T01:45:39.056 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.048+0000 7fc28f7fd740 1 --2- 172.21.15.99:0/27870959 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55da76558070 0x55da7655a500 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:39.056 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.048+0000 7fc28f7fd740 1 --2- 172.21.15.99:0/27870959 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55da76552f00 0x55da76557a00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:39.056 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.048+0000 7fc28f7fd740 1 --2- 172.21.15.99:0/27870959 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55da76552510 0x55da76552930 unknown :-1 s=CLOSED pgs=115 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:39.056 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.048+0000 7fc28f7fd740 1 -- 172.21.15.99:0/27870959 >> 172.21.15.99:0/27870959 conn(0x55da76549670 msgr2=0x55da76549a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:45:39.056 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.048+0000 7fc28f7fd740 1 -- 172.21.15.99:0/27870959 shutdown_connections 2024-03-19T01:45:39.056 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.048+0000 7fc28f7fd740 1 -- 172.21.15.99:0/27870959 wait complete. 2024-03-19T01:45:39.056 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.049+0000 7fc28f7fd740 1 -- start start 2024-03-19T01:45:39.056 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.049+0000 7fc28f7fd740 1 Processor -- start 2024-03-19T01:45:39.056 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.049+0000 7fc28f7fd740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55da76552510 0x55da7655c510 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:39.056 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.049+0000 7fc28f7fd740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55da76552f00 0x55da7655ca50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:39.056 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.049+0000 7fc28f7fd740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55da76558070 0x55da76563a20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:39.057 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.049+0000 7fc28f7fd740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55da7655b020 con 0x55da76552510 2024-03-19T01:45:39.057 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.049+0000 7fc28f7fd740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55da7646edb0 con 0x55da76558070 2024-03-19T01:45:39.057 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.049+0000 7fc28f7fd740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55da7655b1a0 con 0x55da76552f00 2024-03-19T01:45:39.057 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.049+0000 7fc28f7fd740 10 client.0 _ll_register_callbacks cb 0x55da76473e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:45:39.057 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.049+0000 7fc28659c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55da76552510 0x55da7655c510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:39.057 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.049+0000 7fc28659c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55da76552510 0x55da7655c510 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:44894/0 (socket says 172.21.15.99:44894) 2024-03-19T01:45:39.057 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.049+0000 7fc28659c640 1 -- 172.21.15.99:0/808753190 learned_addr learned my addr 172.21.15.99:0/808753190 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:45:39.057 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.049+0000 7fc285d9b640 1 --2- 172.21.15.99:0/808753190 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55da76552f00 0x55da7655ca50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:39.057 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.049+0000 7fc286d9d640 1 --2- 172.21.15.99:0/808753190 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55da76558070 0x55da76563a20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:39.057 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.049+0000 7fc28659c640 1 -- 172.21.15.99:0/808753190 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55da76552f00 msgr2=0x55da7655ca50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:45:39.057 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.049+0000 7fc28659c640 1 --2- 172.21.15.99:0/808753190 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55da76552f00 0x55da7655ca50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:39.057 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.049+0000 7fc28659c640 1 -- 172.21.15.99:0/808753190 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55da76558070 msgr2=0x55da76563a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:45:39.057 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.049+0000 7fc28659c640 1 --2- 172.21.15.99:0/808753190 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55da76558070 0x55da76563a20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:39.058 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.049+0000 7fc28659c640 1 -- 172.21.15.99:0/808753190 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55da7646dc60 con 0x55da76552510 2024-03-19T01:45:39.058 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.049+0000 7fc28659c640 1 --2- 172.21.15.99:0/808753190 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55da76552510 0x55da7655c510 secure :-1 s=READY pgs=437 cs=0 l=1 rev1=1 crypto rx=0x7fc278011d10 tx=0x7fc278004520 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1fe5d16bd29afb3 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:45:39.058 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.049+0000 7fc26dffb640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:45:39.058 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.050+0000 7fc26dffb640 1 -- 172.21.15.99:0/808753190 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc278029070 con 0x55da76552510 2024-03-19T01:45:39.058 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.050+0000 7fc26dffb640 1 -- 172.21.15.99:0/808753190 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7fc278004d50 con 0x55da76552510 2024-03-19T01:45:39.058 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.050+0000 7fc28f7fd740 1 -- 172.21.15.99:0/808753190 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fc278005500 con 0x55da76552510 2024-03-19T01:45:39.058 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.050+0000 7fc26dffb640 1 -- 172.21.15.99:0/808753190 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=26) ==== 24+0+0 (secure 0 0 0) 0x7fc278035050 con 0x55da76552510 2024-03-19T01:45:39.058 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.050+0000 7fc28f7fd740 10 client.15702 fetch_fsmap learned FSMap version 26 2024-03-19T01:45:39.058 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.050+0000 7fc28f7fd740 1 -- 172.21.15.99:0/808753190 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=26,monmap=4+}) -- 0x55da761c7210 con 0x55da76552510 2024-03-19T01:45:39.058 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.050+0000 7fc26dffb640 1 -- 172.21.15.99:0/808753190 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fc278005500 con 0x55da76552510 2024-03-19T01:45:39.058 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.050+0000 7fc26dffb640 1 -- 172.21.15.99:0/808753190 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 26) ==== 8342+0+0 (secure 0 0 0) 0x7fc278023430 con 0x55da76552510 2024-03-19T01:45:39.058 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.050+0000 7fc28f7fd740 10 client.15702 fetch_fsmap finished waiting for FSMap version 26 2024-03-19T01:45:39.058 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.050+0000 7fc28f7fd740 10 client.15702 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:45:39.058 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.050+0000 7fc28f7fd740 20 client.15702 populate_metadata read hostname 'smithi099' 2024-03-19T01:45:39.059 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.050+0000 7fc28f7fd740 1 --2- 172.21.15.99:0/808753190 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55da76568120 0x55da7656a530 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:39.059 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.050+0000 7fc28f7fd740 4 client.15702 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:45:39.059 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.050+0000 7fc28f7fd740 1 -- 172.21.15.99:0/808753190 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55da764c0120 con 0x55da76568120 2024-03-19T01:45:39.059 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.050+0000 7fc285d9b640 1 --2- 172.21.15.99:0/808753190 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55da76568120 0x55da7656a530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:39.059 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.051+0000 7fc285d9b640 1 --2- 172.21.15.99:0/808753190 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55da76568120 0x55da7656a530 crc :-1 s=READY pgs=69 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=1d62f0d41fd6bab9 server_cookie=64e5ed7f53005373 in_seq=0 out_seq=0 2024-03-19T01:45:39.059 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.051+0000 7fc26dffb640 10 client.15702 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:45:39.059 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.051+0000 7fc26dffb640 1 -- 172.21.15.99:0/808753190 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x55da764c0120 con 0x55da76568120 2024-03-19T01:45:39.059 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.052+0000 7fc26dffb640 10 client.15702 handle_command_reply: tid=0 2024-03-19T01:45:39.059 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.052+0000 7fc28f7fd740 1 client.15702 shutdown 2024-03-19T01:45:39.059 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.052+0000 7fc28f7fd740 1 -- 172.21.15.99:0/808753190 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55da76552510 msgr2=0x55da7655c510 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:45:39.059 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.052+0000 7fc28f7fd740 1 --2- 172.21.15.99:0/808753190 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55da76552510 0x55da7655c510 secure :-1 s=READY pgs=437 cs=0 l=1 rev1=1 crypto rx=0x7fc278011d10 tx=0x7fc278004520 comp rx=0 tx=0).stop 2024-03-19T01:45:39.059 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.052+0000 7fc28f7fd740 1 -- 172.21.15.99:0/808753190 shutdown_connections 2024-03-19T01:45:39.059 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.052+0000 7fc28f7fd740 1 --2- 172.21.15.99:0/808753190 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55da76568120 0x55da7656a530 crc :-1 s=READY pgs=69 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:39.060 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.052+0000 7fc28f7fd740 1 --2- 172.21.15.99:0/808753190 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55da76558070 0x55da76563a20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:39.060 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.052+0000 7fc28f7fd740 1 --2- 172.21.15.99:0/808753190 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55da76552f00 0x55da7655ca50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:39.060 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.052+0000 7fc26dffb640 0 client.15702 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:45:39.060 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.052+0000 7fc28f7fd740 1 --2- 172.21.15.99:0/808753190 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55da76552510 0x55da7655c510 unknown :-1 s=CLOSED pgs=437 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:39.060 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.052+0000 7fc28f7fd740 1 -- 172.21.15.99:0/808753190 >> 172.21.15.99:0/808753190 conn(0x55da76549670 msgr2=0x55da7654a8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:45:39.060 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.052+0000 7fc28f7fd740 1 -- 172.21.15.99:0/808753190 shutdown_connections 2024-03-19T01:45:39.060 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.052+0000 7fc28f7fd740 1 -- 172.21.15.99:0/808753190 wait complete. 2024-03-19T01:45:39.060 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.052+0000 7fc28f7fd740 20 client.15702 trim_cache size 0 max 16384 2024-03-19T01:45:39.073 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.068+0000 7fc28f7fd740 1 Processor -- start 2024-03-19T01:45:39.073 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.068+0000 7fc28f7fd740 1 -- start start 2024-03-19T01:45:39.073 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.068+0000 7fc28f7fd740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55da76552510 0x55da7654cfe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:39.073 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.068+0000 7fc28f7fd740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55da76552f00 0x55da765639e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:39.073 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.068+0000 7fc28f7fd740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55da76558070 0x55da7655bb30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:39.073 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.068+0000 7fc28f7fd740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fc2881a9440 con 0x55da76552510 2024-03-19T01:45:39.073 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.068+0000 7fc28f7fd740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fc2881a6d10 con 0x55da76552f00 2024-03-19T01:45:39.073 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.068+0000 7fc28f7fd740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fc2881a4b60 con 0x55da76558070 2024-03-19T01:45:39.073 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.068+0000 7fc28659c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55da76552510 0x55da7654cfe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:39.073 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.068+0000 7fc286d9d640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55da76558070 0x55da7655bb30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:39.073 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.068+0000 7fc28659c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55da76552510 0x55da7654cfe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:44906/0 (socket says 172.21.15.99:44906) 2024-03-19T01:45:39.073 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.068+0000 7fc28659c640 1 -- 172.21.15.99:0/3091077149 learned_addr learned my addr 172.21.15.99:0/3091077149 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:45:39.074 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.068+0000 7fc285d9b640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55da76552f00 0x55da765639e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:39.074 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.068+0000 7fc28659c640 1 -- 172.21.15.99:0/3091077149 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55da76558070 msgr2=0x55da7655bb30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:45:39.074 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.068+0000 7fc28659c640 1 --2- 172.21.15.99:0/3091077149 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55da76558070 0x55da7655bb30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:39.074 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.068+0000 7fc28659c640 1 -- 172.21.15.99:0/3091077149 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55da76552f00 msgr2=0x55da765639e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:45:39.074 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.068+0000 7fc28659c640 1 --2- 172.21.15.99:0/3091077149 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55da76552f00 0x55da765639e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:39.074 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.068+0000 7fc28659c640 1 -- 172.21.15.99:0/3091077149 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55da76293620 con 0x55da76552510 2024-03-19T01:45:39.074 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.068+0000 7fc285d9b640 1 --2- 172.21.15.99:0/3091077149 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55da76552f00 0x55da765639e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:45:39.074 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.068+0000 7fc28659c640 1 --2- 172.21.15.99:0/3091077149 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55da76552510 0x55da7654cfe0 secure :-1 s=READY pgs=438 cs=0 l=1 rev1=1 crypto rx=0x7fc2780238d0 tx=0x7fc278005f90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1cf80fa8cfb86138 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:45:39.074 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.069+0000 7fc26ffff640 1 -- 172.21.15.99:0/3091077149 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc278029070 con 0x55da76552510 2024-03-19T01:45:39.074 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.069+0000 7fc26ffff640 1 -- 172.21.15.99:0/3091077149 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fc278023630 con 0x55da76552510 2024-03-19T01:45:39.074 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.069+0000 7fc26ffff640 1 -- 172.21.15.99:0/3091077149 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc27802e6a0 con 0x55da76552510 2024-03-19T01:45:39.074 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.069+0000 7fc28f7fd740 1 -- 172.21.15.99:0/3091077149 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55da76552510 msgr2=0x55da7654cfe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:45:39.074 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.069+0000 7fc28f7fd740 1 --2- 172.21.15.99:0/3091077149 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55da76552510 0x55da7654cfe0 secure :-1 s=READY pgs=438 cs=0 l=1 rev1=1 crypto rx=0x7fc2780238d0 tx=0x7fc278005f90 comp rx=0 tx=0).stop 2024-03-19T01:45:39.074 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.069+0000 7fc28f7fd740 1 -- 172.21.15.99:0/3091077149 shutdown_connections 2024-03-19T01:45:39.075 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.069+0000 7fc28f7fd740 1 --2- 172.21.15.99:0/3091077149 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55da76552f00 0x55da765639e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:39.075 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.069+0000 7fc28f7fd740 1 --2- 172.21.15.99:0/3091077149 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55da76558070 0x55da7655bb30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:39.075 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.069+0000 7fc28f7fd740 1 --2- 172.21.15.99:0/3091077149 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55da76552510 0x55da7654cfe0 unknown :-1 s=CLOSED pgs=438 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:39.075 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.069+0000 7fc28f7fd740 1 -- 172.21.15.99:0/3091077149 >> 172.21.15.99:0/3091077149 conn(0x55da76549670 msgr2=0x55da7654c8d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:45:39.075 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.069+0000 7fc28f7fd740 1 -- 172.21.15.99:0/3091077149 shutdown_connections 2024-03-19T01:45:39.075 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.069+0000 7fc28f7fd740 1 -- 172.21.15.99:0/3091077149 wait complete. 2024-03-19T01:45:39.075 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.070+0000 7fc28f7fd740 1 -- start start 2024-03-19T01:45:39.075 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.070+0000 7fc28f7fd740 1 Processor -- start 2024-03-19T01:45:39.075 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.070+0000 7fc28f7fd740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55da76552510 0x55da76560f30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:39.075 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.070+0000 7fc28f7fd740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55da76552f00 0x55da76561470 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:39.075 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.070+0000 7fc28f7fd740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55da76558070 0x55da76562480 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:39.075 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.070+0000 7fc28f7fd740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fc2881a51d0 con 0x55da76558070 2024-03-19T01:45:39.075 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.070+0000 7fc28f7fd740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fc2881a6640 con 0x55da76552f00 2024-03-19T01:45:39.075 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.070+0000 7fc28f7fd740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fc2881a58a0 con 0x55da76552510 2024-03-19T01:45:39.076 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.070+0000 7fc28f7fd740 10 client.0 _ll_register_callbacks cb 0x7fc278005e30 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:45:39.076 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.070+0000 7fc286d9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55da76558070 0x55da76562480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:39.076 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.070+0000 7fc28659c640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55da76552510 0x55da76560f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:39.076 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.070+0000 7fc28659c640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55da76552510 0x55da76560f30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:59010/0 (socket says 172.21.15.99:59010) 2024-03-19T01:45:39.076 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.070+0000 7fc286d9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55da76558070 0x55da76562480 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:44910/0 (socket says 172.21.15.99:44910) 2024-03-19T01:45:39.076 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.070+0000 7fc28659c640 1 -- 172.21.15.99:0/1644478345 learned_addr learned my addr 172.21.15.99:0/1644478345 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:45:39.076 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.070+0000 7fc285d9b640 1 --2- 172.21.15.99:0/1644478345 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55da76552f00 0x55da76561470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:39.076 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.070+0000 7fc286d9d640 1 -- 172.21.15.99:0/1644478345 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55da76552510 msgr2=0x55da76560f30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:45:39.076 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.070+0000 7fc286d9d640 1 --2- 172.21.15.99:0/1644478345 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55da76552510 0x55da76560f30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:39.076 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.070+0000 7fc286d9d640 1 -- 172.21.15.99:0/1644478345 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55da76552f00 msgr2=0x55da76561470 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:45:39.076 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.070+0000 7fc286d9d640 1 --2- 172.21.15.99:0/1644478345 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55da76552f00 0x55da76561470 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:39.076 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.070+0000 7fc286d9d640 1 -- 172.21.15.99:0/1644478345 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55da76562a70 con 0x55da76558070 2024-03-19T01:45:39.076 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.070+0000 7fc285d9b640 1 --2- 172.21.15.99:0/1644478345 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55da76552f00 0x55da76561470 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:45:39.077 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.070+0000 7fc28659c640 1 --2- 172.21.15.99:0/1644478345 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55da76552510 0x55da76560f30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:45:39.077 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.070+0000 7fc286d9d640 1 --2- 172.21.15.99:0/1644478345 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55da76558070 0x55da76562480 secure :-1 s=READY pgs=439 cs=0 l=1 rev1=1 crypto rx=0x7fc280034fd0 tx=0x7fc280009500 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7afc3f5421545253 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:45:39.077 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.070+0000 7fc26dffb640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:45:39.077 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.071+0000 7fc26dffb640 1 -- 172.21.15.99:0/1644478345 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc28002caa0 con 0x55da76558070 2024-03-19T01:45:39.077 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.071+0000 7fc26dffb640 1 -- 172.21.15.99:0/1644478345 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7fc28002cc40 con 0x55da76558070 2024-03-19T01:45:39.077 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.071+0000 7fc28f7fd740 1 -- 172.21.15.99:0/1644478345 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fc278023630 con 0x55da76558070 2024-03-19T01:45:39.077 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.071+0000 7fc26dffb640 1 -- 172.21.15.99:0/1644478345 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=26) ==== 24+0+0 (secure 0 0 0) 0x7fc28002ce40 con 0x55da76558070 2024-03-19T01:45:39.077 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.071+0000 7fc28f7fd740 10 client.15708 fetch_fsmap learned FSMap version 26 2024-03-19T01:45:39.077 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.071+0000 7fc28f7fd740 1 -- 172.21.15.99:0/1644478345 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=26,monmap=4+}) -- 0x55da76563df0 con 0x55da76558070 2024-03-19T01:45:39.077 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.071+0000 7fc26dffb640 1 -- 172.21.15.99:0/1644478345 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fc278023630 con 0x55da76558070 2024-03-19T01:45:39.077 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.071+0000 7fc26dffb640 1 -- 172.21.15.99:0/1644478345 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 26) ==== 8342+0+0 (secure 0 0 0) 0x7fc280056020 con 0x55da76558070 2024-03-19T01:45:39.077 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.071+0000 7fc28f7fd740 10 client.15708 fetch_fsmap finished waiting for FSMap version 26 2024-03-19T01:45:39.077 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.071+0000 7fc28f7fd740 10 client.15708 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:45:39.077 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.071+0000 7fc28f7fd740 20 client.15708 populate_metadata read hostname 'smithi099' 2024-03-19T01:45:39.078 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.071+0000 7fc28f7fd740 1 --2- 172.21.15.99:0/1644478345 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55da765ced50 0x55da765d1170 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:39.078 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.071+0000 7fc28f7fd740 4 client.15708 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:45:39.078 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.071+0000 7fc28f7fd740 1 -- 172.21.15.99:0/1644478345 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55da764ba470 con 0x55da765ced50 2024-03-19T01:45:39.078 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.071+0000 7fc28659c640 1 --2- 172.21.15.99:0/1644478345 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55da765ced50 0x55da765d1170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:39.078 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.072+0000 7fc26dffb640 10 client.15708 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:45:39.078 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.072+0000 7fc28659c640 1 --2- 172.21.15.99:0/1644478345 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55da765ced50 0x55da765d1170 crc :-1 s=READY pgs=70 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=8d7363f8ef85ad9d server_cookie=26490fdf743e9fad in_seq=0 out_seq=0 2024-03-19T01:45:39.125 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.124+0000 7fc26dffb640 1 -- 172.21.15.99:0/1644478345 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x55da764ba470 con 0x55da765ced50 2024-03-19T01:45:39.125 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.124+0000 7fc26dffb640 10 client.15708 handle_command_reply: tid=0 2024-03-19T01:45:39.126 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.124+0000 7fc28f7fd740 1 client.15708 shutdown 2024-03-19T01:45:39.126 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.124+0000 7fc28f7fd740 1 -- 172.21.15.99:0/1644478345 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55da76558070 msgr2=0x55da76562480 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:45:39.126 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.124+0000 7fc28f7fd740 1 --2- 172.21.15.99:0/1644478345 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55da76558070 0x55da76562480 secure :-1 s=READY pgs=439 cs=0 l=1 rev1=1 crypto rx=0x7fc280034fd0 tx=0x7fc280009500 comp rx=0 tx=0).stop 2024-03-19T01:45:39.126 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.125+0000 7fc28f7fd740 1 -- 172.21.15.99:0/1644478345 shutdown_connections 2024-03-19T01:45:39.126 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.125+0000 7fc28f7fd740 1 --2- 172.21.15.99:0/1644478345 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55da765ced50 0x55da765d1170 crc :-1 s=READY pgs=70 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:39.126 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.125+0000 7fc26dffb640 0 client.15708 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:45:39.126 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.125+0000 7fc28f7fd740 1 --2- 172.21.15.99:0/1644478345 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55da76552f00 0x55da76561470 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:39.126 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.125+0000 7fc28f7fd740 1 --2- 172.21.15.99:0/1644478345 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55da76558070 0x55da76562480 unknown :-1 s=CLOSED pgs=439 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:39.126 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.125+0000 7fc28f7fd740 1 --2- 172.21.15.99:0/1644478345 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55da76552510 0x55da76560f30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:39.126 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.125+0000 7fc28f7fd740 1 -- 172.21.15.99:0/1644478345 >> 172.21.15.99:0/1644478345 conn(0x55da76549670 msgr2=0x55da7654d160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:45:39.127 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.125+0000 7fc28f7fd740 1 -- 172.21.15.99:0/1644478345 shutdown_connections 2024-03-19T01:45:39.127 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.125+0000 7fc28f7fd740 1 -- 172.21.15.99:0/1644478345 wait complete. 2024-03-19T01:45:39.127 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.125+0000 7fc28f7fd740 20 client.15708 trim_cache size 0 max 16384 2024-03-19T01:45:39.127 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:45:39.127 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:45:39.127 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:45:39.127 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:45:39.128 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.125+0000 7fc28cc14640 1 -- 172.21.15.99:0/935384540 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fc2700a28a0 msgr2=0x7fc2700a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:45:39.128 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.125+0000 7fc28cc14640 1 --2- 172.21.15.99:0/935384540 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fc2700a28a0 0x7fc2700a4d60 secure :-1 s=READY pgs=268 cs=0 l=1 rev1=1 crypto rx=0x7fc280004ab0 tx=0x7fc280004a40 comp rx=0 tx=0).stop 2024-03-19T01:45:39.128 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.125+0000 7fc28cc14640 1 -- 172.21.15.99:0/935384540 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fc288104ca0 msgr2=0x7fc288197f60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:45:39.128 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.125+0000 7fc28cc14640 1 --2- 172.21.15.99:0/935384540 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fc288104ca0 0x7fc288197f60 secure :-1 s=READY pgs=114 cs=0 l=1 rev1=1 crypto rx=0x7fc27c00e9a0 tx=0x7fc27c00ee60 comp rx=0 tx=0).stop 2024-03-19T01:45:39.129 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.125+0000 7fc28cc14640 1 -- 172.21.15.99:0/935384540 shutdown_connections 2024-03-19T01:45:39.129 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.125+0000 7fc28cc14640 1 --2- 172.21.15.99:0/935384540 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fc2700a28a0 0x7fc2700a4d60 unknown :-1 s=CLOSED pgs=268 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:39.129 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.125+0000 7fc28cc14640 1 --2- 172.21.15.99:0/935384540 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fc288104ca0 0x7fc288197f60 unknown :-1 s=CLOSED pgs=114 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:39.129 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.125+0000 7fc28cc14640 1 --2- 172.21.15.99:0/935384540 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fc2881075d0 0x7fc28819efd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:39.129 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.125+0000 7fc28cc14640 1 --2- 172.21.15.99:0/935384540 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc288102370 0x7fc288197a20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:39.129 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.125+0000 7fc28cc14640 1 -- 172.21.15.99:0/935384540 >> 172.21.15.99:0/935384540 conn(0x7fc2880fbcc0 msgr2=0x7fc288102c70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:45:39.129 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.126+0000 7fc28cc14640 1 -- 172.21.15.99:0/935384540 shutdown_connections 2024-03-19T01:45:39.129 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:39.126+0000 7fc28cc14640 1 -- 172.21.15.99:0/935384540 wait complete. 2024-03-19T01:45:39.683 INFO:tasks.workunit.client.0.smithi099.stdout: 1 526124 11.08 MB/sec execute 129 sec latency 26.764 ms 2024-03-19T01:45:40.141 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:45:39 smithi099 ceph-mon[62951]: pgmap v487: 97 pgs: 97 active+clean; 163 MiB data, 1.6 GiB used, 1.0 TiB / 1.0 TiB avail; 795 KiB/s rd, 3.6 MiB/s wr, 280 op/s 2024-03-19T01:45:40.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:45:39 smithi115 ceph-mon[67332]: pgmap v487: 97 pgs: 97 active+clean; 163 MiB data, 1.6 GiB used, 1.0 TiB / 1.0 TiB avail; 795 KiB/s rd, 3.6 MiB/s wr, 280 op/s 2024-03-19T01:45:40.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:45:39 smithi112 ceph-mon[66819]: pgmap v487: 97 pgs: 97 active+clean; 163 MiB data, 1.6 GiB used, 1.0 TiB / 1.0 TiB avail; 795 KiB/s rd, 3.6 MiB/s wr, 280 op/s 2024-03-19T01:45:40.683 INFO:tasks.workunit.client.0.smithi099.stdout: 1 528141 11.05 MB/sec execute 130 sec latency 82.583 ms 2024-03-19T01:45:41.683 INFO:tasks.workunit.client.0.smithi099.stdout: 1 530029 11.06 MB/sec execute 131 sec latency 27.198 ms 2024-03-19T01:45:42.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:45:41 smithi115 ceph-mon[67332]: pgmap v488: 97 pgs: 97 active+clean; 172 MiB data, 1.6 GiB used, 1.0 TiB / 1.0 TiB avail; 655 KiB/s rd, 3.8 MiB/s wr, 247 op/s 2024-03-19T01:45:42.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:45:41 smithi099 ceph-mon[62951]: pgmap v488: 97 pgs: 97 active+clean; 172 MiB data, 1.6 GiB used, 1.0 TiB / 1.0 TiB avail; 655 KiB/s rd, 3.8 MiB/s wr, 247 op/s 2024-03-19T01:45:42.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:45:41 smithi112 ceph-mon[66819]: pgmap v488: 97 pgs: 97 active+clean; 172 MiB data, 1.6 GiB used, 1.0 TiB / 1.0 TiB avail; 655 KiB/s rd, 3.8 MiB/s wr, 247 op/s 2024-03-19T01:45:42.683 INFO:tasks.workunit.client.0.smithi099.stdout: 1 532892 11.10 MB/sec execute 132 sec latency 15.935 ms 2024-03-19T01:45:43.683 INFO:tasks.workunit.client.0.smithi099.stdout: 1 534713 11.07 MB/sec execute 133 sec latency 26.781 ms 2024-03-19T01:45:44.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:45:43 smithi115 ceph-mon[67332]: pgmap v489: 97 pgs: 97 active+clean; 177 MiB data, 1.7 GiB used, 1.0 TiB / 1.0 TiB avail; 846 KiB/s rd, 4.5 MiB/s wr, 271 op/s 2024-03-19T01:45:44.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:45:43 smithi099 ceph-mon[62951]: pgmap v489: 97 pgs: 97 active+clean; 177 MiB data, 1.7 GiB used, 1.0 TiB / 1.0 TiB avail; 846 KiB/s rd, 4.5 MiB/s wr, 271 op/s 2024-03-19T01:45:44.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:45:43 smithi112 ceph-mon[66819]: pgmap v489: 97 pgs: 97 active+clean; 177 MiB data, 1.7 GiB used, 1.0 TiB / 1.0 TiB avail; 846 KiB/s rd, 4.5 MiB/s wr, 271 op/s 2024-03-19T01:45:44.683 INFO:tasks.workunit.client.0.smithi099.stdout: 1 536991 11.09 MB/sec execute 134 sec latency 21.229 ms 2024-03-19T01:45:45.683 INFO:tasks.workunit.client.0.smithi099.stdout: 1 539409 11.11 MB/sec execute 135 sec latency 64.156 ms 2024-03-19T01:45:46.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:45:45 smithi115 ceph-mon[67332]: pgmap v490: 97 pgs: 97 active+clean; 177 MiB data, 1.7 GiB used, 1.0 TiB / 1.0 TiB avail; 613 KiB/s rd, 4.0 MiB/s wr, 222 op/s 2024-03-19T01:45:46.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:45:45 smithi099 ceph-mon[62951]: pgmap v490: 97 pgs: 97 active+clean; 177 MiB data, 1.7 GiB used, 1.0 TiB / 1.0 TiB avail; 613 KiB/s rd, 4.0 MiB/s wr, 222 op/s 2024-03-19T01:45:46.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:45:45 smithi112 ceph-mon[66819]: pgmap v490: 97 pgs: 97 active+clean; 177 MiB data, 1.7 GiB used, 1.0 TiB / 1.0 TiB avail; 613 KiB/s rd, 4.0 MiB/s wr, 222 op/s 2024-03-19T01:45:46.683 INFO:tasks.workunit.client.0.smithi099.stdout: 1 541364 11.11 MB/sec execute 136 sec latency 28.168 ms 2024-03-19T01:45:47.683 INFO:tasks.workunit.client.0.smithi099.stdout: 1 543733 11.13 MB/sec execute 137 sec latency 72.456 ms 2024-03-19T01:45:48.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:45:47 smithi115 ceph-mon[67332]: pgmap v491: 97 pgs: 97 active+clean; 181 MiB data, 1.7 GiB used, 1.0 TiB / 1.0 TiB avail; 896 KiB/s rd, 4.3 MiB/s wr, 279 op/s 2024-03-19T01:45:48.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:45:47 smithi099 ceph-mon[62951]: pgmap v491: 97 pgs: 97 active+clean; 181 MiB data, 1.7 GiB used, 1.0 TiB / 1.0 TiB avail; 896 KiB/s rd, 4.3 MiB/s wr, 279 op/s 2024-03-19T01:45:48.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:45:47 smithi112 ceph-mon[66819]: pgmap v491: 97 pgs: 97 active+clean; 181 MiB data, 1.7 GiB used, 1.0 TiB / 1.0 TiB avail; 896 KiB/s rd, 4.3 MiB/s wr, 279 op/s 2024-03-19T01:45:48.684 INFO:tasks.workunit.client.0.smithi099.stdout: 1 545435 11.09 MB/sec execute 138 sec latency 27.084 ms 2024-03-19T01:45:49.683 INFO:tasks.workunit.client.0.smithi099.stdout: 1 547630 11.11 MB/sec execute 139 sec latency 10.123 ms 2024-03-19T01:45:50.163 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:45:49 smithi112 ceph-mon[66819]: pgmap v492: 97 pgs: 97 active+clean; 183 MiB data, 1.7 GiB used, 1.0 TiB / 1.0 TiB avail; 787 KiB/s rd, 4.0 MiB/s wr, 252 op/s 2024-03-19T01:45:50.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:45:49 smithi115 ceph-mon[67332]: pgmap v492: 97 pgs: 97 active+clean; 183 MiB data, 1.7 GiB used, 1.0 TiB / 1.0 TiB avail; 787 KiB/s rd, 4.0 MiB/s wr, 252 op/s 2024-03-19T01:45:50.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:45:49 smithi099 ceph-mon[62951]: pgmap v492: 97 pgs: 97 active+clean; 183 MiB data, 1.7 GiB used, 1.0 TiB / 1.0 TiB avail; 787 KiB/s rd, 4.0 MiB/s wr, 252 op/s 2024-03-19T01:45:50.683 INFO:tasks.workunit.client.0.smithi099.stdout: 1 550060 11.12 MB/sec execute 140 sec latency 26.542 ms 2024-03-19T01:45:51.684 INFO:tasks.workunit.client.0.smithi099.stdout: 1 551796 11.10 MB/sec execute 141 sec latency 35.998 ms 2024-03-19T01:45:52.217 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.215+0000 7f174207e640 1 -- 172.21.15.99:0/2807750798 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f173c101930 msgr2=0x7f173c101db0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:45:52.218 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.215+0000 7f174207e640 1 --2- 172.21.15.99:0/2807750798 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f173c101930 0x7f173c101db0 secure :-1 s=READY pgs=440 cs=0 l=1 rev1=1 crypto rx=0x7f1724009900 tx=0x7f172402bc30 comp rx=0 tx=0).stop 2024-03-19T01:45:52.218 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.215+0000 7f174207e640 1 -- 172.21.15.99:0/2807750798 shutdown_connections 2024-03-19T01:45:52.218 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.215+0000 7f174207e640 1 --2- 172.21.15.99:0/2807750798 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f173c1022f0 0x7f173c108b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:52.218 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.215+0000 7f174207e640 1 --2- 172.21.15.99:0/2807750798 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f173c101930 0x7f173c101db0 unknown :-1 s=CLOSED pgs=440 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:52.218 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.215+0000 7f174207e640 1 --2- 172.21.15.99:0/2807750798 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f173c100740 0x7f173c100b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:52.218 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.215+0000 7f174207e640 1 -- 172.21.15.99:0/2807750798 >> 172.21.15.99:0/2807750798 conn(0x7f173c0fbcb0 msgr2=0x7f173c0fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:45:52.218 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.215+0000 7f174207e640 1 -- 172.21.15.99:0/2807750798 shutdown_connections 2024-03-19T01:45:52.218 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.215+0000 7f174207e640 1 -- 172.21.15.99:0/2807750798 wait complete. 2024-03-19T01:45:52.218 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.216+0000 7f174207e640 1 Processor -- start 2024-03-19T01:45:52.218 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.216+0000 7f174207e640 1 -- start start 2024-03-19T01:45:52.218 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.216+0000 7f174207e640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f173c100740 0x7f173c1969e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:52.219 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.216+0000 7f174207e640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f173c101930 0x7f173c196f20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:52.219 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.216+0000 7f174207e640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f173c1022f0 0x7f173c19df90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:52.219 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.216+0000 7f174207e640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f173c10c3f0 con 0x7f173c1022f0 2024-03-19T01:45:52.219 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.216+0000 7f174207e640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f173c10c270 con 0x7f173c101930 2024-03-19T01:45:52.219 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.216+0000 7f174207e640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f173c10c570 con 0x7f173c100740 2024-03-19T01:45:52.219 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.216+0000 7f173bfff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f173c1022f0 0x7f173c19df90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:52.219 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.216+0000 7f1733fff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f173c101930 0x7f173c196f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:52.219 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.216+0000 7f173b7fe640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f173c100740 0x7f173c1969e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:52.219 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.216+0000 7f173bfff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f173c1022f0 0x7f173c19df90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:51542/0 (socket says 172.21.15.99:51542) 2024-03-19T01:45:52.219 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.216+0000 7f173bfff640 1 -- 172.21.15.99:0/2312656079 learned_addr learned my addr 172.21.15.99:0/2312656079 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:45:52.219 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.216+0000 7f1733fff640 1 -- 172.21.15.99:0/2312656079 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f173c100740 msgr2=0x7f173c1969e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:45:52.219 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.216+0000 7f1733fff640 1 --2- 172.21.15.99:0/2312656079 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f173c100740 0x7f173c1969e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:52.219 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.216+0000 7f1733fff640 1 -- 172.21.15.99:0/2312656079 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f173c1022f0 msgr2=0x7f173c19df90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:45:52.219 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.216+0000 7f1733fff640 1 --2- 172.21.15.99:0/2312656079 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f173c1022f0 0x7f173c19df90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:52.220 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.216+0000 7f1733fff640 1 -- 172.21.15.99:0/2312656079 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f173c19e600 con 0x7f173c101930 2024-03-19T01:45:52.220 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.216+0000 7f173bfff640 1 --2- 172.21.15.99:0/2312656079 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f173c1022f0 0x7f173c19df90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:45:52.220 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.217+0000 7f1733fff640 1 --2- 172.21.15.99:0/2312656079 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f173c101930 0x7f173c196f20 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7f1724036040 tx=0x7f172402c650 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:45:52.220 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.217+0000 7f17397fa640 1 -- 172.21.15.99:0/2312656079 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1724005000 con 0x7f173c101930 2024-03-19T01:45:52.220 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.217+0000 7f174207e640 1 -- 172.21.15.99:0/2312656079 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f173c068ea0 con 0x7f173c101930 2024-03-19T01:45:52.220 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.217+0000 7f17397fa640 1 -- 172.21.15.99:0/2312656079 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f17240035f0 con 0x7f173c101930 2024-03-19T01:45:52.220 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.217+0000 7f17397fa640 1 -- 172.21.15.99:0/2312656079 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f172403c6b0 con 0x7f173c101930 2024-03-19T01:45:52.220 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.217+0000 7f174207e640 1 -- 172.21.15.99:0/2312656079 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f173c069410 con 0x7f173c101930 2024-03-19T01:45:52.226 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.219+0000 7f1744c67740 1 Processor -- start 2024-03-19T01:45:52.227 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.219+0000 7f17397fa640 1 -- 172.21.15.99:0/2312656079 <== mon.2 v2:172.21.15.112:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f1724003aa0 con 0x7f173c101930 2024-03-19T01:45:52.227 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.219+0000 7f1744c67740 1 -- start start 2024-03-19T01:45:52.227 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.219+0000 7f1744c67740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e985f25100 0x55e985f25520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:52.227 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.219+0000 7f1744c67740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e985f25af0 0x55e985f2a5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:52.227 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.219+0000 7f1744c67740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e985f2ac60 0x55e985f2d0f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:52.227 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.220+0000 7f1744c67740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55e985e89be0 con 0x55e985f25100 2024-03-19T01:45:52.227 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.220+0000 7f1744c67740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55e985e32960 con 0x55e985f25af0 2024-03-19T01:45:52.227 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.220+0000 7f1744c67740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55e985e85c80 con 0x55e985f2ac60 2024-03-19T01:45:52.227 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.220+0000 7f173bfff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e985f25100 0x55e985f25520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:52.227 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.220+0000 7f173bfff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e985f25100 0x55e985f25520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:51558/0 (socket says 172.21.15.99:51558) 2024-03-19T01:45:52.227 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.220+0000 7f173bfff640 1 -- 172.21.15.99:0/2395105954 learned_addr learned my addr 172.21.15.99:0/2395105954 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:45:52.227 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.220+0000 7f1733fff640 1 --2- 172.21.15.99:0/2395105954 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e985f2ac60 0x55e985f2d0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:52.227 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.220+0000 7f173b7fe640 1 --2- 172.21.15.99:0/2395105954 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e985f25af0 0x55e985f2a5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:52.228 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.220+0000 7f173bfff640 1 -- 172.21.15.99:0/2395105954 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e985f2ac60 msgr2=0x55e985f2d0f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:45:52.228 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.220+0000 7f173bfff640 1 --2- 172.21.15.99:0/2395105954 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e985f2ac60 0x55e985f2d0f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:52.228 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.220+0000 7f173bfff640 1 -- 172.21.15.99:0/2395105954 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e985f25af0 msgr2=0x55e985f2a5f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:45:52.228 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.220+0000 7f173bfff640 1 --2- 172.21.15.99:0/2395105954 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e985f25af0 0x55e985f2a5f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:52.228 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.220+0000 7f173bfff640 1 -- 172.21.15.99:0/2395105954 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55e985e29d20 con 0x55e985f25100 2024-03-19T01:45:52.228 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.220+0000 7f1733fff640 1 --2- 172.21.15.99:0/2395105954 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e985f2ac60 0x55e985f2d0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:45:52.228 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.220+0000 7f173b7fe640 1 --2- 172.21.15.99:0/2395105954 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e985f25af0 0x55e985f2a5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:45:52.228 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.220+0000 7f17397fa640 1 --2- 172.21.15.99:0/2312656079 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f17280a28a0 0x7f17280a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:52.228 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.220+0000 7f173b7fe640 1 --2- 172.21.15.99:0/2312656079 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f17280a28a0 0x7f17280a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:52.228 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.220+0000 7f173bfff640 1 --2- 172.21.15.99:0/2395105954 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e985f25100 0x55e985f25520 secure :-1 s=READY pgs=441 cs=0 l=1 rev1=1 crypto rx=0x7f17340118b0 tx=0x7f1734011d70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=53e937d918ff8c65 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:45:52.228 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.220+0000 7f17397fa640 1 -- 172.21.15.99:0/2312656079 <== mon.2 v2:172.21.15.112:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f17240f2b00 con 0x7f173c101930 2024-03-19T01:45:52.228 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.220+0000 7f1731ffb640 1 -- 172.21.15.99:0/2395105954 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f173401e070 con 0x55e985f25100 2024-03-19T01:45:52.228 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.220+0000 7f1731ffb640 1 -- 172.21.15.99:0/2395105954 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f1734011310 con 0x55e985f25100 2024-03-19T01:45:52.229 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.220+0000 7f1731ffb640 1 -- 172.21.15.99:0/2395105954 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1734019e60 con 0x55e985f25100 2024-03-19T01:45:52.229 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.221+0000 7f1744c67740 1 -- 172.21.15.99:0/2395105954 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e985f25100 msgr2=0x55e985f25520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:45:52.229 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.221+0000 7f173b7fe640 1 --2- 172.21.15.99:0/2312656079 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f17280a28a0 0x7f17280a4d60 secure :-1 s=READY pgs=269 cs=0 l=1 rev1=1 crypto rx=0x55e985f25ef0 tx=0x7f172c005f90 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:45:52.229 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.221+0000 7f1744c67740 1 --2- 172.21.15.99:0/2395105954 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e985f25100 0x55e985f25520 secure :-1 s=READY pgs=441 cs=0 l=1 rev1=1 crypto rx=0x7f17340118b0 tx=0x7f1734011d70 comp rx=0 tx=0).stop 2024-03-19T01:45:52.229 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.221+0000 7f1744c67740 1 -- 172.21.15.99:0/2395105954 shutdown_connections 2024-03-19T01:45:52.229 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.221+0000 7f1744c67740 1 --2- 172.21.15.99:0/2395105954 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e985f25af0 0x55e985f2a5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:52.229 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.221+0000 7f1744c67740 1 --2- 172.21.15.99:0/2395105954 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e985f2ac60 0x55e985f2d0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:52.229 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.221+0000 7f1744c67740 1 --2- 172.21.15.99:0/2395105954 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e985f25100 0x55e985f25520 unknown :-1 s=CLOSED pgs=441 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:52.229 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.221+0000 7f1744c67740 1 -- 172.21.15.99:0/2395105954 >> 172.21.15.99:0/2395105954 conn(0x55e985f20670 msgr2=0x55e985f20a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:45:52.229 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.221+0000 7f1744c67740 1 -- 172.21.15.99:0/2395105954 shutdown_connections 2024-03-19T01:45:52.229 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.221+0000 7f1744c67740 1 -- 172.21.15.99:0/2395105954 wait complete. 2024-03-19T01:45:52.229 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.221+0000 7f1744c67740 1 -- start start 2024-03-19T01:45:52.229 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.222+0000 7f1744c67740 1 Processor -- start 2024-03-19T01:45:52.229 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.222+0000 7f1744c67740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e985f25100 0x55e985f2f130 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:52.230 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.222+0000 7f1744c67740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e985f25af0 0x55e985f2f670 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:52.230 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.222+0000 7f1744c67740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e985f2ac60 0x55e985f36640 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:52.230 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.222+0000 7f1744c67740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55e985f2dc10 con 0x55e985f25100 2024-03-19T01:45:52.230 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.222+0000 7f1744c67740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55e985e45db0 con 0x55e985f25af0 2024-03-19T01:45:52.230 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.222+0000 7f1744c67740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55e985f2dd90 con 0x55e985f2ac60 2024-03-19T01:45:52.230 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.222+0000 7f1744c67740 10 client.0 _ll_register_callbacks cb 0x55e985e4ae70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:45:52.233 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.222+0000 7f1733fff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e985f25af0 0x55e985f2f670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:52.234 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.222+0000 7f173b7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e985f25100 0x55e985f2f130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:52.234 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.222+0000 7f173bfff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e985f2ac60 0x55e985f36640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:52.234 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.222+0000 7f1733fff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e985f25af0 0x55e985f2f670 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:40680/0 (socket says 172.21.15.99:40680) 2024-03-19T01:45:52.234 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.222+0000 7f1733fff640 1 -- 172.21.15.99:0/3858161540 learned_addr learned my addr 172.21.15.99:0/3858161540 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:45:52.234 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.222+0000 7f1733fff640 1 -- 172.21.15.99:0/3858161540 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e985f2ac60 msgr2=0x55e985f36640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:45:52.234 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.222+0000 7f1733fff640 1 --2- 172.21.15.99:0/3858161540 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e985f2ac60 0x55e985f36640 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:52.234 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.222+0000 7f1733fff640 1 -- 172.21.15.99:0/3858161540 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e985f25100 msgr2=0x55e985f2f130 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:45:52.234 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.222+0000 7f1733fff640 1 --2- 172.21.15.99:0/3858161540 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e985f25100 0x55e985f2f130 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:52.234 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.222+0000 7f1733fff640 1 -- 172.21.15.99:0/3858161540 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55e985e44c60 con 0x55e985f25af0 2024-03-19T01:45:52.234 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.222+0000 7f173b7fe640 1 --2- 172.21.15.99:0/3858161540 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e985f25100 0x55e985f2f130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:45:52.234 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.222+0000 7f1733fff640 1 --2- 172.21.15.99:0/3858161540 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e985f25af0 0x55e985f2f670 secure :-1 s=READY pgs=117 cs=0 l=1 rev1=1 crypto rx=0x7f17240b9fd0 tx=0x7f1724003910 comp rx=0 tx=0).ready entity=mon.2 client_cookie=42ce1193c290da42 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:45:52.234 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.222+0000 7f171affd640 10 client.0 ms_handle_connect on v2:172.21.15.112:3300/0 2024-03-19T01:45:52.235 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.223+0000 7f171affd640 1 -- 172.21.15.99:0/3858161540 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f17240bb700 con 0x55e985f25af0 2024-03-19T01:45:52.235 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.223+0000 7f171affd640 1 -- 172.21.15.99:0/3858161540 <== mon.2 v2:172.21.15.112:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f17240bf8c0 con 0x55e985f25af0 2024-03-19T01:45:52.235 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.223+0000 7f1744c67740 1 -- 172.21.15.99:0/3858161540 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f1734011310 con 0x55e985f25af0 2024-03-19T01:45:52.235 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.223+0000 7f171affd640 1 -- 172.21.15.99:0/3858161540 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_get_version_reply(handle=1 version=26) ==== 24+0+0 (secure 0 0 0) 0x7f17240bedc0 con 0x55e985f25af0 2024-03-19T01:45:52.235 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.223+0000 7f1744c67740 10 client.25493 fetch_fsmap learned FSMap version 26 2024-03-19T01:45:52.235 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.223+0000 7f1744c67740 1 -- 172.21.15.99:0/3858161540 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,fsmap=26,monmap=4+}) -- 0x55e985b9e210 con 0x55e985f25af0 2024-03-19T01:45:52.235 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.223+0000 7f171affd640 1 -- 172.21.15.99:0/3858161540 <== mon.2 v2:172.21.15.112:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f1734011310 con 0x55e985f25af0 2024-03-19T01:45:52.235 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.223+0000 7f171affd640 1 -- 172.21.15.99:0/3858161540 <== mon.2 v2:172.21.15.112:3300/0 5 ==== fsmap(e 26) ==== 8342+0+0 (secure 0 0 0) 0x7f17240c7770 con 0x55e985f25af0 2024-03-19T01:45:52.235 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.223+0000 7f1744c67740 10 client.25493 fetch_fsmap finished waiting for FSMap version 26 2024-03-19T01:45:52.235 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.223+0000 7f1744c67740 10 client.25493 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:45:52.235 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.223+0000 7f1744c67740 20 client.25493 populate_metadata read hostname 'smithi099' 2024-03-19T01:45:52.235 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.223+0000 7f1744c67740 1 --2- 172.21.15.99:0/3858161540 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e985f3af20 0x55e985f3d380 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:52.235 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.223+0000 7f1744c67740 4 client.25493 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:45:52.235 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.223+0000 7f1744c67740 1 -- 172.21.15.99:0/3858161540 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55e985e97120 con 0x55e985f3af20 2024-03-19T01:45:52.235 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.223+0000 7f173b7fe640 1 --2- 172.21.15.99:0/3858161540 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e985f3af20 0x55e985f3d380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:52.236 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.224+0000 7f173b7fe640 1 --2- 172.21.15.99:0/3858161540 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e985f3af20 0x55e985f3d380 crc :-1 s=READY pgs=71 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=513c30d3c863f335 server_cookie=f102875575ffa891 in_seq=0 out_seq=0 2024-03-19T01:45:52.236 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.224+0000 7f171affd640 10 client.25493 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:45:52.236 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.225+0000 7f171affd640 1 -- 172.21.15.99:0/3858161540 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x55e985e97120 con 0x55e985f3af20 2024-03-19T01:45:52.236 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.225+0000 7f171affd640 10 client.25493 handle_command_reply: tid=0 2024-03-19T01:45:52.236 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.225+0000 7f1744c67740 1 client.25493 shutdown 2024-03-19T01:45:52.236 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.225+0000 7f1744c67740 1 -- 172.21.15.99:0/3858161540 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e985f25af0 msgr2=0x55e985f2f670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:45:52.236 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.225+0000 7f1744c67740 1 --2- 172.21.15.99:0/3858161540 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e985f25af0 0x55e985f2f670 secure :-1 s=READY pgs=117 cs=0 l=1 rev1=1 crypto rx=0x7f17240b9fd0 tx=0x7f1724003910 comp rx=0 tx=0).stop 2024-03-19T01:45:52.236 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.225+0000 7f1744c67740 1 -- 172.21.15.99:0/3858161540 shutdown_connections 2024-03-19T01:45:52.236 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.225+0000 7f1744c67740 1 --2- 172.21.15.99:0/3858161540 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e985f3af20 0x55e985f3d380 crc :-1 s=READY pgs=71 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:52.236 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.225+0000 7f1744c67740 1 --2- 172.21.15.99:0/3858161540 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e985f25af0 0x55e985f2f670 unknown :-1 s=CLOSED pgs=117 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:52.236 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.225+0000 7f1744c67740 1 --2- 172.21.15.99:0/3858161540 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e985f2ac60 0x55e985f36640 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:52.236 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.225+0000 7f1744c67740 1 --2- 172.21.15.99:0/3858161540 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e985f25100 0x55e985f2f130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:52.236 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.225+0000 7f171affd640 0 client.25493 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:45:52.236 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.225+0000 7f1744c67740 1 -- 172.21.15.99:0/3858161540 >> 172.21.15.99:0/3858161540 conn(0x55e985f20670 msgr2=0x55e985f218c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:45:52.236 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.225+0000 7f1744c67740 1 -- 172.21.15.99:0/3858161540 shutdown_connections 2024-03-19T01:45:52.237 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.225+0000 7f1744c67740 1 -- 172.21.15.99:0/3858161540 wait complete. 2024-03-19T01:45:52.237 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.225+0000 7f1744c67740 20 client.25493 trim_cache size 0 max 16384 2024-03-19T01:45:52.244 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.240+0000 7f1744c67740 1 Processor -- start 2024-03-19T01:45:52.244 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.240+0000 7f1744c67740 1 -- start start 2024-03-19T01:45:52.244 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.240+0000 7f1744c67740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e985f20670 0x55e985f1fab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:52.244 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.240+0000 7f1744c67740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e985f25100 0x55e985f36600 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:52.244 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.240+0000 7f1744c67740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e985f25af0 0x55e985f23fe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:52.244 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.240+0000 7f1744c67740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f173c1a9420 con 0x55e985f25af0 2024-03-19T01:45:52.244 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.240+0000 7f1744c67740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f173c1a6cf0 con 0x55e985f25100 2024-03-19T01:45:52.244 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.240+0000 7f1744c67740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f173c1a4b40 con 0x55e985f20670 2024-03-19T01:45:52.244 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.240+0000 7f173bfff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e985f25af0 0x55e985f23fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:52.245 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.240+0000 7f173bfff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e985f25af0 0x55e985f23fe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:51582/0 (socket says 172.21.15.99:51582) 2024-03-19T01:45:52.245 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.240+0000 7f173bfff640 1 -- 172.21.15.99:0/2578112180 learned_addr learned my addr 172.21.15.99:0/2578112180 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:45:52.245 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.240+0000 7f173b7fe640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e985f20670 0x55e985f1fab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:52.245 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.240+0000 7f1733fff640 1 --2- 172.21.15.99:0/2578112180 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e985f25100 0x55e985f36600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:52.245 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.240+0000 7f173bfff640 1 -- 172.21.15.99:0/2578112180 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e985f20670 msgr2=0x55e985f1fab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:45:52.245 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.240+0000 7f173bfff640 1 --2- 172.21.15.99:0/2578112180 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e985f20670 0x55e985f1fab0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:52.245 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.240+0000 7f173bfff640 1 -- 172.21.15.99:0/2578112180 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e985f25100 msgr2=0x55e985f36600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:45:52.245 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.240+0000 7f173bfff640 1 --2- 172.21.15.99:0/2578112180 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e985f25100 0x55e985f36600 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:52.245 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.240+0000 7f173bfff640 1 -- 172.21.15.99:0/2578112180 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55e985c6a620 con 0x55e985f25af0 2024-03-19T01:45:52.245 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.240+0000 7f1733fff640 1 --2- 172.21.15.99:0/2578112180 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e985f25100 0x55e985f36600 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:45:52.245 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.240+0000 7f173bfff640 1 --2- 172.21.15.99:0/2578112180 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e985f25af0 0x55e985f23fe0 secure :-1 s=READY pgs=442 cs=0 l=1 rev1=1 crypto rx=0x7f17340118b0 tx=0x7f1734011360 comp rx=0 tx=0).ready entity=mon.0 client_cookie=23a349875bd3fde0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:45:52.245 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.240+0000 7f17317fa640 1 -- 172.21.15.99:0/2578112180 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f173401e040 con 0x55e985f25af0 2024-03-19T01:45:52.245 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.240+0000 7f17317fa640 1 -- 172.21.15.99:0/2578112180 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f173402a070 con 0x55e985f25af0 2024-03-19T01:45:52.245 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.240+0000 7f17317fa640 1 -- 172.21.15.99:0/2578112180 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f17340053b0 con 0x55e985f25af0 2024-03-19T01:45:52.246 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.241+0000 7f1744c67740 1 -- 172.21.15.99:0/2578112180 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e985f25af0 msgr2=0x55e985f23fe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:45:52.246 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.241+0000 7f1744c67740 1 --2- 172.21.15.99:0/2578112180 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e985f25af0 0x55e985f23fe0 secure :-1 s=READY pgs=442 cs=0 l=1 rev1=1 crypto rx=0x7f17340118b0 tx=0x7f1734011360 comp rx=0 tx=0).stop 2024-03-19T01:45:52.246 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.241+0000 7f1744c67740 1 -- 172.21.15.99:0/2578112180 shutdown_connections 2024-03-19T01:45:52.246 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.241+0000 7f1744c67740 1 --2- 172.21.15.99:0/2578112180 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e985f25100 0x55e985f36600 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:52.246 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.241+0000 7f1744c67740 1 --2- 172.21.15.99:0/2578112180 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e985f25af0 0x55e985f23fe0 unknown :-1 s=CLOSED pgs=442 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:52.246 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.241+0000 7f1744c67740 1 --2- 172.21.15.99:0/2578112180 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e985f20670 0x55e985f1fab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:52.246 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.241+0000 7f1744c67740 1 -- 172.21.15.99:0/2578112180 >> 172.21.15.99:0/2578112180 conn(0x55e985f1bd20 msgr2=0x55e985f238d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:45:52.246 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.241+0000 7f1744c67740 1 -- 172.21.15.99:0/2578112180 shutdown_connections 2024-03-19T01:45:52.246 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.241+0000 7f1744c67740 1 -- 172.21.15.99:0/2578112180 wait complete. 2024-03-19T01:45:52.246 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.241+0000 7f1744c67740 1 -- start start 2024-03-19T01:45:52.246 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.242+0000 7f1744c67740 1 Processor -- start 2024-03-19T01:45:52.246 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.242+0000 7f1744c67740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e985f20670 0x55e985f33bf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:52.246 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.242+0000 7f1744c67740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e985f25100 0x55e985f34130 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:52.246 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.242+0000 7f1744c67740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e985f25af0 0x55e985f350f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:52.247 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.242+0000 7f1744c67740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f173c1a51b0 con 0x55e985f20670 2024-03-19T01:45:52.247 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.242+0000 7f1744c67740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f173c1a6620 con 0x55e985f25100 2024-03-19T01:45:52.247 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.242+0000 7f1744c67740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f173c1a5880 con 0x55e985f25af0 2024-03-19T01:45:52.247 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.242+0000 7f1744c67740 10 client.0 _ll_register_callbacks cb 0x7f17240f62d0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:45:52.247 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.242+0000 7f173b7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e985f20670 0x55e985f33bf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:52.247 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.242+0000 7f173b7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e985f20670 0x55e985f33bf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:51584/0 (socket says 172.21.15.99:51584) 2024-03-19T01:45:52.247 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.242+0000 7f173b7fe640 1 -- 172.21.15.99:0/2388390422 learned_addr learned my addr 172.21.15.99:0/2388390422 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:45:52.247 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.242+0000 7f1733fff640 1 --2- 172.21.15.99:0/2388390422 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e985f25100 0x55e985f34130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:52.247 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.242+0000 7f173bfff640 1 --2- 172.21.15.99:0/2388390422 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e985f25af0 0x55e985f350f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:52.247 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.242+0000 7f173b7fe640 1 -- 172.21.15.99:0/2388390422 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e985f25af0 msgr2=0x55e985f350f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:45:52.247 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.242+0000 7f173b7fe640 1 --2- 172.21.15.99:0/2388390422 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e985f25af0 0x55e985f350f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:52.247 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.242+0000 7f173b7fe640 1 -- 172.21.15.99:0/2388390422 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e985f25100 msgr2=0x55e985f34130 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:45:52.247 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.242+0000 7f173b7fe640 1 --2- 172.21.15.99:0/2388390422 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e985f25100 0x55e985f34130 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:52.247 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.242+0000 7f173b7fe640 1 -- 172.21.15.99:0/2388390422 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55e985f356e0 con 0x55e985f20670 2024-03-19T01:45:52.248 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.242+0000 7f173b7fe640 1 --2- 172.21.15.99:0/2388390422 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e985f20670 0x55e985f33bf0 secure :-1 s=READY pgs=443 cs=0 l=1 rev1=1 crypto rx=0x55e985f26560 tx=0x7f172c0046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=608d91545a4c6e9 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:45:52.248 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.242+0000 7f171affd640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:45:52.248 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.242+0000 7f171affd640 1 -- 172.21.15.99:0/2388390422 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f172c039070 con 0x55e985f20670 2024-03-19T01:45:52.248 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.243+0000 7f171affd640 1 -- 172.21.15.99:0/2388390422 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f172c00fad0 con 0x55e985f20670 2024-03-19T01:45:52.248 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.243+0000 7f1744c67740 1 -- 172.21.15.99:0/2388390422 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f173402a070 con 0x55e985f20670 2024-03-19T01:45:52.248 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.243+0000 7f171affd640 1 -- 172.21.15.99:0/2388390422 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=26) ==== 24+0+0 (secure 0 0 0) 0x7f172c045050 con 0x55e985f20670 2024-03-19T01:45:52.248 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.243+0000 7f1744c67740 10 client.15726 fetch_fsmap learned FSMap version 26 2024-03-19T01:45:52.248 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.243+0000 7f1744c67740 1 -- 172.21.15.99:0/2388390422 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=26,monmap=4+}) -- 0x55e985fa1280 con 0x55e985f20670 2024-03-19T01:45:52.248 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.243+0000 7f171affd640 1 -- 172.21.15.99:0/2388390422 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f173402a070 con 0x55e985f20670 2024-03-19T01:45:52.248 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.243+0000 7f171affd640 1 -- 172.21.15.99:0/2388390422 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 26) ==== 8342+0+0 (secure 0 0 0) 0x7f172c030700 con 0x55e985f20670 2024-03-19T01:45:52.248 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.243+0000 7f1744c67740 10 client.15726 fetch_fsmap finished waiting for FSMap version 26 2024-03-19T01:45:52.248 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.243+0000 7f1744c67740 10 client.15726 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:45:52.248 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.243+0000 7f1744c67740 20 client.15726 populate_metadata read hostname 'smithi099' 2024-03-19T01:45:52.248 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.243+0000 7f1744c67740 1 --2- 172.21.15.99:0/2388390422 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e985f2ac60 0x55e985fa3c70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:45:52.248 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.243+0000 7f1744c67740 4 client.15726 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:45:52.249 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.243+0000 7f1744c67740 1 -- 172.21.15.99:0/2388390422 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55e985e91470 con 0x55e985f2ac60 2024-03-19T01:45:52.249 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.243+0000 7f1733fff640 1 --2- 172.21.15.99:0/2388390422 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e985f2ac60 0x55e985fa3c70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:45:52.249 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.244+0000 7f171affd640 10 client.15726 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:45:52.249 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.244+0000 7f1733fff640 1 --2- 172.21.15.99:0/2388390422 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e985f2ac60 0x55e985fa3c70 crc :-1 s=READY pgs=72 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=e872ec4a0025ebf6 server_cookie=d63e9cb3784c92a3 in_seq=0 out_seq=0 2024-03-19T01:45:52.294 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.294+0000 7f171affd640 1 -- 172.21.15.99:0/2388390422 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x55e985e91470 con 0x55e985f2ac60 2024-03-19T01:45:52.295 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.294+0000 7f171affd640 10 client.15726 handle_command_reply: tid=0 2024-03-19T01:45:52.295 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.294+0000 7f1744c67740 1 client.15726 shutdown 2024-03-19T01:45:52.295 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.294+0000 7f1744c67740 1 -- 172.21.15.99:0/2388390422 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e985f20670 msgr2=0x55e985f33bf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:45:52.295 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.294+0000 7f1744c67740 1 --2- 172.21.15.99:0/2388390422 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e985f20670 0x55e985f33bf0 secure :-1 s=READY pgs=443 cs=0 l=1 rev1=1 crypto rx=0x55e985f26560 tx=0x7f172c0046c0 comp rx=0 tx=0).stop 2024-03-19T01:45:52.295 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.294+0000 7f1744c67740 1 -- 172.21.15.99:0/2388390422 shutdown_connections 2024-03-19T01:45:52.295 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.294+0000 7f1744c67740 1 --2- 172.21.15.99:0/2388390422 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e985f2ac60 0x55e985fa3c70 crc :-1 s=READY pgs=72 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:52.295 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.294+0000 7f1744c67740 1 --2- 172.21.15.99:0/2388390422 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e985f25100 0x55e985f34130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:52.295 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.294+0000 7f171affd640 0 client.15726 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:45:52.295 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.294+0000 7f1744c67740 1 --2- 172.21.15.99:0/2388390422 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e985f25af0 0x55e985f350f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:52.295 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.294+0000 7f1744c67740 1 --2- 172.21.15.99:0/2388390422 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e985f20670 0x55e985f33bf0 unknown :-1 s=CLOSED pgs=443 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:52.295 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.294+0000 7f1744c67740 1 -- 172.21.15.99:0/2388390422 >> 172.21.15.99:0/2388390422 conn(0x55e985f1bd20 msgr2=0x55e985f24160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:45:52.295 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.294+0000 7f1744c67740 1 -- 172.21.15.99:0/2388390422 shutdown_connections 2024-03-19T01:45:52.295 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.294+0000 7f1744c67740 1 -- 172.21.15.99:0/2388390422 wait complete. 2024-03-19T01:45:52.296 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.294+0000 7f1744c67740 20 client.15726 trim_cache size 0 max 16384 2024-03-19T01:45:52.296 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:45:52.296 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:45:52.296 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:45:52.296 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:45:52.297 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.295+0000 7f174207e640 1 -- 172.21.15.99:0/2312656079 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f17280a28a0 msgr2=0x7f17280a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:45:52.297 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.295+0000 7f174207e640 1 --2- 172.21.15.99:0/2312656079 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f17280a28a0 0x7f17280a4d60 secure :-1 s=READY pgs=269 cs=0 l=1 rev1=1 crypto rx=0x55e985f25ef0 tx=0x7f172c005f90 comp rx=0 tx=0).stop 2024-03-19T01:45:52.297 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.295+0000 7f174207e640 1 -- 172.21.15.99:0/2312656079 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f173c101930 msgr2=0x7f173c196f20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:45:52.297 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.295+0000 7f174207e640 1 --2- 172.21.15.99:0/2312656079 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f173c101930 0x7f173c196f20 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7f1724036040 tx=0x7f172402c650 comp rx=0 tx=0).stop 2024-03-19T01:45:52.297 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.295+0000 7f174207e640 1 -- 172.21.15.99:0/2312656079 shutdown_connections 2024-03-19T01:45:52.297 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.295+0000 7f174207e640 1 --2- 172.21.15.99:0/2312656079 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f17280a28a0 0x7f17280a4d60 unknown :-1 s=CLOSED pgs=269 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:52.297 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.295+0000 7f174207e640 1 --2- 172.21.15.99:0/2312656079 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f173c101930 0x7f173c196f20 unknown :-1 s=CLOSED pgs=116 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:52.298 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.295+0000 7f174207e640 1 --2- 172.21.15.99:0/2312656079 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f173c1022f0 0x7f173c19df90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:52.298 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.295+0000 7f174207e640 1 --2- 172.21.15.99:0/2312656079 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f173c100740 0x7f173c1969e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:45:52.298 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.295+0000 7f174207e640 1 -- 172.21.15.99:0/2312656079 >> 172.21.15.99:0/2312656079 conn(0x7f173c0fbcb0 msgr2=0x7f173c104c30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:45:52.298 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.295+0000 7f174207e640 1 -- 172.21.15.99:0/2312656079 shutdown_connections 2024-03-19T01:45:52.298 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:45:52.295+0000 7f174207e640 1 -- 172.21.15.99:0/2312656079 wait complete. 2024-03-19T01:45:52.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:45:51 smithi115 ceph-mon[67332]: pgmap v493: 97 pgs: 97 active+clean; 184 MiB data, 1.8 GiB used, 1.0 TiB / 1.0 TiB avail; 745 KiB/s rd, 3.1 MiB/s wr, 227 op/s 2024-03-19T01:45:52.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:45:51 smithi099 ceph-mon[62951]: pgmap v493: 97 pgs: 97 active+clean; 184 MiB data, 1.8 GiB used, 1.0 TiB / 1.0 TiB avail; 745 KiB/s rd, 3.1 MiB/s wr, 227 op/s 2024-03-19T01:45:52.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:45:51 smithi112 ceph-mon[66819]: pgmap v493: 97 pgs: 97 active+clean; 184 MiB data, 1.8 GiB used, 1.0 TiB / 1.0 TiB avail; 745 KiB/s rd, 3.1 MiB/s wr, 227 op/s 2024-03-19T01:45:52.683 INFO:tasks.workunit.client.0.smithi099.stdout: 1 554148 11.13 MB/sec execute 142 sec latency 78.507 ms 2024-03-19T01:45:53.684 INFO:tasks.workunit.client.0.smithi099.stdout: 1 555683 11.10 MB/sec execute 143 sec latency 26.431 ms 2024-03-19T01:45:54.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:45:53 smithi115 ceph-mon[67332]: pgmap v494: 97 pgs: 97 active+clean; 186 MiB data, 1.8 GiB used, 1.0 TiB / 1.0 TiB avail; 994 KiB/s rd, 2.9 MiB/s wr, 259 op/s 2024-03-19T01:45:54.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:45:53 smithi099 ceph-mon[62951]: pgmap v494: 97 pgs: 97 active+clean; 186 MiB data, 1.8 GiB used, 1.0 TiB / 1.0 TiB avail; 994 KiB/s rd, 2.9 MiB/s wr, 259 op/s 2024-03-19T01:45:54.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:45:53 smithi112 ceph-mon[66819]: pgmap v494: 97 pgs: 97 active+clean; 186 MiB data, 1.8 GiB used, 1.0 TiB / 1.0 TiB avail; 994 KiB/s rd, 2.9 MiB/s wr, 259 op/s 2024-03-19T01:45:54.685 INFO:tasks.workunit.client.0.smithi099.stdout: 1 558132 11.12 MB/sec execute 144 sec latency 9.752 ms 2024-03-19T01:45:55.684 INFO:tasks.workunit.client.0.smithi099.stdout: 1 560599 11.13 MB/sec execute 145 sec latency 24.682 ms 2024-03-19T01:45:56.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:45:55 smithi115 ceph-mon[67332]: pgmap v495: 97 pgs: 97 active+clean; 187 MiB data, 1.8 GiB used, 1.0 TiB / 1.0 TiB avail; 797 KiB/s rd, 2.2 MiB/s wr, 225 op/s 2024-03-19T01:45:56.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:45:55 smithi099 ceph-mon[62951]: pgmap v495: 97 pgs: 97 active+clean; 187 MiB data, 1.8 GiB used, 1.0 TiB / 1.0 TiB avail; 797 KiB/s rd, 2.2 MiB/s wr, 225 op/s 2024-03-19T01:45:56.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:45:55 smithi112 ceph-mon[66819]: pgmap v495: 97 pgs: 97 active+clean; 187 MiB data, 1.8 GiB used, 1.0 TiB / 1.0 TiB avail; 797 KiB/s rd, 2.2 MiB/s wr, 225 op/s 2024-03-19T01:45:56.684 INFO:tasks.workunit.client.0.smithi099.stdout: 1 562420 11.11 MB/sec execute 146 sec latency 40.119 ms 2024-03-19T01:45:57.684 INFO:tasks.workunit.client.0.smithi099.stdout: 1 565018 11.15 MB/sec execute 147 sec latency 69.592 ms 2024-03-19T01:45:58.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:45:57 smithi115 ceph-mon[67332]: pgmap v496: 97 pgs: 97 active+clean; 186 MiB data, 1.8 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 3.0 MiB/s wr, 303 op/s 2024-03-19T01:45:58.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:45:57 smithi099 ceph-mon[62951]: pgmap v496: 97 pgs: 97 active+clean; 186 MiB data, 1.8 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 3.0 MiB/s wr, 303 op/s 2024-03-19T01:45:58.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:45:57 smithi112 ceph-mon[66819]: pgmap v496: 97 pgs: 97 active+clean; 186 MiB data, 1.8 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 3.0 MiB/s wr, 303 op/s 2024-03-19T01:45:58.684 INFO:tasks.workunit.client.0.smithi099.stdout: 1 566709 11.12 MB/sec execute 148 sec latency 32.039 ms 2024-03-19T01:45:59.684 INFO:tasks.workunit.client.0.smithi099.stdout: 1 568938 11.13 MB/sec execute 149 sec latency 27.728 ms 2024-03-19T01:46:00.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:45:59 smithi115 ceph-mon[67332]: pgmap v497: 97 pgs: 97 active+clean; 186 MiB data, 1.8 GiB used, 1.0 TiB / 1.0 TiB avail; 874 KiB/s rd, 2.8 MiB/s wr, 259 op/s 2024-03-19T01:46:00.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:45:59 smithi099 ceph-mon[62951]: pgmap v497: 97 pgs: 97 active+clean; 186 MiB data, 1.8 GiB used, 1.0 TiB / 1.0 TiB avail; 874 KiB/s rd, 2.8 MiB/s wr, 259 op/s 2024-03-19T01:46:00.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:45:59 smithi112 ceph-mon[66819]: pgmap v497: 97 pgs: 97 active+clean; 186 MiB data, 1.8 GiB used, 1.0 TiB / 1.0 TiB avail; 874 KiB/s rd, 2.8 MiB/s wr, 259 op/s 2024-03-19T01:46:00.684 INFO:tasks.workunit.client.0.smithi099.stdout: 1 571668 11.15 MB/sec execute 150 sec latency 14.902 ms 2024-03-19T01:46:01.684 INFO:tasks.workunit.client.0.smithi099.stdout: 1 573297 11.15 MB/sec execute 151 sec latency 38.762 ms 2024-03-19T01:46:02.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:46:01 smithi115 ceph-mon[67332]: pgmap v498: 97 pgs: 97 active+clean; 187 MiB data, 1.8 GiB used, 1.0 TiB / 1.0 TiB avail; 839 KiB/s rd, 2.8 MiB/s wr, 252 op/s 2024-03-19T01:46:02.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:46:01 smithi099 ceph-mon[62951]: pgmap v498: 97 pgs: 97 active+clean; 187 MiB data, 1.8 GiB used, 1.0 TiB / 1.0 TiB avail; 839 KiB/s rd, 2.8 MiB/s wr, 252 op/s 2024-03-19T01:46:02.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:46:01 smithi112 ceph-mon[66819]: pgmap v498: 97 pgs: 97 active+clean; 187 MiB data, 1.8 GiB used, 1.0 TiB / 1.0 TiB avail; 839 KiB/s rd, 2.8 MiB/s wr, 252 op/s 2024-03-19T01:46:02.686 INFO:tasks.workunit.client.0.smithi099.stdout: 1 575660 11.16 MB/sec execute 152 sec latency 74.145 ms 2024-03-19T01:46:03.684 INFO:tasks.workunit.client.0.smithi099.stdout: 1 577209 11.13 MB/sec execute 153 sec latency 34.883 ms 2024-03-19T01:46:04.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:46:03 smithi115 ceph-mon[67332]: pgmap v499: 97 pgs: 97 active+clean; 189 MiB data, 1.8 GiB used, 1.0 TiB / 1.0 TiB avail; 1020 KiB/s rd, 3.0 MiB/s wr, 291 op/s 2024-03-19T01:46:04.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:46:03 smithi099 ceph-mon[62951]: pgmap v499: 97 pgs: 97 active+clean; 189 MiB data, 1.8 GiB used, 1.0 TiB / 1.0 TiB avail; 1020 KiB/s rd, 3.0 MiB/s wr, 291 op/s 2024-03-19T01:46:04.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:46:03 smithi112 ceph-mon[66819]: pgmap v499: 97 pgs: 97 active+clean; 189 MiB data, 1.8 GiB used, 1.0 TiB / 1.0 TiB avail; 1020 KiB/s rd, 3.0 MiB/s wr, 291 op/s 2024-03-19T01:46:04.684 INFO:tasks.workunit.client.0.smithi099.stdout: 1 579431 11.14 MB/sec execute 154 sec latency 11.247 ms 2024-03-19T01:46:05.384 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.381+0000 7f549c3f4640 1 -- 172.21.15.99:0/1523494833 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5494101880 msgr2=0x7f5494101ca0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:46:05.384 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.381+0000 7f549c3f4640 1 --2- 172.21.15.99:0/1523494833 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5494101880 0x7f5494101ca0 secure :-1 s=READY pgs=444 cs=0 l=1 rev1=1 crypto rx=0x7f5490009900 tx=0x7f549002bc30 comp rx=0 tx=0).stop 2024-03-19T01:46:05.384 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.381+0000 7f549c3f4640 1 -- 172.21.15.99:0/1523494833 shutdown_connections 2024-03-19T01:46:05.384 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.381+0000 7f549c3f4640 1 --2- 172.21.15.99:0/1523494833 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5494103430 0x7f5494109cc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:05.384 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.381+0000 7f549c3f4640 1 --2- 172.21.15.99:0/1523494833 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5494102a70 0x7f5494102ef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:05.385 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.381+0000 7f549c3f4640 1 --2- 172.21.15.99:0/1523494833 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5494101880 0x7f5494101ca0 unknown :-1 s=CLOSED pgs=444 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:05.385 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.381+0000 7f549c3f4640 1 -- 172.21.15.99:0/1523494833 >> 172.21.15.99:0/1523494833 conn(0x7f54940fcdf0 msgr2=0x7f54940ff230 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:46:05.385 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.381+0000 7f549c3f4640 1 -- 172.21.15.99:0/1523494833 shutdown_connections 2024-03-19T01:46:05.385 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.381+0000 7f549c3f4640 1 -- 172.21.15.99:0/1523494833 wait complete. 2024-03-19T01:46:05.385 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.382+0000 7f549c3f4640 1 Processor -- start 2024-03-19T01:46:05.385 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.382+0000 7f549c3f4640 1 -- start start 2024-03-19T01:46:05.385 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.382+0000 7f549c3f4640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5494101880 0x7f5494197b30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:05.385 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.382+0000 7f549c3f4640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5494102a70 0x7f5494198070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:05.385 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.382+0000 7f549c3f4640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5494103430 0x7f549419f090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:05.385 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.382+0000 7f549c3f4640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f549410d4f0 con 0x7f5494103430 2024-03-19T01:46:05.385 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.382+0000 7f549c3f4640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f549410d370 con 0x7f5494102a70 2024-03-19T01:46:05.385 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.382+0000 7f549c3f4640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f549410d670 con 0x7f5494101880 2024-03-19T01:46:05.385 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.382+0000 7f549a991640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5494103430 0x7f549419f090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:05.386 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.382+0000 7f549998f640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5494102a70 0x7f5494198070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:05.386 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.382+0000 7f549a991640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5494103430 0x7f549419f090 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:35814/0 (socket says 172.21.15.99:35814) 2024-03-19T01:46:05.386 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.382+0000 7f549a991640 1 -- 172.21.15.99:0/850034200 learned_addr learned my addr 172.21.15.99:0/850034200 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:46:05.386 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.382+0000 7f549a190640 1 --2- 172.21.15.99:0/850034200 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5494101880 0x7f5494197b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:05.386 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.382+0000 7f549a991640 1 -- 172.21.15.99:0/850034200 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5494101880 msgr2=0x7f5494197b30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:46:05.386 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.382+0000 7f549a991640 1 --2- 172.21.15.99:0/850034200 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5494101880 0x7f5494197b30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:05.386 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.382+0000 7f549a991640 1 -- 172.21.15.99:0/850034200 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5494102a70 msgr2=0x7f5494198070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:46:05.386 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.382+0000 7f549a991640 1 --2- 172.21.15.99:0/850034200 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5494102a70 0x7f5494198070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:05.386 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.382+0000 7f549a991640 1 -- 172.21.15.99:0/850034200 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f549419f700 con 0x7f5494103430 2024-03-19T01:46:05.386 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.382+0000 7f549a991640 1 --2- 172.21.15.99:0/850034200 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5494103430 0x7f549419f090 secure :-1 s=READY pgs=445 cs=0 l=1 rev1=1 crypto rx=0x7f548c00eb10 tx=0x7f548c00c2d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:46:05.386 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.383+0000 7f548b7fe640 1 -- 172.21.15.99:0/850034200 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f548c004b20 con 0x7f5494103430 2024-03-19T01:46:05.386 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.383+0000 7f548b7fe640 1 -- 172.21.15.99:0/850034200 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f548c004cc0 con 0x7f5494103430 2024-03-19T01:46:05.386 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.383+0000 7f549c3f4640 1 -- 172.21.15.99:0/850034200 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f549419f9f0 con 0x7f5494103430 2024-03-19T01:46:05.386 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.383+0000 7f548b7fe640 1 -- 172.21.15.99:0/850034200 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f548c010760 con 0x7f5494103430 2024-03-19T01:46:05.387 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.383+0000 7f549c3f4640 1 -- 172.21.15.99:0/850034200 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f549419ff40 con 0x7f5494103430 2024-03-19T01:46:05.393 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.385+0000 7f548b7fe640 1 -- 172.21.15.99:0/850034200 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f548c005020 con 0x7f5494103430 2024-03-19T01:46:05.393 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.385+0000 7f549efe5740 1 Processor -- start 2024-03-19T01:46:05.393 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.385+0000 7f549efe5740 1 -- start start 2024-03-19T01:46:05.393 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.385+0000 7f549efe5740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557deafbc100 0x557deafbc520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:05.393 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.385+0000 7f549efe5740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557deafbcaf0 0x557deafc15f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:05.393 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.385+0000 7f549efe5740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557deafc1c60 0x557deafc40f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:05.393 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.385+0000 7f549efe5740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x557deaf20be0 con 0x557deafbcaf0 2024-03-19T01:46:05.393 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.385+0000 7f549efe5740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x557deaec9960 con 0x557deafbc100 2024-03-19T01:46:05.393 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.385+0000 7f549efe5740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x557deaf1cc80 con 0x557deafc1c60 2024-03-19T01:46:05.394 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.385+0000 7f548b7fe640 1 --2- 172.21.15.99:0/850034200 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f54800a2920 0x7f54800a4de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:05.394 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.386+0000 7f549a190640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557deafbcaf0 0x557deafc15f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:05.394 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.386+0000 7f549a190640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557deafbcaf0 0x557deafc15f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:35826/0 (socket says 172.21.15.99:35826) 2024-03-19T01:46:05.394 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.386+0000 7f549998f640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557deafbc100 0x557deafbc520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:05.394 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.386+0000 7f549a190640 1 -- 172.21.15.99:0/3757281447 learned_addr learned my addr 172.21.15.99:0/3757281447 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:46:05.394 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.386+0000 7f549998f640 1 --2- 172.21.15.99:0/3757281447 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557deafc1c60 0x557deafc40f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:05.394 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.386+0000 7f549a991640 1 --2- 172.21.15.99:0/850034200 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f54800a2920 0x7f54800a4de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:05.394 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.386+0000 7f548b7fe640 1 -- 172.21.15.99:0/850034200 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f548c0cef80 con 0x7f5494103430 2024-03-19T01:46:05.394 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.386+0000 7f549998f640 1 -- 172.21.15.99:0/3757281447 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557deafc1c60 msgr2=0x557deafc40f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:46:05.394 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.386+0000 7f549998f640 1 --2- 172.21.15.99:0/3757281447 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557deafc1c60 0x557deafc40f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:05.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.386+0000 7f549998f640 1 -- 172.21.15.99:0/3757281447 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557deafbcaf0 msgr2=0x557deafc15f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:46:05.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.386+0000 7f549998f640 1 --2- 172.21.15.99:0/3757281447 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557deafbcaf0 0x557deafc15f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:05.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.386+0000 7f549998f640 1 -- 172.21.15.99:0/3757281447 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x557deaec0d20 con 0x557deafbc100 2024-03-19T01:46:05.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.386+0000 7f549998f640 1 --2- 172.21.15.99:0/3757281447 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557deafbc100 0x557deafbc520 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 crypto rx=0x7f5494198ff0 tx=0x7f5484004730 comp rx=0 tx=0).ready entity=mon.2 client_cookie=47f6bc1d0f9687cc server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:46:05.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.386+0000 7f549a991640 1 --2- 172.21.15.99:0/850034200 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f54800a2920 0x7f54800a4de0 secure :-1 s=READY pgs=270 cs=0 l=1 rev1=1 crypto rx=0x7f548c0ccdb0 tx=0x7f548c086040 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:46:05.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.386+0000 7f5488bfe640 1 -- 172.21.15.99:0/3757281447 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5484004050 con 0x557deafbc100 2024-03-19T01:46:05.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.386+0000 7f5488bfe640 1 -- 172.21.15.99:0/3757281447 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f548400fca0 con 0x557deafbc100 2024-03-19T01:46:05.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.386+0000 7f5488bfe640 1 -- 172.21.15.99:0/3757281447 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5484003950 con 0x557deafbc100 2024-03-19T01:46:05.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.387+0000 7f549efe5740 1 -- 172.21.15.99:0/3757281447 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557deafbc100 msgr2=0x557deafbc520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:46:05.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.387+0000 7f549efe5740 1 --2- 172.21.15.99:0/3757281447 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557deafbc100 0x557deafbc520 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 crypto rx=0x7f5494198ff0 tx=0x7f5484004730 comp rx=0 tx=0).stop 2024-03-19T01:46:05.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.387+0000 7f549efe5740 1 -- 172.21.15.99:0/3757281447 shutdown_connections 2024-03-19T01:46:05.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.387+0000 7f549efe5740 1 --2- 172.21.15.99:0/3757281447 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557deafc1c60 0x557deafc40f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:05.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.387+0000 7f549efe5740 1 --2- 172.21.15.99:0/3757281447 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557deafbcaf0 0x557deafc15f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:05.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.387+0000 7f549efe5740 1 --2- 172.21.15.99:0/3757281447 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557deafbc100 0x557deafbc520 unknown :-1 s=CLOSED pgs=118 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:05.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.387+0000 7f549efe5740 1 -- 172.21.15.99:0/3757281447 >> 172.21.15.99:0/3757281447 conn(0x557deafb7670 msgr2=0x557deafb7a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:46:05.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.387+0000 7f549efe5740 1 -- 172.21.15.99:0/3757281447 shutdown_connections 2024-03-19T01:46:05.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.387+0000 7f549efe5740 1 -- 172.21.15.99:0/3757281447 wait complete. 2024-03-19T01:46:05.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.387+0000 7f549efe5740 1 -- start start 2024-03-19T01:46:05.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.388+0000 7f549efe5740 1 Processor -- start 2024-03-19T01:46:05.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.388+0000 7f549efe5740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557deafbc100 0x557deafc6130 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:05.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.388+0000 7f549efe5740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557deafbcaf0 0x557deafc6670 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:05.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.388+0000 7f549efe5740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557deafc1c60 0x557deafcd6b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:05.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.388+0000 7f549efe5740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x557deafc4c10 con 0x557deafbcaf0 2024-03-19T01:46:05.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.388+0000 7f549efe5740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x557deaedcdb0 con 0x557deafc1c60 2024-03-19T01:46:05.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.388+0000 7f549efe5740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x557deafc4d90 con 0x557deafbc100 2024-03-19T01:46:05.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.388+0000 7f549efe5740 10 client.0 _ll_register_callbacks cb 0x557deaee1e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:46:05.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.388+0000 7f549a190640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557deafbcaf0 0x557deafc6670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:05.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.388+0000 7f549998f640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557deafbc100 0x557deafc6130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:05.397 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.388+0000 7f549a190640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557deafbcaf0 0x557deafc6670 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:35830/0 (socket says 172.21.15.99:35830) 2024-03-19T01:46:05.397 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.388+0000 7f549a190640 1 -- 172.21.15.99:0/104407235 learned_addr learned my addr 172.21.15.99:0/104407235 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:46:05.397 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.388+0000 7f549998f640 1 --2- 172.21.15.99:0/104407235 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557deafc1c60 0x557deafcd6b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:05.397 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.388+0000 7f549a190640 1 -- 172.21.15.99:0/104407235 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557deafbc100 msgr2=0x557deafc6130 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:46:05.397 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.388+0000 7f549a190640 1 --2- 172.21.15.99:0/104407235 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557deafbc100 0x557deafc6130 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:05.397 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.388+0000 7f549a190640 1 -- 172.21.15.99:0/104407235 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557deafc1c60 msgr2=0x557deafcd6b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:46:05.397 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.388+0000 7f549a190640 1 --2- 172.21.15.99:0/104407235 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557deafc1c60 0x557deafcd6b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:05.397 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.388+0000 7f549a190640 1 -- 172.21.15.99:0/104407235 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x557deaedbc60 con 0x557deafbcaf0 2024-03-19T01:46:05.397 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.388+0000 7f549a190640 1 --2- 172.21.15.99:0/104407235 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557deafbcaf0 0x557deafc6670 secure :-1 s=READY pgs=446 cs=0 l=1 rev1=1 crypto rx=0x7f5490009230 tx=0x7f5490004270 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7b34f91838c95135 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:46:05.397 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.388+0000 7f547dffb640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:46:05.397 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.388+0000 7f547dffb640 1 -- 172.21.15.99:0/104407235 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5490005d50 con 0x557deafbcaf0 2024-03-19T01:46:05.397 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.388+0000 7f547dffb640 1 -- 172.21.15.99:0/104407235 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f5490005220 con 0x557deafbcaf0 2024-03-19T01:46:05.397 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.388+0000 7f549efe5740 1 -- 172.21.15.99:0/104407235 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f548400fca0 con 0x557deafbcaf0 2024-03-19T01:46:05.397 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.389+0000 7f547dffb640 1 -- 172.21.15.99:0/104407235 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=26) ==== 24+0+0 (secure 0 0 0) 0x7f5490005420 con 0x557deafbcaf0 2024-03-19T01:46:05.398 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.389+0000 7f549efe5740 10 client.15738 fetch_fsmap learned FSMap version 26 2024-03-19T01:46:05.398 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.389+0000 7f549efe5740 1 -- 172.21.15.99:0/104407235 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=26,monmap=4+}) -- 0x557deac35210 con 0x557deafbcaf0 2024-03-19T01:46:05.398 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.389+0000 7f547dffb640 1 -- 172.21.15.99:0/104407235 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f548400fca0 con 0x557deafbcaf0 2024-03-19T01:46:05.398 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.389+0000 7f547dffb640 1 -- 172.21.15.99:0/104407235 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 26) ==== 8342+0+0 (secure 0 0 0) 0x7f5490003850 con 0x557deafbcaf0 2024-03-19T01:46:05.398 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.389+0000 7f549efe5740 10 client.15738 fetch_fsmap finished waiting for FSMap version 26 2024-03-19T01:46:05.398 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.389+0000 7f549efe5740 10 client.15738 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:46:05.398 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.389+0000 7f549efe5740 20 client.15738 populate_metadata read hostname 'smithi099' 2024-03-19T01:46:05.398 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.389+0000 7f549efe5740 1 --2- 172.21.15.99:0/104407235 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x557deafd2000 0x557deafd4440 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:05.398 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.389+0000 7f549efe5740 4 client.15738 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:46:05.398 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.389+0000 7f549efe5740 1 -- 172.21.15.99:0/104407235 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x557deaf2e120 con 0x557deafd2000 2024-03-19T01:46:05.398 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.389+0000 7f549998f640 1 --2- 172.21.15.99:0/104407235 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x557deafd2000 0x557deafd4440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:05.398 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.390+0000 7f549998f640 1 --2- 172.21.15.99:0/104407235 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x557deafd2000 0x557deafd4440 crc :-1 s=READY pgs=73 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=ef754297b8fc8e0d server_cookie=624b2c30e15bd52 in_seq=0 out_seq=0 2024-03-19T01:46:05.398 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.390+0000 7f547dffb640 10 client.15738 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:46:05.398 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.390+0000 7f547dffb640 1 -- 172.21.15.99:0/104407235 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x557deaf2e120 con 0x557deafd2000 2024-03-19T01:46:05.399 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.390+0000 7f547dffb640 10 client.15738 handle_command_reply: tid=0 2024-03-19T01:46:05.399 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.390+0000 7f549efe5740 1 client.15738 shutdown 2024-03-19T01:46:05.399 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.391+0000 7f549efe5740 1 -- 172.21.15.99:0/104407235 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557deafbcaf0 msgr2=0x557deafc6670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:46:05.399 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.391+0000 7f549efe5740 1 --2- 172.21.15.99:0/104407235 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557deafbcaf0 0x557deafc6670 secure :-1 s=READY pgs=446 cs=0 l=1 rev1=1 crypto rx=0x7f5490009230 tx=0x7f5490004270 comp rx=0 tx=0).stop 2024-03-19T01:46:05.399 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.391+0000 7f549efe5740 1 -- 172.21.15.99:0/104407235 shutdown_connections 2024-03-19T01:46:05.399 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.391+0000 7f549efe5740 1 --2- 172.21.15.99:0/104407235 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x557deafd2000 0x557deafd4440 crc :-1 s=READY pgs=73 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:05.399 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.391+0000 7f549efe5740 1 --2- 172.21.15.99:0/104407235 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557deafc1c60 0x557deafcd6b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:05.399 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.391+0000 7f549efe5740 1 --2- 172.21.15.99:0/104407235 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557deafbcaf0 0x557deafc6670 unknown :-1 s=CLOSED pgs=446 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:05.399 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.391+0000 7f547dffb640 0 client.15738 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:46:05.399 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.391+0000 7f549efe5740 1 --2- 172.21.15.99:0/104407235 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557deafbc100 0x557deafc6130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:05.399 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.391+0000 7f549efe5740 1 -- 172.21.15.99:0/104407235 >> 172.21.15.99:0/104407235 conn(0x557deafb7670 msgr2=0x557deafb88c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:46:05.399 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.391+0000 7f549efe5740 1 -- 172.21.15.99:0/104407235 shutdown_connections 2024-03-19T01:46:05.399 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.391+0000 7f549efe5740 1 -- 172.21.15.99:0/104407235 wait complete. 2024-03-19T01:46:05.399 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.391+0000 7f549efe5740 20 client.15738 trim_cache size 0 max 16384 2024-03-19T01:46:05.412 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.407+0000 7f549efe5740 1 Processor -- start 2024-03-19T01:46:05.412 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.407+0000 7f549efe5740 1 -- start start 2024-03-19T01:46:05.413 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.407+0000 7f549efe5740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557deafbc100 0x557deafb6ab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:05.413 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.407+0000 7f549efe5740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557deafbcaf0 0x557deafcd670 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:05.413 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.407+0000 7f549efe5740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557deafc1c60 0x557deafbafe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:05.413 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.407+0000 7f549efe5740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f54941ab630 con 0x557deafbcaf0 2024-03-19T01:46:05.413 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.407+0000 7f549efe5740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f54941a8f00 con 0x557deafbc100 2024-03-19T01:46:05.413 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.407+0000 7f549efe5740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f54941a6d50 con 0x557deafc1c60 2024-03-19T01:46:05.413 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.407+0000 7f549a190640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557deafbcaf0 0x557deafcd670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:05.413 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.407+0000 7f549a190640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557deafbcaf0 0x557deafcd670 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:35840/0 (socket says 172.21.15.99:35840) 2024-03-19T01:46:05.413 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.407+0000 7f549a190640 1 -- 172.21.15.99:0/4209475383 learned_addr learned my addr 172.21.15.99:0/4209475383 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:46:05.413 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.407+0000 7f549998f640 1 --2- 172.21.15.99:0/4209475383 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557deafbc100 0x557deafb6ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:05.413 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.407+0000 7f549998f640 1 --2- 172.21.15.99:0/4209475383 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557deafc1c60 0x557deafbafe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:05.413 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.407+0000 7f549a190640 1 -- 172.21.15.99:0/4209475383 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557deafc1c60 msgr2=0x557deafbafe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:46:05.413 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.407+0000 7f549a190640 1 --2- 172.21.15.99:0/4209475383 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557deafc1c60 0x557deafbafe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:05.414 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.407+0000 7f549a190640 1 -- 172.21.15.99:0/4209475383 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557deafbc100 msgr2=0x557deafb6ab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:46:05.414 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.407+0000 7f549a190640 1 --2- 172.21.15.99:0/4209475383 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557deafbc100 0x557deafb6ab0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:05.414 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.407+0000 7f549a190640 1 -- 172.21.15.99:0/4209475383 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x557dead01620 con 0x557deafbcaf0 2024-03-19T01:46:05.414 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.407+0000 7f549998f640 1 --2- 172.21.15.99:0/4209475383 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557deafbc100 0x557deafb6ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-19T01:46:05.414 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.407+0000 7f549a190640 1 --2- 172.21.15.99:0/4209475383 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557deafbcaf0 0x557deafcd670 secure :-1 s=READY pgs=447 cs=0 l=1 rev1=1 crypto rx=0x7f549002c170 tx=0x7f5490000bc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=961c2987cd66fed5 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:46:05.414 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.407+0000 7f547ffff640 1 -- 172.21.15.99:0/4209475383 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5490054070 con 0x557deafbcaf0 2024-03-19T01:46:05.414 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.407+0000 7f547ffff640 1 -- 172.21.15.99:0/4209475383 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f549004b030 con 0x557deafbcaf0 2024-03-19T01:46:05.414 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.407+0000 7f547ffff640 1 -- 172.21.15.99:0/4209475383 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5490034a10 con 0x557deafbcaf0 2024-03-19T01:46:05.414 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.408+0000 7f549efe5740 1 -- 172.21.15.99:0/4209475383 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557deafbcaf0 msgr2=0x557deafcd670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:46:05.414 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.408+0000 7f549efe5740 1 --2- 172.21.15.99:0/4209475383 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557deafbcaf0 0x557deafcd670 secure :-1 s=READY pgs=447 cs=0 l=1 rev1=1 crypto rx=0x7f549002c170 tx=0x7f5490000bc0 comp rx=0 tx=0).stop 2024-03-19T01:46:05.414 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.408+0000 7f549efe5740 1 -- 172.21.15.99:0/4209475383 shutdown_connections 2024-03-19T01:46:05.414 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.408+0000 7f549efe5740 1 --2- 172.21.15.99:0/4209475383 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557deafc1c60 0x557deafbafe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:05.414 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.408+0000 7f549efe5740 1 --2- 172.21.15.99:0/4209475383 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557deafbcaf0 0x557deafcd670 unknown :-1 s=CLOSED pgs=447 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:05.414 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.408+0000 7f549efe5740 1 --2- 172.21.15.99:0/4209475383 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557deafbc100 0x557deafb6ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:05.415 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.408+0000 7f549efe5740 1 -- 172.21.15.99:0/4209475383 >> 172.21.15.99:0/4209475383 conn(0x557deafb7670 msgr2=0x557deafba8d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:46:05.415 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.408+0000 7f549efe5740 1 -- 172.21.15.99:0/4209475383 shutdown_connections 2024-03-19T01:46:05.415 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.408+0000 7f549efe5740 1 -- 172.21.15.99:0/4209475383 wait complete. 2024-03-19T01:46:05.415 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.408+0000 7f549efe5740 1 -- start start 2024-03-19T01:46:05.415 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.409+0000 7f549efe5740 1 Processor -- start 2024-03-19T01:46:05.415 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.409+0000 7f549efe5740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557deafbc100 0x557deafcafb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:05.415 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.409+0000 7f549efe5740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557deafbcaf0 0x557deafcb4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:05.415 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.409+0000 7f549efe5740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557deafc1c60 0x557deafcc500 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:05.415 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.409+0000 7f549efe5740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f54941a73c0 con 0x557deafc1c60 2024-03-19T01:46:05.415 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.409+0000 7f549efe5740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f54941a8830 con 0x557deafbcaf0 2024-03-19T01:46:05.415 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.409+0000 7f549efe5740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f54941a7a90 con 0x557deafbc100 2024-03-19T01:46:05.415 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.409+0000 7f549efe5740 10 client.0 _ll_register_callbacks cb 0x7f5490009780 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:46:05.415 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.409+0000 7f549998f640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557deafbc100 0x557deafcafb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:05.415 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.409+0000 7f549a190640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557deafbcaf0 0x557deafcb4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:05.416 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.409+0000 7f549998f640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557deafc1c60 0x557deafcc500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:05.416 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.409+0000 7f549a190640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557deafbcaf0 0x557deafcb4f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:43104/0 (socket says 172.21.15.99:43104) 2024-03-19T01:46:05.416 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.409+0000 7f549a190640 1 -- 172.21.15.99:0/2643911354 learned_addr learned my addr 172.21.15.99:0/2643911354 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:46:05.416 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.409+0000 7f549998f640 1 -- 172.21.15.99:0/2643911354 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557deafbc100 msgr2=0x557deafcafb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:46:05.416 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.409+0000 7f549998f640 1 --2- 172.21.15.99:0/2643911354 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557deafbc100 0x557deafcafb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:05.416 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.409+0000 7f549998f640 1 -- 172.21.15.99:0/2643911354 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557deafbcaf0 msgr2=0x557deafcb4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:46:05.416 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.409+0000 7f549998f640 1 --2- 172.21.15.99:0/2643911354 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557deafbcaf0 0x557deafcb4f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:05.416 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.409+0000 7f549998f640 1 -- 172.21.15.99:0/2643911354 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x557deafccaf0 con 0x557deafc1c60 2024-03-19T01:46:05.416 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.409+0000 7f549a190640 1 --2- 172.21.15.99:0/2643911354 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557deafbcaf0 0x557deafcb4f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:46:05.416 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.409+0000 7f549998f640 1 --2- 172.21.15.99:0/2643911354 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557deafc1c60 0x557deafcc500 secure :-1 s=READY pgs=448 cs=0 l=1 rev1=1 crypto rx=0x7f5484009fd0 tx=0x7f5484004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ce9bfbda0afabf6f server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:46:05.416 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.409+0000 7f547dffb640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:46:05.416 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.409+0000 7f547dffb640 1 -- 172.21.15.99:0/2643911354 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5484030720 con 0x557deafc1c60 2024-03-19T01:46:05.416 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.409+0000 7f547dffb640 1 -- 172.21.15.99:0/2643911354 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f5484011050 con 0x557deafc1c60 2024-03-19T01:46:05.416 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.410+0000 7f549efe5740 1 -- 172.21.15.99:0/2643911354 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f549004b030 con 0x557deafc1c60 2024-03-19T01:46:05.416 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.410+0000 7f547dffb640 1 -- 172.21.15.99:0/2643911354 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=26) ==== 24+0+0 (secure 0 0 0) 0x7f5484044050 con 0x557deafc1c60 2024-03-19T01:46:05.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.410+0000 7f549efe5740 10 client.15744 fetch_fsmap learned FSMap version 26 2024-03-19T01:46:05.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.410+0000 7f549efe5740 1 -- 172.21.15.99:0/2643911354 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=26,monmap=4+}) -- 0x557deb038610 con 0x557deafc1c60 2024-03-19T01:46:05.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.410+0000 7f547dffb640 1 -- 172.21.15.99:0/2643911354 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f549004b030 con 0x557deafc1c60 2024-03-19T01:46:05.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.410+0000 7f547dffb640 1 -- 172.21.15.99:0/2643911354 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 26) ==== 8342+0+0 (secure 0 0 0) 0x7f548403b070 con 0x557deafc1c60 2024-03-19T01:46:05.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.410+0000 7f549efe5740 10 client.15744 fetch_fsmap finished waiting for FSMap version 26 2024-03-19T01:46:05.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.410+0000 7f549efe5740 10 client.15744 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:46:05.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.410+0000 7f549efe5740 20 client.15744 populate_metadata read hostname 'smithi099' 2024-03-19T01:46:05.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.410+0000 7f549efe5740 1 --2- 172.21.15.99:0/2643911354 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x557deafd2000 0x557deb03ae00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:05.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.410+0000 7f549efe5740 4 client.15744 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:46:05.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.410+0000 7f549efe5740 1 -- 172.21.15.99:0/2643911354 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x557deaf28470 con 0x557deafd2000 2024-03-19T01:46:05.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.410+0000 7f549a190640 1 --2- 172.21.15.99:0/2643911354 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x557deafd2000 0x557deb03ae00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:05.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.411+0000 7f547dffb640 10 client.15744 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:46:05.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.411+0000 7f549a190640 1 --2- 172.21.15.99:0/2643911354 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x557deafd2000 0x557deb03ae00 crc :-1 s=READY pgs=74 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=9a1825e8e66399f5 server_cookie=3b9bb88bea73bce9 in_seq=0 out_seq=0 2024-03-19T01:46:05.462 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.460+0000 7f547dffb640 1 -- 172.21.15.99:0/2643911354 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x557deaf28470 con 0x557deafd2000 2024-03-19T01:46:05.462 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.460+0000 7f547dffb640 10 client.15744 handle_command_reply: tid=0 2024-03-19T01:46:05.462 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.460+0000 7f549efe5740 1 client.15744 shutdown 2024-03-19T01:46:05.462 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.461+0000 7f549efe5740 1 -- 172.21.15.99:0/2643911354 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557deafc1c60 msgr2=0x557deafcc500 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:46:05.462 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.461+0000 7f549efe5740 1 --2- 172.21.15.99:0/2643911354 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557deafc1c60 0x557deafcc500 secure :-1 s=READY pgs=448 cs=0 l=1 rev1=1 crypto rx=0x7f5484009fd0 tx=0x7f5484004730 comp rx=0 tx=0).stop 2024-03-19T01:46:05.463 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.461+0000 7f549efe5740 1 -- 172.21.15.99:0/2643911354 shutdown_connections 2024-03-19T01:46:05.463 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.461+0000 7f549efe5740 1 --2- 172.21.15.99:0/2643911354 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x557deafd2000 0x557deb03ae00 crc :-1 s=READY pgs=74 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:05.463 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.461+0000 7f549efe5740 1 --2- 172.21.15.99:0/2643911354 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557deafbcaf0 0x557deafcb4f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:05.463 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.461+0000 7f549efe5740 1 --2- 172.21.15.99:0/2643911354 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557deafc1c60 0x557deafcc500 unknown :-1 s=CLOSED pgs=448 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:05.463 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.461+0000 7f547dffb640 0 client.15744 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:46:05.463 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.461+0000 7f549efe5740 1 --2- 172.21.15.99:0/2643911354 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557deafbc100 0x557deafcafb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:05.463 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.461+0000 7f549efe5740 1 -- 172.21.15.99:0/2643911354 >> 172.21.15.99:0/2643911354 conn(0x557deafb7670 msgr2=0x557deafba8d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:46:05.463 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.461+0000 7f549efe5740 1 -- 172.21.15.99:0/2643911354 shutdown_connections 2024-03-19T01:46:05.463 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.461+0000 7f549efe5740 1 -- 172.21.15.99:0/2643911354 wait complete. 2024-03-19T01:46:05.463 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.461+0000 7f549efe5740 20 client.15744 trim_cache size 0 max 16384 2024-03-19T01:46:05.463 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:46:05.464 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:46:05.464 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:46:05.464 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:46:05.464 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.461+0000 7f549c3f4640 1 -- 172.21.15.99:0/850034200 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f54800a2920 msgr2=0x7f54800a4de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:46:05.464 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.461+0000 7f549c3f4640 1 --2- 172.21.15.99:0/850034200 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f54800a2920 0x7f54800a4de0 secure :-1 s=READY pgs=270 cs=0 l=1 rev1=1 crypto rx=0x7f548c0ccdb0 tx=0x7f548c086040 comp rx=0 tx=0).stop 2024-03-19T01:46:05.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.461+0000 7f549c3f4640 1 -- 172.21.15.99:0/850034200 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5494103430 msgr2=0x7f549419f090 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:46:05.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.461+0000 7f549c3f4640 1 --2- 172.21.15.99:0/850034200 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5494103430 0x7f549419f090 secure :-1 s=READY pgs=445 cs=0 l=1 rev1=1 crypto rx=0x7f548c00eb10 tx=0x7f548c00c2d0 comp rx=0 tx=0).stop 2024-03-19T01:46:05.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.461+0000 7f549c3f4640 1 -- 172.21.15.99:0/850034200 shutdown_connections 2024-03-19T01:46:05.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.461+0000 7f549c3f4640 1 --2- 172.21.15.99:0/850034200 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f54800a2920 0x7f54800a4de0 unknown :-1 s=CLOSED pgs=270 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:05.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.461+0000 7f549c3f4640 1 --2- 172.21.15.99:0/850034200 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5494102a70 0x7f5494198070 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:05.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.461+0000 7f549c3f4640 1 --2- 172.21.15.99:0/850034200 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5494103430 0x7f549419f090 unknown :-1 s=CLOSED pgs=445 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:05.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.461+0000 7f549c3f4640 1 --2- 172.21.15.99:0/850034200 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5494101880 0x7f5494197b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:05.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.461+0000 7f549c3f4640 1 -- 172.21.15.99:0/850034200 >> 172.21.15.99:0/850034200 conn(0x7f54940fcdf0 msgr2=0x7f5494105d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:46:05.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.461+0000 7f549c3f4640 1 -- 172.21.15.99:0/850034200 shutdown_connections 2024-03-19T01:46:05.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:05.461+0000 7f549c3f4640 1 -- 172.21.15.99:0/850034200 wait complete. 2024-03-19T01:46:05.684 INFO:tasks.workunit.client.0.smithi099.stdout: 1 581559 11.12 MB/sec execute 155 sec latency 40.561 ms 2024-03-19T01:46:06.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:46:05 smithi115 ceph-mon[67332]: pgmap v500: 97 pgs: 97 active+clean; 189 MiB data, 1.8 GiB used, 1.0 TiB / 1.0 TiB avail; 757 KiB/s rd, 2.7 MiB/s wr, 239 op/s 2024-03-19T01:46:06.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:46:05 smithi099 ceph-mon[62951]: pgmap v500: 97 pgs: 97 active+clean; 189 MiB data, 1.8 GiB used, 1.0 TiB / 1.0 TiB avail; 757 KiB/s rd, 2.7 MiB/s wr, 239 op/s 2024-03-19T01:46:06.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:46:05 smithi112 ceph-mon[66819]: pgmap v500: 97 pgs: 97 active+clean; 189 MiB data, 1.8 GiB used, 1.0 TiB / 1.0 TiB avail; 757 KiB/s rd, 2.7 MiB/s wr, 239 op/s 2024-03-19T01:46:06.685 INFO:tasks.workunit.client.0.smithi099.stdout: 1 583439 11.13 MB/sec execute 156 sec latency 41.244 ms 2024-03-19T01:46:07.685 INFO:tasks.workunit.client.0.smithi099.stdout: 1 585702 11.14 MB/sec execute 157 sec latency 51.374 ms 2024-03-19T01:46:08.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:46:07 smithi115 ceph-mon[67332]: pgmap v501: 97 pgs: 97 active+clean; 192 MiB data, 1.9 GiB used, 1.0 TiB / 1.0 TiB avail; 1024 KiB/s rd, 3.3 MiB/s wr, 304 op/s 2024-03-19T01:46:08.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:46:07 smithi099 ceph-mon[62951]: pgmap v501: 97 pgs: 97 active+clean; 192 MiB data, 1.9 GiB used, 1.0 TiB / 1.0 TiB avail; 1024 KiB/s rd, 3.3 MiB/s wr, 304 op/s 2024-03-19T01:46:08.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:46:07 smithi112 ceph-mon[66819]: pgmap v501: 97 pgs: 97 active+clean; 192 MiB data, 1.9 GiB used, 1.0 TiB / 1.0 TiB avail; 1024 KiB/s rd, 3.3 MiB/s wr, 304 op/s 2024-03-19T01:46:08.685 INFO:tasks.workunit.client.0.smithi099.stdout: 1 587506 11.14 MB/sec execute 158 sec latency 40.492 ms 2024-03-19T01:46:09.685 INFO:tasks.workunit.client.0.smithi099.stdout: 1 590021 11.15 MB/sec execute 159 sec latency 12.347 ms 2024-03-19T01:46:10.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:46:09 smithi115 ceph-mon[67332]: pgmap v502: 97 pgs: 97 active+clean; 193 MiB data, 1.9 GiB used, 1.0 TiB / 1.0 TiB avail; 745 KiB/s rd, 2.7 MiB/s wr, 239 op/s 2024-03-19T01:46:10.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:46:09 smithi099 ceph-mon[62951]: pgmap v502: 97 pgs: 97 active+clean; 193 MiB data, 1.9 GiB used, 1.0 TiB / 1.0 TiB avail; 745 KiB/s rd, 2.7 MiB/s wr, 239 op/s 2024-03-19T01:46:10.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:46:09 smithi112 ceph-mon[66819]: pgmap v502: 97 pgs: 97 active+clean; 193 MiB data, 1.9 GiB used, 1.0 TiB / 1.0 TiB avail; 745 KiB/s rd, 2.7 MiB/s wr, 239 op/s 2024-03-19T01:46:10.685 INFO:tasks.workunit.client.0.smithi099.stdout: 1 592223 11.13 MB/sec execute 160 sec latency 38.869 ms 2024-03-19T01:46:11.685 INFO:tasks.workunit.client.0.smithi099.stdout: 1 594236 11.15 MB/sec execute 161 sec latency 26.871 ms 2024-03-19T01:46:12.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:46:11 smithi115 ceph-mon[67332]: pgmap v503: 97 pgs: 97 active+clean; 194 MiB data, 1.9 GiB used, 1.0 TiB / 1.0 TiB avail; 805 KiB/s rd, 2.9 MiB/s wr, 267 op/s 2024-03-19T01:46:12.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:46:11 smithi099 ceph-mon[62951]: pgmap v503: 97 pgs: 97 active+clean; 194 MiB data, 1.9 GiB used, 1.0 TiB / 1.0 TiB avail; 805 KiB/s rd, 2.9 MiB/s wr, 267 op/s 2024-03-19T01:46:12.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:46:11 smithi112 ceph-mon[66819]: pgmap v503: 97 pgs: 97 active+clean; 194 MiB data, 1.9 GiB used, 1.0 TiB / 1.0 TiB avail; 805 KiB/s rd, 2.9 MiB/s wr, 267 op/s 2024-03-19T01:46:12.685 INFO:tasks.workunit.client.0.smithi099.stdout: 1 596820 11.17 MB/sec execute 162 sec latency 69.023 ms 2024-03-19T01:46:13.685 INFO:tasks.workunit.client.0.smithi099.stdout: 1 598537 11.15 MB/sec execute 163 sec latency 28.282 ms 2024-03-19T01:46:14.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:46:13 smithi115 ceph-mon[67332]: pgmap v504: 97 pgs: 97 active+clean; 194 MiB data, 1.9 GiB used, 1.0 TiB / 1.0 TiB avail; 991 KiB/s rd, 3.1 MiB/s wr, 282 op/s 2024-03-19T01:46:14.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:46:13 smithi099 ceph-mon[62951]: pgmap v504: 97 pgs: 97 active+clean; 194 MiB data, 1.9 GiB used, 1.0 TiB / 1.0 TiB avail; 991 KiB/s rd, 3.1 MiB/s wr, 282 op/s 2024-03-19T01:46:14.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:46:13 smithi112 ceph-mon[66819]: pgmap v504: 97 pgs: 97 active+clean; 194 MiB data, 1.9 GiB used, 1.0 TiB / 1.0 TiB avail; 991 KiB/s rd, 3.1 MiB/s wr, 282 op/s 2024-03-19T01:46:14.685 INFO:tasks.workunit.client.0.smithi099.stdout: 1 600815 11.16 MB/sec execute 164 sec latency 14.265 ms 2024-03-19T01:46:15.685 INFO:tasks.workunit.client.0.smithi099.stdout: 1 603235 11.17 MB/sec execute 165 sec latency 23.970 ms 2024-03-19T01:46:16.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:46:15 smithi115 ceph-mon[67332]: pgmap v505: 97 pgs: 97 active+clean; 194 MiB data, 1.9 GiB used, 1.0 TiB / 1.0 TiB avail; 720 KiB/s rd, 2.6 MiB/s wr, 231 op/s 2024-03-19T01:46:16.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:46:15 smithi099 ceph-mon[62951]: pgmap v505: 97 pgs: 97 active+clean; 194 MiB data, 1.9 GiB used, 1.0 TiB / 1.0 TiB avail; 720 KiB/s rd, 2.6 MiB/s wr, 231 op/s 2024-03-19T01:46:16.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:46:15 smithi112 ceph-mon[66819]: pgmap v505: 97 pgs: 97 active+clean; 194 MiB data, 1.9 GiB used, 1.0 TiB / 1.0 TiB avail; 720 KiB/s rd, 2.6 MiB/s wr, 231 op/s 2024-03-19T01:46:16.685 INFO:tasks.workunit.client.0.smithi099.stdout: 1 605228 11.17 MB/sec execute 166 sec latency 28.838 ms 2024-03-19T01:46:17.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:46:16 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:46:17.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:46:16 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:46:17.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:46:16 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:46:17.685 INFO:tasks.workunit.client.0.smithi099.stdout: 1 607485 11.18 MB/sec execute 167 sec latency 107.297 ms 2024-03-19T01:46:18.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:46:17 smithi115 ceph-mon[67332]: pgmap v506: 97 pgs: 97 active+clean; 198 MiB data, 1.9 GiB used, 1.0 TiB / 1.0 TiB avail; 1012 KiB/s rd, 3.4 MiB/s wr, 296 op/s 2024-03-19T01:46:18.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:46:17 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:46:18.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:46:17 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:46:18.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:46:17 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:46:18.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:46:17 smithi099 ceph-mon[62951]: pgmap v506: 97 pgs: 97 active+clean; 198 MiB data, 1.9 GiB used, 1.0 TiB / 1.0 TiB avail; 1012 KiB/s rd, 3.4 MiB/s wr, 296 op/s 2024-03-19T01:46:18.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:46:17 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:46:18.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:46:17 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:46:18.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:46:17 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:46:18.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:46:17 smithi112 ceph-mon[66819]: pgmap v506: 97 pgs: 97 active+clean; 198 MiB data, 1.9 GiB used, 1.0 TiB / 1.0 TiB avail; 1012 KiB/s rd, 3.4 MiB/s wr, 296 op/s 2024-03-19T01:46:18.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:46:17 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:46:18.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:46:17 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:46:18.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:46:17 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:46:18.553 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.551+0000 7fd67b0c6640 1 -- 172.21.15.99:0/2020462797 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd6740ffb50 msgr2=0x7fd6740fff70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:46:18.553 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.551+0000 7fd67b0c6640 1 --2- 172.21.15.99:0/2020462797 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd6740ffb50 0x7fd6740fff70 secure :-1 s=READY pgs=449 cs=0 l=1 rev1=1 crypto rx=0x7fd668009930 tx=0x7fd66802bc30 comp rx=0 tx=0).stop 2024-03-19T01:46:18.553 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.551+0000 7fd67b0c6640 1 -- 172.21.15.99:0/2020462797 shutdown_connections 2024-03-19T01:46:18.553 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.551+0000 7fd67b0c6640 1 --2- 172.21.15.99:0/2020462797 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd674103580 0x7fd674109e10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:18.553 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.551+0000 7fd67b0c6640 1 --2- 172.21.15.99:0/2020462797 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd674102bc0 0x7fd674103040 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:18.553 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.551+0000 7fd67b0c6640 1 --2- 172.21.15.99:0/2020462797 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd6740ffb50 0x7fd6740fff70 unknown :-1 s=CLOSED pgs=449 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:18.553 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.551+0000 7fd67b0c6640 1 -- 172.21.15.99:0/2020462797 >> 172.21.15.99:0/2020462797 conn(0x7fd6740fb440 msgr2=0x7fd6740fd8a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:46:18.553 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.551+0000 7fd67b0c6640 1 -- 172.21.15.99:0/2020462797 shutdown_connections 2024-03-19T01:46:18.553 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.551+0000 7fd67b0c6640 1 -- 172.21.15.99:0/2020462797 wait complete. 2024-03-19T01:46:18.554 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.551+0000 7fd67b0c6640 1 Processor -- start 2024-03-19T01:46:18.554 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.551+0000 7fd67b0c6640 1 -- start start 2024-03-19T01:46:18.554 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.551+0000 7fd67b0c6640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd6740ffb50 0x7fd674102530 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:18.554 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.551+0000 7fd67b0c6640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd674102bc0 0x7fd674100b70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:18.554 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.551+0000 7fd67b0c6640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd674103580 0x7fd6741010b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:18.554 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.551+0000 7fd67b0c6640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fd67410d500 con 0x7fd674102bc0 2024-03-19T01:46:18.554 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.551+0000 7fd67b0c6640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fd67410d380 con 0x7fd6740ffb50 2024-03-19T01:46:18.554 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.551+0000 7fd67b0c6640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fd67410d680 con 0x7fd674103580 2024-03-19T01:46:18.554 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.552+0000 7fd673fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd674102bc0 0x7fd674100b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:18.554 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.552+0000 7fd678e62640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd6740ffb50 0x7fd674102530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:18.554 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.552+0000 7fd673fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd674102bc0 0x7fd674100b70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:60656/0 (socket says 172.21.15.99:60656) 2024-03-19T01:46:18.554 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.552+0000 7fd679663640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd674103580 0x7fd6741010b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:18.554 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.552+0000 7fd673fff640 1 -- 172.21.15.99:0/2478646663 learned_addr learned my addr 172.21.15.99:0/2478646663 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:46:18.554 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.552+0000 7fd678e62640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd6740ffb50 0x7fd674102530 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:38512/0 (socket says 172.21.15.99:38512) 2024-03-19T01:46:18.555 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.552+0000 7fd673fff640 1 -- 172.21.15.99:0/2478646663 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd674103580 msgr2=0x7fd6741010b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:46:18.555 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.552+0000 7fd673fff640 1 --2- 172.21.15.99:0/2478646663 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd674103580 0x7fd6741010b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:18.555 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.552+0000 7fd673fff640 1 -- 172.21.15.99:0/2478646663 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd6740ffb50 msgr2=0x7fd674102530 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:46:18.555 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.552+0000 7fd673fff640 1 --2- 172.21.15.99:0/2478646663 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd6740ffb50 0x7fd674102530 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:18.555 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.552+0000 7fd673fff640 1 -- 172.21.15.99:0/2478646663 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd674101940 con 0x7fd674102bc0 2024-03-19T01:46:18.555 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.552+0000 7fd673fff640 1 --2- 172.21.15.99:0/2478646663 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd674102bc0 0x7fd674100b70 secure :-1 s=READY pgs=450 cs=0 l=1 rev1=1 crypto rx=0x7fd66000e9f0 tx=0x7fd66000edf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:46:18.555 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.552+0000 7fd671ffb640 1 -- 172.21.15.99:0/2478646663 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd66000cd80 con 0x7fd674102bc0 2024-03-19T01:46:18.555 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.552+0000 7fd67b0c6640 1 -- 172.21.15.99:0/2478646663 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd6741a1c70 con 0x7fd674102bc0 2024-03-19T01:46:18.555 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.552+0000 7fd671ffb640 1 -- 172.21.15.99:0/2478646663 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd660005500 con 0x7fd674102bc0 2024-03-19T01:46:18.555 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.552+0000 7fd671ffb640 1 -- 172.21.15.99:0/2478646663 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd660010690 con 0x7fd674102bc0 2024-03-19T01:46:18.555 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.552+0000 7fd67b0c6640 1 -- 172.21.15.99:0/2478646663 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fd6741a2170 con 0x7fd674102bc0 2024-03-19T01:46:18.560 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.555+0000 7fd671ffb640 1 -- 172.21.15.99:0/2478646663 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7fd6600046c0 con 0x7fd674102bc0 2024-03-19T01:46:18.560 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.555+0000 7fd67dd8f740 1 Processor -- start 2024-03-19T01:46:18.560 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.555+0000 7fd671ffb640 1 --2- 172.21.15.99:0/2478646663 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd6640a28a0 0x7fd6640a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:18.560 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.555+0000 7fd67dd8f740 1 -- start start 2024-03-19T01:46:18.560 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.555+0000 7fd679663640 1 --2- 172.21.15.99:0/2478646663 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd6640a28a0 0x7fd6640a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:18.560 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.555+0000 7fd67dd8f740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e1d3e5b100 0x55e1d3e5b520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:18.561 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.555+0000 7fd67dd8f740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e1d3e5baf0 0x55e1d3e605f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:18.561 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.555+0000 7fd671ffb640 1 -- 172.21.15.99:0/2478646663 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7fd6600cef20 con 0x7fd674102bc0 2024-03-19T01:46:18.561 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.555+0000 7fd67dd8f740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e1d3e60c60 0x55e1d3e630f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:18.561 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.555+0000 7fd67dd8f740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55e1d3dbfbe0 con 0x55e1d3e5b100 2024-03-19T01:46:18.561 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.555+0000 7fd67dd8f740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55e1d3d68960 con 0x55e1d3e60c60 2024-03-19T01:46:18.561 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.555+0000 7fd67dd8f740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55e1d3dbbc80 con 0x55e1d3e5baf0 2024-03-19T01:46:18.561 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.555+0000 7fd678e62640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e1d3e5b100 0x55e1d3e5b520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:18.561 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.555+0000 7fd673fff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e1d3e5baf0 0x55e1d3e605f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:18.561 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.555+0000 7fd679663640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e1d3e60c60 0x55e1d3e630f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:18.561 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.556+0000 7fd673fff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e1d3e5baf0 0x55e1d3e605f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:57250/0 (socket says 172.21.15.99:57250) 2024-03-19T01:46:18.561 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.556+0000 7fd673fff640 1 -- 172.21.15.99:0/510030128 learned_addr learned my addr 172.21.15.99:0/510030128 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:46:18.561 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.556+0000 7fd673fff640 1 -- 172.21.15.99:0/510030128 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e1d3e60c60 msgr2=0x55e1d3e630f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:46:18.561 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.556+0000 7fd673fff640 1 --2- 172.21.15.99:0/510030128 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e1d3e60c60 0x55e1d3e630f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:18.561 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.556+0000 7fd673fff640 1 -- 172.21.15.99:0/510030128 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e1d3e5b100 msgr2=0x55e1d3e5b520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:46:18.562 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.556+0000 7fd673fff640 1 --2- 172.21.15.99:0/510030128 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e1d3e5b100 0x55e1d3e5b520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:18.562 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.556+0000 7fd673fff640 1 -- 172.21.15.99:0/510030128 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55e1d3d5fd20 con 0x55e1d3e5baf0 2024-03-19T01:46:18.562 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.556+0000 7fd679663640 1 --2- 172.21.15.99:0/2478646663 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd6640a28a0 0x7fd6640a4d60 secure :-1 s=READY pgs=271 cs=0 l=1 rev1=1 crypto rx=0x7fd66c005fd0 tx=0x7fd66c011bc0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:46:18.562 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.556+0000 7fd673fff640 1 --2- 172.21.15.99:0/510030128 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e1d3e5baf0 0x55e1d3e605f0 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 crypto rx=0x7fd660097ce0 tx=0x7fd66009b1e0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=e3a580fefedd825b server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:46:18.562 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.556+0000 7fd65effd640 1 -- 172.21.15.99:0/510030128 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd66009f9d0 con 0x55e1d3e5baf0 2024-03-19T01:46:18.562 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.556+0000 7fd65effd640 1 -- 172.21.15.99:0/510030128 <== mon.1 v2:172.21.15.115:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd6600b4070 con 0x55e1d3e5baf0 2024-03-19T01:46:18.562 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.556+0000 7fd65effd640 1 -- 172.21.15.99:0/510030128 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd6600a3b90 con 0x55e1d3e5baf0 2024-03-19T01:46:18.562 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.557+0000 7fd67dd8f740 1 -- 172.21.15.99:0/510030128 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e1d3e5baf0 msgr2=0x55e1d3e605f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:46:18.562 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.557+0000 7fd67dd8f740 1 --2- 172.21.15.99:0/510030128 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e1d3e5baf0 0x55e1d3e605f0 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 crypto rx=0x7fd660097ce0 tx=0x7fd66009b1e0 comp rx=0 tx=0).stop 2024-03-19T01:46:18.562 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.557+0000 7fd67dd8f740 1 -- 172.21.15.99:0/510030128 shutdown_connections 2024-03-19T01:46:18.562 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.557+0000 7fd67dd8f740 1 --2- 172.21.15.99:0/510030128 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e1d3e60c60 0x55e1d3e630f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:18.562 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.557+0000 7fd67dd8f740 1 --2- 172.21.15.99:0/510030128 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e1d3e5baf0 0x55e1d3e605f0 unknown :-1 s=CLOSED pgs=88 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:18.562 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.557+0000 7fd67dd8f740 1 --2- 172.21.15.99:0/510030128 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e1d3e5b100 0x55e1d3e5b520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:18.562 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.557+0000 7fd67dd8f740 1 -- 172.21.15.99:0/510030128 >> 172.21.15.99:0/510030128 conn(0x55e1d3e56670 msgr2=0x55e1d3e56a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:46:18.563 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.557+0000 7fd67dd8f740 1 -- 172.21.15.99:0/510030128 shutdown_connections 2024-03-19T01:46:18.563 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.557+0000 7fd67dd8f740 1 -- 172.21.15.99:0/510030128 wait complete. 2024-03-19T01:46:18.563 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.557+0000 7fd67dd8f740 1 -- start start 2024-03-19T01:46:18.563 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.557+0000 7fd67dd8f740 1 Processor -- start 2024-03-19T01:46:18.563 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.558+0000 7fd67dd8f740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e1d3e5b100 0x55e1d3e650b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:18.563 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.558+0000 7fd67dd8f740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e1d3e5baf0 0x55e1d3e655f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:18.563 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.558+0000 7fd67dd8f740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e1d3e60c60 0x55e1d3e6c630 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:18.563 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.558+0000 7fd67dd8f740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55e1d3e63c10 con 0x55e1d3e60c60 2024-03-19T01:46:18.563 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.558+0000 7fd67dd8f740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55e1d3d7bdb0 con 0x55e1d3e5baf0 2024-03-19T01:46:18.563 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.558+0000 7fd67dd8f740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55e1d3e63d90 con 0x55e1d3e5b100 2024-03-19T01:46:18.563 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.558+0000 7fd67dd8f740 10 client.0 _ll_register_callbacks cb 0x55e1d3d80e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:46:18.563 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.558+0000 7fd673fff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e1d3e5baf0 0x55e1d3e655f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:18.563 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.558+0000 7fd678e62640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e1d3e5b100 0x55e1d3e650b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:18.563 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.558+0000 7fd673fff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e1d3e5baf0 0x55e1d3e655f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:38534/0 (socket says 172.21.15.99:38534) 2024-03-19T01:46:18.563 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.558+0000 7fd679663640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e1d3e60c60 0x55e1d3e6c630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:18.564 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.558+0000 7fd673fff640 1 -- 172.21.15.99:0/1375593880 learned_addr learned my addr 172.21.15.99:0/1375593880 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:46:18.564 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.558+0000 7fd673fff640 1 -- 172.21.15.99:0/1375593880 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e1d3e5b100 msgr2=0x55e1d3e650b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:46:18.564 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.558+0000 7fd673fff640 1 --2- 172.21.15.99:0/1375593880 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e1d3e5b100 0x55e1d3e650b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:18.564 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.558+0000 7fd673fff640 1 -- 172.21.15.99:0/1375593880 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e1d3e60c60 msgr2=0x55e1d3e6c630 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:46:18.564 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.558+0000 7fd673fff640 1 --2- 172.21.15.99:0/1375593880 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e1d3e60c60 0x55e1d3e6c630 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:18.564 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.558+0000 7fd673fff640 1 -- 172.21.15.99:0/1375593880 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55e1d3d7ac60 con 0x55e1d3e5baf0 2024-03-19T01:46:18.564 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.558+0000 7fd679663640 1 --2- 172.21.15.99:0/1375593880 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e1d3e60c60 0x55e1d3e6c630 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:46:18.564 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.558+0000 7fd673fff640 1 --2- 172.21.15.99:0/1375593880 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e1d3e5baf0 0x55e1d3e655f0 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7fd66009b810 tx=0x7fd660004a40 comp rx=0 tx=0).ready entity=mon.2 client_cookie=3f1e05971b8ae856 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:46:18.564 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.558+0000 7fd653fff640 10 client.0 ms_handle_connect on v2:172.21.15.112:3300/0 2024-03-19T01:46:18.564 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.559+0000 7fd653fff640 1 -- 172.21.15.99:0/1375593880 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd6600a3d90 con 0x55e1d3e5baf0 2024-03-19T01:46:18.564 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.559+0000 7fd653fff640 1 -- 172.21.15.99:0/1375593880 <== mon.2 v2:172.21.15.112:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7fd66009cbb0 con 0x55e1d3e5baf0 2024-03-19T01:46:18.564 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.559+0000 7fd67dd8f740 1 -- 172.21.15.99:0/1375593880 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fd6600b4070 con 0x55e1d3e5baf0 2024-03-19T01:46:18.564 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.559+0000 7fd653fff640 1 -- 172.21.15.99:0/1375593880 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_get_version_reply(handle=1 version=26) ==== 24+0+0 (secure 0 0 0) 0x7fd6600a4020 con 0x55e1d3e5baf0 2024-03-19T01:46:18.564 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.559+0000 7fd67dd8f740 10 client.25529 fetch_fsmap learned FSMap version 26 2024-03-19T01:46:18.564 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.559+0000 7fd67dd8f740 1 -- 172.21.15.99:0/1375593880 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,fsmap=26,monmap=4+}) -- 0x55e1d3ad4210 con 0x55e1d3e5baf0 2024-03-19T01:46:18.565 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.559+0000 7fd653fff640 1 -- 172.21.15.99:0/1375593880 <== mon.2 v2:172.21.15.112:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd6600b4070 con 0x55e1d3e5baf0 2024-03-19T01:46:18.565 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.559+0000 7fd653fff640 1 -- 172.21.15.99:0/1375593880 <== mon.2 v2:172.21.15.112:3300/0 5 ==== fsmap(e 26) ==== 8342+0+0 (secure 0 0 0) 0x7fd660097040 con 0x55e1d3e5baf0 2024-03-19T01:46:18.565 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.559+0000 7fd67dd8f740 10 client.25529 fetch_fsmap finished waiting for FSMap version 26 2024-03-19T01:46:18.565 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.559+0000 7fd67dd8f740 10 client.25529 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:46:18.565 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.559+0000 7fd67dd8f740 20 client.25529 populate_metadata read hostname 'smithi099' 2024-03-19T01:46:18.565 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.559+0000 7fd67dd8f740 1 --2- 172.21.15.99:0/1375593880 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e1d3e70b20 0x55e1d3e72f10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:18.565 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.559+0000 7fd67dd8f740 4 client.25529 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:46:18.565 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.559+0000 7fd67dd8f740 1 -- 172.21.15.99:0/1375593880 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55e1d3dcd120 con 0x55e1d3e70b20 2024-03-19T01:46:18.565 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.559+0000 7fd678e62640 1 --2- 172.21.15.99:0/1375593880 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e1d3e70b20 0x55e1d3e72f10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:18.565 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.560+0000 7fd678e62640 1 --2- 172.21.15.99:0/1375593880 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e1d3e70b20 0x55e1d3e72f10 crc :-1 s=READY pgs=75 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=c56a11f757ddf162 server_cookie=20d37f0e37278904 in_seq=0 out_seq=0 2024-03-19T01:46:18.565 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.560+0000 7fd653fff640 10 client.25529 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:46:18.566 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.561+0000 7fd653fff640 1 -- 172.21.15.99:0/1375593880 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x55e1d3dcd120 con 0x55e1d3e70b20 2024-03-19T01:46:18.567 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.561+0000 7fd653fff640 10 client.25529 handle_command_reply: tid=0 2024-03-19T01:46:18.567 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.561+0000 7fd67dd8f740 1 client.25529 shutdown 2024-03-19T01:46:18.567 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.562+0000 7fd67dd8f740 1 -- 172.21.15.99:0/1375593880 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e1d3e5baf0 msgr2=0x55e1d3e655f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:46:18.567 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.562+0000 7fd67dd8f740 1 --2- 172.21.15.99:0/1375593880 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e1d3e5baf0 0x55e1d3e655f0 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7fd66009b810 tx=0x7fd660004a40 comp rx=0 tx=0).stop 2024-03-19T01:46:18.567 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.562+0000 7fd67dd8f740 1 -- 172.21.15.99:0/1375593880 shutdown_connections 2024-03-19T01:46:18.567 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.562+0000 7fd67dd8f740 1 --2- 172.21.15.99:0/1375593880 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e1d3e70b20 0x55e1d3e72f10 crc :-1 s=READY pgs=75 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:18.567 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.562+0000 7fd67dd8f740 1 --2- 172.21.15.99:0/1375593880 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e1d3e5baf0 0x55e1d3e655f0 unknown :-1 s=CLOSED pgs=119 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:18.567 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.562+0000 7fd653fff640 0 client.25529 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:46:18.567 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.562+0000 7fd67dd8f740 1 --2- 172.21.15.99:0/1375593880 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e1d3e60c60 0x55e1d3e6c630 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:18.567 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.562+0000 7fd67dd8f740 1 --2- 172.21.15.99:0/1375593880 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e1d3e5b100 0x55e1d3e650b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:18.567 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.562+0000 7fd67dd8f740 1 -- 172.21.15.99:0/1375593880 >> 172.21.15.99:0/1375593880 conn(0x55e1d3e56670 msgr2=0x55e1d3e578c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:46:18.567 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.562+0000 7fd67dd8f740 1 -- 172.21.15.99:0/1375593880 shutdown_connections 2024-03-19T01:46:18.568 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.562+0000 7fd67dd8f740 1 -- 172.21.15.99:0/1375593880 wait complete. 2024-03-19T01:46:18.568 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.562+0000 7fd67dd8f740 20 client.25529 trim_cache size 0 max 16384 2024-03-19T01:46:18.581 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.577+0000 7fd67dd8f740 1 Processor -- start 2024-03-19T01:46:18.581 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.577+0000 7fd67dd8f740 1 -- start start 2024-03-19T01:46:18.581 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.577+0000 7fd67dd8f740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e1d3e5b100 0x55e1d3e600e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:18.581 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.577+0000 7fd67dd8f740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e1d3e5baf0 0x55e1d3e6c5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:18.581 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.577+0000 7fd67dd8f740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e1d3e60c60 0x55e1d3e5c560 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:18.581 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.577+0000 7fd67dd8f740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fd6741ad950 con 0x55e1d3e60c60 2024-03-19T01:46:18.581 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.577+0000 7fd67dd8f740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fd6741ab220 con 0x55e1d3e5baf0 2024-03-19T01:46:18.582 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.577+0000 7fd67dd8f740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fd6741a9070 con 0x55e1d3e5b100 2024-03-19T01:46:18.582 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.577+0000 7fd679663640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e1d3e60c60 0x55e1d3e5c560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:18.582 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.577+0000 7fd673fff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e1d3e5baf0 0x55e1d3e6c5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:18.582 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.577+0000 7fd678e62640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e1d3e5b100 0x55e1d3e600e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:18.582 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.577+0000 7fd679663640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e1d3e60c60 0x55e1d3e5c560 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:60694/0 (socket says 172.21.15.99:60694) 2024-03-19T01:46:18.582 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.577+0000 7fd678e62640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e1d3e5b100 0x55e1d3e600e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:57264/0 (socket says 172.21.15.99:57264) 2024-03-19T01:46:18.582 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.577+0000 7fd673fff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e1d3e5baf0 0x55e1d3e6c5f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:38544/0 (socket says 172.21.15.99:38544) 2024-03-19T01:46:18.582 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.577+0000 7fd679663640 1 -- 172.21.15.99:0/3522087733 learned_addr learned my addr 172.21.15.99:0/3522087733 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:46:18.582 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.577+0000 7fd679663640 1 -- 172.21.15.99:0/3522087733 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e1d3e5b100 msgr2=0x55e1d3e600e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:46:18.582 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.577+0000 7fd679663640 1 --2- 172.21.15.99:0/3522087733 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e1d3e5b100 0x55e1d3e600e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:18.582 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.577+0000 7fd679663640 1 -- 172.21.15.99:0/3522087733 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e1d3e5baf0 msgr2=0x55e1d3e6c5f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:46:18.582 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.577+0000 7fd679663640 1 --2- 172.21.15.99:0/3522087733 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e1d3e5baf0 0x55e1d3e6c5f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:18.582 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.577+0000 7fd679663640 1 -- 172.21.15.99:0/3522087733 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55e1d3ba0620 con 0x55e1d3e60c60 2024-03-19T01:46:18.582 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.578+0000 7fd679663640 1 --2- 172.21.15.99:0/3522087733 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e1d3e60c60 0x55e1d3e5c560 secure :-1 s=READY pgs=451 cs=0 l=1 rev1=1 crypto rx=0x7fd66c00ffc0 tx=0x7fd66c01f000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=efeb6a30d1678d66 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:46:18.583 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.578+0000 7fd65e7fc640 1 -- 172.21.15.99:0/3522087733 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd66c02d070 con 0x55e1d3e60c60 2024-03-19T01:46:18.583 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.578+0000 7fd65e7fc640 1 -- 172.21.15.99:0/3522087733 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd66c028ad0 con 0x55e1d3e60c60 2024-03-19T01:46:18.583 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.578+0000 7fd65e7fc640 1 -- 172.21.15.99:0/3522087733 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd66c01a040 con 0x55e1d3e60c60 2024-03-19T01:46:18.583 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.578+0000 7fd67dd8f740 1 -- 172.21.15.99:0/3522087733 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e1d3e60c60 msgr2=0x55e1d3e5c560 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:46:18.583 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.578+0000 7fd67dd8f740 1 --2- 172.21.15.99:0/3522087733 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e1d3e60c60 0x55e1d3e5c560 secure :-1 s=READY pgs=451 cs=0 l=1 rev1=1 crypto rx=0x7fd66c00ffc0 tx=0x7fd66c01f000 comp rx=0 tx=0).stop 2024-03-19T01:46:18.583 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.578+0000 7fd67dd8f740 1 -- 172.21.15.99:0/3522087733 shutdown_connections 2024-03-19T01:46:18.583 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.578+0000 7fd67dd8f740 1 --2- 172.21.15.99:0/3522087733 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e1d3e5baf0 0x55e1d3e6c5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:18.583 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.578+0000 7fd67dd8f740 1 --2- 172.21.15.99:0/3522087733 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e1d3e60c60 0x55e1d3e5c560 unknown :-1 s=CLOSED pgs=451 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:18.583 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.578+0000 7fd67dd8f740 1 --2- 172.21.15.99:0/3522087733 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e1d3e5b100 0x55e1d3e600e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:18.583 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.578+0000 7fd67dd8f740 1 -- 172.21.15.99:0/3522087733 >> 172.21.15.99:0/3522087733 conn(0x55e1d3e56670 msgr2=0x55e1d3e55ab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:46:18.583 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.578+0000 7fd67dd8f740 1 -- 172.21.15.99:0/3522087733 shutdown_connections 2024-03-19T01:46:18.583 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.578+0000 7fd67dd8f740 1 -- 172.21.15.99:0/3522087733 wait complete. 2024-03-19T01:46:18.583 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.579+0000 7fd67dd8f740 1 -- start start 2024-03-19T01:46:18.583 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.579+0000 7fd67dd8f740 1 Processor -- start 2024-03-19T01:46:18.584 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.579+0000 7fd67dd8f740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e1d3e5b100 0x55e1d3e6a110 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:18.584 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.579+0000 7fd67dd8f740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e1d3e5baf0 0x55e1d3e6a650 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:18.584 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.579+0000 7fd67dd8f740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e1d3e60c60 0x55e1d3e6b660 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:18.584 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.579+0000 7fd67dd8f740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fd6741a96e0 con 0x55e1d3e5baf0 2024-03-19T01:46:18.584 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.579+0000 7fd67dd8f740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fd6741aab50 con 0x55e1d3e5b100 2024-03-19T01:46:18.584 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.579+0000 7fd67dd8f740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fd6741a9db0 con 0x55e1d3e60c60 2024-03-19T01:46:18.584 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.579+0000 7fd67dd8f740 10 client.0 _ll_register_callbacks cb 0x7fd660005e30 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:46:18.584 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.579+0000 7fd678e62640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e1d3e5b100 0x55e1d3e6a110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:18.584 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.579+0000 7fd673fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e1d3e5baf0 0x55e1d3e6a650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:18.584 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.579+0000 7fd678e62640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e1d3e5b100 0x55e1d3e6a110 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:38548/0 (socket says 172.21.15.99:38548) 2024-03-19T01:46:18.584 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.579+0000 7fd678e62640 1 -- 172.21.15.99:0/2202938542 learned_addr learned my addr 172.21.15.99:0/2202938542 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:46:18.584 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.579+0000 7fd679663640 1 --2- 172.21.15.99:0/2202938542 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e1d3e60c60 0x55e1d3e6b660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:18.585 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.580+0000 7fd678e62640 1 -- 172.21.15.99:0/2202938542 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e1d3e60c60 msgr2=0x55e1d3e6b660 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:46:18.585 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.580+0000 7fd678e62640 1 --2- 172.21.15.99:0/2202938542 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e1d3e60c60 0x55e1d3e6b660 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:18.585 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.580+0000 7fd678e62640 1 -- 172.21.15.99:0/2202938542 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e1d3e5baf0 msgr2=0x55e1d3e6a650 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:46:18.585 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.580+0000 7fd678e62640 1 --2- 172.21.15.99:0/2202938542 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e1d3e5baf0 0x55e1d3e6a650 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:18.585 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.580+0000 7fd678e62640 1 -- 172.21.15.99:0/2202938542 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55e1d3e6bc50 con 0x55e1d3e5b100 2024-03-19T01:46:18.585 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.580+0000 7fd673fff640 1 --2- 172.21.15.99:0/2202938542 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e1d3e5baf0 0x55e1d3e6a650 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:46:18.585 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.580+0000 7fd678e62640 1 --2- 172.21.15.99:0/2202938542 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e1d3e5b100 0x55e1d3e6a110 secure :-1 s=READY pgs=120 cs=0 l=1 rev1=1 crypto rx=0x7fd668009900 tx=0x7fd66802cc60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=a45e4d47c4d0a2f5 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:46:18.585 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.580+0000 7fd653fff640 10 client.0 ms_handle_connect on v2:172.21.15.112:3300/0 2024-03-19T01:46:18.585 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.580+0000 7fd653fff640 1 -- 172.21.15.99:0/2202938542 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd66804a030 con 0x55e1d3e5b100 2024-03-19T01:46:18.585 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.580+0000 7fd67dd8f740 1 -- 172.21.15.99:0/2202938542 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fd66c028ad0 con 0x55e1d3e5b100 2024-03-19T01:46:18.585 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.580+0000 7fd653fff640 1 -- 172.21.15.99:0/2202938542 <== mon.2 v2:172.21.15.112:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7fd668046d20 con 0x55e1d3e5b100 2024-03-19T01:46:18.585 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.580+0000 7fd653fff640 1 -- 172.21.15.99:0/2202938542 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_get_version_reply(handle=1 version=26) ==== 24+0+0 (secure 0 0 0) 0x7fd66802cd90 con 0x55e1d3e5b100 2024-03-19T01:46:18.585 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.580+0000 7fd67dd8f740 10 client.25535 fetch_fsmap learned FSMap version 26 2024-03-19T01:46:18.585 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.580+0000 7fd67dd8f740 1 -- 172.21.15.99:0/2202938542 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,fsmap=26,monmap=4+}) -- 0x55e1d3ed7200 con 0x55e1d3e5b100 2024-03-19T01:46:18.586 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.580+0000 7fd653fff640 1 -- 172.21.15.99:0/2202938542 <== mon.2 v2:172.21.15.112:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd66c028ad0 con 0x55e1d3e5b100 2024-03-19T01:46:18.586 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.580+0000 7fd653fff640 1 -- 172.21.15.99:0/2202938542 <== mon.2 v2:172.21.15.112:3300/0 5 ==== fsmap(e 26) ==== 8342+0+0 (secure 0 0 0) 0x7fd668046070 con 0x55e1d3e5b100 2024-03-19T01:46:18.586 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.580+0000 7fd67dd8f740 10 client.25535 fetch_fsmap finished waiting for FSMap version 26 2024-03-19T01:46:18.586 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.580+0000 7fd67dd8f740 10 client.25535 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:46:18.586 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.580+0000 7fd67dd8f740 20 client.25535 populate_metadata read hostname 'smithi099' 2024-03-19T01:46:18.586 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.580+0000 7fd67dd8f740 1 --2- 172.21.15.99:0/2202938542 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e1d3ed7c20 0x55e1d3eda040 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:18.586 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.580+0000 7fd67dd8f740 4 client.25535 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:46:18.586 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.580+0000 7fd67dd8f740 1 -- 172.21.15.99:0/2202938542 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55e1d3dc7470 con 0x55e1d3ed7c20 2024-03-19T01:46:18.586 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.581+0000 7fd673fff640 1 --2- 172.21.15.99:0/2202938542 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e1d3ed7c20 0x55e1d3eda040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:18.586 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.581+0000 7fd673fff640 1 --2- 172.21.15.99:0/2202938542 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e1d3ed7c20 0x55e1d3eda040 crc :-1 s=READY pgs=76 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=d0f581118806aa63 server_cookie=7a965a20df96b687 in_seq=0 out_seq=0 2024-03-19T01:46:18.586 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.581+0000 7fd653fff640 10 client.25535 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:46:18.628 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.627+0000 7fd653fff640 1 -- 172.21.15.99:0/2202938542 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x55e1d3dc7470 con 0x55e1d3ed7c20 2024-03-19T01:46:18.629 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.627+0000 7fd653fff640 10 client.25535 handle_command_reply: tid=0 2024-03-19T01:46:18.629 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.627+0000 7fd67dd8f740 1 client.25535 shutdown 2024-03-19T01:46:18.629 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.628+0000 7fd67dd8f740 1 -- 172.21.15.99:0/2202938542 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e1d3e5b100 msgr2=0x55e1d3e6a110 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:46:18.629 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.628+0000 7fd67dd8f740 1 --2- 172.21.15.99:0/2202938542 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e1d3e5b100 0x55e1d3e6a110 secure :-1 s=READY pgs=120 cs=0 l=1 rev1=1 crypto rx=0x7fd668009900 tx=0x7fd66802cc60 comp rx=0 tx=0).stop 2024-03-19T01:46:18.629 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.628+0000 7fd67dd8f740 1 -- 172.21.15.99:0/2202938542 shutdown_connections 2024-03-19T01:46:18.629 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.628+0000 7fd67dd8f740 1 --2- 172.21.15.99:0/2202938542 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e1d3ed7c20 0x55e1d3eda040 crc :-1 s=READY pgs=76 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:18.629 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.628+0000 7fd67dd8f740 1 --2- 172.21.15.99:0/2202938542 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e1d3e60c60 0x55e1d3e6b660 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:18.629 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.628+0000 7fd67dd8f740 1 --2- 172.21.15.99:0/2202938542 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e1d3e5baf0 0x55e1d3e6a650 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:18.629 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.628+0000 7fd653fff640 0 client.25535 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:46:18.629 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.628+0000 7fd67dd8f740 1 --2- 172.21.15.99:0/2202938542 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e1d3e5b100 0x55e1d3e6a110 unknown :-1 s=CLOSED pgs=120 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:18.629 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.628+0000 7fd67dd8f740 1 -- 172.21.15.99:0/2202938542 >> 172.21.15.99:0/2202938542 conn(0x55e1d3e56670 msgr2=0x55e1d3e55ab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:46:18.629 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.628+0000 7fd67dd8f740 1 -- 172.21.15.99:0/2202938542 shutdown_connections 2024-03-19T01:46:18.629 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.628+0000 7fd67dd8f740 1 -- 172.21.15.99:0/2202938542 wait complete. 2024-03-19T01:46:18.629 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.628+0000 7fd67dd8f740 20 client.25535 trim_cache size 0 max 16384 2024-03-19T01:46:18.630 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:46:18.630 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:46:18.630 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:46:18.630 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:46:18.631 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.629+0000 7fd67b0c6640 1 -- 172.21.15.99:0/2478646663 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd6640a28a0 msgr2=0x7fd6640a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:46:18.631 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.629+0000 7fd67b0c6640 1 --2- 172.21.15.99:0/2478646663 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd6640a28a0 0x7fd6640a4d60 secure :-1 s=READY pgs=271 cs=0 l=1 rev1=1 crypto rx=0x7fd66c005fd0 tx=0x7fd66c011bc0 comp rx=0 tx=0).stop 2024-03-19T01:46:18.631 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.629+0000 7fd67b0c6640 1 -- 172.21.15.99:0/2478646663 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd674102bc0 msgr2=0x7fd674100b70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:46:18.631 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.629+0000 7fd67b0c6640 1 --2- 172.21.15.99:0/2478646663 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd674102bc0 0x7fd674100b70 secure :-1 s=READY pgs=450 cs=0 l=1 rev1=1 crypto rx=0x7fd66000e9f0 tx=0x7fd66000edf0 comp rx=0 tx=0).stop 2024-03-19T01:46:18.631 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.629+0000 7fd67b0c6640 1 -- 172.21.15.99:0/2478646663 shutdown_connections 2024-03-19T01:46:18.631 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.629+0000 7fd67b0c6640 1 --2- 172.21.15.99:0/2478646663 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd674103580 0x7fd6741010b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:18.631 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.629+0000 7fd67b0c6640 1 --2- 172.21.15.99:0/2478646663 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd674102bc0 0x7fd674100b70 unknown :-1 s=CLOSED pgs=450 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:18.631 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.629+0000 7fd67b0c6640 1 --2- 172.21.15.99:0/2478646663 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd6640a28a0 0x7fd6640a4d60 unknown :-1 s=CLOSED pgs=271 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:18.631 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.629+0000 7fd67b0c6640 1 --2- 172.21.15.99:0/2478646663 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd6740ffb50 0x7fd674102530 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:18.632 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.629+0000 7fd67b0c6640 1 -- 172.21.15.99:0/2478646663 >> 172.21.15.99:0/2478646663 conn(0x7fd6740fb440 msgr2=0x7fd674105f40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:46:18.632 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.629+0000 7fd67b0c6640 1 -- 172.21.15.99:0/2478646663 shutdown_connections 2024-03-19T01:46:18.632 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:18.629+0000 7fd67b0c6640 1 -- 172.21.15.99:0/2478646663 wait complete. 2024-03-19T01:46:18.685 INFO:tasks.workunit.client.0.smithi099.stdout: 1 609030 11.16 MB/sec execute 168 sec latency 35.055 ms 2024-03-19T01:46:19.685 INFO:tasks.workunit.client.0.smithi099.stdout: 1 611440 11.17 MB/sec execute 169 sec latency 10.266 ms 2024-03-19T01:46:20.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:46:19 smithi115 ceph-mon[67332]: pgmap v507: 97 pgs: 97 active+clean; 199 MiB data, 1.9 GiB used, 1.0 TiB / 1.0 TiB avail; 748 KiB/s rd, 2.8 MiB/s wr, 232 op/s 2024-03-19T01:46:20.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:46:19 smithi099 ceph-mon[62951]: pgmap v507: 97 pgs: 97 active+clean; 199 MiB data, 1.9 GiB used, 1.0 TiB / 1.0 TiB avail; 748 KiB/s rd, 2.8 MiB/s wr, 232 op/s 2024-03-19T01:46:20.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:46:19 smithi112 ceph-mon[66819]: pgmap v507: 97 pgs: 97 active+clean; 199 MiB data, 1.9 GiB used, 1.0 TiB / 1.0 TiB avail; 748 KiB/s rd, 2.8 MiB/s wr, 232 op/s 2024-03-19T01:46:20.685 INFO:tasks.workunit.client.0.smithi099.stdout: 1 613886 11.18 MB/sec execute 170 sec latency 36.245 ms 2024-03-19T01:46:21.685 INFO:tasks.workunit.client.0.smithi099.stdout: 1 615862 11.18 MB/sec execute 171 sec latency 32.884 ms 2024-03-19T01:46:22.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:46:21 smithi115 ceph-mon[67332]: pgmap v508: 97 pgs: 97 active+clean; 201 MiB data, 1.9 GiB used, 1.0 TiB / 1.0 TiB avail; 969 KiB/s rd, 3.1 MiB/s wr, 275 op/s 2024-03-19T01:46:22.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:46:21 smithi099 ceph-mon[62951]: pgmap v508: 97 pgs: 97 active+clean; 201 MiB data, 1.9 GiB used, 1.0 TiB / 1.0 TiB avail; 969 KiB/s rd, 3.1 MiB/s wr, 275 op/s 2024-03-19T01:46:22.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:46:21 smithi112 ceph-mon[66819]: pgmap v508: 97 pgs: 97 active+clean; 201 MiB data, 1.9 GiB used, 1.0 TiB / 1.0 TiB avail; 969 KiB/s rd, 3.1 MiB/s wr, 275 op/s 2024-03-19T01:46:22.685 INFO:tasks.workunit.client.0.smithi099.stdout: 1 618210 11.20 MB/sec execute 172 sec latency 73.949 ms 2024-03-19T01:46:23.685 INFO:tasks.workunit.client.0.smithi099.stdout: 1 620183 11.17 MB/sec execute 173 sec latency 82.420 ms 2024-03-19T01:46:24.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:46:23 smithi115 ceph-mon[67332]: pgmap v509: 97 pgs: 97 active+clean; 201 MiB data, 1.9 GiB used, 1.0 TiB / 1.0 TiB avail; 917 KiB/s rd, 3.2 MiB/s wr, 255 op/s 2024-03-19T01:46:24.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:46:23 smithi099 ceph-mon[62951]: pgmap v509: 97 pgs: 97 active+clean; 201 MiB data, 1.9 GiB used, 1.0 TiB / 1.0 TiB avail; 917 KiB/s rd, 3.2 MiB/s wr, 255 op/s 2024-03-19T01:46:24.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:46:23 smithi112 ceph-mon[66819]: pgmap v509: 97 pgs: 97 active+clean; 201 MiB data, 1.9 GiB used, 1.0 TiB / 1.0 TiB avail; 917 KiB/s rd, 3.2 MiB/s wr, 255 op/s 2024-03-19T01:46:24.685 INFO:tasks.workunit.client.0.smithi099.stdout: 1 622272 11.18 MB/sec execute 174 sec latency 37.469 ms 2024-03-19T01:46:25.685 INFO:tasks.workunit.client.0.smithi099.stdout: 1 625179 11.22 MB/sec execute 175 sec latency 16.717 ms 2024-03-19T01:46:26.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:46:25 smithi115 ceph-mon[67332]: pgmap v510: 97 pgs: 97 active+clean; 201 MiB data, 1.9 GiB used, 1.0 TiB / 1.0 TiB avail; 774 KiB/s rd, 2.9 MiB/s wr, 250 op/s 2024-03-19T01:46:26.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:46:25 smithi099 ceph-mon[62951]: pgmap v510: 97 pgs: 97 active+clean; 201 MiB data, 1.9 GiB used, 1.0 TiB / 1.0 TiB avail; 774 KiB/s rd, 2.9 MiB/s wr, 250 op/s 2024-03-19T01:46:26.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:46:25 smithi112 ceph-mon[66819]: pgmap v510: 97 pgs: 97 active+clean; 201 MiB data, 1.9 GiB used, 1.0 TiB / 1.0 TiB avail; 774 KiB/s rd, 2.9 MiB/s wr, 250 op/s 2024-03-19T01:46:26.685 INFO:tasks.workunit.client.0.smithi099.stdout: 1 627218 11.20 MB/sec execute 176 sec latency 26.004 ms 2024-03-19T01:46:27.685 INFO:tasks.workunit.client.0.smithi099.stdout: 1 629300 11.21 MB/sec execute 177 sec latency 25.053 ms 2024-03-19T01:46:28.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:46:27 smithi115 ceph-mon[67332]: pgmap v511: 97 pgs: 97 active+clean; 204 MiB data, 1.9 GiB used, 1.0 TiB / 1.0 TiB avail; 1024 KiB/s rd, 3.6 MiB/s wr, 303 op/s 2024-03-19T01:46:28.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:46:27 smithi099 ceph-mon[62951]: pgmap v511: 97 pgs: 97 active+clean; 204 MiB data, 1.9 GiB used, 1.0 TiB / 1.0 TiB avail; 1024 KiB/s rd, 3.6 MiB/s wr, 303 op/s 2024-03-19T01:46:28.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:46:27 smithi112 ceph-mon[66819]: pgmap v511: 97 pgs: 97 active+clean; 204 MiB data, 1.9 GiB used, 1.0 TiB / 1.0 TiB avail; 1024 KiB/s rd, 3.6 MiB/s wr, 303 op/s 2024-03-19T01:46:28.685 INFO:tasks.workunit.client.0.smithi099.stdout: 1 631615 11.22 MB/sec execute 178 sec latency 109.455 ms 2024-03-19T01:46:29.685 INFO:tasks.workunit.client.0.smithi099.stdout: 1 633317 11.20 MB/sec execute 179 sec latency 69.615 ms 2024-03-19T01:46:30.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:46:29 smithi115 ceph-mon[67332]: pgmap v512: 97 pgs: 97 active+clean; 206 MiB data, 1.9 GiB used, 1.0 TiB / 1.0 TiB avail; 744 KiB/s rd, 2.8 MiB/s wr, 244 op/s 2024-03-19T01:46:30.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:46:29 smithi099 ceph-mon[62951]: pgmap v512: 97 pgs: 97 active+clean; 206 MiB data, 1.9 GiB used, 1.0 TiB / 1.0 TiB avail; 744 KiB/s rd, 2.8 MiB/s wr, 244 op/s 2024-03-19T01:46:30.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:46:29 smithi112 ceph-mon[66819]: pgmap v512: 97 pgs: 97 active+clean; 206 MiB data, 1.9 GiB used, 1.0 TiB / 1.0 TiB avail; 744 KiB/s rd, 2.8 MiB/s wr, 244 op/s 2024-03-19T01:46:30.685 INFO:tasks.workunit.client.0.smithi099.stdout: 1 635994 11.23 MB/sec execute 180 sec latency 16.623 ms 2024-03-19T01:46:31.686 INFO:tasks.workunit.client.0.smithi099.stdout: 1 638151 11.21 MB/sec execute 181 sec latency 27.453 ms 2024-03-19T01:46:31.723 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.721+0000 7fbe6339d640 1 -- 172.21.15.99:0/3107724527 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbe5c104b00 msgr2=0x7fbe5c106ef0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:46:31.723 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.721+0000 7fbe6339d640 1 --2- 172.21.15.99:0/3107724527 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbe5c104b00 0x7fbe5c106ef0 secure :-1 s=READY pgs=452 cs=0 l=1 rev1=1 crypto rx=0x7fbe4c009900 tx=0x7fbe4c02bc30 comp rx=0 tx=0).stop 2024-03-19T01:46:31.723 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.721+0000 7fbe6339d640 1 -- 172.21.15.99:0/3107724527 shutdown_connections 2024-03-19T01:46:31.723 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.721+0000 7fbe6339d640 1 --2- 172.21.15.99:0/3107724527 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fbe5c107430 0x7fbe5c109820 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:31.723 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.721+0000 7fbe6339d640 1 --2- 172.21.15.99:0/3107724527 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbe5c104b00 0x7fbe5c106ef0 unknown :-1 s=CLOSED pgs=452 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:31.723 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.721+0000 7fbe6339d640 1 --2- 172.21.15.99:0/3107724527 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fbe5c1021d0 0x7fbe5c1045c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:31.723 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.721+0000 7fbe6339d640 1 -- 172.21.15.99:0/3107724527 >> 172.21.15.99:0/3107724527 conn(0x7fbe5c0fbb20 msgr2=0x7fbe5c0fdf60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:46:31.723 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.721+0000 7fbe6339d640 1 -- 172.21.15.99:0/3107724527 shutdown_connections 2024-03-19T01:46:31.723 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.721+0000 7fbe6339d640 1 -- 172.21.15.99:0/3107724527 wait complete. 2024-03-19T01:46:31.723 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.721+0000 7fbe6339d640 1 Processor -- start 2024-03-19T01:46:31.723 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.721+0000 7fbe6339d640 1 -- start start 2024-03-19T01:46:31.723 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.721+0000 7fbe6339d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbe5c1021d0 0x7fbe5c1979f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:31.724 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.721+0000 7fbe6339d640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fbe5c104b00 0x7fbe5c197f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:31.724 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.721+0000 7fbe6339d640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fbe5c107430 0x7fbe5c19efa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:31.724 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.721+0000 7fbe6339d640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fbe5c10d320 con 0x7fbe5c1021d0 2024-03-19T01:46:31.724 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.721+0000 7fbe6339d640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fbe5c10d1a0 con 0x7fbe5c104b00 2024-03-19T01:46:31.724 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.721+0000 7fbe6339d640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fbe5c10d4a0 con 0x7fbe5c107430 2024-03-19T01:46:31.724 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.721+0000 7fbe6239b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbe5c1021d0 0x7fbe5c1979f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:31.724 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.722+0000 7fbe6239b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbe5c1021d0 0x7fbe5c1979f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:41152/0 (socket says 172.21.15.99:41152) 2024-03-19T01:46:31.724 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.722+0000 7fbe61b9a640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fbe5c104b00 0x7fbe5c197f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:31.724 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.722+0000 7fbe6239b640 1 -- 172.21.15.99:0/918171857 learned_addr learned my addr 172.21.15.99:0/918171857 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:46:31.724 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.722+0000 7fbe62b9c640 1 --2- 172.21.15.99:0/918171857 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fbe5c107430 0x7fbe5c19efa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:31.724 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.722+0000 7fbe6239b640 1 -- 172.21.15.99:0/918171857 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fbe5c107430 msgr2=0x7fbe5c19efa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:46:31.724 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.722+0000 7fbe6239b640 1 --2- 172.21.15.99:0/918171857 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fbe5c107430 0x7fbe5c19efa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:31.724 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.722+0000 7fbe6239b640 1 -- 172.21.15.99:0/918171857 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fbe5c104b00 msgr2=0x7fbe5c197f30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:46:31.724 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.722+0000 7fbe6239b640 1 --2- 172.21.15.99:0/918171857 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fbe5c104b00 0x7fbe5c197f30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:31.725 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.722+0000 7fbe6239b640 1 -- 172.21.15.99:0/918171857 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fbe5c19f610 con 0x7fbe5c1021d0 2024-03-19T01:46:31.725 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.722+0000 7fbe61b9a640 1 --2- 172.21.15.99:0/918171857 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fbe5c104b00 0x7fbe5c197f30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:46:31.725 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.722+0000 7fbe6239b640 1 --2- 172.21.15.99:0/918171857 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbe5c1021d0 0x7fbe5c1979f0 secure :-1 s=READY pgs=453 cs=0 l=1 rev1=1 crypto rx=0x7fbe5800e9f0 tx=0x7fbe5800edf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:46:31.725 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.722+0000 7fbe537fe640 1 -- 172.21.15.99:0/918171857 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbe5800cd80 con 0x7fbe5c1021d0 2024-03-19T01:46:31.725 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.722+0000 7fbe537fe640 1 -- 172.21.15.99:0/918171857 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fbe58005500 con 0x7fbe5c1021d0 2024-03-19T01:46:31.725 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.722+0000 7fbe6339d640 1 -- 172.21.15.99:0/918171857 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fbe5c19f900 con 0x7fbe5c1021d0 2024-03-19T01:46:31.725 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.722+0000 7fbe537fe640 1 -- 172.21.15.99:0/918171857 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbe58010690 con 0x7fbe5c1021d0 2024-03-19T01:46:31.725 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.722+0000 7fbe6339d640 1 -- 172.21.15.99:0/918171857 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fbe5c19fdd0 con 0x7fbe5c1021d0 2024-03-19T01:46:31.732 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.725+0000 7fbe537fe640 1 -- 172.21.15.99:0/918171857 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7fbe580046c0 con 0x7fbe5c1021d0 2024-03-19T01:46:31.732 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.725+0000 7fbe671eb740 1 Processor -- start 2024-03-19T01:46:31.732 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.725+0000 7fbe671eb740 1 -- start start 2024-03-19T01:46:31.732 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.726+0000 7fbe671eb740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5608bef406f0 0x5608bef40b10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:31.732 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.726+0000 7fbe671eb740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5608bef410e0 0x5608bef45be0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:31.732 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.726+0000 7fbe671eb740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5608bef46250 0x5608bef486e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:31.732 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.726+0000 7fbe671eb740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x5608beea0be0 con 0x5608bef410e0 2024-03-19T01:46:31.732 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.726+0000 7fbe671eb740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x5608bee49960 con 0x5608bef406f0 2024-03-19T01:46:31.732 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.726+0000 7fbe671eb740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x5608bee9cc80 con 0x5608bef46250 2024-03-19T01:46:31.732 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.726+0000 7fbe537fe640 1 --2- 172.21.15.99:0/918171857 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fbe480a27f0 0x7fbe480a4cb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:31.732 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.726+0000 7fbe6239b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5608bef410e0 0x5608bef45be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:31.733 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.726+0000 7fbe62b9c640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5608bef406f0 0x5608bef40b10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:31.733 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.726+0000 7fbe61b9a640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5608bef46250 0x5608bef486e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:31.733 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.726+0000 7fbe62b9c640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5608bef406f0 0x5608bef40b10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:45698/0 (socket says 172.21.15.99:45698) 2024-03-19T01:46:31.733 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.726+0000 7fbe62b9c640 1 -- 172.21.15.99:0/330836881 learned_addr learned my addr 172.21.15.99:0/330836881 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:46:31.733 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.726+0000 7fbe62b9c640 1 --2- 172.21.15.99:0/918171857 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fbe480a27f0 0x7fbe480a4cb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:31.733 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.726+0000 7fbe537fe640 1 -- 172.21.15.99:0/918171857 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7fbe580cee90 con 0x7fbe5c1021d0 2024-03-19T01:46:31.733 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.726+0000 7fbe62b9c640 1 -- 172.21.15.99:0/330836881 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5608bef46250 msgr2=0x5608bef486e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:46:31.733 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.726+0000 7fbe62b9c640 1 --2- 172.21.15.99:0/330836881 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5608bef46250 0x5608bef486e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:31.733 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.726+0000 7fbe62b9c640 1 -- 172.21.15.99:0/330836881 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5608bef410e0 msgr2=0x5608bef45be0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:46:31.733 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.726+0000 7fbe62b9c640 1 --2- 172.21.15.99:0/330836881 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5608bef410e0 0x5608bef45be0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:31.733 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.726+0000 7fbe62b9c640 1 -- 172.21.15.99:0/330836881 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5608bee40d20 con 0x5608bef406f0 2024-03-19T01:46:31.733 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.726+0000 7fbe62b9c640 1 --2- 172.21.15.99:0/918171857 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fbe480a27f0 0x7fbe480a4cb0 secure :-1 s=READY pgs=272 cs=0 l=1 rev1=1 crypto rx=0x7fbe5400fc90 tx=0x7fbe54018000 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:46:31.734 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.726+0000 7fbe62b9c640 1 --2- 172.21.15.99:0/330836881 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5608bef406f0 0x5608bef40b10 secure :-1 s=READY pgs=121 cs=0 l=1 rev1=1 crypto rx=0x7fbe54011e00 tx=0x7fbe5400f790 comp rx=0 tx=0).ready entity=mon.2 client_cookie=9a542fbc30c5abd6 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:46:31.734 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.727+0000 7fbe50bfe640 1 -- 172.21.15.99:0/330836881 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbe54022070 con 0x5608bef406f0 2024-03-19T01:46:31.734 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.727+0000 7fbe50bfe640 1 -- 172.21.15.99:0/330836881 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fbe5400ed10 con 0x5608bef406f0 2024-03-19T01:46:31.734 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.727+0000 7fbe50bfe640 1 -- 172.21.15.99:0/330836881 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbe54005930 con 0x5608bef406f0 2024-03-19T01:46:31.734 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.727+0000 7fbe671eb740 1 -- 172.21.15.99:0/330836881 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5608bef406f0 msgr2=0x5608bef40b10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:46:31.734 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.727+0000 7fbe671eb740 1 --2- 172.21.15.99:0/330836881 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5608bef406f0 0x5608bef40b10 secure :-1 s=READY pgs=121 cs=0 l=1 rev1=1 crypto rx=0x7fbe54011e00 tx=0x7fbe5400f790 comp rx=0 tx=0).stop 2024-03-19T01:46:31.734 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.727+0000 7fbe671eb740 1 -- 172.21.15.99:0/330836881 shutdown_connections 2024-03-19T01:46:31.734 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.727+0000 7fbe671eb740 1 --2- 172.21.15.99:0/330836881 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5608bef46250 0x5608bef486e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:31.734 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.727+0000 7fbe671eb740 1 --2- 172.21.15.99:0/330836881 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5608bef410e0 0x5608bef45be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:31.734 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.727+0000 7fbe671eb740 1 --2- 172.21.15.99:0/330836881 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5608bef406f0 0x5608bef40b10 unknown :-1 s=CLOSED pgs=121 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:31.734 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.727+0000 7fbe671eb740 1 -- 172.21.15.99:0/330836881 >> 172.21.15.99:0/330836881 conn(0x5608bef37850 msgr2=0x5608bef37c50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:46:31.734 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.727+0000 7fbe671eb740 1 -- 172.21.15.99:0/330836881 shutdown_connections 2024-03-19T01:46:31.734 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.727+0000 7fbe671eb740 1 -- 172.21.15.99:0/330836881 wait complete. 2024-03-19T01:46:31.734 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.728+0000 7fbe671eb740 1 -- start start 2024-03-19T01:46:31.735 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.728+0000 7fbe671eb740 1 Processor -- start 2024-03-19T01:46:31.735 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.728+0000 7fbe671eb740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5608bef406f0 0x5608bef4a660 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:31.735 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.728+0000 7fbe671eb740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5608bef410e0 0x5608bef4aba0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:31.735 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.728+0000 7fbe671eb740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5608bef46250 0x5608bef51be0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:31.735 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.728+0000 7fbe671eb740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x5608bef491a0 con 0x5608bef406f0 2024-03-19T01:46:31.735 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.728+0000 7fbe671eb740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x5608bee5cdb0 con 0x5608bef46250 2024-03-19T01:46:31.735 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.728+0000 7fbe671eb740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x5608bef49320 con 0x5608bef410e0 2024-03-19T01:46:31.735 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.728+0000 7fbe671eb740 10 client.0 _ll_register_callbacks cb 0x5608bee61e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:46:31.735 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.728+0000 7fbe6239b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5608bef406f0 0x5608bef4a660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:31.735 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.728+0000 7fbe6239b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5608bef406f0 0x5608bef4a660 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:41160/0 (socket says 172.21.15.99:41160) 2024-03-19T01:46:31.735 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.728+0000 7fbe61b9a640 1 --2- 172.21.15.99:0/2152108011 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5608bef410e0 0x5608bef4aba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:31.735 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.728+0000 7fbe6239b640 1 -- 172.21.15.99:0/2152108011 learned_addr learned my addr 172.21.15.99:0/2152108011 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:46:31.735 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.728+0000 7fbe62b9c640 1 --2- 172.21.15.99:0/2152108011 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5608bef46250 0x5608bef51be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:31.735 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.728+0000 7fbe6239b640 1 -- 172.21.15.99:0/2152108011 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5608bef410e0 msgr2=0x5608bef4aba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:46:31.736 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.728+0000 7fbe6239b640 1 --2- 172.21.15.99:0/2152108011 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5608bef410e0 0x5608bef4aba0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:31.736 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.728+0000 7fbe6239b640 1 -- 172.21.15.99:0/2152108011 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5608bef46250 msgr2=0x5608bef51be0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:46:31.736 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.728+0000 7fbe6239b640 1 --2- 172.21.15.99:0/2152108011 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5608bef46250 0x5608bef51be0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:31.736 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.728+0000 7fbe6239b640 1 -- 172.21.15.99:0/2152108011 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5608bee5bc60 con 0x5608bef406f0 2024-03-19T01:46:31.736 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.729+0000 7fbe6239b640 1 --2- 172.21.15.99:0/2152108011 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5608bef406f0 0x5608bef4a660 secure :-1 s=READY pgs=454 cs=0 l=1 rev1=1 crypto rx=0x7fbe58097c50 tx=0x7fbe5806b3d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8a1855bddeeb5c1e server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:46:31.736 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.729+0000 7fbe45ffb640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:46:31.736 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.729+0000 7fbe45ffb640 1 -- 172.21.15.99:0/2152108011 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbe5809bef0 con 0x5608bef406f0 2024-03-19T01:46:31.736 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.729+0000 7fbe45ffb640 1 -- 172.21.15.99:0/2152108011 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7fbe5806d110 con 0x5608bef406f0 2024-03-19T01:46:31.736 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.729+0000 7fbe671eb740 1 -- 172.21.15.99:0/2152108011 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fbe5400ed10 con 0x5608bef406f0 2024-03-19T01:46:31.736 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.729+0000 7fbe45ffb640 1 -- 172.21.15.99:0/2152108011 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=26) ==== 24+0+0 (secure 0 0 0) 0x7fbe5809b7f0 con 0x5608bef406f0 2024-03-19T01:46:31.736 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.729+0000 7fbe671eb740 10 client.15774 fetch_fsmap learned FSMap version 26 2024-03-19T01:46:31.736 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.729+0000 7fbe671eb740 1 -- 172.21.15.99:0/2152108011 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=26,monmap=4+}) -- 0x5608bebb5210 con 0x5608bef406f0 2024-03-19T01:46:31.736 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.729+0000 7fbe45ffb640 1 -- 172.21.15.99:0/2152108011 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fbe5400ed10 con 0x5608bef406f0 2024-03-19T01:46:31.737 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.729+0000 7fbe45ffb640 1 -- 172.21.15.99:0/2152108011 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 26) ==== 8342+0+0 (secure 0 0 0) 0x7fbe580bb040 con 0x5608bef406f0 2024-03-19T01:46:31.737 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.729+0000 7fbe671eb740 10 client.15774 fetch_fsmap finished waiting for FSMap version 26 2024-03-19T01:46:31.737 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.730+0000 7fbe671eb740 10 client.15774 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:46:31.737 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.730+0000 7fbe671eb740 20 client.15774 populate_metadata read hostname 'smithi099' 2024-03-19T01:46:31.737 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.730+0000 7fbe671eb740 1 --2- 172.21.15.99:0/2152108011 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5608bef56470 0x5608bef588a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:31.737 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.730+0000 7fbe671eb740 4 client.15774 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:46:31.737 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.730+0000 7fbe671eb740 1 -- 172.21.15.99:0/2152108011 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5608beeae120 con 0x5608bef56470 2024-03-19T01:46:31.737 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.730+0000 7fbe61b9a640 1 --2- 172.21.15.99:0/2152108011 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5608bef56470 0x5608bef588a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:31.737 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.730+0000 7fbe61b9a640 1 --2- 172.21.15.99:0/2152108011 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5608bef56470 0x5608bef588a0 crc :-1 s=READY pgs=77 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=7637400c71c0aa9e server_cookie=8a036e2d2661391e in_seq=0 out_seq=0 2024-03-19T01:46:31.737 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.730+0000 7fbe45ffb640 10 client.15774 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:46:31.737 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.731+0000 7fbe45ffb640 1 -- 172.21.15.99:0/2152108011 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x5608beeae120 con 0x5608bef56470 2024-03-19T01:46:31.737 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.731+0000 7fbe45ffb640 10 client.15774 handle_command_reply: tid=0 2024-03-19T01:46:31.737 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.731+0000 7fbe671eb740 1 client.15774 shutdown 2024-03-19T01:46:31.737 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.731+0000 7fbe671eb740 1 -- 172.21.15.99:0/2152108011 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5608bef406f0 msgr2=0x5608bef4a660 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:46:31.737 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.731+0000 7fbe671eb740 1 --2- 172.21.15.99:0/2152108011 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5608bef406f0 0x5608bef4a660 secure :-1 s=READY pgs=454 cs=0 l=1 rev1=1 crypto rx=0x7fbe58097c50 tx=0x7fbe5806b3d0 comp rx=0 tx=0).stop 2024-03-19T01:46:31.738 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.731+0000 7fbe671eb740 1 -- 172.21.15.99:0/2152108011 shutdown_connections 2024-03-19T01:46:31.738 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.731+0000 7fbe671eb740 1 --2- 172.21.15.99:0/2152108011 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5608bef56470 0x5608bef588a0 crc :-1 s=READY pgs=77 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:31.738 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.731+0000 7fbe671eb740 1 --2- 172.21.15.99:0/2152108011 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5608bef46250 0x5608bef51be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:31.738 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.731+0000 7fbe671eb740 1 --2- 172.21.15.99:0/2152108011 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5608bef410e0 0x5608bef4aba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:31.738 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.731+0000 7fbe45ffb640 0 client.15774 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:46:31.738 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.731+0000 7fbe671eb740 1 --2- 172.21.15.99:0/2152108011 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5608bef406f0 0x5608bef4a660 unknown :-1 s=CLOSED pgs=454 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:31.738 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.731+0000 7fbe671eb740 1 -- 172.21.15.99:0/2152108011 >> 172.21.15.99:0/2152108011 conn(0x5608bef37850 msgr2=0x5608bef36ab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:46:31.738 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.731+0000 7fbe671eb740 1 -- 172.21.15.99:0/2152108011 shutdown_connections 2024-03-19T01:46:31.738 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.731+0000 7fbe671eb740 1 -- 172.21.15.99:0/2152108011 wait complete. 2024-03-19T01:46:31.738 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.731+0000 7fbe671eb740 20 client.15774 trim_cache size 0 max 16384 2024-03-19T01:46:31.750 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.746+0000 7fbe671eb740 1 Processor -- start 2024-03-19T01:46:31.750 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.746+0000 7fbe671eb740 1 -- start start 2024-03-19T01:46:31.751 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.746+0000 7fbe671eb740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5608bef406f0 0x5608bef4a980 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:31.753 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.746+0000 7fbe671eb740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5608bef410e0 0x5608bef51ba0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:31.753 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.746+0000 7fbe671eb740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5608bef46250 0x5608bef36220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:31.753 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.746+0000 7fbe671eb740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fbe5c1a93d0 con 0x5608bef46250 2024-03-19T01:46:31.753 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.746+0000 7fbe671eb740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fbe5c1a6ca0 con 0x5608bef410e0 2024-03-19T01:46:31.753 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.746+0000 7fbe671eb740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fbe5c1a4af0 con 0x5608bef406f0 2024-03-19T01:46:31.753 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.746+0000 7fbe62b9c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5608bef46250 0x5608bef36220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:31.753 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.746+0000 7fbe61b9a640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5608bef410e0 0x5608bef51ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:31.753 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.746+0000 7fbe62b9c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5608bef46250 0x5608bef36220 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:41164/0 (socket says 172.21.15.99:41164) 2024-03-19T01:46:31.753 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.746+0000 7fbe6239b640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5608bef406f0 0x5608bef4a980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:31.754 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.746+0000 7fbe62b9c640 1 -- 172.21.15.99:0/1109250864 learned_addr learned my addr 172.21.15.99:0/1109250864 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:46:31.754 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.746+0000 7fbe61b9a640 1 -- 172.21.15.99:0/1109250864 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5608bef406f0 msgr2=0x5608bef4a980 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:46:31.754 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.746+0000 7fbe61b9a640 1 --2- 172.21.15.99:0/1109250864 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5608bef406f0 0x5608bef4a980 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:31.754 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.747+0000 7fbe61b9a640 1 -- 172.21.15.99:0/1109250864 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5608bef46250 msgr2=0x5608bef36220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:46:31.754 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.747+0000 7fbe61b9a640 1 --2- 172.21.15.99:0/1109250864 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5608bef46250 0x5608bef36220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:31.754 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.747+0000 7fbe61b9a640 1 -- 172.21.15.99:0/1109250864 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5608bec81620 con 0x5608bef410e0 2024-03-19T01:46:31.754 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.747+0000 7fbe62b9c640 1 --2- 172.21.15.99:0/1109250864 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5608bef46250 0x5608bef36220 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:46:31.754 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.747+0000 7fbe61b9a640 1 --2- 172.21.15.99:0/1109250864 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5608bef410e0 0x5608bef51ba0 secure :-1 s=READY pgs=122 cs=0 l=1 rev1=1 crypto rx=0x7fbe4c02c170 tx=0x7fbe4c005fd0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=25f33c5ff0353dfa server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:46:31.754 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.747+0000 7fbe47fff640 1 -- 172.21.15.99:0/1109250864 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbe4c046b10 con 0x5608bef410e0 2024-03-19T01:46:31.754 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.747+0000 7fbe47fff640 1 -- 172.21.15.99:0/1109250864 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fbe4c046cb0 con 0x5608bef410e0 2024-03-19T01:46:31.754 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.747+0000 7fbe47fff640 1 -- 172.21.15.99:0/1109250864 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbe4c0052a0 con 0x5608bef410e0 2024-03-19T01:46:31.754 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.748+0000 7fbe671eb740 1 -- 172.21.15.99:0/1109250864 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5608bef410e0 msgr2=0x5608bef51ba0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:46:31.754 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.748+0000 7fbe671eb740 1 --2- 172.21.15.99:0/1109250864 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5608bef410e0 0x5608bef51ba0 secure :-1 s=READY pgs=122 cs=0 l=1 rev1=1 crypto rx=0x7fbe4c02c170 tx=0x7fbe4c005fd0 comp rx=0 tx=0).stop 2024-03-19T01:46:31.754 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.748+0000 7fbe671eb740 1 -- 172.21.15.99:0/1109250864 shutdown_connections 2024-03-19T01:46:31.755 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.748+0000 7fbe671eb740 1 --2- 172.21.15.99:0/1109250864 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5608bef410e0 0x5608bef51ba0 unknown :-1 s=CLOSED pgs=122 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:31.755 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.748+0000 7fbe671eb740 1 --2- 172.21.15.99:0/1109250864 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5608bef46250 0x5608bef36220 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:31.755 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.748+0000 7fbe671eb740 1 --2- 172.21.15.99:0/1109250864 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5608bef406f0 0x5608bef4a980 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:31.755 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.748+0000 7fbe671eb740 1 -- 172.21.15.99:0/1109250864 >> 172.21.15.99:0/1109250864 conn(0x5608bef37850 msgr2=0x5608bef4a270 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:46:31.755 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.748+0000 7fbe671eb740 1 -- 172.21.15.99:0/1109250864 shutdown_connections 2024-03-19T01:46:31.755 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.748+0000 7fbe671eb740 1 -- 172.21.15.99:0/1109250864 wait complete. 2024-03-19T01:46:31.755 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.748+0000 7fbe671eb740 1 -- start start 2024-03-19T01:46:31.755 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.748+0000 7fbe671eb740 1 Processor -- start 2024-03-19T01:46:31.755 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.748+0000 7fbe671eb740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5608bef406f0 0x5608bef4f310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:31.755 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.748+0000 7fbe671eb740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5608bef410e0 0x5608bef4f850 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:31.755 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.748+0000 7fbe671eb740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5608bef46250 0x5608bef50810 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:31.755 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.748+0000 7fbe671eb740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fbe5c1a5160 con 0x5608bef46250 2024-03-19T01:46:31.755 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.748+0000 7fbe671eb740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fbe5c1a65d0 con 0x5608bef406f0 2024-03-19T01:46:31.755 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.748+0000 7fbe671eb740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fbe5c1a5830 con 0x5608bef410e0 2024-03-19T01:46:31.756 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.748+0000 7fbe671eb740 10 client.0 _ll_register_callbacks cb 0x7fbe58004ae0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:46:31.756 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.748+0000 7fbe62b9c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5608bef46250 0x5608bef50810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:31.756 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.748+0000 7fbe6239b640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5608bef406f0 0x5608bef4f310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:31.756 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.748+0000 7fbe62b9c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5608bef46250 0x5608bef50810 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:41170/0 (socket says 172.21.15.99:41170) 2024-03-19T01:46:31.756 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.748+0000 7fbe61b9a640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5608bef410e0 0x5608bef4f850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:31.756 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.748+0000 7fbe62b9c640 1 -- 172.21.15.99:0/1428011776 learned_addr learned my addr 172.21.15.99:0/1428011776 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:46:31.756 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.749+0000 7fbe6239b640 1 -- 172.21.15.99:0/1428011776 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5608bef410e0 msgr2=0x5608bef4f850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:46:31.756 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.749+0000 7fbe6239b640 1 --2- 172.21.15.99:0/1428011776 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5608bef410e0 0x5608bef4f850 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:31.756 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.749+0000 7fbe6239b640 1 -- 172.21.15.99:0/1428011776 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5608bef46250 msgr2=0x5608bef50810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:46:31.756 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.749+0000 7fbe6239b640 1 --2- 172.21.15.99:0/1428011776 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5608bef46250 0x5608bef50810 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:31.756 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.749+0000 7fbe6239b640 1 -- 172.21.15.99:0/1428011776 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5608bef50e00 con 0x5608bef406f0 2024-03-19T01:46:31.756 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.749+0000 7fbe62b9c640 1 --2- 172.21.15.99:0/1428011776 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5608bef46250 0x5608bef50810 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:46:31.756 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.749+0000 7fbe6239b640 1 --2- 172.21.15.99:0/1428011776 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5608bef406f0 0x5608bef4f310 secure :-1 s=READY pgs=123 cs=0 l=1 rev1=1 crypto rx=0x7fbe5800cfd0 tx=0x7fbe5809b6f0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=4e7480854b72ce2a server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:46:31.757 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.749+0000 7fbe45ffb640 10 client.0 ms_handle_connect on v2:172.21.15.112:3300/0 2024-03-19T01:46:31.757 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.749+0000 7fbe45ffb640 1 -- 172.21.15.99:0/1428011776 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbe580a3d90 con 0x5608bef406f0 2024-03-19T01:46:31.757 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.749+0000 7fbe45ffb640 1 -- 172.21.15.99:0/1428011776 <== mon.2 v2:172.21.15.112:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7fbe5809ae50 con 0x5608bef406f0 2024-03-19T01:46:31.757 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.749+0000 7fbe671eb740 1 -- 172.21.15.99:0/1428011776 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fbe4c046cb0 con 0x5608bef406f0 2024-03-19T01:46:31.757 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.749+0000 7fbe45ffb640 1 -- 172.21.15.99:0/1428011776 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_get_version_reply(handle=1 version=26) ==== 24+0+0 (secure 0 0 0) 0x7fbe5809b330 con 0x5608bef406f0 2024-03-19T01:46:31.757 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.749+0000 7fbe671eb740 10 client.25553 fetch_fsmap learned FSMap version 26 2024-03-19T01:46:31.757 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.749+0000 7fbe671eb740 1 -- 172.21.15.99:0/1428011776 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,fsmap=26,monmap=4+}) -- 0x5608befbc810 con 0x5608bef406f0 2024-03-19T01:46:31.757 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.749+0000 7fbe45ffb640 1 -- 172.21.15.99:0/1428011776 <== mon.2 v2:172.21.15.112:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fbe4c046cb0 con 0x5608bef406f0 2024-03-19T01:46:31.757 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.749+0000 7fbe45ffb640 1 -- 172.21.15.99:0/1428011776 <== mon.2 v2:172.21.15.112:3300/0 5 ==== fsmap(e 26) ==== 8342+0+0 (secure 0 0 0) 0x7fbe580bb040 con 0x5608bef406f0 2024-03-19T01:46:31.757 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.749+0000 7fbe671eb740 10 client.25553 fetch_fsmap finished waiting for FSMap version 26 2024-03-19T01:46:31.757 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.749+0000 7fbe671eb740 10 client.25553 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:46:31.757 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.749+0000 7fbe671eb740 20 client.25553 populate_metadata read hostname 'smithi099' 2024-03-19T01:46:31.757 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.749+0000 7fbe671eb740 1 --2- 172.21.15.99:0/1428011776 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5608befbce00 0x5608befbf240 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:31.757 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.749+0000 7fbe671eb740 4 client.25553 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:46:31.758 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.749+0000 7fbe671eb740 1 -- 172.21.15.99:0/1428011776 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x5608beea8470 con 0x5608befbce00 2024-03-19T01:46:31.758 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.750+0000 7fbe61b9a640 1 --2- 172.21.15.99:0/1428011776 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5608befbce00 0x5608befbf240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:31.758 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.750+0000 7fbe45ffb640 10 client.25553 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:46:31.758 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.750+0000 7fbe61b9a640 1 --2- 172.21.15.99:0/1428011776 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5608befbce00 0x5608befbf240 crc :-1 s=READY pgs=78 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=16070340f4b18c09 server_cookie=e7e51b00c50ecfa5 in_seq=0 out_seq=0 2024-03-19T01:46:31.804 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.803+0000 7fbe45ffb640 1 -- 172.21.15.99:0/1428011776 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x5608beea8470 con 0x5608befbce00 2024-03-19T01:46:31.804 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.803+0000 7fbe45ffb640 10 client.25553 handle_command_reply: tid=0 2024-03-19T01:46:31.804 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.803+0000 7fbe671eb740 1 client.25553 shutdown 2024-03-19T01:46:31.805 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.803+0000 7fbe671eb740 1 -- 172.21.15.99:0/1428011776 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5608bef406f0 msgr2=0x5608bef4f310 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:46:31.805 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.803+0000 7fbe671eb740 1 --2- 172.21.15.99:0/1428011776 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5608bef406f0 0x5608bef4f310 secure :-1 s=READY pgs=123 cs=0 l=1 rev1=1 crypto rx=0x7fbe5800cfd0 tx=0x7fbe5809b6f0 comp rx=0 tx=0).stop 2024-03-19T01:46:31.805 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.803+0000 7fbe671eb740 1 -- 172.21.15.99:0/1428011776 shutdown_connections 2024-03-19T01:46:31.805 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.803+0000 7fbe671eb740 1 --2- 172.21.15.99:0/1428011776 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5608befbce00 0x5608befbf240 crc :-1 s=READY pgs=78 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:31.805 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.803+0000 7fbe671eb740 1 --2- 172.21.15.99:0/1428011776 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5608bef46250 0x5608bef50810 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:31.805 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.803+0000 7fbe45ffb640 0 client.25553 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:46:31.805 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.803+0000 7fbe671eb740 1 --2- 172.21.15.99:0/1428011776 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5608bef410e0 0x5608bef4f850 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:31.805 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.803+0000 7fbe671eb740 1 --2- 172.21.15.99:0/1428011776 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5608bef406f0 0x5608bef4f310 unknown :-1 s=CLOSED pgs=123 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:31.805 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.803+0000 7fbe671eb740 1 -- 172.21.15.99:0/1428011776 >> 172.21.15.99:0/1428011776 conn(0x5608bef37850 msgr2=0x5608bef36220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:46:31.805 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.803+0000 7fbe671eb740 1 -- 172.21.15.99:0/1428011776 shutdown_connections 2024-03-19T01:46:31.805 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.803+0000 7fbe671eb740 1 -- 172.21.15.99:0/1428011776 wait complete. 2024-03-19T01:46:31.805 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.804+0000 7fbe671eb740 20 client.25553 trim_cache size 0 max 16384 2024-03-19T01:46:31.806 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:46:31.806 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:46:31.806 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:46:31.806 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:46:31.807 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.804+0000 7fbe6339d640 1 -- 172.21.15.99:0/918171857 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fbe480a27f0 msgr2=0x7fbe480a4cb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:46:31.807 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.804+0000 7fbe6339d640 1 --2- 172.21.15.99:0/918171857 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fbe480a27f0 0x7fbe480a4cb0 secure :-1 s=READY pgs=272 cs=0 l=1 rev1=1 crypto rx=0x7fbe5400fc90 tx=0x7fbe54018000 comp rx=0 tx=0).stop 2024-03-19T01:46:31.807 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.804+0000 7fbe6339d640 1 -- 172.21.15.99:0/918171857 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbe5c1021d0 msgr2=0x7fbe5c1979f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:46:31.807 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.804+0000 7fbe6339d640 1 --2- 172.21.15.99:0/918171857 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbe5c1021d0 0x7fbe5c1979f0 secure :-1 s=READY pgs=453 cs=0 l=1 rev1=1 crypto rx=0x7fbe5800e9f0 tx=0x7fbe5800edf0 comp rx=0 tx=0).stop 2024-03-19T01:46:31.807 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.804+0000 7fbe6339d640 1 -- 172.21.15.99:0/918171857 shutdown_connections 2024-03-19T01:46:31.807 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.804+0000 7fbe6339d640 1 --2- 172.21.15.99:0/918171857 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fbe480a27f0 0x7fbe480a4cb0 unknown :-1 s=CLOSED pgs=272 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:31.807 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.804+0000 7fbe6339d640 1 --2- 172.21.15.99:0/918171857 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fbe5c104b00 0x7fbe5c197f30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:31.807 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.804+0000 7fbe6339d640 1 --2- 172.21.15.99:0/918171857 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fbe5c107430 0x7fbe5c19efa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:31.807 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.804+0000 7fbe6339d640 1 --2- 172.21.15.99:0/918171857 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbe5c1021d0 0x7fbe5c1979f0 unknown :-1 s=CLOSED pgs=453 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:31.808 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.804+0000 7fbe6339d640 1 -- 172.21.15.99:0/918171857 >> 172.21.15.99:0/918171857 conn(0x7fbe5c0fbb20 msgr2=0x7fbe5c102c90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:46:31.808 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.804+0000 7fbe6339d640 1 -- 172.21.15.99:0/918171857 shutdown_connections 2024-03-19T01:46:31.808 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:31.804+0000 7fbe6339d640 1 -- 172.21.15.99:0/918171857 wait complete. 2024-03-19T01:46:32.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:46:31 smithi115 ceph-mon[67332]: pgmap v513: 97 pgs: 97 active+clean; 208 MiB data, 1.9 GiB used, 1.0 TiB / 1.0 TiB avail; 889 KiB/s rd, 3.0 MiB/s wr, 270 op/s 2024-03-19T01:46:32.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:46:31 smithi099 ceph-mon[62951]: pgmap v513: 97 pgs: 97 active+clean; 208 MiB data, 1.9 GiB used, 1.0 TiB / 1.0 TiB avail; 889 KiB/s rd, 3.0 MiB/s wr, 270 op/s 2024-03-19T01:46:32.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:46:31 smithi112 ceph-mon[66819]: pgmap v513: 97 pgs: 97 active+clean; 208 MiB data, 1.9 GiB used, 1.0 TiB / 1.0 TiB avail; 889 KiB/s rd, 3.0 MiB/s wr, 270 op/s 2024-03-19T01:46:32.686 INFO:tasks.workunit.client.0.smithi099.stdout: 1 639989 11.22 MB/sec execute 182 sec latency 32.276 ms 2024-03-19T01:46:33.686 INFO:tasks.workunit.client.0.smithi099.stdout: 1 642233 11.22 MB/sec execute 183 sec latency 52.196 ms 2024-03-19T01:46:34.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:46:33 smithi115 ceph-mon[67332]: pgmap v514: 97 pgs: 97 active+clean; 210 MiB data, 1.9 GiB used, 1.0 TiB / 1.0 TiB avail; 817 KiB/s rd, 3.1 MiB/s wr, 254 op/s 2024-03-19T01:46:34.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:46:33 smithi099 ceph-mon[62951]: pgmap v514: 97 pgs: 97 active+clean; 210 MiB data, 1.9 GiB used, 1.0 TiB / 1.0 TiB avail; 817 KiB/s rd, 3.1 MiB/s wr, 254 op/s 2024-03-19T01:46:34.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:46:33 smithi112 ceph-mon[66819]: pgmap v514: 97 pgs: 97 active+clean; 210 MiB data, 1.9 GiB used, 1.0 TiB / 1.0 TiB avail; 817 KiB/s rd, 3.1 MiB/s wr, 254 op/s 2024-03-19T01:46:34.686 INFO:tasks.workunit.client.0.smithi099.stdout: 1 643988 11.21 MB/sec execute 184 sec latency 38.145 ms 2024-03-19T01:46:35.686 INFO:tasks.workunit.client.0.smithi099.stdout: 1 646648 11.24 MB/sec execute 185 sec latency 16.400 ms 2024-03-19T01:46:36.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:46:35 smithi115 ceph-mon[67332]: pgmap v515: 97 pgs: 97 active+clean; 206 MiB data, 1.9 GiB used, 1.0 TiB / 1.0 TiB avail; 794 KiB/s rd, 2.7 MiB/s wr, 246 op/s 2024-03-19T01:46:36.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:46:35 smithi099 ceph-mon[62951]: pgmap v515: 97 pgs: 97 active+clean; 206 MiB data, 1.9 GiB used, 1.0 TiB / 1.0 TiB avail; 794 KiB/s rd, 2.7 MiB/s wr, 246 op/s 2024-03-19T01:46:36.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:46:35 smithi112 ceph-mon[66819]: pgmap v515: 97 pgs: 97 active+clean; 206 MiB data, 1.9 GiB used, 1.0 TiB / 1.0 TiB avail; 794 KiB/s rd, 2.7 MiB/s wr, 246 op/s 2024-03-19T01:46:36.686 INFO:tasks.workunit.client.0.smithi099.stdout: 1 648805 11.22 MB/sec execute 186 sec latency 31.360 ms 2024-03-19T01:46:37.686 INFO:tasks.workunit.client.0.smithi099.stdout: 1 650814 11.23 MB/sec execute 187 sec latency 26.427 ms 2024-03-19T01:46:38.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:46:37 smithi115 ceph-mon[67332]: pgmap v516: 97 pgs: 97 active+clean; 209 MiB data, 1.9 GiB used, 1.0 TiB / 1.0 TiB avail; 1.0 MiB/s rd, 3.3 MiB/s wr, 322 op/s 2024-03-19T01:46:38.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:46:37 smithi099 ceph-mon[62951]: pgmap v516: 97 pgs: 97 active+clean; 209 MiB data, 1.9 GiB used, 1.0 TiB / 1.0 TiB avail; 1.0 MiB/s rd, 3.3 MiB/s wr, 322 op/s 2024-03-19T01:46:38.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:46:37 smithi112 ceph-mon[66819]: pgmap v516: 97 pgs: 97 active+clean; 209 MiB data, 1.9 GiB used, 1.0 TiB / 1.0 TiB avail; 1.0 MiB/s rd, 3.3 MiB/s wr, 322 op/s 2024-03-19T01:46:38.686 INFO:tasks.workunit.client.0.smithi099.stdout: 1 653402 11.25 MB/sec execute 188 sec latency 78.318 ms 2024-03-19T01:46:39.686 INFO:tasks.workunit.client.0.smithi099.stdout: 1 655049 11.23 MB/sec execute 189 sec latency 38.228 ms 2024-03-19T01:46:40.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:46:39 smithi115 ceph-mon[67332]: pgmap v517: 97 pgs: 97 active+clean; 209 MiB data, 2.0 GiB used, 1.0 TiB / 1.0 TiB avail; 829 KiB/s rd, 2.9 MiB/s wr, 274 op/s 2024-03-19T01:46:40.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:46:39 smithi099 ceph-mon[62951]: pgmap v517: 97 pgs: 97 active+clean; 209 MiB data, 2.0 GiB used, 1.0 TiB / 1.0 TiB avail; 829 KiB/s rd, 2.9 MiB/s wr, 274 op/s 2024-03-19T01:46:40.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:46:39 smithi112 ceph-mon[66819]: pgmap v517: 97 pgs: 97 active+clean; 209 MiB data, 2.0 GiB used, 1.0 TiB / 1.0 TiB avail; 829 KiB/s rd, 2.9 MiB/s wr, 274 op/s 2024-03-19T01:46:40.686 INFO:tasks.workunit.client.0.smithi099.stdout: 1 657431 11.25 MB/sec execute 190 sec latency 10.353 ms 2024-03-19T01:46:41.686 INFO:tasks.workunit.client.0.smithi099.stdout: 1 659585 11.23 MB/sec execute 191 sec latency 39.610 ms 2024-03-19T01:46:42.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:46:41 smithi115 ceph-mon[67332]: pgmap v518: 97 pgs: 97 active+clean; 209 MiB data, 2.0 GiB used, 1.0 TiB / 1.0 TiB avail; 792 KiB/s rd, 3.1 MiB/s wr, 265 op/s 2024-03-19T01:46:42.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:46:41 smithi099 ceph-mon[62951]: pgmap v518: 97 pgs: 97 active+clean; 209 MiB data, 2.0 GiB used, 1.0 TiB / 1.0 TiB avail; 792 KiB/s rd, 3.1 MiB/s wr, 265 op/s 2024-03-19T01:46:42.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:46:41 smithi112 ceph-mon[66819]: pgmap v518: 97 pgs: 97 active+clean; 209 MiB data, 2.0 GiB used, 1.0 TiB / 1.0 TiB avail; 792 KiB/s rd, 3.1 MiB/s wr, 265 op/s 2024-03-19T01:46:42.686 INFO:tasks.workunit.client.0.smithi099.stdout: 1 661432 11.23 MB/sec execute 192 sec latency 56.541 ms 2024-03-19T01:46:43.686 INFO:tasks.workunit.client.0.smithi099.stdout: 1 663880 11.24 MB/sec execute 193 sec latency 59.164 ms 2024-03-19T01:46:44.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:46:43 smithi115 ceph-mon[67332]: pgmap v519: 97 pgs: 97 active+clean; 212 MiB data, 2.0 GiB used, 1.0 TiB / 1.0 TiB avail; 922 KiB/s rd, 3.2 MiB/s wr, 286 op/s 2024-03-19T01:46:44.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:46:43 smithi099 ceph-mon[62951]: pgmap v519: 97 pgs: 97 active+clean; 212 MiB data, 2.0 GiB used, 1.0 TiB / 1.0 TiB avail; 922 KiB/s rd, 3.2 MiB/s wr, 286 op/s 2024-03-19T01:46:44.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:46:43 smithi112 ceph-mon[66819]: pgmap v519: 97 pgs: 97 active+clean; 212 MiB data, 2.0 GiB used, 1.0 TiB / 1.0 TiB avail; 922 KiB/s rd, 3.2 MiB/s wr, 286 op/s 2024-03-19T01:46:44.686 INFO:tasks.workunit.client.0.smithi099.stdout: 1 665592 11.24 MB/sec execute 194 sec latency 26.632 ms 2024-03-19T01:46:44.898 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.897+0000 7fdd64dc6640 1 -- 172.21.15.99:0/71413927 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fdd600ffea0 msgr2=0x7fdd60106ff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:46:44.898 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.897+0000 7fdd64dc6640 1 --2- 172.21.15.99:0/71413927 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fdd600ffea0 0x7fdd60106ff0 secure :-1 s=READY pgs=455 cs=0 l=1 rev1=1 crypto rx=0x7fdd48009900 tx=0x7fdd4802bc30 comp rx=0 tx=0).stop 2024-03-19T01:46:44.898 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.897+0000 7fdd64dc6640 1 -- 172.21.15.99:0/71413927 shutdown_connections 2024-03-19T01:46:44.898 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.897+0000 7fdd64dc6640 1 --2- 172.21.15.99:0/71413927 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fdd60107530 0x7fdd60109980 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:44.898 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.897+0000 7fdd64dc6640 1 --2- 172.21.15.99:0/71413927 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fdd600ffea0 0x7fdd60106ff0 unknown :-1 s=CLOSED pgs=455 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:44.898 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.897+0000 7fdd64dc6640 1 --2- 172.21.15.99:0/71413927 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fdd600ff4b0 0x7fdd600ff8d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:44.898 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.897+0000 7fdd64dc6640 1 -- 172.21.15.99:0/71413927 >> 172.21.15.99:0/71413927 conn(0x7fdd600fb000 msgr2=0x7fdd600fd440 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:46:44.898 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.897+0000 7fdd64dc6640 1 -- 172.21.15.99:0/71413927 shutdown_connections 2024-03-19T01:46:44.898 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.898+0000 7fdd64dc6640 1 -- 172.21.15.99:0/71413927 wait complete. 2024-03-19T01:46:44.899 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.898+0000 7fdd64dc6640 1 Processor -- start 2024-03-19T01:46:44.900 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.898+0000 7fdd64dc6640 1 -- start start 2024-03-19T01:46:44.901 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.899+0000 7fdd64dc6640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fdd600ff4b0 0x7fdd60197b40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:44.901 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.899+0000 7fdd64dc6640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fdd600ffea0 0x7fdd60198080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:44.901 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.899+0000 7fdd64dc6640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fdd60107530 0x7fdd6019f0a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:44.901 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.899+0000 7fdd64dc6640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fdd6010d4f0 con 0x7fdd600ff4b0 2024-03-19T01:46:44.901 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.899+0000 7fdd64dc6640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fdd6010d370 con 0x7fdd600ffea0 2024-03-19T01:46:44.901 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.899+0000 7fdd64dc6640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fdd6010d670 con 0x7fdd60107530 2024-03-19T01:46:44.901 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.899+0000 7fdd5e59c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fdd600ff4b0 0x7fdd60197b40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:44.901 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.899+0000 7fdd5ed9d640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fdd60107530 0x7fdd6019f0a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:44.901 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.899+0000 7fdd5e59c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fdd600ff4b0 0x7fdd60197b40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:41948/0 (socket says 172.21.15.99:41948) 2024-03-19T01:46:44.901 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.899+0000 7fdd55d9b640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fdd600ffea0 0x7fdd60198080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:44.901 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.899+0000 7fdd5ed9d640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fdd60107530 0x7fdd6019f0a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:43904/0 (socket says 172.21.15.99:43904) 2024-03-19T01:46:44.901 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.899+0000 7fdd5e59c640 1 -- 172.21.15.99:0/1736964112 learned_addr learned my addr 172.21.15.99:0/1736964112 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:46:44.901 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.899+0000 7fdd5ed9d640 1 -- 172.21.15.99:0/1736964112 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fdd600ffea0 msgr2=0x7fdd60198080 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:46:44.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.899+0000 7fdd5ed9d640 1 --2- 172.21.15.99:0/1736964112 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fdd600ffea0 0x7fdd60198080 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:44.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.899+0000 7fdd5ed9d640 1 -- 172.21.15.99:0/1736964112 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fdd600ff4b0 msgr2=0x7fdd60197b40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:46:44.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.899+0000 7fdd5ed9d640 1 --2- 172.21.15.99:0/1736964112 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fdd600ff4b0 0x7fdd60197b40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:44.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.899+0000 7fdd5ed9d640 1 -- 172.21.15.99:0/1736964112 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fdd6019f710 con 0x7fdd60107530 2024-03-19T01:46:44.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.899+0000 7fdd55d9b640 1 --2- 172.21.15.99:0/1736964112 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fdd600ffea0 0x7fdd60198080 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:46:44.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.899+0000 7fdd5e59c640 1 --2- 172.21.15.99:0/1736964112 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fdd600ff4b0 0x7fdd60197b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:46:44.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.899+0000 7fdd5ed9d640 1 --2- 172.21.15.99:0/1736964112 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fdd60107530 0x7fdd6019f0a0 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x7fdd58005fd0 tx=0x7fdd5800ef10 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:46:44.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.899+0000 7fdd57fff640 1 -- 172.21.15.99:0/1736964112 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdd5800cdf0 con 0x7fdd60107530 2024-03-19T01:46:44.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.899+0000 7fdd64dc6640 1 -- 172.21.15.99:0/1736964112 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fdd6019fa00 con 0x7fdd60107530 2024-03-19T01:46:44.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.900+0000 7fdd57fff640 1 -- 172.21.15.99:0/1736964112 <== mon.1 v2:172.21.15.115:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fdd58005020 con 0x7fdd60107530 2024-03-19T01:46:44.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.900+0000 7fdd64dc6640 1 -- 172.21.15.99:0/1736964112 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fdd6019ff20 con 0x7fdd60107530 2024-03-19T01:46:44.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.900+0000 7fdd57fff640 1 -- 172.21.15.99:0/1736964112 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdd580106d0 con 0x7fdd60107530 2024-03-19T01:46:44.909 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.902+0000 7fdd679a8740 1 Processor -- start 2024-03-19T01:46:44.909 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.902+0000 7fdd57fff640 1 -- 172.21.15.99:0/1736964112 <== mon.1 v2:172.21.15.115:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7fdd580054d0 con 0x7fdd60107530 2024-03-19T01:46:44.909 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.902+0000 7fdd679a8740 1 -- start start 2024-03-19T01:46:44.909 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.902+0000 7fdd679a8740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560a6859f100 0x560a6859f520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:44.909 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.903+0000 7fdd679a8740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560a6859faf0 0x560a685a45f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:44.909 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.903+0000 7fdd679a8740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560a685a4c60 0x560a685a70f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:44.909 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.903+0000 7fdd679a8740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x560a68503be0 con 0x560a6859faf0 2024-03-19T01:46:44.909 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.903+0000 7fdd679a8740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x560a684ac960 con 0x560a6859f100 2024-03-19T01:46:44.909 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.903+0000 7fdd679a8740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x560a684ffc80 con 0x560a685a4c60 2024-03-19T01:46:44.909 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.903+0000 7fdd5e59c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560a6859faf0 0x560a685a45f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:44.910 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.903+0000 7fdd5e59c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560a6859faf0 0x560a685a45f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:41954/0 (socket says 172.21.15.99:41954) 2024-03-19T01:46:44.910 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.903+0000 7fdd5e59c640 1 -- 172.21.15.99:0/1056458744 learned_addr learned my addr 172.21.15.99:0/1056458744 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:46:44.910 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.903+0000 7fdd55d9b640 1 --2- 172.21.15.99:0/1056458744 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560a6859f100 0x560a6859f520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:44.910 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.903+0000 7fdd55d9b640 1 --2- 172.21.15.99:0/1056458744 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560a685a4c60 0x560a685a70f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:44.910 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.903+0000 7fdd57fff640 1 --2- 172.21.15.99:0/1736964112 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fdd4c0a28a0 0x7fdd4c0a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:44.910 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.903+0000 7fdd5e59c640 1 -- 172.21.15.99:0/1056458744 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560a685a4c60 msgr2=0x560a685a70f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:46:44.910 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.903+0000 7fdd5e59c640 1 --2- 172.21.15.99:0/1056458744 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560a685a4c60 0x560a685a70f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:44.910 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.903+0000 7fdd5e59c640 1 -- 172.21.15.99:0/1056458744 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560a6859f100 msgr2=0x560a6859f520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:46:44.910 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.903+0000 7fdd5e59c640 1 --2- 172.21.15.99:0/1056458744 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560a6859f100 0x560a6859f520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:44.910 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.903+0000 7fdd5e59c640 1 -- 172.21.15.99:0/1056458744 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x560a684a3d20 con 0x560a6859faf0 2024-03-19T01:46:44.910 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.903+0000 7fdd55d9b640 1 --2- 172.21.15.99:0/1056458744 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560a685a4c60 0x560a685a70f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-19T01:46:44.910 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.903+0000 7fdd5ed9d640 1 --2- 172.21.15.99:0/1736964112 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fdd4c0a28a0 0x7fdd4c0a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:44.910 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.903+0000 7fdd5e59c640 1 --2- 172.21.15.99:0/1056458744 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560a6859faf0 0x560a685a45f0 secure :-1 s=READY pgs=456 cs=0 l=1 rev1=1 crypto rx=0x7fdd50011630 tx=0x7fdd50011af0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a00fea87e99fd30a server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:46:44.911 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.903+0000 7fdd57fff640 1 -- 172.21.15.99:0/1736964112 <== mon.1 v2:172.21.15.115:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7fdd580cf180 con 0x7fdd60107530 2024-03-19T01:46:44.911 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.903+0000 7fdd54d99640 1 -- 172.21.15.99:0/1056458744 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdd5001e070 con 0x560a6859faf0 2024-03-19T01:46:44.911 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.904+0000 7fdd54d99640 1 -- 172.21.15.99:0/1056458744 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fdd5000fbd0 con 0x560a6859faf0 2024-03-19T01:46:44.911 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.904+0000 7fdd54d99640 1 -- 172.21.15.99:0/1056458744 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdd50019d00 con 0x560a6859faf0 2024-03-19T01:46:44.911 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.904+0000 7fdd679a8740 1 -- 172.21.15.99:0/1056458744 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560a6859faf0 msgr2=0x560a685a45f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:46:44.911 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.904+0000 7fdd5ed9d640 1 --2- 172.21.15.99:0/1736964112 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fdd4c0a28a0 0x7fdd4c0a4d60 secure :-1 s=READY pgs=273 cs=0 l=1 rev1=1 crypto rx=0x7fdd580a3d50 tx=0x7fdd5806d760 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:46:44.911 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.904+0000 7fdd679a8740 1 --2- 172.21.15.99:0/1056458744 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560a6859faf0 0x560a685a45f0 secure :-1 s=READY pgs=456 cs=0 l=1 rev1=1 crypto rx=0x7fdd50011630 tx=0x7fdd50011af0 comp rx=0 tx=0).stop 2024-03-19T01:46:44.911 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.904+0000 7fdd679a8740 1 -- 172.21.15.99:0/1056458744 shutdown_connections 2024-03-19T01:46:44.911 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.904+0000 7fdd679a8740 1 --2- 172.21.15.99:0/1056458744 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560a685a4c60 0x560a685a70f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:44.911 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.904+0000 7fdd679a8740 1 --2- 172.21.15.99:0/1056458744 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560a6859faf0 0x560a685a45f0 unknown :-1 s=CLOSED pgs=456 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:44.911 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.904+0000 7fdd679a8740 1 --2- 172.21.15.99:0/1056458744 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560a6859f100 0x560a6859f520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:44.911 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.904+0000 7fdd679a8740 1 -- 172.21.15.99:0/1056458744 >> 172.21.15.99:0/1056458744 conn(0x560a6859a670 msgr2=0x560a6859aa70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:46:44.911 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.904+0000 7fdd679a8740 1 -- 172.21.15.99:0/1056458744 shutdown_connections 2024-03-19T01:46:44.911 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.904+0000 7fdd679a8740 1 -- 172.21.15.99:0/1056458744 wait complete. 2024-03-19T01:46:44.911 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.905+0000 7fdd679a8740 1 -- start start 2024-03-19T01:46:44.912 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.905+0000 7fdd679a8740 1 Processor -- start 2024-03-19T01:46:44.912 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.905+0000 7fdd679a8740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560a6859f100 0x560a685a9110 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:44.912 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.905+0000 7fdd679a8740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560a6859faf0 0x560a685a9650 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:44.912 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.905+0000 7fdd679a8740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560a685a4c60 0x560a685b0690 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:44.912 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.905+0000 7fdd679a8740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x560a685a7c10 con 0x560a685a4c60 2024-03-19T01:46:44.912 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.905+0000 7fdd679a8740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x560a684bfdb0 con 0x560a6859f100 2024-03-19T01:46:44.912 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.905+0000 7fdd679a8740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x560a685a7d90 con 0x560a6859faf0 2024-03-19T01:46:44.912 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.905+0000 7fdd679a8740 10 client.0 _ll_register_callbacks cb 0x560a684c4e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:46:44.912 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.905+0000 7fdd5e59c640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560a6859faf0 0x560a685a9650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:44.912 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.905+0000 7fdd55d9b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560a685a4c60 0x560a685b0690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:44.912 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.905+0000 7fdd55d9b640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560a6859f100 0x560a685a9110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:44.912 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.905+0000 7fdd5e59c640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560a6859faf0 0x560a685a9650 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:43924/0 (socket says 172.21.15.99:43924) 2024-03-19T01:46:44.912 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.905+0000 7fdd5e59c640 1 -- 172.21.15.99:0/1911850593 learned_addr learned my addr 172.21.15.99:0/1911850593 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:46:44.912 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.905+0000 7fdd55d9b640 1 -- 172.21.15.99:0/1911850593 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560a6859faf0 msgr2=0x560a685a9650 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:46:44.913 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.905+0000 7fdd55d9b640 1 --2- 172.21.15.99:0/1911850593 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560a6859faf0 0x560a685a9650 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:44.913 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.905+0000 7fdd55d9b640 1 -- 172.21.15.99:0/1911850593 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560a685a4c60 msgr2=0x560a685b0690 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:46:44.913 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.905+0000 7fdd55d9b640 1 --2- 172.21.15.99:0/1911850593 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560a685a4c60 0x560a685b0690 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:44.913 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.905+0000 7fdd55d9b640 1 -- 172.21.15.99:0/1911850593 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x560a684bec60 con 0x560a6859f100 2024-03-19T01:46:44.913 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.906+0000 7fdd45ffb640 10 client.0 ms_handle_connect on v2:172.21.15.112:3300/0 2024-03-19T01:46:44.913 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.906+0000 7fdd55d9b640 1 --2- 172.21.15.99:0/1911850593 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560a6859f100 0x560a685a9110 secure :-1 s=READY pgs=124 cs=0 l=1 rev1=1 crypto rx=0x7fdd48009230 tx=0x7fdd48005b50 comp rx=0 tx=0).ready entity=mon.2 client_cookie=351962ee56f75e35 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:46:44.913 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.906+0000 7fdd45ffb640 1 -- 172.21.15.99:0/1911850593 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdd48034440 con 0x560a6859f100 2024-03-19T01:46:44.913 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.906+0000 7fdd45ffb640 1 -- 172.21.15.99:0/1911850593 <== mon.2 v2:172.21.15.112:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7fdd480345e0 con 0x560a6859f100 2024-03-19T01:46:44.913 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.906+0000 7fdd679a8740 1 -- 172.21.15.99:0/1911850593 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fdd5000fbd0 con 0x560a6859f100 2024-03-19T01:46:44.913 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.906+0000 7fdd45ffb640 1 -- 172.21.15.99:0/1911850593 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_get_version_reply(handle=1 version=26) ==== 24+0+0 (secure 0 0 0) 0x7fdd48055050 con 0x560a6859f100 2024-03-19T01:46:44.913 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.906+0000 7fdd679a8740 10 client.25565 fetch_fsmap learned FSMap version 26 2024-03-19T01:46:44.913 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.906+0000 7fdd679a8740 1 -- 172.21.15.99:0/1911850593 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,fsmap=26,monmap=4+}) -- 0x560a68218210 con 0x560a6859f100 2024-03-19T01:46:44.913 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.906+0000 7fdd45ffb640 1 -- 172.21.15.99:0/1911850593 <== mon.2 v2:172.21.15.112:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fdd5000fbd0 con 0x560a6859f100 2024-03-19T01:46:44.913 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.906+0000 7fdd45ffb640 1 -- 172.21.15.99:0/1911850593 <== mon.2 v2:172.21.15.112:3300/0 5 ==== fsmap(e 26) ==== 8342+0+0 (secure 0 0 0) 0x7fdd4804c070 con 0x560a6859f100 2024-03-19T01:46:44.913 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.906+0000 7fdd679a8740 10 client.25565 fetch_fsmap finished waiting for FSMap version 26 2024-03-19T01:46:44.914 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.906+0000 7fdd679a8740 10 client.25565 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:46:44.914 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.906+0000 7fdd679a8740 20 client.25565 populate_metadata read hostname 'smithi099' 2024-03-19T01:46:44.914 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.906+0000 7fdd679a8740 1 --2- 172.21.15.99:0/1911850593 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x560a685b4ec0 0x560a685b72f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:44.914 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.906+0000 7fdd679a8740 4 client.25565 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:46:44.914 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.906+0000 7fdd679a8740 1 -- 172.21.15.99:0/1911850593 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x560a68511120 con 0x560a685b4ec0 2024-03-19T01:46:44.914 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.907+0000 7fdd5e59c640 1 --2- 172.21.15.99:0/1911850593 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x560a685b4ec0 0x560a685b72f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:44.914 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.907+0000 7fdd45ffb640 10 client.25565 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:46:44.914 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.907+0000 7fdd5e59c640 1 --2- 172.21.15.99:0/1911850593 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x560a685b4ec0 0x560a685b72f0 crc :-1 s=READY pgs=79 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=fe7b15970ba49b77 server_cookie=3fde870bd56ef248 in_seq=0 out_seq=0 2024-03-19T01:46:44.914 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.907+0000 7fdd45ffb640 1 -- 172.21.15.99:0/1911850593 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x560a68511120 con 0x560a685b4ec0 2024-03-19T01:46:44.914 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.908+0000 7fdd45ffb640 10 client.25565 handle_command_reply: tid=0 2024-03-19T01:46:44.914 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.908+0000 7fdd679a8740 1 client.25565 shutdown 2024-03-19T01:46:44.914 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.908+0000 7fdd679a8740 1 -- 172.21.15.99:0/1911850593 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560a6859f100 msgr2=0x560a685a9110 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:46:44.914 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.908+0000 7fdd679a8740 1 --2- 172.21.15.99:0/1911850593 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560a6859f100 0x560a685a9110 secure :-1 s=READY pgs=124 cs=0 l=1 rev1=1 crypto rx=0x7fdd48009230 tx=0x7fdd48005b50 comp rx=0 tx=0).stop 2024-03-19T01:46:44.914 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.908+0000 7fdd679a8740 1 -- 172.21.15.99:0/1911850593 shutdown_connections 2024-03-19T01:46:44.915 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.908+0000 7fdd679a8740 1 --2- 172.21.15.99:0/1911850593 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x560a685b4ec0 0x560a685b72f0 crc :-1 s=READY pgs=79 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:44.915 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.908+0000 7fdd679a8740 1 --2- 172.21.15.99:0/1911850593 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560a685a4c60 0x560a685b0690 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:44.915 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.908+0000 7fdd45ffb640 0 client.25565 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:46:44.915 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.908+0000 7fdd679a8740 1 --2- 172.21.15.99:0/1911850593 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560a6859faf0 0x560a685a9650 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:44.915 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.908+0000 7fdd679a8740 1 --2- 172.21.15.99:0/1911850593 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560a6859f100 0x560a685a9110 unknown :-1 s=CLOSED pgs=124 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:44.915 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.908+0000 7fdd679a8740 1 -- 172.21.15.99:0/1911850593 >> 172.21.15.99:0/1911850593 conn(0x560a6859a670 msgr2=0x560a6859b8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:46:44.915 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.908+0000 7fdd679a8740 1 -- 172.21.15.99:0/1911850593 shutdown_connections 2024-03-19T01:46:44.915 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.908+0000 7fdd679a8740 1 -- 172.21.15.99:0/1911850593 wait complete. 2024-03-19T01:46:44.915 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.908+0000 7fdd679a8740 20 client.25565 trim_cache size 0 max 16384 2024-03-19T01:46:44.925 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.924+0000 7fdd679a8740 1 Processor -- start 2024-03-19T01:46:44.925 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.924+0000 7fdd679a8740 1 -- start start 2024-03-19T01:46:44.925 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.924+0000 7fdd679a8740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560a6859f100 0x560a68599ab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:44.925 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.924+0000 7fdd679a8740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560a6859faf0 0x560a6859dfe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:44.925 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.924+0000 7fdd679a8740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560a685a4c60 0x560a685bbbf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:44.925 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.924+0000 7fdd679a8740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fdd601ab600 con 0x560a685a4c60 2024-03-19T01:46:44.926 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.924+0000 7fdd679a8740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fdd601a8ed0 con 0x560a6859f100 2024-03-19T01:46:44.926 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.924+0000 7fdd679a8740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fdd601a6d20 con 0x560a6859faf0 2024-03-19T01:46:44.926 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.925+0000 7fdd5e59c640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560a6859faf0 0x560a6859dfe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:44.926 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.925+0000 7fdd55d9b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560a685a4c60 0x560a685bbbf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:44.926 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.925+0000 7fdd5e59c640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560a6859faf0 0x560a6859dfe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:43936/0 (socket says 172.21.15.99:43936) 2024-03-19T01:46:44.926 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.925+0000 7fdd5e59c640 1 -- 172.21.15.99:0/1624522240 learned_addr learned my addr 172.21.15.99:0/1624522240 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:46:44.926 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.925+0000 7fdd55d9b640 1 --2- 172.21.15.99:0/1624522240 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560a6859f100 0x560a68599ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:44.926 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.925+0000 7fdd5e59c640 1 -- 172.21.15.99:0/1624522240 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560a6859f100 msgr2=0x560a68599ab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:46:44.926 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.925+0000 7fdd5e59c640 1 --2- 172.21.15.99:0/1624522240 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560a6859f100 0x560a68599ab0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:44.926 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.925+0000 7fdd5e59c640 1 -- 172.21.15.99:0/1624522240 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560a685a4c60 msgr2=0x560a685bbbf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:46:44.926 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.925+0000 7fdd5e59c640 1 --2- 172.21.15.99:0/1624522240 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560a685a4c60 0x560a685bbbf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:44.926 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.925+0000 7fdd5e59c640 1 -- 172.21.15.99:0/1624522240 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x560a682e4620 con 0x560a6859faf0 2024-03-19T01:46:44.926 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.925+0000 7fdd55d9b640 1 --2- 172.21.15.99:0/1624522240 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560a6859f100 0x560a68599ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:46:44.927 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.925+0000 7fdd5e59c640 1 --2- 172.21.15.99:0/1624522240 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560a6859faf0 0x560a6859dfe0 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 crypto rx=0x7fdd5000f3c0 tx=0x7fdd50009ef0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=9215055a7bd33d0c server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:46:44.927 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.925+0000 7fdd47fff640 1 -- 172.21.15.99:0/1624522240 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdd5001e040 con 0x560a6859faf0 2024-03-19T01:46:44.927 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.925+0000 7fdd47fff640 1 -- 172.21.15.99:0/1624522240 <== mon.1 v2:172.21.15.115:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fdd5002f070 con 0x560a6859faf0 2024-03-19T01:46:44.927 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.925+0000 7fdd47fff640 1 -- 172.21.15.99:0/1624522240 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdd50005040 con 0x560a6859faf0 2024-03-19T01:46:44.930 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.926+0000 7fdd679a8740 1 -- 172.21.15.99:0/1624522240 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560a6859faf0 msgr2=0x560a6859dfe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:46:44.930 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.926+0000 7fdd679a8740 1 --2- 172.21.15.99:0/1624522240 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560a6859faf0 0x560a6859dfe0 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 crypto rx=0x7fdd5000f3c0 tx=0x7fdd50009ef0 comp rx=0 tx=0).stop 2024-03-19T01:46:44.930 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.926+0000 7fdd679a8740 1 -- 172.21.15.99:0/1624522240 shutdown_connections 2024-03-19T01:46:44.930 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.926+0000 7fdd679a8740 1 --2- 172.21.15.99:0/1624522240 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560a685a4c60 0x560a685bbbf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:44.931 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.926+0000 7fdd679a8740 1 --2- 172.21.15.99:0/1624522240 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560a6859faf0 0x560a6859dfe0 unknown :-1 s=CLOSED pgs=90 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:44.931 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.926+0000 7fdd679a8740 1 --2- 172.21.15.99:0/1624522240 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560a6859f100 0x560a68599ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:44.931 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.926+0000 7fdd679a8740 1 -- 172.21.15.99:0/1624522240 >> 172.21.15.99:0/1624522240 conn(0x560a6859a670 msgr2=0x560a6859d8d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:46:44.931 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.926+0000 7fdd679a8740 1 -- 172.21.15.99:0/1624522240 shutdown_connections 2024-03-19T01:46:44.931 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.926+0000 7fdd679a8740 1 -- 172.21.15.99:0/1624522240 wait complete. 2024-03-19T01:46:44.931 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.926+0000 7fdd679a8740 1 -- start start 2024-03-19T01:46:44.931 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.926+0000 7fdd679a8740 1 Processor -- start 2024-03-19T01:46:44.931 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.927+0000 7fdd679a8740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560a6859f100 0x560a685ae160 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:44.931 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.927+0000 7fdd679a8740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560a6859faf0 0x560a685ae6a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:44.931 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.927+0000 7fdd679a8740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560a685a4c60 0x560a685af6b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:44.931 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.927+0000 7fdd679a8740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fdd601a7390 con 0x560a6859f100 2024-03-19T01:46:44.931 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.927+0000 7fdd679a8740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fdd601a8800 con 0x560a685a4c60 2024-03-19T01:46:44.931 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.927+0000 7fdd679a8740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fdd601a7a60 con 0x560a6859faf0 2024-03-19T01:46:44.931 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.927+0000 7fdd679a8740 10 client.0 _ll_register_callbacks cb 0x7fdd48009780 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:46:44.932 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.927+0000 7fdd55d9b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560a6859f100 0x560a685ae160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:44.932 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.927+0000 7fdd55d9b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560a6859f100 0x560a685ae160 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:41986/0 (socket says 172.21.15.99:41986) 2024-03-19T01:46:44.932 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.927+0000 7fdd55d9b640 1 -- 172.21.15.99:0/74228133 learned_addr learned my addr 172.21.15.99:0/74228133 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:46:44.932 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.927+0000 7fdd5e59c640 1 --2- 172.21.15.99:0/74228133 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560a6859faf0 0x560a685ae6a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:44.932 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.927+0000 7fdd55d9b640 1 --2- 172.21.15.99:0/74228133 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560a685a4c60 0x560a685af6b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:44.932 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.927+0000 7fdd55d9b640 1 -- 172.21.15.99:0/74228133 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560a6859faf0 msgr2=0x560a685ae6a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:46:44.932 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.927+0000 7fdd55d9b640 1 --2- 172.21.15.99:0/74228133 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560a6859faf0 0x560a685ae6a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:44.932 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.927+0000 7fdd55d9b640 1 -- 172.21.15.99:0/74228133 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560a685a4c60 msgr2=0x560a685af6b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:46:44.932 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.927+0000 7fdd55d9b640 1 --2- 172.21.15.99:0/74228133 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560a685a4c60 0x560a685af6b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:44.932 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.927+0000 7fdd55d9b640 1 -- 172.21.15.99:0/74228133 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x560a685afca0 con 0x560a6859f100 2024-03-19T01:46:44.932 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.927+0000 7fdd55d9b640 1 --2- 172.21.15.99:0/74228133 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560a6859f100 0x560a685ae160 secure :-1 s=READY pgs=457 cs=0 l=1 rev1=1 crypto rx=0x7fdd4802c7b0 tx=0x7fdd4802c6d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5da9bcf76ec4acb7 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:46:44.932 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.927+0000 7fdd45ffb640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:46:44.932 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.927+0000 7fdd45ffb640 1 -- 172.21.15.99:0/74228133 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdd48003cd0 con 0x560a6859f100 2024-03-19T01:46:44.932 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.927+0000 7fdd45ffb640 1 -- 172.21.15.99:0/74228133 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7fdd48034440 con 0x560a6859f100 2024-03-19T01:46:44.932 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.927+0000 7fdd679a8740 1 -- 172.21.15.99:0/74228133 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fdd5002f070 con 0x560a6859f100 2024-03-19T01:46:44.933 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.927+0000 7fdd45ffb640 1 -- 172.21.15.99:0/74228133 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=26) ==== 24+0+0 (secure 0 0 0) 0x7fdd48003e70 con 0x560a6859f100 2024-03-19T01:46:44.933 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.928+0000 7fdd679a8740 10 client.15798 fetch_fsmap learned FSMap version 26 2024-03-19T01:46:44.933 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.928+0000 7fdd679a8740 1 -- 172.21.15.99:0/74228133 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=26,monmap=4+}) -- 0x560a6861b990 con 0x560a6859f100 2024-03-19T01:46:44.933 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.928+0000 7fdd45ffb640 1 -- 172.21.15.99:0/74228133 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fdd5002f070 con 0x560a6859f100 2024-03-19T01:46:44.933 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.928+0000 7fdd45ffb640 1 -- 172.21.15.99:0/74228133 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 26) ==== 8342+0+0 (secure 0 0 0) 0x7fdd4804c040 con 0x560a6859f100 2024-03-19T01:46:44.933 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.928+0000 7fdd679a8740 10 client.15798 fetch_fsmap finished waiting for FSMap version 26 2024-03-19T01:46:44.933 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.928+0000 7fdd679a8740 10 client.15798 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:46:44.933 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.928+0000 7fdd679a8740 20 client.15798 populate_metadata read hostname 'smithi099' 2024-03-19T01:46:44.933 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.928+0000 7fdd679a8740 1 --2- 172.21.15.99:0/74228133 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x560a6861c3c0 0x560a6861e7e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:44.933 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.928+0000 7fdd679a8740 4 client.15798 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:46:44.933 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.928+0000 7fdd679a8740 1 -- 172.21.15.99:0/74228133 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x560a6850b470 con 0x560a6861c3c0 2024-03-19T01:46:44.933 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.928+0000 7fdd5e59c640 1 --2- 172.21.15.99:0/74228133 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x560a6861c3c0 0x560a6861e7e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:44.933 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.929+0000 7fdd45ffb640 10 client.15798 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:46:44.934 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.929+0000 7fdd5e59c640 1 --2- 172.21.15.99:0/74228133 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x560a6861c3c0 0x560a6861e7e0 crc :-1 s=READY pgs=80 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=324d25e184b00e94 server_cookie=eac1b364fad955 in_seq=0 out_seq=0 2024-03-19T01:46:44.978 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.978+0000 7fdd45ffb640 1 -- 172.21.15.99:0/74228133 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x560a6850b470 con 0x560a6861c3c0 2024-03-19T01:46:44.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.978+0000 7fdd45ffb640 10 client.15798 handle_command_reply: tid=0 2024-03-19T01:46:44.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.978+0000 7fdd679a8740 1 client.15798 shutdown 2024-03-19T01:46:44.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.978+0000 7fdd679a8740 1 -- 172.21.15.99:0/74228133 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560a6859f100 msgr2=0x560a685ae160 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:46:44.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.978+0000 7fdd679a8740 1 --2- 172.21.15.99:0/74228133 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560a6859f100 0x560a685ae160 secure :-1 s=READY pgs=457 cs=0 l=1 rev1=1 crypto rx=0x7fdd4802c7b0 tx=0x7fdd4802c6d0 comp rx=0 tx=0).stop 2024-03-19T01:46:44.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.978+0000 7fdd679a8740 1 -- 172.21.15.99:0/74228133 shutdown_connections 2024-03-19T01:46:44.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.978+0000 7fdd679a8740 1 --2- 172.21.15.99:0/74228133 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x560a6861c3c0 0x560a6861e7e0 crc :-1 s=READY pgs=80 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:44.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.978+0000 7fdd679a8740 1 --2- 172.21.15.99:0/74228133 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560a685a4c60 0x560a685af6b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:44.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.978+0000 7fdd679a8740 1 --2- 172.21.15.99:0/74228133 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560a6859faf0 0x560a685ae6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:44.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.978+0000 7fdd679a8740 1 --2- 172.21.15.99:0/74228133 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560a6859f100 0x560a685ae160 unknown :-1 s=CLOSED pgs=457 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:44.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.978+0000 7fdd679a8740 1 -- 172.21.15.99:0/74228133 >> 172.21.15.99:0/74228133 conn(0x560a6859a670 msgr2=0x560a6859e160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:46:44.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.978+0000 7fdd45ffb640 0 client.15798 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:46:44.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.978+0000 7fdd679a8740 1 -- 172.21.15.99:0/74228133 shutdown_connections 2024-03-19T01:46:44.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.978+0000 7fdd679a8740 1 -- 172.21.15.99:0/74228133 wait complete. 2024-03-19T01:46:44.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.978+0000 7fdd679a8740 20 client.15798 trim_cache size 0 max 16384 2024-03-19T01:46:44.980 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:46:44.980 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:46:44.980 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:46:44.980 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:46:44.981 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.979+0000 7fdd64dc6640 1 -- 172.21.15.99:0/1736964112 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fdd4c0a28a0 msgr2=0x7fdd4c0a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:46:44.981 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.979+0000 7fdd64dc6640 1 --2- 172.21.15.99:0/1736964112 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fdd4c0a28a0 0x7fdd4c0a4d60 secure :-1 s=READY pgs=273 cs=0 l=1 rev1=1 crypto rx=0x7fdd580a3d50 tx=0x7fdd5806d760 comp rx=0 tx=0).stop 2024-03-19T01:46:44.981 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.979+0000 7fdd64dc6640 1 -- 172.21.15.99:0/1736964112 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fdd60107530 msgr2=0x7fdd6019f0a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:46:44.981 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.979+0000 7fdd64dc6640 1 --2- 172.21.15.99:0/1736964112 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fdd60107530 0x7fdd6019f0a0 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x7fdd58005fd0 tx=0x7fdd5800ef10 comp rx=0 tx=0).stop 2024-03-19T01:46:44.981 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.979+0000 7fdd64dc6640 1 -- 172.21.15.99:0/1736964112 shutdown_connections 2024-03-19T01:46:44.981 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.979+0000 7fdd64dc6640 1 --2- 172.21.15.99:0/1736964112 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fdd4c0a28a0 0x7fdd4c0a4d60 unknown :-1 s=CLOSED pgs=273 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:44.981 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.979+0000 7fdd64dc6640 1 --2- 172.21.15.99:0/1736964112 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fdd600ffea0 0x7fdd60198080 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:44.981 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.979+0000 7fdd64dc6640 1 --2- 172.21.15.99:0/1736964112 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fdd60107530 0x7fdd6019f0a0 unknown :-1 s=CLOSED pgs=89 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:44.981 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.979+0000 7fdd64dc6640 1 --2- 172.21.15.99:0/1736964112 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fdd600ff4b0 0x7fdd60197b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:44.982 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.979+0000 7fdd64dc6640 1 -- 172.21.15.99:0/1736964112 >> 172.21.15.99:0/1736964112 conn(0x7fdd600fb000 msgr2=0x7fdd60102fd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:46:44.982 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.979+0000 7fdd64dc6640 1 -- 172.21.15.99:0/1736964112 shutdown_connections 2024-03-19T01:46:44.982 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:44.979+0000 7fdd64dc6640 1 -- 172.21.15.99:0/1736964112 wait complete. 2024-03-19T01:46:45.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:46:45 smithi115 ceph-mon[67332]: pgmap v520: 97 pgs: 97 active+clean; 213 MiB data, 2.0 GiB used, 1.0 TiB / 1.0 TiB avail; 790 KiB/s rd, 2.7 MiB/s wr, 246 op/s 2024-03-19T01:46:45.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:46:45 smithi099 ceph-mon[62951]: pgmap v520: 97 pgs: 97 active+clean; 213 MiB data, 2.0 GiB used, 1.0 TiB / 1.0 TiB avail; 790 KiB/s rd, 2.7 MiB/s wr, 246 op/s 2024-03-19T01:46:45.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:46:45 smithi112 ceph-mon[66819]: pgmap v520: 97 pgs: 97 active+clean; 213 MiB data, 2.0 GiB used, 1.0 TiB / 1.0 TiB avail; 790 KiB/s rd, 2.7 MiB/s wr, 246 op/s 2024-03-19T01:46:45.686 INFO:tasks.workunit.client.0.smithi099.stdout: 1 668017 11.25 MB/sec execute 195 sec latency 25.686 ms 2024-03-19T01:46:46.686 INFO:tasks.workunit.client.0.smithi099.stdout: 1 670181 11.23 MB/sec execute 196 sec latency 19.530 ms 2024-03-19T01:46:47.686 INFO:tasks.workunit.client.0.smithi099.stdout: 1 672075 11.24 MB/sec execute 197 sec latency 36.470 ms 2024-03-19T01:46:48.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:46:47 smithi115 ceph-mon[67332]: pgmap v521: 97 pgs: 97 active+clean; 213 MiB data, 2.0 GiB used, 1.0 TiB / 1.0 TiB avail; 861 KiB/s rd, 3.3 MiB/s wr, 273 op/s 2024-03-19T01:46:48.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:46:47 smithi099 ceph-mon[62951]: pgmap v521: 97 pgs: 97 active+clean; 213 MiB data, 2.0 GiB used, 1.0 TiB / 1.0 TiB avail; 861 KiB/s rd, 3.3 MiB/s wr, 273 op/s 2024-03-19T01:46:48.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:46:47 smithi112 ceph-mon[66819]: pgmap v521: 97 pgs: 97 active+clean; 213 MiB data, 2.0 GiB used, 1.0 TiB / 1.0 TiB avail; 861 KiB/s rd, 3.3 MiB/s wr, 273 op/s 2024-03-19T01:46:48.686 INFO:tasks.workunit.client.0.smithi099.stdout: 1 674580 11.25 MB/sec execute 198 sec latency 65.153 ms 2024-03-19T01:46:49.686 INFO:tasks.workunit.client.0.smithi099.stdout: 1 676360 11.24 MB/sec execute 199 sec latency 26.207 ms 2024-03-19T01:46:50.164 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:46:49 smithi112 ceph-mon[66819]: pgmap v522: 97 pgs: 97 active+clean; 216 MiB data, 2.0 GiB used, 1.0 TiB / 1.0 TiB avail; 768 KiB/s rd, 3.0 MiB/s wr, 249 op/s 2024-03-19T01:46:50.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:46:49 smithi115 ceph-mon[67332]: pgmap v522: 97 pgs: 97 active+clean; 216 MiB data, 2.0 GiB used, 1.0 TiB / 1.0 TiB avail; 768 KiB/s rd, 3.0 MiB/s wr, 249 op/s 2024-03-19T01:46:50.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:46:49 smithi099 ceph-mon[62951]: pgmap v522: 97 pgs: 97 active+clean; 216 MiB data, 2.0 GiB used, 1.0 TiB / 1.0 TiB avail; 768 KiB/s rd, 3.0 MiB/s wr, 249 op/s 2024-03-19T01:46:50.686 INFO:tasks.workunit.client.0.smithi099.stdout: 1 678762 11.26 MB/sec execute 200 sec latency 10.452 ms 2024-03-19T01:46:51.686 INFO:tasks.workunit.client.0.smithi099.stdout: 1 681022 11.26 MB/sec execute 201 sec latency 33.653 ms 2024-03-19T01:46:52.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:46:51 smithi115 ceph-mon[67332]: pgmap v523: 97 pgs: 97 active+clean; 217 MiB data, 2.0 GiB used, 1.0 TiB / 1.0 TiB avail; 758 KiB/s rd, 3.0 MiB/s wr, 248 op/s 2024-03-19T01:46:52.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:46:51 smithi099 ceph-mon[62951]: pgmap v523: 97 pgs: 97 active+clean; 217 MiB data, 2.0 GiB used, 1.0 TiB / 1.0 TiB avail; 758 KiB/s rd, 3.0 MiB/s wr, 248 op/s 2024-03-19T01:46:52.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:46:51 smithi112 ceph-mon[66819]: pgmap v523: 97 pgs: 97 active+clean; 217 MiB data, 2.0 GiB used, 1.0 TiB / 1.0 TiB avail; 758 KiB/s rd, 3.0 MiB/s wr, 248 op/s 2024-03-19T01:46:52.686 INFO:tasks.workunit.client.0.smithi099.stdout: 1 683045 11.25 MB/sec execute 202 sec latency 25.855 ms 2024-03-19T01:46:53.686 INFO:tasks.workunit.client.0.smithi099.stdout: 1 685628 11.28 MB/sec execute 203 sec latency 41.374 ms 2024-03-19T01:46:54.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:46:53 smithi115 ceph-mon[67332]: pgmap v524: 97 pgs: 97 active+clean; 218 MiB data, 2.0 GiB used, 1.0 TiB / 1.0 TiB avail; 958 KiB/s rd, 2.9 MiB/s wr, 274 op/s 2024-03-19T01:46:54.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:46:53 smithi099 ceph-mon[62951]: pgmap v524: 97 pgs: 97 active+clean; 218 MiB data, 2.0 GiB used, 1.0 TiB / 1.0 TiB avail; 958 KiB/s rd, 2.9 MiB/s wr, 274 op/s 2024-03-19T01:46:54.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:46:53 smithi112 ceph-mon[66819]: pgmap v524: 97 pgs: 97 active+clean; 218 MiB data, 2.0 GiB used, 1.0 TiB / 1.0 TiB avail; 958 KiB/s rd, 2.9 MiB/s wr, 274 op/s 2024-03-19T01:46:54.690 INFO:tasks.workunit.client.0.smithi099.stdout: 1 687356 11.25 MB/sec execute 204 sec latency 30.314 ms 2024-03-19T01:46:55.687 INFO:tasks.workunit.client.0.smithi099.stdout: 1 689542 11.26 MB/sec execute 205 sec latency 29.272 ms 2024-03-19T01:46:56.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:46:55 smithi115 ceph-mon[67332]: pgmap v525: 97 pgs: 97 active+clean; 219 MiB data, 2.0 GiB used, 1.0 TiB / 1.0 TiB avail; 679 KiB/s rd, 2.6 MiB/s wr, 225 op/s 2024-03-19T01:46:56.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:46:55 smithi099 ceph-mon[62951]: pgmap v525: 97 pgs: 97 active+clean; 219 MiB data, 2.0 GiB used, 1.0 TiB / 1.0 TiB avail; 679 KiB/s rd, 2.6 MiB/s wr, 225 op/s 2024-03-19T01:46:56.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:46:55 smithi112 ceph-mon[66819]: pgmap v525: 97 pgs: 97 active+clean; 219 MiB data, 2.0 GiB used, 1.0 TiB / 1.0 TiB avail; 679 KiB/s rd, 2.6 MiB/s wr, 225 op/s 2024-03-19T01:46:56.687 INFO:tasks.workunit.client.0.smithi099.stdout: 1 692136 11.27 MB/sec execute 206 sec latency 33.226 ms 2024-03-19T01:46:57.687 INFO:tasks.workunit.client.0.smithi099.stdout: 1 693901 11.27 MB/sec execute 207 sec latency 29.322 ms 2024-03-19T01:46:58.070 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.070+0000 7fe10d061640 1 -- 172.21.15.99:0/2941859543 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe108101900 msgr2=0x7fe108101d80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:46:58.070 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.070+0000 7fe10d061640 1 --2- 172.21.15.99:0/2941859543 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe108101900 0x7fe108101d80 secure :-1 s=READY pgs=458 cs=0 l=1 rev1=1 crypto rx=0x7fe0fc009860 tx=0x7fe0fc02bb70 comp rx=0 tx=0).stop 2024-03-19T01:46:58.070 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.070+0000 7fe10d061640 1 -- 172.21.15.99:0/2941859543 shutdown_connections 2024-03-19T01:46:58.071 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.070+0000 7fe10d061640 1 --2- 172.21.15.99:0/2941859543 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fe1081022c0 0x7fe108108b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:58.071 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.070+0000 7fe10d061640 1 --2- 172.21.15.99:0/2941859543 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe108101900 0x7fe108101d80 unknown :-1 s=CLOSED pgs=458 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:58.071 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.070+0000 7fe10d061640 1 --2- 172.21.15.99:0/2941859543 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fe108100710 0x7fe108100b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:58.071 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.070+0000 7fe10d061640 1 -- 172.21.15.99:0/2941859543 >> 172.21.15.99:0/2941859543 conn(0x7fe1080fbc80 msgr2=0x7fe1080fe0c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:46:58.071 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.070+0000 7fe10d061640 1 -- 172.21.15.99:0/2941859543 shutdown_connections 2024-03-19T01:46:58.071 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.070+0000 7fe10d061640 1 -- 172.21.15.99:0/2941859543 wait complete. 2024-03-19T01:46:58.071 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.070+0000 7fe10d061640 1 Processor -- start 2024-03-19T01:46:58.073 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.071+0000 7fe10d061640 1 -- start start 2024-03-19T01:46:58.073 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.071+0000 7fe10d061640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fe108100710 0x7fe108069a10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:58.073 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.071+0000 7fe10d061640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fe108101900 0x7fe10819c7c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:58.073 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.071+0000 7fe10d061640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe1081022c0 0x7fe10819ede0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:58.074 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.071+0000 7fe10d061640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fe10810c390 con 0x7fe1081022c0 2024-03-19T01:46:58.074 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.071+0000 7fe10d061640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fe10810c210 con 0x7fe108101900 2024-03-19T01:46:58.074 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.071+0000 7fe10d061640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fe10810c510 con 0x7fe108100710 2024-03-19T01:46:58.074 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.071+0000 7fe10759e640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe1081022c0 0x7fe10819ede0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:58.074 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.071+0000 7fe106d9d640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fe108100710 0x7fe108069a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:58.074 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.071+0000 7fe10759e640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe1081022c0 0x7fe10819ede0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:33774/0 (socket says 172.21.15.99:33774) 2024-03-19T01:46:58.074 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.071+0000 7fe10659c640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fe108101900 0x7fe10819c7c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:58.074 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.071+0000 7fe10759e640 1 -- 172.21.15.99:0/4252951649 learned_addr learned my addr 172.21.15.99:0/4252951649 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:46:58.074 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.071+0000 7fe10759e640 1 -- 172.21.15.99:0/4252951649 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fe108100710 msgr2=0x7fe108069a10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:46:58.074 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.071+0000 7fe10759e640 1 --2- 172.21.15.99:0/4252951649 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fe108100710 0x7fe108069a10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:58.074 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.071+0000 7fe10759e640 1 -- 172.21.15.99:0/4252951649 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fe108101900 msgr2=0x7fe10819c7c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:46:58.074 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.071+0000 7fe10759e640 1 --2- 172.21.15.99:0/4252951649 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fe108101900 0x7fe10819c7c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:58.074 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.071+0000 7fe10759e640 1 -- 172.21.15.99:0/4252951649 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fe10819f6f0 con 0x7fe1081022c0 2024-03-19T01:46:58.075 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.071+0000 7fe10659c640 1 --2- 172.21.15.99:0/4252951649 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fe108101900 0x7fe10819c7c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:46:58.075 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.071+0000 7fe10759e640 1 --2- 172.21.15.99:0/4252951649 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe1081022c0 0x7fe10819ede0 secure :-1 s=READY pgs=459 cs=0 l=1 rev1=1 crypto rx=0x7fe100009fd0 tx=0x7fe10000c800 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:46:58.075 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.072+0000 7fe0effff640 1 -- 172.21.15.99:0/4252951649 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe100004b20 con 0x7fe1081022c0 2024-03-19T01:46:58.075 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.072+0000 7fe0effff640 1 -- 172.21.15.99:0/4252951649 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe100003e50 con 0x7fe1081022c0 2024-03-19T01:46:58.075 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.072+0000 7fe10d061640 1 -- 172.21.15.99:0/4252951649 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fe10819f980 con 0x7fe1081022c0 2024-03-19T01:46:58.075 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.072+0000 7fe0effff640 1 -- 172.21.15.99:0/4252951649 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe10000b630 con 0x7fe1081022c0 2024-03-19T01:46:58.075 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.072+0000 7fe10d061640 1 -- 172.21.15.99:0/4252951649 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fe10819fea0 con 0x7fe1081022c0 2024-03-19T01:46:58.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.074+0000 7fe0effff640 1 -- 172.21.15.99:0/4252951649 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7fe10001d020 con 0x7fe1081022c0 2024-03-19T01:46:58.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.074+0000 7fe10fc54740 1 Processor -- start 2024-03-19T01:46:58.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.074+0000 7fe10fc54740 1 -- start start 2024-03-19T01:46:58.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.074+0000 7fe10fc54740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55bb5da36100 0x55bb5da36520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:58.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.074+0000 7fe10fc54740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55bb5da36af0 0x55bb5da3b5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:58.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.074+0000 7fe10fc54740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55bb5da3bc60 0x55bb5da3e0f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:58.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.074+0000 7fe10fc54740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55bb5d99abe0 con 0x55bb5da3bc60 2024-03-19T01:46:58.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.074+0000 7fe10fc54740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55bb5d943960 con 0x55bb5da36af0 2024-03-19T01:46:58.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.074+0000 7fe10fc54740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55bb5d996c80 con 0x55bb5da36100 2024-03-19T01:46:58.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.075+0000 7fe10659c640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55bb5da36100 0x55bb5da36520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:58.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.075+0000 7fe106d9d640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55bb5da36af0 0x55bb5da3b5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:58.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.075+0000 7fe0effff640 1 --2- 172.21.15.99:0/4252951649 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fe0f00a28a0 0x7fe0f00a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:58.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.075+0000 7fe10659c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55bb5da3bc60 0x55bb5da3e0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:58.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.075+0000 7fe106d9d640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55bb5da36af0 0x55bb5da3b5f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:52676/0 (socket says 172.21.15.99:52676) 2024-03-19T01:46:58.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.075+0000 7fe106d9d640 1 -- 172.21.15.99:0/1627618328 learned_addr learned my addr 172.21.15.99:0/1627618328 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:46:58.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.075+0000 7fe10759e640 1 --2- 172.21.15.99:0/4252951649 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fe0f00a28a0 0x7fe0f00a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:58.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.075+0000 7fe10659c640 1 -- 172.21.15.99:0/1627618328 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55bb5da36100 msgr2=0x55bb5da36520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:46:58.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.075+0000 7fe10659c640 1 --2- 172.21.15.99:0/1627618328 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55bb5da36100 0x55bb5da36520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:58.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.075+0000 7fe10659c640 1 -- 172.21.15.99:0/1627618328 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55bb5da36af0 msgr2=0x55bb5da3b5f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:46:58.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.075+0000 7fe10659c640 1 --2- 172.21.15.99:0/1627618328 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55bb5da36af0 0x55bb5da3b5f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:58.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.075+0000 7fe0effff640 1 -- 172.21.15.99:0/4252951649 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7fe1000cea10 con 0x7fe1081022c0 2024-03-19T01:46:58.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.075+0000 7fe10659c640 1 -- 172.21.15.99:0/1627618328 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55bb5d93ad20 con 0x55bb5da3bc60 2024-03-19T01:46:58.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.075+0000 7fe10659c640 1 --2- 172.21.15.99:0/1627618328 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55bb5da3bc60 0x55bb5da3e0f0 secure :-1 s=READY pgs=460 cs=0 l=1 rev1=1 crypto rx=0x7fe0fc0349e0 tx=0x7fe0fc034ad0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4a894f924f6a0941 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:46:58.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.075+0000 7fe10759e640 1 --2- 172.21.15.99:0/4252951649 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fe0f00a28a0 0x7fe0f00a4d60 secure :-1 s=READY pgs=274 cs=0 l=1 rev1=1 crypto rx=0x7fe10009b070 tx=0x7fe100004310 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:46:58.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.075+0000 7fe0ed7fa640 1 -- 172.21.15.99:0/1627618328 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe0fc04a070 con 0x55bb5da3bc60 2024-03-19T01:46:58.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.075+0000 7fe0ed7fa640 1 -- 172.21.15.99:0/1627618328 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe0fc02c510 con 0x55bb5da3bc60 2024-03-19T01:46:58.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.075+0000 7fe0ed7fa640 1 -- 172.21.15.99:0/1627618328 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe0fc039040 con 0x55bb5da3bc60 2024-03-19T01:46:58.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.076+0000 7fe10fc54740 1 -- 172.21.15.99:0/1627618328 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55bb5da3bc60 msgr2=0x55bb5da3e0f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:46:58.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.076+0000 7fe10fc54740 1 --2- 172.21.15.99:0/1627618328 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55bb5da3bc60 0x55bb5da3e0f0 secure :-1 s=READY pgs=460 cs=0 l=1 rev1=1 crypto rx=0x7fe0fc0349e0 tx=0x7fe0fc034ad0 comp rx=0 tx=0).stop 2024-03-19T01:46:58.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.076+0000 7fe10fc54740 1 -- 172.21.15.99:0/1627618328 shutdown_connections 2024-03-19T01:46:58.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.076+0000 7fe10fc54740 1 --2- 172.21.15.99:0/1627618328 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55bb5da36af0 0x55bb5da3b5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:58.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.076+0000 7fe10fc54740 1 --2- 172.21.15.99:0/1627618328 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55bb5da3bc60 0x55bb5da3e0f0 unknown :-1 s=CLOSED pgs=460 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:58.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.076+0000 7fe10fc54740 1 --2- 172.21.15.99:0/1627618328 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55bb5da36100 0x55bb5da36520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:58.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.076+0000 7fe10fc54740 1 -- 172.21.15.99:0/1627618328 >> 172.21.15.99:0/1627618328 conn(0x55bb5da31670 msgr2=0x55bb5da31a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:46:58.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.076+0000 7fe10fc54740 1 -- 172.21.15.99:0/1627618328 shutdown_connections 2024-03-19T01:46:58.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.076+0000 7fe10fc54740 1 -- 172.21.15.99:0/1627618328 wait complete. 2024-03-19T01:46:58.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.076+0000 7fe10fc54740 1 -- start start 2024-03-19T01:46:58.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.076+0000 7fe10fc54740 1 Processor -- start 2024-03-19T01:46:58.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.077+0000 7fe10fc54740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55bb5da36100 0x55bb5da400e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:58.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.077+0000 7fe10fc54740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55bb5da36af0 0x55bb5da40620 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:58.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.077+0000 7fe10fc54740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55bb5da3bc60 0x55bb5da47660 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:58.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.077+0000 7fe10fc54740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55bb5da3ebb0 con 0x55bb5da36af0 2024-03-19T01:46:58.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.077+0000 7fe10fc54740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55bb5d956db0 con 0x55bb5da36100 2024-03-19T01:46:58.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.077+0000 7fe10fc54740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55bb5da3ed30 con 0x55bb5da3bc60 2024-03-19T01:46:58.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.077+0000 7fe106d9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55bb5da36af0 0x55bb5da40620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:58.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.077+0000 7fe10659c640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55bb5da36100 0x55bb5da400e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:58.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.077+0000 7fe106d9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55bb5da36af0 0x55bb5da40620 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:33790/0 (socket says 172.21.15.99:33790) 2024-03-19T01:46:58.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.077+0000 7fe10fc54740 10 client.0 _ll_register_callbacks cb 0x55bb5d95be70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:46:58.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.077+0000 7fe106d9d640 1 -- 172.21.15.99:0/3399312116 learned_addr learned my addr 172.21.15.99:0/3399312116 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:46:58.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.077+0000 7fe10659c640 1 --2- 172.21.15.99:0/3399312116 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55bb5da3bc60 0x55bb5da47660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:58.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.077+0000 7fe106d9d640 1 -- 172.21.15.99:0/3399312116 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55bb5da3bc60 msgr2=0x55bb5da47660 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:46:58.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.077+0000 7fe106d9d640 1 --2- 172.21.15.99:0/3399312116 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55bb5da3bc60 0x55bb5da47660 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:58.086 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.077+0000 7fe106d9d640 1 -- 172.21.15.99:0/3399312116 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55bb5da36100 msgr2=0x55bb5da400e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:46:58.086 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.077+0000 7fe106d9d640 1 --2- 172.21.15.99:0/3399312116 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55bb5da36100 0x55bb5da400e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:58.086 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.077+0000 7fe106d9d640 1 -- 172.21.15.99:0/3399312116 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55bb5d955c60 con 0x55bb5da36af0 2024-03-19T01:46:58.086 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.077+0000 7fe10659c640 1 --2- 172.21.15.99:0/3399312116 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55bb5da36100 0x55bb5da400e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:46:58.086 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.077+0000 7fe106d9d640 1 --2- 172.21.15.99:0/3399312116 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55bb5da36af0 0x55bb5da40620 secure :-1 s=READY pgs=461 cs=0 l=1 rev1=1 crypto rx=0x7fe0f8012700 tx=0x7fe0f8012bc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=997ec9c33fe7c601 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:46:58.086 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.077+0000 7fe0eaff5640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:46:58.086 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.077+0000 7fe0eaff5640 1 -- 172.21.15.99:0/3399312116 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe0f8004050 con 0x55bb5da36af0 2024-03-19T01:46:58.086 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.077+0000 7fe0eaff5640 1 -- 172.21.15.99:0/3399312116 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7fe0f80041f0 con 0x55bb5da36af0 2024-03-19T01:46:58.086 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.077+0000 7fe10fc54740 1 -- 172.21.15.99:0/3399312116 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fe0fc02c510 con 0x55bb5da36af0 2024-03-19T01:46:58.086 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.078+0000 7fe0eaff5640 1 -- 172.21.15.99:0/3399312116 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7fe0f80043f0 con 0x55bb5da36af0 2024-03-19T01:46:58.086 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.078+0000 7fe10fc54740 10 client.15810 fetch_fsmap learned FSMap version 27 2024-03-19T01:46:58.086 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.078+0000 7fe10fc54740 1 -- 172.21.15.99:0/3399312116 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x55bb5d6af210 con 0x55bb5da36af0 2024-03-19T01:46:58.086 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.078+0000 7fe0eaff5640 1 -- 172.21.15.99:0/3399312116 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe0fc02c510 con 0x55bb5da36af0 2024-03-19T01:46:58.086 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.078+0000 7fe0eaff5640 1 -- 172.21.15.99:0/3399312116 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 27) ==== 8338+0+0 (secure 0 0 0) 0x7fe0f8003c00 con 0x55bb5da36af0 2024-03-19T01:46:58.087 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.078+0000 7fe10fc54740 10 client.15810 fetch_fsmap finished waiting for FSMap version 27 2024-03-19T01:46:58.087 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.078+0000 7fe10fc54740 10 client.15810 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:46:58.087 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.078+0000 7fe10fc54740 20 client.15810 populate_metadata read hostname 'smithi099' 2024-03-19T01:46:58.087 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.078+0000 7fe10fc54740 1 --2- 172.21.15.99:0/3399312116 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55bb5da4bfd0 0x55bb5da4e450 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:58.087 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.078+0000 7fe10fc54740 4 client.15810 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:46:58.087 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.078+0000 7fe10fc54740 1 -- 172.21.15.99:0/3399312116 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55bb5d9a8120 con 0x55bb5da4bfd0 2024-03-19T01:46:58.087 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.078+0000 7fe10659c640 1 --2- 172.21.15.99:0/3399312116 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55bb5da4bfd0 0x55bb5da4e450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:58.087 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.079+0000 7fe10659c640 1 --2- 172.21.15.99:0/3399312116 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55bb5da4bfd0 0x55bb5da4e450 crc :-1 s=READY pgs=81 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=9014d8c750f71f81 server_cookie=c924ad4569480567 in_seq=0 out_seq=0 2024-03-19T01:46:58.087 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.079+0000 7fe0eaff5640 10 client.15810 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:46:58.087 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.079+0000 7fe0eaff5640 1 -- 172.21.15.99:0/3399312116 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x55bb5d9a8120 con 0x55bb5da4bfd0 2024-03-19T01:46:58.087 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.079+0000 7fe0eaff5640 10 client.15810 handle_command_reply: tid=0 2024-03-19T01:46:58.087 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.079+0000 7fe10fc54740 1 client.15810 shutdown 2024-03-19T01:46:58.087 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.079+0000 7fe10fc54740 1 -- 172.21.15.99:0/3399312116 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55bb5da36af0 msgr2=0x55bb5da40620 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:46:58.087 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.079+0000 7fe10fc54740 1 --2- 172.21.15.99:0/3399312116 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55bb5da36af0 0x55bb5da40620 secure :-1 s=READY pgs=461 cs=0 l=1 rev1=1 crypto rx=0x7fe0f8012700 tx=0x7fe0f8012bc0 comp rx=0 tx=0).stop 2024-03-19T01:46:58.088 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.079+0000 7fe10fc54740 1 -- 172.21.15.99:0/3399312116 shutdown_connections 2024-03-19T01:46:58.088 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.079+0000 7fe10fc54740 1 --2- 172.21.15.99:0/3399312116 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55bb5da4bfd0 0x55bb5da4e450 crc :-1 s=READY pgs=81 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:58.088 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.079+0000 7fe10fc54740 1 --2- 172.21.15.99:0/3399312116 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55bb5da3bc60 0x55bb5da47660 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:58.088 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.079+0000 7fe0eaff5640 0 client.15810 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:46:58.088 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.079+0000 7fe10fc54740 1 --2- 172.21.15.99:0/3399312116 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55bb5da36af0 0x55bb5da40620 unknown :-1 s=CLOSED pgs=461 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:58.088 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.079+0000 7fe10fc54740 1 --2- 172.21.15.99:0/3399312116 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55bb5da36100 0x55bb5da400e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:58.088 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.079+0000 7fe10fc54740 1 -- 172.21.15.99:0/3399312116 >> 172.21.15.99:0/3399312116 conn(0x55bb5da31670 msgr2=0x55bb5da328c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:46:58.088 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.079+0000 7fe10fc54740 1 -- 172.21.15.99:0/3399312116 shutdown_connections 2024-03-19T01:46:58.088 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.079+0000 7fe10fc54740 1 -- 172.21.15.99:0/3399312116 wait complete. 2024-03-19T01:46:58.088 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.080+0000 7fe10fc54740 20 client.15810 trim_cache size 0 max 16384 2024-03-19T01:46:58.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.094+0000 7fe10fc54740 1 Processor -- start 2024-03-19T01:46:58.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.094+0000 7fe10fc54740 1 -- start start 2024-03-19T01:46:58.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.094+0000 7fe10fc54740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55bb5da36100 0x55bb5da37510 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:58.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.094+0000 7fe10fc54740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55bb5da36af0 0x55bb5da47620 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:58.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.094+0000 7fe10fc54740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55bb5da3bc60 0x55bb5da37a50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:58.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.094+0000 7fe10fc54740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fe1081ab5f0 con 0x55bb5da36100 2024-03-19T01:46:58.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.094+0000 7fe10fc54740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fe1081a8ec0 con 0x55bb5da36af0 2024-03-19T01:46:58.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.094+0000 7fe10fc54740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fe1081a6d10 con 0x55bb5da3bc60 2024-03-19T01:46:58.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.094+0000 7fe10659c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55bb5da36100 0x55bb5da37510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:58.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.094+0000 7fe106d9d640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55bb5da36af0 0x55bb5da47620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:58.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.094+0000 7fe106d9d640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55bb5da36af0 0x55bb5da47620 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:52694/0 (socket says 172.21.15.99:52694) 2024-03-19T01:46:58.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.094+0000 7fe106d9d640 1 -- 172.21.15.99:0/1206832156 learned_addr learned my addr 172.21.15.99:0/1206832156 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:46:58.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.094+0000 7fe10659c640 1 --2- 172.21.15.99:0/1206832156 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55bb5da3bc60 0x55bb5da37a50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:58.100 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.094+0000 7fe106d9d640 1 -- 172.21.15.99:0/1206832156 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55bb5da3bc60 msgr2=0x55bb5da37a50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:46:58.100 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.094+0000 7fe106d9d640 1 --2- 172.21.15.99:0/1206832156 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55bb5da3bc60 0x55bb5da37a50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:58.100 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.094+0000 7fe106d9d640 1 -- 172.21.15.99:0/1206832156 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55bb5da36100 msgr2=0x55bb5da37510 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:46:58.100 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.094+0000 7fe106d9d640 1 --2- 172.21.15.99:0/1206832156 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55bb5da36100 0x55bb5da37510 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:58.100 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.094+0000 7fe106d9d640 1 -- 172.21.15.99:0/1206832156 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55bb5d77b620 con 0x55bb5da36af0 2024-03-19T01:46:58.100 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.095+0000 7fe106d9d640 1 --2- 172.21.15.99:0/1206832156 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55bb5da36af0 0x55bb5da47620 secure :-1 s=READY pgs=125 cs=0 l=1 rev1=1 crypto rx=0x7fe0f8000c00 tx=0x7fe0f8010950 comp rx=0 tx=0).ready entity=mon.2 client_cookie=83270de4e568439f server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:46:58.100 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.095+0000 7fe0ecff9640 1 -- 172.21.15.99:0/1206832156 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe0f8010b00 con 0x55bb5da36af0 2024-03-19T01:46:58.100 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.095+0000 7fe0ecff9640 1 -- 172.21.15.99:0/1206832156 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe0f8022030 con 0x55bb5da36af0 2024-03-19T01:46:58.100 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.095+0000 7fe0ecff9640 1 -- 172.21.15.99:0/1206832156 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe0f800fc90 con 0x55bb5da36af0 2024-03-19T01:46:58.100 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.096+0000 7fe10fc54740 1 -- 172.21.15.99:0/1206832156 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55bb5da36af0 msgr2=0x55bb5da47620 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:46:58.100 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.096+0000 7fe10fc54740 1 --2- 172.21.15.99:0/1206832156 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55bb5da36af0 0x55bb5da47620 secure :-1 s=READY pgs=125 cs=0 l=1 rev1=1 crypto rx=0x7fe0f8000c00 tx=0x7fe0f8010950 comp rx=0 tx=0).stop 2024-03-19T01:46:58.100 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.096+0000 7fe10fc54740 1 -- 172.21.15.99:0/1206832156 shutdown_connections 2024-03-19T01:46:58.100 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.096+0000 7fe10fc54740 1 --2- 172.21.15.99:0/1206832156 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55bb5da36af0 0x55bb5da47620 unknown :-1 s=CLOSED pgs=125 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:58.100 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.096+0000 7fe10fc54740 1 --2- 172.21.15.99:0/1206832156 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55bb5da3bc60 0x55bb5da37a50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:58.101 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.096+0000 7fe10fc54740 1 --2- 172.21.15.99:0/1206832156 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55bb5da36100 0x55bb5da37510 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:58.101 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.096+0000 7fe10fc54740 1 -- 172.21.15.99:0/1206832156 >> 172.21.15.99:0/1206832156 conn(0x55bb5da31670 msgr2=0x55bb5da30220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:46:58.101 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.096+0000 7fe10fc54740 1 -- 172.21.15.99:0/1206832156 shutdown_connections 2024-03-19T01:46:58.101 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.096+0000 7fe10fc54740 1 -- 172.21.15.99:0/1206832156 wait complete. 2024-03-19T01:46:58.101 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.096+0000 7fe10fc54740 1 -- start start 2024-03-19T01:46:58.101 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.096+0000 7fe10fc54740 1 Processor -- start 2024-03-19T01:46:58.101 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.096+0000 7fe10fc54740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55bb5da36100 0x55bb5da45140 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:58.101 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.096+0000 7fe10fc54740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55bb5da36af0 0x55bb5da45680 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:58.101 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.096+0000 7fe10fc54740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55bb5da3bc60 0x55bb5da46690 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:58.101 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.096+0000 7fe10fc54740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fe1081a7380 con 0x55bb5da36af0 2024-03-19T01:46:58.101 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.096+0000 7fe10fc54740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fe1081a87f0 con 0x55bb5da36100 2024-03-19T01:46:58.101 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.096+0000 7fe10fc54740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fe1081a7a50 con 0x55bb5da3bc60 2024-03-19T01:46:58.101 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.096+0000 7fe10fc54740 10 client.0 _ll_register_callbacks cb 0x7fe0f8009eb0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:46:58.101 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.096+0000 7fe106d9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55bb5da36af0 0x55bb5da45680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:58.102 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.096+0000 7fe106d9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55bb5da36af0 0x55bb5da45680 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:33814/0 (socket says 172.21.15.99:33814) 2024-03-19T01:46:58.102 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.096+0000 7fe106d9d640 1 -- 172.21.15.99:0/1054912888 learned_addr learned my addr 172.21.15.99:0/1054912888 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:46:58.102 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.096+0000 7fe10659c640 1 --2- 172.21.15.99:0/1054912888 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55bb5da36100 0x55bb5da45140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:58.102 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.096+0000 7fe10659c640 1 --2- 172.21.15.99:0/1054912888 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55bb5da3bc60 0x55bb5da46690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:58.102 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.096+0000 7fe106d9d640 1 -- 172.21.15.99:0/1054912888 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55bb5da3bc60 msgr2=0x55bb5da46690 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:46:58.102 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.096+0000 7fe106d9d640 1 --2- 172.21.15.99:0/1054912888 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55bb5da3bc60 0x55bb5da46690 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:58.102 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.096+0000 7fe106d9d640 1 -- 172.21.15.99:0/1054912888 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55bb5da36100 msgr2=0x55bb5da45140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:46:58.102 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.096+0000 7fe106d9d640 1 --2- 172.21.15.99:0/1054912888 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55bb5da36100 0x55bb5da45140 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:58.102 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.096+0000 7fe106d9d640 1 -- 172.21.15.99:0/1054912888 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55bb5da46c80 con 0x55bb5da36af0 2024-03-19T01:46:58.102 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.096+0000 7fe10659c640 1 --2- 172.21.15.99:0/1054912888 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55bb5da3bc60 0x55bb5da46690 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-19T01:46:58.102 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.097+0000 7fe106d9d640 1 --2- 172.21.15.99:0/1054912888 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55bb5da36af0 0x55bb5da45680 secure :-1 s=READY pgs=462 cs=0 l=1 rev1=1 crypto rx=0x7fe0f8012420 tx=0x7fe0f8005f90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4881fe487ab3eb6e server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:46:58.102 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.097+0000 7fe0eaff5640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:46:58.103 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.097+0000 7fe0eaff5640 1 -- 172.21.15.99:0/1054912888 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe0f8034060 con 0x55bb5da36af0 2024-03-19T01:46:58.103 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.097+0000 7fe0eaff5640 1 -- 172.21.15.99:0/1054912888 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7fe0f800fc20 con 0x55bb5da36af0 2024-03-19T01:46:58.103 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.097+0000 7fe10fc54740 1 -- 172.21.15.99:0/1054912888 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fe0f8022030 con 0x55bb5da36af0 2024-03-19T01:46:58.103 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.097+0000 7fe0eaff5640 1 -- 172.21.15.99:0/1054912888 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7fe0f800fe40 con 0x55bb5da36af0 2024-03-19T01:46:58.103 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.097+0000 7fe10fc54740 10 client.15816 fetch_fsmap learned FSMap version 27 2024-03-19T01:46:58.103 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.097+0000 7fe10fc54740 1 -- 172.21.15.99:0/1054912888 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x55bb5dab26c0 con 0x55bb5da36af0 2024-03-19T01:46:58.103 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.097+0000 7fe0eaff5640 1 -- 172.21.15.99:0/1054912888 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe0f8022030 con 0x55bb5da36af0 2024-03-19T01:46:58.103 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.097+0000 7fe0eaff5640 1 -- 172.21.15.99:0/1054912888 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 27) ==== 8338+0+0 (secure 0 0 0) 0x7fe0f802e030 con 0x55bb5da36af0 2024-03-19T01:46:58.103 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.097+0000 7fe10fc54740 10 client.15816 fetch_fsmap finished waiting for FSMap version 27 2024-03-19T01:46:58.103 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.097+0000 7fe10fc54740 10 client.15816 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:46:58.103 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.097+0000 7fe10fc54740 20 client.15816 populate_metadata read hostname 'smithi099' 2024-03-19T01:46:58.103 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.097+0000 7fe10fc54740 1 --2- 172.21.15.99:0/1054912888 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55bb5dab2fe0 0x55bb5dab5400 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:46:58.103 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.097+0000 7fe10fc54740 4 client.15816 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:46:58.103 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.097+0000 7fe10fc54740 1 -- 172.21.15.99:0/1054912888 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55bb5d9a2470 con 0x55bb5dab2fe0 2024-03-19T01:46:58.103 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.097+0000 7fe10659c640 1 --2- 172.21.15.99:0/1054912888 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55bb5dab2fe0 0x55bb5dab5400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:46:58.104 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.098+0000 7fe0eaff5640 10 client.15816 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:46:58.104 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.098+0000 7fe10659c640 1 --2- 172.21.15.99:0/1054912888 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55bb5dab2fe0 0x55bb5dab5400 crc :-1 s=READY pgs=82 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=e5d2c089c775c21f server_cookie=cc2962d4318bebf in_seq=0 out_seq=0 2024-03-19T01:46:58.151 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.150+0000 7fe0eaff5640 1 -- 172.21.15.99:0/1054912888 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x55bb5d9a2470 con 0x55bb5dab2fe0 2024-03-19T01:46:58.151 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.150+0000 7fe0eaff5640 10 client.15816 handle_command_reply: tid=0 2024-03-19T01:46:58.151 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.150+0000 7fe10fc54740 1 client.15816 shutdown 2024-03-19T01:46:58.151 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.150+0000 7fe10fc54740 1 -- 172.21.15.99:0/1054912888 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55bb5da36af0 msgr2=0x55bb5da45680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:46:58.151 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.150+0000 7fe10fc54740 1 --2- 172.21.15.99:0/1054912888 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55bb5da36af0 0x55bb5da45680 secure :-1 s=READY pgs=462 cs=0 l=1 rev1=1 crypto rx=0x7fe0f8012420 tx=0x7fe0f8005f90 comp rx=0 tx=0).stop 2024-03-19T01:46:58.151 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.150+0000 7fe10fc54740 1 -- 172.21.15.99:0/1054912888 shutdown_connections 2024-03-19T01:46:58.151 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.150+0000 7fe10fc54740 1 --2- 172.21.15.99:0/1054912888 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55bb5dab2fe0 0x55bb5dab5400 crc :-1 s=READY pgs=82 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:58.151 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.150+0000 7fe10fc54740 1 --2- 172.21.15.99:0/1054912888 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55bb5da3bc60 0x55bb5da46690 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:58.151 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.150+0000 7fe10fc54740 1 --2- 172.21.15.99:0/1054912888 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55bb5da36af0 0x55bb5da45680 unknown :-1 s=CLOSED pgs=462 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:58.151 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.150+0000 7fe0eaff5640 0 client.15816 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:46:58.152 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.150+0000 7fe10fc54740 1 --2- 172.21.15.99:0/1054912888 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55bb5da36100 0x55bb5da45140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:58.152 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.150+0000 7fe10fc54740 1 -- 172.21.15.99:0/1054912888 >> 172.21.15.99:0/1054912888 conn(0x55bb5da31670 msgr2=0x55bb5da348d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:46:58.152 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.151+0000 7fe10fc54740 1 -- 172.21.15.99:0/1054912888 shutdown_connections 2024-03-19T01:46:58.152 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.151+0000 7fe10fc54740 1 -- 172.21.15.99:0/1054912888 wait complete. 2024-03-19T01:46:58.152 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.151+0000 7fe10fc54740 20 client.15816 trim_cache size 0 max 16384 2024-03-19T01:46:58.152 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:46:58.152 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:46:58.153 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:46:58.153 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:46:58.154 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.151+0000 7fe10d061640 1 -- 172.21.15.99:0/4252951649 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fe0f00a28a0 msgr2=0x7fe0f00a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:46:58.154 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.151+0000 7fe10d061640 1 --2- 172.21.15.99:0/4252951649 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fe0f00a28a0 0x7fe0f00a4d60 secure :-1 s=READY pgs=274 cs=0 l=1 rev1=1 crypto rx=0x7fe10009b070 tx=0x7fe100004310 comp rx=0 tx=0).stop 2024-03-19T01:46:58.154 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.151+0000 7fe10d061640 1 -- 172.21.15.99:0/4252951649 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe1081022c0 msgr2=0x7fe10819ede0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:46:58.154 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.151+0000 7fe10d061640 1 --2- 172.21.15.99:0/4252951649 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe1081022c0 0x7fe10819ede0 secure :-1 s=READY pgs=459 cs=0 l=1 rev1=1 crypto rx=0x7fe100009fd0 tx=0x7fe10000c800 comp rx=0 tx=0).stop 2024-03-19T01:46:58.154 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.151+0000 7fe10d061640 1 -- 172.21.15.99:0/4252951649 shutdown_connections 2024-03-19T01:46:58.154 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.151+0000 7fe10d061640 1 --2- 172.21.15.99:0/4252951649 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fe0f00a28a0 0x7fe0f00a4d60 unknown :-1 s=CLOSED pgs=274 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:58.155 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.151+0000 7fe10d061640 1 --2- 172.21.15.99:0/4252951649 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fe108101900 0x7fe10819c7c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:58.155 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.151+0000 7fe10d061640 1 --2- 172.21.15.99:0/4252951649 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe1081022c0 0x7fe10819ede0 unknown :-1 s=CLOSED pgs=459 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:58.155 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.151+0000 7fe10d061640 1 --2- 172.21.15.99:0/4252951649 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fe108100710 0x7fe108069a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:46:58.155 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.151+0000 7fe10d061640 1 -- 172.21.15.99:0/4252951649 >> 172.21.15.99:0/4252951649 conn(0x7fe1080fbc80 msgr2=0x7fe108104c00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:46:58.155 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.151+0000 7fe10d061640 1 -- 172.21.15.99:0/4252951649 shutdown_connections 2024-03-19T01:46:58.155 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:46:58.151+0000 7fe10d061640 1 -- 172.21.15.99:0/4252951649 wait complete. 2024-03-19T01:46:58.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:46:57 smithi115 ceph-mon[67332]: pgmap v526: 97 pgs: 97 active+clean; 221 MiB data, 2.1 GiB used, 1.0 TiB / 1.0 TiB avail; 942 KiB/s rd, 3.3 MiB/s wr, 286 op/s 2024-03-19T01:46:58.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:46:57 smithi115 ceph-mon[67332]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T01:46:58.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:46:57 smithi099 ceph-mon[62951]: pgmap v526: 97 pgs: 97 active+clean; 221 MiB data, 2.1 GiB used, 1.0 TiB / 1.0 TiB avail; 942 KiB/s rd, 3.3 MiB/s wr, 286 op/s 2024-03-19T01:46:58.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:46:57 smithi099 ceph-mon[62951]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T01:46:58.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:46:57 smithi112 ceph-mon[66819]: pgmap v526: 97 pgs: 97 active+clean; 221 MiB data, 2.1 GiB used, 1.0 TiB / 1.0 TiB avail; 942 KiB/s rd, 3.3 MiB/s wr, 286 op/s 2024-03-19T01:46:58.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:46:57 smithi112 ceph-mon[66819]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T01:46:58.687 INFO:tasks.workunit.client.0.smithi099.stdout: 1 696266 11.28 MB/sec execute 208 sec latency 77.915 ms 2024-03-19T01:46:59.687 INFO:tasks.workunit.client.0.smithi099.stdout: 1 697901 11.26 MB/sec execute 209 sec latency 23.358 ms 2024-03-19T01:47:00.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:46:59 smithi115 ceph-mon[67332]: pgmap v527: 97 pgs: 97 active+clean; 222 MiB data, 2.1 GiB used, 1.0 TiB / 1.0 TiB avail; 870 KiB/s rd, 2.9 MiB/s wr, 260 op/s 2024-03-19T01:47:00.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:46:59 smithi099 ceph-mon[62951]: pgmap v527: 97 pgs: 97 active+clean; 222 MiB data, 2.1 GiB used, 1.0 TiB / 1.0 TiB avail; 870 KiB/s rd, 2.9 MiB/s wr, 260 op/s 2024-03-19T01:47:00.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:46:59 smithi112 ceph-mon[66819]: pgmap v527: 97 pgs: 97 active+clean; 222 MiB data, 2.1 GiB used, 1.0 TiB / 1.0 TiB avail; 870 KiB/s rd, 2.9 MiB/s wr, 260 op/s 2024-03-19T01:47:00.687 INFO:tasks.workunit.client.0.smithi099.stdout: 1 700139 11.27 MB/sec execute 210 sec latency 11.325 ms 2024-03-19T01:47:01.687 INFO:tasks.workunit.client.0.smithi099.stdout: 1 702502 11.27 MB/sec execute 211 sec latency 33.517 ms 2024-03-19T01:47:02.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:47:01 smithi115 ceph-mon[67332]: pgmap v528: 97 pgs: 97 active+clean; 225 MiB data, 2.1 GiB used, 1.0 TiB / 1.0 TiB avail; 891 KiB/s rd, 2.9 MiB/s wr, 259 op/s 2024-03-19T01:47:02.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:47:01 smithi099 ceph-mon[62951]: pgmap v528: 97 pgs: 97 active+clean; 225 MiB data, 2.1 GiB used, 1.0 TiB / 1.0 TiB avail; 891 KiB/s rd, 2.9 MiB/s wr, 259 op/s 2024-03-19T01:47:02.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:47:01 smithi112 ceph-mon[66819]: pgmap v528: 97 pgs: 97 active+clean; 225 MiB data, 2.1 GiB used, 1.0 TiB / 1.0 TiB avail; 891 KiB/s rd, 2.9 MiB/s wr, 259 op/s 2024-03-19T01:47:02.687 INFO:tasks.workunit.client.0.smithi099.stdout: 1 704285 11.26 MB/sec execute 212 sec latency 26.856 ms 2024-03-19T01:47:03.687 INFO:tasks.workunit.client.0.smithi099.stdout: 1 706748 11.28 MB/sec execute 213 sec latency 77.945 ms 2024-03-19T01:47:04.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:47:03 smithi115 ceph-mon[67332]: pgmap v529: 97 pgs: 97 active+clean; 225 MiB data, 2.1 GiB used, 1.0 TiB / 1.0 TiB avail; 922 KiB/s rd, 3.0 MiB/s wr, 270 op/s 2024-03-19T01:47:04.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:47:03 smithi099 ceph-mon[62951]: pgmap v529: 97 pgs: 97 active+clean; 225 MiB data, 2.1 GiB used, 1.0 TiB / 1.0 TiB avail; 922 KiB/s rd, 3.0 MiB/s wr, 270 op/s 2024-03-19T01:47:04.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:47:03 smithi112 ceph-mon[66819]: pgmap v529: 97 pgs: 97 active+clean; 225 MiB data, 2.1 GiB used, 1.0 TiB / 1.0 TiB avail; 922 KiB/s rd, 3.0 MiB/s wr, 270 op/s 2024-03-19T01:47:04.687 INFO:tasks.workunit.client.0.smithi099.stdout: 1 708267 11.26 MB/sec execute 214 sec latency 31.343 ms 2024-03-19T01:47:05.687 INFO:tasks.workunit.client.0.smithi099.stdout: 1 710615 11.27 MB/sec execute 215 sec latency 12.339 ms 2024-03-19T01:47:06.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:47:05 smithi115 ceph-mon[67332]: pgmap v530: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 751 KiB/s rd, 2.7 MiB/s wr, 250 op/s 2024-03-19T01:47:06.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:47:05 smithi099 ceph-mon[62951]: pgmap v530: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 751 KiB/s rd, 2.7 MiB/s wr, 250 op/s 2024-03-19T01:47:06.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:47:05 smithi112 ceph-mon[66819]: pgmap v530: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 751 KiB/s rd, 2.7 MiB/s wr, 250 op/s 2024-03-19T01:47:06.687 INFO:tasks.workunit.client.0.smithi099.stdout: 1 712679 11.25 MB/sec execute 216 sec latency 29.390 ms 2024-03-19T01:47:07.687 INFO:tasks.workunit.client.0.smithi099.stdout: 1 714486 11.26 MB/sec execute 217 sec latency 68.929 ms 2024-03-19T01:47:08.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:47:07 smithi115 ceph-mon[67332]: pgmap v531: 97 pgs: 97 active+clean; 227 MiB data, 2.1 GiB used, 1.0 TiB / 1.0 TiB avail; 1.0 MiB/s rd, 3.4 MiB/s wr, 306 op/s 2024-03-19T01:47:08.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:47:07 smithi099 ceph-mon[62951]: pgmap v531: 97 pgs: 97 active+clean; 227 MiB data, 2.1 GiB used, 1.0 TiB / 1.0 TiB avail; 1.0 MiB/s rd, 3.4 MiB/s wr, 306 op/s 2024-03-19T01:47:08.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:47:07 smithi112 ceph-mon[66819]: pgmap v531: 97 pgs: 97 active+clean; 227 MiB data, 2.1 GiB used, 1.0 TiB / 1.0 TiB avail; 1.0 MiB/s rd, 3.4 MiB/s wr, 306 op/s 2024-03-19T01:47:08.687 INFO:tasks.workunit.client.0.smithi099.stdout: 1 716932 11.27 MB/sec execute 218 sec latency 23.074 ms 2024-03-19T01:47:09.687 INFO:tasks.workunit.client.0.smithi099.stdout: 1 718287 11.25 MB/sec execute 219 sec latency 52.505 ms 2024-03-19T01:47:10.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:47:09 smithi115 ceph-mon[67332]: pgmap v532: 97 pgs: 97 active+clean; 228 MiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 770 KiB/s rd, 2.8 MiB/s wr, 250 op/s 2024-03-19T01:47:10.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:47:09 smithi099 ceph-mon[62951]: pgmap v532: 97 pgs: 97 active+clean; 228 MiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 770 KiB/s rd, 2.8 MiB/s wr, 250 op/s 2024-03-19T01:47:10.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:47:09 smithi112 ceph-mon[66819]: pgmap v532: 97 pgs: 97 active+clean; 228 MiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 770 KiB/s rd, 2.8 MiB/s wr, 250 op/s 2024-03-19T01:47:10.687 INFO:tasks.workunit.client.0.smithi099.stdout: 1 720974 11.27 MB/sec execute 220 sec latency 15.288 ms 2024-03-19T01:47:11.246 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.243+0000 7fd3c57ab640 1 -- 172.21.15.99:0/3785285027 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd3c0102a50 msgr2=0x7fd3c0102ed0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:47:11.246 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.243+0000 7fd3c57ab640 1 --2- 172.21.15.99:0/3785285027 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd3c0102a50 0x7fd3c0102ed0 secure :-1 s=READY pgs=463 cs=0 l=1 rev1=1 crypto rx=0x7fd3a8009900 tx=0x7fd3a802bc30 comp rx=0 tx=0).stop 2024-03-19T01:47:11.246 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.243+0000 7fd3c57ab640 1 -- 172.21.15.99:0/3785285027 shutdown_connections 2024-03-19T01:47:11.246 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.243+0000 7fd3c57ab640 1 --2- 172.21.15.99:0/3785285027 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd3c0103410 0x7fd3c0109ca0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:11.246 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.243+0000 7fd3c57ab640 1 --2- 172.21.15.99:0/3785285027 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd3c0102a50 0x7fd3c0102ed0 unknown :-1 s=CLOSED pgs=463 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:11.246 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.243+0000 7fd3c57ab640 1 --2- 172.21.15.99:0/3785285027 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd3c0101860 0x7fd3c0101c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:11.246 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.244+0000 7fd3c57ab640 1 -- 172.21.15.99:0/3785285027 >> 172.21.15.99:0/3785285027 conn(0x7fd3c00fcdd0 msgr2=0x7fd3c00ff210 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:47:11.246 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.244+0000 7fd3c57ab640 1 -- 172.21.15.99:0/3785285027 shutdown_connections 2024-03-19T01:47:11.246 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.244+0000 7fd3c57ab640 1 -- 172.21.15.99:0/3785285027 wait complete. 2024-03-19T01:47:11.247 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.244+0000 7fd3c57ab640 1 Processor -- start 2024-03-19T01:47:11.247 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.244+0000 7fd3c57ab640 1 -- start start 2024-03-19T01:47:11.247 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.245+0000 7fd3c57ab640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd3c0101860 0x7fd3c0197b20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:11.247 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.245+0000 7fd3c57ab640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd3c0102a50 0x7fd3c0198060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:11.247 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.245+0000 7fd3c57ab640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd3c0103410 0x7fd3c019f0d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:11.247 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.245+0000 7fd3c57ab640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fd3c010d4e0 con 0x7fd3c0101860 2024-03-19T01:47:11.247 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.245+0000 7fd3c57ab640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fd3c010d360 con 0x7fd3c0103410 2024-03-19T01:47:11.247 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.245+0000 7fd3c57ab640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fd3c010d660 con 0x7fd3c0102a50 2024-03-19T01:47:11.247 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.245+0000 7fd3beffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd3c0101860 0x7fd3c0197b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:11.247 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.245+0000 7fd3beffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd3c0101860 0x7fd3c0197b20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:50724/0 (socket says 172.21.15.99:50724) 2024-03-19T01:47:11.247 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.245+0000 7fd3be7fc640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd3c0102a50 0x7fd3c0198060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:11.247 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.245+0000 7fd3beffd640 1 -- 172.21.15.99:0/246341089 learned_addr learned my addr 172.21.15.99:0/246341089 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:47:11.247 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.245+0000 7fd3bf7fe640 1 --2- 172.21.15.99:0/246341089 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd3c0103410 0x7fd3c019f0d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:11.247 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.245+0000 7fd3beffd640 1 -- 172.21.15.99:0/246341089 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd3c0102a50 msgr2=0x7fd3c0198060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:47:11.247 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.245+0000 7fd3beffd640 1 --2- 172.21.15.99:0/246341089 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd3c0102a50 0x7fd3c0198060 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:11.248 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.245+0000 7fd3beffd640 1 -- 172.21.15.99:0/246341089 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd3c0103410 msgr2=0x7fd3c019f0d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:47:11.248 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.245+0000 7fd3beffd640 1 --2- 172.21.15.99:0/246341089 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd3c0103410 0x7fd3c019f0d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:11.248 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.245+0000 7fd3beffd640 1 -- 172.21.15.99:0/246341089 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd3c019f740 con 0x7fd3c0101860 2024-03-19T01:47:11.248 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.245+0000 7fd3be7fc640 1 --2- 172.21.15.99:0/246341089 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd3c0102a50 0x7fd3c0198060 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:47:11.248 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.245+0000 7fd3beffd640 1 --2- 172.21.15.99:0/246341089 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd3c0101860 0x7fd3c0197b20 secure :-1 s=READY pgs=464 cs=0 l=1 rev1=1 crypto rx=0x7fd3b800eb70 tx=0x7fd3b800c6a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:47:11.248 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.245+0000 7fd3b7fff640 1 -- 172.21.15.99:0/246341089 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd3b8003b70 con 0x7fd3c0101860 2024-03-19T01:47:11.248 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.245+0000 7fd3c57ab640 1 -- 172.21.15.99:0/246341089 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd3c019fa30 con 0x7fd3c0101860 2024-03-19T01:47:11.248 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.245+0000 7fd3b7fff640 1 -- 172.21.15.99:0/246341089 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd3b800ee70 con 0x7fd3c0101860 2024-03-19T01:47:11.248 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.245+0000 7fd3b7fff640 1 -- 172.21.15.99:0/246341089 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd3b8010830 con 0x7fd3c0101860 2024-03-19T01:47:11.248 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.245+0000 7fd3c57ab640 1 -- 172.21.15.99:0/246341089 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fd3c019ff50 con 0x7fd3c0101860 2024-03-19T01:47:11.253 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.248+0000 7fd3b7fff640 1 -- 172.21.15.99:0/246341089 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7fd3b8004020 con 0x7fd3c0101860 2024-03-19T01:47:11.253 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.248+0000 7fd3c838b740 1 Processor -- start 2024-03-19T01:47:11.253 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.248+0000 7fd3c838b740 1 -- start start 2024-03-19T01:47:11.253 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.249+0000 7fd3c838b740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563c10ef2100 0x563c10ef2520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:11.254 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.249+0000 7fd3c838b740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563c10ef2af0 0x563c10ef75f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:11.254 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.249+0000 7fd3c838b740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563c10ef7c60 0x563c10efa0f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:11.254 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.249+0000 7fd3c838b740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x563c10e56be0 con 0x563c10ef2af0 2024-03-19T01:47:11.254 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.249+0000 7fd3c838b740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x563c10dff960 con 0x563c10ef2100 2024-03-19T01:47:11.254 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.249+0000 7fd3c838b740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x563c10e52c80 con 0x563c10ef7c60 2024-03-19T01:47:11.254 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.249+0000 7fd3beffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563c10ef2af0 0x563c10ef75f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:11.254 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.249+0000 7fd3bf7fe640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563c10ef2100 0x563c10ef2520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:11.254 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.249+0000 7fd3bf7fe640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563c10ef2100 0x563c10ef2520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:32956/0 (socket says 172.21.15.99:32956) 2024-03-19T01:47:11.254 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.249+0000 7fd3be7fc640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563c10ef7c60 0x563c10efa0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:11.254 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.249+0000 7fd3bf7fe640 1 -- 172.21.15.99:0/3073811641 learned_addr learned my addr 172.21.15.99:0/3073811641 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:47:11.254 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.249+0000 7fd3b7fff640 1 --2- 172.21.15.99:0/246341089 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd3ac0a28a0 0x7fd3ac0a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:11.254 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.249+0000 7fd3beffd640 1 -- 172.21.15.99:0/3073811641 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563c10ef7c60 msgr2=0x563c10efa0f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:47:11.254 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.249+0000 7fd3beffd640 1 --2- 172.21.15.99:0/3073811641 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563c10ef7c60 0x563c10efa0f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:11.255 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.249+0000 7fd3beffd640 1 -- 172.21.15.99:0/3073811641 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563c10ef2100 msgr2=0x563c10ef2520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:47:11.255 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.249+0000 7fd3beffd640 1 --2- 172.21.15.99:0/3073811641 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563c10ef2100 0x563c10ef2520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:11.255 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.249+0000 7fd3beffd640 1 -- 172.21.15.99:0/3073811641 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x563c10df6d20 con 0x563c10ef2af0 2024-03-19T01:47:11.255 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.249+0000 7fd3bf7fe640 1 --2- 172.21.15.99:0/3073811641 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563c10ef2100 0x563c10ef2520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:47:11.255 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.249+0000 7fd3be7fc640 1 --2- 172.21.15.99:0/3073811641 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563c10ef7c60 0x563c10efa0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:47:11.255 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.249+0000 7fd3bf7fe640 1 --2- 172.21.15.99:0/246341089 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd3ac0a28a0 0x7fd3ac0a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:11.255 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.249+0000 7fd3b7fff640 1 -- 172.21.15.99:0/246341089 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7fd3b80cf110 con 0x7fd3c0101860 2024-03-19T01:47:11.255 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.249+0000 7fd3beffd640 1 --2- 172.21.15.99:0/3073811641 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563c10ef2af0 0x563c10ef75f0 secure :-1 s=READY pgs=465 cs=0 l=1 rev1=1 crypto rx=0x7fd3b80ccba0 tx=0x7fd3b8021000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a83e404afce7b072 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:47:11.255 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.250+0000 7fd3bf7fe640 1 --2- 172.21.15.99:0/246341089 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd3ac0a28a0 0x7fd3ac0a4d60 secure :-1 s=READY pgs=279 cs=0 l=1 rev1=1 crypto rx=0x563c10ef3350 tx=0x7fd3b00046c0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:47:11.255 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.250+0000 7fd3b57fa640 1 -- 172.21.15.99:0/3073811641 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd3b809b840 con 0x563c10ef2af0 2024-03-19T01:47:11.255 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.250+0000 7fd3b57fa640 1 -- 172.21.15.99:0/3073811641 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd3b809c170 con 0x563c10ef2af0 2024-03-19T01:47:11.255 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.250+0000 7fd3b57fa640 1 -- 172.21.15.99:0/3073811641 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd3b809b3d0 con 0x563c10ef2af0 2024-03-19T01:47:11.255 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.250+0000 7fd3c838b740 1 -- 172.21.15.99:0/3073811641 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563c10ef2af0 msgr2=0x563c10ef75f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:47:11.255 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.250+0000 7fd3c838b740 1 --2- 172.21.15.99:0/3073811641 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563c10ef2af0 0x563c10ef75f0 secure :-1 s=READY pgs=465 cs=0 l=1 rev1=1 crypto rx=0x7fd3b80ccba0 tx=0x7fd3b8021000 comp rx=0 tx=0).stop 2024-03-19T01:47:11.255 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.250+0000 7fd3c838b740 1 -- 172.21.15.99:0/3073811641 shutdown_connections 2024-03-19T01:47:11.256 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.250+0000 7fd3c838b740 1 --2- 172.21.15.99:0/3073811641 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563c10ef7c60 0x563c10efa0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:11.256 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.250+0000 7fd3c838b740 1 --2- 172.21.15.99:0/3073811641 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563c10ef2af0 0x563c10ef75f0 unknown :-1 s=CLOSED pgs=465 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:11.256 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.250+0000 7fd3c838b740 1 --2- 172.21.15.99:0/3073811641 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563c10ef2100 0x563c10ef2520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:11.256 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.250+0000 7fd3c838b740 1 -- 172.21.15.99:0/3073811641 >> 172.21.15.99:0/3073811641 conn(0x563c10eed670 msgr2=0x563c10eeda70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:47:11.256 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.251+0000 7fd3c838b740 1 -- 172.21.15.99:0/3073811641 shutdown_connections 2024-03-19T01:47:11.256 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.251+0000 7fd3c838b740 1 -- 172.21.15.99:0/3073811641 wait complete. 2024-03-19T01:47:11.256 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.251+0000 7fd3c838b740 1 -- start start 2024-03-19T01:47:11.256 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.251+0000 7fd3c838b740 1 Processor -- start 2024-03-19T01:47:11.256 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.251+0000 7fd3c838b740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563c10ef2100 0x563c10efc020 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:11.256 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.251+0000 7fd3c838b740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563c10ef2af0 0x563c10efc560 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:11.256 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.251+0000 7fd3c838b740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563c10ef7c60 0x563c10f035a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:11.257 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.251+0000 7fd3c838b740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x563c10efabb0 con 0x563c10ef7c60 2024-03-19T01:47:11.257 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.251+0000 7fd3c838b740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x563c10e12db0 con 0x563c10ef2af0 2024-03-19T01:47:11.257 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.251+0000 7fd3c838b740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x563c10efad30 con 0x563c10ef2100 2024-03-19T01:47:11.257 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.251+0000 7fd3c838b740 10 client.0 _ll_register_callbacks cb 0x563c10e17e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:47:11.257 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.251+0000 7fd3beffd640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563c10ef2100 0x563c10efc020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:11.257 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.251+0000 7fd3be7fc640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563c10ef2af0 0x563c10efc560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:11.257 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.251+0000 7fd3bf7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563c10ef7c60 0x563c10f035a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:11.257 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.251+0000 7fd3be7fc640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563c10ef2af0 0x563c10efc560 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:32964/0 (socket says 172.21.15.99:32964) 2024-03-19T01:47:11.257 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.251+0000 7fd3beffd640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563c10ef2100 0x563c10efc020 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:46666/0 (socket says 172.21.15.99:46666) 2024-03-19T01:47:11.257 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.251+0000 7fd3be7fc640 1 -- 172.21.15.99:0/945959800 learned_addr learned my addr 172.21.15.99:0/945959800 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:47:11.257 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.252+0000 7fd3be7fc640 1 -- 172.21.15.99:0/945959800 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563c10ef2100 msgr2=0x563c10efc020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:47:11.257 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.252+0000 7fd3be7fc640 1 --2- 172.21.15.99:0/945959800 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563c10ef2100 0x563c10efc020 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:11.257 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.252+0000 7fd3be7fc640 1 -- 172.21.15.99:0/945959800 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563c10ef7c60 msgr2=0x563c10f035a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:47:11.257 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.252+0000 7fd3be7fc640 1 --2- 172.21.15.99:0/945959800 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563c10ef7c60 0x563c10f035a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:11.257 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.252+0000 7fd3be7fc640 1 -- 172.21.15.99:0/945959800 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x563c10e11c60 con 0x563c10ef2af0 2024-03-19T01:47:11.258 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.252+0000 7fd3bf7fe640 1 --2- 172.21.15.99:0/945959800 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563c10ef7c60 0x563c10f035a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:47:11.258 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.252+0000 7fd3be7fc640 1 --2- 172.21.15.99:0/945959800 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563c10ef2af0 0x563c10efc560 secure :-1 s=READY pgs=126 cs=0 l=1 rev1=1 crypto rx=0x7fd3a8036040 tx=0x7fd3a8005250 comp rx=0 tx=0).ready entity=mon.2 client_cookie=746ec94559067e44 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:47:11.258 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.252+0000 7fd3a67fc640 10 client.0 ms_handle_connect on v2:172.21.15.112:3300/0 2024-03-19T01:47:11.258 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.252+0000 7fd3a67fc640 1 -- 172.21.15.99:0/945959800 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd3a8005d50 con 0x563c10ef2af0 2024-03-19T01:47:11.258 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.252+0000 7fd3a67fc640 1 -- 172.21.15.99:0/945959800 <== mon.2 v2:172.21.15.112:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7fd3a80046c0 con 0x563c10ef2af0 2024-03-19T01:47:11.258 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.252+0000 7fd3c838b740 1 -- 172.21.15.99:0/945959800 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fd3b809c170 con 0x563c10ef2af0 2024-03-19T01:47:11.258 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.252+0000 7fd3a67fc640 1 -- 172.21.15.99:0/945959800 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7fd3a80048c0 con 0x563c10ef2af0 2024-03-19T01:47:11.258 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.252+0000 7fd3c838b740 10 client.25601 fetch_fsmap learned FSMap version 27 2024-03-19T01:47:11.258 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.252+0000 7fd3c838b740 1 -- 172.21.15.99:0/945959800 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x563c10b6b210 con 0x563c10ef2af0 2024-03-19T01:47:11.258 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.252+0000 7fd3a67fc640 1 -- 172.21.15.99:0/945959800 <== mon.2 v2:172.21.15.112:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd3b809c170 con 0x563c10ef2af0 2024-03-19T01:47:11.258 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.253+0000 7fd3a67fc640 1 -- 172.21.15.99:0/945959800 <== mon.2 v2:172.21.15.112:3300/0 5 ==== fsmap(e 27) ==== 8338+0+0 (secure 0 0 0) 0x7fd3a8003b60 con 0x563c10ef2af0 2024-03-19T01:47:11.258 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.253+0000 7fd3c838b740 10 client.25601 fetch_fsmap finished waiting for FSMap version 27 2024-03-19T01:47:11.258 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.253+0000 7fd3c838b740 10 client.25601 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:47:11.258 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.253+0000 7fd3c838b740 20 client.25601 populate_metadata read hostname 'smithi099' 2024-03-19T01:47:11.259 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.253+0000 7fd3c838b740 1 --2- 172.21.15.99:0/945959800 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x563c10f07e90 0x563c10f0a2a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:11.259 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.253+0000 7fd3c838b740 4 client.25601 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:47:11.259 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.253+0000 7fd3c838b740 1 -- 172.21.15.99:0/945959800 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x563c10e64120 con 0x563c10f07e90 2024-03-19T01:47:11.259 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.253+0000 7fd3beffd640 1 --2- 172.21.15.99:0/945959800 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x563c10f07e90 0x563c10f0a2a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:11.259 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.253+0000 7fd3beffd640 1 --2- 172.21.15.99:0/945959800 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x563c10f07e90 0x563c10f0a2a0 crc :-1 s=READY pgs=83 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=7e162441c6f60aa8 server_cookie=f3f9203e0186d850 in_seq=0 out_seq=0 2024-03-19T01:47:11.259 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.253+0000 7fd3a67fc640 10 client.25601 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:47:11.260 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.254+0000 7fd3a67fc640 1 -- 172.21.15.99:0/945959800 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x563c10e64120 con 0x563c10f07e90 2024-03-19T01:47:11.260 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.254+0000 7fd3a67fc640 10 client.25601 handle_command_reply: tid=0 2024-03-19T01:47:11.260 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.254+0000 7fd3c838b740 1 client.25601 shutdown 2024-03-19T01:47:11.260 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.254+0000 7fd3c838b740 1 -- 172.21.15.99:0/945959800 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563c10ef2af0 msgr2=0x563c10efc560 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:47:11.260 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.254+0000 7fd3c838b740 1 --2- 172.21.15.99:0/945959800 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563c10ef2af0 0x563c10efc560 secure :-1 s=READY pgs=126 cs=0 l=1 rev1=1 crypto rx=0x7fd3a8036040 tx=0x7fd3a8005250 comp rx=0 tx=0).stop 2024-03-19T01:47:11.260 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.254+0000 7fd3c838b740 1 -- 172.21.15.99:0/945959800 shutdown_connections 2024-03-19T01:47:11.260 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.254+0000 7fd3c838b740 1 --2- 172.21.15.99:0/945959800 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x563c10f07e90 0x563c10f0a2a0 crc :-1 s=READY pgs=83 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:11.260 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.254+0000 7fd3c838b740 1 --2- 172.21.15.99:0/945959800 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563c10ef2af0 0x563c10efc560 unknown :-1 s=CLOSED pgs=126 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:11.260 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.254+0000 7fd3c838b740 1 --2- 172.21.15.99:0/945959800 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563c10ef7c60 0x563c10f035a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:11.260 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.254+0000 7fd3a67fc640 0 client.25601 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:47:11.261 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.254+0000 7fd3c838b740 1 --2- 172.21.15.99:0/945959800 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563c10ef2100 0x563c10efc020 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:11.261 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.254+0000 7fd3c838b740 1 -- 172.21.15.99:0/945959800 >> 172.21.15.99:0/945959800 conn(0x563c10eed670 msgr2=0x563c10eee8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:47:11.261 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.254+0000 7fd3c838b740 1 -- 172.21.15.99:0/945959800 shutdown_connections 2024-03-19T01:47:11.261 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.254+0000 7fd3c838b740 1 -- 172.21.15.99:0/945959800 wait complete. 2024-03-19T01:47:11.261 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.255+0000 7fd3c838b740 20 client.25601 trim_cache size 0 max 16384 2024-03-19T01:47:11.272 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.271+0000 7fd3c838b740 1 Processor -- start 2024-03-19T01:47:11.272 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.271+0000 7fd3c838b740 1 -- start start 2024-03-19T01:47:11.272 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.271+0000 7fd3c838b740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563c10ef2100 0x563c10eec220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:11.272 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.271+0000 7fd3c838b740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563c10ef2af0 0x563c10f03560 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:11.272 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.271+0000 7fd3c838b740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563c10ef7c60 0x563c10eec760 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:11.272 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.271+0000 7fd3c838b740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fd3c01ab750 con 0x563c10ef7c60 2024-03-19T01:47:11.272 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.271+0000 7fd3c838b740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fd3c01a9020 con 0x563c10ef2af0 2024-03-19T01:47:11.272 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.271+0000 7fd3c838b740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fd3c01a6e70 con 0x563c10ef2100 2024-03-19T01:47:11.272 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.271+0000 7fd3bf7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563c10ef7c60 0x563c10eec760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:11.272 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.271+0000 7fd3beffd640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563c10ef2100 0x563c10eec220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:11.272 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.271+0000 7fd3be7fc640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563c10ef2af0 0x563c10f03560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:11.273 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.271+0000 7fd3beffd640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563c10ef2100 0x563c10eec220 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:46678/0 (socket says 172.21.15.99:46678) 2024-03-19T01:47:11.273 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.271+0000 7fd3beffd640 1 -- 172.21.15.99:0/4094168927 learned_addr learned my addr 172.21.15.99:0/4094168927 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:47:11.273 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.271+0000 7fd3beffd640 1 -- 172.21.15.99:0/4094168927 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563c10ef2af0 msgr2=0x563c10f03560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:47:11.273 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.271+0000 7fd3beffd640 1 --2- 172.21.15.99:0/4094168927 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563c10ef2af0 0x563c10f03560 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:11.273 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.271+0000 7fd3beffd640 1 -- 172.21.15.99:0/4094168927 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563c10ef7c60 msgr2=0x563c10eec760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:47:11.273 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.271+0000 7fd3beffd640 1 --2- 172.21.15.99:0/4094168927 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563c10ef7c60 0x563c10eec760 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:11.273 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.271+0000 7fd3beffd640 1 -- 172.21.15.99:0/4094168927 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x563c10c37620 con 0x563c10ef2100 2024-03-19T01:47:11.273 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.272+0000 7fd3be7fc640 1 --2- 172.21.15.99:0/4094168927 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563c10ef2af0 0x563c10f03560 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:47:11.273 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.272+0000 7fd3bf7fe640 1 --2- 172.21.15.99:0/4094168927 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563c10ef7c60 0x563c10eec760 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:47:11.273 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.272+0000 7fd3beffd640 1 --2- 172.21.15.99:0/4094168927 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563c10ef2100 0x563c10eec220 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7fd3b80ccf40 tx=0x7fd3b80d2880 comp rx=0 tx=0).ready entity=mon.1 client_cookie=f6542fec48e2c28 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:47:11.273 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.272+0000 7fd3b4ff9640 1 -- 172.21.15.99:0/4094168927 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd3b809b5b0 con 0x563c10ef2100 2024-03-19T01:47:11.273 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.272+0000 7fd3b4ff9640 1 -- 172.21.15.99:0/4094168927 <== mon.1 v2:172.21.15.115:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd3b809c170 con 0x563c10ef2100 2024-03-19T01:47:11.273 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.272+0000 7fd3b4ff9640 1 -- 172.21.15.99:0/4094168927 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd3b809ba70 con 0x563c10ef2100 2024-03-19T01:47:11.276 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.273+0000 7fd3c838b740 1 -- 172.21.15.99:0/4094168927 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563c10ef2100 msgr2=0x563c10eec220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:47:11.277 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.273+0000 7fd3c838b740 1 --2- 172.21.15.99:0/4094168927 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563c10ef2100 0x563c10eec220 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7fd3b80ccf40 tx=0x7fd3b80d2880 comp rx=0 tx=0).stop 2024-03-19T01:47:11.277 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.273+0000 7fd3c838b740 1 -- 172.21.15.99:0/4094168927 shutdown_connections 2024-03-19T01:47:11.277 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.273+0000 7fd3c838b740 1 --2- 172.21.15.99:0/4094168927 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563c10ef2af0 0x563c10f03560 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:11.277 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.273+0000 7fd3c838b740 1 --2- 172.21.15.99:0/4094168927 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563c10ef7c60 0x563c10eec760 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:11.277 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.273+0000 7fd3c838b740 1 --2- 172.21.15.99:0/4094168927 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563c10ef2100 0x563c10eec220 unknown :-1 s=CLOSED pgs=91 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:11.277 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.273+0000 7fd3c838b740 1 -- 172.21.15.99:0/4094168927 >> 172.21.15.99:0/4094168927 conn(0x563c10eed670 msgr2=0x563c10efc020 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:47:11.277 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.273+0000 7fd3c838b740 1 -- 172.21.15.99:0/4094168927 shutdown_connections 2024-03-19T01:47:11.277 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.273+0000 7fd3c838b740 1 -- 172.21.15.99:0/4094168927 wait complete. 2024-03-19T01:47:11.277 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.273+0000 7fd3c838b740 1 -- start start 2024-03-19T01:47:11.277 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.273+0000 7fd3c838b740 1 Processor -- start 2024-03-19T01:47:11.277 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.273+0000 7fd3c838b740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563c10ef2100 0x563c10f00bb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:11.277 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.273+0000 7fd3c838b740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563c10ef2af0 0x563c10f010f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:11.277 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.273+0000 7fd3c838b740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563c10ef7c60 0x563c10f02100 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:11.277 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.273+0000 7fd3c838b740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fd3c01a74e0 con 0x563c10ef7c60 2024-03-19T01:47:11.278 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.273+0000 7fd3c838b740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fd3c01a8950 con 0x563c10ef2100 2024-03-19T01:47:11.278 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.273+0000 7fd3c838b740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fd3c01a7bb0 con 0x563c10ef2af0 2024-03-19T01:47:11.278 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.273+0000 7fd3c838b740 10 client.0 _ll_register_callbacks cb 0x7fd3a8009780 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:47:11.278 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.273+0000 7fd3bf7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563c10ef7c60 0x563c10f02100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:11.278 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.273+0000 7fd3bf7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563c10ef7c60 0x563c10f02100 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:50740/0 (socket says 172.21.15.99:50740) 2024-03-19T01:47:11.278 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.273+0000 7fd3bf7fe640 1 -- 172.21.15.99:0/15385559 learned_addr learned my addr 172.21.15.99:0/15385559 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:47:11.278 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.274+0000 7fd3be7fc640 1 --2- 172.21.15.99:0/15385559 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563c10ef2af0 0x563c10f010f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:11.278 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.274+0000 7fd3beffd640 1 --2- 172.21.15.99:0/15385559 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563c10ef2100 0x563c10f00bb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:11.278 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.274+0000 7fd3bf7fe640 1 -- 172.21.15.99:0/15385559 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563c10ef2af0 msgr2=0x563c10f010f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:47:11.278 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.274+0000 7fd3bf7fe640 1 --2- 172.21.15.99:0/15385559 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563c10ef2af0 0x563c10f010f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:11.278 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.274+0000 7fd3bf7fe640 1 -- 172.21.15.99:0/15385559 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563c10ef2100 msgr2=0x563c10f00bb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:47:11.278 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.274+0000 7fd3bf7fe640 1 --2- 172.21.15.99:0/15385559 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563c10ef2100 0x563c10f00bb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:11.278 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.274+0000 7fd3bf7fe640 1 -- 172.21.15.99:0/15385559 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x563c10f026f0 con 0x563c10ef7c60 2024-03-19T01:47:11.279 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.274+0000 7fd3bf7fe640 1 --2- 172.21.15.99:0/15385559 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563c10ef7c60 0x563c10f02100 secure :-1 s=READY pgs=466 cs=0 l=1 rev1=1 crypto rx=0x7fd3b001c700 tx=0x7fd3b0004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=dda657a7b0f3c269 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:47:11.279 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.274+0000 7fd3a67fc640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:47:11.279 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.274+0000 7fd3a67fc640 1 -- 172.21.15.99:0/15385559 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd3b002d020 con 0x563c10ef7c60 2024-03-19T01:47:11.279 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.274+0000 7fd3a67fc640 1 -- 172.21.15.99:0/15385559 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7fd3b0029900 con 0x563c10ef7c60 2024-03-19T01:47:11.279 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.274+0000 7fd3c838b740 1 -- 172.21.15.99:0/15385559 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fd3b809c170 con 0x563c10ef7c60 2024-03-19T01:47:11.279 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.274+0000 7fd3a67fc640 1 -- 172.21.15.99:0/15385559 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7fd3b0038050 con 0x563c10ef7c60 2024-03-19T01:47:11.279 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.274+0000 7fd3c838b740 10 client.15834 fetch_fsmap learned FSMap version 27 2024-03-19T01:47:11.279 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.274+0000 7fd3c838b740 1 -- 172.21.15.99:0/15385559 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x563c10f6e1e0 con 0x563c10ef7c60 2024-03-19T01:47:11.279 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.274+0000 7fd3a67fc640 1 -- 172.21.15.99:0/15385559 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd3b809c170 con 0x563c10ef7c60 2024-03-19T01:47:11.279 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.275+0000 7fd3a67fc640 1 -- 172.21.15.99:0/15385559 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 27) ==== 8338+0+0 (secure 0 0 0) 0x7fd3b0028bd0 con 0x563c10ef7c60 2024-03-19T01:47:11.279 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.275+0000 7fd3c838b740 10 client.15834 fetch_fsmap finished waiting for FSMap version 27 2024-03-19T01:47:11.279 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.275+0000 7fd3c838b740 10 client.15834 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:47:11.279 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.275+0000 7fd3c838b740 20 client.15834 populate_metadata read hostname 'smithi099' 2024-03-19T01:47:11.279 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.275+0000 7fd3c838b740 1 --2- 172.21.15.99:0/15385559 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x563c10f6ec60 0x563c10f71080 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:11.280 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.275+0000 7fd3c838b740 4 client.15834 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:47:11.280 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.275+0000 7fd3c838b740 1 -- 172.21.15.99:0/15385559 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x563c10e5e470 con 0x563c10f6ec60 2024-03-19T01:47:11.280 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.275+0000 7fd3beffd640 1 --2- 172.21.15.99:0/15385559 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x563c10f6ec60 0x563c10f71080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:11.280 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.275+0000 7fd3a67fc640 10 client.15834 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:47:11.280 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.275+0000 7fd3beffd640 1 --2- 172.21.15.99:0/15385559 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x563c10f6ec60 0x563c10f71080 crc :-1 s=READY pgs=84 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=c985e44d448ac475 server_cookie=c8732305dc374e30 in_seq=0 out_seq=0 2024-03-19T01:47:11.329 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.329+0000 7fd3a67fc640 1 -- 172.21.15.99:0/15385559 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x563c10e5e470 con 0x563c10f6ec60 2024-03-19T01:47:11.329 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.329+0000 7fd3a67fc640 10 client.15834 handle_command_reply: tid=0 2024-03-19T01:47:11.329 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.329+0000 7fd3c838b740 1 client.15834 shutdown 2024-03-19T01:47:11.329 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.329+0000 7fd3c838b740 1 -- 172.21.15.99:0/15385559 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563c10ef7c60 msgr2=0x563c10f02100 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:47:11.330 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.329+0000 7fd3c838b740 1 --2- 172.21.15.99:0/15385559 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563c10ef7c60 0x563c10f02100 secure :-1 s=READY pgs=466 cs=0 l=1 rev1=1 crypto rx=0x7fd3b001c700 tx=0x7fd3b0004730 comp rx=0 tx=0).stop 2024-03-19T01:47:11.330 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.329+0000 7fd3c838b740 1 -- 172.21.15.99:0/15385559 shutdown_connections 2024-03-19T01:47:11.330 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.329+0000 7fd3c838b740 1 --2- 172.21.15.99:0/15385559 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x563c10f6ec60 0x563c10f71080 crc :-1 s=READY pgs=84 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:11.330 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.329+0000 7fd3c838b740 1 --2- 172.21.15.99:0/15385559 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563c10ef7c60 0x563c10f02100 unknown :-1 s=CLOSED pgs=466 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:11.330 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.329+0000 7fd3c838b740 1 --2- 172.21.15.99:0/15385559 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563c10ef2af0 0x563c10f010f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:11.330 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.329+0000 7fd3a67fc640 0 client.15834 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:47:11.330 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.329+0000 7fd3c838b740 1 --2- 172.21.15.99:0/15385559 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563c10ef2100 0x563c10f00bb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:11.330 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.329+0000 7fd3c838b740 1 -- 172.21.15.99:0/15385559 >> 172.21.15.99:0/15385559 conn(0x563c10eed670 msgr2=0x563c10eec220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:47:11.330 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.329+0000 7fd3c838b740 1 -- 172.21.15.99:0/15385559 shutdown_connections 2024-03-19T01:47:11.330 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.329+0000 7fd3c838b740 1 -- 172.21.15.99:0/15385559 wait complete. 2024-03-19T01:47:11.330 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.329+0000 7fd3c838b740 20 client.15834 trim_cache size 0 max 16384 2024-03-19T01:47:11.331 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:47:11.331 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:47:11.331 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:47:11.331 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:47:11.331 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.330+0000 7fd3c57ab640 1 -- 172.21.15.99:0/246341089 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd3ac0a28a0 msgr2=0x7fd3ac0a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:47:11.331 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.330+0000 7fd3c57ab640 1 --2- 172.21.15.99:0/246341089 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd3ac0a28a0 0x7fd3ac0a4d60 secure :-1 s=READY pgs=279 cs=0 l=1 rev1=1 crypto rx=0x563c10ef3350 tx=0x7fd3b00046c0 comp rx=0 tx=0).stop 2024-03-19T01:47:11.332 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.330+0000 7fd3c57ab640 1 -- 172.21.15.99:0/246341089 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd3c0101860 msgr2=0x7fd3c0197b20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:47:11.332 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.330+0000 7fd3c57ab640 1 --2- 172.21.15.99:0/246341089 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd3c0101860 0x7fd3c0197b20 secure :-1 s=READY pgs=464 cs=0 l=1 rev1=1 crypto rx=0x7fd3b800eb70 tx=0x7fd3b800c6a0 comp rx=0 tx=0).stop 2024-03-19T01:47:11.332 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.330+0000 7fd3c57ab640 1 -- 172.21.15.99:0/246341089 shutdown_connections 2024-03-19T01:47:11.332 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.330+0000 7fd3c57ab640 1 --2- 172.21.15.99:0/246341089 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd3ac0a28a0 0x7fd3ac0a4d60 unknown :-1 s=CLOSED pgs=279 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:11.332 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.330+0000 7fd3c57ab640 1 --2- 172.21.15.99:0/246341089 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd3c0103410 0x7fd3c019f0d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:11.332 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.330+0000 7fd3c57ab640 1 --2- 172.21.15.99:0/246341089 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd3c0102a50 0x7fd3c0198060 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:11.332 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.330+0000 7fd3c57ab640 1 --2- 172.21.15.99:0/246341089 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd3c0101860 0x7fd3c0197b20 unknown :-1 s=CLOSED pgs=464 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:11.332 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.330+0000 7fd3c57ab640 1 -- 172.21.15.99:0/246341089 >> 172.21.15.99:0/246341089 conn(0x7fd3c00fcdd0 msgr2=0x7fd3c0105d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:47:11.332 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.330+0000 7fd3c57ab640 1 -- 172.21.15.99:0/246341089 shutdown_connections 2024-03-19T01:47:11.332 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:11.330+0000 7fd3c57ab640 1 -- 172.21.15.99:0/246341089 wait complete. 2024-03-19T01:47:11.687 INFO:tasks.workunit.client.0.smithi099.stdout: 1 722731 11.25 MB/sec execute 221 sec latency 39.704 ms 2024-03-19T01:47:11.806 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:47:11 smithi115 ceph-mon[67332]: pgmap v533: 97 pgs: 97 active+clean; 231 MiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 931 KiB/s rd, 2.9 MiB/s wr, 283 op/s 2024-03-19T01:47:11.840 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:47:11 smithi099 ceph-mon[62951]: pgmap v533: 97 pgs: 97 active+clean; 231 MiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 931 KiB/s rd, 2.9 MiB/s wr, 283 op/s 2024-03-19T01:47:11.861 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:47:11 smithi112 ceph-mon[66819]: pgmap v533: 97 pgs: 97 active+clean; 231 MiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 931 KiB/s rd, 2.9 MiB/s wr, 283 op/s 2024-03-19T01:47:12.687 INFO:tasks.workunit.client.0.smithi099.stdout: 1 724968 11.26 MB/sec execute 222 sec latency 18.567 ms 2024-03-19T01:47:13.687 INFO:tasks.workunit.client.0.smithi099.stdout: 1 727414 11.27 MB/sec execute 223 sec latency 40.341 ms 2024-03-19T01:47:14.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:47:13 smithi115 ceph-mon[67332]: pgmap v534: 97 pgs: 97 active+clean; 231 MiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 817 KiB/s rd, 3.2 MiB/s wr, 251 op/s 2024-03-19T01:47:14.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:47:13 smithi099 ceph-mon[62951]: pgmap v534: 97 pgs: 97 active+clean; 231 MiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 817 KiB/s rd, 3.2 MiB/s wr, 251 op/s 2024-03-19T01:47:14.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:47:13 smithi112 ceph-mon[66819]: pgmap v534: 97 pgs: 97 active+clean; 231 MiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 817 KiB/s rd, 3.2 MiB/s wr, 251 op/s 2024-03-19T01:47:14.687 INFO:tasks.workunit.client.0.smithi099.stdout: 1 728926 11.25 MB/sec execute 224 sec latency 82.410 ms 2024-03-19T01:47:15.687 INFO:tasks.workunit.client.0.smithi099.stdout: 1 731755 11.28 MB/sec execute 225 sec latency 12.802 ms 2024-03-19T01:47:16.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:47:15 smithi115 ceph-mon[67332]: pgmap v535: 97 pgs: 97 active+clean; 228 MiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 777 KiB/s rd, 3.0 MiB/s wr, 237 op/s 2024-03-19T01:47:16.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:47:15 smithi099 ceph-mon[62951]: pgmap v535: 97 pgs: 97 active+clean; 228 MiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 777 KiB/s rd, 3.0 MiB/s wr, 237 op/s 2024-03-19T01:47:16.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:47:15 smithi112 ceph-mon[66819]: pgmap v535: 97 pgs: 97 active+clean; 228 MiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 777 KiB/s rd, 3.0 MiB/s wr, 237 op/s 2024-03-19T01:47:16.687 INFO:tasks.workunit.client.0.smithi099.stdout: 1 733925 11.26 MB/sec execute 226 sec latency 27.348 ms 2024-03-19T01:47:17.688 INFO:tasks.workunit.client.0.smithi099.stdout: 1 735838 11.27 MB/sec execute 227 sec latency 23.817 ms 2024-03-19T01:47:18.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:47:17 smithi115 ceph-mon[67332]: pgmap v536: 97 pgs: 97 active+clean; 231 MiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 3.7 MiB/s wr, 316 op/s 2024-03-19T01:47:18.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:47:17 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:47:18.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:47:17 smithi099 ceph-mon[62951]: pgmap v536: 97 pgs: 97 active+clean; 231 MiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 3.7 MiB/s wr, 316 op/s 2024-03-19T01:47:18.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:47:17 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:47:18.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:47:17 smithi112 ceph-mon[66819]: pgmap v536: 97 pgs: 97 active+clean; 231 MiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 3.7 MiB/s wr, 316 op/s 2024-03-19T01:47:18.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:47:17 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:47:18.688 INFO:tasks.workunit.client.0.smithi099.stdout: 1 738547 11.29 MB/sec execute 228 sec latency 15.546 ms 2024-03-19T01:47:19.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:47:18 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:47:19.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:47:18 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:47:19.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:47:18 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:47:19.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:47:18 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:47:19.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:47:18 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:47:19.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:47:18 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:47:19.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:47:18 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:47:19.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:47:18 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:47:19.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:47:18 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:47:19.687 INFO:tasks.workunit.client.0.smithi099.stdout: 1 740034 11.27 MB/sec execute 229 sec latency 61.347 ms 2024-03-19T01:47:20.207 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:47:19 smithi115 ceph-mon[67332]: pgmap v537: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 800 KiB/s rd, 3.1 MiB/s wr, 269 op/s 2024-03-19T01:47:20.263 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:47:19 smithi112 ceph-mon[66819]: pgmap v537: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 800 KiB/s rd, 3.1 MiB/s wr, 269 op/s 2024-03-19T01:47:20.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:47:19 smithi099 ceph-mon[62951]: pgmap v537: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 800 KiB/s rd, 3.1 MiB/s wr, 269 op/s 2024-03-19T01:47:20.688 INFO:tasks.workunit.client.0.smithi099.stdout: 1 742486 11.29 MB/sec execute 230 sec latency 23.860 ms 2024-03-19T01:47:21.688 INFO:tasks.workunit.client.0.smithi099.stdout: 1 744608 11.27 MB/sec execute 231 sec latency 28.617 ms 2024-03-19T01:47:22.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:47:21 smithi115 ceph-mon[67332]: pgmap v538: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 765 KiB/s rd, 3.0 MiB/s wr, 260 op/s 2024-03-19T01:47:22.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:47:21 smithi099 ceph-mon[62951]: pgmap v538: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 765 KiB/s rd, 3.0 MiB/s wr, 260 op/s 2024-03-19T01:47:22.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:47:21 smithi112 ceph-mon[66819]: pgmap v538: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 765 KiB/s rd, 3.0 MiB/s wr, 260 op/s 2024-03-19T01:47:22.688 INFO:tasks.workunit.client.0.smithi099.stdout: 1 746551 11.27 MB/sec execute 232 sec latency 27.229 ms 2024-03-19T01:47:23.688 INFO:tasks.workunit.client.0.smithi099.stdout: 1 749304 11.30 MB/sec execute 233 sec latency 34.821 ms 2024-03-19T01:47:24.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:47:23 smithi115 ceph-mon[67332]: pgmap v539: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 749 KiB/s rd, 3.1 MiB/s wr, 238 op/s 2024-03-19T01:47:24.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:47:23 smithi099 ceph-mon[62951]: pgmap v539: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 749 KiB/s rd, 3.1 MiB/s wr, 238 op/s 2024-03-19T01:47:24.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:47:23 smithi112 ceph-mon[66819]: pgmap v539: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 749 KiB/s rd, 3.1 MiB/s wr, 238 op/s 2024-03-19T01:47:24.425 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.423+0000 7fa20028a640 1 -- 172.21.15.99:0/2414910919 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa1f8100740 msgr2=0x7fa1f8100b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:47:24.425 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.423+0000 7fa20028a640 1 --2- 172.21.15.99:0/2414910919 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa1f8100740 0x7fa1f8100b60 secure :-1 s=READY pgs=467 cs=0 l=1 rev1=1 crypto rx=0x7fa1f4009900 tx=0x7fa1f402bc30 comp rx=0 tx=0).stop 2024-03-19T01:47:24.425 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.423+0000 7fa20028a640 1 -- 172.21.15.99:0/2414910919 shutdown_connections 2024-03-19T01:47:24.425 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.423+0000 7fa20028a640 1 --2- 172.21.15.99:0/2414910919 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa1f81022f0 0x7fa1f8108b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:24.425 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.423+0000 7fa20028a640 1 --2- 172.21.15.99:0/2414910919 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa1f8101930 0x7fa1f8101db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:24.425 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.423+0000 7fa20028a640 1 --2- 172.21.15.99:0/2414910919 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa1f8100740 0x7fa1f8100b60 unknown :-1 s=CLOSED pgs=467 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:24.425 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.423+0000 7fa20028a640 1 -- 172.21.15.99:0/2414910919 >> 172.21.15.99:0/2414910919 conn(0x7fa1f80fbcb0 msgr2=0x7fa1f80fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:47:24.425 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.423+0000 7fa20028a640 1 -- 172.21.15.99:0/2414910919 shutdown_connections 2024-03-19T01:47:24.426 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.423+0000 7fa20028a640 1 -- 172.21.15.99:0/2414910919 wait complete. 2024-03-19T01:47:24.426 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.424+0000 7fa20028a640 1 Processor -- start 2024-03-19T01:47:24.426 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.424+0000 7fa20028a640 1 -- start start 2024-03-19T01:47:24.426 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.424+0000 7fa20028a640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa1f8100740 0x7fa1f8197ae0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:24.426 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.424+0000 7fa20028a640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa1f8101930 0x7fa1f8198020 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:24.426 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.424+0000 7fa20028a640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa1f81022f0 0x7fa1f819f090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:24.426 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.424+0000 7fa20028a640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fa1f810d4e0 con 0x7fa1f8100740 2024-03-19T01:47:24.426 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.424+0000 7fa20028a640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fa1f810d360 con 0x7fa1f8101930 2024-03-19T01:47:24.426 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.424+0000 7fa20028a640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fa1f810d660 con 0x7fa1f81022f0 2024-03-19T01:47:24.426 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.424+0000 7fa1fe026640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa1f8100740 0x7fa1f8197ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:24.426 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.424+0000 7fa1fe026640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa1f8100740 0x7fa1f8197ae0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:56690/0 (socket says 172.21.15.99:56690) 2024-03-19T01:47:24.426 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.424+0000 7fa1fe026640 1 -- 172.21.15.99:0/3627570085 learned_addr learned my addr 172.21.15.99:0/3627570085 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:47:24.426 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.424+0000 7fa1fd825640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa1f8101930 0x7fa1f8198020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:24.426 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.424+0000 7fa1fe827640 1 --2- 172.21.15.99:0/3627570085 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa1f81022f0 0x7fa1f819f090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:24.426 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.424+0000 7fa1fe026640 1 -- 172.21.15.99:0/3627570085 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa1f81022f0 msgr2=0x7fa1f819f090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:47:24.427 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.424+0000 7fa1fe026640 1 --2- 172.21.15.99:0/3627570085 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa1f81022f0 0x7fa1f819f090 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:24.427 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.424+0000 7fa1fe026640 1 -- 172.21.15.99:0/3627570085 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa1f8101930 msgr2=0x7fa1f8198020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:47:24.427 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.424+0000 7fa1fe026640 1 --2- 172.21.15.99:0/3627570085 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa1f8101930 0x7fa1f8198020 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:24.427 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.424+0000 7fa1fe026640 1 -- 172.21.15.99:0/3627570085 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa1f819f700 con 0x7fa1f8100740 2024-03-19T01:47:24.427 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.424+0000 7fa1fd825640 1 --2- 172.21.15.99:0/3627570085 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa1f8101930 0x7fa1f8198020 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:47:24.427 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.424+0000 7fa1fe827640 1 --2- 172.21.15.99:0/3627570085 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa1f81022f0 0x7fa1f819f090 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:47:24.427 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.424+0000 7fa1fe026640 1 --2- 172.21.15.99:0/3627570085 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa1f8100740 0x7fa1f8197ae0 secure :-1 s=READY pgs=468 cs=0 l=1 rev1=1 crypto rx=0x7fa1f4000c00 tx=0x7fa1f402c640 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:47:24.427 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.425+0000 7fa1ef7fe640 1 -- 172.21.15.99:0/3627570085 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa1f4041070 con 0x7fa1f8100740 2024-03-19T01:47:24.427 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.425+0000 7fa20028a640 1 -- 172.21.15.99:0/3627570085 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa1f819f990 con 0x7fa1f8100740 2024-03-19T01:47:24.427 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.425+0000 7fa1ef7fe640 1 -- 172.21.15.99:0/3627570085 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa1f4004450 con 0x7fa1f8100740 2024-03-19T01:47:24.427 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.425+0000 7fa20028a640 1 -- 172.21.15.99:0/3627570085 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fa1f819fe80 con 0x7fa1f8100740 2024-03-19T01:47:24.427 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.425+0000 7fa1ef7fe640 1 -- 172.21.15.99:0/3627570085 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa1f404b060 con 0x7fa1f8100740 2024-03-19T01:47:24.434 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.427+0000 7fa1ef7fe640 1 -- 172.21.15.99:0/3627570085 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7fa1f4044070 con 0x7fa1f8100740 2024-03-19T01:47:24.434 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.427+0000 7fa202e71740 1 Processor -- start 2024-03-19T01:47:24.434 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.427+0000 7fa202e71740 1 -- start start 2024-03-19T01:47:24.434 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.428+0000 7fa202e71740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c2a92d9100 0x55c2a92d9520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:24.434 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.428+0000 7fa202e71740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c2a92d9af0 0x55c2a92de5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:24.434 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.428+0000 7fa202e71740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c2a92dec60 0x55c2a92e10f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:24.434 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.428+0000 7fa202e71740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55c2a923dbe0 con 0x55c2a92d9100 2024-03-19T01:47:24.434 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.428+0000 7fa202e71740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55c2a91e6960 con 0x55c2a92d9af0 2024-03-19T01:47:24.434 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.428+0000 7fa202e71740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55c2a9239c80 con 0x55c2a92dec60 2024-03-19T01:47:24.434 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.428+0000 7fa1ef7fe640 1 --2- 172.21.15.99:0/3627570085 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa1e40a28a0 0x7fa1e40a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:24.434 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.428+0000 7fa1fe827640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c2a92d9100 0x55c2a92d9520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:24.435 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.428+0000 7fa1fd825640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c2a92dec60 0x55c2a92e10f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:24.435 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.428+0000 7fa1fe026640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c2a92d9af0 0x55c2a92de5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:24.435 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.428+0000 7fa1fd825640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c2a92dec60 0x55c2a92e10f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:60920/0 (socket says 172.21.15.99:60920) 2024-03-19T01:47:24.435 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.428+0000 7fa1fe827640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c2a92d9100 0x55c2a92d9520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:56696/0 (socket says 172.21.15.99:56696) 2024-03-19T01:47:24.435 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.428+0000 7fa1fd825640 1 -- 172.21.15.99:0/864475812 learned_addr learned my addr 172.21.15.99:0/864475812 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:47:24.435 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.428+0000 7fa1fe827640 1 --2- 172.21.15.99:0/3627570085 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa1e40a28a0 0x7fa1e40a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:24.435 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.428+0000 7fa1ef7fe640 1 -- 172.21.15.99:0/3627570085 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7fa1f40f0510 con 0x7fa1f8100740 2024-03-19T01:47:24.435 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.428+0000 7fa1fd825640 1 -- 172.21.15.99:0/864475812 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c2a92d9af0 msgr2=0x55c2a92de5f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:47:24.435 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.428+0000 7fa1fd825640 1 --2- 172.21.15.99:0/864475812 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c2a92d9af0 0x55c2a92de5f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:24.435 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.428+0000 7fa1fd825640 1 -- 172.21.15.99:0/864475812 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c2a92d9100 msgr2=0x55c2a92d9520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:47:24.435 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.428+0000 7fa1fd825640 1 --2- 172.21.15.99:0/864475812 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c2a92d9100 0x55c2a92d9520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:24.435 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.428+0000 7fa1fd825640 1 -- 172.21.15.99:0/864475812 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55c2a91ddd20 con 0x55c2a92dec60 2024-03-19T01:47:24.435 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.428+0000 7fa1fe026640 1 --2- 172.21.15.99:0/864475812 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c2a92d9af0 0x55c2a92de5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:47:24.435 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.428+0000 7fa1fe827640 1 --2- 172.21.15.99:0/3627570085 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa1e40a28a0 0x7fa1e40a4d60 secure :-1 s=READY pgs=280 cs=0 l=1 rev1=1 crypto rx=0x7fa1f00099e0 tx=0x7fa1f0009500 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:47:24.436 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.428+0000 7fa1fd825640 1 --2- 172.21.15.99:0/864475812 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c2a92dec60 0x55c2a92e10f0 secure :-1 s=READY pgs=92 cs=0 l=1 rev1=1 crypto rx=0x7fa1e80114f0 tx=0x7fa1e80118f0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=63f05e4163b8308b server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:47:24.436 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.429+0000 7fa1ecbfe640 1 -- 172.21.15.99:0/864475812 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa1e8004020 con 0x55c2a92dec60 2024-03-19T01:47:24.436 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.429+0000 7fa1ecbfe640 1 -- 172.21.15.99:0/864475812 <== mon.1 v2:172.21.15.115:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa1e80041c0 con 0x55c2a92dec60 2024-03-19T01:47:24.436 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.429+0000 7fa1ecbfe640 1 -- 172.21.15.99:0/864475812 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa1e800f850 con 0x55c2a92dec60 2024-03-19T01:47:24.436 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.429+0000 7fa202e71740 1 -- 172.21.15.99:0/864475812 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c2a92dec60 msgr2=0x55c2a92e10f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:47:24.436 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.429+0000 7fa202e71740 1 --2- 172.21.15.99:0/864475812 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c2a92dec60 0x55c2a92e10f0 secure :-1 s=READY pgs=92 cs=0 l=1 rev1=1 crypto rx=0x7fa1e80114f0 tx=0x7fa1e80118f0 comp rx=0 tx=0).stop 2024-03-19T01:47:24.436 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.429+0000 7fa202e71740 1 -- 172.21.15.99:0/864475812 shutdown_connections 2024-03-19T01:47:24.436 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.429+0000 7fa202e71740 1 --2- 172.21.15.99:0/864475812 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c2a92d9af0 0x55c2a92de5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:24.436 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.429+0000 7fa202e71740 1 --2- 172.21.15.99:0/864475812 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c2a92dec60 0x55c2a92e10f0 unknown :-1 s=CLOSED pgs=92 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:24.436 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.429+0000 7fa202e71740 1 --2- 172.21.15.99:0/864475812 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c2a92d9100 0x55c2a92d9520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:24.436 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.429+0000 7fa202e71740 1 -- 172.21.15.99:0/864475812 >> 172.21.15.99:0/864475812 conn(0x55c2a92d4670 msgr2=0x55c2a92d4a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:47:24.436 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.429+0000 7fa202e71740 1 -- 172.21.15.99:0/864475812 shutdown_connections 2024-03-19T01:47:24.436 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.429+0000 7fa202e71740 1 -- 172.21.15.99:0/864475812 wait complete. 2024-03-19T01:47:24.436 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.430+0000 7fa202e71740 1 -- start start 2024-03-19T01:47:24.437 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.430+0000 7fa202e71740 1 Processor -- start 2024-03-19T01:47:24.437 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.430+0000 7fa202e71740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c2a92d9100 0x55c2a92e3130 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:24.437 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.430+0000 7fa202e71740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c2a92d9af0 0x55c2a92e3670 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:24.437 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.430+0000 7fa202e71740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c2a92dec60 0x55c2a92ea640 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:24.437 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.430+0000 7fa202e71740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55c2a92e1c10 con 0x55c2a92d9af0 2024-03-19T01:47:24.437 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.430+0000 7fa202e71740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55c2a91f9db0 con 0x55c2a92d9100 2024-03-19T01:47:24.437 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.430+0000 7fa202e71740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55c2a92e1d90 con 0x55c2a92dec60 2024-03-19T01:47:24.437 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.430+0000 7fa202e71740 10 client.0 _ll_register_callbacks cb 0x55c2a91fee70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:47:24.437 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.430+0000 7fa1fd825640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c2a92d9af0 0x55c2a92e3670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:24.437 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.430+0000 7fa1fe026640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c2a92d9100 0x55c2a92e3130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:24.437 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.430+0000 7fa1fd825640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c2a92d9af0 0x55c2a92e3670 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:56708/0 (socket says 172.21.15.99:56708) 2024-03-19T01:47:24.437 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.430+0000 7fa1fd825640 1 -- 172.21.15.99:0/4080261351 learned_addr learned my addr 172.21.15.99:0/4080261351 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:47:24.437 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.430+0000 7fa1fe827640 1 --2- 172.21.15.99:0/4080261351 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c2a92dec60 0x55c2a92ea640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:24.437 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.430+0000 7fa1fd825640 1 -- 172.21.15.99:0/4080261351 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c2a92dec60 msgr2=0x55c2a92ea640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:47:24.438 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.430+0000 7fa1fd825640 1 --2- 172.21.15.99:0/4080261351 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c2a92dec60 0x55c2a92ea640 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:24.438 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.430+0000 7fa1fd825640 1 -- 172.21.15.99:0/4080261351 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c2a92d9100 msgr2=0x55c2a92e3130 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:47:24.438 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.430+0000 7fa1fd825640 1 --2- 172.21.15.99:0/4080261351 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c2a92d9100 0x55c2a92e3130 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:24.438 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.430+0000 7fa1fd825640 1 -- 172.21.15.99:0/4080261351 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55c2a91f8c60 con 0x55c2a92d9af0 2024-03-19T01:47:24.438 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.430+0000 7fa1fe026640 1 --2- 172.21.15.99:0/4080261351 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c2a92d9100 0x55c2a92e3130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:47:24.438 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.431+0000 7fa1fd825640 1 --2- 172.21.15.99:0/4080261351 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c2a92d9af0 0x55c2a92e3670 secure :-1 s=READY pgs=469 cs=0 l=1 rev1=1 crypto rx=0x7fa1e8009710 tx=0x7fa1e80055c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=31860e6f6a23f0d4 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:47:24.438 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.431+0000 7fa1e1ffb640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:47:24.438 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.431+0000 7fa1e1ffb640 1 -- 172.21.15.99:0/4080261351 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa1e8027070 con 0x55c2a92d9af0 2024-03-19T01:47:24.438 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.431+0000 7fa1e1ffb640 1 -- 172.21.15.99:0/4080261351 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7fa1e80035f0 con 0x55c2a92d9af0 2024-03-19T01:47:24.438 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.431+0000 7fa202e71740 1 -- 172.21.15.99:0/4080261351 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fa1e80041c0 con 0x55c2a92d9af0 2024-03-19T01:47:24.438 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.431+0000 7fa1e1ffb640 1 -- 172.21.15.99:0/4080261351 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7fa1e8033050 con 0x55c2a92d9af0 2024-03-19T01:47:24.438 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.431+0000 7fa202e71740 10 client.15846 fetch_fsmap learned FSMap version 27 2024-03-19T01:47:24.438 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.431+0000 7fa202e71740 1 -- 172.21.15.99:0/4080261351 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x55c2a8f52210 con 0x55c2a92d9af0 2024-03-19T01:47:24.438 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.431+0000 7fa1e1ffb640 1 -- 172.21.15.99:0/4080261351 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa1e80041c0 con 0x55c2a92d9af0 2024-03-19T01:47:24.439 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.431+0000 7fa1e1ffb640 1 -- 172.21.15.99:0/4080261351 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 27) ==== 8338+0+0 (secure 0 0 0) 0x7fa1e800f550 con 0x55c2a92d9af0 2024-03-19T01:47:24.439 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.431+0000 7fa202e71740 10 client.15846 fetch_fsmap finished waiting for FSMap version 27 2024-03-19T01:47:24.439 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.431+0000 7fa202e71740 10 client.15846 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:47:24.439 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.431+0000 7fa202e71740 20 client.15846 populate_metadata read hostname 'smithi099' 2024-03-19T01:47:24.439 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.431+0000 7fa202e71740 1 --2- 172.21.15.99:0/4080261351 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55c2a92eec60 0x55c2a92f1100 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:24.439 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.431+0000 7fa202e71740 4 client.15846 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:47:24.439 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.431+0000 7fa202e71740 1 -- 172.21.15.99:0/4080261351 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55c2a924b120 con 0x55c2a92eec60 2024-03-19T01:47:24.439 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.432+0000 7fa1fe026640 1 --2- 172.21.15.99:0/4080261351 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55c2a92eec60 0x55c2a92f1100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:24.439 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.432+0000 7fa1fe026640 1 --2- 172.21.15.99:0/4080261351 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55c2a92eec60 0x55c2a92f1100 crc :-1 s=READY pgs=85 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=266f645e9c90098a server_cookie=591b3f77335be10b in_seq=0 out_seq=0 2024-03-19T01:47:24.439 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.432+0000 7fa1e1ffb640 10 client.15846 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:47:24.439 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.433+0000 7fa1e1ffb640 1 -- 172.21.15.99:0/4080261351 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x55c2a924b120 con 0x55c2a92eec60 2024-03-19T01:47:24.439 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.433+0000 7fa1e1ffb640 10 client.15846 handle_command_reply: tid=0 2024-03-19T01:47:24.439 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.433+0000 7fa202e71740 1 client.15846 shutdown 2024-03-19T01:47:24.439 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.433+0000 7fa202e71740 1 -- 172.21.15.99:0/4080261351 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c2a92d9af0 msgr2=0x55c2a92e3670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:47:24.440 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.433+0000 7fa202e71740 1 --2- 172.21.15.99:0/4080261351 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c2a92d9af0 0x55c2a92e3670 secure :-1 s=READY pgs=469 cs=0 l=1 rev1=1 crypto rx=0x7fa1e8009710 tx=0x7fa1e80055c0 comp rx=0 tx=0).stop 2024-03-19T01:47:24.440 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.433+0000 7fa202e71740 1 -- 172.21.15.99:0/4080261351 shutdown_connections 2024-03-19T01:47:24.440 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.433+0000 7fa202e71740 1 --2- 172.21.15.99:0/4080261351 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55c2a92eec60 0x55c2a92f1100 crc :-1 s=READY pgs=85 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:24.440 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.433+0000 7fa202e71740 1 --2- 172.21.15.99:0/4080261351 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c2a92dec60 0x55c2a92ea640 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:24.440 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.433+0000 7fa202e71740 1 --2- 172.21.15.99:0/4080261351 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c2a92d9af0 0x55c2a92e3670 unknown :-1 s=CLOSED pgs=469 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:24.440 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.433+0000 7fa1e1ffb640 0 client.15846 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:47:24.440 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.433+0000 7fa202e71740 1 --2- 172.21.15.99:0/4080261351 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c2a92d9100 0x55c2a92e3130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:24.440 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.433+0000 7fa202e71740 1 -- 172.21.15.99:0/4080261351 >> 172.21.15.99:0/4080261351 conn(0x55c2a92d4670 msgr2=0x55c2a92d58c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:47:24.440 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.433+0000 7fa202e71740 1 -- 172.21.15.99:0/4080261351 shutdown_connections 2024-03-19T01:47:24.440 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.433+0000 7fa202e71740 1 -- 172.21.15.99:0/4080261351 wait complete. 2024-03-19T01:47:24.440 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.433+0000 7fa202e71740 20 client.15846 trim_cache size 0 max 16384 2024-03-19T01:47:24.449 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.448+0000 7fa202e71740 1 Processor -- start 2024-03-19T01:47:24.450 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.448+0000 7fa202e71740 1 -- start start 2024-03-19T01:47:24.450 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.449+0000 7fa202e71740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c2a92d9100 0x55c2a92da380 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:24.450 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.449+0000 7fa202e71740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c2a92d9af0 0x55c2a92ea600 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:24.450 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.449+0000 7fa202e71740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c2a92dec60 0x55c2a92da8c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:24.450 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.449+0000 7fa202e71740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fa1f81ab640 con 0x55c2a92dec60 2024-03-19T01:47:24.450 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.449+0000 7fa202e71740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fa1f81a8f10 con 0x55c2a92d9100 2024-03-19T01:47:24.450 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.449+0000 7fa202e71740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fa1f81a6d60 con 0x55c2a92d9af0 2024-03-19T01:47:24.450 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.449+0000 7fa1fe827640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c2a92dec60 0x55c2a92da8c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:24.450 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.449+0000 7fa1fe827640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c2a92dec60 0x55c2a92da8c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:56716/0 (socket says 172.21.15.99:56716) 2024-03-19T01:47:24.450 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.449+0000 7fa1fe827640 1 -- 172.21.15.99:0/442904209 learned_addr learned my addr 172.21.15.99:0/442904209 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:47:24.450 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.449+0000 7fa1fe026640 1 --2- 172.21.15.99:0/442904209 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c2a92d9100 0x55c2a92da380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:24.450 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.449+0000 7fa1fd825640 1 --2- 172.21.15.99:0/442904209 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c2a92d9af0 0x55c2a92ea600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:24.450 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.449+0000 7fa1fe827640 1 -- 172.21.15.99:0/442904209 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c2a92d9af0 msgr2=0x55c2a92ea600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:47:24.451 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.449+0000 7fa1fe827640 1 --2- 172.21.15.99:0/442904209 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c2a92d9af0 0x55c2a92ea600 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:24.451 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.449+0000 7fa1fe827640 1 -- 172.21.15.99:0/442904209 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c2a92d9100 msgr2=0x55c2a92da380 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:47:24.451 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.449+0000 7fa1fe827640 1 --2- 172.21.15.99:0/442904209 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c2a92d9100 0x55c2a92da380 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:24.451 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.449+0000 7fa1fe827640 1 -- 172.21.15.99:0/442904209 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55c2a901e620 con 0x55c2a92dec60 2024-03-19T01:47:24.451 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.449+0000 7fa1fe026640 1 --2- 172.21.15.99:0/442904209 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c2a92d9100 0x55c2a92da380 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:47:24.451 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.449+0000 7fa1fe827640 1 --2- 172.21.15.99:0/442904209 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c2a92dec60 0x55c2a92da8c0 secure :-1 s=READY pgs=470 cs=0 l=1 rev1=1 crypto rx=0x7fa1f000ff70 tx=0x7fa1f000e000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=558c12bf681a1db server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:47:24.451 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.449+0000 7fa1e3fff640 1 -- 172.21.15.99:0/442904209 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa1f00047d0 con 0x55c2a92dec60 2024-03-19T01:47:24.451 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.450+0000 7fa1e3fff640 1 -- 172.21.15.99:0/442904209 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa1f0011020 con 0x55c2a92dec60 2024-03-19T01:47:24.451 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.450+0000 7fa1e3fff640 1 -- 172.21.15.99:0/442904209 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa1f0010430 con 0x55c2a92dec60 2024-03-19T01:47:24.454 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.450+0000 7fa202e71740 1 -- 172.21.15.99:0/442904209 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c2a92dec60 msgr2=0x55c2a92da8c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:47:24.455 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.450+0000 7fa202e71740 1 --2- 172.21.15.99:0/442904209 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c2a92dec60 0x55c2a92da8c0 secure :-1 s=READY pgs=470 cs=0 l=1 rev1=1 crypto rx=0x7fa1f000ff70 tx=0x7fa1f000e000 comp rx=0 tx=0).stop 2024-03-19T01:47:24.455 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.450+0000 7fa202e71740 1 -- 172.21.15.99:0/442904209 shutdown_connections 2024-03-19T01:47:24.455 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.450+0000 7fa202e71740 1 --2- 172.21.15.99:0/442904209 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c2a92dec60 0x55c2a92da8c0 unknown :-1 s=CLOSED pgs=470 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:24.455 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.450+0000 7fa202e71740 1 --2- 172.21.15.99:0/442904209 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c2a92d9af0 0x55c2a92ea600 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:24.455 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.450+0000 7fa202e71740 1 --2- 172.21.15.99:0/442904209 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c2a92d9100 0x55c2a92da380 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:24.455 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.450+0000 7fa202e71740 1 -- 172.21.15.99:0/442904209 >> 172.21.15.99:0/442904209 conn(0x55c2a92d4670 msgr2=0x55c2a92d3220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:47:24.455 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.450+0000 7fa202e71740 1 -- 172.21.15.99:0/442904209 shutdown_connections 2024-03-19T01:47:24.455 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.450+0000 7fa202e71740 1 -- 172.21.15.99:0/442904209 wait complete. 2024-03-19T01:47:24.455 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.451+0000 7fa202e71740 1 -- start start 2024-03-19T01:47:24.455 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.451+0000 7fa202e71740 1 Processor -- start 2024-03-19T01:47:24.455 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.451+0000 7fa202e71740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c2a92d9100 0x55c2a92e7d10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:24.455 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.451+0000 7fa202e71740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c2a92d9af0 0x55c2a92e8250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:24.455 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.451+0000 7fa202e71740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c2a92dec60 0x55c2a92e91f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:24.455 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.451+0000 7fa202e71740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fa1f81a73d0 con 0x55c2a92d9100 2024-03-19T01:47:24.456 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.451+0000 7fa202e71740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fa1f81a8840 con 0x55c2a92d9af0 2024-03-19T01:47:24.456 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.451+0000 7fa202e71740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fa1f81a7aa0 con 0x55c2a92dec60 2024-03-19T01:47:24.456 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.451+0000 7fa202e71740 10 client.0 _ll_register_callbacks cb 0x7fa1f0011760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:47:24.456 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.451+0000 7fa1fe026640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c2a92d9100 0x55c2a92e7d10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:24.456 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.451+0000 7fa1fe026640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c2a92d9100 0x55c2a92e7d10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:56732/0 (socket says 172.21.15.99:56732) 2024-03-19T01:47:24.456 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.451+0000 7fa1fe026640 1 -- 172.21.15.99:0/2644245572 learned_addr learned my addr 172.21.15.99:0/2644245572 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:47:24.456 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.451+0000 7fa1fd825640 1 --2- 172.21.15.99:0/2644245572 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c2a92d9af0 0x55c2a92e8250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:24.456 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.451+0000 7fa1fe827640 1 --2- 172.21.15.99:0/2644245572 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c2a92dec60 0x55c2a92e91f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:24.456 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.451+0000 7fa1fe026640 1 -- 172.21.15.99:0/2644245572 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c2a92dec60 msgr2=0x55c2a92e91f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:47:24.456 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.451+0000 7fa1fe026640 1 --2- 172.21.15.99:0/2644245572 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c2a92dec60 0x55c2a92e91f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:24.456 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.451+0000 7fa1fe026640 1 -- 172.21.15.99:0/2644245572 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c2a92d9af0 msgr2=0x55c2a92e8250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:47:24.456 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.451+0000 7fa1fe026640 1 --2- 172.21.15.99:0/2644245572 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c2a92d9af0 0x55c2a92e8250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:24.456 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.451+0000 7fa1fe026640 1 -- 172.21.15.99:0/2644245572 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55c2a92e97e0 con 0x55c2a92d9100 2024-03-19T01:47:24.456 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.451+0000 7fa1fe026640 1 --2- 172.21.15.99:0/2644245572 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c2a92d9100 0x55c2a92e7d10 secure :-1 s=READY pgs=471 cs=0 l=1 rev1=1 crypto rx=0x7fa1f402cec0 tx=0x7fa1f4087d70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=38572a96e3fe51bd server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:47:24.457 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.451+0000 7fa1e1ffb640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:47:24.457 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.452+0000 7fa1e1ffb640 1 -- 172.21.15.99:0/2644245572 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa1f4004c80 con 0x55c2a92d9100 2024-03-19T01:47:24.457 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.452+0000 7fa1e1ffb640 1 -- 172.21.15.99:0/2644245572 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7fa1f40035f0 con 0x55c2a92d9100 2024-03-19T01:47:24.457 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.452+0000 7fa202e71740 1 -- 172.21.15.99:0/2644245572 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fa1f0011020 con 0x55c2a92d9100 2024-03-19T01:47:24.457 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.452+0000 7fa1e1ffb640 1 -- 172.21.15.99:0/2644245572 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7fa1f40bcfb0 con 0x55c2a92d9100 2024-03-19T01:47:24.457 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.452+0000 7fa202e71740 10 client.15852 fetch_fsmap learned FSMap version 27 2024-03-19T01:47:24.457 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.452+0000 7fa202e71740 1 -- 172.21.15.99:0/2644245572 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x55c2a93550d0 con 0x55c2a92d9100 2024-03-19T01:47:24.457 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.452+0000 7fa1e1ffb640 1 -- 172.21.15.99:0/2644245572 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa1f0011020 con 0x55c2a92d9100 2024-03-19T01:47:24.457 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.452+0000 7fa1e1ffb640 1 -- 172.21.15.99:0/2644245572 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 27) ==== 8338+0+0 (secure 0 0 0) 0x7fa1f40bd210 con 0x55c2a92d9100 2024-03-19T01:47:24.457 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.452+0000 7fa202e71740 10 client.15852 fetch_fsmap finished waiting for FSMap version 27 2024-03-19T01:47:24.457 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.452+0000 7fa202e71740 10 client.15852 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:47:24.457 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.452+0000 7fa202e71740 20 client.15852 populate_metadata read hostname 'smithi099' 2024-03-19T01:47:24.457 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.452+0000 7fa202e71740 1 --2- 172.21.15.99:0/2644245572 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55c2a9355a00 0x55c2a9357e20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:24.457 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.452+0000 7fa202e71740 4 client.15852 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:47:24.458 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.452+0000 7fa202e71740 1 -- 172.21.15.99:0/2644245572 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55c2a9245470 con 0x55c2a9355a00 2024-03-19T01:47:24.458 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.452+0000 7fa1fd825640 1 --2- 172.21.15.99:0/2644245572 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55c2a9355a00 0x55c2a9357e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:24.458 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.453+0000 7fa1fd825640 1 --2- 172.21.15.99:0/2644245572 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55c2a9355a00 0x55c2a9357e20 crc :-1 s=READY pgs=86 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=2ad28ccd1eea497b server_cookie=9ed3ce144bb07df in_seq=0 out_seq=0 2024-03-19T01:47:24.458 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.453+0000 7fa1e1ffb640 10 client.15852 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:47:24.502 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.501+0000 7fa1e1ffb640 1 -- 172.21.15.99:0/2644245572 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x55c2a9245470 con 0x55c2a9355a00 2024-03-19T01:47:24.502 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.501+0000 7fa1e1ffb640 10 client.15852 handle_command_reply: tid=0 2024-03-19T01:47:24.502 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.501+0000 7fa202e71740 1 client.15852 shutdown 2024-03-19T01:47:24.502 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.502+0000 7fa202e71740 1 -- 172.21.15.99:0/2644245572 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c2a92d9100 msgr2=0x55c2a92e7d10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:47:24.503 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.502+0000 7fa202e71740 1 --2- 172.21.15.99:0/2644245572 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c2a92d9100 0x55c2a92e7d10 secure :-1 s=READY pgs=471 cs=0 l=1 rev1=1 crypto rx=0x7fa1f402cec0 tx=0x7fa1f4087d70 comp rx=0 tx=0).stop 2024-03-19T01:47:24.503 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.502+0000 7fa202e71740 1 -- 172.21.15.99:0/2644245572 shutdown_connections 2024-03-19T01:47:24.503 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.502+0000 7fa202e71740 1 --2- 172.21.15.99:0/2644245572 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55c2a9355a00 0x55c2a9357e20 crc :-1 s=READY pgs=86 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:24.503 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.502+0000 7fa202e71740 1 --2- 172.21.15.99:0/2644245572 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c2a92d9af0 0x55c2a92e8250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:24.503 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.502+0000 7fa202e71740 1 --2- 172.21.15.99:0/2644245572 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c2a92dec60 0x55c2a92e91f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:24.503 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.502+0000 7fa202e71740 1 --2- 172.21.15.99:0/2644245572 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c2a92d9100 0x55c2a92e7d10 unknown :-1 s=CLOSED pgs=471 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:24.503 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.502+0000 7fa1e1ffb640 0 client.15852 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:47:24.503 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.502+0000 7fa202e71740 1 -- 172.21.15.99:0/2644245572 >> 172.21.15.99:0/2644245572 conn(0x55c2a92d4670 msgr2=0x55c2a92d78d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:47:24.503 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.502+0000 7fa202e71740 1 -- 172.21.15.99:0/2644245572 shutdown_connections 2024-03-19T01:47:24.503 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.502+0000 7fa202e71740 1 -- 172.21.15.99:0/2644245572 wait complete. 2024-03-19T01:47:24.503 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.502+0000 7fa202e71740 20 client.15852 trim_cache size 0 max 16384 2024-03-19T01:47:24.503 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:47:24.504 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:47:24.504 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:47:24.504 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:47:24.504 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.502+0000 7fa20028a640 1 -- 172.21.15.99:0/3627570085 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa1e40a28a0 msgr2=0x7fa1e40a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:47:24.505 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.502+0000 7fa20028a640 1 --2- 172.21.15.99:0/3627570085 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa1e40a28a0 0x7fa1e40a4d60 secure :-1 s=READY pgs=280 cs=0 l=1 rev1=1 crypto rx=0x7fa1f00099e0 tx=0x7fa1f0009500 comp rx=0 tx=0).stop 2024-03-19T01:47:24.505 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.502+0000 7fa20028a640 1 -- 172.21.15.99:0/3627570085 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa1f8100740 msgr2=0x7fa1f8197ae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:47:24.505 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.502+0000 7fa20028a640 1 --2- 172.21.15.99:0/3627570085 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa1f8100740 0x7fa1f8197ae0 secure :-1 s=READY pgs=468 cs=0 l=1 rev1=1 crypto rx=0x7fa1f4000c00 tx=0x7fa1f402c640 comp rx=0 tx=0).stop 2024-03-19T01:47:24.505 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.502+0000 7fa20028a640 1 -- 172.21.15.99:0/3627570085 shutdown_connections 2024-03-19T01:47:24.505 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.502+0000 7fa20028a640 1 --2- 172.21.15.99:0/3627570085 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa1e40a28a0 0x7fa1e40a4d60 unknown :-1 s=CLOSED pgs=280 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:24.505 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.502+0000 7fa20028a640 1 --2- 172.21.15.99:0/3627570085 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa1f8101930 0x7fa1f8198020 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:24.505 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.502+0000 7fa20028a640 1 --2- 172.21.15.99:0/3627570085 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa1f81022f0 0x7fa1f819f090 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:24.505 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.502+0000 7fa20028a640 1 --2- 172.21.15.99:0/3627570085 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa1f8100740 0x7fa1f8197ae0 unknown :-1 s=CLOSED pgs=468 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:24.505 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.502+0000 7fa20028a640 1 -- 172.21.15.99:0/3627570085 >> 172.21.15.99:0/3627570085 conn(0x7fa1f80fbcb0 msgr2=0x7fa1f8104c30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:47:24.505 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.502+0000 7fa20028a640 1 -- 172.21.15.99:0/3627570085 shutdown_connections 2024-03-19T01:47:24.505 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:24.502+0000 7fa20028a640 1 -- 172.21.15.99:0/3627570085 wait complete. 2024-03-19T01:47:24.688 INFO:tasks.workunit.client.0.smithi099.stdout: 1 750778 11.28 MB/sec execute 234 sec latency 66.961 ms 2024-03-19T01:47:25.688 INFO:tasks.workunit.client.0.smithi099.stdout: 1 753170 11.29 MB/sec execute 235 sec latency 22.014 ms 2024-03-19T01:47:26.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:47:25 smithi115 ceph-mon[67332]: pgmap v540: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 864 KiB/s rd, 2.8 MiB/s wr, 252 op/s 2024-03-19T01:47:26.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:47:25 smithi099 ceph-mon[62951]: pgmap v540: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 864 KiB/s rd, 2.8 MiB/s wr, 252 op/s 2024-03-19T01:47:26.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:47:25 smithi112 ceph-mon[66819]: pgmap v540: 97 pgs: 97 active+clean; 236 MiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 864 KiB/s rd, 2.8 MiB/s wr, 252 op/s 2024-03-19T01:47:26.688 INFO:tasks.workunit.client.0.smithi099.stdout: 1 755553 11.29 MB/sec execute 236 sec latency 36.164 ms 2024-03-19T01:47:27.688 INFO:tasks.workunit.client.0.smithi099.stdout: 1 757504 11.28 MB/sec execute 237 sec latency 36.445 ms 2024-03-19T01:47:28.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:47:27 smithi115 ceph-mon[67332]: pgmap v541: 97 pgs: 97 active+clean; 238 MiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 988 KiB/s rd, 3.3 MiB/s wr, 308 op/s 2024-03-19T01:47:28.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:47:27 smithi099 ceph-mon[62951]: pgmap v541: 97 pgs: 97 active+clean; 238 MiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 988 KiB/s rd, 3.3 MiB/s wr, 308 op/s 2024-03-19T01:47:28.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:47:27 smithi112 ceph-mon[66819]: pgmap v541: 97 pgs: 97 active+clean; 238 MiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 988 KiB/s rd, 3.3 MiB/s wr, 308 op/s 2024-03-19T01:47:28.688 INFO:tasks.workunit.client.0.smithi099.stdout: 1 760158 11.31 MB/sec execute 238 sec latency 26.774 ms 2024-03-19T01:47:29.688 INFO:tasks.workunit.client.0.smithi099.stdout: 1 762200 11.29 MB/sec execute 239 sec latency 79.140 ms 2024-03-19T01:47:30.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:47:29 smithi115 ceph-mon[67332]: pgmap v542: 97 pgs: 97 active+clean; 238 MiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 704 KiB/s rd, 2.8 MiB/s wr, 236 op/s 2024-03-19T01:47:30.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:47:29 smithi099 ceph-mon[62951]: pgmap v542: 97 pgs: 97 active+clean; 238 MiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 704 KiB/s rd, 2.8 MiB/s wr, 236 op/s 2024-03-19T01:47:30.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:47:29 smithi112 ceph-mon[66819]: pgmap v542: 97 pgs: 97 active+clean; 238 MiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 704 KiB/s rd, 2.8 MiB/s wr, 236 op/s 2024-03-19T01:47:30.688 INFO:tasks.workunit.client.0.smithi099.stdout: 1 764018 11.29 MB/sec execute 240 sec latency 23.521 ms 2024-03-19T01:47:31.688 INFO:tasks.workunit.client.0.smithi099.stdout: 1 766918 11.32 MB/sec execute 241 sec latency 23.889 ms 2024-03-19T01:47:32.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:47:31 smithi115 ceph-mon[67332]: pgmap v543: 97 pgs: 97 active+clean; 242 MiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 923 KiB/s rd, 3.0 MiB/s wr, 271 op/s 2024-03-19T01:47:32.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:47:31 smithi099 ceph-mon[62951]: pgmap v543: 97 pgs: 97 active+clean; 242 MiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 923 KiB/s rd, 3.0 MiB/s wr, 271 op/s 2024-03-19T01:47:32.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:47:31 smithi112 ceph-mon[66819]: pgmap v543: 97 pgs: 97 active+clean; 242 MiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 923 KiB/s rd, 3.0 MiB/s wr, 271 op/s 2024-03-19T01:47:32.688 INFO:tasks.workunit.client.0.smithi099.stdout: 1 768652 11.30 MB/sec execute 242 sec latency 35.817 ms 2024-03-19T01:47:33.688 INFO:tasks.workunit.client.0.smithi099.stdout: 1 771048 11.31 MB/sec execute 243 sec latency 9.675 ms 2024-03-19T01:47:34.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:47:33 smithi115 ceph-mon[67332]: pgmap v544: 97 pgs: 97 active+clean; 243 MiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 3.5 MiB/s wr, 304 op/s 2024-03-19T01:47:34.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:47:33 smithi099 ceph-mon[62951]: pgmap v544: 97 pgs: 97 active+clean; 243 MiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 3.5 MiB/s wr, 304 op/s 2024-03-19T01:47:34.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:47:33 smithi112 ceph-mon[66819]: pgmap v544: 97 pgs: 97 active+clean; 243 MiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 3.5 MiB/s wr, 304 op/s 2024-03-19T01:47:34.688 INFO:tasks.workunit.client.0.smithi099.stdout: 1 773262 11.31 MB/sec execute 244 sec latency 65.249 ms 2024-03-19T01:47:35.688 INFO:tasks.workunit.client.0.smithi099.stdout: 1 775156 11.31 MB/sec execute 245 sec latency 37.448 ms 2024-03-19T01:47:36.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:47:35 smithi115 ceph-mon[67332]: pgmap v545: 97 pgs: 97 active+clean; 243 MiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 902 KiB/s rd, 2.9 MiB/s wr, 275 op/s 2024-03-19T01:47:36.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:47:35 smithi099 ceph-mon[62951]: pgmap v545: 97 pgs: 97 active+clean; 243 MiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 902 KiB/s rd, 2.9 MiB/s wr, 275 op/s 2024-03-19T01:47:36.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:47:35 smithi112 ceph-mon[66819]: pgmap v545: 97 pgs: 97 active+clean; 243 MiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 902 KiB/s rd, 2.9 MiB/s wr, 275 op/s 2024-03-19T01:47:36.688 INFO:tasks.workunit.client.0.smithi099.stdout: 1 777855 11.33 MB/sec execute 246 sec latency 12.871 ms 2024-03-19T01:47:37.595 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.591+0000 7f52578c0640 1 -- 172.21.15.99:0/757727007 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5250101930 msgr2=0x7f5250101db0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:47:37.595 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.591+0000 7f52578c0640 1 --2- 172.21.15.99:0/757727007 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5250101930 0x7f5250101db0 secure :-1 s=READY pgs=472 cs=0 l=1 rev1=1 crypto rx=0x7f5240009900 tx=0x7f524002bc30 comp rx=0 tx=0).stop 2024-03-19T01:47:37.595 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.591+0000 7f52578c0640 1 -- 172.21.15.99:0/757727007 shutdown_connections 2024-03-19T01:47:37.595 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.591+0000 7f52578c0640 1 --2- 172.21.15.99:0/757727007 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f52501022f0 0x7f5250108b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:37.595 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.591+0000 7f52578c0640 1 --2- 172.21.15.99:0/757727007 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5250101930 0x7f5250101db0 unknown :-1 s=CLOSED pgs=472 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:37.595 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.591+0000 7f52578c0640 1 --2- 172.21.15.99:0/757727007 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5250100740 0x7f5250100b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:37.595 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.591+0000 7f52578c0640 1 -- 172.21.15.99:0/757727007 >> 172.21.15.99:0/757727007 conn(0x7f52500fbcb0 msgr2=0x7f52500fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:47:37.595 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.591+0000 7f52578c0640 1 -- 172.21.15.99:0/757727007 shutdown_connections 2024-03-19T01:47:37.595 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.591+0000 7f52578c0640 1 -- 172.21.15.99:0/757727007 wait complete. 2024-03-19T01:47:37.595 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.592+0000 7f52578c0640 1 Processor -- start 2024-03-19T01:47:37.595 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.592+0000 7f52578c0640 1 -- start start 2024-03-19T01:47:37.596 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.592+0000 7f52578c0640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5250100740 0x7f5250069a10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:37.596 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.592+0000 7f52578c0640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5250101930 0x7f525019ebf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:37.596 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.592+0000 7f52578c0640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f52501022f0 0x7f52501a1140 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:37.596 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.592+0000 7f52578c0640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f525010c3c0 con 0x7f5250101930 2024-03-19T01:47:37.596 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.592+0000 7f52578c0640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f525010c240 con 0x7f52501022f0 2024-03-19T01:47:37.596 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.592+0000 7f52578c0640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f525010c540 con 0x7f5250100740 2024-03-19T01:47:37.596 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.592+0000 7f5254e5b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5250101930 0x7f525019ebf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:37.596 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.592+0000 7f525565c640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5250100740 0x7f5250069a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:37.596 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.592+0000 7f5254e5b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5250101930 0x7f525019ebf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:57352/0 (socket says 172.21.15.99:57352) 2024-03-19T01:47:37.596 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.592+0000 7f5254e5b640 1 -- 172.21.15.99:0/1083422414 learned_addr learned my addr 172.21.15.99:0/1083422414 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:47:37.596 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.592+0000 7f5255e5d640 1 --2- 172.21.15.99:0/1083422414 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f52501022f0 0x7f52501a1140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:37.596 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.592+0000 7f525565c640 1 -- 172.21.15.99:0/1083422414 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f52501022f0 msgr2=0x7f52501a1140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:47:37.596 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.592+0000 7f525565c640 1 --2- 172.21.15.99:0/1083422414 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f52501022f0 0x7f52501a1140 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:37.596 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.592+0000 7f525565c640 1 -- 172.21.15.99:0/1083422414 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5250101930 msgr2=0x7f525019ebf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:47:37.597 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.592+0000 7f525565c640 1 --2- 172.21.15.99:0/1083422414 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5250101930 0x7f525019ebf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:37.597 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.592+0000 7f525565c640 1 -- 172.21.15.99:0/1083422414 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f52501a1930 con 0x7f5250100740 2024-03-19T01:47:37.597 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.592+0000 7f5254e5b640 1 --2- 172.21.15.99:0/1083422414 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5250101930 0x7f525019ebf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:47:37.597 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.592+0000 7f5255e5d640 1 --2- 172.21.15.99:0/1083422414 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f52501022f0 0x7f52501a1140 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:47:37.597 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.592+0000 7f525565c640 1 --2- 172.21.15.99:0/1083422414 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5250100740 0x7f5250069a10 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7f524c005fd0 tx=0x7f524c00ef90 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:47:37.597 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.593+0000 7f52467fc640 1 -- 172.21.15.99:0/1083422414 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f524c003970 con 0x7f5250100740 2024-03-19T01:47:37.597 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.593+0000 7f52578c0640 1 -- 172.21.15.99:0/1083422414 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f52501a1bc0 con 0x7f5250100740 2024-03-19T01:47:37.597 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.593+0000 7f52467fc640 1 -- 172.21.15.99:0/1083422414 <== mon.1 v2:172.21.15.115:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f524c004030 con 0x7f5250100740 2024-03-19T01:47:37.597 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.593+0000 7f52467fc640 1 -- 172.21.15.99:0/1083422414 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f524c0107f0 con 0x7f5250100740 2024-03-19T01:47:37.597 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.593+0000 7f52578c0640 1 -- 172.21.15.99:0/1083422414 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f52501a2130 con 0x7f5250100740 2024-03-19T01:47:37.604 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.595+0000 7f52467fc640 1 -- 172.21.15.99:0/1083422414 <== mon.1 v2:172.21.15.115:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f524c0041e0 con 0x7f5250100740 2024-03-19T01:47:37.604 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.595+0000 7f525a585740 1 Processor -- start 2024-03-19T01:47:37.604 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.595+0000 7f525a585740 1 -- start start 2024-03-19T01:47:37.604 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.595+0000 7f525a585740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56480b6ec100 0x56480b6ec520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:37.604 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.595+0000 7f525a585740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56480b6ecaf0 0x56480b6f15f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:37.604 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.595+0000 7f525a585740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56480b6f1c60 0x56480b6f40f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:37.604 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.595+0000 7f525a585740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x56480b650be0 con 0x56480b6ecaf0 2024-03-19T01:47:37.604 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.595+0000 7f525a585740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x56480b5f9960 con 0x56480b6f1c60 2024-03-19T01:47:37.604 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.595+0000 7f525a585740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x56480b64cc80 con 0x56480b6ec100 2024-03-19T01:47:37.605 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.595+0000 7f525565c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56480b6ecaf0 0x56480b6f15f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:37.605 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.596+0000 7f5254e5b640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56480b6f1c60 0x56480b6f40f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:37.605 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.596+0000 7f5254e5b640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56480b6f1c60 0x56480b6f40f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:34706/0 (socket says 172.21.15.99:34706) 2024-03-19T01:47:37.605 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.596+0000 7f5254e5b640 1 -- 172.21.15.99:0/3962114734 learned_addr learned my addr 172.21.15.99:0/3962114734 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:47:37.605 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.596+0000 7f5255e5d640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56480b6ec100 0x56480b6ec520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:37.605 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.596+0000 7f52467fc640 1 --2- 172.21.15.99:0/1083422414 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f523c0a28d0 0x7f523c0a4d90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:37.605 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.596+0000 7f5254e5b640 1 -- 172.21.15.99:0/3962114734 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56480b6ec100 msgr2=0x56480b6ec520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:47:37.605 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.596+0000 7f5254e5b640 1 --2- 172.21.15.99:0/3962114734 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56480b6ec100 0x56480b6ec520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:37.605 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.596+0000 7f5254e5b640 1 -- 172.21.15.99:0/3962114734 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56480b6ecaf0 msgr2=0x56480b6f15f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:47:37.605 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.596+0000 7f5254e5b640 1 --2- 172.21.15.99:0/3962114734 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56480b6ecaf0 0x56480b6f15f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:37.605 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.596+0000 7f5254e5b640 1 -- 172.21.15.99:0/3962114734 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x56480b5f0d20 con 0x56480b6f1c60 2024-03-19T01:47:37.605 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.596+0000 7f5255e5d640 1 --2- 172.21.15.99:0/1083422414 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f523c0a28d0 0x7f523c0a4d90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:37.605 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.596+0000 7f52467fc640 1 -- 172.21.15.99:0/1083422414 <== mon.1 v2:172.21.15.115:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f524c0cf260 con 0x7f5250100740 2024-03-19T01:47:37.606 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.596+0000 7f5254e5b640 1 --2- 172.21.15.99:0/3962114734 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56480b6f1c60 0x56480b6f40f0 secure :-1 s=READY pgs=127 cs=0 l=1 rev1=1 crypto rx=0x7f5240009230 tx=0x7f524002cd30 comp rx=0 tx=0).ready entity=mon.2 client_cookie=74770ba41560091d server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:47:37.606 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.596+0000 7f523b7fe640 1 -- 172.21.15.99:0/3962114734 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5240042070 con 0x56480b6f1c60 2024-03-19T01:47:37.606 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.596+0000 7f523b7fe640 1 -- 172.21.15.99:0/3962114734 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f5240004ba0 con 0x56480b6f1c60 2024-03-19T01:47:37.606 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.596+0000 7f523b7fe640 1 -- 172.21.15.99:0/3962114734 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f52400052f0 con 0x56480b6f1c60 2024-03-19T01:47:37.606 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.597+0000 7f525a585740 1 -- 172.21.15.99:0/3962114734 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56480b6f1c60 msgr2=0x56480b6f40f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:47:37.606 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.597+0000 7f5255e5d640 1 --2- 172.21.15.99:0/1083422414 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f523c0a28d0 0x7f523c0a4d90 secure :-1 s=READY pgs=281 cs=0 l=1 rev1=1 crypto rx=0x56480b6ed350 tx=0x7f52480046c0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:47:37.606 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.597+0000 7f525a585740 1 --2- 172.21.15.99:0/3962114734 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56480b6f1c60 0x56480b6f40f0 secure :-1 s=READY pgs=127 cs=0 l=1 rev1=1 crypto rx=0x7f5240009230 tx=0x7f524002cd30 comp rx=0 tx=0).stop 2024-03-19T01:47:37.606 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.597+0000 7f525a585740 1 -- 172.21.15.99:0/3962114734 shutdown_connections 2024-03-19T01:47:37.606 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.597+0000 7f525a585740 1 --2- 172.21.15.99:0/3962114734 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56480b6f1c60 0x56480b6f40f0 unknown :-1 s=CLOSED pgs=127 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:37.606 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.597+0000 7f525a585740 1 --2- 172.21.15.99:0/3962114734 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56480b6ecaf0 0x56480b6f15f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:37.606 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.597+0000 7f525a585740 1 --2- 172.21.15.99:0/3962114734 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56480b6ec100 0x56480b6ec520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:37.606 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.597+0000 7f525a585740 1 -- 172.21.15.99:0/3962114734 >> 172.21.15.99:0/3962114734 conn(0x56480b6e7670 msgr2=0x56480b6e7a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:47:37.606 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.597+0000 7f525a585740 1 -- 172.21.15.99:0/3962114734 shutdown_connections 2024-03-19T01:47:37.607 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.597+0000 7f525a585740 1 -- 172.21.15.99:0/3962114734 wait complete. 2024-03-19T01:47:37.607 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.597+0000 7f525a585740 1 -- start start 2024-03-19T01:47:37.607 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.597+0000 7f525a585740 1 Processor -- start 2024-03-19T01:47:37.607 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.598+0000 7f525a585740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56480b6ec100 0x56480b6f6080 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:37.607 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.598+0000 7f525a585740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56480b6ecaf0 0x56480b6f65c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:37.607 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.598+0000 7f525a585740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56480b6f1c60 0x56480b6fd600 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:37.607 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.598+0000 7f525a585740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x56480b6f4bb0 con 0x56480b6ec100 2024-03-19T01:47:37.607 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.598+0000 7f525a585740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x56480b60cdb0 con 0x56480b6f1c60 2024-03-19T01:47:37.607 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.598+0000 7f525a585740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x56480b6f4d30 con 0x56480b6ecaf0 2024-03-19T01:47:37.607 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.598+0000 7f525a585740 10 client.0 _ll_register_callbacks cb 0x56480b611e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:47:37.607 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.598+0000 7f525565c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56480b6ec100 0x56480b6f6080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:37.607 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.598+0000 7f5255e5d640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56480b6f1c60 0x56480b6fd600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:37.607 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.598+0000 7f525565c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56480b6ec100 0x56480b6f6080 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:57374/0 (socket says 172.21.15.99:57374) 2024-03-19T01:47:37.608 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.598+0000 7f525565c640 1 -- 172.21.15.99:0/1913284258 learned_addr learned my addr 172.21.15.99:0/1913284258 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:47:37.608 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.598+0000 7f5254e5b640 1 --2- 172.21.15.99:0/1913284258 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56480b6ecaf0 0x56480b6f65c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:37.608 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.598+0000 7f525565c640 1 -- 172.21.15.99:0/1913284258 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56480b6ecaf0 msgr2=0x56480b6f65c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:47:37.608 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.598+0000 7f525565c640 1 --2- 172.21.15.99:0/1913284258 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56480b6ecaf0 0x56480b6f65c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:37.608 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.598+0000 7f525565c640 1 -- 172.21.15.99:0/1913284258 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56480b6f1c60 msgr2=0x56480b6fd600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:47:37.608 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.598+0000 7f525565c640 1 --2- 172.21.15.99:0/1913284258 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56480b6f1c60 0x56480b6fd600 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:37.608 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.598+0000 7f525565c640 1 -- 172.21.15.99:0/1913284258 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x56480b60bc60 con 0x56480b6ec100 2024-03-19T01:47:37.608 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.598+0000 7f5255e5d640 1 --2- 172.21.15.99:0/1913284258 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56480b6f1c60 0x56480b6fd600 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:47:37.608 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.598+0000 7f525565c640 1 --2- 172.21.15.99:0/1913284258 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56480b6ec100 0x56480b6f6080 secure :-1 s=READY pgs=473 cs=0 l=1 rev1=1 crypto rx=0x7f524c098020 tx=0x7f524c0096c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=edc694ee59a0de7a server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:47:37.608 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.598+0000 7f5238ff9640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:47:37.608 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.598+0000 7f5238ff9640 1 -- 172.21.15.99:0/1913284258 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f524c09b700 con 0x56480b6ec100 2024-03-19T01:47:37.608 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.599+0000 7f5238ff9640 1 -- 172.21.15.99:0/1913284258 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f524c004530 con 0x56480b6ec100 2024-03-19T01:47:37.608 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.599+0000 7f525a585740 1 -- 172.21.15.99:0/1913284258 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f5240004ba0 con 0x56480b6ec100 2024-03-19T01:47:37.609 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.599+0000 7f5238ff9640 1 -- 172.21.15.99:0/1913284258 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7f524c09b520 con 0x56480b6ec100 2024-03-19T01:47:37.609 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.599+0000 7f525a585740 10 client.15864 fetch_fsmap learned FSMap version 27 2024-03-19T01:47:37.609 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.599+0000 7f525a585740 1 -- 172.21.15.99:0/1913284258 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x56480b365210 con 0x56480b6ec100 2024-03-19T01:47:37.609 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.599+0000 7f5238ff9640 1 -- 172.21.15.99:0/1913284258 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f5240004ba0 con 0x56480b6ec100 2024-03-19T01:47:37.609 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.599+0000 7f5238ff9640 1 -- 172.21.15.99:0/1913284258 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 27) ==== 8338+0+0 (secure 0 0 0) 0x7f524c0ba250 con 0x56480b6ec100 2024-03-19T01:47:37.609 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.599+0000 7f525a585740 10 client.15864 fetch_fsmap finished waiting for FSMap version 27 2024-03-19T01:47:37.609 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.599+0000 7f525a585740 10 client.15864 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:47:37.609 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.599+0000 7f525a585740 20 client.15864 populate_metadata read hostname 'smithi099' 2024-03-19T01:47:37.609 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.599+0000 7f525a585740 1 --2- 172.21.15.99:0/1913284258 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56480b701f90 0x56480b7043a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:37.609 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.599+0000 7f525a585740 4 client.15864 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:47:37.609 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.599+0000 7f525a585740 1 -- 172.21.15.99:0/1913284258 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x56480b65e120 con 0x56480b701f90 2024-03-19T01:47:37.609 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.599+0000 7f5254e5b640 1 --2- 172.21.15.99:0/1913284258 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56480b701f90 0x56480b7043a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:37.609 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.600+0000 7f5254e5b640 1 --2- 172.21.15.99:0/1913284258 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56480b701f90 0x56480b7043a0 crc :-1 s=READY pgs=87 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=f2542a3e9df5fc45 server_cookie=6bb2c9601ebc0c77 in_seq=0 out_seq=0 2024-03-19T01:47:37.610 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.600+0000 7f5238ff9640 10 client.15864 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:47:37.610 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.601+0000 7f5238ff9640 1 -- 172.21.15.99:0/1913284258 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x56480b65e120 con 0x56480b701f90 2024-03-19T01:47:37.610 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.601+0000 7f5238ff9640 10 client.15864 handle_command_reply: tid=0 2024-03-19T01:47:37.610 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.601+0000 7f525a585740 1 client.15864 shutdown 2024-03-19T01:47:37.610 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.601+0000 7f525a585740 1 -- 172.21.15.99:0/1913284258 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56480b6ec100 msgr2=0x56480b6f6080 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:47:37.610 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.601+0000 7f525a585740 1 --2- 172.21.15.99:0/1913284258 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56480b6ec100 0x56480b6f6080 secure :-1 s=READY pgs=473 cs=0 l=1 rev1=1 crypto rx=0x7f524c098020 tx=0x7f524c0096c0 comp rx=0 tx=0).stop 2024-03-19T01:47:37.610 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.601+0000 7f525a585740 1 -- 172.21.15.99:0/1913284258 shutdown_connections 2024-03-19T01:47:37.610 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.601+0000 7f525a585740 1 --2- 172.21.15.99:0/1913284258 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56480b701f90 0x56480b7043a0 crc :-1 s=READY pgs=87 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:37.610 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.601+0000 7f525a585740 1 --2- 172.21.15.99:0/1913284258 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56480b6f1c60 0x56480b6fd600 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:37.610 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.601+0000 7f5238ff9640 0 client.15864 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:47:37.610 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.601+0000 7f525a585740 1 --2- 172.21.15.99:0/1913284258 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56480b6ecaf0 0x56480b6f65c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:37.610 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.601+0000 7f525a585740 1 --2- 172.21.15.99:0/1913284258 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56480b6ec100 0x56480b6f6080 unknown :-1 s=CLOSED pgs=473 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:37.610 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.601+0000 7f525a585740 1 -- 172.21.15.99:0/1913284258 >> 172.21.15.99:0/1913284258 conn(0x56480b6e7670 msgr2=0x56480b6e88c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:47:37.611 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.601+0000 7f525a585740 1 -- 172.21.15.99:0/1913284258 shutdown_connections 2024-03-19T01:47:37.611 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.601+0000 7f525a585740 1 -- 172.21.15.99:0/1913284258 wait complete. 2024-03-19T01:47:37.611 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.601+0000 7f525a585740 20 client.15864 trim_cache size 0 max 16384 2024-03-19T01:47:37.619 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.616+0000 7f525a585740 1 Processor -- start 2024-03-19T01:47:37.619 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.617+0000 7f525a585740 1 -- start start 2024-03-19T01:47:37.619 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.617+0000 7f525a585740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56480b6ec100 0x56480b6f6520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:37.619 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.617+0000 7f525a585740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56480b6ecaf0 0x56480b6fd5c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:37.619 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.617+0000 7f525a585740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56480b6f1c60 0x56480b6eafe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:37.620 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.617+0000 7f525a585740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f52501ad6b0 con 0x56480b6ecaf0 2024-03-19T01:47:37.620 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.617+0000 7f525a585740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f52501aaf80 con 0x56480b6ec100 2024-03-19T01:47:37.620 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.617+0000 7f525a585740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f52501a8dd0 con 0x56480b6f1c60 2024-03-19T01:47:37.620 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.617+0000 7f5254e5b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56480b6ecaf0 0x56480b6fd5c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:37.620 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.617+0000 7f525565c640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56480b6ec100 0x56480b6f6520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:37.620 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.617+0000 7f5255e5d640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56480b6f1c60 0x56480b6eafe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:37.620 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.617+0000 7f5254e5b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56480b6ecaf0 0x56480b6fd5c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:57376/0 (socket says 172.21.15.99:57376) 2024-03-19T01:47:37.620 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.617+0000 7f5254e5b640 1 -- 172.21.15.99:0/1370740039 learned_addr learned my addr 172.21.15.99:0/1370740039 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:47:37.620 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.617+0000 7f525565c640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56480b6ec100 0x56480b6f6520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:34714/0 (socket says 172.21.15.99:34714) 2024-03-19T01:47:37.620 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.617+0000 7f5254e5b640 1 -- 172.21.15.99:0/1370740039 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56480b6f1c60 msgr2=0x56480b6eafe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:47:37.620 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.617+0000 7f5254e5b640 1 --2- 172.21.15.99:0/1370740039 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56480b6f1c60 0x56480b6eafe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:37.620 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.617+0000 7f5254e5b640 1 -- 172.21.15.99:0/1370740039 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56480b6ec100 msgr2=0x56480b6f6520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:47:37.620 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.617+0000 7f5254e5b640 1 --2- 172.21.15.99:0/1370740039 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56480b6ec100 0x56480b6f6520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:37.621 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.617+0000 7f5254e5b640 1 -- 172.21.15.99:0/1370740039 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x56480b431620 con 0x56480b6ecaf0 2024-03-19T01:47:37.621 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.617+0000 7f525565c640 1 --2- 172.21.15.99:0/1370740039 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56480b6ec100 0x56480b6f6520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:47:37.621 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.617+0000 7f5254e5b640 1 --2- 172.21.15.99:0/1370740039 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56480b6ecaf0 0x56480b6fd5c0 secure :-1 s=READY pgs=474 cs=0 l=1 rev1=1 crypto rx=0x7f5240009430 tx=0x7f524002bc30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a066507e38c67024 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:47:37.621 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.617+0000 7f523affd640 1 -- 172.21.15.99:0/1370740039 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5240059070 con 0x56480b6ecaf0 2024-03-19T01:47:37.621 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.618+0000 7f523affd640 1 -- 172.21.15.99:0/1370740039 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f5240009520 con 0x56480b6ecaf0 2024-03-19T01:47:37.621 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.618+0000 7f523affd640 1 -- 172.21.15.99:0/1370740039 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5240042040 con 0x56480b6ecaf0 2024-03-19T01:47:37.621 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.618+0000 7f525a585740 1 -- 172.21.15.99:0/1370740039 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56480b6ecaf0 msgr2=0x56480b6fd5c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:47:37.621 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.618+0000 7f525a585740 1 --2- 172.21.15.99:0/1370740039 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56480b6ecaf0 0x56480b6fd5c0 secure :-1 s=READY pgs=474 cs=0 l=1 rev1=1 crypto rx=0x7f5240009430 tx=0x7f524002bc30 comp rx=0 tx=0).stop 2024-03-19T01:47:37.621 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.618+0000 7f525a585740 1 -- 172.21.15.99:0/1370740039 shutdown_connections 2024-03-19T01:47:37.621 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.618+0000 7f525a585740 1 --2- 172.21.15.99:0/1370740039 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56480b6f1c60 0x56480b6eafe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:37.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.618+0000 7f525a585740 1 --2- 172.21.15.99:0/1370740039 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56480b6ecaf0 0x56480b6fd5c0 unknown :-1 s=CLOSED pgs=474 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:37.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.618+0000 7f525a585740 1 --2- 172.21.15.99:0/1370740039 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56480b6ec100 0x56480b6f6520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:37.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.618+0000 7f525a585740 1 -- 172.21.15.99:0/1370740039 >> 172.21.15.99:0/1370740039 conn(0x56480b6e7670 msgr2=0x56480b6ea8d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:47:37.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.618+0000 7f525a585740 1 -- 172.21.15.99:0/1370740039 shutdown_connections 2024-03-19T01:47:37.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.618+0000 7f525a585740 1 -- 172.21.15.99:0/1370740039 wait complete. 2024-03-19T01:47:37.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.618+0000 7f525a585740 1 -- start start 2024-03-19T01:47:37.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.619+0000 7f525a585740 1 Processor -- start 2024-03-19T01:47:37.625 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.619+0000 7f525a585740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56480b6ec100 0x56480b6faf60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:37.625 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.619+0000 7f525a585740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56480b6ecaf0 0x56480b6fb4a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:37.625 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.619+0000 7f525a585740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56480b6f1c60 0x56480b6fc4b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:37.625 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.619+0000 7f525a585740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f52501a9440 con 0x56480b6ec100 2024-03-19T01:47:37.625 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.619+0000 7f525a585740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f52501aa8b0 con 0x56480b6ecaf0 2024-03-19T01:47:37.625 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.619+0000 7f525a585740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f52501a9b10 con 0x56480b6f1c60 2024-03-19T01:47:37.626 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.619+0000 7f525a585740 10 client.0 _ll_register_callbacks cb 0x7f524c004830 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:47:37.626 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.619+0000 7f525565c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56480b6ec100 0x56480b6faf60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:37.626 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.619+0000 7f5254e5b640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56480b6ecaf0 0x56480b6fb4a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:37.626 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.619+0000 7f525565c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56480b6ec100 0x56480b6faf60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:57384/0 (socket says 172.21.15.99:57384) 2024-03-19T01:47:37.626 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.619+0000 7f525565c640 1 -- 172.21.15.99:0/2750057851 learned_addr learned my addr 172.21.15.99:0/2750057851 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:47:37.626 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.619+0000 7f5254e5b640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56480b6ecaf0 0x56480b6fb4a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:34730/0 (socket says 172.21.15.99:34730) 2024-03-19T01:47:37.626 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.619+0000 7f5255e5d640 1 --2- 172.21.15.99:0/2750057851 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56480b6f1c60 0x56480b6fc4b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:37.626 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.619+0000 7f525565c640 1 -- 172.21.15.99:0/2750057851 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56480b6f1c60 msgr2=0x56480b6fc4b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:47:37.626 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.619+0000 7f525565c640 1 --2- 172.21.15.99:0/2750057851 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56480b6f1c60 0x56480b6fc4b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:37.626 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.619+0000 7f525565c640 1 -- 172.21.15.99:0/2750057851 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56480b6ecaf0 msgr2=0x56480b6fb4a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:47:37.626 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.619+0000 7f525565c640 1 --2- 172.21.15.99:0/2750057851 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56480b6ecaf0 0x56480b6fb4a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:37.626 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.619+0000 7f525565c640 1 -- 172.21.15.99:0/2750057851 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x56480b6fcaa0 con 0x56480b6ec100 2024-03-19T01:47:37.626 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.619+0000 7f5254e5b640 1 --2- 172.21.15.99:0/2750057851 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56480b6ecaf0 0x56480b6fb4a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:47:37.627 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.620+0000 7f525565c640 1 --2- 172.21.15.99:0/2750057851 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56480b6ec100 0x56480b6faf60 secure :-1 s=READY pgs=475 cs=0 l=1 rev1=1 crypto rx=0x7f524c09bb50 tx=0x7f524c09b430 comp rx=0 tx=0).ready entity=mon.0 client_cookie=72814f7a450e669c server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:47:37.627 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.620+0000 7f5238ff9640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:47:37.627 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.620+0000 7f5238ff9640 1 -- 172.21.15.99:0/2750057851 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f524c097e60 con 0x56480b6ec100 2024-03-19T01:47:37.627 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.620+0000 7f5238ff9640 1 -- 172.21.15.99:0/2750057851 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f524c003d10 con 0x56480b6ec100 2024-03-19T01:47:37.627 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.620+0000 7f525a585740 1 -- 172.21.15.99:0/2750057851 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f5240009520 con 0x56480b6ec100 2024-03-19T01:47:37.627 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.620+0000 7f5238ff9640 1 -- 172.21.15.99:0/2750057851 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7f524c09c2c0 con 0x56480b6ec100 2024-03-19T01:47:37.627 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.620+0000 7f525a585740 10 client.15870 fetch_fsmap learned FSMap version 27 2024-03-19T01:47:37.627 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.620+0000 7f525a585740 1 -- 172.21.15.99:0/2750057851 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x56480b768600 con 0x56480b6ec100 2024-03-19T01:47:37.627 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.620+0000 7f5238ff9640 1 -- 172.21.15.99:0/2750057851 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f5240009520 con 0x56480b6ec100 2024-03-19T01:47:37.627 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.620+0000 7f5238ff9640 1 -- 172.21.15.99:0/2750057851 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 27) ==== 8338+0+0 (secure 0 0 0) 0x7f524c0ba280 con 0x56480b6ec100 2024-03-19T01:47:37.627 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.620+0000 7f525a585740 10 client.15870 fetch_fsmap finished waiting for FSMap version 27 2024-03-19T01:47:37.627 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.620+0000 7f525a585740 10 client.15870 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:47:37.627 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.620+0000 7f525a585740 20 client.15870 populate_metadata read hostname 'smithi099' 2024-03-19T01:47:37.627 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.620+0000 7f525a585740 1 --2- 172.21.15.99:0/2750057851 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56480b768ec0 0x56480b76b2e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:37.627 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.620+0000 7f525a585740 4 client.15870 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:47:37.628 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.620+0000 7f525a585740 1 -- 172.21.15.99:0/2750057851 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x56480b658470 con 0x56480b768ec0 2024-03-19T01:47:37.628 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.620+0000 7f5254e5b640 1 --2- 172.21.15.99:0/2750057851 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56480b768ec0 0x56480b76b2e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:37.628 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.621+0000 7f5238ff9640 10 client.15870 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:47:37.628 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:37.621+0000 7f5254e5b640 1 --2- 172.21.15.99:0/2750057851 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56480b768ec0 0x56480b76b2e0 crc :-1 s=READY pgs=88 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=aea8921c3f815070 server_cookie=edfafd52ba323797 in_seq=0 out_seq=0 2024-03-19T01:47:37.688 INFO:tasks.workunit.client.0.smithi099.stdout: 1 779992 11.31 MB/sec execute 247 sec latency 63.106 ms 2024-03-19T01:47:38.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:47:37 smithi115 ceph-mon[67332]: pgmap v546: 97 pgs: 97 active+clean; 244 MiB data, 2.3 GiB used, 1.0 TiB / 1.0 TiB avail; 816 KiB/s rd, 3.2 MiB/s wr, 257 op/s 2024-03-19T01:47:38.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:47:37 smithi099 ceph-mon[62951]: pgmap v546: 97 pgs: 97 active+clean; 244 MiB data, 2.3 GiB used, 1.0 TiB / 1.0 TiB avail; 816 KiB/s rd, 3.2 MiB/s wr, 257 op/s 2024-03-19T01:47:38.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:47:37 smithi112 ceph-mon[66819]: pgmap v546: 97 pgs: 97 active+clean; 244 MiB data, 2.3 GiB used, 1.0 TiB / 1.0 TiB avail; 816 KiB/s rd, 3.2 MiB/s wr, 257 op/s 2024-03-19T01:47:38.564 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:38.562+0000 7f5238ff9640 1 -- 172.21.15.99:0/2750057851 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x56480b658470 con 0x56480b768ec0 2024-03-19T01:47:38.564 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:38.562+0000 7f5238ff9640 10 client.15870 handle_command_reply: tid=0 2024-03-19T01:47:38.564 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:38.562+0000 7f525a585740 1 client.15870 shutdown 2024-03-19T01:47:38.564 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:38.562+0000 7f525a585740 1 -- 172.21.15.99:0/2750057851 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56480b6ec100 msgr2=0x56480b6faf60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:47:38.564 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:38.562+0000 7f525a585740 1 --2- 172.21.15.99:0/2750057851 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56480b6ec100 0x56480b6faf60 secure :-1 s=READY pgs=475 cs=0 l=1 rev1=1 crypto rx=0x7f524c09bb50 tx=0x7f524c09b430 comp rx=0 tx=0).stop 2024-03-19T01:47:38.564 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:38.562+0000 7f525a585740 1 -- 172.21.15.99:0/2750057851 shutdown_connections 2024-03-19T01:47:38.564 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:38.562+0000 7f525a585740 1 --2- 172.21.15.99:0/2750057851 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56480b768ec0 0x56480b76b2e0 crc :-1 s=READY pgs=88 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:38.564 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:38.563+0000 7f525a585740 1 --2- 172.21.15.99:0/2750057851 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56480b6ecaf0 0x56480b6fb4a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:38.564 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:38.563+0000 7f525a585740 1 --2- 172.21.15.99:0/2750057851 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56480b6f1c60 0x56480b6fc4b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:38.565 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:38.563+0000 7f5238ff9640 0 client.15870 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:47:38.565 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:38.563+0000 7f525a585740 1 --2- 172.21.15.99:0/2750057851 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56480b6ec100 0x56480b6faf60 unknown :-1 s=CLOSED pgs=475 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:38.565 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:38.563+0000 7f525a585740 1 -- 172.21.15.99:0/2750057851 >> 172.21.15.99:0/2750057851 conn(0x56480b6e7670 msgr2=0x56480b6f6310 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:47:38.565 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:38.563+0000 7f525a585740 1 -- 172.21.15.99:0/2750057851 shutdown_connections 2024-03-19T01:47:38.565 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:38.563+0000 7f525a585740 1 -- 172.21.15.99:0/2750057851 wait complete. 2024-03-19T01:47:38.565 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:38.563+0000 7f525a585740 20 client.15870 trim_cache size 0 max 16384 2024-03-19T01:47:38.565 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:47:38.565 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:47:38.565 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:47:38.565 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:47:38.566 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:38.563+0000 7f52578c0640 1 -- 172.21.15.99:0/1083422414 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f523c0a28d0 msgr2=0x7f523c0a4d90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:47:38.566 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:38.563+0000 7f52578c0640 1 --2- 172.21.15.99:0/1083422414 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f523c0a28d0 0x7f523c0a4d90 secure :-1 s=READY pgs=281 cs=0 l=1 rev1=1 crypto rx=0x56480b6ed350 tx=0x7f52480046c0 comp rx=0 tx=0).stop 2024-03-19T01:47:38.566 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:38.563+0000 7f52578c0640 1 -- 172.21.15.99:0/1083422414 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5250100740 msgr2=0x7f5250069a10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:47:38.566 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:38.563+0000 7f52578c0640 1 --2- 172.21.15.99:0/1083422414 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5250100740 0x7f5250069a10 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7f524c005fd0 tx=0x7f524c00ef90 comp rx=0 tx=0).stop 2024-03-19T01:47:38.566 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:38.563+0000 7f52578c0640 1 -- 172.21.15.99:0/1083422414 shutdown_connections 2024-03-19T01:47:38.567 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:38.563+0000 7f52578c0640 1 --2- 172.21.15.99:0/1083422414 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f523c0a28d0 0x7f523c0a4d90 unknown :-1 s=CLOSED pgs=281 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:38.567 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:38.563+0000 7f52578c0640 1 --2- 172.21.15.99:0/1083422414 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f52501022f0 0x7f52501a1140 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:38.567 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:38.563+0000 7f52578c0640 1 --2- 172.21.15.99:0/1083422414 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5250101930 0x7f525019ebf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:38.567 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:38.563+0000 7f52578c0640 1 --2- 172.21.15.99:0/1083422414 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5250100740 0x7f5250069a10 unknown :-1 s=CLOSED pgs=93 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:38.567 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:38.563+0000 7f52578c0640 1 -- 172.21.15.99:0/1083422414 >> 172.21.15.99:0/1083422414 conn(0x7f52500fbcb0 msgr2=0x7f5250104c30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:47:38.567 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:38.563+0000 7f52578c0640 1 -- 172.21.15.99:0/1083422414 shutdown_connections 2024-03-19T01:47:38.567 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:38.563+0000 7f52578c0640 1 -- 172.21.15.99:0/1083422414 wait complete. 2024-03-19T01:47:38.689 INFO:tasks.workunit.client.0.smithi099.stdout: 1 781941 11.32 MB/sec execute 248 sec latency 44.853 ms 2024-03-19T01:47:39.688 INFO:tasks.workunit.client.0.smithi099.stdout: 1 784500 11.33 MB/sec execute 249 sec latency 74.066 ms 2024-03-19T01:47:40.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:47:39 smithi115 ceph-mon[67332]: pgmap v547: 97 pgs: 97 active+clean; 247 MiB data, 2.3 GiB used, 1.0 TiB / 1.0 TiB avail; 882 KiB/s rd, 3.0 MiB/s wr, 246 op/s 2024-03-19T01:47:40.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:47:39 smithi099 ceph-mon[62951]: pgmap v547: 97 pgs: 97 active+clean; 247 MiB data, 2.3 GiB used, 1.0 TiB / 1.0 TiB avail; 882 KiB/s rd, 3.0 MiB/s wr, 246 op/s 2024-03-19T01:47:40.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:47:39 smithi112 ceph-mon[66819]: pgmap v547: 97 pgs: 97 active+clean; 247 MiB data, 2.3 GiB used, 1.0 TiB / 1.0 TiB avail; 882 KiB/s rd, 3.0 MiB/s wr, 246 op/s 2024-03-19T01:47:40.688 INFO:tasks.workunit.client.0.smithi099.stdout: 1 786240 11.32 MB/sec execute 250 sec latency 34.739 ms 2024-03-19T01:47:41.689 INFO:tasks.workunit.client.0.smithi099.stdout: 1 788692 11.33 MB/sec execute 251 sec latency 21.011 ms 2024-03-19T01:47:42.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:47:41 smithi115 ceph-mon[67332]: pgmap v548: 97 pgs: 97 active+clean; 247 MiB data, 2.3 GiB used, 1.0 TiB / 1.0 TiB avail; 901 KiB/s rd, 3.0 MiB/s wr, 268 op/s 2024-03-19T01:47:42.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:47:41 smithi099 ceph-mon[62951]: pgmap v548: 97 pgs: 97 active+clean; 247 MiB data, 2.3 GiB used, 1.0 TiB / 1.0 TiB avail; 901 KiB/s rd, 3.0 MiB/s wr, 268 op/s 2024-03-19T01:47:42.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:47:41 smithi112 ceph-mon[66819]: pgmap v548: 97 pgs: 97 active+clean; 247 MiB data, 2.3 GiB used, 1.0 TiB / 1.0 TiB avail; 901 KiB/s rd, 3.0 MiB/s wr, 268 op/s 2024-03-19T01:47:42.689 INFO:tasks.workunit.client.0.smithi099.stdout: 1 791158 11.34 MB/sec execute 252 sec latency 27.452 ms 2024-03-19T01:47:43.689 INFO:tasks.workunit.client.0.smithi099.stdout: 1 793157 11.34 MB/sec execute 253 sec latency 24.780 ms 2024-03-19T01:47:44.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:47:43 smithi115 ceph-mon[67332]: pgmap v549: 97 pgs: 97 active+clean; 248 MiB data, 2.3 GiB used, 1.0 TiB / 1.0 TiB avail; 812 KiB/s rd, 2.9 MiB/s wr, 249 op/s 2024-03-19T01:47:44.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:47:43 smithi099 ceph-mon[62951]: pgmap v549: 97 pgs: 97 active+clean; 248 MiB data, 2.3 GiB used, 1.0 TiB / 1.0 TiB avail; 812 KiB/s rd, 2.9 MiB/s wr, 249 op/s 2024-03-19T01:47:44.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:47:43 smithi112 ceph-mon[66819]: pgmap v549: 97 pgs: 97 active+clean; 248 MiB data, 2.3 GiB used, 1.0 TiB / 1.0 TiB avail; 812 KiB/s rd, 2.9 MiB/s wr, 249 op/s 2024-03-19T01:47:44.689 INFO:tasks.workunit.client.0.smithi099.stdout: 1 795545 11.35 MB/sec execute 254 sec latency 70.284 ms 2024-03-19T01:47:45.689 INFO:tasks.workunit.client.0.smithi099.stdout: 1 797097 11.32 MB/sec execute 255 sec latency 68.151 ms 2024-03-19T01:47:46.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:47:45 smithi115 ceph-mon[67332]: pgmap v550: 97 pgs: 97 active+clean; 248 MiB data, 2.3 GiB used, 1.0 TiB / 1.0 TiB avail; 643 KiB/s rd, 2.5 MiB/s wr, 222 op/s 2024-03-19T01:47:46.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:47:45 smithi099 ceph-mon[62951]: pgmap v550: 97 pgs: 97 active+clean; 248 MiB data, 2.3 GiB used, 1.0 TiB / 1.0 TiB avail; 643 KiB/s rd, 2.5 MiB/s wr, 222 op/s 2024-03-19T01:47:46.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:47:45 smithi112 ceph-mon[66819]: pgmap v550: 97 pgs: 97 active+clean; 248 MiB data, 2.3 GiB used, 1.0 TiB / 1.0 TiB avail; 643 KiB/s rd, 2.5 MiB/s wr, 222 op/s 2024-03-19T01:47:46.689 INFO:tasks.workunit.client.0.smithi099.stdout: 1 799398 11.33 MB/sec execute 256 sec latency 76.273 ms 2024-03-19T01:47:47.689 INFO:tasks.workunit.client.0.smithi099.stdout: 1 801861 11.34 MB/sec execute 257 sec latency 21.159 ms 2024-03-19T01:47:48.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:47:47 smithi115 ceph-mon[67332]: pgmap v551: 97 pgs: 97 active+clean; 253 MiB data, 2.4 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 3.6 MiB/s wr, 308 op/s 2024-03-19T01:47:48.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:47:47 smithi099 ceph-mon[62951]: pgmap v551: 97 pgs: 97 active+clean; 253 MiB data, 2.4 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 3.6 MiB/s wr, 308 op/s 2024-03-19T01:47:48.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:47:47 smithi112 ceph-mon[66819]: pgmap v551: 97 pgs: 97 active+clean; 253 MiB data, 2.4 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 3.6 MiB/s wr, 308 op/s 2024-03-19T01:47:48.689 INFO:tasks.workunit.client.0.smithi099.stdout: 1 803670 11.33 MB/sec execute 258 sec latency 37.667 ms 2024-03-19T01:47:49.689 INFO:tasks.workunit.client.0.smithi099.stdout: 1 806242 11.35 MB/sec execute 259 sec latency 31.597 ms 2024-03-19T01:47:50.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:47:49 smithi115 ceph-mon[67332]: pgmap v552: 97 pgs: 97 active+clean; 253 MiB data, 2.4 GiB used, 1.0 TiB / 1.0 TiB avail; 910 KiB/s rd, 3.0 MiB/s wr, 283 op/s 2024-03-19T01:47:50.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:47:49 smithi099 ceph-mon[62951]: pgmap v552: 97 pgs: 97 active+clean; 253 MiB data, 2.4 GiB used, 1.0 TiB / 1.0 TiB avail; 910 KiB/s rd, 3.0 MiB/s wr, 283 op/s 2024-03-19T01:47:50.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:47:49 smithi112 ceph-mon[66819]: pgmap v552: 97 pgs: 97 active+clean; 253 MiB data, 2.4 GiB used, 1.0 TiB / 1.0 TiB avail; 910 KiB/s rd, 3.0 MiB/s wr, 283 op/s 2024-03-19T01:47:50.689 INFO:tasks.workunit.client.0.smithi099.stdout: 1 807649 11.33 MB/sec execute 260 sec latency 88.610 ms 2024-03-19T01:47:51.654 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.652+0000 7f0b3fdb2640 1 -- 172.21.15.99:0/242319515 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0b38100760 msgr2=0x7f0b38100b80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:47:51.655 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.652+0000 7f0b3fdb2640 1 --2- 172.21.15.99:0/242319515 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0b38100760 0x7f0b38100b80 secure :-1 s=READY pgs=476 cs=0 l=1 rev1=1 crypto rx=0x7f0b30009ca0 tx=0x7f0b3002c0f0 comp rx=0 tx=0).stop 2024-03-19T01:47:51.655 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.652+0000 7f0b3fdb2640 1 -- 172.21.15.99:0/242319515 shutdown_connections 2024-03-19T01:47:51.655 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.652+0000 7f0b3fdb2640 1 --2- 172.21.15.99:0/242319515 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f0b38102310 0x7f0b38108ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:51.655 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.652+0000 7f0b3fdb2640 1 --2- 172.21.15.99:0/242319515 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f0b38101950 0x7f0b38101dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:51.655 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.652+0000 7f0b3fdb2640 1 --2- 172.21.15.99:0/242319515 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0b38100760 0x7f0b38100b80 unknown :-1 s=CLOSED pgs=476 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:51.655 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.652+0000 7f0b3fdb2640 1 -- 172.21.15.99:0/242319515 >> 172.21.15.99:0/242319515 conn(0x7f0b380fbcd0 msgr2=0x7f0b380fe110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:47:51.655 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.652+0000 7f0b3fdb2640 1 -- 172.21.15.99:0/242319515 shutdown_connections 2024-03-19T01:47:51.655 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.652+0000 7f0b3fdb2640 1 -- 172.21.15.99:0/242319515 wait complete. 2024-03-19T01:47:51.655 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.653+0000 7f0b3fdb2640 1 Processor -- start 2024-03-19T01:47:51.655 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.653+0000 7f0b3fdb2640 1 -- start start 2024-03-19T01:47:51.655 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.653+0000 7f0b3fdb2640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f0b38100760 0x7f0b38197a80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:51.655 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.653+0000 7f0b3fdb2640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f0b38101950 0x7f0b38197fc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:51.655 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.653+0000 7f0b3fdb2640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0b38102310 0x7f0b3819f030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:51.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.653+0000 7f0b3fdb2640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f0b3810c3b0 con 0x7f0b38102310 2024-03-19T01:47:51.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.653+0000 7f0b3fdb2640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f0b3810c230 con 0x7f0b38100760 2024-03-19T01:47:51.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.653+0000 7f0b3fdb2640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f0b3810c530 con 0x7f0b38101950 2024-03-19T01:47:51.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.653+0000 7f0b3e34f640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0b38102310 0x7f0b3819f030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:51.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.653+0000 7f0b3db4e640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f0b38100760 0x7f0b38197a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:51.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.653+0000 7f0b3d34d640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f0b38101950 0x7f0b38197fc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:51.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.653+0000 7f0b3e34f640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0b38102310 0x7f0b3819f030 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:53212/0 (socket says 172.21.15.99:53212) 2024-03-19T01:47:51.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.653+0000 7f0b3e34f640 1 -- 172.21.15.99:0/870313465 learned_addr learned my addr 172.21.15.99:0/870313465 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:47:51.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.654+0000 7f0b3e34f640 1 -- 172.21.15.99:0/870313465 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f0b38101950 msgr2=0x7f0b38197fc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:47:51.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.654+0000 7f0b3e34f640 1 --2- 172.21.15.99:0/870313465 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f0b38101950 0x7f0b38197fc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:51.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.654+0000 7f0b3e34f640 1 -- 172.21.15.99:0/870313465 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f0b38100760 msgr2=0x7f0b38197a80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:47:51.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.654+0000 7f0b3e34f640 1 --2- 172.21.15.99:0/870313465 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f0b38100760 0x7f0b38197a80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:51.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.654+0000 7f0b3e34f640 1 -- 172.21.15.99:0/870313465 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f0b3819f6a0 con 0x7f0b38102310 2024-03-19T01:47:51.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.654+0000 7f0b3d34d640 1 --2- 172.21.15.99:0/870313465 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f0b38101950 0x7f0b38197fc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:47:51.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.654+0000 7f0b3db4e640 1 --2- 172.21.15.99:0/870313465 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f0b38100760 0x7f0b38197a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:47:51.657 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.654+0000 7f0b3e34f640 1 --2- 172.21.15.99:0/870313465 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0b38102310 0x7f0b3819f030 secure :-1 s=READY pgs=477 cs=0 l=1 rev1=1 crypto rx=0x7f0b3400e970 tx=0x7f0b3400ed70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:47:51.657 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.654+0000 7f0b2affd640 1 -- 172.21.15.99:0/870313465 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0b3400ccc0 con 0x7f0b38102310 2024-03-19T01:47:51.657 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.654+0000 7f0b2affd640 1 -- 172.21.15.99:0/870313465 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f0b3400ce60 con 0x7f0b38102310 2024-03-19T01:47:51.657 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.654+0000 7f0b3fdb2640 1 -- 172.21.15.99:0/870313465 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f0b3819f990 con 0x7f0b38102310 2024-03-19T01:47:51.657 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.654+0000 7f0b2affd640 1 -- 172.21.15.99:0/870313465 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0b34010490 con 0x7f0b38102310 2024-03-19T01:47:51.657 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.654+0000 7f0b3fdb2640 1 -- 172.21.15.99:0/870313465 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f0b3819fe90 con 0x7f0b38102310 2024-03-19T01:47:51.664 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.656+0000 7f0b2affd640 1 -- 172.21.15.99:0/870313465 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f0b34005020 con 0x7f0b38102310 2024-03-19T01:47:51.664 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.656+0000 7f0b4299f740 1 Processor -- start 2024-03-19T01:47:51.664 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.657+0000 7f0b4299f740 1 -- start start 2024-03-19T01:47:51.664 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.657+0000 7f0b2affd640 1 --2- 172.21.15.99:0/870313465 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f0b240a28a0 0x7f0b240a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:51.664 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.657+0000 7f0b4299f740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55b32a654100 0x55b32a654520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:51.664 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.657+0000 7f0b4299f740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55b32a654af0 0x55b32a6595f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:51.664 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.657+0000 7f0b4299f740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55b32a659c60 0x55b32a65c0f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:51.664 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.657+0000 7f0b4299f740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55b32a5b8be0 con 0x55b32a659c60 2024-03-19T01:47:51.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.657+0000 7f0b4299f740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55b32a561960 con 0x55b32a654af0 2024-03-19T01:47:51.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.657+0000 7f0b4299f740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55b32a5b4c80 con 0x55b32a654100 2024-03-19T01:47:51.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.657+0000 7f0b2affd640 1 -- 172.21.15.99:0/870313465 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f0b340cef20 con 0x7f0b38102310 2024-03-19T01:47:51.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.657+0000 7f0b3d34d640 1 --2- 172.21.15.99:0/870313465 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f0b240a28a0 0x7f0b240a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:51.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.657+0000 7f0b3e34f640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55b32a659c60 0x55b32a65c0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:51.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.657+0000 7f0b3db4e640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55b32a654100 0x55b32a654520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:51.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.657+0000 7f0b3db4e640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55b32a654100 0x55b32a654520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:47008/0 (socket says 172.21.15.99:47008) 2024-03-19T01:47:51.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.657+0000 7f0b3db4e640 1 -- 172.21.15.99:0/4242024569 learned_addr learned my addr 172.21.15.99:0/4242024569 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:47:51.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.657+0000 7f0b3e34f640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55b32a659c60 0x55b32a65c0f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:53228/0 (socket says 172.21.15.99:53228) 2024-03-19T01:47:51.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.657+0000 7f0b3d34d640 1 --2- 172.21.15.99:0/4242024569 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55b32a654af0 0x55b32a6595f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:51.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.657+0000 7f0b3db4e640 1 -- 172.21.15.99:0/4242024569 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55b32a654af0 msgr2=0x55b32a6595f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:47:51.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.657+0000 7f0b3db4e640 1 --2- 172.21.15.99:0/4242024569 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55b32a654af0 0x55b32a6595f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:51.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.657+0000 7f0b3db4e640 1 -- 172.21.15.99:0/4242024569 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55b32a659c60 msgr2=0x55b32a65c0f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:47:51.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.657+0000 7f0b3db4e640 1 --2- 172.21.15.99:0/4242024569 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55b32a659c60 0x55b32a65c0f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:51.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.657+0000 7f0b3db4e640 1 -- 172.21.15.99:0/4242024569 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55b32a558d20 con 0x55b32a654100 2024-03-19T01:47:51.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.657+0000 7f0b3e34f640 1 --2- 172.21.15.99:0/4242024569 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55b32a659c60 0x55b32a65c0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:47:51.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.657+0000 7f0b3d34d640 1 --2- 172.21.15.99:0/870313465 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f0b240a28a0 0x7f0b240a4d60 secure :-1 s=READY pgs=282 cs=0 l=1 rev1=1 crypto rx=0x7f0b2c009730 tx=0x7f0b2c009ef0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:47:51.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.658+0000 7f0b3db4e640 1 --2- 172.21.15.99:0/4242024569 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55b32a654100 0x55b32a654520 secure :-1 s=READY pgs=94 cs=0 l=1 rev1=1 crypto rx=0x7f0b30009ca0 tx=0x7f0b30005ef0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=ce151a075e6ec08c server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:47:51.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.658+0000 7f0b23fff640 1 -- 172.21.15.99:0/4242024569 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0b30043070 con 0x55b32a654100 2024-03-19T01:47:51.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.658+0000 7f0b23fff640 1 -- 172.21.15.99:0/4242024569 <== mon.1 v2:172.21.15.115:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f0b3002cca0 con 0x55b32a654100 2024-03-19T01:47:51.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.658+0000 7f0b23fff640 1 -- 172.21.15.99:0/4242024569 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0b30039040 con 0x55b32a654100 2024-03-19T01:47:51.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.658+0000 7f0b4299f740 1 -- 172.21.15.99:0/4242024569 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55b32a654100 msgr2=0x55b32a654520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:47:51.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.658+0000 7f0b4299f740 1 --2- 172.21.15.99:0/4242024569 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55b32a654100 0x55b32a654520 secure :-1 s=READY pgs=94 cs=0 l=1 rev1=1 crypto rx=0x7f0b30009ca0 tx=0x7f0b30005ef0 comp rx=0 tx=0).stop 2024-03-19T01:47:51.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.659+0000 7f0b4299f740 1 -- 172.21.15.99:0/4242024569 shutdown_connections 2024-03-19T01:47:51.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.659+0000 7f0b4299f740 1 --2- 172.21.15.99:0/4242024569 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55b32a654af0 0x55b32a6595f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:51.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.659+0000 7f0b4299f740 1 --2- 172.21.15.99:0/4242024569 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55b32a659c60 0x55b32a65c0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:51.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.659+0000 7f0b4299f740 1 --2- 172.21.15.99:0/4242024569 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55b32a654100 0x55b32a654520 unknown :-1 s=CLOSED pgs=94 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:51.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.659+0000 7f0b4299f740 1 -- 172.21.15.99:0/4242024569 >> 172.21.15.99:0/4242024569 conn(0x55b32a64f670 msgr2=0x55b32a64fa70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:47:51.667 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.659+0000 7f0b4299f740 1 -- 172.21.15.99:0/4242024569 shutdown_connections 2024-03-19T01:47:51.667 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.659+0000 7f0b4299f740 1 -- 172.21.15.99:0/4242024569 wait complete. 2024-03-19T01:47:51.667 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.659+0000 7f0b4299f740 1 -- start start 2024-03-19T01:47:51.667 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.659+0000 7f0b4299f740 1 Processor -- start 2024-03-19T01:47:51.667 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.659+0000 7f0b4299f740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55b32a654100 0x55b32a65e040 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:51.667 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.659+0000 7f0b4299f740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55b32a654af0 0x55b32a65e580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:51.667 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.659+0000 7f0b4299f740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55b32a659c60 0x55b32a6655c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:51.667 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.659+0000 7f0b4299f740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55b32a65cbb0 con 0x55b32a654af0 2024-03-19T01:47:51.667 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.659+0000 7f0b4299f740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55b32a574db0 con 0x55b32a659c60 2024-03-19T01:47:51.667 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.659+0000 7f0b4299f740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55b32a65cd30 con 0x55b32a654100 2024-03-19T01:47:51.667 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.659+0000 7f0b4299f740 10 client.0 _ll_register_callbacks cb 0x55b32a579e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:47:51.667 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.659+0000 7f0b3d34d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55b32a654af0 0x55b32a65e580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:51.667 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.660+0000 7f0b3d34d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55b32a654af0 0x55b32a65e580 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:53242/0 (socket says 172.21.15.99:53242) 2024-03-19T01:47:51.667 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.660+0000 7f0b3db4e640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55b32a654100 0x55b32a65e040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:51.667 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.660+0000 7f0b3e34f640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55b32a659c60 0x55b32a6655c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:51.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.660+0000 7f0b3d34d640 1 -- 172.21.15.99:0/372909222 learned_addr learned my addr 172.21.15.99:0/372909222 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:47:51.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.660+0000 7f0b3d34d640 1 -- 172.21.15.99:0/372909222 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55b32a654100 msgr2=0x55b32a65e040 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:47:51.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.660+0000 7f0b3d34d640 1 --2- 172.21.15.99:0/372909222 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55b32a654100 0x55b32a65e040 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:51.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.660+0000 7f0b3d34d640 1 -- 172.21.15.99:0/372909222 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55b32a659c60 msgr2=0x55b32a6655c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:47:51.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.660+0000 7f0b3d34d640 1 --2- 172.21.15.99:0/372909222 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55b32a659c60 0x55b32a6655c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:51.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.660+0000 7f0b3d34d640 1 -- 172.21.15.99:0/372909222 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55b32a573c60 con 0x55b32a654af0 2024-03-19T01:47:51.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.660+0000 7f0b3e34f640 1 --2- 172.21.15.99:0/372909222 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55b32a659c60 0x55b32a6655c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:47:51.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.660+0000 7f0b3d34d640 1 --2- 172.21.15.99:0/372909222 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55b32a654af0 0x55b32a65e580 secure :-1 s=READY pgs=478 cs=0 l=1 rev1=1 crypto rx=0x7f0b2c01f6a0 tx=0x7f0b2c01d000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4db39a72ddf4d48b server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:47:51.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.660+0000 7f0b217fa640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:47:51.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.660+0000 7f0b217fa640 1 -- 172.21.15.99:0/372909222 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0b2c0217a0 con 0x55b32a654af0 2024-03-19T01:47:51.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.660+0000 7f0b217fa640 1 -- 172.21.15.99:0/372909222 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f0b2c021940 con 0x55b32a654af0 2024-03-19T01:47:51.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.660+0000 7f0b4299f740 1 -- 172.21.15.99:0/372909222 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f0b3002cca0 con 0x55b32a654af0 2024-03-19T01:47:51.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.660+0000 7f0b217fa640 1 -- 172.21.15.99:0/372909222 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7f0b2c021b40 con 0x55b32a654af0 2024-03-19T01:47:51.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.660+0000 7f0b4299f740 10 client.15882 fetch_fsmap learned FSMap version 27 2024-03-19T01:47:51.669 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.660+0000 7f0b4299f740 1 -- 172.21.15.99:0/372909222 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x55b32a2cd210 con 0x55b32a654af0 2024-03-19T01:47:51.669 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.660+0000 7f0b217fa640 1 -- 172.21.15.99:0/372909222 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f0b3002cca0 con 0x55b32a654af0 2024-03-19T01:47:51.669 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.661+0000 7f0b217fa640 1 -- 172.21.15.99:0/372909222 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 27) ==== 8338+0+0 (secure 0 0 0) 0x7f0b2c020050 con 0x55b32a654af0 2024-03-19T01:47:51.669 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.661+0000 7f0b4299f740 10 client.15882 fetch_fsmap finished waiting for FSMap version 27 2024-03-19T01:47:51.669 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.661+0000 7f0b4299f740 10 client.15882 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:47:51.669 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.661+0000 7f0b4299f740 20 client.15882 populate_metadata read hostname 'smithi099' 2024-03-19T01:47:51.669 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.661+0000 7f0b4299f740 1 --2- 172.21.15.99:0/372909222 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55b32a669f30 0x55b32a66c360 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:51.669 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.661+0000 7f0b4299f740 4 client.15882 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:47:51.669 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.661+0000 7f0b4299f740 1 -- 172.21.15.99:0/372909222 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55b32a5c6120 con 0x55b32a669f30 2024-03-19T01:47:51.669 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.661+0000 7f0b3db4e640 1 --2- 172.21.15.99:0/372909222 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55b32a669f30 0x55b32a66c360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:51.669 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.661+0000 7f0b217fa640 10 client.15882 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:47:51.669 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.661+0000 7f0b3db4e640 1 --2- 172.21.15.99:0/372909222 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55b32a669f30 0x55b32a66c360 crc :-1 s=READY pgs=89 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=c2efb8390493c3ec server_cookie=69b19bcaf0119e57 in_seq=0 out_seq=0 2024-03-19T01:47:51.669 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.662+0000 7f0b217fa640 1 -- 172.21.15.99:0/372909222 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x55b32a5c6120 con 0x55b32a669f30 2024-03-19T01:47:51.669 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.662+0000 7f0b217fa640 10 client.15882 handle_command_reply: tid=0 2024-03-19T01:47:51.669 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.662+0000 7f0b4299f740 1 client.15882 shutdown 2024-03-19T01:47:51.670 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.662+0000 7f0b4299f740 1 -- 172.21.15.99:0/372909222 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55b32a654af0 msgr2=0x55b32a65e580 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:47:51.670 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.662+0000 7f0b4299f740 1 --2- 172.21.15.99:0/372909222 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55b32a654af0 0x55b32a65e580 secure :-1 s=READY pgs=478 cs=0 l=1 rev1=1 crypto rx=0x7f0b2c01f6a0 tx=0x7f0b2c01d000 comp rx=0 tx=0).stop 2024-03-19T01:47:51.670 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.662+0000 7f0b4299f740 1 -- 172.21.15.99:0/372909222 shutdown_connections 2024-03-19T01:47:51.670 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.662+0000 7f0b4299f740 1 --2- 172.21.15.99:0/372909222 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55b32a669f30 0x55b32a66c360 crc :-1 s=READY pgs=89 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:51.670 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.662+0000 7f0b4299f740 1 --2- 172.21.15.99:0/372909222 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55b32a659c60 0x55b32a6655c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:51.670 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.662+0000 7f0b4299f740 1 --2- 172.21.15.99:0/372909222 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55b32a654af0 0x55b32a65e580 unknown :-1 s=CLOSED pgs=478 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:51.670 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.662+0000 7f0b217fa640 0 client.15882 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:47:51.670 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.662+0000 7f0b4299f740 1 --2- 172.21.15.99:0/372909222 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55b32a654100 0x55b32a65e040 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:51.670 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.662+0000 7f0b4299f740 1 -- 172.21.15.99:0/372909222 >> 172.21.15.99:0/372909222 conn(0x55b32a64f670 msgr2=0x55b32a6508c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:47:51.670 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.662+0000 7f0b4299f740 1 -- 172.21.15.99:0/372909222 shutdown_connections 2024-03-19T01:47:51.670 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.662+0000 7f0b4299f740 1 -- 172.21.15.99:0/372909222 wait complete. 2024-03-19T01:47:51.670 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.662+0000 7f0b4299f740 20 client.15882 trim_cache size 0 max 16384 2024-03-19T01:47:51.683 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.677+0000 7f0b4299f740 1 Processor -- start 2024-03-19T01:47:51.683 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.677+0000 7f0b4299f740 1 -- start start 2024-03-19T01:47:51.683 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.678+0000 7f0b4299f740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55b32a654100 0x55b32a64eab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:51.683 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.678+0000 7f0b4299f740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55b32a654af0 0x55b32a665580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:51.683 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.678+0000 7f0b4299f740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55b32a659c60 0x55b32a65e040 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:51.683 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.678+0000 7f0b4299f740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f0b381ab640 con 0x55b32a659c60 2024-03-19T01:47:51.683 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.678+0000 7f0b4299f740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f0b381a8f10 con 0x55b32a654100 2024-03-19T01:47:51.683 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.678+0000 7f0b4299f740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f0b381a6d60 con 0x55b32a654af0 2024-03-19T01:47:51.683 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.678+0000 7f0b3e34f640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55b32a659c60 0x55b32a65e040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:51.683 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.678+0000 7f0b3db4e640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55b32a654100 0x55b32a64eab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:51.683 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.678+0000 7f0b3e34f640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55b32a659c60 0x55b32a65e040 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:53258/0 (socket says 172.21.15.99:53258) 2024-03-19T01:47:51.683 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.678+0000 7f0b3e34f640 1 -- 172.21.15.99:0/3332293327 learned_addr learned my addr 172.21.15.99:0/3332293327 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:47:51.684 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.678+0000 7f0b3d34d640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55b32a654af0 0x55b32a665580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:51.684 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.678+0000 7f0b3e34f640 1 -- 172.21.15.99:0/3332293327 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55b32a654af0 msgr2=0x55b32a665580 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:47:51.684 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.678+0000 7f0b3e34f640 1 --2- 172.21.15.99:0/3332293327 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55b32a654af0 0x55b32a665580 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:51.684 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.678+0000 7f0b3e34f640 1 -- 172.21.15.99:0/3332293327 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55b32a654100 msgr2=0x55b32a64eab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:47:51.684 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.678+0000 7f0b3e34f640 1 --2- 172.21.15.99:0/3332293327 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55b32a654100 0x55b32a64eab0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:51.684 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.678+0000 7f0b3e34f640 1 -- 172.21.15.99:0/3332293327 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55b32a399620 con 0x55b32a659c60 2024-03-19T01:47:51.684 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.678+0000 7f0b3db4e640 1 --2- 172.21.15.99:0/3332293327 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55b32a654100 0x55b32a64eab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:47:51.684 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.678+0000 7f0b3e34f640 1 --2- 172.21.15.99:0/3332293327 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55b32a659c60 0x55b32a65e040 secure :-1 s=READY pgs=479 cs=0 l=1 rev1=1 crypto rx=0x7f0b34098250 tx=0x7f0b34097b20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=fc4d21b4668ef09c server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:47:51.684 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.678+0000 7f0b237fe640 1 -- 172.21.15.99:0/3332293327 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0b3409f9d0 con 0x55b32a659c60 2024-03-19T01:47:51.684 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.678+0000 7f0b237fe640 1 -- 172.21.15.99:0/3332293327 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f0b340a4020 con 0x55b32a659c60 2024-03-19T01:47:51.684 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.678+0000 7f0b237fe640 1 -- 172.21.15.99:0/3332293327 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0b340860b0 con 0x55b32a659c60 2024-03-19T01:47:51.684 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.679+0000 7f0b4299f740 1 -- 172.21.15.99:0/3332293327 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55b32a659c60 msgr2=0x55b32a65e040 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:47:51.684 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.679+0000 7f0b4299f740 1 --2- 172.21.15.99:0/3332293327 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55b32a659c60 0x55b32a65e040 secure :-1 s=READY pgs=479 cs=0 l=1 rev1=1 crypto rx=0x7f0b34098250 tx=0x7f0b34097b20 comp rx=0 tx=0).stop 2024-03-19T01:47:51.684 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.679+0000 7f0b4299f740 1 -- 172.21.15.99:0/3332293327 shutdown_connections 2024-03-19T01:47:51.685 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.679+0000 7f0b4299f740 1 --2- 172.21.15.99:0/3332293327 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55b32a659c60 0x55b32a65e040 unknown :-1 s=CLOSED pgs=479 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:51.685 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.679+0000 7f0b4299f740 1 --2- 172.21.15.99:0/3332293327 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55b32a654af0 0x55b32a665580 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:51.685 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.679+0000 7f0b4299f740 1 --2- 172.21.15.99:0/3332293327 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55b32a654100 0x55b32a64eab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:51.685 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.679+0000 7f0b4299f740 1 -- 172.21.15.99:0/3332293327 >> 172.21.15.99:0/3332293327 conn(0x55b32a64f670 msgr2=0x55b32a6528d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:47:51.685 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.679+0000 7f0b4299f740 1 -- 172.21.15.99:0/3332293327 shutdown_connections 2024-03-19T01:47:51.685 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.679+0000 7f0b4299f740 1 -- 172.21.15.99:0/3332293327 wait complete. 2024-03-19T01:47:51.685 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.679+0000 7f0b4299f740 1 -- start start 2024-03-19T01:47:51.685 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.679+0000 7f0b4299f740 1 Processor -- start 2024-03-19T01:47:51.685 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.680+0000 7f0b4299f740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55b32a654100 0x55b32a664de0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:51.685 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.680+0000 7f0b4299f740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55b32a654af0 0x55b32a665320 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:51.685 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.680+0000 7f0b4299f740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55b32a659c60 0x55b32a663440 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:51.686 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.680+0000 7f0b4299f740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f0b381a73d0 con 0x55b32a654100 2024-03-19T01:47:51.686 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.680+0000 7f0b4299f740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f0b381a8840 con 0x55b32a654af0 2024-03-19T01:47:51.686 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.680+0000 7f0b4299f740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f0b381a7aa0 con 0x55b32a659c60 2024-03-19T01:47:51.686 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.680+0000 7f0b4299f740 10 client.0 _ll_register_callbacks cb 0x7f0b2c009bb0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:47:51.686 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.680+0000 7f0b3db4e640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55b32a654100 0x55b32a664de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:51.686 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.680+0000 7f0b3db4e640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55b32a654100 0x55b32a664de0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:53264/0 (socket says 172.21.15.99:53264) 2024-03-19T01:47:51.686 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.680+0000 7f0b3db4e640 1 -- 172.21.15.99:0/1946713398 learned_addr learned my addr 172.21.15.99:0/1946713398 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:47:51.686 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.680+0000 7f0b3d34d640 1 --2- 172.21.15.99:0/1946713398 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55b32a654af0 0x55b32a665320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:51.686 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.680+0000 7f0b3e34f640 1 --2- 172.21.15.99:0/1946713398 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55b32a659c60 0x55b32a663440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:51.686 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.680+0000 7f0b3db4e640 1 -- 172.21.15.99:0/1946713398 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55b32a659c60 msgr2=0x55b32a663440 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:47:51.686 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.680+0000 7f0b3db4e640 1 --2- 172.21.15.99:0/1946713398 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55b32a659c60 0x55b32a663440 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:51.686 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.680+0000 7f0b3db4e640 1 -- 172.21.15.99:0/1946713398 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55b32a654af0 msgr2=0x55b32a665320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:47:51.686 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.680+0000 7f0b3db4e640 1 --2- 172.21.15.99:0/1946713398 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55b32a654af0 0x55b32a665320 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:51.686 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.680+0000 7f0b3db4e640 1 -- 172.21.15.99:0/1946713398 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55b32a663a80 con 0x55b32a654100 2024-03-19T01:47:51.686 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.680+0000 7f0b3d34d640 1 --2- 172.21.15.99:0/1946713398 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55b32a654af0 0x55b32a665320 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:47:51.687 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.680+0000 7f0b3db4e640 1 --2- 172.21.15.99:0/1946713398 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55b32a654100 0x55b32a664de0 secure :-1 s=READY pgs=480 cs=0 l=1 rev1=1 crypto rx=0x7f0b300324c0 tx=0x7f0b3002c0f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d1d062e948dd7bb4 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:47:51.687 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.680+0000 7f0b217fa640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:47:51.687 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.680+0000 7f0b217fa640 1 -- 172.21.15.99:0/1946713398 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0b30056070 con 0x55b32a654100 2024-03-19T01:47:51.687 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.680+0000 7f0b217fa640 1 -- 172.21.15.99:0/1946713398 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f0b30032690 con 0x55b32a654100 2024-03-19T01:47:51.687 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.680+0000 7f0b4299f740 1 -- 172.21.15.99:0/1946713398 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f0b340a4020 con 0x55b32a654100 2024-03-19T01:47:51.687 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.680+0000 7f0b217fa640 1 -- 172.21.15.99:0/1946713398 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7f0b30061050 con 0x55b32a654100 2024-03-19T01:47:51.687 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.681+0000 7f0b4299f740 10 client.15888 fetch_fsmap learned FSMap version 27 2024-03-19T01:47:51.687 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.681+0000 7f0b4299f740 1 -- 172.21.15.99:0/1946713398 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x55b32a6d0ca0 con 0x55b32a654100 2024-03-19T01:47:51.687 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.681+0000 7f0b217fa640 1 -- 172.21.15.99:0/1946713398 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f0b340a4020 con 0x55b32a654100 2024-03-19T01:47:51.687 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.681+0000 7f0b217fa640 1 -- 172.21.15.99:0/1946713398 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 27) ==== 8338+0+0 (secure 0 0 0) 0x7f0b30052610 con 0x55b32a654100 2024-03-19T01:47:51.687 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.681+0000 7f0b4299f740 10 client.15888 fetch_fsmap finished waiting for FSMap version 27 2024-03-19T01:47:51.687 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.681+0000 7f0b4299f740 10 client.15888 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:47:51.687 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.681+0000 7f0b4299f740 20 client.15888 populate_metadata read hostname 'smithi099' 2024-03-19T01:47:51.687 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.681+0000 7f0b4299f740 1 --2- 172.21.15.99:0/1946713398 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55b32a6d1210 0x55b32a6d3600 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:47:51.687 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.681+0000 7f0b4299f740 4 client.15888 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:47:51.687 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.681+0000 7f0b4299f740 1 -- 172.21.15.99:0/1946713398 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55b32a5c0470 con 0x55b32a6d1210 2024-03-19T01:47:51.688 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.681+0000 7f0b3d34d640 1 --2- 172.21.15.99:0/1946713398 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55b32a6d1210 0x55b32a6d3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:47:51.688 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.681+0000 7f0b217fa640 10 client.15888 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:47:51.688 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.681+0000 7f0b3d34d640 1 --2- 172.21.15.99:0/1946713398 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55b32a6d1210 0x55b32a6d3600 crc :-1 s=READY pgs=90 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=6f34330a5b36dad9 server_cookie=abfe82ad74734c44 in_seq=0 out_seq=0 2024-03-19T01:47:51.689 INFO:tasks.workunit.client.0.smithi099.stdout: 1 810047 11.34 MB/sec execute 261 sec latency 8.870 ms 2024-03-19T01:47:51.741 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.740+0000 7f0b217fa640 1 -- 172.21.15.99:0/1946713398 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x55b32a5c0470 con 0x55b32a6d1210 2024-03-19T01:47:51.741 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.740+0000 7f0b217fa640 10 client.15888 handle_command_reply: tid=0 2024-03-19T01:47:51.741 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.740+0000 7f0b4299f740 1 client.15888 shutdown 2024-03-19T01:47:51.741 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.740+0000 7f0b4299f740 1 -- 172.21.15.99:0/1946713398 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55b32a654100 msgr2=0x55b32a664de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:47:51.741 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.740+0000 7f0b4299f740 1 --2- 172.21.15.99:0/1946713398 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55b32a654100 0x55b32a664de0 secure :-1 s=READY pgs=480 cs=0 l=1 rev1=1 crypto rx=0x7f0b300324c0 tx=0x7f0b3002c0f0 comp rx=0 tx=0).stop 2024-03-19T01:47:51.741 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.740+0000 7f0b4299f740 1 -- 172.21.15.99:0/1946713398 shutdown_connections 2024-03-19T01:47:51.741 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.740+0000 7f0b4299f740 1 --2- 172.21.15.99:0/1946713398 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55b32a6d1210 0x55b32a6d3600 crc :-1 s=READY pgs=90 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:51.741 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.740+0000 7f0b4299f740 1 --2- 172.21.15.99:0/1946713398 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55b32a654af0 0x55b32a665320 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:51.741 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.740+0000 7f0b4299f740 1 --2- 172.21.15.99:0/1946713398 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55b32a659c60 0x55b32a663440 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:51.741 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.740+0000 7f0b217fa640 0 client.15888 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:47:51.741 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.740+0000 7f0b4299f740 1 --2- 172.21.15.99:0/1946713398 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55b32a654100 0x55b32a664de0 unknown :-1 s=CLOSED pgs=480 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:51.742 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.740+0000 7f0b4299f740 1 -- 172.21.15.99:0/1946713398 >> 172.21.15.99:0/1946713398 conn(0x55b32a64f670 msgr2=0x55b32a653160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:47:51.742 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.740+0000 7f0b4299f740 1 -- 172.21.15.99:0/1946713398 shutdown_connections 2024-03-19T01:47:51.742 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.740+0000 7f0b4299f740 1 -- 172.21.15.99:0/1946713398 wait complete. 2024-03-19T01:47:51.742 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.740+0000 7f0b4299f740 20 client.15888 trim_cache size 0 max 16384 2024-03-19T01:47:51.742 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:47:51.742 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:47:51.742 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:47:51.742 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:47:51.743 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.741+0000 7f0b3fdb2640 1 -- 172.21.15.99:0/870313465 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f0b240a28a0 msgr2=0x7f0b240a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:47:51.743 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.741+0000 7f0b3fdb2640 1 --2- 172.21.15.99:0/870313465 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f0b240a28a0 0x7f0b240a4d60 secure :-1 s=READY pgs=282 cs=0 l=1 rev1=1 crypto rx=0x7f0b2c009730 tx=0x7f0b2c009ef0 comp rx=0 tx=0).stop 2024-03-19T01:47:51.743 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.741+0000 7f0b3fdb2640 1 -- 172.21.15.99:0/870313465 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0b38102310 msgr2=0x7f0b3819f030 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:47:51.743 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.741+0000 7f0b3fdb2640 1 --2- 172.21.15.99:0/870313465 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0b38102310 0x7f0b3819f030 secure :-1 s=READY pgs=477 cs=0 l=1 rev1=1 crypto rx=0x7f0b3400e970 tx=0x7f0b3400ed70 comp rx=0 tx=0).stop 2024-03-19T01:47:51.743 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.741+0000 7f0b3fdb2640 1 -- 172.21.15.99:0/870313465 shutdown_connections 2024-03-19T01:47:51.744 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.741+0000 7f0b3fdb2640 1 --2- 172.21.15.99:0/870313465 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0b38102310 0x7f0b3819f030 unknown :-1 s=CLOSED pgs=477 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:51.744 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.741+0000 7f0b3fdb2640 1 --2- 172.21.15.99:0/870313465 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f0b38101950 0x7f0b38197fc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:51.744 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.741+0000 7f0b3fdb2640 1 --2- 172.21.15.99:0/870313465 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f0b240a28a0 0x7f0b240a4d60 unknown :-1 s=CLOSED pgs=282 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:51.744 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.741+0000 7f0b3fdb2640 1 --2- 172.21.15.99:0/870313465 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f0b38100760 0x7f0b38197a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:47:51.744 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.741+0000 7f0b3fdb2640 1 -- 172.21.15.99:0/870313465 >> 172.21.15.99:0/870313465 conn(0x7f0b380fbcd0 msgr2=0x7f0b380fd6d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:47:51.744 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.741+0000 7f0b3fdb2640 1 -- 172.21.15.99:0/870313465 shutdown_connections 2024-03-19T01:47:51.744 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:47:51.741+0000 7f0b3fdb2640 1 -- 172.21.15.99:0/870313465 wait complete. 2024-03-19T01:47:52.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:47:51 smithi115 ceph-mon[67332]: pgmap v553: 97 pgs: 97 active+clean; 258 MiB data, 2.4 GiB used, 1.0 TiB / 1.0 TiB avail; 954 KiB/s rd, 3.1 MiB/s wr, 283 op/s 2024-03-19T01:47:52.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:47:51 smithi099 ceph-mon[62951]: pgmap v553: 97 pgs: 97 active+clean; 258 MiB data, 2.4 GiB used, 1.0 TiB / 1.0 TiB avail; 954 KiB/s rd, 3.1 MiB/s wr, 283 op/s 2024-03-19T01:47:52.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:47:51 smithi112 ceph-mon[66819]: pgmap v553: 97 pgs: 97 active+clean; 258 MiB data, 2.4 GiB used, 1.0 TiB / 1.0 TiB avail; 954 KiB/s rd, 3.1 MiB/s wr, 283 op/s 2024-03-19T01:47:52.689 INFO:tasks.workunit.client.0.smithi099.stdout: 1 812396 11.34 MB/sec execute 262 sec latency 18.630 ms 2024-03-19T01:47:53.689 INFO:tasks.workunit.client.0.smithi099.stdout: 1 814137 11.33 MB/sec execute 263 sec latency 27.356 ms 2024-03-19T01:47:54.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:47:53 smithi115 ceph-mon[67332]: pgmap v554: 97 pgs: 97 active+clean; 259 MiB data, 2.4 GiB used, 1.0 TiB / 1.0 TiB avail; 970 KiB/s rd, 3.1 MiB/s wr, 263 op/s 2024-03-19T01:47:54.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:47:53 smithi099 ceph-mon[62951]: pgmap v554: 97 pgs: 97 active+clean; 259 MiB data, 2.4 GiB used, 1.0 TiB / 1.0 TiB avail; 970 KiB/s rd, 3.1 MiB/s wr, 263 op/s 2024-03-19T01:47:54.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:47:53 smithi112 ceph-mon[66819]: pgmap v554: 97 pgs: 97 active+clean; 259 MiB data, 2.4 GiB used, 1.0 TiB / 1.0 TiB avail; 970 KiB/s rd, 3.1 MiB/s wr, 263 op/s 2024-03-19T01:47:54.689 INFO:tasks.workunit.client.0.smithi099.stdout: 1 816837 11.35 MB/sec execute 264 sec latency 13.455 ms 2024-03-19T01:47:55.689 INFO:tasks.workunit.client.0.smithi099.stdout: 1 818152 11.33 MB/sec execute 265 sec latency 72.406 ms 2024-03-19T01:47:56.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:47:55 smithi115 ceph-mon[67332]: pgmap v555: 97 pgs: 97 active+clean; 258 MiB data, 2.4 GiB used, 1.0 TiB / 1.0 TiB avail; 820 KiB/s rd, 2.7 MiB/s wr, 234 op/s 2024-03-19T01:47:56.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:47:55 smithi099 ceph-mon[62951]: pgmap v555: 97 pgs: 97 active+clean; 258 MiB data, 2.4 GiB used, 1.0 TiB / 1.0 TiB avail; 820 KiB/s rd, 2.7 MiB/s wr, 234 op/s 2024-03-19T01:47:56.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:47:55 smithi112 ceph-mon[66819]: pgmap v555: 97 pgs: 97 active+clean; 258 MiB data, 2.4 GiB used, 1.0 TiB / 1.0 TiB avail; 820 KiB/s rd, 2.7 MiB/s wr, 234 op/s 2024-03-19T01:47:56.689 INFO:tasks.workunit.client.0.smithi099.stdout: 1 820573 11.34 MB/sec execute 266 sec latency 10.581 ms 2024-03-19T01:47:57.689 INFO:tasks.workunit.client.0.smithi099.stdout: 1 822883 11.34 MB/sec execute 267 sec latency 33.275 ms 2024-03-19T01:47:58.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:47:57 smithi115 ceph-mon[67332]: pgmap v556: 97 pgs: 97 active+clean; 259 MiB data, 2.4 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 3.4 MiB/s wr, 321 op/s 2024-03-19T01:47:58.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:47:57 smithi099 ceph-mon[62951]: pgmap v556: 97 pgs: 97 active+clean; 259 MiB data, 2.4 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 3.4 MiB/s wr, 321 op/s 2024-03-19T01:47:58.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:47:57 smithi112 ceph-mon[66819]: pgmap v556: 97 pgs: 97 active+clean; 259 MiB data, 2.4 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 3.4 MiB/s wr, 321 op/s 2024-03-19T01:47:58.689 INFO:tasks.workunit.client.0.smithi099.stdout: 1 824662 11.33 MB/sec execute 268 sec latency 25.845 ms 2024-03-19T01:47:59.689 INFO:tasks.workunit.client.0.smithi099.stdout: 1 827446 11.35 MB/sec execute 269 sec latency 12.651 ms 2024-03-19T01:48:00.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:47:59 smithi115 ceph-mon[67332]: pgmap v557: 97 pgs: 97 active+clean; 259 MiB data, 2.4 GiB used, 1.0 TiB / 1.0 TiB avail; 675 KiB/s rd, 2.6 MiB/s wr, 240 op/s 2024-03-19T01:48:00.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:47:59 smithi099 ceph-mon[62951]: pgmap v557: 97 pgs: 97 active+clean; 259 MiB data, 2.4 GiB used, 1.0 TiB / 1.0 TiB avail; 675 KiB/s rd, 2.6 MiB/s wr, 240 op/s 2024-03-19T01:48:00.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:47:59 smithi112 ceph-mon[66819]: pgmap v557: 97 pgs: 97 active+clean; 259 MiB data, 2.4 GiB used, 1.0 TiB / 1.0 TiB avail; 675 KiB/s rd, 2.6 MiB/s wr, 240 op/s 2024-03-19T01:48:00.689 INFO:tasks.workunit.client.0.smithi099.stdout: 1 828750 11.33 MB/sec execute 270 sec latency 83.265 ms 2024-03-19T01:48:01.689 INFO:tasks.workunit.client.0.smithi099.stdout: 1 831234 11.34 MB/sec execute 271 sec latency 11.585 ms 2024-03-19T01:48:02.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:48:01 smithi115 ceph-mon[67332]: pgmap v558: 97 pgs: 97 active+clean; 263 MiB data, 2.4 GiB used, 1.0 TiB / 1.0 TiB avail; 893 KiB/s rd, 3.0 MiB/s wr, 275 op/s 2024-03-19T01:48:02.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:48:01 smithi099 ceph-mon[62951]: pgmap v558: 97 pgs: 97 active+clean; 263 MiB data, 2.4 GiB used, 1.0 TiB / 1.0 TiB avail; 893 KiB/s rd, 3.0 MiB/s wr, 275 op/s 2024-03-19T01:48:02.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:48:01 smithi112 ceph-mon[66819]: pgmap v558: 97 pgs: 97 active+clean; 263 MiB data, 2.4 GiB used, 1.0 TiB / 1.0 TiB avail; 893 KiB/s rd, 3.0 MiB/s wr, 275 op/s 2024-03-19T01:48:02.689 INFO:tasks.workunit.client.0.smithi099.stdout: 1 833555 11.34 MB/sec execute 272 sec latency 23.921 ms 2024-03-19T01:48:03.689 INFO:tasks.workunit.client.0.smithi099.stdout: 1 835387 11.34 MB/sec execute 273 sec latency 26.003 ms 2024-03-19T01:48:04.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:48:03 smithi115 ceph-mon[67332]: pgmap v559: 97 pgs: 97 active+clean; 265 MiB data, 2.4 GiB used, 1.0 TiB / 1.0 TiB avail; 838 KiB/s rd, 3.0 MiB/s wr, 264 op/s 2024-03-19T01:48:04.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:48:03 smithi099 ceph-mon[62951]: pgmap v559: 97 pgs: 97 active+clean; 265 MiB data, 2.4 GiB used, 1.0 TiB / 1.0 TiB avail; 838 KiB/s rd, 3.0 MiB/s wr, 264 op/s 2024-03-19T01:48:04.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:48:03 smithi112 ceph-mon[66819]: pgmap v559: 97 pgs: 97 active+clean; 265 MiB data, 2.4 GiB used, 1.0 TiB / 1.0 TiB avail; 838 KiB/s rd, 3.0 MiB/s wr, 264 op/s 2024-03-19T01:48:04.689 INFO:tasks.workunit.client.0.smithi099.stdout: 1 838151 11.36 MB/sec execute 274 sec latency 14.182 ms 2024-03-19T01:48:04.834 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.832+0000 7f344957e640 1 -- 172.21.15.99:0/3865840934 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3444102330 msgr2=0x7f3444108bc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:48:04.834 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.832+0000 7f344957e640 1 --2- 172.21.15.99:0/3865840934 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3444102330 0x7f3444108bc0 secure :-1 s=READY pgs=481 cs=0 l=1 rev1=1 crypto rx=0x7f343c009ca0 tx=0x7f343c02c0f0 comp rx=0 tx=0).stop 2024-03-19T01:48:04.834 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.832+0000 7f344957e640 1 -- 172.21.15.99:0/3865840934 shutdown_connections 2024-03-19T01:48:04.834 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.832+0000 7f344957e640 1 --2- 172.21.15.99:0/3865840934 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3444102330 0x7f3444108bc0 unknown :-1 s=CLOSED pgs=481 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:04.834 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.832+0000 7f344957e640 1 --2- 172.21.15.99:0/3865840934 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f3444101970 0x7f3444101df0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:04.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.832+0000 7f344957e640 1 --2- 172.21.15.99:0/3865840934 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3444100780 0x7f3444100ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:04.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.832+0000 7f344957e640 1 -- 172.21.15.99:0/3865840934 >> 172.21.15.99:0/3865840934 conn(0x7f34440fbcf0 msgr2=0x7f34440fe130 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:48:04.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.832+0000 7f344957e640 1 -- 172.21.15.99:0/3865840934 shutdown_connections 2024-03-19T01:48:04.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.832+0000 7f344957e640 1 -- 172.21.15.99:0/3865840934 wait complete. 2024-03-19T01:48:04.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.832+0000 7f344957e640 1 Processor -- start 2024-03-19T01:48:04.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.832+0000 7f344957e640 1 -- start start 2024-03-19T01:48:04.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.833+0000 7f344957e640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3444100780 0x7f3444197a70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:04.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.833+0000 7f344957e640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3444101970 0x7f3444197fb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:04.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.833+0000 7f344957e640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f3444102330 0x7f344419f020 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:04.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.833+0000 7f344957e640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f344410c400 con 0x7f3444101970 2024-03-19T01:48:04.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.833+0000 7f344957e640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f344410c280 con 0x7f3444100780 2024-03-19T01:48:04.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.833+0000 7f344957e640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f344410c580 con 0x7f3444102330 2024-03-19T01:48:04.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.833+0000 7f343bfff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3444101970 0x7f3444197fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:04.836 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.833+0000 7f343bfff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3444101970 0x7f3444197fb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:56252/0 (socket says 172.21.15.99:56252) 2024-03-19T01:48:04.836 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.833+0000 7f3442ffd640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3444100780 0x7f3444197a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:04.836 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.833+0000 7f343bfff640 1 -- 172.21.15.99:0/1003727193 learned_addr learned my addr 172.21.15.99:0/1003727193 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:48:04.836 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.833+0000 7f34437fe640 1 --2- 172.21.15.99:0/1003727193 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f3444102330 0x7f344419f020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:04.836 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.833+0000 7f3442ffd640 1 -- 172.21.15.99:0/1003727193 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f3444102330 msgr2=0x7f344419f020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:48:04.836 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.833+0000 7f3442ffd640 1 --2- 172.21.15.99:0/1003727193 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f3444102330 0x7f344419f020 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:04.836 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.833+0000 7f3442ffd640 1 -- 172.21.15.99:0/1003727193 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3444101970 msgr2=0x7f3444197fb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:48:04.836 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.833+0000 7f3442ffd640 1 --2- 172.21.15.99:0/1003727193 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3444101970 0x7f3444197fb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:04.836 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.833+0000 7f3442ffd640 1 -- 172.21.15.99:0/1003727193 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f344419f690 con 0x7f3444100780 2024-03-19T01:48:04.836 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.833+0000 7f343bfff640 1 --2- 172.21.15.99:0/1003727193 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3444101970 0x7f3444197fb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:48:04.836 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.833+0000 7f3442ffd640 1 --2- 172.21.15.99:0/1003727193 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3444100780 0x7f3444197a70 secure :-1 s=READY pgs=128 cs=0 l=1 rev1=1 crypto rx=0x7f343400e9c0 tx=0x7f343400ee80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:48:04.836 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.834+0000 7f3440ff9640 1 -- 172.21.15.99:0/1003727193 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f343400cd80 con 0x7f3444100780 2024-03-19T01:48:04.836 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.834+0000 7f3440ff9640 1 -- 172.21.15.99:0/1003727193 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f3434010910 con 0x7f3444100780 2024-03-19T01:48:04.836 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.834+0000 7f344957e640 1 -- 172.21.15.99:0/1003727193 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f344419f980 con 0x7f3444100780 2024-03-19T01:48:04.837 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.834+0000 7f3440ff9640 1 -- 172.21.15.99:0/1003727193 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3434018980 con 0x7f3444100780 2024-03-19T01:48:04.837 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.834+0000 7f344957e640 1 -- 172.21.15.99:0/1003727193 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f344419fed0 con 0x7f3444100780 2024-03-19T01:48:04.843 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.836+0000 7f3440ff9640 1 -- 172.21.15.99:0/1003727193 <== mon.2 v2:172.21.15.112:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f343401f020 con 0x7f3444100780 2024-03-19T01:48:04.843 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.836+0000 7f344c169740 1 Processor -- start 2024-03-19T01:48:04.843 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.836+0000 7f344c169740 1 -- start start 2024-03-19T01:48:04.844 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.836+0000 7f344c169740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564c82968100 0x564c82968520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:04.844 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.836+0000 7f344c169740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564c82968af0 0x564c8296d5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:04.844 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.836+0000 7f344c169740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564c8296dc60 0x564c829700f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:04.844 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.836+0000 7f344c169740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x564c828ccbe0 con 0x564c82968af0 2024-03-19T01:48:04.844 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.836+0000 7f344c169740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x564c82875960 con 0x564c82968100 2024-03-19T01:48:04.844 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.836+0000 7f344c169740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x564c828c8c80 con 0x564c8296dc60 2024-03-19T01:48:04.844 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.837+0000 7f34437fe640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564c82968100 0x564c82968520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:04.844 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.837+0000 7f3442ffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564c82968af0 0x564c8296d5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:04.844 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.837+0000 7f34437fe640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564c82968100 0x564c82968520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:56106/0 (socket says 172.21.15.99:56106) 2024-03-19T01:48:04.844 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.837+0000 7f343bfff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564c8296dc60 0x564c829700f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:04.844 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.837+0000 7f34437fe640 1 -- 172.21.15.99:0/3608492434 learned_addr learned my addr 172.21.15.99:0/3608492434 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:48:04.844 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.837+0000 7f3440ff9640 1 --2- 172.21.15.99:0/1003727193 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f34300a28a0 0x7f34300a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:04.844 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.837+0000 7f34437fe640 1 --2- 172.21.15.99:0/1003727193 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f34300a28a0 0x7f34300a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:04.844 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.837+0000 7f3442ffd640 1 -- 172.21.15.99:0/3608492434 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564c8296dc60 msgr2=0x564c829700f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:48:04.845 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.837+0000 7f3442ffd640 1 --2- 172.21.15.99:0/3608492434 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564c8296dc60 0x564c829700f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:04.845 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.837+0000 7f3442ffd640 1 -- 172.21.15.99:0/3608492434 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564c82968100 msgr2=0x564c82968520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:48:04.845 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.837+0000 7f3442ffd640 1 --2- 172.21.15.99:0/3608492434 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564c82968100 0x564c82968520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:04.845 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.837+0000 7f3442ffd640 1 -- 172.21.15.99:0/3608492434 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x564c8286cd20 con 0x564c82968af0 2024-03-19T01:48:04.845 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.837+0000 7f343bfff640 1 --2- 172.21.15.99:0/3608492434 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564c8296dc60 0x564c829700f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:48:04.845 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.837+0000 7f3440ff9640 1 -- 172.21.15.99:0/1003727193 <== mon.2 v2:172.21.15.112:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f34340ceaf0 con 0x7f3444100780 2024-03-19T01:48:04.845 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.837+0000 7f3442ffd640 1 --2- 172.21.15.99:0/3608492434 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564c82968af0 0x564c8296d5f0 secure :-1 s=READY pgs=482 cs=0 l=1 rev1=1 crypto rx=0x7f34340cc520 tx=0x7f343400efb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1b2fb76546a45161 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:48:04.845 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.837+0000 7f34437fe640 1 --2- 172.21.15.99:0/1003727193 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f34300a28a0 0x7f34300a4d60 secure :-1 s=READY pgs=284 cs=0 l=1 rev1=1 crypto rx=0x564c82969350 tx=0x7f343c02c0f0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:48:04.845 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.837+0000 7f34397fa640 1 -- 172.21.15.99:0/3608492434 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3434000ed0 con 0x564c82968af0 2024-03-19T01:48:04.845 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.837+0000 7f34397fa640 1 -- 172.21.15.99:0/3608492434 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f343409baf0 con 0x564c82968af0 2024-03-19T01:48:04.845 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.837+0000 7f34397fa640 1 -- 172.21.15.99:0/3608492434 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3434099780 con 0x564c82968af0 2024-03-19T01:48:04.845 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.838+0000 7f344c169740 1 -- 172.21.15.99:0/3608492434 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564c82968af0 msgr2=0x564c8296d5f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:48:04.845 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.838+0000 7f344c169740 1 --2- 172.21.15.99:0/3608492434 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564c82968af0 0x564c8296d5f0 secure :-1 s=READY pgs=482 cs=0 l=1 rev1=1 crypto rx=0x7f34340cc520 tx=0x7f343400efb0 comp rx=0 tx=0).stop 2024-03-19T01:48:04.845 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.838+0000 7f344c169740 1 -- 172.21.15.99:0/3608492434 shutdown_connections 2024-03-19T01:48:04.845 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.838+0000 7f344c169740 1 --2- 172.21.15.99:0/3608492434 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564c8296dc60 0x564c829700f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:04.846 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.838+0000 7f344c169740 1 --2- 172.21.15.99:0/3608492434 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564c82968af0 0x564c8296d5f0 unknown :-1 s=CLOSED pgs=482 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:04.846 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.838+0000 7f344c169740 1 --2- 172.21.15.99:0/3608492434 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564c82968100 0x564c82968520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:04.846 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.838+0000 7f344c169740 1 -- 172.21.15.99:0/3608492434 >> 172.21.15.99:0/3608492434 conn(0x564c82963670 msgr2=0x564c82963a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:48:04.846 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.838+0000 7f344c169740 1 -- 172.21.15.99:0/3608492434 shutdown_connections 2024-03-19T01:48:04.846 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.838+0000 7f344c169740 1 -- 172.21.15.99:0/3608492434 wait complete. 2024-03-19T01:48:04.846 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.839+0000 7f344c169740 1 -- start start 2024-03-19T01:48:04.846 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.839+0000 7f344c169740 1 Processor -- start 2024-03-19T01:48:04.846 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.839+0000 7f344c169740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564c82968100 0x564c829720a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:04.846 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.839+0000 7f344c169740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564c82968af0 0x564c829725e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:04.846 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.839+0000 7f344c169740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564c8296dc60 0x564c82979620 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:04.846 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.839+0000 7f344c169740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x564c82970bb0 con 0x564c8296dc60 2024-03-19T01:48:04.846 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.839+0000 7f344c169740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x564c82888db0 con 0x564c82968100 2024-03-19T01:48:04.846 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.839+0000 7f344c169740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x564c82970d30 con 0x564c82968af0 2024-03-19T01:48:04.846 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.839+0000 7f34437fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564c8296dc60 0x564c82979620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:04.847 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.839+0000 7f343bfff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564c82968af0 0x564c829725e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:04.847 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.839+0000 7f3442ffd640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564c82968100 0x564c829720a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:04.847 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.839+0000 7f34437fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564c8296dc60 0x564c82979620 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:56270/0 (socket says 172.21.15.99:56270) 2024-03-19T01:48:04.847 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.839+0000 7f34437fe640 1 -- 172.21.15.99:0/3644876194 learned_addr learned my addr 172.21.15.99:0/3644876194 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:48:04.847 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.839+0000 7f344c169740 10 client.0 _ll_register_callbacks cb 0x564c8288de70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:48:04.847 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.839+0000 7f34437fe640 1 -- 172.21.15.99:0/3644876194 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564c82968af0 msgr2=0x564c829725e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:48:04.847 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.839+0000 7f34437fe640 1 --2- 172.21.15.99:0/3644876194 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564c82968af0 0x564c829725e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:04.847 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.839+0000 7f34437fe640 1 -- 172.21.15.99:0/3644876194 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564c82968100 msgr2=0x564c829720a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:48:04.847 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.839+0000 7f34437fe640 1 --2- 172.21.15.99:0/3644876194 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564c82968100 0x564c829720a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:04.847 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.839+0000 7f34437fe640 1 -- 172.21.15.99:0/3644876194 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x564c82887c60 con 0x564c8296dc60 2024-03-19T01:48:04.847 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.839+0000 7f343bfff640 1 --2- 172.21.15.99:0/3644876194 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564c82968af0 0x564c829725e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:48:04.848 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.840+0000 7f34437fe640 1 --2- 172.21.15.99:0/3644876194 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564c8296dc60 0x564c82979620 secure :-1 s=READY pgs=483 cs=0 l=1 rev1=1 crypto rx=0x7f343c037040 tx=0x7f343c0095c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=af403b2e86dca7e5 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:48:04.848 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.840+0000 7f342a7fc640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:48:04.848 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.840+0000 7f342a7fc640 1 -- 172.21.15.99:0/3644876194 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f343c053060 con 0x564c8296dc60 2024-03-19T01:48:04.848 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.840+0000 7f344c169740 1 -- 172.21.15.99:0/3644876194 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f343409baf0 con 0x564c8296dc60 2024-03-19T01:48:04.848 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.840+0000 7f342a7fc640 1 -- 172.21.15.99:0/3644876194 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f343c004240 con 0x564c8296dc60 2024-03-19T01:48:04.848 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.840+0000 7f342a7fc640 1 -- 172.21.15.99:0/3644876194 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7f343c047070 con 0x564c8296dc60 2024-03-19T01:48:04.848 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.840+0000 7f344c169740 10 client.15900 fetch_fsmap learned FSMap version 27 2024-03-19T01:48:04.848 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.840+0000 7f344c169740 1 -- 172.21.15.99:0/3644876194 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x564c825e1210 con 0x564c8296dc60 2024-03-19T01:48:04.848 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.840+0000 7f342a7fc640 1 -- 172.21.15.99:0/3644876194 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f343409baf0 con 0x564c8296dc60 2024-03-19T01:48:04.848 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.840+0000 7f342a7fc640 1 -- 172.21.15.99:0/3644876194 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 27) ==== 8338+0+0 (secure 0 0 0) 0x7f343c03f070 con 0x564c8296dc60 2024-03-19T01:48:04.848 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.840+0000 7f344c169740 10 client.15900 fetch_fsmap finished waiting for FSMap version 27 2024-03-19T01:48:04.848 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.840+0000 7f344c169740 10 client.15900 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:48:04.848 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.840+0000 7f344c169740 20 client.15900 populate_metadata read hostname 'smithi099' 2024-03-19T01:48:04.848 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.840+0000 7f344c169740 1 --2- 172.21.15.99:0/3644876194 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x564c8297df70 0x564c82980360 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:04.849 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.840+0000 7f344c169740 4 client.15900 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:48:04.849 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.840+0000 7f344c169740 1 -- 172.21.15.99:0/3644876194 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x564c828da120 con 0x564c8297df70 2024-03-19T01:48:04.849 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.841+0000 7f3442ffd640 1 --2- 172.21.15.99:0/3644876194 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x564c8297df70 0x564c82980360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:04.849 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.841+0000 7f3442ffd640 1 --2- 172.21.15.99:0/3644876194 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x564c8297df70 0x564c82980360 crc :-1 s=READY pgs=91 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=d9cc087bdd5557bd server_cookie=9643a9b80f6cebc in_seq=0 out_seq=0 2024-03-19T01:48:04.849 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.841+0000 7f342a7fc640 10 client.15900 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:48:04.849 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.842+0000 7f342a7fc640 1 -- 172.21.15.99:0/3644876194 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x564c828da120 con 0x564c8297df70 2024-03-19T01:48:04.849 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.842+0000 7f342a7fc640 10 client.15900 handle_command_reply: tid=0 2024-03-19T01:48:04.849 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.842+0000 7f344c169740 1 client.15900 shutdown 2024-03-19T01:48:04.849 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.842+0000 7f344c169740 1 -- 172.21.15.99:0/3644876194 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564c8296dc60 msgr2=0x564c82979620 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:48:04.849 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.842+0000 7f344c169740 1 --2- 172.21.15.99:0/3644876194 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564c8296dc60 0x564c82979620 secure :-1 s=READY pgs=483 cs=0 l=1 rev1=1 crypto rx=0x7f343c037040 tx=0x7f343c0095c0 comp rx=0 tx=0).stop 2024-03-19T01:48:04.849 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.842+0000 7f344c169740 1 -- 172.21.15.99:0/3644876194 shutdown_connections 2024-03-19T01:48:04.849 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.842+0000 7f344c169740 1 --2- 172.21.15.99:0/3644876194 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x564c8297df70 0x564c82980360 crc :-1 s=READY pgs=91 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:04.849 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.842+0000 7f344c169740 1 --2- 172.21.15.99:0/3644876194 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564c8296dc60 0x564c82979620 unknown :-1 s=CLOSED pgs=483 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:04.849 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.842+0000 7f344c169740 1 --2- 172.21.15.99:0/3644876194 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564c82968af0 0x564c829725e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:04.850 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.842+0000 7f344c169740 1 --2- 172.21.15.99:0/3644876194 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564c82968100 0x564c829720a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:04.850 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.842+0000 7f342a7fc640 0 client.15900 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:48:04.850 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.842+0000 7f344c169740 1 -- 172.21.15.99:0/3644876194 >> 172.21.15.99:0/3644876194 conn(0x564c82963670 msgr2=0x564c829648c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:48:04.850 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.842+0000 7f344c169740 1 -- 172.21.15.99:0/3644876194 shutdown_connections 2024-03-19T01:48:04.850 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.842+0000 7f344c169740 1 -- 172.21.15.99:0/3644876194 wait complete. 2024-03-19T01:48:04.850 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.842+0000 7f344c169740 20 client.15900 trim_cache size 0 max 16384 2024-03-19T01:48:04.863 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.858+0000 7f344c169740 1 Processor -- start 2024-03-19T01:48:04.863 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.858+0000 7f344c169740 1 -- start start 2024-03-19T01:48:04.863 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.858+0000 7f344c169740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564c82968100 0x564c82962ab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:04.863 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.858+0000 7f344c169740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564c82968af0 0x564c829795e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:04.863 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.858+0000 7f344c169740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564c8296dc60 0x564c82966fe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:04.864 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.858+0000 7f344c169740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f34441ab700 con 0x564c8296dc60 2024-03-19T01:48:04.864 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.858+0000 7f344c169740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f34441a8fd0 con 0x564c82968100 2024-03-19T01:48:04.864 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.858+0000 7f344c169740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f34441a6e20 con 0x564c82968af0 2024-03-19T01:48:04.864 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.859+0000 7f34437fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564c8296dc60 0x564c82966fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:04.864 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.859+0000 7f343bfff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564c82968af0 0x564c829795e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:04.864 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.859+0000 7f34437fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564c8296dc60 0x564c82966fe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:56286/0 (socket says 172.21.15.99:56286) 2024-03-19T01:48:04.864 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.859+0000 7f3442ffd640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564c82968100 0x564c82962ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:04.864 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.859+0000 7f34437fe640 1 -- 172.21.15.99:0/4093007516 learned_addr learned my addr 172.21.15.99:0/4093007516 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:48:04.864 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.859+0000 7f343bfff640 1 -- 172.21.15.99:0/4093007516 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564c82968100 msgr2=0x564c82962ab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:48:04.864 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.859+0000 7f343bfff640 1 --2- 172.21.15.99:0/4093007516 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564c82968100 0x564c82962ab0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:04.864 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.859+0000 7f343bfff640 1 -- 172.21.15.99:0/4093007516 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564c8296dc60 msgr2=0x564c82966fe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:48:04.864 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.859+0000 7f343bfff640 1 --2- 172.21.15.99:0/4093007516 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564c8296dc60 0x564c82966fe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:04.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.859+0000 7f343bfff640 1 -- 172.21.15.99:0/4093007516 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x564c826ad620 con 0x564c82968af0 2024-03-19T01:48:04.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.859+0000 7f34437fe640 1 --2- 172.21.15.99:0/4093007516 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564c8296dc60 0x564c82966fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:48:04.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.859+0000 7f3442ffd640 1 --2- 172.21.15.99:0/4093007516 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564c82968100 0x564c82962ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:48:04.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.859+0000 7f343bfff640 1 --2- 172.21.15.99:0/4093007516 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564c82968af0 0x564c829795e0 secure :-1 s=READY pgs=95 cs=0 l=1 rev1=1 crypto rx=0x564c829645e0 tx=0x7f342c005f90 comp rx=0 tx=0).ready entity=mon.1 client_cookie=bbc106b78c3ea552 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:48:04.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.859+0000 7f3438ff9640 1 -- 172.21.15.99:0/4093007516 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f342c00fa70 con 0x564c82968af0 2024-03-19T01:48:04.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.859+0000 7f3438ff9640 1 -- 172.21.15.99:0/4093007516 <== mon.1 v2:172.21.15.115:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f342c021070 con 0x564c82968af0 2024-03-19T01:48:04.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.859+0000 7f3438ff9640 1 -- 172.21.15.99:0/4093007516 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f342c01c400 con 0x564c82968af0 2024-03-19T01:48:04.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.860+0000 7f344c169740 1 -- 172.21.15.99:0/4093007516 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564c82968af0 msgr2=0x564c829795e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:48:04.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.860+0000 7f344c169740 1 --2- 172.21.15.99:0/4093007516 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564c82968af0 0x564c829795e0 secure :-1 s=READY pgs=95 cs=0 l=1 rev1=1 crypto rx=0x564c829645e0 tx=0x7f342c005f90 comp rx=0 tx=0).stop 2024-03-19T01:48:04.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.860+0000 7f344c169740 1 -- 172.21.15.99:0/4093007516 shutdown_connections 2024-03-19T01:48:04.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.860+0000 7f344c169740 1 --2- 172.21.15.99:0/4093007516 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564c8296dc60 0x564c82966fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:04.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.860+0000 7f344c169740 1 --2- 172.21.15.99:0/4093007516 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564c82968af0 0x564c829795e0 unknown :-1 s=CLOSED pgs=95 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:04.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.860+0000 7f344c169740 1 --2- 172.21.15.99:0/4093007516 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564c82968100 0x564c82962ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:04.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.860+0000 7f344c169740 1 -- 172.21.15.99:0/4093007516 >> 172.21.15.99:0/4093007516 conn(0x564c82963670 msgr2=0x564c829668d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:48:04.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.860+0000 7f344c169740 1 -- 172.21.15.99:0/4093007516 shutdown_connections 2024-03-19T01:48:04.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.860+0000 7f344c169740 1 -- 172.21.15.99:0/4093007516 wait complete. 2024-03-19T01:48:04.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.860+0000 7f344c169740 1 -- start start 2024-03-19T01:48:04.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.861+0000 7f344c169740 1 Processor -- start 2024-03-19T01:48:04.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.861+0000 7f344c169740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564c82968100 0x564c82976d10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:04.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.861+0000 7f344c169740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564c82968af0 0x564c82977250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:04.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.861+0000 7f344c169740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564c8296dc60 0x564c82978210 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:04.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.861+0000 7f344c169740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f34441a7490 con 0x564c82968af0 2024-03-19T01:48:04.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.861+0000 7f344c169740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f34441a8900 con 0x564c82968100 2024-03-19T01:48:04.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.861+0000 7f344c169740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f34441a7b60 con 0x564c8296dc60 2024-03-19T01:48:04.867 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.861+0000 7f344c169740 10 client.0 _ll_register_callbacks cb 0x7f342c005e30 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:48:04.867 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.861+0000 7f343bfff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564c82968af0 0x564c82977250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:04.867 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.861+0000 7f3442ffd640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564c82968100 0x564c82976d10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:04.867 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.861+0000 7f343bfff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564c82968af0 0x564c82977250 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:56292/0 (socket says 172.21.15.99:56292) 2024-03-19T01:48:04.867 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.861+0000 7f343bfff640 1 -- 172.21.15.99:0/1780328463 learned_addr learned my addr 172.21.15.99:0/1780328463 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:48:04.867 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.861+0000 7f34437fe640 1 --2- 172.21.15.99:0/1780328463 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564c8296dc60 0x564c82978210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:04.867 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.861+0000 7f343bfff640 1 -- 172.21.15.99:0/1780328463 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564c8296dc60 msgr2=0x564c82978210 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:48:04.867 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.861+0000 7f343bfff640 1 --2- 172.21.15.99:0/1780328463 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564c8296dc60 0x564c82978210 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:04.867 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.861+0000 7f343bfff640 1 -- 172.21.15.99:0/1780328463 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564c82968100 msgr2=0x564c82976d10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:48:04.867 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.861+0000 7f343bfff640 1 --2- 172.21.15.99:0/1780328463 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564c82968100 0x564c82976d10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:04.867 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.861+0000 7f343bfff640 1 -- 172.21.15.99:0/1780328463 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x564c82978800 con 0x564c82968af0 2024-03-19T01:48:04.867 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.861+0000 7f3442ffd640 1 --2- 172.21.15.99:0/1780328463 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564c82968100 0x564c82976d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:48:04.868 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.861+0000 7f343bfff640 1 --2- 172.21.15.99:0/1780328463 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564c82968af0 0x564c82977250 secure :-1 s=READY pgs=484 cs=0 l=1 rev1=1 crypto rx=0x7f342c020000 tx=0x7f342c0056d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=620a34e89007f184 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:48:04.868 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.861+0000 7f342a7fc640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:48:04.868 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.862+0000 7f342a7fc640 1 -- 172.21.15.99:0/1780328463 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f342c010840 con 0x564c82968af0 2024-03-19T01:48:04.868 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.862+0000 7f344c169740 1 -- 172.21.15.99:0/1780328463 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f342c021070 con 0x564c82968af0 2024-03-19T01:48:04.868 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.862+0000 7f342a7fc640 1 -- 172.21.15.99:0/1780328463 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f342c02f470 con 0x564c82968af0 2024-03-19T01:48:04.868 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.862+0000 7f342a7fc640 1 -- 172.21.15.99:0/1780328463 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7f342c012250 con 0x564c82968af0 2024-03-19T01:48:04.868 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.862+0000 7f344c169740 10 client.15906 fetch_fsmap learned FSMap version 27 2024-03-19T01:48:04.868 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.862+0000 7f344c169740 1 -- 172.21.15.99:0/1780328463 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x564c829e42a0 con 0x564c82968af0 2024-03-19T01:48:04.868 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.862+0000 7f342a7fc640 1 -- 172.21.15.99:0/1780328463 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f342c021070 con 0x564c82968af0 2024-03-19T01:48:04.868 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.862+0000 7f342a7fc640 1 -- 172.21.15.99:0/1780328463 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 27) ==== 8338+0+0 (secure 0 0 0) 0x7f342c0257c0 con 0x564c82968af0 2024-03-19T01:48:04.868 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.862+0000 7f344c169740 10 client.15906 fetch_fsmap finished waiting for FSMap version 27 2024-03-19T01:48:04.868 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.862+0000 7f344c169740 10 client.15906 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:48:04.869 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.862+0000 7f344c169740 20 client.15906 populate_metadata read hostname 'smithi099' 2024-03-19T01:48:04.869 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.862+0000 7f344c169740 1 --2- 172.21.15.99:0/1780328463 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x564c829e4b20 0x564c829e6f40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:04.869 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.862+0000 7f344c169740 4 client.15906 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:48:04.869 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.862+0000 7f344c169740 1 -- 172.21.15.99:0/1780328463 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x564c828d4470 con 0x564c829e4b20 2024-03-19T01:48:04.869 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.862+0000 7f3442ffd640 1 --2- 172.21.15.99:0/1780328463 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x564c829e4b20 0x564c829e6f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:04.869 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.863+0000 7f342a7fc640 10 client.15906 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:48:04.869 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.863+0000 7f3442ffd640 1 --2- 172.21.15.99:0/1780328463 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x564c829e4b20 0x564c829e6f40 crc :-1 s=READY pgs=92 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=325bc83c85ff8df1 server_cookie=39f32429abcab0e2 in_seq=0 out_seq=0 2024-03-19T01:48:04.921 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.920+0000 7f342a7fc640 1 -- 172.21.15.99:0/1780328463 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x564c828d4470 con 0x564c829e4b20 2024-03-19T01:48:04.921 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.920+0000 7f342a7fc640 10 client.15906 handle_command_reply: tid=0 2024-03-19T01:48:04.921 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.920+0000 7f344c169740 1 client.15906 shutdown 2024-03-19T01:48:04.921 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.921+0000 7f344c169740 1 -- 172.21.15.99:0/1780328463 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564c82968af0 msgr2=0x564c82977250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:48:04.921 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.921+0000 7f344c169740 1 --2- 172.21.15.99:0/1780328463 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564c82968af0 0x564c82977250 secure :-1 s=READY pgs=484 cs=0 l=1 rev1=1 crypto rx=0x7f342c020000 tx=0x7f342c0056d0 comp rx=0 tx=0).stop 2024-03-19T01:48:04.921 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.921+0000 7f344c169740 1 -- 172.21.15.99:0/1780328463 shutdown_connections 2024-03-19T01:48:04.922 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.921+0000 7f344c169740 1 --2- 172.21.15.99:0/1780328463 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x564c829e4b20 0x564c829e6f40 crc :-1 s=READY pgs=92 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:04.922 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.921+0000 7f344c169740 1 --2- 172.21.15.99:0/1780328463 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564c8296dc60 0x564c82978210 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:04.922 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.921+0000 7f344c169740 1 --2- 172.21.15.99:0/1780328463 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564c82968af0 0x564c82977250 unknown :-1 s=CLOSED pgs=484 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:04.922 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.921+0000 7f342a7fc640 0 client.15906 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:48:04.922 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.921+0000 7f344c169740 1 --2- 172.21.15.99:0/1780328463 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564c82968100 0x564c82976d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:04.922 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.921+0000 7f344c169740 1 -- 172.21.15.99:0/1780328463 >> 172.21.15.99:0/1780328463 conn(0x564c82963670 msgr2=0x564c82967160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:48:04.922 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.921+0000 7f344c169740 1 -- 172.21.15.99:0/1780328463 shutdown_connections 2024-03-19T01:48:04.922 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.921+0000 7f344c169740 1 -- 172.21.15.99:0/1780328463 wait complete. 2024-03-19T01:48:04.922 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.921+0000 7f344c169740 20 client.15906 trim_cache size 0 max 16384 2024-03-19T01:48:04.923 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:48:04.923 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:48:04.923 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:48:04.923 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:48:04.924 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.921+0000 7f344957e640 1 -- 172.21.15.99:0/1003727193 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f34300a28a0 msgr2=0x7f34300a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:48:04.924 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.921+0000 7f344957e640 1 --2- 172.21.15.99:0/1003727193 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f34300a28a0 0x7f34300a4d60 secure :-1 s=READY pgs=284 cs=0 l=1 rev1=1 crypto rx=0x564c82969350 tx=0x7f343c02c0f0 comp rx=0 tx=0).stop 2024-03-19T01:48:04.924 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.921+0000 7f344957e640 1 -- 172.21.15.99:0/1003727193 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3444100780 msgr2=0x7f3444197a70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:48:04.924 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.921+0000 7f344957e640 1 --2- 172.21.15.99:0/1003727193 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3444100780 0x7f3444197a70 secure :-1 s=READY pgs=128 cs=0 l=1 rev1=1 crypto rx=0x7f343400e9c0 tx=0x7f343400ee80 comp rx=0 tx=0).stop 2024-03-19T01:48:04.924 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.921+0000 7f344957e640 1 -- 172.21.15.99:0/1003727193 shutdown_connections 2024-03-19T01:48:04.924 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.921+0000 7f344957e640 1 --2- 172.21.15.99:0/1003727193 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f3444102330 0x7f344419f020 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:04.924 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.921+0000 7f344957e640 1 --2- 172.21.15.99:0/1003727193 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3444101970 0x7f3444197fb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:04.924 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.921+0000 7f344957e640 1 --2- 172.21.15.99:0/1003727193 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f34300a28a0 0x7f34300a4d60 unknown :-1 s=CLOSED pgs=284 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:04.924 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.921+0000 7f344957e640 1 --2- 172.21.15.99:0/1003727193 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3444100780 0x7f3444197a70 unknown :-1 s=CLOSED pgs=128 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:04.925 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.921+0000 7f344957e640 1 -- 172.21.15.99:0/1003727193 >> 172.21.15.99:0/1003727193 conn(0x7f34440fbcf0 msgr2=0x7f34440fd690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:48:04.925 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.922+0000 7f344957e640 1 -- 172.21.15.99:0/1003727193 shutdown_connections 2024-03-19T01:48:04.925 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:04.922+0000 7f344957e640 1 -- 172.21.15.99:0/1003727193 wait complete. 2024-03-19T01:48:05.689 INFO:tasks.workunit.client.0.smithi099.stdout: 1 839611 11.34 MB/sec execute 275 sec latency 114.333 ms 2024-03-19T01:48:06.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:48:05 smithi115 ceph-mon[67332]: pgmap v560: 97 pgs: 97 active+clean; 265 MiB data, 2.4 GiB used, 1.0 TiB / 1.0 TiB avail; 735 KiB/s rd, 2.7 MiB/s wr, 241 op/s 2024-03-19T01:48:06.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:48:05 smithi099 ceph-mon[62951]: pgmap v560: 97 pgs: 97 active+clean; 265 MiB data, 2.4 GiB used, 1.0 TiB / 1.0 TiB avail; 735 KiB/s rd, 2.7 MiB/s wr, 241 op/s 2024-03-19T01:48:06.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:48:05 smithi112 ceph-mon[66819]: pgmap v560: 97 pgs: 97 active+clean; 265 MiB data, 2.4 GiB used, 1.0 TiB / 1.0 TiB avail; 735 KiB/s rd, 2.7 MiB/s wr, 241 op/s 2024-03-19T01:48:06.689 INFO:tasks.workunit.client.0.smithi099.stdout: 1 841958 11.34 MB/sec execute 276 sec latency 20.815 ms 2024-03-19T01:48:07.689 INFO:tasks.workunit.client.0.smithi099.stdout: 1 844439 11.35 MB/sec execute 277 sec latency 41.508 ms 2024-03-19T01:48:08.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:48:07 smithi115 ceph-mon[67332]: pgmap v561: 97 pgs: 97 active+clean; 265 MiB data, 2.4 GiB used, 1.0 TiB / 1.0 TiB avail; 1.0 MiB/s rd, 3.7 MiB/s wr, 335 op/s 2024-03-19T01:48:08.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:48:07 smithi099 ceph-mon[62951]: pgmap v561: 97 pgs: 97 active+clean; 265 MiB data, 2.4 GiB used, 1.0 TiB / 1.0 TiB avail; 1.0 MiB/s rd, 3.7 MiB/s wr, 335 op/s 2024-03-19T01:48:08.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:48:07 smithi112 ceph-mon[66819]: pgmap v561: 97 pgs: 97 active+clean; 265 MiB data, 2.4 GiB used, 1.0 TiB / 1.0 TiB avail; 1.0 MiB/s rd, 3.7 MiB/s wr, 335 op/s 2024-03-19T01:48:08.690 INFO:tasks.workunit.client.0.smithi099.stdout: 1 846300 11.34 MB/sec execute 278 sec latency 27.138 ms 2024-03-19T01:48:09.690 INFO:tasks.workunit.client.0.smithi099.stdout: 1 848873 11.36 MB/sec execute 279 sec latency 20.346 ms 2024-03-19T01:48:10.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:48:09 smithi115 ceph-mon[67332]: pgmap v562: 97 pgs: 97 active+clean; 265 MiB data, 2.4 GiB used, 1.0 TiB / 1.0 TiB avail; 783 KiB/s rd, 3.0 MiB/s wr, 256 op/s 2024-03-19T01:48:10.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:48:09 smithi099 ceph-mon[62951]: pgmap v562: 97 pgs: 97 active+clean; 265 MiB data, 2.4 GiB used, 1.0 TiB / 1.0 TiB avail; 783 KiB/s rd, 3.0 MiB/s wr, 256 op/s 2024-03-19T01:48:10.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:48:09 smithi112 ceph-mon[66819]: pgmap v562: 97 pgs: 97 active+clean; 265 MiB data, 2.4 GiB used, 1.0 TiB / 1.0 TiB avail; 783 KiB/s rd, 3.0 MiB/s wr, 256 op/s 2024-03-19T01:48:10.690 INFO:tasks.workunit.client.0.smithi099.stdout: 1 850764 11.34 MB/sec execute 280 sec latency 87.807 ms 2024-03-19T01:48:11.690 INFO:tasks.workunit.client.0.smithi099.stdout: 1 852777 11.35 MB/sec execute 281 sec latency 28.855 ms 2024-03-19T01:48:12.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:48:11 smithi115 ceph-mon[67332]: pgmap v563: 97 pgs: 97 active+clean; 263 MiB data, 2.5 GiB used, 1.0 TiB / 1.0 TiB avail; 824 KiB/s rd, 3.0 MiB/s wr, 264 op/s 2024-03-19T01:48:12.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:48:11 smithi099 ceph-mon[62951]: pgmap v563: 97 pgs: 97 active+clean; 263 MiB data, 2.5 GiB used, 1.0 TiB / 1.0 TiB avail; 824 KiB/s rd, 3.0 MiB/s wr, 264 op/s 2024-03-19T01:48:12.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:48:11 smithi112 ceph-mon[66819]: pgmap v563: 97 pgs: 97 active+clean; 263 MiB data, 2.5 GiB used, 1.0 TiB / 1.0 TiB avail; 824 KiB/s rd, 3.0 MiB/s wr, 264 op/s 2024-03-19T01:48:12.690 INFO:tasks.workunit.client.0.smithi099.stdout: 1 855619 11.37 MB/sec execute 282 sec latency 12.586 ms 2024-03-19T01:48:13.690 INFO:tasks.workunit.client.0.smithi099.stdout: 1 857247 11.35 MB/sec execute 283 sec latency 67.225 ms 2024-03-19T01:48:14.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:48:13 smithi115 ceph-mon[67332]: pgmap v564: 97 pgs: 97 active+clean; 266 MiB data, 2.5 GiB used, 1.0 TiB / 1.0 TiB avail; 975 KiB/s rd, 3.2 MiB/s wr, 281 op/s 2024-03-19T01:48:14.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:48:13 smithi099 ceph-mon[62951]: pgmap v564: 97 pgs: 97 active+clean; 266 MiB data, 2.5 GiB used, 1.0 TiB / 1.0 TiB avail; 975 KiB/s rd, 3.2 MiB/s wr, 281 op/s 2024-03-19T01:48:14.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:48:13 smithi112 ceph-mon[66819]: pgmap v564: 97 pgs: 97 active+clean; 266 MiB data, 2.5 GiB used, 1.0 TiB / 1.0 TiB avail; 975 KiB/s rd, 3.2 MiB/s wr, 281 op/s 2024-03-19T01:48:14.690 INFO:tasks.workunit.client.0.smithi099.stdout: 1 859675 11.36 MB/sec execute 284 sec latency 10.704 ms 2024-03-19T01:48:15.690 INFO:tasks.workunit.client.0.smithi099.stdout: 1 861755 11.35 MB/sec execute 285 sec latency 70.645 ms 2024-03-19T01:48:16.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:48:15 smithi115 ceph-mon[67332]: pgmap v565: 97 pgs: 97 active+clean; 266 MiB data, 2.5 GiB used, 1.0 TiB / 1.0 TiB avail; 794 KiB/s rd, 2.8 MiB/s wr, 241 op/s 2024-03-19T01:48:16.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:48:15 smithi099 ceph-mon[62951]: pgmap v565: 97 pgs: 97 active+clean; 266 MiB data, 2.5 GiB used, 1.0 TiB / 1.0 TiB avail; 794 KiB/s rd, 2.8 MiB/s wr, 241 op/s 2024-03-19T01:48:16.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:48:15 smithi112 ceph-mon[66819]: pgmap v565: 97 pgs: 97 active+clean; 266 MiB data, 2.5 GiB used, 1.0 TiB / 1.0 TiB avail; 794 KiB/s rd, 2.8 MiB/s wr, 241 op/s 2024-03-19T01:48:16.690 INFO:tasks.workunit.client.0.smithi099.stdout: 1 863658 11.35 MB/sec execute 286 sec latency 29.011 ms 2024-03-19T01:48:17.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:48:17 smithi115 ceph-mon[67332]: pgmap v566: 97 pgs: 97 active+clean; 268 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 1.0 MiB/s rd, 3.4 MiB/s wr, 304 op/s 2024-03-19T01:48:17.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:48:17 smithi099 ceph-mon[62951]: pgmap v566: 97 pgs: 97 active+clean; 268 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 1.0 MiB/s rd, 3.4 MiB/s wr, 304 op/s 2024-03-19T01:48:17.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:48:17 smithi112 ceph-mon[66819]: pgmap v566: 97 pgs: 97 active+clean; 268 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 1.0 MiB/s rd, 3.4 MiB/s wr, 304 op/s 2024-03-19T01:48:17.690 INFO:tasks.workunit.client.0.smithi099.stdout: 1 866384 11.37 MB/sec execute 287 sec latency 43.844 ms 2024-03-19T01:48:18.012 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.010+0000 7f7edaf59640 1 -- 172.21.15.99:0/2955463255 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7ed4100750 msgr2=0x7f7ed4100b70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:48:18.013 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.010+0000 7f7edaf59640 1 --2- 172.21.15.99:0/2955463255 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7ed4100750 0x7f7ed4100b70 secure :-1 s=READY pgs=485 cs=0 l=1 rev1=1 crypto rx=0x7f7ec4009900 tx=0x7f7ec402bc30 comp rx=0 tx=0).stop 2024-03-19T01:48:18.013 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.010+0000 7f7edaf59640 1 -- 172.21.15.99:0/2955463255 shutdown_connections 2024-03-19T01:48:18.013 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.010+0000 7f7edaf59640 1 --2- 172.21.15.99:0/2955463255 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f7ed4101940 0x7f7ed4101dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:18.013 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.010+0000 7f7edaf59640 1 --2- 172.21.15.99:0/2955463255 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f7ed4102300 0x7f7ed4108b90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:18.013 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.010+0000 7f7edaf59640 1 --2- 172.21.15.99:0/2955463255 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7ed4100750 0x7f7ed4100b70 unknown :-1 s=CLOSED pgs=485 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:18.013 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.010+0000 7f7edaf59640 1 -- 172.21.15.99:0/2955463255 >> 172.21.15.99:0/2955463255 conn(0x7f7ed40fbcc0 msgr2=0x7f7ed40fe100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:48:18.013 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.010+0000 7f7edaf59640 1 -- 172.21.15.99:0/2955463255 shutdown_connections 2024-03-19T01:48:18.013 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.010+0000 7f7edaf59640 1 -- 172.21.15.99:0/2955463255 wait complete. 2024-03-19T01:48:18.013 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.011+0000 7f7edaf59640 1 Processor -- start 2024-03-19T01:48:18.013 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.011+0000 7f7edaf59640 1 -- start start 2024-03-19T01:48:18.013 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.011+0000 7f7edaf59640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7ed4100750 0x7f7ed4197ab0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:18.013 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.011+0000 7f7edaf59640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f7ed4101940 0x7f7ed4197ff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:18.014 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.011+0000 7f7edaf59640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f7ed4102300 0x7f7ed419f060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:18.014 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.011+0000 7f7edaf59640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f7ed410c3a0 con 0x7f7ed4100750 2024-03-19T01:48:18.014 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.011+0000 7f7edaf59640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f7ed410c220 con 0x7f7ed4101940 2024-03-19T01:48:18.014 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.011+0000 7f7edaf59640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f7ed410c520 con 0x7f7ed4102300 2024-03-19T01:48:18.014 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.011+0000 7f7ed8cf5640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7ed4100750 0x7f7ed4197ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:18.014 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.011+0000 7f7ed8cf5640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7ed4100750 0x7f7ed4197ab0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:53302/0 (socket says 172.21.15.99:53302) 2024-03-19T01:48:18.014 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.011+0000 7f7ed8cf5640 1 -- 172.21.15.99:0/247500977 learned_addr learned my addr 172.21.15.99:0/247500977 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:48:18.014 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.011+0000 7f7ed3fff640 1 --2- 172.21.15.99:0/247500977 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f7ed4101940 0x7f7ed4197ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:18.014 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.011+0000 7f7ed94f6640 1 --2- 172.21.15.99:0/247500977 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f7ed4102300 0x7f7ed419f060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:18.014 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.011+0000 7f7ed8cf5640 1 -- 172.21.15.99:0/247500977 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f7ed4102300 msgr2=0x7f7ed419f060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:48:18.014 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.011+0000 7f7ed8cf5640 1 --2- 172.21.15.99:0/247500977 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f7ed4102300 0x7f7ed419f060 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:18.014 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.011+0000 7f7ed8cf5640 1 -- 172.21.15.99:0/247500977 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f7ed4101940 msgr2=0x7f7ed4197ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:48:18.014 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.011+0000 7f7ed8cf5640 1 --2- 172.21.15.99:0/247500977 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f7ed4101940 0x7f7ed4197ff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:18.015 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.011+0000 7f7ed8cf5640 1 -- 172.21.15.99:0/247500977 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7ed419f6d0 con 0x7f7ed4100750 2024-03-19T01:48:18.015 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.011+0000 7f7ed3fff640 1 --2- 172.21.15.99:0/247500977 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f7ed4101940 0x7f7ed4197ff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:48:18.015 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.011+0000 7f7ed8cf5640 1 --2- 172.21.15.99:0/247500977 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7ed4100750 0x7f7ed4197ab0 secure :-1 s=READY pgs=486 cs=0 l=1 rev1=1 crypto rx=0x7f7ec4000c00 tx=0x7f7ec402c640 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:48:18.015 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.012+0000 7f7ed1ffb640 1 -- 172.21.15.99:0/247500977 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7ec4041070 con 0x7f7ed4100750 2024-03-19T01:48:18.015 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.012+0000 7f7edaf59640 1 -- 172.21.15.99:0/247500977 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f7ed419f960 con 0x7f7ed4100750 2024-03-19T01:48:18.015 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.012+0000 7f7ed1ffb640 1 -- 172.21.15.99:0/247500977 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7ec4004450 con 0x7f7ed4100750 2024-03-19T01:48:18.015 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.012+0000 7f7ed1ffb640 1 -- 172.21.15.99:0/247500977 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7ec4034440 con 0x7f7ed4100750 2024-03-19T01:48:18.015 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.012+0000 7f7edaf59640 1 -- 172.21.15.99:0/247500977 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f7ed419fe50 con 0x7f7ed4100750 2024-03-19T01:48:18.022 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.014+0000 7f7ed1ffb640 1 -- 172.21.15.99:0/247500977 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f7ec4044070 con 0x7f7ed4100750 2024-03-19T01:48:18.023 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.014+0000 7f7eddc15740 1 Processor -- start 2024-03-19T01:48:18.023 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.014+0000 7f7eddc15740 1 -- start start 2024-03-19T01:48:18.023 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.014+0000 7f7eddc15740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562c796fe100 0x562c796fe520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:18.023 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.014+0000 7f7eddc15740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562c796feaf0 0x562c797035f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:18.023 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.014+0000 7f7eddc15740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562c79703c60 0x562c797060f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:18.023 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.014+0000 7f7eddc15740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x562c79662be0 con 0x562c79703c60 2024-03-19T01:48:18.023 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.014+0000 7f7eddc15740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x562c7960b960 con 0x562c796fe100 2024-03-19T01:48:18.023 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.014+0000 7f7eddc15740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x562c7965ec80 con 0x562c796feaf0 2024-03-19T01:48:18.023 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.015+0000 7f7ed1ffb640 1 --2- 172.21.15.99:0/247500977 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f7ec00a28a0 0x7f7ec00a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:18.023 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.015+0000 7f7ed8cf5640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562c796feaf0 0x562c797035f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:18.023 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.015+0000 7f7ed3fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562c79703c60 0x562c797060f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:18.023 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.015+0000 7f7ed8cf5640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562c796feaf0 0x562c797035f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:59888/0 (socket says 172.21.15.99:59888) 2024-03-19T01:48:18.023 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.015+0000 7f7ed8cf5640 1 -- 172.21.15.99:0/1035670727 learned_addr learned my addr 172.21.15.99:0/1035670727 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:48:18.024 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.015+0000 7f7ed94f6640 1 --2- 172.21.15.99:0/1035670727 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562c796fe100 0x562c796fe520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:18.024 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.015+0000 7f7ed94f6640 1 --2- 172.21.15.99:0/247500977 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f7ec00a28a0 0x7f7ec00a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:18.024 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.015+0000 7f7ed1ffb640 1 -- 172.21.15.99:0/247500977 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f7ec40f0560 con 0x7f7ed4100750 2024-03-19T01:48:18.024 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.015+0000 7f7ed8cf5640 1 -- 172.21.15.99:0/1035670727 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562c796fe100 msgr2=0x562c796fe520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:48:18.024 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.015+0000 7f7ed8cf5640 1 --2- 172.21.15.99:0/1035670727 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562c796fe100 0x562c796fe520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:18.024 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.015+0000 7f7ed8cf5640 1 -- 172.21.15.99:0/1035670727 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562c79703c60 msgr2=0x562c797060f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:48:18.024 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.015+0000 7f7ed8cf5640 1 --2- 172.21.15.99:0/1035670727 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562c79703c60 0x562c797060f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:18.024 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.015+0000 7f7ed8cf5640 1 -- 172.21.15.99:0/1035670727 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x562c79602d20 con 0x562c796feaf0 2024-03-19T01:48:18.024 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.015+0000 7f7ed3fff640 1 --2- 172.21.15.99:0/1035670727 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562c79703c60 0x562c797060f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:48:18.024 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.015+0000 7f7ed94f6640 1 --2- 172.21.15.99:0/247500977 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f7ec00a28a0 0x7f7ec00a4d60 secure :-1 s=READY pgs=285 cs=0 l=1 rev1=1 crypto rx=0x562c796ff350 tx=0x7f7ecc009500 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:48:18.024 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.015+0000 7f7ed8cf5640 1 --2- 172.21.15.99:0/1035670727 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562c796feaf0 0x562c797035f0 secure :-1 s=READY pgs=96 cs=0 l=1 rev1=1 crypto rx=0x7f7ec40db9c0 tx=0x7f7ec402bc30 comp rx=0 tx=0).ready entity=mon.1 client_cookie=1c54b42908ad6d90 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:48:18.024 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.015+0000 7f7ecaffd640 1 -- 172.21.15.99:0/1035670727 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7ec404b070 con 0x562c796feaf0 2024-03-19T01:48:18.024 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.015+0000 7f7ecaffd640 1 -- 172.21.15.99:0/1035670727 <== mon.1 v2:172.21.15.115:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7ec4004c80 con 0x562c796feaf0 2024-03-19T01:48:18.024 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.015+0000 7f7ecaffd640 1 -- 172.21.15.99:0/1035670727 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7ec40b9670 con 0x562c796feaf0 2024-03-19T01:48:18.024 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.016+0000 7f7eddc15740 1 -- 172.21.15.99:0/1035670727 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562c796feaf0 msgr2=0x562c797035f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:48:18.024 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.016+0000 7f7eddc15740 1 --2- 172.21.15.99:0/1035670727 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562c796feaf0 0x562c797035f0 secure :-1 s=READY pgs=96 cs=0 l=1 rev1=1 crypto rx=0x7f7ec40db9c0 tx=0x7f7ec402bc30 comp rx=0 tx=0).stop 2024-03-19T01:48:18.025 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.016+0000 7f7eddc15740 1 -- 172.21.15.99:0/1035670727 shutdown_connections 2024-03-19T01:48:18.025 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.016+0000 7f7eddc15740 1 --2- 172.21.15.99:0/1035670727 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562c79703c60 0x562c797060f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:18.025 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.016+0000 7f7eddc15740 1 --2- 172.21.15.99:0/1035670727 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562c796feaf0 0x562c797035f0 unknown :-1 s=CLOSED pgs=96 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:18.025 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.016+0000 7f7eddc15740 1 --2- 172.21.15.99:0/1035670727 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562c796fe100 0x562c796fe520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:18.025 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.016+0000 7f7eddc15740 1 -- 172.21.15.99:0/1035670727 >> 172.21.15.99:0/1035670727 conn(0x562c796f9670 msgr2=0x562c796f9a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:48:18.025 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.016+0000 7f7eddc15740 1 -- 172.21.15.99:0/1035670727 shutdown_connections 2024-03-19T01:48:18.025 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.016+0000 7f7eddc15740 1 -- 172.21.15.99:0/1035670727 wait complete. 2024-03-19T01:48:18.025 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.016+0000 7f7eddc15740 1 -- start start 2024-03-19T01:48:18.025 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.016+0000 7f7eddc15740 1 Processor -- start 2024-03-19T01:48:18.025 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.017+0000 7f7eddc15740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562c796fe100 0x562c79707e20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:18.025 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.017+0000 7f7eddc15740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562c796feaf0 0x562c79708360 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:18.025 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.017+0000 7f7eddc15740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562c79703c60 0x562c7970f3a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:18.025 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.017+0000 7f7eddc15740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x562c797069d0 con 0x562c796fe100 2024-03-19T01:48:18.025 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.017+0000 7f7eddc15740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x562c7961edb0 con 0x562c796feaf0 2024-03-19T01:48:18.026 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.017+0000 7f7eddc15740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x562c79706b50 con 0x562c79703c60 2024-03-19T01:48:18.026 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.017+0000 7f7eddc15740 10 client.0 _ll_register_callbacks cb 0x562c79623e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:48:18.026 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.017+0000 7f7ed8cf5640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562c796fe100 0x562c79707e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:18.026 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.017+0000 7f7ed8cf5640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562c796fe100 0x562c79707e20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:53316/0 (socket says 172.21.15.99:53316) 2024-03-19T01:48:18.026 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.017+0000 7f7ed8cf5640 1 -- 172.21.15.99:0/3306974735 learned_addr learned my addr 172.21.15.99:0/3306974735 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:48:18.026 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.017+0000 7f7ed94f6640 1 --2- 172.21.15.99:0/3306974735 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562c79703c60 0x562c7970f3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:18.026 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.017+0000 7f7ed3fff640 1 --2- 172.21.15.99:0/3306974735 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562c796feaf0 0x562c79708360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:18.026 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.017+0000 7f7ed8cf5640 1 -- 172.21.15.99:0/3306974735 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562c79703c60 msgr2=0x562c7970f3a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:48:18.026 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.017+0000 7f7ed8cf5640 1 --2- 172.21.15.99:0/3306974735 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562c79703c60 0x562c7970f3a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:18.026 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.017+0000 7f7ed8cf5640 1 -- 172.21.15.99:0/3306974735 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562c796feaf0 msgr2=0x562c79708360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:48:18.026 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.017+0000 7f7ed8cf5640 1 --2- 172.21.15.99:0/3306974735 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562c796feaf0 0x562c79708360 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:18.026 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.017+0000 7f7ed8cf5640 1 -- 172.21.15.99:0/3306974735 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x562c7961dc60 con 0x562c796fe100 2024-03-19T01:48:18.026 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.017+0000 7f7ed94f6640 1 --2- 172.21.15.99:0/3306974735 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562c79703c60 0x562c7970f3a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:48:18.026 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.017+0000 7f7ed3fff640 1 --2- 172.21.15.99:0/3306974735 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562c796feaf0 0x562c79708360 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:48:18.027 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.017+0000 7f7ed8cf5640 1 --2- 172.21.15.99:0/3306974735 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562c796fe100 0x562c79707e20 secure :-1 s=READY pgs=487 cs=0 l=1 rev1=1 crypto rx=0x7f7ec40bcd00 tx=0x7f7ec40c56e0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=cc523399202e9df server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:48:18.027 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.017+0000 7f7eb3fff640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:48:18.027 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.018+0000 7f7eb3fff640 1 -- 172.21.15.99:0/3306974735 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7ec404b040 con 0x562c796fe100 2024-03-19T01:48:18.027 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.018+0000 7f7eb3fff640 1 -- 172.21.15.99:0/3306974735 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f7ec40c51d0 con 0x562c796fe100 2024-03-19T01:48:18.027 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.018+0000 7f7eddc15740 1 -- 172.21.15.99:0/3306974735 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f7ec4004c80 con 0x562c796fe100 2024-03-19T01:48:18.027 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.018+0000 7f7eb3fff640 1 -- 172.21.15.99:0/3306974735 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7f7ec40bd000 con 0x562c796fe100 2024-03-19T01:48:18.027 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.018+0000 7f7eddc15740 10 client.15918 fetch_fsmap learned FSMap version 27 2024-03-19T01:48:18.027 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.018+0000 7f7eddc15740 1 -- 172.21.15.99:0/3306974735 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x562c79377210 con 0x562c796fe100 2024-03-19T01:48:18.027 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.018+0000 7f7eb3fff640 1 -- 172.21.15.99:0/3306974735 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7ec4004c80 con 0x562c796fe100 2024-03-19T01:48:18.027 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.018+0000 7f7eb3fff640 1 -- 172.21.15.99:0/3306974735 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 27) ==== 8338+0+0 (secure 0 0 0) 0x7f7ec40d3030 con 0x562c796fe100 2024-03-19T01:48:18.027 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.018+0000 7f7eddc15740 10 client.15918 fetch_fsmap finished waiting for FSMap version 27 2024-03-19T01:48:18.027 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.018+0000 7f7eddc15740 10 client.15918 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:48:18.027 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.018+0000 7f7eddc15740 20 client.15918 populate_metadata read hostname 'smithi099' 2024-03-19T01:48:18.027 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.018+0000 7f7eddc15740 1 --2- 172.21.15.99:0/3306974735 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x562c79713aa0 0x562c79715e90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:18.028 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.018+0000 7f7eddc15740 4 client.15918 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:48:18.028 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.018+0000 7f7eddc15740 1 -- 172.21.15.99:0/3306974735 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x562c79670120 con 0x562c79713aa0 2024-03-19T01:48:18.028 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.018+0000 7f7ed3fff640 1 --2- 172.21.15.99:0/3306974735 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x562c79713aa0 0x562c79715e90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:18.028 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.019+0000 7f7eb3fff640 10 client.15918 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:48:18.028 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.019+0000 7f7ed3fff640 1 --2- 172.21.15.99:0/3306974735 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x562c79713aa0 0x562c79715e90 crc :-1 s=READY pgs=93 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=77fc918eed8003a8 server_cookie=61ed0f6f1d6defc7 in_seq=0 out_seq=0 2024-03-19T01:48:18.028 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.019+0000 7f7eb3fff640 1 -- 172.21.15.99:0/3306974735 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x562c79670120 con 0x562c79713aa0 2024-03-19T01:48:18.028 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.019+0000 7f7eb3fff640 10 client.15918 handle_command_reply: tid=0 2024-03-19T01:48:18.028 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.019+0000 7f7eddc15740 1 client.15918 shutdown 2024-03-19T01:48:18.028 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.020+0000 7f7eddc15740 1 -- 172.21.15.99:0/3306974735 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562c796fe100 msgr2=0x562c79707e20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:48:18.028 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.020+0000 7f7eddc15740 1 --2- 172.21.15.99:0/3306974735 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562c796fe100 0x562c79707e20 secure :-1 s=READY pgs=487 cs=0 l=1 rev1=1 crypto rx=0x7f7ec40bcd00 tx=0x7f7ec40c56e0 comp rx=0 tx=0).stop 2024-03-19T01:48:18.028 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.020+0000 7f7eddc15740 1 -- 172.21.15.99:0/3306974735 shutdown_connections 2024-03-19T01:48:18.028 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.020+0000 7f7eddc15740 1 --2- 172.21.15.99:0/3306974735 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x562c79713aa0 0x562c79715e90 crc :-1 s=READY pgs=93 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:18.028 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.020+0000 7f7eddc15740 1 --2- 172.21.15.99:0/3306974735 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562c796feaf0 0x562c79708360 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:18.028 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.020+0000 7f7eddc15740 1 --2- 172.21.15.99:0/3306974735 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562c79703c60 0x562c7970f3a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:18.028 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.020+0000 7f7eddc15740 1 --2- 172.21.15.99:0/3306974735 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562c796fe100 0x562c79707e20 unknown :-1 s=CLOSED pgs=487 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:18.029 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.020+0000 7f7eddc15740 1 -- 172.21.15.99:0/3306974735 >> 172.21.15.99:0/3306974735 conn(0x562c796f9670 msgr2=0x562c796fa8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:48:18.029 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.020+0000 7f7eb3fff640 0 client.15918 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:48:18.029 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.020+0000 7f7eddc15740 1 -- 172.21.15.99:0/3306974735 shutdown_connections 2024-03-19T01:48:18.029 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.020+0000 7f7eddc15740 1 -- 172.21.15.99:0/3306974735 wait complete. 2024-03-19T01:48:18.029 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.020+0000 7f7eddc15740 20 client.15918 trim_cache size 0 max 16384 2024-03-19T01:48:18.040 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.036+0000 7f7eddc15740 1 Processor -- start 2024-03-19T01:48:18.040 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.036+0000 7f7eddc15740 1 -- start start 2024-03-19T01:48:18.040 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.036+0000 7f7eddc15740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562c796fe100 0x562c79708140 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:18.040 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.036+0000 7f7eddc15740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562c796feaf0 0x562c7970f360 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:18.040 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.036+0000 7f7eddc15740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562c79703c60 0x562c796f8220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:18.040 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.036+0000 7f7eddc15740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f7ed41ab610 con 0x562c796feaf0 2024-03-19T01:48:18.040 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.036+0000 7f7eddc15740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f7ed41a8ee0 con 0x562c79703c60 2024-03-19T01:48:18.040 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.036+0000 7f7eddc15740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f7ed41a6d30 con 0x562c796fe100 2024-03-19T01:48:18.041 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.036+0000 7f7ed3fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562c796feaf0 0x562c7970f360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:18.041 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.036+0000 7f7ed3fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562c796feaf0 0x562c7970f360 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:53328/0 (socket says 172.21.15.99:53328) 2024-03-19T01:48:18.041 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.036+0000 7f7ed8cf5640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562c796fe100 0x562c79708140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:18.041 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.036+0000 7f7ed3fff640 1 -- 172.21.15.99:0/1806618663 learned_addr learned my addr 172.21.15.99:0/1806618663 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:48:18.041 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.036+0000 7f7ed94f6640 1 --2- 172.21.15.99:0/1806618663 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562c79703c60 0x562c796f8220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:18.041 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.036+0000 7f7ed3fff640 1 -- 172.21.15.99:0/1806618663 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562c796fe100 msgr2=0x562c79708140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:48:18.041 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.036+0000 7f7ed3fff640 1 --2- 172.21.15.99:0/1806618663 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562c796fe100 0x562c79708140 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:18.041 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.036+0000 7f7ed3fff640 1 -- 172.21.15.99:0/1806618663 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562c79703c60 msgr2=0x562c796f8220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:48:18.041 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.036+0000 7f7ed3fff640 1 --2- 172.21.15.99:0/1806618663 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562c79703c60 0x562c796f8220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:18.041 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.036+0000 7f7ed3fff640 1 -- 172.21.15.99:0/1806618663 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x562c79443620 con 0x562c796feaf0 2024-03-19T01:48:18.041 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.036+0000 7f7ed3fff640 1 --2- 172.21.15.99:0/1806618663 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562c796feaf0 0x562c7970f360 secure :-1 s=READY pgs=488 cs=0 l=1 rev1=1 crypto rx=0x562c79709300 tx=0x7f7ebc005f90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=975270e0ed9a2acd server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:48:18.041 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.037+0000 7f7eca7fc640 1 -- 172.21.15.99:0/1806618663 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7ebc012e60 con 0x562c796feaf0 2024-03-19T01:48:18.041 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.037+0000 7f7eca7fc640 1 -- 172.21.15.99:0/1806618663 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7ebc026020 con 0x562c796feaf0 2024-03-19T01:48:18.042 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.037+0000 7f7eca7fc640 1 -- 172.21.15.99:0/1806618663 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7ebc010430 con 0x562c796feaf0 2024-03-19T01:48:18.042 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.037+0000 7f7eddc15740 1 -- 172.21.15.99:0/1806618663 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562c796feaf0 msgr2=0x562c7970f360 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:48:18.042 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.037+0000 7f7eddc15740 1 --2- 172.21.15.99:0/1806618663 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562c796feaf0 0x562c7970f360 secure :-1 s=READY pgs=488 cs=0 l=1 rev1=1 crypto rx=0x562c79709300 tx=0x7f7ebc005f90 comp rx=0 tx=0).stop 2024-03-19T01:48:18.042 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.037+0000 7f7eddc15740 1 -- 172.21.15.99:0/1806618663 shutdown_connections 2024-03-19T01:48:18.042 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.037+0000 7f7eddc15740 1 --2- 172.21.15.99:0/1806618663 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562c79703c60 0x562c796f8220 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:18.042 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.037+0000 7f7eddc15740 1 --2- 172.21.15.99:0/1806618663 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562c796feaf0 0x562c7970f360 unknown :-1 s=CLOSED pgs=488 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:18.042 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.037+0000 7f7eddc15740 1 --2- 172.21.15.99:0/1806618663 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562c796fe100 0x562c79708140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:18.042 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.037+0000 7f7eddc15740 1 -- 172.21.15.99:0/1806618663 >> 172.21.15.99:0/1806618663 conn(0x562c796f9670 msgr2=0x562c79707a30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:48:18.042 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.037+0000 7f7eddc15740 1 -- 172.21.15.99:0/1806618663 shutdown_connections 2024-03-19T01:48:18.042 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.037+0000 7f7eddc15740 1 -- 172.21.15.99:0/1806618663 wait complete. 2024-03-19T01:48:18.042 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.038+0000 7f7eddc15740 1 -- start start 2024-03-19T01:48:18.042 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.038+0000 7f7eddc15740 1 Processor -- start 2024-03-19T01:48:18.042 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.038+0000 7f7eddc15740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562c796fe100 0x562c7970f020 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:18.042 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.038+0000 7f7eddc15740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562c796feaf0 0x562c7970bac0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:18.043 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.038+0000 7f7eddc15740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562c79703c60 0x562c7970c000 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:18.043 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.038+0000 7f7eddc15740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f7ed41a73a0 con 0x562c796fe100 2024-03-19T01:48:18.043 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.038+0000 7f7eddc15740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f7ed41a8810 con 0x562c79703c60 2024-03-19T01:48:18.043 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.038+0000 7f7eddc15740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f7ed41a7a70 con 0x562c796feaf0 2024-03-19T01:48:18.043 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.038+0000 7f7eddc15740 10 client.0 _ll_register_callbacks cb 0x7f7ebc009570 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:48:18.043 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.038+0000 7f7ed8cf5640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562c796fe100 0x562c7970f020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:18.043 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.038+0000 7f7ed8cf5640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562c796fe100 0x562c7970f020 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:53344/0 (socket says 172.21.15.99:53344) 2024-03-19T01:48:18.043 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.038+0000 7f7ed8cf5640 1 -- 172.21.15.99:0/3544288406 learned_addr learned my addr 172.21.15.99:0/3544288406 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:48:18.043 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.038+0000 7f7ed3fff640 1 --2- 172.21.15.99:0/3544288406 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562c796feaf0 0x562c7970bac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:18.043 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.038+0000 7f7ed94f6640 1 --2- 172.21.15.99:0/3544288406 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562c79703c60 0x562c7970c000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:18.043 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.038+0000 7f7ed8cf5640 1 -- 172.21.15.99:0/3544288406 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562c796feaf0 msgr2=0x562c7970bac0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:48:18.043 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.038+0000 7f7ed8cf5640 1 --2- 172.21.15.99:0/3544288406 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562c796feaf0 0x562c7970bac0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:18.043 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.038+0000 7f7ed8cf5640 1 -- 172.21.15.99:0/3544288406 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562c79703c60 msgr2=0x562c7970c000 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:48:18.043 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.038+0000 7f7ed8cf5640 1 --2- 172.21.15.99:0/3544288406 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562c79703c60 0x562c7970c000 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:18.044 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.038+0000 7f7ed8cf5640 1 -- 172.21.15.99:0/3544288406 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x562c7970c5f0 con 0x562c796fe100 2024-03-19T01:48:18.044 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.039+0000 7f7ed8cf5640 1 --2- 172.21.15.99:0/3544288406 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562c796fe100 0x562c7970f020 secure :-1 s=READY pgs=489 cs=0 l=1 rev1=1 crypto rx=0x7f7ec40ee390 tx=0x7f7ec40c3260 comp rx=0 tx=0).ready entity=mon.0 client_cookie=bdca94cb86b5f531 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:48:18.044 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.039+0000 7f7eb3fff640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:48:18.044 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.039+0000 7f7eb3fff640 1 -- 172.21.15.99:0/3544288406 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7ec404b070 con 0x562c796fe100 2024-03-19T01:48:18.044 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.039+0000 7f7eb3fff640 1 -- 172.21.15.99:0/3544288406 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f7ec40bc450 con 0x562c796fe100 2024-03-19T01:48:18.044 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.039+0000 7f7eddc15740 1 -- 172.21.15.99:0/3544288406 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f7ebc026020 con 0x562c796fe100 2024-03-19T01:48:18.044 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.039+0000 7f7eb3fff640 1 -- 172.21.15.99:0/3544288406 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7f7ec40bcdd0 con 0x562c796fe100 2024-03-19T01:48:18.044 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.039+0000 7f7eddc15740 10 client.15924 fetch_fsmap learned FSMap version 27 2024-03-19T01:48:18.044 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.039+0000 7f7eddc15740 1 -- 172.21.15.99:0/3544288406 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x562c7970c5f0 con 0x562c796fe100 2024-03-19T01:48:18.044 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.039+0000 7f7eb3fff640 1 -- 172.21.15.99:0/3544288406 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7ebc026020 con 0x562c796fe100 2024-03-19T01:48:18.044 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.039+0000 7f7eb3fff640 1 -- 172.21.15.99:0/3544288406 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 27) ==== 8338+0+0 (secure 0 0 0) 0x7f7ec40d3040 con 0x562c796fe100 2024-03-19T01:48:18.044 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.039+0000 7f7eddc15740 10 client.15924 fetch_fsmap finished waiting for FSMap version 27 2024-03-19T01:48:18.044 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.039+0000 7f7eddc15740 10 client.15924 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:48:18.044 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.039+0000 7f7eddc15740 20 client.15924 populate_metadata read hostname 'smithi099' 2024-03-19T01:48:18.045 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.039+0000 7f7eddc15740 1 --2- 172.21.15.99:0/3544288406 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x562c797792b0 0x562c7977b6d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:18.045 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.039+0000 7f7eddc15740 4 client.15924 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:48:18.045 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.039+0000 7f7eddc15740 1 -- 172.21.15.99:0/3544288406 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x562c7966a470 con 0x562c797792b0 2024-03-19T01:48:18.045 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.039+0000 7f7ed3fff640 1 --2- 172.21.15.99:0/3544288406 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x562c797792b0 0x562c7977b6d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:18.045 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.040+0000 7f7eb3fff640 10 client.15924 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:48:18.045 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.040+0000 7f7ed3fff640 1 --2- 172.21.15.99:0/3544288406 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x562c797792b0 0x562c7977b6d0 crc :-1 s=READY pgs=94 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=b107eab339663385 server_cookie=f322b3c18a07c5f7 in_seq=0 out_seq=0 2024-03-19T01:48:18.102 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.102+0000 7f7eb3fff640 1 -- 172.21.15.99:0/3544288406 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x562c7966a470 con 0x562c797792b0 2024-03-19T01:48:18.103 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.102+0000 7f7eb3fff640 10 client.15924 handle_command_reply: tid=0 2024-03-19T01:48:18.103 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.102+0000 7f7eddc15740 1 client.15924 shutdown 2024-03-19T01:48:18.103 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.102+0000 7f7eddc15740 1 -- 172.21.15.99:0/3544288406 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562c796fe100 msgr2=0x562c7970f020 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:48:18.103 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.102+0000 7f7eddc15740 1 --2- 172.21.15.99:0/3544288406 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562c796fe100 0x562c7970f020 secure :-1 s=READY pgs=489 cs=0 l=1 rev1=1 crypto rx=0x7f7ec40ee390 tx=0x7f7ec40c3260 comp rx=0 tx=0).stop 2024-03-19T01:48:18.103 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.102+0000 7f7eddc15740 1 -- 172.21.15.99:0/3544288406 shutdown_connections 2024-03-19T01:48:18.103 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.102+0000 7f7eddc15740 1 --2- 172.21.15.99:0/3544288406 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x562c797792b0 0x562c7977b6d0 crc :-1 s=READY pgs=94 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:18.103 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.102+0000 7f7eddc15740 1 --2- 172.21.15.99:0/3544288406 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562c79703c60 0x562c7970c000 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:18.103 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.102+0000 7f7eddc15740 1 --2- 172.21.15.99:0/3544288406 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562c796feaf0 0x562c7970bac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:18.103 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.102+0000 7f7eddc15740 1 --2- 172.21.15.99:0/3544288406 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562c796fe100 0x562c7970f020 unknown :-1 s=CLOSED pgs=489 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:18.103 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.102+0000 7f7eb3fff640 0 client.15924 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:48:18.103 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.102+0000 7f7eddc15740 1 -- 172.21.15.99:0/3544288406 >> 172.21.15.99:0/3544288406 conn(0x562c796f9670 msgr2=0x562c796f8220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:48:18.103 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.102+0000 7f7eddc15740 1 -- 172.21.15.99:0/3544288406 shutdown_connections 2024-03-19T01:48:18.103 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.102+0000 7f7eddc15740 1 -- 172.21.15.99:0/3544288406 wait complete. 2024-03-19T01:48:18.104 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.103+0000 7f7eddc15740 20 client.15924 trim_cache size 0 max 16384 2024-03-19T01:48:18.104 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:48:18.104 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:48:18.104 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:48:18.104 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:48:18.105 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.103+0000 7f7edaf59640 1 -- 172.21.15.99:0/247500977 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f7ec00a28a0 msgr2=0x7f7ec00a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:48:18.105 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.103+0000 7f7edaf59640 1 --2- 172.21.15.99:0/247500977 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f7ec00a28a0 0x7f7ec00a4d60 secure :-1 s=READY pgs=285 cs=0 l=1 rev1=1 crypto rx=0x562c796ff350 tx=0x7f7ecc009500 comp rx=0 tx=0).stop 2024-03-19T01:48:18.105 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.103+0000 7f7edaf59640 1 -- 172.21.15.99:0/247500977 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7ed4100750 msgr2=0x7f7ed4197ab0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:48:18.105 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.103+0000 7f7edaf59640 1 --2- 172.21.15.99:0/247500977 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7ed4100750 0x7f7ed4197ab0 secure :-1 s=READY pgs=486 cs=0 l=1 rev1=1 crypto rx=0x7f7ec4000c00 tx=0x7f7ec402c640 comp rx=0 tx=0).stop 2024-03-19T01:48:18.105 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.103+0000 7f7edaf59640 1 -- 172.21.15.99:0/247500977 shutdown_connections 2024-03-19T01:48:18.105 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.103+0000 7f7edaf59640 1 --2- 172.21.15.99:0/247500977 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f7ec00a28a0 0x7f7ec00a4d60 unknown :-1 s=CLOSED pgs=285 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:18.105 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.103+0000 7f7edaf59640 1 --2- 172.21.15.99:0/247500977 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f7ed4101940 0x7f7ed4197ff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:18.105 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.103+0000 7f7edaf59640 1 --2- 172.21.15.99:0/247500977 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f7ed4102300 0x7f7ed419f060 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:18.105 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.103+0000 7f7edaf59640 1 --2- 172.21.15.99:0/247500977 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7ed4100750 0x7f7ed4197ab0 unknown :-1 s=CLOSED pgs=486 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:18.106 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.103+0000 7f7edaf59640 1 -- 172.21.15.99:0/247500977 >> 172.21.15.99:0/247500977 conn(0x7f7ed40fbcc0 msgr2=0x7f7ed40fd6d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:48:18.106 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.103+0000 7f7edaf59640 1 -- 172.21.15.99:0/247500977 shutdown_connections 2024-03-19T01:48:18.106 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:18.103+0000 7f7edaf59640 1 -- 172.21.15.99:0/247500977 wait complete. 2024-03-19T01:48:18.690 INFO:tasks.workunit.client.0.smithi099.stdout: 1 867997 11.35 MB/sec execute 288 sec latency 25.812 ms 2024-03-19T01:48:19.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:48:18 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:48:19.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:48:18 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:48:19.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:48:18 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:48:19.690 INFO:tasks.workunit.client.0.smithi099.stdout: 1 870268 11.36 MB/sec execute 289 sec latency 29.012 ms 2024-03-19T01:48:20.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:48:19 smithi099 ceph-mon[62951]: pgmap v567: 97 pgs: 97 active+clean; 268 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 688 KiB/s rd, 2.6 MiB/s wr, 213 op/s 2024-03-19T01:48:20.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:48:19 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:48:20.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:48:19 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:48:20.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:48:19 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:48:20.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:48:19 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:48:20.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:48:19 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:48:20.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:48:19 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:48:20.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:48:19 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.9", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:48:20.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:48:19 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.10", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:48:20.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:48:19 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:48:20.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:48:19 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:48:20.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:48:19 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:48:20.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:48:19 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.11", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:48:20.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:48:19 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:48:20.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:48:19 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:48:20.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:48:19 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:48:20.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:48:19 smithi112 ceph-mon[66819]: pgmap v567: 97 pgs: 97 active+clean; 268 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 688 KiB/s rd, 2.6 MiB/s wr, 213 op/s 2024-03-19T01:48:20.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:48:19 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:48:20.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:48:19 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:48:20.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:48:19 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:48:20.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:48:19 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:48:20.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:48:19 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:48:20.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:48:19 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:48:20.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:48:19 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.9", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:48:20.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:48:19 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.10", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:48:20.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:48:19 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:48:20.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:48:19 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:48:20.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:48:19 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:48:20.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:48:19 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.11", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:48:20.113 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:48:19 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:48:20.113 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:48:19 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:48:20.113 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:48:19 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:48:20.207 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:48:19 smithi115 ceph-mon[67332]: pgmap v567: 97 pgs: 97 active+clean; 268 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 688 KiB/s rd, 2.6 MiB/s wr, 213 op/s 2024-03-19T01:48:20.207 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:48:19 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:48:20.207 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:48:19 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:48:20.207 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:48:19 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:48:20.207 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:48:19 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:48:20.207 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:48:19 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:48:20.207 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:48:19 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:48:20.208 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:48:19 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.9", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:48:20.208 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:48:19 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.10", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:48:20.208 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:48:19 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:48:20.208 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:48:19 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:48:20.208 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:48:19 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:48:20.208 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:48:19 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.11", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:48:20.208 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:48:19 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:48:20.208 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:48:19 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:48:20.208 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:48:19 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:48:20.690 INFO:tasks.workunit.client.0.smithi099.stdout: 1 872279 11.35 MB/sec execute 290 sec latency 45.223 ms 2024-03-19T01:48:21.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:48:20 smithi099 ceph-mon[62951]: Adjusting osd_memory_target on smithi099 to 176.2M 2024-03-19T01:48:21.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:48:20 smithi099 ceph-mon[62951]: Adjusting osd_memory_target on smithi112 to 174.1M 2024-03-19T01:48:21.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:48:20 smithi099 ceph-mon[62951]: Unable to set osd_memory_target on smithi099 to 184763801: error parsing value: Value '184763801' is below minimum 939524096 2024-03-19T01:48:21.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:48:20 smithi099 ceph-mon[62951]: Unable to set osd_memory_target on smithi112 to 182611968: error parsing value: Value '182611968' is below minimum 939524096 2024-03-19T01:48:21.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:48:20 smithi099 ceph-mon[62951]: Adjusting osd_memory_target on smithi115 to 174.1M 2024-03-19T01:48:21.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:48:20 smithi099 ceph-mon[62951]: Unable to set osd_memory_target on smithi115 to 182618419: error parsing value: Value '182618419' is below minimum 939524096 2024-03-19T01:48:21.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:48:20 smithi112 ceph-mon[66819]: Adjusting osd_memory_target on smithi099 to 176.2M 2024-03-19T01:48:21.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:48:20 smithi112 ceph-mon[66819]: Adjusting osd_memory_target on smithi112 to 174.1M 2024-03-19T01:48:21.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:48:20 smithi112 ceph-mon[66819]: Unable to set osd_memory_target on smithi099 to 184763801: error parsing value: Value '184763801' is below minimum 939524096 2024-03-19T01:48:21.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:48:20 smithi112 ceph-mon[66819]: Unable to set osd_memory_target on smithi112 to 182611968: error parsing value: Value '182611968' is below minimum 939524096 2024-03-19T01:48:21.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:48:20 smithi112 ceph-mon[66819]: Adjusting osd_memory_target on smithi115 to 174.1M 2024-03-19T01:48:21.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:48:20 smithi112 ceph-mon[66819]: Unable to set osd_memory_target on smithi115 to 182618419: error parsing value: Value '182618419' is below minimum 939524096 2024-03-19T01:48:21.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:48:20 smithi115 ceph-mon[67332]: Adjusting osd_memory_target on smithi099 to 176.2M 2024-03-19T01:48:21.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:48:20 smithi115 ceph-mon[67332]: Adjusting osd_memory_target on smithi112 to 174.1M 2024-03-19T01:48:21.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:48:20 smithi115 ceph-mon[67332]: Unable to set osd_memory_target on smithi099 to 184763801: error parsing value: Value '184763801' is below minimum 939524096 2024-03-19T01:48:21.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:48:20 smithi115 ceph-mon[67332]: Unable to set osd_memory_target on smithi112 to 182611968: error parsing value: Value '182611968' is below minimum 939524096 2024-03-19T01:48:21.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:48:20 smithi115 ceph-mon[67332]: Adjusting osd_memory_target on smithi115 to 174.1M 2024-03-19T01:48:21.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:48:20 smithi115 ceph-mon[67332]: Unable to set osd_memory_target on smithi115 to 182618419: error parsing value: Value '182618419' is below minimum 939524096 2024-03-19T01:48:21.690 INFO:tasks.workunit.client.0.smithi099.stdout: 1 874259 11.35 MB/sec execute 291 sec latency 29.231 ms 2024-03-19T01:48:22.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:48:21 smithi099 ceph-mon[62951]: pgmap v568: 97 pgs: 97 active+clean; 265 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 953 KiB/s rd, 2.8 MiB/s wr, 256 op/s 2024-03-19T01:48:22.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:48:21 smithi112 ceph-mon[66819]: pgmap v568: 97 pgs: 97 active+clean; 265 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 953 KiB/s rd, 2.8 MiB/s wr, 256 op/s 2024-03-19T01:48:22.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:48:21 smithi115 ceph-mon[67332]: pgmap v568: 97 pgs: 97 active+clean; 265 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 953 KiB/s rd, 2.8 MiB/s wr, 256 op/s 2024-03-19T01:48:22.690 INFO:tasks.workunit.client.0.smithi099.stdout: 1 877138 11.38 MB/sec execute 292 sec latency 10.447 ms 2024-03-19T01:48:23.690 INFO:tasks.workunit.client.0.smithi099.stdout: 1 878905 11.36 MB/sec execute 293 sec latency 38.193 ms 2024-03-19T01:48:24.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:48:23 smithi115 ceph-mon[67332]: pgmap v569: 97 pgs: 97 active+clean; 255 MiB data, 2.5 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 3.2 MiB/s wr, 294 op/s 2024-03-19T01:48:24.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:48:23 smithi099 ceph-mon[62951]: pgmap v569: 97 pgs: 97 active+clean; 255 MiB data, 2.5 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 3.2 MiB/s wr, 294 op/s 2024-03-19T01:48:24.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:48:23 smithi112 ceph-mon[66819]: pgmap v569: 97 pgs: 97 active+clean; 255 MiB data, 2.5 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 3.2 MiB/s wr, 294 op/s 2024-03-19T01:48:24.690 INFO:tasks.workunit.client.0.smithi099.stdout: 1 881069 11.37 MB/sec execute 294 sec latency 39.248 ms 2024-03-19T01:48:25.690 INFO:tasks.workunit.client.0.smithi099.stdout: 1 883407 11.37 MB/sec execute 295 sec latency 51.847 ms 2024-03-19T01:48:26.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:48:25 smithi115 ceph-mon[67332]: pgmap v570: 97 pgs: 97 active+clean; 247 MiB data, 2.5 GiB used, 1.0 TiB / 1.0 TiB avail; 751 KiB/s rd, 2.5 MiB/s wr, 241 op/s 2024-03-19T01:48:26.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:48:25 smithi099 ceph-mon[62951]: pgmap v570: 97 pgs: 97 active+clean; 247 MiB data, 2.5 GiB used, 1.0 TiB / 1.0 TiB avail; 751 KiB/s rd, 2.5 MiB/s wr, 241 op/s 2024-03-19T01:48:26.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:48:25 smithi112 ceph-mon[66819]: pgmap v570: 97 pgs: 97 active+clean; 247 MiB data, 2.5 GiB used, 1.0 TiB / 1.0 TiB avail; 751 KiB/s rd, 2.5 MiB/s wr, 241 op/s 2024-03-19T01:48:26.690 INFO:tasks.workunit.client.0.smithi099.stdout: 1 885268 11.36 MB/sec execute 296 sec latency 44.978 ms 2024-03-19T01:48:27.690 INFO:tasks.workunit.client.0.smithi099.stdout: 1 887939 11.38 MB/sec execute 297 sec latency 12.027 ms 2024-03-19T01:48:28.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:48:27 smithi115 ceph-mon[67332]: pgmap v571: 97 pgs: 97 active+clean; 249 MiB data, 2.5 GiB used, 1.0 TiB / 1.0 TiB avail; 834 KiB/s rd, 3.1 MiB/s wr, 269 op/s 2024-03-19T01:48:28.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:48:27 smithi099 ceph-mon[62951]: pgmap v571: 97 pgs: 97 active+clean; 249 MiB data, 2.5 GiB used, 1.0 TiB / 1.0 TiB avail; 834 KiB/s rd, 3.1 MiB/s wr, 269 op/s 2024-03-19T01:48:28.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:48:27 smithi112 ceph-mon[66819]: pgmap v571: 97 pgs: 97 active+clean; 249 MiB data, 2.5 GiB used, 1.0 TiB / 1.0 TiB avail; 834 KiB/s rd, 3.1 MiB/s wr, 269 op/s 2024-03-19T01:48:28.690 INFO:tasks.workunit.client.0.smithi099.stdout: 1 890161 11.37 MB/sec execute 298 sec latency 39.791 ms 2024-03-19T01:48:29.690 INFO:tasks.workunit.client.0.smithi099.stdout: 1 892118 11.37 MB/sec execute 299 sec latency 27.529 ms 2024-03-19T01:48:30.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:48:29 smithi115 ceph-mon[67332]: pgmap v572: 97 pgs: 97 active+clean; 253 MiB data, 2.5 GiB used, 1.0 TiB / 1.0 TiB avail; 845 KiB/s rd, 3.0 MiB/s wr, 260 op/s 2024-03-19T01:48:30.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:48:29 smithi099 ceph-mon[62951]: pgmap v572: 97 pgs: 97 active+clean; 253 MiB data, 2.5 GiB used, 1.0 TiB / 1.0 TiB avail; 845 KiB/s rd, 3.0 MiB/s wr, 260 op/s 2024-03-19T01:48:30.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:48:29 smithi112 ceph-mon[66819]: pgmap v572: 97 pgs: 97 active+clean; 253 MiB data, 2.5 GiB used, 1.0 TiB / 1.0 TiB avail; 845 KiB/s rd, 3.0 MiB/s wr, 260 op/s 2024-03-19T01:48:30.691 INFO:tasks.workunit.client.0.smithi099.stdout: 1 894804 11.39 MB/sec execute 300 sec latency 86.576 ms 2024-03-19T01:48:31.193 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.193+0000 7f8a30f62640 1 -- 172.21.15.99:0/690997042 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8a2c102a60 msgr2=0x7f8a2c102ee0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:48:31.193 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.193+0000 7f8a30f62640 1 --2- 172.21.15.99:0/690997042 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8a2c102a60 0x7f8a2c102ee0 secure :-1 s=READY pgs=129 cs=0 l=1 rev1=1 crypto rx=0x7f8a14009900 tx=0x7f8a1402bc30 comp rx=0 tx=0).stop 2024-03-19T01:48:31.193 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.193+0000 7f8a30f62640 1 -- 172.21.15.99:0/690997042 shutdown_connections 2024-03-19T01:48:31.193 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.193+0000 7f8a30f62640 1 --2- 172.21.15.99:0/690997042 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8a2c102a60 0x7f8a2c102ee0 unknown :-1 s=CLOSED pgs=129 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:31.193 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.193+0000 7f8a30f62640 1 --2- 172.21.15.99:0/690997042 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8a2c103420 0x7f8a2c109cb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:31.193 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.193+0000 7f8a30f62640 1 --2- 172.21.15.99:0/690997042 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8a2c101870 0x7f8a2c101c90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:31.193 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.193+0000 7f8a30f62640 1 -- 172.21.15.99:0/690997042 >> 172.21.15.99:0/690997042 conn(0x7f8a2c0fcde0 msgr2=0x7f8a2c0ff220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:48:31.193 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.193+0000 7f8a30f62640 1 -- 172.21.15.99:0/690997042 shutdown_connections 2024-03-19T01:48:31.193 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.193+0000 7f8a30f62640 1 -- 172.21.15.99:0/690997042 wait complete. 2024-03-19T01:48:31.195 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.193+0000 7f8a30f62640 1 Processor -- start 2024-03-19T01:48:31.196 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.194+0000 7f8a30f62640 1 -- start start 2024-03-19T01:48:31.196 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.194+0000 7f8a30f62640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8a2c101870 0x7f8a2c197b10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:31.196 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.194+0000 7f8a30f62640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8a2c102a60 0x7f8a2c198050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:31.196 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.194+0000 7f8a30f62640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8a2c103420 0x7f8a2c19f0c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:31.196 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.194+0000 7f8a30f62640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f8a2c10d4d0 con 0x7f8a2c102a60 2024-03-19T01:48:31.196 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.194+0000 7f8a30f62640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f8a2c10d350 con 0x7f8a2c101870 2024-03-19T01:48:31.196 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.194+0000 7f8a30f62640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f8a2c10d650 con 0x7f8a2c103420 2024-03-19T01:48:31.196 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.194+0000 7f8a2a59c640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8a2c101870 0x7f8a2c197b10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:31.196 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.194+0000 7f8a2a59c640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8a2c101870 0x7f8a2c197b10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:55048/0 (socket says 172.21.15.99:55048) 2024-03-19T01:48:31.196 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.194+0000 7f8a2a59c640 1 -- 172.21.15.99:0/3866801129 learned_addr learned my addr 172.21.15.99:0/3866801129 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:48:31.196 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.194+0000 7f8a2ad9d640 1 --2- 172.21.15.99:0/3866801129 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8a2c103420 0x7f8a2c19f0c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:31.196 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.194+0000 7f8a23fff640 1 --2- 172.21.15.99:0/3866801129 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8a2c102a60 0x7f8a2c198050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:31.196 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.194+0000 7f8a2a59c640 1 -- 172.21.15.99:0/3866801129 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8a2c103420 msgr2=0x7f8a2c19f0c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:48:31.196 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.194+0000 7f8a2a59c640 1 --2- 172.21.15.99:0/3866801129 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8a2c103420 0x7f8a2c19f0c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:31.197 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.194+0000 7f8a2a59c640 1 -- 172.21.15.99:0/3866801129 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8a2c102a60 msgr2=0x7f8a2c198050 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:48:31.197 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.194+0000 7f8a2a59c640 1 --2- 172.21.15.99:0/3866801129 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8a2c102a60 0x7f8a2c198050 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:31.197 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.194+0000 7f8a2a59c640 1 -- 172.21.15.99:0/3866801129 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f8a2c19f730 con 0x7f8a2c101870 2024-03-19T01:48:31.197 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.195+0000 7f8a2a59c640 1 --2- 172.21.15.99:0/3866801129 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8a2c101870 0x7f8a2c197b10 secure :-1 s=READY pgs=130 cs=0 l=1 rev1=1 crypto rx=0x7f8a1c00e9c0 tx=0x7f8a1c00ee80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:48:31.197 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.195+0000 7f8a237fe640 1 -- 172.21.15.99:0/3866801129 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8a1c00cde0 con 0x7f8a2c101870 2024-03-19T01:48:31.197 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.195+0000 7f8a30f62640 1 -- 172.21.15.99:0/3866801129 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f8a2c19fa20 con 0x7f8a2c101870 2024-03-19T01:48:31.197 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.195+0000 7f8a237fe640 1 -- 172.21.15.99:0/3866801129 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f8a1c010910 con 0x7f8a2c101870 2024-03-19T01:48:31.197 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.195+0000 7f8a237fe640 1 -- 172.21.15.99:0/3866801129 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8a1c018a10 con 0x7f8a2c101870 2024-03-19T01:48:31.197 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.195+0000 7f8a30f62640 1 -- 172.21.15.99:0/3866801129 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f8a2c19ff90 con 0x7f8a2c101870 2024-03-19T01:48:31.199 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.198+0000 7f8a237fe640 1 -- 172.21.15.99:0/3866801129 <== mon.2 v2:172.21.15.112:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f8a1c01f020 con 0x7f8a2c101870 2024-03-19T01:48:31.199 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.198+0000 7f8a33c2b740 1 Processor -- start 2024-03-19T01:48:31.199 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.198+0000 7f8a33c2b740 1 -- start start 2024-03-19T01:48:31.199 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.198+0000 7f8a33c2b740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5605e2238100 0x5605e2238520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:31.200 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.198+0000 7f8a33c2b740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5605e2238af0 0x5605e223d5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:31.200 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.198+0000 7f8a33c2b740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5605e223dc60 0x5605e22400f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:31.200 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.198+0000 7f8a33c2b740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x5605e219cbe0 con 0x5605e223dc60 2024-03-19T01:48:31.200 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.198+0000 7f8a33c2b740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x5605e2145960 con 0x5605e2238af0 2024-03-19T01:48:31.200 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.198+0000 7f8a33c2b740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x5605e2198c80 con 0x5605e2238100 2024-03-19T01:48:31.200 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.198+0000 7f8a237fe640 1 --2- 172.21.15.99:0/3866801129 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f8a180a28a0 0x7f8a180a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:31.200 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.198+0000 7f8a2a59c640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5605e2238af0 0x5605e223d5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:31.200 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.199+0000 7f8a2ad9d640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5605e2238100 0x5605e2238520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:31.200 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.199+0000 7f8a23fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5605e223dc60 0x5605e22400f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:31.200 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.199+0000 7f8a2ad9d640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5605e2238100 0x5605e2238520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:36780/0 (socket says 172.21.15.99:36780) 2024-03-19T01:48:31.200 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.199+0000 7f8a2ad9d640 1 -- 172.21.15.99:0/3882975656 learned_addr learned my addr 172.21.15.99:0/3882975656 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:48:31.200 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.199+0000 7f8a2ad9d640 1 --2- 172.21.15.99:0/3866801129 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f8a180a28a0 0x7f8a180a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:31.200 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.199+0000 7f8a2ad9d640 1 -- 172.21.15.99:0/3882975656 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5605e2238af0 msgr2=0x5605e223d5f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:48:31.200 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.199+0000 7f8a237fe640 1 -- 172.21.15.99:0/3866801129 <== mon.2 v2:172.21.15.112:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f8a1c0cec60 con 0x7f8a2c101870 2024-03-19T01:48:31.200 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.199+0000 7f8a2ad9d640 1 --2- 172.21.15.99:0/3882975656 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5605e2238af0 0x5605e223d5f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:31.201 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.199+0000 7f8a2ad9d640 1 -- 172.21.15.99:0/3882975656 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5605e223dc60 msgr2=0x5605e22400f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:48:31.201 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.199+0000 7f8a2ad9d640 1 --2- 172.21.15.99:0/3882975656 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5605e223dc60 0x5605e22400f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:31.201 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.199+0000 7f8a2ad9d640 1 -- 172.21.15.99:0/3882975656 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5605e213cd20 con 0x5605e2238100 2024-03-19T01:48:31.201 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.199+0000 7f8a2ad9d640 1 --2- 172.21.15.99:0/3866801129 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f8a180a28a0 0x7f8a180a4d60 secure :-1 s=READY pgs=288 cs=0 l=1 rev1=1 crypto rx=0x7f8a2400fc90 tx=0x7f8a2401e000 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:48:31.201 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.199+0000 7f8a2ad9d640 1 --2- 172.21.15.99:0/3882975656 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5605e2238100 0x5605e2238520 secure :-1 s=READY pgs=97 cs=0 l=1 rev1=1 crypto rx=0x7f8a24011e20 tx=0x7f8a2400f790 comp rx=0 tx=0).ready entity=mon.1 client_cookie=d3bd05726e709447 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:48:31.201 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.199+0000 7f8a20ff9640 1 -- 172.21.15.99:0/3882975656 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8a2401dcb0 con 0x5605e2238100 2024-03-19T01:48:31.201 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.199+0000 7f8a20ff9640 1 -- 172.21.15.99:0/3882975656 <== mon.1 v2:172.21.15.115:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f8a2401de50 con 0x5605e2238100 2024-03-19T01:48:31.201 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.199+0000 7f8a20ff9640 1 -- 172.21.15.99:0/3882975656 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8a24004240 con 0x5605e2238100 2024-03-19T01:48:31.206 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.200+0000 7f8a33c2b740 1 -- 172.21.15.99:0/3882975656 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5605e2238100 msgr2=0x5605e2238520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:48:31.206 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.200+0000 7f8a33c2b740 1 --2- 172.21.15.99:0/3882975656 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5605e2238100 0x5605e2238520 secure :-1 s=READY pgs=97 cs=0 l=1 rev1=1 crypto rx=0x7f8a24011e20 tx=0x7f8a2400f790 comp rx=0 tx=0).stop 2024-03-19T01:48:31.206 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.200+0000 7f8a33c2b740 1 -- 172.21.15.99:0/3882975656 shutdown_connections 2024-03-19T01:48:31.206 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.200+0000 7f8a33c2b740 1 --2- 172.21.15.99:0/3882975656 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5605e2238af0 0x5605e223d5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:31.206 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.200+0000 7f8a33c2b740 1 --2- 172.21.15.99:0/3882975656 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5605e223dc60 0x5605e22400f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:31.206 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.200+0000 7f8a33c2b740 1 --2- 172.21.15.99:0/3882975656 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5605e2238100 0x5605e2238520 unknown :-1 s=CLOSED pgs=97 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:31.206 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.200+0000 7f8a33c2b740 1 -- 172.21.15.99:0/3882975656 >> 172.21.15.99:0/3882975656 conn(0x5605e2233670 msgr2=0x5605e2233a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:48:31.206 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.200+0000 7f8a33c2b740 1 -- 172.21.15.99:0/3882975656 shutdown_connections 2024-03-19T01:48:31.206 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.200+0000 7f8a33c2b740 1 -- 172.21.15.99:0/3882975656 wait complete. 2024-03-19T01:48:31.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.200+0000 7f8a33c2b740 1 -- start start 2024-03-19T01:48:31.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.201+0000 7f8a33c2b740 1 Processor -- start 2024-03-19T01:48:31.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.201+0000 7f8a33c2b740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5605e2238100 0x5605e2241fb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:31.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.201+0000 7f8a33c2b740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5605e2238af0 0x5605e22424f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:31.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.201+0000 7f8a33c2b740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5605e223dc60 0x5605e2249530 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:31.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.201+0000 7f8a33c2b740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x5605e2240b50 con 0x5605e223dc60 2024-03-19T01:48:31.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.201+0000 7f8a33c2b740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x5605e2158db0 con 0x5605e2238100 2024-03-19T01:48:31.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.201+0000 7f8a33c2b740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x5605e2240cd0 con 0x5605e2238af0 2024-03-19T01:48:31.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.201+0000 7f8a33c2b740 10 client.0 _ll_register_callbacks cb 0x5605e215de70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:48:31.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.201+0000 7f8a2ad9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5605e223dc60 0x5605e2249530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:31.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.201+0000 7f8a2ad9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5605e223dc60 0x5605e2249530 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:56048/0 (socket says 172.21.15.99:56048) 2024-03-19T01:48:31.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.201+0000 7f8a2ad9d640 1 -- 172.21.15.99:0/726092685 learned_addr learned my addr 172.21.15.99:0/726092685 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:48:31.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.201+0000 7f8a23fff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5605e2238af0 0x5605e22424f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:31.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.201+0000 7f8a2a59c640 1 --2- 172.21.15.99:0/726092685 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5605e2238100 0x5605e2241fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:31.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.201+0000 7f8a2ad9d640 1 -- 172.21.15.99:0/726092685 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5605e2238af0 msgr2=0x5605e22424f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:48:31.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.201+0000 7f8a2ad9d640 1 --2- 172.21.15.99:0/726092685 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5605e2238af0 0x5605e22424f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:31.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.201+0000 7f8a2ad9d640 1 -- 172.21.15.99:0/726092685 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5605e2238100 msgr2=0x5605e2241fb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:48:31.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.201+0000 7f8a2ad9d640 1 --2- 172.21.15.99:0/726092685 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5605e2238100 0x5605e2241fb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:31.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.201+0000 7f8a2ad9d640 1 -- 172.21.15.99:0/726092685 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5605e2157c60 con 0x5605e223dc60 2024-03-19T01:48:31.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.201+0000 7f8a23fff640 1 --2- 172.21.15.99:0/726092685 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5605e2238af0 0x5605e22424f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:48:31.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.201+0000 7f8a2a59c640 1 --2- 172.21.15.99:0/726092685 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5605e2238100 0x5605e2241fb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:48:31.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.201+0000 7f8a2ad9d640 1 --2- 172.21.15.99:0/726092685 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5605e223dc60 0x5605e2249530 secure :-1 s=READY pgs=490 cs=0 l=1 rev1=1 crypto rx=0x7f8a2400ffa0 tx=0x7f8a24011780 comp rx=0 tx=0).ready entity=mon.0 client_cookie=837db84f57aa0641 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:48:31.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.201+0000 7f8a11ffb640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:48:31.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.201+0000 7f8a11ffb640 1 -- 172.21.15.99:0/726092685 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8a2402a030 con 0x5605e223dc60 2024-03-19T01:48:31.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.202+0000 7f8a11ffb640 1 -- 172.21.15.99:0/726092685 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f8a24011930 con 0x5605e223dc60 2024-03-19T01:48:31.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.202+0000 7f8a33c2b740 1 -- 172.21.15.99:0/726092685 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f8a2401de50 con 0x5605e223dc60 2024-03-19T01:48:31.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.202+0000 7f8a11ffb640 1 -- 172.21.15.99:0/726092685 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7f8a24003be0 con 0x5605e223dc60 2024-03-19T01:48:31.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.202+0000 7f8a33c2b740 10 client.15936 fetch_fsmap learned FSMap version 27 2024-03-19T01:48:31.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.202+0000 7f8a33c2b740 1 -- 172.21.15.99:0/726092685 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x5605e1eb1210 con 0x5605e223dc60 2024-03-19T01:48:31.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.202+0000 7f8a11ffb640 1 -- 172.21.15.99:0/726092685 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f8a2401de50 con 0x5605e223dc60 2024-03-19T01:48:31.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.202+0000 7f8a11ffb640 1 -- 172.21.15.99:0/726092685 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 27) ==== 8338+0+0 (secure 0 0 0) 0x7f8a24004620 con 0x5605e223dc60 2024-03-19T01:48:31.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.202+0000 7f8a33c2b740 10 client.15936 fetch_fsmap finished waiting for FSMap version 27 2024-03-19T01:48:31.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.202+0000 7f8a33c2b740 10 client.15936 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:48:31.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.202+0000 7f8a33c2b740 20 client.15936 populate_metadata read hostname 'smithi099' 2024-03-19T01:48:31.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.202+0000 7f8a33c2b740 1 --2- 172.21.15.99:0/726092685 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5605e224dda0 0x5605e2250190 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:31.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.202+0000 7f8a33c2b740 4 client.15936 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:48:31.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.202+0000 7f8a33c2b740 1 -- 172.21.15.99:0/726092685 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5605e21aa120 con 0x5605e224dda0 2024-03-19T01:48:31.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.202+0000 7f8a2a59c640 1 --2- 172.21.15.99:0/726092685 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5605e224dda0 0x5605e2250190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:31.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.203+0000 7f8a11ffb640 10 client.15936 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:48:31.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.203+0000 7f8a2a59c640 1 --2- 172.21.15.99:0/726092685 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5605e224dda0 0x5605e2250190 crc :-1 s=READY pgs=95 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=416a1ea1365a873c server_cookie=442c027339fadac4 in_seq=0 out_seq=0 2024-03-19T01:48:31.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.203+0000 7f8a11ffb640 1 -- 172.21.15.99:0/726092685 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x5605e21aa120 con 0x5605e224dda0 2024-03-19T01:48:31.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.203+0000 7f8a11ffb640 10 client.15936 handle_command_reply: tid=0 2024-03-19T01:48:31.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.204+0000 7f8a33c2b740 1 client.15936 shutdown 2024-03-19T01:48:31.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.204+0000 7f8a33c2b740 1 -- 172.21.15.99:0/726092685 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5605e223dc60 msgr2=0x5605e2249530 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:48:31.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.204+0000 7f8a33c2b740 1 --2- 172.21.15.99:0/726092685 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5605e223dc60 0x5605e2249530 secure :-1 s=READY pgs=490 cs=0 l=1 rev1=1 crypto rx=0x7f8a2400ffa0 tx=0x7f8a24011780 comp rx=0 tx=0).stop 2024-03-19T01:48:31.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.204+0000 7f8a33c2b740 1 -- 172.21.15.99:0/726092685 shutdown_connections 2024-03-19T01:48:31.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.204+0000 7f8a33c2b740 1 --2- 172.21.15.99:0/726092685 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5605e224dda0 0x5605e2250190 crc :-1 s=READY pgs=95 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:31.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.204+0000 7f8a33c2b740 1 --2- 172.21.15.99:0/726092685 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5605e223dc60 0x5605e2249530 unknown :-1 s=CLOSED pgs=490 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:31.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.204+0000 7f8a11ffb640 0 client.15936 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:48:31.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.204+0000 7f8a33c2b740 1 --2- 172.21.15.99:0/726092685 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5605e2238af0 0x5605e22424f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:31.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.204+0000 7f8a33c2b740 1 --2- 172.21.15.99:0/726092685 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5605e2238100 0x5605e2241fb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:31.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.204+0000 7f8a33c2b740 1 -- 172.21.15.99:0/726092685 >> 172.21.15.99:0/726092685 conn(0x5605e2233670 msgr2=0x5605e22348c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:48:31.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.204+0000 7f8a33c2b740 1 -- 172.21.15.99:0/726092685 shutdown_connections 2024-03-19T01:48:31.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.204+0000 7f8a33c2b740 1 -- 172.21.15.99:0/726092685 wait complete. 2024-03-19T01:48:31.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.204+0000 7f8a33c2b740 20 client.15936 trim_cache size 0 max 16384 2024-03-19T01:48:31.223 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.218+0000 7f8a33c2b740 1 Processor -- start 2024-03-19T01:48:31.223 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.219+0000 7f8a33c2b740 1 -- start start 2024-03-19T01:48:31.223 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.219+0000 7f8a33c2b740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5605e2238100 0x5605e2232220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:31.223 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.219+0000 7f8a33c2b740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5605e2238af0 0x5605e2232760 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:31.223 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.219+0000 7f8a33c2b740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5605e223dc60 0x5605e2239350 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:31.223 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.219+0000 7f8a33c2b740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f8a2c1ab730 con 0x5605e2238100 2024-03-19T01:48:31.223 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.219+0000 7f8a33c2b740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f8a2c1a9000 con 0x5605e2238af0 2024-03-19T01:48:31.223 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.219+0000 7f8a33c2b740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f8a2c1a6e50 con 0x5605e223dc60 2024-03-19T01:48:31.223 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.219+0000 7f8a2a59c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5605e2238100 0x5605e2232220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:31.223 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.219+0000 7f8a2a59c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5605e2238100 0x5605e2232220 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:56064/0 (socket says 172.21.15.99:56064) 2024-03-19T01:48:31.223 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.219+0000 7f8a2a59c640 1 -- 172.21.15.99:0/1367578899 learned_addr learned my addr 172.21.15.99:0/1367578899 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:48:31.223 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.219+0000 7f8a2ad9d640 1 --2- 172.21.15.99:0/1367578899 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5605e223dc60 0x5605e2239350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:31.224 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.219+0000 7f8a23fff640 1 --2- 172.21.15.99:0/1367578899 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5605e2238af0 0x5605e2232760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:31.224 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.219+0000 7f8a2a59c640 1 -- 172.21.15.99:0/1367578899 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5605e223dc60 msgr2=0x5605e2239350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:48:31.224 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.219+0000 7f8a2a59c640 1 --2- 172.21.15.99:0/1367578899 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5605e223dc60 0x5605e2239350 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:31.224 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.219+0000 7f8a2a59c640 1 -- 172.21.15.99:0/1367578899 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5605e2238af0 msgr2=0x5605e2232760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:48:31.224 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.219+0000 7f8a2a59c640 1 --2- 172.21.15.99:0/1367578899 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5605e2238af0 0x5605e2232760 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:31.224 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.219+0000 7f8a2a59c640 1 -- 172.21.15.99:0/1367578899 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5605e1f7d620 con 0x5605e2238100 2024-03-19T01:48:31.224 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.219+0000 7f8a2a59c640 1 --2- 172.21.15.99:0/1367578899 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5605e2238100 0x5605e2232220 secure :-1 s=READY pgs=491 cs=0 l=1 rev1=1 crypto rx=0x7f8a1c0058c0 tx=0x7f8a1c0cc5f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b7f63da72ebd52cf server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:48:31.224 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.219+0000 7f8a13fff640 1 -- 172.21.15.99:0/1367578899 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8a1c09b5c0 con 0x5605e2238100 2024-03-19T01:48:31.224 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.219+0000 7f8a13fff640 1 -- 172.21.15.99:0/1367578899 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f8a1c09f710 con 0x5605e2238100 2024-03-19T01:48:31.224 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.219+0000 7f8a13fff640 1 -- 172.21.15.99:0/1367578899 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8a1c010ac0 con 0x5605e2238100 2024-03-19T01:48:31.224 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.220+0000 7f8a33c2b740 1 -- 172.21.15.99:0/1367578899 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5605e2238100 msgr2=0x5605e2232220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:48:31.224 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.220+0000 7f8a33c2b740 1 --2- 172.21.15.99:0/1367578899 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5605e2238100 0x5605e2232220 secure :-1 s=READY pgs=491 cs=0 l=1 rev1=1 crypto rx=0x7f8a1c0058c0 tx=0x7f8a1c0cc5f0 comp rx=0 tx=0).stop 2024-03-19T01:48:31.224 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.220+0000 7f8a33c2b740 1 -- 172.21.15.99:0/1367578899 shutdown_connections 2024-03-19T01:48:31.224 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.220+0000 7f8a33c2b740 1 --2- 172.21.15.99:0/1367578899 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5605e2238af0 0x5605e2232760 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:31.224 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.220+0000 7f8a33c2b740 1 --2- 172.21.15.99:0/1367578899 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5605e223dc60 0x5605e2239350 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:31.225 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.220+0000 7f8a33c2b740 1 --2- 172.21.15.99:0/1367578899 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5605e2238100 0x5605e2232220 unknown :-1 s=CLOSED pgs=491 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:31.225 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.220+0000 7f8a33c2b740 1 -- 172.21.15.99:0/1367578899 >> 172.21.15.99:0/1367578899 conn(0x5605e2233670 msgr2=0x5605e223d0a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:48:31.225 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.220+0000 7f8a33c2b740 1 -- 172.21.15.99:0/1367578899 shutdown_connections 2024-03-19T01:48:31.225 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.220+0000 7f8a33c2b740 1 -- 172.21.15.99:0/1367578899 wait complete. 2024-03-19T01:48:31.225 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.220+0000 7f8a33c2b740 1 -- start start 2024-03-19T01:48:31.225 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.221+0000 7f8a33c2b740 1 Processor -- start 2024-03-19T01:48:31.225 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.221+0000 7f8a33c2b740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5605e2238100 0x5605e22af0f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:31.225 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.221+0000 7f8a33c2b740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5605e2238af0 0x5605e22af630 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:31.225 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.221+0000 7f8a33c2b740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5605e223dc60 0x5605e22b6670 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:31.225 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.221+0000 7f8a33c2b740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f8a2c1a74c0 con 0x5605e223dc60 2024-03-19T01:48:31.225 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.221+0000 7f8a33c2b740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f8a2c1a8930 con 0x5605e2238100 2024-03-19T01:48:31.225 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.221+0000 7f8a33c2b740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f8a2c1a7b90 con 0x5605e2238af0 2024-03-19T01:48:31.225 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.221+0000 7f8a33c2b740 10 client.0 _ll_register_callbacks cb 0x7f8a1c005e30 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:48:31.225 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.221+0000 7f8a2ad9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5605e223dc60 0x5605e22b6670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:31.225 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.221+0000 7f8a23fff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5605e2238af0 0x5605e22af630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:31.226 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.221+0000 7f8a2a59c640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5605e2238100 0x5605e22af0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:31.226 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.221+0000 7f8a23fff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5605e2238af0 0x5605e22af630 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:36804/0 (socket says 172.21.15.99:36804) 2024-03-19T01:48:31.226 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.221+0000 7f8a23fff640 1 -- 172.21.15.99:0/680293938 learned_addr learned my addr 172.21.15.99:0/680293938 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:48:31.226 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.221+0000 7f8a2ad9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5605e223dc60 0x5605e22b6670 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:56074/0 (socket says 172.21.15.99:56074) 2024-03-19T01:48:31.226 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.221+0000 7f8a23fff640 1 -- 172.21.15.99:0/680293938 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5605e2238100 msgr2=0x5605e22af0f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:48:31.226 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.221+0000 7f8a23fff640 1 --2- 172.21.15.99:0/680293938 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5605e2238100 0x5605e22af0f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:31.226 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.221+0000 7f8a23fff640 1 -- 172.21.15.99:0/680293938 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5605e223dc60 msgr2=0x5605e22b6670 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:48:31.226 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.221+0000 7f8a23fff640 1 --2- 172.21.15.99:0/680293938 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5605e223dc60 0x5605e22b6670 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:31.226 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.221+0000 7f8a23fff640 1 -- 172.21.15.99:0/680293938 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5605e22b6c60 con 0x5605e2238af0 2024-03-19T01:48:31.226 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.221+0000 7f8a2ad9d640 1 --2- 172.21.15.99:0/680293938 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5605e223dc60 0x5605e22b6670 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:48:31.226 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.221+0000 7f8a23fff640 1 --2- 172.21.15.99:0/680293938 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5605e2238af0 0x5605e22af630 secure :-1 s=READY pgs=98 cs=0 l=1 rev1=1 crypto rx=0x7f8a1402cfd0 tx=0x7f8a14004c20 comp rx=0 tx=0).ready entity=mon.1 client_cookie=f07c6cafaf99020b server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:48:31.226 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.221+0000 7f8a11ffb640 10 client.0 ms_handle_connect on v2:172.21.15.115:3300/0 2024-03-19T01:48:31.226 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.222+0000 7f8a11ffb640 1 -- 172.21.15.99:0/680293938 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8a1404d060 con 0x5605e2238af0 2024-03-19T01:48:31.226 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.222+0000 7f8a33c2b740 1 -- 172.21.15.99:0/680293938 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f8a1c09f710 con 0x5605e2238af0 2024-03-19T01:48:31.227 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.222+0000 7f8a11ffb640 1 -- 172.21.15.99:0/680293938 <== mon.1 v2:172.21.15.115:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f8a14004260 con 0x5605e2238af0 2024-03-19T01:48:31.227 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.222+0000 7f8a11ffb640 1 -- 172.21.15.99:0/680293938 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7f8a14004d10 con 0x5605e2238af0 2024-03-19T01:48:31.227 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.222+0000 7f8a33c2b740 10 client.25762 fetch_fsmap learned FSMap version 27 2024-03-19T01:48:31.227 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.222+0000 7f8a33c2b740 1 -- 172.21.15.99:0/680293938 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x5605e22b8010 con 0x5605e2238af0 2024-03-19T01:48:31.227 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.222+0000 7f8a11ffb640 1 -- 172.21.15.99:0/680293938 <== mon.1 v2:172.21.15.115:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f8a1c09f710 con 0x5605e2238af0 2024-03-19T01:48:31.227 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.222+0000 7f8a11ffb640 1 -- 172.21.15.99:0/680293938 <== mon.1 v2:172.21.15.115:3300/0 5 ==== fsmap(e 27) ==== 8338+0+0 (secure 0 0 0) 0x7f8a140039c0 con 0x5605e2238af0 2024-03-19T01:48:31.227 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.222+0000 7f8a33c2b740 10 client.25762 fetch_fsmap finished waiting for FSMap version 27 2024-03-19T01:48:31.227 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.222+0000 7f8a33c2b740 10 client.25762 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:48:31.227 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.222+0000 7f8a33c2b740 20 client.25762 populate_metadata read hostname 'smithi099' 2024-03-19T01:48:31.227 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.222+0000 7f8a33c2b740 1 --2- 172.21.15.99:0/680293938 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5605e224dda0 0x5605e22baa80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:31.227 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.222+0000 7f8a33c2b740 4 client.25762 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:48:31.227 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.222+0000 7f8a33c2b740 1 -- 172.21.15.99:0/680293938 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x5605e21a4470 con 0x5605e224dda0 2024-03-19T01:48:31.227 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.222+0000 7f8a2a59c640 1 --2- 172.21.15.99:0/680293938 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5605e224dda0 0x5605e22baa80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:31.227 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.223+0000 7f8a11ffb640 10 client.25762 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:48:31.227 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.223+0000 7f8a2a59c640 1 --2- 172.21.15.99:0/680293938 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5605e224dda0 0x5605e22baa80 crc :-1 s=READY pgs=96 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=c530f06d0b30a144 server_cookie=301a85fdfa3c6a10 in_seq=0 out_seq=0 2024-03-19T01:48:31.335 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.335+0000 7f8a11ffb640 1 -- 172.21.15.99:0/680293938 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x5605e21a4470 con 0x5605e224dda0 2024-03-19T01:48:31.336 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.335+0000 7f8a11ffb640 10 client.25762 handle_command_reply: tid=0 2024-03-19T01:48:31.336 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.335+0000 7f8a33c2b740 1 client.25762 shutdown 2024-03-19T01:48:31.336 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.335+0000 7f8a33c2b740 1 -- 172.21.15.99:0/680293938 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5605e2238af0 msgr2=0x5605e22af630 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:48:31.336 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.335+0000 7f8a33c2b740 1 --2- 172.21.15.99:0/680293938 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5605e2238af0 0x5605e22af630 secure :-1 s=READY pgs=98 cs=0 l=1 rev1=1 crypto rx=0x7f8a1402cfd0 tx=0x7f8a14004c20 comp rx=0 tx=0).stop 2024-03-19T01:48:31.336 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.335+0000 7f8a33c2b740 1 -- 172.21.15.99:0/680293938 shutdown_connections 2024-03-19T01:48:31.336 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.335+0000 7f8a33c2b740 1 --2- 172.21.15.99:0/680293938 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5605e224dda0 0x5605e22baa80 crc :-1 s=READY pgs=96 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:31.336 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.335+0000 7f8a33c2b740 1 --2- 172.21.15.99:0/680293938 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5605e223dc60 0x5605e22b6670 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:31.336 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.335+0000 7f8a33c2b740 1 --2- 172.21.15.99:0/680293938 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5605e2238af0 0x5605e22af630 unknown :-1 s=CLOSED pgs=98 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:31.336 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.335+0000 7f8a33c2b740 1 --2- 172.21.15.99:0/680293938 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5605e2238100 0x5605e22af0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:31.336 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.335+0000 7f8a11ffb640 0 client.25762 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:48:31.336 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.335+0000 7f8a33c2b740 1 -- 172.21.15.99:0/680293938 >> 172.21.15.99:0/680293938 conn(0x5605e2233670 msgr2=0x5605e2241ee0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:48:31.336 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.335+0000 7f8a33c2b740 1 -- 172.21.15.99:0/680293938 shutdown_connections 2024-03-19T01:48:31.337 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.335+0000 7f8a33c2b740 1 -- 172.21.15.99:0/680293938 wait complete. 2024-03-19T01:48:31.337 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.335+0000 7f8a33c2b740 20 client.25762 trim_cache size 0 max 16384 2024-03-19T01:48:31.337 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:48:31.337 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:48:31.337 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:48:31.337 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:48:31.338 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.336+0000 7f8a30f62640 1 -- 172.21.15.99:0/3866801129 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f8a180a28a0 msgr2=0x7f8a180a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:48:31.338 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.336+0000 7f8a30f62640 1 --2- 172.21.15.99:0/3866801129 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f8a180a28a0 0x7f8a180a4d60 secure :-1 s=READY pgs=288 cs=0 l=1 rev1=1 crypto rx=0x7f8a2400fc90 tx=0x7f8a2401e000 comp rx=0 tx=0).stop 2024-03-19T01:48:31.338 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.336+0000 7f8a30f62640 1 -- 172.21.15.99:0/3866801129 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8a2c101870 msgr2=0x7f8a2c197b10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:48:31.339 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.336+0000 7f8a30f62640 1 --2- 172.21.15.99:0/3866801129 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8a2c101870 0x7f8a2c197b10 secure :-1 s=READY pgs=130 cs=0 l=1 rev1=1 crypto rx=0x7f8a1c00e9c0 tx=0x7f8a1c00ee80 comp rx=0 tx=0).stop 2024-03-19T01:48:31.339 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.336+0000 7f8a30f62640 1 -- 172.21.15.99:0/3866801129 shutdown_connections 2024-03-19T01:48:31.339 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.336+0000 7f8a30f62640 1 --2- 172.21.15.99:0/3866801129 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8a2c103420 0x7f8a2c19f0c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:31.339 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.336+0000 7f8a30f62640 1 --2- 172.21.15.99:0/3866801129 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8a2c102a60 0x7f8a2c198050 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:31.339 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.336+0000 7f8a30f62640 1 --2- 172.21.15.99:0/3866801129 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f8a180a28a0 0x7f8a180a4d60 unknown :-1 s=CLOSED pgs=288 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:31.339 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.336+0000 7f8a30f62640 1 --2- 172.21.15.99:0/3866801129 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8a2c101870 0x7f8a2c197b10 unknown :-1 s=CLOSED pgs=130 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:31.339 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.337+0000 7f8a30f62640 1 -- 172.21.15.99:0/3866801129 >> 172.21.15.99:0/3866801129 conn(0x7f8a2c0fcde0 msgr2=0x7f8a2c105d00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:48:31.339 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.337+0000 7f8a30f62640 1 -- 172.21.15.99:0/3866801129 shutdown_connections 2024-03-19T01:48:31.339 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:31.337+0000 7f8a30f62640 1 -- 172.21.15.99:0/3866801129 wait complete. 2024-03-19T01:48:31.690 INFO:tasks.workunit.client.0.smithi099.stdout: 1 896266 11.37 MB/sec execute 301 sec latency 58.817 ms 2024-03-19T01:48:32.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:48:31 smithi115 ceph-mon[67332]: pgmap v573: 97 pgs: 97 active+clean; 252 MiB data, 2.5 GiB used, 1.0 TiB / 1.0 TiB avail; 887 KiB/s rd, 2.9 MiB/s wr, 266 op/s 2024-03-19T01:48:32.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:48:31 smithi099 ceph-mon[62951]: pgmap v573: 97 pgs: 97 active+clean; 252 MiB data, 2.5 GiB used, 1.0 TiB / 1.0 TiB avail; 887 KiB/s rd, 2.9 MiB/s wr, 266 op/s 2024-03-19T01:48:32.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:48:31 smithi112 ceph-mon[66819]: pgmap v573: 97 pgs: 97 active+clean; 252 MiB data, 2.5 GiB used, 1.0 TiB / 1.0 TiB avail; 887 KiB/s rd, 2.9 MiB/s wr, 266 op/s 2024-03-19T01:48:32.690 INFO:tasks.workunit.client.0.smithi099.stdout: 1 898664 11.38 MB/sec execute 302 sec latency 9.711 ms 2024-03-19T01:48:33.691 INFO:tasks.workunit.client.0.smithi099.stdout: 1 900957 11.38 MB/sec execute 303 sec latency 31.372 ms 2024-03-19T01:48:34.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:48:33 smithi115 ceph-mon[67332]: pgmap v574: 97 pgs: 97 active+clean; 256 MiB data, 2.5 GiB used, 1.0 TiB / 1.0 TiB avail; 937 KiB/s rd, 3.1 MiB/s wr, 280 op/s 2024-03-19T01:48:34.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:48:33 smithi099 ceph-mon[62951]: pgmap v574: 97 pgs: 97 active+clean; 256 MiB data, 2.5 GiB used, 1.0 TiB / 1.0 TiB avail; 937 KiB/s rd, 3.1 MiB/s wr, 280 op/s 2024-03-19T01:48:34.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:48:33 smithi112 ceph-mon[66819]: pgmap v574: 97 pgs: 97 active+clean; 256 MiB data, 2.5 GiB used, 1.0 TiB / 1.0 TiB avail; 937 KiB/s rd, 3.1 MiB/s wr, 280 op/s 2024-03-19T01:48:34.691 INFO:tasks.workunit.client.0.smithi099.stdout: 1 902808 11.38 MB/sec execute 304 sec latency 28.876 ms 2024-03-19T01:48:35.691 INFO:tasks.workunit.client.0.smithi099.stdout: 1 905502 11.39 MB/sec execute 305 sec latency 28.231 ms 2024-03-19T01:48:36.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:48:35 smithi115 ceph-mon[67332]: pgmap v575: 97 pgs: 97 active+clean; 256 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 726 KiB/s rd, 2.7 MiB/s wr, 235 op/s 2024-03-19T01:48:36.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:48:35 smithi099 ceph-mon[62951]: pgmap v575: 97 pgs: 97 active+clean; 256 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 726 KiB/s rd, 2.7 MiB/s wr, 235 op/s 2024-03-19T01:48:36.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:48:35 smithi112 ceph-mon[66819]: pgmap v575: 97 pgs: 97 active+clean; 256 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 726 KiB/s rd, 2.7 MiB/s wr, 235 op/s 2024-03-19T01:48:36.691 INFO:tasks.workunit.client.0.smithi099.stdout: 1 906738 11.37 MB/sec execute 306 sec latency 86.759 ms 2024-03-19T01:48:37.691 INFO:tasks.workunit.client.0.smithi099.stdout: 1 909143 11.38 MB/sec execute 307 sec latency 24.707 ms 2024-03-19T01:48:38.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:48:37 smithi115 ceph-mon[67332]: pgmap v576: 97 pgs: 97 active+clean; 258 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 1011 KiB/s rd, 3.3 MiB/s wr, 307 op/s 2024-03-19T01:48:38.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:48:37 smithi099 ceph-mon[62951]: pgmap v576: 97 pgs: 97 active+clean; 258 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 1011 KiB/s rd, 3.3 MiB/s wr, 307 op/s 2024-03-19T01:48:38.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:48:37 smithi112 ceph-mon[66819]: pgmap v576: 97 pgs: 97 active+clean; 258 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 1011 KiB/s rd, 3.3 MiB/s wr, 307 op/s 2024-03-19T01:48:38.691 INFO:tasks.workunit.client.0.smithi099.stdout: 1 911228 11.37 MB/sec execute 308 sec latency 28.195 ms 2024-03-19T01:48:39.691 INFO:tasks.workunit.client.0.smithi099.stdout: 1 913098 11.37 MB/sec execute 309 sec latency 17.518 ms 2024-03-19T01:48:40.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:48:39 smithi115 ceph-mon[67332]: pgmap v577: 97 pgs: 97 active+clean; 257 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 930 KiB/s rd, 2.9 MiB/s wr, 287 op/s 2024-03-19T01:48:40.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:48:39 smithi099 ceph-mon[62951]: pgmap v577: 97 pgs: 97 active+clean; 257 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 930 KiB/s rd, 2.9 MiB/s wr, 287 op/s 2024-03-19T01:48:40.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:48:39 smithi112 ceph-mon[66819]: pgmap v577: 97 pgs: 97 active+clean; 257 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 930 KiB/s rd, 2.9 MiB/s wr, 287 op/s 2024-03-19T01:48:40.691 INFO:tasks.workunit.client.0.smithi099.stdout: 1 915744 11.38 MB/sec execute 310 sec latency 23.934 ms 2024-03-19T01:48:41.691 INFO:tasks.workunit.client.0.smithi099.stdout: 1 916759 11.37 MB/sec execute 311 sec latency 62.361 ms 2024-03-19T01:48:42.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:48:41 smithi115 ceph-mon[67332]: pgmap v578: 97 pgs: 97 active+clean; 258 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 712 KiB/s rd, 2.7 MiB/s wr, 247 op/s 2024-03-19T01:48:42.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:48:41 smithi099 ceph-mon[62951]: pgmap v578: 97 pgs: 97 active+clean; 258 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 712 KiB/s rd, 2.7 MiB/s wr, 247 op/s 2024-03-19T01:48:42.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:48:41 smithi112 ceph-mon[66819]: pgmap v578: 97 pgs: 97 active+clean; 258 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 712 KiB/s rd, 2.7 MiB/s wr, 247 op/s 2024-03-19T01:48:42.691 INFO:tasks.workunit.client.0.smithi099.stdout: 1 919022 11.41 MB/sec execute 312 sec latency 15.575 ms 2024-03-19T01:48:43.691 INFO:tasks.workunit.client.0.smithi099.stdout: 1 920914 11.42 MB/sec execute 313 sec latency 64.697 ms 2024-03-19T01:48:44.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:48:43 smithi115 ceph-mon[67332]: pgmap v579: 97 pgs: 97 active+clean; 259 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 905 KiB/s rd, 3.2 MiB/s wr, 278 op/s 2024-03-19T01:48:44.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:48:43 smithi099 ceph-mon[62951]: pgmap v579: 97 pgs: 97 active+clean; 259 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 905 KiB/s rd, 3.2 MiB/s wr, 278 op/s 2024-03-19T01:48:44.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:48:43 smithi112 ceph-mon[66819]: pgmap v579: 97 pgs: 97 active+clean; 259 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 905 KiB/s rd, 3.2 MiB/s wr, 278 op/s 2024-03-19T01:48:44.432 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.429+0000 7ff643c12640 1 -- 172.21.15.99:0/3966998404 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff63c100740 msgr2=0x7ff63c100b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:48:44.432 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.429+0000 7ff643c12640 1 --2- 172.21.15.99:0/3966998404 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff63c100740 0x7ff63c100b60 secure :-1 s=READY pgs=492 cs=0 l=1 rev1=1 crypto rx=0x7ff638009900 tx=0x7ff63802bc30 comp rx=0 tx=0).stop 2024-03-19T01:48:44.432 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.430+0000 7ff643c12640 1 -- 172.21.15.99:0/3966998404 shutdown_connections 2024-03-19T01:48:44.432 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.430+0000 7ff643c12640 1 --2- 172.21.15.99:0/3966998404 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff63c1022f0 0x7ff63c108b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:44.432 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.430+0000 7ff643c12640 1 --2- 172.21.15.99:0/3966998404 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff63c101930 0x7ff63c101db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:44.432 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.430+0000 7ff643c12640 1 --2- 172.21.15.99:0/3966998404 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff63c100740 0x7ff63c100b60 unknown :-1 s=CLOSED pgs=492 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:44.432 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.430+0000 7ff643c12640 1 -- 172.21.15.99:0/3966998404 >> 172.21.15.99:0/3966998404 conn(0x7ff63c0fbcb0 msgr2=0x7ff63c0fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:48:44.432 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.430+0000 7ff643c12640 1 -- 172.21.15.99:0/3966998404 shutdown_connections 2024-03-19T01:48:44.432 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.430+0000 7ff643c12640 1 -- 172.21.15.99:0/3966998404 wait complete. 2024-03-19T01:48:44.432 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.430+0000 7ff643c12640 1 Processor -- start 2024-03-19T01:48:44.433 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.430+0000 7ff643c12640 1 -- start start 2024-03-19T01:48:44.433 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.430+0000 7ff643c12640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff63c100740 0x7ff63c069a10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:44.433 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.430+0000 7ff643c12640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff63c101930 0x7ff63c19c850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:44.433 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.430+0000 7ff643c12640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff63c1022f0 0x7ff63c19ee70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:44.433 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.430+0000 7ff643c12640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7ff63c10c390 con 0x7ff63c1022f0 2024-03-19T01:48:44.433 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.430+0000 7ff643c12640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7ff63c10c210 con 0x7ff63c101930 2024-03-19T01:48:44.433 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.430+0000 7ff643c12640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7ff63c10c510 con 0x7ff63c100740 2024-03-19T01:48:44.433 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.430+0000 7ff6421af640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff63c1022f0 0x7ff63c19ee70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:44.433 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.430+0000 7ff6421af640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff63c1022f0 0x7ff63c19ee70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:53144/0 (socket says 172.21.15.99:53144) 2024-03-19T01:48:44.433 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.430+0000 7ff6421af640 1 -- 172.21.15.99:0/4010474583 learned_addr learned my addr 172.21.15.99:0/4010474583 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:48:44.433 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.430+0000 7ff6411ad640 1 --2- 172.21.15.99:0/4010474583 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff63c101930 0x7ff63c19c850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:44.433 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.430+0000 7ff6419ae640 1 --2- 172.21.15.99:0/4010474583 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff63c100740 0x7ff63c069a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:44.433 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.431+0000 7ff6421af640 1 -- 172.21.15.99:0/4010474583 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff63c100740 msgr2=0x7ff63c069a10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:48:44.434 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.431+0000 7ff6421af640 1 --2- 172.21.15.99:0/4010474583 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff63c100740 0x7ff63c069a10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:44.434 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.431+0000 7ff6421af640 1 -- 172.21.15.99:0/4010474583 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff63c101930 msgr2=0x7ff63c19c850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:48:44.434 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.431+0000 7ff6421af640 1 --2- 172.21.15.99:0/4010474583 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff63c101930 0x7ff63c19c850 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:44.434 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.431+0000 7ff6421af640 1 -- 172.21.15.99:0/4010474583 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ff63c19f780 con 0x7ff63c1022f0 2024-03-19T01:48:44.434 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.431+0000 7ff6421af640 1 --2- 172.21.15.99:0/4010474583 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff63c1022f0 0x7ff63c19ee70 secure :-1 s=READY pgs=493 cs=0 l=1 rev1=1 crypto rx=0x7ff63400eb10 tx=0x7ff63400c2d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:48:44.434 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.431+0000 7ff632ffd640 1 -- 172.21.15.99:0/4010474583 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff634004b20 con 0x7ff63c1022f0 2024-03-19T01:48:44.434 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.431+0000 7ff643c12640 1 -- 172.21.15.99:0/4010474583 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ff63c19fa10 con 0x7ff63c1022f0 2024-03-19T01:48:44.434 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.431+0000 7ff632ffd640 1 -- 172.21.15.99:0/4010474583 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7ff634004cc0 con 0x7ff63c1022f0 2024-03-19T01:48:44.434 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.431+0000 7ff632ffd640 1 -- 172.21.15.99:0/4010474583 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff634010760 con 0x7ff63c1022f0 2024-03-19T01:48:44.434 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.431+0000 7ff643c12640 1 -- 172.21.15.99:0/4010474583 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7ff63c19ff60 con 0x7ff63c1022f0 2024-03-19T01:48:44.441 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.433+0000 7ff632ffd640 1 -- 172.21.15.99:0/4010474583 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7ff634005020 con 0x7ff63c1022f0 2024-03-19T01:48:44.441 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.434+0000 7ff646803740 1 Processor -- start 2024-03-19T01:48:44.441 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.434+0000 7ff646803740 1 -- start start 2024-03-19T01:48:44.441 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.434+0000 7ff632ffd640 1 --2- 172.21.15.99:0/4010474583 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff6280a28a0 0x7ff6280a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:44.441 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.434+0000 7ff646803740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561c24fa6100 0x561c24fa6520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:44.441 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.434+0000 7ff646803740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561c24fa6af0 0x561c24fab5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:44.441 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.434+0000 7ff646803740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561c24fabc60 0x561c24fae0f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:44.442 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.434+0000 7ff646803740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x561c24f0abe0 con 0x561c24fa6100 2024-03-19T01:48:44.442 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.434+0000 7ff646803740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x561c24eb3960 con 0x561c24fa6af0 2024-03-19T01:48:44.442 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.434+0000 7ff646803740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x561c24f06c80 con 0x561c24fabc60 2024-03-19T01:48:44.442 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.434+0000 7ff6411ad640 1 --2- 172.21.15.99:0/4010474583 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff6280a28a0 0x7ff6280a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:44.442 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.434+0000 7ff632ffd640 1 -- 172.21.15.99:0/4010474583 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7ff6340cf010 con 0x7ff63c1022f0 2024-03-19T01:48:44.442 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.434+0000 7ff6411ad640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561c24fa6af0 0x561c24fab5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:44.442 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.434+0000 7ff6419ae640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561c24fa6100 0x561c24fa6520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:44.442 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.434+0000 7ff6419ae640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561c24fa6100 0x561c24fa6520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:53154/0 (socket says 172.21.15.99:53154) 2024-03-19T01:48:44.442 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.434+0000 7ff6419ae640 1 -- 172.21.15.99:0/3309683376 learned_addr learned my addr 172.21.15.99:0/3309683376 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:48:44.442 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.434+0000 7ff6421af640 1 --2- 172.21.15.99:0/3309683376 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561c24fabc60 0x561c24fae0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:44.442 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.435+0000 7ff6419ae640 1 -- 172.21.15.99:0/3309683376 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561c24fabc60 msgr2=0x561c24fae0f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:48:44.442 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.435+0000 7ff6419ae640 1 --2- 172.21.15.99:0/3309683376 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561c24fabc60 0x561c24fae0f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:44.442 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.435+0000 7ff6419ae640 1 -- 172.21.15.99:0/3309683376 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561c24fa6af0 msgr2=0x561c24fab5f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:48:44.443 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.435+0000 7ff6419ae640 1 --2- 172.21.15.99:0/3309683376 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561c24fa6af0 0x561c24fab5f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:44.443 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.435+0000 7ff6419ae640 1 -- 172.21.15.99:0/3309683376 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x561c24eaad20 con 0x561c24fa6100 2024-03-19T01:48:44.443 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.435+0000 7ff6411ad640 1 --2- 172.21.15.99:0/3309683376 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561c24fa6af0 0x561c24fab5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:48:44.443 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.435+0000 7ff6411ad640 1 --2- 172.21.15.99:0/4010474583 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff6280a28a0 0x7ff6280a4d60 secure :-1 s=READY pgs=290 cs=0 l=1 rev1=1 crypto rx=0x7ff62c009fd0 tx=0x7ff62c013460 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:48:44.443 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.435+0000 7ff6419ae640 1 --2- 172.21.15.99:0/3309683376 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561c24fa6100 0x561c24fa6520 secure :-1 s=READY pgs=494 cs=0 l=1 rev1=1 crypto rx=0x7ff638009900 tx=0x7ff638009560 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e63fc64330e756ae server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:48:44.443 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.435+0000 7ff627fff640 1 -- 172.21.15.99:0/3309683376 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff63802cb80 con 0x561c24fa6100 2024-03-19T01:48:44.443 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.435+0000 7ff627fff640 1 -- 172.21.15.99:0/3309683376 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7ff638005040 con 0x561c24fa6100 2024-03-19T01:48:44.443 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.435+0000 7ff627fff640 1 -- 172.21.15.99:0/3309683376 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff638004920 con 0x561c24fa6100 2024-03-19T01:48:44.443 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.436+0000 7ff646803740 1 -- 172.21.15.99:0/3309683376 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561c24fa6100 msgr2=0x561c24fa6520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:48:44.443 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.436+0000 7ff646803740 1 --2- 172.21.15.99:0/3309683376 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561c24fa6100 0x561c24fa6520 secure :-1 s=READY pgs=494 cs=0 l=1 rev1=1 crypto rx=0x7ff638009900 tx=0x7ff638009560 comp rx=0 tx=0).stop 2024-03-19T01:48:44.443 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.436+0000 7ff646803740 1 -- 172.21.15.99:0/3309683376 shutdown_connections 2024-03-19T01:48:44.443 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.436+0000 7ff646803740 1 --2- 172.21.15.99:0/3309683376 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561c24fa6af0 0x561c24fab5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:44.443 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.436+0000 7ff646803740 1 --2- 172.21.15.99:0/3309683376 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561c24fabc60 0x561c24fae0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:44.443 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.436+0000 7ff646803740 1 --2- 172.21.15.99:0/3309683376 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561c24fa6100 0x561c24fa6520 unknown :-1 s=CLOSED pgs=494 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:44.444 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.436+0000 7ff646803740 1 -- 172.21.15.99:0/3309683376 >> 172.21.15.99:0/3309683376 conn(0x561c24fa1670 msgr2=0x561c24fa1a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:48:44.444 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.436+0000 7ff646803740 1 -- 172.21.15.99:0/3309683376 shutdown_connections 2024-03-19T01:48:44.444 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.436+0000 7ff646803740 1 -- 172.21.15.99:0/3309683376 wait complete. 2024-03-19T01:48:44.444 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.436+0000 7ff646803740 1 -- start start 2024-03-19T01:48:44.444 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.436+0000 7ff646803740 1 Processor -- start 2024-03-19T01:48:44.444 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.437+0000 7ff646803740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561c24fa6100 0x561c24fb00b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:44.444 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.437+0000 7ff646803740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561c24fa6af0 0x561c24fb05f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:44.444 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.437+0000 7ff646803740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561c24fabc60 0x561c24fb7630 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:44.444 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.437+0000 7ff646803740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x561c24faebb0 con 0x561c24fa6100 2024-03-19T01:48:44.444 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.437+0000 7ff646803740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x561c24ec6db0 con 0x561c24fa6af0 2024-03-19T01:48:44.444 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.437+0000 7ff646803740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x561c24faed30 con 0x561c24fabc60 2024-03-19T01:48:44.444 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.437+0000 7ff6419ae640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561c24fa6100 0x561c24fb00b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:44.444 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.437+0000 7ff646803740 10 client.0 _ll_register_callbacks cb 0x561c24ecbe70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:48:44.445 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.437+0000 7ff6419ae640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561c24fa6100 0x561c24fb00b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:53164/0 (socket says 172.21.15.99:53164) 2024-03-19T01:48:44.445 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.437+0000 7ff6419ae640 1 -- 172.21.15.99:0/3524413074 learned_addr learned my addr 172.21.15.99:0/3524413074 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:48:44.445 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.437+0000 7ff6421af640 1 --2- 172.21.15.99:0/3524413074 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561c24fabc60 0x561c24fb7630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:44.445 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.437+0000 7ff6411ad640 1 --2- 172.21.15.99:0/3524413074 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561c24fa6af0 0x561c24fb05f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:44.445 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.437+0000 7ff6419ae640 1 -- 172.21.15.99:0/3524413074 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561c24fabc60 msgr2=0x561c24fb7630 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:48:44.445 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.437+0000 7ff6419ae640 1 --2- 172.21.15.99:0/3524413074 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561c24fabc60 0x561c24fb7630 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:44.445 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.437+0000 7ff6419ae640 1 -- 172.21.15.99:0/3524413074 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561c24fa6af0 msgr2=0x561c24fb05f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:48:44.445 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.437+0000 7ff6419ae640 1 --2- 172.21.15.99:0/3524413074 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561c24fa6af0 0x561c24fb05f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:44.445 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.437+0000 7ff6419ae640 1 -- 172.21.15.99:0/3524413074 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x561c24ec5c60 con 0x561c24fa6100 2024-03-19T01:48:44.445 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.437+0000 7ff6419ae640 1 --2- 172.21.15.99:0/3524413074 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561c24fa6100 0x561c24fb00b0 secure :-1 s=READY pgs=495 cs=0 l=1 rev1=1 crypto rx=0x7ff638036040 tx=0x7ff6380054a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=768a1baaf0178a9 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:48:44.445 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.437+0000 7ff6257fa640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:48:44.445 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.437+0000 7ff6257fa640 1 -- 172.21.15.99:0/3524413074 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff638004ad0 con 0x561c24fa6100 2024-03-19T01:48:44.445 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.437+0000 7ff6257fa640 1 -- 172.21.15.99:0/3524413074 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7ff638004c70 con 0x561c24fa6100 2024-03-19T01:48:44.445 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.437+0000 7ff646803740 1 -- 172.21.15.99:0/3524413074 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7ff638005040 con 0x561c24fa6100 2024-03-19T01:48:44.446 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.437+0000 7ff6257fa640 1 -- 172.21.15.99:0/3524413074 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7ff638004e70 con 0x561c24fa6100 2024-03-19T01:48:44.446 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.437+0000 7ff646803740 10 client.15954 fetch_fsmap learned FSMap version 27 2024-03-19T01:48:44.446 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.437+0000 7ff646803740 1 -- 172.21.15.99:0/3524413074 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x561c24c1f210 con 0x561c24fa6100 2024-03-19T01:48:44.446 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.438+0000 7ff6257fa640 1 -- 172.21.15.99:0/3524413074 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7ff638005040 con 0x561c24fa6100 2024-03-19T01:48:44.446 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.438+0000 7ff6257fa640 1 -- 172.21.15.99:0/3524413074 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 27) ==== 8338+0+0 (secure 0 0 0) 0x7ff638045420 con 0x561c24fa6100 2024-03-19T01:48:44.446 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.438+0000 7ff646803740 10 client.15954 fetch_fsmap finished waiting for FSMap version 27 2024-03-19T01:48:44.446 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.438+0000 7ff646803740 10 client.15954 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:48:44.446 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.438+0000 7ff646803740 20 client.15954 populate_metadata read hostname 'smithi099' 2024-03-19T01:48:44.446 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.438+0000 7ff646803740 1 --2- 172.21.15.99:0/3524413074 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x561c24fbbb20 0x561c24fbdf10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:44.446 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.438+0000 7ff646803740 4 client.15954 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:48:44.446 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.438+0000 7ff646803740 1 -- 172.21.15.99:0/3524413074 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x561c24f18120 con 0x561c24fbbb20 2024-03-19T01:48:44.446 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.438+0000 7ff6411ad640 1 --2- 172.21.15.99:0/3524413074 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x561c24fbbb20 0x561c24fbdf10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:44.446 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.438+0000 7ff6257fa640 10 client.15954 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:48:44.446 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.438+0000 7ff6411ad640 1 --2- 172.21.15.99:0/3524413074 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x561c24fbbb20 0x561c24fbdf10 crc :-1 s=READY pgs=97 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=626979ebebe26dc3 server_cookie=2624d4865f66ab28 in_seq=0 out_seq=0 2024-03-19T01:48:44.447 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.439+0000 7ff6257fa640 1 -- 172.21.15.99:0/3524413074 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x561c24f18120 con 0x561c24fbbb20 2024-03-19T01:48:44.447 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.439+0000 7ff6257fa640 10 client.15954 handle_command_reply: tid=0 2024-03-19T01:48:44.447 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.439+0000 7ff646803740 1 client.15954 shutdown 2024-03-19T01:48:44.447 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.439+0000 7ff646803740 1 -- 172.21.15.99:0/3524413074 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561c24fa6100 msgr2=0x561c24fb00b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:48:44.447 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.439+0000 7ff646803740 1 --2- 172.21.15.99:0/3524413074 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561c24fa6100 0x561c24fb00b0 secure :-1 s=READY pgs=495 cs=0 l=1 rev1=1 crypto rx=0x7ff638036040 tx=0x7ff6380054a0 comp rx=0 tx=0).stop 2024-03-19T01:48:44.447 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.439+0000 7ff646803740 1 -- 172.21.15.99:0/3524413074 shutdown_connections 2024-03-19T01:48:44.447 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.439+0000 7ff646803740 1 --2- 172.21.15.99:0/3524413074 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x561c24fbbb20 0x561c24fbdf10 crc :-1 s=READY pgs=97 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:44.447 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.439+0000 7ff646803740 1 --2- 172.21.15.99:0/3524413074 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561c24fa6af0 0x561c24fb05f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:44.447 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.439+0000 7ff6257fa640 0 client.15954 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:48:44.447 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.439+0000 7ff646803740 1 --2- 172.21.15.99:0/3524413074 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561c24fabc60 0x561c24fb7630 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:44.447 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.439+0000 7ff646803740 1 --2- 172.21.15.99:0/3524413074 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561c24fa6100 0x561c24fb00b0 unknown :-1 s=CLOSED pgs=495 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:44.447 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.439+0000 7ff646803740 1 -- 172.21.15.99:0/3524413074 >> 172.21.15.99:0/3524413074 conn(0x561c24fa1670 msgr2=0x561c24fa28c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:48:44.447 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.439+0000 7ff646803740 1 -- 172.21.15.99:0/3524413074 shutdown_connections 2024-03-19T01:48:44.448 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.439+0000 7ff646803740 1 -- 172.21.15.99:0/3524413074 wait complete. 2024-03-19T01:48:44.448 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.439+0000 7ff646803740 20 client.15954 trim_cache size 0 max 16384 2024-03-19T01:48:44.460 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.455+0000 7ff646803740 1 Processor -- start 2024-03-19T01:48:44.460 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.455+0000 7ff646803740 1 -- start start 2024-03-19T01:48:44.460 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.455+0000 7ff646803740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561c24fa6100 0x561c24fa4fe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:44.460 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.455+0000 7ff646803740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561c24fa6af0 0x561c24fb7550 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:44.460 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.455+0000 7ff646803740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561c24fabc60 0x561c24faf6d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:44.460 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.455+0000 7ff646803740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7ff63c1ab620 con 0x561c24fa6af0 2024-03-19T01:48:44.461 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.455+0000 7ff646803740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7ff63c1a8ef0 con 0x561c24fa6100 2024-03-19T01:48:44.461 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.455+0000 7ff646803740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7ff63c1a6d40 con 0x561c24fabc60 2024-03-19T01:48:44.461 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.455+0000 7ff6411ad640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561c24fa6af0 0x561c24fb7550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:44.461 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.455+0000 7ff6411ad640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561c24fa6af0 0x561c24fb7550 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:53180/0 (socket says 172.21.15.99:53180) 2024-03-19T01:48:44.461 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.455+0000 7ff6411ad640 1 -- 172.21.15.99:0/2707748435 learned_addr learned my addr 172.21.15.99:0/2707748435 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:48:44.461 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.455+0000 7ff6419ae640 1 --2- 172.21.15.99:0/2707748435 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561c24fa6100 0x561c24fa4fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:44.461 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.455+0000 7ff6421af640 1 --2- 172.21.15.99:0/2707748435 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561c24fabc60 0x561c24faf6d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:44.461 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.456+0000 7ff6411ad640 1 -- 172.21.15.99:0/2707748435 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561c24fabc60 msgr2=0x561c24faf6d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:48:44.461 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.456+0000 7ff6411ad640 1 --2- 172.21.15.99:0/2707748435 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561c24fabc60 0x561c24faf6d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:44.461 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.456+0000 7ff6411ad640 1 -- 172.21.15.99:0/2707748435 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561c24fa6100 msgr2=0x561c24fa4fe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:48:44.461 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.456+0000 7ff6411ad640 1 --2- 172.21.15.99:0/2707748435 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561c24fa6100 0x561c24fa4fe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:44.461 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.456+0000 7ff6411ad640 1 -- 172.21.15.99:0/2707748435 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x561c24ceb620 con 0x561c24fa6af0 2024-03-19T01:48:44.461 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.456+0000 7ff6411ad640 1 --2- 172.21.15.99:0/2707748435 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561c24fa6af0 0x561c24fb7550 secure :-1 s=READY pgs=496 cs=0 l=1 rev1=1 crypto rx=0x7ff62c013670 tx=0x7ff62c005f90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ad5c45bdd45f21f0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:48:44.462 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.456+0000 7ff6277fe640 1 -- 172.21.15.99:0/2707748435 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff62c00fd60 con 0x561c24fa6af0 2024-03-19T01:48:44.462 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.456+0000 7ff6277fe640 1 -- 172.21.15.99:0/2707748435 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7ff62c010de0 con 0x561c24fa6af0 2024-03-19T01:48:44.462 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.456+0000 7ff6277fe640 1 -- 172.21.15.99:0/2707748435 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff62c02a6c0 con 0x561c24fa6af0 2024-03-19T01:48:44.462 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.456+0000 7ff646803740 1 -- 172.21.15.99:0/2707748435 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561c24fa6af0 msgr2=0x561c24fb7550 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:48:44.462 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.456+0000 7ff646803740 1 --2- 172.21.15.99:0/2707748435 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561c24fa6af0 0x561c24fb7550 secure :-1 s=READY pgs=496 cs=0 l=1 rev1=1 crypto rx=0x7ff62c013670 tx=0x7ff62c005f90 comp rx=0 tx=0).stop 2024-03-19T01:48:44.462 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.457+0000 7ff646803740 1 -- 172.21.15.99:0/2707748435 shutdown_connections 2024-03-19T01:48:44.462 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.457+0000 7ff646803740 1 --2- 172.21.15.99:0/2707748435 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561c24fabc60 0x561c24faf6d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:44.462 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.457+0000 7ff646803740 1 --2- 172.21.15.99:0/2707748435 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561c24fa6af0 0x561c24fb7550 unknown :-1 s=CLOSED pgs=496 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:44.462 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.457+0000 7ff646803740 1 --2- 172.21.15.99:0/2707748435 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561c24fa6100 0x561c24fa4fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:44.462 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.457+0000 7ff646803740 1 -- 172.21.15.99:0/2707748435 >> 172.21.15.99:0/2707748435 conn(0x561c24fa1670 msgr2=0x561c24fa48d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:48:44.462 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.457+0000 7ff646803740 1 -- 172.21.15.99:0/2707748435 shutdown_connections 2024-03-19T01:48:44.462 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.457+0000 7ff646803740 1 -- 172.21.15.99:0/2707748435 wait complete. 2024-03-19T01:48:44.462 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.457+0000 7ff646803740 1 -- start start 2024-03-19T01:48:44.462 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.457+0000 7ff646803740 1 Processor -- start 2024-03-19T01:48:44.463 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.457+0000 7ff646803740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561c24fa6100 0x561c24fb4aa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:44.463 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.457+0000 7ff646803740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561c24fa6af0 0x561c24fb4fe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:44.463 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.457+0000 7ff646803740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561c24fabc60 0x561c24fb5f80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:44.463 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.457+0000 7ff646803740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7ff63c1a73b0 con 0x561c24fabc60 2024-03-19T01:48:44.463 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.457+0000 7ff646803740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7ff63c1a8820 con 0x561c24fa6100 2024-03-19T01:48:44.463 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.457+0000 7ff646803740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7ff63c1a7a80 con 0x561c24fa6af0 2024-03-19T01:48:44.463 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.457+0000 7ff646803740 10 client.0 _ll_register_callbacks cb 0x7ff62c009ea0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:48:44.463 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.457+0000 7ff6421af640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561c24fabc60 0x561c24fb5f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:44.463 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.457+0000 7ff6421af640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561c24fabc60 0x561c24fb5f80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:53194/0 (socket says 172.21.15.99:53194) 2024-03-19T01:48:44.463 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.457+0000 7ff6421af640 1 -- 172.21.15.99:0/2876797963 learned_addr learned my addr 172.21.15.99:0/2876797963 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:48:44.463 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.457+0000 7ff6411ad640 1 --2- 172.21.15.99:0/2876797963 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561c24fa6af0 0x561c24fb4fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:44.463 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.457+0000 7ff6419ae640 1 --2- 172.21.15.99:0/2876797963 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561c24fa6100 0x561c24fb4aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:44.463 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.458+0000 7ff6421af640 1 -- 172.21.15.99:0/2876797963 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561c24fa6af0 msgr2=0x561c24fb4fe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:48:44.464 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.458+0000 7ff6421af640 1 --2- 172.21.15.99:0/2876797963 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561c24fa6af0 0x561c24fb4fe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:44.464 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.458+0000 7ff6421af640 1 -- 172.21.15.99:0/2876797963 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561c24fa6100 msgr2=0x561c24fb4aa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:48:44.464 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.458+0000 7ff6421af640 1 --2- 172.21.15.99:0/2876797963 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561c24fa6100 0x561c24fb4aa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:44.464 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.458+0000 7ff6421af640 1 -- 172.21.15.99:0/2876797963 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x561c24fb6570 con 0x561c24fabc60 2024-03-19T01:48:44.464 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.458+0000 7ff6421af640 1 --2- 172.21.15.99:0/2876797963 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561c24fabc60 0x561c24fb5f80 secure :-1 s=READY pgs=497 cs=0 l=1 rev1=1 crypto rx=0x7ff634097dd0 tx=0x7ff634086040 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d9d639d49787e4dc server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:48:44.464 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.458+0000 7ff6257fa640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:48:44.464 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.458+0000 7ff6257fa640 1 -- 172.21.15.99:0/2876797963 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff6340b9060 con 0x561c24fabc60 2024-03-19T01:48:44.464 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.458+0000 7ff6257fa640 1 -- 172.21.15.99:0/2876797963 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7ff6340a3e80 con 0x561c24fabc60 2024-03-19T01:48:44.464 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.458+0000 7ff646803740 1 -- 172.21.15.99:0/2876797963 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7ff62c010de0 con 0x561c24fabc60 2024-03-19T01:48:44.464 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.458+0000 7ff6257fa640 1 -- 172.21.15.99:0/2876797963 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7ff6340ba250 con 0x561c24fabc60 2024-03-19T01:48:44.464 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.458+0000 7ff646803740 10 client.15960 fetch_fsmap learned FSMap version 27 2024-03-19T01:48:44.464 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.458+0000 7ff646803740 1 -- 172.21.15.99:0/2876797963 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x561c25021e10 con 0x561c24fabc60 2024-03-19T01:48:44.464 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.458+0000 7ff6257fa640 1 -- 172.21.15.99:0/2876797963 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7ff62c010de0 con 0x561c24fabc60 2024-03-19T01:48:44.464 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.458+0000 7ff6257fa640 1 -- 172.21.15.99:0/2876797963 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 27) ==== 8338+0+0 (secure 0 0 0) 0x7ff6340860b0 con 0x561c24fabc60 2024-03-19T01:48:44.464 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.458+0000 7ff646803740 10 client.15960 fetch_fsmap finished waiting for FSMap version 27 2024-03-19T01:48:44.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.458+0000 7ff646803740 10 client.15960 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:48:44.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.458+0000 7ff646803740 20 client.15960 populate_metadata read hostname 'smithi099' 2024-03-19T01:48:44.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.458+0000 7ff646803740 1 --2- 172.21.15.99:0/2876797963 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x561c25022620 0x561c25024a40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:44.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.458+0000 7ff646803740 4 client.15960 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:48:44.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.458+0000 7ff646803740 1 -- 172.21.15.99:0/2876797963 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x561c24f12470 con 0x561c25022620 2024-03-19T01:48:44.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.459+0000 7ff6419ae640 1 --2- 172.21.15.99:0/2876797963 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x561c25022620 0x561c25024a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:44.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.459+0000 7ff6419ae640 1 --2- 172.21.15.99:0/2876797963 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x561c25022620 0x561c25024a40 crc :-1 s=READY pgs=98 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=fa106bbaafe3a7b2 server_cookie=bd5635f8d88cd461 in_seq=0 out_seq=0 2024-03-19T01:48:44.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.459+0000 7ff6257fa640 10 client.15960 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:48:44.517 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.517+0000 7ff6257fa640 1 -- 172.21.15.99:0/2876797963 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x561c24f12470 con 0x561c25022620 2024-03-19T01:48:44.518 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.517+0000 7ff6257fa640 10 client.15960 handle_command_reply: tid=0 2024-03-19T01:48:44.518 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.517+0000 7ff646803740 1 client.15960 shutdown 2024-03-19T01:48:44.518 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.517+0000 7ff646803740 1 -- 172.21.15.99:0/2876797963 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561c24fabc60 msgr2=0x561c24fb5f80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:48:44.518 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.517+0000 7ff646803740 1 --2- 172.21.15.99:0/2876797963 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561c24fabc60 0x561c24fb5f80 secure :-1 s=READY pgs=497 cs=0 l=1 rev1=1 crypto rx=0x7ff634097dd0 tx=0x7ff634086040 comp rx=0 tx=0).stop 2024-03-19T01:48:44.518 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.517+0000 7ff646803740 1 -- 172.21.15.99:0/2876797963 shutdown_connections 2024-03-19T01:48:44.518 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.517+0000 7ff646803740 1 --2- 172.21.15.99:0/2876797963 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x561c25022620 0x561c25024a40 crc :-1 s=READY pgs=98 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:44.518 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.517+0000 7ff6257fa640 0 client.15960 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:48:44.518 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.517+0000 7ff646803740 1 --2- 172.21.15.99:0/2876797963 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561c24fabc60 0x561c24fb5f80 unknown :-1 s=CLOSED pgs=497 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:44.518 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.517+0000 7ff646803740 1 --2- 172.21.15.99:0/2876797963 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561c24fa6af0 0x561c24fb4fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:44.518 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.517+0000 7ff646803740 1 --2- 172.21.15.99:0/2876797963 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561c24fa6100 0x561c24fb4aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:44.518 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.517+0000 7ff646803740 1 -- 172.21.15.99:0/2876797963 >> 172.21.15.99:0/2876797963 conn(0x561c24fa1670 msgr2=0x561c24fa5160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:48:44.518 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.517+0000 7ff646803740 1 -- 172.21.15.99:0/2876797963 shutdown_connections 2024-03-19T01:48:44.519 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.517+0000 7ff646803740 1 -- 172.21.15.99:0/2876797963 wait complete. 2024-03-19T01:48:44.519 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.517+0000 7ff646803740 20 client.15960 trim_cache size 0 max 16384 2024-03-19T01:48:44.519 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:48:44.519 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:48:44.519 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:48:44.519 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:48:44.520 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.518+0000 7ff643c12640 1 -- 172.21.15.99:0/4010474583 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff6280a28a0 msgr2=0x7ff6280a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:48:44.520 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.518+0000 7ff643c12640 1 --2- 172.21.15.99:0/4010474583 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff6280a28a0 0x7ff6280a4d60 secure :-1 s=READY pgs=290 cs=0 l=1 rev1=1 crypto rx=0x7ff62c009fd0 tx=0x7ff62c013460 comp rx=0 tx=0).stop 2024-03-19T01:48:44.520 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.518+0000 7ff643c12640 1 -- 172.21.15.99:0/4010474583 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff63c1022f0 msgr2=0x7ff63c19ee70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:48:44.520 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.518+0000 7ff643c12640 1 --2- 172.21.15.99:0/4010474583 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff63c1022f0 0x7ff63c19ee70 secure :-1 s=READY pgs=493 cs=0 l=1 rev1=1 crypto rx=0x7ff63400eb10 tx=0x7ff63400c2d0 comp rx=0 tx=0).stop 2024-03-19T01:48:44.520 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.518+0000 7ff643c12640 1 -- 172.21.15.99:0/4010474583 shutdown_connections 2024-03-19T01:48:44.520 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.518+0000 7ff643c12640 1 --2- 172.21.15.99:0/4010474583 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff6280a28a0 0x7ff6280a4d60 unknown :-1 s=CLOSED pgs=290 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:44.520 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.518+0000 7ff643c12640 1 --2- 172.21.15.99:0/4010474583 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff63c101930 0x7ff63c19c850 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:44.520 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.518+0000 7ff643c12640 1 --2- 172.21.15.99:0/4010474583 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff63c1022f0 0x7ff63c19ee70 unknown :-1 s=CLOSED pgs=493 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:44.521 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.518+0000 7ff643c12640 1 --2- 172.21.15.99:0/4010474583 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff63c100740 0x7ff63c069a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:44.521 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.518+0000 7ff643c12640 1 -- 172.21.15.99:0/4010474583 >> 172.21.15.99:0/4010474583 conn(0x7ff63c0fbcb0 msgr2=0x7ff63c104c30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:48:44.521 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.518+0000 7ff643c12640 1 -- 172.21.15.99:0/4010474583 shutdown_connections 2024-03-19T01:48:44.521 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:44.518+0000 7ff643c12640 1 -- 172.21.15.99:0/4010474583 wait complete. 2024-03-19T01:48:44.691 INFO:tasks.workunit.client.0.smithi099.stdout: 1 923452 11.42 MB/sec execute 314 sec latency 29.447 ms 2024-03-19T01:48:45.691 INFO:tasks.workunit.client.0.smithi099.stdout: 1 925296 11.42 MB/sec execute 315 sec latency 20.300 ms 2024-03-19T01:48:46.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:48:45 smithi115 ceph-mon[67332]: pgmap v580: 97 pgs: 97 active+clean; 261 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 768 KiB/s rd, 2.8 MiB/s wr, 236 op/s 2024-03-19T01:48:46.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:48:45 smithi099 ceph-mon[62951]: pgmap v580: 97 pgs: 97 active+clean; 261 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 768 KiB/s rd, 2.8 MiB/s wr, 236 op/s 2024-03-19T01:48:46.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:48:45 smithi112 ceph-mon[66819]: pgmap v580: 97 pgs: 97 active+clean; 261 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 768 KiB/s rd, 2.8 MiB/s wr, 236 op/s 2024-03-19T01:48:46.691 INFO:tasks.workunit.client.0.smithi099.stdout: 1 927456 11.42 MB/sec execute 316 sec latency 64.245 ms 2024-03-19T01:48:47.691 INFO:tasks.workunit.client.0.smithi099.stdout: 1 929005 11.41 MB/sec execute 317 sec latency 37.167 ms 2024-03-19T01:48:48.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:48:47 smithi115 ceph-mon[67332]: pgmap v581: 97 pgs: 97 active+clean; 245 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 813 KiB/s rd, 3.4 MiB/s wr, 288 op/s 2024-03-19T01:48:48.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:48:47 smithi099 ceph-mon[62951]: pgmap v581: 97 pgs: 97 active+clean; 245 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 813 KiB/s rd, 3.4 MiB/s wr, 288 op/s 2024-03-19T01:48:48.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:48:47 smithi112 ceph-mon[66819]: pgmap v581: 97 pgs: 97 active+clean; 245 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 813 KiB/s rd, 3.4 MiB/s wr, 288 op/s 2024-03-19T01:48:48.691 INFO:tasks.workunit.client.0.smithi099.stdout: 1 931326 11.42 MB/sec execute 318 sec latency 21.991 ms 2024-03-19T01:48:49.691 INFO:tasks.workunit.client.0.smithi099.stdout: 1 933313 11.40 MB/sec execute 319 sec latency 112.378 ms 2024-03-19T01:48:50.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:48:49 smithi115 ceph-mon[67332]: pgmap v582: 97 pgs: 97 active+clean; 246 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 691 KiB/s rd, 3.1 MiB/s wr, 251 op/s 2024-03-19T01:48:50.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:48:49 smithi099 ceph-mon[62951]: pgmap v582: 97 pgs: 97 active+clean; 246 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 691 KiB/s rd, 3.1 MiB/s wr, 251 op/s 2024-03-19T01:48:50.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:48:49 smithi112 ceph-mon[66819]: pgmap v582: 97 pgs: 97 active+clean; 246 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 691 KiB/s rd, 3.1 MiB/s wr, 251 op/s 2024-03-19T01:48:50.691 INFO:tasks.workunit.client.0.smithi099.stdout: 1 935222 11.41 MB/sec execute 320 sec latency 42.530 ms 2024-03-19T01:48:51.691 INFO:tasks.workunit.client.0.smithi099.stdout: 1 937364 11.41 MB/sec execute 321 sec latency 90.447 ms 2024-03-19T01:48:52.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:48:51 smithi115 ceph-mon[67332]: pgmap v583: 97 pgs: 97 active+clean; 246 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 672 KiB/s rd, 3.1 MiB/s wr, 247 op/s 2024-03-19T01:48:52.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:48:51 smithi099 ceph-mon[62951]: pgmap v583: 97 pgs: 97 active+clean; 246 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 672 KiB/s rd, 3.1 MiB/s wr, 247 op/s 2024-03-19T01:48:52.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:48:51 smithi112 ceph-mon[66819]: pgmap v583: 97 pgs: 97 active+clean; 246 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 672 KiB/s rd, 3.1 MiB/s wr, 247 op/s 2024-03-19T01:48:52.691 INFO:tasks.workunit.client.0.smithi099.stdout: 1 939072 11.40 MB/sec execute 322 sec latency 35.777 ms 2024-03-19T01:48:53.691 INFO:tasks.workunit.client.0.smithi099.stdout: 1 941718 11.41 MB/sec execute 323 sec latency 14.659 ms 2024-03-19T01:48:54.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:48:53 smithi115 ceph-mon[67332]: pgmap v584: 97 pgs: 97 active+clean; 249 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 838 KiB/s rd, 3.6 MiB/s wr, 305 op/s 2024-03-19T01:48:54.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:48:53 smithi099 ceph-mon[62951]: pgmap v584: 97 pgs: 97 active+clean; 249 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 838 KiB/s rd, 3.6 MiB/s wr, 305 op/s 2024-03-19T01:48:54.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:48:53 smithi112 ceph-mon[66819]: pgmap v584: 97 pgs: 97 active+clean; 249 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 838 KiB/s rd, 3.6 MiB/s wr, 305 op/s 2024-03-19T01:48:54.691 INFO:tasks.workunit.client.0.smithi099.stdout: 1 943478 11.40 MB/sec execute 324 sec latency 33.615 ms 2024-03-19T01:48:55.692 INFO:tasks.workunit.client.0.smithi099.stdout: 1 945698 11.41 MB/sec execute 325 sec latency 15.662 ms 2024-03-19T01:48:56.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:48:55 smithi115 ceph-mon[67332]: pgmap v585: 97 pgs: 97 active+clean; 249 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 602 KiB/s rd, 2.9 MiB/s wr, 252 op/s 2024-03-19T01:48:56.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:48:55 smithi099 ceph-mon[62951]: pgmap v585: 97 pgs: 97 active+clean; 249 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 602 KiB/s rd, 2.9 MiB/s wr, 252 op/s 2024-03-19T01:48:56.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:48:55 smithi112 ceph-mon[66819]: pgmap v585: 97 pgs: 97 active+clean; 249 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 602 KiB/s rd, 2.9 MiB/s wr, 252 op/s 2024-03-19T01:48:56.691 INFO:tasks.workunit.client.0.smithi099.stdout: 1 947713 11.40 MB/sec execute 326 sec latency 124.326 ms 2024-03-19T01:48:57.612 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.610+0000 7f922c322640 1 -- 172.21.15.99:0/47578800 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9224104c10 msgr2=0x7f9224107000 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:48:57.612 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.610+0000 7f922c322640 1 --2- 172.21.15.99:0/47578800 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9224104c10 0x7f9224107000 secure :-1 s=READY pgs=498 cs=0 l=1 rev1=1 crypto rx=0x7f9214009900 tx=0x7f921402bc30 comp rx=0 tx=0).stop 2024-03-19T01:48:57.613 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.610+0000 7f922c322640 1 -- 172.21.15.99:0/47578800 shutdown_connections 2024-03-19T01:48:57.613 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.610+0000 7f922c322640 1 --2- 172.21.15.99:0/47578800 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9224107540 0x7f9224109970 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:57.613 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.610+0000 7f922c322640 1 --2- 172.21.15.99:0/47578800 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9224104c10 0x7f9224107000 unknown :-1 s=CLOSED pgs=498 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:57.613 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.610+0000 7f922c322640 1 --2- 172.21.15.99:0/47578800 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f92241022e0 0x7f92241046d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:57.613 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.610+0000 7f922c322640 1 -- 172.21.15.99:0/47578800 >> 172.21.15.99:0/47578800 conn(0x7f92240fbcb0 msgr2=0x7f92240fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:48:57.613 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.611+0000 7f922c322640 1 -- 172.21.15.99:0/47578800 shutdown_connections 2024-03-19T01:48:57.613 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.611+0000 7f922c322640 1 -- 172.21.15.99:0/47578800 wait complete. 2024-03-19T01:48:57.613 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.611+0000 7f922c322640 1 Processor -- start 2024-03-19T01:48:57.613 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.611+0000 7f922c322640 1 -- start start 2024-03-19T01:48:57.613 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.611+0000 7f922c322640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f92241022e0 0x7f9224197ae0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:57.613 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.611+0000 7f922c322640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9224104c10 0x7f9224198020 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:57.613 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.611+0000 7f922c322640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9224107540 0x7f922419f090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:57.613 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.611+0000 7f922c322640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f922410d4d0 con 0x7f9224107540 2024-03-19T01:48:57.614 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.611+0000 7f922c322640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f922410d350 con 0x7f9224104c10 2024-03-19T01:48:57.614 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.611+0000 7f922c322640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f922410d650 con 0x7f92241022e0 2024-03-19T01:48:57.614 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.611+0000 7f922a8bf640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9224107540 0x7f922419f090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:57.614 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.611+0000 7f922a8bf640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9224107540 0x7f922419f090 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:36126/0 (socket says 172.21.15.99:36126) 2024-03-19T01:48:57.614 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.611+0000 7f922a8bf640 1 -- 172.21.15.99:0/2471158207 learned_addr learned my addr 172.21.15.99:0/2471158207 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:48:57.614 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.611+0000 7f922a0be640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f92241022e0 0x7f9224197ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:57.614 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.611+0000 7f92298bd640 1 --2- 172.21.15.99:0/2471158207 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9224104c10 0x7f9224198020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:57.614 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.612+0000 7f922a8bf640 1 -- 172.21.15.99:0/2471158207 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f92241022e0 msgr2=0x7f9224197ae0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:48:57.614 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.612+0000 7f922a8bf640 1 --2- 172.21.15.99:0/2471158207 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f92241022e0 0x7f9224197ae0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:57.614 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.612+0000 7f922a8bf640 1 -- 172.21.15.99:0/2471158207 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9224104c10 msgr2=0x7f9224198020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:48:57.614 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.612+0000 7f922a8bf640 1 --2- 172.21.15.99:0/2471158207 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9224104c10 0x7f9224198020 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:57.614 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.612+0000 7f922a8bf640 1 -- 172.21.15.99:0/2471158207 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f922419f700 con 0x7f9224107540 2024-03-19T01:48:57.614 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.612+0000 7f922a8bf640 1 --2- 172.21.15.99:0/2471158207 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9224107540 0x7f922419f090 secure :-1 s=READY pgs=499 cs=0 l=1 rev1=1 crypto rx=0x7f921c00eb10 tx=0x7f921c00c2d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:48:57.614 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.612+0000 7f921b7fe640 1 -- 172.21.15.99:0/2471158207 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f921c004b20 con 0x7f9224107540 2024-03-19T01:48:57.615 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.612+0000 7f921b7fe640 1 -- 172.21.15.99:0/2471158207 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f921c004cc0 con 0x7f9224107540 2024-03-19T01:48:57.615 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.612+0000 7f922c322640 1 -- 172.21.15.99:0/2471158207 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f922419f9f0 con 0x7f9224107540 2024-03-19T01:48:57.615 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.612+0000 7f921b7fe640 1 -- 172.21.15.99:0/2471158207 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f921c010760 con 0x7f9224107540 2024-03-19T01:48:57.615 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.612+0000 7f922c322640 1 -- 172.21.15.99:0/2471158207 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f922419ff10 con 0x7f9224107540 2024-03-19T01:48:57.621 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.614+0000 7f921b7fe640 1 -- 172.21.15.99:0/2471158207 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f921c005020 con 0x7f9224107540 2024-03-19T01:48:57.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.615+0000 7f922efed740 1 Processor -- start 2024-03-19T01:48:57.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.615+0000 7f922efed740 1 -- start start 2024-03-19T01:48:57.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.615+0000 7f922efed740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563ab5a85100 0x563ab5a85520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:57.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.615+0000 7f922efed740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563ab5a85af0 0x563ab5a8a5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:57.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.615+0000 7f921b7fe640 1 --2- 172.21.15.99:0/2471158207 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f92100a28a0 0x7f92100a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:57.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.615+0000 7f922efed740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563ab5a8ac60 0x563ab5a8d0f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:57.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.615+0000 7f922efed740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x563ab59e9be0 con 0x563ab5a85100 2024-03-19T01:48:57.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.615+0000 7f922efed740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x563ab5992960 con 0x563ab5a8ac60 2024-03-19T01:48:57.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.615+0000 7f922efed740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x563ab59e5c80 con 0x563ab5a85af0 2024-03-19T01:48:57.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.615+0000 7f92298bd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563ab5a85100 0x563ab5a85520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:57.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.615+0000 7f92298bd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563ab5a85100 0x563ab5a85520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:36138/0 (socket says 172.21.15.99:36138) 2024-03-19T01:48:57.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.615+0000 7f92298bd640 1 -- 172.21.15.99:0/1196521409 learned_addr learned my addr 172.21.15.99:0/1196521409 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:48:57.623 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.615+0000 7f922a8bf640 1 --2- 172.21.15.99:0/1196521409 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563ab5a8ac60 0x563ab5a8d0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:57.623 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.615+0000 7f922a0be640 1 --2- 172.21.15.99:0/2471158207 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f92100a28a0 0x7f92100a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:57.623 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.615+0000 7f92298bd640 1 --2- 172.21.15.99:0/1196521409 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563ab5a85af0 0x563ab5a8a5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:57.623 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.616+0000 7f921b7fe640 1 -- 172.21.15.99:0/2471158207 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f921c0cf180 con 0x7f9224107540 2024-03-19T01:48:57.623 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.616+0000 7f92298bd640 1 -- 172.21.15.99:0/1196521409 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563ab5a85af0 msgr2=0x563ab5a8a5f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:48:57.623 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.616+0000 7f92298bd640 1 --2- 172.21.15.99:0/1196521409 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563ab5a85af0 0x563ab5a8a5f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:57.623 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.616+0000 7f92298bd640 1 -- 172.21.15.99:0/1196521409 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563ab5a8ac60 msgr2=0x563ab5a8d0f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:48:57.623 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.616+0000 7f92298bd640 1 --2- 172.21.15.99:0/1196521409 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563ab5a8ac60 0x563ab5a8d0f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:57.623 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.616+0000 7f92298bd640 1 -- 172.21.15.99:0/1196521409 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x563ab5989d20 con 0x563ab5a85100 2024-03-19T01:48:57.623 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.616+0000 7f922a8bf640 1 --2- 172.21.15.99:0/1196521409 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563ab5a8ac60 0x563ab5a8d0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:48:57.623 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.616+0000 7f92298bd640 1 --2- 172.21.15.99:0/1196521409 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563ab5a85100 0x563ab5a85520 secure :-1 s=READY pgs=500 cs=0 l=1 rev1=1 crypto rx=0x563ab5a86560 tx=0x7f9214005b50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=576f402533fd1ba0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:48:57.623 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.616+0000 7f922a0be640 1 --2- 172.21.15.99:0/2471158207 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f92100a28a0 0x7f92100a4d60 secure :-1 s=READY pgs=291 cs=0 l=1 rev1=1 crypto rx=0x7f9220009bd0 tx=0x7f9220005f90 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:48:57.623 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.616+0000 7f9218bfe640 1 -- 172.21.15.99:0/1196521409 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9214004220 con 0x563ab5a85100 2024-03-19T01:48:57.623 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.616+0000 7f9218bfe640 1 -- 172.21.15.99:0/1196521409 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f92140043c0 con 0x563ab5a85100 2024-03-19T01:48:57.624 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.616+0000 7f9218bfe640 1 -- 172.21.15.99:0/1196521409 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9214034680 con 0x563ab5a85100 2024-03-19T01:48:57.624 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.617+0000 7f922efed740 1 -- 172.21.15.99:0/1196521409 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563ab5a85100 msgr2=0x563ab5a85520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:48:57.624 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.617+0000 7f922efed740 1 --2- 172.21.15.99:0/1196521409 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563ab5a85100 0x563ab5a85520 secure :-1 s=READY pgs=500 cs=0 l=1 rev1=1 crypto rx=0x563ab5a86560 tx=0x7f9214005b50 comp rx=0 tx=0).stop 2024-03-19T01:48:57.624 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.617+0000 7f922efed740 1 -- 172.21.15.99:0/1196521409 shutdown_connections 2024-03-19T01:48:57.624 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.617+0000 7f922efed740 1 --2- 172.21.15.99:0/1196521409 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563ab5a8ac60 0x563ab5a8d0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:57.624 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.617+0000 7f922efed740 1 --2- 172.21.15.99:0/1196521409 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563ab5a85af0 0x563ab5a8a5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:57.624 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.617+0000 7f922efed740 1 --2- 172.21.15.99:0/1196521409 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563ab5a85100 0x563ab5a85520 unknown :-1 s=CLOSED pgs=500 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:57.624 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.617+0000 7f922efed740 1 -- 172.21.15.99:0/1196521409 >> 172.21.15.99:0/1196521409 conn(0x563ab5a80670 msgr2=0x563ab5a80a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:48:57.624 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.617+0000 7f922efed740 1 -- 172.21.15.99:0/1196521409 shutdown_connections 2024-03-19T01:48:57.624 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.617+0000 7f922efed740 1 -- 172.21.15.99:0/1196521409 wait complete. 2024-03-19T01:48:57.624 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.617+0000 7f922efed740 1 -- start start 2024-03-19T01:48:57.624 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.618+0000 7f922efed740 1 Processor -- start 2024-03-19T01:48:57.624 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.618+0000 7f922efed740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563ab5a85100 0x563ab5a8f0b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:57.625 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.618+0000 7f922efed740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563ab5a85af0 0x563ab5a8f5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:57.625 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.618+0000 7f922efed740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563ab5a8ac60 0x563ab5a96630 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:57.625 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.618+0000 7f922efed740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x563ab5a8db50 con 0x563ab5a85af0 2024-03-19T01:48:57.625 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.618+0000 7f922efed740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x563ab59a5db0 con 0x563ab5a85100 2024-03-19T01:48:57.625 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.618+0000 7f922efed740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x563ab5a8dcd0 con 0x563ab5a8ac60 2024-03-19T01:48:57.625 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.618+0000 7f922efed740 10 client.0 _ll_register_callbacks cb 0x563ab59aae70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:48:57.625 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.618+0000 7f92298bd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563ab5a85af0 0x563ab5a8f5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:57.625 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.618+0000 7f922a0be640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563ab5a85100 0x563ab5a8f0b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:57.625 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.618+0000 7f92298bd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563ab5a85af0 0x563ab5a8f5f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:36144/0 (socket says 172.21.15.99:36144) 2024-03-19T01:48:57.625 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.618+0000 7f92298bd640 1 -- 172.21.15.99:0/1258291568 learned_addr learned my addr 172.21.15.99:0/1258291568 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:48:57.625 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.618+0000 7f922a8bf640 1 --2- 172.21.15.99:0/1258291568 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563ab5a8ac60 0x563ab5a96630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:57.625 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.618+0000 7f92298bd640 1 -- 172.21.15.99:0/1258291568 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563ab5a8ac60 msgr2=0x563ab5a96630 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:48:57.625 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.618+0000 7f92298bd640 1 --2- 172.21.15.99:0/1258291568 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563ab5a8ac60 0x563ab5a96630 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:57.625 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.618+0000 7f92298bd640 1 -- 172.21.15.99:0/1258291568 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563ab5a85100 msgr2=0x563ab5a8f0b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:48:57.626 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.618+0000 7f92298bd640 1 --2- 172.21.15.99:0/1258291568 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563ab5a85100 0x563ab5a8f0b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:57.626 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.618+0000 7f92298bd640 1 -- 172.21.15.99:0/1258291568 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x563ab59a4c60 con 0x563ab5a85af0 2024-03-19T01:48:57.626 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.618+0000 7f922a0be640 1 --2- 172.21.15.99:0/1258291568 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563ab5a85100 0x563ab5a8f0b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:48:57.626 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.618+0000 7f92298bd640 1 --2- 172.21.15.99:0/1258291568 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563ab5a85af0 0x563ab5a8f5f0 secure :-1 s=READY pgs=501 cs=0 l=1 rev1=1 crypto rx=0x7f9214036040 tx=0x7f9214005fd0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b19dc68894360056 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:48:57.626 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.618+0000 7f920dffb640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:48:57.626 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.618+0000 7f920dffb640 1 -- 172.21.15.99:0/1258291568 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9214005250 con 0x563ab5a85af0 2024-03-19T01:48:57.626 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.619+0000 7f922efed740 1 -- 172.21.15.99:0/1258291568 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f92140043c0 con 0x563ab5a85af0 2024-03-19T01:48:57.626 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.619+0000 7f920dffb640 1 -- 172.21.15.99:0/1258291568 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f92140053f0 con 0x563ab5a85af0 2024-03-19T01:48:57.626 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.619+0000 7f920dffb640 1 -- 172.21.15.99:0/1258291568 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7f92140055f0 con 0x563ab5a85af0 2024-03-19T01:48:57.626 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.619+0000 7f922efed740 10 client.15972 fetch_fsmap learned FSMap version 27 2024-03-19T01:48:57.626 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.619+0000 7f922efed740 1 -- 172.21.15.99:0/1258291568 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x563ab56fe210 con 0x563ab5a85af0 2024-03-19T01:48:57.626 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.619+0000 7f920dffb640 1 -- 172.21.15.99:0/1258291568 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f92140043c0 con 0x563ab5a85af0 2024-03-19T01:48:57.626 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.619+0000 7f920dffb640 1 -- 172.21.15.99:0/1258291568 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 27) ==== 8338+0+0 (secure 0 0 0) 0x7f921404f4d0 con 0x563ab5a85af0 2024-03-19T01:48:57.627 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.619+0000 7f922efed740 10 client.15972 fetch_fsmap finished waiting for FSMap version 27 2024-03-19T01:48:57.627 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.619+0000 7f922efed740 10 client.15972 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:48:57.627 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.619+0000 7f922efed740 20 client.15972 populate_metadata read hostname 'smithi099' 2024-03-19T01:48:57.627 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.619+0000 7f922efed740 1 --2- 172.21.15.99:0/1258291568 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x563ab5a9ab20 0x563ab5a9cf10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:57.627 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.619+0000 7f922efed740 4 client.15972 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:48:57.627 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.619+0000 7f922efed740 1 -- 172.21.15.99:0/1258291568 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x563ab59f7120 con 0x563ab5a9ab20 2024-03-19T01:48:57.627 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.619+0000 7f922a0be640 1 --2- 172.21.15.99:0/1258291568 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x563ab5a9ab20 0x563ab5a9cf10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:57.627 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.620+0000 7f920dffb640 10 client.15972 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:48:57.627 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.620+0000 7f922a0be640 1 --2- 172.21.15.99:0/1258291568 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x563ab5a9ab20 0x563ab5a9cf10 crc :-1 s=READY pgs=99 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=2b012f7b09a459f7 server_cookie=714518de99b4c83f in_seq=0 out_seq=0 2024-03-19T01:48:57.627 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.621+0000 7f920dffb640 1 -- 172.21.15.99:0/1258291568 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x563ab59f7120 con 0x563ab5a9ab20 2024-03-19T01:48:57.627 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.621+0000 7f920dffb640 10 client.15972 handle_command_reply: tid=0 2024-03-19T01:48:57.627 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.621+0000 7f922efed740 1 client.15972 shutdown 2024-03-19T01:48:57.627 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.621+0000 7f922efed740 1 -- 172.21.15.99:0/1258291568 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563ab5a85af0 msgr2=0x563ab5a8f5f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:48:57.627 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.621+0000 7f922efed740 1 --2- 172.21.15.99:0/1258291568 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563ab5a85af0 0x563ab5a8f5f0 secure :-1 s=READY pgs=501 cs=0 l=1 rev1=1 crypto rx=0x7f9214036040 tx=0x7f9214005fd0 comp rx=0 tx=0).stop 2024-03-19T01:48:57.628 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.621+0000 7f922efed740 1 -- 172.21.15.99:0/1258291568 shutdown_connections 2024-03-19T01:48:57.628 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.621+0000 7f922efed740 1 --2- 172.21.15.99:0/1258291568 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x563ab5a9ab20 0x563ab5a9cf10 crc :-1 s=READY pgs=99 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:57.628 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.621+0000 7f920dffb640 0 client.15972 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:48:57.628 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.621+0000 7f922efed740 1 --2- 172.21.15.99:0/1258291568 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563ab5a8ac60 0x563ab5a96630 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:57.628 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.621+0000 7f922efed740 1 --2- 172.21.15.99:0/1258291568 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563ab5a85af0 0x563ab5a8f5f0 unknown :-1 s=CLOSED pgs=501 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:57.628 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.621+0000 7f922efed740 1 --2- 172.21.15.99:0/1258291568 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563ab5a85100 0x563ab5a8f0b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:57.628 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.621+0000 7f922efed740 1 -- 172.21.15.99:0/1258291568 >> 172.21.15.99:0/1258291568 conn(0x563ab5a80670 msgr2=0x563ab5a818c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:48:57.628 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.621+0000 7f922efed740 1 -- 172.21.15.99:0/1258291568 shutdown_connections 2024-03-19T01:48:57.628 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.621+0000 7f922efed740 1 -- 172.21.15.99:0/1258291568 wait complete. 2024-03-19T01:48:57.628 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.621+0000 7f922efed740 20 client.15972 trim_cache size 0 max 16384 2024-03-19T01:48:57.641 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.636+0000 7f922efed740 1 Processor -- start 2024-03-19T01:48:57.641 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.636+0000 7f922efed740 1 -- start start 2024-03-19T01:48:57.641 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.637+0000 7f922efed740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563ab5a80670 0x563ab5a7f930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:57.641 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.637+0000 7f922efed740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563ab5a85100 0x563ab5a965f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:57.642 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.637+0000 7f922efed740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563ab5a85af0 0x563ab5a86510 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:57.642 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.637+0000 7f922efed740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f92241ab660 con 0x563ab5a80670 2024-03-19T01:48:57.642 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.637+0000 7f922efed740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f92241a8f30 con 0x563ab5a85af0 2024-03-19T01:48:57.642 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.637+0000 7f922efed740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f92241a6d80 con 0x563ab5a85100 2024-03-19T01:48:57.642 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.637+0000 7f922a0be640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563ab5a80670 0x563ab5a7f930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:57.642 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.637+0000 7f92298bd640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563ab5a85100 0x563ab5a965f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:57.642 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.637+0000 7f922a0be640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563ab5a80670 0x563ab5a7f930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:36146/0 (socket says 172.21.15.99:36146) 2024-03-19T01:48:57.642 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.637+0000 7f92298bd640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563ab5a85100 0x563ab5a965f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:42666/0 (socket says 172.21.15.99:42666) 2024-03-19T01:48:57.642 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.637+0000 7f922a0be640 1 -- 172.21.15.99:0/1745938599 learned_addr learned my addr 172.21.15.99:0/1745938599 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:48:57.642 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.637+0000 7f922a8bf640 1 --2- 172.21.15.99:0/1745938599 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563ab5a85af0 0x563ab5a86510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:57.642 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.637+0000 7f92298bd640 1 -- 172.21.15.99:0/1745938599 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563ab5a85af0 msgr2=0x563ab5a86510 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:48:57.642 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.637+0000 7f92298bd640 1 --2- 172.21.15.99:0/1745938599 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563ab5a85af0 0x563ab5a86510 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:57.642 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.637+0000 7f92298bd640 1 -- 172.21.15.99:0/1745938599 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563ab5a80670 msgr2=0x563ab5a7f930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:48:57.643 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.637+0000 7f92298bd640 1 --2- 172.21.15.99:0/1745938599 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563ab5a80670 0x563ab5a7f930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:57.643 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.637+0000 7f92298bd640 1 -- 172.21.15.99:0/1745938599 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x563ab57ca620 con 0x563ab5a85100 2024-03-19T01:48:57.643 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.637+0000 7f922a0be640 1 --2- 172.21.15.99:0/1745938599 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563ab5a80670 0x563ab5a7f930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:48:57.643 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.637+0000 7f922a8bf640 1 --2- 172.21.15.99:0/1745938599 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563ab5a85af0 0x563ab5a86510 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:48:57.643 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.637+0000 7f92298bd640 1 --2- 172.21.15.99:0/1745938599 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563ab5a85100 0x563ab5a965f0 secure :-1 s=READY pgs=99 cs=0 l=1 rev1=1 crypto rx=0x7f921402cfd0 tx=0x7f921402bc30 comp rx=0 tx=0).ready entity=mon.1 client_cookie=ca85c928009efdca server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:48:57.643 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.637+0000 7f920ffff640 1 -- 172.21.15.99:0/1745938599 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9214005790 con 0x563ab5a85100 2024-03-19T01:48:57.643 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.637+0000 7f920ffff640 1 -- 172.21.15.99:0/1745938599 <== mon.1 v2:172.21.15.115:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f921405b020 con 0x563ab5a85100 2024-03-19T01:48:57.643 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.637+0000 7f920ffff640 1 -- 172.21.15.99:0/1745938599 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f921404fac0 con 0x563ab5a85100 2024-03-19T01:48:57.643 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.638+0000 7f922efed740 1 -- 172.21.15.99:0/1745938599 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563ab5a85100 msgr2=0x563ab5a965f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:48:57.643 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.638+0000 7f922efed740 1 --2- 172.21.15.99:0/1745938599 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563ab5a85100 0x563ab5a965f0 secure :-1 s=READY pgs=99 cs=0 l=1 rev1=1 crypto rx=0x7f921402cfd0 tx=0x7f921402bc30 comp rx=0 tx=0).stop 2024-03-19T01:48:57.643 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.638+0000 7f922efed740 1 -- 172.21.15.99:0/1745938599 shutdown_connections 2024-03-19T01:48:57.643 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.638+0000 7f922efed740 1 --2- 172.21.15.99:0/1745938599 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563ab5a85af0 0x563ab5a86510 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:57.643 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.638+0000 7f922efed740 1 --2- 172.21.15.99:0/1745938599 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563ab5a85100 0x563ab5a965f0 unknown :-1 s=CLOSED pgs=99 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:57.643 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.638+0000 7f922efed740 1 --2- 172.21.15.99:0/1745938599 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563ab5a80670 0x563ab5a7f930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:57.644 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.638+0000 7f922efed740 1 -- 172.21.15.99:0/1745938599 >> 172.21.15.99:0/1745938599 conn(0x563ab5a7bd20 msgr2=0x563ab5a7f220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:48:57.644 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.638+0000 7f922efed740 1 -- 172.21.15.99:0/1745938599 shutdown_connections 2024-03-19T01:48:57.644 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.638+0000 7f922efed740 1 -- 172.21.15.99:0/1745938599 wait complete. 2024-03-19T01:48:57.644 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.638+0000 7f922efed740 1 -- start start 2024-03-19T01:48:57.644 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.639+0000 7f922efed740 1 Processor -- start 2024-03-19T01:48:57.644 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.639+0000 7f922efed740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563ab5a80670 0x563ab5a93fb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:57.644 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.639+0000 7f922efed740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563ab5a85100 0x563ab5a944f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:57.644 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.639+0000 7f922efed740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563ab5a85af0 0x563ab5a95500 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:57.644 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.639+0000 7f922efed740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f92241a73f0 con 0x563ab5a85af0 2024-03-19T01:48:57.644 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.639+0000 7f922efed740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f92241a8860 con 0x563ab5a85100 2024-03-19T01:48:57.644 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.639+0000 7f922efed740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f92241a7ac0 con 0x563ab5a80670 2024-03-19T01:48:57.644 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.639+0000 7f922efed740 10 client.0 _ll_register_callbacks cb 0x7f921c09b8b0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:48:57.644 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.639+0000 7f922a0be640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563ab5a80670 0x563ab5a93fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:57.645 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.639+0000 7f922a8bf640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563ab5a85af0 0x563ab5a95500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:57.645 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.639+0000 7f922a0be640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563ab5a80670 0x563ab5a93fb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:42676/0 (socket says 172.21.15.99:42676) 2024-03-19T01:48:57.645 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.639+0000 7f922a0be640 1 -- 172.21.15.99:0/2783267840 learned_addr learned my addr 172.21.15.99:0/2783267840 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:48:57.645 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.639+0000 7f92298bd640 1 --2- 172.21.15.99:0/2783267840 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563ab5a85100 0x563ab5a944f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:57.645 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.639+0000 7f922a0be640 1 -- 172.21.15.99:0/2783267840 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563ab5a85100 msgr2=0x563ab5a944f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:48:57.645 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.639+0000 7f922a0be640 1 --2- 172.21.15.99:0/2783267840 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563ab5a85100 0x563ab5a944f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:57.645 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.639+0000 7f922a0be640 1 -- 172.21.15.99:0/2783267840 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563ab5a85af0 msgr2=0x563ab5a95500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:48:57.645 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.639+0000 7f922a0be640 1 --2- 172.21.15.99:0/2783267840 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563ab5a85af0 0x563ab5a95500 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:57.645 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.639+0000 7f922a0be640 1 -- 172.21.15.99:0/2783267840 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({osdmap=0}) -- 0x563ab5a95af0 con 0x563ab5a80670 2024-03-19T01:48:57.645 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.639+0000 7f922a8bf640 1 --2- 172.21.15.99:0/2783267840 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563ab5a85af0 0x563ab5a95500 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:48:57.645 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.639+0000 7f922a0be640 1 --2- 172.21.15.99:0/2783267840 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563ab5a80670 0x563ab5a93fb0 secure :-1 s=READY pgs=100 cs=0 l=1 rev1=1 crypto rx=0x7f922000e920 tx=0x7f9220004730 comp rx=0 tx=0).ready entity=mon.1 client_cookie=de0bdd3f41a8e71c server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:48:57.645 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.639+0000 7f920dffb640 10 client.0 ms_handle_connect on v2:172.21.15.115:3300/0 2024-03-19T01:48:57.645 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.639+0000 7f920dffb640 1 -- 172.21.15.99:0/2783267840 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9220029800 con 0x563ab5a80670 2024-03-19T01:48:57.646 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.640+0000 7f922efed740 1 -- 172.21.15.99:0/2783267840 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f921405b020 con 0x563ab5a80670 2024-03-19T01:48:57.646 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.640+0000 7f920dffb640 1 -- 172.21.15.99:0/2783267840 <== mon.1 v2:172.21.15.115:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f922001d410 con 0x563ab5a80670 2024-03-19T01:48:57.646 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.640+0000 7f920dffb640 1 -- 172.21.15.99:0/2783267840 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7f922001d610 con 0x563ab5a80670 2024-03-19T01:48:57.646 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.640+0000 7f922efed740 10 client.25798 fetch_fsmap learned FSMap version 27 2024-03-19T01:48:57.646 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.640+0000 7f922efed740 1 -- 172.21.15.99:0/2783267840 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x563ab5b01010 con 0x563ab5a80670 2024-03-19T01:48:57.646 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.640+0000 7f920dffb640 1 -- 172.21.15.99:0/2783267840 <== mon.1 v2:172.21.15.115:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f921405b020 con 0x563ab5a80670 2024-03-19T01:48:57.646 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.640+0000 7f920dffb640 1 -- 172.21.15.99:0/2783267840 <== mon.1 v2:172.21.15.115:3300/0 5 ==== fsmap(e 27) ==== 8338+0+0 (secure 0 0 0) 0x7f9220030070 con 0x563ab5a80670 2024-03-19T01:48:57.646 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.640+0000 7f922efed740 10 client.25798 fetch_fsmap finished waiting for FSMap version 27 2024-03-19T01:48:57.646 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.640+0000 7f922efed740 10 client.25798 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:48:57.646 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.640+0000 7f922efed740 20 client.25798 populate_metadata read hostname 'smithi099' 2024-03-19T01:48:57.646 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.640+0000 7f922efed740 1 --2- 172.21.15.99:0/2783267840 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x563ab5a8ac60 0x563ab5b03a30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:48:57.646 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.640+0000 7f922efed740 4 client.25798 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:48:57.646 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.640+0000 7f922efed740 1 -- 172.21.15.99:0/2783267840 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x563ab59f1470 con 0x563ab5a8ac60 2024-03-19T01:48:57.646 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.640+0000 7f92298bd640 1 --2- 172.21.15.99:0/2783267840 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x563ab5a8ac60 0x563ab5b03a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:48:57.647 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.641+0000 7f920dffb640 10 client.25798 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:48:57.647 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.641+0000 7f92298bd640 1 --2- 172.21.15.99:0/2783267840 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x563ab5a8ac60 0x563ab5b03a30 crc :-1 s=READY pgs=100 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=b53b999262d322f0 server_cookie=6d492d4e0957bae4 in_seq=0 out_seq=0 2024-03-19T01:48:57.691 INFO:tasks.workunit.client.0.smithi099.stdout: 1 949711 11.40 MB/sec execute 327 sec latency 35.913 ms 2024-03-19T01:48:57.700 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.699+0000 7f920dffb640 1 -- 172.21.15.99:0/2783267840 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x563ab59f1470 con 0x563ab5a8ac60 2024-03-19T01:48:57.700 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.699+0000 7f920dffb640 10 client.25798 handle_command_reply: tid=0 2024-03-19T01:48:57.700 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.699+0000 7f922efed740 1 client.25798 shutdown 2024-03-19T01:48:57.700 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.699+0000 7f922efed740 1 -- 172.21.15.99:0/2783267840 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563ab5a80670 msgr2=0x563ab5a93fb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:48:57.700 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.699+0000 7f922efed740 1 --2- 172.21.15.99:0/2783267840 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563ab5a80670 0x563ab5a93fb0 secure :-1 s=READY pgs=100 cs=0 l=1 rev1=1 crypto rx=0x7f922000e920 tx=0x7f9220004730 comp rx=0 tx=0).stop 2024-03-19T01:48:57.700 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.699+0000 7f922efed740 1 -- 172.21.15.99:0/2783267840 shutdown_connections 2024-03-19T01:48:57.700 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.699+0000 7f922efed740 1 --2- 172.21.15.99:0/2783267840 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x563ab5a8ac60 0x563ab5b03a30 crc :-1 s=READY pgs=100 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:57.701 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.699+0000 7f922efed740 1 --2- 172.21.15.99:0/2783267840 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563ab5a85100 0x563ab5a944f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:57.701 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.699+0000 7f920dffb640 0 client.25798 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:48:57.701 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.699+0000 7f922efed740 1 --2- 172.21.15.99:0/2783267840 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563ab5a85af0 0x563ab5a95500 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:57.701 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.699+0000 7f922efed740 1 --2- 172.21.15.99:0/2783267840 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563ab5a80670 0x563ab5a93fb0 unknown :-1 s=CLOSED pgs=100 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:57.701 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.699+0000 7f922efed740 1 -- 172.21.15.99:0/2783267840 >> 172.21.15.99:0/2783267840 conn(0x563ab5a7bd20 msgr2=0x563ab5a7fab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:48:57.701 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.700+0000 7f922efed740 1 -- 172.21.15.99:0/2783267840 shutdown_connections 2024-03-19T01:48:57.701 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.700+0000 7f922efed740 1 -- 172.21.15.99:0/2783267840 wait complete. 2024-03-19T01:48:57.701 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.700+0000 7f922efed740 20 client.25798 trim_cache size 0 max 16384 2024-03-19T01:48:57.701 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:48:57.701 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:48:57.702 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:48:57.702 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:48:57.702 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.700+0000 7f922c322640 1 -- 172.21.15.99:0/2471158207 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f92100a28a0 msgr2=0x7f92100a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:48:57.702 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.700+0000 7f922c322640 1 --2- 172.21.15.99:0/2471158207 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f92100a28a0 0x7f92100a4d60 secure :-1 s=READY pgs=291 cs=0 l=1 rev1=1 crypto rx=0x7f9220009bd0 tx=0x7f9220005f90 comp rx=0 tx=0).stop 2024-03-19T01:48:57.702 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.700+0000 7f922c322640 1 -- 172.21.15.99:0/2471158207 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9224107540 msgr2=0x7f922419f090 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:48:57.703 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.700+0000 7f922c322640 1 --2- 172.21.15.99:0/2471158207 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9224107540 0x7f922419f090 secure :-1 s=READY pgs=499 cs=0 l=1 rev1=1 crypto rx=0x7f921c00eb10 tx=0x7f921c00c2d0 comp rx=0 tx=0).stop 2024-03-19T01:48:57.703 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.701+0000 7f922c322640 1 -- 172.21.15.99:0/2471158207 shutdown_connections 2024-03-19T01:48:57.703 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.701+0000 7f922c322640 1 --2- 172.21.15.99:0/2471158207 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f92100a28a0 0x7f92100a4d60 unknown :-1 s=CLOSED pgs=291 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:57.703 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.701+0000 7f922c322640 1 --2- 172.21.15.99:0/2471158207 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9224104c10 0x7f9224198020 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:57.703 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.701+0000 7f922c322640 1 --2- 172.21.15.99:0/2471158207 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9224107540 0x7f922419f090 unknown :-1 s=CLOSED pgs=499 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:57.703 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.701+0000 7f922c322640 1 --2- 172.21.15.99:0/2471158207 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f92241022e0 0x7f9224197ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:48:57.703 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.701+0000 7f922c322640 1 -- 172.21.15.99:0/2471158207 >> 172.21.15.99:0/2471158207 conn(0x7f92240fbcb0 msgr2=0x7f9224102c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:48:57.703 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.701+0000 7f922c322640 1 -- 172.21.15.99:0/2471158207 shutdown_connections 2024-03-19T01:48:57.703 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:48:57.701+0000 7f922c322640 1 -- 172.21.15.99:0/2471158207 wait complete. 2024-03-19T01:48:58.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:48:57 smithi115 ceph-mon[67332]: pgmap v586: 97 pgs: 97 active+clean; 251 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 668 KiB/s rd, 3.3 MiB/s wr, 280 op/s 2024-03-19T01:48:58.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:48:57 smithi099 ceph-mon[62951]: pgmap v586: 97 pgs: 97 active+clean; 251 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 668 KiB/s rd, 3.3 MiB/s wr, 280 op/s 2024-03-19T01:48:58.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:48:57 smithi112 ceph-mon[66819]: pgmap v586: 97 pgs: 97 active+clean; 251 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 668 KiB/s rd, 3.3 MiB/s wr, 280 op/s 2024-03-19T01:48:58.692 INFO:tasks.workunit.client.0.smithi099.stdout: 1 952466 11.42 MB/sec execute 328 sec latency 15.303 ms 2024-03-19T01:48:59.692 INFO:tasks.workunit.client.0.smithi099.stdout: 1 954542 11.40 MB/sec execute 329 sec latency 37.494 ms 2024-03-19T01:49:00.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:48:59 smithi115 ceph-mon[67332]: pgmap v587: 97 pgs: 97 active+clean; 253 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 624 KiB/s rd, 2.9 MiB/s wr, 232 op/s 2024-03-19T01:49:00.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:48:59 smithi099 ceph-mon[62951]: pgmap v587: 97 pgs: 97 active+clean; 253 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 624 KiB/s rd, 2.9 MiB/s wr, 232 op/s 2024-03-19T01:49:00.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:48:59 smithi112 ceph-mon[66819]: pgmap v587: 97 pgs: 97 active+clean; 253 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 624 KiB/s rd, 2.9 MiB/s wr, 232 op/s 2024-03-19T01:49:00.692 INFO:tasks.workunit.client.0.smithi099.stdout: 1 956533 11.41 MB/sec execute 330 sec latency 28.512 ms 2024-03-19T01:49:01.692 INFO:tasks.workunit.client.0.smithi099.stdout: 1 959082 11.42 MB/sec execute 331 sec latency 85.194 ms 2024-03-19T01:49:02.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:49:01 smithi115 ceph-mon[67332]: pgmap v588: 97 pgs: 97 active+clean; 256 MiB data, 2.7 GiB used, 1.0 TiB / 1.0 TiB avail; 717 KiB/s rd, 3.0 MiB/s wr, 253 op/s 2024-03-19T01:49:02.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:49:01 smithi099 ceph-mon[62951]: pgmap v588: 97 pgs: 97 active+clean; 256 MiB data, 2.7 GiB used, 1.0 TiB / 1.0 TiB avail; 717 KiB/s rd, 3.0 MiB/s wr, 253 op/s 2024-03-19T01:49:02.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:49:01 smithi112 ceph-mon[66819]: pgmap v588: 97 pgs: 97 active+clean; 256 MiB data, 2.7 GiB used, 1.0 TiB / 1.0 TiB avail; 717 KiB/s rd, 3.0 MiB/s wr, 253 op/s 2024-03-19T01:49:02.692 INFO:tasks.workunit.client.0.smithi099.stdout: 1 960695 11.41 MB/sec execute 332 sec latency 32.604 ms 2024-03-19T01:49:03.692 INFO:tasks.workunit.client.0.smithi099.stdout: 1 963208 11.42 MB/sec execute 333 sec latency 9.789 ms 2024-03-19T01:49:04.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:49:03 smithi115 ceph-mon[67332]: pgmap v589: 97 pgs: 97 active+clean; 254 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 753 KiB/s rd, 3.1 MiB/s wr, 285 op/s 2024-03-19T01:49:04.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:49:03 smithi099 ceph-mon[62951]: pgmap v589: 97 pgs: 97 active+clean; 254 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 753 KiB/s rd, 3.1 MiB/s wr, 285 op/s 2024-03-19T01:49:04.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:49:03 smithi112 ceph-mon[66819]: pgmap v589: 97 pgs: 97 active+clean; 254 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 753 KiB/s rd, 3.1 MiB/s wr, 285 op/s 2024-03-19T01:49:04.692 INFO:tasks.workunit.client.0.smithi099.stdout: 1 965275 11.41 MB/sec execute 334 sec latency 40.516 ms 2024-03-19T01:49:05.692 INFO:tasks.workunit.client.0.smithi099.stdout: 1 967099 11.41 MB/sec execute 335 sec latency 52.850 ms 2024-03-19T01:49:06.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:49:05 smithi115 ceph-mon[67332]: pgmap v590: 97 pgs: 97 active+clean; 253 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 542 KiB/s rd, 2.3 MiB/s wr, 208 op/s 2024-03-19T01:49:06.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:49:05 smithi099 ceph-mon[62951]: pgmap v590: 97 pgs: 97 active+clean; 253 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 542 KiB/s rd, 2.3 MiB/s wr, 208 op/s 2024-03-19T01:49:06.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:49:05 smithi112 ceph-mon[66819]: pgmap v590: 97 pgs: 97 active+clean; 253 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 542 KiB/s rd, 2.3 MiB/s wr, 208 op/s 2024-03-19T01:49:06.692 INFO:tasks.workunit.client.0.smithi099.stdout: 1 969405 11.41 MB/sec execute 336 sec latency 91.641 ms 2024-03-19T01:49:07.692 INFO:tasks.workunit.client.0.smithi099.stdout: 1 971213 11.40 MB/sec execute 337 sec latency 39.133 ms 2024-03-19T01:49:08.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:49:07 smithi115 ceph-mon[67332]: pgmap v591: 97 pgs: 97 active+clean; 251 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 840 KiB/s rd, 3.0 MiB/s wr, 290 op/s 2024-03-19T01:49:08.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:49:07 smithi099 ceph-mon[62951]: pgmap v591: 97 pgs: 97 active+clean; 251 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 840 KiB/s rd, 3.0 MiB/s wr, 290 op/s 2024-03-19T01:49:08.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:49:07 smithi112 ceph-mon[66819]: pgmap v591: 97 pgs: 97 active+clean; 251 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 840 KiB/s rd, 3.0 MiB/s wr, 290 op/s 2024-03-19T01:49:08.692 INFO:tasks.workunit.client.0.smithi099.stdout: 1 973790 11.42 MB/sec execute 338 sec latency 18.339 ms 2024-03-19T01:49:09.692 INFO:tasks.workunit.client.0.smithi099.stdout: 1 975803 11.41 MB/sec execute 339 sec latency 83.970 ms 2024-03-19T01:49:10.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:49:09 smithi115 ceph-mon[67332]: pgmap v592: 97 pgs: 97 active+clean; 233 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 595 KiB/s rd, 2.7 MiB/s wr, 237 op/s 2024-03-19T01:49:10.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:49:09 smithi099 ceph-mon[62951]: pgmap v592: 97 pgs: 97 active+clean; 233 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 595 KiB/s rd, 2.7 MiB/s wr, 237 op/s 2024-03-19T01:49:10.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:49:09 smithi112 ceph-mon[66819]: pgmap v592: 97 pgs: 97 active+clean; 233 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 595 KiB/s rd, 2.7 MiB/s wr, 237 op/s 2024-03-19T01:49:10.692 INFO:tasks.workunit.client.0.smithi099.stdout: 1 977610 11.41 MB/sec execute 340 sec latency 24.300 ms 2024-03-19T01:49:10.796 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.793+0000 7f965fd87640 1 -- 172.21.15.99:0/3476488441 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9658100740 msgr2=0x7f9658100b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:49:10.796 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.793+0000 7f965fd87640 1 --2- 172.21.15.99:0/3476488441 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9658100740 0x7f9658100b60 secure :-1 s=READY pgs=502 cs=0 l=1 rev1=1 crypto rx=0x7f9654009900 tx=0x7f965402bc30 comp rx=0 tx=0).stop 2024-03-19T01:49:10.796 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.793+0000 7f965fd87640 1 -- 172.21.15.99:0/3476488441 shutdown_connections 2024-03-19T01:49:10.796 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.793+0000 7f965fd87640 1 --2- 172.21.15.99:0/3476488441 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9658101930 0x7f9658101db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:10.796 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.793+0000 7f965fd87640 1 --2- 172.21.15.99:0/3476488441 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f96581022f0 0x7f9658108b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:10.796 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.793+0000 7f965fd87640 1 --2- 172.21.15.99:0/3476488441 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9658100740 0x7f9658100b60 unknown :-1 s=CLOSED pgs=502 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:10.796 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.793+0000 7f965fd87640 1 -- 172.21.15.99:0/3476488441 >> 172.21.15.99:0/3476488441 conn(0x7f96580fbcb0 msgr2=0x7f96580fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:49:10.796 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.793+0000 7f965fd87640 1 -- 172.21.15.99:0/3476488441 shutdown_connections 2024-03-19T01:49:10.796 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.793+0000 7f965fd87640 1 -- 172.21.15.99:0/3476488441 wait complete. 2024-03-19T01:49:10.796 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.793+0000 7f965fd87640 1 Processor -- start 2024-03-19T01:49:10.797 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.793+0000 7f965fd87640 1 -- start start 2024-03-19T01:49:10.797 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.793+0000 7f965fd87640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9658100740 0x7f9658197b00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:10.797 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.794+0000 7f965fd87640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9658101930 0x7f9658198040 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:10.797 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.794+0000 7f965fd87640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f96581022f0 0x7f965819f0b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:10.797 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.794+0000 7f965fd87640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f965810c3c0 con 0x7f9658101930 2024-03-19T01:49:10.797 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.794+0000 7f965fd87640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f965810c240 con 0x7f96581022f0 2024-03-19T01:49:10.797 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.794+0000 7f965fd87640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f965810c540 con 0x7f9658100740 2024-03-19T01:49:10.797 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.794+0000 7f965d322640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9658101930 0x7f9658198040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:10.797 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.794+0000 7f965db23640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9658100740 0x7f9658197b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:10.797 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.794+0000 7f965d322640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9658101930 0x7f9658198040 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:45360/0 (socket says 172.21.15.99:45360) 2024-03-19T01:49:10.797 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.794+0000 7f965d322640 1 -- 172.21.15.99:0/3225198097 learned_addr learned my addr 172.21.15.99:0/3225198097 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:49:10.797 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.794+0000 7f965e324640 1 --2- 172.21.15.99:0/3225198097 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f96581022f0 0x7f965819f0b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:10.797 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.794+0000 7f965d322640 1 -- 172.21.15.99:0/3225198097 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9658100740 msgr2=0x7f9658197b00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:49:10.797 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.794+0000 7f965d322640 1 --2- 172.21.15.99:0/3225198097 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9658100740 0x7f9658197b00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:10.798 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.794+0000 7f965d322640 1 -- 172.21.15.99:0/3225198097 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f96581022f0 msgr2=0x7f965819f0b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:49:10.798 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.794+0000 7f965d322640 1 --2- 172.21.15.99:0/3225198097 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f96581022f0 0x7f965819f0b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:10.798 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.794+0000 7f965d322640 1 -- 172.21.15.99:0/3225198097 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f965819f720 con 0x7f9658101930 2024-03-19T01:49:10.798 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.794+0000 7f965e324640 1 --2- 172.21.15.99:0/3225198097 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f96581022f0 0x7f965819f0b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:49:10.798 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.794+0000 7f965d322640 1 --2- 172.21.15.99:0/3225198097 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9658101930 0x7f9658198040 secure :-1 s=READY pgs=503 cs=0 l=1 rev1=1 crypto rx=0x7f964800e9f0 tx=0x7f964800eeb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:49:10.798 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.794+0000 7f964effd640 1 -- 172.21.15.99:0/3225198097 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f964800cd80 con 0x7f9658101930 2024-03-19T01:49:10.798 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.794+0000 7f964effd640 1 -- 172.21.15.99:0/3225198097 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f9648010910 con 0x7f9658101930 2024-03-19T01:49:10.798 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.794+0000 7f964effd640 1 -- 172.21.15.99:0/3225198097 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9648018990 con 0x7f9658101930 2024-03-19T01:49:10.798 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.794+0000 7f965fd87640 1 -- 172.21.15.99:0/3225198097 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f965819fa10 con 0x7f9658101930 2024-03-19T01:49:10.798 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.794+0000 7f965fd87640 1 -- 172.21.15.99:0/3225198097 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f965819ff10 con 0x7f9658101930 2024-03-19T01:49:10.805 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.797+0000 7f964effd640 1 -- 172.21.15.99:0/3225198097 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f9648010ac0 con 0x7f9658101930 2024-03-19T01:49:10.805 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.797+0000 7f9662971740 1 Processor -- start 2024-03-19T01:49:10.805 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.797+0000 7f9662971740 1 -- start start 2024-03-19T01:49:10.805 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.798+0000 7f964effd640 1 --2- 172.21.15.99:0/3225198097 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f96440a28a0 0x7f96440a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:10.805 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.798+0000 7f9662971740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ea986ac100 0x55ea986ac520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:10.805 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.798+0000 7f9662971740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ea986acaf0 0x55ea986b15f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:10.805 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.798+0000 7f9662971740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ea986b1c60 0x55ea986b40f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:10.805 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.798+0000 7f9662971740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55ea98610be0 con 0x55ea986acaf0 2024-03-19T01:49:10.805 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.798+0000 7f9662971740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55ea985b9960 con 0x55ea986b1c60 2024-03-19T01:49:10.806 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.798+0000 7f9662971740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55ea9860cc80 con 0x55ea986ac100 2024-03-19T01:49:10.806 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.798+0000 7f965db23640 1 --2- 172.21.15.99:0/3225198097 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f96440a28a0 0x7f96440a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:10.806 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.798+0000 7f965e324640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ea986ac100 0x55ea986ac520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:10.806 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.798+0000 7f965d322640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ea986acaf0 0x55ea986b15f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:10.806 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.798+0000 7f965e324640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ea986ac100 0x55ea986ac520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:34950/0 (socket says 172.21.15.99:34950) 2024-03-19T01:49:10.806 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.798+0000 7f965e324640 1 -- 172.21.15.99:0/4119595610 learned_addr learned my addr 172.21.15.99:0/4119595610 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:49:10.806 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.798+0000 7f964effd640 1 -- 172.21.15.99:0/3225198097 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f96480cedd0 con 0x7f9658101930 2024-03-19T01:49:10.806 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.798+0000 7f965e324640 1 --2- 172.21.15.99:0/4119595610 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ea986b1c60 0x55ea986b40f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:10.806 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.798+0000 7f965d322640 1 -- 172.21.15.99:0/4119595610 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ea986ac100 msgr2=0x55ea986ac520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:49:10.806 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.798+0000 7f965d322640 1 --2- 172.21.15.99:0/4119595610 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ea986ac100 0x55ea986ac520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:10.806 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.798+0000 7f965d322640 1 -- 172.21.15.99:0/4119595610 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ea986b1c60 msgr2=0x55ea986b40f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:49:10.806 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.798+0000 7f965d322640 1 --2- 172.21.15.99:0/4119595610 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ea986b1c60 0x55ea986b40f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:10.806 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.798+0000 7f965d322640 1 -- 172.21.15.99:0/4119595610 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55ea985b0d20 con 0x55ea986acaf0 2024-03-19T01:49:10.806 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.798+0000 7f965e324640 1 --2- 172.21.15.99:0/4119595610 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ea986b1c60 0x55ea986b40f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:49:10.807 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.798+0000 7f965db23640 1 --2- 172.21.15.99:0/3225198097 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f96440a28a0 0x7f96440a4d60 secure :-1 s=READY pgs=292 cs=0 l=1 rev1=1 crypto rx=0x7f9654000c00 tx=0x7f9654005fd0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:49:10.807 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.798+0000 7f965d322640 1 --2- 172.21.15.99:0/4119595610 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ea986acaf0 0x55ea986b15f0 secure :-1 s=READY pgs=504 cs=0 l=1 rev1=1 crypto rx=0x7f96480cc8e0 tx=0x7f9648018420 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d9c899eb9ac3b21a server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:49:10.807 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.798+0000 7f9643fff640 1 -- 172.21.15.99:0/4119595610 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9648026040 con 0x55ea986acaf0 2024-03-19T01:49:10.807 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.798+0000 7f9643fff640 1 -- 172.21.15.99:0/4119595610 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f964809b7e0 con 0x55ea986acaf0 2024-03-19T01:49:10.807 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.798+0000 7f9643fff640 1 -- 172.21.15.99:0/4119595610 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9648097e50 con 0x55ea986acaf0 2024-03-19T01:49:10.807 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.799+0000 7f9662971740 1 -- 172.21.15.99:0/4119595610 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ea986acaf0 msgr2=0x55ea986b15f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:49:10.807 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.799+0000 7f9662971740 1 --2- 172.21.15.99:0/4119595610 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ea986acaf0 0x55ea986b15f0 secure :-1 s=READY pgs=504 cs=0 l=1 rev1=1 crypto rx=0x7f96480cc8e0 tx=0x7f9648018420 comp rx=0 tx=0).stop 2024-03-19T01:49:10.807 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.799+0000 7f9662971740 1 -- 172.21.15.99:0/4119595610 shutdown_connections 2024-03-19T01:49:10.807 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.799+0000 7f9662971740 1 --2- 172.21.15.99:0/4119595610 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ea986b1c60 0x55ea986b40f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:10.807 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.799+0000 7f9662971740 1 --2- 172.21.15.99:0/4119595610 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ea986acaf0 0x55ea986b15f0 unknown :-1 s=CLOSED pgs=504 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:10.807 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.799+0000 7f9662971740 1 --2- 172.21.15.99:0/4119595610 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ea986ac100 0x55ea986ac520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:10.807 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.799+0000 7f9662971740 1 -- 172.21.15.99:0/4119595610 >> 172.21.15.99:0/4119595610 conn(0x55ea986a7670 msgr2=0x55ea986a7a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:49:10.807 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.799+0000 7f9662971740 1 -- 172.21.15.99:0/4119595610 shutdown_connections 2024-03-19T01:49:10.808 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.799+0000 7f9662971740 1 -- 172.21.15.99:0/4119595610 wait complete. 2024-03-19T01:49:10.808 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.799+0000 7f9662971740 1 -- start start 2024-03-19T01:49:10.808 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.800+0000 7f9662971740 1 Processor -- start 2024-03-19T01:49:10.808 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.800+0000 7f9662971740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ea986ac100 0x55ea986b6070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:10.808 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.800+0000 7f9662971740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ea986acaf0 0x55ea986b65b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:10.808 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.800+0000 7f9662971740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ea986b1c60 0x55ea986bd5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:10.808 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.800+0000 7f9662971740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55ea986b4bb0 con 0x55ea986b1c60 2024-03-19T01:49:10.808 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.800+0000 7f9662971740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55ea985ccdb0 con 0x55ea986ac100 2024-03-19T01:49:10.808 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.800+0000 7f9662971740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55ea986b4d30 con 0x55ea986acaf0 2024-03-19T01:49:10.808 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.800+0000 7f9662971740 10 client.0 _ll_register_callbacks cb 0x55ea985d1e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:49:10.808 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.800+0000 7f965e324640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ea986b1c60 0x55ea986bd5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:10.808 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.800+0000 7f965e324640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ea986b1c60 0x55ea986bd5f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:45372/0 (socket says 172.21.15.99:45372) 2024-03-19T01:49:10.808 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.800+0000 7f965e324640 1 -- 172.21.15.99:0/2544495745 learned_addr learned my addr 172.21.15.99:0/2544495745 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:49:10.809 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.800+0000 7f965d322640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ea986acaf0 0x55ea986b65b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:10.809 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.800+0000 7f965db23640 1 --2- 172.21.15.99:0/2544495745 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ea986ac100 0x55ea986b6070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:10.809 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.800+0000 7f965e324640 1 -- 172.21.15.99:0/2544495745 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ea986acaf0 msgr2=0x55ea986b65b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:49:10.809 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.800+0000 7f965e324640 1 --2- 172.21.15.99:0/2544495745 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ea986acaf0 0x55ea986b65b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:10.809 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.800+0000 7f965e324640 1 -- 172.21.15.99:0/2544495745 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ea986ac100 msgr2=0x55ea986b6070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:49:10.809 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.800+0000 7f965e324640 1 --2- 172.21.15.99:0/2544495745 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ea986ac100 0x55ea986b6070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:10.809 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.800+0000 7f965e324640 1 -- 172.21.15.99:0/2544495745 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55ea985cbc60 con 0x55ea986b1c60 2024-03-19T01:49:10.809 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.800+0000 7f965e324640 1 --2- 172.21.15.99:0/2544495745 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ea986b1c60 0x55ea986bd5f0 secure :-1 s=READY pgs=505 cs=0 l=1 rev1=1 crypto rx=0x7f9650003a60 tx=0x7f9650011750 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b38a5f8db4e80e67 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:49:10.809 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.800+0000 7f96417fa640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:49:10.809 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.801+0000 7f96417fa640 1 -- 172.21.15.99:0/2544495745 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9650004d20 con 0x55ea986b1c60 2024-03-19T01:49:10.809 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.801+0000 7f96417fa640 1 -- 172.21.15.99:0/2544495745 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f9650005020 con 0x55ea986b1c60 2024-03-19T01:49:10.809 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.801+0000 7f9662971740 1 -- 172.21.15.99:0/2544495745 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f964809b7e0 con 0x55ea986b1c60 2024-03-19T01:49:10.809 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.801+0000 7f96417fa640 1 -- 172.21.15.99:0/2544495745 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7f9650004810 con 0x55ea986b1c60 2024-03-19T01:49:10.809 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.801+0000 7f9662971740 10 client.15990 fetch_fsmap learned FSMap version 27 2024-03-19T01:49:10.810 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.801+0000 7f9662971740 1 -- 172.21.15.99:0/2544495745 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x55ea98325210 con 0x55ea986b1c60 2024-03-19T01:49:10.810 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.801+0000 7f96417fa640 1 -- 172.21.15.99:0/2544495745 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f964809b7e0 con 0x55ea986b1c60 2024-03-19T01:49:10.810 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.801+0000 7f96417fa640 1 -- 172.21.15.99:0/2544495745 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 27) ==== 8338+0+0 (secure 0 0 0) 0x7f965001d070 con 0x55ea986b1c60 2024-03-19T01:49:10.810 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.801+0000 7f9662971740 10 client.15990 fetch_fsmap finished waiting for FSMap version 27 2024-03-19T01:49:10.810 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.801+0000 7f9662971740 10 client.15990 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:49:10.810 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.801+0000 7f9662971740 20 client.15990 populate_metadata read hostname 'smithi099' 2024-03-19T01:49:10.810 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.801+0000 7f9662971740 1 --2- 172.21.15.99:0/2544495745 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55ea986c1f90 0x55ea986c43c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:10.810 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.801+0000 7f9662971740 4 client.15990 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:49:10.810 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.801+0000 7f9662971740 1 -- 172.21.15.99:0/2544495745 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55ea9861e120 con 0x55ea986c1f90 2024-03-19T01:49:10.810 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.801+0000 7f965db23640 1 --2- 172.21.15.99:0/2544495745 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55ea986c1f90 0x55ea986c43c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:10.810 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.802+0000 7f96417fa640 10 client.15990 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:49:10.810 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.802+0000 7f965db23640 1 --2- 172.21.15.99:0/2544495745 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55ea986c1f90 0x55ea986c43c0 crc :-1 s=READY pgs=101 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=1ae5db69e5b907e2 server_cookie=f498b36457f41595 in_seq=0 out_seq=0 2024-03-19T01:49:10.810 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.802+0000 7f96417fa640 1 -- 172.21.15.99:0/2544495745 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x55ea9861e120 con 0x55ea986c1f90 2024-03-19T01:49:10.810 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.802+0000 7f96417fa640 10 client.15990 handle_command_reply: tid=0 2024-03-19T01:49:10.811 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.802+0000 7f9662971740 1 client.15990 shutdown 2024-03-19T01:49:10.811 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.803+0000 7f9662971740 1 -- 172.21.15.99:0/2544495745 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ea986b1c60 msgr2=0x55ea986bd5f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:49:10.811 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.803+0000 7f9662971740 1 --2- 172.21.15.99:0/2544495745 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ea986b1c60 0x55ea986bd5f0 secure :-1 s=READY pgs=505 cs=0 l=1 rev1=1 crypto rx=0x7f9650003a60 tx=0x7f9650011750 comp rx=0 tx=0).stop 2024-03-19T01:49:10.811 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.803+0000 7f9662971740 1 -- 172.21.15.99:0/2544495745 shutdown_connections 2024-03-19T01:49:10.811 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.803+0000 7f9662971740 1 --2- 172.21.15.99:0/2544495745 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55ea986c1f90 0x55ea986c43c0 crc :-1 s=READY pgs=101 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:10.811 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.803+0000 7f9662971740 1 --2- 172.21.15.99:0/2544495745 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ea986b1c60 0x55ea986bd5f0 unknown :-1 s=CLOSED pgs=505 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:10.811 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.803+0000 7f9662971740 1 --2- 172.21.15.99:0/2544495745 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ea986acaf0 0x55ea986b65b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:10.811 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.803+0000 7f96417fa640 0 client.15990 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:49:10.811 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.803+0000 7f9662971740 1 --2- 172.21.15.99:0/2544495745 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ea986ac100 0x55ea986b6070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:10.811 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.803+0000 7f9662971740 1 -- 172.21.15.99:0/2544495745 >> 172.21.15.99:0/2544495745 conn(0x55ea986a7670 msgr2=0x55ea986a88c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:49:10.811 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.803+0000 7f9662971740 1 -- 172.21.15.99:0/2544495745 shutdown_connections 2024-03-19T01:49:10.811 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.803+0000 7f9662971740 1 -- 172.21.15.99:0/2544495745 wait complete. 2024-03-19T01:49:10.812 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.803+0000 7f9662971740 20 client.15990 trim_cache size 0 max 16384 2024-03-19T01:49:10.823 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.818+0000 7f9662971740 1 Processor -- start 2024-03-19T01:49:10.823 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.818+0000 7f9662971740 1 -- start start 2024-03-19T01:49:10.823 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.818+0000 7f9662971740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ea986ac100 0x55ea986a6220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:10.823 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.818+0000 7f9662971740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ea986acaf0 0x55ea986bd5b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:10.823 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.818+0000 7f9662971740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ea986b1c60 0x55ea986a6760 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:10.823 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.818+0000 7f9662971740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f96581ab6e0 con 0x55ea986acaf0 2024-03-19T01:49:10.823 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.818+0000 7f9662971740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f96581a8fb0 con 0x55ea986ac100 2024-03-19T01:49:10.823 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.818+0000 7f9662971740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f96581a6e00 con 0x55ea986b1c60 2024-03-19T01:49:10.823 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.818+0000 7f965d322640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ea986acaf0 0x55ea986bd5b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:10.824 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.818+0000 7f965d322640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ea986acaf0 0x55ea986bd5b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:45388/0 (socket says 172.21.15.99:45388) 2024-03-19T01:49:10.824 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.818+0000 7f965d322640 1 -- 172.21.15.99:0/4059895736 learned_addr learned my addr 172.21.15.99:0/4059895736 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:49:10.824 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.818+0000 7f965e324640 1 --2- 172.21.15.99:0/4059895736 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ea986b1c60 0x55ea986a6760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:10.824 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.818+0000 7f965db23640 1 --2- 172.21.15.99:0/4059895736 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ea986ac100 0x55ea986a6220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:10.824 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.818+0000 7f965d322640 1 -- 172.21.15.99:0/4059895736 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ea986b1c60 msgr2=0x55ea986a6760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:49:10.824 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.818+0000 7f965d322640 1 --2- 172.21.15.99:0/4059895736 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ea986b1c60 0x55ea986a6760 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:10.824 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.818+0000 7f965d322640 1 -- 172.21.15.99:0/4059895736 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ea986ac100 msgr2=0x55ea986a6220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:49:10.824 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.819+0000 7f965d322640 1 --2- 172.21.15.99:0/4059895736 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ea986ac100 0x55ea986a6220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:10.824 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.819+0000 7f965d322640 1 -- 172.21.15.99:0/4059895736 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55ea983f1620 con 0x55ea986acaf0 2024-03-19T01:49:10.824 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.819+0000 7f965d322640 1 --2- 172.21.15.99:0/4059895736 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ea986acaf0 0x55ea986bd5b0 secure :-1 s=READY pgs=506 cs=0 l=1 rev1=1 crypto rx=0x7f96480ccc00 tx=0x7f9648000bc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=bd7713ba3386f2f2 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:49:10.824 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.819+0000 7f96437fe640 1 -- 172.21.15.99:0/4059895736 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f96480b8070 con 0x55ea986acaf0 2024-03-19T01:49:10.824 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.819+0000 7f96437fe640 1 -- 172.21.15.99:0/4059895736 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f9648099ae0 con 0x55ea986acaf0 2024-03-19T01:49:10.824 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.819+0000 7f96437fe640 1 -- 172.21.15.99:0/4059895736 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f964809ac30 con 0x55ea986acaf0 2024-03-19T01:49:10.824 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.819+0000 7f9662971740 1 -- 172.21.15.99:0/4059895736 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ea986acaf0 msgr2=0x55ea986bd5b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:49:10.825 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.819+0000 7f9662971740 1 --2- 172.21.15.99:0/4059895736 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ea986acaf0 0x55ea986bd5b0 secure :-1 s=READY pgs=506 cs=0 l=1 rev1=1 crypto rx=0x7f96480ccc00 tx=0x7f9648000bc0 comp rx=0 tx=0).stop 2024-03-19T01:49:10.825 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.819+0000 7f9662971740 1 -- 172.21.15.99:0/4059895736 shutdown_connections 2024-03-19T01:49:10.825 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.819+0000 7f9662971740 1 --2- 172.21.15.99:0/4059895736 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ea986b1c60 0x55ea986a6760 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:10.825 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.819+0000 7f9662971740 1 --2- 172.21.15.99:0/4059895736 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ea986acaf0 0x55ea986bd5b0 unknown :-1 s=CLOSED pgs=506 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:10.825 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.819+0000 7f9662971740 1 --2- 172.21.15.99:0/4059895736 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ea986ac100 0x55ea986a6220 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:10.825 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.819+0000 7f9662971740 1 -- 172.21.15.99:0/4059895736 >> 172.21.15.99:0/4059895736 conn(0x55ea986a7670 msgr2=0x55ea986a7a50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:49:10.825 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.819+0000 7f9662971740 1 -- 172.21.15.99:0/4059895736 shutdown_connections 2024-03-19T01:49:10.825 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.819+0000 7f9662971740 1 -- 172.21.15.99:0/4059895736 wait complete. 2024-03-19T01:49:10.825 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.820+0000 7f9662971740 1 -- start start 2024-03-19T01:49:10.825 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.820+0000 7f9662971740 1 Processor -- start 2024-03-19T01:49:10.825 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.820+0000 7f9662971740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ea986ac100 0x55ea986bad90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:10.825 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.820+0000 7f9662971740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ea986acaf0 0x55ea986bb2d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:10.825 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.820+0000 7f9662971740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ea986b1c60 0x55ea986bc220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:10.825 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.820+0000 7f9662971740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f96581a7470 con 0x55ea986ac100 2024-03-19T01:49:10.825 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.820+0000 7f9662971740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f96581a88e0 con 0x55ea986b1c60 2024-03-19T01:49:10.826 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.820+0000 7f9662971740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f96581a7b40 con 0x55ea986acaf0 2024-03-19T01:49:10.826 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.820+0000 7f9662971740 10 client.0 _ll_register_callbacks cb 0x7f9650003970 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:49:10.826 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.820+0000 7f965db23640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ea986ac100 0x55ea986bad90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:10.826 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.820+0000 7f965db23640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ea986ac100 0x55ea986bad90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:45396/0 (socket says 172.21.15.99:45396) 2024-03-19T01:49:10.826 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.820+0000 7f965db23640 1 -- 172.21.15.99:0/3556448256 learned_addr learned my addr 172.21.15.99:0/3556448256 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:49:10.826 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.820+0000 7f965e324640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ea986b1c60 0x55ea986bc220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:10.826 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.820+0000 7f965d322640 1 --2- 172.21.15.99:0/3556448256 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ea986acaf0 0x55ea986bb2d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:10.826 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.820+0000 7f965db23640 1 -- 172.21.15.99:0/3556448256 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ea986acaf0 msgr2=0x55ea986bb2d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:49:10.826 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.820+0000 7f965db23640 1 --2- 172.21.15.99:0/3556448256 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ea986acaf0 0x55ea986bb2d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:10.826 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.821+0000 7f965db23640 1 -- 172.21.15.99:0/3556448256 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ea986b1c60 msgr2=0x55ea986bc220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:49:10.826 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.821+0000 7f965db23640 1 --2- 172.21.15.99:0/3556448256 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ea986b1c60 0x55ea986bc220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:10.827 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.821+0000 7f965db23640 1 -- 172.21.15.99:0/3556448256 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55ea986bc810 con 0x55ea986ac100 2024-03-19T01:49:10.827 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.821+0000 7f965db23640 1 --2- 172.21.15.99:0/3556448256 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ea986ac100 0x55ea986bad90 secure :-1 s=READY pgs=507 cs=0 l=1 rev1=1 crypto rx=0x7f965402c7b0 tx=0x7f965403e000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=442be3d72a482dac server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:49:10.827 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.821+0000 7f96417fa640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:49:10.827 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.821+0000 7f96417fa640 1 -- 172.21.15.99:0/3556448256 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9654054070 con 0x55ea986ac100 2024-03-19T01:49:10.827 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.821+0000 7f96417fa640 1 -- 172.21.15.99:0/3556448256 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f9654050c30 con 0x55ea986ac100 2024-03-19T01:49:10.827 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.821+0000 7f9662971740 1 -- 172.21.15.99:0/3556448256 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f9648099ae0 con 0x55ea986ac100 2024-03-19T01:49:10.827 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.821+0000 7f96417fa640 1 -- 172.21.15.99:0/3556448256 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7f9654050e30 con 0x55ea986ac100 2024-03-19T01:49:10.827 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.821+0000 7f9662971740 10 client.15996 fetch_fsmap learned FSMap version 27 2024-03-19T01:49:10.827 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.821+0000 7f9662971740 1 -- 172.21.15.99:0/3556448256 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x55ea98728300 con 0x55ea986ac100 2024-03-19T01:49:10.827 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.821+0000 7f96417fa640 1 -- 172.21.15.99:0/3556448256 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f9648099ae0 con 0x55ea986ac100 2024-03-19T01:49:10.827 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.821+0000 7f96417fa640 1 -- 172.21.15.99:0/3556448256 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 27) ==== 8338+0+0 (secure 0 0 0) 0x7f96540046c0 con 0x55ea986ac100 2024-03-19T01:49:10.827 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.821+0000 7f9662971740 10 client.15996 fetch_fsmap finished waiting for FSMap version 27 2024-03-19T01:49:10.827 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.821+0000 7f9662971740 10 client.15996 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:49:10.827 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.821+0000 7f9662971740 20 client.15996 populate_metadata read hostname 'smithi099' 2024-03-19T01:49:10.828 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.821+0000 7f9662971740 1 --2- 172.21.15.99:0/3556448256 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55ea98728c40 0x55ea9872b060 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:10.828 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.821+0000 7f9662971740 4 client.15996 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:49:10.828 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.821+0000 7f9662971740 1 -- 172.21.15.99:0/3556448256 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55ea98618470 con 0x55ea98728c40 2024-03-19T01:49:10.828 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.822+0000 7f965d322640 1 --2- 172.21.15.99:0/3556448256 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55ea98728c40 0x55ea9872b060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:10.828 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.822+0000 7f96417fa640 10 client.15996 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:49:10.828 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.822+0000 7f965d322640 1 --2- 172.21.15.99:0/3556448256 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55ea98728c40 0x55ea9872b060 crc :-1 s=READY pgs=102 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=a1cad78c90f1e743 server_cookie=6792d00410e3dfa4 in_seq=0 out_seq=0 2024-03-19T01:49:10.883 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.881+0000 7f96417fa640 1 -- 172.21.15.99:0/3556448256 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x55ea98618470 con 0x55ea98728c40 2024-03-19T01:49:10.883 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.881+0000 7f96417fa640 10 client.15996 handle_command_reply: tid=0 2024-03-19T01:49:10.883 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.881+0000 7f9662971740 1 client.15996 shutdown 2024-03-19T01:49:10.883 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.881+0000 7f9662971740 1 -- 172.21.15.99:0/3556448256 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ea986ac100 msgr2=0x55ea986bad90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:49:10.883 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.881+0000 7f9662971740 1 --2- 172.21.15.99:0/3556448256 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ea986ac100 0x55ea986bad90 secure :-1 s=READY pgs=507 cs=0 l=1 rev1=1 crypto rx=0x7f965402c7b0 tx=0x7f965403e000 comp rx=0 tx=0).stop 2024-03-19T01:49:10.883 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.882+0000 7f9662971740 1 -- 172.21.15.99:0/3556448256 shutdown_connections 2024-03-19T01:49:10.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.882+0000 7f9662971740 1 --2- 172.21.15.99:0/3556448256 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55ea98728c40 0x55ea9872b060 crc :-1 s=READY pgs=102 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:10.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.882+0000 7f9662971740 1 --2- 172.21.15.99:0/3556448256 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ea986b1c60 0x55ea986bc220 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:10.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.882+0000 7f9662971740 1 --2- 172.21.15.99:0/3556448256 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ea986acaf0 0x55ea986bb2d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:10.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.882+0000 7f96417fa640 0 client.15996 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:49:10.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.882+0000 7f9662971740 1 --2- 172.21.15.99:0/3556448256 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ea986ac100 0x55ea986bad90 unknown :-1 s=CLOSED pgs=507 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:10.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.882+0000 7f9662971740 1 -- 172.21.15.99:0/3556448256 >> 172.21.15.99:0/3556448256 conn(0x55ea986a7670 msgr2=0x55ea986a6220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:49:10.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.882+0000 7f9662971740 1 -- 172.21.15.99:0/3556448256 shutdown_connections 2024-03-19T01:49:10.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.882+0000 7f9662971740 1 -- 172.21.15.99:0/3556448256 wait complete. 2024-03-19T01:49:10.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.882+0000 7f9662971740 20 client.15996 trim_cache size 0 max 16384 2024-03-19T01:49:10.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:49:10.885 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:49:10.885 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:49:10.885 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:49:10.886 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.882+0000 7f965fd87640 1 -- 172.21.15.99:0/3225198097 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f96440a28a0 msgr2=0x7f96440a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:49:10.886 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.882+0000 7f965fd87640 1 --2- 172.21.15.99:0/3225198097 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f96440a28a0 0x7f96440a4d60 secure :-1 s=READY pgs=292 cs=0 l=1 rev1=1 crypto rx=0x7f9654000c00 tx=0x7f9654005fd0 comp rx=0 tx=0).stop 2024-03-19T01:49:10.886 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.882+0000 7f965fd87640 1 -- 172.21.15.99:0/3225198097 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9658101930 msgr2=0x7f9658198040 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:49:10.886 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.882+0000 7f965fd87640 1 --2- 172.21.15.99:0/3225198097 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9658101930 0x7f9658198040 secure :-1 s=READY pgs=503 cs=0 l=1 rev1=1 crypto rx=0x7f964800e9f0 tx=0x7f964800eeb0 comp rx=0 tx=0).stop 2024-03-19T01:49:10.886 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.882+0000 7f965fd87640 1 -- 172.21.15.99:0/3225198097 shutdown_connections 2024-03-19T01:49:10.886 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.882+0000 7f965fd87640 1 --2- 172.21.15.99:0/3225198097 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f96440a28a0 0x7f96440a4d60 unknown :-1 s=CLOSED pgs=292 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:10.886 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.882+0000 7f965fd87640 1 --2- 172.21.15.99:0/3225198097 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f96581022f0 0x7f965819f0b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:10.886 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.882+0000 7f965fd87640 1 --2- 172.21.15.99:0/3225198097 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9658101930 0x7f9658198040 unknown :-1 s=CLOSED pgs=503 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:10.886 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.882+0000 7f965fd87640 1 --2- 172.21.15.99:0/3225198097 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9658100740 0x7f9658197b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:10.886 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.882+0000 7f965fd87640 1 -- 172.21.15.99:0/3225198097 >> 172.21.15.99:0/3225198097 conn(0x7f96580fbcb0 msgr2=0x7f9658068ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:49:10.886 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.882+0000 7f965fd87640 1 -- 172.21.15.99:0/3225198097 shutdown_connections 2024-03-19T01:49:10.886 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:10.882+0000 7f965fd87640 1 -- 172.21.15.99:0/3225198097 wait complete. 2024-03-19T01:49:11.692 INFO:tasks.workunit.client.0.smithi099.stdout: 1 979774 11.41 MB/sec execute 341 sec latency 65.532 ms 2024-03-19T01:49:12.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:49:11 smithi115 ceph-mon[67332]: pgmap v593: 97 pgs: 97 active+clean; 235 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 839 KiB/s rd, 2.8 MiB/s wr, 285 op/s 2024-03-19T01:49:12.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:49:11 smithi099 ceph-mon[62951]: pgmap v593: 97 pgs: 97 active+clean; 235 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 839 KiB/s rd, 2.8 MiB/s wr, 285 op/s 2024-03-19T01:49:12.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:49:11 smithi112 ceph-mon[66819]: pgmap v593: 97 pgs: 97 active+clean; 235 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 839 KiB/s rd, 2.8 MiB/s wr, 285 op/s 2024-03-19T01:49:12.692 INFO:tasks.workunit.client.0.smithi099.stdout: 1 981460 11.40 MB/sec execute 342 sec latency 37.098 ms 2024-03-19T01:49:13.692 INFO:tasks.workunit.client.0.smithi099.stdout: 1 984200 11.42 MB/sec execute 343 sec latency 21.815 ms 2024-03-19T01:49:14.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:49:13 smithi115 ceph-mon[67332]: pgmap v594: 97 pgs: 97 active+clean; 235 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 621 KiB/s rd, 2.9 MiB/s wr, 262 op/s 2024-03-19T01:49:14.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:49:13 smithi099 ceph-mon[62951]: pgmap v594: 97 pgs: 97 active+clean; 235 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 621 KiB/s rd, 2.9 MiB/s wr, 262 op/s 2024-03-19T01:49:14.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:49:13 smithi112 ceph-mon[66819]: pgmap v594: 97 pgs: 97 active+clean; 235 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 621 KiB/s rd, 2.9 MiB/s wr, 262 op/s 2024-03-19T01:49:14.692 INFO:tasks.workunit.client.0.smithi099.stdout: 1 985812 11.40 MB/sec execute 344 sec latency 40.626 ms 2024-03-19T01:49:15.692 INFO:tasks.workunit.client.0.smithi099.stdout: 1 988115 11.41 MB/sec execute 345 sec latency 27.379 ms 2024-03-19T01:49:16.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:49:15 smithi115 ceph-mon[67332]: pgmap v595: 97 pgs: 97 active+clean; 235 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 581 KiB/s rd, 2.5 MiB/s wr, 222 op/s 2024-03-19T01:49:16.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:49:15 smithi099 ceph-mon[62951]: pgmap v595: 97 pgs: 97 active+clean; 235 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 581 KiB/s rd, 2.5 MiB/s wr, 222 op/s 2024-03-19T01:49:16.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:49:15 smithi112 ceph-mon[66819]: pgmap v595: 97 pgs: 97 active+clean; 235 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 581 KiB/s rd, 2.5 MiB/s wr, 222 op/s 2024-03-19T01:49:16.693 INFO:tasks.workunit.client.0.smithi099.stdout: 1 990612 11.41 MB/sec execute 346 sec latency 35.767 ms 2024-03-19T01:49:17.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:49:17 smithi115 ceph-mon[67332]: pgmap v596: 97 pgs: 97 active+clean; 238 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 868 KiB/s rd, 3.3 MiB/s wr, 309 op/s 2024-03-19T01:49:17.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:49:17 smithi099 ceph-mon[62951]: pgmap v596: 97 pgs: 97 active+clean; 238 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 868 KiB/s rd, 3.3 MiB/s wr, 309 op/s 2024-03-19T01:49:17.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:49:17 smithi112 ceph-mon[66819]: pgmap v596: 97 pgs: 97 active+clean; 238 MiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 868 KiB/s rd, 3.3 MiB/s wr, 309 op/s 2024-03-19T01:49:17.693 INFO:tasks.workunit.client.0.smithi099.stdout: 1 992106 11.40 MB/sec execute 347 sec latency 62.187 ms 2024-03-19T01:49:18.693 INFO:tasks.workunit.client.0.smithi099.stdout: 1 994915 11.42 MB/sec execute 348 sec latency 11.319 ms 2024-03-19T01:49:19.693 INFO:tasks.workunit.client.0.smithi099.stdout: 1 996667 11.40 MB/sec execute 349 sec latency 27.590 ms 2024-03-19T01:49:20.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:49:19 smithi115 ceph-mon[67332]: pgmap v597: 97 pgs: 97 active+clean; 247 MiB data, 2.7 GiB used, 1.0 TiB / 1.0 TiB avail; 632 KiB/s rd, 3.5 MiB/s wr, 255 op/s 2024-03-19T01:49:20.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:49:19 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:49:20.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:49:19 smithi099 ceph-mon[62951]: pgmap v597: 97 pgs: 97 active+clean; 247 MiB data, 2.7 GiB used, 1.0 TiB / 1.0 TiB avail; 632 KiB/s rd, 3.5 MiB/s wr, 255 op/s 2024-03-19T01:49:20.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:49:19 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:49:20.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:49:19 smithi112 ceph-mon[66819]: pgmap v597: 97 pgs: 97 active+clean; 247 MiB data, 2.7 GiB used, 1.0 TiB / 1.0 TiB avail; 632 KiB/s rd, 3.5 MiB/s wr, 255 op/s 2024-03-19T01:49:20.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:49:19 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:49:20.693 INFO:tasks.workunit.client.0.smithi099.stdout: 1 998796 11.41 MB/sec execute 350 sec latency 23.059 ms 2024-03-19T01:49:21.693 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1001237 11.41 MB/sec execute 351 sec latency 29.417 ms 2024-03-19T01:49:21.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:49:21 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:49:21.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:49:21 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:49:21.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:49:21 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:49:21.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:49:21 smithi115 ceph-mon[67332]: pgmap v598: 97 pgs: 97 active+clean; 253 MiB data, 2.7 GiB used, 1.0 TiB / 1.0 TiB avail; 635 KiB/s rd, 4.1 MiB/s wr, 256 op/s 2024-03-19T01:49:21.840 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:49:21 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:49:21.840 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:49:21 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:49:21.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:49:21 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:49:21.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:49:21 smithi099 ceph-mon[62951]: pgmap v598: 97 pgs: 97 active+clean; 253 MiB data, 2.7 GiB used, 1.0 TiB / 1.0 TiB avail; 635 KiB/s rd, 4.1 MiB/s wr, 256 op/s 2024-03-19T01:49:21.861 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:49:21 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:49:21.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:49:21 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:49:21.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:49:21 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:49:21.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:49:21 smithi112 ceph-mon[66819]: pgmap v598: 97 pgs: 97 active+clean; 253 MiB data, 2.7 GiB used, 1.0 TiB / 1.0 TiB avail; 635 KiB/s rd, 4.1 MiB/s wr, 256 op/s 2024-03-19T01:49:22.693 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1002748 11.40 MB/sec execute 352 sec latency 76.672 ms 2024-03-19T01:49:23.693 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1005400 11.42 MB/sec execute 353 sec latency 27.867 ms 2024-03-19T01:49:23.974 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.972+0000 7f478759e640 1 -- 172.21.15.99:0/851608365 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f47881049f0 msgr2=0x7f4788106de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:49:23.974 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.972+0000 7f478759e640 1 --2- 172.21.15.99:0/851608365 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f47881049f0 0x7f4788106de0 secure :-1 s=READY pgs=508 cs=0 l=1 rev1=1 crypto rx=0x7f477c009900 tx=0x7f477c02bc30 comp rx=0 tx=0).stop 2024-03-19T01:49:23.975 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.972+0000 7f478759e640 1 -- 172.21.15.99:0/851608365 shutdown_connections 2024-03-19T01:49:23.975 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.972+0000 7f478759e640 1 --2- 172.21.15.99:0/851608365 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4788107320 0x7f4788109770 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:23.975 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.972+0000 7f478759e640 1 --2- 172.21.15.99:0/851608365 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f47881049f0 0x7f4788106de0 unknown :-1 s=CLOSED pgs=508 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:23.975 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.973+0000 7f478759e640 1 --2- 172.21.15.99:0/851608365 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f47881020c0 0x7f47881044b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:23.975 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.973+0000 7f478759e640 1 -- 172.21.15.99:0/851608365 >> 172.21.15.99:0/851608365 conn(0x7f47880fbad0 msgr2=0x7f47880fdef0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:49:23.975 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.973+0000 7f478759e640 1 -- 172.21.15.99:0/851608365 shutdown_connections 2024-03-19T01:49:23.975 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.973+0000 7f478759e640 1 -- 172.21.15.99:0/851608365 wait complete. 2024-03-19T01:49:23.975 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.973+0000 7f478759e640 1 Processor -- start 2024-03-19T01:49:23.975 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.973+0000 7f478759e640 1 -- start start 2024-03-19T01:49:23.975 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.973+0000 7f478759e640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f47881020c0 0x7f47881979a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:23.975 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.973+0000 7f478759e640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f47881049f0 0x7f4788197ee0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:23.975 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.973+0000 7f478759e640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4788107320 0x7f478819ef50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:23.976 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.973+0000 7f478759e640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f478810d2c0 con 0x7f47881049f0 2024-03-19T01:49:23.976 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.973+0000 7f478759e640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f478810d140 con 0x7f4788107320 2024-03-19T01:49:23.976 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.973+0000 7f478759e640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f478810d440 con 0x7f47881020c0 2024-03-19T01:49:23.976 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.974+0000 7f4785d9b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f47881049f0 0x7f4788197ee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:23.976 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.974+0000 7f478659c640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f47881020c0 0x7f47881979a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:23.976 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.974+0000 7f4786d9d640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4788107320 0x7f478819ef50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:23.976 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.974+0000 7f4785d9b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f47881049f0 0x7f4788197ee0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:49216/0 (socket says 172.21.15.99:49216) 2024-03-19T01:49:23.976 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.974+0000 7f4786d9d640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4788107320 0x7f478819ef50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:39960/0 (socket says 172.21.15.99:39960) 2024-03-19T01:49:23.976 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.974+0000 7f4785d9b640 1 -- 172.21.15.99:0/1044060445 learned_addr learned my addr 172.21.15.99:0/1044060445 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:49:23.976 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.974+0000 7f4786d9d640 1 -- 172.21.15.99:0/1044060445 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f47881020c0 msgr2=0x7f47881979a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:49:23.976 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.974+0000 7f4786d9d640 1 --2- 172.21.15.99:0/1044060445 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f47881020c0 0x7f47881979a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:23.976 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.974+0000 7f4786d9d640 1 -- 172.21.15.99:0/1044060445 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f47881049f0 msgr2=0x7f4788197ee0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:49:23.976 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.974+0000 7f4786d9d640 1 --2- 172.21.15.99:0/1044060445 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f47881049f0 0x7f4788197ee0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:23.976 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.974+0000 7f4786d9d640 1 -- 172.21.15.99:0/1044060445 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f478819f5c0 con 0x7f4788107320 2024-03-19T01:49:23.976 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.974+0000 7f4785d9b640 1 --2- 172.21.15.99:0/1044060445 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f47881049f0 0x7f4788197ee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:49:23.977 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.974+0000 7f4786d9d640 1 --2- 172.21.15.99:0/1044060445 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4788107320 0x7f478819ef50 secure :-1 s=READY pgs=131 cs=0 l=1 rev1=1 crypto rx=0x7f477800e970 tx=0x7f477800ee30 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:49:23.977 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.974+0000 7f47777fe640 1 -- 172.21.15.99:0/1044060445 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f477800ccc0 con 0x7f4788107320 2024-03-19T01:49:23.977 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.974+0000 7f47777fe640 1 -- 172.21.15.99:0/1044060445 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f47780035f0 con 0x7f4788107320 2024-03-19T01:49:23.977 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.974+0000 7f478759e640 1 -- 172.21.15.99:0/1044060445 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f478819f8b0 con 0x7f4788107320 2024-03-19T01:49:23.977 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.974+0000 7f47777fe640 1 -- 172.21.15.99:0/1044060445 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f477800ce60 con 0x7f4788107320 2024-03-19T01:49:23.977 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.974+0000 7f478759e640 1 -- 172.21.15.99:0/1044060445 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f478819fd80 con 0x7f4788107320 2024-03-19T01:49:23.984 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.976+0000 7f47777fe640 1 -- 172.21.15.99:0/1044060445 <== mon.2 v2:172.21.15.112:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f47780037a0 con 0x7f4788107320 2024-03-19T01:49:23.984 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.977+0000 7f47777fe640 1 --2- 172.21.15.99:0/1044060445 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f47700a27f0 0x7f47700a4cb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:23.985 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.977+0000 7f478f5ec740 1 Processor -- start 2024-03-19T01:49:23.985 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.977+0000 7f478659c640 1 --2- 172.21.15.99:0/1044060445 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f47700a27f0 0x7f47700a4cb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:23.985 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.977+0000 7f47777fe640 1 -- 172.21.15.99:0/1044060445 <== mon.2 v2:172.21.15.112:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f47780cece0 con 0x7f4788107320 2024-03-19T01:49:23.985 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.977+0000 7f478f5ec740 1 -- start start 2024-03-19T01:49:23.985 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.977+0000 7f478f5ec740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55cfe1b082e0 0x55cfe1b08700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:23.985 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.977+0000 7f478f5ec740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55cfe1b08cd0 0x55cfe1b0d7d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:23.985 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.977+0000 7f478f5ec740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55cfe1b0de40 0x55cfe1b102d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:23.985 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.977+0000 7f478f5ec740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55cfe1a6cbe0 con 0x55cfe1b08cd0 2024-03-19T01:49:23.985 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.977+0000 7f478f5ec740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55cfe1a15960 con 0x55cfe1b0de40 2024-03-19T01:49:23.985 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.977+0000 7f478f5ec740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55cfe1a68c80 con 0x55cfe1b082e0 2024-03-19T01:49:23.985 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.977+0000 7f478659c640 1 --2- 172.21.15.99:0/1044060445 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f47700a27f0 0x7f47700a4cb0 secure :-1 s=READY pgs=293 cs=0 l=1 rev1=1 crypto rx=0x7f4780005fd0 tx=0x7f4780009500 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:49:23.985 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.977+0000 7f4785d9b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55cfe1b08cd0 0x55cfe1b0d7d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:23.985 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.977+0000 7f478659c640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55cfe1b082e0 0x55cfe1b08700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:23.985 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.977+0000 7f478659c640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55cfe1b082e0 0x55cfe1b08700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:39534/0 (socket says 172.21.15.99:39534) 2024-03-19T01:49:23.986 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.977+0000 7f4785d9b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55cfe1b08cd0 0x55cfe1b0d7d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:49226/0 (socket says 172.21.15.99:49226) 2024-03-19T01:49:23.986 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.977+0000 7f478659c640 1 -- 172.21.15.99:0/244613908 learned_addr learned my addr 172.21.15.99:0/244613908 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:49:23.986 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.977+0000 7f4786d9d640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55cfe1b0de40 0x55cfe1b102d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:23.986 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.978+0000 7f478659c640 1 -- 172.21.15.99:0/244613908 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55cfe1b0de40 msgr2=0x55cfe1b102d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:49:23.986 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.978+0000 7f478659c640 1 --2- 172.21.15.99:0/244613908 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55cfe1b0de40 0x55cfe1b102d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:23.986 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.978+0000 7f478659c640 1 -- 172.21.15.99:0/244613908 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55cfe1b08cd0 msgr2=0x55cfe1b0d7d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:49:23.986 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.978+0000 7f478659c640 1 --2- 172.21.15.99:0/244613908 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55cfe1b08cd0 0x55cfe1b0d7d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:23.986 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.978+0000 7f478659c640 1 -- 172.21.15.99:0/244613908 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55cfe1a0cd20 con 0x55cfe1b082e0 2024-03-19T01:49:23.986 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.978+0000 7f4785d9b640 1 --2- 172.21.15.99:0/244613908 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55cfe1b08cd0 0x55cfe1b0d7d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:49:23.986 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.978+0000 7f478659c640 1 --2- 172.21.15.99:0/244613908 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55cfe1b082e0 0x55cfe1b08700 secure :-1 s=READY pgs=101 cs=0 l=1 rev1=1 crypto rx=0x7f478000e750 tx=0x7f4780014000 comp rx=0 tx=0).ready entity=mon.1 client_cookie=ce419fa9b3af478d server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:49:23.986 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.978+0000 7f4774ff9640 1 -- 172.21.15.99:0/244613908 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f478001d630 con 0x55cfe1b082e0 2024-03-19T01:49:23.986 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.978+0000 7f4774ff9640 1 -- 172.21.15.99:0/244613908 <== mon.1 v2:172.21.15.115:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f478001dcb0 con 0x55cfe1b082e0 2024-03-19T01:49:23.986 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.978+0000 7f4774ff9640 1 -- 172.21.15.99:0/244613908 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f478001ed80 con 0x55cfe1b082e0 2024-03-19T01:49:23.987 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.979+0000 7f478f5ec740 1 -- 172.21.15.99:0/244613908 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55cfe1b082e0 msgr2=0x55cfe1b08700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:49:23.987 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.979+0000 7f478f5ec740 1 --2- 172.21.15.99:0/244613908 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55cfe1b082e0 0x55cfe1b08700 secure :-1 s=READY pgs=101 cs=0 l=1 rev1=1 crypto rx=0x7f478000e750 tx=0x7f4780014000 comp rx=0 tx=0).stop 2024-03-19T01:49:23.987 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.979+0000 7f478f5ec740 1 -- 172.21.15.99:0/244613908 shutdown_connections 2024-03-19T01:49:23.987 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.979+0000 7f478f5ec740 1 --2- 172.21.15.99:0/244613908 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55cfe1b0de40 0x55cfe1b102d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:23.987 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.979+0000 7f478f5ec740 1 --2- 172.21.15.99:0/244613908 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55cfe1b08cd0 0x55cfe1b0d7d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:23.987 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.979+0000 7f478f5ec740 1 --2- 172.21.15.99:0/244613908 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55cfe1b082e0 0x55cfe1b08700 unknown :-1 s=CLOSED pgs=101 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:23.987 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.979+0000 7f478f5ec740 1 -- 172.21.15.99:0/244613908 >> 172.21.15.99:0/244613908 conn(0x55cfe1b03850 msgr2=0x55cfe1b03c50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:49:23.987 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.979+0000 7f478f5ec740 1 -- 172.21.15.99:0/244613908 shutdown_connections 2024-03-19T01:49:23.987 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.979+0000 7f478f5ec740 1 -- 172.21.15.99:0/244613908 wait complete. 2024-03-19T01:49:23.987 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.979+0000 7f478f5ec740 1 -- start start 2024-03-19T01:49:23.987 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.979+0000 7f478f5ec740 1 Processor -- start 2024-03-19T01:49:23.987 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.979+0000 7f478f5ec740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55cfe1b082e0 0x55cfe1b12320 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:23.987 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.979+0000 7f478f5ec740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55cfe1b08cd0 0x55cfe1b12860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:23.987 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.979+0000 7f478f5ec740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55cfe1b0de40 0x55cfe1b19830 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:23.987 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.979+0000 7f478f5ec740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55cfe1b10df0 con 0x55cfe1b0de40 2024-03-19T01:49:23.988 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.979+0000 7f478f5ec740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55cfe1a28db0 con 0x55cfe1b082e0 2024-03-19T01:49:23.988 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.979+0000 7f478f5ec740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55cfe1b10f70 con 0x55cfe1b08cd0 2024-03-19T01:49:23.988 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.979+0000 7f478f5ec740 10 client.0 _ll_register_callbacks cb 0x55cfe1a2de70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:49:23.988 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.980+0000 7f478659c640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55cfe1b082e0 0x55cfe1b12320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:23.988 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.980+0000 7f4785d9b640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55cfe1b08cd0 0x55cfe1b12860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:23.988 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.980+0000 7f478659c640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55cfe1b082e0 0x55cfe1b12320 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:39972/0 (socket says 172.21.15.99:39972) 2024-03-19T01:49:23.988 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.980+0000 7f478659c640 1 -- 172.21.15.99:0/2173513078 learned_addr learned my addr 172.21.15.99:0/2173513078 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:49:23.988 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.980+0000 7f4786d9d640 1 --2- 172.21.15.99:0/2173513078 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55cfe1b0de40 0x55cfe1b19830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:23.988 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.980+0000 7f478659c640 1 -- 172.21.15.99:0/2173513078 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55cfe1b08cd0 msgr2=0x55cfe1b12860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:49:23.988 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.980+0000 7f478659c640 1 --2- 172.21.15.99:0/2173513078 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55cfe1b08cd0 0x55cfe1b12860 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:23.988 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.980+0000 7f478659c640 1 -- 172.21.15.99:0/2173513078 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55cfe1b0de40 msgr2=0x55cfe1b19830 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:49:23.988 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.980+0000 7f478659c640 1 --2- 172.21.15.99:0/2173513078 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55cfe1b0de40 0x55cfe1b19830 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:23.988 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.980+0000 7f478659c640 1 -- 172.21.15.99:0/2173513078 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55cfe1a27c60 con 0x55cfe1b082e0 2024-03-19T01:49:23.989 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.980+0000 7f4785d9b640 1 --2- 172.21.15.99:0/2173513078 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55cfe1b08cd0 0x55cfe1b12860 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:49:23.989 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.980+0000 7f4786d9d640 1 --2- 172.21.15.99:0/2173513078 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55cfe1b0de40 0x55cfe1b19830 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:49:23.989 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.980+0000 7f478659c640 1 --2- 172.21.15.99:0/2173513078 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55cfe1b082e0 0x55cfe1b12320 secure :-1 s=READY pgs=132 cs=0 l=1 rev1=1 crypto rx=0x7f4780028040 tx=0x7f478001ef90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=f5d73a67331da6ba server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:49:23.989 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.980+0000 7f476dffb640 10 client.0 ms_handle_connect on v2:172.21.15.112:3300/0 2024-03-19T01:49:23.989 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.980+0000 7f476dffb640 1 -- 172.21.15.99:0/2173513078 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4780033060 con 0x55cfe1b082e0 2024-03-19T01:49:23.989 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.980+0000 7f476dffb640 1 -- 172.21.15.99:0/2173513078 <== mon.2 v2:172.21.15.112:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f4780004a40 con 0x55cfe1b082e0 2024-03-19T01:49:23.989 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.980+0000 7f478f5ec740 1 -- 172.21.15.99:0/2173513078 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f478001dcb0 con 0x55cfe1b082e0 2024-03-19T01:49:23.989 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.980+0000 7f476dffb640 1 -- 172.21.15.99:0/2173513078 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7f4780004c40 con 0x55cfe1b082e0 2024-03-19T01:49:23.989 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.980+0000 7f478f5ec740 10 client.25781 fetch_fsmap learned FSMap version 27 2024-03-19T01:49:23.989 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.980+0000 7f478f5ec740 1 -- 172.21.15.99:0/2173513078 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x55cfe1781210 con 0x55cfe1b082e0 2024-03-19T01:49:23.989 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.981+0000 7f476dffb640 1 -- 172.21.15.99:0/2173513078 <== mon.2 v2:172.21.15.112:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f478001dcb0 con 0x55cfe1b082e0 2024-03-19T01:49:23.989 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.981+0000 7f476dffb640 1 -- 172.21.15.99:0/2173513078 <== mon.2 v2:172.21.15.112:3300/0 5 ==== fsmap(e 27) ==== 8338+0+0 (secure 0 0 0) 0x7f4780004040 con 0x55cfe1b082e0 2024-03-19T01:49:23.989 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.981+0000 7f478f5ec740 10 client.25781 fetch_fsmap finished waiting for FSMap version 27 2024-03-19T01:49:23.989 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.981+0000 7f478f5ec740 10 client.25781 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:49:23.989 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.981+0000 7f478f5ec740 20 client.25781 populate_metadata read hostname 'smithi099' 2024-03-19T01:49:23.990 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.981+0000 7f478f5ec740 1 --2- 172.21.15.99:0/2173513078 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55cfe1b1de30 0x55cfe1b20220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:23.990 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.981+0000 7f478f5ec740 4 client.25781 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:49:23.990 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.981+0000 7f478f5ec740 1 -- 172.21.15.99:0/2173513078 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55cfe1a7a120 con 0x55cfe1b1de30 2024-03-19T01:49:23.990 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.981+0000 7f4785d9b640 1 --2- 172.21.15.99:0/2173513078 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55cfe1b1de30 0x55cfe1b20220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:23.990 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.981+0000 7f4785d9b640 1 --2- 172.21.15.99:0/2173513078 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55cfe1b1de30 0x55cfe1b20220 crc :-1 s=READY pgs=103 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=5cb5493690dfb498 server_cookie=a21eef4531642c6e in_seq=0 out_seq=0 2024-03-19T01:49:23.990 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.981+0000 7f476dffb640 10 client.25781 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:49:23.990 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.982+0000 7f476dffb640 1 -- 172.21.15.99:0/2173513078 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x55cfe1a7a120 con 0x55cfe1b1de30 2024-03-19T01:49:23.990 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.982+0000 7f476dffb640 10 client.25781 handle_command_reply: tid=0 2024-03-19T01:49:23.990 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.982+0000 7f478f5ec740 1 client.25781 shutdown 2024-03-19T01:49:23.990 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.982+0000 7f478f5ec740 1 -- 172.21.15.99:0/2173513078 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55cfe1b082e0 msgr2=0x55cfe1b12320 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:49:23.990 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.982+0000 7f478f5ec740 1 --2- 172.21.15.99:0/2173513078 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55cfe1b082e0 0x55cfe1b12320 secure :-1 s=READY pgs=132 cs=0 l=1 rev1=1 crypto rx=0x7f4780028040 tx=0x7f478001ef90 comp rx=0 tx=0).stop 2024-03-19T01:49:23.990 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.982+0000 7f478f5ec740 1 -- 172.21.15.99:0/2173513078 shutdown_connections 2024-03-19T01:49:23.990 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.982+0000 7f478f5ec740 1 --2- 172.21.15.99:0/2173513078 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55cfe1b1de30 0x55cfe1b20220 crc :-1 s=READY pgs=103 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:23.990 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.982+0000 7f478f5ec740 1 --2- 172.21.15.99:0/2173513078 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55cfe1b0de40 0x55cfe1b19830 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:23.991 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.982+0000 7f476dffb640 0 client.25781 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:49:23.991 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.982+0000 7f478f5ec740 1 --2- 172.21.15.99:0/2173513078 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55cfe1b08cd0 0x55cfe1b12860 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:23.991 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.982+0000 7f478f5ec740 1 --2- 172.21.15.99:0/2173513078 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55cfe1b082e0 0x55cfe1b12320 unknown :-1 s=CLOSED pgs=132 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:23.991 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.983+0000 7f478f5ec740 1 -- 172.21.15.99:0/2173513078 >> 172.21.15.99:0/2173513078 conn(0x55cfe1b03850 msgr2=0x55cfe1b02ab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:49:23.991 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.983+0000 7f478f5ec740 1 -- 172.21.15.99:0/2173513078 shutdown_connections 2024-03-19T01:49:23.991 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.983+0000 7f478f5ec740 1 -- 172.21.15.99:0/2173513078 wait complete. 2024-03-19T01:49:23.991 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.983+0000 7f478f5ec740 20 client.25781 trim_cache size 0 max 16384 2024-03-19T01:49:24.002 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.997+0000 7f478f5ec740 1 Processor -- start 2024-03-19T01:49:24.003 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.998+0000 7f478f5ec740 1 -- start start 2024-03-19T01:49:24.003 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.998+0000 7f478f5ec740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55cfe1b082e0 0x55cfe1b127c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:24.003 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.998+0000 7f478f5ec740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55cfe1b08cd0 0x55cfe1b197f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:24.003 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.998+0000 7f478f5ec740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55cfe1b0de40 0x55cfe1b02930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:24.003 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.998+0000 7f478f5ec740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f47881ab530 con 0x55cfe1b08cd0 2024-03-19T01:49:24.003 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.998+0000 7f478f5ec740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f47881a8e00 con 0x55cfe1b082e0 2024-03-19T01:49:24.004 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.998+0000 7f478f5ec740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f47881a6c50 con 0x55cfe1b0de40 2024-03-19T01:49:24.004 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.998+0000 7f4785d9b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55cfe1b08cd0 0x55cfe1b197f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:24.004 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.998+0000 7f478659c640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55cfe1b082e0 0x55cfe1b127c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:24.004 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.998+0000 7f4785d9b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55cfe1b08cd0 0x55cfe1b197f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:49232/0 (socket says 172.21.15.99:49232) 2024-03-19T01:49:24.004 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.998+0000 7f4786d9d640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55cfe1b0de40 0x55cfe1b02930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:24.004 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.998+0000 7f4785d9b640 1 -- 172.21.15.99:0/2236078440 learned_addr learned my addr 172.21.15.99:0/2236078440 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:49:24.004 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.998+0000 7f4785d9b640 1 -- 172.21.15.99:0/2236078440 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55cfe1b0de40 msgr2=0x55cfe1b02930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:49:24.004 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.998+0000 7f4785d9b640 1 --2- 172.21.15.99:0/2236078440 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55cfe1b0de40 0x55cfe1b02930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:24.004 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.998+0000 7f4785d9b640 1 -- 172.21.15.99:0/2236078440 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55cfe1b082e0 msgr2=0x55cfe1b127c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:49:24.004 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.998+0000 7f4785d9b640 1 --2- 172.21.15.99:0/2236078440 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55cfe1b082e0 0x55cfe1b127c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:24.004 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.998+0000 7f4785d9b640 1 -- 172.21.15.99:0/2236078440 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55cfe184d620 con 0x55cfe1b08cd0 2024-03-19T01:49:24.004 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.998+0000 7f478659c640 1 --2- 172.21.15.99:0/2236078440 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55cfe1b082e0 0x55cfe1b127c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:49:24.004 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.998+0000 7f4785d9b640 1 --2- 172.21.15.99:0/2236078440 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55cfe1b08cd0 0x55cfe1b197f0 secure :-1 s=READY pgs=509 cs=0 l=1 rev1=1 crypto rx=0x7f477c02c8e0 tx=0x7f477c005b50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=159e5d80c9a0621e server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:49:24.005 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.998+0000 7f476ffff640 1 -- 172.21.15.99:0/2236078440 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f477c04c070 con 0x55cfe1b08cd0 2024-03-19T01:49:24.005 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.998+0000 7f476ffff640 1 -- 172.21.15.99:0/2236078440 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f477c048b80 con 0x55cfe1b08cd0 2024-03-19T01:49:24.005 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.998+0000 7f476ffff640 1 -- 172.21.15.99:0/2236078440 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f477c004ed0 con 0x55cfe1b08cd0 2024-03-19T01:49:24.005 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.999+0000 7f478f5ec740 1 -- 172.21.15.99:0/2236078440 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55cfe1b08cd0 msgr2=0x55cfe1b197f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:49:24.005 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.999+0000 7f478f5ec740 1 --2- 172.21.15.99:0/2236078440 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55cfe1b08cd0 0x55cfe1b197f0 secure :-1 s=READY pgs=509 cs=0 l=1 rev1=1 crypto rx=0x7f477c02c8e0 tx=0x7f477c005b50 comp rx=0 tx=0).stop 2024-03-19T01:49:24.005 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.999+0000 7f478f5ec740 1 -- 172.21.15.99:0/2236078440 shutdown_connections 2024-03-19T01:49:24.005 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.999+0000 7f478f5ec740 1 --2- 172.21.15.99:0/2236078440 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55cfe1b0de40 0x55cfe1b02930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:24.005 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.999+0000 7f478f5ec740 1 --2- 172.21.15.99:0/2236078440 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55cfe1b08cd0 0x55cfe1b197f0 unknown :-1 s=CLOSED pgs=509 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:24.005 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.999+0000 7f478f5ec740 1 --2- 172.21.15.99:0/2236078440 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55cfe1b082e0 0x55cfe1b127c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:24.005 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.999+0000 7f478f5ec740 1 -- 172.21.15.99:0/2236078440 >> 172.21.15.99:0/2236078440 conn(0x55cfe1b03850 msgr2=0x55cfe1b02220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:49:24.005 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.999+0000 7f478f5ec740 1 -- 172.21.15.99:0/2236078440 shutdown_connections 2024-03-19T01:49:24.005 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.999+0000 7f478f5ec740 1 -- 172.21.15.99:0/2236078440 wait complete. 2024-03-19T01:49:24.005 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.999+0000 7f478f5ec740 1 -- start start 2024-03-19T01:49:24.005 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:23.999+0000 7f478f5ec740 1 Processor -- start 2024-03-19T01:49:24.006 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.000+0000 7f478f5ec740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55cfe1b082e0 0x55cfe1b16fc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:24.006 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.000+0000 7f478f5ec740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55cfe1b08cd0 0x55cfe1b17500 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:24.006 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.000+0000 7f478f5ec740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55cfe1b0de40 0x55cfe1b18510 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:24.006 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.000+0000 7f478f5ec740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f47881a72c0 con 0x55cfe1b08cd0 2024-03-19T01:49:24.006 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.000+0000 7f478f5ec740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f47881a8730 con 0x55cfe1b0de40 2024-03-19T01:49:24.006 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.000+0000 7f478f5ec740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f47881a7990 con 0x55cfe1b082e0 2024-03-19T01:49:24.006 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.000+0000 7f478f5ec740 10 client.0 _ll_register_callbacks cb 0x7f47800093a0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:49:24.006 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.000+0000 7f4785d9b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55cfe1b08cd0 0x55cfe1b17500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:24.006 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.000+0000 7f4785d9b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55cfe1b08cd0 0x55cfe1b17500 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:49242/0 (socket says 172.21.15.99:49242) 2024-03-19T01:49:24.006 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.000+0000 7f478659c640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55cfe1b082e0 0x55cfe1b16fc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:24.006 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.000+0000 7f4785d9b640 1 -- 172.21.15.99:0/1809794653 learned_addr learned my addr 172.21.15.99:0/1809794653 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:49:24.006 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.000+0000 7f4786d9d640 1 --2- 172.21.15.99:0/1809794653 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55cfe1b0de40 0x55cfe1b18510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:24.006 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.000+0000 7f4785d9b640 1 -- 172.21.15.99:0/1809794653 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55cfe1b082e0 msgr2=0x55cfe1b16fc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:49:24.007 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.000+0000 7f4785d9b640 1 --2- 172.21.15.99:0/1809794653 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55cfe1b082e0 0x55cfe1b16fc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:24.007 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.000+0000 7f4785d9b640 1 -- 172.21.15.99:0/1809794653 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55cfe1b0de40 msgr2=0x55cfe1b18510 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:49:24.007 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.000+0000 7f4785d9b640 1 --2- 172.21.15.99:0/1809794653 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55cfe1b0de40 0x55cfe1b18510 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:24.007 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.000+0000 7f4785d9b640 1 -- 172.21.15.99:0/1809794653 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55cfe1b18b00 con 0x55cfe1b08cd0 2024-03-19T01:49:24.007 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.000+0000 7f4786d9d640 1 --2- 172.21.15.99:0/1809794653 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55cfe1b0de40 0x55cfe1b18510 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-19T01:49:24.007 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.000+0000 7f4785d9b640 1 --2- 172.21.15.99:0/1809794653 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55cfe1b08cd0 0x55cfe1b17500 secure :-1 s=READY pgs=510 cs=0 l=1 rev1=1 crypto rx=0x7f477c0096c0 tx=0x7f477c004740 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1f1e20ff6998a9ed server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:49:24.007 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.000+0000 7f476dffb640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:49:24.007 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.000+0000 7f476dffb640 1 -- 172.21.15.99:0/1809794653 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f477c055030 con 0x55cfe1b08cd0 2024-03-19T01:49:24.007 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.000+0000 7f476dffb640 1 -- 172.21.15.99:0/1809794653 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f477c048410 con 0x55cfe1b08cd0 2024-03-19T01:49:24.007 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.000+0000 7f478f5ec740 1 -- 172.21.15.99:0/1809794653 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f477c048b80 con 0x55cfe1b08cd0 2024-03-19T01:49:24.008 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.000+0000 7f476dffb640 1 -- 172.21.15.99:0/1809794653 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7f477c04c050 con 0x55cfe1b08cd0 2024-03-19T01:49:24.008 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.000+0000 7f478f5ec740 10 client.16014 fetch_fsmap learned FSMap version 27 2024-03-19T01:49:24.008 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.000+0000 7f478f5ec740 1 -- 172.21.15.99:0/1809794653 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x55cfe1b84100 con 0x55cfe1b08cd0 2024-03-19T01:49:24.008 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.001+0000 7f476dffb640 1 -- 172.21.15.99:0/1809794653 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f477c048b80 con 0x55cfe1b08cd0 2024-03-19T01:49:24.008 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.001+0000 7f476dffb640 1 -- 172.21.15.99:0/1809794653 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 27) ==== 8338+0+0 (secure 0 0 0) 0x7f477c047070 con 0x55cfe1b08cd0 2024-03-19T01:49:24.008 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.001+0000 7f478f5ec740 10 client.16014 fetch_fsmap finished waiting for FSMap version 27 2024-03-19T01:49:24.008 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.001+0000 7f478f5ec740 10 client.16014 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:49:24.008 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.001+0000 7f478f5ec740 20 client.16014 populate_metadata read hostname 'smithi099' 2024-03-19T01:49:24.008 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.001+0000 7f478f5ec740 1 --2- 172.21.15.99:0/1809794653 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55cfe1b84ad0 0x55cfe1b86ef0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:24.008 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.001+0000 7f478f5ec740 4 client.16014 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:49:24.008 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.001+0000 7f478f5ec740 1 -- 172.21.15.99:0/1809794653 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55cfe1a74470 con 0x55cfe1b84ad0 2024-03-19T01:49:24.008 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.001+0000 7f478659c640 1 --2- 172.21.15.99:0/1809794653 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55cfe1b84ad0 0x55cfe1b86ef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:24.008 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.001+0000 7f478659c640 1 --2- 172.21.15.99:0/1809794653 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55cfe1b84ad0 0x55cfe1b86ef0 crc :-1 s=READY pgs=104 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=43213325e4c644cb server_cookie=5cacb77c2bd32526 in_seq=0 out_seq=0 2024-03-19T01:49:24.009 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.001+0000 7f476dffb640 10 client.16014 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:49:24.062 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.061+0000 7f476dffb640 1 -- 172.21.15.99:0/1809794653 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x55cfe1a74470 con 0x55cfe1b84ad0 2024-03-19T01:49:24.063 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.061+0000 7f476dffb640 10 client.16014 handle_command_reply: tid=0 2024-03-19T01:49:24.063 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.062+0000 7f478f5ec740 1 client.16014 shutdown 2024-03-19T01:49:24.063 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.062+0000 7f478f5ec740 1 -- 172.21.15.99:0/1809794653 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55cfe1b08cd0 msgr2=0x55cfe1b17500 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:49:24.063 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.062+0000 7f478f5ec740 1 --2- 172.21.15.99:0/1809794653 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55cfe1b08cd0 0x55cfe1b17500 secure :-1 s=READY pgs=510 cs=0 l=1 rev1=1 crypto rx=0x7f477c0096c0 tx=0x7f477c004740 comp rx=0 tx=0).stop 2024-03-19T01:49:24.063 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.062+0000 7f478f5ec740 1 -- 172.21.15.99:0/1809794653 shutdown_connections 2024-03-19T01:49:24.063 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.062+0000 7f478f5ec740 1 --2- 172.21.15.99:0/1809794653 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55cfe1b84ad0 0x55cfe1b86ef0 crc :-1 s=READY pgs=104 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:24.063 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.062+0000 7f478f5ec740 1 --2- 172.21.15.99:0/1809794653 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55cfe1b0de40 0x55cfe1b18510 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:24.063 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.062+0000 7f476dffb640 0 client.16014 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:49:24.063 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.062+0000 7f478f5ec740 1 --2- 172.21.15.99:0/1809794653 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55cfe1b08cd0 0x55cfe1b17500 unknown :-1 s=CLOSED pgs=510 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:24.063 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.062+0000 7f478f5ec740 1 --2- 172.21.15.99:0/1809794653 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55cfe1b082e0 0x55cfe1b16fc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:24.063 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.062+0000 7f478f5ec740 1 -- 172.21.15.99:0/1809794653 >> 172.21.15.99:0/1809794653 conn(0x55cfe1b03850 msgr2=0x55cfe1b12640 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:49:24.063 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.062+0000 7f478f5ec740 1 -- 172.21.15.99:0/1809794653 shutdown_connections 2024-03-19T01:49:24.063 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.062+0000 7f478f5ec740 1 -- 172.21.15.99:0/1809794653 wait complete. 2024-03-19T01:49:24.063 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.062+0000 7f478f5ec740 20 client.16014 trim_cache size 0 max 16384 2024-03-19T01:49:24.064 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:49:24.064 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:49:24.064 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:49:24.064 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:49:24.065 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.063+0000 7f478759e640 1 -- 172.21.15.99:0/1044060445 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f47700a27f0 msgr2=0x7f47700a4cb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:49:24.065 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.063+0000 7f478759e640 1 --2- 172.21.15.99:0/1044060445 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f47700a27f0 0x7f47700a4cb0 secure :-1 s=READY pgs=293 cs=0 l=1 rev1=1 crypto rx=0x7f4780005fd0 tx=0x7f4780009500 comp rx=0 tx=0).stop 2024-03-19T01:49:24.065 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.063+0000 7f478759e640 1 -- 172.21.15.99:0/1044060445 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4788107320 msgr2=0x7f478819ef50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:49:24.065 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.063+0000 7f478759e640 1 --2- 172.21.15.99:0/1044060445 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4788107320 0x7f478819ef50 secure :-1 s=READY pgs=131 cs=0 l=1 rev1=1 crypto rx=0x7f477800e970 tx=0x7f477800ee30 comp rx=0 tx=0).stop 2024-03-19T01:49:24.065 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.063+0000 7f478759e640 1 -- 172.21.15.99:0/1044060445 shutdown_connections 2024-03-19T01:49:24.065 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.063+0000 7f478759e640 1 --2- 172.21.15.99:0/1044060445 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f47700a27f0 0x7f47700a4cb0 unknown :-1 s=CLOSED pgs=293 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:24.065 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.063+0000 7f478759e640 1 --2- 172.21.15.99:0/1044060445 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4788107320 0x7f478819ef50 unknown :-1 s=CLOSED pgs=131 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:24.065 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.063+0000 7f478759e640 1 --2- 172.21.15.99:0/1044060445 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f47881049f0 0x7f4788197ee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:24.065 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.063+0000 7f478759e640 1 --2- 172.21.15.99:0/1044060445 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f47881020c0 0x7f47881979a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:24.066 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.063+0000 7f478759e640 1 -- 172.21.15.99:0/1044060445 >> 172.21.15.99:0/1044060445 conn(0x7f47880fbad0 msgr2=0x7f4788102a00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:49:24.066 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.063+0000 7f478759e640 1 -- 172.21.15.99:0/1044060445 shutdown_connections 2024-03-19T01:49:24.066 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:24.063+0000 7f478759e640 1 -- 172.21.15.99:0/1044060445 wait complete. 2024-03-19T01:49:24.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:49:23 smithi115 ceph-mon[67332]: pgmap v599: 97 pgs: 97 active+clean; 262 MiB data, 2.7 GiB used, 1.0 TiB / 1.0 TiB avail; 730 KiB/s rd, 4.8 MiB/s wr, 270 op/s 2024-03-19T01:49:24.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:49:23 smithi099 ceph-mon[62951]: pgmap v599: 97 pgs: 97 active+clean; 262 MiB data, 2.7 GiB used, 1.0 TiB / 1.0 TiB avail; 730 KiB/s rd, 4.8 MiB/s wr, 270 op/s 2024-03-19T01:49:24.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:49:23 smithi112 ceph-mon[66819]: pgmap v599: 97 pgs: 97 active+clean; 262 MiB data, 2.7 GiB used, 1.0 TiB / 1.0 TiB avail; 730 KiB/s rd, 4.8 MiB/s wr, 270 op/s 2024-03-19T01:49:24.693 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1006886 11.40 MB/sec execute 354 sec latency 44.193 ms 2024-03-19T01:49:25.693 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1009267 11.41 MB/sec execute 355 sec latency 21.295 ms 2024-03-19T01:49:26.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:49:25 smithi115 ceph-mon[67332]: pgmap v600: 97 pgs: 97 active+clean; 263 MiB data, 2.7 GiB used, 1.0 TiB / 1.0 TiB avail; 724 KiB/s rd, 4.4 MiB/s wr, 243 op/s 2024-03-19T01:49:26.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:49:25 smithi099 ceph-mon[62951]: pgmap v600: 97 pgs: 97 active+clean; 263 MiB data, 2.7 GiB used, 1.0 TiB / 1.0 TiB avail; 724 KiB/s rd, 4.4 MiB/s wr, 243 op/s 2024-03-19T01:49:26.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:49:25 smithi112 ceph-mon[66819]: pgmap v600: 97 pgs: 97 active+clean; 263 MiB data, 2.7 GiB used, 1.0 TiB / 1.0 TiB avail; 724 KiB/s rd, 4.4 MiB/s wr, 243 op/s 2024-03-19T01:49:26.693 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1011336 11.40 MB/sec execute 356 sec latency 33.686 ms 2024-03-19T01:49:27.693 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1012974 11.40 MB/sec execute 357 sec latency 83.241 ms 2024-03-19T01:49:28.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:49:27 smithi115 ceph-mon[67332]: pgmap v601: 97 pgs: 97 active+clean; 258 MiB data, 2.7 GiB used, 1.0 TiB / 1.0 TiB avail; 830 KiB/s rd, 4.8 MiB/s wr, 290 op/s 2024-03-19T01:49:28.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:49:27 smithi099 ceph-mon[62951]: pgmap v601: 97 pgs: 97 active+clean; 258 MiB data, 2.7 GiB used, 1.0 TiB / 1.0 TiB avail; 830 KiB/s rd, 4.8 MiB/s wr, 290 op/s 2024-03-19T01:49:28.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:49:27 smithi112 ceph-mon[66819]: pgmap v601: 97 pgs: 97 active+clean; 258 MiB data, 2.7 GiB used, 1.0 TiB / 1.0 TiB avail; 830 KiB/s rd, 4.8 MiB/s wr, 290 op/s 2024-03-19T01:49:28.693 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1015108 11.39 MB/sec execute 358 sec latency 31.239 ms 2024-03-19T01:49:29.693 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1016986 11.39 MB/sec execute 359 sec latency 33.873 ms 2024-03-19T01:49:30.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:49:29 smithi115 ceph-mon[67332]: pgmap v602: 97 pgs: 97 active+clean; 260 MiB data, 2.7 GiB used, 1.0 TiB / 1.0 TiB avail; 817 KiB/s rd, 4.4 MiB/s wr, 270 op/s 2024-03-19T01:49:30.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:49:29 smithi099 ceph-mon[62951]: pgmap v602: 97 pgs: 97 active+clean; 260 MiB data, 2.7 GiB used, 1.0 TiB / 1.0 TiB avail; 817 KiB/s rd, 4.4 MiB/s wr, 270 op/s 2024-03-19T01:49:30.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:49:29 smithi112 ceph-mon[66819]: pgmap v602: 97 pgs: 97 active+clean; 260 MiB data, 2.7 GiB used, 1.0 TiB / 1.0 TiB avail; 817 KiB/s rd, 4.4 MiB/s wr, 270 op/s 2024-03-19T01:49:30.693 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1019732 11.41 MB/sec execute 360 sec latency 17.101 ms 2024-03-19T01:49:31.693 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1021555 11.40 MB/sec execute 361 sec latency 28.203 ms 2024-03-19T01:49:32.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:49:31 smithi115 ceph-mon[67332]: pgmap v603: 97 pgs: 97 active+clean; 260 MiB data, 2.7 GiB used, 1.0 TiB / 1.0 TiB avail; 759 KiB/s rd, 3.7 MiB/s wr, 251 op/s 2024-03-19T01:49:32.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:49:31 smithi099 ceph-mon[62951]: pgmap v603: 97 pgs: 97 active+clean; 260 MiB data, 2.7 GiB used, 1.0 TiB / 1.0 TiB avail; 759 KiB/s rd, 3.7 MiB/s wr, 251 op/s 2024-03-19T01:49:32.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:49:31 smithi112 ceph-mon[66819]: pgmap v603: 97 pgs: 97 active+clean; 260 MiB data, 2.7 GiB used, 1.0 TiB / 1.0 TiB avail; 759 KiB/s rd, 3.7 MiB/s wr, 251 op/s 2024-03-19T01:49:32.693 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1023535 11.40 MB/sec execute 362 sec latency 71.792 ms 2024-03-19T01:49:33.693 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1025742 11.39 MB/sec execute 363 sec latency 39.610 ms 2024-03-19T01:49:34.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:49:33 smithi115 ceph-mon[67332]: pgmap v604: 97 pgs: 97 active+clean; 262 MiB data, 2.8 GiB used, 1.0 TiB / 1.0 TiB avail; 978 KiB/s rd, 3.5 MiB/s wr, 296 op/s 2024-03-19T01:49:34.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:49:33 smithi099 ceph-mon[62951]: pgmap v604: 97 pgs: 97 active+clean; 262 MiB data, 2.8 GiB used, 1.0 TiB / 1.0 TiB avail; 978 KiB/s rd, 3.5 MiB/s wr, 296 op/s 2024-03-19T01:49:34.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:49:33 smithi112 ceph-mon[66819]: pgmap v604: 97 pgs: 97 active+clean; 262 MiB data, 2.8 GiB used, 1.0 TiB / 1.0 TiB avail; 978 KiB/s rd, 3.5 MiB/s wr, 296 op/s 2024-03-19T01:49:34.693 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1027743 11.40 MB/sec execute 364 sec latency 31.199 ms 2024-03-19T01:49:35.693 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1030440 11.41 MB/sec execute 365 sec latency 20.229 ms 2024-03-19T01:49:36.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:49:35 smithi115 ceph-mon[67332]: pgmap v605: 97 pgs: 97 active+clean; 262 MiB data, 2.8 GiB used, 1.0 TiB / 1.0 TiB avail; 701 KiB/s rd, 2.2 MiB/s wr, 225 op/s 2024-03-19T01:49:36.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:49:35 smithi099 ceph-mon[62951]: pgmap v605: 97 pgs: 97 active+clean; 262 MiB data, 2.8 GiB used, 1.0 TiB / 1.0 TiB avail; 701 KiB/s rd, 2.2 MiB/s wr, 225 op/s 2024-03-19T01:49:36.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:49:35 smithi112 ceph-mon[66819]: pgmap v605: 97 pgs: 97 active+clean; 262 MiB data, 2.8 GiB used, 1.0 TiB / 1.0 TiB avail; 701 KiB/s rd, 2.2 MiB/s wr, 225 op/s 2024-03-19T01:49:36.694 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1032373 11.40 MB/sec execute 366 sec latency 26.691 ms 2024-03-19T01:49:37.155 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.153+0000 7f66fd681640 1 -- 172.21.15.99:0/2813072206 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f66f8100740 msgr2=0x7f66f8100b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:49:37.155 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.153+0000 7f66fd681640 1 --2- 172.21.15.99:0/2813072206 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f66f8100740 0x7f66f8100b60 secure :-1 s=READY pgs=511 cs=0 l=1 rev1=1 crypto rx=0x7f66f0009900 tx=0x7f66f002bc30 comp rx=0 tx=0).stop 2024-03-19T01:49:37.155 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.153+0000 7f66fd681640 1 -- 172.21.15.99:0/2813072206 shutdown_connections 2024-03-19T01:49:37.155 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.153+0000 7f66fd681640 1 --2- 172.21.15.99:0/2813072206 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f66f8101930 0x7f66f8101db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:37.155 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.153+0000 7f66fd681640 1 --2- 172.21.15.99:0/2813072206 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f66f81022f0 0x7f66f8108b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:37.156 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.153+0000 7f66fd681640 1 --2- 172.21.15.99:0/2813072206 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f66f8100740 0x7f66f8100b60 unknown :-1 s=CLOSED pgs=511 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:37.156 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.153+0000 7f66fd681640 1 -- 172.21.15.99:0/2813072206 >> 172.21.15.99:0/2813072206 conn(0x7f66f80fbcb0 msgr2=0x7f66f80fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:49:37.156 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.153+0000 7f66fd681640 1 -- 172.21.15.99:0/2813072206 shutdown_connections 2024-03-19T01:49:37.156 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.153+0000 7f66fd681640 1 -- 172.21.15.99:0/2813072206 wait complete. 2024-03-19T01:49:37.156 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.154+0000 7f66fd681640 1 Processor -- start 2024-03-19T01:49:37.156 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.154+0000 7f66fd681640 1 -- start start 2024-03-19T01:49:37.156 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.154+0000 7f66fd681640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f66f8100740 0x7f66f8199d20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:37.156 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.154+0000 7f66fd681640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f66f8101930 0x7f66f819a260 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:37.156 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.154+0000 7f66fd681640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f66f81022f0 0x7f66f81a1290 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:37.156 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.154+0000 7f66fd681640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f66f810c390 con 0x7f66f81022f0 2024-03-19T01:49:37.156 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.154+0000 7f66fd681640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f66f810c210 con 0x7f66f8100740 2024-03-19T01:49:37.156 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.154+0000 7f66fd681640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f66f810c510 con 0x7f66f8101930 2024-03-19T01:49:37.156 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.154+0000 7f66f6ffd640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f66f8100740 0x7f66f8199d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:37.157 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.154+0000 7f66f77fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f66f81022f0 0x7f66f81a1290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:37.157 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.154+0000 7f66f6ffd640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f66f8100740 0x7f66f8199d20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:48342/0 (socket says 172.21.15.99:48342) 2024-03-19T01:49:37.157 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.154+0000 7f66f6ffd640 1 -- 172.21.15.99:0/608351261 learned_addr learned my addr 172.21.15.99:0/608351261 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:49:37.157 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.154+0000 7f66f67fc640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f66f8101930 0x7f66f819a260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:37.157 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.155+0000 7f66f6ffd640 1 -- 172.21.15.99:0/608351261 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f66f8101930 msgr2=0x7f66f819a260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:49:37.157 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.155+0000 7f66f6ffd640 1 --2- 172.21.15.99:0/608351261 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f66f8101930 0x7f66f819a260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:37.157 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.155+0000 7f66f6ffd640 1 -- 172.21.15.99:0/608351261 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f66f81022f0 msgr2=0x7f66f81a1290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:49:37.157 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.155+0000 7f66f6ffd640 1 --2- 172.21.15.99:0/608351261 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f66f81022f0 0x7f66f81a1290 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:37.157 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.155+0000 7f66f6ffd640 1 -- 172.21.15.99:0/608351261 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f66f81a1930 con 0x7f66f8100740 2024-03-19T01:49:37.157 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.155+0000 7f66f6ffd640 1 --2- 172.21.15.99:0/608351261 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f66f8100740 0x7f66f8199d20 secure :-1 s=READY pgs=133 cs=0 l=1 rev1=1 crypto rx=0x7f66f0000c00 tx=0x7f66f002c640 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:49:37.157 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.155+0000 7f66dffff640 1 -- 172.21.15.99:0/608351261 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f66f0041070 con 0x7f66f8100740 2024-03-19T01:49:37.157 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.155+0000 7f66fd681640 1 -- 172.21.15.99:0/608351261 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f66f81a1bc0 con 0x7f66f8100740 2024-03-19T01:49:37.157 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.155+0000 7f66dffff640 1 -- 172.21.15.99:0/608351261 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f66f0004450 con 0x7f66f8100740 2024-03-19T01:49:37.157 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.155+0000 7f66dffff640 1 -- 172.21.15.99:0/608351261 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f66f0034440 con 0x7f66f8100740 2024-03-19T01:49:37.158 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.155+0000 7f66fd681640 1 -- 172.21.15.99:0/608351261 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f66f81a2100 con 0x7f66f8100740 2024-03-19T01:49:37.161 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.157+0000 7f66dffff640 1 -- 172.21.15.99:0/608351261 <== mon.2 v2:172.21.15.112:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f66f0044070 con 0x7f66f8100740 2024-03-19T01:49:37.161 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.157+0000 7f670026b740 1 Processor -- start 2024-03-19T01:49:37.161 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.157+0000 7f670026b740 1 -- start start 2024-03-19T01:49:37.161 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.157+0000 7f670026b740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564717ca0100 0x564717ca0520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:37.161 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.157+0000 7f670026b740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564717ca0af0 0x564717ca55f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:37.161 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.158+0000 7f670026b740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564717ca5c60 0x564717ca80f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:37.161 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.158+0000 7f670026b740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x564717c04be0 con 0x564717ca0100 2024-03-19T01:49:37.161 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.158+0000 7f670026b740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x564717bad960 con 0x564717ca0af0 2024-03-19T01:49:37.161 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.158+0000 7f670026b740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x564717c00c80 con 0x564717ca5c60 2024-03-19T01:49:37.161 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.158+0000 7f66dffff640 1 --2- 172.21.15.99:0/608351261 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f66e00a28a0 0x7f66e00a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:37.161 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.158+0000 7f66f77fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564717ca0100 0x564717ca0520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:37.162 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.158+0000 7f66f77fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564717ca0100 0x564717ca0520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:38106/0 (socket says 172.21.15.99:38106) 2024-03-19T01:49:37.162 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.158+0000 7f66f67fc640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564717ca5c60 0x564717ca80f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:37.162 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.158+0000 7f66f77fe640 1 -- 172.21.15.99:0/53238697 learned_addr learned my addr 172.21.15.99:0/53238697 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:49:37.162 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.158+0000 7f66f6ffd640 1 --2- 172.21.15.99:0/53238697 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564717ca0af0 0x564717ca55f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:37.162 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.158+0000 7f66dffff640 1 -- 172.21.15.99:0/608351261 <== mon.2 v2:172.21.15.112:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f66f00f05e0 con 0x7f66f8100740 2024-03-19T01:49:37.162 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.158+0000 7f66f77fe640 1 --2- 172.21.15.99:0/608351261 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f66e00a28a0 0x7f66e00a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:37.162 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.158+0000 7f66f67fc640 1 -- 172.21.15.99:0/53238697 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564717ca0af0 msgr2=0x564717ca55f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:49:37.162 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.158+0000 7f66f67fc640 1 --2- 172.21.15.99:0/53238697 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564717ca0af0 0x564717ca55f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:37.162 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.158+0000 7f66f67fc640 1 -- 172.21.15.99:0/53238697 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564717ca0100 msgr2=0x564717ca0520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:49:37.162 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.158+0000 7f66f67fc640 1 --2- 172.21.15.99:0/53238697 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564717ca0100 0x564717ca0520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:37.162 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.158+0000 7f66f67fc640 1 -- 172.21.15.99:0/53238697 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x564717ba4d20 con 0x564717ca5c60 2024-03-19T01:49:37.162 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.158+0000 7f66f77fe640 1 --2- 172.21.15.99:0/53238697 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564717ca0100 0x564717ca0520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:49:37.163 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.158+0000 7f66f67fc640 1 --2- 172.21.15.99:0/53238697 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564717ca5c60 0x564717ca80f0 secure :-1 s=READY pgs=102 cs=0 l=1 rev1=1 crypto rx=0x7f66ec009fa0 tx=0x7f66ec0118d0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=432019ee548d943f server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:49:37.163 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.158+0000 7f66f77fe640 1 --2- 172.21.15.99:0/608351261 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f66e00a28a0 0x7f66e00a4d60 secure :-1 s=READY pgs=296 cs=0 l=1 rev1=1 crypto rx=0x564717ca1350 tx=0x7f66e8004730 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:49:37.163 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.158+0000 7f66dd7fa640 1 -- 172.21.15.99:0/53238697 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f66ec003c30 con 0x564717ca5c60 2024-03-19T01:49:37.163 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.158+0000 7f66dd7fa640 1 -- 172.21.15.99:0/53238697 <== mon.1 v2:172.21.15.115:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f66ec003dd0 con 0x564717ca5c60 2024-03-19T01:49:37.163 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.158+0000 7f66dd7fa640 1 -- 172.21.15.99:0/53238697 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f66ec00fb00 con 0x564717ca5c60 2024-03-19T01:49:37.163 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.159+0000 7f670026b740 1 -- 172.21.15.99:0/53238697 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564717ca5c60 msgr2=0x564717ca80f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:49:37.163 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.159+0000 7f670026b740 1 --2- 172.21.15.99:0/53238697 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564717ca5c60 0x564717ca80f0 secure :-1 s=READY pgs=102 cs=0 l=1 rev1=1 crypto rx=0x7f66ec009fa0 tx=0x7f66ec0118d0 comp rx=0 tx=0).stop 2024-03-19T01:49:37.163 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.159+0000 7f670026b740 1 -- 172.21.15.99:0/53238697 shutdown_connections 2024-03-19T01:49:37.163 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.159+0000 7f670026b740 1 --2- 172.21.15.99:0/53238697 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564717ca0af0 0x564717ca55f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:37.163 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.159+0000 7f670026b740 1 --2- 172.21.15.99:0/53238697 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564717ca5c60 0x564717ca80f0 unknown :-1 s=CLOSED pgs=102 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:37.163 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.159+0000 7f670026b740 1 --2- 172.21.15.99:0/53238697 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564717ca0100 0x564717ca0520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:37.163 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.159+0000 7f670026b740 1 -- 172.21.15.99:0/53238697 >> 172.21.15.99:0/53238697 conn(0x564717c9b670 msgr2=0x564717c9ba70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:49:37.163 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.159+0000 7f670026b740 1 -- 172.21.15.99:0/53238697 shutdown_connections 2024-03-19T01:49:37.164 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.159+0000 7f670026b740 1 -- 172.21.15.99:0/53238697 wait complete. 2024-03-19T01:49:37.164 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.159+0000 7f670026b740 1 -- start start 2024-03-19T01:49:37.164 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.160+0000 7f670026b740 1 Processor -- start 2024-03-19T01:49:37.164 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.160+0000 7f670026b740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564717ca0100 0x564717caa190 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:37.164 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.160+0000 7f670026b740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564717ca0af0 0x564717caa6d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:37.164 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.160+0000 7f670026b740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564717ca5c60 0x564717cb1710 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:37.164 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.160+0000 7f670026b740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x564717ca8c10 con 0x564717ca0af0 2024-03-19T01:49:37.164 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.160+0000 7f670026b740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x564717bc0db0 con 0x564717ca0100 2024-03-19T01:49:37.164 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.160+0000 7f670026b740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x564717ca8d90 con 0x564717ca5c60 2024-03-19T01:49:37.165 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.160+0000 7f670026b740 10 client.0 _ll_register_callbacks cb 0x564717bc5e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:49:37.165 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.164+0000 7f66f6ffd640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564717ca0100 0x564717caa190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:37.165 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.164+0000 7f66f67fc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564717ca0af0 0x564717caa6d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:37.165 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.165+0000 7f66f67fc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564717ca0af0 0x564717caa6d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:38112/0 (socket says 172.21.15.99:38112) 2024-03-19T01:49:37.165 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.165+0000 7f66f67fc640 1 -- 172.21.15.99:0/1284494708 learned_addr learned my addr 172.21.15.99:0/1284494708 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:49:37.166 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.167+0000 7f66f77fe640 1 --2- 172.21.15.99:0/1284494708 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564717ca5c60 0x564717cb1710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:37.167 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.167+0000 7f66f67fc640 1 -- 172.21.15.99:0/1284494708 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564717ca5c60 msgr2=0x564717cb1710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:49:37.167 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.167+0000 7f66f67fc640 1 --2- 172.21.15.99:0/1284494708 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564717ca5c60 0x564717cb1710 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:37.167 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.167+0000 7f66f67fc640 1 -- 172.21.15.99:0/1284494708 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564717ca0100 msgr2=0x564717caa190 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:49:37.167 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.167+0000 7f66f67fc640 1 --2- 172.21.15.99:0/1284494708 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564717ca0100 0x564717caa190 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:37.167 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.167+0000 7f66f67fc640 1 -- 172.21.15.99:0/1284494708 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x564717bbfc60 con 0x564717ca0af0 2024-03-19T01:49:37.168 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.168+0000 7f66f67fc640 1 --2- 172.21.15.99:0/1284494708 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564717ca0af0 0x564717caa6d0 secure :-1 s=READY pgs=512 cs=0 l=1 rev1=1 crypto rx=0x7f66ec01b040 tx=0x7f66ec004c70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b17d4163e6add628 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:49:37.169 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.170+0000 7f66daff5640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:49:37.176 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.176+0000 7f66daff5640 1 -- 172.21.15.99:0/1284494708 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f66ec028020 con 0x564717ca0af0 2024-03-19T01:49:37.176 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.176+0000 7f670026b740 1 -- 172.21.15.99:0/1284494708 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f66ec003dd0 con 0x564717ca0af0 2024-03-19T01:49:37.176 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.176+0000 7f66daff5640 1 -- 172.21.15.99:0/1284494708 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f66ec00fa50 con 0x564717ca0af0 2024-03-19T01:49:37.180 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.178+0000 7f66daff5640 1 -- 172.21.15.99:0/1284494708 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7f66ec033050 con 0x564717ca0af0 2024-03-19T01:49:37.180 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.178+0000 7f670026b740 10 client.16026 fetch_fsmap learned FSMap version 27 2024-03-19T01:49:37.180 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.178+0000 7f670026b740 1 -- 172.21.15.99:0/1284494708 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x564717919210 con 0x564717ca0af0 2024-03-19T01:49:37.180 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.180+0000 7f66daff5640 1 -- 172.21.15.99:0/1284494708 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f66ec003dd0 con 0x564717ca0af0 2024-03-19T01:49:37.181 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.180+0000 7f66daff5640 1 -- 172.21.15.99:0/1284494708 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 27) ==== 8338+0+0 (secure 0 0 0) 0x7f66ec005040 con 0x564717ca0af0 2024-03-19T01:49:37.183 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.181+0000 7f670026b740 10 client.16026 fetch_fsmap finished waiting for FSMap version 27 2024-03-19T01:49:37.183 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.181+0000 7f670026b740 10 client.16026 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:49:37.183 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.181+0000 7f670026b740 20 client.16026 populate_metadata read hostname 'smithi099' 2024-03-19T01:49:37.183 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.181+0000 7f670026b740 1 --2- 172.21.15.99:0/1284494708 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x564717cb5e50 0x564717cb8260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:37.183 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.182+0000 7f670026b740 4 client.16026 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:49:37.184 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.182+0000 7f670026b740 1 -- 172.21.15.99:0/1284494708 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x564717c12120 con 0x564717cb5e50 2024-03-19T01:49:37.184 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.183+0000 7f66f6ffd640 1 --2- 172.21.15.99:0/1284494708 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x564717cb5e50 0x564717cb8260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:37.186 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.186+0000 7f66f6ffd640 1 --2- 172.21.15.99:0/1284494708 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x564717cb5e50 0x564717cb8260 crc :-1 s=READY pgs=105 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=7d68e3cbece505da server_cookie=b176dc2e9f0a342a in_seq=0 out_seq=0 2024-03-19T01:49:37.186 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.186+0000 7f66daff5640 10 client.16026 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:49:37.189 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.189+0000 7f66daff5640 1 -- 172.21.15.99:0/1284494708 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x564717c12120 con 0x564717cb5e50 2024-03-19T01:49:37.189 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.189+0000 7f66daff5640 10 client.16026 handle_command_reply: tid=0 2024-03-19T01:49:37.189 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.189+0000 7f670026b740 1 client.16026 shutdown 2024-03-19T01:49:37.190 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.189+0000 7f670026b740 1 -- 172.21.15.99:0/1284494708 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564717ca0af0 msgr2=0x564717caa6d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:49:37.190 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.189+0000 7f670026b740 1 --2- 172.21.15.99:0/1284494708 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564717ca0af0 0x564717caa6d0 secure :-1 s=READY pgs=512 cs=0 l=1 rev1=1 crypto rx=0x7f66ec01b040 tx=0x7f66ec004c70 comp rx=0 tx=0).stop 2024-03-19T01:49:37.190 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.189+0000 7f670026b740 1 -- 172.21.15.99:0/1284494708 shutdown_connections 2024-03-19T01:49:37.190 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.189+0000 7f670026b740 1 --2- 172.21.15.99:0/1284494708 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x564717cb5e50 0x564717cb8260 crc :-1 s=READY pgs=105 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:37.190 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.189+0000 7f670026b740 1 --2- 172.21.15.99:0/1284494708 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564717ca5c60 0x564717cb1710 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:37.190 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.189+0000 7f670026b740 1 --2- 172.21.15.99:0/1284494708 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564717ca0af0 0x564717caa6d0 unknown :-1 s=CLOSED pgs=512 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:37.190 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.189+0000 7f670026b740 1 --2- 172.21.15.99:0/1284494708 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564717ca0100 0x564717caa190 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:37.190 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.189+0000 7f670026b740 1 -- 172.21.15.99:0/1284494708 >> 172.21.15.99:0/1284494708 conn(0x564717c9b670 msgr2=0x564717c9c8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:49:37.190 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.189+0000 7f66daff5640 0 client.16026 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:49:37.191 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.191+0000 7f670026b740 1 -- 172.21.15.99:0/1284494708 shutdown_connections 2024-03-19T01:49:37.191 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.191+0000 7f670026b740 1 -- 172.21.15.99:0/1284494708 wait complete. 2024-03-19T01:49:37.191 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.191+0000 7f670026b740 20 client.16026 trim_cache size 0 max 16384 2024-03-19T01:49:37.212 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.207+0000 7f670026b740 1 Processor -- start 2024-03-19T01:49:37.212 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.207+0000 7f670026b740 1 -- start start 2024-03-19T01:49:37.212 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.208+0000 7f670026b740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564717ca0100 0x564717c9a930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:37.213 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.208+0000 7f670026b740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564717ca0af0 0x564717cb16d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:37.213 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.208+0000 7f670026b740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564717ca5c60 0x564717ca1380 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:37.213 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.208+0000 7f670026b740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f66f81ad980 con 0x564717ca0100 2024-03-19T01:49:37.213 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.208+0000 7f670026b740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f66f81ab250 con 0x564717ca5c60 2024-03-19T01:49:37.213 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.208+0000 7f670026b740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f66f81a90a0 con 0x564717ca0af0 2024-03-19T01:49:37.213 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.208+0000 7f66f6ffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564717ca0100 0x564717c9a930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:37.213 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.208+0000 7f66f6ffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564717ca0100 0x564717c9a930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:38116/0 (socket says 172.21.15.99:38116) 2024-03-19T01:49:37.213 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.208+0000 7f66f6ffd640 1 -- 172.21.15.99:0/2448350194 learned_addr learned my addr 172.21.15.99:0/2448350194 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:49:37.213 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.208+0000 7f66f67fc640 1 --2- 172.21.15.99:0/2448350194 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564717ca0af0 0x564717cb16d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:37.213 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.208+0000 7f66f77fe640 1 --2- 172.21.15.99:0/2448350194 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564717ca5c60 0x564717ca1380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:37.213 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.208+0000 7f66f6ffd640 1 -- 172.21.15.99:0/2448350194 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564717ca0af0 msgr2=0x564717cb16d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:49:37.214 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.208+0000 7f66f6ffd640 1 --2- 172.21.15.99:0/2448350194 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564717ca0af0 0x564717cb16d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:37.214 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.208+0000 7f66f6ffd640 1 -- 172.21.15.99:0/2448350194 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564717ca5c60 msgr2=0x564717ca1380 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:49:37.214 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.208+0000 7f66f6ffd640 1 --2- 172.21.15.99:0/2448350194 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564717ca5c60 0x564717ca1380 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:37.214 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.208+0000 7f66f6ffd640 1 -- 172.21.15.99:0/2448350194 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5647179e5620 con 0x564717ca0100 2024-03-19T01:49:37.214 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.208+0000 7f66f6ffd640 1 --2- 172.21.15.99:0/2448350194 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564717ca0100 0x564717c9a930 secure :-1 s=READY pgs=513 cs=0 l=1 rev1=1 crypto rx=0x7f66f00ee0a0 tx=0x7f66f00e7990 comp rx=0 tx=0).ready entity=mon.0 client_cookie=59efa9e12336e289 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:49:37.214 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.208+0000 7f66dcff9640 1 -- 172.21.15.99:0/2448350194 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f66f004b040 con 0x564717ca0100 2024-03-19T01:49:37.214 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.208+0000 7f66dcff9640 1 -- 172.21.15.99:0/2448350194 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f66f0004c80 con 0x564717ca0100 2024-03-19T01:49:37.214 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.209+0000 7f66dcff9640 1 -- 172.21.15.99:0/2448350194 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f66f009be60 con 0x564717ca0100 2024-03-19T01:49:37.214 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.209+0000 7f670026b740 1 -- 172.21.15.99:0/2448350194 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564717ca0100 msgr2=0x564717c9a930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:49:37.214 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.209+0000 7f670026b740 1 --2- 172.21.15.99:0/2448350194 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564717ca0100 0x564717c9a930 secure :-1 s=READY pgs=513 cs=0 l=1 rev1=1 crypto rx=0x7f66f00ee0a0 tx=0x7f66f00e7990 comp rx=0 tx=0).stop 2024-03-19T01:49:37.214 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.209+0000 7f670026b740 1 -- 172.21.15.99:0/2448350194 shutdown_connections 2024-03-19T01:49:37.214 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.209+0000 7f670026b740 1 --2- 172.21.15.99:0/2448350194 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564717ca5c60 0x564717ca1380 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:37.215 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.209+0000 7f670026b740 1 --2- 172.21.15.99:0/2448350194 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564717ca0af0 0x564717cb16d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:37.215 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.209+0000 7f670026b740 1 --2- 172.21.15.99:0/2448350194 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564717ca0100 0x564717c9a930 unknown :-1 s=CLOSED pgs=513 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:37.215 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.209+0000 7f670026b740 1 -- 172.21.15.99:0/2448350194 >> 172.21.15.99:0/2448350194 conn(0x564717c9b670 msgr2=0x564717c9a220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:49:37.215 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.209+0000 7f670026b740 1 -- 172.21.15.99:0/2448350194 shutdown_connections 2024-03-19T01:49:37.215 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.209+0000 7f670026b740 1 -- 172.21.15.99:0/2448350194 wait complete. 2024-03-19T01:49:37.215 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.210+0000 7f670026b740 1 -- start start 2024-03-19T01:49:37.215 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.210+0000 7f670026b740 1 Processor -- start 2024-03-19T01:49:37.215 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.210+0000 7f670026b740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564717ca0100 0x564717caeb10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:37.215 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.210+0000 7f670026b740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564717ca0af0 0x564717caf050 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:37.215 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.210+0000 7f670026b740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564717ca5c60 0x564717cb0010 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:37.215 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.210+0000 7f670026b740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f66f81a9710 con 0x564717ca0af0 2024-03-19T01:49:37.216 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.210+0000 7f670026b740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f66f81aab80 con 0x564717ca0100 2024-03-19T01:49:37.216 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.210+0000 7f670026b740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f66f81a9de0 con 0x564717ca5c60 2024-03-19T01:49:37.216 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.210+0000 7f670026b740 10 client.0 _ll_register_callbacks cb 0x7f66e80097d0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:49:37.216 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.210+0000 7f66f67fc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564717ca0af0 0x564717caf050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:37.216 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.210+0000 7f66f67fc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564717ca0af0 0x564717caf050 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:38132/0 (socket says 172.21.15.99:38132) 2024-03-19T01:49:37.216 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.210+0000 7f66f67fc640 1 -- 172.21.15.99:0/4116352804 learned_addr learned my addr 172.21.15.99:0/4116352804 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:49:37.216 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.210+0000 7f66f6ffd640 1 --2- 172.21.15.99:0/4116352804 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564717ca0100 0x564717caeb10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:37.216 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.210+0000 7f66f77fe640 1 --2- 172.21.15.99:0/4116352804 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564717ca5c60 0x564717cb0010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:37.216 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.210+0000 7f66f67fc640 1 -- 172.21.15.99:0/4116352804 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564717ca5c60 msgr2=0x564717cb0010 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:49:37.216 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.210+0000 7f66f67fc640 1 --2- 172.21.15.99:0/4116352804 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564717ca5c60 0x564717cb0010 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:37.216 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.210+0000 7f66f67fc640 1 -- 172.21.15.99:0/4116352804 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564717ca0100 msgr2=0x564717caeb10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:49:37.217 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.210+0000 7f66f67fc640 1 --2- 172.21.15.99:0/4116352804 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564717ca0100 0x564717caeb10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:37.217 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.210+0000 7f66f67fc640 1 -- 172.21.15.99:0/4116352804 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x564717cb0600 con 0x564717ca0af0 2024-03-19T01:49:37.217 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.210+0000 7f66f67fc640 1 --2- 172.21.15.99:0/4116352804 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564717ca0af0 0x564717caf050 secure :-1 s=READY pgs=514 cs=0 l=1 rev1=1 crypto rx=0x7f66ec005410 tx=0x7f66ec0094f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=855dddc0159dadd8 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:49:37.217 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.210+0000 7f66daff5640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:49:37.217 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.211+0000 7f66daff5640 1 -- 172.21.15.99:0/4116352804 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f66ec03c060 con 0x564717ca0af0 2024-03-19T01:49:37.217 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.211+0000 7f670026b740 1 -- 172.21.15.99:0/4116352804 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f66f0004c80 con 0x564717ca0af0 2024-03-19T01:49:37.217 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.211+0000 7f66daff5640 1 -- 172.21.15.99:0/4116352804 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f66ec005880 con 0x564717ca0af0 2024-03-19T01:49:37.217 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.211+0000 7f66daff5640 1 -- 172.21.15.99:0/4116352804 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7f66ec028040 con 0x564717ca0af0 2024-03-19T01:49:37.217 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.211+0000 7f670026b740 10 client.16032 fetch_fsmap learned FSMap version 27 2024-03-19T01:49:37.217 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.211+0000 7f670026b740 1 -- 172.21.15.99:0/4116352804 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x564717cb1940 con 0x564717ca0af0 2024-03-19T01:49:37.217 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.211+0000 7f66daff5640 1 -- 172.21.15.99:0/4116352804 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f66f0004c80 con 0x564717ca0af0 2024-03-19T01:49:37.217 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.211+0000 7f66daff5640 1 -- 172.21.15.99:0/4116352804 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 27) ==== 8338+0+0 (secure 0 0 0) 0x7f66ec004000 con 0x564717ca0af0 2024-03-19T01:49:37.217 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.211+0000 7f670026b740 10 client.16032 fetch_fsmap finished waiting for FSMap version 27 2024-03-19T01:49:37.218 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.211+0000 7f670026b740 10 client.16032 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:49:37.218 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.211+0000 7f670026b740 20 client.16032 populate_metadata read hostname 'smithi099' 2024-03-19T01:49:37.218 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.211+0000 7f670026b740 1 --2- 172.21.15.99:0/4116352804 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x564717d250a0 0x564717d274c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:37.218 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.211+0000 7f670026b740 4 client.16032 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:49:37.218 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.211+0000 7f670026b740 1 -- 172.21.15.99:0/4116352804 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x564717c0c470 con 0x564717d250a0 2024-03-19T01:49:37.218 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.211+0000 7f66f6ffd640 1 --2- 172.21.15.99:0/4116352804 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x564717d250a0 0x564717d274c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:37.218 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.212+0000 7f66f6ffd640 1 --2- 172.21.15.99:0/4116352804 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x564717d250a0 0x564717d274c0 crc :-1 s=READY pgs=106 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=b449222d89a3efc5 server_cookie=67e3e185a44833e6 in_seq=0 out_seq=0 2024-03-19T01:49:37.218 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:37.212+0000 7f66daff5640 10 client.16032 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:49:37.693 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1034182 11.40 MB/sec execute 367 sec latency 58.359 ms 2024-03-19T01:49:38.286 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:38.286+0000 7f66daff5640 1 -- 172.21.15.99:0/4116352804 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x564717c0c470 con 0x564717d250a0 2024-03-19T01:49:38.287 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:38.286+0000 7f66daff5640 10 client.16032 handle_command_reply: tid=0 2024-03-19T01:49:38.287 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:38.286+0000 7f670026b740 1 client.16032 shutdown 2024-03-19T01:49:38.287 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:38.286+0000 7f670026b740 1 -- 172.21.15.99:0/4116352804 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564717ca0af0 msgr2=0x564717caf050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:49:38.287 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:38.286+0000 7f670026b740 1 --2- 172.21.15.99:0/4116352804 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564717ca0af0 0x564717caf050 secure :-1 s=READY pgs=514 cs=0 l=1 rev1=1 crypto rx=0x7f66ec005410 tx=0x7f66ec0094f0 comp rx=0 tx=0).stop 2024-03-19T01:49:38.287 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:38.286+0000 7f670026b740 1 -- 172.21.15.99:0/4116352804 shutdown_connections 2024-03-19T01:49:38.287 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:38.286+0000 7f670026b740 1 --2- 172.21.15.99:0/4116352804 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x564717d250a0 0x564717d274c0 crc :-1 s=READY pgs=106 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:38.287 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:38.286+0000 7f670026b740 1 --2- 172.21.15.99:0/4116352804 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564717ca5c60 0x564717cb0010 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:38.287 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:38.286+0000 7f66daff5640 0 client.16032 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:49:38.287 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:38.286+0000 7f670026b740 1 --2- 172.21.15.99:0/4116352804 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564717ca0af0 0x564717caf050 unknown :-1 s=CLOSED pgs=514 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:38.287 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:38.286+0000 7f670026b740 1 --2- 172.21.15.99:0/4116352804 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564717ca0100 0x564717caeb10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:38.287 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:38.286+0000 7f670026b740 1 -- 172.21.15.99:0/4116352804 >> 172.21.15.99:0/4116352804 conn(0x564717c9b670 msgr2=0x564717c9e8d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:49:38.287 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:38.286+0000 7f670026b740 1 -- 172.21.15.99:0/4116352804 shutdown_connections 2024-03-19T01:49:38.288 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:38.286+0000 7f670026b740 1 -- 172.21.15.99:0/4116352804 wait complete. 2024-03-19T01:49:38.288 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:38.286+0000 7f670026b740 20 client.16032 trim_cache size 0 max 16384 2024-03-19T01:49:38.288 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:49:38.288 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:49:38.288 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:49:38.288 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:49:38.289 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:38.287+0000 7f66fd681640 1 -- 172.21.15.99:0/608351261 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f66e00a28a0 msgr2=0x7f66e00a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:49:38.289 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:38.287+0000 7f66fd681640 1 --2- 172.21.15.99:0/608351261 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f66e00a28a0 0x7f66e00a4d60 secure :-1 s=READY pgs=296 cs=0 l=1 rev1=1 crypto rx=0x564717ca1350 tx=0x7f66e8004730 comp rx=0 tx=0).stop 2024-03-19T01:49:38.289 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:38.287+0000 7f66fd681640 1 -- 172.21.15.99:0/608351261 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f66f8100740 msgr2=0x7f66f8199d20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:49:38.289 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:38.287+0000 7f66fd681640 1 --2- 172.21.15.99:0/608351261 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f66f8100740 0x7f66f8199d20 secure :-1 s=READY pgs=133 cs=0 l=1 rev1=1 crypto rx=0x7f66f0000c00 tx=0x7f66f002c640 comp rx=0 tx=0).stop 2024-03-19T01:49:38.289 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:38.287+0000 7f66fd681640 1 -- 172.21.15.99:0/608351261 shutdown_connections 2024-03-19T01:49:38.289 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:38.287+0000 7f66fd681640 1 --2- 172.21.15.99:0/608351261 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f66f81022f0 0x7f66f81a1290 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:38.289 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:38.287+0000 7f66fd681640 1 --2- 172.21.15.99:0/608351261 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f66f8101930 0x7f66f819a260 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:38.289 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:38.287+0000 7f66fd681640 1 --2- 172.21.15.99:0/608351261 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f66e00a28a0 0x7f66e00a4d60 unknown :-1 s=CLOSED pgs=296 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:38.290 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:38.287+0000 7f66fd681640 1 --2- 172.21.15.99:0/608351261 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f66f8100740 0x7f66f8199d20 unknown :-1 s=CLOSED pgs=133 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:38.290 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:38.287+0000 7f66fd681640 1 -- 172.21.15.99:0/608351261 >> 172.21.15.99:0/608351261 conn(0x7f66f80fbcb0 msgr2=0x7f66f80fd710 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:49:38.290 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:38.287+0000 7f66fd681640 1 -- 172.21.15.99:0/608351261 shutdown_connections 2024-03-19T01:49:38.290 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:38.287+0000 7f66fd681640 1 -- 172.21.15.99:0/608351261 wait complete. 2024-03-19T01:49:38.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:49:37 smithi115 ceph-mon[67332]: pgmap v606: 97 pgs: 97 active+clean; 263 MiB data, 2.8 GiB used, 1.0 TiB / 1.0 TiB avail; 772 KiB/s rd, 2.8 MiB/s wr, 245 op/s 2024-03-19T01:49:38.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:49:37 smithi099 ceph-mon[62951]: pgmap v606: 97 pgs: 97 active+clean; 263 MiB data, 2.8 GiB used, 1.0 TiB / 1.0 TiB avail; 772 KiB/s rd, 2.8 MiB/s wr, 245 op/s 2024-03-19T01:49:38.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:49:37 smithi112 ceph-mon[66819]: pgmap v606: 97 pgs: 97 active+clean; 263 MiB data, 2.8 GiB used, 1.0 TiB / 1.0 TiB avail; 772 KiB/s rd, 2.8 MiB/s wr, 245 op/s 2024-03-19T01:49:38.694 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1036233 11.39 MB/sec execute 368 sec latency 31.938 ms 2024-03-19T01:49:39.694 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1038195 11.39 MB/sec execute 369 sec latency 27.328 ms 2024-03-19T01:49:40.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:49:39 smithi115 ceph-mon[67332]: pgmap v607: 97 pgs: 97 active+clean; 265 MiB data, 2.8 GiB used, 1.0 TiB / 1.0 TiB avail; 889 KiB/s rd, 2.9 MiB/s wr, 248 op/s 2024-03-19T01:49:40.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:49:39 smithi099 ceph-mon[62951]: pgmap v607: 97 pgs: 97 active+clean; 265 MiB data, 2.8 GiB used, 1.0 TiB / 1.0 TiB avail; 889 KiB/s rd, 2.9 MiB/s wr, 248 op/s 2024-03-19T01:49:40.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:49:39 smithi112 ceph-mon[66819]: pgmap v607: 97 pgs: 97 active+clean; 265 MiB data, 2.8 GiB used, 1.0 TiB / 1.0 TiB avail; 889 KiB/s rd, 2.9 MiB/s wr, 248 op/s 2024-03-19T01:49:40.694 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1040863 11.41 MB/sec execute 370 sec latency 15.121 ms 2024-03-19T01:49:41.694 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1042433 11.40 MB/sec execute 371 sec latency 33.384 ms 2024-03-19T01:49:42.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:49:41 smithi115 ceph-mon[67332]: pgmap v608: 97 pgs: 97 active+clean; 267 MiB data, 2.8 GiB used, 1.0 TiB / 1.0 TiB avail; 807 KiB/s rd, 2.8 MiB/s wr, 235 op/s 2024-03-19T01:49:42.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:49:41 smithi099 ceph-mon[62951]: pgmap v608: 97 pgs: 97 active+clean; 267 MiB data, 2.8 GiB used, 1.0 TiB / 1.0 TiB avail; 807 KiB/s rd, 2.8 MiB/s wr, 235 op/s 2024-03-19T01:49:42.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:49:41 smithi112 ceph-mon[66819]: pgmap v608: 97 pgs: 97 active+clean; 267 MiB data, 2.8 GiB used, 1.0 TiB / 1.0 TiB avail; 807 KiB/s rd, 2.8 MiB/s wr, 235 op/s 2024-03-19T01:49:42.694 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1044664 11.40 MB/sec execute 372 sec latency 59.840 ms 2024-03-19T01:49:43.694 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1046074 11.39 MB/sec execute 373 sec latency 44.297 ms 2024-03-19T01:49:44.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:49:43 smithi115 ceph-mon[67332]: pgmap v609: 97 pgs: 97 active+clean; 268 MiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 906 KiB/s rd, 3.2 MiB/s wr, 273 op/s 2024-03-19T01:49:44.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:49:43 smithi099 ceph-mon[62951]: pgmap v609: 97 pgs: 97 active+clean; 268 MiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 906 KiB/s rd, 3.2 MiB/s wr, 273 op/s 2024-03-19T01:49:44.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:49:43 smithi112 ceph-mon[66819]: pgmap v609: 97 pgs: 97 active+clean; 268 MiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 906 KiB/s rd, 3.2 MiB/s wr, 273 op/s 2024-03-19T01:49:44.694 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1048418 11.39 MB/sec execute 374 sec latency 26.576 ms 2024-03-19T01:49:45.694 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1050742 11.39 MB/sec execute 375 sec latency 31.125 ms 2024-03-19T01:49:46.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:49:45 smithi115 ceph-mon[67332]: pgmap v610: 97 pgs: 97 active+clean; 263 MiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 684 KiB/s rd, 2.5 MiB/s wr, 215 op/s 2024-03-19T01:49:46.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:49:45 smithi099 ceph-mon[62951]: pgmap v610: 97 pgs: 97 active+clean; 263 MiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 684 KiB/s rd, 2.5 MiB/s wr, 215 op/s 2024-03-19T01:49:46.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:49:45 smithi112 ceph-mon[66819]: pgmap v610: 97 pgs: 97 active+clean; 263 MiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 684 KiB/s rd, 2.5 MiB/s wr, 215 op/s 2024-03-19T01:49:46.694 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1052568 11.39 MB/sec execute 376 sec latency 37.498 ms 2024-03-19T01:49:47.694 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1055127 11.40 MB/sec execute 377 sec latency 94.950 ms 2024-03-19T01:49:48.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:49:47 smithi115 ceph-mon[67332]: pgmap v611: 97 pgs: 97 active+clean; 250 MiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 944 KiB/s rd, 3.2 MiB/s wr, 268 op/s 2024-03-19T01:49:48.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:49:47 smithi099 ceph-mon[62951]: pgmap v611: 97 pgs: 97 active+clean; 250 MiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 944 KiB/s rd, 3.2 MiB/s wr, 268 op/s 2024-03-19T01:49:48.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:49:47 smithi112 ceph-mon[66819]: pgmap v611: 97 pgs: 97 active+clean; 250 MiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 944 KiB/s rd, 3.2 MiB/s wr, 268 op/s 2024-03-19T01:49:48.694 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1056714 11.39 MB/sec execute 378 sec latency 30.338 ms 2024-03-19T01:49:49.694 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1059076 11.40 MB/sec execute 379 sec latency 11.077 ms 2024-03-19T01:49:50.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:49:49 smithi115 ceph-mon[67332]: pgmap v612: 97 pgs: 97 active+clean; 249 MiB data, 2.8 GiB used, 1.0 TiB / 1.0 TiB avail; 973 KiB/s rd, 3.0 MiB/s wr, 283 op/s 2024-03-19T01:49:50.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:49:49 smithi099 ceph-mon[62951]: pgmap v612: 97 pgs: 97 active+clean; 249 MiB data, 2.8 GiB used, 1.0 TiB / 1.0 TiB avail; 973 KiB/s rd, 3.0 MiB/s wr, 283 op/s 2024-03-19T01:49:50.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:49:49 smithi112 ceph-mon[66819]: pgmap v612: 97 pgs: 97 active+clean; 249 MiB data, 2.8 GiB used, 1.0 TiB / 1.0 TiB avail; 973 KiB/s rd, 3.0 MiB/s wr, 283 op/s 2024-03-19T01:49:50.694 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1061476 11.39 MB/sec execute 380 sec latency 26.892 ms 2024-03-19T01:49:51.377 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.375+0000 7efe2f473640 1 -- 172.21.15.99:0/1965378891 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7efe281022f0 msgr2=0x7efe28108b80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:49:51.378 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.375+0000 7efe2f473640 1 --2- 172.21.15.99:0/1965378891 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7efe281022f0 0x7efe28108b80 secure :-1 s=READY pgs=515 cs=0 l=1 rev1=1 crypto rx=0x7efe20009ca0 tx=0x7efe2002c0f0 comp rx=0 tx=0).stop 2024-03-19T01:49:51.378 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.375+0000 7efe2f473640 1 -- 172.21.15.99:0/1965378891 shutdown_connections 2024-03-19T01:49:51.378 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.375+0000 7efe2f473640 1 --2- 172.21.15.99:0/1965378891 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7efe281022f0 0x7efe28108b80 unknown :-1 s=CLOSED pgs=515 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:51.378 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.375+0000 7efe2f473640 1 --2- 172.21.15.99:0/1965378891 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7efe28101930 0x7efe28101db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:51.378 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.375+0000 7efe2f473640 1 --2- 172.21.15.99:0/1965378891 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7efe28100740 0x7efe28100b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:51.378 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.375+0000 7efe2f473640 1 -- 172.21.15.99:0/1965378891 >> 172.21.15.99:0/1965378891 conn(0x7efe280fbcb0 msgr2=0x7efe280fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:49:51.378 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.375+0000 7efe2f473640 1 -- 172.21.15.99:0/1965378891 shutdown_connections 2024-03-19T01:49:51.378 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.375+0000 7efe2f473640 1 -- 172.21.15.99:0/1965378891 wait complete. 2024-03-19T01:49:51.379 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.376+0000 7efe2f473640 1 Processor -- start 2024-03-19T01:49:51.379 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.376+0000 7efe2f473640 1 -- start start 2024-03-19T01:49:51.379 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.376+0000 7efe2f473640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7efe28100740 0x7efe28069a30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:51.379 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.376+0000 7efe2f473640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7efe28101930 0x7efe2819c750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:51.379 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.376+0000 7efe2f473640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7efe281022f0 0x7efe2819ed90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:51.379 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.376+0000 7efe2f473640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7efe2810c3a0 con 0x7efe281022f0 2024-03-19T01:49:51.379 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.376+0000 7efe2f473640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7efe2810c220 con 0x7efe28100740 2024-03-19T01:49:51.379 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.376+0000 7efe2f473640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7efe2810c520 con 0x7efe28101930 2024-03-19T01:49:51.379 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.376+0000 7efe2da10640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7efe281022f0 0x7efe2819ed90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:51.379 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.376+0000 7efe2d20f640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7efe28100740 0x7efe28069a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:51.379 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.376+0000 7efe2da10640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7efe281022f0 0x7efe2819ed90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:59386/0 (socket says 172.21.15.99:59386) 2024-03-19T01:49:51.380 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.376+0000 7efe2d20f640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7efe28100740 0x7efe28069a30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:33620/0 (socket says 172.21.15.99:33620) 2024-03-19T01:49:51.380 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.376+0000 7efe2da10640 1 -- 172.21.15.99:0/1841925824 learned_addr learned my addr 172.21.15.99:0/1841925824 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:49:51.380 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.376+0000 7efe2ca0e640 1 --2- 172.21.15.99:0/1841925824 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7efe28101930 0x7efe2819c750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:51.380 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.376+0000 7efe2da10640 1 -- 172.21.15.99:0/1841925824 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7efe28101930 msgr2=0x7efe2819c750 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:49:51.380 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.376+0000 7efe2da10640 1 --2- 172.21.15.99:0/1841925824 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7efe28101930 0x7efe2819c750 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:51.380 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.376+0000 7efe2da10640 1 -- 172.21.15.99:0/1841925824 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7efe28100740 msgr2=0x7efe28069a30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:49:51.380 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.376+0000 7efe2da10640 1 --2- 172.21.15.99:0/1841925824 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7efe28100740 0x7efe28069a30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:51.380 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.376+0000 7efe2da10640 1 -- 172.21.15.99:0/1841925824 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7efe2819f650 con 0x7efe281022f0 2024-03-19T01:49:51.380 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.377+0000 7efe2d20f640 1 --2- 172.21.15.99:0/1841925824 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7efe28100740 0x7efe28069a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:49:51.380 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.377+0000 7efe2da10640 1 --2- 172.21.15.99:0/1841925824 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7efe281022f0 0x7efe2819ed90 secure :-1 s=READY pgs=516 cs=0 l=1 rev1=1 crypto rx=0x7efe20037040 tx=0x7efe20032f40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:49:51.380 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.377+0000 7efe1e7fc640 1 -- 172.21.15.99:0/1841925824 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efe20041020 con 0x7efe281022f0 2024-03-19T01:49:51.380 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.377+0000 7efe1e7fc640 1 -- 172.21.15.99:0/1841925824 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7efe20034cd0 con 0x7efe281022f0 2024-03-19T01:49:51.381 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.377+0000 7efe2f473640 1 -- 172.21.15.99:0/1841925824 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7efe2819f8e0 con 0x7efe281022f0 2024-03-19T01:49:51.381 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.377+0000 7efe1e7fc640 1 -- 172.21.15.99:0/1841925824 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efe20039040 con 0x7efe281022f0 2024-03-19T01:49:51.381 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.377+0000 7efe2f473640 1 -- 172.21.15.99:0/1841925824 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7efe2819fdd0 con 0x7efe281022f0 2024-03-19T01:49:51.388 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.379+0000 7efe1e7fc640 1 -- 172.21.15.99:0/1841925824 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7efe20044020 con 0x7efe281022f0 2024-03-19T01:49:51.388 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.380+0000 7efe3205e740 1 Processor -- start 2024-03-19T01:49:51.389 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.380+0000 7efe3205e740 1 -- start start 2024-03-19T01:49:51.389 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.380+0000 7efe1e7fc640 1 --2- 172.21.15.99:0/1841925824 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7efe140a28a0 0x7efe140a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:51.389 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.380+0000 7efe3205e740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5560c79aa100 0x5560c79aa520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:51.389 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.380+0000 7efe3205e740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5560c79aaaf0 0x5560c79af5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:51.389 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.380+0000 7efe3205e740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5560c79afc60 0x5560c79b20f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:51.389 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.380+0000 7efe3205e740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x5560c790ebe0 con 0x5560c79aa100 2024-03-19T01:49:51.389 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.380+0000 7efe3205e740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x5560c78b7960 con 0x5560c79afc60 2024-03-19T01:49:51.389 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.380+0000 7efe3205e740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x5560c790ac80 con 0x5560c79aaaf0 2024-03-19T01:49:51.389 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.380+0000 7efe2d20f640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5560c79aa100 0x5560c79aa520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:51.389 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.380+0000 7efe2ca0e640 1 --2- 172.21.15.99:0/1841925824 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7efe140a28a0 0x7efe140a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:51.389 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.380+0000 7efe2d20f640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5560c79aa100 0x5560c79aa520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:59394/0 (socket says 172.21.15.99:59394) 2024-03-19T01:49:51.389 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.380+0000 7efe2d20f640 1 -- 172.21.15.99:0/3979317387 learned_addr learned my addr 172.21.15.99:0/3979317387 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:49:51.390 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.380+0000 7efe2da10640 1 --2- 172.21.15.99:0/3979317387 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5560c79afc60 0x5560c79b20f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:51.390 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.380+0000 7efe2ca0e640 1 --2- 172.21.15.99:0/3979317387 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5560c79aaaf0 0x5560c79af5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:51.390 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.380+0000 7efe1e7fc640 1 -- 172.21.15.99:0/1841925824 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7efe200f0940 con 0x7efe281022f0 2024-03-19T01:49:51.390 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.380+0000 7efe2d20f640 1 -- 172.21.15.99:0/3979317387 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5560c79aaaf0 msgr2=0x5560c79af5f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:49:51.390 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.380+0000 7efe2d20f640 1 --2- 172.21.15.99:0/3979317387 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5560c79aaaf0 0x5560c79af5f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:51.390 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.380+0000 7efe2d20f640 1 -- 172.21.15.99:0/3979317387 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5560c79afc60 msgr2=0x5560c79b20f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:49:51.390 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.380+0000 7efe2d20f640 1 --2- 172.21.15.99:0/3979317387 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5560c79afc60 0x5560c79b20f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:51.390 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.380+0000 7efe2d20f640 1 -- 172.21.15.99:0/3979317387 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5560c78aed20 con 0x5560c79aa100 2024-03-19T01:49:51.390 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.380+0000 7efe2da10640 1 --2- 172.21.15.99:0/3979317387 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5560c79afc60 0x5560c79b20f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:49:51.390 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.380+0000 7efe2ca0e640 1 --2- 172.21.15.99:0/1841925824 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7efe140a28a0 0x7efe140a4d60 secure :-1 s=READY pgs=297 cs=0 l=1 rev1=1 crypto rx=0x7efe280697a0 tx=0x7efe180098b0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:49:51.390 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.381+0000 7efe2d20f640 1 --2- 172.21.15.99:0/3979317387 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5560c79aa100 0x5560c79aa520 secure :-1 s=READY pgs=517 cs=0 l=1 rev1=1 crypto rx=0x7efe240114c0 tx=0x7efe24011980 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6f08b7d28e97947c server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:49:51.391 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.381+0000 7efe137fe640 1 -- 172.21.15.99:0/3979317387 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efe24004020 con 0x5560c79aa100 2024-03-19T01:49:51.391 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.381+0000 7efe137fe640 1 -- 172.21.15.99:0/3979317387 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7efe24005500 con 0x5560c79aa100 2024-03-19T01:49:51.391 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.381+0000 7efe137fe640 1 -- 172.21.15.99:0/3979317387 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efe24004d80 con 0x5560c79aa100 2024-03-19T01:49:51.391 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.381+0000 7efe3205e740 1 -- 172.21.15.99:0/3979317387 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5560c79aa100 msgr2=0x5560c79aa520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:49:51.391 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.381+0000 7efe3205e740 1 --2- 172.21.15.99:0/3979317387 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5560c79aa100 0x5560c79aa520 secure :-1 s=READY pgs=517 cs=0 l=1 rev1=1 crypto rx=0x7efe240114c0 tx=0x7efe24011980 comp rx=0 tx=0).stop 2024-03-19T01:49:51.391 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.381+0000 7efe3205e740 1 -- 172.21.15.99:0/3979317387 shutdown_connections 2024-03-19T01:49:51.391 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.381+0000 7efe3205e740 1 --2- 172.21.15.99:0/3979317387 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5560c79afc60 0x5560c79b20f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:51.391 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.381+0000 7efe3205e740 1 --2- 172.21.15.99:0/3979317387 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5560c79aaaf0 0x5560c79af5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:51.391 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.382+0000 7efe3205e740 1 --2- 172.21.15.99:0/3979317387 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5560c79aa100 0x5560c79aa520 unknown :-1 s=CLOSED pgs=517 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:51.391 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.382+0000 7efe3205e740 1 -- 172.21.15.99:0/3979317387 >> 172.21.15.99:0/3979317387 conn(0x5560c79a5670 msgr2=0x5560c79a5a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:49:51.391 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.382+0000 7efe3205e740 1 -- 172.21.15.99:0/3979317387 shutdown_connections 2024-03-19T01:49:51.391 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.382+0000 7efe3205e740 1 -- 172.21.15.99:0/3979317387 wait complete. 2024-03-19T01:49:51.392 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.382+0000 7efe3205e740 1 -- start start 2024-03-19T01:49:51.392 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.382+0000 7efe3205e740 1 Processor -- start 2024-03-19T01:49:51.392 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.382+0000 7efe3205e740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5560c79aa100 0x5560c79b40d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:51.392 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.382+0000 7efe3205e740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5560c79aaaf0 0x5560c79b4610 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:51.392 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.382+0000 7efe3205e740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5560c79afc60 0x5560c79bb650 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:51.392 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.382+0000 7efe3205e740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x5560c79b2c10 con 0x5560c79aaaf0 2024-03-19T01:49:51.392 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.382+0000 7efe3205e740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x5560c78cadb0 con 0x5560c79aa100 2024-03-19T01:49:51.392 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.382+0000 7efe3205e740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x5560c79b2d90 con 0x5560c79afc60 2024-03-19T01:49:51.392 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.382+0000 7efe3205e740 10 client.0 _ll_register_callbacks cb 0x5560c78cfe70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:49:51.392 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.382+0000 7efe2ca0e640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5560c79aaaf0 0x5560c79b4610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:51.392 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.382+0000 7efe2d20f640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5560c79aa100 0x5560c79b40d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:51.392 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.382+0000 7efe2ca0e640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5560c79aaaf0 0x5560c79b4610 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:59400/0 (socket says 172.21.15.99:59400) 2024-03-19T01:49:51.393 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.382+0000 7efe2ca0e640 1 -- 172.21.15.99:0/495526637 learned_addr learned my addr 172.21.15.99:0/495526637 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:49:51.393 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.382+0000 7efe2da10640 1 --2- 172.21.15.99:0/495526637 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5560c79afc60 0x5560c79bb650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:51.393 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.383+0000 7efe2d20f640 1 -- 172.21.15.99:0/495526637 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5560c79afc60 msgr2=0x5560c79bb650 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:49:51.393 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.383+0000 7efe2d20f640 1 --2- 172.21.15.99:0/495526637 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5560c79afc60 0x5560c79bb650 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:51.393 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.383+0000 7efe2d20f640 1 -- 172.21.15.99:0/495526637 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5560c79aaaf0 msgr2=0x5560c79b4610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:49:51.393 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.383+0000 7efe2d20f640 1 --2- 172.21.15.99:0/495526637 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5560c79aaaf0 0x5560c79b4610 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:51.393 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.383+0000 7efe2d20f640 1 -- 172.21.15.99:0/495526637 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5560c78c9c60 con 0x5560c79aa100 2024-03-19T01:49:51.393 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.383+0000 7efe2ca0e640 1 --2- 172.21.15.99:0/495526637 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5560c79aaaf0 0x5560c79b4610 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:49:51.393 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.383+0000 7efe2d20f640 1 --2- 172.21.15.99:0/495526637 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5560c79aa100 0x5560c79b40d0 secure :-1 s=READY pgs=134 cs=0 l=1 rev1=1 crypto rx=0x7efe24009530 tx=0x7efe24005270 comp rx=0 tx=0).ready entity=mon.2 client_cookie=9ea8dacac4c78df8 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:49:51.393 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.383+0000 7efe10ff9640 10 client.0 ms_handle_connect on v2:172.21.15.112:3300/0 2024-03-19T01:49:51.393 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.383+0000 7efe10ff9640 1 -- 172.21.15.99:0/495526637 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efe24028020 con 0x5560c79aa100 2024-03-19T01:49:51.393 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.383+0000 7efe10ff9640 1 -- 172.21.15.99:0/495526637 <== mon.2 v2:172.21.15.112:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7efe2400f5c0 con 0x5560c79aa100 2024-03-19T01:49:51.394 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.383+0000 7efe3205e740 1 -- 172.21.15.99:0/495526637 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7efe24005500 con 0x5560c79aa100 2024-03-19T01:49:51.394 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.383+0000 7efe10ff9640 1 -- 172.21.15.99:0/495526637 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7efe24033050 con 0x5560c79aa100 2024-03-19T01:49:51.394 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.383+0000 7efe3205e740 10 client.25814 fetch_fsmap learned FSMap version 27 2024-03-19T01:49:51.394 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.383+0000 7efe3205e740 1 -- 172.21.15.99:0/495526637 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x5560c7623210 con 0x5560c79aa100 2024-03-19T01:49:51.394 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.383+0000 7efe10ff9640 1 -- 172.21.15.99:0/495526637 <== mon.2 v2:172.21.15.112:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7efe24005500 con 0x5560c79aa100 2024-03-19T01:49:51.394 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.384+0000 7efe10ff9640 1 -- 172.21.15.99:0/495526637 <== mon.2 v2:172.21.15.112:3300/0 5 ==== fsmap(e 27) ==== 8338+0+0 (secure 0 0 0) 0x7efe2402f4a0 con 0x5560c79aa100 2024-03-19T01:49:51.394 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.384+0000 7efe3205e740 10 client.25814 fetch_fsmap finished waiting for FSMap version 27 2024-03-19T01:49:51.394 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.384+0000 7efe3205e740 10 client.25814 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:49:51.394 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.384+0000 7efe3205e740 20 client.25814 populate_metadata read hostname 'smithi099' 2024-03-19T01:49:51.394 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.384+0000 7efe3205e740 1 --2- 172.21.15.99:0/495526637 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5560c79bfd80 0x5560c79c2170 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:51.394 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.384+0000 7efe3205e740 4 client.25814 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:49:51.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.384+0000 7efe3205e740 1 -- 172.21.15.99:0/495526637 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5560c791c120 con 0x5560c79bfd80 2024-03-19T01:49:51.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.384+0000 7efe2ca0e640 1 --2- 172.21.15.99:0/495526637 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5560c79bfd80 0x5560c79c2170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:51.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.384+0000 7efe10ff9640 10 client.25814 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:49:51.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.384+0000 7efe2ca0e640 1 --2- 172.21.15.99:0/495526637 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5560c79bfd80 0x5560c79c2170 crc :-1 s=READY pgs=107 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=e070ca1a9809b39 server_cookie=3269898413e1032a in_seq=0 out_seq=0 2024-03-19T01:49:51.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.385+0000 7efe10ff9640 1 -- 172.21.15.99:0/495526637 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x5560c791c120 con 0x5560c79bfd80 2024-03-19T01:49:51.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.385+0000 7efe10ff9640 10 client.25814 handle_command_reply: tid=0 2024-03-19T01:49:51.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.385+0000 7efe3205e740 1 client.25814 shutdown 2024-03-19T01:49:51.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.385+0000 7efe3205e740 1 -- 172.21.15.99:0/495526637 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5560c79aa100 msgr2=0x5560c79b40d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:49:51.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.385+0000 7efe3205e740 1 --2- 172.21.15.99:0/495526637 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5560c79aa100 0x5560c79b40d0 secure :-1 s=READY pgs=134 cs=0 l=1 rev1=1 crypto rx=0x7efe24009530 tx=0x7efe24005270 comp rx=0 tx=0).stop 2024-03-19T01:49:51.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.385+0000 7efe3205e740 1 -- 172.21.15.99:0/495526637 shutdown_connections 2024-03-19T01:49:51.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.385+0000 7efe3205e740 1 --2- 172.21.15.99:0/495526637 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5560c79bfd80 0x5560c79c2170 crc :-1 s=READY pgs=107 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:51.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.385+0000 7efe3205e740 1 --2- 172.21.15.99:0/495526637 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5560c79afc60 0x5560c79bb650 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:51.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.385+0000 7efe3205e740 1 --2- 172.21.15.99:0/495526637 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5560c79aaaf0 0x5560c79b4610 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:51.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.385+0000 7efe10ff9640 0 client.25814 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:49:51.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.385+0000 7efe3205e740 1 --2- 172.21.15.99:0/495526637 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5560c79aa100 0x5560c79b40d0 unknown :-1 s=CLOSED pgs=134 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:51.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.385+0000 7efe3205e740 1 -- 172.21.15.99:0/495526637 >> 172.21.15.99:0/495526637 conn(0x5560c79a5670 msgr2=0x5560c79a68c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:49:51.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.385+0000 7efe3205e740 1 -- 172.21.15.99:0/495526637 shutdown_connections 2024-03-19T01:49:51.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.385+0000 7efe3205e740 1 -- 172.21.15.99:0/495526637 wait complete. 2024-03-19T01:49:51.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.385+0000 7efe3205e740 20 client.25814 trim_cache size 0 max 16384 2024-03-19T01:49:51.405 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.400+0000 7efe3205e740 1 Processor -- start 2024-03-19T01:49:51.405 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.400+0000 7efe3205e740 1 -- start start 2024-03-19T01:49:51.405 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.400+0000 7efe3205e740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5560c79aa100 0x5560c79a8fe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:51.405 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.400+0000 7efe3205e740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5560c79aaaf0 0x5560c79bb610 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:51.405 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.400+0000 7efe3205e740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5560c79afc60 0x5560c79b36f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:51.405 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.400+0000 7efe3205e740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7efe281a9450 con 0x5560c79aaaf0 2024-03-19T01:49:51.405 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.400+0000 7efe3205e740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7efe281a6d20 con 0x5560c79aa100 2024-03-19T01:49:51.405 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.400+0000 7efe3205e740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7efe281a4b70 con 0x5560c79afc60 2024-03-19T01:49:51.405 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.400+0000 7efe2ca0e640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5560c79aaaf0 0x5560c79bb610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:51.406 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.400+0000 7efe2ca0e640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5560c79aaaf0 0x5560c79bb610 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:59412/0 (socket says 172.21.15.99:59412) 2024-03-19T01:49:51.406 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.400+0000 7efe2ca0e640 1 -- 172.21.15.99:0/3640834395 learned_addr learned my addr 172.21.15.99:0/3640834395 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:49:51.406 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.400+0000 7efe2d20f640 1 --2- 172.21.15.99:0/3640834395 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5560c79aa100 0x5560c79a8fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:51.406 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.400+0000 7efe2da10640 1 --2- 172.21.15.99:0/3640834395 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5560c79afc60 0x5560c79b36f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:51.406 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.401+0000 7efe2ca0e640 1 -- 172.21.15.99:0/3640834395 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5560c79afc60 msgr2=0x5560c79b36f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:49:51.406 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.401+0000 7efe2ca0e640 1 --2- 172.21.15.99:0/3640834395 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5560c79afc60 0x5560c79b36f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:51.406 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.401+0000 7efe2ca0e640 1 -- 172.21.15.99:0/3640834395 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5560c79aa100 msgr2=0x5560c79a8fe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:49:51.406 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.401+0000 7efe2ca0e640 1 --2- 172.21.15.99:0/3640834395 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5560c79aa100 0x5560c79a8fe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:51.406 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.401+0000 7efe2ca0e640 1 -- 172.21.15.99:0/3640834395 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5560c76ef620 con 0x5560c79aaaf0 2024-03-19T01:49:51.406 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.401+0000 7efe2d20f640 1 --2- 172.21.15.99:0/3640834395 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5560c79aa100 0x5560c79a8fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:49:51.406 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.401+0000 7efe2ca0e640 1 --2- 172.21.15.99:0/3640834395 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5560c79aaaf0 0x5560c79bb610 secure :-1 s=READY pgs=518 cs=0 l=1 rev1=1 crypto rx=0x5560c79c29b0 tx=0x7efe18005f90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2ce010eca16e9b31 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:49:51.407 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.401+0000 7efe12ffd640 1 -- 172.21.15.99:0/3640834395 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efe1802dba0 con 0x5560c79aaaf0 2024-03-19T01:49:51.407 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.401+0000 7efe12ffd640 1 -- 172.21.15.99:0/3640834395 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7efe1802dd40 con 0x5560c79aaaf0 2024-03-19T01:49:51.407 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.401+0000 7efe12ffd640 1 -- 172.21.15.99:0/3640834395 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efe18005280 con 0x5560c79aaaf0 2024-03-19T01:49:51.407 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.402+0000 7efe3205e740 1 -- 172.21.15.99:0/3640834395 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5560c79aaaf0 msgr2=0x5560c79bb610 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:49:51.407 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.402+0000 7efe3205e740 1 --2- 172.21.15.99:0/3640834395 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5560c79aaaf0 0x5560c79bb610 secure :-1 s=READY pgs=518 cs=0 l=1 rev1=1 crypto rx=0x5560c79c29b0 tx=0x7efe18005f90 comp rx=0 tx=0).stop 2024-03-19T01:49:51.407 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.402+0000 7efe3205e740 1 -- 172.21.15.99:0/3640834395 shutdown_connections 2024-03-19T01:49:51.407 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.402+0000 7efe3205e740 1 --2- 172.21.15.99:0/3640834395 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5560c79afc60 0x5560c79b36f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:51.407 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.402+0000 7efe3205e740 1 --2- 172.21.15.99:0/3640834395 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5560c79aaaf0 0x5560c79bb610 unknown :-1 s=CLOSED pgs=518 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:51.407 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.402+0000 7efe3205e740 1 --2- 172.21.15.99:0/3640834395 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5560c79aa100 0x5560c79a8fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:51.407 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.402+0000 7efe3205e740 1 -- 172.21.15.99:0/3640834395 >> 172.21.15.99:0/3640834395 conn(0x5560c79a5670 msgr2=0x5560c79a88d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:49:51.407 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.402+0000 7efe3205e740 1 -- 172.21.15.99:0/3640834395 shutdown_connections 2024-03-19T01:49:51.407 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.402+0000 7efe3205e740 1 -- 172.21.15.99:0/3640834395 wait complete. 2024-03-19T01:49:51.408 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.402+0000 7efe3205e740 1 -- start start 2024-03-19T01:49:51.408 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.402+0000 7efe3205e740 1 Processor -- start 2024-03-19T01:49:51.408 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.402+0000 7efe3205e740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5560c79aa100 0x5560c79b8a80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:51.408 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.402+0000 7efe3205e740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5560c79aaaf0 0x5560c79b8fc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:51.408 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.402+0000 7efe3205e740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5560c79afc60 0x5560c79b9fd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:51.408 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.402+0000 7efe3205e740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7efe281a51e0 con 0x5560c79afc60 2024-03-19T01:49:51.408 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.402+0000 7efe3205e740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7efe281a6650 con 0x5560c79aa100 2024-03-19T01:49:51.408 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.402+0000 7efe3205e740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7efe281a58b0 con 0x5560c79aaaf0 2024-03-19T01:49:51.408 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.402+0000 7efe3205e740 10 client.0 _ll_register_callbacks cb 0x7efe20049f90 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:49:51.408 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.402+0000 7efe2d20f640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5560c79aa100 0x5560c79b8a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:51.408 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.402+0000 7efe2da10640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5560c79afc60 0x5560c79b9fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:51.408 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.402+0000 7efe2ca0e640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5560c79aaaf0 0x5560c79b8fc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:51.409 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.402+0000 7efe2da10640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5560c79afc60 0x5560c79b9fd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:59428/0 (socket says 172.21.15.99:59428) 2024-03-19T01:49:51.409 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.402+0000 7efe2da10640 1 -- 172.21.15.99:0/1725495846 learned_addr learned my addr 172.21.15.99:0/1725495846 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:49:51.409 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.402+0000 7efe2d20f640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5560c79aa100 0x5560c79b8a80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:33656/0 (socket says 172.21.15.99:33656) 2024-03-19T01:49:51.409 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.403+0000 7efe2da10640 1 -- 172.21.15.99:0/1725495846 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5560c79aaaf0 msgr2=0x5560c79b8fc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:49:51.410 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.403+0000 7efe2da10640 1 --2- 172.21.15.99:0/1725495846 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5560c79aaaf0 0x5560c79b8fc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:51.410 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.403+0000 7efe2da10640 1 -- 172.21.15.99:0/1725495846 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5560c79aa100 msgr2=0x5560c79b8a80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:49:51.410 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.403+0000 7efe2da10640 1 --2- 172.21.15.99:0/1725495846 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5560c79aa100 0x5560c79b8a80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:51.410 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.403+0000 7efe2da10640 1 -- 172.21.15.99:0/1725495846 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5560c79ba5c0 con 0x5560c79afc60 2024-03-19T01:49:51.410 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.403+0000 7efe2da10640 1 --2- 172.21.15.99:0/1725495846 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5560c79afc60 0x5560c79b9fd0 secure :-1 s=READY pgs=519 cs=0 l=1 rev1=1 crypto rx=0x7efe200ee400 tx=0x7efe2004c620 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d406eca1d51ab6ce server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:49:51.410 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.403+0000 7efe10ff9640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:49:51.410 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.403+0000 7efe10ff9640 1 -- 172.21.15.99:0/1725495846 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efe200c1480 con 0x5560c79afc60 2024-03-19T01:49:51.411 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.403+0000 7efe10ff9640 1 -- 172.21.15.99:0/1725495846 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7efe20003970 con 0x5560c79afc60 2024-03-19T01:49:51.411 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.403+0000 7efe3205e740 1 -- 172.21.15.99:0/1725495846 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7efe1802dd40 con 0x5560c79afc60 2024-03-19T01:49:51.411 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.403+0000 7efe10ff9640 1 -- 172.21.15.99:0/1725495846 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7efe200c5a40 con 0x5560c79afc60 2024-03-19T01:49:51.411 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.403+0000 7efe3205e740 10 client.16050 fetch_fsmap learned FSMap version 27 2024-03-19T01:49:51.412 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.403+0000 7efe3205e740 1 -- 172.21.15.99:0/1725495846 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x5560c79bb9a0 con 0x5560c79afc60 2024-03-19T01:49:51.412 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.403+0000 7efe10ff9640 1 -- 172.21.15.99:0/1725495846 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7efe1802dd40 con 0x5560c79afc60 2024-03-19T01:49:51.412 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.403+0000 7efe10ff9640 1 -- 172.21.15.99:0/1725495846 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 27) ==== 8338+0+0 (secure 0 0 0) 0x7efe2003c030 con 0x5560c79afc60 2024-03-19T01:49:51.412 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.403+0000 7efe3205e740 10 client.16050 fetch_fsmap finished waiting for FSMap version 27 2024-03-19T01:49:51.412 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.403+0000 7efe3205e740 10 client.16050 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:49:51.412 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.403+0000 7efe3205e740 20 client.16050 populate_metadata read hostname 'smithi099' 2024-03-19T01:49:51.413 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.404+0000 7efe3205e740 1 --2- 172.21.15.99:0/1725495846 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5560c7a26890 0x5560c7a28cb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:49:51.413 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.404+0000 7efe3205e740 4 client.16050 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:49:51.413 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.404+0000 7efe3205e740 1 -- 172.21.15.99:0/1725495846 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x5560c7916470 con 0x5560c7a26890 2024-03-19T01:49:51.413 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.404+0000 7efe2d20f640 1 --2- 172.21.15.99:0/1725495846 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5560c7a26890 0x5560c7a28cb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:49:51.413 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.404+0000 7efe10ff9640 10 client.16050 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:49:51.413 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.404+0000 7efe2d20f640 1 --2- 172.21.15.99:0/1725495846 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5560c7a26890 0x5560c7a28cb0 crc :-1 s=READY pgs=108 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=83278faf27f867a9 server_cookie=f8056bcf77c02906 in_seq=0 out_seq=0 2024-03-19T01:49:51.694 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1063273 11.39 MB/sec execute 381 sec latency 29.061 ms 2024-03-19T01:49:51.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.768+0000 7efe10ff9640 1 -- 172.21.15.99:0/1725495846 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x5560c7916470 con 0x5560c7a26890 2024-03-19T01:49:51.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.768+0000 7efe10ff9640 10 client.16050 handle_command_reply: tid=0 2024-03-19T01:49:51.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.768+0000 7efe3205e740 1 client.16050 shutdown 2024-03-19T01:49:51.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.768+0000 7efe3205e740 1 -- 172.21.15.99:0/1725495846 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5560c79afc60 msgr2=0x5560c79b9fd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:49:51.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.768+0000 7efe3205e740 1 --2- 172.21.15.99:0/1725495846 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5560c79afc60 0x5560c79b9fd0 secure :-1 s=READY pgs=519 cs=0 l=1 rev1=1 crypto rx=0x7efe200ee400 tx=0x7efe2004c620 comp rx=0 tx=0).stop 2024-03-19T01:49:51.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.768+0000 7efe3205e740 1 -- 172.21.15.99:0/1725495846 shutdown_connections 2024-03-19T01:49:51.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.768+0000 7efe3205e740 1 --2- 172.21.15.99:0/1725495846 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5560c7a26890 0x5560c7a28cb0 crc :-1 s=READY pgs=108 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:51.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.768+0000 7efe3205e740 1 --2- 172.21.15.99:0/1725495846 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5560c79afc60 0x5560c79b9fd0 unknown :-1 s=CLOSED pgs=519 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:51.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.768+0000 7efe3205e740 1 --2- 172.21.15.99:0/1725495846 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5560c79aaaf0 0x5560c79b8fc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:51.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.768+0000 7efe10ff9640 0 client.16050 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:49:51.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.768+0000 7efe3205e740 1 --2- 172.21.15.99:0/1725495846 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5560c79aa100 0x5560c79b8a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:51.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.768+0000 7efe3205e740 1 -- 172.21.15.99:0/1725495846 >> 172.21.15.99:0/1725495846 conn(0x5560c79a5670 msgr2=0x5560c79a9160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:49:51.770 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.768+0000 7efe3205e740 1 -- 172.21.15.99:0/1725495846 shutdown_connections 2024-03-19T01:49:51.770 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.768+0000 7efe3205e740 1 -- 172.21.15.99:0/1725495846 wait complete. 2024-03-19T01:49:51.770 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.768+0000 7efe3205e740 20 client.16050 trim_cache size 0 max 16384 2024-03-19T01:49:51.770 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:49:51.770 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:49:51.770 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:49:51.770 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:49:51.771 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.769+0000 7efe2f473640 1 -- 172.21.15.99:0/1841925824 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7efe140a28a0 msgr2=0x7efe140a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:49:51.771 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.769+0000 7efe2f473640 1 --2- 172.21.15.99:0/1841925824 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7efe140a28a0 0x7efe140a4d60 secure :-1 s=READY pgs=297 cs=0 l=1 rev1=1 crypto rx=0x7efe280697a0 tx=0x7efe180098b0 comp rx=0 tx=0).stop 2024-03-19T01:49:51.771 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.769+0000 7efe2f473640 1 -- 172.21.15.99:0/1841925824 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7efe281022f0 msgr2=0x7efe2819ed90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:49:51.771 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.769+0000 7efe2f473640 1 --2- 172.21.15.99:0/1841925824 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7efe281022f0 0x7efe2819ed90 secure :-1 s=READY pgs=516 cs=0 l=1 rev1=1 crypto rx=0x7efe20037040 tx=0x7efe20032f40 comp rx=0 tx=0).stop 2024-03-19T01:49:51.771 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.769+0000 7efe2f473640 1 -- 172.21.15.99:0/1841925824 shutdown_connections 2024-03-19T01:49:51.771 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.769+0000 7efe2f473640 1 --2- 172.21.15.99:0/1841925824 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7efe281022f0 0x7efe2819ed90 unknown :-1 s=CLOSED pgs=516 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:51.771 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.769+0000 7efe2f473640 1 --2- 172.21.15.99:0/1841925824 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7efe28101930 0x7efe2819c750 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:51.771 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.769+0000 7efe2f473640 1 --2- 172.21.15.99:0/1841925824 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7efe140a28a0 0x7efe140a4d60 unknown :-1 s=CLOSED pgs=297 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:51.772 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.769+0000 7efe2f473640 1 --2- 172.21.15.99:0/1841925824 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7efe28100740 0x7efe28069a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:49:51.772 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.769+0000 7efe2f473640 1 -- 172.21.15.99:0/1841925824 >> 172.21.15.99:0/1841925824 conn(0x7efe280fbcb0 msgr2=0x7efe28104bd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:49:51.772 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.769+0000 7efe2f473640 1 -- 172.21.15.99:0/1841925824 shutdown_connections 2024-03-19T01:49:51.772 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:49:51.769+0000 7efe2f473640 1 -- 172.21.15.99:0/1841925824 wait complete. 2024-03-19T01:49:52.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:49:51 smithi115 ceph-mon[67332]: pgmap v613: 97 pgs: 97 active+clean; 249 MiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 754 KiB/s rd, 2.8 MiB/s wr, 240 op/s 2024-03-19T01:49:52.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:49:51 smithi099 ceph-mon[62951]: pgmap v613: 97 pgs: 97 active+clean; 249 MiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 754 KiB/s rd, 2.8 MiB/s wr, 240 op/s 2024-03-19T01:49:52.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:49:51 smithi112 ceph-mon[66819]: pgmap v613: 97 pgs: 97 active+clean; 249 MiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 754 KiB/s rd, 2.8 MiB/s wr, 240 op/s 2024-03-19T01:49:52.694 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1065829 11.40 MB/sec execute 382 sec latency 60.329 ms 2024-03-19T01:49:53.694 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1067514 11.39 MB/sec execute 383 sec latency 35.034 ms 2024-03-19T01:49:54.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:49:53 smithi115 ceph-mon[67332]: pgmap v614: 97 pgs: 97 active+clean; 252 MiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 923 KiB/s rd, 3.1 MiB/s wr, 272 op/s 2024-03-19T01:49:54.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:49:53 smithi099 ceph-mon[62951]: pgmap v614: 97 pgs: 97 active+clean; 252 MiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 923 KiB/s rd, 3.1 MiB/s wr, 272 op/s 2024-03-19T01:49:54.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:49:53 smithi112 ceph-mon[66819]: pgmap v614: 97 pgs: 97 active+clean; 252 MiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 923 KiB/s rd, 3.1 MiB/s wr, 272 op/s 2024-03-19T01:49:54.694 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1069831 11.40 MB/sec execute 384 sec latency 28.698 ms 2024-03-19T01:49:55.695 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1072458 11.40 MB/sec execute 385 sec latency 39.202 ms 2024-03-19T01:49:56.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:49:55 smithi115 ceph-mon[67332]: pgmap v615: 97 pgs: 97 active+clean; 253 MiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 820 KiB/s rd, 2.7 MiB/s wr, 238 op/s 2024-03-19T01:49:56.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:49:55 smithi099 ceph-mon[62951]: pgmap v615: 97 pgs: 97 active+clean; 253 MiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 820 KiB/s rd, 2.7 MiB/s wr, 238 op/s 2024-03-19T01:49:56.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:49:55 smithi112 ceph-mon[66819]: pgmap v615: 97 pgs: 97 active+clean; 253 MiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 820 KiB/s rd, 2.7 MiB/s wr, 238 op/s 2024-03-19T01:49:56.694 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1074390 11.40 MB/sec execute 386 sec latency 27.546 ms 2024-03-19T01:49:57.695 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1076745 11.41 MB/sec execute 387 sec latency 32.771 ms 2024-03-19T01:49:58.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:49:57 smithi115 ceph-mon[67332]: pgmap v616: 97 pgs: 97 active+clean; 254 MiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 947 KiB/s rd, 3.2 MiB/s wr, 278 op/s 2024-03-19T01:49:58.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:49:57 smithi099 ceph-mon[62951]: pgmap v616: 97 pgs: 97 active+clean; 254 MiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 947 KiB/s rd, 3.2 MiB/s wr, 278 op/s 2024-03-19T01:49:58.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:49:57 smithi112 ceph-mon[66819]: pgmap v616: 97 pgs: 97 active+clean; 254 MiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 947 KiB/s rd, 3.2 MiB/s wr, 278 op/s 2024-03-19T01:49:58.695 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1078778 11.40 MB/sec execute 388 sec latency 62.959 ms 2024-03-19T01:49:59.695 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1080694 11.40 MB/sec execute 389 sec latency 32.880 ms 2024-03-19T01:50:00.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:49:59 smithi115 ceph-mon[67332]: pgmap v617: 97 pgs: 97 active+clean; 257 MiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 895 KiB/s rd, 3.1 MiB/s wr, 281 op/s 2024-03-19T01:50:00.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:49:59 smithi099 ceph-mon[62951]: pgmap v617: 97 pgs: 97 active+clean; 257 MiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 895 KiB/s rd, 3.1 MiB/s wr, 281 op/s 2024-03-19T01:50:00.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:49:59 smithi112 ceph-mon[66819]: pgmap v617: 97 pgs: 97 active+clean; 257 MiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 895 KiB/s rd, 3.1 MiB/s wr, 281 op/s 2024-03-19T01:50:00.695 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1083489 11.41 MB/sec execute 390 sec latency 13.387 ms 2024-03-19T01:50:01.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:50:00 smithi115 ceph-mon[67332]: overall HEALTH_OK 2024-03-19T01:50:01.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:50:00 smithi099 ceph-mon[62951]: overall HEALTH_OK 2024-03-19T01:50:01.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:50:00 smithi112 ceph-mon[66819]: overall HEALTH_OK 2024-03-19T01:50:01.695 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1085282 11.40 MB/sec execute 391 sec latency 27.857 ms 2024-03-19T01:50:02.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:50:01 smithi115 ceph-mon[67332]: pgmap v618: 97 pgs: 97 active+clean; 259 MiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 899 KiB/s rd, 3.0 MiB/s wr, 272 op/s 2024-03-19T01:50:02.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:50:01 smithi099 ceph-mon[62951]: pgmap v618: 97 pgs: 97 active+clean; 259 MiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 899 KiB/s rd, 3.0 MiB/s wr, 272 op/s 2024-03-19T01:50:02.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:50:01 smithi112 ceph-mon[66819]: pgmap v618: 97 pgs: 97 active+clean; 259 MiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 899 KiB/s rd, 3.0 MiB/s wr, 272 op/s 2024-03-19T01:50:02.695 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1087510 11.41 MB/sec execute 392 sec latency 10.260 ms 2024-03-19T01:50:03.695 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1089363 11.40 MB/sec execute 393 sec latency 94.858 ms 2024-03-19T01:50:04.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:50:03 smithi115 ceph-mon[67332]: pgmap v619: 97 pgs: 97 active+clean; 259 MiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 1022 KiB/s rd, 3.3 MiB/s wr, 296 op/s 2024-03-19T01:50:04.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:50:03 smithi099 ceph-mon[62951]: pgmap v619: 97 pgs: 97 active+clean; 259 MiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 1022 KiB/s rd, 3.3 MiB/s wr, 296 op/s 2024-03-19T01:50:04.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:50:03 smithi112 ceph-mon[66819]: pgmap v619: 97 pgs: 97 active+clean; 259 MiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 1022 KiB/s rd, 3.3 MiB/s wr, 296 op/s 2024-03-19T01:50:04.695 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1091311 11.40 MB/sec execute 394 sec latency 25.625 ms 2024-03-19T01:50:04.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.863+0000 7fa02cbdf640 1 -- 172.21.15.99:0/3252327375 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa028101950 msgr2=0x7fa028101dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:50:04.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.863+0000 7fa02cbdf640 1 --2- 172.21.15.99:0/3252327375 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa028101950 0x7fa028101dd0 secure :-1 s=READY pgs=520 cs=0 l=1 rev1=1 crypto rx=0x7fa01c009900 tx=0x7fa01c02bc30 comp rx=0 tx=0).stop 2024-03-19T01:50:04.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.863+0000 7fa02cbdf640 1 -- 172.21.15.99:0/3252327375 shutdown_connections 2024-03-19T01:50:04.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.863+0000 7fa02cbdf640 1 --2- 172.21.15.99:0/3252327375 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa028102310 0x7fa028108ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:04.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.863+0000 7fa02cbdf640 1 --2- 172.21.15.99:0/3252327375 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa028101950 0x7fa028101dd0 unknown :-1 s=CLOSED pgs=520 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:04.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.863+0000 7fa02cbdf640 1 --2- 172.21.15.99:0/3252327375 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa028100760 0x7fa028100b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:04.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.863+0000 7fa02cbdf640 1 -- 172.21.15.99:0/3252327375 >> 172.21.15.99:0/3252327375 conn(0x7fa0280fbcd0 msgr2=0x7fa0280fe110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:50:04.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.863+0000 7fa02cbdf640 1 -- 172.21.15.99:0/3252327375 shutdown_connections 2024-03-19T01:50:04.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.863+0000 7fa02cbdf640 1 -- 172.21.15.99:0/3252327375 wait complete. 2024-03-19T01:50:04.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.864+0000 7fa02cbdf640 1 Processor -- start 2024-03-19T01:50:04.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.864+0000 7fa02cbdf640 1 -- start start 2024-03-19T01:50:04.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.864+0000 7fa02cbdf640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa028100760 0x7fa028197ad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:04.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.864+0000 7fa02cbdf640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa028101950 0x7fa028198010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:04.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.864+0000 7fa02cbdf640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa028102310 0x7fa02819f080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:04.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.864+0000 7fa02cbdf640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fa02810c3c0 con 0x7fa028102310 2024-03-19T01:50:04.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.864+0000 7fa02cbdf640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fa02810c240 con 0x7fa028100760 2024-03-19T01:50:04.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.864+0000 7fa02cbdf640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fa02810c540 con 0x7fa028101950 2024-03-19T01:50:04.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.864+0000 7fa02659c640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa028100760 0x7fa028197ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:04.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.864+0000 7fa026d9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa028102310 0x7fa02819f080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:04.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.864+0000 7fa02659c640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa028100760 0x7fa028197ad0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:44382/0 (socket says 172.21.15.99:44382) 2024-03-19T01:50:04.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.864+0000 7fa026d9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa028102310 0x7fa02819f080 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:60300/0 (socket says 172.21.15.99:60300) 2024-03-19T01:50:04.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.864+0000 7fa025d9b640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa028101950 0x7fa028198010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:04.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.864+0000 7fa02659c640 1 -- 172.21.15.99:0/3729152216 learned_addr learned my addr 172.21.15.99:0/3729152216 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:50:04.867 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.864+0000 7fa02659c640 1 -- 172.21.15.99:0/3729152216 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa028101950 msgr2=0x7fa028198010 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:50:04.867 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.864+0000 7fa02659c640 1 --2- 172.21.15.99:0/3729152216 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa028101950 0x7fa028198010 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:04.867 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.864+0000 7fa02659c640 1 -- 172.21.15.99:0/3729152216 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa028102310 msgr2=0x7fa02819f080 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:50:04.867 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.864+0000 7fa02659c640 1 --2- 172.21.15.99:0/3729152216 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa028102310 0x7fa02819f080 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:04.867 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.864+0000 7fa02659c640 1 -- 172.21.15.99:0/3729152216 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa02819f6f0 con 0x7fa028100760 2024-03-19T01:50:04.867 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.864+0000 7fa026d9d640 1 --2- 172.21.15.99:0/3729152216 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa028102310 0x7fa02819f080 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:50:04.867 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.864+0000 7fa02659c640 1 --2- 172.21.15.99:0/3729152216 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa028100760 0x7fa028197ad0 secure :-1 s=READY pgs=135 cs=0 l=1 rev1=1 crypto rx=0x7fa01800e9c0 tx=0x7fa01800ee80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:50:04.867 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.864+0000 7fa0177fe640 1 -- 172.21.15.99:0/3729152216 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa01800cd80 con 0x7fa028100760 2024-03-19T01:50:04.867 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.865+0000 7fa02cbdf640 1 -- 172.21.15.99:0/3729152216 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa02819f9e0 con 0x7fa028100760 2024-03-19T01:50:04.867 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.865+0000 7fa0177fe640 1 -- 172.21.15.99:0/3729152216 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa0180035f0 con 0x7fa028100760 2024-03-19T01:50:04.867 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.865+0000 7fa0177fe640 1 -- 172.21.15.99:0/3729152216 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa018010690 con 0x7fa028100760 2024-03-19T01:50:04.867 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.865+0000 7fa02cbdf640 1 -- 172.21.15.99:0/3729152216 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fa02819ff00 con 0x7fa028100760 2024-03-19T01:50:04.874 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.867+0000 7fa0177fe640 1 -- 172.21.15.99:0/3729152216 <== mon.2 v2:172.21.15.112:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7fa0180037a0 con 0x7fa028100760 2024-03-19T01:50:04.874 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.867+0000 7fa02f7c8740 1 Processor -- start 2024-03-19T01:50:04.874 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.867+0000 7fa02f7c8740 1 -- start start 2024-03-19T01:50:04.874 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.867+0000 7fa02f7c8740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559329ae4100 0x559329ae4520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:04.874 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.867+0000 7fa02f7c8740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559329ae4af0 0x559329ae95f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:04.874 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.867+0000 7fa02f7c8740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559329ae9c60 0x559329aec0f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:04.874 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.867+0000 7fa02f7c8740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x559329a48be0 con 0x559329ae4100 2024-03-19T01:50:04.874 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.867+0000 7fa02f7c8740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x5593299f1960 con 0x559329ae4af0 2024-03-19T01:50:04.875 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.867+0000 7fa02f7c8740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x559329a44c80 con 0x559329ae9c60 2024-03-19T01:50:04.875 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.868+0000 7fa026d9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559329ae4100 0x559329ae4520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:04.875 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.868+0000 7fa0177fe640 1 --2- 172.21.15.99:0/3729152216 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa0100a28a0 0x7fa0100a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:04.875 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.868+0000 7fa02659c640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559329ae4af0 0x559329ae95f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:04.875 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.868+0000 7fa025d9b640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559329ae9c60 0x559329aec0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:04.875 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.868+0000 7fa026d9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559329ae4100 0x559329ae4520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:60316/0 (socket says 172.21.15.99:60316) 2024-03-19T01:50:04.875 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.868+0000 7fa026d9d640 1 -- 172.21.15.99:0/1460478195 learned_addr learned my addr 172.21.15.99:0/1460478195 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:50:04.875 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.868+0000 7fa026d9d640 1 --2- 172.21.15.99:0/3729152216 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa0100a28a0 0x7fa0100a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:04.875 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.868+0000 7fa0177fe640 1 -- 172.21.15.99:0/3729152216 <== mon.2 v2:172.21.15.112:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7fa0180ceeb0 con 0x7fa028100760 2024-03-19T01:50:04.875 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.868+0000 7fa02659c640 1 -- 172.21.15.99:0/1460478195 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559329ae9c60 msgr2=0x559329aec0f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:50:04.875 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.868+0000 7fa02659c640 1 --2- 172.21.15.99:0/1460478195 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559329ae9c60 0x559329aec0f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:04.875 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.868+0000 7fa02659c640 1 -- 172.21.15.99:0/1460478195 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559329ae4100 msgr2=0x559329ae4520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:50:04.875 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.868+0000 7fa02659c640 1 --2- 172.21.15.99:0/1460478195 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559329ae4100 0x559329ae4520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:04.876 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.868+0000 7fa02659c640 1 -- 172.21.15.99:0/1460478195 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5593299e8d20 con 0x559329ae4af0 2024-03-19T01:50:04.876 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.868+0000 7fa026d9d640 1 --2- 172.21.15.99:0/3729152216 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa0100a28a0 0x7fa0100a4d60 secure :-1 s=READY pgs=299 cs=0 l=1 rev1=1 crypto rx=0x7fa020009fd0 tx=0x7fa0200134c0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:50:04.876 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.868+0000 7fa02659c640 1 --2- 172.21.15.99:0/1460478195 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559329ae4af0 0x559329ae95f0 secure :-1 s=READY pgs=136 cs=0 l=1 rev1=1 crypto rx=0x7fa0180cc8e0 tx=0x7fa01809b110 comp rx=0 tx=0).ready entity=mon.2 client_cookie=892c7bd26a3ada99 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:50:04.876 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.868+0000 7fa014ff9640 1 -- 172.21.15.99:0/1460478195 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa01809beb0 con 0x559329ae4af0 2024-03-19T01:50:04.876 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.868+0000 7fa014ff9640 1 -- 172.21.15.99:0/1460478195 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa01809f900 con 0x559329ae4af0 2024-03-19T01:50:04.876 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.868+0000 7fa014ff9640 1 -- 172.21.15.99:0/1460478195 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa018097e20 con 0x559329ae4af0 2024-03-19T01:50:04.876 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.869+0000 7fa02f7c8740 1 -- 172.21.15.99:0/1460478195 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559329ae4af0 msgr2=0x559329ae95f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:50:04.876 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.869+0000 7fa02f7c8740 1 --2- 172.21.15.99:0/1460478195 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559329ae4af0 0x559329ae95f0 secure :-1 s=READY pgs=136 cs=0 l=1 rev1=1 crypto rx=0x7fa0180cc8e0 tx=0x7fa01809b110 comp rx=0 tx=0).stop 2024-03-19T01:50:04.876 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.869+0000 7fa02f7c8740 1 -- 172.21.15.99:0/1460478195 shutdown_connections 2024-03-19T01:50:04.876 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.869+0000 7fa02f7c8740 1 --2- 172.21.15.99:0/1460478195 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559329ae4af0 0x559329ae95f0 unknown :-1 s=CLOSED pgs=136 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:04.876 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.869+0000 7fa02f7c8740 1 --2- 172.21.15.99:0/1460478195 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559329ae9c60 0x559329aec0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:04.876 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.869+0000 7fa02f7c8740 1 --2- 172.21.15.99:0/1460478195 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559329ae4100 0x559329ae4520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:04.876 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.869+0000 7fa02f7c8740 1 -- 172.21.15.99:0/1460478195 >> 172.21.15.99:0/1460478195 conn(0x559329adf670 msgr2=0x559329adfa70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:50:04.876 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.869+0000 7fa02f7c8740 1 -- 172.21.15.99:0/1460478195 shutdown_connections 2024-03-19T01:50:04.876 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.869+0000 7fa02f7c8740 1 -- 172.21.15.99:0/1460478195 wait complete. 2024-03-19T01:50:04.876 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.869+0000 7fa02f7c8740 1 -- start start 2024-03-19T01:50:04.877 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.870+0000 7fa02f7c8740 1 Processor -- start 2024-03-19T01:50:04.877 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.870+0000 7fa02f7c8740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559329ae4100 0x559329aee050 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:04.877 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.870+0000 7fa02f7c8740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559329ae4af0 0x559329aee590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:04.877 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.870+0000 7fa02f7c8740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559329ae9c60 0x559329af55d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:04.877 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.870+0000 7fa02f7c8740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x559329aecb50 con 0x559329ae9c60 2024-03-19T01:50:04.877 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.870+0000 7fa02f7c8740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x559329a04db0 con 0x559329ae4100 2024-03-19T01:50:04.877 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.870+0000 7fa02f7c8740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x559329aeccd0 con 0x559329ae4af0 2024-03-19T01:50:04.877 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.870+0000 7fa02f7c8740 10 client.0 _ll_register_callbacks cb 0x559329a09e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:50:04.877 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.870+0000 7fa02659c640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559329ae4100 0x559329aee050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:04.877 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.870+0000 7fa026d9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559329ae9c60 0x559329af55d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:04.877 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.870+0000 7fa025d9b640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559329ae4af0 0x559329aee590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:04.877 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.870+0000 7fa02659c640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559329ae4100 0x559329aee050 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:44388/0 (socket says 172.21.15.99:44388) 2024-03-19T01:50:04.877 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.870+0000 7fa02659c640 1 -- 172.21.15.99:0/2724998646 learned_addr learned my addr 172.21.15.99:0/2724998646 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:50:04.877 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.870+0000 7fa02659c640 1 -- 172.21.15.99:0/2724998646 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559329ae4af0 msgr2=0x559329aee590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:50:04.878 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.870+0000 7fa02659c640 1 --2- 172.21.15.99:0/2724998646 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559329ae4af0 0x559329aee590 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:04.878 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.870+0000 7fa02659c640 1 -- 172.21.15.99:0/2724998646 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559329ae9c60 msgr2=0x559329af55d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:50:04.878 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.870+0000 7fa02659c640 1 --2- 172.21.15.99:0/2724998646 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559329ae9c60 0x559329af55d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:04.878 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.870+0000 7fa02659c640 1 -- 172.21.15.99:0/2724998646 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x559329a03c60 con 0x559329ae4100 2024-03-19T01:50:04.878 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.870+0000 7fa02659c640 1 --2- 172.21.15.99:0/2724998646 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559329ae4100 0x559329aee050 secure :-1 s=READY pgs=137 cs=0 l=1 rev1=1 crypto rx=0x7fa01809b4b0 tx=0x7fa018097a50 comp rx=0 tx=0).ready entity=mon.2 client_cookie=39420ed15aae3b17 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:50:04.878 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.870+0000 7fa00dffb640 10 client.0 ms_handle_connect on v2:172.21.15.112:3300/0 2024-03-19T01:50:04.878 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.871+0000 7fa00dffb640 1 -- 172.21.15.99:0/2724998646 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa0180b3070 con 0x559329ae4100 2024-03-19T01:50:04.878 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.871+0000 7fa00dffb640 1 -- 172.21.15.99:0/2724998646 <== mon.2 v2:172.21.15.112:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7fa01809ba10 con 0x559329ae4100 2024-03-19T01:50:04.878 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.871+0000 7fa02f7c8740 1 -- 172.21.15.99:0/2724998646 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fa01809f900 con 0x559329ae4100 2024-03-19T01:50:04.878 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.871+0000 7fa00dffb640 1 -- 172.21.15.99:0/2724998646 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7fa01809beb0 con 0x559329ae4100 2024-03-19T01:50:04.878 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.871+0000 7fa02f7c8740 10 client.25832 fetch_fsmap learned FSMap version 27 2024-03-19T01:50:04.878 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.871+0000 7fa02f7c8740 1 -- 172.21.15.99:0/2724998646 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x55932975d210 con 0x559329ae4100 2024-03-19T01:50:04.878 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.871+0000 7fa00dffb640 1 -- 172.21.15.99:0/2724998646 <== mon.2 v2:172.21.15.112:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa01809f900 con 0x559329ae4100 2024-03-19T01:50:04.879 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.871+0000 7fa00dffb640 1 -- 172.21.15.99:0/2724998646 <== mon.2 v2:172.21.15.112:3300/0 5 ==== fsmap(e 27) ==== 8338+0+0 (secure 0 0 0) 0x7fa018097590 con 0x559329ae4100 2024-03-19T01:50:04.879 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.871+0000 7fa02f7c8740 10 client.25832 fetch_fsmap finished waiting for FSMap version 27 2024-03-19T01:50:04.879 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.871+0000 7fa02f7c8740 10 client.25832 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:50:04.879 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.871+0000 7fa02f7c8740 20 client.25832 populate_metadata read hostname 'smithi099' 2024-03-19T01:50:04.879 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.871+0000 7fa02f7c8740 1 --2- 172.21.15.99:0/2724998646 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x559329af9ac0 0x559329afbeb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:04.879 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.871+0000 7fa02f7c8740 4 client.25832 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:50:04.879 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.871+0000 7fa02f7c8740 1 -- 172.21.15.99:0/2724998646 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x559329a56120 con 0x559329af9ac0 2024-03-19T01:50:04.879 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.871+0000 7fa025d9b640 1 --2- 172.21.15.99:0/2724998646 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x559329af9ac0 0x559329afbeb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:04.879 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.872+0000 7fa00dffb640 10 client.25832 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:50:04.879 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.872+0000 7fa025d9b640 1 --2- 172.21.15.99:0/2724998646 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x559329af9ac0 0x559329afbeb0 crc :-1 s=READY pgs=109 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=7daf812823d4d860 server_cookie=11b67b197fbdf5b1 in_seq=0 out_seq=0 2024-03-19T01:50:04.879 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.872+0000 7fa00dffb640 1 -- 172.21.15.99:0/2724998646 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x559329a56120 con 0x559329af9ac0 2024-03-19T01:50:04.879 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.872+0000 7fa00dffb640 10 client.25832 handle_command_reply: tid=0 2024-03-19T01:50:04.879 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.872+0000 7fa02f7c8740 1 client.25832 shutdown 2024-03-19T01:50:04.879 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.873+0000 7fa02f7c8740 1 -- 172.21.15.99:0/2724998646 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559329ae4100 msgr2=0x559329aee050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:50:04.879 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.873+0000 7fa02f7c8740 1 --2- 172.21.15.99:0/2724998646 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559329ae4100 0x559329aee050 secure :-1 s=READY pgs=137 cs=0 l=1 rev1=1 crypto rx=0x7fa01809b4b0 tx=0x7fa018097a50 comp rx=0 tx=0).stop 2024-03-19T01:50:04.880 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.873+0000 7fa02f7c8740 1 -- 172.21.15.99:0/2724998646 shutdown_connections 2024-03-19T01:50:04.880 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.873+0000 7fa02f7c8740 1 --2- 172.21.15.99:0/2724998646 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x559329af9ac0 0x559329afbeb0 crc :-1 s=READY pgs=109 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:04.880 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.873+0000 7fa02f7c8740 1 --2- 172.21.15.99:0/2724998646 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559329ae9c60 0x559329af55d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:04.880 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.873+0000 7fa02f7c8740 1 --2- 172.21.15.99:0/2724998646 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559329ae4af0 0x559329aee590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:04.880 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.873+0000 7fa02f7c8740 1 --2- 172.21.15.99:0/2724998646 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559329ae4100 0x559329aee050 unknown :-1 s=CLOSED pgs=137 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:04.880 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.873+0000 7fa00dffb640 0 client.25832 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:50:04.880 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.873+0000 7fa02f7c8740 1 -- 172.21.15.99:0/2724998646 >> 172.21.15.99:0/2724998646 conn(0x559329adf670 msgr2=0x559329ae08c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:50:04.880 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.873+0000 7fa02f7c8740 1 -- 172.21.15.99:0/2724998646 shutdown_connections 2024-03-19T01:50:04.880 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.873+0000 7fa02f7c8740 1 -- 172.21.15.99:0/2724998646 wait complete. 2024-03-19T01:50:04.880 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.873+0000 7fa02f7c8740 20 client.25832 trim_cache size 0 max 16384 2024-03-19T01:50:04.888 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.888+0000 7fa02f7c8740 1 Processor -- start 2024-03-19T01:50:04.889 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.888+0000 7fa02f7c8740 1 -- start start 2024-03-19T01:50:04.889 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.888+0000 7fa02f7c8740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559329ae4100 0x559329aee370 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:04.889 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.888+0000 7fa02f7c8740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559329ae4af0 0x559329af5590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:04.889 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.888+0000 7fa02f7c8740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559329ae9c60 0x559329ae5560 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:04.889 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.888+0000 7fa02f7c8740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fa0281a9450 con 0x559329ae4100 2024-03-19T01:50:04.889 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.888+0000 7fa02f7c8740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fa0281a6d20 con 0x559329ae9c60 2024-03-19T01:50:04.889 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.888+0000 7fa02f7c8740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fa0281a4b70 con 0x559329ae4af0 2024-03-19T01:50:04.889 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.888+0000 7fa02659c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559329ae4100 0x559329aee370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:04.889 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.888+0000 7fa02659c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559329ae4100 0x559329aee370 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:60334/0 (socket says 172.21.15.99:60334) 2024-03-19T01:50:04.889 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.888+0000 7fa025d9b640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559329ae4af0 0x559329af5590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:04.889 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.888+0000 7fa026d9d640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559329ae9c60 0x559329ae5560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:04.889 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.888+0000 7fa02659c640 1 -- 172.21.15.99:0/817436761 learned_addr learned my addr 172.21.15.99:0/817436761 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:50:04.889 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.888+0000 7fa025d9b640 1 -- 172.21.15.99:0/817436761 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559329ae9c60 msgr2=0x559329ae5560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:50:04.890 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.888+0000 7fa025d9b640 1 --2- 172.21.15.99:0/817436761 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559329ae9c60 0x559329ae5560 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:04.890 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.888+0000 7fa025d9b640 1 -- 172.21.15.99:0/817436761 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559329ae4100 msgr2=0x559329aee370 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:50:04.890 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.888+0000 7fa025d9b640 1 --2- 172.21.15.99:0/817436761 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559329ae4100 0x559329aee370 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:04.890 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.888+0000 7fa025d9b640 1 -- 172.21.15.99:0/817436761 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x559329829620 con 0x559329ae4af0 2024-03-19T01:50:04.890 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.888+0000 7fa02659c640 1 --2- 172.21.15.99:0/817436761 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559329ae4100 0x559329aee370 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:50:04.890 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.889+0000 7fa025d9b640 1 --2- 172.21.15.99:0/817436761 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559329ae4af0 0x559329af5590 secure :-1 s=READY pgs=103 cs=0 l=1 rev1=1 crypto rx=0x7fa01c009230 tx=0x7fa01c005fd0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=fba2a6df786c6f5b server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:50:04.890 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.889+0000 7fa00ffff640 1 -- 172.21.15.99:0/817436761 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa01c02cb20 con 0x559329ae4af0 2024-03-19T01:50:04.890 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.889+0000 7fa00ffff640 1 -- 172.21.15.99:0/817436761 <== mon.1 v2:172.21.15.115:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa01c02ccc0 con 0x559329ae4af0 2024-03-19T01:50:04.890 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.889+0000 7fa00ffff640 1 -- 172.21.15.99:0/817436761 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa01c004480 con 0x559329ae4af0 2024-03-19T01:50:04.895 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.889+0000 7fa02f7c8740 1 -- 172.21.15.99:0/817436761 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559329ae4af0 msgr2=0x559329af5590 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:50:04.895 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.889+0000 7fa02f7c8740 1 --2- 172.21.15.99:0/817436761 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559329ae4af0 0x559329af5590 secure :-1 s=READY pgs=103 cs=0 l=1 rev1=1 crypto rx=0x7fa01c009230 tx=0x7fa01c005fd0 comp rx=0 tx=0).stop 2024-03-19T01:50:04.895 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.890+0000 7fa02f7c8740 1 -- 172.21.15.99:0/817436761 shutdown_connections 2024-03-19T01:50:04.895 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.890+0000 7fa02f7c8740 1 --2- 172.21.15.99:0/817436761 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559329ae9c60 0x559329ae5560 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:04.895 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.890+0000 7fa02f7c8740 1 --2- 172.21.15.99:0/817436761 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559329ae4af0 0x559329af5590 unknown :-1 s=CLOSED pgs=103 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:04.895 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.890+0000 7fa02f7c8740 1 --2- 172.21.15.99:0/817436761 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559329ae4100 0x559329aee370 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:04.895 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.890+0000 7fa02f7c8740 1 -- 172.21.15.99:0/817436761 >> 172.21.15.99:0/817436761 conn(0x559329adf670 msgr2=0x559329aedc60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:50:04.895 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.890+0000 7fa02f7c8740 1 -- 172.21.15.99:0/817436761 shutdown_connections 2024-03-19T01:50:04.895 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.890+0000 7fa02f7c8740 1 -- 172.21.15.99:0/817436761 wait complete. 2024-03-19T01:50:04.895 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.890+0000 7fa02f7c8740 1 -- start start 2024-03-19T01:50:04.895 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.890+0000 7fa02f7c8740 1 Processor -- start 2024-03-19T01:50:04.895 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.890+0000 7fa02f7c8740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559329ae4100 0x559329af31f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:04.896 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.890+0000 7fa02f7c8740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559329ae4af0 0x559329af3730 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:04.896 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.890+0000 7fa02f7c8740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559329ae9c60 0x559329af4740 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:04.896 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.890+0000 7fa02f7c8740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fa0281a51e0 con 0x559329ae4af0 2024-03-19T01:50:04.896 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.890+0000 7fa02f7c8740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fa0281a6650 con 0x559329ae9c60 2024-03-19T01:50:04.896 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.890+0000 7fa02f7c8740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fa0281a58b0 con 0x559329ae4100 2024-03-19T01:50:04.896 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.890+0000 7fa02f7c8740 10 client.0 _ll_register_callbacks cb 0x7fa020013850 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:50:04.896 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.890+0000 7fa025d9b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559329ae4af0 0x559329af3730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:04.896 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.890+0000 7fa02659c640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559329ae4100 0x559329af31f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:04.896 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.890+0000 7fa025d9b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559329ae4af0 0x559329af3730 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:60342/0 (socket says 172.21.15.99:60342) 2024-03-19T01:50:04.896 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.891+0000 7fa025d9b640 1 -- 172.21.15.99:0/3941469032 learned_addr learned my addr 172.21.15.99:0/3941469032 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:50:04.896 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.891+0000 7fa026d9d640 1 --2- 172.21.15.99:0/3941469032 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559329ae9c60 0x559329af4740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:04.896 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.891+0000 7fa025d9b640 1 -- 172.21.15.99:0/3941469032 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559329ae4100 msgr2=0x559329af31f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:50:04.896 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.891+0000 7fa025d9b640 1 --2- 172.21.15.99:0/3941469032 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559329ae4100 0x559329af31f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:04.896 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.891+0000 7fa025d9b640 1 -- 172.21.15.99:0/3941469032 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559329ae9c60 msgr2=0x559329af4740 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:50:04.897 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.891+0000 7fa025d9b640 1 --2- 172.21.15.99:0/3941469032 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559329ae9c60 0x559329af4740 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:04.897 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.891+0000 7fa025d9b640 1 -- 172.21.15.99:0/3941469032 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x559329af4d30 con 0x559329ae4af0 2024-03-19T01:50:04.897 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.891+0000 7fa025d9b640 1 --2- 172.21.15.99:0/3941469032 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559329ae4af0 0x559329af3730 secure :-1 s=READY pgs=521 cs=0 l=1 rev1=1 crypto rx=0x7fa01c009900 tx=0x7fa01c02cba0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=85a87320559cfbe4 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:50:04.897 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.891+0000 7fa00dffb640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:50:04.897 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.891+0000 7fa00dffb640 1 -- 172.21.15.99:0/3941469032 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa01c052020 con 0x559329ae4af0 2024-03-19T01:50:04.897 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.891+0000 7fa00dffb640 1 -- 172.21.15.99:0/3941469032 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7fa01c0041c0 con 0x559329ae4af0 2024-03-19T01:50:04.897 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.891+0000 7fa02f7c8740 1 -- 172.21.15.99:0/3941469032 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fa01c02ccc0 con 0x559329ae4af0 2024-03-19T01:50:04.897 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.891+0000 7fa00dffb640 1 -- 172.21.15.99:0/3941469032 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7fa01c0043c0 con 0x559329ae4af0 2024-03-19T01:50:04.897 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.891+0000 7fa02f7c8740 10 client.16068 fetch_fsmap learned FSMap version 27 2024-03-19T01:50:04.897 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.891+0000 7fa02f7c8740 1 -- 172.21.15.99:0/3941469032 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x559329b60180 con 0x559329ae4af0 2024-03-19T01:50:04.897 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.892+0000 7fa00dffb640 1 -- 172.21.15.99:0/3941469032 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa01c02ccc0 con 0x559329ae4af0 2024-03-19T01:50:04.897 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.892+0000 7fa00dffb640 1 -- 172.21.15.99:0/3941469032 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 27) ==== 8338+0+0 (secure 0 0 0) 0x7fa01c0046e0 con 0x559329ae4af0 2024-03-19T01:50:04.897 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.892+0000 7fa02f7c8740 10 client.16068 fetch_fsmap finished waiting for FSMap version 27 2024-03-19T01:50:04.898 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.892+0000 7fa02f7c8740 10 client.16068 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:50:04.898 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.892+0000 7fa02f7c8740 20 client.16068 populate_metadata read hostname 'smithi099' 2024-03-19T01:50:04.898 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.892+0000 7fa02f7c8740 1 --2- 172.21.15.99:0/3941469032 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x559329b60ac0 0x559329b62ee0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:04.898 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.892+0000 7fa02f7c8740 4 client.16068 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:50:04.898 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.892+0000 7fa02f7c8740 1 -- 172.21.15.99:0/3941469032 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x559329a50470 con 0x559329b60ac0 2024-03-19T01:50:04.898 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.892+0000 7fa02659c640 1 --2- 172.21.15.99:0/3941469032 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x559329b60ac0 0x559329b62ee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:04.898 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.892+0000 7fa00dffb640 10 client.16068 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:50:04.898 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.892+0000 7fa02659c640 1 --2- 172.21.15.99:0/3941469032 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x559329b60ac0 0x559329b62ee0 crc :-1 s=READY pgs=110 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=cb73118eb144aec server_cookie=6b5c1829a8fdeedd in_seq=0 out_seq=0 2024-03-19T01:50:04.953 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.953+0000 7fa00dffb640 1 -- 172.21.15.99:0/3941469032 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x559329a50470 con 0x559329b60ac0 2024-03-19T01:50:04.953 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.953+0000 7fa00dffb640 10 client.16068 handle_command_reply: tid=0 2024-03-19T01:50:04.953 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.953+0000 7fa02f7c8740 1 client.16068 shutdown 2024-03-19T01:50:04.953 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.953+0000 7fa02f7c8740 1 -- 172.21.15.99:0/3941469032 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559329ae4af0 msgr2=0x559329af3730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:50:04.954 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.953+0000 7fa02f7c8740 1 --2- 172.21.15.99:0/3941469032 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559329ae4af0 0x559329af3730 secure :-1 s=READY pgs=521 cs=0 l=1 rev1=1 crypto rx=0x7fa01c009900 tx=0x7fa01c02cba0 comp rx=0 tx=0).stop 2024-03-19T01:50:04.954 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.953+0000 7fa02f7c8740 1 -- 172.21.15.99:0/3941469032 shutdown_connections 2024-03-19T01:50:04.954 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.953+0000 7fa02f7c8740 1 --2- 172.21.15.99:0/3941469032 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x559329b60ac0 0x559329b62ee0 crc :-1 s=READY pgs=110 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:04.954 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.953+0000 7fa02f7c8740 1 --2- 172.21.15.99:0/3941469032 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559329ae9c60 0x559329af4740 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:04.954 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.953+0000 7fa00dffb640 0 client.16068 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:50:04.954 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.953+0000 7fa02f7c8740 1 --2- 172.21.15.99:0/3941469032 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559329ae4af0 0x559329af3730 unknown :-1 s=CLOSED pgs=521 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:04.954 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.953+0000 7fa02f7c8740 1 --2- 172.21.15.99:0/3941469032 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559329ae4100 0x559329af31f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:04.954 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.953+0000 7fa02f7c8740 1 -- 172.21.15.99:0/3941469032 >> 172.21.15.99:0/3941469032 conn(0x559329adf670 msgr2=0x559329ae28d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:50:04.954 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.953+0000 7fa02f7c8740 1 -- 172.21.15.99:0/3941469032 shutdown_connections 2024-03-19T01:50:04.954 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.953+0000 7fa02f7c8740 1 -- 172.21.15.99:0/3941469032 wait complete. 2024-03-19T01:50:04.954 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.953+0000 7fa02f7c8740 20 client.16068 trim_cache size 0 max 16384 2024-03-19T01:50:04.955 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:50:04.955 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:50:04.955 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:50:04.955 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:50:04.956 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.953+0000 7fa02cbdf640 1 -- 172.21.15.99:0/3729152216 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa0100a28a0 msgr2=0x7fa0100a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:50:04.956 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.953+0000 7fa02cbdf640 1 --2- 172.21.15.99:0/3729152216 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa0100a28a0 0x7fa0100a4d60 secure :-1 s=READY pgs=299 cs=0 l=1 rev1=1 crypto rx=0x7fa020009fd0 tx=0x7fa0200134c0 comp rx=0 tx=0).stop 2024-03-19T01:50:04.956 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.953+0000 7fa02cbdf640 1 -- 172.21.15.99:0/3729152216 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa028100760 msgr2=0x7fa028197ad0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:50:04.956 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.953+0000 7fa02cbdf640 1 --2- 172.21.15.99:0/3729152216 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa028100760 0x7fa028197ad0 secure :-1 s=READY pgs=135 cs=0 l=1 rev1=1 crypto rx=0x7fa01800e9c0 tx=0x7fa01800ee80 comp rx=0 tx=0).stop 2024-03-19T01:50:04.956 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.953+0000 7fa02cbdf640 1 -- 172.21.15.99:0/3729152216 shutdown_connections 2024-03-19T01:50:04.956 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.953+0000 7fa02cbdf640 1 --2- 172.21.15.99:0/3729152216 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa028102310 0x7fa02819f080 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:04.956 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.953+0000 7fa02cbdf640 1 --2- 172.21.15.99:0/3729152216 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa028101950 0x7fa028198010 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:04.956 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.953+0000 7fa02cbdf640 1 --2- 172.21.15.99:0/3729152216 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa0100a28a0 0x7fa0100a4d60 unknown :-1 s=CLOSED pgs=299 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:04.956 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.954+0000 7fa02cbdf640 1 --2- 172.21.15.99:0/3729152216 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa028100760 0x7fa028197ad0 unknown :-1 s=CLOSED pgs=135 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:04.956 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.954+0000 7fa02cbdf640 1 -- 172.21.15.99:0/3729152216 >> 172.21.15.99:0/3729152216 conn(0x7fa0280fbcd0 msgr2=0x7fa0280fd6f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:50:04.956 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.954+0000 7fa02cbdf640 1 -- 172.21.15.99:0/3729152216 shutdown_connections 2024-03-19T01:50:04.956 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:04.954+0000 7fa02cbdf640 1 -- 172.21.15.99:0/3729152216 wait complete. 2024-03-19T01:50:05.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:50:04 smithi115 ceph-mon[67332]: pgmap v620: 97 pgs: 97 active+clean; 260 MiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 706 KiB/s rd, 2.7 MiB/s wr, 222 op/s 2024-03-19T01:50:05.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:50:04 smithi099 ceph-mon[62951]: pgmap v620: 97 pgs: 97 active+clean; 260 MiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 706 KiB/s rd, 2.7 MiB/s wr, 222 op/s 2024-03-19T01:50:05.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:50:04 smithi112 ceph-mon[66819]: pgmap v620: 97 pgs: 97 active+clean; 260 MiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 706 KiB/s rd, 2.7 MiB/s wr, 222 op/s 2024-03-19T01:50:05.695 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1093998 11.41 MB/sec execute 395 sec latency 16.495 ms 2024-03-19T01:50:06.695 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1095676 11.40 MB/sec execute 396 sec latency 32.557 ms 2024-03-19T01:50:07.695 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1098044 11.41 MB/sec execute 397 sec latency 9.956 ms 2024-03-19T01:50:08.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:50:07 smithi115 ceph-mon[67332]: pgmap v621: 97 pgs: 97 active+clean; 262 MiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 1019 KiB/s rd, 3.2 MiB/s wr, 288 op/s 2024-03-19T01:50:08.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:50:07 smithi099 ceph-mon[62951]: pgmap v621: 97 pgs: 97 active+clean; 262 MiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 1019 KiB/s rd, 3.2 MiB/s wr, 288 op/s 2024-03-19T01:50:08.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:50:07 smithi112 ceph-mon[66819]: pgmap v621: 97 pgs: 97 active+clean; 262 MiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 1019 KiB/s rd, 3.2 MiB/s wr, 288 op/s 2024-03-19T01:50:08.695 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1099800 11.39 MB/sec execute 398 sec latency 43.731 ms 2024-03-19T01:50:09.695 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1101752 11.40 MB/sec execute 399 sec latency 27.891 ms 2024-03-19T01:50:10.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:50:09 smithi115 ceph-mon[67332]: pgmap v622: 97 pgs: 97 active+clean; 260 MiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 892 KiB/s rd, 2.8 MiB/s wr, 264 op/s 2024-03-19T01:50:10.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:50:09 smithi099 ceph-mon[62951]: pgmap v622: 97 pgs: 97 active+clean; 260 MiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 892 KiB/s rd, 2.8 MiB/s wr, 264 op/s 2024-03-19T01:50:10.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:50:09 smithi112 ceph-mon[66819]: pgmap v622: 97 pgs: 97 active+clean; 260 MiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 892 KiB/s rd, 2.8 MiB/s wr, 264 op/s 2024-03-19T01:50:10.695 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1104166 11.40 MB/sec execute 400 sec latency 18.287 ms 2024-03-19T01:50:11.695 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1106078 11.40 MB/sec execute 401 sec latency 27.442 ms 2024-03-19T01:50:12.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:50:11 smithi115 ceph-mon[67332]: pgmap v623: 97 pgs: 97 active+clean; 264 MiB data, 3.0 GiB used, 1.0 TiB / 1.0 TiB avail; 827 KiB/s rd, 2.9 MiB/s wr, 265 op/s 2024-03-19T01:50:12.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:50:11 smithi099 ceph-mon[62951]: pgmap v623: 97 pgs: 97 active+clean; 264 MiB data, 3.0 GiB used, 1.0 TiB / 1.0 TiB avail; 827 KiB/s rd, 2.9 MiB/s wr, 265 op/s 2024-03-19T01:50:12.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:50:11 smithi112 ceph-mon[66819]: pgmap v623: 97 pgs: 97 active+clean; 264 MiB data, 3.0 GiB used, 1.0 TiB / 1.0 TiB avail; 827 KiB/s rd, 2.9 MiB/s wr, 265 op/s 2024-03-19T01:50:12.695 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1108562 11.41 MB/sec execute 402 sec latency 13.432 ms 2024-03-19T01:50:13.695 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1109927 11.39 MB/sec execute 403 sec latency 63.638 ms 2024-03-19T01:50:14.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:50:13 smithi115 ceph-mon[67332]: pgmap v624: 97 pgs: 97 active+clean; 264 MiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 774 KiB/s rd, 3.0 MiB/s wr, 256 op/s 2024-03-19T01:50:14.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:50:13 smithi099 ceph-mon[62951]: pgmap v624: 97 pgs: 97 active+clean; 264 MiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 774 KiB/s rd, 3.0 MiB/s wr, 256 op/s 2024-03-19T01:50:14.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:50:13 smithi112 ceph-mon[66819]: pgmap v624: 97 pgs: 97 active+clean; 264 MiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 774 KiB/s rd, 3.0 MiB/s wr, 256 op/s 2024-03-19T01:50:14.695 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1112209 11.40 MB/sec execute 404 sec latency 40.684 ms 2024-03-19T01:50:15.695 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1114391 11.39 MB/sec execute 405 sec latency 28.858 ms 2024-03-19T01:50:16.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:50:15 smithi115 ceph-mon[67332]: pgmap v625: 97 pgs: 97 active+clean; 266 MiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 753 KiB/s rd, 2.6 MiB/s wr, 238 op/s 2024-03-19T01:50:16.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:50:15 smithi099 ceph-mon[62951]: pgmap v625: 97 pgs: 97 active+clean; 266 MiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 753 KiB/s rd, 2.6 MiB/s wr, 238 op/s 2024-03-19T01:50:16.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:50:15 smithi112 ceph-mon[66819]: pgmap v625: 97 pgs: 97 active+clean; 266 MiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 753 KiB/s rd, 2.6 MiB/s wr, 238 op/s 2024-03-19T01:50:16.695 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1116005 11.39 MB/sec execute 406 sec latency 62.245 ms 2024-03-19T01:50:17.696 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1118533 11.40 MB/sec execute 407 sec latency 11.923 ms 2024-03-19T01:50:18.050 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.048+0000 7ff0a7f05640 1 -- 172.21.15.99:0/3321611464 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff0a0103420 msgr2=0x7ff0a0109cb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:50:18.051 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.048+0000 7ff0a7f05640 1 --2- 172.21.15.99:0/3321611464 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff0a0103420 0x7ff0a0109cb0 secure :-1 s=READY pgs=522 cs=0 l=1 rev1=1 crypto rx=0x7ff098009ca0 tx=0x7ff09802c0f0 comp rx=0 tx=0).stop 2024-03-19T01:50:18.051 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.049+0000 7ff0a7f05640 1 -- 172.21.15.99:0/3321611464 shutdown_connections 2024-03-19T01:50:18.051 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.049+0000 7ff0a7f05640 1 --2- 172.21.15.99:0/3321611464 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff0a0103420 0x7ff0a0109cb0 unknown :-1 s=CLOSED pgs=522 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:18.051 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.049+0000 7ff0a7f05640 1 --2- 172.21.15.99:0/3321611464 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff0a0102a60 0x7ff0a0102ee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:18.051 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.049+0000 7ff0a7f05640 1 --2- 172.21.15.99:0/3321611464 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff0a0101870 0x7ff0a0101c90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:18.051 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.049+0000 7ff0a7f05640 1 -- 172.21.15.99:0/3321611464 >> 172.21.15.99:0/3321611464 conn(0x7ff0a00fcde0 msgr2=0x7ff0a00ff220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:50:18.051 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.049+0000 7ff0a7f05640 1 -- 172.21.15.99:0/3321611464 shutdown_connections 2024-03-19T01:50:18.051 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.049+0000 7ff0a7f05640 1 -- 172.21.15.99:0/3321611464 wait complete. 2024-03-19T01:50:18.051 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.049+0000 7ff0a7f05640 1 Processor -- start 2024-03-19T01:50:18.051 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.049+0000 7ff0a7f05640 1 -- start start 2024-03-19T01:50:18.051 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.049+0000 7ff0a7f05640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff0a0101870 0x7ff0a0197a70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:18.052 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.049+0000 7ff0a7f05640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff0a0102a60 0x7ff0a0197fb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:18.052 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.049+0000 7ff0a7f05640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff0a0103420 0x7ff0a019f020 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:18.052 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.049+0000 7ff0a7f05640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7ff0a010d4f0 con 0x7ff0a0102a60 2024-03-19T01:50:18.052 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.049+0000 7ff0a7f05640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7ff0a010d370 con 0x7ff0a0103420 2024-03-19T01:50:18.052 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.049+0000 7ff0a7f05640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7ff0a010d670 con 0x7ff0a0101870 2024-03-19T01:50:18.052 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.049+0000 7ff0a5ca1640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff0a0101870 0x7ff0a0197a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:18.052 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.050+0000 7ff0a54a0640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff0a0102a60 0x7ff0a0197fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:18.052 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.050+0000 7ff0a64a2640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff0a0103420 0x7ff0a019f020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:18.052 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.050+0000 7ff0a5ca1640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff0a0101870 0x7ff0a0197a70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:40744/0 (socket says 172.21.15.99:40744) 2024-03-19T01:50:18.052 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.050+0000 7ff0a5ca1640 1 -- 172.21.15.99:0/3044258732 learned_addr learned my addr 172.21.15.99:0/3044258732 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:50:18.052 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.050+0000 7ff0a5ca1640 1 -- 172.21.15.99:0/3044258732 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff0a0103420 msgr2=0x7ff0a019f020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:50:18.052 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.050+0000 7ff0a5ca1640 1 --2- 172.21.15.99:0/3044258732 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff0a0103420 0x7ff0a019f020 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:18.052 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.050+0000 7ff0a5ca1640 1 -- 172.21.15.99:0/3044258732 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff0a0102a60 msgr2=0x7ff0a0197fb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:50:18.052 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.050+0000 7ff0a5ca1640 1 --2- 172.21.15.99:0/3044258732 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff0a0102a60 0x7ff0a0197fb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:18.053 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.050+0000 7ff0a5ca1640 1 -- 172.21.15.99:0/3044258732 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ff0a019f690 con 0x7ff0a0101870 2024-03-19T01:50:18.053 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.050+0000 7ff0a54a0640 1 --2- 172.21.15.99:0/3044258732 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff0a0102a60 0x7ff0a0197fb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:50:18.053 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.050+0000 7ff0a64a2640 1 --2- 172.21.15.99:0/3044258732 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff0a0103420 0x7ff0a019f020 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:50:18.053 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.050+0000 7ff0a5ca1640 1 --2- 172.21.15.99:0/3044258732 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff0a0101870 0x7ff0a0197a70 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 crypto rx=0x7ff09000e9c0 tx=0x7ff09000ee80 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:50:18.053 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.050+0000 7ff096ffd640 1 -- 172.21.15.99:0/3044258732 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff09000cd80 con 0x7ff0a0101870 2024-03-19T01:50:18.053 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.050+0000 7ff096ffd640 1 -- 172.21.15.99:0/3044258732 <== mon.1 v2:172.21.15.115:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7ff0900035f0 con 0x7ff0a0101870 2024-03-19T01:50:18.053 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.050+0000 7ff0a7f05640 1 -- 172.21.15.99:0/3044258732 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ff0a019f980 con 0x7ff0a0101870 2024-03-19T01:50:18.053 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.050+0000 7ff096ffd640 1 -- 172.21.15.99:0/3044258732 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff090010690 con 0x7ff0a0101870 2024-03-19T01:50:18.053 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.050+0000 7ff0a7f05640 1 -- 172.21.15.99:0/3044258732 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7ff0a019ff20 con 0x7ff0a0101870 2024-03-19T01:50:18.059 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.053+0000 7ff096ffd640 1 -- 172.21.15.99:0/3044258732 <== mon.1 v2:172.21.15.115:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7ff0900037a0 con 0x7ff0a0101870 2024-03-19T01:50:18.059 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.053+0000 7ff0aabc5740 1 Processor -- start 2024-03-19T01:50:18.059 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.053+0000 7ff0aabc5740 1 -- start start 2024-03-19T01:50:18.059 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.054+0000 7ff0aabc5740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563c1c45a100 0x563c1c45a520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:18.059 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.054+0000 7ff0aabc5740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563c1c45aaf0 0x563c1c45f5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:18.059 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.054+0000 7ff0aabc5740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563c1c45fc60 0x563c1c4620f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:18.059 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.054+0000 7ff0aabc5740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x563c1c3bebe0 con 0x563c1c45aaf0 2024-03-19T01:50:18.059 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.054+0000 7ff0aabc5740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x563c1c367960 con 0x563c1c45fc60 2024-03-19T01:50:18.059 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.054+0000 7ff0aabc5740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x563c1c3bac80 con 0x563c1c45a100 2024-03-19T01:50:18.059 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.054+0000 7ff0a5ca1640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563c1c45aaf0 0x563c1c45f5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:18.060 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.054+0000 7ff0a64a2640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563c1c45a100 0x563c1c45a520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:18.060 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.054+0000 7ff0a54a0640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563c1c45fc60 0x563c1c4620f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:18.060 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.054+0000 7ff0a5ca1640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563c1c45aaf0 0x563c1c45f5f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:49556/0 (socket says 172.21.15.99:49556) 2024-03-19T01:50:18.060 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.054+0000 7ff0a64a2640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563c1c45a100 0x563c1c45a520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:40750/0 (socket says 172.21.15.99:40750) 2024-03-19T01:50:18.060 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.054+0000 7ff0a5ca1640 1 -- 172.21.15.99:0/2115241706 learned_addr learned my addr 172.21.15.99:0/2115241706 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:50:18.060 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.054+0000 7ff096ffd640 1 --2- 172.21.15.99:0/3044258732 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff08c0a2920 0x7ff08c0a4de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:18.060 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.054+0000 7ff0a64a2640 1 --2- 172.21.15.99:0/3044258732 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff08c0a2920 0x7ff08c0a4de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:18.060 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.054+0000 7ff0a54a0640 1 -- 172.21.15.99:0/2115241706 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563c1c45a100 msgr2=0x563c1c45a520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:50:18.060 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.054+0000 7ff0a54a0640 1 --2- 172.21.15.99:0/2115241706 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563c1c45a100 0x563c1c45a520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:18.060 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.054+0000 7ff0a54a0640 1 -- 172.21.15.99:0/2115241706 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563c1c45aaf0 msgr2=0x563c1c45f5f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:50:18.060 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.054+0000 7ff0a54a0640 1 --2- 172.21.15.99:0/2115241706 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563c1c45aaf0 0x563c1c45f5f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:18.061 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.054+0000 7ff0a54a0640 1 -- 172.21.15.99:0/2115241706 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x563c1c35ed20 con 0x563c1c45fc60 2024-03-19T01:50:18.061 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.054+0000 7ff096ffd640 1 -- 172.21.15.99:0/3044258732 <== mon.1 v2:172.21.15.115:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7ff0900cedd0 con 0x7ff0a0101870 2024-03-19T01:50:18.061 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.054+0000 7ff0a54a0640 1 --2- 172.21.15.99:0/2115241706 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563c1c45fc60 0x563c1c4620f0 secure :-1 s=READY pgs=138 cs=0 l=1 rev1=1 crypto rx=0x7ff09c009fa0 tx=0x7ff09c0118d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=cde926b5543774ab server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:50:18.061 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.055+0000 7ff0a64a2640 1 --2- 172.21.15.99:0/3044258732 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff08c0a2920 0x7ff08c0a4de0 secure :-1 s=READY pgs=301 cs=0 l=1 rev1=1 crypto rx=0x7ff098005e30 tx=0x7ff09802c0f0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:50:18.061 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.055+0000 7ff08bfff640 1 -- 172.21.15.99:0/2115241706 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff09c004310 con 0x563c1c45fc60 2024-03-19T01:50:18.061 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.055+0000 7ff08bfff640 1 -- 172.21.15.99:0/2115241706 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7ff09c0044b0 con 0x563c1c45fc60 2024-03-19T01:50:18.061 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.055+0000 7ff08bfff640 1 -- 172.21.15.99:0/2115241706 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff09c0035f0 con 0x563c1c45fc60 2024-03-19T01:50:18.061 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.055+0000 7ff0aabc5740 1 -- 172.21.15.99:0/2115241706 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563c1c45fc60 msgr2=0x563c1c4620f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:50:18.061 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.055+0000 7ff0aabc5740 1 --2- 172.21.15.99:0/2115241706 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563c1c45fc60 0x563c1c4620f0 secure :-1 s=READY pgs=138 cs=0 l=1 rev1=1 crypto rx=0x7ff09c009fa0 tx=0x7ff09c0118d0 comp rx=0 tx=0).stop 2024-03-19T01:50:18.061 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.055+0000 7ff0aabc5740 1 -- 172.21.15.99:0/2115241706 shutdown_connections 2024-03-19T01:50:18.061 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.055+0000 7ff0aabc5740 1 --2- 172.21.15.99:0/2115241706 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563c1c45fc60 0x563c1c4620f0 unknown :-1 s=CLOSED pgs=138 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:18.061 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.055+0000 7ff0aabc5740 1 --2- 172.21.15.99:0/2115241706 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563c1c45aaf0 0x563c1c45f5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:18.062 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.055+0000 7ff0aabc5740 1 --2- 172.21.15.99:0/2115241706 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563c1c45a100 0x563c1c45a520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:18.062 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.055+0000 7ff0aabc5740 1 -- 172.21.15.99:0/2115241706 >> 172.21.15.99:0/2115241706 conn(0x563c1c455670 msgr2=0x563c1c455a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:50:18.062 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.055+0000 7ff0aabc5740 1 -- 172.21.15.99:0/2115241706 shutdown_connections 2024-03-19T01:50:18.062 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.056+0000 7ff0aabc5740 1 -- 172.21.15.99:0/2115241706 wait complete. 2024-03-19T01:50:18.062 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.056+0000 7ff0aabc5740 1 -- start start 2024-03-19T01:50:18.062 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.056+0000 7ff0aabc5740 1 Processor -- start 2024-03-19T01:50:18.062 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.056+0000 7ff0aabc5740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563c1c45a100 0x563c1c4640e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:18.062 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.056+0000 7ff0aabc5740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563c1c45aaf0 0x563c1c464620 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:18.062 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.057+0000 7ff0aabc5740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563c1c45fc60 0x563c1c46b660 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:18.062 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.057+0000 7ff0aabc5740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x563c1c462c10 con 0x563c1c45fc60 2024-03-19T01:50:18.062 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.057+0000 7ff0aabc5740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x563c1c37adb0 con 0x563c1c45a100 2024-03-19T01:50:18.062 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.057+0000 7ff0aabc5740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x563c1c462d90 con 0x563c1c45aaf0 2024-03-19T01:50:18.062 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.057+0000 7ff0aabc5740 10 client.0 _ll_register_callbacks cb 0x563c1c37fe70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:50:18.063 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.057+0000 7ff0a5ca1640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563c1c45a100 0x563c1c4640e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:18.063 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.057+0000 7ff0a54a0640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563c1c45aaf0 0x563c1c464620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:18.063 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.057+0000 7ff0a5ca1640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563c1c45a100 0x563c1c4640e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:55824/0 (socket says 172.21.15.99:55824) 2024-03-19T01:50:18.063 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.057+0000 7ff0a5ca1640 1 -- 172.21.15.99:0/486294516 learned_addr learned my addr 172.21.15.99:0/486294516 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:50:18.063 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.057+0000 7ff0a64a2640 1 --2- 172.21.15.99:0/486294516 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563c1c45fc60 0x563c1c46b660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:18.063 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.057+0000 7ff0a5ca1640 1 -- 172.21.15.99:0/486294516 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563c1c45aaf0 msgr2=0x563c1c464620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:50:18.063 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.057+0000 7ff0a5ca1640 1 --2- 172.21.15.99:0/486294516 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563c1c45aaf0 0x563c1c464620 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:18.063 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.057+0000 7ff0a5ca1640 1 -- 172.21.15.99:0/486294516 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563c1c45fc60 msgr2=0x563c1c46b660 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:50:18.063 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.057+0000 7ff0a5ca1640 1 --2- 172.21.15.99:0/486294516 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563c1c45fc60 0x563c1c46b660 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:18.063 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.057+0000 7ff0a5ca1640 1 -- 172.21.15.99:0/486294516 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x563c1c379c60 con 0x563c1c45a100 2024-03-19T01:50:18.063 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.057+0000 7ff0a64a2640 1 --2- 172.21.15.99:0/486294516 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563c1c45fc60 0x563c1c46b660 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-19T01:50:18.063 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.057+0000 7ff0a54a0640 1 --2- 172.21.15.99:0/486294516 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563c1c45aaf0 0x563c1c464620 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:50:18.063 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.057+0000 7ff0a5ca1640 1 --2- 172.21.15.99:0/486294516 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563c1c45a100 0x563c1c4640e0 secure :-1 s=READY pgs=139 cs=0 l=1 rev1=1 crypto rx=0x7ff0900cc8e0 tx=0x7ff090097310 comp rx=0 tx=0).ready entity=mon.2 client_cookie=8a6682b3f6863d8a server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:50:18.063 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.057+0000 7ff0897fa640 10 client.0 ms_handle_connect on v2:172.21.15.112:3300/0 2024-03-19T01:50:18.064 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.057+0000 7ff0897fa640 1 -- 172.21.15.99:0/486294516 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff09009b090 con 0x563c1c45a100 2024-03-19T01:50:18.064 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.057+0000 7ff0897fa640 1 -- 172.21.15.99:0/486294516 <== mon.2 v2:172.21.15.112:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7ff090003b30 con 0x563c1c45a100 2024-03-19T01:50:18.064 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.057+0000 7ff0aabc5740 1 -- 172.21.15.99:0/486294516 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7ff09c0044b0 con 0x563c1c45a100 2024-03-19T01:50:18.064 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.058+0000 7ff0897fa640 1 -- 172.21.15.99:0/486294516 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7ff09009be30 con 0x563c1c45a100 2024-03-19T01:50:18.064 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.058+0000 7ff0aabc5740 10 client.25850 fetch_fsmap learned FSMap version 27 2024-03-19T01:50:18.064 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.058+0000 7ff0aabc5740 1 -- 172.21.15.99:0/486294516 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x563c1c0d3210 con 0x563c1c45a100 2024-03-19T01:50:18.064 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.058+0000 7ff0897fa640 1 -- 172.21.15.99:0/486294516 <== mon.2 v2:172.21.15.112:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7ff09c0044b0 con 0x563c1c45a100 2024-03-19T01:50:18.064 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.058+0000 7ff0897fa640 1 -- 172.21.15.99:0/486294516 <== mon.2 v2:172.21.15.112:3300/0 5 ==== fsmap(e 27) ==== 8338+0+0 (secure 0 0 0) 0x7ff09009ab80 con 0x563c1c45a100 2024-03-19T01:50:18.064 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.058+0000 7ff0aabc5740 10 client.25850 fetch_fsmap finished waiting for FSMap version 27 2024-03-19T01:50:18.064 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.058+0000 7ff0aabc5740 10 client.25850 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:50:18.064 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.058+0000 7ff0aabc5740 20 client.25850 populate_metadata read hostname 'smithi099' 2024-03-19T01:50:18.064 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.058+0000 7ff0aabc5740 1 --2- 172.21.15.99:0/486294516 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x563c1c46ff10 0x563c1c472360 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:18.064 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.058+0000 7ff0aabc5740 4 client.25850 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:50:18.064 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.058+0000 7ff0aabc5740 1 -- 172.21.15.99:0/486294516 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x563c1c3cc120 con 0x563c1c46ff10 2024-03-19T01:50:18.064 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.058+0000 7ff0a54a0640 1 --2- 172.21.15.99:0/486294516 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x563c1c46ff10 0x563c1c472360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:18.065 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.059+0000 7ff0897fa640 10 client.25850 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:50:18.065 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.059+0000 7ff0a54a0640 1 --2- 172.21.15.99:0/486294516 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x563c1c46ff10 0x563c1c472360 crc :-1 s=READY pgs=111 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=a4961a08e2b895f8 server_cookie=f9e12e0a75aa80fc in_seq=0 out_seq=0 2024-03-19T01:50:18.066 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.060+0000 7ff0897fa640 1 -- 172.21.15.99:0/486294516 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x563c1c3cc120 con 0x563c1c46ff10 2024-03-19T01:50:18.066 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.060+0000 7ff0897fa640 10 client.25850 handle_command_reply: tid=0 2024-03-19T01:50:18.066 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.060+0000 7ff0aabc5740 1 client.25850 shutdown 2024-03-19T01:50:18.067 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.060+0000 7ff0aabc5740 1 -- 172.21.15.99:0/486294516 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563c1c45a100 msgr2=0x563c1c4640e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:50:18.067 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.060+0000 7ff0aabc5740 1 --2- 172.21.15.99:0/486294516 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563c1c45a100 0x563c1c4640e0 secure :-1 s=READY pgs=139 cs=0 l=1 rev1=1 crypto rx=0x7ff0900cc8e0 tx=0x7ff090097310 comp rx=0 tx=0).stop 2024-03-19T01:50:18.067 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.060+0000 7ff0aabc5740 1 -- 172.21.15.99:0/486294516 shutdown_connections 2024-03-19T01:50:18.067 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.060+0000 7ff0aabc5740 1 --2- 172.21.15.99:0/486294516 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x563c1c46ff10 0x563c1c472360 crc :-1 s=READY pgs=111 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:18.067 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.060+0000 7ff0aabc5740 1 --2- 172.21.15.99:0/486294516 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563c1c45fc60 0x563c1c46b660 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:18.067 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.060+0000 7ff0aabc5740 1 --2- 172.21.15.99:0/486294516 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563c1c45aaf0 0x563c1c464620 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:18.067 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.060+0000 7ff0897fa640 0 client.25850 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:50:18.067 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.060+0000 7ff0aabc5740 1 --2- 172.21.15.99:0/486294516 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563c1c45a100 0x563c1c4640e0 unknown :-1 s=CLOSED pgs=139 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:18.067 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.060+0000 7ff0aabc5740 1 -- 172.21.15.99:0/486294516 >> 172.21.15.99:0/486294516 conn(0x563c1c455670 msgr2=0x563c1c4568c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:50:18.067 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.060+0000 7ff0aabc5740 1 -- 172.21.15.99:0/486294516 shutdown_connections 2024-03-19T01:50:18.067 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.060+0000 7ff0aabc5740 1 -- 172.21.15.99:0/486294516 wait complete. 2024-03-19T01:50:18.067 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.060+0000 7ff0aabc5740 20 client.25850 trim_cache size 0 max 16384 2024-03-19T01:50:18.078 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.077+0000 7ff0aabc5740 1 Processor -- start 2024-03-19T01:50:18.078 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.077+0000 7ff0aabc5740 1 -- start start 2024-03-19T01:50:18.078 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.077+0000 7ff0aabc5740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563c1c45a100 0x563c1c464580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:18.078 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.077+0000 7ff0aabc5740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563c1c45aaf0 0x563c1c46b5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:18.078 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.077+0000 7ff0aabc5740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563c1c45fc60 0x563c1c458fe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:18.078 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.077+0000 7ff0aabc5740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7ff0a01ab630 con 0x563c1c45fc60 2024-03-19T01:50:18.078 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.077+0000 7ff0aabc5740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7ff0a01a8f00 con 0x563c1c45aaf0 2024-03-19T01:50:18.078 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.077+0000 7ff0aabc5740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7ff0a01a6d50 con 0x563c1c45a100 2024-03-19T01:50:18.079 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.078+0000 7ff0a54a0640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563c1c45aaf0 0x563c1c46b5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:18.079 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.078+0000 7ff0a5ca1640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563c1c45a100 0x563c1c464580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:18.079 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.078+0000 7ff0a64a2640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563c1c45fc60 0x563c1c458fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:18.079 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.078+0000 7ff0a54a0640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563c1c45aaf0 0x563c1c46b5f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:55832/0 (socket says 172.21.15.99:55832) 2024-03-19T01:50:18.079 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.078+0000 7ff0a54a0640 1 -- 172.21.15.99:0/3529195766 learned_addr learned my addr 172.21.15.99:0/3529195766 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:50:18.079 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.078+0000 7ff0a54a0640 1 -- 172.21.15.99:0/3529195766 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563c1c45a100 msgr2=0x563c1c464580 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:50:18.079 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.078+0000 7ff0a54a0640 1 --2- 172.21.15.99:0/3529195766 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563c1c45a100 0x563c1c464580 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:18.079 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.078+0000 7ff0a54a0640 1 -- 172.21.15.99:0/3529195766 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563c1c45fc60 msgr2=0x563c1c458fe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:50:18.079 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.078+0000 7ff0a54a0640 1 --2- 172.21.15.99:0/3529195766 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563c1c45fc60 0x563c1c458fe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:18.079 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.078+0000 7ff0a54a0640 1 -- 172.21.15.99:0/3529195766 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x563c1c19f620 con 0x563c1c45aaf0 2024-03-19T01:50:18.079 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.078+0000 7ff0a54a0640 1 --2- 172.21.15.99:0/3529195766 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563c1c45aaf0 0x563c1c46b5f0 secure :-1 s=READY pgs=140 cs=0 l=1 rev1=1 crypto rx=0x7ff09c009380 tx=0x7ff09c0045b0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=5c7a4d0d50c9d59 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:50:18.079 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.078+0000 7ff08b7fe640 1 -- 172.21.15.99:0/3529195766 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff09c01d040 con 0x563c1c45aaf0 2024-03-19T01:50:18.079 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.078+0000 7ff08b7fe640 1 -- 172.21.15.99:0/3529195766 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7ff09c009b30 con 0x563c1c45aaf0 2024-03-19T01:50:18.079 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.078+0000 7ff08b7fe640 1 -- 172.21.15.99:0/3529195766 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff09c003670 con 0x563c1c45aaf0 2024-03-19T01:50:18.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.079+0000 7ff0aabc5740 1 -- 172.21.15.99:0/3529195766 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563c1c45aaf0 msgr2=0x563c1c46b5f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:50:18.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.079+0000 7ff0aabc5740 1 --2- 172.21.15.99:0/3529195766 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563c1c45aaf0 0x563c1c46b5f0 secure :-1 s=READY pgs=140 cs=0 l=1 rev1=1 crypto rx=0x7ff09c009380 tx=0x7ff09c0045b0 comp rx=0 tx=0).stop 2024-03-19T01:50:18.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.079+0000 7ff0aabc5740 1 -- 172.21.15.99:0/3529195766 shutdown_connections 2024-03-19T01:50:18.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.079+0000 7ff0aabc5740 1 --2- 172.21.15.99:0/3529195766 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563c1c45aaf0 0x563c1c46b5f0 unknown :-1 s=CLOSED pgs=140 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:18.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.079+0000 7ff0aabc5740 1 --2- 172.21.15.99:0/3529195766 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563c1c45fc60 0x563c1c458fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:18.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.079+0000 7ff0aabc5740 1 --2- 172.21.15.99:0/3529195766 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563c1c45a100 0x563c1c464580 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:18.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.079+0000 7ff0aabc5740 1 -- 172.21.15.99:0/3529195766 >> 172.21.15.99:0/3529195766 conn(0x563c1c455670 msgr2=0x563c1c4588d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:50:18.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.079+0000 7ff0aabc5740 1 -- 172.21.15.99:0/3529195766 shutdown_connections 2024-03-19T01:50:18.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.079+0000 7ff0aabc5740 1 -- 172.21.15.99:0/3529195766 wait complete. 2024-03-19T01:50:18.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.079+0000 7ff0aabc5740 1 -- start start 2024-03-19T01:50:18.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.079+0000 7ff0aabc5740 1 Processor -- start 2024-03-19T01:50:18.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.080+0000 7ff0aabc5740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563c1c45a100 0x563c1c468fc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:18.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.080+0000 7ff0aabc5740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563c1c45aaf0 0x563c1c469500 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:18.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.080+0000 7ff0aabc5740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563c1c45fc60 0x563c1c46a510 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:18.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.080+0000 7ff0aabc5740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7ff0a01a73c0 con 0x563c1c45a100 2024-03-19T01:50:18.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.080+0000 7ff0aabc5740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7ff0a01a8830 con 0x563c1c45aaf0 2024-03-19T01:50:18.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.080+0000 7ff0aabc5740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7ff0a01a7a90 con 0x563c1c45fc60 2024-03-19T01:50:18.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.080+0000 7ff0aabc5740 10 client.0 _ll_register_callbacks cb 0x7ff0900097b0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:50:18.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.080+0000 7ff0a5ca1640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563c1c45a100 0x563c1c468fc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:18.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.080+0000 7ff0a5ca1640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563c1c45a100 0x563c1c468fc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:49596/0 (socket says 172.21.15.99:49596) 2024-03-19T01:50:18.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.080+0000 7ff0a5ca1640 1 -- 172.21.15.99:0/3368413655 learned_addr learned my addr 172.21.15.99:0/3368413655 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:50:18.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.080+0000 7ff0a54a0640 1 --2- 172.21.15.99:0/3368413655 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563c1c45aaf0 0x563c1c469500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:18.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.080+0000 7ff0a64a2640 1 --2- 172.21.15.99:0/3368413655 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563c1c45fc60 0x563c1c46a510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:18.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.080+0000 7ff0a5ca1640 1 -- 172.21.15.99:0/3368413655 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563c1c45fc60 msgr2=0x563c1c46a510 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:50:18.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.080+0000 7ff0a5ca1640 1 --2- 172.21.15.99:0/3368413655 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563c1c45fc60 0x563c1c46a510 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:18.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.080+0000 7ff0a5ca1640 1 -- 172.21.15.99:0/3368413655 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563c1c45aaf0 msgr2=0x563c1c469500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:50:18.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.080+0000 7ff0a5ca1640 1 --2- 172.21.15.99:0/3368413655 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563c1c45aaf0 0x563c1c469500 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:18.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.080+0000 7ff0a5ca1640 1 -- 172.21.15.99:0/3368413655 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x563c1c46ab00 con 0x563c1c45a100 2024-03-19T01:50:18.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.080+0000 7ff0a5ca1640 1 --2- 172.21.15.99:0/3368413655 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563c1c45a100 0x563c1c468fc0 secure :-1 s=READY pgs=523 cs=0 l=1 rev1=1 crypto rx=0x7ff090097b90 tx=0x7ff090005f10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a206380b847cc936 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:50:18.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.080+0000 7ff0897fa640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:50:18.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.080+0000 7ff0897fa640 1 -- 172.21.15.99:0/3368413655 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff09009b090 con 0x563c1c45a100 2024-03-19T01:50:18.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.080+0000 7ff0897fa640 1 -- 172.21.15.99:0/3368413655 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7ff09009ab80 con 0x563c1c45a100 2024-03-19T01:50:18.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.080+0000 7ff0aabc5740 1 -- 172.21.15.99:0/3368413655 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7ff09c009b30 con 0x563c1c45a100 2024-03-19T01:50:18.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.081+0000 7ff0897fa640 1 -- 172.21.15.99:0/3368413655 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7ff09009b500 con 0x563c1c45a100 2024-03-19T01:50:18.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.081+0000 7ff0aabc5740 10 client.16086 fetch_fsmap learned FSMap version 27 2024-03-19T01:50:18.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.081+0000 7ff0aabc5740 1 -- 172.21.15.99:0/3368413655 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x563c1c4d6580 con 0x563c1c45a100 2024-03-19T01:50:18.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.081+0000 7ff0897fa640 1 -- 172.21.15.99:0/3368413655 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7ff09c009b30 con 0x563c1c45a100 2024-03-19T01:50:18.086 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.081+0000 7ff0897fa640 1 -- 172.21.15.99:0/3368413655 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 27) ==== 8338+0+0 (secure 0 0 0) 0x7ff0900d1070 con 0x563c1c45a100 2024-03-19T01:50:18.086 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.081+0000 7ff0aabc5740 10 client.16086 fetch_fsmap finished waiting for FSMap version 27 2024-03-19T01:50:18.086 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.081+0000 7ff0aabc5740 10 client.16086 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:50:18.086 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.081+0000 7ff0aabc5740 20 client.16086 populate_metadata read hostname 'smithi099' 2024-03-19T01:50:18.086 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.081+0000 7ff0aabc5740 1 --2- 172.21.15.99:0/3368413655 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x563c1c4d6ef0 0x563c1c4d9310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:18.086 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.081+0000 7ff0aabc5740 4 client.16086 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:50:18.086 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.081+0000 7ff0aabc5740 1 -- 172.21.15.99:0/3368413655 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x563c1c3c6470 con 0x563c1c4d6ef0 2024-03-19T01:50:18.086 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.081+0000 7ff0a54a0640 1 --2- 172.21.15.99:0/3368413655 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x563c1c4d6ef0 0x563c1c4d9310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:18.086 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.082+0000 7ff0897fa640 10 client.16086 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:50:18.086 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.082+0000 7ff0a54a0640 1 --2- 172.21.15.99:0/3368413655 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x563c1c4d6ef0 0x563c1c4d9310 crc :-1 s=READY pgs=112 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=5deada11537b647e server_cookie=85123f774dc29170 in_seq=0 out_seq=0 2024-03-19T01:50:18.176 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.175+0000 7ff0897fa640 1 -- 172.21.15.99:0/3368413655 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x563c1c3c6470 con 0x563c1c4d6ef0 2024-03-19T01:50:18.176 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.175+0000 7ff0897fa640 10 client.16086 handle_command_reply: tid=0 2024-03-19T01:50:18.176 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.175+0000 7ff0aabc5740 1 client.16086 shutdown 2024-03-19T01:50:18.176 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.175+0000 7ff0aabc5740 1 -- 172.21.15.99:0/3368413655 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563c1c45a100 msgr2=0x563c1c468fc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:50:18.176 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.175+0000 7ff0aabc5740 1 --2- 172.21.15.99:0/3368413655 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563c1c45a100 0x563c1c468fc0 secure :-1 s=READY pgs=523 cs=0 l=1 rev1=1 crypto rx=0x7ff090097b90 tx=0x7ff090005f10 comp rx=0 tx=0).stop 2024-03-19T01:50:18.176 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.175+0000 7ff0aabc5740 1 -- 172.21.15.99:0/3368413655 shutdown_connections 2024-03-19T01:50:18.176 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.175+0000 7ff0aabc5740 1 --2- 172.21.15.99:0/3368413655 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x563c1c4d6ef0 0x563c1c4d9310 crc :-1 s=READY pgs=112 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:18.176 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.175+0000 7ff0aabc5740 1 --2- 172.21.15.99:0/3368413655 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563c1c45aaf0 0x563c1c469500 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:18.176 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.175+0000 7ff0aabc5740 1 --2- 172.21.15.99:0/3368413655 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563c1c45fc60 0x563c1c46a510 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:18.176 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.175+0000 7ff0897fa640 0 client.16086 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:50:18.177 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.175+0000 7ff0aabc5740 1 --2- 172.21.15.99:0/3368413655 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563c1c45a100 0x563c1c468fc0 unknown :-1 s=CLOSED pgs=523 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:18.177 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.175+0000 7ff0aabc5740 1 -- 172.21.15.99:0/3368413655 >> 172.21.15.99:0/3368413655 conn(0x563c1c455670 msgr2=0x563c1c459160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:50:18.177 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.175+0000 7ff0aabc5740 1 -- 172.21.15.99:0/3368413655 shutdown_connections 2024-03-19T01:50:18.177 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.175+0000 7ff0aabc5740 1 -- 172.21.15.99:0/3368413655 wait complete. 2024-03-19T01:50:18.177 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.176+0000 7ff0aabc5740 20 client.16086 trim_cache size 0 max 16384 2024-03-19T01:50:18.177 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:50:18.177 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:50:18.177 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:50:18.177 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:50:18.178 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.176+0000 7ff0a7f05640 1 -- 172.21.15.99:0/3044258732 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff08c0a2920 msgr2=0x7ff08c0a4de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:50:18.178 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.176+0000 7ff0a7f05640 1 --2- 172.21.15.99:0/3044258732 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff08c0a2920 0x7ff08c0a4de0 secure :-1 s=READY pgs=301 cs=0 l=1 rev1=1 crypto rx=0x7ff098005e30 tx=0x7ff09802c0f0 comp rx=0 tx=0).stop 2024-03-19T01:50:18.178 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.176+0000 7ff0a7f05640 1 -- 172.21.15.99:0/3044258732 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff0a0101870 msgr2=0x7ff0a0197a70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:50:18.178 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.176+0000 7ff0a7f05640 1 --2- 172.21.15.99:0/3044258732 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff0a0101870 0x7ff0a0197a70 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 crypto rx=0x7ff09000e9c0 tx=0x7ff09000ee80 comp rx=0 tx=0).stop 2024-03-19T01:50:18.178 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.176+0000 7ff0a7f05640 1 -- 172.21.15.99:0/3044258732 shutdown_connections 2024-03-19T01:50:18.179 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.176+0000 7ff0a7f05640 1 --2- 172.21.15.99:0/3044258732 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff08c0a2920 0x7ff08c0a4de0 unknown :-1 s=CLOSED pgs=301 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:18.179 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.176+0000 7ff0a7f05640 1 --2- 172.21.15.99:0/3044258732 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff0a0103420 0x7ff0a019f020 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:18.179 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.176+0000 7ff0a7f05640 1 --2- 172.21.15.99:0/3044258732 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff0a0102a60 0x7ff0a0197fb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:18.179 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.176+0000 7ff0a7f05640 1 --2- 172.21.15.99:0/3044258732 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff0a0101870 0x7ff0a0197a70 unknown :-1 s=CLOSED pgs=104 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:18.179 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.176+0000 7ff0a7f05640 1 -- 172.21.15.99:0/3044258732 >> 172.21.15.99:0/3044258732 conn(0x7ff0a00fcde0 msgr2=0x7ff0a0105d00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:50:18.179 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.176+0000 7ff0a7f05640 1 -- 172.21.15.99:0/3044258732 shutdown_connections 2024-03-19T01:50:18.179 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:18.176+0000 7ff0a7f05640 1 -- 172.21.15.99:0/3044258732 wait complete. 2024-03-19T01:50:18.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:50:17 smithi115 ceph-mon[67332]: pgmap v626: 97 pgs: 97 active+clean; 266 MiB data, 3.0 GiB used, 1.0 TiB / 1.0 TiB avail; 811 KiB/s rd, 3.0 MiB/s wr, 256 op/s 2024-03-19T01:50:18.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:50:17 smithi099 ceph-mon[62951]: pgmap v626: 97 pgs: 97 active+clean; 266 MiB data, 3.0 GiB used, 1.0 TiB / 1.0 TiB avail; 811 KiB/s rd, 3.0 MiB/s wr, 256 op/s 2024-03-19T01:50:18.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:50:17 smithi112 ceph-mon[66819]: pgmap v626: 97 pgs: 97 active+clean; 266 MiB data, 3.0 GiB used, 1.0 TiB / 1.0 TiB avail; 811 KiB/s rd, 3.0 MiB/s wr, 256 op/s 2024-03-19T01:50:18.696 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1119847 11.38 MB/sec execute 408 sec latency 99.618 ms 2024-03-19T01:50:19.696 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1122645 11.40 MB/sec execute 409 sec latency 22.907 ms 2024-03-19T01:50:20.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:50:19 smithi115 ceph-mon[67332]: pgmap v627: 97 pgs: 97 active+clean; 268 MiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 769 KiB/s rd, 2.7 MiB/s wr, 236 op/s 2024-03-19T01:50:20.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:50:19 smithi099 ceph-mon[62951]: pgmap v627: 97 pgs: 97 active+clean; 268 MiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 769 KiB/s rd, 2.7 MiB/s wr, 236 op/s 2024-03-19T01:50:20.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:50:19 smithi112 ceph-mon[66819]: pgmap v627: 97 pgs: 97 active+clean; 268 MiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 769 KiB/s rd, 2.7 MiB/s wr, 236 op/s 2024-03-19T01:50:20.696 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1124794 11.39 MB/sec execute 410 sec latency 39.184 ms 2024-03-19T01:50:21.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:50:20 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:50:21.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:50:20 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:50:21.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:50:20 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:50:21.696 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1126564 11.39 MB/sec execute 411 sec latency 60.046 ms 2024-03-19T01:50:22.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:50:21 smithi115 ceph-mon[67332]: pgmap v628: 97 pgs: 97 active+clean; 269 MiB data, 3.0 GiB used, 1.0 TiB / 1.0 TiB avail; 811 KiB/s rd, 2.9 MiB/s wr, 248 op/s 2024-03-19T01:50:22.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:50:21 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:50:22.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:50:21 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:50:22.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:50:21 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:50:22.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:50:21 smithi099 ceph-mon[62951]: pgmap v628: 97 pgs: 97 active+clean; 269 MiB data, 3.0 GiB used, 1.0 TiB / 1.0 TiB avail; 811 KiB/s rd, 2.9 MiB/s wr, 248 op/s 2024-03-19T01:50:22.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:50:21 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:50:22.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:50:21 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:50:22.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:50:21 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:50:22.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:50:21 smithi112 ceph-mon[66819]: pgmap v628: 97 pgs: 97 active+clean; 269 MiB data, 3.0 GiB used, 1.0 TiB / 1.0 TiB avail; 811 KiB/s rd, 2.9 MiB/s wr, 248 op/s 2024-03-19T01:50:22.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:50:21 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:50:22.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:50:21 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:50:22.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:50:21 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:50:22.696 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1129303 11.40 MB/sec execute 412 sec latency 10.482 ms 2024-03-19T01:50:23.696 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1130686 11.39 MB/sec execute 413 sec latency 90.112 ms 2024-03-19T01:50:24.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:50:23 smithi115 ceph-mon[67332]: pgmap v629: 97 pgs: 97 active+clean; 267 MiB data, 3.0 GiB used, 1.0 TiB / 1.0 TiB avail; 925 KiB/s rd, 3.1 MiB/s wr, 263 op/s 2024-03-19T01:50:24.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:50:23 smithi099 ceph-mon[62951]: pgmap v629: 97 pgs: 97 active+clean; 267 MiB data, 3.0 GiB used, 1.0 TiB / 1.0 TiB avail; 925 KiB/s rd, 3.1 MiB/s wr, 263 op/s 2024-03-19T01:50:24.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:50:23 smithi112 ceph-mon[66819]: pgmap v629: 97 pgs: 97 active+clean; 267 MiB data, 3.0 GiB used, 1.0 TiB / 1.0 TiB avail; 925 KiB/s rd, 3.1 MiB/s wr, 263 op/s 2024-03-19T01:50:24.696 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1133371 11.40 MB/sec execute 414 sec latency 22.272 ms 2024-03-19T01:50:25.696 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1135446 11.39 MB/sec execute 415 sec latency 57.119 ms 2024-03-19T01:50:26.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:50:25 smithi115 ceph-mon[67332]: pgmap v630: 97 pgs: 97 active+clean; 267 MiB data, 3.0 GiB used, 1.0 TiB / 1.0 TiB avail; 840 KiB/s rd, 2.6 MiB/s wr, 235 op/s 2024-03-19T01:50:26.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:50:25 smithi099 ceph-mon[62951]: pgmap v630: 97 pgs: 97 active+clean; 267 MiB data, 3.0 GiB used, 1.0 TiB / 1.0 TiB avail; 840 KiB/s rd, 2.6 MiB/s wr, 235 op/s 2024-03-19T01:50:26.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:50:25 smithi112 ceph-mon[66819]: pgmap v630: 97 pgs: 97 active+clean; 267 MiB data, 3.0 GiB used, 1.0 TiB / 1.0 TiB avail; 840 KiB/s rd, 2.6 MiB/s wr, 235 op/s 2024-03-19T01:50:26.696 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1137252 11.39 MB/sec execute 416 sec latency 61.608 ms 2024-03-19T01:50:27.696 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1140015 11.40 MB/sec execute 417 sec latency 11.142 ms 2024-03-19T01:50:28.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:50:27 smithi115 ceph-mon[67332]: pgmap v631: 97 pgs: 97 active+clean; 266 MiB data, 3.0 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 3.3 MiB/s wr, 287 op/s 2024-03-19T01:50:28.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:50:27 smithi099 ceph-mon[62951]: pgmap v631: 97 pgs: 97 active+clean; 266 MiB data, 3.0 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 3.3 MiB/s wr, 287 op/s 2024-03-19T01:50:28.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:50:27 smithi112 ceph-mon[66819]: pgmap v631: 97 pgs: 97 active+clean; 266 MiB data, 3.0 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 3.3 MiB/s wr, 287 op/s 2024-03-19T01:50:28.696 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1141482 11.39 MB/sec execute 418 sec latency 54.621 ms 2024-03-19T01:50:29.696 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1144047 11.40 MB/sec execute 419 sec latency 24.772 ms 2024-03-19T01:50:30.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:50:29 smithi115 ceph-mon[67332]: pgmap v632: 97 pgs: 97 active+clean; 266 MiB data, 3.0 GiB used, 1.0 TiB / 1.0 TiB avail; 1.0 MiB/s rd, 2.9 MiB/s wr, 286 op/s 2024-03-19T01:50:30.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:50:29 smithi099 ceph-mon[62951]: pgmap v632: 97 pgs: 97 active+clean; 266 MiB data, 3.0 GiB used, 1.0 TiB / 1.0 TiB avail; 1.0 MiB/s rd, 2.9 MiB/s wr, 286 op/s 2024-03-19T01:50:30.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:50:29 smithi112 ceph-mon[66819]: pgmap v632: 97 pgs: 97 active+clean; 266 MiB data, 3.0 GiB used, 1.0 TiB / 1.0 TiB avail; 1.0 MiB/s rd, 2.9 MiB/s wr, 286 op/s 2024-03-19T01:50:30.696 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1146173 11.39 MB/sec execute 420 sec latency 26.731 ms 2024-03-19T01:50:31.277 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.266+0000 7f59e9759640 1 -- 172.21.15.99:0/422466894 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f59e4107560 msgr2=0x7f59e4109970 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:50:31.277 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.266+0000 7f59e9759640 1 --2- 172.21.15.99:0/422466894 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f59e4107560 0x7f59e4109970 secure :-1 s=READY pgs=524 cs=0 l=1 rev1=1 crypto rx=0x7f59dc009ca0 tx=0x7f59dc02c0f0 comp rx=0 tx=0).stop 2024-03-19T01:50:31.277 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.267+0000 7f59e9759640 1 -- 172.21.15.99:0/422466894 shutdown_connections 2024-03-19T01:50:31.277 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.267+0000 7f59e9759640 1 --2- 172.21.15.99:0/422466894 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f59e4104c30 0x7f59e4107020 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:31.277 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.267+0000 7f59e9759640 1 --2- 172.21.15.99:0/422466894 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f59e4107560 0x7f59e4109970 unknown :-1 s=CLOSED pgs=524 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:31.277 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.267+0000 7f59e9759640 1 --2- 172.21.15.99:0/422466894 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f59e4102300 0x7f59e41046f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:31.277 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.267+0000 7f59e9759640 1 -- 172.21.15.99:0/422466894 >> 172.21.15.99:0/422466894 conn(0x7f59e40fbcb0 msgr2=0x7f59e40fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:50:31.277 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.267+0000 7f59e9759640 1 -- 172.21.15.99:0/422466894 shutdown_connections 2024-03-19T01:50:31.277 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.267+0000 7f59e9759640 1 -- 172.21.15.99:0/422466894 wait complete. 2024-03-19T01:50:31.277 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.267+0000 7f59e9759640 1 Processor -- start 2024-03-19T01:50:31.278 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.267+0000 7f59e9759640 1 -- start start 2024-03-19T01:50:31.278 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.267+0000 7f59e9759640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f59e4102300 0x7f59e4197aa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:31.278 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.267+0000 7f59e9759640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f59e4104c30 0x7f59e4197fe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:31.278 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.267+0000 7f59e9759640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f59e4107560 0x7f59e419f050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:31.278 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.267+0000 7f59e9759640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f59e410d4c0 con 0x7f59e4104c30 2024-03-19T01:50:31.278 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.267+0000 7f59e9759640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f59e410d340 con 0x7f59e4102300 2024-03-19T01:50:31.278 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.267+0000 7f59e9759640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f59e410d640 con 0x7f59e4107560 2024-03-19T01:50:31.278 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.268+0000 7f59e27fc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f59e4104c30 0x7f59e4197fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:31.278 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.268+0000 7f59e27fc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f59e4104c30 0x7f59e4197fe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:35960/0 (socket says 172.21.15.99:35960) 2024-03-19T01:50:31.278 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.268+0000 7f59e27fc640 1 -- 172.21.15.99:0/2150815964 learned_addr learned my addr 172.21.15.99:0/2150815964 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:50:31.278 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.268+0000 7f59e2ffd640 1 --2- 172.21.15.99:0/2150815964 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f59e4102300 0x7f59e4197aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:31.278 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.268+0000 7f59e37fe640 1 --2- 172.21.15.99:0/2150815964 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f59e4107560 0x7f59e419f050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:31.278 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.268+0000 7f59e27fc640 1 -- 172.21.15.99:0/2150815964 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f59e4107560 msgr2=0x7f59e419f050 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:50:31.279 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.268+0000 7f59e27fc640 1 --2- 172.21.15.99:0/2150815964 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f59e4107560 0x7f59e419f050 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:31.279 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.268+0000 7f59e27fc640 1 -- 172.21.15.99:0/2150815964 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f59e4102300 msgr2=0x7f59e4197aa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:50:31.279 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.268+0000 7f59e27fc640 1 --2- 172.21.15.99:0/2150815964 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f59e4102300 0x7f59e4197aa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:31.279 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.268+0000 7f59e27fc640 1 -- 172.21.15.99:0/2150815964 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f59e419f6c0 con 0x7f59e4104c30 2024-03-19T01:50:31.279 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.268+0000 7f59e27fc640 1 --2- 172.21.15.99:0/2150815964 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f59e4104c30 0x7f59e4197fe0 secure :-1 s=READY pgs=525 cs=0 l=1 rev1=1 crypto rx=0x7f59d800e9c0 tx=0x7f59d800ee80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:50:31.279 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.268+0000 7f59cbfff640 1 -- 172.21.15.99:0/2150815964 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f59d800cd80 con 0x7f59e4104c30 2024-03-19T01:50:31.279 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.268+0000 7f59cbfff640 1 -- 172.21.15.99:0/2150815964 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f59d8005500 con 0x7f59e4104c30 2024-03-19T01:50:31.279 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.268+0000 7f59e9759640 1 -- 172.21.15.99:0/2150815964 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f59e419f9b0 con 0x7f59e4104c30 2024-03-19T01:50:31.279 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.268+0000 7f59cbfff640 1 -- 172.21.15.99:0/2150815964 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f59d8010690 con 0x7f59e4104c30 2024-03-19T01:50:31.279 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.268+0000 7f59e9759640 1 -- 172.21.15.99:0/2150815964 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f59e419ff00 con 0x7f59e4104c30 2024-03-19T01:50:31.279 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.271+0000 7f59cbfff640 1 -- 172.21.15.99:0/2150815964 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f59d80046c0 con 0x7f59e4104c30 2024-03-19T01:50:31.279 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.271+0000 7f59ec412740 1 Processor -- start 2024-03-19T01:50:31.279 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.271+0000 7f59ec412740 1 -- start start 2024-03-19T01:50:31.280 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.271+0000 7f59ec412740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559712e56100 0x559712e56520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:31.280 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.271+0000 7f59ec412740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559712e56af0 0x559712e5b5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:31.280 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.271+0000 7f59ec412740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559712e5bc60 0x559712e5e0f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:31.280 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.271+0000 7f59ec412740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x559712dbabe0 con 0x559712e5bc60 2024-03-19T01:50:31.280 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.271+0000 7f59ec412740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x559712d63960 con 0x559712e56af0 2024-03-19T01:50:31.280 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.271+0000 7f59ec412740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x559712db6c80 con 0x559712e56100 2024-03-19T01:50:31.280 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.271+0000 7f59e27fc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559712e5bc60 0x559712e5e0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:31.280 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.271+0000 7f59e37fe640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559712e56100 0x559712e56520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:31.280 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.272+0000 7f59e2ffd640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559712e56af0 0x559712e5b5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:31.280 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.272+0000 7f59e37fe640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559712e56100 0x559712e56520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:52974/0 (socket says 172.21.15.99:52974) 2024-03-19T01:50:31.280 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.272+0000 7f59e37fe640 1 -- 172.21.15.99:0/1148692504 learned_addr learned my addr 172.21.15.99:0/1148692504 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:50:31.280 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.272+0000 7f59cbfff640 1 --2- 172.21.15.99:0/2150815964 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f59cc0a28a0 0x7f59cc0a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:31.280 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.272+0000 7f59e37fe640 1 --2- 172.21.15.99:0/2150815964 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f59cc0a28a0 0x7f59cc0a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:31.280 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.272+0000 7f59e2ffd640 1 -- 172.21.15.99:0/1148692504 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559712e56100 msgr2=0x559712e56520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:50:31.281 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.272+0000 7f59e2ffd640 1 --2- 172.21.15.99:0/1148692504 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559712e56100 0x559712e56520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:31.281 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.272+0000 7f59e2ffd640 1 -- 172.21.15.99:0/1148692504 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559712e5bc60 msgr2=0x559712e5e0f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:50:31.281 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.272+0000 7f59e2ffd640 1 --2- 172.21.15.99:0/1148692504 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559712e5bc60 0x559712e5e0f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:31.281 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.272+0000 7f59e2ffd640 1 -- 172.21.15.99:0/1148692504 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x559712d5ad20 con 0x559712e56af0 2024-03-19T01:50:31.281 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.272+0000 7f59cbfff640 1 -- 172.21.15.99:0/2150815964 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f59d80cee00 con 0x7f59e4104c30 2024-03-19T01:50:31.281 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.272+0000 7f59e27fc640 1 --2- 172.21.15.99:0/1148692504 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559712e5bc60 0x559712e5e0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:50:31.281 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.272+0000 7f59e2ffd640 1 --2- 172.21.15.99:0/1148692504 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559712e56af0 0x559712e5b5f0 secure :-1 s=READY pgs=141 cs=0 l=1 rev1=1 crypto rx=0x7f59d4009c20 tx=0x7f59d4011810 comp rx=0 tx=0).ready entity=mon.2 client_cookie=c07df91b1ae628be server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:50:31.281 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.272+0000 7f59e37fe640 1 --2- 172.21.15.99:0/2150815964 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f59cc0a28a0 0x7f59cc0a4d60 secure :-1 s=READY pgs=302 cs=0 l=1 rev1=1 crypto rx=0x559712e57350 tx=0x7f59dc02c0f0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:50:31.281 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.272+0000 7f59c97fa640 1 -- 172.21.15.99:0/1148692504 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f59d4004050 con 0x559712e56af0 2024-03-19T01:50:31.281 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.272+0000 7f59c97fa640 1 -- 172.21.15.99:0/1148692504 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f59d4005500 con 0x559712e56af0 2024-03-19T01:50:31.281 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.272+0000 7f59c97fa640 1 -- 172.21.15.99:0/1148692504 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f59d4004dd0 con 0x559712e56af0 2024-03-19T01:50:31.281 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.273+0000 7f59ec412740 1 -- 172.21.15.99:0/1148692504 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559712e56af0 msgr2=0x559712e5b5f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:50:31.281 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.273+0000 7f59ec412740 1 --2- 172.21.15.99:0/1148692504 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559712e56af0 0x559712e5b5f0 secure :-1 s=READY pgs=141 cs=0 l=1 rev1=1 crypto rx=0x7f59d4009c20 tx=0x7f59d4011810 comp rx=0 tx=0).stop 2024-03-19T01:50:31.281 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.273+0000 7f59ec412740 1 -- 172.21.15.99:0/1148692504 shutdown_connections 2024-03-19T01:50:31.282 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.273+0000 7f59ec412740 1 --2- 172.21.15.99:0/1148692504 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559712e56af0 0x559712e5b5f0 unknown :-1 s=CLOSED pgs=141 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:31.282 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.273+0000 7f59ec412740 1 --2- 172.21.15.99:0/1148692504 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559712e5bc60 0x559712e5e0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:31.282 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.273+0000 7f59ec412740 1 --2- 172.21.15.99:0/1148692504 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559712e56100 0x559712e56520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:31.282 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.273+0000 7f59ec412740 1 -- 172.21.15.99:0/1148692504 >> 172.21.15.99:0/1148692504 conn(0x559712e51670 msgr2=0x559712e51a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:50:31.282 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.273+0000 7f59ec412740 1 -- 172.21.15.99:0/1148692504 shutdown_connections 2024-03-19T01:50:31.282 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.273+0000 7f59ec412740 1 -- 172.21.15.99:0/1148692504 wait complete. 2024-03-19T01:50:31.282 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.273+0000 7f59ec412740 1 -- start start 2024-03-19T01:50:31.282 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.273+0000 7f59ec412740 1 Processor -- start 2024-03-19T01:50:31.282 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.274+0000 7f59ec412740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559712e56100 0x559712e600e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:31.282 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.274+0000 7f59ec412740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559712e56af0 0x559712e60620 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:31.282 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.274+0000 7f59ec412740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559712e5bc60 0x559712e67660 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:31.282 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.274+0000 7f59ec412740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x559712e5ec10 con 0x559712e56af0 2024-03-19T01:50:31.282 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.274+0000 7f59ec412740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x559712d76db0 con 0x559712e5bc60 2024-03-19T01:50:31.282 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.274+0000 7f59ec412740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x559712e5ed90 con 0x559712e56100 2024-03-19T01:50:31.283 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.274+0000 7f59ec412740 10 client.0 _ll_register_callbacks cb 0x559712d7be70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:50:31.283 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.274+0000 7f59e27fc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559712e56af0 0x559712e60620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:31.283 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.274+0000 7f59e27fc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559712e56af0 0x559712e60620 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:35982/0 (socket says 172.21.15.99:35982) 2024-03-19T01:50:31.283 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.274+0000 7f59e27fc640 1 -- 172.21.15.99:0/2167809554 learned_addr learned my addr 172.21.15.99:0/2167809554 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:50:31.283 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.274+0000 7f59e2ffd640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559712e56100 0x559712e600e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:31.283 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.274+0000 7f59e37fe640 1 --2- 172.21.15.99:0/2167809554 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559712e5bc60 0x559712e67660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:31.283 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.274+0000 7f59e27fc640 1 -- 172.21.15.99:0/2167809554 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559712e56100 msgr2=0x559712e600e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:50:31.283 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.274+0000 7f59e27fc640 1 --2- 172.21.15.99:0/2167809554 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559712e56100 0x559712e600e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:31.283 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.274+0000 7f59e27fc640 1 -- 172.21.15.99:0/2167809554 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559712e5bc60 msgr2=0x559712e67660 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:50:31.283 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.274+0000 7f59e27fc640 1 --2- 172.21.15.99:0/2167809554 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559712e5bc60 0x559712e67660 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:31.283 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.274+0000 7f59e27fc640 1 -- 172.21.15.99:0/2167809554 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x559712d75c60 con 0x559712e56af0 2024-03-19T01:50:31.283 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.274+0000 7f59e2ffd640 1 --2- 172.21.15.99:0/2167809554 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559712e56100 0x559712e600e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:50:31.283 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.274+0000 7f59e27fc640 1 --2- 172.21.15.99:0/2167809554 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559712e56af0 0x559712e60620 secure :-1 s=READY pgs=526 cs=0 l=1 rev1=1 crypto rx=0x7f59d80cc8c0 tx=0x7f59d80973d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=17cfc972b6266e52 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:50:31.283 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.274+0000 7f59c6ff5640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:50:31.284 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.274+0000 7f59c6ff5640 1 -- 172.21.15.99:0/2167809554 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f59d809b0c0 con 0x559712e56af0 2024-03-19T01:50:31.284 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.274+0000 7f59c6ff5640 1 -- 172.21.15.99:0/2167809554 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f59d809b9c0 con 0x559712e56af0 2024-03-19T01:50:31.284 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.274+0000 7f59ec412740 1 -- 172.21.15.99:0/2167809554 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f59d4005500 con 0x559712e56af0 2024-03-19T01:50:31.284 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.275+0000 7f59c6ff5640 1 -- 172.21.15.99:0/2167809554 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7f59d809be60 con 0x559712e56af0 2024-03-19T01:50:31.284 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.275+0000 7f59ec412740 10 client.16098 fetch_fsmap learned FSMap version 27 2024-03-19T01:50:31.284 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.275+0000 7f59ec412740 1 -- 172.21.15.99:0/2167809554 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x559712acf210 con 0x559712e56af0 2024-03-19T01:50:31.284 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.275+0000 7f59c6ff5640 1 -- 172.21.15.99:0/2167809554 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f59d4005500 con 0x559712e56af0 2024-03-19T01:50:31.284 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.275+0000 7f59c6ff5640 1 -- 172.21.15.99:0/2167809554 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 27) ==== 8338+0+0 (secure 0 0 0) 0x7f59d80a3a70 con 0x559712e56af0 2024-03-19T01:50:31.284 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.275+0000 7f59ec412740 10 client.16098 fetch_fsmap finished waiting for FSMap version 27 2024-03-19T01:50:31.284 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.275+0000 7f59ec412740 10 client.16098 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:50:31.284 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.275+0000 7f59ec412740 20 client.16098 populate_metadata read hostname 'smithi099' 2024-03-19T01:50:31.284 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.275+0000 7f59ec412740 1 --2- 172.21.15.99:0/2167809554 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x559712e6bfa0 0x559712e6e430 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:31.284 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.275+0000 7f59ec412740 4 client.16098 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:50:31.284 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.275+0000 7f59ec412740 1 -- 172.21.15.99:0/2167809554 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x559712dc8120 con 0x559712e6bfa0 2024-03-19T01:50:31.284 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.275+0000 7f59e2ffd640 1 --2- 172.21.15.99:0/2167809554 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x559712e6bfa0 0x559712e6e430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:31.285 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.276+0000 7f59e2ffd640 1 --2- 172.21.15.99:0/2167809554 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x559712e6bfa0 0x559712e6e430 crc :-1 s=READY pgs=113 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=a975be653c797335 server_cookie=c43c40358e4949be in_seq=0 out_seq=0 2024-03-19T01:50:31.285 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.276+0000 7f59c6ff5640 10 client.16098 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:50:31.285 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.276+0000 7f59c6ff5640 1 -- 172.21.15.99:0/2167809554 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x559712dc8120 con 0x559712e6bfa0 2024-03-19T01:50:31.285 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.276+0000 7f59c6ff5640 10 client.16098 handle_command_reply: tid=0 2024-03-19T01:50:31.285 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.277+0000 7f59ec412740 1 client.16098 shutdown 2024-03-19T01:50:31.285 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.277+0000 7f59ec412740 1 -- 172.21.15.99:0/2167809554 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559712e56af0 msgr2=0x559712e60620 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:50:31.285 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.277+0000 7f59ec412740 1 --2- 172.21.15.99:0/2167809554 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559712e56af0 0x559712e60620 secure :-1 s=READY pgs=526 cs=0 l=1 rev1=1 crypto rx=0x7f59d80cc8c0 tx=0x7f59d80973d0 comp rx=0 tx=0).stop 2024-03-19T01:50:31.285 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.277+0000 7f59ec412740 1 -- 172.21.15.99:0/2167809554 shutdown_connections 2024-03-19T01:50:31.285 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.277+0000 7f59ec412740 1 --2- 172.21.15.99:0/2167809554 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x559712e6bfa0 0x559712e6e430 crc :-1 s=READY pgs=113 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:31.285 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.277+0000 7f59ec412740 1 --2- 172.21.15.99:0/2167809554 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559712e5bc60 0x559712e67660 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:31.285 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.277+0000 7f59ec412740 1 --2- 172.21.15.99:0/2167809554 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559712e56af0 0x559712e60620 unknown :-1 s=CLOSED pgs=526 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:31.285 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.277+0000 7f59ec412740 1 --2- 172.21.15.99:0/2167809554 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559712e56100 0x559712e600e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:31.285 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.277+0000 7f59c6ff5640 0 client.16098 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:50:31.285 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.277+0000 7f59ec412740 1 -- 172.21.15.99:0/2167809554 >> 172.21.15.99:0/2167809554 conn(0x559712e51670 msgr2=0x559712e528c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:50:31.286 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.277+0000 7f59ec412740 1 -- 172.21.15.99:0/2167809554 shutdown_connections 2024-03-19T01:50:31.286 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.277+0000 7f59ec412740 1 -- 172.21.15.99:0/2167809554 wait complete. 2024-03-19T01:50:31.286 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.277+0000 7f59ec412740 20 client.16098 trim_cache size 0 max 16384 2024-03-19T01:50:31.296 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.292+0000 7f59ec412740 1 Processor -- start 2024-03-19T01:50:31.296 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.292+0000 7f59ec412740 1 -- start start 2024-03-19T01:50:31.297 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.292+0000 7f59ec412740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559712e56100 0x559712e57590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:31.297 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.292+0000 7f59ec412740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559712e56af0 0x559712e67620 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:31.297 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.292+0000 7f59ec412740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559712e5bc60 0x559712e57ad0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:31.297 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.292+0000 7f59ec412740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f59e41ab6e0 con 0x559712e5bc60 2024-03-19T01:50:31.297 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.292+0000 7f59ec412740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f59e41a8fb0 con 0x559712e56100 2024-03-19T01:50:31.297 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.292+0000 7f59ec412740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f59e41a6e00 con 0x559712e56af0 2024-03-19T01:50:31.297 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.292+0000 7f59e37fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559712e5bc60 0x559712e57ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:31.297 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.292+0000 7f59e37fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559712e5bc60 0x559712e57ad0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:35992/0 (socket says 172.21.15.99:35992) 2024-03-19T01:50:31.297 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.292+0000 7f59e37fe640 1 -- 172.21.15.99:0/4279106636 learned_addr learned my addr 172.21.15.99:0/4279106636 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:50:31.297 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.292+0000 7f59e2ffd640 1 --2- 172.21.15.99:0/4279106636 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559712e56100 0x559712e57590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:31.297 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.293+0000 7f59e27fc640 1 --2- 172.21.15.99:0/4279106636 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559712e56af0 0x559712e67620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:31.297 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.293+0000 7f59e37fe640 1 -- 172.21.15.99:0/4279106636 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559712e56af0 msgr2=0x559712e67620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:50:31.297 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.293+0000 7f59e37fe640 1 --2- 172.21.15.99:0/4279106636 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559712e56af0 0x559712e67620 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:31.298 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.293+0000 7f59e37fe640 1 -- 172.21.15.99:0/4279106636 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559712e56100 msgr2=0x559712e57590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:50:31.298 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.293+0000 7f59e37fe640 1 --2- 172.21.15.99:0/4279106636 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559712e56100 0x559712e57590 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:31.298 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.293+0000 7f59e37fe640 1 -- 172.21.15.99:0/4279106636 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x559712e4cee0 con 0x559712e5bc60 2024-03-19T01:50:31.298 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.293+0000 7f59e37fe640 1 --2- 172.21.15.99:0/4279106636 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559712e5bc60 0x559712e57ad0 secure :-1 s=READY pgs=527 cs=0 l=1 rev1=1 crypto rx=0x7f59dc037040 tx=0x7f59dc009360 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b3c9c1e7b68e8aac server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:50:31.298 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.293+0000 7f59c8ff9640 1 -- 172.21.15.99:0/4279106636 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f59dc039040 con 0x559712e5bc60 2024-03-19T01:50:31.298 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.293+0000 7f59c8ff9640 1 -- 172.21.15.99:0/4279106636 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f59dc054070 con 0x559712e5bc60 2024-03-19T01:50:31.298 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.293+0000 7f59c8ff9640 1 -- 172.21.15.99:0/4279106636 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f59dc04f6b0 con 0x559712e5bc60 2024-03-19T01:50:31.298 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.294+0000 7f59ec412740 1 -- 172.21.15.99:0/4279106636 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559712e5bc60 msgr2=0x559712e57ad0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:50:31.298 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.294+0000 7f59ec412740 1 --2- 172.21.15.99:0/4279106636 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559712e5bc60 0x559712e57ad0 secure :-1 s=READY pgs=527 cs=0 l=1 rev1=1 crypto rx=0x7f59dc037040 tx=0x7f59dc009360 comp rx=0 tx=0).stop 2024-03-19T01:50:31.298 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.294+0000 7f59ec412740 1 -- 172.21.15.99:0/4279106636 shutdown_connections 2024-03-19T01:50:31.298 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.294+0000 7f59ec412740 1 --2- 172.21.15.99:0/4279106636 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559712e5bc60 0x559712e57ad0 unknown :-1 s=CLOSED pgs=527 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:31.298 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.294+0000 7f59ec412740 1 --2- 172.21.15.99:0/4279106636 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559712e56af0 0x559712e67620 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:31.298 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.294+0000 7f59ec412740 1 --2- 172.21.15.99:0/4279106636 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559712e56100 0x559712e57590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:31.299 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.294+0000 7f59ec412740 1 -- 172.21.15.99:0/4279106636 >> 172.21.15.99:0/4279106636 conn(0x559712e51670 msgr2=0x559712e50220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:50:31.299 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.294+0000 7f59ec412740 1 -- 172.21.15.99:0/4279106636 shutdown_connections 2024-03-19T01:50:31.299 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.294+0000 7f59ec412740 1 -- 172.21.15.99:0/4279106636 wait complete. 2024-03-19T01:50:31.299 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.294+0000 7f59ec412740 1 -- start start 2024-03-19T01:50:31.299 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.294+0000 7f59ec412740 1 Processor -- start 2024-03-19T01:50:31.299 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.294+0000 7f59ec412740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559712e56100 0x559712e651c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:31.299 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.294+0000 7f59ec412740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559712e56af0 0x559712e65700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:31.299 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.294+0000 7f59ec412740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559712e5bc60 0x559712e66710 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:31.299 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.294+0000 7f59ec412740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f59e41a7470 con 0x559712e5bc60 2024-03-19T01:50:31.299 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.294+0000 7f59ec412740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f59e41a88e0 con 0x559712e56100 2024-03-19T01:50:31.299 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.294+0000 7f59ec412740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f59e41a7b40 con 0x559712e56af0 2024-03-19T01:50:31.299 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.294+0000 7f59ec412740 10 client.0 _ll_register_callbacks cb 0x7f59d8009790 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:50:31.299 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.294+0000 7f59e37fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559712e5bc60 0x559712e66710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:31.299 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.294+0000 7f59e2ffd640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559712e56100 0x559712e651c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:31.299 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.294+0000 7f59e37fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559712e5bc60 0x559712e66710 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:36004/0 (socket says 172.21.15.99:36004) 2024-03-19T01:50:31.300 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.294+0000 7f59e37fe640 1 -- 172.21.15.99:0/2817497727 learned_addr learned my addr 172.21.15.99:0/2817497727 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:50:31.300 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.294+0000 7f59e2ffd640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559712e56100 0x559712e651c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:60970/0 (socket says 172.21.15.99:60970) 2024-03-19T01:50:31.300 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.294+0000 7f59e27fc640 1 --2- 172.21.15.99:0/2817497727 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559712e56af0 0x559712e65700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:31.300 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.295+0000 7f59e37fe640 1 -- 172.21.15.99:0/2817497727 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559712e56af0 msgr2=0x559712e65700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:50:31.300 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.295+0000 7f59e37fe640 1 --2- 172.21.15.99:0/2817497727 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559712e56af0 0x559712e65700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:31.300 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.295+0000 7f59e37fe640 1 -- 172.21.15.99:0/2817497727 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559712e56100 msgr2=0x559712e651c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:50:31.300 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.295+0000 7f59e37fe640 1 --2- 172.21.15.99:0/2817497727 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559712e56100 0x559712e651c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:31.300 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.295+0000 7f59e37fe640 1 -- 172.21.15.99:0/2817497727 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x559712b9b620 con 0x559712e5bc60 2024-03-19T01:50:31.300 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.295+0000 7f59e37fe640 1 --2- 172.21.15.99:0/2817497727 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559712e5bc60 0x559712e66710 secure :-1 s=READY pgs=528 cs=0 l=1 rev1=1 crypto rx=0x7f59dc02c630 tx=0x7f59dc034840 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d1f2f6cb04889f86 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:50:31.300 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.295+0000 7f59c6ff5640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:50:31.300 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.295+0000 7f59c6ff5640 1 -- 172.21.15.99:0/2817497727 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f59dc039040 con 0x559712e5bc60 2024-03-19T01:50:31.300 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.295+0000 7f59c6ff5640 1 -- 172.21.15.99:0/2817497727 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f59dc034dc0 con 0x559712e5bc60 2024-03-19T01:50:31.300 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.295+0000 7f59ec412740 1 -- 172.21.15.99:0/2817497727 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f59dc054070 con 0x559712e5bc60 2024-03-19T01:50:31.301 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.295+0000 7f59c6ff5640 1 -- 172.21.15.99:0/2817497727 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7f59dc04f450 con 0x559712e5bc60 2024-03-19T01:50:31.301 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.295+0000 7f59ec412740 10 client.16104 fetch_fsmap learned FSMap version 27 2024-03-19T01:50:31.301 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.295+0000 7f59ec412740 1 -- 172.21.15.99:0/2817497727 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x559712ed2670 con 0x559712e5bc60 2024-03-19T01:50:31.301 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.295+0000 7f59c6ff5640 1 -- 172.21.15.99:0/2817497727 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f59dc054070 con 0x559712e5bc60 2024-03-19T01:50:31.301 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.296+0000 7f59c6ff5640 1 -- 172.21.15.99:0/2817497727 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 27) ==== 8338+0+0 (secure 0 0 0) 0x7f59dc03f070 con 0x559712e5bc60 2024-03-19T01:50:31.301 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.296+0000 7f59ec412740 10 client.16104 fetch_fsmap finished waiting for FSMap version 27 2024-03-19T01:50:31.301 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.296+0000 7f59ec412740 10 client.16104 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:50:31.301 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.296+0000 7f59ec412740 20 client.16104 populate_metadata read hostname 'smithi099' 2024-03-19T01:50:31.301 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.296+0000 7f59ec412740 1 --2- 172.21.15.99:0/2817497727 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x559712ed30a0 0x559712ed54c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:31.301 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.296+0000 7f59ec412740 4 client.16104 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:50:31.301 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.296+0000 7f59ec412740 1 -- 172.21.15.99:0/2817497727 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x559712dc2470 con 0x559712ed30a0 2024-03-19T01:50:31.301 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.296+0000 7f59e2ffd640 1 --2- 172.21.15.99:0/2817497727 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x559712ed30a0 0x559712ed54c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:31.301 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.296+0000 7f59e2ffd640 1 --2- 172.21.15.99:0/2817497727 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x559712ed30a0 0x559712ed54c0 crc :-1 s=READY pgs=114 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=6e8e2c80b5e0584a server_cookie=405206e2913d2d8c in_seq=0 out_seq=0 2024-03-19T01:50:31.301 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.296+0000 7f59c6ff5640 10 client.16104 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:50:31.356 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.355+0000 7f59c6ff5640 1 -- 172.21.15.99:0/2817497727 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x559712dc2470 con 0x559712ed30a0 2024-03-19T01:50:31.356 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.355+0000 7f59c6ff5640 10 client.16104 handle_command_reply: tid=0 2024-03-19T01:50:31.356 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.355+0000 7f59ec412740 1 client.16104 shutdown 2024-03-19T01:50:31.356 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.355+0000 7f59ec412740 1 -- 172.21.15.99:0/2817497727 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559712e5bc60 msgr2=0x559712e66710 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:50:31.356 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.355+0000 7f59ec412740 1 --2- 172.21.15.99:0/2817497727 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559712e5bc60 0x559712e66710 secure :-1 s=READY pgs=528 cs=0 l=1 rev1=1 crypto rx=0x7f59dc02c630 tx=0x7f59dc034840 comp rx=0 tx=0).stop 2024-03-19T01:50:31.356 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.355+0000 7f59ec412740 1 -- 172.21.15.99:0/2817497727 shutdown_connections 2024-03-19T01:50:31.356 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.355+0000 7f59ec412740 1 --2- 172.21.15.99:0/2817497727 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x559712ed30a0 0x559712ed54c0 crc :-1 s=READY pgs=114 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:31.357 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.355+0000 7f59ec412740 1 --2- 172.21.15.99:0/2817497727 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559712e5bc60 0x559712e66710 unknown :-1 s=CLOSED pgs=528 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:31.357 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.355+0000 7f59ec412740 1 --2- 172.21.15.99:0/2817497727 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559712e56af0 0x559712e65700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:31.357 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.355+0000 7f59c6ff5640 0 client.16104 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:50:31.357 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.355+0000 7f59ec412740 1 --2- 172.21.15.99:0/2817497727 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559712e56100 0x559712e651c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:31.357 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.355+0000 7f59ec412740 1 -- 172.21.15.99:0/2817497727 >> 172.21.15.99:0/2817497727 conn(0x559712e51670 msgr2=0x559712e57590 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:50:31.357 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.356+0000 7f59ec412740 1 -- 172.21.15.99:0/2817497727 shutdown_connections 2024-03-19T01:50:31.357 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.356+0000 7f59ec412740 1 -- 172.21.15.99:0/2817497727 wait complete. 2024-03-19T01:50:31.357 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.356+0000 7f59ec412740 20 client.16104 trim_cache size 0 max 16384 2024-03-19T01:50:31.357 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:50:31.357 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:50:31.358 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:50:31.358 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:50:31.358 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.356+0000 7f59e9759640 1 -- 172.21.15.99:0/2150815964 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f59cc0a28a0 msgr2=0x7f59cc0a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:50:31.359 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.356+0000 7f59e9759640 1 --2- 172.21.15.99:0/2150815964 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f59cc0a28a0 0x7f59cc0a4d60 secure :-1 s=READY pgs=302 cs=0 l=1 rev1=1 crypto rx=0x559712e57350 tx=0x7f59dc02c0f0 comp rx=0 tx=0).stop 2024-03-19T01:50:31.359 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.356+0000 7f59e9759640 1 -- 172.21.15.99:0/2150815964 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f59e4104c30 msgr2=0x7f59e4197fe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:50:31.359 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.356+0000 7f59e9759640 1 --2- 172.21.15.99:0/2150815964 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f59e4104c30 0x7f59e4197fe0 secure :-1 s=READY pgs=525 cs=0 l=1 rev1=1 crypto rx=0x7f59d800e9c0 tx=0x7f59d800ee80 comp rx=0 tx=0).stop 2024-03-19T01:50:31.359 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.356+0000 7f59e9759640 1 -- 172.21.15.99:0/2150815964 shutdown_connections 2024-03-19T01:50:31.359 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.356+0000 7f59e9759640 1 --2- 172.21.15.99:0/2150815964 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f59e4107560 0x7f59e419f050 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:31.359 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.356+0000 7f59e9759640 1 --2- 172.21.15.99:0/2150815964 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f59e4104c30 0x7f59e4197fe0 unknown :-1 s=CLOSED pgs=525 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:31.359 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.356+0000 7f59e9759640 1 --2- 172.21.15.99:0/2150815964 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f59cc0a28a0 0x7f59cc0a4d60 unknown :-1 s=CLOSED pgs=302 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:31.359 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.356+0000 7f59e9759640 1 --2- 172.21.15.99:0/2150815964 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f59e4102300 0x7f59e4197aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:31.359 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.356+0000 7f59e9759640 1 -- 172.21.15.99:0/2150815964 >> 172.21.15.99:0/2150815964 conn(0x7f59e40fbcb0 msgr2=0x7f59e4102c60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:50:31.359 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.356+0000 7f59e9759640 1 -- 172.21.15.99:0/2150815964 shutdown_connections 2024-03-19T01:50:31.359 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:31.356+0000 7f59e9759640 1 -- 172.21.15.99:0/2150815964 wait complete. 2024-03-19T01:50:31.696 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1148003 11.39 MB/sec execute 421 sec latency 32.108 ms 2024-03-19T01:50:32.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:50:31 smithi115 ceph-mon[67332]: pgmap v633: 97 pgs: 97 active+clean; 255 MiB data, 3.0 GiB used, 1.0 TiB / 1.0 TiB avail; 801 KiB/s rd, 3.1 MiB/s wr, 251 op/s 2024-03-19T01:50:32.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:50:31 smithi099 ceph-mon[62951]: pgmap v633: 97 pgs: 97 active+clean; 255 MiB data, 3.0 GiB used, 1.0 TiB / 1.0 TiB avail; 801 KiB/s rd, 3.1 MiB/s wr, 251 op/s 2024-03-19T01:50:32.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:50:31 smithi112 ceph-mon[66819]: pgmap v633: 97 pgs: 97 active+clean; 255 MiB data, 3.0 GiB used, 1.0 TiB / 1.0 TiB avail; 801 KiB/s rd, 3.1 MiB/s wr, 251 op/s 2024-03-19T01:50:32.696 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1150664 11.40 MB/sec execute 422 sec latency 11.054 ms 2024-03-19T01:50:33.696 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1152022 11.39 MB/sec execute 423 sec latency 80.065 ms 2024-03-19T01:50:34.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:50:33 smithi115 ceph-mon[67332]: pgmap v634: 97 pgs: 97 active+clean; 250 MiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 3.4 MiB/s wr, 305 op/s 2024-03-19T01:50:34.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:50:33 smithi099 ceph-mon[62951]: pgmap v634: 97 pgs: 97 active+clean; 250 MiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 3.4 MiB/s wr, 305 op/s 2024-03-19T01:50:34.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:50:33 smithi112 ceph-mon[66819]: pgmap v634: 97 pgs: 97 active+clean; 250 MiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 3.4 MiB/s wr, 305 op/s 2024-03-19T01:50:34.696 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1154604 11.40 MB/sec execute 424 sec latency 31.317 ms 2024-03-19T01:50:35.696 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1156258 11.39 MB/sec execute 425 sec latency 34.574 ms 2024-03-19T01:50:36.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:50:35 smithi099 ceph-mon[62951]: pgmap v635: 97 pgs: 97 active+clean; 251 MiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 774 KiB/s rd, 2.8 MiB/s wr, 236 op/s 2024-03-19T01:50:36.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:50:35 smithi112 ceph-mon[66819]: pgmap v635: 97 pgs: 97 active+clean; 251 MiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 774 KiB/s rd, 2.8 MiB/s wr, 236 op/s 2024-03-19T01:50:36.414 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:50:35 smithi115 ceph-mon[67332]: pgmap v635: 97 pgs: 97 active+clean; 251 MiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 774 KiB/s rd, 2.8 MiB/s wr, 236 op/s 2024-03-19T01:50:36.696 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1158435 11.39 MB/sec execute 426 sec latency 29.914 ms 2024-03-19T01:50:37.696 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1160518 11.39 MB/sec execute 427 sec latency 31.408 ms 2024-03-19T01:50:38.320 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:50:37 smithi115 ceph-mon[67332]: pgmap v636: 97 pgs: 97 active+clean; 253 MiB data, 3.0 GiB used, 1.0 TiB / 1.0 TiB avail; 1002 KiB/s rd, 3.4 MiB/s wr, 294 op/s 2024-03-19T01:50:38.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:50:37 smithi099 ceph-mon[62951]: pgmap v636: 97 pgs: 97 active+clean; 253 MiB data, 3.0 GiB used, 1.0 TiB / 1.0 TiB avail; 1002 KiB/s rd, 3.4 MiB/s wr, 294 op/s 2024-03-19T01:50:38.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:50:37 smithi112 ceph-mon[66819]: pgmap v636: 97 pgs: 97 active+clean; 253 MiB data, 3.0 GiB used, 1.0 TiB / 1.0 TiB avail; 1002 KiB/s rd, 3.4 MiB/s wr, 294 op/s 2024-03-19T01:50:38.696 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1162197 11.39 MB/sec execute 428 sec latency 96.339 ms 2024-03-19T01:50:39.696 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1164497 11.39 MB/sec execute 429 sec latency 100.356 ms 2024-03-19T01:50:40.324 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:50:39 smithi115 ceph-mon[67332]: pgmap v637: 97 pgs: 97 active+clean; 252 MiB data, 3.0 GiB used, 1.0 TiB / 1.0 TiB avail; 680 KiB/s rd, 2.6 MiB/s wr, 240 op/s 2024-03-19T01:50:40.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:50:39 smithi099 ceph-mon[62951]: pgmap v637: 97 pgs: 97 active+clean; 252 MiB data, 3.0 GiB used, 1.0 TiB / 1.0 TiB avail; 680 KiB/s rd, 2.6 MiB/s wr, 240 op/s 2024-03-19T01:50:40.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:50:39 smithi112 ceph-mon[66819]: pgmap v637: 97 pgs: 97 active+clean; 252 MiB data, 3.0 GiB used, 1.0 TiB / 1.0 TiB avail; 680 KiB/s rd, 2.6 MiB/s wr, 240 op/s 2024-03-19T01:50:40.696 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1166297 11.38 MB/sec execute 430 sec latency 40.615 ms 2024-03-19T01:50:41.696 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1168641 11.39 MB/sec execute 431 sec latency 120.315 ms 2024-03-19T01:50:42.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:50:41 smithi115 ceph-mon[67332]: pgmap v638: 97 pgs: 97 active+clean; 255 MiB data, 3.0 GiB used, 1.0 TiB / 1.0 TiB avail; 895 KiB/s rd, 3.0 MiB/s wr, 273 op/s 2024-03-19T01:50:42.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:50:41 smithi099 ceph-mon[62951]: pgmap v638: 97 pgs: 97 active+clean; 255 MiB data, 3.0 GiB used, 1.0 TiB / 1.0 TiB avail; 895 KiB/s rd, 3.0 MiB/s wr, 273 op/s 2024-03-19T01:50:42.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:50:41 smithi112 ceph-mon[66819]: pgmap v638: 97 pgs: 97 active+clean; 255 MiB data, 3.0 GiB used, 1.0 TiB / 1.0 TiB avail; 895 KiB/s rd, 3.0 MiB/s wr, 273 op/s 2024-03-19T01:50:42.696 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1170090 11.38 MB/sec execute 432 sec latency 39.090 ms 2024-03-19T01:50:43.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:50:42 smithi115 ceph-mon[67332]: pgmap v639: 97 pgs: 97 active+clean; 255 MiB data, 3.0 GiB used, 1.0 TiB / 1.0 TiB avail; 895 KiB/s rd, 3.0 MiB/s wr, 271 op/s 2024-03-19T01:50:43.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:50:42 smithi099 ceph-mon[62951]: pgmap v639: 97 pgs: 97 active+clean; 255 MiB data, 3.0 GiB used, 1.0 TiB / 1.0 TiB avail; 895 KiB/s rd, 3.0 MiB/s wr, 271 op/s 2024-03-19T01:50:43.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:50:42 smithi112 ceph-mon[66819]: pgmap v639: 97 pgs: 97 active+clean; 255 MiB data, 3.0 GiB used, 1.0 TiB / 1.0 TiB avail; 895 KiB/s rd, 3.0 MiB/s wr, 271 op/s 2024-03-19T01:50:43.697 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1172430 11.39 MB/sec execute 433 sec latency 12.271 ms 2024-03-19T01:50:44.452 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.449+0000 7fa4dc3d4640 1 -- 172.21.15.99:0/1444418590 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa4d4101940 msgr2=0x7fa4d4101dc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:50:44.452 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.449+0000 7fa4dc3d4640 1 --2- 172.21.15.99:0/1444418590 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa4d4101940 0x7fa4d4101dc0 secure :-1 s=READY pgs=529 cs=0 l=1 rev1=1 crypto rx=0x7fa4c4009900 tx=0x7fa4c402bc30 comp rx=0 tx=0).stop 2024-03-19T01:50:44.452 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.449+0000 7fa4dc3d4640 1 -- 172.21.15.99:0/1444418590 shutdown_connections 2024-03-19T01:50:44.453 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.449+0000 7fa4dc3d4640 1 --2- 172.21.15.99:0/1444418590 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa4d4102300 0x7fa4d4108b90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:44.453 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.449+0000 7fa4dc3d4640 1 --2- 172.21.15.99:0/1444418590 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa4d4101940 0x7fa4d4101dc0 unknown :-1 s=CLOSED pgs=529 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:44.453 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.449+0000 7fa4dc3d4640 1 --2- 172.21.15.99:0/1444418590 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa4d4100750 0x7fa4d4100b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:44.453 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.449+0000 7fa4dc3d4640 1 -- 172.21.15.99:0/1444418590 >> 172.21.15.99:0/1444418590 conn(0x7fa4d40fbcc0 msgr2=0x7fa4d40fe100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:50:44.453 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.449+0000 7fa4dc3d4640 1 -- 172.21.15.99:0/1444418590 shutdown_connections 2024-03-19T01:50:44.453 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.449+0000 7fa4dc3d4640 1 -- 172.21.15.99:0/1444418590 wait complete. 2024-03-19T01:50:44.453 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.450+0000 7fa4dc3d4640 1 Processor -- start 2024-03-19T01:50:44.453 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.450+0000 7fa4dc3d4640 1 -- start start 2024-03-19T01:50:44.453 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.450+0000 7fa4dc3d4640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa4d4100750 0x7fa4d4196960 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:44.453 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.450+0000 7fa4dc3d4640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa4d4101940 0x7fa4d4196ea0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:44.453 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.450+0000 7fa4dc3d4640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa4d4102300 0x7fa4d419df10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:44.453 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.450+0000 7fa4dc3d4640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fa4d410c3d0 con 0x7fa4d4102300 2024-03-19T01:50:44.453 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.450+0000 7fa4dc3d4640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fa4d410c250 con 0x7fa4d4100750 2024-03-19T01:50:44.454 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.450+0000 7fa4dc3d4640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fa4d410c550 con 0x7fa4d4101940 2024-03-19T01:50:44.454 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.450+0000 7fa4da971640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa4d4102300 0x7fa4d419df10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:44.454 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.450+0000 7fa4da971640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa4d4102300 0x7fa4d419df10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:36614/0 (socket says 172.21.15.99:36614) 2024-03-19T01:50:44.454 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.450+0000 7fa4da971640 1 -- 172.21.15.99:0/1080122192 learned_addr learned my addr 172.21.15.99:0/1080122192 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:50:44.454 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.450+0000 7fa4da170640 1 --2- 172.21.15.99:0/1080122192 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa4d4100750 0x7fa4d4196960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:44.454 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.450+0000 7fa4d996f640 1 --2- 172.21.15.99:0/1080122192 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa4d4101940 0x7fa4d4196ea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:44.454 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.451+0000 7fa4da971640 1 -- 172.21.15.99:0/1080122192 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa4d4101940 msgr2=0x7fa4d4196ea0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:50:44.454 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.451+0000 7fa4da971640 1 --2- 172.21.15.99:0/1080122192 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa4d4101940 0x7fa4d4196ea0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:44.454 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.451+0000 7fa4da971640 1 -- 172.21.15.99:0/1080122192 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa4d4100750 msgr2=0x7fa4d4196960 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:50:44.454 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.451+0000 7fa4da971640 1 --2- 172.21.15.99:0/1080122192 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa4d4100750 0x7fa4d4196960 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:44.454 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.451+0000 7fa4da971640 1 -- 172.21.15.99:0/1080122192 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa4d419e580 con 0x7fa4d4102300 2024-03-19T01:50:44.454 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.451+0000 7fa4da971640 1 --2- 172.21.15.99:0/1080122192 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa4d4102300 0x7fa4d419df10 secure :-1 s=READY pgs=530 cs=0 l=1 rev1=1 crypto rx=0x7fa4cc00eb10 tx=0x7fa4cc00c2d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:50:44.454 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.451+0000 7fa4cb7fe640 1 -- 172.21.15.99:0/1080122192 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa4cc004b20 con 0x7fa4d4102300 2024-03-19T01:50:44.455 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.451+0000 7fa4cb7fe640 1 -- 172.21.15.99:0/1080122192 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa4cc004cc0 con 0x7fa4d4102300 2024-03-19T01:50:44.455 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.451+0000 7fa4cb7fe640 1 -- 172.21.15.99:0/1080122192 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa4cc010760 con 0x7fa4d4102300 2024-03-19T01:50:44.455 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.451+0000 7fa4dc3d4640 1 -- 172.21.15.99:0/1080122192 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa4d419e870 con 0x7fa4d4102300 2024-03-19T01:50:44.455 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.451+0000 7fa4dc3d4640 1 -- 172.21.15.99:0/1080122192 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fa4d419ed90 con 0x7fa4d4102300 2024-03-19T01:50:44.460 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.453+0000 7fa4cb7fe640 1 -- 172.21.15.99:0/1080122192 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7fa4cc005020 con 0x7fa4d4102300 2024-03-19T01:50:44.460 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.454+0000 7fa4defbe740 1 Processor -- start 2024-03-19T01:50:44.460 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.454+0000 7fa4defbe740 1 -- start start 2024-03-19T01:50:44.460 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.454+0000 7fa4defbe740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5649b975f310 0x5649b975f730 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:44.460 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.454+0000 7fa4defbe740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5649b975fd00 0x5649b9764800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:44.460 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.454+0000 7fa4defbe740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5649b9764e70 0x5649b9767300 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:44.460 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.454+0000 7fa4defbe740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x5649b96c1be0 con 0x5649b9764e70 2024-03-19T01:50:44.460 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.454+0000 7fa4defbe740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x5649b966a960 con 0x5649b975fd00 2024-03-19T01:50:44.460 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.454+0000 7fa4defbe740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x5649b96bdc80 con 0x5649b975f310 2024-03-19T01:50:44.461 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.454+0000 7fa4cb7fe640 1 --2- 172.21.15.99:0/1080122192 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa4c00a28a0 0x7fa4c00a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:44.461 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.454+0000 7fa4da170640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5649b975fd00 0x5649b9764800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:44.461 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.454+0000 7fa4d996f640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5649b975f310 0x5649b975f730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:44.461 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.454+0000 7fa4d996f640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5649b9764e70 0x5649b9767300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:44.461 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.454+0000 7fa4da170640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5649b975fd00 0x5649b9764800 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:51108/0 (socket says 172.21.15.99:51108) 2024-03-19T01:50:44.461 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.454+0000 7fa4da170640 1 -- 172.21.15.99:0/2860589334 learned_addr learned my addr 172.21.15.99:0/2860589334 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:50:44.461 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.454+0000 7fa4da971640 1 --2- 172.21.15.99:0/1080122192 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa4c00a28a0 0x7fa4c00a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:44.461 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.454+0000 7fa4da170640 1 -- 172.21.15.99:0/2860589334 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5649b975f310 msgr2=0x5649b975f730 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:50:44.461 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.454+0000 7fa4da170640 1 --2- 172.21.15.99:0/2860589334 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5649b975f310 0x5649b975f730 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:44.461 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.454+0000 7fa4da170640 1 -- 172.21.15.99:0/2860589334 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5649b9764e70 msgr2=0x5649b9767300 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:50:44.461 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.454+0000 7fa4da170640 1 --2- 172.21.15.99:0/2860589334 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5649b9764e70 0x5649b9767300 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:44.461 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.454+0000 7fa4da170640 1 -- 172.21.15.99:0/2860589334 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5649b9661d20 con 0x5649b975fd00 2024-03-19T01:50:44.461 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.455+0000 7fa4cb7fe640 1 -- 172.21.15.99:0/1080122192 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7fa4cc0cf180 con 0x7fa4d4102300 2024-03-19T01:50:44.461 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.455+0000 7fa4da170640 1 --2- 172.21.15.99:0/2860589334 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5649b975fd00 0x5649b9764800 secure :-1 s=READY pgs=142 cs=0 l=1 rev1=1 crypto rx=0x7fa4d0009ca0 tx=0x7fa4d0011810 comp rx=0 tx=0).ready entity=mon.2 client_cookie=77af1247aef7e3d3 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:50:44.462 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.455+0000 7fa4da971640 1 --2- 172.21.15.99:0/1080122192 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa4c00a28a0 0x7fa4c00a4d60 secure :-1 s=READY pgs=304 cs=0 l=1 rev1=1 crypto rx=0x7fa4cc0a3d50 tx=0x7fa4cc0964a0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:50:44.462 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.455+0000 7fa4c8bfe640 1 -- 172.21.15.99:0/2860589334 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa4d0009e50 con 0x5649b975fd00 2024-03-19T01:50:44.462 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.455+0000 7fa4c8bfe640 1 -- 172.21.15.99:0/2860589334 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa4d00035f0 con 0x5649b975fd00 2024-03-19T01:50:44.462 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.455+0000 7fa4c8bfe640 1 -- 172.21.15.99:0/2860589334 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa4d0019630 con 0x5649b975fd00 2024-03-19T01:50:44.462 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.456+0000 7fa4defbe740 1 -- 172.21.15.99:0/2860589334 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5649b975fd00 msgr2=0x5649b9764800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:50:44.462 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.456+0000 7fa4defbe740 1 --2- 172.21.15.99:0/2860589334 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5649b975fd00 0x5649b9764800 secure :-1 s=READY pgs=142 cs=0 l=1 rev1=1 crypto rx=0x7fa4d0009ca0 tx=0x7fa4d0011810 comp rx=0 tx=0).stop 2024-03-19T01:50:44.462 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.456+0000 7fa4defbe740 1 -- 172.21.15.99:0/2860589334 shutdown_connections 2024-03-19T01:50:44.462 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.456+0000 7fa4defbe740 1 --2- 172.21.15.99:0/2860589334 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5649b975fd00 0x5649b9764800 unknown :-1 s=CLOSED pgs=142 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:44.462 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.456+0000 7fa4defbe740 1 --2- 172.21.15.99:0/2860589334 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5649b9764e70 0x5649b9767300 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:44.462 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.456+0000 7fa4defbe740 1 --2- 172.21.15.99:0/2860589334 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5649b975f310 0x5649b975f730 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:44.462 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.456+0000 7fa4defbe740 1 -- 172.21.15.99:0/2860589334 >> 172.21.15.99:0/2860589334 conn(0x5649b9758670 msgr2=0x5649b9758a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:50:44.462 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.456+0000 7fa4defbe740 1 -- 172.21.15.99:0/2860589334 shutdown_connections 2024-03-19T01:50:44.462 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.456+0000 7fa4defbe740 1 -- 172.21.15.99:0/2860589334 wait complete. 2024-03-19T01:50:44.462 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.456+0000 7fa4defbe740 1 -- start start 2024-03-19T01:50:44.463 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.456+0000 7fa4defbe740 1 Processor -- start 2024-03-19T01:50:44.463 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.456+0000 7fa4defbe740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5649b975f310 0x5649b9769370 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:44.463 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.456+0000 7fa4defbe740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5649b975fd00 0x5649b97698b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:44.463 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.456+0000 7fa4defbe740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5649b9764e70 0x5649b97708f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:44.463 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.456+0000 7fa4defbe740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x5649b9767e20 con 0x5649b9764e70 2024-03-19T01:50:44.463 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.456+0000 7fa4defbe740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x5649b967ddb0 con 0x5649b975fd00 2024-03-19T01:50:44.463 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.456+0000 7fa4defbe740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x5649b9767fa0 con 0x5649b975f310 2024-03-19T01:50:44.463 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.457+0000 7fa4d996f640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5649b9764e70 0x5649b97708f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:44.463 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.457+0000 7fa4defbe740 10 client.0 _ll_register_callbacks cb 0x5649b9682e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:50:44.463 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.457+0000 7fa4d996f640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5649b9764e70 0x5649b97708f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:36630/0 (socket says 172.21.15.99:36630) 2024-03-19T01:50:44.463 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.457+0000 7fa4d996f640 1 -- 172.21.15.99:0/77245332 learned_addr learned my addr 172.21.15.99:0/77245332 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:50:44.463 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.457+0000 7fa4da170640 1 --2- 172.21.15.99:0/77245332 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5649b975fd00 0x5649b97698b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:44.463 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.457+0000 7fa4d996f640 1 --2- 172.21.15.99:0/77245332 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5649b975f310 0x5649b9769370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:44.463 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.457+0000 7fa4d996f640 1 -- 172.21.15.99:0/77245332 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5649b975f310 msgr2=0x5649b9769370 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:50:44.464 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.457+0000 7fa4d996f640 1 --2- 172.21.15.99:0/77245332 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5649b975f310 0x5649b9769370 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:44.464 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.457+0000 7fa4d996f640 1 -- 172.21.15.99:0/77245332 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5649b975fd00 msgr2=0x5649b97698b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:50:44.464 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.457+0000 7fa4d996f640 1 --2- 172.21.15.99:0/77245332 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5649b975fd00 0x5649b97698b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:44.464 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.457+0000 7fa4d996f640 1 -- 172.21.15.99:0/77245332 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5649b967cc60 con 0x5649b9764e70 2024-03-19T01:50:44.464 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.457+0000 7fa4d996f640 1 --2- 172.21.15.99:0/77245332 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5649b9764e70 0x5649b97708f0 secure :-1 s=READY pgs=531 cs=0 l=1 rev1=1 crypto rx=0x7fa4c4036040 tx=0x7fa4c4044040 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1af75d09601d1e8f server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:50:44.464 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.457+0000 7fa4bdffb640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:50:44.464 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.457+0000 7fa4bdffb640 1 -- 172.21.15.99:0/77245332 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa4c4003c10 con 0x5649b9764e70 2024-03-19T01:50:44.464 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.457+0000 7fa4bdffb640 1 -- 172.21.15.99:0/77245332 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7fa4c4003db0 con 0x5649b9764e70 2024-03-19T01:50:44.464 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.457+0000 7fa4defbe740 1 -- 172.21.15.99:0/77245332 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fa4d00035f0 con 0x5649b9764e70 2024-03-19T01:50:44.464 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.457+0000 7fa4bdffb640 1 -- 172.21.15.99:0/77245332 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7fa4c402ccc0 con 0x5649b9764e70 2024-03-19T01:50:44.464 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.457+0000 7fa4defbe740 10 client.16116 fetch_fsmap learned FSMap version 27 2024-03-19T01:50:44.464 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.457+0000 7fa4defbe740 1 -- 172.21.15.99:0/77245332 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x5649b93d6210 con 0x5649b9764e70 2024-03-19T01:50:44.464 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.457+0000 7fa4bdffb640 1 -- 172.21.15.99:0/77245332 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa4d00035f0 con 0x5649b9764e70 2024-03-19T01:50:44.464 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.458+0000 7fa4bdffb640 1 -- 172.21.15.99:0/77245332 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 27) ==== 8338+0+0 (secure 0 0 0) 0x7fa4c4057080 con 0x5649b9764e70 2024-03-19T01:50:44.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.458+0000 7fa4defbe740 10 client.16116 fetch_fsmap finished waiting for FSMap version 27 2024-03-19T01:50:44.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.458+0000 7fa4defbe740 10 client.16116 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:50:44.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.458+0000 7fa4defbe740 20 client.16116 populate_metadata read hostname 'smithi099' 2024-03-19T01:50:44.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.458+0000 7fa4defbe740 1 --2- 172.21.15.99:0/77245332 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5649b9775180 0x5649b97775d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:44.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.458+0000 7fa4defbe740 4 client.16116 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:50:44.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.458+0000 7fa4defbe740 1 -- 172.21.15.99:0/77245332 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5649b96cf120 con 0x5649b9775180 2024-03-19T01:50:44.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.458+0000 7fa4da170640 1 --2- 172.21.15.99:0/77245332 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5649b9775180 0x5649b97775d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:44.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.458+0000 7fa4bdffb640 10 client.16116 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:50:44.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.458+0000 7fa4da170640 1 --2- 172.21.15.99:0/77245332 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5649b9775180 0x5649b97775d0 crc :-1 s=READY pgs=115 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=c509c9b8f7115514 server_cookie=f295e6c854711a6d in_seq=0 out_seq=0 2024-03-19T01:50:44.466 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.460+0000 7fa4bdffb640 1 -- 172.21.15.99:0/77245332 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x5649b96cf120 con 0x5649b9775180 2024-03-19T01:50:44.466 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.460+0000 7fa4bdffb640 10 client.16116 handle_command_reply: tid=0 2024-03-19T01:50:44.466 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.460+0000 7fa4defbe740 1 client.16116 shutdown 2024-03-19T01:50:44.466 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.460+0000 7fa4defbe740 1 -- 172.21.15.99:0/77245332 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5649b9764e70 msgr2=0x5649b97708f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:50:44.467 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.460+0000 7fa4defbe740 1 --2- 172.21.15.99:0/77245332 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5649b9764e70 0x5649b97708f0 secure :-1 s=READY pgs=531 cs=0 l=1 rev1=1 crypto rx=0x7fa4c4036040 tx=0x7fa4c4044040 comp rx=0 tx=0).stop 2024-03-19T01:50:44.467 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.460+0000 7fa4defbe740 1 -- 172.21.15.99:0/77245332 shutdown_connections 2024-03-19T01:50:44.467 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.460+0000 7fa4defbe740 1 --2- 172.21.15.99:0/77245332 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5649b9775180 0x5649b97775d0 crc :-1 s=READY pgs=115 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:44.467 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.460+0000 7fa4defbe740 1 --2- 172.21.15.99:0/77245332 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5649b975fd00 0x5649b97698b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:44.467 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.460+0000 7fa4defbe740 1 --2- 172.21.15.99:0/77245332 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5649b9764e70 0x5649b97708f0 unknown :-1 s=CLOSED pgs=531 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:44.467 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.460+0000 7fa4defbe740 1 --2- 172.21.15.99:0/77245332 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5649b975f310 0x5649b9769370 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:44.467 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.460+0000 7fa4bdffb640 0 client.16116 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:50:44.467 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.460+0000 7fa4defbe740 1 -- 172.21.15.99:0/77245332 >> 172.21.15.99:0/77245332 conn(0x5649b9758670 msgr2=0x5649b97598c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:50:44.467 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.460+0000 7fa4defbe740 1 -- 172.21.15.99:0/77245332 shutdown_connections 2024-03-19T01:50:44.467 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.460+0000 7fa4defbe740 1 -- 172.21.15.99:0/77245332 wait complete. 2024-03-19T01:50:44.467 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.461+0000 7fa4defbe740 20 client.16116 trim_cache size 0 max 16384 2024-03-19T01:50:44.481 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.475+0000 7fa4defbe740 1 Processor -- start 2024-03-19T01:50:44.481 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.475+0000 7fa4defbe740 1 -- start start 2024-03-19T01:50:44.481 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.476+0000 7fa4defbe740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5649b975f310 0x5649b975bfe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:44.481 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.476+0000 7fa4defbe740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5649b975fd00 0x5649b97708b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:44.482 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.476+0000 7fa4defbe740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5649b9764e70 0x5649b977bf20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:44.482 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.476+0000 7fa4defbe740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fa4d41a9400 con 0x5649b975f310 2024-03-19T01:50:44.482 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.476+0000 7fa4defbe740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fa4d41a6cd0 con 0x5649b9764e70 2024-03-19T01:50:44.482 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.476+0000 7fa4defbe740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fa4d41a4b20 con 0x5649b975fd00 2024-03-19T01:50:44.482 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.476+0000 7fa4d996f640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5649b975f310 0x5649b975bfe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:44.482 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.476+0000 7fa4d996f640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5649b975f310 0x5649b975bfe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:36634/0 (socket says 172.21.15.99:36634) 2024-03-19T01:50:44.482 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.476+0000 7fa4d996f640 1 -- 172.21.15.99:0/2471867772 learned_addr learned my addr 172.21.15.99:0/2471867772 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:50:44.482 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.476+0000 7fa4da170640 1 --2- 172.21.15.99:0/2471867772 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5649b975fd00 0x5649b97708b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:44.482 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.476+0000 7fa4d996f640 1 --2- 172.21.15.99:0/2471867772 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5649b9764e70 0x5649b977bf20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:44.482 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.476+0000 7fa4d996f640 1 -- 172.21.15.99:0/2471867772 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5649b975fd00 msgr2=0x5649b97708b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:50:44.482 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.476+0000 7fa4d996f640 1 --2- 172.21.15.99:0/2471867772 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5649b975fd00 0x5649b97708b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:44.482 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.476+0000 7fa4d996f640 1 -- 172.21.15.99:0/2471867772 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5649b9764e70 msgr2=0x5649b977bf20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:50:44.482 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.476+0000 7fa4d996f640 1 --2- 172.21.15.99:0/2471867772 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5649b9764e70 0x5649b977bf20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:44.482 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.476+0000 7fa4d996f640 1 -- 172.21.15.99:0/2471867772 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5649b94a2620 con 0x5649b975f310 2024-03-19T01:50:44.483 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.476+0000 7fa4d996f640 1 --2- 172.21.15.99:0/2471867772 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5649b975f310 0x5649b975bfe0 secure :-1 s=READY pgs=532 cs=0 l=1 rev1=1 crypto rx=0x7fa4c4041490 tx=0x7fa4c4005fd0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=29ec1d97f1699b6 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:50:44.483 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.476+0000 7fa4bffff640 1 -- 172.21.15.99:0/2471867772 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa4c4009520 con 0x5649b975f310 2024-03-19T01:50:44.483 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.476+0000 7fa4bffff640 1 -- 172.21.15.99:0/2471867772 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa4c405c070 con 0x5649b975f310 2024-03-19T01:50:44.483 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.476+0000 7fa4bffff640 1 -- 172.21.15.99:0/2471867772 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa4c4034440 con 0x5649b975f310 2024-03-19T01:50:44.483 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.477+0000 7fa4defbe740 1 -- 172.21.15.99:0/2471867772 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5649b975f310 msgr2=0x5649b975bfe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:50:44.483 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.477+0000 7fa4defbe740 1 --2- 172.21.15.99:0/2471867772 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5649b975f310 0x5649b975bfe0 secure :-1 s=READY pgs=532 cs=0 l=1 rev1=1 crypto rx=0x7fa4c4041490 tx=0x7fa4c4005fd0 comp rx=0 tx=0).stop 2024-03-19T01:50:44.483 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.477+0000 7fa4defbe740 1 -- 172.21.15.99:0/2471867772 shutdown_connections 2024-03-19T01:50:44.483 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.477+0000 7fa4defbe740 1 --2- 172.21.15.99:0/2471867772 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5649b9764e70 0x5649b977bf20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:44.483 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.477+0000 7fa4defbe740 1 --2- 172.21.15.99:0/2471867772 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5649b975fd00 0x5649b97708b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:44.483 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.477+0000 7fa4defbe740 1 --2- 172.21.15.99:0/2471867772 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5649b975f310 0x5649b975bfe0 unknown :-1 s=CLOSED pgs=532 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:44.483 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.477+0000 7fa4defbe740 1 -- 172.21.15.99:0/2471867772 >> 172.21.15.99:0/2471867772 conn(0x5649b9758670 msgr2=0x5649b975b8d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:50:44.483 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.477+0000 7fa4defbe740 1 -- 172.21.15.99:0/2471867772 shutdown_connections 2024-03-19T01:50:44.483 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.477+0000 7fa4defbe740 1 -- 172.21.15.99:0/2471867772 wait complete. 2024-03-19T01:50:44.483 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.477+0000 7fa4defbe740 1 -- start start 2024-03-19T01:50:44.484 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.477+0000 7fa4defbe740 1 Processor -- start 2024-03-19T01:50:44.484 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.478+0000 7fa4defbe740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5649b975f310 0x5649b9770170 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:44.484 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.478+0000 7fa4defbe740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5649b975fd00 0x5649b97706b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:44.484 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.478+0000 7fa4defbe740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5649b9764e70 0x5649b976e7d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:44.484 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.478+0000 7fa4defbe740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fa4d41a5190 con 0x5649b9764e70 2024-03-19T01:50:44.484 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.478+0000 7fa4defbe740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fa4d41a6600 con 0x5649b975fd00 2024-03-19T01:50:44.484 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.478+0000 7fa4defbe740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fa4d41a5860 con 0x5649b975f310 2024-03-19T01:50:44.484 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.478+0000 7fa4defbe740 10 client.0 _ll_register_callbacks cb 0x7fa4c4004520 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:50:44.484 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.478+0000 7fa4d996f640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5649b9764e70 0x5649b976e7d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:44.484 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.478+0000 7fa4d996f640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5649b9764e70 0x5649b976e7d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:36638/0 (socket says 172.21.15.99:36638) 2024-03-19T01:50:44.484 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.478+0000 7fa4d996f640 1 -- 172.21.15.99:0/420678698 learned_addr learned my addr 172.21.15.99:0/420678698 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:50:44.484 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.478+0000 7fa4da170640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5649b975fd00 0x5649b97706b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:44.484 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.478+0000 7fa4d996f640 1 --2- 172.21.15.99:0/420678698 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5649b975f310 0x5649b9770170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:44.484 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.478+0000 7fa4d996f640 1 -- 172.21.15.99:0/420678698 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5649b975f310 msgr2=0x5649b9770170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:50:44.485 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.478+0000 7fa4d996f640 1 --2- 172.21.15.99:0/420678698 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5649b975f310 0x5649b9770170 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:44.485 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.478+0000 7fa4d996f640 1 -- 172.21.15.99:0/420678698 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5649b975fd00 msgr2=0x5649b97706b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:50:44.485 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.478+0000 7fa4d996f640 1 --2- 172.21.15.99:0/420678698 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5649b975fd00 0x5649b97706b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:44.485 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.478+0000 7fa4d996f640 1 -- 172.21.15.99:0/420678698 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5649b976ef10 con 0x5649b9764e70 2024-03-19T01:50:44.485 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.478+0000 7fa4da170640 1 --2- 172.21.15.99:0/420678698 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5649b975fd00 0x5649b97706b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:50:44.485 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.478+0000 7fa4d996f640 1 --2- 172.21.15.99:0/420678698 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5649b9764e70 0x5649b976e7d0 secure :-1 s=READY pgs=533 cs=0 l=1 rev1=1 crypto rx=0x7fa4c4041560 tx=0x7fa4c4043040 comp rx=0 tx=0).ready entity=mon.0 client_cookie=54d966a5465fb9f9 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:50:44.485 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.478+0000 7fa4bdffb640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:50:44.485 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.478+0000 7fa4bdffb640 1 -- 172.21.15.99:0/420678698 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa4c4064060 con 0x5649b9764e70 2024-03-19T01:50:44.485 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.478+0000 7fa4bdffb640 1 -- 172.21.15.99:0/420678698 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7fa4c4034440 con 0x5649b9764e70 2024-03-19T01:50:44.485 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.478+0000 7fa4defbe740 1 -- 172.21.15.99:0/420678698 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fa4c405c070 con 0x5649b9764e70 2024-03-19T01:50:44.485 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.478+0000 7fa4bdffb640 1 -- 172.21.15.99:0/420678698 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7fa4c4003e50 con 0x5649b9764e70 2024-03-19T01:50:44.485 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.478+0000 7fa4defbe740 10 client.16122 fetch_fsmap learned FSMap version 27 2024-03-19T01:50:44.485 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.478+0000 7fa4defbe740 1 -- 172.21.15.99:0/420678698 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x5649b97dbf90 con 0x5649b9764e70 2024-03-19T01:50:44.485 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.479+0000 7fa4bdffb640 1 -- 172.21.15.99:0/420678698 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa4c405c070 con 0x5649b9764e70 2024-03-19T01:50:44.485 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.479+0000 7fa4bdffb640 1 -- 172.21.15.99:0/420678698 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 27) ==== 8338+0+0 (secure 0 0 0) 0x7fa4c4057040 con 0x5649b9764e70 2024-03-19T01:50:44.486 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.479+0000 7fa4defbe740 10 client.16122 fetch_fsmap finished waiting for FSMap version 27 2024-03-19T01:50:44.486 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.479+0000 7fa4defbe740 10 client.16122 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:50:44.486 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.479+0000 7fa4defbe740 20 client.16122 populate_metadata read hostname 'smithi099' 2024-03-19T01:50:44.486 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.479+0000 7fa4defbe740 1 --2- 172.21.15.99:0/420678698 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5649b97dc8d0 0x5649b97decc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:44.486 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.479+0000 7fa4defbe740 4 client.16122 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:50:44.486 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.479+0000 7fa4defbe740 1 -- 172.21.15.99:0/420678698 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x5649b96c9470 con 0x5649b97dc8d0 2024-03-19T01:50:44.486 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.479+0000 7fa4da170640 1 --2- 172.21.15.99:0/420678698 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5649b97dc8d0 0x5649b97decc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:44.486 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.479+0000 7fa4da170640 1 --2- 172.21.15.99:0/420678698 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5649b97dc8d0 0x5649b97decc0 crc :-1 s=READY pgs=116 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=330a19ce6c20d8d8 server_cookie=cb1939e09ded458 in_seq=0 out_seq=0 2024-03-19T01:50:44.486 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.479+0000 7fa4bdffb640 10 client.16122 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:50:44.537 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.536+0000 7fa4bdffb640 1 -- 172.21.15.99:0/420678698 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x5649b96c9470 con 0x5649b97dc8d0 2024-03-19T01:50:44.537 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.536+0000 7fa4bdffb640 10 client.16122 handle_command_reply: tid=0 2024-03-19T01:50:44.537 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.536+0000 7fa4defbe740 1 client.16122 shutdown 2024-03-19T01:50:44.538 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.536+0000 7fa4defbe740 1 -- 172.21.15.99:0/420678698 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5649b9764e70 msgr2=0x5649b976e7d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:50:44.538 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.536+0000 7fa4defbe740 1 --2- 172.21.15.99:0/420678698 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5649b9764e70 0x5649b976e7d0 secure :-1 s=READY pgs=533 cs=0 l=1 rev1=1 crypto rx=0x7fa4c4041560 tx=0x7fa4c4043040 comp rx=0 tx=0).stop 2024-03-19T01:50:44.538 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.536+0000 7fa4defbe740 1 -- 172.21.15.99:0/420678698 shutdown_connections 2024-03-19T01:50:44.538 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.536+0000 7fa4defbe740 1 --2- 172.21.15.99:0/420678698 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5649b97dc8d0 0x5649b97decc0 crc :-1 s=READY pgs=116 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:44.538 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.536+0000 7fa4defbe740 1 --2- 172.21.15.99:0/420678698 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5649b975fd00 0x5649b97706b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:44.538 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.536+0000 7fa4defbe740 1 --2- 172.21.15.99:0/420678698 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5649b9764e70 0x5649b976e7d0 unknown :-1 s=CLOSED pgs=533 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:44.538 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.536+0000 7fa4bdffb640 0 client.16122 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:50:44.538 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.536+0000 7fa4defbe740 1 --2- 172.21.15.99:0/420678698 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5649b975f310 0x5649b9770170 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:44.538 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.536+0000 7fa4defbe740 1 -- 172.21.15.99:0/420678698 >> 172.21.15.99:0/420678698 conn(0x5649b9758670 msgr2=0x5649b975c160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:50:44.538 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.536+0000 7fa4defbe740 1 -- 172.21.15.99:0/420678698 shutdown_connections 2024-03-19T01:50:44.538 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.536+0000 7fa4defbe740 1 -- 172.21.15.99:0/420678698 wait complete. 2024-03-19T01:50:44.538 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.536+0000 7fa4defbe740 20 client.16122 trim_cache size 0 max 16384 2024-03-19T01:50:44.539 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:50:44.539 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:50:44.539 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:50:44.539 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:50:44.540 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.537+0000 7fa4dc3d4640 1 -- 172.21.15.99:0/1080122192 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa4c00a28a0 msgr2=0x7fa4c00a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:50:44.540 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.537+0000 7fa4dc3d4640 1 --2- 172.21.15.99:0/1080122192 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa4c00a28a0 0x7fa4c00a4d60 secure :-1 s=READY pgs=304 cs=0 l=1 rev1=1 crypto rx=0x7fa4cc0a3d50 tx=0x7fa4cc0964a0 comp rx=0 tx=0).stop 2024-03-19T01:50:44.540 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.537+0000 7fa4dc3d4640 1 -- 172.21.15.99:0/1080122192 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa4d4102300 msgr2=0x7fa4d419df10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:50:44.541 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.537+0000 7fa4dc3d4640 1 --2- 172.21.15.99:0/1080122192 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa4d4102300 0x7fa4d419df10 secure :-1 s=READY pgs=530 cs=0 l=1 rev1=1 crypto rx=0x7fa4cc00eb10 tx=0x7fa4cc00c2d0 comp rx=0 tx=0).stop 2024-03-19T01:50:44.541 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.537+0000 7fa4dc3d4640 1 -- 172.21.15.99:0/1080122192 shutdown_connections 2024-03-19T01:50:44.541 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.537+0000 7fa4dc3d4640 1 --2- 172.21.15.99:0/1080122192 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa4d4102300 0x7fa4d419df10 unknown :-1 s=CLOSED pgs=530 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:44.541 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.537+0000 7fa4dc3d4640 1 --2- 172.21.15.99:0/1080122192 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa4d4101940 0x7fa4d4196ea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:44.541 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.537+0000 7fa4dc3d4640 1 --2- 172.21.15.99:0/1080122192 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa4c00a28a0 0x7fa4c00a4d60 unknown :-1 s=CLOSED pgs=304 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:44.541 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.537+0000 7fa4dc3d4640 1 --2- 172.21.15.99:0/1080122192 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa4d4100750 0x7fa4d4196960 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:44.541 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.537+0000 7fa4dc3d4640 1 -- 172.21.15.99:0/1080122192 >> 172.21.15.99:0/1080122192 conn(0x7fa4d40fbcc0 msgr2=0x7fa4d4104b70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:50:44.541 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.537+0000 7fa4dc3d4640 1 -- 172.21.15.99:0/1080122192 shutdown_connections 2024-03-19T01:50:44.541 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:44.537+0000 7fa4dc3d4640 1 -- 172.21.15.99:0/1080122192 wait complete. 2024-03-19T01:50:44.697 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1173768 11.37 MB/sec execute 434 sec latency 39.249 ms 2024-03-19T01:50:45.697 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1176030 11.38 MB/sec execute 435 sec latency 27.715 ms 2024-03-19T01:50:46.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:50:45 smithi115 ceph-mon[67332]: pgmap v640: 97 pgs: 97 active+clean; 257 MiB data, 3.0 GiB used, 1.0 TiB / 1.0 TiB avail; 703 KiB/s rd, 2.5 MiB/s wr, 228 op/s 2024-03-19T01:50:46.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:50:45 smithi099 ceph-mon[62951]: pgmap v640: 97 pgs: 97 active+clean; 257 MiB data, 3.0 GiB used, 1.0 TiB / 1.0 TiB avail; 703 KiB/s rd, 2.5 MiB/s wr, 228 op/s 2024-03-19T01:50:46.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:50:45 smithi112 ceph-mon[66819]: pgmap v640: 97 pgs: 97 active+clean; 257 MiB data, 3.0 GiB used, 1.0 TiB / 1.0 TiB avail; 703 KiB/s rd, 2.5 MiB/s wr, 228 op/s 2024-03-19T01:50:46.697 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1178034 11.37 MB/sec execute 436 sec latency 28.662 ms 2024-03-19T01:50:47.697 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1179780 11.37 MB/sec execute 437 sec latency 9.469 ms 2024-03-19T01:50:48.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:50:47 smithi115 ceph-mon[67332]: pgmap v641: 97 pgs: 97 active+clean; 257 MiB data, 3.1 GiB used, 1.0 TiB / 1.0 TiB avail; 821 KiB/s rd, 2.9 MiB/s wr, 263 op/s 2024-03-19T01:50:48.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:50:47 smithi099 ceph-mon[62951]: pgmap v641: 97 pgs: 97 active+clean; 257 MiB data, 3.1 GiB used, 1.0 TiB / 1.0 TiB avail; 821 KiB/s rd, 2.9 MiB/s wr, 263 op/s 2024-03-19T01:50:48.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:50:47 smithi112 ceph-mon[66819]: pgmap v641: 97 pgs: 97 active+clean; 257 MiB data, 3.1 GiB used, 1.0 TiB / 1.0 TiB avail; 821 KiB/s rd, 2.9 MiB/s wr, 263 op/s 2024-03-19T01:50:48.697 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1181966 11.37 MB/sec execute 438 sec latency 62.403 ms 2024-03-19T01:50:49.699 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1183488 11.36 MB/sec execute 439 sec latency 49.771 ms 2024-03-19T01:50:50.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:50:49 smithi115 ceph-mon[67332]: pgmap v642: 97 pgs: 97 active+clean; 259 MiB data, 3.1 GiB used, 1.0 TiB / 1.0 TiB avail; 792 KiB/s rd, 2.7 MiB/s wr, 254 op/s 2024-03-19T01:50:50.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:50:49 smithi099 ceph-mon[62951]: pgmap v642: 97 pgs: 97 active+clean; 259 MiB data, 3.1 GiB used, 1.0 TiB / 1.0 TiB avail; 792 KiB/s rd, 2.7 MiB/s wr, 254 op/s 2024-03-19T01:50:50.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:50:49 smithi112 ceph-mon[66819]: pgmap v642: 97 pgs: 97 active+clean; 259 MiB data, 3.1 GiB used, 1.0 TiB / 1.0 TiB avail; 792 KiB/s rd, 2.7 MiB/s wr, 254 op/s 2024-03-19T01:50:50.697 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1185758 11.36 MB/sec execute 440 sec latency 16.742 ms 2024-03-19T01:50:51.697 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1187473 11.36 MB/sec execute 441 sec latency 37.381 ms 2024-03-19T01:50:52.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:50:51 smithi115 ceph-mon[67332]: pgmap v643: 97 pgs: 97 active+clean; 260 MiB data, 3.1 GiB used, 1.0 TiB / 1.0 TiB avail; 799 KiB/s rd, 2.8 MiB/s wr, 255 op/s 2024-03-19T01:50:52.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:50:51 smithi099 ceph-mon[62951]: pgmap v643: 97 pgs: 97 active+clean; 260 MiB data, 3.1 GiB used, 1.0 TiB / 1.0 TiB avail; 799 KiB/s rd, 2.8 MiB/s wr, 255 op/s 2024-03-19T01:50:52.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:50:51 smithi112 ceph-mon[66819]: pgmap v643: 97 pgs: 97 active+clean; 260 MiB data, 3.1 GiB used, 1.0 TiB / 1.0 TiB avail; 799 KiB/s rd, 2.8 MiB/s wr, 255 op/s 2024-03-19T01:50:52.697 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1189946 11.37 MB/sec execute 442 sec latency 20.248 ms 2024-03-19T01:50:53.697 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1191488 11.36 MB/sec execute 443 sec latency 28.888 ms 2024-03-19T01:50:54.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:50:53 smithi115 ceph-mon[67332]: pgmap v644: 97 pgs: 97 active+clean; 261 MiB data, 3.1 GiB used, 1.0 TiB / 1.0 TiB avail; 735 KiB/s rd, 3.0 MiB/s wr, 246 op/s 2024-03-19T01:50:54.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:50:53 smithi099 ceph-mon[62951]: pgmap v644: 97 pgs: 97 active+clean; 261 MiB data, 3.1 GiB used, 1.0 TiB / 1.0 TiB avail; 735 KiB/s rd, 3.0 MiB/s wr, 246 op/s 2024-03-19T01:50:54.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:50:53 smithi112 ceph-mon[66819]: pgmap v644: 97 pgs: 97 active+clean; 261 MiB data, 3.1 GiB used, 1.0 TiB / 1.0 TiB avail; 735 KiB/s rd, 3.0 MiB/s wr, 246 op/s 2024-03-19T01:50:54.697 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1193645 11.36 MB/sec execute 444 sec latency 90.507 ms 2024-03-19T01:50:55.697 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1195040 11.35 MB/sec execute 445 sec latency 66.633 ms 2024-03-19T01:50:56.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:50:55 smithi115 ceph-mon[67332]: pgmap v645: 97 pgs: 97 active+clean; 261 MiB data, 3.1 GiB used, 1.0 TiB / 1.0 TiB avail; 692 KiB/s rd, 2.8 MiB/s wr, 228 op/s 2024-03-19T01:50:56.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:50:55 smithi099 ceph-mon[62951]: pgmap v645: 97 pgs: 97 active+clean; 261 MiB data, 3.1 GiB used, 1.0 TiB / 1.0 TiB avail; 692 KiB/s rd, 2.8 MiB/s wr, 228 op/s 2024-03-19T01:50:56.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:50:55 smithi112 ceph-mon[66819]: pgmap v645: 97 pgs: 97 active+clean; 261 MiB data, 3.1 GiB used, 1.0 TiB / 1.0 TiB avail; 692 KiB/s rd, 2.8 MiB/s wr, 228 op/s 2024-03-19T01:50:56.697 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1197349 11.36 MB/sec execute 446 sec latency 23.786 ms 2024-03-19T01:50:57.628 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.625+0000 7f84fb0ff640 1 -- 172.21.15.99:0/800560626 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f84f4102300 msgr2=0x7f84f41046f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:50:57.628 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.625+0000 7f84fb0ff640 1 --2- 172.21.15.99:0/800560626 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f84f4102300 0x7f84f41046f0 secure :-1 s=READY pgs=143 cs=0 l=1 rev1=1 crypto rx=0x7f84e8009900 tx=0x7f84e802bc30 comp rx=0 tx=0).stop 2024-03-19T01:50:57.628 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.625+0000 7f84fb0ff640 1 -- 172.21.15.99:0/800560626 shutdown_connections 2024-03-19T01:50:57.628 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.626+0000 7f84fb0ff640 1 --2- 172.21.15.99:0/800560626 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f84f4107560 0x7f84f4109970 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:57.628 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.626+0000 7f84fb0ff640 1 --2- 172.21.15.99:0/800560626 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f84f4104c30 0x7f84f4107020 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:57.628 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.626+0000 7f84fb0ff640 1 --2- 172.21.15.99:0/800560626 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f84f4102300 0x7f84f41046f0 unknown :-1 s=CLOSED pgs=143 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:57.628 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.626+0000 7f84fb0ff640 1 -- 172.21.15.99:0/800560626 >> 172.21.15.99:0/800560626 conn(0x7f84f40fbcb0 msgr2=0x7f84f40fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:50:57.628 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.626+0000 7f84fb0ff640 1 -- 172.21.15.99:0/800560626 shutdown_connections 2024-03-19T01:50:57.628 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.626+0000 7f84fb0ff640 1 -- 172.21.15.99:0/800560626 wait complete. 2024-03-19T01:50:57.628 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.626+0000 7f84fb0ff640 1 Processor -- start 2024-03-19T01:50:57.629 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.626+0000 7f84fb0ff640 1 -- start start 2024-03-19T01:50:57.629 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.626+0000 7f84fb0ff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f84f4102300 0x7f84f4197b00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:57.629 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.626+0000 7f84fb0ff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f84f4104c30 0x7f84f4198040 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:57.629 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.626+0000 7f84fb0ff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f84f4107560 0x7f84f419f0b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:57.629 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.626+0000 7f84fb0ff640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f84f410d4e0 con 0x7f84f4107560 2024-03-19T01:50:57.629 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.626+0000 7f84fb0ff640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f84f410d360 con 0x7f84f4102300 2024-03-19T01:50:57.629 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.626+0000 7f84fb0ff640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f84f410d660 con 0x7f84f4104c30 2024-03-19T01:50:57.629 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.626+0000 7f84f969c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f84f4107560 0x7f84f419f0b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:57.629 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.626+0000 7f84f3fff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f84f4104c30 0x7f84f4198040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:57.629 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.626+0000 7f84f8e9b640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f84f4102300 0x7f84f4197b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:57.629 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.626+0000 7f84f969c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f84f4107560 0x7f84f419f0b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:37078/0 (socket says 172.21.15.99:37078) 2024-03-19T01:50:57.629 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.626+0000 7f84f3fff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f84f4104c30 0x7f84f4198040 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:53110/0 (socket says 172.21.15.99:53110) 2024-03-19T01:50:57.629 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.626+0000 7f84f969c640 1 -- 172.21.15.99:0/3795257389 learned_addr learned my addr 172.21.15.99:0/3795257389 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:50:57.629 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.627+0000 7f84f8e9b640 1 -- 172.21.15.99:0/3795257389 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f84f4104c30 msgr2=0x7f84f4198040 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:50:57.630 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.627+0000 7f84f8e9b640 1 --2- 172.21.15.99:0/3795257389 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f84f4104c30 0x7f84f4198040 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:57.630 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.627+0000 7f84f8e9b640 1 -- 172.21.15.99:0/3795257389 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f84f4107560 msgr2=0x7f84f419f0b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:50:57.630 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.627+0000 7f84f8e9b640 1 --2- 172.21.15.99:0/3795257389 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f84f4107560 0x7f84f419f0b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:57.630 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.627+0000 7f84f8e9b640 1 -- 172.21.15.99:0/3795257389 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f84f419f720 con 0x7f84f4102300 2024-03-19T01:50:57.630 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.627+0000 7f84f969c640 1 --2- 172.21.15.99:0/3795257389 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f84f4107560 0x7f84f419f0b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:50:57.630 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.627+0000 7f84f8e9b640 1 --2- 172.21.15.99:0/3795257389 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f84f4102300 0x7f84f4197b00 secure :-1 s=READY pgs=144 cs=0 l=1 rev1=1 crypto rx=0x7f84e8009230 tx=0x7f84e802c650 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:50:57.630 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.627+0000 7f84f1ffb640 1 -- 172.21.15.99:0/3795257389 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f84e8041070 con 0x7f84f4102300 2024-03-19T01:50:57.630 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.627+0000 7f84fb0ff640 1 -- 172.21.15.99:0/3795257389 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f84f419f9b0 con 0x7f84f4102300 2024-03-19T01:50:57.630 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.627+0000 7f84f1ffb640 1 -- 172.21.15.99:0/3795257389 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f84e8004450 con 0x7f84f4102300 2024-03-19T01:50:57.630 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.627+0000 7f84f1ffb640 1 -- 172.21.15.99:0/3795257389 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f84e8003df0 con 0x7f84f4102300 2024-03-19T01:50:57.630 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.627+0000 7f84fb0ff640 1 -- 172.21.15.99:0/3795257389 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f84f419fea0 con 0x7f84f4102300 2024-03-19T01:50:57.636 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.630+0000 7f84fddc9740 1 Processor -- start 2024-03-19T01:50:57.636 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.630+0000 7f84f1ffb640 1 -- 172.21.15.99:0/3795257389 <== mon.2 v2:172.21.15.112:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f84e8044070 con 0x7f84f4102300 2024-03-19T01:50:57.636 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.630+0000 7f84fddc9740 1 -- start start 2024-03-19T01:50:57.636 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.630+0000 7f84fddc9740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5598ab6c3100 0x5598ab6c3520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:57.636 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.630+0000 7f84fddc9740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5598ab6c3af0 0x5598ab6c85f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:57.636 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.630+0000 7f84fddc9740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5598ab6c8c60 0x5598ab6cb0f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:57.636 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.630+0000 7f84fddc9740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x5598ab627be0 con 0x5598ab6c8c60 2024-03-19T01:50:57.636 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.630+0000 7f84fddc9740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x5598ab5d0960 con 0x5598ab6c3af0 2024-03-19T01:50:57.637 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.630+0000 7f84fddc9740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x5598ab623c80 con 0x5598ab6c3100 2024-03-19T01:50:57.637 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.630+0000 7f84f8e9b640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5598ab6c3af0 0x5598ab6c85f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:57.637 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.630+0000 7f84f3fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5598ab6c8c60 0x5598ab6cb0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:57.637 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.630+0000 7f84f969c640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5598ab6c3100 0x5598ab6c3520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:57.637 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.630+0000 7f84f8e9b640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5598ab6c3af0 0x5598ab6c85f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:42274/0 (socket says 172.21.15.99:42274) 2024-03-19T01:50:57.637 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.630+0000 7f84f3fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5598ab6c8c60 0x5598ab6cb0f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:37082/0 (socket says 172.21.15.99:37082) 2024-03-19T01:50:57.637 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.630+0000 7f84f8e9b640 1 -- 172.21.15.99:0/1370102195 learned_addr learned my addr 172.21.15.99:0/1370102195 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:50:57.637 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.630+0000 7f84f8e9b640 1 -- 172.21.15.99:0/1370102195 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5598ab6c3100 msgr2=0x5598ab6c3520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:50:57.637 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.630+0000 7f84f1ffb640 1 --2- 172.21.15.99:0/3795257389 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f84e40a28a0 0x7f84e40a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:57.637 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.630+0000 7f84f8e9b640 1 --2- 172.21.15.99:0/1370102195 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5598ab6c3100 0x5598ab6c3520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:57.637 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.630+0000 7f84f8e9b640 1 -- 172.21.15.99:0/1370102195 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5598ab6c8c60 msgr2=0x5598ab6cb0f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:50:57.637 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.630+0000 7f84f8e9b640 1 --2- 172.21.15.99:0/1370102195 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5598ab6c8c60 0x5598ab6cb0f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:57.637 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.630+0000 7f84f8e9b640 1 -- 172.21.15.99:0/1370102195 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5598ab5c7d20 con 0x5598ab6c3af0 2024-03-19T01:50:57.638 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.630+0000 7f84f969c640 1 --2- 172.21.15.99:0/3795257389 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f84e40a28a0 0x7f84e40a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:57.638 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.630+0000 7f84f1ffb640 1 -- 172.21.15.99:0/3795257389 <== mon.2 v2:172.21.15.112:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f84e80f04f0 con 0x7f84f4102300 2024-03-19T01:50:57.638 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.631+0000 7f84f8e9b640 1 --2- 172.21.15.99:0/1370102195 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5598ab6c3af0 0x5598ab6c85f0 secure :-1 s=READY pgs=145 cs=0 l=1 rev1=1 crypto rx=0x7f84e80edef0 tx=0x7f84e80b9220 comp rx=0 tx=0).ready entity=mon.2 client_cookie=3f33767d1a833ce8 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:50:57.638 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.631+0000 7f84deffd640 1 -- 172.21.15.99:0/1370102195 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f84e804b070 con 0x5598ab6c3af0 2024-03-19T01:50:57.638 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.631+0000 7f84deffd640 1 -- 172.21.15.99:0/1370102195 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f84e80bf070 con 0x5598ab6c3af0 2024-03-19T01:50:57.638 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.631+0000 7f84deffd640 1 -- 172.21.15.99:0/1370102195 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f84e80bccd0 con 0x5598ab6c3af0 2024-03-19T01:50:57.638 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.631+0000 7f84fddc9740 1 -- 172.21.15.99:0/1370102195 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5598ab6c3af0 msgr2=0x5598ab6c85f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:50:57.638 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.631+0000 7f84f969c640 1 --2- 172.21.15.99:0/3795257389 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f84e40a28a0 0x7f84e40a4d60 secure :-1 s=READY pgs=305 cs=0 l=1 rev1=1 crypto rx=0x5598ab6c4350 tx=0x7f84ec004a40 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:50:57.638 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.631+0000 7f84fddc9740 1 --2- 172.21.15.99:0/1370102195 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5598ab6c3af0 0x5598ab6c85f0 secure :-1 s=READY pgs=145 cs=0 l=1 rev1=1 crypto rx=0x7f84e80edef0 tx=0x7f84e80b9220 comp rx=0 tx=0).stop 2024-03-19T01:50:57.638 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.631+0000 7f84fddc9740 1 -- 172.21.15.99:0/1370102195 shutdown_connections 2024-03-19T01:50:57.638 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.631+0000 7f84fddc9740 1 --2- 172.21.15.99:0/1370102195 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5598ab6c3af0 0x5598ab6c85f0 unknown :-1 s=CLOSED pgs=145 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:57.638 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.631+0000 7f84fddc9740 1 --2- 172.21.15.99:0/1370102195 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5598ab6c8c60 0x5598ab6cb0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:57.638 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.631+0000 7f84fddc9740 1 --2- 172.21.15.99:0/1370102195 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5598ab6c3100 0x5598ab6c3520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:57.638 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.631+0000 7f84fddc9740 1 -- 172.21.15.99:0/1370102195 >> 172.21.15.99:0/1370102195 conn(0x5598ab6be670 msgr2=0x5598ab6bea70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:50:57.639 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.631+0000 7f84fddc9740 1 -- 172.21.15.99:0/1370102195 shutdown_connections 2024-03-19T01:50:57.639 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.631+0000 7f84fddc9740 1 -- 172.21.15.99:0/1370102195 wait complete. 2024-03-19T01:50:57.639 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.632+0000 7f84fddc9740 1 -- start start 2024-03-19T01:50:57.639 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.632+0000 7f84fddc9740 1 Processor -- start 2024-03-19T01:50:57.639 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.632+0000 7f84fddc9740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5598ab6c3100 0x5598ab6cce80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:57.639 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.632+0000 7f84fddc9740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5598ab6c3af0 0x5598ab6cd3c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:57.639 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.632+0000 7f84fddc9740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5598ab6c8c60 0x5598ab6d4390 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:57.639 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.632+0000 7f84fddc9740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x5598ab6cb9d0 con 0x5598ab6c3af0 2024-03-19T01:50:57.639 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.632+0000 7f84fddc9740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x5598ab5e3db0 con 0x5598ab6c8c60 2024-03-19T01:50:57.639 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.632+0000 7f84fddc9740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x5598ab6cbb50 con 0x5598ab6c3100 2024-03-19T01:50:57.639 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.632+0000 7f84fddc9740 10 client.0 _ll_register_callbacks cb 0x5598ab5e8e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:50:57.639 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.632+0000 7f84f3fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5598ab6c3af0 0x5598ab6cd3c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:57.639 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.632+0000 7f84f3fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5598ab6c3af0 0x5598ab6cd3c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:37092/0 (socket says 172.21.15.99:37092) 2024-03-19T01:50:57.639 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.632+0000 7f84f969c640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5598ab6c8c60 0x5598ab6d4390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:57.639 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.632+0000 7f84f3fff640 1 -- 172.21.15.99:0/1164064042 learned_addr learned my addr 172.21.15.99:0/1164064042 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:50:57.640 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.632+0000 7f84f8e9b640 1 --2- 172.21.15.99:0/1164064042 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5598ab6c3100 0x5598ab6cce80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:57.640 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.633+0000 7f84f3fff640 1 -- 172.21.15.99:0/1164064042 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5598ab6c3100 msgr2=0x5598ab6cce80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:50:57.640 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.633+0000 7f84f3fff640 1 --2- 172.21.15.99:0/1164064042 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5598ab6c3100 0x5598ab6cce80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:57.640 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.633+0000 7f84f3fff640 1 -- 172.21.15.99:0/1164064042 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5598ab6c8c60 msgr2=0x5598ab6d4390 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:50:57.640 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.633+0000 7f84f3fff640 1 --2- 172.21.15.99:0/1164064042 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5598ab6c8c60 0x5598ab6d4390 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:57.640 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.633+0000 7f84f3fff640 1 -- 172.21.15.99:0/1164064042 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5598ab5e2c60 con 0x5598ab6c3af0 2024-03-19T01:50:57.640 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.633+0000 7f84f969c640 1 --2- 172.21.15.99:0/1164064042 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5598ab6c8c60 0x5598ab6d4390 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:50:57.640 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.633+0000 7f84f3fff640 1 --2- 172.21.15.99:0/1164064042 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5598ab6c3af0 0x5598ab6cd3c0 secure :-1 s=READY pgs=534 cs=0 l=1 rev1=1 crypto rx=0x7f84e0012700 tx=0x7f84e0012bc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9ff230e12b07f54b server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:50:57.640 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.633+0000 7f84d3fff640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:50:57.640 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.633+0000 7f84d3fff640 1 -- 172.21.15.99:0/1164064042 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f84e0004020 con 0x5598ab6c3af0 2024-03-19T01:50:57.640 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.633+0000 7f84d3fff640 1 -- 172.21.15.99:0/1164064042 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f84e00041c0 con 0x5598ab6c3af0 2024-03-19T01:50:57.640 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.633+0000 7f84fddc9740 1 -- 172.21.15.99:0/1164064042 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f84e80bf070 con 0x5598ab6c3af0 2024-03-19T01:50:57.640 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.633+0000 7f84d3fff640 1 -- 172.21.15.99:0/1164064042 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7f84e00043c0 con 0x5598ab6c3af0 2024-03-19T01:50:57.640 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.633+0000 7f84fddc9740 10 client.16134 fetch_fsmap learned FSMap version 27 2024-03-19T01:50:57.641 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.633+0000 7f84fddc9740 1 -- 172.21.15.99:0/1164064042 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x5598ab33c210 con 0x5598ab6c3af0 2024-03-19T01:50:57.641 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.633+0000 7f84d3fff640 1 -- 172.21.15.99:0/1164064042 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f84e80bf070 con 0x5598ab6c3af0 2024-03-19T01:50:57.641 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.633+0000 7f84d3fff640 1 -- 172.21.15.99:0/1164064042 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 27) ==== 8338+0+0 (secure 0 0 0) 0x7f84e0003a40 con 0x5598ab6c3af0 2024-03-19T01:50:57.641 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.633+0000 7f84fddc9740 10 client.16134 fetch_fsmap finished waiting for FSMap version 27 2024-03-19T01:50:57.641 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.633+0000 7f84fddc9740 10 client.16134 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:50:57.641 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.633+0000 7f84fddc9740 20 client.16134 populate_metadata read hostname 'smithi099' 2024-03-19T01:50:57.641 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.633+0000 7f84fddc9740 1 --2- 172.21.15.99:0/1164064042 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5598ab6d8d40 0x5598ab6db1c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:57.641 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.633+0000 7f84fddc9740 4 client.16134 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:50:57.641 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.633+0000 7f84fddc9740 1 -- 172.21.15.99:0/1164064042 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5598ab635120 con 0x5598ab6d8d40 2024-03-19T01:50:57.641 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.634+0000 7f84f8e9b640 1 --2- 172.21.15.99:0/1164064042 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5598ab6d8d40 0x5598ab6db1c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:57.641 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.634+0000 7f84f8e9b640 1 --2- 172.21.15.99:0/1164064042 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5598ab6d8d40 0x5598ab6db1c0 crc :-1 s=READY pgs=117 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=1de9f02edb87daec server_cookie=cf67d8bb17efbc4a in_seq=0 out_seq=0 2024-03-19T01:50:57.641 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.634+0000 7f84d3fff640 10 client.16134 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:50:57.641 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.635+0000 7f84d3fff640 1 -- 172.21.15.99:0/1164064042 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x5598ab635120 con 0x5598ab6d8d40 2024-03-19T01:50:57.641 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.635+0000 7f84d3fff640 10 client.16134 handle_command_reply: tid=0 2024-03-19T01:50:57.641 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.635+0000 7f84fddc9740 1 client.16134 shutdown 2024-03-19T01:50:57.642 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.635+0000 7f84fddc9740 1 -- 172.21.15.99:0/1164064042 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5598ab6c3af0 msgr2=0x5598ab6cd3c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:50:57.642 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.635+0000 7f84fddc9740 1 --2- 172.21.15.99:0/1164064042 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5598ab6c3af0 0x5598ab6cd3c0 secure :-1 s=READY pgs=534 cs=0 l=1 rev1=1 crypto rx=0x7f84e0012700 tx=0x7f84e0012bc0 comp rx=0 tx=0).stop 2024-03-19T01:50:57.642 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.635+0000 7f84fddc9740 1 -- 172.21.15.99:0/1164064042 shutdown_connections 2024-03-19T01:50:57.642 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.635+0000 7f84fddc9740 1 --2- 172.21.15.99:0/1164064042 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5598ab6d8d40 0x5598ab6db1c0 crc :-1 s=READY pgs=117 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:57.642 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.635+0000 7f84fddc9740 1 --2- 172.21.15.99:0/1164064042 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5598ab6c8c60 0x5598ab6d4390 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:57.642 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.635+0000 7f84d3fff640 0 client.16134 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:50:57.642 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.635+0000 7f84fddc9740 1 --2- 172.21.15.99:0/1164064042 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5598ab6c3af0 0x5598ab6cd3c0 unknown :-1 s=CLOSED pgs=534 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:57.642 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.635+0000 7f84fddc9740 1 --2- 172.21.15.99:0/1164064042 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5598ab6c3100 0x5598ab6cce80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:57.642 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.635+0000 7f84fddc9740 1 -- 172.21.15.99:0/1164064042 >> 172.21.15.99:0/1164064042 conn(0x5598ab6be670 msgr2=0x5598ab6bf8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:50:57.642 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.635+0000 7f84fddc9740 1 -- 172.21.15.99:0/1164064042 shutdown_connections 2024-03-19T01:50:57.642 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.635+0000 7f84fddc9740 1 -- 172.21.15.99:0/1164064042 wait complete. 2024-03-19T01:50:57.642 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.635+0000 7f84fddc9740 20 client.16134 trim_cache size 0 max 16384 2024-03-19T01:50:57.655 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.650+0000 7f84fddc9740 1 Processor -- start 2024-03-19T01:50:57.655 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.650+0000 7f84fddc9740 1 -- start start 2024-03-19T01:50:57.655 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.651+0000 7f84fddc9740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5598ab6c3100 0x5598ab6c4380 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:57.655 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.651+0000 7f84fddc9740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5598ab6c3af0 0x5598ab6d4350 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:57.655 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.651+0000 7f84fddc9740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5598ab6c8c60 0x5598ab6c48c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:57.655 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.651+0000 7f84fddc9740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f84f41ab6c0 con 0x5598ab6c3af0 2024-03-19T01:50:57.655 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.651+0000 7f84fddc9740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f84f41a8f90 con 0x5598ab6c3100 2024-03-19T01:50:57.655 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.651+0000 7f84fddc9740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f84f41a6de0 con 0x5598ab6c8c60 2024-03-19T01:50:57.655 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.651+0000 7f84f3fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5598ab6c3af0 0x5598ab6d4350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:57.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.651+0000 7f84f3fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5598ab6c3af0 0x5598ab6d4350 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:37096/0 (socket says 172.21.15.99:37096) 2024-03-19T01:50:57.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.651+0000 7f84f8e9b640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5598ab6c3100 0x5598ab6c4380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:57.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.651+0000 7f84f3fff640 1 -- 172.21.15.99:0/476366502 learned_addr learned my addr 172.21.15.99:0/476366502 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:50:57.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.651+0000 7f84f969c640 1 --2- 172.21.15.99:0/476366502 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5598ab6c8c60 0x5598ab6c48c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:57.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.651+0000 7f84f3fff640 1 -- 172.21.15.99:0/476366502 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5598ab6c8c60 msgr2=0x5598ab6c48c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:50:57.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.651+0000 7f84f3fff640 1 --2- 172.21.15.99:0/476366502 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5598ab6c8c60 0x5598ab6c48c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:57.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.651+0000 7f84f3fff640 1 -- 172.21.15.99:0/476366502 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5598ab6c3100 msgr2=0x5598ab6c4380 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:50:57.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.651+0000 7f84f3fff640 1 --2- 172.21.15.99:0/476366502 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5598ab6c3100 0x5598ab6c4380 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:57.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.651+0000 7f84f3fff640 1 -- 172.21.15.99:0/476366502 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5598ab408620 con 0x5598ab6c3af0 2024-03-19T01:50:57.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.651+0000 7f84f8e9b640 1 --2- 172.21.15.99:0/476366502 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5598ab6c3100 0x5598ab6c4380 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:50:57.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.651+0000 7f84f3fff640 1 --2- 172.21.15.99:0/476366502 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5598ab6c3af0 0x5598ab6d4350 secure :-1 s=READY pgs=535 cs=0 l=1 rev1=1 crypto rx=0x7f84e00059f0 tx=0x7f84e0009620 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8d727cd3f8faedea server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:50:57.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.651+0000 7f84de7fc640 1 -- 172.21.15.99:0/476366502 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f84e0009e60 con 0x5598ab6c3af0 2024-03-19T01:50:57.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.651+0000 7f84de7fc640 1 -- 172.21.15.99:0/476366502 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f84e001ce40 con 0x5598ab6c3af0 2024-03-19T01:50:57.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.651+0000 7f84de7fc640 1 -- 172.21.15.99:0/476366502 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f84e0003a80 con 0x5598ab6c3af0 2024-03-19T01:50:57.657 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.652+0000 7f84fddc9740 1 -- 172.21.15.99:0/476366502 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5598ab6c3af0 msgr2=0x5598ab6d4350 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:50:57.657 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.652+0000 7f84fddc9740 1 --2- 172.21.15.99:0/476366502 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5598ab6c3af0 0x5598ab6d4350 secure :-1 s=READY pgs=535 cs=0 l=1 rev1=1 crypto rx=0x7f84e00059f0 tx=0x7f84e0009620 comp rx=0 tx=0).stop 2024-03-19T01:50:57.657 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.652+0000 7f84fddc9740 1 -- 172.21.15.99:0/476366502 shutdown_connections 2024-03-19T01:50:57.657 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.652+0000 7f84fddc9740 1 --2- 172.21.15.99:0/476366502 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5598ab6c8c60 0x5598ab6c48c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:57.657 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.652+0000 7f84fddc9740 1 --2- 172.21.15.99:0/476366502 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5598ab6c3af0 0x5598ab6d4350 unknown :-1 s=CLOSED pgs=535 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:57.657 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.652+0000 7f84fddc9740 1 --2- 172.21.15.99:0/476366502 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5598ab6c3100 0x5598ab6c4380 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:57.657 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.652+0000 7f84fddc9740 1 -- 172.21.15.99:0/476366502 >> 172.21.15.99:0/476366502 conn(0x5598ab6be670 msgr2=0x5598ab6bd220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:50:57.657 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.652+0000 7f84fddc9740 1 -- 172.21.15.99:0/476366502 shutdown_connections 2024-03-19T01:50:57.657 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.652+0000 7f84fddc9740 1 -- 172.21.15.99:0/476366502 wait complete. 2024-03-19T01:50:57.657 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.652+0000 7f84fddc9740 1 -- start start 2024-03-19T01:50:57.657 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.652+0000 7f84fddc9740 1 Processor -- start 2024-03-19T01:50:57.657 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.653+0000 7f84fddc9740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5598ab6c3100 0x5598ab6d1d60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:57.657 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.653+0000 7f84fddc9740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5598ab6c3af0 0x5598ab6d22a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:57.657 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.653+0000 7f84fddc9740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5598ab6c8c60 0x5598ab6d32b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:57.658 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.653+0000 7f84fddc9740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f84f41a7450 con 0x5598ab6c8c60 2024-03-19T01:50:57.658 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.653+0000 7f84fddc9740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f84f41a88c0 con 0x5598ab6c3af0 2024-03-19T01:50:57.658 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.653+0000 7f84fddc9740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f84f41a7b20 con 0x5598ab6c3100 2024-03-19T01:50:57.658 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.653+0000 7f84fddc9740 10 client.0 _ll_register_callbacks cb 0x7f84e0009910 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:50:57.658 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.653+0000 7f84f969c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5598ab6c8c60 0x5598ab6d32b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:57.658 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.653+0000 7f84f969c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5598ab6c8c60 0x5598ab6d32b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:37106/0 (socket says 172.21.15.99:37106) 2024-03-19T01:50:57.658 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.653+0000 7f84f969c640 1 -- 172.21.15.99:0/4160521268 learned_addr learned my addr 172.21.15.99:0/4160521268 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:50:57.658 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.653+0000 7f84f8e9b640 1 --2- 172.21.15.99:0/4160521268 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5598ab6c3100 0x5598ab6d1d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:57.658 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.653+0000 7f84f3fff640 1 --2- 172.21.15.99:0/4160521268 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5598ab6c3af0 0x5598ab6d22a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:57.658 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.653+0000 7f84f969c640 1 -- 172.21.15.99:0/4160521268 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5598ab6c3100 msgr2=0x5598ab6d1d60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:50:57.658 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.653+0000 7f84f969c640 1 --2- 172.21.15.99:0/4160521268 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5598ab6c3100 0x5598ab6d1d60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:57.658 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.653+0000 7f84f969c640 1 -- 172.21.15.99:0/4160521268 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5598ab6c3af0 msgr2=0x5598ab6d22a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:50:57.658 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.653+0000 7f84f969c640 1 --2- 172.21.15.99:0/4160521268 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5598ab6c3af0 0x5598ab6d22a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:57.659 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.653+0000 7f84f969c640 1 -- 172.21.15.99:0/4160521268 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5598ab6d38a0 con 0x5598ab6c8c60 2024-03-19T01:50:57.659 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.653+0000 7f84f3fff640 1 --2- 172.21.15.99:0/4160521268 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5598ab6c3af0 0x5598ab6d22a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:50:57.659 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.653+0000 7f84f8e9b640 1 --2- 172.21.15.99:0/4160521268 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5598ab6c3100 0x5598ab6d1d60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:50:57.659 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.653+0000 7f84f969c640 1 --2- 172.21.15.99:0/4160521268 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5598ab6c8c60 0x5598ab6d32b0 secure :-1 s=READY pgs=536 cs=0 l=1 rev1=1 crypto rx=0x7f84ec01cc40 tx=0x7f84ec004ab0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1303a1a4dac47dda server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:50:57.659 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.653+0000 7f84d3fff640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:50:57.659 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.653+0000 7f84d3fff640 1 -- 172.21.15.99:0/4160521268 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f84ec01c050 con 0x5598ab6c8c60 2024-03-19T01:50:57.659 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.653+0000 7f84d3fff640 1 -- 172.21.15.99:0/4160521268 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f84ec01c1f0 con 0x5598ab6c8c60 2024-03-19T01:50:57.659 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.653+0000 7f84fddc9740 1 -- 172.21.15.99:0/4160521268 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x5598ab62f470 con 0x5598ab6c8c60 2024-03-19T01:50:57.659 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.653+0000 7f84d3fff640 1 -- 172.21.15.99:0/4160521268 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7f84ec02a440 con 0x5598ab6c8c60 2024-03-19T01:50:57.659 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.653+0000 7f84fddc9740 10 client.16140 fetch_fsmap learned FSMap version 27 2024-03-19T01:50:57.659 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.653+0000 7f84fddc9740 1 -- 172.21.15.99:0/4160521268 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x5598ab73f490 con 0x5598ab6c8c60 2024-03-19T01:50:57.659 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.654+0000 7f84d3fff640 1 -- 172.21.15.99:0/4160521268 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x5598ab62f470 con 0x5598ab6c8c60 2024-03-19T01:50:57.659 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.654+0000 7f84d3fff640 1 -- 172.21.15.99:0/4160521268 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 27) ==== 8338+0+0 (secure 0 0 0) 0x7f84ec01b550 con 0x5598ab6c8c60 2024-03-19T01:50:57.660 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.654+0000 7f84fddc9740 10 client.16140 fetch_fsmap finished waiting for FSMap version 27 2024-03-19T01:50:57.660 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.654+0000 7f84fddc9740 10 client.16140 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:50:57.660 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.654+0000 7f84fddc9740 20 client.16140 populate_metadata read hostname 'smithi099' 2024-03-19T01:50:57.660 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.654+0000 7f84fddc9740 1 --2- 172.21.15.99:0/4160521268 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5598ab6d8d40 0x5598ab741f50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:50:57.660 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.654+0000 7f84fddc9740 4 client.16140 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:50:57.660 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.654+0000 7f84fddc9740 1 -- 172.21.15.99:0/4160521268 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x5598ab5ad770 con 0x5598ab6d8d40 2024-03-19T01:50:57.660 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.654+0000 7f84f8e9b640 1 --2- 172.21.15.99:0/4160521268 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5598ab6d8d40 0x5598ab741f50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:50:57.660 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.654+0000 7f84d3fff640 10 client.16140 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:50:57.660 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.654+0000 7f84f8e9b640 1 --2- 172.21.15.99:0/4160521268 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5598ab6d8d40 0x5598ab741f50 crc :-1 s=READY pgs=118 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=dbb1777b5173ec7 server_cookie=629f3bd8fda90522 in_seq=0 out_seq=0 2024-03-19T01:50:57.698 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1199385 11.35 MB/sec execute 447 sec latency 31.751 ms 2024-03-19T01:50:57.711 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.710+0000 7f84d3fff640 1 -- 172.21.15.99:0/4160521268 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x5598ab5ad770 con 0x5598ab6d8d40 2024-03-19T01:50:57.711 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.710+0000 7f84d3fff640 10 client.16140 handle_command_reply: tid=0 2024-03-19T01:50:57.711 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.710+0000 7f84fddc9740 1 client.16140 shutdown 2024-03-19T01:50:57.711 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.711+0000 7f84fddc9740 1 -- 172.21.15.99:0/4160521268 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5598ab6c8c60 msgr2=0x5598ab6d32b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:50:57.711 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.711+0000 7f84fddc9740 1 --2- 172.21.15.99:0/4160521268 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5598ab6c8c60 0x5598ab6d32b0 secure :-1 s=READY pgs=536 cs=0 l=1 rev1=1 crypto rx=0x7f84ec01cc40 tx=0x7f84ec004ab0 comp rx=0 tx=0).stop 2024-03-19T01:50:57.711 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.711+0000 7f84fddc9740 1 -- 172.21.15.99:0/4160521268 shutdown_connections 2024-03-19T01:50:57.711 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.711+0000 7f84fddc9740 1 --2- 172.21.15.99:0/4160521268 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5598ab6d8d40 0x5598ab741f50 crc :-1 s=READY pgs=118 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:57.711 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.711+0000 7f84fddc9740 1 --2- 172.21.15.99:0/4160521268 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5598ab6c3af0 0x5598ab6d22a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:57.711 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.711+0000 7f84d3fff640 0 client.16140 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:50:57.711 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.711+0000 7f84fddc9740 1 --2- 172.21.15.99:0/4160521268 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5598ab6c8c60 0x5598ab6d32b0 unknown :-1 s=CLOSED pgs=536 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:57.711 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.711+0000 7f84fddc9740 1 --2- 172.21.15.99:0/4160521268 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5598ab6c3100 0x5598ab6d1d60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:57.711 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.711+0000 7f84fddc9740 1 -- 172.21.15.99:0/4160521268 >> 172.21.15.99:0/4160521268 conn(0x5598ab6be670 msgr2=0x5598ab6c18d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:50:57.712 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.711+0000 7f84fddc9740 1 -- 172.21.15.99:0/4160521268 shutdown_connections 2024-03-19T01:50:57.712 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.711+0000 7f84fddc9740 1 -- 172.21.15.99:0/4160521268 wait complete. 2024-03-19T01:50:57.712 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.711+0000 7f84fddc9740 20 client.16140 trim_cache size 0 max 16384 2024-03-19T01:50:57.712 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:50:57.712 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:50:57.712 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:50:57.712 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:50:57.713 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.711+0000 7f84fb0ff640 1 -- 172.21.15.99:0/3795257389 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f84e40a28a0 msgr2=0x7f84e40a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:50:57.713 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.711+0000 7f84fb0ff640 1 --2- 172.21.15.99:0/3795257389 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f84e40a28a0 0x7f84e40a4d60 secure :-1 s=READY pgs=305 cs=0 l=1 rev1=1 crypto rx=0x5598ab6c4350 tx=0x7f84ec004a40 comp rx=0 tx=0).stop 2024-03-19T01:50:57.713 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.711+0000 7f84fb0ff640 1 -- 172.21.15.99:0/3795257389 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f84f4102300 msgr2=0x7f84f4197b00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:50:57.713 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.711+0000 7f84fb0ff640 1 --2- 172.21.15.99:0/3795257389 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f84f4102300 0x7f84f4197b00 secure :-1 s=READY pgs=144 cs=0 l=1 rev1=1 crypto rx=0x7f84e8009230 tx=0x7f84e802c650 comp rx=0 tx=0).stop 2024-03-19T01:50:57.713 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.711+0000 7f84fb0ff640 1 -- 172.21.15.99:0/3795257389 shutdown_connections 2024-03-19T01:50:57.713 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.711+0000 7f84fb0ff640 1 --2- 172.21.15.99:0/3795257389 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f84f4107560 0x7f84f419f0b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:57.713 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.711+0000 7f84fb0ff640 1 --2- 172.21.15.99:0/3795257389 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f84f4104c30 0x7f84f4198040 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:57.713 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.711+0000 7f84fb0ff640 1 --2- 172.21.15.99:0/3795257389 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f84e40a28a0 0x7f84e40a4d60 unknown :-1 s=CLOSED pgs=305 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:57.713 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.711+0000 7f84fb0ff640 1 --2- 172.21.15.99:0/3795257389 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f84f4102300 0x7f84f4197b00 unknown :-1 s=CLOSED pgs=144 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:50:57.714 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.711+0000 7f84fb0ff640 1 -- 172.21.15.99:0/3795257389 >> 172.21.15.99:0/3795257389 conn(0x7f84f40fbcb0 msgr2=0x7f84f4102c40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:50:57.714 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.712+0000 7f84fb0ff640 1 -- 172.21.15.99:0/3795257389 shutdown_connections 2024-03-19T01:50:57.714 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:50:57.712+0000 7f84fb0ff640 1 -- 172.21.15.99:0/3795257389 wait complete. 2024-03-19T01:50:58.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:50:57 smithi115 ceph-mon[67332]: pgmap v646: 97 pgs: 97 active+clean; 264 MiB data, 3.1 GiB used, 1.0 TiB / 1.0 TiB avail; 900 KiB/s rd, 3.1 MiB/s wr, 256 op/s 2024-03-19T01:50:58.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:50:57 smithi099 ceph-mon[62951]: pgmap v646: 97 pgs: 97 active+clean; 264 MiB data, 3.1 GiB used, 1.0 TiB / 1.0 TiB avail; 900 KiB/s rd, 3.1 MiB/s wr, 256 op/s 2024-03-19T01:50:58.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:50:57 smithi112 ceph-mon[66819]: pgmap v646: 97 pgs: 97 active+clean; 264 MiB data, 3.1 GiB used, 1.0 TiB / 1.0 TiB avail; 900 KiB/s rd, 3.1 MiB/s wr, 256 op/s 2024-03-19T01:50:58.698 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1201151 11.35 MB/sec execute 448 sec latency 42.116 ms 2024-03-19T01:50:59.698 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1202971 11.34 MB/sec execute 449 sec latency 70.546 ms 2024-03-19T01:51:00.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:50:59 smithi115 ceph-mon[67332]: pgmap v647: 97 pgs: 97 active+clean; 264 MiB data, 3.1 GiB used, 1.0 TiB / 1.0 TiB avail; 784 KiB/s rd, 2.8 MiB/s wr, 231 op/s 2024-03-19T01:51:00.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:50:59 smithi099 ceph-mon[62951]: pgmap v647: 97 pgs: 97 active+clean; 264 MiB data, 3.1 GiB used, 1.0 TiB / 1.0 TiB avail; 784 KiB/s rd, 2.8 MiB/s wr, 231 op/s 2024-03-19T01:51:00.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:50:59 smithi112 ceph-mon[66819]: pgmap v647: 97 pgs: 97 active+clean; 264 MiB data, 3.1 GiB used, 1.0 TiB / 1.0 TiB avail; 784 KiB/s rd, 2.8 MiB/s wr, 231 op/s 2024-03-19T01:51:00.698 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1204758 11.34 MB/sec execute 450 sec latency 34.161 ms 2024-03-19T01:51:01.698 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1207208 11.34 MB/sec execute 451 sec latency 14.545 ms 2024-03-19T01:51:02.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:51:01 smithi115 ceph-mon[67332]: pgmap v648: 97 pgs: 97 active+clean; 263 MiB data, 3.1 GiB used, 1.0 TiB / 1.0 TiB avail; 853 KiB/s rd, 2.9 MiB/s wr, 243 op/s 2024-03-19T01:51:02.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:51:01 smithi099 ceph-mon[62951]: pgmap v648: 97 pgs: 97 active+clean; 263 MiB data, 3.1 GiB used, 1.0 TiB / 1.0 TiB avail; 853 KiB/s rd, 2.9 MiB/s wr, 243 op/s 2024-03-19T01:51:02.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:51:01 smithi112 ceph-mon[66819]: pgmap v648: 97 pgs: 97 active+clean; 263 MiB data, 3.1 GiB used, 1.0 TiB / 1.0 TiB avail; 853 KiB/s rd, 2.9 MiB/s wr, 243 op/s 2024-03-19T01:51:02.699 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1208973 11.34 MB/sec execute 452 sec latency 24.163 ms 2024-03-19T01:51:03.699 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1211431 11.35 MB/sec execute 453 sec latency 11.849 ms 2024-03-19T01:51:04.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:51:03 smithi115 ceph-mon[67332]: pgmap v649: 97 pgs: 97 active+clean; 263 MiB data, 3.2 GiB used, 1.0 TiB / 1.0 TiB avail; 905 KiB/s rd, 3.2 MiB/s wr, 271 op/s 2024-03-19T01:51:04.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:51:03 smithi099 ceph-mon[62951]: pgmap v649: 97 pgs: 97 active+clean; 263 MiB data, 3.2 GiB used, 1.0 TiB / 1.0 TiB avail; 905 KiB/s rd, 3.2 MiB/s wr, 271 op/s 2024-03-19T01:51:04.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:51:03 smithi112 ceph-mon[66819]: pgmap v649: 97 pgs: 97 active+clean; 263 MiB data, 3.2 GiB used, 1.0 TiB / 1.0 TiB avail; 905 KiB/s rd, 3.2 MiB/s wr, 271 op/s 2024-03-19T01:51:04.699 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1212974 11.34 MB/sec execute 454 sec latency 79.363 ms 2024-03-19T01:51:05.699 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1215092 11.34 MB/sec execute 455 sec latency 48.982 ms 2024-03-19T01:51:06.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:51:05 smithi115 ceph-mon[67332]: pgmap v650: 97 pgs: 97 active+clean; 263 MiB data, 3.2 GiB used, 1.0 TiB / 1.0 TiB avail; 705 KiB/s rd, 2.6 MiB/s wr, 223 op/s 2024-03-19T01:51:06.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:51:05 smithi099 ceph-mon[62951]: pgmap v650: 97 pgs: 97 active+clean; 263 MiB data, 3.2 GiB used, 1.0 TiB / 1.0 TiB avail; 705 KiB/s rd, 2.6 MiB/s wr, 223 op/s 2024-03-19T01:51:06.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:51:05 smithi112 ceph-mon[66819]: pgmap v650: 97 pgs: 97 active+clean; 263 MiB data, 3.2 GiB used, 1.0 TiB / 1.0 TiB avail; 705 KiB/s rd, 2.6 MiB/s wr, 223 op/s 2024-03-19T01:51:06.699 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1217221 11.33 MB/sec execute 456 sec latency 31.006 ms 2024-03-19T01:51:07.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:51:07 smithi115 ceph-mon[67332]: pgmap v651: 97 pgs: 97 active+clean; 267 MiB data, 3.2 GiB used, 1.0 TiB / 1.0 TiB avail; 1.0 MiB/s rd, 3.2 MiB/s wr, 298 op/s 2024-03-19T01:51:07.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:51:07 smithi099 ceph-mon[62951]: pgmap v651: 97 pgs: 97 active+clean; 267 MiB data, 3.2 GiB used, 1.0 TiB / 1.0 TiB avail; 1.0 MiB/s rd, 3.2 MiB/s wr, 298 op/s 2024-03-19T01:51:07.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:51:07 smithi112 ceph-mon[66819]: pgmap v651: 97 pgs: 97 active+clean; 267 MiB data, 3.2 GiB used, 1.0 TiB / 1.0 TiB avail; 1.0 MiB/s rd, 3.2 MiB/s wr, 298 op/s 2024-03-19T01:51:07.699 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1219094 11.33 MB/sec execute 457 sec latency 27.994 ms 2024-03-19T01:51:08.699 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1221761 11.35 MB/sec execute 458 sec latency 16.560 ms 2024-03-19T01:51:09.699 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1223225 11.34 MB/sec execute 459 sec latency 50.636 ms 2024-03-19T01:51:10.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:51:09 smithi115 ceph-mon[67332]: pgmap v652: 97 pgs: 97 active+clean; 267 MiB data, 3.2 GiB used, 1.0 TiB / 1.0 TiB avail; 693 KiB/s rd, 2.8 MiB/s wr, 243 op/s 2024-03-19T01:51:10.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:51:09 smithi099 ceph-mon[62951]: pgmap v652: 97 pgs: 97 active+clean; 267 MiB data, 3.2 GiB used, 1.0 TiB / 1.0 TiB avail; 693 KiB/s rd, 2.8 MiB/s wr, 243 op/s 2024-03-19T01:51:10.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:51:09 smithi112 ceph-mon[66819]: pgmap v652: 97 pgs: 97 active+clean; 267 MiB data, 3.2 GiB used, 1.0 TiB / 1.0 TiB avail; 693 KiB/s rd, 2.8 MiB/s wr, 243 op/s 2024-03-19T01:51:10.699 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1225593 11.34 MB/sec execute 460 sec latency 35.355 ms 2024-03-19T01:51:10.804 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.802+0000 7f5f15444640 1 -- 172.21.15.99:0/876736238 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5f10101930 msgr2=0x7f5f10101db0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:51:10.804 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.802+0000 7f5f15444640 1 --2- 172.21.15.99:0/876736238 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5f10101930 0x7f5f10101db0 secure :-1 s=READY pgs=537 cs=0 l=1 rev1=1 crypto rx=0x7f5ef8009900 tx=0x7f5ef802bc30 comp rx=0 tx=0).stop 2024-03-19T01:51:10.804 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.802+0000 7f5f15444640 1 -- 172.21.15.99:0/876736238 shutdown_connections 2024-03-19T01:51:10.804 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.802+0000 7f5f15444640 1 --2- 172.21.15.99:0/876736238 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5f101022f0 0x7f5f10108b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:10.804 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.802+0000 7f5f15444640 1 --2- 172.21.15.99:0/876736238 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5f10101930 0x7f5f10101db0 unknown :-1 s=CLOSED pgs=537 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:10.804 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.802+0000 7f5f15444640 1 --2- 172.21.15.99:0/876736238 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5f10100740 0x7f5f10100b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:10.804 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.802+0000 7f5f15444640 1 -- 172.21.15.99:0/876736238 >> 172.21.15.99:0/876736238 conn(0x7f5f100fbcb0 msgr2=0x7f5f100fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:51:10.805 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.802+0000 7f5f15444640 1 -- 172.21.15.99:0/876736238 shutdown_connections 2024-03-19T01:51:10.805 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.802+0000 7f5f15444640 1 -- 172.21.15.99:0/876736238 wait complete. 2024-03-19T01:51:10.805 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.803+0000 7f5f15444640 1 Processor -- start 2024-03-19T01:51:10.805 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.803+0000 7f5f15444640 1 -- start start 2024-03-19T01:51:10.805 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.803+0000 7f5f15444640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5f10100740 0x7f5f10069a30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:10.805 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.803+0000 7f5f15444640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5f10101930 0x7f5f1019c7d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:10.805 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.803+0000 7f5f15444640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5f101022f0 0x7f5f1019ee10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:10.805 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.803+0000 7f5f15444640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f5f1010c3c0 con 0x7f5f10101930 2024-03-19T01:51:10.805 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.803+0000 7f5f15444640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f5f1010c240 con 0x7f5f10100740 2024-03-19T01:51:10.805 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.803+0000 7f5f15444640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f5f1010c540 con 0x7f5f101022f0 2024-03-19T01:51:10.805 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.803+0000 7f5f07fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5f10101930 0x7f5f1019c7d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:10.805 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.803+0000 7f5f07fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5f10101930 0x7f5f1019c7d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:38744/0 (socket says 172.21.15.99:38744) 2024-03-19T01:51:10.805 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.803+0000 7f5f07fff640 1 -- 172.21.15.99:0/280731356 learned_addr learned my addr 172.21.15.99:0/280731356 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:51:10.805 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.803+0000 7f5f0effd640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5f10100740 0x7f5f10069a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:10.806 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.803+0000 7f5f0f7fe640 1 --2- 172.21.15.99:0/280731356 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5f101022f0 0x7f5f1019ee10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:10.806 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.803+0000 7f5f0effd640 1 -- 172.21.15.99:0/280731356 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5f101022f0 msgr2=0x7f5f1019ee10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:51:10.806 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.803+0000 7f5f0effd640 1 --2- 172.21.15.99:0/280731356 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5f101022f0 0x7f5f1019ee10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:10.806 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.803+0000 7f5f0effd640 1 -- 172.21.15.99:0/280731356 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5f10101930 msgr2=0x7f5f1019c7d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:51:10.806 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.803+0000 7f5f0effd640 1 --2- 172.21.15.99:0/280731356 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5f10101930 0x7f5f1019c7d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:10.806 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.803+0000 7f5f0effd640 1 -- 172.21.15.99:0/280731356 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f5f1019f6d0 con 0x7f5f10100740 2024-03-19T01:51:10.806 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.804+0000 7f5f0effd640 1 --2- 172.21.15.99:0/280731356 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5f10100740 0x7f5f10069a30 secure :-1 s=READY pgs=146 cs=0 l=1 rev1=1 crypto rx=0x7f5f0000e9c0 tx=0x7f5f0000ee80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:51:10.806 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.804+0000 7f5f0cff9640 1 -- 172.21.15.99:0/280731356 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5f0000ccc0 con 0x7f5f10100740 2024-03-19T01:51:10.806 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.804+0000 7f5f15444640 1 -- 172.21.15.99:0/280731356 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f5f1019f9c0 con 0x7f5f10100740 2024-03-19T01:51:10.806 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.804+0000 7f5f0cff9640 1 -- 172.21.15.99:0/280731356 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f5f000035f0 con 0x7f5f10100740 2024-03-19T01:51:10.806 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.804+0000 7f5f0cff9640 1 -- 172.21.15.99:0/280731356 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5f0000ce60 con 0x7f5f10100740 2024-03-19T01:51:10.806 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.804+0000 7f5f15444640 1 -- 172.21.15.99:0/280731356 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f5f1019ff30 con 0x7f5f10100740 2024-03-19T01:51:10.813 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.806+0000 7f5f0cff9640 1 -- 172.21.15.99:0/280731356 <== mon.2 v2:172.21.15.112:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f5f000037a0 con 0x7f5f10100740 2024-03-19T01:51:10.813 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.806+0000 7f5f0cff9640 1 --2- 172.21.15.99:0/280731356 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f5efc0a28a0 0x7f5efc0a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:10.813 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.806+0000 7f5f1802d740 1 Processor -- start 2024-03-19T01:51:10.813 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.807+0000 7f5f1802d740 1 -- start start 2024-03-19T01:51:10.813 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.807+0000 7f5f07fff640 1 --2- 172.21.15.99:0/280731356 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f5efc0a28a0 0x7f5efc0a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:10.813 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.807+0000 7f5f0cff9640 1 -- 172.21.15.99:0/280731356 <== mon.2 v2:172.21.15.112:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f5f000cee10 con 0x7f5f10100740 2024-03-19T01:51:10.813 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.807+0000 7f5f1802d740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x558eb5e82510 0x558eb5e82930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:10.813 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.807+0000 7f5f1802d740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x558eb5e82f00 0x558eb5e87a00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:10.813 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.807+0000 7f5f1802d740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x558eb5e88070 0x558eb5e8a500 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:10.813 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.807+0000 7f5f1802d740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x558eb5de2be0 con 0x558eb5e82f00 2024-03-19T01:51:10.814 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.807+0000 7f5f1802d740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x558eb5d8b960 con 0x558eb5e88070 2024-03-19T01:51:10.814 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.807+0000 7f5f1802d740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x558eb5ddec80 con 0x558eb5e82510 2024-03-19T01:51:10.814 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.807+0000 7f5f0effd640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x558eb5e82510 0x558eb5e82930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:10.814 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.807+0000 7f5f0effd640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x558eb5e82510 0x558eb5e82930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:58730/0 (socket says 172.21.15.99:58730) 2024-03-19T01:51:10.814 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.807+0000 7f5f07fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x558eb5e82f00 0x558eb5e87a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:10.814 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.807+0000 7f5f0effd640 1 -- 172.21.15.99:0/2673398914 learned_addr learned my addr 172.21.15.99:0/2673398914 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:51:10.814 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.807+0000 7f5f0f7fe640 1 --2- 172.21.15.99:0/2673398914 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x558eb5e88070 0x558eb5e8a500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:10.814 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.807+0000 7f5f07fff640 1 --2- 172.21.15.99:0/280731356 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f5efc0a28a0 0x7f5efc0a4d60 secure :-1 s=READY pgs=307 cs=0 l=1 rev1=1 crypto rx=0x7f5ef8000c00 tx=0x7f5ef8005dd0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:51:10.814 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.807+0000 7f5f0effd640 1 -- 172.21.15.99:0/2673398914 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x558eb5e88070 msgr2=0x558eb5e8a500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:51:10.814 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.807+0000 7f5f0effd640 1 --2- 172.21.15.99:0/2673398914 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x558eb5e88070 0x558eb5e8a500 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:10.814 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.807+0000 7f5f0effd640 1 -- 172.21.15.99:0/2673398914 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x558eb5e82f00 msgr2=0x558eb5e87a00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:51:10.814 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.807+0000 7f5f0effd640 1 --2- 172.21.15.99:0/2673398914 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x558eb5e82f00 0x558eb5e87a00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:10.814 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.807+0000 7f5f0effd640 1 -- 172.21.15.99:0/2673398914 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x558eb5d82d20 con 0x558eb5e82510 2024-03-19T01:51:10.814 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.807+0000 7f5f07fff640 1 --2- 172.21.15.99:0/2673398914 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x558eb5e82f00 0x558eb5e87a00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:51:10.815 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.807+0000 7f5f0f7fe640 1 --2- 172.21.15.99:0/2673398914 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x558eb5e88070 0x558eb5e8a500 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:51:10.815 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.808+0000 7f5f0effd640 1 --2- 172.21.15.99:0/2673398914 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x558eb5e82510 0x558eb5e82930 secure :-1 s=READY pgs=105 cs=0 l=1 rev1=1 crypto rx=0x7f5f000a39e0 tx=0x7f5f0009be70 comp rx=0 tx=0).ready entity=mon.1 client_cookie=7a9e7b168acdde06 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:51:10.815 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.808+0000 7f5f057fa640 1 -- 172.21.15.99:0/2673398914 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5f0009f8c0 con 0x558eb5e82510 2024-03-19T01:51:10.815 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.808+0000 7f5f057fa640 1 -- 172.21.15.99:0/2673398914 <== mon.1 v2:172.21.15.115:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f5f000a3f10 con 0x558eb5e82510 2024-03-19T01:51:10.815 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.808+0000 7f5f057fa640 1 -- 172.21.15.99:0/2673398914 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5f000a3ce0 con 0x558eb5e82510 2024-03-19T01:51:10.815 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.808+0000 7f5f1802d740 1 -- 172.21.15.99:0/2673398914 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x558eb5e82510 msgr2=0x558eb5e82930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:51:10.815 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.808+0000 7f5f1802d740 1 --2- 172.21.15.99:0/2673398914 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x558eb5e82510 0x558eb5e82930 secure :-1 s=READY pgs=105 cs=0 l=1 rev1=1 crypto rx=0x7f5f000a39e0 tx=0x7f5f0009be70 comp rx=0 tx=0).stop 2024-03-19T01:51:10.815 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.809+0000 7f5f1802d740 1 -- 172.21.15.99:0/2673398914 shutdown_connections 2024-03-19T01:51:10.815 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.809+0000 7f5f1802d740 1 --2- 172.21.15.99:0/2673398914 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x558eb5e88070 0x558eb5e8a500 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:10.815 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.809+0000 7f5f1802d740 1 --2- 172.21.15.99:0/2673398914 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x558eb5e82f00 0x558eb5e87a00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:10.815 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.809+0000 7f5f1802d740 1 --2- 172.21.15.99:0/2673398914 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x558eb5e82510 0x558eb5e82930 unknown :-1 s=CLOSED pgs=105 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:10.815 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.809+0000 7f5f1802d740 1 -- 172.21.15.99:0/2673398914 >> 172.21.15.99:0/2673398914 conn(0x558eb5e79670 msgr2=0x558eb5e79a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:51:10.815 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.809+0000 7f5f1802d740 1 -- 172.21.15.99:0/2673398914 shutdown_connections 2024-03-19T01:51:10.815 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.809+0000 7f5f1802d740 1 -- 172.21.15.99:0/2673398914 wait complete. 2024-03-19T01:51:10.815 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.809+0000 7f5f1802d740 1 -- start start 2024-03-19T01:51:10.816 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.809+0000 7f5f1802d740 1 Processor -- start 2024-03-19T01:51:10.816 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.809+0000 7f5f1802d740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x558eb5e82510 0x558eb5e8c520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:10.816 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.809+0000 7f5f1802d740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x558eb5e82f00 0x558eb5e8ca60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:10.816 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.809+0000 7f5f1802d740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x558eb5e88070 0x558eb5e93a30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:10.816 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.809+0000 7f5f1802d740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x558eb5e8afc0 con 0x558eb5e88070 2024-03-19T01:51:10.816 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.809+0000 7f5f1802d740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x558eb5d9edb0 con 0x558eb5e82f00 2024-03-19T01:51:10.816 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.809+0000 7f5f1802d740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x558eb5e8b140 con 0x558eb5e82510 2024-03-19T01:51:10.816 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.809+0000 7f5f0f7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x558eb5e88070 0x558eb5e93a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:10.816 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.809+0000 7f5f1802d740 10 client.0 _ll_register_callbacks cb 0x558eb5da3e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:51:10.816 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.809+0000 7f5f07fff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x558eb5e82f00 0x558eb5e8ca60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:10.816 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.809+0000 7f5f0effd640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x558eb5e82510 0x558eb5e8c520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:10.816 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.809+0000 7f5f0f7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x558eb5e88070 0x558eb5e93a30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:38762/0 (socket says 172.21.15.99:38762) 2024-03-19T01:51:10.816 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.809+0000 7f5f0f7fe640 1 -- 172.21.15.99:0/1602883168 learned_addr learned my addr 172.21.15.99:0/1602883168 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:51:10.816 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.810+0000 7f5f0f7fe640 1 -- 172.21.15.99:0/1602883168 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x558eb5e82510 msgr2=0x558eb5e8c520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:51:10.817 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.810+0000 7f5f0f7fe640 1 --2- 172.21.15.99:0/1602883168 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x558eb5e82510 0x558eb5e8c520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:10.817 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.810+0000 7f5f0f7fe640 1 -- 172.21.15.99:0/1602883168 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x558eb5e82f00 msgr2=0x558eb5e8ca60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:51:10.817 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.810+0000 7f5f0f7fe640 1 --2- 172.21.15.99:0/1602883168 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x558eb5e82f00 0x558eb5e8ca60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:10.817 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.810+0000 7f5f0f7fe640 1 -- 172.21.15.99:0/1602883168 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x558eb5d9dc60 con 0x558eb5e88070 2024-03-19T01:51:10.817 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.810+0000 7f5f07fff640 1 --2- 172.21.15.99:0/1602883168 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x558eb5e82f00 0x558eb5e8ca60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:51:10.817 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.810+0000 7f5f0f7fe640 1 --2- 172.21.15.99:0/1602883168 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x558eb5e88070 0x558eb5e93a30 secure :-1 s=READY pgs=538 cs=0 l=1 rev1=1 crypto rx=0x7f5f08011d10 tx=0x7f5f0800f790 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a15d0b3241a28529 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:51:10.817 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.810+0000 7f5ef67fc640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:51:10.817 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.810+0000 7f5ef67fc640 1 -- 172.21.15.99:0/1602883168 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5f0801e060 con 0x558eb5e88070 2024-03-19T01:51:10.817 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.810+0000 7f5ef67fc640 1 -- 172.21.15.99:0/1602883168 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f5f08003970 con 0x558eb5e88070 2024-03-19T01:51:10.817 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.810+0000 7f5f1802d740 1 -- 172.21.15.99:0/1602883168 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f5f000a3f10 con 0x558eb5e88070 2024-03-19T01:51:10.817 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.810+0000 7f5ef67fc640 1 -- 172.21.15.99:0/1602883168 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7f5f08009e30 con 0x558eb5e88070 2024-03-19T01:51:10.817 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.810+0000 7f5f1802d740 10 client.16152 fetch_fsmap learned FSMap version 27 2024-03-19T01:51:10.817 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.810+0000 7f5f1802d740 1 -- 172.21.15.99:0/1602883168 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x558eb5af7210 con 0x558eb5e88070 2024-03-19T01:51:10.817 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.810+0000 7f5ef67fc640 1 -- 172.21.15.99:0/1602883168 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f5f000a3f10 con 0x558eb5e88070 2024-03-19T01:51:10.817 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.810+0000 7f5ef67fc640 1 -- 172.21.15.99:0/1602883168 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 27) ==== 8338+0+0 (secure 0 0 0) 0x7f5f08019070 con 0x558eb5e88070 2024-03-19T01:51:10.818 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.810+0000 7f5f1802d740 10 client.16152 fetch_fsmap finished waiting for FSMap version 27 2024-03-19T01:51:10.818 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.810+0000 7f5f1802d740 10 client.16152 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:51:10.818 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.811+0000 7f5f1802d740 20 client.16152 populate_metadata read hostname 'smithi099' 2024-03-19T01:51:10.818 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.811+0000 7f5f1802d740 1 --2- 172.21.15.99:0/1602883168 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x558eb5e98450 0x558eb5e9a860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:10.818 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.811+0000 7f5f1802d740 4 client.16152 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:51:10.818 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.811+0000 7f5f1802d740 1 -- 172.21.15.99:0/1602883168 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x558eb5df0120 con 0x558eb5e98450 2024-03-19T01:51:10.818 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.811+0000 7f5f0effd640 1 --2- 172.21.15.99:0/1602883168 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x558eb5e98450 0x558eb5e9a860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:10.818 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.811+0000 7f5f0effd640 1 --2- 172.21.15.99:0/1602883168 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x558eb5e98450 0x558eb5e9a860 crc :-1 s=READY pgs=119 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=3a12b1775a3831e9 server_cookie=a034b417c766c6d0 in_seq=0 out_seq=0 2024-03-19T01:51:10.818 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.811+0000 7f5ef67fc640 10 client.16152 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:51:10.818 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.812+0000 7f5ef67fc640 1 -- 172.21.15.99:0/1602883168 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x558eb5df0120 con 0x558eb5e98450 2024-03-19T01:51:10.818 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.812+0000 7f5ef67fc640 10 client.16152 handle_command_reply: tid=0 2024-03-19T01:51:10.818 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.812+0000 7f5f1802d740 1 client.16152 shutdown 2024-03-19T01:51:10.818 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.812+0000 7f5f1802d740 1 -- 172.21.15.99:0/1602883168 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x558eb5e88070 msgr2=0x558eb5e93a30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:51:10.818 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.812+0000 7f5f1802d740 1 --2- 172.21.15.99:0/1602883168 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x558eb5e88070 0x558eb5e93a30 secure :-1 s=READY pgs=538 cs=0 l=1 rev1=1 crypto rx=0x7f5f08011d10 tx=0x7f5f0800f790 comp rx=0 tx=0).stop 2024-03-19T01:51:10.819 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.812+0000 7f5f1802d740 1 -- 172.21.15.99:0/1602883168 shutdown_connections 2024-03-19T01:51:10.819 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.812+0000 7f5f1802d740 1 --2- 172.21.15.99:0/1602883168 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x558eb5e98450 0x558eb5e9a860 crc :-1 s=READY pgs=119 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:10.819 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.812+0000 7f5f1802d740 1 --2- 172.21.15.99:0/1602883168 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x558eb5e82f00 0x558eb5e8ca60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:10.819 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.812+0000 7f5f1802d740 1 --2- 172.21.15.99:0/1602883168 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x558eb5e88070 0x558eb5e93a30 unknown :-1 s=CLOSED pgs=538 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:10.819 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.812+0000 7f5f1802d740 1 --2- 172.21.15.99:0/1602883168 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x558eb5e82510 0x558eb5e8c520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:10.819 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.812+0000 7f5ef67fc640 0 client.16152 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:51:10.819 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.812+0000 7f5f1802d740 1 -- 172.21.15.99:0/1602883168 >> 172.21.15.99:0/1602883168 conn(0x558eb5e79670 msgr2=0x558eb5e7a8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:51:10.819 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.812+0000 7f5f1802d740 1 -- 172.21.15.99:0/1602883168 shutdown_connections 2024-03-19T01:51:10.819 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.812+0000 7f5f1802d740 1 -- 172.21.15.99:0/1602883168 wait complete. 2024-03-19T01:51:10.819 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.812+0000 7f5f1802d740 20 client.16152 trim_cache size 0 max 16384 2024-03-19T01:51:10.833 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.828+0000 7f5f1802d740 1 Processor -- start 2024-03-19T01:51:10.833 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.828+0000 7f5f1802d740 1 -- start start 2024-03-19T01:51:10.833 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.828+0000 7f5f1802d740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x558eb5e82510 0x558eb5e78220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:10.833 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.829+0000 7f5f1802d740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x558eb5e82f00 0x558eb5e939f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:10.833 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.829+0000 7f5f1802d740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x558eb5e88070 0x558eb5e78760 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:10.833 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.829+0000 7f5f1802d740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f5f101a9420 con 0x558eb5e82510 2024-03-19T01:51:10.833 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.829+0000 7f5f1802d740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f5f101a6cf0 con 0x558eb5e82f00 2024-03-19T01:51:10.833 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.829+0000 7f5f1802d740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f5f101a4b40 con 0x558eb5e88070 2024-03-19T01:51:10.833 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.829+0000 7f5f0effd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x558eb5e82510 0x558eb5e78220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:10.833 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.829+0000 7f5f0effd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x558eb5e82510 0x558eb5e78220 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:38768/0 (socket says 172.21.15.99:38768) 2024-03-19T01:51:10.834 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.829+0000 7f5f0effd640 1 -- 172.21.15.99:0/243143472 learned_addr learned my addr 172.21.15.99:0/243143472 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:51:10.834 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.829+0000 7f5f07fff640 1 --2- 172.21.15.99:0/243143472 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x558eb5e82f00 0x558eb5e939f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:10.834 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.829+0000 7f5f0f7fe640 1 --2- 172.21.15.99:0/243143472 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x558eb5e88070 0x558eb5e78760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:10.834 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.829+0000 7f5f0effd640 1 -- 172.21.15.99:0/243143472 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x558eb5e88070 msgr2=0x558eb5e78760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:51:10.834 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.829+0000 7f5f0effd640 1 --2- 172.21.15.99:0/243143472 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x558eb5e88070 0x558eb5e78760 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:10.834 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.829+0000 7f5f0effd640 1 -- 172.21.15.99:0/243143472 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x558eb5e82f00 msgr2=0x558eb5e939f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:51:10.834 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.829+0000 7f5f0effd640 1 --2- 172.21.15.99:0/243143472 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x558eb5e82f00 0x558eb5e939f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:10.834 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.829+0000 7f5f0effd640 1 -- 172.21.15.99:0/243143472 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x558eb5bc3620 con 0x558eb5e82510 2024-03-19T01:51:10.834 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.829+0000 7f5f07fff640 1 --2- 172.21.15.99:0/243143472 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x558eb5e82f00 0x558eb5e939f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:51:10.834 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.829+0000 7f5f0effd640 1 --2- 172.21.15.99:0/243143472 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x558eb5e82510 0x558eb5e78220 secure :-1 s=READY pgs=539 cs=0 l=1 rev1=1 crypto rx=0x7f5f00097bd0 tx=0x7f5f00097a90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ea004b3f4a9abf5f server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:51:10.834 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.829+0000 7f5f04ff9640 1 -- 172.21.15.99:0/243143472 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5f0007f070 con 0x558eb5e82510 2024-03-19T01:51:10.834 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.829+0000 7f5f04ff9640 1 -- 172.21.15.99:0/243143472 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f5f0009f8c0 con 0x558eb5e82510 2024-03-19T01:51:10.834 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.829+0000 7f5f04ff9640 1 -- 172.21.15.99:0/243143472 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5f000a3a80 con 0x558eb5e82510 2024-03-19T01:51:10.834 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.830+0000 7f5f1802d740 1 -- 172.21.15.99:0/243143472 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x558eb5e82510 msgr2=0x558eb5e78220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:51:10.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.830+0000 7f5f1802d740 1 --2- 172.21.15.99:0/243143472 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x558eb5e82510 0x558eb5e78220 secure :-1 s=READY pgs=539 cs=0 l=1 rev1=1 crypto rx=0x7f5f00097bd0 tx=0x7f5f00097a90 comp rx=0 tx=0).stop 2024-03-19T01:51:10.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.830+0000 7f5f1802d740 1 -- 172.21.15.99:0/243143472 shutdown_connections 2024-03-19T01:51:10.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.830+0000 7f5f1802d740 1 --2- 172.21.15.99:0/243143472 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x558eb5e82f00 0x558eb5e939f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:10.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.830+0000 7f5f1802d740 1 --2- 172.21.15.99:0/243143472 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x558eb5e88070 0x558eb5e78760 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:10.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.830+0000 7f5f1802d740 1 --2- 172.21.15.99:0/243143472 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x558eb5e82510 0x558eb5e78220 unknown :-1 s=CLOSED pgs=539 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:10.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.830+0000 7f5f1802d740 1 -- 172.21.15.99:0/243143472 >> 172.21.15.99:0/243143472 conn(0x558eb5e79670 msgr2=0x558eb5e9a1c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:51:10.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.830+0000 7f5f1802d740 1 -- 172.21.15.99:0/243143472 shutdown_connections 2024-03-19T01:51:10.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.830+0000 7f5f1802d740 1 -- 172.21.15.99:0/243143472 wait complete. 2024-03-19T01:51:10.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.830+0000 7f5f1802d740 1 -- start start 2024-03-19T01:51:10.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.830+0000 7f5f1802d740 1 Processor -- start 2024-03-19T01:51:10.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.831+0000 7f5f1802d740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x558eb5e82510 0x558eb5e90b70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:10.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.831+0000 7f5f1802d740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x558eb5e82f00 0x558eb5e910b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:10.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.831+0000 7f5f1802d740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x558eb5e88070 0x558eb5e920c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:10.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.831+0000 7f5f1802d740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f5f101a51b0 con 0x558eb5e88070 2024-03-19T01:51:10.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.831+0000 7f5f1802d740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f5f101a6620 con 0x558eb5e82f00 2024-03-19T01:51:10.836 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.831+0000 7f5f1802d740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f5f101a5880 con 0x558eb5e82510 2024-03-19T01:51:10.836 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.831+0000 7f5f1802d740 10 client.0 _ll_register_callbacks cb 0x7f5f08009cc0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:51:10.836 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.831+0000 7f5f0f7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x558eb5e88070 0x558eb5e920c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:10.836 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.831+0000 7f5f0effd640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x558eb5e82510 0x558eb5e90b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:10.836 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.831+0000 7f5f0f7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x558eb5e88070 0x558eb5e920c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:38782/0 (socket says 172.21.15.99:38782) 2024-03-19T01:51:10.836 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.831+0000 7f5f0f7fe640 1 -- 172.21.15.99:0/4174978405 learned_addr learned my addr 172.21.15.99:0/4174978405 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:51:10.836 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.831+0000 7f5f07fff640 1 --2- 172.21.15.99:0/4174978405 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x558eb5e82f00 0x558eb5e910b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:10.836 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.831+0000 7f5f0effd640 1 -- 172.21.15.99:0/4174978405 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x558eb5e82f00 msgr2=0x558eb5e910b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:51:10.836 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.831+0000 7f5f0effd640 1 --2- 172.21.15.99:0/4174978405 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x558eb5e82f00 0x558eb5e910b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:10.837 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.831+0000 7f5f0effd640 1 -- 172.21.15.99:0/4174978405 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x558eb5e88070 msgr2=0x558eb5e920c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:51:10.837 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.831+0000 7f5f0effd640 1 --2- 172.21.15.99:0/4174978405 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x558eb5e88070 0x558eb5e920c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:10.837 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.831+0000 7f5f0effd640 1 -- 172.21.15.99:0/4174978405 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({osdmap=0}) -- 0x558eb5e926b0 con 0x558eb5e82510 2024-03-19T01:51:10.837 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.831+0000 7f5f0f7fe640 1 --2- 172.21.15.99:0/4174978405 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x558eb5e88070 0x558eb5e920c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:51:10.837 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.831+0000 7f5f0effd640 1 --2- 172.21.15.99:0/4174978405 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x558eb5e82510 0x558eb5e90b70 secure :-1 s=READY pgs=106 cs=0 l=1 rev1=1 crypto rx=0x7f5f00097bd0 tx=0x7f5f0009b7e0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=852abaf0dc056f28 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:51:10.837 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.831+0000 7f5ef67fc640 10 client.0 ms_handle_connect on v2:172.21.15.115:3300/0 2024-03-19T01:51:10.837 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.831+0000 7f5ef67fc640 1 -- 172.21.15.99:0/4174978405 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5f0007f070 con 0x558eb5e82510 2024-03-19T01:51:10.837 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.831+0000 7f5f1802d740 1 -- 172.21.15.99:0/4174978405 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f5f0009f8c0 con 0x558eb5e82510 2024-03-19T01:51:10.837 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.831+0000 7f5ef67fc640 1 -- 172.21.15.99:0/4174978405 <== mon.1 v2:172.21.15.115:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f5f000860b0 con 0x558eb5e82510 2024-03-19T01:51:10.837 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.832+0000 7f5ef67fc640 1 -- 172.21.15.99:0/4174978405 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7f5f000a3a80 con 0x558eb5e82510 2024-03-19T01:51:10.837 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.832+0000 7f5f1802d740 10 client.25978 fetch_fsmap learned FSMap version 27 2024-03-19T01:51:10.837 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.832+0000 7f5f1802d740 1 -- 172.21.15.99:0/4174978405 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x558eb5e939c0 con 0x558eb5e82510 2024-03-19T01:51:10.838 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.832+0000 7f5ef67fc640 1 -- 172.21.15.99:0/4174978405 <== mon.1 v2:172.21.15.115:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f5f0009f8c0 con 0x558eb5e82510 2024-03-19T01:51:10.838 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.832+0000 7f5ef67fc640 1 -- 172.21.15.99:0/4174978405 <== mon.1 v2:172.21.15.115:3300/0 5 ==== fsmap(e 27) ==== 8338+0+0 (secure 0 0 0) 0x7f5f00082030 con 0x558eb5e82510 2024-03-19T01:51:10.838 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.832+0000 7f5f1802d740 10 client.25978 fetch_fsmap finished waiting for FSMap version 27 2024-03-19T01:51:10.838 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.832+0000 7f5f1802d740 10 client.25978 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:51:10.838 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.832+0000 7f5f1802d740 20 client.25978 populate_metadata read hostname 'smithi099' 2024-03-19T01:51:10.838 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.832+0000 7f5f1802d740 1 --2- 172.21.15.99:0/4174978405 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x558eb5efe7a0 0x558eb5f00b90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:10.838 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.832+0000 7f5f1802d740 4 client.25978 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:51:10.838 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.832+0000 7f5f1802d740 1 -- 172.21.15.99:0/4174978405 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x558eb5dea470 con 0x558eb5efe7a0 2024-03-19T01:51:10.838 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.832+0000 7f5f07fff640 1 --2- 172.21.15.99:0/4174978405 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x558eb5efe7a0 0x558eb5f00b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:10.838 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.833+0000 7f5f07fff640 1 --2- 172.21.15.99:0/4174978405 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x558eb5efe7a0 0x558eb5f00b90 crc :-1 s=READY pgs=120 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=64b8626ef7acf0a5 server_cookie=be65100f40716378 in_seq=0 out_seq=0 2024-03-19T01:51:10.838 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.833+0000 7f5ef67fc640 10 client.25978 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:51:10.897 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.896+0000 7f5ef67fc640 1 -- 172.21.15.99:0/4174978405 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x558eb5dea470 con 0x558eb5efe7a0 2024-03-19T01:51:10.898 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.896+0000 7f5ef67fc640 10 client.25978 handle_command_reply: tid=0 2024-03-19T01:51:10.898 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.896+0000 7f5f1802d740 1 client.25978 shutdown 2024-03-19T01:51:10.898 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.896+0000 7f5f1802d740 1 -- 172.21.15.99:0/4174978405 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x558eb5e82510 msgr2=0x558eb5e90b70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:51:10.898 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.896+0000 7f5f1802d740 1 --2- 172.21.15.99:0/4174978405 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x558eb5e82510 0x558eb5e90b70 secure :-1 s=READY pgs=106 cs=0 l=1 rev1=1 crypto rx=0x7f5f00097bd0 tx=0x7f5f0009b7e0 comp rx=0 tx=0).stop 2024-03-19T01:51:10.898 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.896+0000 7f5f1802d740 1 -- 172.21.15.99:0/4174978405 shutdown_connections 2024-03-19T01:51:10.899 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.896+0000 7f5f1802d740 1 --2- 172.21.15.99:0/4174978405 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x558eb5efe7a0 0x558eb5f00b90 crc :-1 s=READY pgs=120 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:10.899 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.896+0000 7f5f1802d740 1 --2- 172.21.15.99:0/4174978405 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x558eb5e82f00 0x558eb5e910b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:10.899 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.896+0000 7f5f1802d740 1 --2- 172.21.15.99:0/4174978405 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x558eb5e88070 0x558eb5e920c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:10.899 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.896+0000 7f5f1802d740 1 --2- 172.21.15.99:0/4174978405 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x558eb5e82510 0x558eb5e90b70 unknown :-1 s=CLOSED pgs=106 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:10.900 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.896+0000 7f5ef67fc640 0 client.25978 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:51:10.900 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.896+0000 7f5f1802d740 1 -- 172.21.15.99:0/4174978405 >> 172.21.15.99:0/4174978405 conn(0x558eb5e79670 msgr2=0x558eb5e78220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:51:10.900 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.896+0000 7f5f1802d740 1 -- 172.21.15.99:0/4174978405 shutdown_connections 2024-03-19T01:51:10.900 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.896+0000 7f5f1802d740 1 -- 172.21.15.99:0/4174978405 wait complete. 2024-03-19T01:51:10.900 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.896+0000 7f5f1802d740 20 client.25978 trim_cache size 0 max 16384 2024-03-19T01:51:10.900 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:51:10.900 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:51:10.900 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:51:10.900 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:51:10.901 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.897+0000 7f5f15444640 1 -- 172.21.15.99:0/280731356 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f5efc0a28a0 msgr2=0x7f5efc0a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:51:10.901 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.897+0000 7f5f15444640 1 --2- 172.21.15.99:0/280731356 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f5efc0a28a0 0x7f5efc0a4d60 secure :-1 s=READY pgs=307 cs=0 l=1 rev1=1 crypto rx=0x7f5ef8000c00 tx=0x7f5ef8005dd0 comp rx=0 tx=0).stop 2024-03-19T01:51:10.901 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.897+0000 7f5f15444640 1 -- 172.21.15.99:0/280731356 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5f10100740 msgr2=0x7f5f10069a30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:51:10.901 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.897+0000 7f5f15444640 1 --2- 172.21.15.99:0/280731356 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5f10100740 0x7f5f10069a30 secure :-1 s=READY pgs=146 cs=0 l=1 rev1=1 crypto rx=0x7f5f0000e9c0 tx=0x7f5f0000ee80 comp rx=0 tx=0).stop 2024-03-19T01:51:10.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.897+0000 7f5f15444640 1 -- 172.21.15.99:0/280731356 shutdown_connections 2024-03-19T01:51:10.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.897+0000 7f5f15444640 1 --2- 172.21.15.99:0/280731356 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5f101022f0 0x7f5f1019ee10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:10.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.897+0000 7f5f15444640 1 --2- 172.21.15.99:0/280731356 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5f10101930 0x7f5f1019c7d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:10.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.897+0000 7f5f15444640 1 --2- 172.21.15.99:0/280731356 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f5efc0a28a0 0x7f5efc0a4d60 unknown :-1 s=CLOSED pgs=307 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:10.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.897+0000 7f5f15444640 1 --2- 172.21.15.99:0/280731356 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5f10100740 0x7f5f10069a30 unknown :-1 s=CLOSED pgs=146 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:10.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.897+0000 7f5f15444640 1 -- 172.21.15.99:0/280731356 >> 172.21.15.99:0/280731356 conn(0x7f5f100fbcb0 msgr2=0x7f5f10104c30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:51:10.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.897+0000 7f5f15444640 1 -- 172.21.15.99:0/280731356 shutdown_connections 2024-03-19T01:51:10.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:10.897+0000 7f5f15444640 1 -- 172.21.15.99:0/280731356 wait complete. 2024-03-19T01:51:11.699 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1227661 11.33 MB/sec execute 461 sec latency 23.540 ms 2024-03-19T01:51:11.806 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:51:11 smithi115 ceph-mon[67332]: pgmap v653: 97 pgs: 97 active+clean; 266 MiB data, 3.2 GiB used, 1.0 TiB / 1.0 TiB avail; 696 KiB/s rd, 2.9 MiB/s wr, 236 op/s 2024-03-19T01:51:11.840 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:51:11 smithi099 ceph-mon[62951]: pgmap v653: 97 pgs: 97 active+clean; 266 MiB data, 3.2 GiB used, 1.0 TiB / 1.0 TiB avail; 696 KiB/s rd, 2.9 MiB/s wr, 236 op/s 2024-03-19T01:51:11.861 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:51:11 smithi112 ceph-mon[66819]: pgmap v653: 97 pgs: 97 active+clean; 266 MiB data, 3.2 GiB used, 1.0 TiB / 1.0 TiB avail; 696 KiB/s rd, 2.9 MiB/s wr, 236 op/s 2024-03-19T01:51:12.699 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1229560 11.34 MB/sec execute 462 sec latency 25.489 ms 2024-03-19T01:51:13.699 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1232193 11.34 MB/sec execute 463 sec latency 14.265 ms 2024-03-19T01:51:14.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:51:13 smithi115 ceph-mon[67332]: pgmap v654: 97 pgs: 97 active+clean; 270 MiB data, 3.2 GiB used, 1.0 TiB / 1.0 TiB avail; 775 KiB/s rd, 3.1 MiB/s wr, 261 op/s 2024-03-19T01:51:14.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:51:13 smithi099 ceph-mon[62951]: pgmap v654: 97 pgs: 97 active+clean; 270 MiB data, 3.2 GiB used, 1.0 TiB / 1.0 TiB avail; 775 KiB/s rd, 3.1 MiB/s wr, 261 op/s 2024-03-19T01:51:14.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:51:13 smithi112 ceph-mon[66819]: pgmap v654: 97 pgs: 97 active+clean; 270 MiB data, 3.2 GiB used, 1.0 TiB / 1.0 TiB avail; 775 KiB/s rd, 3.1 MiB/s wr, 261 op/s 2024-03-19T01:51:14.699 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1233777 11.34 MB/sec execute 464 sec latency 51.046 ms 2024-03-19T01:51:15.699 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1236159 11.34 MB/sec execute 465 sec latency 48.155 ms 2024-03-19T01:51:16.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:51:15 smithi115 ceph-mon[67332]: pgmap v655: 97 pgs: 97 active+clean; 270 MiB data, 3.2 GiB used, 1.0 TiB / 1.0 TiB avail; 690 KiB/s rd, 2.7 MiB/s wr, 227 op/s 2024-03-19T01:51:16.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:51:15 smithi099 ceph-mon[62951]: pgmap v655: 97 pgs: 97 active+clean; 270 MiB data, 3.2 GiB used, 1.0 TiB / 1.0 TiB avail; 690 KiB/s rd, 2.7 MiB/s wr, 227 op/s 2024-03-19T01:51:16.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:51:15 smithi112 ceph-mon[66819]: pgmap v655: 97 pgs: 97 active+clean; 270 MiB data, 3.2 GiB used, 1.0 TiB / 1.0 TiB avail; 690 KiB/s rd, 2.7 MiB/s wr, 227 op/s 2024-03-19T01:51:16.699 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1237913 11.33 MB/sec execute 466 sec latency 40.094 ms 2024-03-19T01:51:17.699 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1240017 11.34 MB/sec execute 467 sec latency 38.429 ms 2024-03-19T01:51:18.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:51:17 smithi115 ceph-mon[67332]: pgmap v656: 97 pgs: 97 active+clean; 273 MiB data, 3.3 GiB used, 1.0 TiB / 1.0 TiB avail; 1.0 MiB/s rd, 3.3 MiB/s wr, 302 op/s 2024-03-19T01:51:18.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:51:17 smithi099 ceph-mon[62951]: pgmap v656: 97 pgs: 97 active+clean; 273 MiB data, 3.3 GiB used, 1.0 TiB / 1.0 TiB avail; 1.0 MiB/s rd, 3.3 MiB/s wr, 302 op/s 2024-03-19T01:51:18.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:51:17 smithi112 ceph-mon[66819]: pgmap v656: 97 pgs: 97 active+clean; 273 MiB data, 3.3 GiB used, 1.0 TiB / 1.0 TiB avail; 1.0 MiB/s rd, 3.3 MiB/s wr, 302 op/s 2024-03-19T01:51:18.699 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1242216 11.34 MB/sec execute 468 sec latency 52.529 ms 2024-03-19T01:51:19.699 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1243852 11.33 MB/sec execute 469 sec latency 74.559 ms 2024-03-19T01:51:20.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:51:19 smithi115 ceph-mon[67332]: pgmap v657: 97 pgs: 97 active+clean; 269 MiB data, 3.3 GiB used, 1.0 TiB / 1.0 TiB avail; 712 KiB/s rd, 2.7 MiB/s wr, 238 op/s 2024-03-19T01:51:20.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:51:19 smithi099 ceph-mon[62951]: pgmap v657: 97 pgs: 97 active+clean; 269 MiB data, 3.3 GiB used, 1.0 TiB / 1.0 TiB avail; 712 KiB/s rd, 2.7 MiB/s wr, 238 op/s 2024-03-19T01:51:20.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:51:19 smithi112 ceph-mon[66819]: pgmap v657: 97 pgs: 97 active+clean; 269 MiB data, 3.3 GiB used, 1.0 TiB / 1.0 TiB avail; 712 KiB/s rd, 2.7 MiB/s wr, 238 op/s 2024-03-19T01:51:20.699 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1246232 11.33 MB/sec execute 470 sec latency 23.695 ms 2024-03-19T01:51:21.699 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1247983 11.33 MB/sec execute 471 sec latency 49.043 ms 2024-03-19T01:51:22.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:51:21 smithi115 ceph-mon[67332]: pgmap v658: 97 pgs: 97 active+clean; 262 MiB data, 3.2 GiB used, 1.0 TiB / 1.0 TiB avail; 715 KiB/s rd, 2.9 MiB/s wr, 236 op/s 2024-03-19T01:51:22.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:51:21 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:51:22.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:51:21 smithi099 ceph-mon[62951]: pgmap v658: 97 pgs: 97 active+clean; 262 MiB data, 3.2 GiB used, 1.0 TiB / 1.0 TiB avail; 715 KiB/s rd, 2.9 MiB/s wr, 236 op/s 2024-03-19T01:51:22.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:51:21 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:51:22.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:51:21 smithi112 ceph-mon[66819]: pgmap v658: 97 pgs: 97 active+clean; 262 MiB data, 3.2 GiB used, 1.0 TiB / 1.0 TiB avail; 715 KiB/s rd, 2.9 MiB/s wr, 236 op/s 2024-03-19T01:51:22.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:51:21 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:51:22.699 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1250340 11.34 MB/sec execute 472 sec latency 12.696 ms 2024-03-19T01:51:23.700 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1251802 11.32 MB/sec execute 473 sec latency 28.462 ms 2024-03-19T01:51:23.993 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.991+0000 7f2e6e0ae640 1 -- 172.21.15.99:0/2162836585 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2e681022f0 msgr2=0x7f2e68108b80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:51:23.993 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.991+0000 7f2e6e0ae640 1 --2- 172.21.15.99:0/2162836585 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2e681022f0 0x7f2e68108b80 secure :-1 s=READY pgs=540 cs=0 l=1 rev1=1 crypto rx=0x7f2e60009ca0 tx=0x7f2e6002c0f0 comp rx=0 tx=0).stop 2024-03-19T01:51:23.994 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.991+0000 7f2e6e0ae640 1 -- 172.21.15.99:0/2162836585 shutdown_connections 2024-03-19T01:51:23.994 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.992+0000 7f2e6e0ae640 1 --2- 172.21.15.99:0/2162836585 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2e681022f0 0x7f2e68108b80 unknown :-1 s=CLOSED pgs=540 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:23.994 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.992+0000 7f2e6e0ae640 1 --2- 172.21.15.99:0/2162836585 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f2e68101930 0x7f2e68101db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:23.994 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.992+0000 7f2e6e0ae640 1 --2- 172.21.15.99:0/2162836585 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f2e68100740 0x7f2e68100b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:23.994 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.992+0000 7f2e6e0ae640 1 -- 172.21.15.99:0/2162836585 >> 172.21.15.99:0/2162836585 conn(0x7f2e680fbcb0 msgr2=0x7f2e680fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:51:23.994 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.992+0000 7f2e6e0ae640 1 -- 172.21.15.99:0/2162836585 shutdown_connections 2024-03-19T01:51:23.994 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.992+0000 7f2e6e0ae640 1 -- 172.21.15.99:0/2162836585 wait complete. 2024-03-19T01:51:23.994 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.992+0000 7f2e6e0ae640 1 Processor -- start 2024-03-19T01:51:23.994 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.992+0000 7f2e6e0ae640 1 -- start start 2024-03-19T01:51:23.994 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.992+0000 7f2e6e0ae640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f2e68100740 0x7f2e68197a40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:23.994 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.992+0000 7f2e6e0ae640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f2e68101930 0x7f2e68197f80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:23.994 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.992+0000 7f2e6e0ae640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2e681022f0 0x7f2e6819eff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:23.994 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.992+0000 7f2e6e0ae640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f2e6810c3a0 con 0x7f2e681022f0 2024-03-19T01:51:23.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.992+0000 7f2e6e0ae640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f2e6810c220 con 0x7f2e68100740 2024-03-19T01:51:23.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.992+0000 7f2e6e0ae640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f2e6810c520 con 0x7f2e68101930 2024-03-19T01:51:23.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.993+0000 7f2e67fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2e681022f0 0x7f2e6819eff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:23.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.993+0000 7f2e677fe640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f2e68100740 0x7f2e68197a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:23.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.993+0000 7f2e67fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2e681022f0 0x7f2e6819eff0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:44372/0 (socket says 172.21.15.99:44372) 2024-03-19T01:51:23.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.993+0000 7f2e67fff640 1 -- 172.21.15.99:0/328392622 learned_addr learned my addr 172.21.15.99:0/328392622 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:51:23.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.993+0000 7f2e66ffd640 1 --2- 172.21.15.99:0/328392622 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f2e68101930 0x7f2e68197f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:23.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.993+0000 7f2e67fff640 1 -- 172.21.15.99:0/328392622 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f2e68101930 msgr2=0x7f2e68197f80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:51:23.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.993+0000 7f2e67fff640 1 --2- 172.21.15.99:0/328392622 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f2e68101930 0x7f2e68197f80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:23.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.993+0000 7f2e67fff640 1 -- 172.21.15.99:0/328392622 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f2e68100740 msgr2=0x7f2e68197a40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:51:23.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.993+0000 7f2e67fff640 1 --2- 172.21.15.99:0/328392622 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f2e68100740 0x7f2e68197a40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:23.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.993+0000 7f2e67fff640 1 -- 172.21.15.99:0/328392622 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f2e6819f660 con 0x7f2e681022f0 2024-03-19T01:51:23.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.993+0000 7f2e67fff640 1 --2- 172.21.15.99:0/328392622 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2e681022f0 0x7f2e6819eff0 secure :-1 s=READY pgs=541 cs=0 l=1 rev1=1 crypto rx=0x7f2e60037040 tx=0x7f2e60032f40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:51:23.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.993+0000 7f2e64ff9640 1 -- 172.21.15.99:0/328392622 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2e60041020 con 0x7f2e681022f0 2024-03-19T01:51:23.996 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.993+0000 7f2e6e0ae640 1 -- 172.21.15.99:0/328392622 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f2e6819f8f0 con 0x7f2e681022f0 2024-03-19T01:51:23.996 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.993+0000 7f2e6e0ae640 1 -- 172.21.15.99:0/328392622 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f2e6819fe30 con 0x7f2e681022f0 2024-03-19T01:51:23.996 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.993+0000 7f2e64ff9640 1 -- 172.21.15.99:0/328392622 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f2e60034920 con 0x7f2e681022f0 2024-03-19T01:51:23.996 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.993+0000 7f2e64ff9640 1 -- 172.21.15.99:0/328392622 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2e60039040 con 0x7f2e681022f0 2024-03-19T01:51:24.003 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.996+0000 7f2e64ff9640 1 -- 172.21.15.99:0/328392622 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f2e60044020 con 0x7f2e681022f0 2024-03-19T01:51:24.003 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.996+0000 7f2e70d70740 1 Processor -- start 2024-03-19T01:51:24.003 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.996+0000 7f2e70d70740 1 -- start start 2024-03-19T01:51:24.003 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.996+0000 7f2e70d70740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559b30896100 0x559b30896520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:24.003 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.996+0000 7f2e70d70740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559b30896af0 0x559b3089b5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:24.003 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.996+0000 7f2e70d70740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559b3089bc60 0x559b3089e0f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:24.003 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.996+0000 7f2e70d70740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x559b307fabe0 con 0x559b3089bc60 2024-03-19T01:51:24.003 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.996+0000 7f2e70d70740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x559b307a3960 con 0x559b30896100 2024-03-19T01:51:24.003 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.996+0000 7f2e70d70740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x559b307f6c80 con 0x559b30896af0 2024-03-19T01:51:24.004 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.996+0000 7f2e66ffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559b3089bc60 0x559b3089e0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:24.004 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.996+0000 7f2e66ffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559b3089bc60 0x559b3089e0f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:44374/0 (socket says 172.21.15.99:44374) 2024-03-19T01:51:24.004 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.996+0000 7f2e66ffd640 1 -- 172.21.15.99:0/3390831111 learned_addr learned my addr 172.21.15.99:0/3390831111 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:51:24.004 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.996+0000 7f2e677fe640 1 --2- 172.21.15.99:0/3390831111 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559b30896af0 0x559b3089b5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:24.004 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.996+0000 7f2e66ffd640 1 --2- 172.21.15.99:0/3390831111 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559b30896100 0x559b30896520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:24.004 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.996+0000 7f2e64ff9640 1 --2- 172.21.15.99:0/328392622 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f2e500a28a0 0x7f2e500a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:24.004 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.997+0000 7f2e66ffd640 1 -- 172.21.15.99:0/3390831111 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559b30896af0 msgr2=0x559b3089b5f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:51:24.004 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.997+0000 7f2e66ffd640 1 --2- 172.21.15.99:0/3390831111 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559b30896af0 0x559b3089b5f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:24.004 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.997+0000 7f2e66ffd640 1 -- 172.21.15.99:0/3390831111 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559b30896100 msgr2=0x559b30896520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:51:24.004 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.997+0000 7f2e66ffd640 1 --2- 172.21.15.99:0/3390831111 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559b30896100 0x559b30896520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:24.004 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.997+0000 7f2e66ffd640 1 -- 172.21.15.99:0/3390831111 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x559b3079ad20 con 0x559b3089bc60 2024-03-19T01:51:24.004 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.997+0000 7f2e67fff640 1 --2- 172.21.15.99:0/328392622 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f2e500a28a0 0x7f2e500a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:24.004 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.997+0000 7f2e66ffd640 1 --2- 172.21.15.99:0/3390831111 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559b3089bc60 0x559b3089e0f0 secure :-1 s=READY pgs=542 cs=0 l=1 rev1=1 crypto rx=0x7f2e5c009fd0 tx=0x7f2e5c005850 comp rx=0 tx=0).ready entity=mon.0 client_cookie=81a8681ca7f6fe47 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:51:24.005 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.997+0000 7f2e64ff9640 1 -- 172.21.15.99:0/328392622 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f2e600f0210 con 0x7f2e681022f0 2024-03-19T01:51:24.005 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.997+0000 7f2e4dffb640 1 -- 172.21.15.99:0/3390831111 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2e5c024070 con 0x559b3089bc60 2024-03-19T01:51:24.005 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.997+0000 7f2e4dffb640 1 -- 172.21.15.99:0/3390831111 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f2e5c00f790 con 0x559b3089bc60 2024-03-19T01:51:24.005 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.997+0000 7f2e4dffb640 1 -- 172.21.15.99:0/3390831111 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2e5c01f8c0 con 0x559b3089bc60 2024-03-19T01:51:24.005 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.998+0000 7f2e70d70740 1 -- 172.21.15.99:0/3390831111 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559b3089bc60 msgr2=0x559b3089e0f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:51:24.005 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.998+0000 7f2e67fff640 1 --2- 172.21.15.99:0/328392622 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f2e500a28a0 0x7f2e500a4d60 secure :-1 s=READY pgs=308 cs=0 l=1 rev1=1 crypto rx=0x7f2e600b95d0 tx=0x7f2e6002cf90 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:51:24.005 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.998+0000 7f2e70d70740 1 --2- 172.21.15.99:0/3390831111 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559b3089bc60 0x559b3089e0f0 secure :-1 s=READY pgs=542 cs=0 l=1 rev1=1 crypto rx=0x7f2e5c009fd0 tx=0x7f2e5c005850 comp rx=0 tx=0).stop 2024-03-19T01:51:24.005 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.998+0000 7f2e70d70740 1 -- 172.21.15.99:0/3390831111 shutdown_connections 2024-03-19T01:51:24.005 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.998+0000 7f2e70d70740 1 --2- 172.21.15.99:0/3390831111 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559b3089bc60 0x559b3089e0f0 unknown :-1 s=CLOSED pgs=542 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:24.005 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.998+0000 7f2e70d70740 1 --2- 172.21.15.99:0/3390831111 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559b30896af0 0x559b3089b5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:24.005 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.998+0000 7f2e70d70740 1 --2- 172.21.15.99:0/3390831111 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559b30896100 0x559b30896520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:24.005 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.998+0000 7f2e70d70740 1 -- 172.21.15.99:0/3390831111 >> 172.21.15.99:0/3390831111 conn(0x559b30891670 msgr2=0x559b30891a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:51:24.005 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.998+0000 7f2e70d70740 1 -- 172.21.15.99:0/3390831111 shutdown_connections 2024-03-19T01:51:24.005 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.998+0000 7f2e70d70740 1 -- 172.21.15.99:0/3390831111 wait complete. 2024-03-19T01:51:24.005 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.998+0000 7f2e70d70740 1 -- start start 2024-03-19T01:51:24.006 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.998+0000 7f2e70d70740 1 Processor -- start 2024-03-19T01:51:24.006 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.998+0000 7f2e70d70740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559b30896100 0x559b308a0140 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:24.006 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.999+0000 7f2e70d70740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559b30896af0 0x559b308a0680 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:24.006 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.999+0000 7f2e70d70740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559b3089bc60 0x559b308a76c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:24.006 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.999+0000 7f2e70d70740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x559b3089ec10 con 0x559b30896af0 2024-03-19T01:51:24.006 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.999+0000 7f2e70d70740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x559b307b6db0 con 0x559b3089bc60 2024-03-19T01:51:24.006 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.999+0000 7f2e70d70740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x559b3089ed90 con 0x559b30896100 2024-03-19T01:51:24.006 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.999+0000 7f2e70d70740 10 client.0 _ll_register_callbacks cb 0x559b307bbe70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:51:24.006 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.999+0000 7f2e677fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559b30896af0 0x559b308a0680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:24.006 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.999+0000 7f2e677fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559b30896af0 0x559b308a0680 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:44382/0 (socket says 172.21.15.99:44382) 2024-03-19T01:51:24.006 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.999+0000 7f2e677fe640 1 -- 172.21.15.99:0/3966102139 learned_addr learned my addr 172.21.15.99:0/3966102139 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:51:24.006 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.999+0000 7f2e66ffd640 1 --2- 172.21.15.99:0/3966102139 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559b30896100 0x559b308a0140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:24.007 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.999+0000 7f2e66ffd640 1 --2- 172.21.15.99:0/3966102139 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559b3089bc60 0x559b308a76c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:24.007 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.999+0000 7f2e677fe640 1 -- 172.21.15.99:0/3966102139 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559b30896100 msgr2=0x559b308a0140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:51:24.007 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.999+0000 7f2e677fe640 1 --2- 172.21.15.99:0/3966102139 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559b30896100 0x559b308a0140 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:24.007 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.999+0000 7f2e677fe640 1 -- 172.21.15.99:0/3966102139 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559b3089bc60 msgr2=0x559b308a76c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:51:24.007 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.999+0000 7f2e677fe640 1 --2- 172.21.15.99:0/3966102139 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559b3089bc60 0x559b308a76c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:24.007 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.999+0000 7f2e677fe640 1 -- 172.21.15.99:0/3966102139 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x559b307b5c60 con 0x559b30896af0 2024-03-19T01:51:24.007 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.999+0000 7f2e677fe640 1 --2- 172.21.15.99:0/3966102139 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559b30896af0 0x559b308a0680 secure :-1 s=READY pgs=543 cs=0 l=1 rev1=1 crypto rx=0x7f2e58012700 tx=0x7f2e58012bc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=13d4766073fdef63 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:51:24.007 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.999+0000 7f2e46ffd640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:51:24.007 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.999+0000 7f2e46ffd640 1 -- 172.21.15.99:0/3966102139 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2e58004050 con 0x559b30896af0 2024-03-19T01:51:24.007 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.999+0000 7f2e46ffd640 1 -- 172.21.15.99:0/3966102139 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f2e580041f0 con 0x559b30896af0 2024-03-19T01:51:24.007 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.999+0000 7f2e70d70740 1 -- 172.21.15.99:0/3966102139 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f2e5c00f790 con 0x559b30896af0 2024-03-19T01:51:24.007 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.999+0000 7f2e46ffd640 1 -- 172.21.15.99:0/3966102139 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7f2e580043f0 con 0x559b30896af0 2024-03-19T01:51:24.007 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.999+0000 7f2e70d70740 10 client.16170 fetch_fsmap learned FSMap version 27 2024-03-19T01:51:24.007 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:23.999+0000 7f2e70d70740 1 -- 172.21.15.99:0/3966102139 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x559b3050f210 con 0x559b30896af0 2024-03-19T01:51:24.008 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.000+0000 7f2e46ffd640 1 -- 172.21.15.99:0/3966102139 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f2e5c00f790 con 0x559b30896af0 2024-03-19T01:51:24.008 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.000+0000 7f2e46ffd640 1 -- 172.21.15.99:0/3966102139 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 27) ==== 8338+0+0 (secure 0 0 0) 0x7f2e58003bc0 con 0x559b30896af0 2024-03-19T01:51:24.008 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.000+0000 7f2e70d70740 10 client.16170 fetch_fsmap finished waiting for FSMap version 27 2024-03-19T01:51:24.008 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.000+0000 7f2e70d70740 10 client.16170 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:51:24.008 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.000+0000 7f2e70d70740 20 client.16170 populate_metadata read hostname 'smithi099' 2024-03-19T01:51:24.008 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.000+0000 7f2e70d70740 1 --2- 172.21.15.99:0/3966102139 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x559b308ac030 0x559b308ae490 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:24.008 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.000+0000 7f2e70d70740 4 client.16170 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:51:24.008 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.000+0000 7f2e70d70740 1 -- 172.21.15.99:0/3966102139 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x559b30808120 con 0x559b308ac030 2024-03-19T01:51:24.008 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.000+0000 7f2e66ffd640 1 --2- 172.21.15.99:0/3966102139 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x559b308ac030 0x559b308ae490 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:24.008 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.000+0000 7f2e66ffd640 1 --2- 172.21.15.99:0/3966102139 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x559b308ac030 0x559b308ae490 crc :-1 s=READY pgs=121 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=48a49be93b5e9d76 server_cookie=a8453c2991cf5a4a in_seq=0 out_seq=0 2024-03-19T01:51:24.008 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.000+0000 7f2e46ffd640 10 client.16170 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:51:24.008 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.001+0000 7f2e46ffd640 1 -- 172.21.15.99:0/3966102139 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x559b30808120 con 0x559b308ac030 2024-03-19T01:51:24.008 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.001+0000 7f2e46ffd640 10 client.16170 handle_command_reply: tid=0 2024-03-19T01:51:24.009 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.001+0000 7f2e70d70740 1 client.16170 shutdown 2024-03-19T01:51:24.009 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.001+0000 7f2e70d70740 1 -- 172.21.15.99:0/3966102139 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559b30896af0 msgr2=0x559b308a0680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:51:24.009 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.001+0000 7f2e70d70740 1 --2- 172.21.15.99:0/3966102139 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559b30896af0 0x559b308a0680 secure :-1 s=READY pgs=543 cs=0 l=1 rev1=1 crypto rx=0x7f2e58012700 tx=0x7f2e58012bc0 comp rx=0 tx=0).stop 2024-03-19T01:51:24.009 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.001+0000 7f2e70d70740 1 -- 172.21.15.99:0/3966102139 shutdown_connections 2024-03-19T01:51:24.009 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.001+0000 7f2e70d70740 1 --2- 172.21.15.99:0/3966102139 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x559b308ac030 0x559b308ae490 crc :-1 s=READY pgs=121 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:24.009 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.001+0000 7f2e70d70740 1 --2- 172.21.15.99:0/3966102139 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559b3089bc60 0x559b308a76c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:24.009 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.001+0000 7f2e70d70740 1 --2- 172.21.15.99:0/3966102139 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559b30896af0 0x559b308a0680 unknown :-1 s=CLOSED pgs=543 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:24.009 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.001+0000 7f2e46ffd640 0 client.16170 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:51:24.009 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.001+0000 7f2e70d70740 1 --2- 172.21.15.99:0/3966102139 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559b30896100 0x559b308a0140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:24.009 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.001+0000 7f2e70d70740 1 -- 172.21.15.99:0/3966102139 >> 172.21.15.99:0/3966102139 conn(0x559b30891670 msgr2=0x559b308928c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:51:24.009 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.002+0000 7f2e70d70740 1 -- 172.21.15.99:0/3966102139 shutdown_connections 2024-03-19T01:51:24.009 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.002+0000 7f2e70d70740 1 -- 172.21.15.99:0/3966102139 wait complete. 2024-03-19T01:51:24.009 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.002+0000 7f2e70d70740 20 client.16170 trim_cache size 0 max 16384 2024-03-19T01:51:24.023 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.017+0000 7f2e70d70740 1 Processor -- start 2024-03-19T01:51:24.023 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.017+0000 7f2e70d70740 1 -- start start 2024-03-19T01:51:24.023 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.018+0000 7f2e70d70740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559b30896100 0x559b30894fe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:24.023 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.018+0000 7f2e70d70740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559b30896af0 0x559b308a7680 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:24.023 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.018+0000 7f2e70d70740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559b3089bc60 0x559b308b2d90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:24.023 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.018+0000 7f2e70d70740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f2e54010e70 con 0x559b3089bc60 2024-03-19T01:51:24.023 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.018+0000 7f2e70d70740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f2e5400e740 con 0x559b30896100 2024-03-19T01:51:24.023 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.018+0000 7f2e70d70740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f2e5400c590 con 0x559b30896af0 2024-03-19T01:51:24.023 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.018+0000 7f2e66ffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559b3089bc60 0x559b308b2d90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:24.024 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.018+0000 7f2e66ffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559b3089bc60 0x559b308b2d90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:44386/0 (socket says 172.21.15.99:44386) 2024-03-19T01:51:24.024 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.018+0000 7f2e66ffd640 1 -- 172.21.15.99:0/1090535801 learned_addr learned my addr 172.21.15.99:0/1090535801 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:51:24.024 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.018+0000 7f2e66ffd640 1 --2- 172.21.15.99:0/1090535801 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559b30896100 0x559b30894fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:24.024 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.018+0000 7f2e677fe640 1 --2- 172.21.15.99:0/1090535801 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559b30896af0 0x559b308a7680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:24.024 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.018+0000 7f2e66ffd640 1 -- 172.21.15.99:0/1090535801 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559b30896af0 msgr2=0x559b308a7680 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:51:24.024 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.018+0000 7f2e66ffd640 1 --2- 172.21.15.99:0/1090535801 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559b30896af0 0x559b308a7680 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:24.024 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.018+0000 7f2e66ffd640 1 -- 172.21.15.99:0/1090535801 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559b30896100 msgr2=0x559b30894fe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:51:24.024 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.018+0000 7f2e66ffd640 1 --2- 172.21.15.99:0/1090535801 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559b30896100 0x559b30894fe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:24.024 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.018+0000 7f2e66ffd640 1 -- 172.21.15.99:0/1090535801 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x559b305db620 con 0x559b3089bc60 2024-03-19T01:51:24.024 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.018+0000 7f2e66ffd640 1 --2- 172.21.15.99:0/1090535801 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559b3089bc60 0x559b308b2d90 secure :-1 s=READY pgs=544 cs=0 l=1 rev1=1 crypto rx=0x7f2e5c010010 tx=0x7f2e5c005850 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d476d5cd39c56054 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:51:24.024 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.018+0000 7f2e4d7fa640 1 -- 172.21.15.99:0/1090535801 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2e5c024040 con 0x559b3089bc60 2024-03-19T01:51:24.024 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.018+0000 7f2e4d7fa640 1 -- 172.21.15.99:0/1090535801 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f2e5c005600 con 0x559b3089bc60 2024-03-19T01:51:24.024 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.018+0000 7f2e4d7fa640 1 -- 172.21.15.99:0/1090535801 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2e5c033440 con 0x559b3089bc60 2024-03-19T01:51:24.024 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.019+0000 7f2e70d70740 1 -- 172.21.15.99:0/1090535801 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559b3089bc60 msgr2=0x559b308b2d90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:51:24.025 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.019+0000 7f2e70d70740 1 --2- 172.21.15.99:0/1090535801 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559b3089bc60 0x559b308b2d90 secure :-1 s=READY pgs=544 cs=0 l=1 rev1=1 crypto rx=0x7f2e5c010010 tx=0x7f2e5c005850 comp rx=0 tx=0).stop 2024-03-19T01:51:24.025 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.019+0000 7f2e70d70740 1 -- 172.21.15.99:0/1090535801 shutdown_connections 2024-03-19T01:51:24.025 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.019+0000 7f2e70d70740 1 --2- 172.21.15.99:0/1090535801 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559b3089bc60 0x559b308b2d90 unknown :-1 s=CLOSED pgs=544 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:24.025 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.019+0000 7f2e70d70740 1 --2- 172.21.15.99:0/1090535801 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559b30896af0 0x559b308a7680 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:24.025 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.019+0000 7f2e70d70740 1 --2- 172.21.15.99:0/1090535801 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559b30896100 0x559b30894fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:24.025 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.019+0000 7f2e70d70740 1 -- 172.21.15.99:0/1090535801 >> 172.21.15.99:0/1090535801 conn(0x559b30891670 msgr2=0x559b308948d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:51:24.025 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.019+0000 7f2e70d70740 1 -- 172.21.15.99:0/1090535801 shutdown_connections 2024-03-19T01:51:24.025 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.019+0000 7f2e70d70740 1 -- 172.21.15.99:0/1090535801 wait complete. 2024-03-19T01:51:24.025 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.019+0000 7f2e70d70740 1 -- start start 2024-03-19T01:51:24.025 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.020+0000 7f2e70d70740 1 Processor -- start 2024-03-19T01:51:24.025 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.020+0000 7f2e70d70740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559b30896100 0x559b308a5460 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:24.025 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.020+0000 7f2e70d70740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559b30896af0 0x559b308a59a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:24.025 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.020+0000 7f2e70d70740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559b3089bc60 0x559b308a69b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:24.025 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.020+0000 7f2e70d70740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f2e5400cc00 con 0x559b3089bc60 2024-03-19T01:51:24.026 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.020+0000 7f2e70d70740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f2e5400e070 con 0x559b30896af0 2024-03-19T01:51:24.026 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.020+0000 7f2e70d70740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f2e5400d2d0 con 0x559b30896100 2024-03-19T01:51:24.026 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.020+0000 7f2e70d70740 10 client.0 _ll_register_callbacks cb 0x7f2e58009930 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:51:24.026 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.020+0000 7f2e66ffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559b3089bc60 0x559b308a69b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:24.026 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.020+0000 7f2e677fe640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559b30896af0 0x559b308a59a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:24.026 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.020+0000 7f2e66ffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559b3089bc60 0x559b308a69b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:44396/0 (socket says 172.21.15.99:44396) 2024-03-19T01:51:24.026 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.020+0000 7f2e66ffd640 1 -- 172.21.15.99:0/1602434495 learned_addr learned my addr 172.21.15.99:0/1602434495 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:51:24.026 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.020+0000 7f2e66ffd640 1 --2- 172.21.15.99:0/1602434495 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559b30896100 0x559b308a5460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:24.026 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.020+0000 7f2e66ffd640 1 -- 172.21.15.99:0/1602434495 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559b30896100 msgr2=0x559b308a5460 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:51:24.026 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.020+0000 7f2e66ffd640 1 --2- 172.21.15.99:0/1602434495 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559b30896100 0x559b308a5460 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:24.026 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.020+0000 7f2e66ffd640 1 -- 172.21.15.99:0/1602434495 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559b30896af0 msgr2=0x559b308a59a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:51:24.026 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.020+0000 7f2e66ffd640 1 --2- 172.21.15.99:0/1602434495 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559b30896af0 0x559b308a59a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:24.026 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.020+0000 7f2e66ffd640 1 -- 172.21.15.99:0/1602434495 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x559b308a6fa0 con 0x559b3089bc60 2024-03-19T01:51:24.026 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.020+0000 7f2e677fe640 1 --2- 172.21.15.99:0/1602434495 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559b30896af0 0x559b308a59a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:51:24.027 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.021+0000 7f2e66ffd640 1 --2- 172.21.15.99:0/1602434495 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559b3089bc60 0x559b308a69b0 secure :-1 s=READY pgs=545 cs=0 l=1 rev1=1 crypto rx=0x7f2e5c00f990 tx=0x7f2e5c0115f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2a7e9b4d92123bca server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:51:24.027 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.021+0000 7f2e46ffd640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:51:24.027 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.021+0000 7f2e46ffd640 1 -- 172.21.15.99:0/1602434495 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2e5c024070 con 0x559b3089bc60 2024-03-19T01:51:24.027 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.021+0000 7f2e46ffd640 1 -- 172.21.15.99:0/1602434495 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f2e5c0050d0 con 0x559b3089bc60 2024-03-19T01:51:24.027 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.021+0000 7f2e70d70740 1 -- 172.21.15.99:0/1602434495 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f2e5c005600 con 0x559b3089bc60 2024-03-19T01:51:24.027 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.021+0000 7f2e46ffd640 1 -- 172.21.15.99:0/1602434495 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7f2e5c0052d0 con 0x559b3089bc60 2024-03-19T01:51:24.027 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.021+0000 7f2e70d70740 10 client.16176 fetch_fsmap learned FSMap version 27 2024-03-19T01:51:24.027 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.021+0000 7f2e70d70740 1 -- 172.21.15.99:0/1602434495 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x559b30912a20 con 0x559b3089bc60 2024-03-19T01:51:24.027 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.021+0000 7f2e46ffd640 1 -- 172.21.15.99:0/1602434495 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f2e5c005600 con 0x559b3089bc60 2024-03-19T01:51:24.027 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.021+0000 7f2e46ffd640 1 -- 172.21.15.99:0/1602434495 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 27) ==== 8338+0+0 (secure 0 0 0) 0x7f2e5c01fad0 con 0x559b3089bc60 2024-03-19T01:51:24.027 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.021+0000 7f2e70d70740 10 client.16176 fetch_fsmap finished waiting for FSMap version 27 2024-03-19T01:51:24.027 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.021+0000 7f2e70d70740 10 client.16176 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:51:24.027 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.021+0000 7f2e70d70740 20 client.16176 populate_metadata read hostname 'smithi099' 2024-03-19T01:51:24.027 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.021+0000 7f2e70d70740 1 --2- 172.21.15.99:0/1602434495 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x559b30913190 0x559b309155b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:24.028 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.021+0000 7f2e70d70740 4 client.16176 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:51:24.028 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.021+0000 7f2e70d70740 1 -- 172.21.15.99:0/1602434495 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x559b30802470 con 0x559b30913190 2024-03-19T01:51:24.028 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.021+0000 7f2e677fe640 1 --2- 172.21.15.99:0/1602434495 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x559b30913190 0x559b309155b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:24.028 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.022+0000 7f2e46ffd640 10 client.16176 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:51:24.028 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.022+0000 7f2e677fe640 1 --2- 172.21.15.99:0/1602434495 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x559b30913190 0x559b309155b0 crc :-1 s=READY pgs=122 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=f3e2368fc15341ed server_cookie=7aca77e719e3e59e in_seq=0 out_seq=0 2024-03-19T01:51:24.095 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.094+0000 7f2e46ffd640 1 -- 172.21.15.99:0/1602434495 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x559b30802470 con 0x559b30913190 2024-03-19T01:51:24.095 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.094+0000 7f2e46ffd640 10 client.16176 handle_command_reply: tid=0 2024-03-19T01:51:24.096 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.094+0000 7f2e70d70740 1 client.16176 shutdown 2024-03-19T01:51:24.096 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.094+0000 7f2e70d70740 1 -- 172.21.15.99:0/1602434495 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559b3089bc60 msgr2=0x559b308a69b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:51:24.096 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.094+0000 7f2e70d70740 1 --2- 172.21.15.99:0/1602434495 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559b3089bc60 0x559b308a69b0 secure :-1 s=READY pgs=545 cs=0 l=1 rev1=1 crypto rx=0x7f2e5c00f990 tx=0x7f2e5c0115f0 comp rx=0 tx=0).stop 2024-03-19T01:51:24.096 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.094+0000 7f2e70d70740 1 -- 172.21.15.99:0/1602434495 shutdown_connections 2024-03-19T01:51:24.096 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.094+0000 7f2e70d70740 1 --2- 172.21.15.99:0/1602434495 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x559b30913190 0x559b309155b0 crc :-1 s=READY pgs=122 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:24.096 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.094+0000 7f2e70d70740 1 --2- 172.21.15.99:0/1602434495 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x559b30896af0 0x559b308a59a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:24.096 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.094+0000 7f2e46ffd640 0 client.16176 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:51:24.096 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.094+0000 7f2e70d70740 1 --2- 172.21.15.99:0/1602434495 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x559b3089bc60 0x559b308a69b0 unknown :-1 s=CLOSED pgs=545 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:24.096 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.094+0000 7f2e70d70740 1 --2- 172.21.15.99:0/1602434495 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x559b30896100 0x559b308a5460 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:24.096 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.094+0000 7f2e70d70740 1 -- 172.21.15.99:0/1602434495 >> 172.21.15.99:0/1602434495 conn(0x559b30891670 msgr2=0x559b3089f430 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:51:24.096 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.095+0000 7f2e70d70740 1 -- 172.21.15.99:0/1602434495 shutdown_connections 2024-03-19T01:51:24.096 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.095+0000 7f2e70d70740 1 -- 172.21.15.99:0/1602434495 wait complete. 2024-03-19T01:51:24.096 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.095+0000 7f2e70d70740 20 client.16176 trim_cache size 0 max 16384 2024-03-19T01:51:24.097 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:51:24.097 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:51:24.097 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:51:24.097 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:51:24.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.095+0000 7f2e6e0ae640 1 -- 172.21.15.99:0/328392622 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f2e500a28a0 msgr2=0x7f2e500a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:51:24.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.095+0000 7f2e6e0ae640 1 --2- 172.21.15.99:0/328392622 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f2e500a28a0 0x7f2e500a4d60 secure :-1 s=READY pgs=308 cs=0 l=1 rev1=1 crypto rx=0x7f2e600b95d0 tx=0x7f2e6002cf90 comp rx=0 tx=0).stop 2024-03-19T01:51:24.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.095+0000 7f2e6e0ae640 1 -- 172.21.15.99:0/328392622 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2e681022f0 msgr2=0x7f2e6819eff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:51:24.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.095+0000 7f2e6e0ae640 1 --2- 172.21.15.99:0/328392622 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2e681022f0 0x7f2e6819eff0 secure :-1 s=READY pgs=541 cs=0 l=1 rev1=1 crypto rx=0x7f2e60037040 tx=0x7f2e60032f40 comp rx=0 tx=0).stop 2024-03-19T01:51:24.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.095+0000 7f2e6e0ae640 1 -- 172.21.15.99:0/328392622 shutdown_connections 2024-03-19T01:51:24.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.095+0000 7f2e6e0ae640 1 --2- 172.21.15.99:0/328392622 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2e681022f0 0x7f2e6819eff0 unknown :-1 s=CLOSED pgs=541 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:24.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.095+0000 7f2e6e0ae640 1 --2- 172.21.15.99:0/328392622 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f2e68101930 0x7f2e68197f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:24.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.095+0000 7f2e6e0ae640 1 --2- 172.21.15.99:0/328392622 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f2e500a28a0 0x7f2e500a4d60 unknown :-1 s=CLOSED pgs=308 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:24.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.095+0000 7f2e6e0ae640 1 --2- 172.21.15.99:0/328392622 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f2e68100740 0x7f2e68197a40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:24.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.095+0000 7f2e6e0ae640 1 -- 172.21.15.99:0/328392622 >> 172.21.15.99:0/328392622 conn(0x7f2e680fbcb0 msgr2=0x7f2e680fd670 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:51:24.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.095+0000 7f2e6e0ae640 1 -- 172.21.15.99:0/328392622 shutdown_connections 2024-03-19T01:51:24.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:24.095+0000 7f2e6e0ae640 1 -- 172.21.15.99:0/328392622 wait complete. 2024-03-19T01:51:24.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:51:23 smithi115 ceph-mon[67332]: pgmap v659: 97 pgs: 97 active+clean; 253 MiB data, 3.2 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 3.2 MiB/s wr, 296 op/s 2024-03-19T01:51:24.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:51:23 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:51:24.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:51:23 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:51:24.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:51:23 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:51:24.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:51:23 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:51:24.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:51:23 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:51:24.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:51:23 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:51:24.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:51:23 smithi099 ceph-mon[62951]: pgmap v659: 97 pgs: 97 active+clean; 253 MiB data, 3.2 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 3.2 MiB/s wr, 296 op/s 2024-03-19T01:51:24.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:51:23 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:51:24.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:51:23 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:51:24.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:51:23 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:51:24.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:51:23 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:51:24.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:51:23 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:51:24.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:51:23 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:51:24.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:51:23 smithi112 ceph-mon[66819]: pgmap v659: 97 pgs: 97 active+clean; 253 MiB data, 3.2 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 3.2 MiB/s wr, 296 op/s 2024-03-19T01:51:24.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:51:23 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:51:24.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:51:23 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:51:24.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:51:23 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:51:24.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:51:23 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:51:24.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:51:23 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:51:24.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:51:23 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:51:24.700 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1253982 11.33 MB/sec execute 474 sec latency 23.493 ms 2024-03-19T01:51:25.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:51:24 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:51:25.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:51:24 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:51:25.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:51:24 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:51:25.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:51:24 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:51:25.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:51:24 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:51:25.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:51:24 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:51:25.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:51:24 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:51:25.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:51:24 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:51:25.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:51:24 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:51:25.700 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1255633 11.32 MB/sec execute 475 sec latency 101.294 ms 2024-03-19T01:51:26.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:51:25 smithi115 ceph-mon[67332]: pgmap v660: 97 pgs: 97 active+clean; 254 MiB data, 3.2 GiB used, 1.0 TiB / 1.0 TiB avail; 732 KiB/s rd, 2.6 MiB/s wr, 211 op/s 2024-03-19T01:51:26.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:51:25 smithi099 ceph-mon[62951]: pgmap v660: 97 pgs: 97 active+clean; 254 MiB data, 3.2 GiB used, 1.0 TiB / 1.0 TiB avail; 732 KiB/s rd, 2.6 MiB/s wr, 211 op/s 2024-03-19T01:51:26.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:51:25 smithi112 ceph-mon[66819]: pgmap v660: 97 pgs: 97 active+clean; 254 MiB data, 3.2 GiB used, 1.0 TiB / 1.0 TiB avail; 732 KiB/s rd, 2.6 MiB/s wr, 211 op/s 2024-03-19T01:51:26.700 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1257767 11.32 MB/sec execute 476 sec latency 43.847 ms 2024-03-19T01:51:27.700 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1260180 11.33 MB/sec execute 477 sec latency 35.762 ms 2024-03-19T01:51:28.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:51:27 smithi115 ceph-mon[67332]: pgmap v661: 97 pgs: 97 active+clean; 258 MiB data, 3.2 GiB used, 1.0 TiB / 1.0 TiB avail; 928 KiB/s rd, 3.2 MiB/s wr, 259 op/s 2024-03-19T01:51:28.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:51:27 smithi099 ceph-mon[62951]: pgmap v661: 97 pgs: 97 active+clean; 258 MiB data, 3.2 GiB used, 1.0 TiB / 1.0 TiB avail; 928 KiB/s rd, 3.2 MiB/s wr, 259 op/s 2024-03-19T01:51:28.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:51:27 smithi112 ceph-mon[66819]: pgmap v661: 97 pgs: 97 active+clean; 258 MiB data, 3.2 GiB used, 1.0 TiB / 1.0 TiB avail; 928 KiB/s rd, 3.2 MiB/s wr, 259 op/s 2024-03-19T01:51:28.700 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1261943 11.32 MB/sec execute 478 sec latency 40.475 ms 2024-03-19T01:51:29.700 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1264584 11.33 MB/sec execute 479 sec latency 17.736 ms 2024-03-19T01:51:30.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:51:29 smithi115 ceph-mon[67332]: pgmap v662: 97 pgs: 97 active+clean; 259 MiB data, 3.3 GiB used, 1.0 TiB / 1.0 TiB avail; 733 KiB/s rd, 2.7 MiB/s wr, 223 op/s 2024-03-19T01:51:30.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:51:29 smithi099 ceph-mon[62951]: pgmap v662: 97 pgs: 97 active+clean; 259 MiB data, 3.3 GiB used, 1.0 TiB / 1.0 TiB avail; 733 KiB/s rd, 2.7 MiB/s wr, 223 op/s 2024-03-19T01:51:30.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:51:29 smithi112 ceph-mon[66819]: pgmap v662: 97 pgs: 97 active+clean; 259 MiB data, 3.3 GiB used, 1.0 TiB / 1.0 TiB avail; 733 KiB/s rd, 2.7 MiB/s wr, 223 op/s 2024-03-19T01:51:30.700 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1266124 11.32 MB/sec execute 480 sec latency 43.217 ms 2024-03-19T01:51:31.700 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1268385 11.32 MB/sec execute 481 sec latency 42.541 ms 2024-03-19T01:51:32.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:51:31 smithi115 ceph-mon[67332]: pgmap v663: 97 pgs: 97 active+clean; 259 MiB data, 3.3 GiB used, 1.0 TiB / 1.0 TiB avail; 728 KiB/s rd, 2.7 MiB/s wr, 219 op/s 2024-03-19T01:51:32.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:51:31 smithi099 ceph-mon[62951]: pgmap v663: 97 pgs: 97 active+clean; 259 MiB data, 3.3 GiB used, 1.0 TiB / 1.0 TiB avail; 728 KiB/s rd, 2.7 MiB/s wr, 219 op/s 2024-03-19T01:51:32.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:51:31 smithi112 ceph-mon[66819]: pgmap v663: 97 pgs: 97 active+clean; 259 MiB data, 3.3 GiB used, 1.0 TiB / 1.0 TiB avail; 728 KiB/s rd, 2.7 MiB/s wr, 219 op/s 2024-03-19T01:51:32.700 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1270613 11.32 MB/sec execute 482 sec latency 16.003 ms 2024-03-19T01:51:33.700 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1272367 11.32 MB/sec execute 483 sec latency 35.022 ms 2024-03-19T01:51:34.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:51:33 smithi115 ceph-mon[67332]: pgmap v664: 97 pgs: 97 active+clean; 262 MiB data, 3.3 GiB used, 1.0 TiB / 1.0 TiB avail; 1.0 MiB/s rd, 2.9 MiB/s wr, 282 op/s 2024-03-19T01:51:34.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:51:33 smithi099 ceph-mon[62951]: pgmap v664: 97 pgs: 97 active+clean; 262 MiB data, 3.3 GiB used, 1.0 TiB / 1.0 TiB avail; 1.0 MiB/s rd, 2.9 MiB/s wr, 282 op/s 2024-03-19T01:51:34.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:51:33 smithi112 ceph-mon[66819]: pgmap v664: 97 pgs: 97 active+clean; 262 MiB data, 3.3 GiB used, 1.0 TiB / 1.0 TiB avail; 1.0 MiB/s rd, 2.9 MiB/s wr, 282 op/s 2024-03-19T01:51:34.700 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1274898 11.33 MB/sec execute 484 sec latency 26.983 ms 2024-03-19T01:51:35.700 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1276322 11.32 MB/sec execute 485 sec latency 60.135 ms 2024-03-19T01:51:36.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:51:35 smithi115 ceph-mon[67332]: pgmap v665: 97 pgs: 97 active+clean; 262 MiB data, 3.3 GiB used, 1.0 TiB / 1.0 TiB avail; 837 KiB/s rd, 2.5 MiB/s wr, 245 op/s 2024-03-19T01:51:36.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:51:35 smithi099 ceph-mon[62951]: pgmap v665: 97 pgs: 97 active+clean; 262 MiB data, 3.3 GiB used, 1.0 TiB / 1.0 TiB avail; 837 KiB/s rd, 2.5 MiB/s wr, 245 op/s 2024-03-19T01:51:36.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:51:35 smithi112 ceph-mon[66819]: pgmap v665: 97 pgs: 97 active+clean; 262 MiB data, 3.3 GiB used, 1.0 TiB / 1.0 TiB avail; 837 KiB/s rd, 2.5 MiB/s wr, 245 op/s 2024-03-19T01:51:36.700 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1278707 11.32 MB/sec execute 486 sec latency 41.463 ms 2024-03-19T01:51:37.192 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.190+0000 7f4f560f1640 1 -- 172.21.15.99:0/3832772837 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4f50101930 msgr2=0x7f4f50101db0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:51:37.192 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.190+0000 7f4f560f1640 1 --2- 172.21.15.99:0/3832772837 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4f50101930 0x7f4f50101db0 secure :-1 s=READY pgs=546 cs=0 l=1 rev1=1 crypto rx=0x7f4f38009900 tx=0x7f4f3802bc30 comp rx=0 tx=0).stop 2024-03-19T01:51:37.192 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.190+0000 7f4f560f1640 1 -- 172.21.15.99:0/3832772837 shutdown_connections 2024-03-19T01:51:37.192 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.190+0000 7f4f560f1640 1 --2- 172.21.15.99:0/3832772837 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4f501022f0 0x7f4f50108b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:37.192 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.190+0000 7f4f560f1640 1 --2- 172.21.15.99:0/3832772837 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4f50101930 0x7f4f50101db0 unknown :-1 s=CLOSED pgs=546 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:37.193 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.190+0000 7f4f560f1640 1 --2- 172.21.15.99:0/3832772837 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4f50100740 0x7f4f50100b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:37.193 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.190+0000 7f4f560f1640 1 -- 172.21.15.99:0/3832772837 >> 172.21.15.99:0/3832772837 conn(0x7f4f500fbcb0 msgr2=0x7f4f500fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:51:37.193 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.190+0000 7f4f560f1640 1 -- 172.21.15.99:0/3832772837 shutdown_connections 2024-03-19T01:51:37.193 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.190+0000 7f4f560f1640 1 -- 172.21.15.99:0/3832772837 wait complete. 2024-03-19T01:51:37.193 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.190+0000 7f4f560f1640 1 Processor -- start 2024-03-19T01:51:37.193 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.190+0000 7f4f560f1640 1 -- start start 2024-03-19T01:51:37.193 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.191+0000 7f4f560f1640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4f50100740 0x7f4f50197ac0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:37.193 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.191+0000 7f4f560f1640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4f50101930 0x7f4f50198000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:37.193 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.191+0000 7f4f560f1640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4f501022f0 0x7f4f5019f070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:37.194 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.191+0000 7f4f560f1640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f4f5010c3c0 con 0x7f4f50101930 2024-03-19T01:51:37.194 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.191+0000 7f4f560f1640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f4f5010c240 con 0x7f4f50100740 2024-03-19T01:51:37.194 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.191+0000 7f4f560f1640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f4f5010c540 con 0x7f4f501022f0 2024-03-19T01:51:37.194 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.191+0000 7f4f47fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4f50101930 0x7f4f50198000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:37.194 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.191+0000 7f4f47fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4f50101930 0x7f4f50198000 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:43272/0 (socket says 172.21.15.99:43272) 2024-03-19T01:51:37.194 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.191+0000 7f4f4f7fe640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4f50100740 0x7f4f50197ac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:37.194 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.191+0000 7f4f47fff640 1 -- 172.21.15.99:0/2913950611 learned_addr learned my addr 172.21.15.99:0/2913950611 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:51:37.194 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.191+0000 7f4f4ffff640 1 --2- 172.21.15.99:0/2913950611 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4f501022f0 0x7f4f5019f070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:37.194 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.191+0000 7f4f47fff640 1 -- 172.21.15.99:0/2913950611 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4f501022f0 msgr2=0x7f4f5019f070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:51:37.194 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.191+0000 7f4f47fff640 1 --2- 172.21.15.99:0/2913950611 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4f501022f0 0x7f4f5019f070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:37.194 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.191+0000 7f4f47fff640 1 -- 172.21.15.99:0/2913950611 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4f50100740 msgr2=0x7f4f50197ac0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:51:37.195 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.191+0000 7f4f47fff640 1 --2- 172.21.15.99:0/2913950611 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4f50100740 0x7f4f50197ac0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:37.195 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.191+0000 7f4f47fff640 1 -- 172.21.15.99:0/2913950611 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f4f5019f6e0 con 0x7f4f50101930 2024-03-19T01:51:37.195 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.191+0000 7f4f4f7fe640 1 --2- 172.21.15.99:0/2913950611 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4f50100740 0x7f4f50197ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:51:37.195 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.191+0000 7f4f47fff640 1 --2- 172.21.15.99:0/2913950611 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4f50101930 0x7f4f50198000 secure :-1 s=READY pgs=547 cs=0 l=1 rev1=1 crypto rx=0x7f4f38009230 tx=0x7f4f3802c650 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:51:37.195 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.191+0000 7f4f4d7fa640 1 -- 172.21.15.99:0/2913950611 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4f38041070 con 0x7f4f50101930 2024-03-19T01:51:37.195 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.192+0000 7f4f4d7fa640 1 -- 172.21.15.99:0/2913950611 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f4f38004450 con 0x7f4f50101930 2024-03-19T01:51:37.195 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.192+0000 7f4f560f1640 1 -- 172.21.15.99:0/2913950611 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f4f5019f970 con 0x7f4f50101930 2024-03-19T01:51:37.195 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.192+0000 7f4f4d7fa640 1 -- 172.21.15.99:0/2913950611 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4f38003e20 con 0x7f4f50101930 2024-03-19T01:51:37.195 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.192+0000 7f4f560f1640 1 -- 172.21.15.99:0/2913950611 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f4f5019feb0 con 0x7f4f50101930 2024-03-19T01:51:37.202 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.194+0000 7f4f4d7fa640 1 -- 172.21.15.99:0/2913950611 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f4f38044070 con 0x7f4f50101930 2024-03-19T01:51:37.202 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.195+0000 7f4f58db0740 1 Processor -- start 2024-03-19T01:51:37.203 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.195+0000 7f4f58db0740 1 -- start start 2024-03-19T01:51:37.203 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.195+0000 7f4f58db0740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5628a88e7510 0x5628a88e7930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:37.203 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.195+0000 7f4f58db0740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5628a88e7f00 0x5628a88eca00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:37.203 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.195+0000 7f4f58db0740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5628a88ed070 0x5628a88ef500 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:37.203 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.195+0000 7f4f58db0740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x5628a8847be0 con 0x5628a88e7f00 2024-03-19T01:51:37.203 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.195+0000 7f4f58db0740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x5628a87f0960 con 0x5628a88ed070 2024-03-19T01:51:37.203 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.195+0000 7f4f58db0740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x5628a8843c80 con 0x5628a88e7510 2024-03-19T01:51:37.203 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.195+0000 7f4f4d7fa640 1 --2- 172.21.15.99:0/2913950611 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f4f3c0a2920 0x7f4f3c0a4de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:37.203 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.195+0000 7f4f4f7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5628a88e7f00 0x5628a88eca00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:37.203 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.195+0000 7f4f4f7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5628a88e7f00 0x5628a88eca00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:43280/0 (socket says 172.21.15.99:43280) 2024-03-19T01:51:37.203 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.195+0000 7f4f47fff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5628a88ed070 0x5628a88ef500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:37.204 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.195+0000 7f4f4ffff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5628a88e7510 0x5628a88e7930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:37.204 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.195+0000 7f4f4f7fe640 1 -- 172.21.15.99:0/4188207892 learned_addr learned my addr 172.21.15.99:0/4188207892 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:51:37.204 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.195+0000 7f4f4ffff640 1 --2- 172.21.15.99:0/2913950611 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f4f3c0a2920 0x7f4f3c0a4de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:37.204 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.195+0000 7f4f4f7fe640 1 -- 172.21.15.99:0/4188207892 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5628a88e7510 msgr2=0x5628a88e7930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:51:37.204 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.195+0000 7f4f4f7fe640 1 --2- 172.21.15.99:0/4188207892 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5628a88e7510 0x5628a88e7930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:37.204 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.196+0000 7f4f4f7fe640 1 -- 172.21.15.99:0/4188207892 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5628a88ed070 msgr2=0x5628a88ef500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:51:37.204 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.196+0000 7f4f4f7fe640 1 --2- 172.21.15.99:0/4188207892 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5628a88ed070 0x5628a88ef500 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:37.204 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.196+0000 7f4f4f7fe640 1 -- 172.21.15.99:0/4188207892 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5628a87e7d20 con 0x5628a88e7f00 2024-03-19T01:51:37.204 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.196+0000 7f4f4d7fa640 1 -- 172.21.15.99:0/2913950611 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f4f380f05b0 con 0x7f4f50101930 2024-03-19T01:51:37.204 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.196+0000 7f4f4f7fe640 1 --2- 172.21.15.99:0/4188207892 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5628a88e7f00 0x5628a88eca00 secure :-1 s=READY pgs=548 cs=0 l=1 rev1=1 crypto rx=0x7f4f400114f0 tx=0x7f4f400118f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=989f50faa48041a7 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:51:37.204 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.196+0000 7f4f4ffff640 1 --2- 172.21.15.99:0/2913950611 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f4f3c0a2920 0x7f4f3c0a4de0 secure :-1 s=READY pgs=309 cs=0 l=1 rev1=1 crypto rx=0x5628a88e8760 tx=0x7f4f48009500 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:51:37.204 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.196+0000 7f4f45ffb640 1 -- 172.21.15.99:0/4188207892 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4f4001e070 con 0x5628a88e7f00 2024-03-19T01:51:37.204 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.196+0000 7f4f45ffb640 1 -- 172.21.15.99:0/4188207892 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f4f4000f9c0 con 0x5628a88e7f00 2024-03-19T01:51:37.205 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.196+0000 7f4f45ffb640 1 -- 172.21.15.99:0/4188207892 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4f40019b90 con 0x5628a88e7f00 2024-03-19T01:51:37.205 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.197+0000 7f4f58db0740 1 -- 172.21.15.99:0/4188207892 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5628a88e7f00 msgr2=0x5628a88eca00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:51:37.205 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.197+0000 7f4f58db0740 1 --2- 172.21.15.99:0/4188207892 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5628a88e7f00 0x5628a88eca00 secure :-1 s=READY pgs=548 cs=0 l=1 rev1=1 crypto rx=0x7f4f400114f0 tx=0x7f4f400118f0 comp rx=0 tx=0).stop 2024-03-19T01:51:37.205 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.197+0000 7f4f58db0740 1 -- 172.21.15.99:0/4188207892 shutdown_connections 2024-03-19T01:51:37.205 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.197+0000 7f4f58db0740 1 --2- 172.21.15.99:0/4188207892 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5628a88ed070 0x5628a88ef500 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:37.205 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.197+0000 7f4f58db0740 1 --2- 172.21.15.99:0/4188207892 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5628a88e7f00 0x5628a88eca00 unknown :-1 s=CLOSED pgs=548 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:37.205 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.197+0000 7f4f58db0740 1 --2- 172.21.15.99:0/4188207892 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5628a88e7510 0x5628a88e7930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:37.205 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.197+0000 7f4f58db0740 1 -- 172.21.15.99:0/4188207892 >> 172.21.15.99:0/4188207892 conn(0x5628a88de670 msgr2=0x5628a88dea70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:51:37.205 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.197+0000 7f4f58db0740 1 -- 172.21.15.99:0/4188207892 shutdown_connections 2024-03-19T01:51:37.205 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.197+0000 7f4f58db0740 1 -- 172.21.15.99:0/4188207892 wait complete. 2024-03-19T01:51:37.205 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.197+0000 7f4f58db0740 1 -- start start 2024-03-19T01:51:37.205 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.198+0000 7f4f58db0740 1 Processor -- start 2024-03-19T01:51:37.206 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.198+0000 7f4f58db0740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5628a88e7510 0x5628a88ea2c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:37.206 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.198+0000 7f4f58db0740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5628a88e7f00 0x5628a88ea800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:37.206 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.198+0000 7f4f58db0740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5628a88ed070 0x5628a88eb810 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:37.206 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.198+0000 7f4f58db0740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x5628a88e2fe0 con 0x5628a88e7510 2024-03-19T01:51:37.206 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.198+0000 7f4f58db0740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x5628a8803db0 con 0x5628a88e7f00 2024-03-19T01:51:37.206 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.198+0000 7f4f58db0740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x5628a88e3160 con 0x5628a88ed070 2024-03-19T01:51:37.206 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.198+0000 7f4f58db0740 10 client.0 _ll_register_callbacks cb 0x5628a8808e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:51:37.206 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.198+0000 7f4f47fff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5628a88e7f00 0x5628a88ea800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:37.206 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.198+0000 7f4f47fff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5628a88e7f00 0x5628a88ea800 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:34278/0 (socket says 172.21.15.99:34278) 2024-03-19T01:51:37.206 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.198+0000 7f4f47fff640 1 -- 172.21.15.99:0/832418271 learned_addr learned my addr 172.21.15.99:0/832418271 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:51:37.206 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.198+0000 7f4f4f7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5628a88e7510 0x5628a88ea2c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:37.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.198+0000 7f4f4ffff640 1 --2- 172.21.15.99:0/832418271 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5628a88ed070 0x5628a88eb810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:37.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.199+0000 7f4f47fff640 1 -- 172.21.15.99:0/832418271 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5628a88ed070 msgr2=0x5628a88eb810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:51:37.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.199+0000 7f4f47fff640 1 --2- 172.21.15.99:0/832418271 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5628a88ed070 0x5628a88eb810 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:37.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.199+0000 7f4f47fff640 1 -- 172.21.15.99:0/832418271 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5628a88e7510 msgr2=0x5628a88ea2c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:51:37.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.199+0000 7f4f47fff640 1 --2- 172.21.15.99:0/832418271 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5628a88e7510 0x5628a88ea2c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:37.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.199+0000 7f4f47fff640 1 -- 172.21.15.99:0/832418271 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5628a8802c60 con 0x5628a88e7f00 2024-03-19T01:51:37.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.199+0000 7f4f4f7fe640 1 --2- 172.21.15.99:0/832418271 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5628a88e7510 0x5628a88ea2c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:51:37.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.199+0000 7f4f4ffff640 1 --2- 172.21.15.99:0/832418271 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5628a88ed070 0x5628a88eb810 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:51:37.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.199+0000 7f4f47fff640 1 --2- 172.21.15.99:0/832418271 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5628a88e7f00 0x5628a88ea800 secure :-1 s=READY pgs=147 cs=0 l=1 rev1=1 crypto rx=0x7f4f380ee070 tx=0x7f4f38009560 comp rx=0 tx=0).ready entity=mon.2 client_cookie=7ba225b7b62dd640 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:51:37.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.199+0000 7f4f2effd640 10 client.0 ms_handle_connect on v2:172.21.15.112:3300/0 2024-03-19T01:51:37.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.199+0000 7f4f2effd640 1 -- 172.21.15.99:0/832418271 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4f3804b040 con 0x5628a88e7f00 2024-03-19T01:51:37.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.199+0000 7f4f2effd640 1 -- 172.21.15.99:0/832418271 <== mon.2 v2:172.21.15.112:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f4f380bd220 con 0x5628a88e7f00 2024-03-19T01:51:37.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.199+0000 7f4f58db0740 1 -- 172.21.15.99:0/832418271 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f4f4000f9c0 con 0x5628a88e7f00 2024-03-19T01:51:37.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.199+0000 7f4f2effd640 1 -- 172.21.15.99:0/832418271 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7f4f38004c60 con 0x5628a88e7f00 2024-03-19T01:51:37.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.199+0000 7f4f58db0740 10 client.25958 fetch_fsmap learned FSMap version 27 2024-03-19T01:51:37.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.199+0000 7f4f58db0740 1 -- 172.21.15.99:0/832418271 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x5628a855c210 con 0x5628a88e7f00 2024-03-19T01:51:37.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.199+0000 7f4f2effd640 1 -- 172.21.15.99:0/832418271 <== mon.2 v2:172.21.15.112:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f4f4000f9c0 con 0x5628a88e7f00 2024-03-19T01:51:37.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.199+0000 7f4f2effd640 1 -- 172.21.15.99:0/832418271 <== mon.2 v2:172.21.15.112:3300/0 5 ==== fsmap(e 27) ==== 8338+0+0 (secure 0 0 0) 0x7f4f380d2030 con 0x5628a88e7f00 2024-03-19T01:51:37.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.199+0000 7f4f58db0740 10 client.25958 fetch_fsmap finished waiting for FSMap version 27 2024-03-19T01:51:37.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.199+0000 7f4f58db0740 10 client.25958 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:51:37.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.200+0000 7f4f58db0740 20 client.25958 populate_metadata read hostname 'smithi099' 2024-03-19T01:51:37.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.200+0000 7f4f58db0740 1 --2- 172.21.15.99:0/832418271 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5628a88f8f10 0x5628a88fb340 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:37.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.200+0000 7f4f58db0740 4 client.25958 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:51:37.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.200+0000 7f4f58db0740 1 -- 172.21.15.99:0/832418271 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5628a8855120 con 0x5628a88f8f10 2024-03-19T01:51:37.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.200+0000 7f4f4f7fe640 1 --2- 172.21.15.99:0/832418271 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5628a88f8f10 0x5628a88fb340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:37.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.201+0000 7f4f4f7fe640 1 --2- 172.21.15.99:0/832418271 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5628a88f8f10 0x5628a88fb340 crc :-1 s=READY pgs=123 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=ab6dd285c07be81f server_cookie=b33eea8c8238772a in_seq=0 out_seq=0 2024-03-19T01:51:37.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.201+0000 7f4f2effd640 10 client.25958 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:51:37.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.201+0000 7f4f2effd640 1 -- 172.21.15.99:0/832418271 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x5628a8855120 con 0x5628a88f8f10 2024-03-19T01:51:37.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.201+0000 7f4f2effd640 10 client.25958 handle_command_reply: tid=0 2024-03-19T01:51:37.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.201+0000 7f4f58db0740 1 client.25958 shutdown 2024-03-19T01:51:37.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.202+0000 7f4f58db0740 1 -- 172.21.15.99:0/832418271 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5628a88e7f00 msgr2=0x5628a88ea800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:51:37.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.202+0000 7f4f58db0740 1 --2- 172.21.15.99:0/832418271 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5628a88e7f00 0x5628a88ea800 secure :-1 s=READY pgs=147 cs=0 l=1 rev1=1 crypto rx=0x7f4f380ee070 tx=0x7f4f38009560 comp rx=0 tx=0).stop 2024-03-19T01:51:37.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.202+0000 7f4f58db0740 1 -- 172.21.15.99:0/832418271 shutdown_connections 2024-03-19T01:51:37.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.202+0000 7f4f58db0740 1 --2- 172.21.15.99:0/832418271 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5628a88f8f10 0x5628a88fb340 crc :-1 s=READY pgs=123 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:37.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.202+0000 7f4f58db0740 1 --2- 172.21.15.99:0/832418271 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5628a88e7f00 0x5628a88ea800 unknown :-1 s=CLOSED pgs=147 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:37.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.202+0000 7f4f58db0740 1 --2- 172.21.15.99:0/832418271 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5628a88ed070 0x5628a88eb810 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:37.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.202+0000 7f4f2effd640 0 client.25958 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:51:37.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.202+0000 7f4f58db0740 1 --2- 172.21.15.99:0/832418271 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5628a88e7510 0x5628a88ea2c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:37.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.202+0000 7f4f58db0740 1 -- 172.21.15.99:0/832418271 >> 172.21.15.99:0/832418271 conn(0x5628a88de670 msgr2=0x5628a88df8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:51:37.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.202+0000 7f4f58db0740 1 -- 172.21.15.99:0/832418271 shutdown_connections 2024-03-19T01:51:37.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.202+0000 7f4f58db0740 1 -- 172.21.15.99:0/832418271 wait complete. 2024-03-19T01:51:37.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.202+0000 7f4f58db0740 20 client.25958 trim_cache size 0 max 16384 2024-03-19T01:51:37.223 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.218+0000 7f4f58db0740 1 Processor -- start 2024-03-19T01:51:37.223 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.218+0000 7f4f58db0740 1 -- start start 2024-03-19T01:51:37.223 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.218+0000 7f4f58db0740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5628a88e7510 0x5628a88ef460 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:37.223 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.218+0000 7f4f58db0740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5628a88e7f00 0x5628a88dd930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:37.223 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.218+0000 7f4f58db0740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5628a88ed070 0x5628a88e3530 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:37.223 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.218+0000 7f4f58db0740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f4f501a93f0 con 0x5628a88ed070 2024-03-19T01:51:37.223 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.218+0000 7f4f58db0740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f4f501a6cc0 con 0x5628a88e7f00 2024-03-19T01:51:37.223 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.218+0000 7f4f58db0740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f4f501a4b10 con 0x5628a88e7510 2024-03-19T01:51:37.223 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.218+0000 7f4f4ffff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5628a88ed070 0x5628a88e3530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:37.223 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.218+0000 7f4f4ffff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5628a88ed070 0x5628a88e3530 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:43296/0 (socket says 172.21.15.99:43296) 2024-03-19T01:51:37.223 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.218+0000 7f4f4ffff640 1 -- 172.21.15.99:0/3761628582 learned_addr learned my addr 172.21.15.99:0/3761628582 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:51:37.224 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.218+0000 7f4f47fff640 1 --2- 172.21.15.99:0/3761628582 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5628a88e7f00 0x5628a88dd930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:37.224 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.218+0000 7f4f4f7fe640 1 --2- 172.21.15.99:0/3761628582 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5628a88e7510 0x5628a88ef460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:37.224 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.219+0000 7f4f47fff640 1 -- 172.21.15.99:0/3761628582 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5628a88e7510 msgr2=0x5628a88ef460 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:51:37.224 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.219+0000 7f4f47fff640 1 --2- 172.21.15.99:0/3761628582 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5628a88e7510 0x5628a88ef460 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:37.224 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.219+0000 7f4f47fff640 1 -- 172.21.15.99:0/3761628582 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5628a88ed070 msgr2=0x5628a88e3530 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:51:37.224 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.219+0000 7f4f47fff640 1 --2- 172.21.15.99:0/3761628582 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5628a88ed070 0x5628a88e3530 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:37.224 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.219+0000 7f4f47fff640 1 -- 172.21.15.99:0/3761628582 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5628a8628620 con 0x5628a88e7f00 2024-03-19T01:51:37.224 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.219+0000 7f4f4ffff640 1 --2- 172.21.15.99:0/3761628582 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5628a88ed070 0x5628a88e3530 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:51:37.224 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.219+0000 7f4f47fff640 1 --2- 172.21.15.99:0/3761628582 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5628a88e7f00 0x5628a88dd930 secure :-1 s=READY pgs=148 cs=0 l=1 rev1=1 crypto rx=0x7f4f380ee3e0 tx=0x7f4f38005e50 comp rx=0 tx=0).ready entity=mon.2 client_cookie=98f7a89e72e241c4 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:51:37.224 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.219+0000 7f4f457fa640 1 -- 172.21.15.99:0/3761628582 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4f3804b070 con 0x5628a88e7f00 2024-03-19T01:51:37.224 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.219+0000 7f4f457fa640 1 -- 172.21.15.99:0/3761628582 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f4f380c56b0 con 0x5628a88e7f00 2024-03-19T01:51:37.224 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.219+0000 7f4f457fa640 1 -- 172.21.15.99:0/3761628582 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4f380f0160 con 0x5628a88e7f00 2024-03-19T01:51:37.225 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.220+0000 7f4f58db0740 1 -- 172.21.15.99:0/3761628582 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5628a88e7f00 msgr2=0x5628a88dd930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:51:37.225 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.220+0000 7f4f58db0740 1 --2- 172.21.15.99:0/3761628582 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5628a88e7f00 0x5628a88dd930 secure :-1 s=READY pgs=148 cs=0 l=1 rev1=1 crypto rx=0x7f4f380ee3e0 tx=0x7f4f38005e50 comp rx=0 tx=0).stop 2024-03-19T01:51:37.225 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.220+0000 7f4f58db0740 1 -- 172.21.15.99:0/3761628582 shutdown_connections 2024-03-19T01:51:37.225 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.220+0000 7f4f58db0740 1 --2- 172.21.15.99:0/3761628582 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5628a88e7f00 0x5628a88dd930 unknown :-1 s=CLOSED pgs=148 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:37.225 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.220+0000 7f4f58db0740 1 --2- 172.21.15.99:0/3761628582 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5628a88ed070 0x5628a88e3530 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:37.225 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.220+0000 7f4f58db0740 1 --2- 172.21.15.99:0/3761628582 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5628a88e7510 0x5628a88ef460 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:37.225 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.220+0000 7f4f58db0740 1 -- 172.21.15.99:0/3761628582 >> 172.21.15.99:0/3761628582 conn(0x5628a88de670 msgr2=0x5628a88dd220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:51:37.225 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.220+0000 7f4f58db0740 1 -- 172.21.15.99:0/3761628582 shutdown_connections 2024-03-19T01:51:37.225 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.220+0000 7f4f58db0740 1 -- 172.21.15.99:0/3761628582 wait complete. 2024-03-19T01:51:37.225 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.220+0000 7f4f58db0740 1 -- start start 2024-03-19T01:51:37.225 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.220+0000 7f4f58db0740 1 Processor -- start 2024-03-19T01:51:37.225 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.221+0000 7f4f58db0740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5628a88e7510 0x5628a88f2ed0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:37.225 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.221+0000 7f4f58db0740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5628a88e7f00 0x5628a88f3410 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:37.225 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.221+0000 7f4f58db0740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5628a88ed070 0x5628a88f4420 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:37.226 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.221+0000 7f4f58db0740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f4f501a5180 con 0x5628a88ed070 2024-03-19T01:51:37.226 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.221+0000 7f4f58db0740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f4f501a65f0 con 0x5628a88e7510 2024-03-19T01:51:37.226 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.221+0000 7f4f58db0740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f4f501a5850 con 0x5628a88e7f00 2024-03-19T01:51:37.226 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.221+0000 7f4f58db0740 10 client.0 _ll_register_callbacks cb 0x7f4f38009260 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:51:37.226 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.221+0000 7f4f4ffff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5628a88ed070 0x5628a88f4420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:37.226 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.221+0000 7f4f4f7fe640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5628a88e7510 0x5628a88f2ed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:37.226 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.221+0000 7f4f4ffff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5628a88ed070 0x5628a88f4420 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:43312/0 (socket says 172.21.15.99:43312) 2024-03-19T01:51:37.226 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.221+0000 7f4f4ffff640 1 -- 172.21.15.99:0/2986975191 learned_addr learned my addr 172.21.15.99:0/2986975191 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:51:37.226 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.221+0000 7f4f47fff640 1 --2- 172.21.15.99:0/2986975191 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5628a88e7f00 0x5628a88f3410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:37.226 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.221+0000 7f4f4ffff640 1 -- 172.21.15.99:0/2986975191 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5628a88e7f00 msgr2=0x5628a88f3410 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:51:37.226 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.221+0000 7f4f4ffff640 1 --2- 172.21.15.99:0/2986975191 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5628a88e7f00 0x5628a88f3410 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:37.226 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.221+0000 7f4f4ffff640 1 -- 172.21.15.99:0/2986975191 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5628a88e7510 msgr2=0x5628a88f2ed0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:51:37.226 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.221+0000 7f4f4ffff640 1 --2- 172.21.15.99:0/2986975191 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5628a88e7510 0x5628a88f2ed0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:37.227 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.221+0000 7f4f4ffff640 1 -- 172.21.15.99:0/2986975191 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5628a88f4a10 con 0x5628a88ed070 2024-03-19T01:51:37.227 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.221+0000 7f4f4f7fe640 1 --2- 172.21.15.99:0/2986975191 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5628a88e7510 0x5628a88f2ed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:51:37.227 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.221+0000 7f4f4ffff640 1 --2- 172.21.15.99:0/2986975191 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5628a88ed070 0x5628a88f4420 secure :-1 s=READY pgs=549 cs=0 l=1 rev1=1 crypto rx=0x7f4f48011910 tx=0x7f4f48004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=82de15ec796cada5 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:51:37.227 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.221+0000 7f4f2effd640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:51:37.227 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.221+0000 7f4f2effd640 1 -- 172.21.15.99:0/2986975191 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4f48004d20 con 0x5628a88ed070 2024-03-19T01:51:37.227 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.221+0000 7f4f2effd640 1 -- 172.21.15.99:0/2986975191 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f4f48020720 con 0x5628a88ed070 2024-03-19T01:51:37.227 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.221+0000 7f4f58db0740 1 -- 172.21.15.99:0/2986975191 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f4f380c56b0 con 0x5628a88ed070 2024-03-19T01:51:37.227 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.222+0000 7f4f2effd640 1 -- 172.21.15.99:0/2986975191 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7f4f48020920 con 0x5628a88ed070 2024-03-19T01:51:37.227 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.222+0000 7f4f58db0740 10 client.16194 fetch_fsmap learned FSMap version 27 2024-03-19T01:51:37.227 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.222+0000 7f4f58db0740 1 -- 172.21.15.99:0/2986975191 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x5628a895f250 con 0x5628a88ed070 2024-03-19T01:51:37.227 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.222+0000 7f4f2effd640 1 -- 172.21.15.99:0/2986975191 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f4f380c56b0 con 0x5628a88ed070 2024-03-19T01:51:37.227 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.223+0000 7f4f2effd640 1 -- 172.21.15.99:0/2986975191 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 27) ==== 8338+0+0 (secure 0 0 0) 0x7f4f4801d040 con 0x5628a88ed070 2024-03-19T01:51:37.227 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.223+0000 7f4f58db0740 10 client.16194 fetch_fsmap finished waiting for FSMap version 27 2024-03-19T01:51:37.227 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.223+0000 7f4f58db0740 10 client.16194 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:51:37.227 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.223+0000 7f4f58db0740 20 client.16194 populate_metadata read hostname 'smithi099' 2024-03-19T01:51:37.227 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.223+0000 7f4f58db0740 1 --2- 172.21.15.99:0/2986975191 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5628a895fc70 0x5628a8962090 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:37.228 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.223+0000 7f4f58db0740 4 client.16194 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:51:37.228 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.223+0000 7f4f58db0740 1 -- 172.21.15.99:0/2986975191 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x5628a884f470 con 0x5628a895fc70 2024-03-19T01:51:37.228 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.223+0000 7f4f4f7fe640 1 --2- 172.21.15.99:0/2986975191 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5628a895fc70 0x5628a8962090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:37.228 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.223+0000 7f4f4f7fe640 1 --2- 172.21.15.99:0/2986975191 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5628a895fc70 0x5628a8962090 crc :-1 s=READY pgs=124 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=e945f37439d07cf2 server_cookie=26c7f48d07f182d6 in_seq=0 out_seq=0 2024-03-19T01:51:37.228 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.223+0000 7f4f2effd640 10 client.16194 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:51:37.283 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.282+0000 7f4f2effd640 1 -- 172.21.15.99:0/2986975191 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x5628a884f470 con 0x5628a895fc70 2024-03-19T01:51:37.283 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.282+0000 7f4f2effd640 10 client.16194 handle_command_reply: tid=0 2024-03-19T01:51:37.283 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.282+0000 7f4f58db0740 1 client.16194 shutdown 2024-03-19T01:51:37.283 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.282+0000 7f4f58db0740 1 -- 172.21.15.99:0/2986975191 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5628a88ed070 msgr2=0x5628a88f4420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:51:37.283 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.282+0000 7f4f58db0740 1 --2- 172.21.15.99:0/2986975191 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5628a88ed070 0x5628a88f4420 secure :-1 s=READY pgs=549 cs=0 l=1 rev1=1 crypto rx=0x7f4f48011910 tx=0x7f4f48004a40 comp rx=0 tx=0).stop 2024-03-19T01:51:37.283 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.283+0000 7f4f58db0740 1 -- 172.21.15.99:0/2986975191 shutdown_connections 2024-03-19T01:51:37.283 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.283+0000 7f4f58db0740 1 --2- 172.21.15.99:0/2986975191 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5628a895fc70 0x5628a8962090 crc :-1 s=READY pgs=124 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:37.283 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.283+0000 7f4f58db0740 1 --2- 172.21.15.99:0/2986975191 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5628a88ed070 0x5628a88f4420 unknown :-1 s=CLOSED pgs=549 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:37.283 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.283+0000 7f4f58db0740 1 --2- 172.21.15.99:0/2986975191 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5628a88e7f00 0x5628a88f3410 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:37.283 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.283+0000 7f4f58db0740 1 --2- 172.21.15.99:0/2986975191 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5628a88e7510 0x5628a88f2ed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:37.283 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.283+0000 7f4f2effd640 0 client.16194 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:51:37.284 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.283+0000 7f4f58db0740 1 -- 172.21.15.99:0/2986975191 >> 172.21.15.99:0/2986975191 conn(0x5628a88de670 msgr2=0x5628a88e18d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:51:37.284 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.283+0000 7f4f58db0740 1 -- 172.21.15.99:0/2986975191 shutdown_connections 2024-03-19T01:51:37.284 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.283+0000 7f4f58db0740 1 -- 172.21.15.99:0/2986975191 wait complete. 2024-03-19T01:51:37.284 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.283+0000 7f4f58db0740 20 client.16194 trim_cache size 0 max 16384 2024-03-19T01:51:37.284 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:51:37.284 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:51:37.284 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:51:37.284 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:51:37.285 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.283+0000 7f4f560f1640 1 -- 172.21.15.99:0/2913950611 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f4f3c0a2920 msgr2=0x7f4f3c0a4de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:51:37.285 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.283+0000 7f4f560f1640 1 --2- 172.21.15.99:0/2913950611 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f4f3c0a2920 0x7f4f3c0a4de0 secure :-1 s=READY pgs=309 cs=0 l=1 rev1=1 crypto rx=0x5628a88e8760 tx=0x7f4f48009500 comp rx=0 tx=0).stop 2024-03-19T01:51:37.285 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.283+0000 7f4f560f1640 1 -- 172.21.15.99:0/2913950611 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4f50101930 msgr2=0x7f4f50198000 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:51:37.285 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.283+0000 7f4f560f1640 1 --2- 172.21.15.99:0/2913950611 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4f50101930 0x7f4f50198000 secure :-1 s=READY pgs=547 cs=0 l=1 rev1=1 crypto rx=0x7f4f38009230 tx=0x7f4f3802c650 comp rx=0 tx=0).stop 2024-03-19T01:51:37.286 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.284+0000 7f4f560f1640 1 -- 172.21.15.99:0/2913950611 shutdown_connections 2024-03-19T01:51:37.286 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.284+0000 7f4f560f1640 1 --2- 172.21.15.99:0/2913950611 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4f501022f0 0x7f4f5019f070 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:37.286 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.284+0000 7f4f560f1640 1 --2- 172.21.15.99:0/2913950611 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4f50101930 0x7f4f50198000 unknown :-1 s=CLOSED pgs=547 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:37.286 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.284+0000 7f4f560f1640 1 --2- 172.21.15.99:0/2913950611 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f4f3c0a2920 0x7f4f3c0a4de0 unknown :-1 s=CLOSED pgs=309 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:37.286 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.284+0000 7f4f560f1640 1 --2- 172.21.15.99:0/2913950611 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4f50100740 0x7f4f50197ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:37.286 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.284+0000 7f4f560f1640 1 -- 172.21.15.99:0/2913950611 >> 172.21.15.99:0/2913950611 conn(0x7f4f500fbcb0 msgr2=0x7f4f500fd6c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:51:37.286 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.284+0000 7f4f560f1640 1 -- 172.21.15.99:0/2913950611 shutdown_connections 2024-03-19T01:51:37.286 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:37.284+0000 7f4f560f1640 1 -- 172.21.15.99:0/2913950611 wait complete. 2024-03-19T01:51:37.700 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1280172 11.31 MB/sec execute 487 sec latency 36.714 ms 2024-03-19T01:51:38.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:51:37 smithi115 ceph-mon[67332]: pgmap v666: 97 pgs: 97 active+clean; 263 MiB data, 3.3 GiB used, 1.0 TiB / 1.0 TiB avail; 916 KiB/s rd, 2.8 MiB/s wr, 270 op/s 2024-03-19T01:51:38.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:51:37 smithi099 ceph-mon[62951]: pgmap v666: 97 pgs: 97 active+clean; 263 MiB data, 3.3 GiB used, 1.0 TiB / 1.0 TiB avail; 916 KiB/s rd, 2.8 MiB/s wr, 270 op/s 2024-03-19T01:51:38.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:51:37 smithi112 ceph-mon[66819]: pgmap v666: 97 pgs: 97 active+clean; 263 MiB data, 3.3 GiB used, 1.0 TiB / 1.0 TiB avail; 916 KiB/s rd, 2.8 MiB/s wr, 270 op/s 2024-03-19T01:51:38.700 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1282435 11.32 MB/sec execute 488 sec latency 15.925 ms 2024-03-19T01:51:39.700 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1284457 11.31 MB/sec execute 489 sec latency 64.303 ms 2024-03-19T01:51:40.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:51:39 smithi115 ceph-mon[67332]: pgmap v667: 97 pgs: 97 active+clean; 263 MiB data, 3.3 GiB used, 1.0 TiB / 1.0 TiB avail; 743 KiB/s rd, 2.4 MiB/s wr, 227 op/s 2024-03-19T01:51:40.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:51:39 smithi099 ceph-mon[62951]: pgmap v667: 97 pgs: 97 active+clean; 263 MiB data, 3.3 GiB used, 1.0 TiB / 1.0 TiB avail; 743 KiB/s rd, 2.4 MiB/s wr, 227 op/s 2024-03-19T01:51:40.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:51:39 smithi112 ceph-mon[66819]: pgmap v667: 97 pgs: 97 active+clean; 263 MiB data, 3.3 GiB used, 1.0 TiB / 1.0 TiB avail; 743 KiB/s rd, 2.4 MiB/s wr, 227 op/s 2024-03-19T01:51:40.700 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1286165 11.31 MB/sec execute 490 sec latency 44.845 ms 2024-03-19T01:51:41.701 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1288338 11.31 MB/sec execute 491 sec latency 35.555 ms 2024-03-19T01:51:42.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:51:41 smithi115 ceph-mon[67332]: pgmap v668: 97 pgs: 97 active+clean; 267 MiB data, 3.3 GiB used, 1.0 TiB / 1.0 TiB avail; 824 KiB/s rd, 2.7 MiB/s wr, 248 op/s 2024-03-19T01:51:42.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:51:41 smithi099 ceph-mon[62951]: pgmap v668: 97 pgs: 97 active+clean; 267 MiB data, 3.3 GiB used, 1.0 TiB / 1.0 TiB avail; 824 KiB/s rd, 2.7 MiB/s wr, 248 op/s 2024-03-19T01:51:42.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:51:41 smithi112 ceph-mon[66819]: pgmap v668: 97 pgs: 97 active+clean; 267 MiB data, 3.3 GiB used, 1.0 TiB / 1.0 TiB avail; 824 KiB/s rd, 2.7 MiB/s wr, 248 op/s 2024-03-19T01:51:42.700 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1290098 11.31 MB/sec execute 492 sec latency 32.452 ms 2024-03-19T01:51:43.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:51:42 smithi115 ceph-mon[67332]: pgmap v669: 97 pgs: 97 active+clean; 268 MiB data, 3.3 GiB used, 1.0 TiB / 1.0 TiB avail; 919 KiB/s rd, 2.9 MiB/s wr, 276 op/s 2024-03-19T01:51:43.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:51:42 smithi099 ceph-mon[62951]: pgmap v669: 97 pgs: 97 active+clean; 268 MiB data, 3.3 GiB used, 1.0 TiB / 1.0 TiB avail; 919 KiB/s rd, 2.9 MiB/s wr, 276 op/s 2024-03-19T01:51:43.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:51:42 smithi112 ceph-mon[66819]: pgmap v669: 97 pgs: 97 active+clean; 268 MiB data, 3.3 GiB used, 1.0 TiB / 1.0 TiB avail; 919 KiB/s rd, 2.9 MiB/s wr, 276 op/s 2024-03-19T01:51:43.700 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1292758 11.32 MB/sec execute 493 sec latency 15.042 ms 2024-03-19T01:51:44.701 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1294300 11.31 MB/sec execute 494 sec latency 37.850 ms 2024-03-19T01:51:45.701 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1296545 11.31 MB/sec execute 495 sec latency 41.447 ms 2024-03-19T01:51:46.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:51:45 smithi115 ceph-mon[67332]: pgmap v670: 97 pgs: 97 active+clean; 268 MiB data, 3.3 GiB used, 1.0 TiB / 1.0 TiB avail; 569 KiB/s rd, 2.4 MiB/s wr, 206 op/s 2024-03-19T01:51:46.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:51:45 smithi099 ceph-mon[62951]: pgmap v670: 97 pgs: 97 active+clean; 268 MiB data, 3.3 GiB used, 1.0 TiB / 1.0 TiB avail; 569 KiB/s rd, 2.4 MiB/s wr, 206 op/s 2024-03-19T01:51:46.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:51:45 smithi112 ceph-mon[66819]: pgmap v670: 97 pgs: 97 active+clean; 268 MiB data, 3.3 GiB used, 1.0 TiB / 1.0 TiB avail; 569 KiB/s rd, 2.4 MiB/s wr, 206 op/s 2024-03-19T01:51:46.701 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1298392 11.30 MB/sec execute 496 sec latency 42.331 ms 2024-03-19T01:51:47.701 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1300353 11.31 MB/sec execute 497 sec latency 10.881 ms 2024-03-19T01:51:48.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:51:47 smithi115 ceph-mon[67332]: pgmap v671: 97 pgs: 97 active+clean; 268 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 797 KiB/s rd, 2.7 MiB/s wr, 243 op/s 2024-03-19T01:51:48.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:51:47 smithi099 ceph-mon[62951]: pgmap v671: 97 pgs: 97 active+clean; 268 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 797 KiB/s rd, 2.7 MiB/s wr, 243 op/s 2024-03-19T01:51:48.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:51:47 smithi112 ceph-mon[66819]: pgmap v671: 97 pgs: 97 active+clean; 268 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 797 KiB/s rd, 2.7 MiB/s wr, 243 op/s 2024-03-19T01:51:48.701 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1302716 11.31 MB/sec execute 498 sec latency 30.657 ms 2024-03-19T01:51:49.701 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1304232 11.30 MB/sec execute 499 sec latency 39.462 ms 2024-03-19T01:51:50.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:51:49 smithi115 ceph-mon[67332]: pgmap v672: 97 pgs: 97 active+clean; 268 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 719 KiB/s rd, 2.6 MiB/s wr, 224 op/s 2024-03-19T01:51:50.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:51:49 smithi099 ceph-mon[62951]: pgmap v672: 97 pgs: 97 active+clean; 268 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 719 KiB/s rd, 2.6 MiB/s wr, 224 op/s 2024-03-19T01:51:50.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:51:49 smithi112 ceph-mon[66819]: pgmap v672: 97 pgs: 97 active+clean; 268 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 719 KiB/s rd, 2.6 MiB/s wr, 224 op/s 2024-03-19T01:51:50.388 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.385+0000 7f1f9d3ff640 1 -- 172.21.15.99:0/2827939449 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1f981017a0 msgr2=0x7f1f98101c20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:51:50.388 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.385+0000 7f1f9d3ff640 1 --2- 172.21.15.99:0/2827939449 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1f981017a0 0x7f1f98101c20 secure :-1 s=READY pgs=550 cs=0 l=1 rev1=1 crypto rx=0x7f1f80009900 tx=0x7f1f8002bc30 comp rx=0 tx=0).stop 2024-03-19T01:51:50.388 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.385+0000 7f1f9d3ff640 1 -- 172.21.15.99:0/2827939449 shutdown_connections 2024-03-19T01:51:50.388 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.385+0000 7f1f9d3ff640 1 --2- 172.21.15.99:0/2827939449 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1f98102160 0x7f1f981089f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:50.388 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.385+0000 7f1f9d3ff640 1 --2- 172.21.15.99:0/2827939449 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1f981017a0 0x7f1f98101c20 unknown :-1 s=CLOSED pgs=550 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:50.388 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.385+0000 7f1f9d3ff640 1 --2- 172.21.15.99:0/2827939449 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1f981005b0 0x7f1f981009d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:50.388 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.385+0000 7f1f9d3ff640 1 -- 172.21.15.99:0/2827939449 >> 172.21.15.99:0/2827939449 conn(0x7f1f980fbb60 msgr2=0x7f1f980fdf80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:51:50.388 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.385+0000 7f1f9d3ff640 1 -- 172.21.15.99:0/2827939449 shutdown_connections 2024-03-19T01:51:50.388 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.385+0000 7f1f9d3ff640 1 -- 172.21.15.99:0/2827939449 wait complete. 2024-03-19T01:51:50.388 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.386+0000 7f1f9d3ff640 1 Processor -- start 2024-03-19T01:51:50.388 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.386+0000 7f1f9d3ff640 1 -- start start 2024-03-19T01:51:50.389 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.386+0000 7f1f9d3ff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1f981005b0 0x7f1f98104540 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:50.389 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.386+0000 7f1f9d3ff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1f981017a0 0x7f1f98102b80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:50.389 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.386+0000 7f1f9d3ff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1f98102160 0x7f1f981030c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:50.389 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.386+0000 7f1f9d3ff640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f1f9810c230 con 0x7f1f981017a0 2024-03-19T01:51:50.389 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.386+0000 7f1f9d3ff640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f1f9810c0b0 con 0x7f1f98102160 2024-03-19T01:51:50.389 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.386+0000 7f1f9d3ff640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f1f9810c3b0 con 0x7f1f981005b0 2024-03-19T01:51:50.389 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.386+0000 7f1f8e7fc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1f981017a0 0x7f1f98102b80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:50.389 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.386+0000 7f1f96ffd640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1f981005b0 0x7f1f98104540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:50.389 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.386+0000 7f1f8e7fc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1f981017a0 0x7f1f98102b80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:59724/0 (socket says 172.21.15.99:59724) 2024-03-19T01:51:50.389 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.386+0000 7f1f96ffd640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1f981005b0 0x7f1f98104540 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:37658/0 (socket says 172.21.15.99:37658) 2024-03-19T01:51:50.389 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.386+0000 7f1f8e7fc640 1 -- 172.21.15.99:0/3097585968 learned_addr learned my addr 172.21.15.99:0/3097585968 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:51:50.390 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.386+0000 7f1f977fe640 1 --2- 172.21.15.99:0/3097585968 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1f98102160 0x7f1f981030c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:50.390 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.387+0000 7f1f96ffd640 1 -- 172.21.15.99:0/3097585968 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1f98102160 msgr2=0x7f1f981030c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:51:50.390 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.387+0000 7f1f96ffd640 1 --2- 172.21.15.99:0/3097585968 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1f98102160 0x7f1f981030c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:50.390 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.387+0000 7f1f96ffd640 1 -- 172.21.15.99:0/3097585968 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1f981017a0 msgr2=0x7f1f98102b80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:51:50.390 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.387+0000 7f1f96ffd640 1 --2- 172.21.15.99:0/3097585968 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1f981017a0 0x7f1f98102b80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:50.390 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.387+0000 7f1f96ffd640 1 -- 172.21.15.99:0/3097585968 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f1f98103950 con 0x7f1f981005b0 2024-03-19T01:51:50.390 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.387+0000 7f1f8e7fc640 1 --2- 172.21.15.99:0/3097585968 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1f981017a0 0x7f1f98102b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:51:50.390 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.387+0000 7f1f96ffd640 1 --2- 172.21.15.99:0/3097585968 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1f981005b0 0x7f1f98104540 secure :-1 s=READY pgs=107 cs=0 l=1 rev1=1 crypto rx=0x7f1f8800e9c0 tx=0x7f1f8800ee80 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:51:50.390 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.387+0000 7f1f94ff9640 1 -- 172.21.15.99:0/3097585968 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1f8800ccc0 con 0x7f1f981005b0 2024-03-19T01:51:50.390 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.387+0000 7f1f9d3ff640 1 -- 172.21.15.99:0/3097585968 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f1f981a1ac0 con 0x7f1f981005b0 2024-03-19T01:51:50.390 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.387+0000 7f1f94ff9640 1 -- 172.21.15.99:0/3097585968 <== mon.1 v2:172.21.15.115:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f1f880035f0 con 0x7f1f981005b0 2024-03-19T01:51:50.390 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.387+0000 7f1f94ff9640 1 -- 172.21.15.99:0/3097585968 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1f8800ce60 con 0x7f1f981005b0 2024-03-19T01:51:50.391 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.387+0000 7f1f9d3ff640 1 -- 172.21.15.99:0/3097585968 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f1f981a1fe0 con 0x7f1f981005b0 2024-03-19T01:51:50.398 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.389+0000 7f1f94ff9640 1 -- 172.21.15.99:0/3097585968 <== mon.1 v2:172.21.15.115:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f1f880037a0 con 0x7f1f981005b0 2024-03-19T01:51:50.398 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.389+0000 7f1f9ffed740 1 Processor -- start 2024-03-19T01:51:50.398 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.390+0000 7f1f9ffed740 1 -- start start 2024-03-19T01:51:50.398 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.390+0000 7f1f9ffed740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a5c05262e0 0x55a5c0526700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:50.398 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.390+0000 7f1f9ffed740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a5c0526cd0 0x55a5c052b7d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:50.398 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.390+0000 7f1f9ffed740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a5c052be40 0x55a5c052e2d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:50.398 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.390+0000 7f1f9ffed740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55a5c048abe0 con 0x55a5c05262e0 2024-03-19T01:51:50.398 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.390+0000 7f1f9ffed740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55a5c0433960 con 0x55a5c052be40 2024-03-19T01:51:50.398 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.390+0000 7f1f9ffed740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55a5c0486c80 con 0x55a5c0526cd0 2024-03-19T01:51:50.398 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.390+0000 7f1f977fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a5c05262e0 0x55a5c0526700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:50.398 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.390+0000 7f1f8e7fc640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a5c052be40 0x55a5c052e2d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:50.399 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.390+0000 7f1f8e7fc640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a5c052be40 0x55a5c052e2d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:41286/0 (socket says 172.21.15.99:41286) 2024-03-19T01:51:50.399 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.390+0000 7f1f96ffd640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a5c0526cd0 0x55a5c052b7d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:50.399 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.390+0000 7f1f8e7fc640 1 -- 172.21.15.99:0/3266581081 learned_addr learned my addr 172.21.15.99:0/3266581081 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:51:50.399 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.390+0000 7f1f977fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a5c05262e0 0x55a5c0526700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:59726/0 (socket says 172.21.15.99:59726) 2024-03-19T01:51:50.399 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.390+0000 7f1f94ff9640 1 --2- 172.21.15.99:0/3097585968 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f1f840a27f0 0x7f1f840a4cb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:50.399 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.390+0000 7f1f8e7fc640 1 -- 172.21.15.99:0/3266581081 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a5c0526cd0 msgr2=0x55a5c052b7d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:51:50.399 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.390+0000 7f1f8e7fc640 1 --2- 172.21.15.99:0/3266581081 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a5c0526cd0 0x55a5c052b7d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:50.399 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.390+0000 7f1f8e7fc640 1 -- 172.21.15.99:0/3266581081 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a5c05262e0 msgr2=0x55a5c0526700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:51:50.399 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.390+0000 7f1f8e7fc640 1 --2- 172.21.15.99:0/3266581081 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a5c05262e0 0x55a5c0526700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:50.399 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.390+0000 7f1f8e7fc640 1 -- 172.21.15.99:0/3266581081 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55a5c042ad20 con 0x55a5c052be40 2024-03-19T01:51:50.399 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.390+0000 7f1f977fe640 1 --2- 172.21.15.99:0/3266581081 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a5c05262e0 0x55a5c0526700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:51:50.399 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.390+0000 7f1f94ff9640 1 -- 172.21.15.99:0/3097585968 <== mon.1 v2:172.21.15.115:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f1f880cf000 con 0x7f1f981005b0 2024-03-19T01:51:50.399 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.390+0000 7f1f977fe640 1 --2- 172.21.15.99:0/3097585968 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f1f840a27f0 0x7f1f840a4cb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:50.399 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.391+0000 7f1f8e7fc640 1 --2- 172.21.15.99:0/3266581081 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a5c052be40 0x55a5c052e2d0 secure :-1 s=READY pgs=149 cs=0 l=1 rev1=1 crypto rx=0x7f1f80009230 tx=0x7f1f8002cd30 comp rx=0 tx=0).ready entity=mon.2 client_cookie=c37e3e979c88f74f server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:51:50.400 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.391+0000 7f1f8d7fa640 1 -- 172.21.15.99:0/3266581081 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1f80042070 con 0x55a5c052be40 2024-03-19T01:51:50.400 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.391+0000 7f1f8d7fa640 1 -- 172.21.15.99:0/3266581081 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f1f80004ba0 con 0x55a5c052be40 2024-03-19T01:51:50.400 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.391+0000 7f1f8d7fa640 1 -- 172.21.15.99:0/3266581081 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1f800052f0 con 0x55a5c052be40 2024-03-19T01:51:50.400 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.391+0000 7f1f9ffed740 1 -- 172.21.15.99:0/3266581081 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a5c052be40 msgr2=0x55a5c052e2d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:51:50.400 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.391+0000 7f1f977fe640 1 --2- 172.21.15.99:0/3097585968 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f1f840a27f0 0x7f1f840a4cb0 secure :-1 s=READY pgs=310 cs=0 l=1 rev1=1 crypto rx=0x55a5c0527530 tx=0x7f1f90004a40 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:51:50.400 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.391+0000 7f1f9ffed740 1 --2- 172.21.15.99:0/3266581081 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a5c052be40 0x55a5c052e2d0 secure :-1 s=READY pgs=149 cs=0 l=1 rev1=1 crypto rx=0x7f1f80009230 tx=0x7f1f8002cd30 comp rx=0 tx=0).stop 2024-03-19T01:51:50.400 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.392+0000 7f1f9ffed740 1 -- 172.21.15.99:0/3266581081 shutdown_connections 2024-03-19T01:51:50.400 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.392+0000 7f1f9ffed740 1 --2- 172.21.15.99:0/3266581081 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a5c052be40 0x55a5c052e2d0 unknown :-1 s=CLOSED pgs=149 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:50.400 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.392+0000 7f1f9ffed740 1 --2- 172.21.15.99:0/3266581081 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a5c0526cd0 0x55a5c052b7d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:50.400 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.392+0000 7f1f9ffed740 1 --2- 172.21.15.99:0/3266581081 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a5c05262e0 0x55a5c0526700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:50.400 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.392+0000 7f1f9ffed740 1 -- 172.21.15.99:0/3266581081 >> 172.21.15.99:0/3266581081 conn(0x55a5c0521850 msgr2=0x55a5c0521c50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:51:50.400 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.392+0000 7f1f9ffed740 1 -- 172.21.15.99:0/3266581081 shutdown_connections 2024-03-19T01:51:50.400 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.392+0000 7f1f9ffed740 1 -- 172.21.15.99:0/3266581081 wait complete. 2024-03-19T01:51:50.400 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.392+0000 7f1f9ffed740 1 -- start start 2024-03-19T01:51:50.400 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.392+0000 7f1f9ffed740 1 Processor -- start 2024-03-19T01:51:50.401 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.392+0000 7f1f9ffed740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a5c05262e0 0x55a5c0530280 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:50.401 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.392+0000 7f1f9ffed740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a5c0526cd0 0x55a5c05307c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:50.401 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.392+0000 7f1f9ffed740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a5c052be40 0x55a5c0537800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:50.401 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.392+0000 7f1f9ffed740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55a5c052ed90 con 0x55a5c05262e0 2024-03-19T01:51:50.401 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.392+0000 7f1f9ffed740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55a5c0446db0 con 0x55a5c0526cd0 2024-03-19T01:51:50.401 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.392+0000 7f1f9ffed740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55a5c052ef10 con 0x55a5c052be40 2024-03-19T01:51:50.401 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.392+0000 7f1f9ffed740 10 client.0 _ll_register_callbacks cb 0x55a5c044be70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:51:50.401 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.392+0000 7f1f96ffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a5c05262e0 0x55a5c0530280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:50.401 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.392+0000 7f1f8e7fc640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a5c0526cd0 0x55a5c05307c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:50.401 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.392+0000 7f1f96ffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a5c05262e0 0x55a5c0530280 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:59732/0 (socket says 172.21.15.99:59732) 2024-03-19T01:51:50.401 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.392+0000 7f1f96ffd640 1 -- 172.21.15.99:0/2400760160 learned_addr learned my addr 172.21.15.99:0/2400760160 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:51:50.401 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.392+0000 7f1f977fe640 1 --2- 172.21.15.99:0/2400760160 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a5c052be40 0x55a5c0537800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:50.401 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.393+0000 7f1f96ffd640 1 -- 172.21.15.99:0/2400760160 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a5c052be40 msgr2=0x55a5c0537800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:51:50.401 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.393+0000 7f1f96ffd640 1 --2- 172.21.15.99:0/2400760160 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a5c052be40 0x55a5c0537800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:50.402 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.393+0000 7f1f96ffd640 1 -- 172.21.15.99:0/2400760160 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a5c0526cd0 msgr2=0x55a5c05307c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:51:50.402 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.393+0000 7f1f96ffd640 1 --2- 172.21.15.99:0/2400760160 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a5c0526cd0 0x55a5c05307c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:50.402 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.393+0000 7f1f96ffd640 1 -- 172.21.15.99:0/2400760160 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55a5c0445c60 con 0x55a5c05262e0 2024-03-19T01:51:50.402 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.393+0000 7f1f8e7fc640 1 --2- 172.21.15.99:0/2400760160 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a5c0526cd0 0x55a5c05307c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:51:50.402 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.393+0000 7f1f96ffd640 1 --2- 172.21.15.99:0/2400760160 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a5c05262e0 0x55a5c0530280 secure :-1 s=READY pgs=551 cs=0 l=1 rev1=1 crypto rx=0x7f1f88097dc0 tx=0x7f1f880d26c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a6168e7b8d7119c1 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:51:50.402 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.393+0000 7f1f7e7fc640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:51:50.402 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.393+0000 7f1f7e7fc640 1 -- 172.21.15.99:0/2400760160 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1f8809c060 con 0x55a5c05262e0 2024-03-19T01:51:50.402 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.393+0000 7f1f7e7fc640 1 -- 172.21.15.99:0/2400760160 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f1f880d10f0 con 0x55a5c05262e0 2024-03-19T01:51:50.402 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.393+0000 7f1f9ffed740 1 -- 172.21.15.99:0/2400760160 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f1f80004ba0 con 0x55a5c05262e0 2024-03-19T01:51:50.402 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.393+0000 7f1f7e7fc640 1 -- 172.21.15.99:0/2400760160 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7f1f880d12f0 con 0x55a5c05262e0 2024-03-19T01:51:50.402 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.393+0000 7f1f9ffed740 10 client.16206 fetch_fsmap learned FSMap version 27 2024-03-19T01:51:50.402 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.393+0000 7f1f9ffed740 1 -- 172.21.15.99:0/2400760160 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x55a5c019f210 con 0x55a5c05262e0 2024-03-19T01:51:50.402 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.393+0000 7f1f7e7fc640 1 -- 172.21.15.99:0/2400760160 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f1f80004ba0 con 0x55a5c05262e0 2024-03-19T01:51:50.402 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.393+0000 7f1f7e7fc640 1 -- 172.21.15.99:0/2400760160 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 27) ==== 8338+0+0 (secure 0 0 0) 0x7f1f88077020 con 0x55a5c05262e0 2024-03-19T01:51:50.403 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.393+0000 7f1f9ffed740 10 client.16206 fetch_fsmap finished waiting for FSMap version 27 2024-03-19T01:51:50.403 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.393+0000 7f1f9ffed740 10 client.16206 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:51:50.403 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.393+0000 7f1f9ffed740 20 client.16206 populate_metadata read hostname 'smithi099' 2024-03-19T01:51:50.403 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.393+0000 7f1f9ffed740 1 --2- 172.21.15.99:0/2400760160 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55a5c053c130 0x55a5c053e580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:50.403 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.393+0000 7f1f9ffed740 4 client.16206 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:51:50.403 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.393+0000 7f1f9ffed740 1 -- 172.21.15.99:0/2400760160 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55a5c0498120 con 0x55a5c053c130 2024-03-19T01:51:50.403 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.394+0000 7f1f8e7fc640 1 --2- 172.21.15.99:0/2400760160 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55a5c053c130 0x55a5c053e580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:50.403 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.394+0000 7f1f7e7fc640 10 client.16206 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:51:50.403 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.394+0000 7f1f8e7fc640 1 --2- 172.21.15.99:0/2400760160 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55a5c053c130 0x55a5c053e580 crc :-1 s=READY pgs=125 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=c12ac0f1b71ab0f0 server_cookie=a69d5201aa733107 in_seq=0 out_seq=0 2024-03-19T01:51:50.403 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.394+0000 7f1f7e7fc640 1 -- 172.21.15.99:0/2400760160 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x55a5c0498120 con 0x55a5c053c130 2024-03-19T01:51:50.403 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.394+0000 7f1f7e7fc640 10 client.16206 handle_command_reply: tid=0 2024-03-19T01:51:50.403 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.394+0000 7f1f9ffed740 1 client.16206 shutdown 2024-03-19T01:51:50.403 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.395+0000 7f1f9ffed740 1 -- 172.21.15.99:0/2400760160 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a5c05262e0 msgr2=0x55a5c0530280 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:51:50.404 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.395+0000 7f1f9ffed740 1 --2- 172.21.15.99:0/2400760160 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a5c05262e0 0x55a5c0530280 secure :-1 s=READY pgs=551 cs=0 l=1 rev1=1 crypto rx=0x7f1f88097dc0 tx=0x7f1f880d26c0 comp rx=0 tx=0).stop 2024-03-19T01:51:50.404 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.395+0000 7f1f9ffed740 1 -- 172.21.15.99:0/2400760160 shutdown_connections 2024-03-19T01:51:50.404 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.395+0000 7f1f9ffed740 1 --2- 172.21.15.99:0/2400760160 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55a5c053c130 0x55a5c053e580 crc :-1 s=READY pgs=125 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:50.404 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.395+0000 7f1f9ffed740 1 --2- 172.21.15.99:0/2400760160 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a5c0526cd0 0x55a5c05307c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:50.404 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.395+0000 7f1f9ffed740 1 --2- 172.21.15.99:0/2400760160 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a5c052be40 0x55a5c0537800 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:50.404 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.395+0000 7f1f9ffed740 1 --2- 172.21.15.99:0/2400760160 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a5c05262e0 0x55a5c0530280 unknown :-1 s=CLOSED pgs=551 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:50.404 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.395+0000 7f1f7e7fc640 0 client.16206 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:51:50.404 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.395+0000 7f1f9ffed740 1 -- 172.21.15.99:0/2400760160 >> 172.21.15.99:0/2400760160 conn(0x55a5c0521850 msgr2=0x55a5c0520ab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:51:50.404 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.395+0000 7f1f9ffed740 1 -- 172.21.15.99:0/2400760160 shutdown_connections 2024-03-19T01:51:50.404 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.395+0000 7f1f9ffed740 1 -- 172.21.15.99:0/2400760160 wait complete. 2024-03-19T01:51:50.404 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.395+0000 7f1f9ffed740 20 client.16206 trim_cache size 0 max 16384 2024-03-19T01:51:50.414 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.410+0000 7f1f9ffed740 1 Processor -- start 2024-03-19T01:51:50.414 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.410+0000 7f1f9ffed740 1 -- start start 2024-03-19T01:51:50.414 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.410+0000 7f1f9ffed740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a5c05262e0 0x55a5c05305a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:50.414 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.410+0000 7f1f9ffed740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a5c0526cd0 0x55a5c05377c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:50.414 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.410+0000 7f1f9ffed740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a5c052be40 0x55a5c0520220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:50.414 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.410+0000 7f1f9ffed740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f1f981ad680 con 0x55a5c0526cd0 2024-03-19T01:51:50.414 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.410+0000 7f1f8e7fc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a5c0526cd0 0x55a5c05377c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:50.414 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.410+0000 7f1f9ffed740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f1f981aaf50 con 0x55a5c052be40 2024-03-19T01:51:50.414 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.410+0000 7f1f9ffed740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f1f981a8da0 con 0x55a5c05262e0 2024-03-19T01:51:50.415 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.410+0000 7f1f8e7fc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a5c0526cd0 0x55a5c05377c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:59734/0 (socket says 172.21.15.99:59734) 2024-03-19T01:51:50.415 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.410+0000 7f1f8e7fc640 1 -- 172.21.15.99:0/3294445332 learned_addr learned my addr 172.21.15.99:0/3294445332 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:51:50.415 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.410+0000 7f1f96ffd640 1 --2- 172.21.15.99:0/3294445332 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a5c05262e0 0x55a5c05305a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:50.415 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.410+0000 7f1f977fe640 1 --2- 172.21.15.99:0/3294445332 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a5c052be40 0x55a5c0520220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:50.415 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.410+0000 7f1f8e7fc640 1 -- 172.21.15.99:0/3294445332 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a5c05262e0 msgr2=0x55a5c05305a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:51:50.415 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.410+0000 7f1f8e7fc640 1 --2- 172.21.15.99:0/3294445332 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a5c05262e0 0x55a5c05305a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:50.415 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.410+0000 7f1f8e7fc640 1 -- 172.21.15.99:0/3294445332 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a5c052be40 msgr2=0x55a5c0520220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:51:50.415 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.410+0000 7f1f8e7fc640 1 --2- 172.21.15.99:0/3294445332 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a5c052be40 0x55a5c0520220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:50.415 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.410+0000 7f1f8e7fc640 1 -- 172.21.15.99:0/3294445332 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55a5c026b620 con 0x55a5c0526cd0 2024-03-19T01:51:50.415 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.411+0000 7f1f96ffd640 1 --2- 172.21.15.99:0/3294445332 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a5c05262e0 0x55a5c05305a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:51:50.415 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.411+0000 7f1f977fe640 1 --2- 172.21.15.99:0/3294445332 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a5c052be40 0x55a5c0520220 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-19T01:51:50.415 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.411+0000 7f1f8e7fc640 1 --2- 172.21.15.99:0/3294445332 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a5c0526cd0 0x55a5c05377c0 secure :-1 s=READY pgs=552 cs=0 l=1 rev1=1 crypto rx=0x7f1f8002cfd0 tx=0x7f1f8002bc30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=17d6fe4a6845fb24 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:51:50.415 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.411+0000 7f1f8cff9640 1 -- 172.21.15.99:0/3294445332 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1f80042040 con 0x55a5c0526cd0 2024-03-19T01:51:50.415 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.411+0000 7f1f8cff9640 1 -- 172.21.15.99:0/3294445332 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f1f80057020 con 0x55a5c0526cd0 2024-03-19T01:51:50.416 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.411+0000 7f1f8cff9640 1 -- 172.21.15.99:0/3294445332 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1f80005450 con 0x55a5c0526cd0 2024-03-19T01:51:50.416 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.412+0000 7f1f9ffed740 1 -- 172.21.15.99:0/3294445332 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a5c0526cd0 msgr2=0x55a5c05377c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:51:50.416 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.412+0000 7f1f9ffed740 1 --2- 172.21.15.99:0/3294445332 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a5c0526cd0 0x55a5c05377c0 secure :-1 s=READY pgs=552 cs=0 l=1 rev1=1 crypto rx=0x7f1f8002cfd0 tx=0x7f1f8002bc30 comp rx=0 tx=0).stop 2024-03-19T01:51:50.416 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.412+0000 7f1f9ffed740 1 -- 172.21.15.99:0/3294445332 shutdown_connections 2024-03-19T01:51:50.416 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.412+0000 7f1f9ffed740 1 --2- 172.21.15.99:0/3294445332 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a5c052be40 0x55a5c0520220 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:50.416 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.412+0000 7f1f9ffed740 1 --2- 172.21.15.99:0/3294445332 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a5c0526cd0 0x55a5c05377c0 unknown :-1 s=CLOSED pgs=552 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:50.416 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.412+0000 7f1f9ffed740 1 --2- 172.21.15.99:0/3294445332 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a5c05262e0 0x55a5c05305a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:50.416 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.412+0000 7f1f9ffed740 1 -- 172.21.15.99:0/3294445332 >> 172.21.15.99:0/3294445332 conn(0x55a5c0521850 msgr2=0x55a5c052fe90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:51:50.416 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.412+0000 7f1f9ffed740 1 -- 172.21.15.99:0/3294445332 shutdown_connections 2024-03-19T01:51:50.416 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.412+0000 7f1f9ffed740 1 -- 172.21.15.99:0/3294445332 wait complete. 2024-03-19T01:51:50.416 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.412+0000 7f1f9ffed740 1 -- start start 2024-03-19T01:51:50.416 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.412+0000 7f1f9ffed740 1 Processor -- start 2024-03-19T01:51:50.416 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.412+0000 7f1f9ffed740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a5c05262e0 0x55a5c05349b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:50.416 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.412+0000 7f1f9ffed740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a5c0526cd0 0x55a5c0534ef0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:50.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.412+0000 7f1f9ffed740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a5c052be40 0x55a5c0535f00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:50.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.412+0000 7f1f9ffed740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f1f981a9410 con 0x55a5c05262e0 2024-03-19T01:51:50.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.412+0000 7f1f9ffed740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f1f981aa880 con 0x55a5c0526cd0 2024-03-19T01:51:50.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.412+0000 7f1f9ffed740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f1f981a9ae0 con 0x55a5c052be40 2024-03-19T01:51:50.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.412+0000 7f1f9ffed740 10 client.0 _ll_register_callbacks cb 0x7f1f88003e10 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:51:50.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.412+0000 7f1f96ffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a5c05262e0 0x55a5c05349b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:50.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.413+0000 7f1f96ffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a5c05262e0 0x55a5c05349b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:59748/0 (socket says 172.21.15.99:59748) 2024-03-19T01:51:50.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.413+0000 7f1f96ffd640 1 -- 172.21.15.99:0/538043340 learned_addr learned my addr 172.21.15.99:0/538043340 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:51:50.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.413+0000 7f1f8e7fc640 1 --2- 172.21.15.99:0/538043340 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a5c0526cd0 0x55a5c0534ef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:50.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.413+0000 7f1f977fe640 1 --2- 172.21.15.99:0/538043340 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a5c052be40 0x55a5c0535f00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:50.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.413+0000 7f1f96ffd640 1 -- 172.21.15.99:0/538043340 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a5c052be40 msgr2=0x55a5c0535f00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:51:50.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.413+0000 7f1f96ffd640 1 --2- 172.21.15.99:0/538043340 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a5c052be40 0x55a5c0535f00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:50.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.413+0000 7f1f96ffd640 1 -- 172.21.15.99:0/538043340 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a5c0526cd0 msgr2=0x55a5c0534ef0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:51:50.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.413+0000 7f1f96ffd640 1 --2- 172.21.15.99:0/538043340 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a5c0526cd0 0x55a5c0534ef0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:50.418 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.413+0000 7f1f96ffd640 1 -- 172.21.15.99:0/538043340 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55a5c05364f0 con 0x55a5c05262e0 2024-03-19T01:51:50.418 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.413+0000 7f1f96ffd640 1 --2- 172.21.15.99:0/538043340 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a5c05262e0 0x55a5c05349b0 secure :-1 s=READY pgs=553 cs=0 l=1 rev1=1 crypto rx=0x7f1f88012020 tx=0x7f1f88003d40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c3afd72a8bf24b00 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:51:50.418 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.413+0000 7f1f7e7fc640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:51:50.418 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.413+0000 7f1f7e7fc640 1 -- 172.21.15.99:0/538043340 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1f8809c060 con 0x55a5c05262e0 2024-03-19T01:51:50.418 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.413+0000 7f1f7e7fc640 1 -- 172.21.15.99:0/538043340 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f1f88004210 con 0x55a5c05262e0 2024-03-19T01:51:50.418 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.413+0000 7f1f9ffed740 1 -- 172.21.15.99:0/538043340 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f1f80057020 con 0x55a5c05262e0 2024-03-19T01:51:50.418 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.413+0000 7f1f7e7fc640 1 -- 172.21.15.99:0/538043340 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7f1f8809af70 con 0x55a5c05262e0 2024-03-19T01:51:50.418 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.413+0000 7f1f9ffed740 10 client.16212 fetch_fsmap learned FSMap version 27 2024-03-19T01:51:50.418 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.413+0000 7f1f9ffed740 1 -- 172.21.15.99:0/538043340 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x55a5c0537870 con 0x55a5c05262e0 2024-03-19T01:51:50.418 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.413+0000 7f1f7e7fc640 1 -- 172.21.15.99:0/538043340 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f1f80057020 con 0x55a5c05262e0 2024-03-19T01:51:50.418 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.413+0000 7f1f7e7fc640 1 -- 172.21.15.99:0/538043340 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 27) ==== 8338+0+0 (secure 0 0 0) 0x7f1f8809a820 con 0x55a5c05262e0 2024-03-19T01:51:50.418 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.413+0000 7f1f9ffed740 10 client.16212 fetch_fsmap finished waiting for FSMap version 27 2024-03-19T01:51:50.418 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.414+0000 7f1f9ffed740 10 client.16212 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:51:50.418 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.414+0000 7f1f9ffed740 20 client.16212 populate_metadata read hostname 'smithi099' 2024-03-19T01:51:50.419 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.414+0000 7f1f9ffed740 1 --2- 172.21.15.99:0/538043340 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55a5c05a2910 0x55a5c05a4d00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:51:50.419 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.414+0000 7f1f9ffed740 4 client.16212 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:51:50.419 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.414+0000 7f1f9ffed740 1 -- 172.21.15.99:0/538043340 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55a5c0492470 con 0x55a5c05a2910 2024-03-19T01:51:50.419 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.414+0000 7f1f8e7fc640 1 --2- 172.21.15.99:0/538043340 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55a5c05a2910 0x55a5c05a4d00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:51:50.419 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.414+0000 7f1f7e7fc640 10 client.16212 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:51:50.419 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.414+0000 7f1f8e7fc640 1 --2- 172.21.15.99:0/538043340 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55a5c05a2910 0x55a5c05a4d00 crc :-1 s=READY pgs=126 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=e37cd704a4213048 server_cookie=5b2e5d8e15485ad2 in_seq=0 out_seq=0 2024-03-19T01:51:50.494 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.493+0000 7f1f7e7fc640 1 -- 172.21.15.99:0/538043340 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x55a5c0492470 con 0x55a5c05a2910 2024-03-19T01:51:50.494 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.494+0000 7f1f7e7fc640 10 client.16212 handle_command_reply: tid=0 2024-03-19T01:51:50.494 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.494+0000 7f1f9ffed740 1 client.16212 shutdown 2024-03-19T01:51:50.494 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.494+0000 7f1f9ffed740 1 -- 172.21.15.99:0/538043340 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a5c05262e0 msgr2=0x55a5c05349b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:51:50.494 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.494+0000 7f1f9ffed740 1 --2- 172.21.15.99:0/538043340 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a5c05262e0 0x55a5c05349b0 secure :-1 s=READY pgs=553 cs=0 l=1 rev1=1 crypto rx=0x7f1f88012020 tx=0x7f1f88003d40 comp rx=0 tx=0).stop 2024-03-19T01:51:50.494 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.494+0000 7f1f9ffed740 1 -- 172.21.15.99:0/538043340 shutdown_connections 2024-03-19T01:51:50.494 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.494+0000 7f1f9ffed740 1 --2- 172.21.15.99:0/538043340 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55a5c05a2910 0x55a5c05a4d00 crc :-1 s=READY pgs=126 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:50.494 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.494+0000 7f1f7e7fc640 0 client.16212 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:51:50.495 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.494+0000 7f1f9ffed740 1 --2- 172.21.15.99:0/538043340 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a5c0526cd0 0x55a5c0534ef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:50.495 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.494+0000 7f1f9ffed740 1 --2- 172.21.15.99:0/538043340 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a5c052be40 0x55a5c0535f00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:50.495 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.494+0000 7f1f9ffed740 1 --2- 172.21.15.99:0/538043340 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a5c05262e0 0x55a5c05349b0 unknown :-1 s=CLOSED pgs=553 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:50.495 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.494+0000 7f1f9ffed740 1 -- 172.21.15.99:0/538043340 >> 172.21.15.99:0/538043340 conn(0x55a5c0521850 msgr2=0x55a5c05276d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:51:50.495 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.495+0000 7f1f9ffed740 1 -- 172.21.15.99:0/538043340 shutdown_connections 2024-03-19T01:51:50.495 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.495+0000 7f1f9ffed740 1 -- 172.21.15.99:0/538043340 wait complete. 2024-03-19T01:51:50.495 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.495+0000 7f1f9ffed740 20 client.16212 trim_cache size 0 max 16384 2024-03-19T01:51:50.496 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:51:50.496 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:51:50.496 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:51:50.496 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:51:50.497 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.495+0000 7f1f9d3ff640 1 -- 172.21.15.99:0/3097585968 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f1f840a27f0 msgr2=0x7f1f840a4cb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:51:50.497 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.495+0000 7f1f9d3ff640 1 --2- 172.21.15.99:0/3097585968 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f1f840a27f0 0x7f1f840a4cb0 secure :-1 s=READY pgs=310 cs=0 l=1 rev1=1 crypto rx=0x55a5c0527530 tx=0x7f1f90004a40 comp rx=0 tx=0).stop 2024-03-19T01:51:50.497 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.495+0000 7f1f9d3ff640 1 -- 172.21.15.99:0/3097585968 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1f981005b0 msgr2=0x7f1f98104540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:51:50.497 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.495+0000 7f1f9d3ff640 1 --2- 172.21.15.99:0/3097585968 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1f981005b0 0x7f1f98104540 secure :-1 s=READY pgs=107 cs=0 l=1 rev1=1 crypto rx=0x7f1f8800e9c0 tx=0x7f1f8800ee80 comp rx=0 tx=0).stop 2024-03-19T01:51:50.497 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.495+0000 7f1f9d3ff640 1 -- 172.21.15.99:0/3097585968 shutdown_connections 2024-03-19T01:51:50.497 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.495+0000 7f1f9d3ff640 1 --2- 172.21.15.99:0/3097585968 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f1f840a27f0 0x7f1f840a4cb0 unknown :-1 s=CLOSED pgs=310 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:50.497 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.495+0000 7f1f9d3ff640 1 --2- 172.21.15.99:0/3097585968 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1f98102160 0x7f1f981030c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:50.497 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.495+0000 7f1f9d3ff640 1 --2- 172.21.15.99:0/3097585968 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1f981017a0 0x7f1f98102b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:50.497 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.495+0000 7f1f9d3ff640 1 --2- 172.21.15.99:0/3097585968 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1f981005b0 0x7f1f98104540 unknown :-1 s=CLOSED pgs=107 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:51:50.497 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.495+0000 7f1f9d3ff640 1 -- 172.21.15.99:0/3097585968 >> 172.21.15.99:0/3097585968 conn(0x7f1f980fbb60 msgr2=0x7f1f980fd620 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:51:50.497 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.495+0000 7f1f9d3ff640 1 -- 172.21.15.99:0/3097585968 shutdown_connections 2024-03-19T01:51:50.497 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:51:50.495+0000 7f1f9d3ff640 1 -- 172.21.15.99:0/3097585968 wait complete. 2024-03-19T01:51:50.701 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1306130 11.30 MB/sec execute 500 sec latency 54.970 ms 2024-03-19T01:51:51.701 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1307821 11.29 MB/sec execute 501 sec latency 17.122 ms 2024-03-19T01:51:52.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:51:51 smithi115 ceph-mon[67332]: pgmap v673: 97 pgs: 97 active+clean; 271 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 893 KiB/s rd, 2.8 MiB/s wr, 265 op/s 2024-03-19T01:51:52.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:51:51 smithi099 ceph-mon[62951]: pgmap v673: 97 pgs: 97 active+clean; 271 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 893 KiB/s rd, 2.8 MiB/s wr, 265 op/s 2024-03-19T01:51:52.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:51:51 smithi112 ceph-mon[66819]: pgmap v673: 97 pgs: 97 active+clean; 271 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 893 KiB/s rd, 2.8 MiB/s wr, 265 op/s 2024-03-19T01:51:52.701 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1310434 11.30 MB/sec execute 502 sec latency 10.767 ms 2024-03-19T01:51:53.701 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1311978 11.30 MB/sec execute 503 sec latency 35.385 ms 2024-03-19T01:51:54.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:51:53 smithi115 ceph-mon[67332]: pgmap v674: 97 pgs: 97 active+clean; 267 MiB data, 3.3 GiB used, 1.0 TiB / 1.0 TiB avail; 768 KiB/s rd, 2.8 MiB/s wr, 243 op/s 2024-03-19T01:51:54.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:51:53 smithi099 ceph-mon[62951]: pgmap v674: 97 pgs: 97 active+clean; 267 MiB data, 3.3 GiB used, 1.0 TiB / 1.0 TiB avail; 768 KiB/s rd, 2.8 MiB/s wr, 243 op/s 2024-03-19T01:51:54.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:51:53 smithi112 ceph-mon[66819]: pgmap v674: 97 pgs: 97 active+clean; 267 MiB data, 3.3 GiB used, 1.0 TiB / 1.0 TiB avail; 768 KiB/s rd, 2.8 MiB/s wr, 243 op/s 2024-03-19T01:51:54.701 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1314242 11.30 MB/sec execute 504 sec latency 87.790 ms 2024-03-19T01:51:55.701 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1315556 11.29 MB/sec execute 505 sec latency 53.061 ms 2024-03-19T01:51:56.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:51:55 smithi115 ceph-mon[67332]: pgmap v675: 97 pgs: 97 active+clean; 269 MiB data, 3.3 GiB used, 1.0 TiB / 1.0 TiB avail; 896 KiB/s rd, 2.6 MiB/s wr, 250 op/s 2024-03-19T01:51:56.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:51:55 smithi099 ceph-mon[62951]: pgmap v675: 97 pgs: 97 active+clean; 269 MiB data, 3.3 GiB used, 1.0 TiB / 1.0 TiB avail; 896 KiB/s rd, 2.6 MiB/s wr, 250 op/s 2024-03-19T01:51:56.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:51:55 smithi112 ceph-mon[66819]: pgmap v675: 97 pgs: 97 active+clean; 269 MiB data, 3.3 GiB used, 1.0 TiB / 1.0 TiB avail; 896 KiB/s rd, 2.6 MiB/s wr, 250 op/s 2024-03-19T01:51:56.701 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1317798 11.29 MB/sec execute 506 sec latency 40.723 ms 2024-03-19T01:51:57.701 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1319638 11.28 MB/sec execute 507 sec latency 30.083 ms 2024-03-19T01:51:58.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:51:57 smithi115 ceph-mon[67332]: pgmap v676: 97 pgs: 97 active+clean; 271 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 3.2 MiB/s wr, 301 op/s 2024-03-19T01:51:58.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:51:57 smithi099 ceph-mon[62951]: pgmap v676: 97 pgs: 97 active+clean; 271 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 3.2 MiB/s wr, 301 op/s 2024-03-19T01:51:58.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:51:57 smithi112 ceph-mon[66819]: pgmap v676: 97 pgs: 97 active+clean; 271 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 3.2 MiB/s wr, 301 op/s 2024-03-19T01:51:58.701 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1321662 11.29 MB/sec execute 508 sec latency 9.515 ms 2024-03-19T01:51:59.701 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1323696 11.28 MB/sec execute 509 sec latency 44.779 ms 2024-03-19T01:52:00.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:51:59 smithi115 ceph-mon[67332]: pgmap v677: 97 pgs: 97 active+clean; 271 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 739 KiB/s rd, 2.9 MiB/s wr, 238 op/s 2024-03-19T01:52:00.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:51:59 smithi099 ceph-mon[62951]: pgmap v677: 97 pgs: 97 active+clean; 271 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 739 KiB/s rd, 2.9 MiB/s wr, 238 op/s 2024-03-19T01:52:00.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:51:59 smithi112 ceph-mon[66819]: pgmap v677: 97 pgs: 97 active+clean; 271 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 739 KiB/s rd, 2.9 MiB/s wr, 238 op/s 2024-03-19T01:52:00.701 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1325426 11.28 MB/sec execute 510 sec latency 75.419 ms 2024-03-19T01:52:01.701 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1327811 11.28 MB/sec execute 511 sec latency 22.404 ms 2024-03-19T01:52:02.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:52:01 smithi115 ceph-mon[67332]: pgmap v678: 97 pgs: 97 active+clean; 272 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 742 KiB/s rd, 2.9 MiB/s wr, 240 op/s 2024-03-19T01:52:02.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:52:01 smithi099 ceph-mon[62951]: pgmap v678: 97 pgs: 97 active+clean; 272 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 742 KiB/s rd, 2.9 MiB/s wr, 240 op/s 2024-03-19T01:52:02.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:52:01 smithi112 ceph-mon[66819]: pgmap v678: 97 pgs: 97 active+clean; 272 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 742 KiB/s rd, 2.9 MiB/s wr, 240 op/s 2024-03-19T01:52:02.702 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1329363 11.28 MB/sec execute 512 sec latency 55.195 ms 2024-03-19T01:52:03.587 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.585+0000 7fde86c1f640 1 -- 172.21.15.99:0/1921666588 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fde80102a40 msgr2=0x7fde80102ec0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:52:03.587 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.585+0000 7fde86c1f640 1 --2- 172.21.15.99:0/1921666588 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fde80102a40 0x7fde80102ec0 secure :-1 s=READY pgs=108 cs=0 l=1 rev1=1 crypto rx=0x7fde6c009900 tx=0x7fde6c02bc30 comp rx=0 tx=0).stop 2024-03-19T01:52:03.587 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.585+0000 7fde86c1f640 1 -- 172.21.15.99:0/1921666588 shutdown_connections 2024-03-19T01:52:03.587 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.585+0000 7fde86c1f640 1 --2- 172.21.15.99:0/1921666588 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fde80103400 0x7fde80109c90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:03.587 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.585+0000 7fde86c1f640 1 --2- 172.21.15.99:0/1921666588 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fde80102a40 0x7fde80102ec0 unknown :-1 s=CLOSED pgs=108 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:03.587 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.585+0000 7fde86c1f640 1 --2- 172.21.15.99:0/1921666588 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fde80101850 0x7fde80101c70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:03.587 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.585+0000 7fde86c1f640 1 -- 172.21.15.99:0/1921666588 >> 172.21.15.99:0/1921666588 conn(0x7fde800fcde0 msgr2=0x7fde800ff200 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:52:03.588 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.585+0000 7fde86c1f640 1 -- 172.21.15.99:0/1921666588 shutdown_connections 2024-03-19T01:52:03.588 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.585+0000 7fde86c1f640 1 -- 172.21.15.99:0/1921666588 wait complete. 2024-03-19T01:52:03.588 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.585+0000 7fde86c1f640 1 Processor -- start 2024-03-19T01:52:03.588 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.586+0000 7fde86c1f640 1 -- start start 2024-03-19T01:52:03.588 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.586+0000 7fde86c1f640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fde80101850 0x7fde80197a30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:03.588 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.586+0000 7fde86c1f640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fde80102a40 0x7fde80197f70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:03.588 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.586+0000 7fde86c1f640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fde80103400 0x7fde8019efe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:03.588 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.586+0000 7fde86c1f640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fde8010d4b0 con 0x7fde80102a40 2024-03-19T01:52:03.588 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.586+0000 7fde86c1f640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fde8010d330 con 0x7fde80101850 2024-03-19T01:52:03.588 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.586+0000 7fde86c1f640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fde8010d630 con 0x7fde80103400 2024-03-19T01:52:03.588 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.586+0000 7fde7ffff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fde80102a40 0x7fde80197f70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:03.589 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.586+0000 7fde7ffff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fde80102a40 0x7fde80197f70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:59246/0 (socket says 172.21.15.99:59246) 2024-03-19T01:52:03.589 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.586+0000 7fde7ffff640 1 -- 172.21.15.99:0/1334800361 learned_addr learned my addr 172.21.15.99:0/1334800361 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:52:03.589 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.586+0000 7fde849bb640 1 --2- 172.21.15.99:0/1334800361 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fde80101850 0x7fde80197a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:03.589 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.586+0000 7fde851bc640 1 --2- 172.21.15.99:0/1334800361 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fde80103400 0x7fde8019efe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:03.589 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.586+0000 7fde7ffff640 1 -- 172.21.15.99:0/1334800361 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fde80103400 msgr2=0x7fde8019efe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:52:03.589 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.586+0000 7fde7ffff640 1 --2- 172.21.15.99:0/1334800361 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fde80103400 0x7fde8019efe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:03.589 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.586+0000 7fde7ffff640 1 -- 172.21.15.99:0/1334800361 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fde80101850 msgr2=0x7fde80197a30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:52:03.589 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.586+0000 7fde7ffff640 1 --2- 172.21.15.99:0/1334800361 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fde80101850 0x7fde80197a30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:03.589 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.586+0000 7fde7ffff640 1 -- 172.21.15.99:0/1334800361 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fde8019f650 con 0x7fde80102a40 2024-03-19T01:52:03.589 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.586+0000 7fde849bb640 1 --2- 172.21.15.99:0/1334800361 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fde80101850 0x7fde80197a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:52:03.589 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.586+0000 7fde7ffff640 1 --2- 172.21.15.99:0/1334800361 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fde80102a40 0x7fde80197f70 secure :-1 s=READY pgs=554 cs=0 l=1 rev1=1 crypto rx=0x7fde6c009230 tx=0x7fde6c02c650 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:52:03.589 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.587+0000 7fde7dffb640 1 -- 172.21.15.99:0/1334800361 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fde6c041070 con 0x7fde80102a40 2024-03-19T01:52:03.589 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.587+0000 7fde86c1f640 1 -- 172.21.15.99:0/1334800361 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fde8019f8e0 con 0x7fde80102a40 2024-03-19T01:52:03.589 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.587+0000 7fde7dffb640 1 -- 172.21.15.99:0/1334800361 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fde6c004450 con 0x7fde80102a40 2024-03-19T01:52:03.589 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.587+0000 7fde7dffb640 1 -- 172.21.15.99:0/1334800361 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fde6c003e20 con 0x7fde80102a40 2024-03-19T01:52:03.590 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.587+0000 7fde86c1f640 1 -- 172.21.15.99:0/1334800361 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fde8019fdd0 con 0x7fde80102a40 2024-03-19T01:52:03.596 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.589+0000 7fde7dffb640 1 -- 172.21.15.99:0/1334800361 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7fde6c044070 con 0x7fde80102a40 2024-03-19T01:52:03.597 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.589+0000 7fde7dffb640 1 --2- 172.21.15.99:0/1334800361 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fde700a28a0 0x7fde700a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:03.597 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.589+0000 7fde89805740 1 Processor -- start 2024-03-19T01:52:03.597 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.589+0000 7fde849bb640 1 --2- 172.21.15.99:0/1334800361 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fde700a28a0 0x7fde700a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:03.597 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.589+0000 7fde89805740 1 -- start start 2024-03-19T01:52:03.597 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.589+0000 7fde7dffb640 1 -- 172.21.15.99:0/1334800361 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7fde6c0f0650 con 0x7fde80102a40 2024-03-19T01:52:03.597 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.590+0000 7fde89805740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563a16ec5100 0x563a16ec5520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:03.597 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.590+0000 7fde89805740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563a16ec5af0 0x563a16eca5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:03.597 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.590+0000 7fde89805740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563a16ecac60 0x563a16ecd0f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:03.597 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.590+0000 7fde89805740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x563a16e29be0 con 0x563a16ecac60 2024-03-19T01:52:03.597 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.590+0000 7fde89805740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x563a16dd2960 con 0x563a16ec5100 2024-03-19T01:52:03.597 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.590+0000 7fde89805740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x563a16e25c80 con 0x563a16ec5af0 2024-03-19T01:52:03.597 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.590+0000 7fde851bc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563a16ecac60 0x563a16ecd0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:03.597 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.590+0000 7fde7ffff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563a16ec5af0 0x563a16eca5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:03.598 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.590+0000 7fde7ffff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563a16ec5af0 0x563a16eca5f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:51458/0 (socket says 172.21.15.99:51458) 2024-03-19T01:52:03.598 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.590+0000 7fde851bc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563a16ecac60 0x563a16ecd0f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:59262/0 (socket says 172.21.15.99:59262) 2024-03-19T01:52:03.598 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.590+0000 7fde7ffff640 1 -- 172.21.15.99:0/1942903720 learned_addr learned my addr 172.21.15.99:0/1942903720 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:52:03.598 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.590+0000 7fde849bb640 1 --2- 172.21.15.99:0/1942903720 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563a16ec5100 0x563a16ec5520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:03.598 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.590+0000 7fde849bb640 1 --2- 172.21.15.99:0/1334800361 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fde700a28a0 0x7fde700a4d60 secure :-1 s=READY pgs=311 cs=0 l=1 rev1=1 crypto rx=0x7fde801028a0 tx=0x7fde74005f90 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:52:03.598 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.590+0000 7fde7ffff640 1 -- 172.21.15.99:0/1942903720 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563a16ec5100 msgr2=0x563a16ec5520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:52:03.598 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.590+0000 7fde7ffff640 1 --2- 172.21.15.99:0/1942903720 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563a16ec5100 0x563a16ec5520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:03.598 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.590+0000 7fde7ffff640 1 -- 172.21.15.99:0/1942903720 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563a16ecac60 msgr2=0x563a16ecd0f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:52:03.598 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.590+0000 7fde7ffff640 1 --2- 172.21.15.99:0/1942903720 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563a16ecac60 0x563a16ecd0f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:03.598 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.590+0000 7fde7ffff640 1 -- 172.21.15.99:0/1942903720 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x563a16dc9d20 con 0x563a16ec5af0 2024-03-19T01:52:03.598 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.590+0000 7fde851bc640 1 --2- 172.21.15.99:0/1942903720 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563a16ecac60 0x563a16ecd0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:52:03.599 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.590+0000 7fde7ffff640 1 --2- 172.21.15.99:0/1942903720 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563a16ec5af0 0x563a16eca5f0 secure :-1 s=READY pgs=109 cs=0 l=1 rev1=1 crypto rx=0x7fde6c009430 tx=0x7fde6c005e50 comp rx=0 tx=0).ready entity=mon.1 client_cookie=6c0d72889d80ac09 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:52:03.599 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.590+0000 7fde6affd640 1 -- 172.21.15.99:0/1942903720 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fde6c04b070 con 0x563a16ec5af0 2024-03-19T01:52:03.599 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.590+0000 7fde6affd640 1 -- 172.21.15.99:0/1942903720 <== mon.1 v2:172.21.15.115:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fde6c0c1100 con 0x563a16ec5af0 2024-03-19T01:52:03.599 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.591+0000 7fde6affd640 1 -- 172.21.15.99:0/1942903720 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fde6c0b7bd0 con 0x563a16ec5af0 2024-03-19T01:52:03.599 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.591+0000 7fde89805740 1 -- 172.21.15.99:0/1942903720 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563a16ec5af0 msgr2=0x563a16eca5f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:52:03.599 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.591+0000 7fde89805740 1 --2- 172.21.15.99:0/1942903720 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563a16ec5af0 0x563a16eca5f0 secure :-1 s=READY pgs=109 cs=0 l=1 rev1=1 crypto rx=0x7fde6c009430 tx=0x7fde6c005e50 comp rx=0 tx=0).stop 2024-03-19T01:52:03.599 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.591+0000 7fde89805740 1 -- 172.21.15.99:0/1942903720 shutdown_connections 2024-03-19T01:52:03.599 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.591+0000 7fde89805740 1 --2- 172.21.15.99:0/1942903720 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563a16ecac60 0x563a16ecd0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:03.599 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.591+0000 7fde89805740 1 --2- 172.21.15.99:0/1942903720 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563a16ec5af0 0x563a16eca5f0 unknown :-1 s=CLOSED pgs=109 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:03.599 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.591+0000 7fde89805740 1 --2- 172.21.15.99:0/1942903720 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563a16ec5100 0x563a16ec5520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:03.599 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.591+0000 7fde89805740 1 -- 172.21.15.99:0/1942903720 >> 172.21.15.99:0/1942903720 conn(0x563a16ec0670 msgr2=0x563a16ec0a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:52:03.599 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.591+0000 7fde89805740 1 -- 172.21.15.99:0/1942903720 shutdown_connections 2024-03-19T01:52:03.599 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.591+0000 7fde89805740 1 -- 172.21.15.99:0/1942903720 wait complete. 2024-03-19T01:52:03.599 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.592+0000 7fde89805740 1 -- start start 2024-03-19T01:52:03.599 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.592+0000 7fde89805740 1 Processor -- start 2024-03-19T01:52:03.600 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.592+0000 7fde89805740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563a16ec5100 0x563a16ecedf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:03.600 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.592+0000 7fde89805740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563a16ec5af0 0x563a16ecf330 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:03.600 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.592+0000 7fde89805740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563a16ecac60 0x563a16ed6370 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:03.600 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.592+0000 7fde89805740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x563a16ecd9d0 con 0x563a16ecac60 2024-03-19T01:52:03.600 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.592+0000 7fde89805740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x563a16de5db0 con 0x563a16ec5100 2024-03-19T01:52:03.600 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.592+0000 7fde89805740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x563a16ecdb50 con 0x563a16ec5af0 2024-03-19T01:52:03.600 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.592+0000 7fde89805740 10 client.0 _ll_register_callbacks cb 0x563a16deae70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:52:03.600 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.592+0000 7fde851bc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563a16ecac60 0x563a16ed6370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:03.600 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.592+0000 7fde7ffff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563a16ec5af0 0x563a16ecf330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:03.600 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.592+0000 7fde849bb640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563a16ec5100 0x563a16ecedf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:03.600 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.592+0000 7fde7ffff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563a16ec5af0 0x563a16ecf330 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:51460/0 (socket says 172.21.15.99:51460) 2024-03-19T01:52:03.600 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.592+0000 7fde7ffff640 1 -- 172.21.15.99:0/1693238040 learned_addr learned my addr 172.21.15.99:0/1693238040 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:52:03.600 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.592+0000 7fde851bc640 1 -- 172.21.15.99:0/1693238040 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563a16ec5af0 msgr2=0x563a16ecf330 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:52:03.600 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.592+0000 7fde851bc640 1 --2- 172.21.15.99:0/1693238040 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563a16ec5af0 0x563a16ecf330 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:03.601 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.592+0000 7fde851bc640 1 -- 172.21.15.99:0/1693238040 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563a16ec5100 msgr2=0x563a16ecedf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:52:03.601 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.592+0000 7fde851bc640 1 --2- 172.21.15.99:0/1693238040 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563a16ec5100 0x563a16ecedf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:03.601 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.592+0000 7fde851bc640 1 -- 172.21.15.99:0/1693238040 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x563a16de4c60 con 0x563a16ecac60 2024-03-19T01:52:03.601 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.592+0000 7fde849bb640 1 --2- 172.21.15.99:0/1693238040 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563a16ec5100 0x563a16ecedf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:52:03.601 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.593+0000 7fde851bc640 1 --2- 172.21.15.99:0/1693238040 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563a16ecac60 0x563a16ed6370 secure :-1 s=READY pgs=555 cs=0 l=1 rev1=1 crypto rx=0x7fde78005fd0 tx=0x7fde7800f790 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8e40be75402f4a47 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:52:03.601 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.593+0000 7fde5ffff640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:52:03.601 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.593+0000 7fde5ffff640 1 -- 172.21.15.99:0/1693238040 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fde78009e30 con 0x563a16ecac60 2024-03-19T01:52:03.601 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.593+0000 7fde5ffff640 1 -- 172.21.15.99:0/1693238040 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7fde78003970 con 0x563a16ecac60 2024-03-19T01:52:03.601 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.593+0000 7fde89805740 1 -- 172.21.15.99:0/1693238040 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fde6c0c1100 con 0x563a16ecac60 2024-03-19T01:52:03.601 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.593+0000 7fde5ffff640 1 -- 172.21.15.99:0/1693238040 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7fde78003b90 con 0x563a16ecac60 2024-03-19T01:52:03.601 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.593+0000 7fde89805740 10 client.16224 fetch_fsmap learned FSMap version 27 2024-03-19T01:52:03.601 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.593+0000 7fde89805740 1 -- 172.21.15.99:0/1693238040 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x563a16b3e210 con 0x563a16ecac60 2024-03-19T01:52:03.601 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.593+0000 7fde5ffff640 1 -- 172.21.15.99:0/1693238040 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fde6c0c1100 con 0x563a16ecac60 2024-03-19T01:52:03.601 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.593+0000 7fde5ffff640 1 -- 172.21.15.99:0/1693238040 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 27) ==== 8338+0+0 (secure 0 0 0) 0x7fde7801a410 con 0x563a16ecac60 2024-03-19T01:52:03.601 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.593+0000 7fde89805740 10 client.16224 fetch_fsmap finished waiting for FSMap version 27 2024-03-19T01:52:03.601 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.593+0000 7fde89805740 10 client.16224 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:52:03.602 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.593+0000 7fde89805740 20 client.16224 populate_metadata read hostname 'smithi099' 2024-03-19T01:52:03.602 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.593+0000 7fde89805740 1 --2- 172.21.15.99:0/1693238040 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x563a16edacc0 0x563a16edd0b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:03.602 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.593+0000 7fde89805740 4 client.16224 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:52:03.602 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.593+0000 7fde89805740 1 -- 172.21.15.99:0/1693238040 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x563a16e37120 con 0x563a16edacc0 2024-03-19T01:52:03.602 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.594+0000 7fde849bb640 1 --2- 172.21.15.99:0/1693238040 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x563a16edacc0 0x563a16edd0b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:03.602 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.594+0000 7fde5ffff640 10 client.16224 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:52:03.602 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.594+0000 7fde849bb640 1 --2- 172.21.15.99:0/1693238040 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x563a16edacc0 0x563a16edd0b0 crc :-1 s=READY pgs=127 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=5adc432dfa0d5918 server_cookie=885b830fbe872864 in_seq=0 out_seq=0 2024-03-19T01:52:03.602 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.595+0000 7fde5ffff640 1 -- 172.21.15.99:0/1693238040 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x563a16e37120 con 0x563a16edacc0 2024-03-19T01:52:03.602 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.595+0000 7fde5ffff640 10 client.16224 handle_command_reply: tid=0 2024-03-19T01:52:03.602 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.595+0000 7fde89805740 1 client.16224 shutdown 2024-03-19T01:52:03.602 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.595+0000 7fde89805740 1 -- 172.21.15.99:0/1693238040 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563a16ecac60 msgr2=0x563a16ed6370 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:52:03.602 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.595+0000 7fde89805740 1 --2- 172.21.15.99:0/1693238040 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563a16ecac60 0x563a16ed6370 secure :-1 s=READY pgs=555 cs=0 l=1 rev1=1 crypto rx=0x7fde78005fd0 tx=0x7fde7800f790 comp rx=0 tx=0).stop 2024-03-19T01:52:03.602 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.595+0000 7fde89805740 1 -- 172.21.15.99:0/1693238040 shutdown_connections 2024-03-19T01:52:03.602 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.595+0000 7fde89805740 1 --2- 172.21.15.99:0/1693238040 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x563a16edacc0 0x563a16edd0b0 crc :-1 s=READY pgs=127 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:03.603 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.595+0000 7fde89805740 1 --2- 172.21.15.99:0/1693238040 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563a16ecac60 0x563a16ed6370 unknown :-1 s=CLOSED pgs=555 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:03.603 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.595+0000 7fde89805740 1 --2- 172.21.15.99:0/1693238040 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563a16ec5af0 0x563a16ecf330 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:03.603 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.595+0000 7fde89805740 1 --2- 172.21.15.99:0/1693238040 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563a16ec5100 0x563a16ecedf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:03.603 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.595+0000 7fde5ffff640 0 client.16224 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:52:03.603 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.595+0000 7fde89805740 1 -- 172.21.15.99:0/1693238040 >> 172.21.15.99:0/1693238040 conn(0x563a16ec0670 msgr2=0x563a16ec18c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:52:03.603 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.595+0000 7fde89805740 1 -- 172.21.15.99:0/1693238040 shutdown_connections 2024-03-19T01:52:03.603 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.595+0000 7fde89805740 1 -- 172.21.15.99:0/1693238040 wait complete. 2024-03-19T01:52:03.603 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.595+0000 7fde89805740 20 client.16224 trim_cache size 0 max 16384 2024-03-19T01:52:03.615 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.611+0000 7fde89805740 1 Processor -- start 2024-03-19T01:52:03.615 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.611+0000 7fde89805740 1 -- start start 2024-03-19T01:52:03.615 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.611+0000 7fde89805740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563a16ec5100 0x563a16ebf220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:03.615 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.611+0000 7fde89805740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563a16ec5af0 0x563a16ed6330 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:03.615 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.611+0000 7fde89805740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563a16ecac60 0x563a16ebf760 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:03.615 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.611+0000 7fde89805740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fde801ab6a0 con 0x563a16ec5af0 2024-03-19T01:52:03.615 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.611+0000 7fde89805740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fde801a8f70 con 0x563a16ec5100 2024-03-19T01:52:03.615 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.611+0000 7fde89805740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fde801a6dc0 con 0x563a16ecac60 2024-03-19T01:52:03.616 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.611+0000 7fde7ffff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563a16ec5af0 0x563a16ed6330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:03.616 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.611+0000 7fde7ffff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563a16ec5af0 0x563a16ed6330 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:59276/0 (socket says 172.21.15.99:59276) 2024-03-19T01:52:03.616 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.611+0000 7fde849bb640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563a16ec5100 0x563a16ebf220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:03.616 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.611+0000 7fde7ffff640 1 -- 172.21.15.99:0/978024221 learned_addr learned my addr 172.21.15.99:0/978024221 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:52:03.616 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.611+0000 7fde851bc640 1 --2- 172.21.15.99:0/978024221 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563a16ecac60 0x563a16ebf760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:03.616 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.611+0000 7fde849bb640 1 -- 172.21.15.99:0/978024221 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563a16ecac60 msgr2=0x563a16ebf760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:52:03.616 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.611+0000 7fde849bb640 1 --2- 172.21.15.99:0/978024221 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563a16ecac60 0x563a16ebf760 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:03.616 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.611+0000 7fde849bb640 1 -- 172.21.15.99:0/978024221 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563a16ec5af0 msgr2=0x563a16ed6330 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:52:03.616 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.611+0000 7fde849bb640 1 --2- 172.21.15.99:0/978024221 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563a16ec5af0 0x563a16ed6330 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:03.616 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.611+0000 7fde849bb640 1 -- 172.21.15.99:0/978024221 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x563a16c0a620 con 0x563a16ec5100 2024-03-19T01:52:03.616 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.611+0000 7fde851bc640 1 --2- 172.21.15.99:0/978024221 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563a16ecac60 0x563a16ebf760 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:52:03.616 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.611+0000 7fde7ffff640 1 --2- 172.21.15.99:0/978024221 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563a16ec5af0 0x563a16ed6330 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:52:03.616 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.612+0000 7fde849bb640 1 --2- 172.21.15.99:0/978024221 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563a16ec5100 0x563a16ebf220 secure :-1 s=READY pgs=150 cs=0 l=1 rev1=1 crypto rx=0x563a16edd900 tx=0x7fde74014000 comp rx=0 tx=0).ready entity=mon.2 client_cookie=b45069cd4565840d server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:52:03.616 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.612+0000 7fde6a7fc640 1 -- 172.21.15.99:0/978024221 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fde74031070 con 0x563a16ec5100 2024-03-19T01:52:03.616 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.612+0000 7fde6a7fc640 1 -- 172.21.15.99:0/978024221 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fde740035f0 con 0x563a16ec5100 2024-03-19T01:52:03.616 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.612+0000 7fde6a7fc640 1 -- 172.21.15.99:0/978024221 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fde7401c670 con 0x563a16ec5100 2024-03-19T01:52:03.617 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.613+0000 7fde89805740 1 -- 172.21.15.99:0/978024221 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563a16ec5100 msgr2=0x563a16ebf220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:52:03.617 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.613+0000 7fde89805740 1 --2- 172.21.15.99:0/978024221 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563a16ec5100 0x563a16ebf220 secure :-1 s=READY pgs=150 cs=0 l=1 rev1=1 crypto rx=0x563a16edd900 tx=0x7fde74014000 comp rx=0 tx=0).stop 2024-03-19T01:52:03.617 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.613+0000 7fde89805740 1 -- 172.21.15.99:0/978024221 shutdown_connections 2024-03-19T01:52:03.617 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.613+0000 7fde89805740 1 --2- 172.21.15.99:0/978024221 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563a16ecac60 0x563a16ebf760 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:03.617 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.613+0000 7fde89805740 1 --2- 172.21.15.99:0/978024221 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563a16ec5af0 0x563a16ed6330 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:03.617 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.613+0000 7fde89805740 1 --2- 172.21.15.99:0/978024221 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563a16ec5100 0x563a16ebf220 unknown :-1 s=CLOSED pgs=150 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:03.617 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.613+0000 7fde89805740 1 -- 172.21.15.99:0/978024221 >> 172.21.15.99:0/978024221 conn(0x563a16ec0670 msgr2=0x563a16ebb940 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:52:03.617 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.613+0000 7fde89805740 1 -- 172.21.15.99:0/978024221 shutdown_connections 2024-03-19T01:52:03.617 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.613+0000 7fde89805740 1 -- 172.21.15.99:0/978024221 wait complete. 2024-03-19T01:52:03.617 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.613+0000 7fde89805740 1 -- start start 2024-03-19T01:52:03.617 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.613+0000 7fde89805740 1 Processor -- start 2024-03-19T01:52:03.617 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.613+0000 7fde89805740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563a16ec5100 0x563a16ed3190 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:03.617 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.613+0000 7fde89805740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563a16ec5af0 0x563a16ed36d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:03.617 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.613+0000 7fde89805740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563a16ecac60 0x563a16f3efe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:03.617 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.613+0000 7fde89805740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fde801a7430 con 0x563a16ecac60 2024-03-19T01:52:03.618 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.613+0000 7fde89805740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fde801a88a0 con 0x563a16ec5af0 2024-03-19T01:52:03.618 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.613+0000 7fde89805740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fde801a7b00 con 0x563a16ec5100 2024-03-19T01:52:03.618 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.613+0000 7fde89805740 10 client.0 _ll_register_callbacks cb 0x7fde78009cc0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:52:03.618 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.613+0000 7fde851bc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563a16ecac60 0x563a16f3efe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:03.618 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.614+0000 7fde849bb640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563a16ec5100 0x563a16ed3190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:03.618 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.614+0000 7fde851bc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563a16ecac60 0x563a16f3efe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:59292/0 (socket says 172.21.15.99:59292) 2024-03-19T01:52:03.618 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.614+0000 7fde851bc640 1 -- 172.21.15.99:0/2339939962 learned_addr learned my addr 172.21.15.99:0/2339939962 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:52:03.618 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.614+0000 7fde7ffff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563a16ec5af0 0x563a16ed36d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:03.618 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.614+0000 7fde851bc640 1 -- 172.21.15.99:0/2339939962 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563a16ec5100 msgr2=0x563a16ed3190 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:52:03.618 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.614+0000 7fde851bc640 1 --2- 172.21.15.99:0/2339939962 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563a16ec5100 0x563a16ed3190 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:03.618 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.614+0000 7fde851bc640 1 -- 172.21.15.99:0/2339939962 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563a16ec5af0 msgr2=0x563a16ed36d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:52:03.618 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.614+0000 7fde851bc640 1 --2- 172.21.15.99:0/2339939962 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563a16ec5af0 0x563a16ed36d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:03.618 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.614+0000 7fde851bc640 1 -- 172.21.15.99:0/2339939962 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x563a16f3f520 con 0x563a16ecac60 2024-03-19T01:52:03.619 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.614+0000 7fde7ffff640 1 --2- 172.21.15.99:0/2339939962 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563a16ec5af0 0x563a16ed36d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:52:03.619 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.614+0000 7fde851bc640 1 --2- 172.21.15.99:0/2339939962 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563a16ecac60 0x563a16f3efe0 secure :-1 s=READY pgs=556 cs=0 l=1 rev1=1 crypto rx=0x7fde78000f80 tx=0x7fde780119f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c5ed3c6c9c825860 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:52:03.619 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.614+0000 7fde5ffff640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:52:03.619 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.614+0000 7fde5ffff640 1 -- 172.21.15.99:0/2339939962 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fde78028030 con 0x563a16ecac60 2024-03-19T01:52:03.619 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.614+0000 7fde5ffff640 1 -- 172.21.15.99:0/2339939962 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7fde7801a600 con 0x563a16ecac60 2024-03-19T01:52:03.619 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.614+0000 7fde89805740 1 -- 172.21.15.99:0/2339939962 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fde740035f0 con 0x563a16ecac60 2024-03-19T01:52:03.619 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.614+0000 7fde5ffff640 1 -- 172.21.15.99:0/2339939962 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7fde7802d440 con 0x563a16ecac60 2024-03-19T01:52:03.619 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.614+0000 7fde89805740 10 client.16230 fetch_fsmap learned FSMap version 27 2024-03-19T01:52:03.619 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.614+0000 7fde89805740 1 -- 172.21.15.99:0/2339939962 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x563a16f40720 con 0x563a16ecac60 2024-03-19T01:52:03.619 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.614+0000 7fde5ffff640 1 -- 172.21.15.99:0/2339939962 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fde740035f0 con 0x563a16ecac60 2024-03-19T01:52:03.619 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.615+0000 7fde5ffff640 1 -- 172.21.15.99:0/2339939962 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 27) ==== 8338+0+0 (secure 0 0 0) 0x7fde78038730 con 0x563a16ecac60 2024-03-19T01:52:03.619 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.615+0000 7fde89805740 10 client.16230 fetch_fsmap finished waiting for FSMap version 27 2024-03-19T01:52:03.619 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.615+0000 7fde89805740 10 client.16230 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:52:03.619 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.615+0000 7fde89805740 20 client.16230 populate_metadata read hostname 'smithi099' 2024-03-19T01:52:03.619 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.615+0000 7fde89805740 1 --2- 172.21.15.99:0/2339939962 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x563a16f410f0 0x563a16f43510 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:03.620 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.615+0000 7fde89805740 4 client.16230 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:52:03.620 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.615+0000 7fde89805740 1 -- 172.21.15.99:0/2339939962 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x563a16e31470 con 0x563a16f410f0 2024-03-19T01:52:03.620 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.615+0000 7fde849bb640 1 --2- 172.21.15.99:0/2339939962 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x563a16f410f0 0x563a16f43510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:03.620 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.615+0000 7fde5ffff640 10 client.16230 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:52:03.620 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.615+0000 7fde849bb640 1 --2- 172.21.15.99:0/2339939962 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x563a16f410f0 0x563a16f43510 crc :-1 s=READY pgs=128 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=655351c66cefe70f server_cookie=cb402c5b1742c447 in_seq=0 out_seq=0 2024-03-19T01:52:03.676 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.675+0000 7fde5ffff640 1 -- 172.21.15.99:0/2339939962 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x563a16e31470 con 0x563a16f410f0 2024-03-19T01:52:03.676 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.675+0000 7fde5ffff640 10 client.16230 handle_command_reply: tid=0 2024-03-19T01:52:03.676 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.675+0000 7fde89805740 1 client.16230 shutdown 2024-03-19T01:52:03.676 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.676+0000 7fde89805740 1 -- 172.21.15.99:0/2339939962 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563a16ecac60 msgr2=0x563a16f3efe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:52:03.677 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.676+0000 7fde89805740 1 --2- 172.21.15.99:0/2339939962 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563a16ecac60 0x563a16f3efe0 secure :-1 s=READY pgs=556 cs=0 l=1 rev1=1 crypto rx=0x7fde78000f80 tx=0x7fde780119f0 comp rx=0 tx=0).stop 2024-03-19T01:52:03.677 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.676+0000 7fde89805740 1 -- 172.21.15.99:0/2339939962 shutdown_connections 2024-03-19T01:52:03.677 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.676+0000 7fde89805740 1 --2- 172.21.15.99:0/2339939962 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x563a16f410f0 0x563a16f43510 crc :-1 s=READY pgs=128 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:03.677 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.676+0000 7fde89805740 1 --2- 172.21.15.99:0/2339939962 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563a16ec5af0 0x563a16ed36d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:03.677 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.676+0000 7fde89805740 1 --2- 172.21.15.99:0/2339939962 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563a16ecac60 0x563a16f3efe0 unknown :-1 s=CLOSED pgs=556 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:03.677 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.676+0000 7fde89805740 1 --2- 172.21.15.99:0/2339939962 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563a16ec5100 0x563a16ed3190 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:03.677 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.676+0000 7fde5ffff640 0 client.16230 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:52:03.677 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.676+0000 7fde89805740 1 -- 172.21.15.99:0/2339939962 >> 172.21.15.99:0/2339939962 conn(0x563a16ec0670 msgr2=0x563a16ebf220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:52:03.677 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.676+0000 7fde89805740 1 -- 172.21.15.99:0/2339939962 shutdown_connections 2024-03-19T01:52:03.677 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.676+0000 7fde89805740 1 -- 172.21.15.99:0/2339939962 wait complete. 2024-03-19T01:52:03.677 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.676+0000 7fde89805740 20 client.16230 trim_cache size 0 max 16384 2024-03-19T01:52:03.678 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:52:03.678 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:52:03.678 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:52:03.678 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:52:03.679 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.676+0000 7fde86c1f640 1 -- 172.21.15.99:0/1334800361 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fde700a28a0 msgr2=0x7fde700a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:52:03.679 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.676+0000 7fde86c1f640 1 --2- 172.21.15.99:0/1334800361 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fde700a28a0 0x7fde700a4d60 secure :-1 s=READY pgs=311 cs=0 l=1 rev1=1 crypto rx=0x7fde801028a0 tx=0x7fde74005f90 comp rx=0 tx=0).stop 2024-03-19T01:52:03.679 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.676+0000 7fde86c1f640 1 -- 172.21.15.99:0/1334800361 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fde80102a40 msgr2=0x7fde80197f70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:52:03.679 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.676+0000 7fde86c1f640 1 --2- 172.21.15.99:0/1334800361 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fde80102a40 0x7fde80197f70 secure :-1 s=READY pgs=554 cs=0 l=1 rev1=1 crypto rx=0x7fde6c009230 tx=0x7fde6c02c650 comp rx=0 tx=0).stop 2024-03-19T01:52:03.679 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.677+0000 7fde86c1f640 1 -- 172.21.15.99:0/1334800361 shutdown_connections 2024-03-19T01:52:03.679 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.677+0000 7fde86c1f640 1 --2- 172.21.15.99:0/1334800361 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fde80103400 0x7fde8019efe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:03.679 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.677+0000 7fde86c1f640 1 --2- 172.21.15.99:0/1334800361 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fde80102a40 0x7fde80197f70 unknown :-1 s=CLOSED pgs=554 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:03.679 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.677+0000 7fde86c1f640 1 --2- 172.21.15.99:0/1334800361 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fde700a28a0 0x7fde700a4d60 unknown :-1 s=CLOSED pgs=311 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:03.679 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.677+0000 7fde86c1f640 1 --2- 172.21.15.99:0/1334800361 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fde80101850 0x7fde80197a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:03.679 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.677+0000 7fde86c1f640 1 -- 172.21.15.99:0/1334800361 >> 172.21.15.99:0/1334800361 conn(0x7fde800fcde0 msgr2=0x7fde80105c70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:52:03.680 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.677+0000 7fde86c1f640 1 -- 172.21.15.99:0/1334800361 shutdown_connections 2024-03-19T01:52:03.680 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:03.677+0000 7fde86c1f640 1 -- 172.21.15.99:0/1334800361 wait complete. 2024-03-19T01:52:03.701 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1331966 11.29 MB/sec execute 513 sec latency 25.895 ms 2024-03-19T01:52:04.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:52:03 smithi115 ceph-mon[67332]: pgmap v679: 97 pgs: 97 active+clean; 275 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 859 KiB/s rd, 3.0 MiB/s wr, 254 op/s 2024-03-19T01:52:04.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:52:03 smithi099 ceph-mon[62951]: pgmap v679: 97 pgs: 97 active+clean; 275 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 859 KiB/s rd, 3.0 MiB/s wr, 254 op/s 2024-03-19T01:52:04.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:52:03 smithi112 ceph-mon[66819]: pgmap v679: 97 pgs: 97 active+clean; 275 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 859 KiB/s rd, 3.0 MiB/s wr, 254 op/s 2024-03-19T01:52:04.702 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1333928 11.28 MB/sec execute 514 sec latency 15.820 ms 2024-03-19T01:52:05.702 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1335848 11.28 MB/sec execute 515 sec latency 25.704 ms 2024-03-19T01:52:06.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:52:05 smithi115 ceph-mon[67332]: pgmap v680: 97 pgs: 97 active+clean; 275 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 745 KiB/s rd, 2.7 MiB/s wr, 219 op/s 2024-03-19T01:52:06.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:52:05 smithi099 ceph-mon[62951]: pgmap v680: 97 pgs: 97 active+clean; 275 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 745 KiB/s rd, 2.7 MiB/s wr, 219 op/s 2024-03-19T01:52:06.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:52:05 smithi112 ceph-mon[66819]: pgmap v680: 97 pgs: 97 active+clean; 275 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 745 KiB/s rd, 2.7 MiB/s wr, 219 op/s 2024-03-19T01:52:06.702 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1337756 11.28 MB/sec execute 516 sec latency 58.209 ms 2024-03-19T01:52:07.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:52:07 smithi115 ceph-mon[67332]: pgmap v681: 97 pgs: 97 active+clean; 256 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 829 KiB/s rd, 3.2 MiB/s wr, 258 op/s 2024-03-19T01:52:07.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:52:07 smithi099 ceph-mon[62951]: pgmap v681: 97 pgs: 97 active+clean; 256 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 829 KiB/s rd, 3.2 MiB/s wr, 258 op/s 2024-03-19T01:52:07.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:52:07 smithi112 ceph-mon[66819]: pgmap v681: 97 pgs: 97 active+clean; 256 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 829 KiB/s rd, 3.2 MiB/s wr, 258 op/s 2024-03-19T01:52:07.702 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1339646 11.28 MB/sec execute 517 sec latency 32.506 ms 2024-03-19T01:52:08.702 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1342441 11.29 MB/sec execute 518 sec latency 14.890 ms 2024-03-19T01:52:09.702 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1344010 11.28 MB/sec execute 519 sec latency 29.020 ms 2024-03-19T01:52:10.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:52:09 smithi115 ceph-mon[67332]: pgmap v682: 97 pgs: 97 active+clean; 252 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 629 KiB/s rd, 2.7 MiB/s wr, 219 op/s 2024-03-19T01:52:10.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:52:09 smithi099 ceph-mon[62951]: pgmap v682: 97 pgs: 97 active+clean; 252 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 629 KiB/s rd, 2.7 MiB/s wr, 219 op/s 2024-03-19T01:52:10.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:52:09 smithi112 ceph-mon[66819]: pgmap v682: 97 pgs: 97 active+clean; 252 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 629 KiB/s rd, 2.7 MiB/s wr, 219 op/s 2024-03-19T01:52:10.702 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1346283 11.28 MB/sec execute 520 sec latency 18.767 ms 2024-03-19T01:52:11.702 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1347804 11.27 MB/sec execute 521 sec latency 65.399 ms 2024-03-19T01:52:12.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:52:11 smithi115 ceph-mon[67332]: pgmap v683: 97 pgs: 97 active+clean; 254 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 792 KiB/s rd, 2.7 MiB/s wr, 249 op/s 2024-03-19T01:52:12.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:52:11 smithi099 ceph-mon[62951]: pgmap v683: 97 pgs: 97 active+clean; 254 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 792 KiB/s rd, 2.7 MiB/s wr, 249 op/s 2024-03-19T01:52:12.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:52:11 smithi112 ceph-mon[66819]: pgmap v683: 97 pgs: 97 active+clean; 254 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 792 KiB/s rd, 2.7 MiB/s wr, 249 op/s 2024-03-19T01:52:12.702 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1349934 11.28 MB/sec execute 522 sec latency 10.663 ms 2024-03-19T01:52:13.702 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1352213 11.28 MB/sec execute 523 sec latency 27.692 ms 2024-03-19T01:52:14.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:52:13 smithi115 ceph-mon[67332]: pgmap v684: 97 pgs: 97 active+clean; 256 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 1.0 MiB/s rd, 3.1 MiB/s wr, 292 op/s 2024-03-19T01:52:14.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:52:13 smithi099 ceph-mon[62951]: pgmap v684: 97 pgs: 97 active+clean; 256 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 1.0 MiB/s rd, 3.1 MiB/s wr, 292 op/s 2024-03-19T01:52:14.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:52:13 smithi112 ceph-mon[66819]: pgmap v684: 97 pgs: 97 active+clean; 256 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 1.0 MiB/s rd, 3.1 MiB/s wr, 292 op/s 2024-03-19T01:52:14.702 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1353981 11.27 MB/sec execute 524 sec latency 30.709 ms 2024-03-19T01:52:15.702 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1356552 11.28 MB/sec execute 525 sec latency 31.743 ms 2024-03-19T01:52:16.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:52:15 smithi115 ceph-mon[67332]: pgmap v685: 97 pgs: 97 active+clean; 256 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 716 KiB/s rd, 2.6 MiB/s wr, 227 op/s 2024-03-19T01:52:16.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:52:15 smithi099 ceph-mon[62951]: pgmap v685: 97 pgs: 97 active+clean; 256 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 716 KiB/s rd, 2.6 MiB/s wr, 227 op/s 2024-03-19T01:52:16.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:52:15 smithi112 ceph-mon[66819]: pgmap v685: 97 pgs: 97 active+clean; 256 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 716 KiB/s rd, 2.6 MiB/s wr, 227 op/s 2024-03-19T01:52:16.702 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1357945 11.27 MB/sec execute 526 sec latency 57.927 ms 2024-03-19T01:52:16.767 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.763+0000 7f8c65310640 1 -- 172.21.15.99:0/3108140749 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8c60100740 msgr2=0x7f8c60100b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:52:16.767 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.763+0000 7f8c65310640 1 --2- 172.21.15.99:0/3108140749 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8c60100740 0x7f8c60100b60 secure :-1 s=READY pgs=557 cs=0 l=1 rev1=1 crypto rx=0x7f8c50009900 tx=0x7f8c5002bc30 comp rx=0 tx=0).stop 2024-03-19T01:52:16.767 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.763+0000 7f8c65310640 1 -- 172.21.15.99:0/3108140749 shutdown_connections 2024-03-19T01:52:16.767 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.763+0000 7f8c65310640 1 --2- 172.21.15.99:0/3108140749 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8c60101930 0x7f8c60101db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:16.767 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.763+0000 7f8c65310640 1 --2- 172.21.15.99:0/3108140749 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8c601022f0 0x7f8c60108b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:16.768 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.763+0000 7f8c65310640 1 --2- 172.21.15.99:0/3108140749 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8c60100740 0x7f8c60100b60 unknown :-1 s=CLOSED pgs=557 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:16.768 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.763+0000 7f8c65310640 1 -- 172.21.15.99:0/3108140749 >> 172.21.15.99:0/3108140749 conn(0x7f8c600fbcb0 msgr2=0x7f8c600fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:52:16.768 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.763+0000 7f8c65310640 1 -- 172.21.15.99:0/3108140749 shutdown_connections 2024-03-19T01:52:16.768 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.763+0000 7f8c65310640 1 -- 172.21.15.99:0/3108140749 wait complete. 2024-03-19T01:52:16.768 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.764+0000 7f8c65310640 1 Processor -- start 2024-03-19T01:52:16.768 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.764+0000 7f8c65310640 1 -- start start 2024-03-19T01:52:16.768 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.764+0000 7f8c65310640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8c60100740 0x7f8c60197af0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:16.768 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.764+0000 7f8c65310640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8c60101930 0x7f8c60198030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:16.768 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.764+0000 7f8c65310640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8c601022f0 0x7f8c6019f0a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:16.768 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.764+0000 7f8c65310640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f8c6010c410 con 0x7f8c60101930 2024-03-19T01:52:16.768 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.764+0000 7f8c65310640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f8c6010c290 con 0x7f8c601022f0 2024-03-19T01:52:16.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.764+0000 7f8c65310640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f8c6010c590 con 0x7f8c60100740 2024-03-19T01:52:16.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.764+0000 7f8c5e7fc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8c60101930 0x7f8c60198030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:16.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.764+0000 7f8c5effd640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8c60100740 0x7f8c60197af0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:16.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.764+0000 7f8c5e7fc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8c60101930 0x7f8c60198030 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:39844/0 (socket says 172.21.15.99:39844) 2024-03-19T01:52:16.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.764+0000 7f8c5e7fc640 1 -- 172.21.15.99:0/2213656901 learned_addr learned my addr 172.21.15.99:0/2213656901 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:52:16.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.764+0000 7f8c5f7fe640 1 --2- 172.21.15.99:0/2213656901 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8c601022f0 0x7f8c6019f0a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:16.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.765+0000 7f8c5e7fc640 1 -- 172.21.15.99:0/2213656901 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8c60100740 msgr2=0x7f8c60197af0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:52:16.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.765+0000 7f8c5e7fc640 1 --2- 172.21.15.99:0/2213656901 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8c60100740 0x7f8c60197af0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:16.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.765+0000 7f8c5e7fc640 1 -- 172.21.15.99:0/2213656901 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8c601022f0 msgr2=0x7f8c6019f0a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:52:16.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.765+0000 7f8c5e7fc640 1 --2- 172.21.15.99:0/2213656901 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8c601022f0 0x7f8c6019f0a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:16.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.765+0000 7f8c5e7fc640 1 -- 172.21.15.99:0/2213656901 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f8c6019f710 con 0x7f8c60101930 2024-03-19T01:52:16.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.765+0000 7f8c5effd640 1 --2- 172.21.15.99:0/2213656901 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8c60100740 0x7f8c60197af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:52:16.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.765+0000 7f8c5f7fe640 1 --2- 172.21.15.99:0/2213656901 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8c601022f0 0x7f8c6019f0a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:52:16.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.765+0000 7f8c5e7fc640 1 --2- 172.21.15.99:0/2213656901 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8c60101930 0x7f8c60198030 secure :-1 s=READY pgs=558 cs=0 l=1 rev1=1 crypto rx=0x7f8c5400e9f0 tx=0x7f8c5400eeb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:52:16.770 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.765+0000 7f8c47fff640 1 -- 172.21.15.99:0/2213656901 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8c5400cd80 con 0x7f8c60101930 2024-03-19T01:52:16.770 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.765+0000 7f8c65310640 1 -- 172.21.15.99:0/2213656901 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f8c6019fa00 con 0x7f8c60101930 2024-03-19T01:52:16.770 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.765+0000 7f8c47fff640 1 -- 172.21.15.99:0/2213656901 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f8c54010910 con 0x7f8c60101930 2024-03-19T01:52:16.770 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.765+0000 7f8c47fff640 1 -- 172.21.15.99:0/2213656901 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8c540188f0 con 0x7f8c60101930 2024-03-19T01:52:16.770 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.765+0000 7f8c65310640 1 -- 172.21.15.99:0/2213656901 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f8c6019ffa0 con 0x7f8c60101930 2024-03-19T01:52:16.776 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.767+0000 7f8c47fff640 1 -- 172.21.15.99:0/2213656901 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f8c54010ac0 con 0x7f8c60101930 2024-03-19T01:52:16.776 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.768+0000 7f8c67fd8740 1 Processor -- start 2024-03-19T01:52:16.776 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.768+0000 7f8c67fd8740 1 -- start start 2024-03-19T01:52:16.776 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.768+0000 7f8c67fd8740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561a50adc100 0x561a50adc520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:16.776 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.768+0000 7f8c67fd8740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561a50adcaf0 0x561a50ae15f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:16.776 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.768+0000 7f8c67fd8740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561a50ae1c60 0x561a50ae40f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:16.776 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.768+0000 7f8c67fd8740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x561a50a40be0 con 0x561a50ae1c60 2024-03-19T01:52:16.776 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.768+0000 7f8c67fd8740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x561a509e9960 con 0x561a50adcaf0 2024-03-19T01:52:16.777 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.768+0000 7f8c67fd8740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x561a50a3cc80 con 0x561a50adc100 2024-03-19T01:52:16.777 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.768+0000 7f8c5e7fc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561a50ae1c60 0x561a50ae40f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:16.777 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.768+0000 7f8c5effd640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561a50adcaf0 0x561a50ae15f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:16.777 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.768+0000 7f8c5f7fe640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561a50adc100 0x561a50adc520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:16.777 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.768+0000 7f8c5e7fc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561a50ae1c60 0x561a50ae40f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:39852/0 (socket says 172.21.15.99:39852) 2024-03-19T01:52:16.777 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.768+0000 7f8c5e7fc640 1 -- 172.21.15.99:0/1919186685 learned_addr learned my addr 172.21.15.99:0/1919186685 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:52:16.777 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.768+0000 7f8c47fff640 1 --2- 172.21.15.99:0/2213656901 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f8c480a28a0 0x7f8c480a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:16.777 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.768+0000 7f8c5e7fc640 1 -- 172.21.15.99:0/1919186685 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561a50adc100 msgr2=0x561a50adc520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:52:16.777 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.768+0000 7f8c5e7fc640 1 --2- 172.21.15.99:0/1919186685 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561a50adc100 0x561a50adc520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:16.777 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.768+0000 7f8c5e7fc640 1 -- 172.21.15.99:0/1919186685 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561a50adcaf0 msgr2=0x561a50ae15f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:52:16.777 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.768+0000 7f8c5e7fc640 1 --2- 172.21.15.99:0/1919186685 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561a50adcaf0 0x561a50ae15f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:16.777 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.768+0000 7f8c5e7fc640 1 -- 172.21.15.99:0/1919186685 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x561a509e0d20 con 0x561a50ae1c60 2024-03-19T01:52:16.777 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.768+0000 7f8c5effd640 1 --2- 172.21.15.99:0/1919186685 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561a50adcaf0 0x561a50ae15f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:52:16.777 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.768+0000 7f8c5f7fe640 1 --2- 172.21.15.99:0/2213656901 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f8c480a28a0 0x7f8c480a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:16.778 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.768+0000 7f8c47fff640 1 -- 172.21.15.99:0/2213656901 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f8c540cee30 con 0x7f8c60101930 2024-03-19T01:52:16.778 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.769+0000 7f8c5e7fc640 1 --2- 172.21.15.99:0/1919186685 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561a50ae1c60 0x561a50ae40f0 secure :-1 s=READY pgs=559 cs=0 l=1 rev1=1 crypto rx=0x7f8c540703d0 tx=0x7f8c540666c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b19d7f92efd1dcdf server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:52:16.778 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.769+0000 7f8c5f7fe640 1 --2- 172.21.15.99:0/2213656901 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f8c480a28a0 0x7f8c480a4d60 secure :-1 s=READY pgs=312 cs=0 l=1 rev1=1 crypto rx=0x561a50add350 tx=0x7f8c58004a40 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:52:16.778 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.769+0000 7f8c457fa640 1 -- 172.21.15.99:0/1919186685 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8c54026030 con 0x561a50ae1c60 2024-03-19T01:52:16.778 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.769+0000 7f8c457fa640 1 -- 172.21.15.99:0/1919186685 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f8c54070400 con 0x561a50ae1c60 2024-03-19T01:52:16.778 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.769+0000 7f8c457fa640 1 -- 172.21.15.99:0/1919186685 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8c54070dc0 con 0x561a50ae1c60 2024-03-19T01:52:16.778 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.769+0000 7f8c67fd8740 1 -- 172.21.15.99:0/1919186685 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561a50ae1c60 msgr2=0x561a50ae40f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:52:16.778 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.769+0000 7f8c67fd8740 1 --2- 172.21.15.99:0/1919186685 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561a50ae1c60 0x561a50ae40f0 secure :-1 s=READY pgs=559 cs=0 l=1 rev1=1 crypto rx=0x7f8c540703d0 tx=0x7f8c540666c0 comp rx=0 tx=0).stop 2024-03-19T01:52:16.778 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.769+0000 7f8c67fd8740 1 -- 172.21.15.99:0/1919186685 shutdown_connections 2024-03-19T01:52:16.778 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.769+0000 7f8c67fd8740 1 --2- 172.21.15.99:0/1919186685 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561a50adcaf0 0x561a50ae15f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:16.778 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.769+0000 7f8c67fd8740 1 --2- 172.21.15.99:0/1919186685 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561a50ae1c60 0x561a50ae40f0 unknown :-1 s=CLOSED pgs=559 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:16.778 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.769+0000 7f8c67fd8740 1 --2- 172.21.15.99:0/1919186685 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561a50adc100 0x561a50adc520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:16.778 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.769+0000 7f8c67fd8740 1 -- 172.21.15.99:0/1919186685 >> 172.21.15.99:0/1919186685 conn(0x561a50ad7670 msgr2=0x561a50ad7a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:52:16.778 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.769+0000 7f8c67fd8740 1 -- 172.21.15.99:0/1919186685 shutdown_connections 2024-03-19T01:52:16.778 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.769+0000 7f8c67fd8740 1 -- 172.21.15.99:0/1919186685 wait complete. 2024-03-19T01:52:16.779 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.770+0000 7f8c67fd8740 1 -- start start 2024-03-19T01:52:16.779 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.770+0000 7f8c67fd8740 1 Processor -- start 2024-03-19T01:52:16.779 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.770+0000 7f8c67fd8740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561a50adc100 0x561a50ae5ff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:16.779 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.770+0000 7f8c67fd8740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561a50adcaf0 0x561a50ae6530 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:16.779 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.770+0000 7f8c67fd8740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561a50ae1c60 0x561a50aed570 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:16.779 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.770+0000 7f8c67fd8740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x561a50ae4bb0 con 0x561a50adcaf0 2024-03-19T01:52:16.779 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.770+0000 7f8c67fd8740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x561a509fcdb0 con 0x561a50adc100 2024-03-19T01:52:16.779 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.770+0000 7f8c67fd8740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x561a50ae4d30 con 0x561a50ae1c60 2024-03-19T01:52:16.779 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.770+0000 7f8c67fd8740 10 client.0 _ll_register_callbacks cb 0x561a50a01e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:52:16.779 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.770+0000 7f8c5e7fc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561a50adcaf0 0x561a50ae6530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:16.779 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.770+0000 7f8c5e7fc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561a50adcaf0 0x561a50ae6530 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:39862/0 (socket says 172.21.15.99:39862) 2024-03-19T01:52:16.779 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.770+0000 7f8c5e7fc640 1 -- 172.21.15.99:0/2062941737 learned_addr learned my addr 172.21.15.99:0/2062941737 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:52:16.779 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.770+0000 7f8c5effd640 1 --2- 172.21.15.99:0/2062941737 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561a50adc100 0x561a50ae5ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:16.779 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.770+0000 7f8c5f7fe640 1 --2- 172.21.15.99:0/2062941737 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561a50ae1c60 0x561a50aed570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:16.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.770+0000 7f8c5e7fc640 1 -- 172.21.15.99:0/2062941737 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561a50ae1c60 msgr2=0x561a50aed570 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:52:16.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.770+0000 7f8c5e7fc640 1 --2- 172.21.15.99:0/2062941737 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561a50ae1c60 0x561a50aed570 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:16.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.770+0000 7f8c5e7fc640 1 -- 172.21.15.99:0/2062941737 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561a50adc100 msgr2=0x561a50ae5ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:52:16.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.770+0000 7f8c5e7fc640 1 --2- 172.21.15.99:0/2062941737 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561a50adc100 0x561a50ae5ff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:16.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.770+0000 7f8c5e7fc640 1 -- 172.21.15.99:0/2062941737 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x561a509fbc60 con 0x561a50adcaf0 2024-03-19T01:52:16.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.770+0000 7f8c5effd640 1 --2- 172.21.15.99:0/2062941737 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561a50adc100 0x561a50ae5ff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:52:16.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.771+0000 7f8c5e7fc640 1 --2- 172.21.15.99:0/2062941737 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561a50adcaf0 0x561a50ae6530 secure :-1 s=READY pgs=560 cs=0 l=1 rev1=1 crypto rx=0x7f8c5409b310 tx=0x7f8c54009770 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ca203ad26b6e182b server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:52:16.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.771+0000 7f8c42ff5640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:52:16.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.771+0000 7f8c42ff5640 1 -- 172.21.15.99:0/2062941737 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8c540709d0 con 0x561a50adcaf0 2024-03-19T01:52:16.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.771+0000 7f8c42ff5640 1 -- 172.21.15.99:0/2062941737 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f8c5406eb30 con 0x561a50adcaf0 2024-03-19T01:52:16.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.771+0000 7f8c67fd8740 1 -- 172.21.15.99:0/2062941737 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x561a50a4e120 con 0x561a50adcaf0 2024-03-19T01:52:16.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.771+0000 7f8c42ff5640 1 -- 172.21.15.99:0/2062941737 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7f8c5409b840 con 0x561a50adcaf0 2024-03-19T01:52:16.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.771+0000 7f8c67fd8740 10 client.16242 fetch_fsmap learned FSMap version 27 2024-03-19T01:52:16.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.771+0000 7f8c67fd8740 1 -- 172.21.15.99:0/2062941737 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x561a50755210 con 0x561a50adcaf0 2024-03-19T01:52:16.781 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.771+0000 7f8c42ff5640 1 -- 172.21.15.99:0/2062941737 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x561a50a4e120 con 0x561a50adcaf0 2024-03-19T01:52:16.781 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.771+0000 7f8c42ff5640 1 -- 172.21.15.99:0/2062941737 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 27) ==== 8338+0+0 (secure 0 0 0) 0x7f8c54070090 con 0x561a50adcaf0 2024-03-19T01:52:16.781 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.771+0000 7f8c67fd8740 10 client.16242 fetch_fsmap finished waiting for FSMap version 27 2024-03-19T01:52:16.781 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.771+0000 7f8c67fd8740 10 client.16242 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:52:16.781 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.771+0000 7f8c67fd8740 20 client.16242 populate_metadata read hostname 'smithi099' 2024-03-19T01:52:16.781 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.771+0000 7f8c67fd8740 1 --2- 172.21.15.99:0/2062941737 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x561a50af1a60 0x561a50af3e50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:16.781 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.771+0000 7f8c67fd8740 4 client.16242 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:52:16.781 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.771+0000 7f8c67fd8740 1 -- 172.21.15.99:0/2062941737 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x561a50a48470 con 0x561a50af1a60 2024-03-19T01:52:16.781 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.771+0000 7f8c5effd640 1 --2- 172.21.15.99:0/2062941737 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x561a50af1a60 0x561a50af3e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:16.781 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.772+0000 7f8c5effd640 1 --2- 172.21.15.99:0/2062941737 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x561a50af1a60 0x561a50af3e50 crc :-1 s=READY pgs=129 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=84fbc0f64485fd0f server_cookie=7969e6dbc9c11461 in_seq=0 out_seq=0 2024-03-19T01:52:16.781 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.772+0000 7f8c42ff5640 10 client.16242 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:52:16.781 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.773+0000 7f8c42ff5640 1 -- 172.21.15.99:0/2062941737 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x561a50a48470 con 0x561a50af1a60 2024-03-19T01:52:16.781 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.773+0000 7f8c42ff5640 10 client.16242 handle_command_reply: tid=0 2024-03-19T01:52:16.781 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.773+0000 7f8c67fd8740 1 client.16242 shutdown 2024-03-19T01:52:16.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.773+0000 7f8c67fd8740 1 -- 172.21.15.99:0/2062941737 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561a50adcaf0 msgr2=0x561a50ae6530 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:52:16.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.773+0000 7f8c67fd8740 1 --2- 172.21.15.99:0/2062941737 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561a50adcaf0 0x561a50ae6530 secure :-1 s=READY pgs=560 cs=0 l=1 rev1=1 crypto rx=0x7f8c5409b310 tx=0x7f8c54009770 comp rx=0 tx=0).stop 2024-03-19T01:52:16.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.773+0000 7f8c67fd8740 1 -- 172.21.15.99:0/2062941737 shutdown_connections 2024-03-19T01:52:16.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.773+0000 7f8c67fd8740 1 --2- 172.21.15.99:0/2062941737 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x561a50af1a60 0x561a50af3e50 crc :-1 s=READY pgs=129 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:16.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.773+0000 7f8c67fd8740 1 --2- 172.21.15.99:0/2062941737 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561a50ae1c60 0x561a50aed570 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:16.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.773+0000 7f8c67fd8740 1 --2- 172.21.15.99:0/2062941737 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561a50adcaf0 0x561a50ae6530 unknown :-1 s=CLOSED pgs=560 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:16.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.773+0000 7f8c67fd8740 1 --2- 172.21.15.99:0/2062941737 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561a50adc100 0x561a50ae5ff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:16.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.773+0000 7f8c42ff5640 0 client.16242 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:52:16.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.773+0000 7f8c67fd8740 1 -- 172.21.15.99:0/2062941737 >> 172.21.15.99:0/2062941737 conn(0x561a50ad7670 msgr2=0x561a50ad88c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:52:16.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.773+0000 7f8c67fd8740 1 -- 172.21.15.99:0/2062941737 shutdown_connections 2024-03-19T01:52:16.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.773+0000 7f8c67fd8740 1 -- 172.21.15.99:0/2062941737 wait complete. 2024-03-19T01:52:16.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.773+0000 7f8c67fd8740 20 client.16242 trim_cache size 0 max 16384 2024-03-19T01:52:16.794 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.788+0000 7f8c67fd8740 1 Processor -- start 2024-03-19T01:52:16.794 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.788+0000 7f8c67fd8740 1 -- start start 2024-03-19T01:52:16.794 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.789+0000 7f8c67fd8740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561a50adc100 0x561a50af3a70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:16.794 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.789+0000 7f8c67fd8740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561a50adcaf0 0x561a50aed530 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:16.795 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.789+0000 7f8c67fd8740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561a50ae1c60 0x561a50af3fb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:16.795 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.789+0000 7f8c67fd8740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f8c601ab6b0 con 0x561a50ae1c60 2024-03-19T01:52:16.795 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.789+0000 7f8c67fd8740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f8c601a8f80 con 0x561a50adc100 2024-03-19T01:52:16.795 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.789+0000 7f8c67fd8740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f8c601a6dd0 con 0x561a50adcaf0 2024-03-19T01:52:16.795 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.789+0000 7f8c5f7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561a50ae1c60 0x561a50af3fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:16.795 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.789+0000 7f8c5f7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561a50ae1c60 0x561a50af3fb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:39864/0 (socket says 172.21.15.99:39864) 2024-03-19T01:52:16.795 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.789+0000 7f8c5f7fe640 1 -- 172.21.15.99:0/1119669339 learned_addr learned my addr 172.21.15.99:0/1119669339 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:52:16.795 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.789+0000 7f8c5effd640 1 --2- 172.21.15.99:0/1119669339 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561a50adc100 0x561a50af3a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:16.795 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.789+0000 7f8c5e7fc640 1 --2- 172.21.15.99:0/1119669339 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561a50adcaf0 0x561a50aed530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:16.795 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.789+0000 7f8c5f7fe640 1 -- 172.21.15.99:0/1119669339 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561a50adcaf0 msgr2=0x561a50aed530 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:52:16.796 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.789+0000 7f8c5f7fe640 1 --2- 172.21.15.99:0/1119669339 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561a50adcaf0 0x561a50aed530 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:16.796 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.789+0000 7f8c5f7fe640 1 -- 172.21.15.99:0/1119669339 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561a50adc100 msgr2=0x561a50af3a70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:52:16.796 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.789+0000 7f8c5f7fe640 1 --2- 172.21.15.99:0/1119669339 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561a50adc100 0x561a50af3a70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:16.796 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.789+0000 7f8c5f7fe640 1 -- 172.21.15.99:0/1119669339 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x561a50821620 con 0x561a50ae1c60 2024-03-19T01:52:16.796 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.789+0000 7f8c5effd640 1 --2- 172.21.15.99:0/1119669339 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561a50adc100 0x561a50af3a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:52:16.796 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.789+0000 7f8c5f7fe640 1 --2- 172.21.15.99:0/1119669339 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561a50ae1c60 0x561a50af3fb0 secure :-1 s=READY pgs=561 cs=0 l=1 rev1=1 crypto rx=0x7f8c580118d0 tx=0x7f8c5801a000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=98039a09c8e00c5d server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:52:16.796 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.789+0000 7f8c44ff9640 1 -- 172.21.15.99:0/1119669339 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8c58011e50 con 0x561a50ae1c60 2024-03-19T01:52:16.796 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.789+0000 7f8c44ff9640 1 -- 172.21.15.99:0/1119669339 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f8c5801d960 con 0x561a50ae1c60 2024-03-19T01:52:16.796 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.789+0000 7f8c44ff9640 1 -- 172.21.15.99:0/1119669339 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8c580055d0 con 0x561a50ae1c60 2024-03-19T01:52:16.796 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.790+0000 7f8c67fd8740 1 -- 172.21.15.99:0/1119669339 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561a50ae1c60 msgr2=0x561a50af3fb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:52:16.796 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.790+0000 7f8c67fd8740 1 --2- 172.21.15.99:0/1119669339 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561a50ae1c60 0x561a50af3fb0 secure :-1 s=READY pgs=561 cs=0 l=1 rev1=1 crypto rx=0x7f8c580118d0 tx=0x7f8c5801a000 comp rx=0 tx=0).stop 2024-03-19T01:52:16.796 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.790+0000 7f8c67fd8740 1 -- 172.21.15.99:0/1119669339 shutdown_connections 2024-03-19T01:52:16.797 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.790+0000 7f8c67fd8740 1 --2- 172.21.15.99:0/1119669339 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561a50ae1c60 0x561a50af3fb0 unknown :-1 s=CLOSED pgs=561 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:16.797 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.790+0000 7f8c67fd8740 1 --2- 172.21.15.99:0/1119669339 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561a50adcaf0 0x561a50aed530 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:16.797 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.790+0000 7f8c67fd8740 1 --2- 172.21.15.99:0/1119669339 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561a50adc100 0x561a50af3a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:16.797 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.790+0000 7f8c67fd8740 1 -- 172.21.15.99:0/1119669339 >> 172.21.15.99:0/1119669339 conn(0x561a50ad7670 msgr2=0x561a50ae5ff0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:52:16.797 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.790+0000 7f8c67fd8740 1 -- 172.21.15.99:0/1119669339 shutdown_connections 2024-03-19T01:52:16.797 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.790+0000 7f8c67fd8740 1 -- 172.21.15.99:0/1119669339 wait complete. 2024-03-19T01:52:16.797 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.790+0000 7f8c67fd8740 1 -- start start 2024-03-19T01:52:16.797 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.790+0000 7f8c67fd8740 1 Processor -- start 2024-03-19T01:52:16.797 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.791+0000 7f8c67fd8740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561a50adc100 0x561a50aea6c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:16.797 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.791+0000 7f8c67fd8740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561a50adcaf0 0x561a50aeac00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:16.797 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.791+0000 7f8c67fd8740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561a50ae1c60 0x561a50aebc10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:16.797 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.791+0000 7f8c67fd8740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f8c601a7440 con 0x561a50adc100 2024-03-19T01:52:16.797 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.791+0000 7f8c67fd8740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f8c601a88b0 con 0x561a50ae1c60 2024-03-19T01:52:16.798 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.791+0000 7f8c67fd8740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f8c601a7b10 con 0x561a50adcaf0 2024-03-19T01:52:16.798 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.791+0000 7f8c67fd8740 10 client.0 _ll_register_callbacks cb 0x7f8c580093a0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:52:16.798 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.791+0000 7f8c5effd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561a50adc100 0x561a50aea6c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:16.798 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.791+0000 7f8c5effd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561a50adc100 0x561a50aea6c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:39876/0 (socket says 172.21.15.99:39876) 2024-03-19T01:52:16.798 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.791+0000 7f8c5effd640 1 -- 172.21.15.99:0/1914244519 learned_addr learned my addr 172.21.15.99:0/1914244519 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:52:16.798 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.791+0000 7f8c5f7fe640 1 --2- 172.21.15.99:0/1914244519 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561a50ae1c60 0x561a50aebc10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:16.798 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.791+0000 7f8c5e7fc640 1 --2- 172.21.15.99:0/1914244519 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561a50adcaf0 0x561a50aeac00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:16.798 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.791+0000 7f8c5effd640 1 -- 172.21.15.99:0/1914244519 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561a50adcaf0 msgr2=0x561a50aeac00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:52:16.798 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.791+0000 7f8c5effd640 1 --2- 172.21.15.99:0/1914244519 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561a50adcaf0 0x561a50aeac00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:16.798 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.791+0000 7f8c5effd640 1 -- 172.21.15.99:0/1914244519 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561a50ae1c60 msgr2=0x561a50aebc10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:52:16.798 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.791+0000 7f8c5effd640 1 --2- 172.21.15.99:0/1914244519 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561a50ae1c60 0x561a50aebc10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:16.798 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.791+0000 7f8c5effd640 1 -- 172.21.15.99:0/1914244519 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x561a50aec200 con 0x561a50adc100 2024-03-19T01:52:16.798 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.791+0000 7f8c5effd640 1 --2- 172.21.15.99:0/1914244519 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561a50adc100 0x561a50aea6c0 secure :-1 s=READY pgs=562 cs=0 l=1 rev1=1 crypto rx=0x7f8c60101790 tx=0x7f8c50004510 comp rx=0 tx=0).ready entity=mon.0 client_cookie=697d15560014e14f server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:52:16.798 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.791+0000 7f8c42ff5640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:52:16.799 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.791+0000 7f8c42ff5640 1 -- 172.21.15.99:0/1914244519 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8c50051070 con 0x561a50adc100 2024-03-19T01:52:16.799 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.791+0000 7f8c67fd8740 1 -- 172.21.15.99:0/1914244519 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f8c5801d960 con 0x561a50adc100 2024-03-19T01:52:16.799 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.791+0000 7f8c42ff5640 1 -- 172.21.15.99:0/1914244519 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7f8c5004ad20 con 0x561a50adc100 2024-03-19T01:52:16.799 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.791+0000 7f8c42ff5640 1 -- 172.21.15.99:0/1914244519 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7f8c5005d050 con 0x561a50adc100 2024-03-19T01:52:16.799 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.791+0000 7f8c67fd8740 10 client.16248 fetch_fsmap learned FSMap version 27 2024-03-19T01:52:16.799 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.791+0000 7f8c67fd8740 1 -- 172.21.15.99:0/1914244519 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x561a50aed580 con 0x561a50adc100 2024-03-19T01:52:16.799 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.792+0000 7f8c42ff5640 1 -- 172.21.15.99:0/1914244519 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f8c5801d960 con 0x561a50adc100 2024-03-19T01:52:16.799 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.792+0000 7f8c42ff5640 1 -- 172.21.15.99:0/1914244519 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 27) ==== 8338+0+0 (secure 0 0 0) 0x7f8c50004e70 con 0x561a50adc100 2024-03-19T01:52:16.799 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.792+0000 7f8c67fd8740 10 client.16248 fetch_fsmap finished waiting for FSMap version 27 2024-03-19T01:52:16.799 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.792+0000 7f8c67fd8740 10 client.16248 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:52:16.799 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.792+0000 7f8c67fd8740 20 client.16248 populate_metadata read hostname 'smithi099' 2024-03-19T01:52:16.799 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.792+0000 7f8c67fd8740 1 --2- 172.21.15.99:0/1914244519 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x561a50b58200 0x561a50b5a620 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:16.799 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.792+0000 7f8c67fd8740 4 client.16248 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:52:16.799 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.792+0000 7f8c67fd8740 1 -- 172.21.15.99:0/1914244519 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x561a509c6770 con 0x561a50b58200 2024-03-19T01:52:16.800 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.792+0000 7f8c5e7fc640 1 --2- 172.21.15.99:0/1914244519 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x561a50b58200 0x561a50b5a620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:16.800 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.792+0000 7f8c42ff5640 10 client.16248 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:52:16.800 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.792+0000 7f8c5e7fc640 1 --2- 172.21.15.99:0/1914244519 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x561a50b58200 0x561a50b5a620 crc :-1 s=READY pgs=130 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=c19be8f53f4a0ce7 server_cookie=26ad571d61afb05a in_seq=0 out_seq=0 2024-03-19T01:52:16.856 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.854+0000 7f8c42ff5640 1 -- 172.21.15.99:0/1914244519 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x561a509c6770 con 0x561a50b58200 2024-03-19T01:52:16.856 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.854+0000 7f8c42ff5640 10 client.16248 handle_command_reply: tid=0 2024-03-19T01:52:16.856 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.854+0000 7f8c67fd8740 1 client.16248 shutdown 2024-03-19T01:52:16.856 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.855+0000 7f8c67fd8740 1 -- 172.21.15.99:0/1914244519 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561a50adc100 msgr2=0x561a50aea6c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:52:16.856 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.855+0000 7f8c67fd8740 1 --2- 172.21.15.99:0/1914244519 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561a50adc100 0x561a50aea6c0 secure :-1 s=READY pgs=562 cs=0 l=1 rev1=1 crypto rx=0x7f8c60101790 tx=0x7f8c50004510 comp rx=0 tx=0).stop 2024-03-19T01:52:16.856 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.855+0000 7f8c67fd8740 1 -- 172.21.15.99:0/1914244519 shutdown_connections 2024-03-19T01:52:16.856 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.855+0000 7f8c67fd8740 1 --2- 172.21.15.99:0/1914244519 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x561a50b58200 0x561a50b5a620 crc :-1 s=READY pgs=130 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:16.857 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.855+0000 7f8c67fd8740 1 --2- 172.21.15.99:0/1914244519 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561a50ae1c60 0x561a50aebc10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:16.857 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.855+0000 7f8c42ff5640 0 client.16248 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:52:16.857 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.855+0000 7f8c67fd8740 1 --2- 172.21.15.99:0/1914244519 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561a50adcaf0 0x561a50aeac00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:16.857 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.855+0000 7f8c67fd8740 1 --2- 172.21.15.99:0/1914244519 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561a50adc100 0x561a50aea6c0 unknown :-1 s=CLOSED pgs=562 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:16.857 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.855+0000 7f8c67fd8740 1 -- 172.21.15.99:0/1914244519 >> 172.21.15.99:0/1914244519 conn(0x561a50ad7670 msgr2=0x561a50af3a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:52:16.857 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.855+0000 7f8c67fd8740 1 -- 172.21.15.99:0/1914244519 shutdown_connections 2024-03-19T01:52:16.857 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.855+0000 7f8c67fd8740 1 -- 172.21.15.99:0/1914244519 wait complete. 2024-03-19T01:52:16.857 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.855+0000 7f8c67fd8740 20 client.16248 trim_cache size 0 max 16384 2024-03-19T01:52:16.857 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:52:16.857 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:52:16.858 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:52:16.858 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:52:16.859 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.855+0000 7f8c65310640 1 -- 172.21.15.99:0/2213656901 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f8c480a28a0 msgr2=0x7f8c480a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:52:16.859 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.855+0000 7f8c65310640 1 --2- 172.21.15.99:0/2213656901 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f8c480a28a0 0x7f8c480a4d60 secure :-1 s=READY pgs=312 cs=0 l=1 rev1=1 crypto rx=0x561a50add350 tx=0x7f8c58004a40 comp rx=0 tx=0).stop 2024-03-19T01:52:16.859 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.855+0000 7f8c65310640 1 -- 172.21.15.99:0/2213656901 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8c60101930 msgr2=0x7f8c60198030 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:52:16.859 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.855+0000 7f8c65310640 1 --2- 172.21.15.99:0/2213656901 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8c60101930 0x7f8c60198030 secure :-1 s=READY pgs=558 cs=0 l=1 rev1=1 crypto rx=0x7f8c5400e9f0 tx=0x7f8c5400eeb0 comp rx=0 tx=0).stop 2024-03-19T01:52:16.859 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.855+0000 7f8c65310640 1 -- 172.21.15.99:0/2213656901 shutdown_connections 2024-03-19T01:52:16.859 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.855+0000 7f8c65310640 1 --2- 172.21.15.99:0/2213656901 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f8c480a28a0 0x7f8c480a4d60 unknown :-1 s=CLOSED pgs=312 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:16.859 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.855+0000 7f8c65310640 1 --2- 172.21.15.99:0/2213656901 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8c601022f0 0x7f8c6019f0a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:16.859 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.855+0000 7f8c65310640 1 --2- 172.21.15.99:0/2213656901 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8c60101930 0x7f8c60198030 unknown :-1 s=CLOSED pgs=558 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:16.859 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.855+0000 7f8c65310640 1 --2- 172.21.15.99:0/2213656901 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8c60100740 0x7f8c60197af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:16.859 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.855+0000 7f8c65310640 1 -- 172.21.15.99:0/2213656901 >> 172.21.15.99:0/2213656901 conn(0x7f8c600fbcb0 msgr2=0x7f8c600fd680 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:52:16.859 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.856+0000 7f8c65310640 1 -- 172.21.15.99:0/2213656901 shutdown_connections 2024-03-19T01:52:16.859 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:16.856+0000 7f8c65310640 1 -- 172.21.15.99:0/2213656901 wait complete. 2024-03-19T01:52:17.702 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1360300 11.28 MB/sec execute 527 sec latency 35.625 ms 2024-03-19T01:52:18.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:52:17 smithi115 ceph-mon[67332]: pgmap v686: 97 pgs: 97 active+clean; 260 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 812 KiB/s rd, 2.9 MiB/s wr, 252 op/s 2024-03-19T01:52:18.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:52:17 smithi099 ceph-mon[62951]: pgmap v686: 97 pgs: 97 active+clean; 260 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 812 KiB/s rd, 2.9 MiB/s wr, 252 op/s 2024-03-19T01:52:18.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:52:17 smithi112 ceph-mon[66819]: pgmap v686: 97 pgs: 97 active+clean; 260 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 812 KiB/s rd, 2.9 MiB/s wr, 252 op/s 2024-03-19T01:52:18.702 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1361868 11.27 MB/sec execute 528 sec latency 36.359 ms 2024-03-19T01:52:19.702 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1364070 11.27 MB/sec execute 529 sec latency 11.192 ms 2024-03-19T01:52:20.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:52:19 smithi115 ceph-mon[67332]: pgmap v687: 97 pgs: 97 active+clean; 262 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 703 KiB/s rd, 2.6 MiB/s wr, 225 op/s 2024-03-19T01:52:20.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:52:19 smithi099 ceph-mon[62951]: pgmap v687: 97 pgs: 97 active+clean; 262 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 703 KiB/s rd, 2.6 MiB/s wr, 225 op/s 2024-03-19T01:52:20.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:52:19 smithi112 ceph-mon[66819]: pgmap v687: 97 pgs: 97 active+clean; 262 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 703 KiB/s rd, 2.6 MiB/s wr, 225 op/s 2024-03-19T01:52:20.702 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1366089 11.26 MB/sec execute 530 sec latency 28.820 ms 2024-03-19T01:52:21.702 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1367775 11.27 MB/sec execute 531 sec latency 59.261 ms 2024-03-19T01:52:22.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:52:21 smithi115 ceph-mon[67332]: pgmap v688: 97 pgs: 97 active+clean; 262 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 706 KiB/s rd, 2.7 MiB/s wr, 229 op/s 2024-03-19T01:52:22.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:52:21 smithi099 ceph-mon[62951]: pgmap v688: 97 pgs: 97 active+clean; 262 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 706 KiB/s rd, 2.7 MiB/s wr, 229 op/s 2024-03-19T01:52:22.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:52:21 smithi112 ceph-mon[66819]: pgmap v688: 97 pgs: 97 active+clean; 262 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 706 KiB/s rd, 2.7 MiB/s wr, 229 op/s 2024-03-19T01:52:22.702 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1369855 11.26 MB/sec execute 532 sec latency 41.035 ms 2024-03-19T01:52:23.702 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1371640 11.26 MB/sec execute 533 sec latency 42.931 ms 2024-03-19T01:52:24.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:52:23 smithi115 ceph-mon[67332]: pgmap v689: 97 pgs: 97 active+clean; 268 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 857 KiB/s rd, 3.3 MiB/s wr, 268 op/s 2024-03-19T01:52:24.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:52:23 smithi099 ceph-mon[62951]: pgmap v689: 97 pgs: 97 active+clean; 268 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 857 KiB/s rd, 3.3 MiB/s wr, 268 op/s 2024-03-19T01:52:24.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:52:23 smithi112 ceph-mon[66819]: pgmap v689: 97 pgs: 97 active+clean; 268 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 857 KiB/s rd, 3.3 MiB/s wr, 268 op/s 2024-03-19T01:52:24.702 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1374365 11.27 MB/sec execute 534 sec latency 21.884 ms 2024-03-19T01:52:25.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:52:25 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:52:25.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:52:25 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:52:25.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:52:25 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:52:25.702 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1375363 11.28 MB/sec execute 535 sec latency 64.114 ms 2024-03-19T01:52:26.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:52:26 smithi115 ceph-mon[67332]: pgmap v690: 97 pgs: 97 active+clean; 267 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 614 KiB/s rd, 2.6 MiB/s wr, 217 op/s 2024-03-19T01:52:26.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:52:26 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:52:26.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:52:26 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:52:26.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:52:26 smithi099 ceph-mon[62951]: pgmap v690: 97 pgs: 97 active+clean; 267 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 614 KiB/s rd, 2.6 MiB/s wr, 217 op/s 2024-03-19T01:52:26.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:52:26 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:52:26.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:52:26 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:52:26.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:52:26 smithi112 ceph-mon[66819]: pgmap v690: 97 pgs: 97 active+clean; 267 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 614 KiB/s rd, 2.6 MiB/s wr, 217 op/s 2024-03-19T01:52:26.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:52:26 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:52:26.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:52:26 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:52:26.702 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1377421 11.28 MB/sec execute 536 sec latency 58.631 ms 2024-03-19T01:52:27.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:52:27 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:52:27.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:52:27 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:52:27.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:52:27 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:52:27.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:52:27 smithi115 ceph-mon[67332]: pgmap v691: 97 pgs: 97 active+clean; 266 MiB data, 3.5 GiB used, 1.0 TiB / 1.0 TiB avail; 829 KiB/s rd, 3.0 MiB/s wr, 257 op/s 2024-03-19T01:52:27.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:52:27 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:52:27.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:52:27 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:52:27.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:52:27 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:52:27.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:52:27 smithi099 ceph-mon[62951]: pgmap v691: 97 pgs: 97 active+clean; 266 MiB data, 3.5 GiB used, 1.0 TiB / 1.0 TiB avail; 829 KiB/s rd, 3.0 MiB/s wr, 257 op/s 2024-03-19T01:52:27.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:52:27 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:52:27.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:52:27 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:52:27.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:52:27 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:52:27.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:52:27 smithi112 ceph-mon[66819]: pgmap v691: 97 pgs: 97 active+clean; 266 MiB data, 3.5 GiB used, 1.0 TiB / 1.0 TiB avail; 829 KiB/s rd, 3.0 MiB/s wr, 257 op/s 2024-03-19T01:52:27.702 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1379184 11.29 MB/sec execute 537 sec latency 27.828 ms 2024-03-19T01:52:28.702 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1381763 11.29 MB/sec execute 538 sec latency 36.936 ms 2024-03-19T01:52:29.702 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1383113 11.28 MB/sec execute 539 sec latency 113.057 ms 2024-03-19T01:52:29.949 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.947+0000 7ff6c1bba640 1 -- 172.21.15.99:0/812453129 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff6bc1022f0 msgr2=0x7ff6bc108b80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:52:29.949 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.947+0000 7ff6c1bba640 1 --2- 172.21.15.99:0/812453129 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff6bc1022f0 0x7ff6bc108b80 secure :-1 s=READY pgs=151 cs=0 l=1 rev1=1 crypto rx=0x7ff6b4009ca0 tx=0x7ff6b402c0f0 comp rx=0 tx=0).stop 2024-03-19T01:52:29.949 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.947+0000 7ff6c1bba640 1 -- 172.21.15.99:0/812453129 shutdown_connections 2024-03-19T01:52:29.949 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.947+0000 7ff6c1bba640 1 --2- 172.21.15.99:0/812453129 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff6bc1022f0 0x7ff6bc108b80 unknown :-1 s=CLOSED pgs=151 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:29.950 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.947+0000 7ff6c1bba640 1 --2- 172.21.15.99:0/812453129 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff6bc101930 0x7ff6bc101db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:29.950 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.947+0000 7ff6c1bba640 1 --2- 172.21.15.99:0/812453129 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff6bc100740 0x7ff6bc100b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:29.950 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.947+0000 7ff6c1bba640 1 -- 172.21.15.99:0/812453129 >> 172.21.15.99:0/812453129 conn(0x7ff6bc0fbcb0 msgr2=0x7ff6bc0fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:52:29.950 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.947+0000 7ff6c1bba640 1 -- 172.21.15.99:0/812453129 shutdown_connections 2024-03-19T01:52:29.950 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.947+0000 7ff6c1bba640 1 -- 172.21.15.99:0/812453129 wait complete. 2024-03-19T01:52:29.950 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.948+0000 7ff6c1bba640 1 Processor -- start 2024-03-19T01:52:29.951 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.948+0000 7ff6c1bba640 1 -- start start 2024-03-19T01:52:29.951 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.948+0000 7ff6c1bba640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff6bc100740 0x7ff6bc196ad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:29.951 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.948+0000 7ff6c1bba640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff6bc101930 0x7ff6bc197010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:29.951 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.948+0000 7ff6c1bba640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff6bc1022f0 0x7ff6bc19df40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:29.951 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.948+0000 7ff6c1bba640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7ff6bc10c3c0 con 0x7ff6bc100740 2024-03-19T01:52:29.951 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.948+0000 7ff6c1bba640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7ff6bc10c240 con 0x7ff6bc1022f0 2024-03-19T01:52:29.951 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.948+0000 7ff6c1bba640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7ff6bc10c540 con 0x7ff6bc101930 2024-03-19T01:52:29.951 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.948+0000 7ff6bb7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff6bc100740 0x7ff6bc196ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:29.951 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.948+0000 7ff6bbfff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff6bc1022f0 0x7ff6bc19df40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:29.951 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.948+0000 7ff6baffd640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff6bc101930 0x7ff6bc197010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:29.952 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.948+0000 7ff6bb7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff6bc100740 0x7ff6bc196ad0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:39056/0 (socket says 172.21.15.99:39056) 2024-03-19T01:52:29.952 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.948+0000 7ff6bb7fe640 1 -- 172.21.15.99:0/1134690175 learned_addr learned my addr 172.21.15.99:0/1134690175 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:52:29.952 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.948+0000 7ff6bb7fe640 1 -- 172.21.15.99:0/1134690175 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff6bc101930 msgr2=0x7ff6bc197010 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:52:29.952 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.948+0000 7ff6bb7fe640 1 --2- 172.21.15.99:0/1134690175 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff6bc101930 0x7ff6bc197010 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:29.952 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.948+0000 7ff6bb7fe640 1 -- 172.21.15.99:0/1134690175 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff6bc1022f0 msgr2=0x7ff6bc19df40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:52:29.952 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.948+0000 7ff6bb7fe640 1 --2- 172.21.15.99:0/1134690175 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff6bc1022f0 0x7ff6bc19df40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:29.952 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.948+0000 7ff6bb7fe640 1 -- 172.21.15.99:0/1134690175 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ff6bc19e5b0 con 0x7ff6bc100740 2024-03-19T01:52:29.952 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.948+0000 7ff6baffd640 1 --2- 172.21.15.99:0/1134690175 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff6bc101930 0x7ff6bc197010 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:52:29.952 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.948+0000 7ff6bbfff640 1 --2- 172.21.15.99:0/1134690175 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff6bc1022f0 0x7ff6bc19df40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:52:29.952 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.949+0000 7ff6bb7fe640 1 --2- 172.21.15.99:0/1134690175 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff6bc100740 0x7ff6bc196ad0 secure :-1 s=READY pgs=563 cs=0 l=1 rev1=1 crypto rx=0x7ff6ac00e9c0 tx=0x7ff6ac00ee80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:52:29.952 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.949+0000 7ff6b8ff9640 1 -- 172.21.15.99:0/1134690175 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff6ac00cd80 con 0x7ff6bc100740 2024-03-19T01:52:29.952 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.949+0000 7ff6c1bba640 1 -- 172.21.15.99:0/1134690175 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ff6bc19e8a0 con 0x7ff6bc100740 2024-03-19T01:52:29.952 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.949+0000 7ff6b8ff9640 1 -- 172.21.15.99:0/1134690175 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7ff6ac0035f0 con 0x7ff6bc100740 2024-03-19T01:52:29.952 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.949+0000 7ff6b8ff9640 1 -- 172.21.15.99:0/1134690175 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff6ac010690 con 0x7ff6bc100740 2024-03-19T01:52:29.953 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.949+0000 7ff6c1bba640 1 -- 172.21.15.99:0/1134690175 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7ff6bc19edf0 con 0x7ff6bc100740 2024-03-19T01:52:29.959 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.951+0000 7ff6b8ff9640 1 -- 172.21.15.99:0/1134690175 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7ff6ac0037a0 con 0x7ff6bc100740 2024-03-19T01:52:29.959 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.951+0000 7ff6c479d740 1 Processor -- start 2024-03-19T01:52:29.960 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.951+0000 7ff6c479d740 1 -- start start 2024-03-19T01:52:29.960 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.952+0000 7ff6b8ff9640 1 --2- 172.21.15.99:0/1134690175 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff6a40a28a0 0x7ff6a40a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:29.960 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.952+0000 7ff6c479d740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ec0f637100 0x55ec0f637520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:29.960 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.952+0000 7ff6c479d740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ec0f637af0 0x55ec0f63c5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:29.960 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.952+0000 7ff6c479d740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ec0f63cc60 0x55ec0f63f0f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:29.960 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.952+0000 7ff6c479d740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55ec0f59bbe0 con 0x55ec0f63cc60 2024-03-19T01:52:29.960 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.952+0000 7ff6bbfff640 1 --2- 172.21.15.99:0/1134690175 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff6a40a28a0 0x7ff6a40a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:29.960 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.952+0000 7ff6c479d740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55ec0f544960 con 0x55ec0f637af0 2024-03-19T01:52:29.960 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.952+0000 7ff6c479d740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55ec0f597c80 con 0x55ec0f637100 2024-03-19T01:52:29.960 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.952+0000 7ff6b8ff9640 1 -- 172.21.15.99:0/1134690175 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7ff6ac0cee00 con 0x7ff6bc100740 2024-03-19T01:52:29.961 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.952+0000 7ff6bb7fe640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ec0f637100 0x55ec0f637520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:29.961 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.952+0000 7ff6bb7fe640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ec0f637100 0x55ec0f637520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:49134/0 (socket says 172.21.15.99:49134) 2024-03-19T01:52:29.961 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.952+0000 7ff6bb7fe640 1 -- 172.21.15.99:0/1863369899 learned_addr learned my addr 172.21.15.99:0/1863369899 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:52:29.961 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.952+0000 7ff6baffd640 1 --2- 172.21.15.99:0/1863369899 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ec0f637af0 0x55ec0f63c5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:29.961 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.952+0000 7ff6bbfff640 1 --2- 172.21.15.99:0/1863369899 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ec0f63cc60 0x55ec0f63f0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:29.961 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.952+0000 7ff6bb7fe640 1 -- 172.21.15.99:0/1863369899 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ec0f637af0 msgr2=0x55ec0f63c5f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:52:29.961 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.952+0000 7ff6bb7fe640 1 --2- 172.21.15.99:0/1863369899 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ec0f637af0 0x55ec0f63c5f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:29.961 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.952+0000 7ff6bb7fe640 1 -- 172.21.15.99:0/1863369899 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ec0f63cc60 msgr2=0x55ec0f63f0f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:52:29.962 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.952+0000 7ff6bb7fe640 1 --2- 172.21.15.99:0/1863369899 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ec0f63cc60 0x55ec0f63f0f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:29.962 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.952+0000 7ff6bb7fe640 1 -- 172.21.15.99:0/1863369899 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55ec0f53bd20 con 0x55ec0f637100 2024-03-19T01:52:29.962 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.952+0000 7ff6bbfff640 1 --2- 172.21.15.99:0/1134690175 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff6a40a28a0 0x7ff6a40a4d60 secure :-1 s=READY pgs=314 cs=0 l=1 rev1=1 crypto rx=0x7ff6b4006010 tx=0x7ff6b40098d0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:52:29.962 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.952+0000 7ff6bb7fe640 1 --2- 172.21.15.99:0/1863369899 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ec0f637100 0x55ec0f637520 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 crypto rx=0x7ff6ac0a39d0 tx=0x7ff6ac09b530 comp rx=0 tx=0).ready entity=mon.1 client_cookie=608bf50906da1ae1 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:52:29.962 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.952+0000 7ff6a1ffb640 1 -- 172.21.15.99:0/1863369899 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff6ac000ed0 con 0x55ec0f637100 2024-03-19T01:52:29.962 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.953+0000 7ff6a1ffb640 1 -- 172.21.15.99:0/1863369899 <== mon.1 v2:172.21.15.115:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7ff6ac09be60 con 0x55ec0f637100 2024-03-19T01:52:29.962 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.953+0000 7ff6a1ffb640 1 -- 172.21.15.99:0/1863369899 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff6ac099a60 con 0x55ec0f637100 2024-03-19T01:52:29.962 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.953+0000 7ff6c479d740 1 -- 172.21.15.99:0/1863369899 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ec0f637100 msgr2=0x55ec0f637520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:52:29.962 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.953+0000 7ff6c479d740 1 --2- 172.21.15.99:0/1863369899 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ec0f637100 0x55ec0f637520 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 crypto rx=0x7ff6ac0a39d0 tx=0x7ff6ac09b530 comp rx=0 tx=0).stop 2024-03-19T01:52:29.962 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.953+0000 7ff6c479d740 1 -- 172.21.15.99:0/1863369899 shutdown_connections 2024-03-19T01:52:29.962 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.953+0000 7ff6c479d740 1 --2- 172.21.15.99:0/1863369899 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ec0f637af0 0x55ec0f63c5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:29.962 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.953+0000 7ff6c479d740 1 --2- 172.21.15.99:0/1863369899 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ec0f63cc60 0x55ec0f63f0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:29.962 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.953+0000 7ff6c479d740 1 --2- 172.21.15.99:0/1863369899 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ec0f637100 0x55ec0f637520 unknown :-1 s=CLOSED pgs=110 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:29.963 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.953+0000 7ff6c479d740 1 -- 172.21.15.99:0/1863369899 >> 172.21.15.99:0/1863369899 conn(0x55ec0f632670 msgr2=0x55ec0f632a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:52:29.963 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.953+0000 7ff6c479d740 1 -- 172.21.15.99:0/1863369899 shutdown_connections 2024-03-19T01:52:29.963 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.953+0000 7ff6c479d740 1 -- 172.21.15.99:0/1863369899 wait complete. 2024-03-19T01:52:29.963 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.954+0000 7ff6c479d740 1 -- start start 2024-03-19T01:52:29.963 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.954+0000 7ff6c479d740 1 Processor -- start 2024-03-19T01:52:29.963 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.954+0000 7ff6c479d740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ec0f637100 0x55ec0f6410a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:29.963 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.954+0000 7ff6c479d740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ec0f637af0 0x55ec0f6415e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:29.963 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.954+0000 7ff6c479d740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ec0f63cc60 0x55ec0f648620 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:29.963 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.954+0000 7ff6c479d740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55ec0f63fbb0 con 0x55ec0f637af0 2024-03-19T01:52:29.964 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.954+0000 7ff6c479d740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55ec0f557db0 con 0x55ec0f637100 2024-03-19T01:52:29.964 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.954+0000 7ff6c479d740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55ec0f63fd30 con 0x55ec0f63cc60 2024-03-19T01:52:29.964 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.954+0000 7ff6c479d740 10 client.0 _ll_register_callbacks cb 0x55ec0f55ce70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:52:29.964 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.954+0000 7ff6baffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ec0f637af0 0x55ec0f6415e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:29.965 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.954+0000 7ff6baffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ec0f637af0 0x55ec0f6415e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:39068/0 (socket says 172.21.15.99:39068) 2024-03-19T01:52:29.965 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.954+0000 7ff6baffd640 1 -- 172.21.15.99:0/3464286849 learned_addr learned my addr 172.21.15.99:0/3464286849 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:52:29.965 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.954+0000 7ff6bb7fe640 1 --2- 172.21.15.99:0/3464286849 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ec0f637100 0x55ec0f6410a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:29.965 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.954+0000 7ff6bbfff640 1 --2- 172.21.15.99:0/3464286849 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ec0f63cc60 0x55ec0f648620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:29.965 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.954+0000 7ff6baffd640 1 -- 172.21.15.99:0/3464286849 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ec0f63cc60 msgr2=0x55ec0f648620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:52:29.965 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.954+0000 7ff6baffd640 1 --2- 172.21.15.99:0/3464286849 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ec0f63cc60 0x55ec0f648620 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:29.965 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.954+0000 7ff6baffd640 1 -- 172.21.15.99:0/3464286849 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ec0f637100 msgr2=0x55ec0f6410a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:52:29.965 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.954+0000 7ff6baffd640 1 --2- 172.21.15.99:0/3464286849 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ec0f637100 0x55ec0f6410a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:29.965 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.954+0000 7ff6baffd640 1 -- 172.21.15.99:0/3464286849 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55ec0f556c60 con 0x55ec0f637af0 2024-03-19T01:52:29.965 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.955+0000 7ff6baffd640 1 --2- 172.21.15.99:0/3464286849 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ec0f637af0 0x55ec0f6415e0 secure :-1 s=READY pgs=564 cs=0 l=1 rev1=1 crypto rx=0x7ff6b0012d00 tx=0x7ff6b000f790 comp rx=0 tx=0).ready entity=mon.0 client_cookie=fcc8b6af478472a9 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:52:29.966 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.955+0000 7ff69affd640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:52:29.966 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.955+0000 7ff69affd640 1 -- 172.21.15.99:0/3464286849 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff6b0012110 con 0x55ec0f637af0 2024-03-19T01:52:29.966 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.955+0000 7ff6c479d740 1 -- 172.21.15.99:0/3464286849 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7ff6ac09be60 con 0x55ec0f637af0 2024-03-19T01:52:29.966 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.955+0000 7ff69affd640 1 -- 172.21.15.99:0/3464286849 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7ff6b0004050 con 0x55ec0f637af0 2024-03-19T01:52:29.966 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.955+0000 7ff69affd640 1 -- 172.21.15.99:0/3464286849 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7ff6b00122b0 con 0x55ec0f637af0 2024-03-19T01:52:29.966 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.955+0000 7ff6c479d740 10 client.16260 fetch_fsmap learned FSMap version 27 2024-03-19T01:52:29.966 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.955+0000 7ff6c479d740 1 -- 172.21.15.99:0/3464286849 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x55ec0f2b0210 con 0x55ec0f637af0 2024-03-19T01:52:29.966 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.955+0000 7ff69affd640 1 -- 172.21.15.99:0/3464286849 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7ff6ac09be60 con 0x55ec0f637af0 2024-03-19T01:52:29.966 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.955+0000 7ff69affd640 1 -- 172.21.15.99:0/3464286849 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 27) ==== 8338+0+0 (secure 0 0 0) 0x7ff6b0010430 con 0x55ec0f637af0 2024-03-19T01:52:29.966 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.955+0000 7ff6c479d740 10 client.16260 fetch_fsmap finished waiting for FSMap version 27 2024-03-19T01:52:29.966 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.955+0000 7ff6c479d740 10 client.16260 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:52:29.966 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.955+0000 7ff6c479d740 20 client.16260 populate_metadata read hostname 'smithi099' 2024-03-19T01:52:29.966 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.955+0000 7ff6c479d740 1 --2- 172.21.15.99:0/3464286849 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55ec0f64cf40 0x55ec0f64f400 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:29.966 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.955+0000 7ff6c479d740 4 client.16260 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:52:29.967 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.955+0000 7ff6c479d740 1 -- 172.21.15.99:0/3464286849 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55ec0f5a9120 con 0x55ec0f64cf40 2024-03-19T01:52:29.967 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.955+0000 7ff6bb7fe640 1 --2- 172.21.15.99:0/3464286849 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55ec0f64cf40 0x55ec0f64f400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:29.967 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.956+0000 7ff69affd640 10 client.16260 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:52:29.967 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.956+0000 7ff6bb7fe640 1 --2- 172.21.15.99:0/3464286849 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55ec0f64cf40 0x55ec0f64f400 crc :-1 s=READY pgs=131 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=c84ed374bd011b21 server_cookie=e32281cad067df27 in_seq=0 out_seq=0 2024-03-19T01:52:29.967 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.957+0000 7ff69affd640 1 -- 172.21.15.99:0/3464286849 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x55ec0f5a9120 con 0x55ec0f64cf40 2024-03-19T01:52:29.967 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.957+0000 7ff69affd640 10 client.16260 handle_command_reply: tid=0 2024-03-19T01:52:29.967 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.957+0000 7ff6c479d740 1 client.16260 shutdown 2024-03-19T01:52:29.967 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.957+0000 7ff6c479d740 1 -- 172.21.15.99:0/3464286849 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ec0f637af0 msgr2=0x55ec0f6415e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:52:29.967 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.957+0000 7ff6c479d740 1 --2- 172.21.15.99:0/3464286849 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ec0f637af0 0x55ec0f6415e0 secure :-1 s=READY pgs=564 cs=0 l=1 rev1=1 crypto rx=0x7ff6b0012d00 tx=0x7ff6b000f790 comp rx=0 tx=0).stop 2024-03-19T01:52:29.967 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.957+0000 7ff6c479d740 1 -- 172.21.15.99:0/3464286849 shutdown_connections 2024-03-19T01:52:29.967 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.957+0000 7ff6c479d740 1 --2- 172.21.15.99:0/3464286849 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55ec0f64cf40 0x55ec0f64f400 crc :-1 s=READY pgs=131 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:29.967 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.957+0000 7ff6c479d740 1 --2- 172.21.15.99:0/3464286849 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ec0f63cc60 0x55ec0f648620 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:29.967 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.957+0000 7ff6c479d740 1 --2- 172.21.15.99:0/3464286849 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ec0f637af0 0x55ec0f6415e0 unknown :-1 s=CLOSED pgs=564 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:29.967 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.957+0000 7ff69affd640 0 client.16260 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:52:29.968 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.957+0000 7ff6c479d740 1 --2- 172.21.15.99:0/3464286849 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ec0f637100 0x55ec0f6410a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:29.968 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.957+0000 7ff6c479d740 1 -- 172.21.15.99:0/3464286849 >> 172.21.15.99:0/3464286849 conn(0x55ec0f632670 msgr2=0x55ec0f6338c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:52:29.968 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.957+0000 7ff6c479d740 1 -- 172.21.15.99:0/3464286849 shutdown_connections 2024-03-19T01:52:29.968 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.957+0000 7ff6c479d740 1 -- 172.21.15.99:0/3464286849 wait complete. 2024-03-19T01:52:29.968 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.957+0000 7ff6c479d740 20 client.16260 trim_cache size 0 max 16384 2024-03-19T01:52:29.973 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.972+0000 7ff6c479d740 1 Processor -- start 2024-03-19T01:52:29.973 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.972+0000 7ff6c479d740 1 -- start start 2024-03-19T01:52:29.974 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.973+0000 7ff6c479d740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ec0f637100 0x55ec0f631220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:29.974 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.973+0000 7ff6c479d740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ec0f637af0 0x55ec0f6485e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:29.974 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.973+0000 7ff6c479d740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ec0f63cc60 0x55ec0f631760 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:29.974 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.973+0000 7ff6c479d740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7ff6bc1aa5d0 con 0x55ec0f637af0 2024-03-19T01:52:29.974 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.973+0000 7ff6c479d740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7ff6bc1a7ea0 con 0x55ec0f63cc60 2024-03-19T01:52:29.974 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.973+0000 7ff6c479d740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7ff6bc1a5cf0 con 0x55ec0f637100 2024-03-19T01:52:29.974 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.973+0000 7ff6bbfff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ec0f63cc60 0x55ec0f631760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:29.974 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.973+0000 7ff6bbfff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ec0f63cc60 0x55ec0f631760 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:43212/0 (socket says 172.21.15.99:43212) 2024-03-19T01:52:29.974 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.973+0000 7ff6bbfff640 1 -- 172.21.15.99:0/817292752 learned_addr learned my addr 172.21.15.99:0/817292752 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:52:29.974 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.973+0000 7ff6bb7fe640 1 --2- 172.21.15.99:0/817292752 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ec0f637100 0x55ec0f631220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:29.974 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.973+0000 7ff6baffd640 1 --2- 172.21.15.99:0/817292752 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ec0f637af0 0x55ec0f6485e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:29.974 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.973+0000 7ff6bbfff640 1 -- 172.21.15.99:0/817292752 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ec0f637100 msgr2=0x55ec0f631220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:52:29.974 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.973+0000 7ff6bbfff640 1 --2- 172.21.15.99:0/817292752 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ec0f637100 0x55ec0f631220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:29.974 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.973+0000 7ff6bbfff640 1 -- 172.21.15.99:0/817292752 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ec0f637af0 msgr2=0x55ec0f6485e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:52:29.974 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.973+0000 7ff6bbfff640 1 --2- 172.21.15.99:0/817292752 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ec0f637af0 0x55ec0f6485e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:29.975 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.973+0000 7ff6bbfff640 1 -- 172.21.15.99:0/817292752 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55ec0f37c620 con 0x55ec0f63cc60 2024-03-19T01:52:29.975 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.973+0000 7ff6bbfff640 1 --2- 172.21.15.99:0/817292752 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ec0f63cc60 0x55ec0f631760 secure :-1 s=READY pgs=152 cs=0 l=1 rev1=1 crypto rx=0x7ff6b40099c0 tx=0x7ff6b4047000 comp rx=0 tx=0).ready entity=mon.2 client_cookie=67f8b00c4ffaba5c server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:52:29.975 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.973+0000 7ff6a17fa640 1 -- 172.21.15.99:0/817292752 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff6b4053070 con 0x55ec0f63cc60 2024-03-19T01:52:29.975 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.973+0000 7ff6a17fa640 1 -- 172.21.15.99:0/817292752 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7ff6b4032a30 con 0x55ec0f63cc60 2024-03-19T01:52:29.975 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.973+0000 7ff6a17fa640 1 -- 172.21.15.99:0/817292752 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff6b4039040 con 0x55ec0f63cc60 2024-03-19T01:52:29.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.974+0000 7ff6c479d740 1 -- 172.21.15.99:0/817292752 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ec0f63cc60 msgr2=0x55ec0f631760 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:52:29.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.974+0000 7ff6c479d740 1 --2- 172.21.15.99:0/817292752 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ec0f63cc60 0x55ec0f631760 secure :-1 s=READY pgs=152 cs=0 l=1 rev1=1 crypto rx=0x7ff6b40099c0 tx=0x7ff6b4047000 comp rx=0 tx=0).stop 2024-03-19T01:52:29.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.974+0000 7ff6c479d740 1 -- 172.21.15.99:0/817292752 shutdown_connections 2024-03-19T01:52:29.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.974+0000 7ff6c479d740 1 --2- 172.21.15.99:0/817292752 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ec0f63cc60 0x55ec0f631760 unknown :-1 s=CLOSED pgs=152 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:29.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.974+0000 7ff6c479d740 1 --2- 172.21.15.99:0/817292752 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ec0f637af0 0x55ec0f6485e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:29.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.974+0000 7ff6c479d740 1 --2- 172.21.15.99:0/817292752 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ec0f637100 0x55ec0f631220 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:29.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.974+0000 7ff6c479d740 1 -- 172.21.15.99:0/817292752 >> 172.21.15.99:0/817292752 conn(0x55ec0f632670 msgr2=0x55ec0f632a50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:52:29.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.974+0000 7ff6c479d740 1 -- 172.21.15.99:0/817292752 shutdown_connections 2024-03-19T01:52:29.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.974+0000 7ff6c479d740 1 -- 172.21.15.99:0/817292752 wait complete. 2024-03-19T01:52:29.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.975+0000 7ff6c479d740 1 -- start start 2024-03-19T01:52:29.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.975+0000 7ff6c479d740 1 Processor -- start 2024-03-19T01:52:29.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.975+0000 7ff6c479d740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ec0f637100 0x55ec0f645f60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:29.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.975+0000 7ff6c479d740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ec0f637af0 0x55ec0f6464a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:29.980 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.975+0000 7ff6c479d740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ec0f63cc60 0x55ec0f647460 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:29.980 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.975+0000 7ff6c479d740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7ff6bc1a6360 con 0x55ec0f637af0 2024-03-19T01:52:29.980 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.975+0000 7ff6c479d740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7ff6bc1a77d0 con 0x55ec0f637100 2024-03-19T01:52:29.980 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.975+0000 7ff6c479d740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7ff6bc1a6a30 con 0x55ec0f63cc60 2024-03-19T01:52:29.980 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.975+0000 7ff6c479d740 10 client.0 _ll_register_callbacks cb 0x7ff6b0009ce0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:52:29.980 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.975+0000 7ff6baffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ec0f637af0 0x55ec0f6464a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:29.980 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.975+0000 7ff6bb7fe640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ec0f637100 0x55ec0f645f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:29.980 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.975+0000 7ff6baffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ec0f637af0 0x55ec0f6464a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:39090/0 (socket says 172.21.15.99:39090) 2024-03-19T01:52:29.980 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.975+0000 7ff6baffd640 1 -- 172.21.15.99:0/815520235 learned_addr learned my addr 172.21.15.99:0/815520235 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:52:29.980 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.975+0000 7ff6bbfff640 1 --2- 172.21.15.99:0/815520235 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ec0f63cc60 0x55ec0f647460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:29.980 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.975+0000 7ff6bb7fe640 1 -- 172.21.15.99:0/815520235 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ec0f63cc60 msgr2=0x55ec0f647460 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:52:29.980 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.975+0000 7ff6bb7fe640 1 --2- 172.21.15.99:0/815520235 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ec0f63cc60 0x55ec0f647460 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:29.980 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.975+0000 7ff6bb7fe640 1 -- 172.21.15.99:0/815520235 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ec0f637af0 msgr2=0x55ec0f6464a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:52:29.980 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.975+0000 7ff6bb7fe640 1 --2- 172.21.15.99:0/815520235 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ec0f637af0 0x55ec0f6464a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:29.980 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.975+0000 7ff6bb7fe640 1 -- 172.21.15.99:0/815520235 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55ec0f647a50 con 0x55ec0f637100 2024-03-19T01:52:29.981 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.975+0000 7ff6baffd640 1 --2- 172.21.15.99:0/815520235 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ec0f637af0 0x55ec0f6464a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:52:29.981 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.976+0000 7ff6bb7fe640 1 --2- 172.21.15.99:0/815520235 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ec0f637100 0x55ec0f645f60 secure :-1 s=READY pgs=153 cs=0 l=1 rev1=1 crypto rx=0x7ff6ac09b920 tx=0x7ff6ac0973d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=1036ce103b4a12e8 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:52:29.981 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.976+0000 7ff69affd640 10 client.0 ms_handle_connect on v2:172.21.15.112:3300/0 2024-03-19T01:52:29.981 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.976+0000 7ff69affd640 1 -- 172.21.15.99:0/815520235 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff6ac09be60 con 0x55ec0f637100 2024-03-19T01:52:29.981 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.976+0000 7ff6c479d740 1 -- 172.21.15.99:0/815520235 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x55ec0f5a3470 con 0x55ec0f637100 2024-03-19T01:52:29.981 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.976+0000 7ff69affd640 1 -- 172.21.15.99:0/815520235 <== mon.2 v2:172.21.15.112:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8298+0+0 (secure 0 0 0) 0x7ff6ac099a60 con 0x55ec0f637100 2024-03-19T01:52:29.981 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.976+0000 7ff69affd640 1 -- 172.21.15.99:0/815520235 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) ==== 24+0+0 (secure 0 0 0) 0x7ff6ac003b10 con 0x55ec0f637100 2024-03-19T01:52:29.981 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.976+0000 7ff6c479d740 10 client.26036 fetch_fsmap learned FSMap version 27 2024-03-19T01:52:29.981 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.976+0000 7ff6c479d740 1 -- 172.21.15.99:0/815520235 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) -- 0x55ec0f6b35b0 con 0x55ec0f637100 2024-03-19T01:52:29.981 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.976+0000 7ff69affd640 1 -- 172.21.15.99:0/815520235 <== mon.2 v2:172.21.15.112:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x55ec0f5a3470 con 0x55ec0f637100 2024-03-19T01:52:29.981 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.976+0000 7ff69affd640 1 -- 172.21.15.99:0/815520235 <== mon.2 v2:172.21.15.112:3300/0 5 ==== fsmap(e 27) ==== 8338+0+0 (secure 0 0 0) 0x7ff6ac011140 con 0x55ec0f637100 2024-03-19T01:52:29.981 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.976+0000 7ff6c479d740 10 client.26036 fetch_fsmap finished waiting for FSMap version 27 2024-03-19T01:52:29.981 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.976+0000 7ff6c479d740 10 client.26036 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:52:29.981 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.976+0000 7ff6c479d740 20 client.26036 populate_metadata read hostname 'smithi099' 2024-03-19T01:52:29.982 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.976+0000 7ff6c479d740 1 --2- 172.21.15.99:0/815520235 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55ec0f6b3ae0 0x55ec0f6b5ed0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:29.982 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.976+0000 7ff6c479d740 4 client.26036 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:52:29.982 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.976+0000 7ff6c479d740 1 -- 172.21.15.99:0/815520235 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55ec0f521770 con 0x55ec0f6b3ae0 2024-03-19T01:52:29.982 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.977+0000 7ff6baffd640 1 --2- 172.21.15.99:0/815520235 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55ec0f6b3ae0 0x55ec0f6b5ed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:29.982 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.977+0000 7ff6baffd640 1 --2- 172.21.15.99:0/815520235 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55ec0f6b3ae0 0x55ec0f6b5ed0 crc :-1 s=READY pgs=132 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=299929816683f1e7 server_cookie=2ec798b928d983c2 in_seq=0 out_seq=0 2024-03-19T01:52:29.982 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:29.977+0000 7ff69affd640 10 client.26036 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:52:30.047 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:30.047+0000 7ff69affd640 1 -- 172.21.15.99:0/815520235 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x55ec0f521770 con 0x55ec0f6b3ae0 2024-03-19T01:52:30.048 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:30.047+0000 7ff69affd640 10 client.26036 handle_command_reply: tid=0 2024-03-19T01:52:30.048 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:30.047+0000 7ff6c479d740 1 client.26036 shutdown 2024-03-19T01:52:30.048 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:30.047+0000 7ff6c479d740 1 -- 172.21.15.99:0/815520235 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ec0f637100 msgr2=0x55ec0f645f60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:52:30.048 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:30.047+0000 7ff6c479d740 1 --2- 172.21.15.99:0/815520235 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ec0f637100 0x55ec0f645f60 secure :-1 s=READY pgs=153 cs=0 l=1 rev1=1 crypto rx=0x7ff6ac09b920 tx=0x7ff6ac0973d0 comp rx=0 tx=0).stop 2024-03-19T01:52:30.048 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:30.047+0000 7ff6c479d740 1 -- 172.21.15.99:0/815520235 shutdown_connections 2024-03-19T01:52:30.048 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:30.047+0000 7ff6c479d740 1 --2- 172.21.15.99:0/815520235 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55ec0f6b3ae0 0x55ec0f6b5ed0 crc :-1 s=READY pgs=132 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:30.048 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:30.047+0000 7ff6c479d740 1 --2- 172.21.15.99:0/815520235 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ec0f63cc60 0x55ec0f647460 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:30.048 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:30.047+0000 7ff69affd640 0 client.26036 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:52:30.048 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:30.047+0000 7ff6c479d740 1 --2- 172.21.15.99:0/815520235 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ec0f637af0 0x55ec0f6464a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:30.048 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:30.047+0000 7ff6c479d740 1 --2- 172.21.15.99:0/815520235 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ec0f637100 0x55ec0f645f60 unknown :-1 s=CLOSED pgs=153 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:30.048 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:30.047+0000 7ff6c479d740 1 -- 172.21.15.99:0/815520235 >> 172.21.15.99:0/815520235 conn(0x55ec0f632670 msgr2=0x55ec0f641080 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:52:30.048 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:30.047+0000 7ff6c479d740 1 -- 172.21.15.99:0/815520235 shutdown_connections 2024-03-19T01:52:30.048 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:30.047+0000 7ff6c479d740 1 -- 172.21.15.99:0/815520235 wait complete. 2024-03-19T01:52:30.049 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:30.047+0000 7ff6c479d740 20 client.26036 trim_cache size 0 max 16384 2024-03-19T01:52:30.049 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:52:30.049 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:52:30.049 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:52:30.049 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:52:30.050 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:30.048+0000 7ff6c1bba640 1 -- 172.21.15.99:0/1134690175 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff6a40a28a0 msgr2=0x7ff6a40a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:52:30.050 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:30.048+0000 7ff6c1bba640 1 --2- 172.21.15.99:0/1134690175 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff6a40a28a0 0x7ff6a40a4d60 secure :-1 s=READY pgs=314 cs=0 l=1 rev1=1 crypto rx=0x7ff6b4006010 tx=0x7ff6b40098d0 comp rx=0 tx=0).stop 2024-03-19T01:52:30.050 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:30.048+0000 7ff6c1bba640 1 -- 172.21.15.99:0/1134690175 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff6bc100740 msgr2=0x7ff6bc196ad0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:52:30.050 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:30.048+0000 7ff6c1bba640 1 --2- 172.21.15.99:0/1134690175 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff6bc100740 0x7ff6bc196ad0 secure :-1 s=READY pgs=563 cs=0 l=1 rev1=1 crypto rx=0x7ff6ac00e9c0 tx=0x7ff6ac00ee80 comp rx=0 tx=0).stop 2024-03-19T01:52:30.050 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:30.048+0000 7ff6c1bba640 1 -- 172.21.15.99:0/1134690175 shutdown_connections 2024-03-19T01:52:30.050 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:30.048+0000 7ff6c1bba640 1 --2- 172.21.15.99:0/1134690175 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff6a40a28a0 0x7ff6a40a4d60 unknown :-1 s=CLOSED pgs=314 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:30.050 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:30.048+0000 7ff6c1bba640 1 --2- 172.21.15.99:0/1134690175 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff6bc1022f0 0x7ff6bc19df40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:30.050 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:30.048+0000 7ff6c1bba640 1 --2- 172.21.15.99:0/1134690175 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff6bc101930 0x7ff6bc197010 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:30.050 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:30.048+0000 7ff6c1bba640 1 --2- 172.21.15.99:0/1134690175 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff6bc100740 0x7ff6bc196ad0 unknown :-1 s=CLOSED pgs=563 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:30.051 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:30.048+0000 7ff6c1bba640 1 -- 172.21.15.99:0/1134690175 >> 172.21.15.99:0/1134690175 conn(0x7ff6bc0fbcb0 msgr2=0x7ff6bc104bd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:52:30.051 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:30.048+0000 7ff6c1bba640 1 -- 172.21.15.99:0/1134690175 shutdown_connections 2024-03-19T01:52:30.051 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:30.048+0000 7ff6c1bba640 1 -- 172.21.15.99:0/1134690175 wait complete. 2024-03-19T01:52:30.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:52:29 smithi115 ceph-mon[67332]: pgmap v692: 97 pgs: 97 active+clean; 267 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 877 KiB/s rd, 3.0 MiB/s wr, 267 op/s 2024-03-19T01:52:30.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:52:29 smithi099 ceph-mon[62951]: pgmap v692: 97 pgs: 97 active+clean; 267 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 877 KiB/s rd, 3.0 MiB/s wr, 267 op/s 2024-03-19T01:52:30.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:52:29 smithi112 ceph-mon[66819]: pgmap v692: 97 pgs: 97 active+clean; 267 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 877 KiB/s rd, 3.0 MiB/s wr, 267 op/s 2024-03-19T01:52:30.702 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1385662 11.29 MB/sec execute 540 sec latency 46.805 ms 2024-03-19T01:52:30.806 INFO:journalctl@ceph.mds.f.smithi115.stdout:Mar 19 01:52:30 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-f[96845]: ignoring --setuser ceph since I am not root 2024-03-19T01:52:30.806 INFO:journalctl@ceph.mds.f.smithi115.stdout:Mar 19 01:52:30 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-f[96845]: ignoring --setgroup ceph since I am not root 2024-03-19T01:52:30.806 INFO:journalctl@ceph.mds.f.smithi115.stdout:Mar 19 01:52:30 smithi115 ceph-mds[96849]: -- 172.21.15.115:0/3391975049 <== mon.1 v2:172.21.15.115:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5613ce0a9a40 con 0x5613ce07c800 2024-03-19T01:52:30.806 INFO:journalctl@ceph.mds.f.smithi115.stdout:Mar 19 01:52:30 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-f[96845]: starting mds.f at 2024-03-19T01:52:31.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:52:30 smithi115 ceph-mon[67332]: daemon mds.f restarted 2024-03-19T01:52:31.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:52:30 smithi115 ceph-mon[67332]: osdmap e92: 12 total, 12 up, 12 in 2024-03-19T01:52:31.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:52:30 smithi115 ceph-mon[67332]: mds.? [v2:172.21.15.115:6834/1572636732,v1:172.21.15.115:6835/1572636732] up:boot 2024-03-19T01:52:31.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:52:30 smithi115 ceph-mon[67332]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T01:52:31.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:52:30 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "f"}]: dispatch 2024-03-19T01:52:31.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:52:30 smithi099 ceph-mon[62951]: daemon mds.f restarted 2024-03-19T01:52:31.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:52:30 smithi099 ceph-mon[62951]: osdmap e92: 12 total, 12 up, 12 in 2024-03-19T01:52:31.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:52:30 smithi099 ceph-mon[62951]: mds.? [v2:172.21.15.115:6834/1572636732,v1:172.21.15.115:6835/1572636732] up:boot 2024-03-19T01:52:31.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:52:30 smithi099 ceph-mon[62951]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T01:52:31.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:52:30 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "f"}]: dispatch 2024-03-19T01:52:31.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:52:30 smithi112 ceph-mon[66819]: daemon mds.f restarted 2024-03-19T01:52:31.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:52:30 smithi112 ceph-mon[66819]: osdmap e92: 12 total, 12 up, 12 in 2024-03-19T01:52:31.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:52:30 smithi112 ceph-mon[66819]: mds.? [v2:172.21.15.115:6834/1572636732,v1:172.21.15.115:6835/1572636732] up:boot 2024-03-19T01:52:31.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:52:30 smithi112 ceph-mon[66819]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T01:52:31.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:52:30 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "f"}]: dispatch 2024-03-19T01:52:31.703 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1387107 11.28 MB/sec execute 541 sec latency 59.772 ms 2024-03-19T01:52:32.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:52:31 smithi115 ceph-mon[67332]: pgmap v694: 97 pgs: 97 active+clean; 262 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 812 KiB/s rd, 2.7 MiB/s wr, 244 op/s 2024-03-19T01:52:32.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:52:31 smithi099 ceph-mon[62951]: pgmap v694: 97 pgs: 97 active+clean; 262 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 812 KiB/s rd, 2.7 MiB/s wr, 244 op/s 2024-03-19T01:52:32.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:52:31 smithi112 ceph-mon[66819]: pgmap v694: 97 pgs: 97 active+clean; 262 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 812 KiB/s rd, 2.7 MiB/s wr, 244 op/s 2024-03-19T01:52:32.703 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1389439 11.29 MB/sec execute 542 sec latency 43.086 ms 2024-03-19T01:52:33.703 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1391053 11.28 MB/sec execute 543 sec latency 29.166 ms 2024-03-19T01:52:34.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:52:33 smithi115 ceph-mon[67332]: pgmap v695: 97 pgs: 97 active+clean; 255 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 754 KiB/s rd, 2.5 MiB/s wr, 245 op/s 2024-03-19T01:52:34.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:52:33 smithi099 ceph-mon[62951]: pgmap v695: 97 pgs: 97 active+clean; 255 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 754 KiB/s rd, 2.5 MiB/s wr, 245 op/s 2024-03-19T01:52:34.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:52:33 smithi112 ceph-mon[66819]: pgmap v695: 97 pgs: 97 active+clean; 255 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 754 KiB/s rd, 2.5 MiB/s wr, 245 op/s 2024-03-19T01:52:34.703 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1393233 11.28 MB/sec execute 544 sec latency 19.329 ms 2024-03-19T01:52:35.703 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1395369 11.28 MB/sec execute 545 sec latency 29.322 ms 2024-03-19T01:52:36.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:52:35 smithi115 ceph-mon[67332]: pgmap v696: 97 pgs: 97 active+clean; 255 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 754 KiB/s rd, 2.6 MiB/s wr, 255 op/s 2024-03-19T01:52:36.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:52:35 smithi099 ceph-mon[62951]: pgmap v696: 97 pgs: 97 active+clean; 255 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 754 KiB/s rd, 2.6 MiB/s wr, 255 op/s 2024-03-19T01:52:36.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:52:35 smithi112 ceph-mon[66819]: pgmap v696: 97 pgs: 97 active+clean; 255 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 754 KiB/s rd, 2.6 MiB/s wr, 255 op/s 2024-03-19T01:52:36.703 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1397018 11.28 MB/sec execute 546 sec latency 50.948 ms 2024-03-19T01:52:37.703 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1398953 11.27 MB/sec execute 547 sec latency 55.139 ms 2024-03-19T01:52:38.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:52:37 smithi115 ceph-mon[67332]: pgmap v697: 97 pgs: 97 active+clean; 256 MiB data, 3.5 GiB used, 1.0 TiB / 1.0 TiB avail; 769 KiB/s rd, 2.6 MiB/s wr, 281 op/s 2024-03-19T01:52:38.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:52:37 smithi099 ceph-mon[62951]: pgmap v697: 97 pgs: 97 active+clean; 256 MiB data, 3.5 GiB used, 1.0 TiB / 1.0 TiB avail; 769 KiB/s rd, 2.6 MiB/s wr, 281 op/s 2024-03-19T01:52:38.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:52:37 smithi112 ceph-mon[66819]: pgmap v697: 97 pgs: 97 active+clean; 256 MiB data, 3.5 GiB used, 1.0 TiB / 1.0 TiB avail; 769 KiB/s rd, 2.6 MiB/s wr, 281 op/s 2024-03-19T01:52:38.703 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1400809 11.27 MB/sec execute 548 sec latency 27.650 ms 2024-03-19T01:52:39.703 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1403261 11.28 MB/sec execute 549 sec latency 66.496 ms 2024-03-19T01:52:40.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:52:39 smithi115 ceph-mon[67332]: pgmap v698: 97 pgs: 97 active+clean; 254 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 598 KiB/s rd, 2.4 MiB/s wr, 241 op/s 2024-03-19T01:52:40.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:52:39 smithi099 ceph-mon[62951]: pgmap v698: 97 pgs: 97 active+clean; 254 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 598 KiB/s rd, 2.4 MiB/s wr, 241 op/s 2024-03-19T01:52:40.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:52:39 smithi112 ceph-mon[66819]: pgmap v698: 97 pgs: 97 active+clean; 254 MiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 598 KiB/s rd, 2.4 MiB/s wr, 241 op/s 2024-03-19T01:52:40.703 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1404955 11.27 MB/sec execute 550 sec latency 41.716 ms 2024-03-19T01:52:41.703 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1407250 11.28 MB/sec execute 551 sec latency 100.117 ms 2024-03-19T01:52:42.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:52:41 smithi115 ceph-mon[67332]: pgmap v699: 97 pgs: 97 active+clean; 251 MiB data, 3.5 GiB used, 1.0 TiB / 1.0 TiB avail; 750 KiB/s rd, 2.7 MiB/s wr, 257 op/s 2024-03-19T01:52:42.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:52:41 smithi099 ceph-mon[62951]: pgmap v699: 97 pgs: 97 active+clean; 251 MiB data, 3.5 GiB used, 1.0 TiB / 1.0 TiB avail; 750 KiB/s rd, 2.7 MiB/s wr, 257 op/s 2024-03-19T01:52:42.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:52:41 smithi112 ceph-mon[66819]: pgmap v699: 97 pgs: 97 active+clean; 251 MiB data, 3.5 GiB used, 1.0 TiB / 1.0 TiB avail; 750 KiB/s rd, 2.7 MiB/s wr, 257 op/s 2024-03-19T01:52:42.703 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1408760 11.27 MB/sec execute 552 sec latency 37.644 ms 2024-03-19T01:52:43.139 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.137+0000 7f2a01976640 1 -- 172.21.15.99:0/2173844696 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f29fc101930 msgr2=0x7f29fc101db0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:52:43.139 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.137+0000 7f2a01976640 1 --2- 172.21.15.99:0/2173844696 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f29fc101930 0x7f29fc101db0 secure :-1 s=READY pgs=155 cs=0 l=1 rev1=1 crypto rx=0x7f29f0009860 tx=0x7f29f002bb70 comp rx=0 tx=0).stop 2024-03-19T01:52:43.140 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.137+0000 7f2a01976640 1 -- 172.21.15.99:0/2173844696 shutdown_connections 2024-03-19T01:52:43.140 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.137+0000 7f2a01976640 1 --2- 172.21.15.99:0/2173844696 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f29fc101930 0x7f29fc101db0 unknown :-1 s=CLOSED pgs=155 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:43.140 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.137+0000 7f2a01976640 1 --2- 172.21.15.99:0/2173844696 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f29fc1022f0 0x7f29fc108b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:43.140 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.137+0000 7f2a01976640 1 --2- 172.21.15.99:0/2173844696 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f29fc100740 0x7f29fc100b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:43.140 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.137+0000 7f2a01976640 1 -- 172.21.15.99:0/2173844696 >> 172.21.15.99:0/2173844696 conn(0x7f29fc0fbcb0 msgr2=0x7f29fc0fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:52:43.140 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.137+0000 7f2a01976640 1 -- 172.21.15.99:0/2173844696 shutdown_connections 2024-03-19T01:52:43.140 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.137+0000 7f2a01976640 1 -- 172.21.15.99:0/2173844696 wait complete. 2024-03-19T01:52:43.140 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.138+0000 7f2a01976640 1 Processor -- start 2024-03-19T01:52:43.140 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.138+0000 7f2a01976640 1 -- start start 2024-03-19T01:52:43.140 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.138+0000 7f2a01976640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f29fc100740 0x7f29fc1969d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:43.140 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.138+0000 7f2a01976640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f29fc101930 0x7f29fc196f10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:43.140 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.138+0000 7f2a01976640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f29fc1022f0 0x7f29fc19df80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:43.140 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.138+0000 7f2a01976640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f29fc10c3c0 con 0x7f29fc101930 2024-03-19T01:52:43.141 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.138+0000 7f2a01976640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f29fc10c240 con 0x7f29fc1022f0 2024-03-19T01:52:43.141 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.138+0000 7f2a01976640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f29fc10c540 con 0x7f29fc100740 2024-03-19T01:52:43.141 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.138+0000 7f29fa7fc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f29fc101930 0x7f29fc196f10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:43.141 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.138+0000 7f29faffd640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f29fc100740 0x7f29fc1969d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:43.141 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.138+0000 7f29faffd640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f29fc100740 0x7f29fc1969d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:46326/0 (socket says 172.21.15.99:46326) 2024-03-19T01:52:43.141 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.138+0000 7f29faffd640 1 -- 172.21.15.99:0/3508573402 learned_addr learned my addr 172.21.15.99:0/3508573402 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:52:43.141 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.138+0000 7f29fb7fe640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f29fc1022f0 0x7f29fc19df80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:43.141 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.138+0000 7f29fa7fc640 1 --2- 172.21.15.99:0/3508573402 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f29fc101930 0x7f29fc196f10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:37950/0 (socket says 172.21.15.99:37950) 2024-03-19T01:52:43.141 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.139+0000 7f29fa7fc640 1 -- 172.21.15.99:0/3508573402 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f29fc100740 msgr2=0x7f29fc1969d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:52:43.141 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.139+0000 7f29fa7fc640 1 --2- 172.21.15.99:0/3508573402 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f29fc100740 0x7f29fc1969d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:43.141 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.139+0000 7f29fa7fc640 1 -- 172.21.15.99:0/3508573402 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f29fc1022f0 msgr2=0x7f29fc19df80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:52:43.141 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.139+0000 7f29fa7fc640 1 --2- 172.21.15.99:0/3508573402 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f29fc1022f0 0x7f29fc19df80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:43.141 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.139+0000 7f29fa7fc640 1 -- 172.21.15.99:0/3508573402 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f29fc19e5f0 con 0x7f29fc101930 2024-03-19T01:52:43.141 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.139+0000 7f29faffd640 1 --2- 172.21.15.99:0/3508573402 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f29fc100740 0x7f29fc1969d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:52:43.142 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.139+0000 7f29fa7fc640 1 --2- 172.21.15.99:0/3508573402 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f29fc101930 0x7f29fc196f10 secure :-1 s=READY pgs=565 cs=0 l=1 rev1=1 crypto rx=0x7f29f002c0b0 tx=0x7f29f0032430 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:52:43.142 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.139+0000 7f2a00974640 1 -- 172.21.15.99:0/3508573402 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f29f0044020 con 0x7f29fc101930 2024-03-19T01:52:43.142 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.139+0000 7f2a01976640 1 -- 172.21.15.99:0/3508573402 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f29fc068ea0 con 0x7f29fc101930 2024-03-19T01:52:43.142 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.139+0000 7f2a00974640 1 -- 172.21.15.99:0/3508573402 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f29f002c9f0 con 0x7f29fc101930 2024-03-19T01:52:43.142 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.139+0000 7f2a00974640 1 -- 172.21.15.99:0/3508573402 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f29f0039040 con 0x7f29fc101930 2024-03-19T01:52:43.142 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.139+0000 7f2a01976640 1 -- 172.21.15.99:0/3508573402 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f29fc069390 con 0x7f29fc101930 2024-03-19T01:52:43.148 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.141+0000 7f2a00974640 1 -- 172.21.15.99:0/3508573402 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f29f0047020 con 0x7f29fc101930 2024-03-19T01:52:43.149 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.142+0000 7f2a04635740 1 Processor -- start 2024-03-19T01:52:43.149 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.142+0000 7f2a04635740 1 -- start start 2024-03-19T01:52:43.149 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.142+0000 7f2a00974640 1 --2- 172.21.15.99:0/3508573402 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f29e40a28a0 0x7f29e40a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:43.149 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.142+0000 7f2a04635740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e407c1a100 0x55e407c1a520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:43.149 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.142+0000 7f2a04635740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e407c1aaf0 0x55e407c1f5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:43.149 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.142+0000 7f2a04635740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e407c1fc60 0x55e407c220f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:43.149 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.142+0000 7f2a04635740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55e407b7ebe0 con 0x55e407c1aaf0 2024-03-19T01:52:43.149 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.142+0000 7f2a04635740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55e407b27960 con 0x55e407c1fc60 2024-03-19T01:52:43.149 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.142+0000 7f2a04635740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55e407b7ac80 con 0x55e407c1a100 2024-03-19T01:52:43.149 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.142+0000 7f29fb7fe640 1 --2- 172.21.15.99:0/3508573402 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f29e40a28a0 0x7f29e40a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:43.149 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.142+0000 7f29fa7fc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e407c1aaf0 0x55e407c1f5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:43.149 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.142+0000 7f2a00974640 1 -- 172.21.15.99:0/3508573402 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f29f00f2c10 con 0x7f29fc101930 2024-03-19T01:52:43.149 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.142+0000 7f29fa7fc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e407c1aaf0 0x55e407c1f5f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:37964/0 (socket says 172.21.15.99:37964) 2024-03-19T01:52:43.149 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.142+0000 7f29fa7fc640 1 -- 172.21.15.99:0/4244412255 learned_addr learned my addr 172.21.15.99:0/4244412255 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:52:43.150 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.142+0000 7f29faffd640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e407c1a100 0x55e407c1a520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:43.150 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.142+0000 7f29fb7fe640 1 --2- 172.21.15.99:0/4244412255 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e407c1fc60 0x55e407c220f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:43.150 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.142+0000 7f29fa7fc640 1 -- 172.21.15.99:0/4244412255 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e407c1a100 msgr2=0x55e407c1a520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:52:43.150 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.142+0000 7f29fa7fc640 1 --2- 172.21.15.99:0/4244412255 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e407c1a100 0x55e407c1a520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:43.150 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.142+0000 7f29fa7fc640 1 -- 172.21.15.99:0/4244412255 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e407c1fc60 msgr2=0x55e407c220f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:52:43.150 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.142+0000 7f29fa7fc640 1 --2- 172.21.15.99:0/4244412255 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e407c1fc60 0x55e407c220f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:43.150 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.142+0000 7f29fa7fc640 1 -- 172.21.15.99:0/4244412255 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55e407b1ed20 con 0x55e407c1aaf0 2024-03-19T01:52:43.150 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.142+0000 7f29faffd640 1 --2- 172.21.15.99:0/4244412255 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e407c1a100 0x55e407c1a520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:52:43.150 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.143+0000 7f29fb7fe640 1 --2- 172.21.15.99:0/3508573402 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f29e40a28a0 0x7f29e40a4d60 secure :-1 s=READY pgs=316 cs=0 l=1 rev1=1 crypto rx=0x7f29f4005fd0 tx=0x7f29f4009b10 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:52:43.150 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.143+0000 7f29fa7fc640 1 --2- 172.21.15.99:0/4244412255 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e407c1aaf0 0x55e407c1f5f0 secure :-1 s=READY pgs=566 cs=0 l=1 rev1=1 crypto rx=0x7f29f00c77e0 tx=0x7f29f0050000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3af777c5add3b105 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:52:43.150 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.143+0000 7f29e1ffb640 1 -- 172.21.15.99:0/4244412255 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f29f00beed0 con 0x55e407c1aaf0 2024-03-19T01:52:43.150 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.143+0000 7f29e1ffb640 1 -- 172.21.15.99:0/4244412255 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f29f0032e50 con 0x55e407c1aaf0 2024-03-19T01:52:43.150 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.143+0000 7f29e1ffb640 1 -- 172.21.15.99:0/4244412255 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f29f00bd610 con 0x55e407c1aaf0 2024-03-19T01:52:43.151 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.144+0000 7f2a04635740 1 -- 172.21.15.99:0/4244412255 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e407c1aaf0 msgr2=0x55e407c1f5f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:52:43.151 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.144+0000 7f2a04635740 1 --2- 172.21.15.99:0/4244412255 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e407c1aaf0 0x55e407c1f5f0 secure :-1 s=READY pgs=566 cs=0 l=1 rev1=1 crypto rx=0x7f29f00c77e0 tx=0x7f29f0050000 comp rx=0 tx=0).stop 2024-03-19T01:52:43.151 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.144+0000 7f2a04635740 1 -- 172.21.15.99:0/4244412255 shutdown_connections 2024-03-19T01:52:43.151 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.144+0000 7f2a04635740 1 --2- 172.21.15.99:0/4244412255 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e407c1fc60 0x55e407c220f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:43.151 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.144+0000 7f2a04635740 1 --2- 172.21.15.99:0/4244412255 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e407c1aaf0 0x55e407c1f5f0 unknown :-1 s=CLOSED pgs=566 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:43.151 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.144+0000 7f2a04635740 1 --2- 172.21.15.99:0/4244412255 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e407c1a100 0x55e407c1a520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:43.151 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.144+0000 7f2a04635740 1 -- 172.21.15.99:0/4244412255 >> 172.21.15.99:0/4244412255 conn(0x55e407c15670 msgr2=0x55e407c15a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:52:43.151 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.144+0000 7f2a04635740 1 -- 172.21.15.99:0/4244412255 shutdown_connections 2024-03-19T01:52:43.151 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.144+0000 7f2a04635740 1 -- 172.21.15.99:0/4244412255 wait complete. 2024-03-19T01:52:43.151 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.144+0000 7f2a04635740 1 -- start start 2024-03-19T01:52:43.151 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.144+0000 7f2a04635740 1 Processor -- start 2024-03-19T01:52:43.151 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.144+0000 7f2a04635740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e407c1a100 0x55e407c23d50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:43.151 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.144+0000 7f2a04635740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e407c1aaf0 0x55e407c24290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:43.151 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.144+0000 7f2a04635740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e407c1fc60 0x55e407c2b2d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:43.152 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.144+0000 7f2a04635740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55e407c229d0 con 0x55e407c1a100 2024-03-19T01:52:43.152 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.144+0000 7f2a04635740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55e407b3adb0 con 0x55e407c1fc60 2024-03-19T01:52:43.152 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.144+0000 7f2a04635740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55e407c22b50 con 0x55e407c1aaf0 2024-03-19T01:52:43.152 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.144+0000 7f2a04635740 10 client.0 _ll_register_callbacks cb 0x55e407b3fe70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:52:43.152 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.145+0000 7f29faffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e407c1a100 0x55e407c23d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:43.153 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.145+0000 7f29faffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e407c1a100 0x55e407c23d50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:37978/0 (socket says 172.21.15.99:37978) 2024-03-19T01:52:43.153 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.145+0000 7f29faffd640 1 -- 172.21.15.99:0/2579141960 learned_addr learned my addr 172.21.15.99:0/2579141960 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:52:43.153 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.145+0000 7f29fa7fc640 1 --2- 172.21.15.99:0/2579141960 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e407c1aaf0 0x55e407c24290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:43.153 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.145+0000 7f29fb7fe640 1 --2- 172.21.15.99:0/2579141960 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e407c1fc60 0x55e407c2b2d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:43.153 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.145+0000 7f29faffd640 1 -- 172.21.15.99:0/2579141960 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e407c1aaf0 msgr2=0x55e407c24290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:52:43.153 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.145+0000 7f29faffd640 1 --2- 172.21.15.99:0/2579141960 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e407c1aaf0 0x55e407c24290 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:43.154 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.145+0000 7f29faffd640 1 -- 172.21.15.99:0/2579141960 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e407c1fc60 msgr2=0x55e407c2b2d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:52:43.154 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.145+0000 7f29faffd640 1 --2- 172.21.15.99:0/2579141960 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e407c1fc60 0x55e407c2b2d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:43.154 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.145+0000 7f29faffd640 1 -- 172.21.15.99:0/2579141960 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55e407b39c60 con 0x55e407c1a100 2024-03-19T01:52:43.154 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.145+0000 7f29faffd640 1 --2- 172.21.15.99:0/2579141960 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e407c1a100 0x55e407c23d50 secure :-1 s=READY pgs=567 cs=0 l=1 rev1=1 crypto rx=0x7f29ec012d20 tx=0x7f29ec00f790 comp rx=0 tx=0).ready entity=mon.0 client_cookie=20497696ecebef6b server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:52:43.154 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.145+0000 7f29daffd640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:52:43.154 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.145+0000 7f29daffd640 1 -- 172.21.15.99:0/2579141960 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f29ec0120d0 con 0x55e407c1a100 2024-03-19T01:52:43.154 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.145+0000 7f29daffd640 1 -- 172.21.15.99:0/2579141960 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f29ec005020 con 0x55e407c1a100 2024-03-19T01:52:43.154 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.145+0000 7f2a04635740 1 -- 172.21.15.99:0/2579141960 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f29f0032e50 con 0x55e407c1a100 2024-03-19T01:52:43.154 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.145+0000 7f29daffd640 1 -- 172.21.15.99:0/2579141960 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=28) ==== 24+0+0 (secure 0 0 0) 0x7f29ec012270 con 0x55e407c1a100 2024-03-19T01:52:43.154 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.145+0000 7f2a04635740 10 client.16284 fetch_fsmap learned FSMap version 28 2024-03-19T01:52:43.154 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.145+0000 7f2a04635740 1 -- 172.21.15.99:0/2579141960 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=28,monmap=4+}) -- 0x55e407893210 con 0x55e407c1a100 2024-03-19T01:52:43.154 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.145+0000 7f29daffd640 1 -- 172.21.15.99:0/2579141960 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f29f0032e50 con 0x55e407c1a100 2024-03-19T01:52:43.154 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.146+0000 7f29daffd640 1 -- 172.21.15.99:0/2579141960 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 28) ==== 8338+0+0 (secure 0 0 0) 0x7f29ec01c470 con 0x55e407c1a100 2024-03-19T01:52:43.155 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.146+0000 7f2a04635740 10 client.16284 fetch_fsmap finished waiting for FSMap version 28 2024-03-19T01:52:43.155 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.146+0000 7f2a04635740 10 client.16284 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:52:43.155 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.146+0000 7f2a04635740 20 client.16284 populate_metadata read hostname 'smithi099' 2024-03-19T01:52:43.155 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.146+0000 7f2a04635740 1 --2- 172.21.15.99:0/2579141960 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e407c2fc10 0x55e407c32080 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:43.155 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.146+0000 7f2a04635740 4 client.16284 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:52:43.155 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.146+0000 7f2a04635740 1 -- 172.21.15.99:0/2579141960 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55e407b8c120 con 0x55e407c2fc10 2024-03-19T01:52:43.155 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.146+0000 7f29fa7fc640 1 --2- 172.21.15.99:0/2579141960 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e407c2fc10 0x55e407c32080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:43.155 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.146+0000 7f29fa7fc640 1 --2- 172.21.15.99:0/2579141960 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e407c2fc10 0x55e407c32080 crc :-1 s=READY pgs=133 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=2ce8540d5e2be5c4 server_cookie=521c40bb314bc1e0 in_seq=0 out_seq=0 2024-03-19T01:52:43.155 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.146+0000 7f29daffd640 10 client.16284 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:52:43.155 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.147+0000 7f29daffd640 1 -- 172.21.15.99:0/2579141960 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x55e407b8c120 con 0x55e407c2fc10 2024-03-19T01:52:43.155 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.147+0000 7f29daffd640 10 client.16284 handle_command_reply: tid=0 2024-03-19T01:52:43.155 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.147+0000 7f2a04635740 1 client.16284 shutdown 2024-03-19T01:52:43.155 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.147+0000 7f2a04635740 1 -- 172.21.15.99:0/2579141960 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e407c1a100 msgr2=0x55e407c23d50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:52:43.155 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.147+0000 7f2a04635740 1 --2- 172.21.15.99:0/2579141960 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e407c1a100 0x55e407c23d50 secure :-1 s=READY pgs=567 cs=0 l=1 rev1=1 crypto rx=0x7f29ec012d20 tx=0x7f29ec00f790 comp rx=0 tx=0).stop 2024-03-19T01:52:43.156 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.147+0000 7f2a04635740 1 -- 172.21.15.99:0/2579141960 shutdown_connections 2024-03-19T01:52:43.156 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.147+0000 7f2a04635740 1 --2- 172.21.15.99:0/2579141960 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e407c2fc10 0x55e407c32080 crc :-1 s=READY pgs=133 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:43.156 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.147+0000 7f2a04635740 1 --2- 172.21.15.99:0/2579141960 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e407c1fc60 0x55e407c2b2d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:43.156 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.147+0000 7f29daffd640 0 client.16284 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:52:43.156 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.147+0000 7f2a04635740 1 --2- 172.21.15.99:0/2579141960 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e407c1aaf0 0x55e407c24290 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:43.156 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.147+0000 7f2a04635740 1 --2- 172.21.15.99:0/2579141960 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e407c1a100 0x55e407c23d50 unknown :-1 s=CLOSED pgs=567 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:43.156 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.147+0000 7f2a04635740 1 -- 172.21.15.99:0/2579141960 >> 172.21.15.99:0/2579141960 conn(0x55e407c15670 msgr2=0x55e407c168c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:52:43.156 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.147+0000 7f2a04635740 1 -- 172.21.15.99:0/2579141960 shutdown_connections 2024-03-19T01:52:43.156 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.147+0000 7f2a04635740 1 -- 172.21.15.99:0/2579141960 wait complete. 2024-03-19T01:52:43.156 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.147+0000 7f2a04635740 20 client.16284 trim_cache size 0 max 16384 2024-03-19T01:52:43.167 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.162+0000 7f2a04635740 1 Processor -- start 2024-03-19T01:52:43.167 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.162+0000 7f2a04635740 1 -- start start 2024-03-19T01:52:43.167 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.162+0000 7f2a04635740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e407c1a100 0x55e407c18fe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:43.167 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.162+0000 7f2a04635740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e407c1aaf0 0x55e407c2b290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:43.167 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.162+0000 7f2a04635740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e407c1fc60 0x55e407c23040 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:43.167 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.162+0000 7f2a04635740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f29fc1a9490 con 0x55e407c1a100 2024-03-19T01:52:43.167 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.162+0000 7f2a04635740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f29fc1a6d60 con 0x55e407c1aaf0 2024-03-19T01:52:43.167 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.162+0000 7f2a04635740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f29fc1a4bb0 con 0x55e407c1fc60 2024-03-19T01:52:43.167 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.162+0000 7f29faffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e407c1a100 0x55e407c18fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:43.168 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.163+0000 7f29faffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e407c1a100 0x55e407c18fe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:37990/0 (socket says 172.21.15.99:37990) 2024-03-19T01:52:43.168 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.163+0000 7f29faffd640 1 -- 172.21.15.99:0/2445763915 learned_addr learned my addr 172.21.15.99:0/2445763915 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:52:43.168 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.163+0000 7f29fa7fc640 1 --2- 172.21.15.99:0/2445763915 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e407c1aaf0 0x55e407c2b290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:43.168 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.163+0000 7f29fb7fe640 1 --2- 172.21.15.99:0/2445763915 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e407c1fc60 0x55e407c23040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:43.168 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.163+0000 7f29faffd640 1 -- 172.21.15.99:0/2445763915 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e407c1fc60 msgr2=0x55e407c23040 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:52:43.168 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.163+0000 7f29faffd640 1 --2- 172.21.15.99:0/2445763915 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e407c1fc60 0x55e407c23040 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:43.168 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.163+0000 7f29faffd640 1 -- 172.21.15.99:0/2445763915 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e407c1aaf0 msgr2=0x55e407c2b290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:52:43.168 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.163+0000 7f29faffd640 1 --2- 172.21.15.99:0/2445763915 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e407c1aaf0 0x55e407c2b290 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:43.168 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.163+0000 7f29faffd640 1 -- 172.21.15.99:0/2445763915 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55e407c10ee0 con 0x55e407c1a100 2024-03-19T01:52:43.168 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.163+0000 7f29faffd640 1 --2- 172.21.15.99:0/2445763915 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e407c1a100 0x55e407c18fe0 secure :-1 s=READY pgs=568 cs=0 l=1 rev1=1 crypto rx=0x7f29ec0098b0 tx=0x7f29ec005df0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e90075e7167b0ae9 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:52:43.168 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.163+0000 7f29e17fa640 1 -- 172.21.15.99:0/2445763915 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f29ec01c470 con 0x55e407c1a100 2024-03-19T01:52:43.168 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.163+0000 7f29e17fa640 1 -- 172.21.15.99:0/2445763915 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f29ec01c610 con 0x55e407c1a100 2024-03-19T01:52:43.168 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.163+0000 7f29e17fa640 1 -- 172.21.15.99:0/2445763915 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f29ec0035f0 con 0x55e407c1a100 2024-03-19T01:52:43.168 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.164+0000 7f2a04635740 1 -- 172.21.15.99:0/2445763915 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e407c1a100 msgr2=0x55e407c18fe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:52:43.169 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.164+0000 7f2a04635740 1 --2- 172.21.15.99:0/2445763915 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e407c1a100 0x55e407c18fe0 secure :-1 s=READY pgs=568 cs=0 l=1 rev1=1 crypto rx=0x7f29ec0098b0 tx=0x7f29ec005df0 comp rx=0 tx=0).stop 2024-03-19T01:52:43.169 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.164+0000 7f2a04635740 1 -- 172.21.15.99:0/2445763915 shutdown_connections 2024-03-19T01:52:43.169 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.164+0000 7f2a04635740 1 --2- 172.21.15.99:0/2445763915 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e407c1aaf0 0x55e407c2b290 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:43.169 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.164+0000 7f2a04635740 1 --2- 172.21.15.99:0/2445763915 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e407c1fc60 0x55e407c23040 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:43.169 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.164+0000 7f2a04635740 1 --2- 172.21.15.99:0/2445763915 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e407c1a100 0x55e407c18fe0 unknown :-1 s=CLOSED pgs=568 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:43.169 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.164+0000 7f2a04635740 1 -- 172.21.15.99:0/2445763915 >> 172.21.15.99:0/2445763915 conn(0x55e407c15670 msgr2=0x55e407c188d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:52:43.169 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.164+0000 7f2a04635740 1 -- 172.21.15.99:0/2445763915 shutdown_connections 2024-03-19T01:52:43.169 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.164+0000 7f2a04635740 1 -- 172.21.15.99:0/2445763915 wait complete. 2024-03-19T01:52:43.169 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.164+0000 7f2a04635740 1 -- start start 2024-03-19T01:52:43.169 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.164+0000 7f2a04635740 1 Processor -- start 2024-03-19T01:52:43.169 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.165+0000 7f2a04635740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e407c1a100 0x55e407c28760 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:43.169 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.165+0000 7f2a04635740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e407c1aaf0 0x55e407c28ca0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:43.169 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.165+0000 7f2a04635740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e407c1fc60 0x55e407c29cb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:43.169 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.165+0000 7f2a04635740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f29fc1a5220 con 0x55e407c1aaf0 2024-03-19T01:52:43.170 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.165+0000 7f2a04635740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f29fc1a6690 con 0x55e407c1fc60 2024-03-19T01:52:43.170 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.165+0000 7f2a04635740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f29fc1a58f0 con 0x55e407c1a100 2024-03-19T01:52:43.170 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.165+0000 7f2a04635740 10 client.0 _ll_register_callbacks cb 0x7f29ec012930 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:52:43.170 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.165+0000 7f29fa7fc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e407c1aaf0 0x55e407c28ca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:43.170 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.165+0000 7f29fa7fc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e407c1aaf0 0x55e407c28ca0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:38000/0 (socket says 172.21.15.99:38000) 2024-03-19T01:52:43.170 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.165+0000 7f29fa7fc640 1 -- 172.21.15.99:0/1444142020 learned_addr learned my addr 172.21.15.99:0/1444142020 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:52:43.170 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.165+0000 7f29faffd640 1 --2- 172.21.15.99:0/1444142020 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e407c1a100 0x55e407c28760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:43.170 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.165+0000 7f29fb7fe640 1 --2- 172.21.15.99:0/1444142020 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e407c1fc60 0x55e407c29cb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:43.170 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.165+0000 7f29fa7fc640 1 -- 172.21.15.99:0/1444142020 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e407c1a100 msgr2=0x55e407c28760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:52:43.170 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.165+0000 7f29fa7fc640 1 --2- 172.21.15.99:0/1444142020 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e407c1a100 0x55e407c28760 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:43.170 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.165+0000 7f29fa7fc640 1 -- 172.21.15.99:0/1444142020 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e407c1fc60 msgr2=0x55e407c29cb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:52:43.170 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.165+0000 7f29fa7fc640 1 --2- 172.21.15.99:0/1444142020 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e407c1fc60 0x55e407c29cb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:43.170 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.165+0000 7f29fa7fc640 1 -- 172.21.15.99:0/1444142020 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55e40795f620 con 0x55e407c1aaf0 2024-03-19T01:52:43.171 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.165+0000 7f29fa7fc640 1 --2- 172.21.15.99:0/1444142020 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e407c1aaf0 0x55e407c28ca0 secure :-1 s=READY pgs=569 cs=0 l=1 rev1=1 crypto rx=0x7f29f00bf270 tx=0x7f29f0005f90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ef81936289e3c9c8 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:52:43.171 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.165+0000 7f29daffd640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:52:43.171 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.165+0000 7f29daffd640 1 -- 172.21.15.99:0/1444142020 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f29f0032e50 con 0x55e407c1aaf0 2024-03-19T01:52:43.171 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.165+0000 7f29daffd640 1 -- 172.21.15.99:0/1444142020 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f29f00bd870 con 0x55e407c1aaf0 2024-03-19T01:52:43.171 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.165+0000 7f2a04635740 1 -- 172.21.15.99:0/1444142020 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f29ec01c610 con 0x55e407c1aaf0 2024-03-19T01:52:43.171 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.165+0000 7f29daffd640 1 -- 172.21.15.99:0/1444142020 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=28) ==== 24+0+0 (secure 0 0 0) 0x7f29f00bfc70 con 0x55e407c1aaf0 2024-03-19T01:52:43.171 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.166+0000 7f2a04635740 10 client.16290 fetch_fsmap learned FSMap version 28 2024-03-19T01:52:43.171 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.166+0000 7f2a04635740 1 -- 172.21.15.99:0/1444142020 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=28,monmap=4+}) -- 0x55e407c2b3e0 con 0x55e407c1aaf0 2024-03-19T01:52:43.171 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.166+0000 7f29daffd640 1 -- 172.21.15.99:0/1444142020 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f29ec01c610 con 0x55e407c1aaf0 2024-03-19T01:52:43.171 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.166+0000 7f29daffd640 1 -- 172.21.15.99:0/1444142020 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 28) ==== 8338+0+0 (secure 0 0 0) 0x7f29f00ac040 con 0x55e407c1aaf0 2024-03-19T01:52:43.171 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.166+0000 7f2a04635740 10 client.16290 fetch_fsmap finished waiting for FSMap version 28 2024-03-19T01:52:43.171 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.166+0000 7f2a04635740 10 client.16290 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:52:43.171 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.166+0000 7f2a04635740 20 client.16290 populate_metadata read hostname 'smithi099' 2024-03-19T01:52:43.172 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.166+0000 7f2a04635740 1 --2- 172.21.15.99:0/1444142020 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e407c2fc10 0x55e407c980a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:43.172 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.166+0000 7f2a04635740 4 client.16290 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:52:43.172 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.166+0000 7f2a04635740 1 -- 172.21.15.99:0/1444142020 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55e407b86470 con 0x55e407c2fc10 2024-03-19T01:52:43.172 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.166+0000 7f29faffd640 1 --2- 172.21.15.99:0/1444142020 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e407c2fc10 0x55e407c980a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:43.172 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.166+0000 7f29daffd640 10 client.16290 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:52:43.172 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.166+0000 7f29faffd640 1 --2- 172.21.15.99:0/1444142020 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e407c2fc10 0x55e407c980a0 crc :-1 s=READY pgs=134 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=15719d675663ce2 server_cookie=fa97e001278b581e in_seq=0 out_seq=0 2024-03-19T01:52:43.224 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.224+0000 7f29daffd640 1 -- 172.21.15.99:0/1444142020 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x55e407b86470 con 0x55e407c2fc10 2024-03-19T01:52:43.224 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.224+0000 7f29daffd640 10 client.16290 handle_command_reply: tid=0 2024-03-19T01:52:43.225 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.224+0000 7f2a04635740 1 client.16290 shutdown 2024-03-19T01:52:43.225 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.224+0000 7f2a04635740 1 -- 172.21.15.99:0/1444142020 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e407c1aaf0 msgr2=0x55e407c28ca0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:52:43.225 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.224+0000 7f2a04635740 1 --2- 172.21.15.99:0/1444142020 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e407c1aaf0 0x55e407c28ca0 secure :-1 s=READY pgs=569 cs=0 l=1 rev1=1 crypto rx=0x7f29f00bf270 tx=0x7f29f0005f90 comp rx=0 tx=0).stop 2024-03-19T01:52:43.225 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.224+0000 7f2a04635740 1 -- 172.21.15.99:0/1444142020 shutdown_connections 2024-03-19T01:52:43.225 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.224+0000 7f2a04635740 1 --2- 172.21.15.99:0/1444142020 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e407c2fc10 0x55e407c980a0 crc :-1 s=READY pgs=134 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:43.225 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.224+0000 7f2a04635740 1 --2- 172.21.15.99:0/1444142020 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e407c1fc60 0x55e407c29cb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:43.225 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.224+0000 7f2a04635740 1 --2- 172.21.15.99:0/1444142020 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e407c1aaf0 0x55e407c28ca0 unknown :-1 s=CLOSED pgs=569 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:43.225 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.224+0000 7f2a04635740 1 --2- 172.21.15.99:0/1444142020 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e407c1a100 0x55e407c28760 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:43.225 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.224+0000 7f29daffd640 0 client.16290 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:52:43.225 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.224+0000 7f2a04635740 1 -- 172.21.15.99:0/1444142020 >> 172.21.15.99:0/1444142020 conn(0x55e407c15670 msgr2=0x55e407c19160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:52:43.225 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.224+0000 7f2a04635740 1 -- 172.21.15.99:0/1444142020 shutdown_connections 2024-03-19T01:52:43.225 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.224+0000 7f2a04635740 1 -- 172.21.15.99:0/1444142020 wait complete. 2024-03-19T01:52:43.225 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.224+0000 7f2a04635740 20 client.16290 trim_cache size 0 max 16384 2024-03-19T01:52:43.226 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:52:43.226 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:52:43.226 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:52:43.226 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:52:43.227 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.225+0000 7f2a01976640 1 -- 172.21.15.99:0/3508573402 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f29e40a28a0 msgr2=0x7f29e40a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:52:43.227 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.225+0000 7f2a01976640 1 --2- 172.21.15.99:0/3508573402 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f29e40a28a0 0x7f29e40a4d60 secure :-1 s=READY pgs=316 cs=0 l=1 rev1=1 crypto rx=0x7f29f4005fd0 tx=0x7f29f4009b10 comp rx=0 tx=0).stop 2024-03-19T01:52:43.227 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.225+0000 7f2a01976640 1 -- 172.21.15.99:0/3508573402 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f29fc101930 msgr2=0x7f29fc196f10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:52:43.227 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.225+0000 7f2a01976640 1 --2- 172.21.15.99:0/3508573402 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f29fc101930 0x7f29fc196f10 secure :-1 s=READY pgs=565 cs=0 l=1 rev1=1 crypto rx=0x7f29f002c0b0 tx=0x7f29f0032430 comp rx=0 tx=0).stop 2024-03-19T01:52:43.227 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.225+0000 7f2a01976640 1 -- 172.21.15.99:0/3508573402 shutdown_connections 2024-03-19T01:52:43.227 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.225+0000 7f2a01976640 1 --2- 172.21.15.99:0/3508573402 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f29e40a28a0 0x7f29e40a4d60 unknown :-1 s=CLOSED pgs=316 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:43.227 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.225+0000 7f2a01976640 1 --2- 172.21.15.99:0/3508573402 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f29fc1022f0 0x7f29fc19df80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:43.227 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.225+0000 7f2a01976640 1 --2- 172.21.15.99:0/3508573402 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f29fc101930 0x7f29fc196f10 unknown :-1 s=CLOSED pgs=565 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:43.227 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.225+0000 7f2a01976640 1 --2- 172.21.15.99:0/3508573402 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f29fc100740 0x7f29fc1969d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:43.228 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.225+0000 7f2a01976640 1 -- 172.21.15.99:0/3508573402 >> 172.21.15.99:0/3508573402 conn(0x7f29fc0fbcb0 msgr2=0x7f29fc104c30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:52:43.228 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.225+0000 7f2a01976640 1 -- 172.21.15.99:0/3508573402 shutdown_connections 2024-03-19T01:52:43.228 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:43.225+0000 7f2a01976640 1 -- 172.21.15.99:0/3508573402 wait complete. 2024-03-19T01:52:43.703 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1411004 11.27 MB/sec execute 553 sec latency 32.580 ms 2024-03-19T01:52:44.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:52:44 smithi115 ceph-mon[67332]: pgmap v700: 97 pgs: 97 active+clean; 253 MiB data, 3.5 GiB used, 1.0 TiB / 1.0 TiB avail; 825 KiB/s rd, 3.0 MiB/s wr, 281 op/s 2024-03-19T01:52:44.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:52:44 smithi099 ceph-mon[62951]: pgmap v700: 97 pgs: 97 active+clean; 253 MiB data, 3.5 GiB used, 1.0 TiB / 1.0 TiB avail; 825 KiB/s rd, 3.0 MiB/s wr, 281 op/s 2024-03-19T01:52:44.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:52:44 smithi112 ceph-mon[66819]: pgmap v700: 97 pgs: 97 active+clean; 253 MiB data, 3.5 GiB used, 1.0 TiB / 1.0 TiB avail; 825 KiB/s rd, 3.0 MiB/s wr, 281 op/s 2024-03-19T01:52:44.703 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1413298 11.27 MB/sec execute 554 sec latency 20.364 ms 2024-03-19T01:52:45.703 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1415067 11.27 MB/sec execute 555 sec latency 29.606 ms 2024-03-19T01:52:46.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:52:46 smithi115 ceph-mon[67332]: pgmap v701: 97 pgs: 97 active+clean; 253 MiB data, 3.5 GiB used, 1.0 TiB / 1.0 TiB avail; 559 KiB/s rd, 2.4 MiB/s wr, 211 op/s 2024-03-19T01:52:46.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:52:46 smithi099 ceph-mon[62951]: pgmap v701: 97 pgs: 97 active+clean; 253 MiB data, 3.5 GiB used, 1.0 TiB / 1.0 TiB avail; 559 KiB/s rd, 2.4 MiB/s wr, 211 op/s 2024-03-19T01:52:46.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:52:46 smithi112 ceph-mon[66819]: pgmap v701: 97 pgs: 97 active+clean; 253 MiB data, 3.5 GiB used, 1.0 TiB / 1.0 TiB avail; 559 KiB/s rd, 2.4 MiB/s wr, 211 op/s 2024-03-19T01:52:46.703 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1417444 11.27 MB/sec execute 556 sec latency 51.386 ms 2024-03-19T01:52:47.703 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1418909 11.26 MB/sec execute 557 sec latency 37.593 ms 2024-03-19T01:52:48.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:52:48 smithi115 ceph-mon[67332]: pgmap v702: 97 pgs: 97 active+clean; 254 MiB data, 3.5 GiB used, 1.0 TiB / 1.0 TiB avail; 598 KiB/s rd, 2.8 MiB/s wr, 243 op/s 2024-03-19T01:52:48.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:52:48 smithi099 ceph-mon[62951]: pgmap v702: 97 pgs: 97 active+clean; 254 MiB data, 3.5 GiB used, 1.0 TiB / 1.0 TiB avail; 598 KiB/s rd, 2.8 MiB/s wr, 243 op/s 2024-03-19T01:52:48.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:52:48 smithi112 ceph-mon[66819]: pgmap v702: 97 pgs: 97 active+clean; 254 MiB data, 3.5 GiB used, 1.0 TiB / 1.0 TiB avail; 598 KiB/s rd, 2.8 MiB/s wr, 243 op/s 2024-03-19T01:52:48.704 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1421456 11.27 MB/sec execute 558 sec latency 14.604 ms 2024-03-19T01:52:49.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:52:49 smithi115 ceph-mon[67332]: pgmap v703: 97 pgs: 97 active+clean; 256 MiB data, 3.5 GiB used, 1.0 TiB / 1.0 TiB avail; 581 KiB/s rd, 2.8 MiB/s wr, 230 op/s 2024-03-19T01:52:49.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:52:49 smithi099 ceph-mon[62951]: pgmap v703: 97 pgs: 97 active+clean; 256 MiB data, 3.5 GiB used, 1.0 TiB / 1.0 TiB avail; 581 KiB/s rd, 2.8 MiB/s wr, 230 op/s 2024-03-19T01:52:49.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:52:49 smithi112 ceph-mon[66819]: pgmap v703: 97 pgs: 97 active+clean; 256 MiB data, 3.5 GiB used, 1.0 TiB / 1.0 TiB avail; 581 KiB/s rd, 2.8 MiB/s wr, 230 op/s 2024-03-19T01:52:49.704 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1423428 11.26 MB/sec execute 559 sec latency 32.953 ms 2024-03-19T01:52:50.705 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1425191 11.27 MB/sec execute 560 sec latency 99.034 ms 2024-03-19T01:52:51.703 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1427123 11.26 MB/sec execute 561 sec latency 134.236 ms 2024-03-19T01:52:52.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:52:51 smithi115 ceph-mon[67332]: pgmap v704: 97 pgs: 97 active+clean; 258 MiB data, 3.5 GiB used, 1.0 TiB / 1.0 TiB avail; 750 KiB/s rd, 2.9 MiB/s wr, 263 op/s 2024-03-19T01:52:52.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:52:51 smithi099 ceph-mon[62951]: pgmap v704: 97 pgs: 97 active+clean; 258 MiB data, 3.5 GiB used, 1.0 TiB / 1.0 TiB avail; 750 KiB/s rd, 2.9 MiB/s wr, 263 op/s 2024-03-19T01:52:52.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:52:51 smithi112 ceph-mon[66819]: pgmap v704: 97 pgs: 97 active+clean; 258 MiB data, 3.5 GiB used, 1.0 TiB / 1.0 TiB avail; 750 KiB/s rd, 2.9 MiB/s wr, 263 op/s 2024-03-19T01:52:52.704 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1428692 11.26 MB/sec execute 562 sec latency 59.866 ms 2024-03-19T01:52:53.704 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1430736 11.25 MB/sec execute 563 sec latency 35.182 ms 2024-03-19T01:52:54.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:52:53 smithi115 ceph-mon[67332]: pgmap v705: 97 pgs: 97 active+clean; 244 MiB data, 3.5 GiB used, 1.0 TiB / 1.0 TiB avail; 637 KiB/s rd, 3.0 MiB/s wr, 261 op/s 2024-03-19T01:52:54.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:52:53 smithi099 ceph-mon[62951]: pgmap v705: 97 pgs: 97 active+clean; 244 MiB data, 3.5 GiB used, 1.0 TiB / 1.0 TiB avail; 637 KiB/s rd, 3.0 MiB/s wr, 261 op/s 2024-03-19T01:52:54.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:52:53 smithi112 ceph-mon[66819]: pgmap v705: 97 pgs: 97 active+clean; 244 MiB data, 3.5 GiB used, 1.0 TiB / 1.0 TiB avail; 637 KiB/s rd, 3.0 MiB/s wr, 261 op/s 2024-03-19T01:52:54.703 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1432625 11.26 MB/sec execute 564 sec latency 30.191 ms 2024-03-19T01:52:55.704 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1435172 11.26 MB/sec execute 565 sec latency 40.067 ms 2024-03-19T01:52:56.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:52:55 smithi115 ceph-mon[67332]: pgmap v706: 97 pgs: 97 active+clean; 240 MiB data, 3.5 GiB used, 1.0 TiB / 1.0 TiB avail; 462 KiB/s rd, 2.4 MiB/s wr, 204 op/s 2024-03-19T01:52:56.314 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.312+0000 7fa75c3b2640 1 -- 172.21.15.99:0/1733264533 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa754101400 msgr2=0x7fa754108ca0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:52:56.314 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.312+0000 7fa75c3b2640 1 --2- 172.21.15.99:0/1733264533 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa754101400 0x7fa754108ca0 secure :-1 s=READY pgs=570 cs=0 l=1 rev1=1 crypto rx=0x7fa74c009ca0 tx=0x7fa74c02c0f0 comp rx=0 tx=0).stop 2024-03-19T01:52:56.314 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.312+0000 7fa75c3b2640 1 -- 172.21.15.99:0/1733264533 shutdown_connections 2024-03-19T01:52:56.314 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.312+0000 7fa75c3b2640 1 --2- 172.21.15.99:0/1733264533 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa754100a40 0x7fa754100ec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:56.315 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.312+0000 7fa75c3b2640 1 --2- 172.21.15.99:0/1733264533 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa754101400 0x7fa754108ca0 unknown :-1 s=CLOSED pgs=570 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:56.315 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.312+0000 7fa75c3b2640 1 --2- 172.21.15.99:0/1733264533 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa754101d10 0x7fa754102130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:56.315 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.312+0000 7fa75c3b2640 1 -- 172.21.15.99:0/1733264533 >> 172.21.15.99:0/1733264533 conn(0x7fa7540fb450 msgr2=0x7fa7540fd8b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:52:56.315 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.312+0000 7fa75c3b2640 1 -- 172.21.15.99:0/1733264533 shutdown_connections 2024-03-19T01:52:56.315 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.313+0000 7fa75c3b2640 1 -- 172.21.15.99:0/1733264533 wait complete. 2024-03-19T01:52:56.315 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.313+0000 7fa75c3b2640 1 Processor -- start 2024-03-19T01:52:56.315 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.313+0000 7fa75c3b2640 1 -- start start 2024-03-19T01:52:56.315 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.313+0000 7fa75c3b2640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa754100a40 0x7fa754197a60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:56.315 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.313+0000 7fa75c3b2640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa754101400 0x7fa754197fa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:56.315 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.313+0000 7fa75c3b2640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa754101d10 0x7fa75419f010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:56.315 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.313+0000 7fa75c3b2640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fa75410d530 con 0x7fa754101d10 2024-03-19T01:52:56.315 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.313+0000 7fa75c3b2640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fa75410d3b0 con 0x7fa754100a40 2024-03-19T01:52:56.315 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.313+0000 7fa75c3b2640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fa75410d6b0 con 0x7fa754101400 2024-03-19T01:52:56.316 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.313+0000 7fa75a94f640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa754101d10 0x7fa75419f010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:56.316 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.313+0000 7fa75a14e640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa754100a40 0x7fa754197a60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:56.316 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.313+0000 7fa75a94f640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa754101d10 0x7fa75419f010 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:35822/0 (socket says 172.21.15.99:35822) 2024-03-19T01:52:56.316 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.313+0000 7fa75a94f640 1 -- 172.21.15.99:0/2996766525 learned_addr learned my addr 172.21.15.99:0/2996766525 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:52:56.316 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.313+0000 7fa75994d640 1 --2- 172.21.15.99:0/2996766525 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa754101400 0x7fa754197fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:56.316 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.314+0000 7fa75a94f640 1 -- 172.21.15.99:0/2996766525 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa754101400 msgr2=0x7fa754197fa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:52:56.316 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.314+0000 7fa75a94f640 1 --2- 172.21.15.99:0/2996766525 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa754101400 0x7fa754197fa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:56.316 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.314+0000 7fa75a94f640 1 -- 172.21.15.99:0/2996766525 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa754100a40 msgr2=0x7fa754197a60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:52:56.316 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.314+0000 7fa75a94f640 1 --2- 172.21.15.99:0/2996766525 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa754100a40 0x7fa754197a60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:56.316 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.314+0000 7fa75a94f640 1 -- 172.21.15.99:0/2996766525 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa75419f680 con 0x7fa754101d10 2024-03-19T01:52:56.316 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.314+0000 7fa75a14e640 1 --2- 172.21.15.99:0/2996766525 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa754100a40 0x7fa754197a60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:52:56.316 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.314+0000 7fa75a94f640 1 --2- 172.21.15.99:0/2996766525 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa754101d10 0x7fa75419f010 secure :-1 s=READY pgs=571 cs=0 l=1 rev1=1 crypto rx=0x7fa74c037040 tx=0x7fa74c0346f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:52:56.316 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.314+0000 7fa74b7fe640 1 -- 172.21.15.99:0/2996766525 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa74c041020 con 0x7fa754101d10 2024-03-19T01:52:56.316 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.314+0000 7fa74b7fe640 1 -- 172.21.15.99:0/2996766525 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa74c032890 con 0x7fa754101d10 2024-03-19T01:52:56.316 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.314+0000 7fa75c3b2640 1 -- 172.21.15.99:0/2996766525 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa75419f910 con 0x7fa754101d10 2024-03-19T01:52:56.317 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.314+0000 7fa74b7fe640 1 -- 172.21.15.99:0/2996766525 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa74c039040 con 0x7fa754101d10 2024-03-19T01:52:56.317 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.314+0000 7fa75c3b2640 1 -- 172.21.15.99:0/2996766525 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fa75419fe00 con 0x7fa754101d10 2024-03-19T01:52:56.323 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.316+0000 7fa74b7fe640 1 -- 172.21.15.99:0/2996766525 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7fa74c044020 con 0x7fa754101d10 2024-03-19T01:52:56.323 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.316+0000 7fa75ef9f740 1 Processor -- start 2024-03-19T01:52:56.323 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.317+0000 7fa75ef9f740 1 -- start start 2024-03-19T01:52:56.323 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.317+0000 7fa75ef9f740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fcc0c88100 0x55fcc0c88520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:56.323 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.317+0000 7fa75ef9f740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fcc0c88af0 0x55fcc0c8d5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:56.323 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.317+0000 7fa75ef9f740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fcc0c8dc60 0x55fcc0c900f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:56.323 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.317+0000 7fa75ef9f740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55fcc0becbe0 con 0x55fcc0c8dc60 2024-03-19T01:52:56.323 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.317+0000 7fa75ef9f740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55fcc0b95960 con 0x55fcc0c88af0 2024-03-19T01:52:56.323 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.317+0000 7fa75ef9f740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55fcc0be8c80 con 0x55fcc0c88100 2024-03-19T01:52:56.323 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.317+0000 7fa74b7fe640 1 --2- 172.21.15.99:0/2996766525 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa7400a28a0 0x7fa7400a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:56.323 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.317+0000 7fa75994d640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fcc0c88100 0x55fcc0c88520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:56.323 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.317+0000 7fa75a14e640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fcc0c88af0 0x55fcc0c8d5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:56.323 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.317+0000 7fa75a14e640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fcc0c88af0 0x55fcc0c8d5f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:59726/0 (socket says 172.21.15.99:59726) 2024-03-19T01:52:56.324 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.317+0000 7fa75994d640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fcc0c88100 0x55fcc0c88520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:59182/0 (socket says 172.21.15.99:59182) 2024-03-19T01:52:56.324 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.317+0000 7fa75a14e640 1 -- 172.21.15.99:0/792119817 learned_addr learned my addr 172.21.15.99:0/792119817 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:52:56.324 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.317+0000 7fa75a94f640 1 --2- 172.21.15.99:0/2996766525 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa7400a28a0 0x7fa7400a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:56.324 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.317+0000 7fa75994d640 1 --2- 172.21.15.99:0/792119817 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fcc0c8dc60 0x55fcc0c900f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:56.324 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.317+0000 7fa74b7fe640 1 -- 172.21.15.99:0/2996766525 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fa74c0f0850 con 0x7fa754101d10 2024-03-19T01:52:56.324 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.317+0000 7fa75a14e640 1 -- 172.21.15.99:0/792119817 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fcc0c88100 msgr2=0x55fcc0c88520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:52:56.324 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.317+0000 7fa75a14e640 1 --2- 172.21.15.99:0/792119817 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fcc0c88100 0x55fcc0c88520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:56.324 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.317+0000 7fa75a14e640 1 -- 172.21.15.99:0/792119817 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fcc0c8dc60 msgr2=0x55fcc0c900f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:52:56.324 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.317+0000 7fa75a14e640 1 --2- 172.21.15.99:0/792119817 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fcc0c8dc60 0x55fcc0c900f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:56.324 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.317+0000 7fa75a14e640 1 -- 172.21.15.99:0/792119817 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55fcc0b8cd20 con 0x55fcc0c88af0 2024-03-19T01:52:56.324 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.318+0000 7fa75a14e640 1 --2- 172.21.15.99:0/792119817 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fcc0c88af0 0x55fcc0c8d5f0 secure :-1 s=READY pgs=156 cs=0 l=1 rev1=1 crypto rx=0x7fa7500114f0 tx=0x7fa7500119b0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=e368af884e1910b6 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:52:56.324 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.318+0000 7fa75a94f640 1 --2- 172.21.15.99:0/2996766525 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa7400a28a0 0x7fa7400a4d60 secure :-1 s=READY pgs=318 cs=0 l=1 rev1=1 crypto rx=0x7fa74c0ee360 tx=0x7fa74c08ff70 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:52:56.324 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.318+0000 7fa748bfe640 1 -- 172.21.15.99:0/792119817 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa75001e070 con 0x55fcc0c88af0 2024-03-19T01:52:56.324 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.318+0000 7fa748bfe640 1 -- 172.21.15.99:0/792119817 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa7500035f0 con 0x55fcc0c88af0 2024-03-19T01:52:56.325 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.318+0000 7fa748bfe640 1 -- 172.21.15.99:0/792119817 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa75000f7a0 con 0x55fcc0c88af0 2024-03-19T01:52:56.325 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.319+0000 7fa75ef9f740 1 -- 172.21.15.99:0/792119817 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fcc0c88af0 msgr2=0x55fcc0c8d5f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:52:56.325 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.319+0000 7fa75ef9f740 1 --2- 172.21.15.99:0/792119817 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fcc0c88af0 0x55fcc0c8d5f0 secure :-1 s=READY pgs=156 cs=0 l=1 rev1=1 crypto rx=0x7fa7500114f0 tx=0x7fa7500119b0 comp rx=0 tx=0).stop 2024-03-19T01:52:56.325 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.319+0000 7fa75ef9f740 1 -- 172.21.15.99:0/792119817 shutdown_connections 2024-03-19T01:52:56.325 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.319+0000 7fa75ef9f740 1 --2- 172.21.15.99:0/792119817 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fcc0c88af0 0x55fcc0c8d5f0 unknown :-1 s=CLOSED pgs=156 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:56.325 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.319+0000 7fa75ef9f740 1 --2- 172.21.15.99:0/792119817 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fcc0c8dc60 0x55fcc0c900f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:56.325 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.319+0000 7fa75ef9f740 1 --2- 172.21.15.99:0/792119817 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fcc0c88100 0x55fcc0c88520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:56.325 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.319+0000 7fa75ef9f740 1 -- 172.21.15.99:0/792119817 >> 172.21.15.99:0/792119817 conn(0x55fcc0c83670 msgr2=0x55fcc0c83a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:52:56.325 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.319+0000 7fa75ef9f740 1 -- 172.21.15.99:0/792119817 shutdown_connections 2024-03-19T01:52:56.325 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.319+0000 7fa75ef9f740 1 -- 172.21.15.99:0/792119817 wait complete. 2024-03-19T01:52:56.325 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.319+0000 7fa75ef9f740 1 -- start start 2024-03-19T01:52:56.325 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.319+0000 7fa75ef9f740 1 Processor -- start 2024-03-19T01:52:56.325 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.319+0000 7fa75ef9f740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fcc0c88100 0x55fcc0c92110 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:56.325 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.319+0000 7fa75ef9f740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fcc0c88af0 0x55fcc0c92650 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:56.325 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.319+0000 7fa75ef9f740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fcc0c8dc60 0x55fcc0c99690 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:56.326 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.319+0000 7fa75ef9f740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55fcc0c90c10 con 0x55fcc0c88af0 2024-03-19T01:52:56.326 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.319+0000 7fa75ef9f740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55fcc0ba8db0 con 0x55fcc0c88100 2024-03-19T01:52:56.326 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.319+0000 7fa75ef9f740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55fcc0c90d90 con 0x55fcc0c8dc60 2024-03-19T01:52:56.326 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.319+0000 7fa75ef9f740 10 client.0 _ll_register_callbacks cb 0x55fcc0bade70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:52:56.326 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.319+0000 7fa75a14e640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fcc0c88af0 0x55fcc0c92650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:56.326 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.319+0000 7fa75994d640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fcc0c88100 0x55fcc0c92110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:56.326 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.319+0000 7fa75994d640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fcc0c88100 0x55fcc0c92110 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:59742/0 (socket says 172.21.15.99:59742) 2024-03-19T01:52:56.326 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.319+0000 7fa75994d640 1 -- 172.21.15.99:0/16586790 learned_addr learned my addr 172.21.15.99:0/16586790 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:52:56.326 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.319+0000 7fa75994d640 1 --2- 172.21.15.99:0/16586790 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fcc0c8dc60 0x55fcc0c99690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:56.326 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.320+0000 7fa75a14e640 1 -- 172.21.15.99:0/16586790 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fcc0c8dc60 msgr2=0x55fcc0c99690 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:52:56.326 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.320+0000 7fa75a14e640 1 --2- 172.21.15.99:0/16586790 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fcc0c8dc60 0x55fcc0c99690 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:56.326 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.320+0000 7fa75a14e640 1 -- 172.21.15.99:0/16586790 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fcc0c88100 msgr2=0x55fcc0c92110 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:52:56.326 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.320+0000 7fa75a14e640 1 --2- 172.21.15.99:0/16586790 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fcc0c88100 0x55fcc0c92110 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:56.326 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.320+0000 7fa75a14e640 1 -- 172.21.15.99:0/16586790 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55fcc0ba7c60 con 0x55fcc0c88af0 2024-03-19T01:52:56.327 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.320+0000 7fa75994d640 1 --2- 172.21.15.99:0/16586790 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fcc0c88100 0x55fcc0c92110 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:52:56.327 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.320+0000 7fa75a14e640 1 --2- 172.21.15.99:0/16586790 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fcc0c88af0 0x55fcc0c92650 secure :-1 s=READY pgs=572 cs=0 l=1 rev1=1 crypto rx=0x55fcc0c845e0 tx=0x7fa750003a50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1aefbc8af6c7fe76 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:52:56.327 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.320+0000 7fa73dffb640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:52:56.327 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.320+0000 7fa73dffb640 1 -- 172.21.15.99:0/16586790 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa750026020 con 0x55fcc0c88af0 2024-03-19T01:52:56.327 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.320+0000 7fa73dffb640 1 -- 172.21.15.99:0/16586790 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fa750005040 con 0x55fcc0c88af0 2024-03-19T01:52:56.327 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.320+0000 7fa75ef9f740 1 -- 172.21.15.99:0/16586790 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fa7500035f0 con 0x55fcc0c88af0 2024-03-19T01:52:56.327 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.320+0000 7fa73dffb640 1 -- 172.21.15.99:0/16586790 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=28) ==== 24+0+0 (secure 0 0 0) 0x7fa750031050 con 0x55fcc0c88af0 2024-03-19T01:52:56.327 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.320+0000 7fa75ef9f740 10 client.16302 fetch_fsmap learned FSMap version 28 2024-03-19T01:52:56.327 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.320+0000 7fa75ef9f740 1 -- 172.21.15.99:0/16586790 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=28,monmap=4+}) -- 0x55fcc0901210 con 0x55fcc0c88af0 2024-03-19T01:52:56.327 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.320+0000 7fa73dffb640 1 -- 172.21.15.99:0/16586790 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa7500035f0 con 0x55fcc0c88af0 2024-03-19T01:52:56.327 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.320+0000 7fa73dffb640 1 -- 172.21.15.99:0/16586790 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 28) ==== 8338+0+0 (secure 0 0 0) 0x7fa750021030 con 0x55fcc0c88af0 2024-03-19T01:52:56.327 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.320+0000 7fa75ef9f740 10 client.16302 fetch_fsmap finished waiting for FSMap version 28 2024-03-19T01:52:56.327 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.320+0000 7fa75ef9f740 10 client.16302 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:52:56.327 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.320+0000 7fa75ef9f740 20 client.16302 populate_metadata read hostname 'smithi099' 2024-03-19T01:52:56.327 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.321+0000 7fa75ef9f740 1 --2- 172.21.15.99:0/16586790 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55fcc0c9dca0 0x55fcc0ca00b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:56.328 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.321+0000 7fa75ef9f740 4 client.16302 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:52:56.328 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.321+0000 7fa75ef9f740 1 -- 172.21.15.99:0/16586790 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55fcc0bfa120 con 0x55fcc0c9dca0 2024-03-19T01:52:56.328 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.321+0000 7fa75994d640 1 --2- 172.21.15.99:0/16586790 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55fcc0c9dca0 0x55fcc0ca00b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:56.328 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.321+0000 7fa73dffb640 10 client.16302 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:52:56.328 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.321+0000 7fa75994d640 1 --2- 172.21.15.99:0/16586790 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55fcc0c9dca0 0x55fcc0ca00b0 crc :-1 s=READY pgs=135 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=25a9ee1cd7f813ab server_cookie=f79c1975742a4c96 in_seq=0 out_seq=0 2024-03-19T01:52:56.328 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.322+0000 7fa73dffb640 1 -- 172.21.15.99:0/16586790 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x55fcc0bfa120 con 0x55fcc0c9dca0 2024-03-19T01:52:56.328 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.322+0000 7fa73dffb640 10 client.16302 handle_command_reply: tid=0 2024-03-19T01:52:56.328 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.322+0000 7fa75ef9f740 1 client.16302 shutdown 2024-03-19T01:52:56.328 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.322+0000 7fa75ef9f740 1 -- 172.21.15.99:0/16586790 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fcc0c88af0 msgr2=0x55fcc0c92650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:52:56.328 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.322+0000 7fa75ef9f740 1 --2- 172.21.15.99:0/16586790 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fcc0c88af0 0x55fcc0c92650 secure :-1 s=READY pgs=572 cs=0 l=1 rev1=1 crypto rx=0x55fcc0c845e0 tx=0x7fa750003a50 comp rx=0 tx=0).stop 2024-03-19T01:52:56.328 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.322+0000 7fa75ef9f740 1 -- 172.21.15.99:0/16586790 shutdown_connections 2024-03-19T01:52:56.328 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.322+0000 7fa75ef9f740 1 --2- 172.21.15.99:0/16586790 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55fcc0c9dca0 0x55fcc0ca00b0 crc :-1 s=READY pgs=135 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:56.328 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.322+0000 7fa75ef9f740 1 --2- 172.21.15.99:0/16586790 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fcc0c8dc60 0x55fcc0c99690 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:56.328 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.322+0000 7fa73dffb640 0 client.16302 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:52:56.329 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.322+0000 7fa75ef9f740 1 --2- 172.21.15.99:0/16586790 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fcc0c88af0 0x55fcc0c92650 unknown :-1 s=CLOSED pgs=572 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:56.329 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.322+0000 7fa75ef9f740 1 --2- 172.21.15.99:0/16586790 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fcc0c88100 0x55fcc0c92110 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:56.329 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.322+0000 7fa75ef9f740 1 -- 172.21.15.99:0/16586790 >> 172.21.15.99:0/16586790 conn(0x55fcc0c83670 msgr2=0x55fcc0c848c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:52:56.329 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.322+0000 7fa75ef9f740 1 -- 172.21.15.99:0/16586790 shutdown_connections 2024-03-19T01:52:56.329 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.322+0000 7fa75ef9f740 1 -- 172.21.15.99:0/16586790 wait complete. 2024-03-19T01:52:56.329 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.323+0000 7fa75ef9f740 20 client.16302 trim_cache size 0 max 16384 2024-03-19T01:52:56.341 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.338+0000 7fa75ef9f740 1 Processor -- start 2024-03-19T01:52:56.341 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.338+0000 7fa75ef9f740 1 -- start start 2024-03-19T01:52:56.341 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.338+0000 7fa75ef9f740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fcc0c88100 0x55fcc0c82930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:56.341 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.338+0000 7fa75ef9f740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fcc0c88af0 0x55fcc0c99650 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:56.341 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.338+0000 7fa75ef9f740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fcc0c8dc60 0x55fcc0c89560 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:56.341 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.338+0000 7fa75ef9f740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fa7541ab630 con 0x55fcc0c8dc60 2024-03-19T01:52:56.341 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.338+0000 7fa75ef9f740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fa7541a8f00 con 0x55fcc0c88100 2024-03-19T01:52:56.341 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.338+0000 7fa75ef9f740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fa7541a6d50 con 0x55fcc0c88af0 2024-03-19T01:52:56.341 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.338+0000 7fa75994d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fcc0c8dc60 0x55fcc0c89560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:56.341 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.338+0000 7fa75994d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fcc0c8dc60 0x55fcc0c89560 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:35850/0 (socket says 172.21.15.99:35850) 2024-03-19T01:52:56.342 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.338+0000 7fa75994d640 1 -- 172.21.15.99:0/624747479 learned_addr learned my addr 172.21.15.99:0/624747479 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:52:56.342 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.338+0000 7fa75a14e640 1 --2- 172.21.15.99:0/624747479 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fcc0c88af0 0x55fcc0c99650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:56.342 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.338+0000 7fa75994d640 1 --2- 172.21.15.99:0/624747479 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fcc0c88100 0x55fcc0c82930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:56.342 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.338+0000 7fa75994d640 1 -- 172.21.15.99:0/624747479 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fcc0c88af0 msgr2=0x55fcc0c99650 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:52:56.342 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.338+0000 7fa75994d640 1 --2- 172.21.15.99:0/624747479 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fcc0c88af0 0x55fcc0c99650 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:56.343 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.338+0000 7fa75994d640 1 -- 172.21.15.99:0/624747479 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fcc0c88100 msgr2=0x55fcc0c82930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:52:56.343 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.338+0000 7fa75994d640 1 --2- 172.21.15.99:0/624747479 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fcc0c88100 0x55fcc0c82930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:56.343 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.338+0000 7fa75994d640 1 -- 172.21.15.99:0/624747479 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55fcc09cd620 con 0x55fcc0c8dc60 2024-03-19T01:52:56.343 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.339+0000 7fa75994d640 1 --2- 172.21.15.99:0/624747479 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fcc0c8dc60 0x55fcc0c89560 secure :-1 s=READY pgs=573 cs=0 l=1 rev1=1 crypto rx=0x7fa744009a90 tx=0x7fa744004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=96c6f6b1e5c2cba8 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:52:56.343 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.339+0000 7fa73ffff640 1 -- 172.21.15.99:0/624747479 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa744003c00 con 0x55fcc0c8dc60 2024-03-19T01:52:56.343 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.339+0000 7fa73ffff640 1 -- 172.21.15.99:0/624747479 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa744003da0 con 0x55fcc0c8dc60 2024-03-19T01:52:56.343 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.339+0000 7fa73ffff640 1 -- 172.21.15.99:0/624747479 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa744023a40 con 0x55fcc0c8dc60 2024-03-19T01:52:56.343 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.339+0000 7fa75ef9f740 1 -- 172.21.15.99:0/624747479 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fcc0c8dc60 msgr2=0x55fcc0c89560 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:52:56.344 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.339+0000 7fa75ef9f740 1 --2- 172.21.15.99:0/624747479 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fcc0c8dc60 0x55fcc0c89560 secure :-1 s=READY pgs=573 cs=0 l=1 rev1=1 crypto rx=0x7fa744009a90 tx=0x7fa744004730 comp rx=0 tx=0).stop 2024-03-19T01:52:56.344 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.339+0000 7fa75ef9f740 1 -- 172.21.15.99:0/624747479 shutdown_connections 2024-03-19T01:52:56.344 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.339+0000 7fa75ef9f740 1 --2- 172.21.15.99:0/624747479 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fcc0c8dc60 0x55fcc0c89560 unknown :-1 s=CLOSED pgs=573 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:56.344 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.339+0000 7fa75ef9f740 1 --2- 172.21.15.99:0/624747479 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fcc0c88af0 0x55fcc0c99650 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:56.344 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.339+0000 7fa75ef9f740 1 --2- 172.21.15.99:0/624747479 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fcc0c88100 0x55fcc0c82930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:56.344 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.339+0000 7fa75ef9f740 1 -- 172.21.15.99:0/624747479 >> 172.21.15.99:0/624747479 conn(0x55fcc0c83670 msgr2=0x55fcc0c82220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:52:56.344 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.339+0000 7fa75ef9f740 1 -- 172.21.15.99:0/624747479 shutdown_connections 2024-03-19T01:52:56.344 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.339+0000 7fa75ef9f740 1 -- 172.21.15.99:0/624747479 wait complete. 2024-03-19T01:52:56.344 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.340+0000 7fa75ef9f740 1 -- start start 2024-03-19T01:52:56.344 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.340+0000 7fa75ef9f740 1 Processor -- start 2024-03-19T01:52:56.344 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.340+0000 7fa75ef9f740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fcc0c88100 0x55fcc0c96a50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:56.344 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.340+0000 7fa75ef9f740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fcc0c88af0 0x55fcc0c96f90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:56.344 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.340+0000 7fa75ef9f740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fcc0c8dc60 0x55fcc0c97fa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:56.344 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.340+0000 7fa75ef9f740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fa7541a73c0 con 0x55fcc0c88100 2024-03-19T01:52:56.345 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.340+0000 7fa75ef9f740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fa7541a8830 con 0x55fcc0c8dc60 2024-03-19T01:52:56.345 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.340+0000 7fa75ef9f740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fa7541a7a90 con 0x55fcc0c88af0 2024-03-19T01:52:56.345 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.340+0000 7fa75ef9f740 10 client.0 _ll_register_callbacks cb 0x7fa75001c070 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:52:56.345 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.340+0000 7fa75994d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fcc0c88100 0x55fcc0c96a50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:56.345 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.340+0000 7fa75994d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fcc0c88100 0x55fcc0c96a50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:35856/0 (socket says 172.21.15.99:35856) 2024-03-19T01:52:56.345 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.340+0000 7fa75994d640 1 -- 172.21.15.99:0/299305183 learned_addr learned my addr 172.21.15.99:0/299305183 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:52:56.345 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.340+0000 7fa75a14e640 1 --2- 172.21.15.99:0/299305183 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fcc0c88af0 0x55fcc0c96f90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:56.345 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.340+0000 7fa75994d640 1 --2- 172.21.15.99:0/299305183 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fcc0c8dc60 0x55fcc0c97fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:56.345 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.340+0000 7fa75994d640 1 -- 172.21.15.99:0/299305183 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fcc0c88af0 msgr2=0x55fcc0c96f90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:52:56.345 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.340+0000 7fa75994d640 1 --2- 172.21.15.99:0/299305183 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fcc0c88af0 0x55fcc0c96f90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:56.345 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.340+0000 7fa75994d640 1 -- 172.21.15.99:0/299305183 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fcc0c8dc60 msgr2=0x55fcc0c97fa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:52:56.345 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.340+0000 7fa75994d640 1 --2- 172.21.15.99:0/299305183 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fcc0c8dc60 0x55fcc0c97fa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:56.345 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.340+0000 7fa75994d640 1 -- 172.21.15.99:0/299305183 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55fcc0c98590 con 0x55fcc0c88100 2024-03-19T01:52:56.345 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.340+0000 7fa75994d640 1 --2- 172.21.15.99:0/299305183 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fcc0c88100 0x55fcc0c96a50 secure :-1 s=READY pgs=574 cs=0 l=1 rev1=1 crypto rx=0x7fa744006010 tx=0x7fa744009310 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3ccbe5cc6988d446 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:52:56.346 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.340+0000 7fa73dffb640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:52:56.346 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.340+0000 7fa73dffb640 1 -- 172.21.15.99:0/299305183 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa744011250 con 0x55fcc0c88100 2024-03-19T01:52:56.346 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.341+0000 7fa73dffb640 1 -- 172.21.15.99:0/299305183 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fa744028200 con 0x55fcc0c88100 2024-03-19T01:52:56.346 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.341+0000 7fa75ef9f740 1 -- 172.21.15.99:0/299305183 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fa744003da0 con 0x55fcc0c88100 2024-03-19T01:52:56.346 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:52:55 smithi099 ceph-mon[62951]: pgmap v706: 97 pgs: 97 active+clean; 240 MiB data, 3.5 GiB used, 1.0 TiB / 1.0 TiB avail; 462 KiB/s rd, 2.4 MiB/s wr, 204 op/s 2024-03-19T01:52:56.348 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.341+0000 7fa73dffb640 1 -- 172.21.15.99:0/299305183 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=28) ==== 24+0+0 (secure 0 0 0) 0x7fa744028400 con 0x55fcc0c88100 2024-03-19T01:52:56.348 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.341+0000 7fa75ef9f740 10 client.16308 fetch_fsmap learned FSMap version 28 2024-03-19T01:52:56.349 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.341+0000 7fa75ef9f740 1 -- 172.21.15.99:0/299305183 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=28,monmap=4+}) -- 0x55fcc0c998a0 con 0x55fcc0c88100 2024-03-19T01:52:56.349 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.341+0000 7fa73dffb640 1 -- 172.21.15.99:0/299305183 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa744003da0 con 0x55fcc0c88100 2024-03-19T01:52:56.349 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.341+0000 7fa73dffb640 1 -- 172.21.15.99:0/299305183 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 28) ==== 8338+0+0 (secure 0 0 0) 0x7fa744031040 con 0x55fcc0c88100 2024-03-19T01:52:56.349 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.341+0000 7fa75ef9f740 10 client.16308 fetch_fsmap finished waiting for FSMap version 28 2024-03-19T01:52:56.349 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.341+0000 7fa75ef9f740 10 client.16308 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:52:56.349 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.341+0000 7fa75ef9f740 20 client.16308 populate_metadata read hostname 'smithi099' 2024-03-19T01:52:56.349 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.341+0000 7fa75ef9f740 1 --2- 172.21.15.99:0/299305183 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55fcc0d045c0 0x55fcc0d069e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:52:56.349 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.341+0000 7fa75ef9f740 4 client.16308 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:52:56.349 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.341+0000 7fa75ef9f740 1 -- 172.21.15.99:0/299305183 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55fcc0bf4470 con 0x55fcc0d045c0 2024-03-19T01:52:56.350 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.341+0000 7fa75a14e640 1 --2- 172.21.15.99:0/299305183 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55fcc0d045c0 0x55fcc0d069e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:52:56.350 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.342+0000 7fa75a14e640 1 --2- 172.21.15.99:0/299305183 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55fcc0d045c0 0x55fcc0d069e0 crc :-1 s=READY pgs=136 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=481c4e505442e3b0 server_cookie=56759fe7aed55059 in_seq=0 out_seq=0 2024-03-19T01:52:56.350 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.342+0000 7fa73dffb640 10 client.16308 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:52:56.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:52:55 smithi112 ceph-mon[66819]: pgmap v706: 97 pgs: 97 active+clean; 240 MiB data, 3.5 GiB used, 1.0 TiB / 1.0 TiB avail; 462 KiB/s rd, 2.4 MiB/s wr, 204 op/s 2024-03-19T01:52:56.704 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1436746 11.26 MB/sec execute 566 sec latency 29.842 ms 2024-03-19T01:52:56.757 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.756+0000 7fa73dffb640 1 -- 172.21.15.99:0/299305183 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x55fcc0bf4470 con 0x55fcc0d045c0 2024-03-19T01:52:56.757 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.756+0000 7fa73dffb640 10 client.16308 handle_command_reply: tid=0 2024-03-19T01:52:56.757 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.756+0000 7fa75ef9f740 1 client.16308 shutdown 2024-03-19T01:52:56.757 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.756+0000 7fa75ef9f740 1 -- 172.21.15.99:0/299305183 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fcc0c88100 msgr2=0x55fcc0c96a50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:52:56.757 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.756+0000 7fa75ef9f740 1 --2- 172.21.15.99:0/299305183 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fcc0c88100 0x55fcc0c96a50 secure :-1 s=READY pgs=574 cs=0 l=1 rev1=1 crypto rx=0x7fa744006010 tx=0x7fa744009310 comp rx=0 tx=0).stop 2024-03-19T01:52:56.757 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.757+0000 7fa75ef9f740 1 -- 172.21.15.99:0/299305183 shutdown_connections 2024-03-19T01:52:56.757 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.757+0000 7fa75ef9f740 1 --2- 172.21.15.99:0/299305183 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55fcc0d045c0 0x55fcc0d069e0 crc :-1 s=READY pgs=136 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:56.758 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.757+0000 7fa75ef9f740 1 --2- 172.21.15.99:0/299305183 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fcc0c8dc60 0x55fcc0c97fa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:56.758 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.757+0000 7fa73dffb640 0 client.16308 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:52:56.758 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.757+0000 7fa75ef9f740 1 --2- 172.21.15.99:0/299305183 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fcc0c88af0 0x55fcc0c96f90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:56.758 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.757+0000 7fa75ef9f740 1 --2- 172.21.15.99:0/299305183 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fcc0c88100 0x55fcc0c96a50 unknown :-1 s=CLOSED pgs=574 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:56.758 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.757+0000 7fa75ef9f740 1 -- 172.21.15.99:0/299305183 >> 172.21.15.99:0/299305183 conn(0x55fcc0c83670 msgr2=0x55fcc0c868d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:52:56.758 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.757+0000 7fa75ef9f740 1 -- 172.21.15.99:0/299305183 shutdown_connections 2024-03-19T01:52:56.758 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.757+0000 7fa75ef9f740 1 -- 172.21.15.99:0/299305183 wait complete. 2024-03-19T01:52:56.758 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.757+0000 7fa75ef9f740 20 client.16308 trim_cache size 0 max 16384 2024-03-19T01:52:56.758 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:52:56.758 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:52:56.759 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:52:56.759 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:52:56.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.757+0000 7fa75c3b2640 1 -- 172.21.15.99:0/2996766525 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa7400a28a0 msgr2=0x7fa7400a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:52:56.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.757+0000 7fa75c3b2640 1 --2- 172.21.15.99:0/2996766525 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa7400a28a0 0x7fa7400a4d60 secure :-1 s=READY pgs=318 cs=0 l=1 rev1=1 crypto rx=0x7fa74c0ee360 tx=0x7fa74c08ff70 comp rx=0 tx=0).stop 2024-03-19T01:52:56.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.757+0000 7fa75c3b2640 1 -- 172.21.15.99:0/2996766525 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa754101d10 msgr2=0x7fa75419f010 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:52:56.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.757+0000 7fa75c3b2640 1 --2- 172.21.15.99:0/2996766525 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa754101d10 0x7fa75419f010 secure :-1 s=READY pgs=571 cs=0 l=1 rev1=1 crypto rx=0x7fa74c037040 tx=0x7fa74c0346f0 comp rx=0 tx=0).stop 2024-03-19T01:52:56.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.757+0000 7fa75c3b2640 1 -- 172.21.15.99:0/2996766525 shutdown_connections 2024-03-19T01:52:56.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.757+0000 7fa75c3b2640 1 --2- 172.21.15.99:0/2996766525 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa754101d10 0x7fa75419f010 unknown :-1 s=CLOSED pgs=571 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:56.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.757+0000 7fa75c3b2640 1 --2- 172.21.15.99:0/2996766525 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa754101400 0x7fa754197fa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:56.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.757+0000 7fa75c3b2640 1 --2- 172.21.15.99:0/2996766525 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa7400a28a0 0x7fa7400a4d60 unknown :-1 s=CLOSED pgs=318 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:56.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.757+0000 7fa75c3b2640 1 --2- 172.21.15.99:0/2996766525 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa754100a40 0x7fa754197a60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:52:56.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.757+0000 7fa75c3b2640 1 -- 172.21.15.99:0/2996766525 >> 172.21.15.99:0/2996766525 conn(0x7fa7540fb450 msgr2=0x7fa754104c80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:52:56.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.757+0000 7fa75c3b2640 1 -- 172.21.15.99:0/2996766525 shutdown_connections 2024-03-19T01:52:56.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:52:56.757+0000 7fa75c3b2640 1 -- 172.21.15.99:0/2996766525 wait complete. 2024-03-19T01:52:57.704 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1438722 11.25 MB/sec execute 567 sec latency 105.228 ms 2024-03-19T01:52:58.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:52:57 smithi115 ceph-mon[67332]: pgmap v707: 97 pgs: 97 active+clean; 227 MiB data, 3.5 GiB used, 1.0 TiB / 1.0 TiB avail; 667 KiB/s rd, 3.0 MiB/s wr, 254 op/s 2024-03-19T01:52:58.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:52:57 smithi099 ceph-mon[62951]: pgmap v707: 97 pgs: 97 active+clean; 227 MiB data, 3.5 GiB used, 1.0 TiB / 1.0 TiB avail; 667 KiB/s rd, 3.0 MiB/s wr, 254 op/s 2024-03-19T01:52:58.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:52:57 smithi112 ceph-mon[66819]: pgmap v707: 97 pgs: 97 active+clean; 227 MiB data, 3.5 GiB used, 1.0 TiB / 1.0 TiB avail; 667 KiB/s rd, 3.0 MiB/s wr, 254 op/s 2024-03-19T01:52:58.704 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1440353 11.25 MB/sec execute 568 sec latency 30.186 ms 2024-03-19T01:52:59.704 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1442778 11.26 MB/sec execute 569 sec latency 10.919 ms 2024-03-19T01:53:00.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:52:59 smithi115 ceph-mon[67332]: pgmap v708: 97 pgs: 97 active+clean; 235 MiB data, 3.5 GiB used, 1.0 TiB / 1.0 TiB avail; 742 KiB/s rd, 3.2 MiB/s wr, 258 op/s 2024-03-19T01:53:00.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:52:59 smithi099 ceph-mon[62951]: pgmap v708: 97 pgs: 97 active+clean; 235 MiB data, 3.5 GiB used, 1.0 TiB / 1.0 TiB avail; 742 KiB/s rd, 3.2 MiB/s wr, 258 op/s 2024-03-19T01:53:00.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:52:59 smithi112 ceph-mon[66819]: pgmap v708: 97 pgs: 97 active+clean; 235 MiB data, 3.5 GiB used, 1.0 TiB / 1.0 TiB avail; 742 KiB/s rd, 3.2 MiB/s wr, 258 op/s 2024-03-19T01:53:00.704 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1444787 11.25 MB/sec execute 570 sec latency 31.526 ms 2024-03-19T01:53:01.704 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1446588 11.25 MB/sec execute 571 sec latency 10.340 ms 2024-03-19T01:53:02.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:53:02 smithi115 ceph-mon[67332]: pgmap v709: 97 pgs: 97 active+clean; 243 MiB data, 3.5 GiB used, 1.0 TiB / 1.0 TiB avail; 553 KiB/s rd, 3.6 MiB/s wr, 226 op/s 2024-03-19T01:53:02.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:53:02 smithi099 ceph-mon[62951]: pgmap v709: 97 pgs: 97 active+clean; 243 MiB data, 3.5 GiB used, 1.0 TiB / 1.0 TiB avail; 553 KiB/s rd, 3.6 MiB/s wr, 226 op/s 2024-03-19T01:53:02.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:53:02 smithi112 ceph-mon[66819]: pgmap v709: 97 pgs: 97 active+clean; 243 MiB data, 3.5 GiB used, 1.0 TiB / 1.0 TiB avail; 553 KiB/s rd, 3.6 MiB/s wr, 226 op/s 2024-03-19T01:53:02.704 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1448785 11.25 MB/sec execute 572 sec latency 43.408 ms 2024-03-19T01:53:03.704 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1450554 11.25 MB/sec execute 573 sec latency 29.421 ms 2024-03-19T01:53:04.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:53:04 smithi115 ceph-mon[67332]: pgmap v710: 97 pgs: 97 active+clean; 252 MiB data, 3.6 GiB used, 1.0 TiB / 1.0 TiB avail; 660 KiB/s rd, 4.4 MiB/s wr, 260 op/s 2024-03-19T01:53:04.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:53:04 smithi099 ceph-mon[62951]: pgmap v710: 97 pgs: 97 active+clean; 252 MiB data, 3.6 GiB used, 1.0 TiB / 1.0 TiB avail; 660 KiB/s rd, 4.4 MiB/s wr, 260 op/s 2024-03-19T01:53:04.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:53:04 smithi112 ceph-mon[66819]: pgmap v710: 97 pgs: 97 active+clean; 252 MiB data, 3.6 GiB used, 1.0 TiB / 1.0 TiB avail; 660 KiB/s rd, 4.4 MiB/s wr, 260 op/s 2024-03-19T01:53:04.704 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1452950 11.25 MB/sec execute 574 sec latency 130.907 ms 2024-03-19T01:53:05.704 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1454614 11.25 MB/sec execute 575 sec latency 41.578 ms 2024-03-19T01:53:06.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:53:06 smithi115 ceph-mon[67332]: pgmap v711: 97 pgs: 97 active+clean; 252 MiB data, 3.6 GiB used, 1.0 TiB / 1.0 TiB avail; 682 KiB/s rd, 4.2 MiB/s wr, 234 op/s 2024-03-19T01:53:06.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:53:06 smithi099 ceph-mon[62951]: pgmap v711: 97 pgs: 97 active+clean; 252 MiB data, 3.6 GiB used, 1.0 TiB / 1.0 TiB avail; 682 KiB/s rd, 4.2 MiB/s wr, 234 op/s 2024-03-19T01:53:06.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:53:06 smithi112 ceph-mon[66819]: pgmap v711: 97 pgs: 97 active+clean; 252 MiB data, 3.6 GiB used, 1.0 TiB / 1.0 TiB avail; 682 KiB/s rd, 4.2 MiB/s wr, 234 op/s 2024-03-19T01:53:06.704 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1457038 11.25 MB/sec execute 576 sec latency 30.497 ms 2024-03-19T01:53:07.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:53:07 smithi099 ceph-mon[62951]: pgmap v712: 97 pgs: 97 active+clean; 255 MiB data, 3.6 GiB used, 1.0 TiB / 1.0 TiB avail; 953 KiB/s rd, 4.8 MiB/s wr, 304 op/s 2024-03-19T01:53:07.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:53:07 smithi112 ceph-mon[66819]: pgmap v712: 97 pgs: 97 active+clean; 255 MiB data, 3.6 GiB used, 1.0 TiB / 1.0 TiB avail; 953 KiB/s rd, 4.8 MiB/s wr, 304 op/s 2024-03-19T01:53:07.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:53:07 smithi115 ceph-mon[67332]: pgmap v712: 97 pgs: 97 active+clean; 255 MiB data, 3.6 GiB used, 1.0 TiB / 1.0 TiB avail; 953 KiB/s rd, 4.8 MiB/s wr, 304 op/s 2024-03-19T01:53:07.705 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1458917 11.24 MB/sec execute 577 sec latency 85.944 ms 2024-03-19T01:53:08.704 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1460801 11.25 MB/sec execute 578 sec latency 13.056 ms 2024-03-19T01:53:09.704 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1463219 11.25 MB/sec execute 579 sec latency 34.651 ms 2024-03-19T01:53:09.848 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.845+0000 7f3fb379d640 1 -- 172.21.15.99:0/517172437 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3fac101950 msgr2=0x7f3fac101dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:53:09.848 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.845+0000 7f3fb379d640 1 --2- 172.21.15.99:0/517172437 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3fac101950 0x7f3fac101dd0 secure :-1 s=READY pgs=575 cs=0 l=1 rev1=1 crypto rx=0x7f3f9c009900 tx=0x7f3f9c02bc30 comp rx=0 tx=0).stop 2024-03-19T01:53:09.848 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.845+0000 7f3fb379d640 1 -- 172.21.15.99:0/517172437 shutdown_connections 2024-03-19T01:53:09.848 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.845+0000 7f3fb379d640 1 --2- 172.21.15.99:0/517172437 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3fac102310 0x7f3fac108ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:09.848 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.845+0000 7f3fb379d640 1 --2- 172.21.15.99:0/517172437 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3fac101950 0x7f3fac101dd0 unknown :-1 s=CLOSED pgs=575 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:09.848 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.845+0000 7f3fb379d640 1 --2- 172.21.15.99:0/517172437 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f3fac100760 0x7f3fac100b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:09.848 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.845+0000 7f3fb379d640 1 -- 172.21.15.99:0/517172437 >> 172.21.15.99:0/517172437 conn(0x7f3fac0fbcd0 msgr2=0x7f3fac0fe110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:53:09.848 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.846+0000 7f3fb379d640 1 -- 172.21.15.99:0/517172437 shutdown_connections 2024-03-19T01:53:09.848 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.846+0000 7f3fb379d640 1 -- 172.21.15.99:0/517172437 wait complete. 2024-03-19T01:53:09.848 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.846+0000 7f3fb379d640 1 Processor -- start 2024-03-19T01:53:09.848 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.846+0000 7f3fb379d640 1 -- start start 2024-03-19T01:53:09.849 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.846+0000 7f3fb379d640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3fac100760 0x7f3fac197a70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:09.849 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.846+0000 7f3fb379d640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f3fac101950 0x7f3fac197fb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:09.849 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.846+0000 7f3fb379d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3fac102310 0x7f3fac19f020 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:09.849 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.846+0000 7f3fb379d640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f3fac10d500 con 0x7f3fac102310 2024-03-19T01:53:09.849 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.846+0000 7f3fb379d640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f3fac10d380 con 0x7f3fac100760 2024-03-19T01:53:09.849 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.846+0000 7f3fb379d640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f3fac10d680 con 0x7f3fac101950 2024-03-19T01:53:09.849 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.846+0000 7f3fb1d3a640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3fac102310 0x7f3fac19f020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:09.849 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.846+0000 7f3fb1539640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3fac100760 0x7f3fac197a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:09.849 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.846+0000 7f3fb1d3a640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3fac102310 0x7f3fac19f020 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:46848/0 (socket says 172.21.15.99:46848) 2024-03-19T01:53:09.849 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.846+0000 7f3fb1d3a640 1 -- 172.21.15.99:0/143680265 learned_addr learned my addr 172.21.15.99:0/143680265 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:53:09.849 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.847+0000 7f3fb0d38640 1 --2- 172.21.15.99:0/143680265 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f3fac101950 0x7f3fac197fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:09.849 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.847+0000 7f3fb1d3a640 1 -- 172.21.15.99:0/143680265 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f3fac101950 msgr2=0x7f3fac197fb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:53:09.849 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.847+0000 7f3fb1d3a640 1 --2- 172.21.15.99:0/143680265 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f3fac101950 0x7f3fac197fb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:09.849 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.847+0000 7f3fb1d3a640 1 -- 172.21.15.99:0/143680265 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3fac100760 msgr2=0x7f3fac197a70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:53:09.850 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.847+0000 7f3fb1d3a640 1 --2- 172.21.15.99:0/143680265 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3fac100760 0x7f3fac197a70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:09.850 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.847+0000 7f3fb1d3a640 1 -- 172.21.15.99:0/143680265 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f3fac19f690 con 0x7f3fac102310 2024-03-19T01:53:09.850 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.847+0000 7f3fb1539640 1 --2- 172.21.15.99:0/143680265 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3fac100760 0x7f3fac197a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:53:09.850 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.847+0000 7f3fb1d3a640 1 --2- 172.21.15.99:0/143680265 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3fac102310 0x7f3fac19f020 secure :-1 s=READY pgs=576 cs=0 l=1 rev1=1 crypto rx=0x7f3fa400eb10 tx=0x7f3fa400c2d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:53:09.850 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.847+0000 7f3fa27fc640 1 -- 172.21.15.99:0/143680265 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3fa4004b20 con 0x7f3fac102310 2024-03-19T01:53:09.850 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.847+0000 7f3fa27fc640 1 -- 172.21.15.99:0/143680265 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f3fa4004cc0 con 0x7f3fac102310 2024-03-19T01:53:09.850 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.847+0000 7f3fb379d640 1 -- 172.21.15.99:0/143680265 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f3fac19f980 con 0x7f3fac102310 2024-03-19T01:53:09.850 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.847+0000 7f3fa27fc640 1 -- 172.21.15.99:0/143680265 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3fa4010760 con 0x7f3fac102310 2024-03-19T01:53:09.850 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.847+0000 7f3fb379d640 1 -- 172.21.15.99:0/143680265 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f3fac19fe50 con 0x7f3fac102310 2024-03-19T01:53:09.857 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.849+0000 7f3fa27fc640 1 -- 172.21.15.99:0/143680265 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f3fa4005020 con 0x7f3fac102310 2024-03-19T01:53:09.857 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.850+0000 7f3fb6381740 1 Processor -- start 2024-03-19T01:53:09.857 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.850+0000 7f3fb6381740 1 -- start start 2024-03-19T01:53:09.857 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.850+0000 7f3fa27fc640 1 --2- 172.21.15.99:0/143680265 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f3f980a28a0 0x7f3f980a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:09.857 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.850+0000 7f3fb6381740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5574e74eb100 0x5574e74eb520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:09.857 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.850+0000 7f3fb6381740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5574e74ebaf0 0x5574e74f05f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:09.857 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.850+0000 7f3fb6381740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5574e74f0c60 0x5574e74f30f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:09.857 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.850+0000 7f3fb6381740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x5574e744fbe0 con 0x5574e74f0c60 2024-03-19T01:53:09.857 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.850+0000 7f3fb6381740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x5574e73f8960 con 0x5574e74eb100 2024-03-19T01:53:09.857 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.850+0000 7f3fb6381740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x5574e744bc80 con 0x5574e74ebaf0 2024-03-19T01:53:09.858 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.850+0000 7f3fb0d38640 1 --2- 172.21.15.99:0/143680265 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f3f980a28a0 0x7f3f980a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:09.858 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.850+0000 7f3fa27fc640 1 -- 172.21.15.99:0/143680265 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f3fa40cf010 con 0x7f3fac102310 2024-03-19T01:53:09.858 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.850+0000 7f3fb1539640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5574e74eb100 0x5574e74eb520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:09.858 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.850+0000 7f3fb1d3a640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5574e74f0c60 0x5574e74f30f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:09.858 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.850+0000 7f3fb1539640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5574e74eb100 0x5574e74eb520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:50022/0 (socket says 172.21.15.99:50022) 2024-03-19T01:53:09.858 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.850+0000 7f3fb1539640 1 -- 172.21.15.99:0/1420123048 learned_addr learned my addr 172.21.15.99:0/1420123048 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:53:09.858 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.850+0000 7f3fb0d38640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5574e74ebaf0 0x5574e74f05f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:09.858 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.851+0000 7f3fb1d3a640 1 -- 172.21.15.99:0/1420123048 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5574e74ebaf0 msgr2=0x5574e74f05f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:53:09.858 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.851+0000 7f3fb1d3a640 1 --2- 172.21.15.99:0/1420123048 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5574e74ebaf0 0x5574e74f05f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:09.858 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.851+0000 7f3fb1d3a640 1 -- 172.21.15.99:0/1420123048 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5574e74eb100 msgr2=0x5574e74eb520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:53:09.858 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.851+0000 7f3fb1d3a640 1 --2- 172.21.15.99:0/1420123048 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5574e74eb100 0x5574e74eb520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:09.858 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.851+0000 7f3fb1d3a640 1 -- 172.21.15.99:0/1420123048 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5574e73efd20 con 0x5574e74f0c60 2024-03-19T01:53:09.858 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.851+0000 7f3fb1539640 1 --2- 172.21.15.99:0/1420123048 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5574e74eb100 0x5574e74eb520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:53:09.858 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.851+0000 7f3fb0d38640 1 --2- 172.21.15.99:0/143680265 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f3f980a28a0 0x7f3f980a4d60 secure :-1 s=READY pgs=319 cs=0 l=1 rev1=1 crypto rx=0x7f3f9c02c7b0 tx=0x7f3f9c005d50 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:53:09.858 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.851+0000 7f3fb1d3a640 1 --2- 172.21.15.99:0/1420123048 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5574e74f0c60 0x5574e74f30f0 secure :-1 s=READY pgs=577 cs=0 l=1 rev1=1 crypto rx=0x7f3fa40ccb20 tx=0x7f3fa4097c90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=dcfe7dbc9beb82cf server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:53:09.859 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.851+0000 7f3f977fe640 1 -- 172.21.15.99:0/1420123048 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3fa409b2d0 con 0x5574e74f0c60 2024-03-19T01:53:09.859 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.851+0000 7f3f977fe640 1 -- 172.21.15.99:0/1420123048 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f3fa409c070 con 0x5574e74f0c60 2024-03-19T01:53:09.859 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.851+0000 7f3f977fe640 1 -- 172.21.15.99:0/1420123048 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3fa4078e00 con 0x5574e74f0c60 2024-03-19T01:53:09.859 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.852+0000 7f3fb6381740 1 -- 172.21.15.99:0/1420123048 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5574e74f0c60 msgr2=0x5574e74f30f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:53:09.859 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.852+0000 7f3fb6381740 1 --2- 172.21.15.99:0/1420123048 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5574e74f0c60 0x5574e74f30f0 secure :-1 s=READY pgs=577 cs=0 l=1 rev1=1 crypto rx=0x7f3fa40ccb20 tx=0x7f3fa4097c90 comp rx=0 tx=0).stop 2024-03-19T01:53:09.859 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.852+0000 7f3fb6381740 1 -- 172.21.15.99:0/1420123048 shutdown_connections 2024-03-19T01:53:09.859 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.852+0000 7f3fb6381740 1 --2- 172.21.15.99:0/1420123048 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5574e74f0c60 0x5574e74f30f0 unknown :-1 s=CLOSED pgs=577 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:09.859 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.852+0000 7f3fb6381740 1 --2- 172.21.15.99:0/1420123048 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5574e74ebaf0 0x5574e74f05f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:09.859 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.852+0000 7f3fb6381740 1 --2- 172.21.15.99:0/1420123048 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5574e74eb100 0x5574e74eb520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:09.859 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.852+0000 7f3fb6381740 1 -- 172.21.15.99:0/1420123048 >> 172.21.15.99:0/1420123048 conn(0x5574e74e6670 msgr2=0x5574e74e6a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:53:09.859 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.852+0000 7f3fb6381740 1 -- 172.21.15.99:0/1420123048 shutdown_connections 2024-03-19T01:53:09.859 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.852+0000 7f3fb6381740 1 -- 172.21.15.99:0/1420123048 wait complete. 2024-03-19T01:53:09.859 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.852+0000 7f3fb6381740 1 -- start start 2024-03-19T01:53:09.859 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.852+0000 7f3fb6381740 1 Processor -- start 2024-03-19T01:53:09.860 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.852+0000 7f3fb6381740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5574e74eb100 0x5574e74f5070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:09.860 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.852+0000 7f3fb6381740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5574e74ebaf0 0x5574e74f55b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:09.860 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.852+0000 7f3fb6381740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5574e74f0c60 0x5574e74fc5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:09.860 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.852+0000 7f3fb6381740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x5574e74f3c10 con 0x5574e74ebaf0 2024-03-19T01:53:09.860 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.852+0000 7f3fb6381740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x5574e740bdb0 con 0x5574e74f0c60 2024-03-19T01:53:09.860 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.852+0000 7f3fb6381740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x5574e74f3d90 con 0x5574e74eb100 2024-03-19T01:53:09.860 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.852+0000 7f3fb6381740 10 client.0 _ll_register_callbacks cb 0x5574e7410e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:53:09.860 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.853+0000 7f3fb0d38640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5574e74ebaf0 0x5574e74f55b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:09.860 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.853+0000 7f3fb0d38640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5574e74ebaf0 0x5574e74f55b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:46864/0 (socket says 172.21.15.99:46864) 2024-03-19T01:53:09.860 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.853+0000 7f3fb0d38640 1 -- 172.21.15.99:0/1470123109 learned_addr learned my addr 172.21.15.99:0/1470123109 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:53:09.860 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.853+0000 7f3fb1d3a640 1 --2- 172.21.15.99:0/1470123109 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5574e74f0c60 0x5574e74fc5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:09.860 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.853+0000 7f3fb1539640 1 --2- 172.21.15.99:0/1470123109 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5574e74eb100 0x5574e74f5070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:09.860 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.853+0000 7f3fb0d38640 1 -- 172.21.15.99:0/1470123109 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5574e74eb100 msgr2=0x5574e74f5070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:53:09.860 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.853+0000 7f3fb0d38640 1 --2- 172.21.15.99:0/1470123109 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5574e74eb100 0x5574e74f5070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:09.860 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.853+0000 7f3fb0d38640 1 -- 172.21.15.99:0/1470123109 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5574e74f0c60 msgr2=0x5574e74fc5f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:53:09.861 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.853+0000 7f3fb0d38640 1 --2- 172.21.15.99:0/1470123109 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5574e74f0c60 0x5574e74fc5f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:09.861 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.853+0000 7f3fb0d38640 1 -- 172.21.15.99:0/1470123109 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5574e740ac60 con 0x5574e74ebaf0 2024-03-19T01:53:09.861 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.853+0000 7f3fb1d3a640 1 --2- 172.21.15.99:0/1470123109 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5574e74f0c60 0x5574e74fc5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:53:09.861 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.853+0000 7f3fb0d38640 1 --2- 172.21.15.99:0/1470123109 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5574e74ebaf0 0x5574e74f55b0 secure :-1 s=READY pgs=578 cs=0 l=1 rev1=1 crypto rx=0x7f3f9c009900 tx=0x7f3f9c02bc30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=995b3dcc8501eb76 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:53:09.861 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.853+0000 7f3f94ff9640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:53:09.861 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.853+0000 7f3f94ff9640 1 -- 172.21.15.99:0/1470123109 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3f9c005450 con 0x5574e74ebaf0 2024-03-19T01:53:09.861 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.853+0000 7f3f94ff9640 1 -- 172.21.15.99:0/1470123109 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f3f9c0055f0 con 0x5574e74ebaf0 2024-03-19T01:53:09.861 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.853+0000 7f3fb6381740 1 -- 172.21.15.99:0/1470123109 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f3fa409c070 con 0x5574e74ebaf0 2024-03-19T01:53:09.861 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.853+0000 7f3f94ff9640 1 -- 172.21.15.99:0/1470123109 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=28) ==== 24+0+0 (secure 0 0 0) 0x7f3f9c0046c0 con 0x5574e74ebaf0 2024-03-19T01:53:09.861 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.853+0000 7f3fb6381740 10 client.16320 fetch_fsmap learned FSMap version 28 2024-03-19T01:53:09.861 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.853+0000 7f3fb6381740 1 -- 172.21.15.99:0/1470123109 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=28,monmap=4+}) -- 0x5574e7164210 con 0x5574e74ebaf0 2024-03-19T01:53:09.861 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.854+0000 7f3f94ff9640 1 -- 172.21.15.99:0/1470123109 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f3fa409c070 con 0x5574e74ebaf0 2024-03-19T01:53:09.861 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.854+0000 7f3f94ff9640 1 -- 172.21.15.99:0/1470123109 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 28) ==== 8338+0+0 (secure 0 0 0) 0x7f3f9c003b80 con 0x5574e74ebaf0 2024-03-19T01:53:09.861 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.854+0000 7f3fb6381740 10 client.16320 fetch_fsmap finished waiting for FSMap version 28 2024-03-19T01:53:09.861 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.854+0000 7f3fb6381740 10 client.16320 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:53:09.862 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.854+0000 7f3fb6381740 20 client.16320 populate_metadata read hostname 'smithi099' 2024-03-19T01:53:09.862 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.854+0000 7f3fb6381740 1 --2- 172.21.15.99:0/1470123109 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5574e7500f90 0x5574e75033c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:09.862 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.854+0000 7f3fb6381740 4 client.16320 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:53:09.862 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.854+0000 7f3fb6381740 1 -- 172.21.15.99:0/1470123109 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5574e745d120 con 0x5574e7500f90 2024-03-19T01:53:09.862 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.854+0000 7f3fb1539640 1 --2- 172.21.15.99:0/1470123109 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5574e7500f90 0x5574e75033c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:09.862 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.854+0000 7f3fb1539640 1 --2- 172.21.15.99:0/1470123109 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5574e7500f90 0x5574e75033c0 crc :-1 s=READY pgs=137 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=7c72bc5faf915ba6 server_cookie=a2c00e3052c95e01 in_seq=0 out_seq=0 2024-03-19T01:53:09.862 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.854+0000 7f3f94ff9640 10 client.16320 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:53:09.862 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.855+0000 7f3f94ff9640 1 -- 172.21.15.99:0/1470123109 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x5574e745d120 con 0x5574e7500f90 2024-03-19T01:53:09.862 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.855+0000 7f3f94ff9640 10 client.16320 handle_command_reply: tid=0 2024-03-19T01:53:09.862 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.855+0000 7f3fb6381740 1 client.16320 shutdown 2024-03-19T01:53:09.862 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.855+0000 7f3fb6381740 1 -- 172.21.15.99:0/1470123109 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5574e74ebaf0 msgr2=0x5574e74f55b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:53:09.862 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.855+0000 7f3fb6381740 1 --2- 172.21.15.99:0/1470123109 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5574e74ebaf0 0x5574e74f55b0 secure :-1 s=READY pgs=578 cs=0 l=1 rev1=1 crypto rx=0x7f3f9c009900 tx=0x7f3f9c02bc30 comp rx=0 tx=0).stop 2024-03-19T01:53:09.862 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.855+0000 7f3fb6381740 1 -- 172.21.15.99:0/1470123109 shutdown_connections 2024-03-19T01:53:09.862 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.855+0000 7f3fb6381740 1 --2- 172.21.15.99:0/1470123109 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5574e7500f90 0x5574e75033c0 crc :-1 s=READY pgs=137 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:09.863 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.855+0000 7f3fb6381740 1 --2- 172.21.15.99:0/1470123109 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5574e74f0c60 0x5574e74fc5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:09.863 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.855+0000 7f3f94ff9640 0 client.16320 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:53:09.863 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.855+0000 7f3fb6381740 1 --2- 172.21.15.99:0/1470123109 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5574e74ebaf0 0x5574e74f55b0 unknown :-1 s=CLOSED pgs=578 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:09.863 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.855+0000 7f3fb6381740 1 --2- 172.21.15.99:0/1470123109 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5574e74eb100 0x5574e74f5070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:09.863 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.855+0000 7f3fb6381740 1 -- 172.21.15.99:0/1470123109 >> 172.21.15.99:0/1470123109 conn(0x5574e74e6670 msgr2=0x5574e74e78c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:53:09.863 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.855+0000 7f3fb6381740 1 -- 172.21.15.99:0/1470123109 shutdown_connections 2024-03-19T01:53:09.863 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.855+0000 7f3fb6381740 1 -- 172.21.15.99:0/1470123109 wait complete. 2024-03-19T01:53:09.863 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.855+0000 7f3fb6381740 20 client.16320 trim_cache size 0 max 16384 2024-03-19T01:53:09.875 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.870+0000 7f3fb6381740 1 Processor -- start 2024-03-19T01:53:09.876 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.870+0000 7f3fb6381740 1 -- start start 2024-03-19T01:53:09.876 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.870+0000 7f3fb6381740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5574e74eb100 0x5574e74e9fe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:09.876 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.870+0000 7f3fb6381740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5574e74ebaf0 0x5574e74fc5b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:09.876 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.870+0000 7f3fb6381740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5574e74f0c60 0x5574e7507cc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:09.876 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.870+0000 7f3fb6381740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f3fac1ab5c0 con 0x5574e74ebaf0 2024-03-19T01:53:09.876 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.870+0000 7f3fb6381740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f3fac1a8e90 con 0x5574e74f0c60 2024-03-19T01:53:09.876 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.871+0000 7f3fb6381740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f3fac1a6ce0 con 0x5574e74eb100 2024-03-19T01:53:09.876 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.871+0000 7f3fb0d38640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5574e74ebaf0 0x5574e74fc5b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:09.876 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.871+0000 7f3fb0d38640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5574e74ebaf0 0x5574e74fc5b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:46878/0 (socket says 172.21.15.99:46878) 2024-03-19T01:53:09.876 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.871+0000 7f3fb0d38640 1 -- 172.21.15.99:0/4050703347 learned_addr learned my addr 172.21.15.99:0/4050703347 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:53:09.876 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.871+0000 7f3fb1539640 1 --2- 172.21.15.99:0/4050703347 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5574e74eb100 0x5574e74e9fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:09.876 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.871+0000 7f3fb1d3a640 1 --2- 172.21.15.99:0/4050703347 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5574e74f0c60 0x5574e7507cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:09.876 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.871+0000 7f3fb0d38640 1 -- 172.21.15.99:0/4050703347 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5574e74eb100 msgr2=0x5574e74e9fe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:53:09.877 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.871+0000 7f3fb0d38640 1 --2- 172.21.15.99:0/4050703347 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5574e74eb100 0x5574e74e9fe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:09.877 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.871+0000 7f3fb0d38640 1 -- 172.21.15.99:0/4050703347 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5574e74f0c60 msgr2=0x5574e7507cc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:53:09.877 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.871+0000 7f3fb0d38640 1 --2- 172.21.15.99:0/4050703347 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5574e74f0c60 0x5574e7507cc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:09.877 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.871+0000 7f3fb0d38640 1 -- 172.21.15.99:0/4050703347 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5574e7230620 con 0x5574e74ebaf0 2024-03-19T01:53:09.877 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.871+0000 7f3fb1d3a640 1 --2- 172.21.15.99:0/4050703347 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5574e74f0c60 0x5574e7507cc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-19T01:53:09.877 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.871+0000 7f3fb0d38640 1 --2- 172.21.15.99:0/4050703347 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5574e74ebaf0 0x5574e74fc5b0 secure :-1 s=READY pgs=579 cs=0 l=1 rev1=1 crypto rx=0x7f3f9c02ce70 tx=0x7f3f9c005b50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=165f966ae19a9488 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:53:09.877 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.871+0000 7f3f96ffd640 1 -- 172.21.15.99:0/4050703347 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3f9c042630 con 0x5574e74ebaf0 2024-03-19T01:53:09.877 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.871+0000 7f3f96ffd640 1 -- 172.21.15.99:0/4050703347 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f3f9c052070 con 0x5574e74ebaf0 2024-03-19T01:53:09.877 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.871+0000 7f3f96ffd640 1 -- 172.21.15.99:0/4050703347 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3f9c004ac0 con 0x5574e74ebaf0 2024-03-19T01:53:09.877 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.872+0000 7f3fb6381740 1 -- 172.21.15.99:0/4050703347 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5574e74ebaf0 msgr2=0x5574e74fc5b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:53:09.877 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.872+0000 7f3fb6381740 1 --2- 172.21.15.99:0/4050703347 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5574e74ebaf0 0x5574e74fc5b0 secure :-1 s=READY pgs=579 cs=0 l=1 rev1=1 crypto rx=0x7f3f9c02ce70 tx=0x7f3f9c005b50 comp rx=0 tx=0).stop 2024-03-19T01:53:09.877 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.872+0000 7f3fb6381740 1 -- 172.21.15.99:0/4050703347 shutdown_connections 2024-03-19T01:53:09.877 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.872+0000 7f3fb6381740 1 --2- 172.21.15.99:0/4050703347 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5574e74f0c60 0x5574e7507cc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:09.877 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.872+0000 7f3fb6381740 1 --2- 172.21.15.99:0/4050703347 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5574e74ebaf0 0x5574e74fc5b0 unknown :-1 s=CLOSED pgs=579 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:09.878 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.872+0000 7f3fb6381740 1 --2- 172.21.15.99:0/4050703347 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5574e74eb100 0x5574e74e9fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:09.878 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.872+0000 7f3fb6381740 1 -- 172.21.15.99:0/4050703347 >> 172.21.15.99:0/4050703347 conn(0x5574e74e6670 msgr2=0x5574e74e98d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:53:09.878 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.872+0000 7f3fb6381740 1 -- 172.21.15.99:0/4050703347 shutdown_connections 2024-03-19T01:53:09.878 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.872+0000 7f3fb6381740 1 -- 172.21.15.99:0/4050703347 wait complete. 2024-03-19T01:53:09.878 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.872+0000 7f3fb6381740 1 -- start start 2024-03-19T01:53:09.878 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.872+0000 7f3fb6381740 1 Processor -- start 2024-03-19T01:53:09.878 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.872+0000 7f3fb6381740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5574e74eb100 0x5574e74fa0a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:09.878 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.872+0000 7f3fb6381740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5574e74ebaf0 0x5574e74fa5e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:09.878 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.872+0000 7f3fb6381740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5574e74f0c60 0x5574e74fb5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:09.878 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.872+0000 7f3fb6381740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f3fac1a7350 con 0x5574e74f0c60 2024-03-19T01:53:09.878 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.872+0000 7f3fb6381740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f3fac1a87c0 con 0x5574e74ebaf0 2024-03-19T01:53:09.878 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.872+0000 7f3fb6381740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f3fac1a7a20 con 0x5574e74eb100 2024-03-19T01:53:09.878 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.873+0000 7f3fb6381740 10 client.0 _ll_register_callbacks cb 0x7f3f9c02caa0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:53:09.879 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.873+0000 7f3fb1d3a640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5574e74f0c60 0x5574e74fb5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:09.879 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.873+0000 7f3fb1539640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5574e74eb100 0x5574e74fa0a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:09.879 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.873+0000 7f3fb0d38640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5574e74ebaf0 0x5574e74fa5e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:09.879 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.873+0000 7f3fb1539640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5574e74eb100 0x5574e74fa0a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:59434/0 (socket says 172.21.15.99:59434) 2024-03-19T01:53:09.879 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.873+0000 7f3fb0d38640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5574e74ebaf0 0x5574e74fa5e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:50056/0 (socket says 172.21.15.99:50056) 2024-03-19T01:53:09.879 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.873+0000 7f3fb1d3a640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5574e74f0c60 0x5574e74fb5f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:46880/0 (socket says 172.21.15.99:46880) 2024-03-19T01:53:09.879 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.873+0000 7f3fb1539640 1 -- 172.21.15.99:0/1200809755 learned_addr learned my addr 172.21.15.99:0/1200809755 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:53:09.879 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.873+0000 7f3fb0d38640 1 -- 172.21.15.99:0/1200809755 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5574e74eb100 msgr2=0x5574e74fa0a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:53:09.879 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.873+0000 7f3fb0d38640 1 --2- 172.21.15.99:0/1200809755 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5574e74eb100 0x5574e74fa0a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:09.879 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.873+0000 7f3fb0d38640 1 -- 172.21.15.99:0/1200809755 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5574e74f0c60 msgr2=0x5574e74fb5f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:53:09.879 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.873+0000 7f3fb0d38640 1 --2- 172.21.15.99:0/1200809755 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5574e74f0c60 0x5574e74fb5f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:09.879 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.873+0000 7f3fb0d38640 1 -- 172.21.15.99:0/1200809755 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5574e74fbbe0 con 0x5574e74ebaf0 2024-03-19T01:53:09.879 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.873+0000 7f3fb1d3a640 1 --2- 172.21.15.99:0/1200809755 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5574e74f0c60 0x5574e74fb5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:53:09.880 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.873+0000 7f3fb0d38640 1 --2- 172.21.15.99:0/1200809755 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5574e74ebaf0 0x5574e74fa5e0 secure :-1 s=READY pgs=157 cs=0 l=1 rev1=1 crypto rx=0x7f3f9c000c00 tx=0x7f3f9c0058d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=6e302a98ffba547f server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:53:09.880 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.873+0000 7f3f94ff9640 10 client.0 ms_handle_connect on v2:172.21.15.112:3300/0 2024-03-19T01:53:09.880 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.873+0000 7f3f94ff9640 1 -- 172.21.15.99:0/1200809755 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3f9c042d70 con 0x5574e74ebaf0 2024-03-19T01:53:09.880 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.873+0000 7f3f94ff9640 1 -- 172.21.15.99:0/1200809755 <== mon.2 v2:172.21.15.112:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f3f9c004860 con 0x5574e74ebaf0 2024-03-19T01:53:09.880 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.873+0000 7f3fb6381740 1 -- 172.21.15.99:0/1200809755 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f3f9c052070 con 0x5574e74ebaf0 2024-03-19T01:53:09.880 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.873+0000 7f3f94ff9640 1 -- 172.21.15.99:0/1200809755 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_get_version_reply(handle=1 version=28) ==== 24+0+0 (secure 0 0 0) 0x7f3f9c004a60 con 0x5574e74ebaf0 2024-03-19T01:53:09.880 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.873+0000 7f3fb6381740 10 client.26096 fetch_fsmap learned FSMap version 28 2024-03-19T01:53:09.880 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.873+0000 7f3fb6381740 1 -- 172.21.15.99:0/1200809755 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,fsmap=28,monmap=4+}) -- 0x5574e7567700 con 0x5574e74ebaf0 2024-03-19T01:53:09.880 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.874+0000 7f3f94ff9640 1 -- 172.21.15.99:0/1200809755 <== mon.2 v2:172.21.15.112:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f3f9c052070 con 0x5574e74ebaf0 2024-03-19T01:53:09.880 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.874+0000 7f3f94ff9640 1 -- 172.21.15.99:0/1200809755 <== mon.2 v2:172.21.15.112:3300/0 5 ==== fsmap(e 28) ==== 8338+0+0 (secure 0 0 0) 0x7f3f9c06a440 con 0x5574e74ebaf0 2024-03-19T01:53:09.880 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.874+0000 7f3fb6381740 10 client.26096 fetch_fsmap finished waiting for FSMap version 28 2024-03-19T01:53:09.880 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.874+0000 7f3fb6381740 10 client.26096 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:53:09.880 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.874+0000 7f3fb6381740 20 client.26096 populate_metadata read hostname 'smithi099' 2024-03-19T01:53:09.880 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.874+0000 7f3fb6381740 1 --2- 172.21.15.99:0/1200809755 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5574e7500f90 0x5574e7569fa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:09.881 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.874+0000 7f3fb6381740 4 client.26096 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:53:09.881 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.874+0000 7f3fb6381740 1 -- 172.21.15.99:0/1200809755 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x5574e7457470 con 0x5574e7500f90 2024-03-19T01:53:09.881 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.874+0000 7f3fb1539640 1 --2- 172.21.15.99:0/1200809755 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5574e7500f90 0x5574e7569fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:09.881 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.874+0000 7f3fb1539640 1 --2- 172.21.15.99:0/1200809755 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5574e7500f90 0x5574e7569fa0 crc :-1 s=READY pgs=138 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=dfa669984e27057b server_cookie=e07c8f0f2139bfbd in_seq=0 out_seq=0 2024-03-19T01:53:09.881 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.874+0000 7f3f94ff9640 10 client.26096 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:53:09.940 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.940+0000 7f3f94ff9640 1 -- 172.21.15.99:0/1200809755 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x5574e7457470 con 0x5574e7500f90 2024-03-19T01:53:09.941 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.940+0000 7f3f94ff9640 10 client.26096 handle_command_reply: tid=0 2024-03-19T01:53:09.941 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.940+0000 7f3fb6381740 1 client.26096 shutdown 2024-03-19T01:53:09.941 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.940+0000 7f3fb6381740 1 -- 172.21.15.99:0/1200809755 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5574e74ebaf0 msgr2=0x5574e74fa5e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:53:09.941 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.940+0000 7f3fb6381740 1 --2- 172.21.15.99:0/1200809755 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5574e74ebaf0 0x5574e74fa5e0 secure :-1 s=READY pgs=157 cs=0 l=1 rev1=1 crypto rx=0x7f3f9c000c00 tx=0x7f3f9c0058d0 comp rx=0 tx=0).stop 2024-03-19T01:53:09.941 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.940+0000 7f3fb6381740 1 -- 172.21.15.99:0/1200809755 shutdown_connections 2024-03-19T01:53:09.941 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.940+0000 7f3fb6381740 1 --2- 172.21.15.99:0/1200809755 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5574e7500f90 0x5574e7569fa0 crc :-1 s=READY pgs=138 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:09.941 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.940+0000 7f3fb6381740 1 --2- 172.21.15.99:0/1200809755 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5574e74ebaf0 0x5574e74fa5e0 unknown :-1 s=CLOSED pgs=157 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:09.941 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.940+0000 7f3f94ff9640 0 client.26096 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:53:09.941 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.940+0000 7f3fb6381740 1 --2- 172.21.15.99:0/1200809755 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5574e74f0c60 0x5574e74fb5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:09.941 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.940+0000 7f3fb6381740 1 --2- 172.21.15.99:0/1200809755 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5574e74eb100 0x5574e74fa0a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:09.941 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.940+0000 7f3fb6381740 1 -- 172.21.15.99:0/1200809755 >> 172.21.15.99:0/1200809755 conn(0x5574e74e6670 msgr2=0x5574e74ea160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:53:09.941 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.940+0000 7f3fb6381740 1 -- 172.21.15.99:0/1200809755 shutdown_connections 2024-03-19T01:53:09.941 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.940+0000 7f3fb6381740 1 -- 172.21.15.99:0/1200809755 wait complete. 2024-03-19T01:53:09.942 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.940+0000 7f3fb6381740 20 client.26096 trim_cache size 0 max 16384 2024-03-19T01:53:09.942 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:53:09.942 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:53:09.942 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:53:09.942 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:53:09.943 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.941+0000 7f3fb379d640 1 -- 172.21.15.99:0/143680265 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f3f980a28a0 msgr2=0x7f3f980a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:53:09.943 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.941+0000 7f3fb379d640 1 --2- 172.21.15.99:0/143680265 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f3f980a28a0 0x7f3f980a4d60 secure :-1 s=READY pgs=319 cs=0 l=1 rev1=1 crypto rx=0x7f3f9c02c7b0 tx=0x7f3f9c005d50 comp rx=0 tx=0).stop 2024-03-19T01:53:09.943 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.941+0000 7f3fb379d640 1 -- 172.21.15.99:0/143680265 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3fac102310 msgr2=0x7f3fac19f020 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:53:09.943 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.941+0000 7f3fb379d640 1 --2- 172.21.15.99:0/143680265 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3fac102310 0x7f3fac19f020 secure :-1 s=READY pgs=576 cs=0 l=1 rev1=1 crypto rx=0x7f3fa400eb10 tx=0x7f3fa400c2d0 comp rx=0 tx=0).stop 2024-03-19T01:53:09.943 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.941+0000 7f3fb379d640 1 -- 172.21.15.99:0/143680265 shutdown_connections 2024-03-19T01:53:09.944 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.941+0000 7f3fb379d640 1 --2- 172.21.15.99:0/143680265 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3fac102310 0x7f3fac19f020 unknown :-1 s=CLOSED pgs=576 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:09.944 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.941+0000 7f3fb379d640 1 --2- 172.21.15.99:0/143680265 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f3fac101950 0x7f3fac197fb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:09.944 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.941+0000 7f3fb379d640 1 --2- 172.21.15.99:0/143680265 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f3f980a28a0 0x7f3f980a4d60 unknown :-1 s=CLOSED pgs=319 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:09.944 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.941+0000 7f3fb379d640 1 --2- 172.21.15.99:0/143680265 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3fac100760 0x7f3fac197a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:09.944 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.941+0000 7f3fb379d640 1 -- 172.21.15.99:0/143680265 >> 172.21.15.99:0/143680265 conn(0x7f3fac0fbcd0 msgr2=0x7f3fac104bf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:53:09.944 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.941+0000 7f3fb379d640 1 -- 172.21.15.99:0/143680265 shutdown_connections 2024-03-19T01:53:09.944 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:09.941+0000 7f3fb379d640 1 -- 172.21.15.99:0/143680265 wait complete. 2024-03-19T01:53:10.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:53:09 smithi115 ceph-mon[67332]: pgmap v713: 97 pgs: 97 active+clean; 256 MiB data, 3.6 GiB used, 1.0 TiB / 1.0 TiB avail; 753 KiB/s rd, 4.1 MiB/s wr, 255 op/s 2024-03-19T01:53:10.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:53:09 smithi099 ceph-mon[62951]: pgmap v713: 97 pgs: 97 active+clean; 256 MiB data, 3.6 GiB used, 1.0 TiB / 1.0 TiB avail; 753 KiB/s rd, 4.1 MiB/s wr, 255 op/s 2024-03-19T01:53:10.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:53:09 smithi112 ceph-mon[66819]: pgmap v713: 97 pgs: 97 active+clean; 256 MiB data, 3.6 GiB used, 1.0 TiB / 1.0 TiB avail; 753 KiB/s rd, 4.1 MiB/s wr, 255 op/s 2024-03-19T01:53:10.704 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1465143 11.25 MB/sec execute 580 sec latency 26.484 ms 2024-03-19T01:53:11.704 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1467571 11.25 MB/sec execute 581 sec latency 27.268 ms 2024-03-19T01:53:11.806 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:53:11 smithi115 ceph-mon[67332]: pgmap v714: 97 pgs: 97 active+clean; 256 MiB data, 3.6 GiB used, 1.0 TiB / 1.0 TiB avail; 703 KiB/s rd, 3.7 MiB/s wr, 221 op/s 2024-03-19T01:53:11.840 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:53:11 smithi099 ceph-mon[62951]: pgmap v714: 97 pgs: 97 active+clean; 256 MiB data, 3.6 GiB used, 1.0 TiB / 1.0 TiB avail; 703 KiB/s rd, 3.7 MiB/s wr, 221 op/s 2024-03-19T01:53:11.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:53:11 smithi112 ceph-mon[66819]: pgmap v714: 97 pgs: 97 active+clean; 256 MiB data, 3.6 GiB used, 1.0 TiB / 1.0 TiB avail; 703 KiB/s rd, 3.7 MiB/s wr, 221 op/s 2024-03-19T01:53:12.704 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1469077 11.24 MB/sec execute 582 sec latency 56.122 ms 2024-03-19T01:53:13.705 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1471281 11.25 MB/sec execute 583 sec latency 24.569 ms 2024-03-19T01:53:14.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:53:13 smithi115 ceph-mon[67332]: pgmap v715: 97 pgs: 97 active+clean; 258 MiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 820 KiB/s rd, 3.5 MiB/s wr, 259 op/s 2024-03-19T01:53:14.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:53:13 smithi099 ceph-mon[62951]: pgmap v715: 97 pgs: 97 active+clean; 258 MiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 820 KiB/s rd, 3.5 MiB/s wr, 259 op/s 2024-03-19T01:53:14.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:53:13 smithi112 ceph-mon[66819]: pgmap v715: 97 pgs: 97 active+clean; 258 MiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 820 KiB/s rd, 3.5 MiB/s wr, 259 op/s 2024-03-19T01:53:14.705 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1473444 11.24 MB/sec execute 584 sec latency 34.657 ms 2024-03-19T01:53:15.705 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1475317 11.24 MB/sec execute 585 sec latency 36.924 ms 2024-03-19T01:53:16.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:53:15 smithi115 ceph-mon[67332]: pgmap v716: 97 pgs: 97 active+clean; 261 MiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 812 KiB/s rd, 2.6 MiB/s wr, 239 op/s 2024-03-19T01:53:16.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:53:15 smithi099 ceph-mon[62951]: pgmap v716: 97 pgs: 97 active+clean; 261 MiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 812 KiB/s rd, 2.6 MiB/s wr, 239 op/s 2024-03-19T01:53:16.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:53:15 smithi112 ceph-mon[66819]: pgmap v716: 97 pgs: 97 active+clean; 261 MiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 812 KiB/s rd, 2.6 MiB/s wr, 239 op/s 2024-03-19T01:53:16.705 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1478008 11.25 MB/sec execute 586 sec latency 31.257 ms 2024-03-19T01:53:17.705 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1479377 11.25 MB/sec execute 587 sec latency 49.962 ms 2024-03-19T01:53:18.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:53:18 smithi115 ceph-mon[67332]: pgmap v717: 97 pgs: 97 active+clean; 262 MiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 784 KiB/s rd, 2.9 MiB/s wr, 261 op/s 2024-03-19T01:53:18.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:53:18 smithi099 ceph-mon[62951]: pgmap v717: 97 pgs: 97 active+clean; 262 MiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 784 KiB/s rd, 2.9 MiB/s wr, 261 op/s 2024-03-19T01:53:18.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:53:18 smithi112 ceph-mon[66819]: pgmap v717: 97 pgs: 97 active+clean; 262 MiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 784 KiB/s rd, 2.9 MiB/s wr, 261 op/s 2024-03-19T01:53:18.705 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1481755 11.25 MB/sec execute 588 sec latency 27.970 ms 2024-03-19T01:53:19.705 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1483842 11.24 MB/sec execute 589 sec latency 37.949 ms 2024-03-19T01:53:20.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:53:20 smithi115 ceph-mon[67332]: pgmap v718: 97 pgs: 97 active+clean; 263 MiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 651 KiB/s rd, 2.5 MiB/s wr, 217 op/s 2024-03-19T01:53:20.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:53:20 smithi099 ceph-mon[62951]: pgmap v718: 97 pgs: 97 active+clean; 263 MiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 651 KiB/s rd, 2.5 MiB/s wr, 217 op/s 2024-03-19T01:53:20.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:53:20 smithi112 ceph-mon[66819]: pgmap v718: 97 pgs: 97 active+clean; 263 MiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 651 KiB/s rd, 2.5 MiB/s wr, 217 op/s 2024-03-19T01:53:20.705 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1485625 11.25 MB/sec execute 590 sec latency 23.835 ms 2024-03-19T01:53:21.705 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1487815 11.24 MB/sec execute 591 sec latency 36.948 ms 2024-03-19T01:53:22.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:53:22 smithi115 ceph-mon[67332]: pgmap v719: 97 pgs: 97 active+clean; 265 MiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 890 KiB/s rd, 2.9 MiB/s wr, 256 op/s 2024-03-19T01:53:22.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:53:22 smithi099 ceph-mon[62951]: pgmap v719: 97 pgs: 97 active+clean; 265 MiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 890 KiB/s rd, 2.9 MiB/s wr, 256 op/s 2024-03-19T01:53:22.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:53:22 smithi112 ceph-mon[66819]: pgmap v719: 97 pgs: 97 active+clean; 265 MiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 890 KiB/s rd, 2.9 MiB/s wr, 256 op/s 2024-03-19T01:53:22.705 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1489402 11.24 MB/sec execute 592 sec latency 65.050 ms 2024-03-19T01:53:23.037 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.034+0000 7f4f9e977640 1 -- 172.21.15.99:0/1257255417 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4f98101930 msgr2=0x7f4f98101db0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:53:23.038 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.034+0000 7f4f9e977640 1 --2- 172.21.15.99:0/1257255417 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4f98101930 0x7f4f98101db0 secure :-1 s=READY pgs=580 cs=0 l=1 rev1=1 crypto rx=0x7f4f8c009900 tx=0x7f4f8c02bc30 comp rx=0 tx=0).stop 2024-03-19T01:53:23.038 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.034+0000 7f4f9e977640 1 -- 172.21.15.99:0/1257255417 shutdown_connections 2024-03-19T01:53:23.038 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.034+0000 7f4f9e977640 1 --2- 172.21.15.99:0/1257255417 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4f981022f0 0x7f4f98108b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:23.038 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.034+0000 7f4f9e977640 1 --2- 172.21.15.99:0/1257255417 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4f98101930 0x7f4f98101db0 unknown :-1 s=CLOSED pgs=580 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:23.038 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.034+0000 7f4f9e977640 1 --2- 172.21.15.99:0/1257255417 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4f98100740 0x7f4f98100b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:23.039 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.034+0000 7f4f9e977640 1 -- 172.21.15.99:0/1257255417 >> 172.21.15.99:0/1257255417 conn(0x7f4f980fbcb0 msgr2=0x7f4f980fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:53:23.039 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.034+0000 7f4f9e977640 1 -- 172.21.15.99:0/1257255417 shutdown_connections 2024-03-19T01:53:23.039 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.034+0000 7f4f9e977640 1 -- 172.21.15.99:0/1257255417 wait complete. 2024-03-19T01:53:23.039 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.034+0000 7f4f9e977640 1 Processor -- start 2024-03-19T01:53:23.039 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.035+0000 7f4f9e977640 1 -- start start 2024-03-19T01:53:23.039 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.035+0000 7f4f9e977640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4f98100740 0x7f4f98197ac0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:23.039 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.035+0000 7f4f9e977640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4f98101930 0x7f4f98198000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:23.040 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.035+0000 7f4f9e977640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4f981022f0 0x7f4f9819f070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:23.040 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.035+0000 7f4f9e977640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f4f9810c3c0 con 0x7f4f981022f0 2024-03-19T01:53:23.040 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.035+0000 7f4f9e977640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f4f9810c240 con 0x7f4f98101930 2024-03-19T01:53:23.040 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.035+0000 7f4f9e977640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f4f9810c540 con 0x7f4f98100740 2024-03-19T01:53:23.040 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.035+0000 7f4f9cf14640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4f981022f0 0x7f4f9819f070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:23.040 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.035+0000 7f4f9cf14640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4f981022f0 0x7f4f9819f070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:45186/0 (socket says 172.21.15.99:45186) 2024-03-19T01:53:23.040 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.035+0000 7f4f9cf14640 1 -- 172.21.15.99:0/4053767597 learned_addr learned my addr 172.21.15.99:0/4053767597 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:53:23.041 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.035+0000 7f4f97fff640 1 --2- 172.21.15.99:0/4053767597 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4f98100740 0x7f4f98197ac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:23.041 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.035+0000 7f4f977fe640 1 --2- 172.21.15.99:0/4053767597 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4f98101930 0x7f4f98198000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:23.041 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.035+0000 7f4f9cf14640 1 -- 172.21.15.99:0/4053767597 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4f98100740 msgr2=0x7f4f98197ac0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:53:23.041 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.035+0000 7f4f9cf14640 1 --2- 172.21.15.99:0/4053767597 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4f98100740 0x7f4f98197ac0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:23.041 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.035+0000 7f4f9cf14640 1 -- 172.21.15.99:0/4053767597 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4f98101930 msgr2=0x7f4f98198000 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:53:23.041 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.035+0000 7f4f9cf14640 1 --2- 172.21.15.99:0/4053767597 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4f98101930 0x7f4f98198000 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:23.041 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.035+0000 7f4f9cf14640 1 -- 172.21.15.99:0/4053767597 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f4f9819f6e0 con 0x7f4f981022f0 2024-03-19T01:53:23.041 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.036+0000 7f4f9cf14640 1 --2- 172.21.15.99:0/4053767597 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4f981022f0 0x7f4f9819f070 secure :-1 s=READY pgs=581 cs=0 l=1 rev1=1 crypto rx=0x7f4f9000eb10 tx=0x7f4f9000c2d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:53:23.042 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.036+0000 7f4f957fa640 1 -- 172.21.15.99:0/4053767597 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4f90004b20 con 0x7f4f981022f0 2024-03-19T01:53:23.042 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.036+0000 7f4f957fa640 1 -- 172.21.15.99:0/4053767597 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f4f90004cc0 con 0x7f4f981022f0 2024-03-19T01:53:23.042 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.036+0000 7f4f957fa640 1 -- 172.21.15.99:0/4053767597 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4f90010760 con 0x7f4f981022f0 2024-03-19T01:53:23.042 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.036+0000 7f4f9e977640 1 -- 172.21.15.99:0/4053767597 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f4f9819f9d0 con 0x7f4f981022f0 2024-03-19T01:53:23.042 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.036+0000 7f4f9e977640 1 -- 172.21.15.99:0/4053767597 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f4f9819fef0 con 0x7f4f981022f0 2024-03-19T01:53:23.051 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.038+0000 7f4f957fa640 1 -- 172.21.15.99:0/4053767597 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f4f90005020 con 0x7f4f981022f0 2024-03-19T01:53:23.051 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.039+0000 7f4fa1566740 1 Processor -- start 2024-03-19T01:53:23.051 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.039+0000 7f4fa1566740 1 -- start start 2024-03-19T01:53:23.051 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.039+0000 7f4fa1566740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564dcf793100 0x564dcf793520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:23.051 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.039+0000 7f4fa1566740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564dcf793af0 0x564dcf7985f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:23.051 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.039+0000 7f4fa1566740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564dcf798c60 0x564dcf79b0f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:23.051 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.039+0000 7f4fa1566740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x564dcf6f7be0 con 0x564dcf793af0 2024-03-19T01:53:23.051 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.039+0000 7f4fa1566740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x564dcf6a0960 con 0x564dcf793100 2024-03-19T01:53:23.051 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.039+0000 7f4fa1566740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x564dcf6f3c80 con 0x564dcf798c60 2024-03-19T01:53:23.051 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.039+0000 7f4f97fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564dcf793af0 0x564dcf7985f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:23.051 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.039+0000 7f4f957fa640 1 --2- 172.21.15.99:0/4053767597 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f4f800a28a0 0x7f4f800a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:23.051 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.039+0000 7f4f97fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564dcf793af0 0x564dcf7985f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:45198/0 (socket says 172.21.15.99:45198) 2024-03-19T01:53:23.052 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.039+0000 7f4f97fff640 1 -- 172.21.15.99:0/827416909 learned_addr learned my addr 172.21.15.99:0/827416909 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:53:23.052 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.039+0000 7f4f977fe640 1 --2- 172.21.15.99:0/827416909 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564dcf793100 0x564dcf793520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:23.052 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.039+0000 7f4f977fe640 1 --2- 172.21.15.99:0/827416909 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564dcf798c60 0x564dcf79b0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:23.052 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.039+0000 7f4f9cf14640 1 --2- 172.21.15.99:0/4053767597 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f4f800a28a0 0x7f4f800a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:23.052 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.039+0000 7f4f97fff640 1 -- 172.21.15.99:0/827416909 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564dcf798c60 msgr2=0x564dcf79b0f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:53:23.052 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.039+0000 7f4f97fff640 1 --2- 172.21.15.99:0/827416909 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564dcf798c60 0x564dcf79b0f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:23.052 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.039+0000 7f4f97fff640 1 -- 172.21.15.99:0/827416909 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564dcf793100 msgr2=0x564dcf793520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:53:23.052 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.039+0000 7f4f97fff640 1 --2- 172.21.15.99:0/827416909 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564dcf793100 0x564dcf793520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:23.052 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.039+0000 7f4f97fff640 1 -- 172.21.15.99:0/827416909 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x564dcf697d20 con 0x564dcf793af0 2024-03-19T01:53:23.052 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.039+0000 7f4f957fa640 1 -- 172.21.15.99:0/4053767597 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f4f900cf180 con 0x7f4f981022f0 2024-03-19T01:53:23.052 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.040+0000 7f4f97fff640 1 --2- 172.21.15.99:0/827416909 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564dcf793af0 0x564dcf7985f0 secure :-1 s=READY pgs=582 cs=0 l=1 rev1=1 crypto rx=0x7f4f88009ca0 tx=0x7f4f88011810 comp rx=0 tx=0).ready entity=mon.0 client_cookie=63e60418149725a7 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:53:23.052 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.040+0000 7f4f7e7fc640 1 -- 172.21.15.99:0/827416909 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4f88009e50 con 0x564dcf793af0 2024-03-19T01:53:23.053 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.040+0000 7f4f9cf14640 1 --2- 172.21.15.99:0/4053767597 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f4f800a28a0 0x7f4f800a4d60 secure :-1 s=READY pgs=322 cs=0 l=1 rev1=1 crypto rx=0x7f4f9009b7e0 tx=0x7f4f9000efd0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:53:23.053 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.040+0000 7f4f7e7fc640 1 -- 172.21.15.99:0/827416909 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f4f880035f0 con 0x564dcf793af0 2024-03-19T01:53:23.053 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.040+0000 7f4f7e7fc640 1 -- 172.21.15.99:0/827416909 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4f88019670 con 0x564dcf793af0 2024-03-19T01:53:23.053 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.040+0000 7f4fa1566740 1 -- 172.21.15.99:0/827416909 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564dcf793af0 msgr2=0x564dcf7985f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:53:23.053 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.040+0000 7f4fa1566740 1 --2- 172.21.15.99:0/827416909 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564dcf793af0 0x564dcf7985f0 secure :-1 s=READY pgs=582 cs=0 l=1 rev1=1 crypto rx=0x7f4f88009ca0 tx=0x7f4f88011810 comp rx=0 tx=0).stop 2024-03-19T01:53:23.053 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.040+0000 7f4fa1566740 1 -- 172.21.15.99:0/827416909 shutdown_connections 2024-03-19T01:53:23.053 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.041+0000 7f4fa1566740 1 --2- 172.21.15.99:0/827416909 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564dcf798c60 0x564dcf79b0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:23.053 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.041+0000 7f4fa1566740 1 --2- 172.21.15.99:0/827416909 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564dcf793af0 0x564dcf7985f0 unknown :-1 s=CLOSED pgs=582 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:23.053 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.041+0000 7f4fa1566740 1 --2- 172.21.15.99:0/827416909 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564dcf793100 0x564dcf793520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:23.053 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.041+0000 7f4fa1566740 1 -- 172.21.15.99:0/827416909 >> 172.21.15.99:0/827416909 conn(0x564dcf78e670 msgr2=0x564dcf78ea70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:53:23.053 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.041+0000 7f4fa1566740 1 -- 172.21.15.99:0/827416909 shutdown_connections 2024-03-19T01:53:23.053 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.041+0000 7f4fa1566740 1 -- 172.21.15.99:0/827416909 wait complete. 2024-03-19T01:53:23.053 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.041+0000 7f4fa1566740 1 -- start start 2024-03-19T01:53:23.054 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.041+0000 7f4fa1566740 1 Processor -- start 2024-03-19T01:53:23.054 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.041+0000 7f4fa1566740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564dcf793100 0x564dcf79d070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:23.054 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.041+0000 7f4fa1566740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564dcf793af0 0x564dcf79d5b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:23.054 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.041+0000 7f4fa1566740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564dcf798c60 0x564dcf7a45f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:23.054 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.041+0000 7f4fa1566740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x564dcf79bc10 con 0x564dcf798c60 2024-03-19T01:53:23.054 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.041+0000 7f4fa1566740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x564dcf6b3db0 con 0x564dcf793af0 2024-03-19T01:53:23.054 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.041+0000 7f4fa1566740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x564dcf79bd90 con 0x564dcf793100 2024-03-19T01:53:23.054 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.041+0000 7f4fa1566740 10 client.0 _ll_register_callbacks cb 0x564dcf6b8e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:53:23.054 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.042+0000 7f4f977fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564dcf798c60 0x564dcf7a45f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:23.054 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.042+0000 7f4f97fff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564dcf793af0 0x564dcf79d5b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:23.054 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.042+0000 7f4f977fe640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564dcf793100 0x564dcf79d070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:23.054 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.042+0000 7f4f97fff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564dcf793af0 0x564dcf79d5b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:57366/0 (socket says 172.21.15.99:57366) 2024-03-19T01:53:23.055 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.042+0000 7f4f977fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564dcf798c60 0x564dcf7a45f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:45208/0 (socket says 172.21.15.99:45208) 2024-03-19T01:53:23.055 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.042+0000 7f4f97fff640 1 -- 172.21.15.99:0/1005825171 learned_addr learned my addr 172.21.15.99:0/1005825171 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:53:23.055 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.042+0000 7f4f97fff640 1 -- 172.21.15.99:0/1005825171 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564dcf793100 msgr2=0x564dcf79d070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:53:23.055 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.042+0000 7f4f97fff640 1 --2- 172.21.15.99:0/1005825171 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564dcf793100 0x564dcf79d070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:23.055 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.042+0000 7f4f97fff640 1 -- 172.21.15.99:0/1005825171 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564dcf798c60 msgr2=0x564dcf7a45f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:53:23.055 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.042+0000 7f4f97fff640 1 --2- 172.21.15.99:0/1005825171 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564dcf798c60 0x564dcf7a45f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:23.055 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.042+0000 7f4f97fff640 1 -- 172.21.15.99:0/1005825171 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x564dcf6b2c60 con 0x564dcf793af0 2024-03-19T01:53:23.055 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.042+0000 7f4f977fe640 1 --2- 172.21.15.99:0/1005825171 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564dcf798c60 0x564dcf7a45f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:53:23.055 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.042+0000 7f4f97fff640 1 --2- 172.21.15.99:0/1005825171 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564dcf793af0 0x564dcf79d5b0 secure :-1 s=READY pgs=158 cs=0 l=1 rev1=1 crypto rx=0x7f4f88003df0 tx=0x7f4f880039d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=ffb0701a3261cf92 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:53:23.055 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.042+0000 7f4f777fe640 10 client.0 ms_handle_connect on v2:172.21.15.112:3300/0 2024-03-19T01:53:23.055 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.042+0000 7f4f777fe640 1 -- 172.21.15.99:0/1005825171 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4f88028020 con 0x564dcf793af0 2024-03-19T01:53:23.055 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.042+0000 7f4fa1566740 1 -- 172.21.15.99:0/1005825171 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f4f880035f0 con 0x564dcf793af0 2024-03-19T01:53:23.056 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.042+0000 7f4f777fe640 1 -- 172.21.15.99:0/1005825171 <== mon.2 v2:172.21.15.112:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f4f88016040 con 0x564dcf793af0 2024-03-19T01:53:23.056 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.042+0000 7f4f777fe640 1 -- 172.21.15.99:0/1005825171 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_get_version_reply(handle=1 version=28) ==== 24+0+0 (secure 0 0 0) 0x7f4f88033050 con 0x564dcf793af0 2024-03-19T01:53:23.056 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.042+0000 7f4fa1566740 10 client.26108 fetch_fsmap learned FSMap version 28 2024-03-19T01:53:23.056 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.042+0000 7f4fa1566740 1 -- 172.21.15.99:0/1005825171 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,fsmap=28,monmap=4+}) -- 0x564dcf40c210 con 0x564dcf793af0 2024-03-19T01:53:23.056 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.042+0000 7f4f777fe640 1 -- 172.21.15.99:0/1005825171 <== mon.2 v2:172.21.15.112:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f4f880035f0 con 0x564dcf793af0 2024-03-19T01:53:23.056 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.043+0000 7f4f777fe640 1 -- 172.21.15.99:0/1005825171 <== mon.2 v2:172.21.15.112:3300/0 5 ==== fsmap(e 28) ==== 8338+0+0 (secure 0 0 0) 0x7f4f88004990 con 0x564dcf793af0 2024-03-19T01:53:23.056 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.043+0000 7f4fa1566740 10 client.26108 fetch_fsmap finished waiting for FSMap version 28 2024-03-19T01:53:23.056 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.043+0000 7f4fa1566740 10 client.26108 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:53:23.056 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.043+0000 7f4fa1566740 20 client.26108 populate_metadata read hostname 'smithi099' 2024-03-19T01:53:23.056 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.043+0000 7f4fa1566740 1 --2- 172.21.15.99:0/1005825171 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x564dcf7a8d70 0x564dcf7ab160 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:23.057 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.043+0000 7f4fa1566740 4 client.26108 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:53:23.057 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.043+0000 7f4fa1566740 1 -- 172.21.15.99:0/1005825171 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x564dcf705120 con 0x564dcf7a8d70 2024-03-19T01:53:23.057 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.043+0000 7f4f977fe640 1 --2- 172.21.15.99:0/1005825171 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x564dcf7a8d70 0x564dcf7ab160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:23.057 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.043+0000 7f4f977fe640 1 --2- 172.21.15.99:0/1005825171 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x564dcf7a8d70 0x564dcf7ab160 crc :-1 s=READY pgs=139 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=821ab05543f4c891 server_cookie=5afe1ae6d5395fb5 in_seq=0 out_seq=0 2024-03-19T01:53:23.057 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.043+0000 7f4f777fe640 10 client.26108 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:53:23.057 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.044+0000 7f4f777fe640 1 -- 172.21.15.99:0/1005825171 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x564dcf705120 con 0x564dcf7a8d70 2024-03-19T01:53:23.057 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.044+0000 7f4f777fe640 10 client.26108 handle_command_reply: tid=0 2024-03-19T01:53:23.057 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.044+0000 7f4fa1566740 1 client.26108 shutdown 2024-03-19T01:53:23.057 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.044+0000 7f4fa1566740 1 -- 172.21.15.99:0/1005825171 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564dcf793af0 msgr2=0x564dcf79d5b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:53:23.057 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.044+0000 7f4fa1566740 1 --2- 172.21.15.99:0/1005825171 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564dcf793af0 0x564dcf79d5b0 secure :-1 s=READY pgs=158 cs=0 l=1 rev1=1 crypto rx=0x7f4f88003df0 tx=0x7f4f880039d0 comp rx=0 tx=0).stop 2024-03-19T01:53:23.057 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.044+0000 7f4fa1566740 1 -- 172.21.15.99:0/1005825171 shutdown_connections 2024-03-19T01:53:23.057 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.044+0000 7f4fa1566740 1 --2- 172.21.15.99:0/1005825171 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x564dcf7a8d70 0x564dcf7ab160 crc :-1 s=READY pgs=139 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:23.057 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.044+0000 7f4fa1566740 1 --2- 172.21.15.99:0/1005825171 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564dcf793af0 0x564dcf79d5b0 unknown :-1 s=CLOSED pgs=158 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:23.057 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.044+0000 7f4f777fe640 0 client.26108 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:53:23.058 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.044+0000 7f4fa1566740 1 --2- 172.21.15.99:0/1005825171 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564dcf798c60 0x564dcf7a45f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:23.058 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.044+0000 7f4fa1566740 1 --2- 172.21.15.99:0/1005825171 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564dcf793100 0x564dcf79d070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:23.058 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.044+0000 7f4fa1566740 1 -- 172.21.15.99:0/1005825171 >> 172.21.15.99:0/1005825171 conn(0x564dcf78e670 msgr2=0x564dcf78f8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:53:23.058 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.044+0000 7f4fa1566740 1 -- 172.21.15.99:0/1005825171 shutdown_connections 2024-03-19T01:53:23.058 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.044+0000 7f4fa1566740 1 -- 172.21.15.99:0/1005825171 wait complete. 2024-03-19T01:53:23.058 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.044+0000 7f4fa1566740 20 client.26108 trim_cache size 0 max 16384 2024-03-19T01:53:23.061 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.060+0000 7f4fa1566740 1 Processor -- start 2024-03-19T01:53:23.061 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.060+0000 7f4fa1566740 1 -- start start 2024-03-19T01:53:23.061 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.060+0000 7f4fa1566740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564dcf793100 0x564dcf78d220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:23.061 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.060+0000 7f4fa1566740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564dcf793af0 0x564dcf7a45b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:23.061 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.060+0000 7f4fa1566740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564dcf798c60 0x564dcf78d760 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:23.061 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.060+0000 7f4fa1566740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f4f981ab770 con 0x564dcf793af0 2024-03-19T01:53:23.061 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.060+0000 7f4fa1566740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f4f981a9040 con 0x564dcf793100 2024-03-19T01:53:23.062 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.060+0000 7f4fa1566740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f4f981a6e90 con 0x564dcf798c60 2024-03-19T01:53:23.062 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.060+0000 7f4f97fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564dcf793af0 0x564dcf7a45b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:23.062 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.060+0000 7f4f97fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564dcf793af0 0x564dcf7a45b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:45220/0 (socket says 172.21.15.99:45220) 2024-03-19T01:53:23.062 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.060+0000 7f4f97fff640 1 -- 172.21.15.99:0/4019439694 learned_addr learned my addr 172.21.15.99:0/4019439694 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:53:23.062 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.060+0000 7f4f977fe640 1 --2- 172.21.15.99:0/4019439694 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564dcf793100 0x564dcf78d220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:23.062 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.060+0000 7f4f977fe640 1 --2- 172.21.15.99:0/4019439694 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564dcf798c60 0x564dcf78d760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:23.062 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.060+0000 7f4f97fff640 1 -- 172.21.15.99:0/4019439694 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564dcf798c60 msgr2=0x564dcf78d760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:53:23.062 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.060+0000 7f4f97fff640 1 --2- 172.21.15.99:0/4019439694 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564dcf798c60 0x564dcf78d760 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:23.063 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.060+0000 7f4f97fff640 1 -- 172.21.15.99:0/4019439694 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564dcf793100 msgr2=0x564dcf78d220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:53:23.063 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.060+0000 7f4f97fff640 1 --2- 172.21.15.99:0/4019439694 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564dcf793100 0x564dcf78d220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:23.063 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.060+0000 7f4f97fff640 1 -- 172.21.15.99:0/4019439694 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x564dcf4d8620 con 0x564dcf793af0 2024-03-19T01:53:23.063 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.061+0000 7f4f97fff640 1 --2- 172.21.15.99:0/4019439694 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564dcf793af0 0x564dcf7a45b0 secure :-1 s=READY pgs=583 cs=0 l=1 rev1=1 crypto rx=0x7f4f88012fd0 tx=0x7f4f88005f90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5f3470b25ad5535a server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:53:23.063 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.061+0000 7f4f7dffb640 1 -- 172.21.15.99:0/4019439694 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4f88028040 con 0x564dcf793af0 2024-03-19T01:53:23.063 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.061+0000 7f4f7dffb640 1 -- 172.21.15.99:0/4019439694 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f4f88016030 con 0x564dcf793af0 2024-03-19T01:53:23.063 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.061+0000 7f4f7dffb640 1 -- 172.21.15.99:0/4019439694 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4f88023890 con 0x564dcf793af0 2024-03-19T01:53:23.066 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.062+0000 7f4fa1566740 1 -- 172.21.15.99:0/4019439694 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564dcf793af0 msgr2=0x564dcf7a45b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:53:23.066 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.062+0000 7f4fa1566740 1 --2- 172.21.15.99:0/4019439694 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564dcf793af0 0x564dcf7a45b0 secure :-1 s=READY pgs=583 cs=0 l=1 rev1=1 crypto rx=0x7f4f88012fd0 tx=0x7f4f88005f90 comp rx=0 tx=0).stop 2024-03-19T01:53:23.067 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.062+0000 7f4fa1566740 1 -- 172.21.15.99:0/4019439694 shutdown_connections 2024-03-19T01:53:23.067 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.062+0000 7f4fa1566740 1 --2- 172.21.15.99:0/4019439694 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564dcf798c60 0x564dcf78d760 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:23.067 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.062+0000 7f4fa1566740 1 --2- 172.21.15.99:0/4019439694 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564dcf793af0 0x564dcf7a45b0 unknown :-1 s=CLOSED pgs=583 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:23.067 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.062+0000 7f4fa1566740 1 --2- 172.21.15.99:0/4019439694 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564dcf793100 0x564dcf78d220 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:23.067 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.062+0000 7f4fa1566740 1 -- 172.21.15.99:0/4019439694 >> 172.21.15.99:0/4019439694 conn(0x564dcf78e670 msgr2=0x564dcf78ea50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:53:23.067 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.062+0000 7f4fa1566740 1 -- 172.21.15.99:0/4019439694 shutdown_connections 2024-03-19T01:53:23.067 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.062+0000 7f4fa1566740 1 -- 172.21.15.99:0/4019439694 wait complete. 2024-03-19T01:53:23.067 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.062+0000 7f4fa1566740 1 -- start start 2024-03-19T01:53:23.067 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.062+0000 7f4fa1566740 1 Processor -- start 2024-03-19T01:53:23.067 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.062+0000 7f4fa1566740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564dcf793100 0x564dcf7a13d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:23.067 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.062+0000 7f4fa1566740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564dcf793af0 0x564dcf7a1910 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:23.067 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.062+0000 7f4fa1566740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564dcf798c60 0x564dcf80d070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:23.067 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.062+0000 7f4fa1566740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f4f981a7500 con 0x564dcf798c60 2024-03-19T01:53:23.067 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.062+0000 7f4fa1566740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f4f981a8970 con 0x564dcf793af0 2024-03-19T01:53:23.068 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.062+0000 7f4fa1566740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f4f981a7bd0 con 0x564dcf793100 2024-03-19T01:53:23.068 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.062+0000 7f4fa1566740 10 client.0 _ll_register_callbacks cb 0x7f4f900093a0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:53:23.068 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.062+0000 7f4f977fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564dcf798c60 0x564dcf80d070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:23.068 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.063+0000 7f4f97fff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564dcf793af0 0x564dcf7a1910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:23.068 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.063+0000 7f4f97fff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564dcf793af0 0x564dcf7a1910 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:57386/0 (socket says 172.21.15.99:57386) 2024-03-19T01:53:23.068 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.063+0000 7f4f97fff640 1 -- 172.21.15.99:0/3686622165 learned_addr learned my addr 172.21.15.99:0/3686622165 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:53:23.068 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.063+0000 7f4f977fe640 1 --2- 172.21.15.99:0/3686622165 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564dcf798c60 0x564dcf80d070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:45222/0 (socket says 172.21.15.99:45222) 2024-03-19T01:53:23.068 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.063+0000 7f4f977fe640 1 --2- 172.21.15.99:0/3686622165 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564dcf793100 0x564dcf7a13d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:23.068 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.063+0000 7f4f97fff640 1 -- 172.21.15.99:0/3686622165 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564dcf793100 msgr2=0x564dcf7a13d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:53:23.069 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.063+0000 7f4f97fff640 1 --2- 172.21.15.99:0/3686622165 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564dcf793100 0x564dcf7a13d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:23.069 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.063+0000 7f4f97fff640 1 -- 172.21.15.99:0/3686622165 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564dcf798c60 msgr2=0x564dcf80d070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:53:23.069 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.063+0000 7f4f97fff640 1 --2- 172.21.15.99:0/3686622165 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564dcf798c60 0x564dcf80d070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:23.069 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.063+0000 7f4f97fff640 1 -- 172.21.15.99:0/3686622165 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x564dcf7a1f80 con 0x564dcf793af0 2024-03-19T01:53:23.069 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.063+0000 7f4f97fff640 1 --2- 172.21.15.99:0/3686622165 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564dcf793af0 0x564dcf7a1910 secure :-1 s=READY pgs=159 cs=0 l=1 rev1=1 crypto rx=0x7f4f88000c00 tx=0x7f4f88023750 comp rx=0 tx=0).ready entity=mon.2 client_cookie=553dabb734e55 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:53:23.069 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.063+0000 7f4f777fe640 10 client.0 ms_handle_connect on v2:172.21.15.112:3300/0 2024-03-19T01:53:23.069 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.063+0000 7f4f777fe640 1 -- 172.21.15.99:0/3686622165 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4f88028040 con 0x564dcf793af0 2024-03-19T01:53:23.069 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.063+0000 7f4fa1566740 1 -- 172.21.15.99:0/3686622165 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f4f88016030 con 0x564dcf793af0 2024-03-19T01:53:23.069 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.063+0000 7f4f777fe640 1 -- 172.21.15.99:0/3686622165 <== mon.2 v2:172.21.15.112:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f4f88003d40 con 0x564dcf793af0 2024-03-19T01:53:23.069 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.063+0000 7f4f777fe640 1 -- 172.21.15.99:0/3686622165 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_get_version_reply(handle=1 version=28) ==== 24+0+0 (secure 0 0 0) 0x7f4f88023450 con 0x564dcf793af0 2024-03-19T01:53:23.069 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.063+0000 7f4fa1566740 10 client.26114 fetch_fsmap learned FSMap version 28 2024-03-19T01:53:23.069 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.063+0000 7f4fa1566740 1 -- 172.21.15.99:0/3686622165 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,fsmap=28,monmap=4+}) -- 0x564dcf80e6c0 con 0x564dcf793af0 2024-03-19T01:53:23.069 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.063+0000 7f4f777fe640 1 -- 172.21.15.99:0/3686622165 <== mon.2 v2:172.21.15.112:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f4f88016030 con 0x564dcf793af0 2024-03-19T01:53:23.069 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.064+0000 7f4f777fe640 1 -- 172.21.15.99:0/3686622165 <== mon.2 v2:172.21.15.112:3300/0 5 ==== fsmap(e 28) ==== 8338+0+0 (secure 0 0 0) 0x7f4f88004500 con 0x564dcf793af0 2024-03-19T01:53:23.070 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.064+0000 7f4fa1566740 10 client.26114 fetch_fsmap finished waiting for FSMap version 28 2024-03-19T01:53:23.070 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.064+0000 7f4fa1566740 10 client.26114 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:53:23.070 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.064+0000 7f4fa1566740 20 client.26114 populate_metadata read hostname 'smithi099' 2024-03-19T01:53:23.070 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.064+0000 7f4fa1566740 1 --2- 172.21.15.99:0/3686622165 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x564dcf80efe0 0x564dcf811400 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:23.070 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.064+0000 7f4fa1566740 4 client.26114 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:53:23.070 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.064+0000 7f4fa1566740 1 -- 172.21.15.99:0/3686622165 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x564dcf6ff470 con 0x564dcf80efe0 2024-03-19T01:53:23.070 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.064+0000 7f4f977fe640 1 --2- 172.21.15.99:0/3686622165 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x564dcf80efe0 0x564dcf811400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:23.070 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.064+0000 7f4f777fe640 10 client.26114 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:53:23.070 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.064+0000 7f4f977fe640 1 --2- 172.21.15.99:0/3686622165 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x564dcf80efe0 0x564dcf811400 crc :-1 s=READY pgs=140 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=d1fec532129181b server_cookie=7cc981d6afb25565 in_seq=0 out_seq=0 2024-03-19T01:53:23.130 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.129+0000 7f4f777fe640 1 -- 172.21.15.99:0/3686622165 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x564dcf6ff470 con 0x564dcf80efe0 2024-03-19T01:53:23.130 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.129+0000 7f4f777fe640 10 client.26114 handle_command_reply: tid=0 2024-03-19T01:53:23.130 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.129+0000 7f4fa1566740 1 client.26114 shutdown 2024-03-19T01:53:23.130 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.129+0000 7f4fa1566740 1 -- 172.21.15.99:0/3686622165 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564dcf793af0 msgr2=0x564dcf7a1910 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:53:23.130 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.129+0000 7f4fa1566740 1 --2- 172.21.15.99:0/3686622165 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564dcf793af0 0x564dcf7a1910 secure :-1 s=READY pgs=159 cs=0 l=1 rev1=1 crypto rx=0x7f4f88000c00 tx=0x7f4f88023750 comp rx=0 tx=0).stop 2024-03-19T01:53:23.130 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.129+0000 7f4fa1566740 1 -- 172.21.15.99:0/3686622165 shutdown_connections 2024-03-19T01:53:23.130 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.129+0000 7f4fa1566740 1 --2- 172.21.15.99:0/3686622165 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x564dcf80efe0 0x564dcf811400 crc :-1 s=READY pgs=140 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:23.130 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.129+0000 7f4fa1566740 1 --2- 172.21.15.99:0/3686622165 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564dcf793af0 0x564dcf7a1910 unknown :-1 s=CLOSED pgs=159 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:23.130 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.129+0000 7f4fa1566740 1 --2- 172.21.15.99:0/3686622165 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564dcf798c60 0x564dcf80d070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:23.130 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.129+0000 7f4fa1566740 1 --2- 172.21.15.99:0/3686622165 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564dcf793100 0x564dcf7a13d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:23.131 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.129+0000 7f4fa1566740 1 -- 172.21.15.99:0/3686622165 >> 172.21.15.99:0/3686622165 conn(0x564dcf78e670 msgr2=0x564dcf79cf40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:53:23.131 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.129+0000 7f4f777fe640 0 client.26114 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:53:23.131 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.129+0000 7f4fa1566740 1 -- 172.21.15.99:0/3686622165 shutdown_connections 2024-03-19T01:53:23.131 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.129+0000 7f4fa1566740 1 -- 172.21.15.99:0/3686622165 wait complete. 2024-03-19T01:53:23.131 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.129+0000 7f4fa1566740 20 client.26114 trim_cache size 0 max 16384 2024-03-19T01:53:23.131 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:53:23.131 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:53:23.131 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:53:23.131 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:53:23.132 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.130+0000 7f4f9e977640 1 -- 172.21.15.99:0/4053767597 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f4f800a28a0 msgr2=0x7f4f800a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:53:23.132 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.130+0000 7f4f9e977640 1 --2- 172.21.15.99:0/4053767597 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f4f800a28a0 0x7f4f800a4d60 secure :-1 s=READY pgs=322 cs=0 l=1 rev1=1 crypto rx=0x7f4f9009b7e0 tx=0x7f4f9000efd0 comp rx=0 tx=0).stop 2024-03-19T01:53:23.132 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.130+0000 7f4f9e977640 1 -- 172.21.15.99:0/4053767597 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4f981022f0 msgr2=0x7f4f9819f070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:53:23.132 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.130+0000 7f4f9e977640 1 --2- 172.21.15.99:0/4053767597 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4f981022f0 0x7f4f9819f070 secure :-1 s=READY pgs=581 cs=0 l=1 rev1=1 crypto rx=0x7f4f9000eb10 tx=0x7f4f9000c2d0 comp rx=0 tx=0).stop 2024-03-19T01:53:23.132 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.130+0000 7f4f9e977640 1 -- 172.21.15.99:0/4053767597 shutdown_connections 2024-03-19T01:53:23.133 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.130+0000 7f4f9e977640 1 --2- 172.21.15.99:0/4053767597 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f4f800a28a0 0x7f4f800a4d60 unknown :-1 s=CLOSED pgs=322 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:23.133 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.130+0000 7f4f9e977640 1 --2- 172.21.15.99:0/4053767597 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4f98101930 0x7f4f98198000 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:23.133 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.130+0000 7f4f9e977640 1 --2- 172.21.15.99:0/4053767597 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4f981022f0 0x7f4f9819f070 unknown :-1 s=CLOSED pgs=581 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:23.133 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.130+0000 7f4f9e977640 1 --2- 172.21.15.99:0/4053767597 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4f98100740 0x7f4f98197ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:23.133 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.130+0000 7f4f9e977640 1 -- 172.21.15.99:0/4053767597 >> 172.21.15.99:0/4053767597 conn(0x7f4f980fbcb0 msgr2=0x7f4f980fd6c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:53:23.133 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.130+0000 7f4f9e977640 1 -- 172.21.15.99:0/4053767597 shutdown_connections 2024-03-19T01:53:23.133 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:23.130+0000 7f4f9e977640 1 -- 172.21.15.99:0/4053767597 wait complete. 2024-03-19T01:53:23.705 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1491761 11.24 MB/sec execute 593 sec latency 60.501 ms 2024-03-19T01:53:24.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:53:24 smithi115 ceph-mon[67332]: pgmap v720: 97 pgs: 97 active+clean; 264 MiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 866 KiB/s rd, 3.0 MiB/s wr, 274 op/s 2024-03-19T01:53:24.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:53:24 smithi099 ceph-mon[62951]: pgmap v720: 97 pgs: 97 active+clean; 264 MiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 866 KiB/s rd, 3.0 MiB/s wr, 274 op/s 2024-03-19T01:53:24.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:53:24 smithi112 ceph-mon[66819]: pgmap v720: 97 pgs: 97 active+clean; 264 MiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 866 KiB/s rd, 3.0 MiB/s wr, 274 op/s 2024-03-19T01:53:24.705 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1493575 11.24 MB/sec execute 594 sec latency 32.770 ms 2024-03-19T01:53:25.705 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1496041 11.25 MB/sec execute 595 sec latency 11.021 ms 2024-03-19T01:53:26.309 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:53:26 smithi115 ceph-mon[67332]: pgmap v721: 97 pgs: 97 active+clean; 264 MiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 791 KiB/s rd, 2.6 MiB/s wr, 227 op/s 2024-03-19T01:53:26.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:53:26 smithi099 ceph-mon[62951]: pgmap v721: 97 pgs: 97 active+clean; 264 MiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 791 KiB/s rd, 2.6 MiB/s wr, 227 op/s 2024-03-19T01:53:26.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:53:26 smithi112 ceph-mon[66819]: pgmap v721: 97 pgs: 97 active+clean; 264 MiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 791 KiB/s rd, 2.6 MiB/s wr, 227 op/s 2024-03-19T01:53:26.705 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1498042 11.24 MB/sec execute 596 sec latency 26.830 ms 2024-03-19T01:53:27.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:53:27 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:53:27.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:53:27 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:53:27.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:53:27 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:53:27.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:53:27 smithi115 ceph-mon[67332]: pgmap v722: 97 pgs: 97 active+clean; 262 MiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 703 KiB/s rd, 2.9 MiB/s wr, 236 op/s 2024-03-19T01:53:27.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:53:27 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:53:27.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:53:27 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:53:27.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:53:27 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:53:27.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:53:27 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:53:27.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:53:27 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:53:27.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:53:27 smithi099 ceph-mon[62951]: pgmap v722: 97 pgs: 97 active+clean; 262 MiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 703 KiB/s rd, 2.9 MiB/s wr, 236 op/s 2024-03-19T01:53:27.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:53:27 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:53:27.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:53:27 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:53:27.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:53:27 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:53:27.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:53:27 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:53:27.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:53:27 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:53:27.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:53:27 smithi112 ceph-mon[66819]: pgmap v722: 97 pgs: 97 active+clean; 262 MiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 703 KiB/s rd, 2.9 MiB/s wr, 236 op/s 2024-03-19T01:53:27.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:53:27 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:53:27.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:53:27 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:53:27.705 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1499684 11.24 MB/sec execute 597 sec latency 42.621 ms 2024-03-19T01:53:28.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:53:28 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:53:28.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:53:28 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:53:28.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:53:28 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:53:28.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:53:28 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:53:28.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:53:28 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:53:28.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:53:28 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:53:28.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:53:28 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:53:28.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:53:28 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:53:28.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:53:28 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:53:28.705 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1501794 11.24 MB/sec execute 598 sec latency 39.078 ms 2024-03-19T01:53:29.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:53:29 smithi115 ceph-mon[67332]: pgmap v723: 97 pgs: 97 active+clean; 265 MiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 827 KiB/s rd, 2.8 MiB/s wr, 263 op/s 2024-03-19T01:53:29.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:53:29 smithi099 ceph-mon[62951]: pgmap v723: 97 pgs: 97 active+clean; 265 MiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 827 KiB/s rd, 2.8 MiB/s wr, 263 op/s 2024-03-19T01:53:29.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:53:29 smithi112 ceph-mon[66819]: pgmap v723: 97 pgs: 97 active+clean; 265 MiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 827 KiB/s rd, 2.8 MiB/s wr, 263 op/s 2024-03-19T01:53:29.705 INFO:tasks.workunit.client.0.smithi099.stdout: 1 1503703 11.24 MB/sec execute 599 sec latency 30.534 ms 2024-03-19T01:53:30.705 INFO:tasks.workunit.client.0.smithi099.stdout: 1 cleanup 600 sec 2024-03-19T01:53:30.994 INFO:tasks.workunit.client.0.smithi099.stdout: 0 cleanup 600 sec 2024-03-19T01:53:30.995 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-03-19T01:53:30.995 INFO:tasks.workunit.client.0.smithi099.stdout: Operation Count AvgLat MaxLat 2024-03-19T01:53:30.995 INFO:tasks.workunit.client.0.smithi099.stdout: ---------------------------------------- 2024-03-19T01:53:30.995 INFO:tasks.workunit.client.0.smithi099.stdout: NTCreateX 214507 1.111 130.898 2024-03-19T01:53:30.995 INFO:tasks.workunit.client.0.smithi099.stdout: Close 157580 0.002 0.090 2024-03-19T01:53:30.995 INFO:tasks.workunit.client.0.smithi099.stdout: Rename 9088 5.252 134.221 2024-03-19T01:53:30.995 INFO:tasks.workunit.client.0.smithi099.stdout: Unlink 43313 1.991 112.370 2024-03-19T01:53:30.995 INFO:tasks.workunit.client.0.smithi099.stdout: Deltree 6 96.231 149.399 2024-03-19T01:53:30.995 INFO:tasks.workunit.client.0.smithi099.stdout: Mkdir 3 0.011 0.014 2024-03-19T01:53:30.995 INFO:tasks.workunit.client.0.smithi099.stdout: Qpathinfo 194421 0.799 4723.591 2024-03-19T01:53:30.995 INFO:tasks.workunit.client.0.smithi099.stdout: Qfileinfo 34106 0.002 0.041 2024-03-19T01:53:30.995 INFO:tasks.workunit.client.0.smithi099.stdout: Qfsinfo 35678 0.150 9.974 2024-03-19T01:53:30.995 INFO:tasks.workunit.client.0.smithi099.stdout: Sfileinfo 17471 0.018 1.517 2024-03-19T01:53:30.995 INFO:tasks.workunit.client.0.smithi099.stdout: Find 75209 0.182 114.325 2024-03-19T01:53:30.996 INFO:tasks.workunit.client.0.smithi099.stdout: WriteX 107086 0.013 5.155 2024-03-19T01:53:30.996 INFO:tasks.workunit.client.0.smithi099.stdout: ReadX 336572 0.004 20.178 2024-03-19T01:53:30.996 INFO:tasks.workunit.client.0.smithi099.stdout: LockX 700 0.944 1.978 2024-03-19T01:53:30.996 INFO:tasks.workunit.client.0.smithi099.stdout: UnlockX 700 0.783 2.042 2024-03-19T01:53:30.996 INFO:tasks.workunit.client.0.smithi099.stdout: Flush 15036 3.034 352.874 2024-03-19T01:53:30.996 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-03-19T01:53:30.996 INFO:tasks.workunit.client.0.smithi099.stdout:Throughput 11.2371 MB/sec 1 clients 1 procs max_latency=4723.600 ms 2024-03-19T01:53:30.996 INFO:tasks.workunit.client.0.smithi099.stderr:+ dbench 10 2024-03-19T01:53:30.997 INFO:tasks.workunit.client.0.smithi099.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-03-19T01:53:30.997 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-03-19T01:53:30.997 INFO:tasks.workunit.client.0.smithi099.stdout:Running for 600 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 120 secs 2024-03-19T01:53:30.997 INFO:tasks.workunit.client.0.smithi099.stdout:8 of 10 processes prepared for launch 0 sec 2024-03-19T01:53:31.096 INFO:tasks.workunit.client.0.smithi099.stdout:10 of 10 processes prepared for launch 0 sec 2024-03-19T01:53:31.096 INFO:tasks.workunit.client.0.smithi099.stdout:releasing clients 2024-03-19T01:53:32.096 INFO:tasks.workunit.client.0.smithi099.stdout: 10 312 114.33 MB/sec warmup 1 sec latency 79.780 ms 2024-03-19T01:53:32.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:53:32 smithi115 ceph-mon[67332]: pgmap v724: 97 pgs: 97 active+clean; 265 MiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 794 KiB/s rd, 2.8 MiB/s wr, 244 op/s 2024-03-19T01:53:32.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:53:32 smithi099 ceph-mon[62951]: pgmap v724: 97 pgs: 97 active+clean; 265 MiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 794 KiB/s rd, 2.8 MiB/s wr, 244 op/s 2024-03-19T01:53:32.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:53:32 smithi112 ceph-mon[66819]: pgmap v724: 97 pgs: 97 active+clean; 265 MiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 794 KiB/s rd, 2.8 MiB/s wr, 244 op/s 2024-03-19T01:53:33.096 INFO:tasks.workunit.client.0.smithi099.stdout: 10 760 106.97 MB/sec warmup 2 sec latency 100.734 ms 2024-03-19T01:53:33.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:53:33 smithi115 ceph-mon[67332]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T01:53:33.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:53:33 smithi115 ceph-mon[67332]: pgmap v725: 97 pgs: 97 active+clean; 266 MiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 806 KiB/s rd, 3.0 MiB/s wr, 256 op/s 2024-03-19T01:53:33.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:53:33 smithi099 ceph-mon[62951]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T01:53:33.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:53:33 smithi099 ceph-mon[62951]: pgmap v725: 97 pgs: 97 active+clean; 266 MiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 806 KiB/s rd, 3.0 MiB/s wr, 256 op/s 2024-03-19T01:53:33.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:53:33 smithi112 ceph-mon[66819]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T01:53:33.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:53:33 smithi112 ceph-mon[66819]: pgmap v725: 97 pgs: 97 active+clean; 266 MiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 806 KiB/s rd, 3.0 MiB/s wr, 256 op/s 2024-03-19T01:53:34.096 INFO:tasks.workunit.client.0.smithi099.stdout: 10 1288 79.79 MB/sec warmup 3 sec latency 72.608 ms 2024-03-19T01:53:35.096 INFO:tasks.workunit.client.0.smithi099.stdout: 10 1989 69.92 MB/sec warmup 4 sec latency 55.519 ms 2024-03-19T01:53:36.096 INFO:tasks.workunit.client.0.smithi099.stdout: 10 2780 61.95 MB/sec warmup 5 sec latency 61.228 ms 2024-03-19T01:53:36.228 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.226+0000 7fbf29630640 1 -- 172.21.15.99:0/1027837347 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbf24102a40 msgr2=0x7fbf24102ec0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:53:36.228 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.226+0000 7fbf29630640 1 --2- 172.21.15.99:0/1027837347 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbf24102a40 0x7fbf24102ec0 secure :-1 s=READY pgs=584 cs=0 l=1 rev1=1 crypto rx=0x7fbf18009ca0 tx=0x7fbf1802c0f0 comp rx=0 tx=0).stop 2024-03-19T01:53:36.228 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.226+0000 7fbf29630640 1 -- 172.21.15.99:0/1027837347 shutdown_connections 2024-03-19T01:53:36.228 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.226+0000 7fbf29630640 1 --2- 172.21.15.99:0/1027837347 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fbf24103400 0x7fbf24109c90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:36.229 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.226+0000 7fbf29630640 1 --2- 172.21.15.99:0/1027837347 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbf24102a40 0x7fbf24102ec0 unknown :-1 s=CLOSED pgs=584 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:36.229 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.226+0000 7fbf29630640 1 --2- 172.21.15.99:0/1027837347 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fbf24101850 0x7fbf24101c70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:36.229 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.226+0000 7fbf29630640 1 -- 172.21.15.99:0/1027837347 >> 172.21.15.99:0/1027837347 conn(0x7fbf240fcde0 msgr2=0x7fbf240ff200 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:53:36.229 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.226+0000 7fbf29630640 1 -- 172.21.15.99:0/1027837347 shutdown_connections 2024-03-19T01:53:36.229 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.226+0000 7fbf29630640 1 -- 172.21.15.99:0/1027837347 wait complete. 2024-03-19T01:53:36.229 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.227+0000 7fbf29630640 1 Processor -- start 2024-03-19T01:53:36.229 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.227+0000 7fbf29630640 1 -- start start 2024-03-19T01:53:36.229 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.227+0000 7fbf29630640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fbf24101850 0x7fbf24197a50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:36.229 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.227+0000 7fbf29630640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fbf24102a40 0x7fbf24197f90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:36.229 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.227+0000 7fbf29630640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbf24103400 0x7fbf2419f000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:36.229 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.227+0000 7fbf29630640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fbf2410d4d0 con 0x7fbf24103400 2024-03-19T01:53:36.229 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.227+0000 7fbf29630640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fbf2410d350 con 0x7fbf24102a40 2024-03-19T01:53:36.229 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.227+0000 7fbf29630640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fbf2410d650 con 0x7fbf24101850 2024-03-19T01:53:36.230 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.227+0000 7fbf237fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbf24103400 0x7fbf2419f000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:36.230 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.227+0000 7fbf237fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbf24103400 0x7fbf2419f000 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:56678/0 (socket says 172.21.15.99:56678) 2024-03-19T01:53:36.230 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.227+0000 7fbf227fc640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fbf24102a40 0x7fbf24197f90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:36.230 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.227+0000 7fbf237fe640 1 -- 172.21.15.99:0/880346624 learned_addr learned my addr 172.21.15.99:0/880346624 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:53:36.230 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.227+0000 7fbf22ffd640 1 --2- 172.21.15.99:0/880346624 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fbf24101850 0x7fbf24197a50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:36.230 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.228+0000 7fbf237fe640 1 -- 172.21.15.99:0/880346624 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fbf24101850 msgr2=0x7fbf24197a50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:53:36.230 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.228+0000 7fbf237fe640 1 --2- 172.21.15.99:0/880346624 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fbf24101850 0x7fbf24197a50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:36.231 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.228+0000 7fbf237fe640 1 -- 172.21.15.99:0/880346624 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fbf24102a40 msgr2=0x7fbf24197f90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:53:36.231 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.228+0000 7fbf237fe640 1 --2- 172.21.15.99:0/880346624 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fbf24102a40 0x7fbf24197f90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:36.231 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.228+0000 7fbf237fe640 1 -- 172.21.15.99:0/880346624 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fbf2419f670 con 0x7fbf24103400 2024-03-19T01:53:36.231 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.228+0000 7fbf237fe640 1 --2- 172.21.15.99:0/880346624 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbf24103400 0x7fbf2419f000 secure :-1 s=READY pgs=585 cs=0 l=1 rev1=1 crypto rx=0x7fbf1400e970 tx=0x7fbf1400ed70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:53:36.231 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.228+0000 7fbf0bfff640 1 -- 172.21.15.99:0/880346624 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbf1400ccc0 con 0x7fbf24103400 2024-03-19T01:53:36.231 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.228+0000 7fbf29630640 1 -- 172.21.15.99:0/880346624 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fbf2419f960 con 0x7fbf24103400 2024-03-19T01:53:36.232 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.228+0000 7fbf0bfff640 1 -- 172.21.15.99:0/880346624 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fbf1400ce60 con 0x7fbf24103400 2024-03-19T01:53:36.232 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.228+0000 7fbf0bfff640 1 -- 172.21.15.99:0/880346624 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbf14010450 con 0x7fbf24103400 2024-03-19T01:53:36.233 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.228+0000 7fbf29630640 1 -- 172.21.15.99:0/880346624 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fbf2419fed0 con 0x7fbf24103400 2024-03-19T01:53:36.245 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.230+0000 7fbf0bfff640 1 -- 172.21.15.99:0/880346624 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7fbf14005020 con 0x7fbf24103400 2024-03-19T01:53:36.245 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.230+0000 7fbf2c217740 1 Processor -- start 2024-03-19T01:53:36.245 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.230+0000 7fbf2c217740 1 -- start start 2024-03-19T01:53:36.245 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.231+0000 7fbf2c217740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55aeb9d89100 0x55aeb9d89520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:36.246 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.231+0000 7fbf2c217740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55aeb9d89af0 0x55aeb9d8e5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:36.246 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.231+0000 7fbf2c217740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55aeb9d8ec60 0x55aeb9d910f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:36.246 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.231+0000 7fbf2c217740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55aeb9cedbe0 con 0x55aeb9d89af0 2024-03-19T01:53:36.246 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.231+0000 7fbf2c217740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55aeb9c96960 con 0x55aeb9d8ec60 2024-03-19T01:53:36.246 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.231+0000 7fbf2c217740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55aeb9ce9c80 con 0x55aeb9d89100 2024-03-19T01:53:36.246 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.231+0000 7fbf22ffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55aeb9d89af0 0x55aeb9d8e5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:36.246 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.231+0000 7fbf227fc640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55aeb9d89100 0x55aeb9d89520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:36.246 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.231+0000 7fbf22ffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55aeb9d89af0 0x55aeb9d8e5f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:56684/0 (socket says 172.21.15.99:56684) 2024-03-19T01:53:36.246 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.231+0000 7fbf22ffd640 1 -- 172.21.15.99:0/2331542140 learned_addr learned my addr 172.21.15.99:0/2331542140 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:53:36.246 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.231+0000 7fbf0bfff640 1 --2- 172.21.15.99:0/880346624 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fbf0c0a28a0 0x7fbf0c0a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:36.246 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.231+0000 7fbf227fc640 1 --2- 172.21.15.99:0/2331542140 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55aeb9d8ec60 0x55aeb9d910f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:36.247 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.231+0000 7fbf22ffd640 1 -- 172.21.15.99:0/2331542140 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55aeb9d89100 msgr2=0x55aeb9d89520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:53:36.247 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.231+0000 7fbf22ffd640 1 --2- 172.21.15.99:0/2331542140 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55aeb9d89100 0x55aeb9d89520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:36.247 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.231+0000 7fbf237fe640 1 --2- 172.21.15.99:0/880346624 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fbf0c0a28a0 0x7fbf0c0a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:36.247 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.231+0000 7fbf22ffd640 1 -- 172.21.15.99:0/2331542140 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55aeb9d8ec60 msgr2=0x55aeb9d910f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:53:36.247 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.231+0000 7fbf22ffd640 1 --2- 172.21.15.99:0/2331542140 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55aeb9d8ec60 0x55aeb9d910f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:36.247 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.231+0000 7fbf22ffd640 1 -- 172.21.15.99:0/2331542140 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55aeb9c8dd20 con 0x55aeb9d89af0 2024-03-19T01:53:36.247 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.231+0000 7fbf227fc640 1 --2- 172.21.15.99:0/2331542140 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55aeb9d8ec60 0x55aeb9d910f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-19T01:53:36.247 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.231+0000 7fbf0bfff640 1 -- 172.21.15.99:0/880346624 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fbf140cee90 con 0x7fbf24103400 2024-03-19T01:53:36.247 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.231+0000 7fbf22ffd640 1 --2- 172.21.15.99:0/2331542140 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55aeb9d89af0 0x55aeb9d8e5f0 secure :-1 s=READY pgs=586 cs=0 l=1 rev1=1 crypto rx=0x7fbf1c009ca0 tx=0x7fbf1c011810 comp rx=0 tx=0).ready entity=mon.0 client_cookie=78f09f6e17252418 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:53:36.247 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.232+0000 7fbf097fa640 1 -- 172.21.15.99:0/2331542140 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbf1c004220 con 0x55aeb9d89af0 2024-03-19T01:53:36.248 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.232+0000 7fbf097fa640 1 -- 172.21.15.99:0/2331542140 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fbf1c0043c0 con 0x55aeb9d89af0 2024-03-19T01:53:36.248 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.232+0000 7fbf097fa640 1 -- 172.21.15.99:0/2331542140 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbf1c019930 con 0x55aeb9d89af0 2024-03-19T01:53:36.248 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.232+0000 7fbf237fe640 1 --2- 172.21.15.99:0/880346624 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fbf0c0a28a0 0x7fbf0c0a4d60 secure :-1 s=READY pgs=326 cs=0 l=1 rev1=1 crypto rx=0x7fbf140981c0 tx=0x7fbf14090cd0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:53:36.248 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.232+0000 7fbf2c217740 1 -- 172.21.15.99:0/2331542140 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55aeb9d89af0 msgr2=0x55aeb9d8e5f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:53:36.248 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.232+0000 7fbf2c217740 1 --2- 172.21.15.99:0/2331542140 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55aeb9d89af0 0x55aeb9d8e5f0 secure :-1 s=READY pgs=586 cs=0 l=1 rev1=1 crypto rx=0x7fbf1c009ca0 tx=0x7fbf1c011810 comp rx=0 tx=0).stop 2024-03-19T01:53:36.248 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.232+0000 7fbf2c217740 1 -- 172.21.15.99:0/2331542140 shutdown_connections 2024-03-19T01:53:36.248 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.232+0000 7fbf2c217740 1 --2- 172.21.15.99:0/2331542140 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55aeb9d8ec60 0x55aeb9d910f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:36.248 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.232+0000 7fbf2c217740 1 --2- 172.21.15.99:0/2331542140 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55aeb9d89af0 0x55aeb9d8e5f0 unknown :-1 s=CLOSED pgs=586 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:36.248 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.232+0000 7fbf2c217740 1 --2- 172.21.15.99:0/2331542140 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55aeb9d89100 0x55aeb9d89520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:36.248 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.232+0000 7fbf2c217740 1 -- 172.21.15.99:0/2331542140 >> 172.21.15.99:0/2331542140 conn(0x55aeb9d84670 msgr2=0x55aeb9d84a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:53:36.248 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.232+0000 7fbf2c217740 1 -- 172.21.15.99:0/2331542140 shutdown_connections 2024-03-19T01:53:36.248 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.232+0000 7fbf2c217740 1 -- 172.21.15.99:0/2331542140 wait complete. 2024-03-19T01:53:36.248 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.233+0000 7fbf2c217740 1 -- start start 2024-03-19T01:53:36.249 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.233+0000 7fbf2c217740 1 Processor -- start 2024-03-19T01:53:36.249 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.233+0000 7fbf2c217740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55aeb9d89100 0x55aeb9d93160 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:36.249 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.233+0000 7fbf2c217740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55aeb9d89af0 0x55aeb9d936a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:36.249 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.233+0000 7fbf2c217740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55aeb9d8ec60 0x55aeb9d9a6e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:36.249 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.233+0000 7fbf2c217740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55aeb9d91c10 con 0x55aeb9d8ec60 2024-03-19T01:53:36.249 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.233+0000 7fbf2c217740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55aeb9ca9db0 con 0x55aeb9d89100 2024-03-19T01:53:36.249 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.233+0000 7fbf2c217740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55aeb9d91d90 con 0x55aeb9d89af0 2024-03-19T01:53:36.249 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.233+0000 7fbf2c217740 10 client.0 _ll_register_callbacks cb 0x55aeb9caee70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:53:36.249 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.233+0000 7fbf227fc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55aeb9d8ec60 0x55aeb9d9a6e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:36.249 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.233+0000 7fbf227fc640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55aeb9d89100 0x55aeb9d93160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:36.249 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.233+0000 7fbf227fc640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55aeb9d89100 0x55aeb9d93160 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:57708/0 (socket says 172.21.15.99:57708) 2024-03-19T01:53:36.249 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.233+0000 7fbf227fc640 1 -- 172.21.15.99:0/1228183340 learned_addr learned my addr 172.21.15.99:0/1228183340 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:53:36.249 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.233+0000 7fbf22ffd640 1 --2- 172.21.15.99:0/1228183340 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55aeb9d89af0 0x55aeb9d936a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:36.250 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.234+0000 7fbf227fc640 1 -- 172.21.15.99:0/1228183340 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55aeb9d89af0 msgr2=0x55aeb9d936a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:53:36.250 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.234+0000 7fbf227fc640 1 --2- 172.21.15.99:0/1228183340 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55aeb9d89af0 0x55aeb9d936a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:36.250 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.234+0000 7fbf227fc640 1 -- 172.21.15.99:0/1228183340 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55aeb9d89100 msgr2=0x55aeb9d93160 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:53:36.250 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.234+0000 7fbf227fc640 1 --2- 172.21.15.99:0/1228183340 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55aeb9d89100 0x55aeb9d93160 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:36.250 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.234+0000 7fbf227fc640 1 -- 172.21.15.99:0/1228183340 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55aeb9ca8c60 con 0x55aeb9d8ec60 2024-03-19T01:53:36.250 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.234+0000 7fbf227fc640 1 --2- 172.21.15.99:0/1228183340 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55aeb9d8ec60 0x55aeb9d9a6e0 secure :-1 s=READY pgs=587 cs=0 l=1 rev1=1 crypto rx=0x7fbf18005fe0 tx=0x7fbf180095c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=cc304e593e1192a9 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:53:36.250 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.234+0000 7fbf06ff5640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:53:36.250 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.234+0000 7fbf06ff5640 1 -- 172.21.15.99:0/1228183340 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbf18039040 con 0x55aeb9d8ec60 2024-03-19T01:53:36.250 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.234+0000 7fbf06ff5640 1 -- 172.21.15.99:0/1228183340 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fbf18035680 con 0x55aeb9d8ec60 2024-03-19T01:53:36.250 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.234+0000 7fbf2c217740 1 -- 172.21.15.99:0/1228183340 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fbf1c0043c0 con 0x55aeb9d8ec60 2024-03-19T01:53:36.250 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.234+0000 7fbf06ff5640 1 -- 172.21.15.99:0/1228183340 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=30) ==== 24+0+0 (secure 0 0 0) 0x7fbf18055050 con 0x55aeb9d8ec60 2024-03-19T01:53:36.250 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.234+0000 7fbf2c217740 10 client.16356 fetch_fsmap learned FSMap version 30 2024-03-19T01:53:36.250 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.234+0000 7fbf2c217740 1 -- 172.21.15.99:0/1228183340 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=30,monmap=4+}) -- 0x55aeb9a02210 con 0x55aeb9d8ec60 2024-03-19T01:53:36.251 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.234+0000 7fbf06ff5640 1 -- 172.21.15.99:0/1228183340 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fbf1c0043c0 con 0x55aeb9d8ec60 2024-03-19T01:53:36.251 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.234+0000 7fbf06ff5640 1 -- 172.21.15.99:0/1228183340 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 30) ==== 8350+0+0 (secure 0 0 0) 0x7fbf1803f040 con 0x55aeb9d8ec60 2024-03-19T01:53:36.251 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.234+0000 7fbf2c217740 10 client.16356 fetch_fsmap finished waiting for FSMap version 30 2024-03-19T01:53:36.251 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.234+0000 7fbf2c217740 10 client.16356 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:53:36.251 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.234+0000 7fbf2c217740 20 client.16356 populate_metadata read hostname 'smithi099' 2024-03-19T01:53:36.251 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.234+0000 7fbf2c217740 1 --2- 172.21.15.99:0/1228183340 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55aeb9d9ef70 0x55aeb9da13e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:36.251 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.234+0000 7fbf2c217740 4 client.16356 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:53:36.251 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.234+0000 7fbf2c217740 1 -- 172.21.15.99:0/1228183340 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55aeb9cfb120 con 0x55aeb9d9ef70 2024-03-19T01:53:36.251 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.235+0000 7fbf22ffd640 1 --2- 172.21.15.99:0/1228183340 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55aeb9d9ef70 0x55aeb9da13e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:36.251 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.235+0000 7fbf06ff5640 10 client.16356 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:53:36.251 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.235+0000 7fbf22ffd640 1 --2- 172.21.15.99:0/1228183340 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55aeb9d9ef70 0x55aeb9da13e0 crc :-1 s=READY pgs=141 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=3115b92bce09c05e server_cookie=6ad1e558490a0940 in_seq=0 out_seq=0 2024-03-19T01:53:36.251 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.237+0000 7fbf06ff5640 1 -- 172.21.15.99:0/1228183340 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x55aeb9cfb120 con 0x55aeb9d9ef70 2024-03-19T01:53:36.251 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.237+0000 7fbf06ff5640 10 client.16356 handle_command_reply: tid=0 2024-03-19T01:53:36.251 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.237+0000 7fbf2c217740 1 client.16356 shutdown 2024-03-19T01:53:36.252 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.237+0000 7fbf2c217740 1 -- 172.21.15.99:0/1228183340 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55aeb9d8ec60 msgr2=0x55aeb9d9a6e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:53:36.252 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.237+0000 7fbf2c217740 1 --2- 172.21.15.99:0/1228183340 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55aeb9d8ec60 0x55aeb9d9a6e0 secure :-1 s=READY pgs=587 cs=0 l=1 rev1=1 crypto rx=0x7fbf18005fe0 tx=0x7fbf180095c0 comp rx=0 tx=0).stop 2024-03-19T01:53:36.252 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.237+0000 7fbf2c217740 1 -- 172.21.15.99:0/1228183340 shutdown_connections 2024-03-19T01:53:36.252 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.237+0000 7fbf2c217740 1 --2- 172.21.15.99:0/1228183340 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55aeb9d9ef70 0x55aeb9da13e0 crc :-1 s=READY pgs=141 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:36.252 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.237+0000 7fbf2c217740 1 --2- 172.21.15.99:0/1228183340 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55aeb9d8ec60 0x55aeb9d9a6e0 unknown :-1 s=CLOSED pgs=587 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:36.252 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.237+0000 7fbf2c217740 1 --2- 172.21.15.99:0/1228183340 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55aeb9d89af0 0x55aeb9d936a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:36.252 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.237+0000 7fbf06ff5640 0 client.16356 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:53:36.252 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.237+0000 7fbf2c217740 1 --2- 172.21.15.99:0/1228183340 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55aeb9d89100 0x55aeb9d93160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:36.252 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.237+0000 7fbf2c217740 1 -- 172.21.15.99:0/1228183340 >> 172.21.15.99:0/1228183340 conn(0x55aeb9d84670 msgr2=0x55aeb9d858c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:53:36.252 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.237+0000 7fbf2c217740 1 -- 172.21.15.99:0/1228183340 shutdown_connections 2024-03-19T01:53:36.252 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.238+0000 7fbf2c217740 1 -- 172.21.15.99:0/1228183340 wait complete. 2024-03-19T01:53:36.252 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.238+0000 7fbf2c217740 20 client.16356 trim_cache size 0 max 16384 2024-03-19T01:53:36.254 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.254+0000 7fbf2c217740 1 Processor -- start 2024-03-19T01:53:36.254 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.254+0000 7fbf2c217740 1 -- start start 2024-03-19T01:53:36.255 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.254+0000 7fbf2c217740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55aeb9d89100 0x55aeb9d87fe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:36.255 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.254+0000 7fbf2c217740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55aeb9d89af0 0x55aeb9d9a6a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:36.255 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.254+0000 7fbf2c217740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55aeb9da5ce0 0x55aeb9da6170 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:36.255 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.254+0000 7fbf2c217740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fbf241ab700 con 0x55aeb9d89af0 2024-03-19T01:53:36.255 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.254+0000 7fbf2c217740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fbf241a8f40 con 0x55aeb9d89100 2024-03-19T01:53:36.255 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.254+0000 7fbf2c217740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fbf241a6d90 con 0x55aeb9da5ce0 2024-03-19T01:53:36.255 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.254+0000 7fbf22ffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55aeb9d89af0 0x55aeb9d9a6a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:36.255 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.254+0000 7fbf22ffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55aeb9d89af0 0x55aeb9d9a6a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:56708/0 (socket says 172.21.15.99:56708) 2024-03-19T01:53:36.255 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.254+0000 7fbf227fc640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55aeb9d89100 0x55aeb9d87fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:36.255 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.254+0000 7fbf22ffd640 1 -- 172.21.15.99:0/3338657860 learned_addr learned my addr 172.21.15.99:0/3338657860 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:53:36.255 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.254+0000 7fbf227fc640 1 --2- 172.21.15.99:0/3338657860 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55aeb9da5ce0 0x55aeb9da6170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:36.255 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.254+0000 7fbf22ffd640 1 -- 172.21.15.99:0/3338657860 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55aeb9da5ce0 msgr2=0x55aeb9da6170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:53:36.256 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.254+0000 7fbf22ffd640 1 --2- 172.21.15.99:0/3338657860 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55aeb9da5ce0 0x55aeb9da6170 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:36.256 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.254+0000 7fbf22ffd640 1 -- 172.21.15.99:0/3338657860 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55aeb9d89100 msgr2=0x55aeb9d87fe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:53:36.256 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.254+0000 7fbf22ffd640 1 --2- 172.21.15.99:0/3338657860 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55aeb9d89100 0x55aeb9d87fe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:36.256 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.254+0000 7fbf22ffd640 1 -- 172.21.15.99:0/3338657860 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55aeb9ace620 con 0x55aeb9d89af0 2024-03-19T01:53:36.256 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.254+0000 7fbf22ffd640 1 --2- 172.21.15.99:0/3338657860 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55aeb9d89af0 0x55aeb9d9a6a0 secure :-1 s=READY pgs=588 cs=0 l=1 rev1=1 crypto rx=0x7fbf1c009230 tx=0x7fbf1c011810 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9ba3bcbdc6ec341d server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:53:36.256 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.254+0000 7fbf08ff9640 1 -- 172.21.15.99:0/3338657860 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbf1c02ba30 con 0x55aeb9d89af0 2024-03-19T01:53:36.256 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.254+0000 7fbf08ff9640 1 -- 172.21.15.99:0/3338657860 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fbf1c0055d0 con 0x55aeb9d89af0 2024-03-19T01:53:36.256 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.254+0000 7fbf08ff9640 1 -- 172.21.15.99:0/3338657860 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbf1c0035f0 con 0x55aeb9d89af0 2024-03-19T01:53:36.256 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.255+0000 7fbf2c217740 1 -- 172.21.15.99:0/3338657860 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55aeb9d89af0 msgr2=0x55aeb9d9a6a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:53:36.256 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.255+0000 7fbf2c217740 1 --2- 172.21.15.99:0/3338657860 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55aeb9d89af0 0x55aeb9d9a6a0 secure :-1 s=READY pgs=588 cs=0 l=1 rev1=1 crypto rx=0x7fbf1c009230 tx=0x7fbf1c011810 comp rx=0 tx=0).stop 2024-03-19T01:53:36.259 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.255+0000 7fbf2c217740 1 -- 172.21.15.99:0/3338657860 shutdown_connections 2024-03-19T01:53:36.260 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.255+0000 7fbf2c217740 1 --2- 172.21.15.99:0/3338657860 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55aeb9da5ce0 0x55aeb9da6170 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:36.260 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.255+0000 7fbf2c217740 1 --2- 172.21.15.99:0/3338657860 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55aeb9d89af0 0x55aeb9d9a6a0 unknown :-1 s=CLOSED pgs=588 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:36.260 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.255+0000 7fbf2c217740 1 --2- 172.21.15.99:0/3338657860 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55aeb9d89100 0x55aeb9d87fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:36.260 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.255+0000 7fbf2c217740 1 -- 172.21.15.99:0/3338657860 >> 172.21.15.99:0/3338657860 conn(0x55aeb9d84670 msgr2=0x55aeb9d878d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:53:36.260 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.255+0000 7fbf2c217740 1 -- 172.21.15.99:0/3338657860 shutdown_connections 2024-03-19T01:53:36.260 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.255+0000 7fbf2c217740 1 -- 172.21.15.99:0/3338657860 wait complete. 2024-03-19T01:53:36.260 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.255+0000 7fbf2c217740 1 -- start start 2024-03-19T01:53:36.260 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.256+0000 7fbf2c217740 1 Processor -- start 2024-03-19T01:53:36.260 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.256+0000 7fbf2c217740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55aeb9d89100 0x55aeb9d9a1c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:36.260 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.256+0000 7fbf2c217740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55aeb9d89af0 0x55aeb9d987d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:36.260 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.256+0000 7fbf2c217740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55aeb9da5ce0 0x55aeb9d98d10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:36.260 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.256+0000 7fbf2c217740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fbf241a7400 con 0x55aeb9d89af0 2024-03-19T01:53:36.260 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.256+0000 7fbf2c217740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fbf241a8870 con 0x55aeb9d89100 2024-03-19T01:53:36.260 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.256+0000 7fbf2c217740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fbf241a7ad0 con 0x55aeb9da5ce0 2024-03-19T01:53:36.261 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.256+0000 7fbf2c217740 10 client.0 _ll_register_callbacks cb 0x7fbf18009b20 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:53:36.261 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.256+0000 7fbf22ffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55aeb9d89af0 0x55aeb9d987d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:36.261 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.256+0000 7fbf22ffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55aeb9d89af0 0x55aeb9d987d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:56722/0 (socket says 172.21.15.99:56722) 2024-03-19T01:53:36.261 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.256+0000 7fbf22ffd640 1 -- 172.21.15.99:0/1487393550 learned_addr learned my addr 172.21.15.99:0/1487393550 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:53:36.261 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.256+0000 7fbf227fc640 1 --2- 172.21.15.99:0/1487393550 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55aeb9d89100 0x55aeb9d9a1c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:36.261 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.256+0000 7fbf227fc640 1 --2- 172.21.15.99:0/1487393550 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55aeb9da5ce0 0x55aeb9d98d10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:36.261 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.256+0000 7fbf22ffd640 1 -- 172.21.15.99:0/1487393550 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55aeb9da5ce0 msgr2=0x55aeb9d98d10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:53:36.261 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.256+0000 7fbf22ffd640 1 --2- 172.21.15.99:0/1487393550 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55aeb9da5ce0 0x55aeb9d98d10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:36.261 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.256+0000 7fbf22ffd640 1 -- 172.21.15.99:0/1487393550 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55aeb9d89100 msgr2=0x55aeb9d9a1c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:53:36.261 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.256+0000 7fbf22ffd640 1 --2- 172.21.15.99:0/1487393550 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55aeb9d89100 0x55aeb9d9a1c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:36.261 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.256+0000 7fbf22ffd640 1 -- 172.21.15.99:0/1487393550 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55aeb9d99250 con 0x55aeb9d89af0 2024-03-19T01:53:36.261 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.256+0000 7fbf227fc640 1 --2- 172.21.15.99:0/1487393550 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55aeb9d89100 0x55aeb9d9a1c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-19T01:53:36.261 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.256+0000 7fbf22ffd640 1 --2- 172.21.15.99:0/1487393550 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55aeb9d89af0 0x55aeb9d987d0 secure :-1 s=READY pgs=589 cs=0 l=1 rev1=1 crypto rx=0x7fbf1c01b040 tx=0x7fbf1c02b2a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ff456521ccd379cf server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:53:36.261 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.256+0000 7fbf06ff5640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:53:36.262 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.256+0000 7fbf06ff5640 1 -- 172.21.15.99:0/1487393550 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbf1c005300 con 0x55aeb9d89af0 2024-03-19T01:53:36.262 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.256+0000 7fbf06ff5640 1 -- 172.21.15.99:0/1487393550 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fbf1c0035f0 con 0x55aeb9d89af0 2024-03-19T01:53:36.262 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.256+0000 7fbf2c217740 1 -- 172.21.15.99:0/1487393550 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fbf1c0055d0 con 0x55aeb9d89af0 2024-03-19T01:53:36.262 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.257+0000 7fbf06ff5640 1 -- 172.21.15.99:0/1487393550 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=30) ==== 24+0+0 (secure 0 0 0) 0x7fbf1c003810 con 0x55aeb9d89af0 2024-03-19T01:53:36.262 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.257+0000 7fbf2c217740 10 client.16362 fetch_fsmap learned FSMap version 30 2024-03-19T01:53:36.262 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.257+0000 7fbf2c217740 1 -- 172.21.15.99:0/1487393550 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=30,monmap=4+}) -- 0x55aeb9e05fa0 con 0x55aeb9d89af0 2024-03-19T01:53:36.262 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.257+0000 7fbf06ff5640 1 -- 172.21.15.99:0/1487393550 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fbf1c0055d0 con 0x55aeb9d89af0 2024-03-19T01:53:36.262 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.257+0000 7fbf06ff5640 1 -- 172.21.15.99:0/1487393550 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 30) ==== 8350+0+0 (secure 0 0 0) 0x7fbf1c03f430 con 0x55aeb9d89af0 2024-03-19T01:53:36.262 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.257+0000 7fbf2c217740 10 client.16362 fetch_fsmap finished waiting for FSMap version 30 2024-03-19T01:53:36.262 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.257+0000 7fbf2c217740 10 client.16362 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:53:36.262 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.257+0000 7fbf2c217740 20 client.16362 populate_metadata read hostname 'smithi099' 2024-03-19T01:53:36.262 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.257+0000 7fbf2c217740 1 --2- 172.21.15.99:0/1487393550 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55aeb9e067f0 0x55aeb9e08c10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:36.262 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.257+0000 7fbf2c217740 4 client.16362 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:53:36.263 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.257+0000 7fbf2c217740 1 -- 172.21.15.99:0/1487393550 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55aeb9cf5470 con 0x55aeb9e067f0 2024-03-19T01:53:36.263 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.257+0000 7fbf227fc640 1 --2- 172.21.15.99:0/1487393550 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55aeb9e067f0 0x55aeb9e08c10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:36.263 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.258+0000 7fbf227fc640 1 --2- 172.21.15.99:0/1487393550 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55aeb9e067f0 0x55aeb9e08c10 crc :-1 s=READY pgs=142 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=b362c5d465cbb21f server_cookie=466c6c0dd6a3733c in_seq=0 out_seq=0 2024-03-19T01:53:36.263 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.258+0000 7fbf06ff5640 10 client.16362 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:53:36.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:53:36 smithi115 ceph-mon[67332]: pgmap v726: 97 pgs: 97 active+clean; 264 MiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 786 KiB/s rd, 2.8 MiB/s wr, 232 op/s 2024-03-19T01:53:36.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:53:36 smithi099 ceph-mon[62951]: pgmap v726: 97 pgs: 97 active+clean; 264 MiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 786 KiB/s rd, 2.8 MiB/s wr, 232 op/s 2024-03-19T01:53:36.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:53:36 smithi112 ceph-mon[66819]: pgmap v726: 97 pgs: 97 active+clean; 264 MiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 786 KiB/s rd, 2.8 MiB/s wr, 232 op/s 2024-03-19T01:53:36.655 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.655+0000 7fbf06ff5640 1 -- 172.21.15.99:0/1487393550 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x55aeb9cf5470 con 0x55aeb9e067f0 2024-03-19T01:53:36.655 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.655+0000 7fbf06ff5640 10 client.16362 handle_command_reply: tid=0 2024-03-19T01:53:36.655 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.655+0000 7fbf2c217740 1 client.16362 shutdown 2024-03-19T01:53:36.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.655+0000 7fbf2c217740 1 -- 172.21.15.99:0/1487393550 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55aeb9d89af0 msgr2=0x55aeb9d987d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:53:36.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.655+0000 7fbf2c217740 1 --2- 172.21.15.99:0/1487393550 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55aeb9d89af0 0x55aeb9d987d0 secure :-1 s=READY pgs=589 cs=0 l=1 rev1=1 crypto rx=0x7fbf1c01b040 tx=0x7fbf1c02b2a0 comp rx=0 tx=0).stop 2024-03-19T01:53:36.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.655+0000 7fbf2c217740 1 -- 172.21.15.99:0/1487393550 shutdown_connections 2024-03-19T01:53:36.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.655+0000 7fbf2c217740 1 --2- 172.21.15.99:0/1487393550 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55aeb9e067f0 0x55aeb9e08c10 crc :-1 s=READY pgs=142 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:36.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.655+0000 7fbf2c217740 1 --2- 172.21.15.99:0/1487393550 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55aeb9da5ce0 0x55aeb9d98d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:36.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.655+0000 7fbf06ff5640 0 client.16362 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:53:36.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.655+0000 7fbf2c217740 1 --2- 172.21.15.99:0/1487393550 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55aeb9d89af0 0x55aeb9d987d0 unknown :-1 s=CLOSED pgs=589 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:36.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.655+0000 7fbf2c217740 1 --2- 172.21.15.99:0/1487393550 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55aeb9d89100 0x55aeb9d9a1c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:36.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.655+0000 7fbf2c217740 1 -- 172.21.15.99:0/1487393550 >> 172.21.15.99:0/1487393550 conn(0x55aeb9d84670 msgr2=0x55aeb9d88160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:53:36.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.655+0000 7fbf2c217740 1 -- 172.21.15.99:0/1487393550 shutdown_connections 2024-03-19T01:53:36.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.655+0000 7fbf2c217740 1 -- 172.21.15.99:0/1487393550 wait complete. 2024-03-19T01:53:36.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.655+0000 7fbf2c217740 20 client.16362 trim_cache size 0 max 16384 2024-03-19T01:53:36.657 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:53:36.657 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:53:36.657 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:53:36.657 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:53:36.658 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.656+0000 7fbf29630640 1 -- 172.21.15.99:0/880346624 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fbf0c0a28a0 msgr2=0x7fbf0c0a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:53:36.658 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.656+0000 7fbf29630640 1 --2- 172.21.15.99:0/880346624 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fbf0c0a28a0 0x7fbf0c0a4d60 secure :-1 s=READY pgs=326 cs=0 l=1 rev1=1 crypto rx=0x7fbf140981c0 tx=0x7fbf14090cd0 comp rx=0 tx=0).stop 2024-03-19T01:53:36.658 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.656+0000 7fbf29630640 1 -- 172.21.15.99:0/880346624 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbf24103400 msgr2=0x7fbf2419f000 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:53:36.658 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.656+0000 7fbf29630640 1 --2- 172.21.15.99:0/880346624 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbf24103400 0x7fbf2419f000 secure :-1 s=READY pgs=585 cs=0 l=1 rev1=1 crypto rx=0x7fbf1400e970 tx=0x7fbf1400ed70 comp rx=0 tx=0).stop 2024-03-19T01:53:36.658 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.656+0000 7fbf29630640 1 -- 172.21.15.99:0/880346624 shutdown_connections 2024-03-19T01:53:36.658 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.656+0000 7fbf29630640 1 --2- 172.21.15.99:0/880346624 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fbf0c0a28a0 0x7fbf0c0a4d60 unknown :-1 s=CLOSED pgs=326 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:36.658 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.656+0000 7fbf29630640 1 --2- 172.21.15.99:0/880346624 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fbf24102a40 0x7fbf24197f90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:36.658 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.656+0000 7fbf29630640 1 --2- 172.21.15.99:0/880346624 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fbf24103400 0x7fbf2419f000 unknown :-1 s=CLOSED pgs=585 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:36.658 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.656+0000 7fbf29630640 1 --2- 172.21.15.99:0/880346624 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fbf24101850 0x7fbf24197a50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:36.658 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.656+0000 7fbf29630640 1 -- 172.21.15.99:0/880346624 >> 172.21.15.99:0/880346624 conn(0x7fbf240fcde0 msgr2=0x7fbf24105ce0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:53:36.658 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.656+0000 7fbf29630640 1 -- 172.21.15.99:0/880346624 shutdown_connections 2024-03-19T01:53:36.658 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:36.656+0000 7fbf29630640 1 -- 172.21.15.99:0/880346624 wait complete. 2024-03-19T01:53:37.096 INFO:tasks.workunit.client.0.smithi099.stdout: 10 3076 54.58 MB/sec warmup 6 sec latency 276.673 ms 2024-03-19T01:53:37.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:53:37 smithi115 ceph-mon[67332]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T01:53:37.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:53:37 smithi099 ceph-mon[62951]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T01:53:37.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:53:37 smithi112 ceph-mon[66819]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T01:53:38.096 INFO:tasks.workunit.client.0.smithi099.stdout: 10 3634 52.23 MB/sec warmup 7 sec latency 78.398 ms 2024-03-19T01:53:38.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:53:38 smithi115 ceph-mon[67332]: pgmap v727: 97 pgs: 97 active+clean; 265 MiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 1.3 MiB/s rd, 3.8 MiB/s wr, 361 op/s 2024-03-19T01:53:38.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:53:38 smithi115 ceph-mon[67332]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T01:53:38.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:53:38 smithi099 ceph-mon[62951]: pgmap v727: 97 pgs: 97 active+clean; 265 MiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 1.3 MiB/s rd, 3.8 MiB/s wr, 361 op/s 2024-03-19T01:53:38.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:53:38 smithi099 ceph-mon[62951]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T01:53:38.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:53:38 smithi112 ceph-mon[66819]: pgmap v727: 97 pgs: 97 active+clean; 265 MiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 1.3 MiB/s rd, 3.8 MiB/s wr, 361 op/s 2024-03-19T01:53:38.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:53:38 smithi112 ceph-mon[66819]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T01:53:39.096 INFO:tasks.workunit.client.0.smithi099.stdout: 10 4185 51.31 MB/sec warmup 8 sec latency 114.472 ms 2024-03-19T01:53:39.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:53:39 smithi099 ceph-mon[62951]: pgmap v728: 97 pgs: 97 active+clean; 264 MiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 3.6 MiB/s wr, 365 op/s 2024-03-19T01:53:39.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:53:39 smithi112 ceph-mon[66819]: pgmap v728: 97 pgs: 97 active+clean; 264 MiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 3.6 MiB/s wr, 365 op/s 2024-03-19T01:53:39.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:53:39 smithi115 ceph-mon[67332]: pgmap v728: 97 pgs: 97 active+clean; 264 MiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 3.6 MiB/s wr, 365 op/s 2024-03-19T01:53:40.097 INFO:tasks.workunit.client.0.smithi099.stdout: 10 4560 47.77 MB/sec warmup 9 sec latency 61.013 ms 2024-03-19T01:53:41.097 INFO:tasks.workunit.client.0.smithi099.stdout: 10 4981 45.19 MB/sec warmup 10 sec latency 72.799 ms 2024-03-19T01:53:41.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:53:41 smithi099 ceph-mon[62951]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T01:53:41.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:53:41 smithi099 ceph-mon[62951]: pgmap v729: 97 pgs: 97 active+clean; 268 MiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 1.0 MiB/s rd, 3.6 MiB/s wr, 400 op/s 2024-03-19T01:53:41.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:53:41 smithi112 ceph-mon[66819]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T01:53:41.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:53:41 smithi112 ceph-mon[66819]: pgmap v729: 97 pgs: 97 active+clean; 268 MiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 1.0 MiB/s rd, 3.6 MiB/s wr, 400 op/s 2024-03-19T01:53:41.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:53:41 smithi115 ceph-mon[67332]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T01:53:41.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:53:41 smithi115 ceph-mon[67332]: pgmap v729: 97 pgs: 97 active+clean; 268 MiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 1.0 MiB/s rd, 3.6 MiB/s wr, 400 op/s 2024-03-19T01:53:42.097 INFO:tasks.workunit.client.0.smithi099.stdout: 10 5328 42.27 MB/sec warmup 11 sec latency 133.622 ms 2024-03-19T01:53:43.097 INFO:tasks.workunit.client.0.smithi099.stdout: 10 5707 40.00 MB/sec warmup 12 sec latency 100.523 ms 2024-03-19T01:53:43.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:53:43 smithi099 ceph-mon[62951]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T01:53:43.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:53:43 smithi099 ceph-mon[62951]: pgmap v730: 97 pgs: 97 active+clean; 270 MiB data, 3.8 GiB used, 1.0 TiB / 1.0 TiB avail; 1.4 MiB/s rd, 4.4 MiB/s wr, 501 op/s 2024-03-19T01:53:43.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:53:43 smithi112 ceph-mon[66819]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T01:53:43.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:53:43 smithi112 ceph-mon[66819]: pgmap v730: 97 pgs: 97 active+clean; 270 MiB data, 3.8 GiB used, 1.0 TiB / 1.0 TiB avail; 1.4 MiB/s rd, 4.4 MiB/s wr, 501 op/s 2024-03-19T01:53:43.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:53:43 smithi115 ceph-mon[67332]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T01:53:43.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:53:43 smithi115 ceph-mon[67332]: pgmap v730: 97 pgs: 97 active+clean; 270 MiB data, 3.8 GiB used, 1.0 TiB / 1.0 TiB avail; 1.4 MiB/s rd, 4.4 MiB/s wr, 501 op/s 2024-03-19T01:53:44.097 INFO:tasks.workunit.client.0.smithi099.stdout: 10 6101 38.19 MB/sec warmup 13 sec latency 100.947 ms 2024-03-19T01:53:45.097 INFO:tasks.workunit.client.0.smithi099.stdout: 10 6729 37.69 MB/sec warmup 14 sec latency 73.730 ms 2024-03-19T01:53:46.097 INFO:tasks.workunit.client.0.smithi099.stdout: 10 7513 37.34 MB/sec warmup 15 sec latency 70.568 ms 2024-03-19T01:53:46.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:53:46 smithi115 ceph-mon[67332]: pgmap v731: 97 pgs: 97 active+clean; 266 MiB data, 3.8 GiB used, 1.0 TiB / 1.0 TiB avail; 1.2 MiB/s rd, 4.0 MiB/s wr, 448 op/s 2024-03-19T01:53:46.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:53:46 smithi099 ceph-mon[62951]: pgmap v731: 97 pgs: 97 active+clean; 266 MiB data, 3.8 GiB used, 1.0 TiB / 1.0 TiB avail; 1.2 MiB/s rd, 4.0 MiB/s wr, 448 op/s 2024-03-19T01:53:46.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:53:46 smithi112 ceph-mon[66819]: pgmap v731: 97 pgs: 97 active+clean; 266 MiB data, 3.8 GiB used, 1.0 TiB / 1.0 TiB avail; 1.2 MiB/s rd, 4.0 MiB/s wr, 448 op/s 2024-03-19T01:53:47.097 INFO:tasks.workunit.client.0.smithi099.stdout: 10 8131 38.25 MB/sec warmup 16 sec latency 49.690 ms 2024-03-19T01:53:48.097 INFO:tasks.workunit.client.0.smithi099.stdout: 10 8687 38.21 MB/sec warmup 17 sec latency 63.930 ms 2024-03-19T01:53:48.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:53:48 smithi115 ceph-mon[67332]: pgmap v732: 97 pgs: 97 active+clean; 256 MiB data, 3.8 GiB used, 1.0 TiB / 1.0 TiB avail; 1.4 MiB/s rd, 4.7 MiB/s wr, 526 op/s 2024-03-19T01:53:48.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:53:48 smithi099 ceph-mon[62951]: pgmap v732: 97 pgs: 97 active+clean; 256 MiB data, 3.8 GiB used, 1.0 TiB / 1.0 TiB avail; 1.4 MiB/s rd, 4.7 MiB/s wr, 526 op/s 2024-03-19T01:53:48.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:53:48 smithi112 ceph-mon[66819]: pgmap v732: 97 pgs: 97 active+clean; 256 MiB data, 3.8 GiB used, 1.0 TiB / 1.0 TiB avail; 1.4 MiB/s rd, 4.7 MiB/s wr, 526 op/s 2024-03-19T01:53:49.097 INFO:tasks.workunit.client.0.smithi099.stdout: 10 9065 37.50 MB/sec warmup 18 sec latency 135.764 ms 2024-03-19T01:53:49.749 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.746+0000 7f7e92697640 1 -- 172.21.15.99:0/3757799891 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7e8c102330 msgr2=0x7f7e8c108bc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:53:49.750 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.746+0000 7f7e92697640 1 --2- 172.21.15.99:0/3757799891 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7e8c102330 0x7f7e8c108bc0 secure :-1 s=READY pgs=590 cs=0 l=1 rev1=1 crypto rx=0x7f7e84009ca0 tx=0x7f7e8402c0f0 comp rx=0 tx=0).stop 2024-03-19T01:53:49.750 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.746+0000 7f7e92697640 1 -- 172.21.15.99:0/3757799891 shutdown_connections 2024-03-19T01:53:49.750 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.747+0000 7f7e92697640 1 --2- 172.21.15.99:0/3757799891 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f7e8c101970 0x7f7e8c101df0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:49.750 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.747+0000 7f7e92697640 1 --2- 172.21.15.99:0/3757799891 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7e8c102330 0x7f7e8c108bc0 unknown :-1 s=CLOSED pgs=590 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:49.750 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.747+0000 7f7e92697640 1 --2- 172.21.15.99:0/3757799891 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f7e8c100780 0x7f7e8c100ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:49.750 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.747+0000 7f7e92697640 1 -- 172.21.15.99:0/3757799891 >> 172.21.15.99:0/3757799891 conn(0x7f7e8c0fbcf0 msgr2=0x7f7e8c0fe130 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:53:49.750 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.747+0000 7f7e92697640 1 -- 172.21.15.99:0/3757799891 shutdown_connections 2024-03-19T01:53:49.750 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.747+0000 7f7e92697640 1 -- 172.21.15.99:0/3757799891 wait complete. 2024-03-19T01:53:49.750 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.747+0000 7f7e92697640 1 Processor -- start 2024-03-19T01:53:49.750 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.747+0000 7f7e92697640 1 -- start start 2024-03-19T01:53:49.750 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.747+0000 7f7e92697640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f7e8c100780 0x7f7e8c197a50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:49.750 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.747+0000 7f7e92697640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7e8c101970 0x7f7e8c197f90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:49.751 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.747+0000 7f7e92697640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f7e8c102330 0x7f7e8c19f000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:49.751 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.747+0000 7f7e92697640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f7e8c10c3e0 con 0x7f7e8c101970 2024-03-19T01:53:49.751 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.747+0000 7f7e92697640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f7e8c10c260 con 0x7f7e8c102330 2024-03-19T01:53:49.751 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.747+0000 7f7e92697640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f7e8c10c560 con 0x7f7e8c100780 2024-03-19T01:53:49.751 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.748+0000 7f7e8b7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7e8c101970 0x7f7e8c197f90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:49.751 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.748+0000 7f7e8b7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7e8c101970 0x7f7e8c197f90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:35186/0 (socket says 172.21.15.99:35186) 2024-03-19T01:53:49.751 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.748+0000 7f7e8bfff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f7e8c100780 0x7f7e8c197a50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:49.751 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.748+0000 7f7e8b7fe640 1 -- 172.21.15.99:0/154312159 learned_addr learned my addr 172.21.15.99:0/154312159 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:53:49.751 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.748+0000 7f7e90c34640 1 --2- 172.21.15.99:0/154312159 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f7e8c102330 0x7f7e8c19f000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:49.751 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.748+0000 7f7e8b7fe640 1 -- 172.21.15.99:0/154312159 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f7e8c100780 msgr2=0x7f7e8c197a50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:53:49.751 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.748+0000 7f7e8b7fe640 1 --2- 172.21.15.99:0/154312159 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f7e8c100780 0x7f7e8c197a50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:49.751 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.748+0000 7f7e8b7fe640 1 -- 172.21.15.99:0/154312159 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f7e8c102330 msgr2=0x7f7e8c19f000 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:53:49.751 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.748+0000 7f7e8b7fe640 1 --2- 172.21.15.99:0/154312159 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f7e8c102330 0x7f7e8c19f000 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:49.751 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.748+0000 7f7e8b7fe640 1 -- 172.21.15.99:0/154312159 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7e8c19f670 con 0x7f7e8c101970 2024-03-19T01:53:49.752 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.748+0000 7f7e8b7fe640 1 --2- 172.21.15.99:0/154312159 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7e8c101970 0x7f7e8c197f90 secure :-1 s=READY pgs=591 cs=0 l=1 rev1=1 crypto rx=0x7f7e8000e9f0 tx=0x7f7e8000eeb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:53:49.752 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.748+0000 7f7e897fa640 1 -- 172.21.15.99:0/154312159 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7e8000cd80 con 0x7f7e8c101970 2024-03-19T01:53:49.752 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.748+0000 7f7e897fa640 1 -- 172.21.15.99:0/154312159 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7e80010910 con 0x7f7e8c101970 2024-03-19T01:53:49.752 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.748+0000 7f7e92697640 1 -- 172.21.15.99:0/154312159 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f7e8c19f960 con 0x7f7e8c101970 2024-03-19T01:53:49.752 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.748+0000 7f7e897fa640 1 -- 172.21.15.99:0/154312159 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7e80018990 con 0x7f7e8c101970 2024-03-19T01:53:49.752 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.748+0000 7f7e92697640 1 -- 172.21.15.99:0/154312159 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f7e8c19ff00 con 0x7f7e8c101970 2024-03-19T01:53:49.757 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.750+0000 7f7e897fa640 1 -- 172.21.15.99:0/154312159 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f7e80010ac0 con 0x7f7e8c101970 2024-03-19T01:53:49.757 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.751+0000 7f7e94f22740 1 Processor -- start 2024-03-19T01:53:49.757 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.751+0000 7f7e94f22740 1 -- start start 2024-03-19T01:53:49.757 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.751+0000 7f7e94f22740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5621bca91100 0x5621bca91520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:49.758 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.751+0000 7f7e94f22740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5621bca91af0 0x5621bca965f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:49.758 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.751+0000 7f7e94f22740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5621bca96c60 0x5621bca990f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:49.758 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.751+0000 7f7e94f22740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x5621bc9f5be0 con 0x5621bca91af0 2024-03-19T01:53:49.758 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.751+0000 7f7e94f22740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x5621bc99e960 con 0x5621bca91100 2024-03-19T01:53:49.758 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.751+0000 7f7e94f22740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x5621bc9f1c80 con 0x5621bca96c60 2024-03-19T01:53:49.758 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.751+0000 7f7e8bfff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5621bca91af0 0x5621bca965f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:49.758 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.751+0000 7f7e90c34640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5621bca91100 0x5621bca91520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:49.758 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.751+0000 7f7e8bfff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5621bca91af0 0x5621bca965f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:35200/0 (socket says 172.21.15.99:35200) 2024-03-19T01:53:49.758 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.751+0000 7f7e8bfff640 1 -- 172.21.15.99:0/1223120246 learned_addr learned my addr 172.21.15.99:0/1223120246 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:53:49.758 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.751+0000 7f7e8b7fe640 1 --2- 172.21.15.99:0/1223120246 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5621bca96c60 0x5621bca990f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:49.758 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.751+0000 7f7e897fa640 1 --2- 172.21.15.99:0/154312159 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f7e740a2920 0x7f7e740a4de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:49.758 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.751+0000 7f7e8bfff640 1 -- 172.21.15.99:0/1223120246 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5621bca96c60 msgr2=0x5621bca990f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:53:49.758 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.751+0000 7f7e8bfff640 1 --2- 172.21.15.99:0/1223120246 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5621bca96c60 0x5621bca990f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:49.759 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.751+0000 7f7e8bfff640 1 -- 172.21.15.99:0/1223120246 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5621bca91100 msgr2=0x5621bca91520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:53:49.759 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.751+0000 7f7e8bfff640 1 --2- 172.21.15.99:0/1223120246 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5621bca91100 0x5621bca91520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:49.759 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.751+0000 7f7e8bfff640 1 -- 172.21.15.99:0/1223120246 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5621bc995d20 con 0x5621bca91af0 2024-03-19T01:53:49.759 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.751+0000 7f7e90c34640 1 --2- 172.21.15.99:0/154312159 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f7e740a2920 0x7f7e740a4de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:49.759 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.751+0000 7f7e897fa640 1 -- 172.21.15.99:0/154312159 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f7e800ced50 con 0x7f7e8c101970 2024-03-19T01:53:49.759 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.751+0000 7f7e8bfff640 1 --2- 172.21.15.99:0/1223120246 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5621bca91af0 0x5621bca965f0 secure :-1 s=READY pgs=592 cs=0 l=1 rev1=1 crypto rx=0x7f7e7c009ca0 tx=0x7f7e7c011810 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5649a9b7a32554a6 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:53:49.759 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.752+0000 7f7e727fc640 1 -- 172.21.15.99:0/1223120246 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7e7c004220 con 0x5621bca91af0 2024-03-19T01:53:49.759 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.752+0000 7f7e727fc640 1 -- 172.21.15.99:0/1223120246 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7e7c0043c0 con 0x5621bca91af0 2024-03-19T01:53:49.759 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.752+0000 7f7e727fc640 1 -- 172.21.15.99:0/1223120246 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7e7c019930 con 0x5621bca91af0 2024-03-19T01:53:49.759 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.752+0000 7f7e94f22740 1 -- 172.21.15.99:0/1223120246 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5621bca91af0 msgr2=0x5621bca965f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:53:49.759 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.752+0000 7f7e90c34640 1 --2- 172.21.15.99:0/154312159 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f7e740a2920 0x7f7e740a4de0 secure :-1 s=READY pgs=330 cs=0 l=1 rev1=1 crypto rx=0x7f7e8402c630 tx=0x7f7e8402e040 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:53:49.759 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.752+0000 7f7e94f22740 1 --2- 172.21.15.99:0/1223120246 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5621bca91af0 0x5621bca965f0 secure :-1 s=READY pgs=592 cs=0 l=1 rev1=1 crypto rx=0x7f7e7c009ca0 tx=0x7f7e7c011810 comp rx=0 tx=0).stop 2024-03-19T01:53:49.759 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.752+0000 7f7e94f22740 1 -- 172.21.15.99:0/1223120246 shutdown_connections 2024-03-19T01:53:49.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.752+0000 7f7e94f22740 1 --2- 172.21.15.99:0/1223120246 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5621bca96c60 0x5621bca990f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:49.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.752+0000 7f7e94f22740 1 --2- 172.21.15.99:0/1223120246 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5621bca91af0 0x5621bca965f0 unknown :-1 s=CLOSED pgs=592 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:49.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.752+0000 7f7e94f22740 1 --2- 172.21.15.99:0/1223120246 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5621bca91100 0x5621bca91520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:49.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.752+0000 7f7e94f22740 1 -- 172.21.15.99:0/1223120246 >> 172.21.15.99:0/1223120246 conn(0x5621bca8c670 msgr2=0x5621bca8ca70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:53:49.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.752+0000 7f7e94f22740 1 -- 172.21.15.99:0/1223120246 shutdown_connections 2024-03-19T01:53:49.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.752+0000 7f7e94f22740 1 -- 172.21.15.99:0/1223120246 wait complete. 2024-03-19T01:53:49.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.753+0000 7f7e94f22740 1 -- start start 2024-03-19T01:53:49.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.753+0000 7f7e94f22740 1 Processor -- start 2024-03-19T01:53:49.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.753+0000 7f7e94f22740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5621bca91100 0x5621bca9b080 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:49.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.753+0000 7f7e94f22740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5621bca91af0 0x5621bca9b5c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:49.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.753+0000 7f7e94f22740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5621bca96c60 0x5621bcaa2600 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:49.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.753+0000 7f7e94f22740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x5621bca99c10 con 0x5621bca96c60 2024-03-19T01:53:49.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.753+0000 7f7e94f22740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x5621bc9b1db0 con 0x5621bca91100 2024-03-19T01:53:49.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.753+0000 7f7e94f22740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x5621bca99d90 con 0x5621bca91af0 2024-03-19T01:53:49.761 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.753+0000 7f7e94f22740 10 client.0 _ll_register_callbacks cb 0x5621bc9b6e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:53:49.761 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.753+0000 7f7e90c34640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5621bca96c60 0x5621bcaa2600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:49.761 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.753+0000 7f7e8bfff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5621bca91100 0x5621bca9b080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:49.761 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.753+0000 7f7e90c34640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5621bca96c60 0x5621bcaa2600 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:35210/0 (socket says 172.21.15.99:35210) 2024-03-19T01:53:49.761 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.753+0000 7f7e90c34640 1 -- 172.21.15.99:0/1058289102 learned_addr learned my addr 172.21.15.99:0/1058289102 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:53:49.761 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.753+0000 7f7e8bfff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5621bca91100 0x5621bca9b080 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:50982/0 (socket says 172.21.15.99:50982) 2024-03-19T01:53:49.761 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.753+0000 7f7e8b7fe640 1 --2- 172.21.15.99:0/1058289102 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5621bca91af0 0x5621bca9b5c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:49.761 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.754+0000 7f7e90c34640 1 -- 172.21.15.99:0/1058289102 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5621bca91af0 msgr2=0x5621bca9b5c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:53:49.761 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.754+0000 7f7e90c34640 1 --2- 172.21.15.99:0/1058289102 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5621bca91af0 0x5621bca9b5c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:49.761 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.754+0000 7f7e90c34640 1 -- 172.21.15.99:0/1058289102 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5621bca91100 msgr2=0x5621bca9b080 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:53:49.761 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.754+0000 7f7e90c34640 1 --2- 172.21.15.99:0/1058289102 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5621bca91100 0x5621bca9b080 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:49.761 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.754+0000 7f7e90c34640 1 -- 172.21.15.99:0/1058289102 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5621bc9b0c60 con 0x5621bca96c60 2024-03-19T01:53:49.761 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.754+0000 7f7e8bfff640 1 --2- 172.21.15.99:0/1058289102 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5621bca91100 0x5621bca9b080 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:53:49.761 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.754+0000 7f7e90c34640 1 --2- 172.21.15.99:0/1058289102 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5621bca96c60 0x5621bcaa2600 secure :-1 s=READY pgs=593 cs=0 l=1 rev1=1 crypto rx=0x7f7e84009ca0 tx=0x7f7e84032590 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d1e1941199fd2955 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:53:49.762 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.754+0000 7f7e6b7fe640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:53:49.762 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.754+0000 7f7e6b7fe640 1 -- 172.21.15.99:0/1058289102 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7e84039040 con 0x5621bca96c60 2024-03-19T01:53:49.762 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.754+0000 7f7e6b7fe640 1 -- 172.21.15.99:0/1058289102 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f7e84041730 con 0x5621bca96c60 2024-03-19T01:53:49.762 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.754+0000 7f7e94f22740 1 -- 172.21.15.99:0/1058289102 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f7e7c0043c0 con 0x5621bca96c60 2024-03-19T01:53:49.762 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.754+0000 7f7e6b7fe640 1 -- 172.21.15.99:0/1058289102 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=33) ==== 24+0+0 (secure 0 0 0) 0x7f7e84041930 con 0x5621bca96c60 2024-03-19T01:53:49.762 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.754+0000 7f7e94f22740 10 client.16374 fetch_fsmap learned FSMap version 33 2024-03-19T01:53:49.762 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.754+0000 7f7e94f22740 1 -- 172.21.15.99:0/1058289102 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=33,monmap=4+}) -- 0x5621bc70a210 con 0x5621bca96c60 2024-03-19T01:53:49.762 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.754+0000 7f7e6b7fe640 1 -- 172.21.15.99:0/1058289102 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7e7c0043c0 con 0x5621bca96c60 2024-03-19T01:53:49.762 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.755+0000 7f7e6b7fe640 1 -- 172.21.15.99:0/1058289102 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 33) ==== 8362+0+0 (secure 0 0 0) 0x7f7e8403f040 con 0x5621bca96c60 2024-03-19T01:53:49.762 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.755+0000 7f7e94f22740 10 client.16374 fetch_fsmap finished waiting for FSMap version 33 2024-03-19T01:53:49.762 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.755+0000 7f7e94f22740 10 client.16374 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:53:49.762 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.755+0000 7f7e94f22740 20 client.16374 populate_metadata read hostname 'smithi099' 2024-03-19T01:53:49.762 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.755+0000 7f7e94f22740 1 --2- 172.21.15.99:0/1058289102 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5621bcaa7010 0x5621bcaa94a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:49.762 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.755+0000 7f7e94f22740 4 client.16374 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:53:49.763 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.755+0000 7f7e94f22740 1 -- 172.21.15.99:0/1058289102 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5621bca03120 con 0x5621bcaa7010 2024-03-19T01:53:49.763 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.755+0000 7f7e8bfff640 1 --2- 172.21.15.99:0/1058289102 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5621bcaa7010 0x5621bcaa94a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:49.763 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.755+0000 7f7e6b7fe640 10 client.16374 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:53:49.763 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.755+0000 7f7e8bfff640 1 --2- 172.21.15.99:0/1058289102 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5621bcaa7010 0x5621bcaa94a0 crc :-1 s=READY pgs=143 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=fb524bf542ac6965 server_cookie=cecce2449f05d9ea in_seq=0 out_seq=0 2024-03-19T01:53:49.764 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.762+0000 7f7e6b7fe640 1 -- 172.21.15.99:0/1058289102 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x5621bca03120 con 0x5621bcaa7010 2024-03-19T01:53:49.764 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.762+0000 7f7e6b7fe640 10 client.16374 handle_command_reply: tid=0 2024-03-19T01:53:49.764 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.762+0000 7f7e94f22740 1 client.16374 shutdown 2024-03-19T01:53:49.764 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.762+0000 7f7e94f22740 1 -- 172.21.15.99:0/1058289102 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5621bca96c60 msgr2=0x5621bcaa2600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:53:49.764 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.762+0000 7f7e94f22740 1 --2- 172.21.15.99:0/1058289102 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5621bca96c60 0x5621bcaa2600 secure :-1 s=READY pgs=593 cs=0 l=1 rev1=1 crypto rx=0x7f7e84009ca0 tx=0x7f7e84032590 comp rx=0 tx=0).stop 2024-03-19T01:53:49.764 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.762+0000 7f7e94f22740 1 -- 172.21.15.99:0/1058289102 shutdown_connections 2024-03-19T01:53:49.764 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.762+0000 7f7e94f22740 1 --2- 172.21.15.99:0/1058289102 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5621bcaa7010 0x5621bcaa94a0 crc :-1 s=READY pgs=143 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:49.764 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.762+0000 7f7e94f22740 1 --2- 172.21.15.99:0/1058289102 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5621bca96c60 0x5621bcaa2600 unknown :-1 s=CLOSED pgs=593 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:49.764 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.762+0000 7f7e6b7fe640 0 client.16374 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:53:49.765 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.762+0000 7f7e94f22740 1 --2- 172.21.15.99:0/1058289102 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5621bca91af0 0x5621bca9b5c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:49.765 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.762+0000 7f7e94f22740 1 --2- 172.21.15.99:0/1058289102 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5621bca91100 0x5621bca9b080 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:49.765 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.762+0000 7f7e94f22740 1 -- 172.21.15.99:0/1058289102 >> 172.21.15.99:0/1058289102 conn(0x5621bca8c670 msgr2=0x5621bca8d8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:53:49.765 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.762+0000 7f7e94f22740 1 -- 172.21.15.99:0/1058289102 shutdown_connections 2024-03-19T01:53:49.765 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.762+0000 7f7e94f22740 1 -- 172.21.15.99:0/1058289102 wait complete. 2024-03-19T01:53:49.765 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.762+0000 7f7e94f22740 20 client.16374 trim_cache size 0 max 16384 2024-03-19T01:53:49.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.777+0000 7f7e94f22740 1 Processor -- start 2024-03-19T01:53:49.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.777+0000 7f7e94f22740 1 -- start start 2024-03-19T01:53:49.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.778+0000 7f7e94f22740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5621bca91100 0x5621bca92590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:49.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.778+0000 7f7e94f22740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5621bca91af0 0x5621bcaa25c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:49.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.778+0000 7f7e94f22740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5621bca96c60 0x5621bca92ad0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:49.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.778+0000 7f7e94f22740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f7e8c1ab6c0 con 0x5621bca91af0 2024-03-19T01:53:49.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.778+0000 7f7e94f22740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f7e8c1a8ed0 con 0x5621bca96c60 2024-03-19T01:53:49.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.778+0000 7f7e94f22740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f7e8c1a6d20 con 0x5621bca91100 2024-03-19T01:53:49.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.778+0000 7f7e8b7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5621bca91af0 0x5621bcaa25c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:49.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.778+0000 7f7e8b7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5621bca91af0 0x5621bcaa25c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:35224/0 (socket says 172.21.15.99:35224) 2024-03-19T01:53:49.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.778+0000 7f7e8b7fe640 1 -- 172.21.15.99:0/673725481 learned_addr learned my addr 172.21.15.99:0/673725481 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:53:49.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.778+0000 7f7e90c34640 1 --2- 172.21.15.99:0/673725481 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5621bca96c60 0x5621bca92ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:49.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.778+0000 7f7e8bfff640 1 --2- 172.21.15.99:0/673725481 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5621bca91100 0x5621bca92590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:49.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.778+0000 7f7e8b7fe640 1 -- 172.21.15.99:0/673725481 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5621bca91100 msgr2=0x5621bca92590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:53:49.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.778+0000 7f7e8b7fe640 1 --2- 172.21.15.99:0/673725481 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5621bca91100 0x5621bca92590 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:49.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.778+0000 7f7e8b7fe640 1 -- 172.21.15.99:0/673725481 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5621bca96c60 msgr2=0x5621bca92ad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:53:49.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.778+0000 7f7e8b7fe640 1 --2- 172.21.15.99:0/673725481 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5621bca96c60 0x5621bca92ad0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:49.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.778+0000 7f7e8b7fe640 1 -- 172.21.15.99:0/673725481 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5621bc7d6620 con 0x5621bca91af0 2024-03-19T01:53:49.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.778+0000 7f7e8bfff640 1 --2- 172.21.15.99:0/673725481 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5621bca91100 0x5621bca92590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-19T01:53:49.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.778+0000 7f7e8b7fe640 1 --2- 172.21.15.99:0/673725481 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5621bca91af0 0x5621bcaa25c0 secure :-1 s=READY pgs=594 cs=0 l=1 rev1=1 crypto rx=0x7f7e8009b460 tx=0x7f7e800d2830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1ca0993ef4d7e83a server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:53:49.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.778+0000 7f7e71ffb640 1 -- 172.21.15.99:0/673725481 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7e8009b2a0 con 0x5621bca91af0 2024-03-19T01:53:49.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.778+0000 7f7e71ffb640 1 -- 172.21.15.99:0/673725481 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7e8009b760 con 0x5621bca91af0 2024-03-19T01:53:49.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.778+0000 7f7e71ffb640 1 -- 172.21.15.99:0/673725481 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7e8009b0c0 con 0x5621bca91af0 2024-03-19T01:53:49.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.779+0000 7f7e94f22740 1 -- 172.21.15.99:0/673725481 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5621bca91af0 msgr2=0x5621bcaa25c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:53:49.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.779+0000 7f7e94f22740 1 --2- 172.21.15.99:0/673725481 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5621bca91af0 0x5621bcaa25c0 secure :-1 s=READY pgs=594 cs=0 l=1 rev1=1 crypto rx=0x7f7e8009b460 tx=0x7f7e800d2830 comp rx=0 tx=0).stop 2024-03-19T01:53:49.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.779+0000 7f7e94f22740 1 -- 172.21.15.99:0/673725481 shutdown_connections 2024-03-19T01:53:49.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.779+0000 7f7e94f22740 1 --2- 172.21.15.99:0/673725481 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5621bca96c60 0x5621bca92ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:49.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.779+0000 7f7e94f22740 1 --2- 172.21.15.99:0/673725481 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5621bca91af0 0x5621bcaa25c0 unknown :-1 s=CLOSED pgs=594 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:49.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.779+0000 7f7e94f22740 1 --2- 172.21.15.99:0/673725481 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5621bca91100 0x5621bca92590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:49.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.779+0000 7f7e94f22740 1 -- 172.21.15.99:0/673725481 >> 172.21.15.99:0/673725481 conn(0x5621bca8c670 msgr2=0x5621bca8b220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:53:49.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.779+0000 7f7e94f22740 1 -- 172.21.15.99:0/673725481 shutdown_connections 2024-03-19T01:53:49.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.779+0000 7f7e94f22740 1 -- 172.21.15.99:0/673725481 wait complete. 2024-03-19T01:53:49.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.779+0000 7f7e94f22740 1 -- start start 2024-03-19T01:53:49.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.779+0000 7f7e94f22740 1 Processor -- start 2024-03-19T01:53:49.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.780+0000 7f7e94f22740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5621bca91100 0x5621bcaa0040 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:49.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.780+0000 7f7e94f22740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5621bca91af0 0x5621bcaa0580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:49.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.780+0000 7f7e94f22740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5621bca96c60 0x5621bcaa1590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:49.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.780+0000 7f7e94f22740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f7e8c1a7390 con 0x5621bca91af0 2024-03-19T01:53:49.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.780+0000 7f7e94f22740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f7e8c1a8800 con 0x5621bca91100 2024-03-19T01:53:49.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.780+0000 7f7e94f22740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f7e8c1a7a60 con 0x5621bca96c60 2024-03-19T01:53:49.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.780+0000 7f7e94f22740 10 client.0 _ll_register_callbacks cb 0x7f7e7c005e30 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:53:49.786 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.780+0000 7f7e8b7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5621bca91af0 0x5621bcaa0580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:49.786 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.780+0000 7f7e8b7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5621bca91af0 0x5621bcaa0580 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:35226/0 (socket says 172.21.15.99:35226) 2024-03-19T01:53:49.786 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.780+0000 7f7e8bfff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5621bca91100 0x5621bcaa0040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:49.786 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.780+0000 7f7e8b7fe640 1 -- 172.21.15.99:0/1306479250 learned_addr learned my addr 172.21.15.99:0/1306479250 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:53:49.786 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.780+0000 7f7e90c34640 1 --2- 172.21.15.99:0/1306479250 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5621bca96c60 0x5621bcaa1590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:49.786 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.780+0000 7f7e8b7fe640 1 -- 172.21.15.99:0/1306479250 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5621bca96c60 msgr2=0x5621bcaa1590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:53:49.786 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.780+0000 7f7e8b7fe640 1 --2- 172.21.15.99:0/1306479250 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5621bca96c60 0x5621bcaa1590 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:49.786 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.780+0000 7f7e8b7fe640 1 -- 172.21.15.99:0/1306479250 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5621bca91100 msgr2=0x5621bcaa0040 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:53:49.786 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.780+0000 7f7e8b7fe640 1 --2- 172.21.15.99:0/1306479250 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5621bca91100 0x5621bcaa0040 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:49.786 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.780+0000 7f7e8b7fe640 1 -- 172.21.15.99:0/1306479250 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5621bcaa1b80 con 0x5621bca91af0 2024-03-19T01:53:49.786 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.780+0000 7f7e8bfff640 1 --2- 172.21.15.99:0/1306479250 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5621bca91100 0x5621bcaa0040 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:53:49.786 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.780+0000 7f7e8b7fe640 1 --2- 172.21.15.99:0/1306479250 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5621bca91af0 0x5621bcaa0580 secure :-1 s=READY pgs=595 cs=0 l=1 rev1=1 crypto rx=0x7f7e8000c6a0 tx=0x7f7e800a3ed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6fb97694334bbda0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:53:49.786 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.780+0000 7f7e6b7fe640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:53:49.787 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.780+0000 7f7e6b7fe640 1 -- 172.21.15.99:0/1306479250 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7e800b3020 con 0x5621bca91af0 2024-03-19T01:53:49.787 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.780+0000 7f7e6b7fe640 1 -- 172.21.15.99:0/1306479250 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f7e80018b30 con 0x5621bca91af0 2024-03-19T01:53:49.787 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.780+0000 7f7e94f22740 1 -- 172.21.15.99:0/1306479250 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f7e8009b760 con 0x5621bca91af0 2024-03-19T01:53:49.787 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.780+0000 7f7e6b7fe640 1 -- 172.21.15.99:0/1306479250 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=33) ==== 24+0+0 (secure 0 0 0) 0x7f7e8009b2a0 con 0x5621bca91af0 2024-03-19T01:53:49.787 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.781+0000 7f7e94f22740 10 client.16380 fetch_fsmap learned FSMap version 33 2024-03-19T01:53:49.787 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.781+0000 7f7e94f22740 1 -- 172.21.15.99:0/1306479250 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=33,monmap=4+}) -- 0x5621bcb0d730 con 0x5621bca91af0 2024-03-19T01:53:49.787 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.781+0000 7f7e6b7fe640 1 -- 172.21.15.99:0/1306479250 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7e8009b760 con 0x5621bca91af0 2024-03-19T01:53:49.787 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.781+0000 7f7e6b7fe640 1 -- 172.21.15.99:0/1306479250 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 33) ==== 8362+0+0 (secure 0 0 0) 0x7f7e800b4220 con 0x5621bca91af0 2024-03-19T01:53:49.787 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.781+0000 7f7e94f22740 10 client.16380 fetch_fsmap finished waiting for FSMap version 33 2024-03-19T01:53:49.787 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.781+0000 7f7e94f22740 10 client.16380 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:53:49.787 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.781+0000 7f7e94f22740 20 client.16380 populate_metadata read hostname 'smithi099' 2024-03-19T01:53:49.787 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.781+0000 7f7e94f22740 1 --2- 172.21.15.99:0/1306479250 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5621bcb0e180 0x5621bcb105a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:53:49.787 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.781+0000 7f7e94f22740 4 client.16380 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:53:49.787 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.781+0000 7f7e94f22740 1 -- 172.21.15.99:0/1306479250 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x5621bc9fd470 con 0x5621bcb0e180 2024-03-19T01:53:49.787 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.781+0000 7f7e8bfff640 1 --2- 172.21.15.99:0/1306479250 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5621bcb0e180 0x5621bcb105a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:53:49.788 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.781+0000 7f7e6b7fe640 10 client.16380 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:53:49.788 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:49.781+0000 7f7e8bfff640 1 --2- 172.21.15.99:0/1306479250 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5621bcb0e180 0x5621bcb105a0 crc :-1 s=READY pgs=144 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=dfe8901cbafbf58c server_cookie=3e0c1ef2a3580e30 in_seq=0 out_seq=0 2024-03-19T01:53:50.097 INFO:tasks.workunit.client.0.smithi099.stdout: 10 9365 36.13 MB/sec warmup 19 sec latency 206.564 ms 2024-03-19T01:53:50.190 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:50.188+0000 7f7e6b7fe640 1 -- 172.21.15.99:0/1306479250 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x5621bc9fd470 con 0x5621bcb0e180 2024-03-19T01:53:50.190 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:50.188+0000 7f7e6b7fe640 10 client.16380 handle_command_reply: tid=0 2024-03-19T01:53:50.190 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:50.188+0000 7f7e94f22740 1 client.16380 shutdown 2024-03-19T01:53:50.190 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:50.188+0000 7f7e94f22740 1 -- 172.21.15.99:0/1306479250 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5621bca91af0 msgr2=0x5621bcaa0580 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:53:50.191 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:50.188+0000 7f7e94f22740 1 --2- 172.21.15.99:0/1306479250 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5621bca91af0 0x5621bcaa0580 secure :-1 s=READY pgs=595 cs=0 l=1 rev1=1 crypto rx=0x7f7e8000c6a0 tx=0x7f7e800a3ed0 comp rx=0 tx=0).stop 2024-03-19T01:53:50.191 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:50.189+0000 7f7e94f22740 1 -- 172.21.15.99:0/1306479250 shutdown_connections 2024-03-19T01:53:50.191 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:50.189+0000 7f7e94f22740 1 --2- 172.21.15.99:0/1306479250 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5621bcb0e180 0x5621bcb105a0 crc :-1 s=READY pgs=144 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:50.191 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:50.189+0000 7f7e94f22740 1 --2- 172.21.15.99:0/1306479250 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5621bca96c60 0x5621bcaa1590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:50.191 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:50.189+0000 7f7e94f22740 1 --2- 172.21.15.99:0/1306479250 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5621bca91af0 0x5621bcaa0580 unknown :-1 s=CLOSED pgs=595 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:50.191 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:50.189+0000 7f7e6b7fe640 0 client.16380 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:53:50.191 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:50.189+0000 7f7e94f22740 1 --2- 172.21.15.99:0/1306479250 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5621bca91100 0x5621bcaa0040 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:50.191 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:50.189+0000 7f7e94f22740 1 -- 172.21.15.99:0/1306479250 >> 172.21.15.99:0/1306479250 conn(0x5621bca8c670 msgr2=0x5621bca92590 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:53:50.191 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:50.189+0000 7f7e94f22740 1 -- 172.21.15.99:0/1306479250 shutdown_connections 2024-03-19T01:53:50.191 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:50.189+0000 7f7e94f22740 1 -- 172.21.15.99:0/1306479250 wait complete. 2024-03-19T01:53:50.191 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:50.189+0000 7f7e94f22740 20 client.16380 trim_cache size 0 max 16384 2024-03-19T01:53:50.192 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:53:50.192 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:53:50.192 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:53:50.192 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:53:50.193 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:50.189+0000 7f7e92697640 1 -- 172.21.15.99:0/154312159 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f7e740a2920 msgr2=0x7f7e740a4de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:53:50.193 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:50.189+0000 7f7e92697640 1 --2- 172.21.15.99:0/154312159 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f7e740a2920 0x7f7e740a4de0 secure :-1 s=READY pgs=330 cs=0 l=1 rev1=1 crypto rx=0x7f7e8402c630 tx=0x7f7e8402e040 comp rx=0 tx=0).stop 2024-03-19T01:53:50.193 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:50.189+0000 7f7e92697640 1 -- 172.21.15.99:0/154312159 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7e8c101970 msgr2=0x7f7e8c197f90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:53:50.193 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:50.189+0000 7f7e92697640 1 --2- 172.21.15.99:0/154312159 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7e8c101970 0x7f7e8c197f90 secure :-1 s=READY pgs=591 cs=0 l=1 rev1=1 crypto rx=0x7f7e8000e9f0 tx=0x7f7e8000eeb0 comp rx=0 tx=0).stop 2024-03-19T01:53:50.193 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:50.189+0000 7f7e92697640 1 -- 172.21.15.99:0/154312159 shutdown_connections 2024-03-19T01:53:50.193 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:50.189+0000 7f7e92697640 1 --2- 172.21.15.99:0/154312159 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f7e740a2920 0x7f7e740a4de0 unknown :-1 s=CLOSED pgs=330 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:50.193 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:50.190+0000 7f7e92697640 1 --2- 172.21.15.99:0/154312159 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f7e8c102330 0x7f7e8c19f000 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:50.193 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:50.190+0000 7f7e92697640 1 --2- 172.21.15.99:0/154312159 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7e8c101970 0x7f7e8c197f90 unknown :-1 s=CLOSED pgs=591 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:50.193 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:50.190+0000 7f7e92697640 1 --2- 172.21.15.99:0/154312159 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f7e8c100780 0x7f7e8c197a50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:53:50.194 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:50.190+0000 7f7e92697640 1 -- 172.21.15.99:0/154312159 >> 172.21.15.99:0/154312159 conn(0x7f7e8c0fbcf0 msgr2=0x7f7e8c0fd690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:53:50.194 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:50.190+0000 7f7e92697640 1 -- 172.21.15.99:0/154312159 shutdown_connections 2024-03-19T01:53:50.194 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:53:50.190+0000 7f7e92697640 1 -- 172.21.15.99:0/154312159 wait complete. 2024-03-19T01:53:50.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:53:50 smithi115 ceph-mon[67332]: pgmap v733: 97 pgs: 97 active+clean; 258 MiB data, 3.8 GiB used, 1.0 TiB / 1.0 TiB avail; 1.3 MiB/s rd, 4.3 MiB/s wr, 452 op/s 2024-03-19T01:53:50.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:53:50 smithi099 ceph-mon[62951]: pgmap v733: 97 pgs: 97 active+clean; 258 MiB data, 3.8 GiB used, 1.0 TiB / 1.0 TiB avail; 1.3 MiB/s rd, 4.3 MiB/s wr, 452 op/s 2024-03-19T01:53:50.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:53:50 smithi112 ceph-mon[66819]: pgmap v733: 97 pgs: 97 active+clean; 258 MiB data, 3.8 GiB used, 1.0 TiB / 1.0 TiB avail; 1.3 MiB/s rd, 4.3 MiB/s wr, 452 op/s 2024-03-19T01:53:51.097 INFO:tasks.workunit.client.0.smithi099.stdout: 10 9768 35.33 MB/sec warmup 20 sec latency 251.941 ms 2024-03-19T01:53:51.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:53:51 smithi099 ceph-mon[62951]: pgmap v734: 97 pgs: 97 active+clean; 259 MiB data, 3.8 GiB used, 1.0 TiB / 1.0 TiB avail; 1.3 MiB/s rd, 4.3 MiB/s wr, 444 op/s 2024-03-19T01:53:51.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:53:51 smithi112 ceph-mon[66819]: pgmap v734: 97 pgs: 97 active+clean; 259 MiB data, 3.8 GiB used, 1.0 TiB / 1.0 TiB avail; 1.3 MiB/s rd, 4.3 MiB/s wr, 444 op/s 2024-03-19T01:53:51.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:53:51 smithi115 ceph-mon[67332]: pgmap v734: 97 pgs: 97 active+clean; 259 MiB data, 3.8 GiB used, 1.0 TiB / 1.0 TiB avail; 1.3 MiB/s rd, 4.3 MiB/s wr, 444 op/s 2024-03-19T01:53:52.097 INFO:tasks.workunit.client.0.smithi099.stdout: 10 10108 34.52 MB/sec warmup 21 sec latency 129.975 ms 2024-03-19T01:53:53.097 INFO:tasks.workunit.client.0.smithi099.stdout: 10 10483 33.59 MB/sec warmup 22 sec latency 95.821 ms 2024-03-19T01:53:54.097 INFO:tasks.workunit.client.0.smithi099.stdout: 10 10901 32.94 MB/sec warmup 23 sec latency 93.113 ms 2024-03-19T01:53:54.258 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:53:54 smithi115 ceph-mon[67332]: pgmap v735: 97 pgs: 97 active+clean; 254 MiB data, 3.8 GiB used, 1.0 TiB / 1.0 TiB avail; 1.4 MiB/s rd, 4.6 MiB/s wr, 521 op/s 2024-03-19T01:53:54.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:53:54 smithi099 ceph-mon[62951]: pgmap v735: 97 pgs: 97 active+clean; 254 MiB data, 3.8 GiB used, 1.0 TiB / 1.0 TiB avail; 1.4 MiB/s rd, 4.6 MiB/s wr, 521 op/s 2024-03-19T01:53:54.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:53:54 smithi112 ceph-mon[66819]: pgmap v735: 97 pgs: 97 active+clean; 254 MiB data, 3.8 GiB used, 1.0 TiB / 1.0 TiB avail; 1.4 MiB/s rd, 4.6 MiB/s wr, 521 op/s 2024-03-19T01:53:55.097 INFO:tasks.workunit.client.0.smithi099.stdout: 10 11548 32.91 MB/sec warmup 24 sec latency 68.251 ms 2024-03-19T01:53:56.098 INFO:tasks.workunit.client.0.smithi099.stdout: 10 12235 32.67 MB/sec warmup 25 sec latency 54.578 ms 2024-03-19T01:53:56.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:53:56 smithi115 ceph-mon[67332]: pgmap v736: 97 pgs: 97 active+clean; 258 MiB data, 3.8 GiB used, 1.0 TiB / 1.0 TiB avail; 1.3 MiB/s rd, 4.1 MiB/s wr, 484 op/s 2024-03-19T01:53:56.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:53:56 smithi099 ceph-mon[62951]: pgmap v736: 97 pgs: 97 active+clean; 258 MiB data, 3.8 GiB used, 1.0 TiB / 1.0 TiB avail; 1.3 MiB/s rd, 4.1 MiB/s wr, 484 op/s 2024-03-19T01:53:56.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:53:56 smithi112 ceph-mon[66819]: pgmap v736: 97 pgs: 97 active+clean; 258 MiB data, 3.8 GiB used, 1.0 TiB / 1.0 TiB avail; 1.3 MiB/s rd, 4.1 MiB/s wr, 484 op/s 2024-03-19T01:53:57.097 INFO:tasks.workunit.client.0.smithi099.stdout: 10 12825 33.13 MB/sec warmup 26 sec latency 178.334 ms 2024-03-19T01:53:58.097 INFO:tasks.workunit.client.0.smithi099.stdout: 10 13391 33.48 MB/sec warmup 27 sec latency 281.583 ms 2024-03-19T01:53:58.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:53:58 smithi115 ceph-mon[67332]: pgmap v737: 97 pgs: 97 active+clean; 263 MiB data, 3.9 GiB used, 1.0 TiB / 1.0 TiB avail; 1.6 MiB/s rd, 4.8 MiB/s wr, 560 op/s 2024-03-19T01:53:58.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:53:58 smithi099 ceph-mon[62951]: pgmap v737: 97 pgs: 97 active+clean; 263 MiB data, 3.9 GiB used, 1.0 TiB / 1.0 TiB avail; 1.6 MiB/s rd, 4.8 MiB/s wr, 560 op/s 2024-03-19T01:53:58.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:53:58 smithi112 ceph-mon[66819]: pgmap v737: 97 pgs: 97 active+clean; 263 MiB data, 3.9 GiB used, 1.0 TiB / 1.0 TiB avail; 1.6 MiB/s rd, 4.8 MiB/s wr, 560 op/s 2024-03-19T01:53:59.097 INFO:tasks.workunit.client.0.smithi099.stdout: 10 13754 33.16 MB/sec warmup 28 sec latency 177.286 ms 2024-03-19T01:53:59.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:53:59 smithi099 ceph-mon[62951]: pgmap v738: 97 pgs: 97 active+clean; 268 MiB data, 3.9 GiB used, 1.0 TiB / 1.0 TiB avail; 1.6 MiB/s rd, 4.6 MiB/s wr, 529 op/s 2024-03-19T01:53:59.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:53:59 smithi112 ceph-mon[66819]: pgmap v738: 97 pgs: 97 active+clean; 268 MiB data, 3.9 GiB used, 1.0 TiB / 1.0 TiB avail; 1.6 MiB/s rd, 4.6 MiB/s wr, 529 op/s 2024-03-19T01:53:59.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:53:59 smithi115 ceph-mon[67332]: pgmap v738: 97 pgs: 97 active+clean; 268 MiB data, 3.9 GiB used, 1.0 TiB / 1.0 TiB avail; 1.6 MiB/s rd, 4.6 MiB/s wr, 529 op/s 2024-03-19T01:54:00.097 INFO:tasks.workunit.client.0.smithi099.stdout: 10 14125 32.60 MB/sec warmup 29 sec latency 64.177 ms 2024-03-19T01:54:01.097 INFO:tasks.workunit.client.0.smithi099.stdout: 10 14546 32.04 MB/sec warmup 30 sec latency 106.281 ms 2024-03-19T01:54:02.097 INFO:tasks.workunit.client.0.smithi099.stdout: 10 14938 31.57 MB/sec warmup 31 sec latency 83.843 ms 2024-03-19T01:54:02.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:54:02 smithi115 ceph-mon[67332]: pgmap v739: 97 pgs: 97 active+clean; 268 MiB data, 3.9 GiB used, 1.0 TiB / 1.0 TiB avail; 1.2 MiB/s rd, 4.1 MiB/s wr, 478 op/s 2024-03-19T01:54:02.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:54:02 smithi099 ceph-mon[62951]: pgmap v739: 97 pgs: 97 active+clean; 268 MiB data, 3.9 GiB used, 1.0 TiB / 1.0 TiB avail; 1.2 MiB/s rd, 4.1 MiB/s wr, 478 op/s 2024-03-19T01:54:02.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:54:02 smithi112 ceph-mon[66819]: pgmap v739: 97 pgs: 97 active+clean; 268 MiB data, 3.9 GiB used, 1.0 TiB / 1.0 TiB avail; 1.2 MiB/s rd, 4.1 MiB/s wr, 478 op/s 2024-03-19T01:54:03.097 INFO:tasks.workunit.client.0.smithi099.stdout: 10 15362 31.30 MB/sec warmup 32 sec latency 86.550 ms 2024-03-19T01:54:03.285 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.283+0000 7fc028bd3640 1 -- 172.21.15.99:0/3894624043 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc0241022f0 msgr2=0x7fc024108b80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:54:03.285 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.283+0000 7fc028bd3640 1 --2- 172.21.15.99:0/3894624043 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc0241022f0 0x7fc024108b80 secure :-1 s=READY pgs=596 cs=0 l=1 rev1=1 crypto rx=0x7fc01c009ca0 tx=0x7fc01c02c0f0 comp rx=0 tx=0).stop 2024-03-19T01:54:03.286 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.284+0000 7fc028bd3640 1 -- 172.21.15.99:0/3894624043 shutdown_connections 2024-03-19T01:54:03.286 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.284+0000 7fc028bd3640 1 --2- 172.21.15.99:0/3894624043 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fc024101930 0x7fc024101db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:03.286 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.284+0000 7fc028bd3640 1 --2- 172.21.15.99:0/3894624043 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc0241022f0 0x7fc024108b80 unknown :-1 s=CLOSED pgs=596 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:03.286 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.284+0000 7fc028bd3640 1 --2- 172.21.15.99:0/3894624043 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fc024100740 0x7fc024100b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:03.286 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.284+0000 7fc028bd3640 1 -- 172.21.15.99:0/3894624043 >> 172.21.15.99:0/3894624043 conn(0x7fc0240fbcb0 msgr2=0x7fc0240fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:54:03.286 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.284+0000 7fc028bd3640 1 -- 172.21.15.99:0/3894624043 shutdown_connections 2024-03-19T01:54:03.286 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.284+0000 7fc028bd3640 1 -- 172.21.15.99:0/3894624043 wait complete. 2024-03-19T01:54:03.286 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.284+0000 7fc028bd3640 1 Processor -- start 2024-03-19T01:54:03.286 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.284+0000 7fc028bd3640 1 -- start start 2024-03-19T01:54:03.286 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.285+0000 7fc028bd3640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fc024100740 0x7fc024069a30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:03.286 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.285+0000 7fc028bd3640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fc024101930 0x7fc02419c770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:03.286 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.285+0000 7fc028bd3640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc0241022f0 0x7fc02419edb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:03.287 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.285+0000 7fc028bd3640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fc02410c3c0 con 0x7fc0241022f0 2024-03-19T01:54:03.287 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.285+0000 7fc028bd3640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fc02410c240 con 0x7fc024100740 2024-03-19T01:54:03.287 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.285+0000 7fc028bd3640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fc02410c540 con 0x7fc024101930 2024-03-19T01:54:03.287 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.285+0000 7fc022d9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc0241022f0 0x7fc02419edb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:03.287 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.285+0000 7fc022d9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc0241022f0 0x7fc02419edb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:39458/0 (socket says 172.21.15.99:39458) 2024-03-19T01:54:03.287 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.285+0000 7fc02259c640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fc024100740 0x7fc024069a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:03.287 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.285+0000 7fc022d9d640 1 -- 172.21.15.99:0/3377394738 learned_addr learned my addr 172.21.15.99:0/3377394738 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:54:03.287 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.285+0000 7fc021d9b640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fc024101930 0x7fc02419c770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:03.287 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.285+0000 7fc022d9d640 1 -- 172.21.15.99:0/3377394738 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fc024101930 msgr2=0x7fc02419c770 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:54:03.287 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.285+0000 7fc022d9d640 1 --2- 172.21.15.99:0/3377394738 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fc024101930 0x7fc02419c770 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:03.287 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.285+0000 7fc022d9d640 1 -- 172.21.15.99:0/3377394738 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fc024100740 msgr2=0x7fc024069a30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:54:03.287 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.285+0000 7fc022d9d640 1 --2- 172.21.15.99:0/3377394738 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fc024100740 0x7fc024069a30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:03.287 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.285+0000 7fc022d9d640 1 -- 172.21.15.99:0/3377394738 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fc02419f670 con 0x7fc0241022f0 2024-03-19T01:54:03.287 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.285+0000 7fc022d9d640 1 --2- 172.21.15.99:0/3377394738 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc0241022f0 0x7fc02419edb0 secure :-1 s=READY pgs=597 cs=0 l=1 rev1=1 crypto rx=0x7fc01c037040 tx=0x7fc01c032f40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:54:03.288 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.285+0000 7fc0137fe640 1 -- 172.21.15.99:0/3377394738 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc01c041020 con 0x7fc0241022f0 2024-03-19T01:54:03.288 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.285+0000 7fc0137fe640 1 -- 172.21.15.99:0/3377394738 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fc01c034cd0 con 0x7fc0241022f0 2024-03-19T01:54:03.288 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.285+0000 7fc0137fe640 1 -- 172.21.15.99:0/3377394738 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc01c039040 con 0x7fc0241022f0 2024-03-19T01:54:03.288 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.285+0000 7fc028bd3640 1 -- 172.21.15.99:0/3377394738 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fc02419f900 con 0x7fc0241022f0 2024-03-19T01:54:03.288 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.285+0000 7fc028bd3640 1 -- 172.21.15.99:0/3377394738 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fc02419fe40 con 0x7fc0241022f0 2024-03-19T01:54:03.295 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.288+0000 7fc0137fe640 1 -- 172.21.15.99:0/3377394738 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7fc01c044020 con 0x7fc0241022f0 2024-03-19T01:54:03.295 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.288+0000 7fc02b7b6740 1 Processor -- start 2024-03-19T01:54:03.295 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.288+0000 7fc02b7b6740 1 -- start start 2024-03-19T01:54:03.295 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.288+0000 7fc02b7b6740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557992d1f100 0x557992d1f520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:03.295 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.288+0000 7fc02b7b6740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557992d1faf0 0x557992d245f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:03.295 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.288+0000 7fc02b7b6740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557992d24c60 0x557992d270f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:03.296 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.288+0000 7fc02b7b6740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x557992c83be0 con 0x557992d1f100 2024-03-19T01:54:03.296 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.288+0000 7fc02b7b6740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x557992c2c960 con 0x557992d24c60 2024-03-19T01:54:03.296 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.288+0000 7fc02b7b6740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x557992c7fc80 con 0x557992d1faf0 2024-03-19T01:54:03.296 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.288+0000 7fc0137fe640 1 --2- 172.21.15.99:0/3377394738 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fc00c0a2920 0x7fc00c0a4de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:03.296 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.288+0000 7fc021d9b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557992d1f100 0x557992d1f520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:03.296 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.288+0000 7fc021d9b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557992d1f100 0x557992d1f520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:39466/0 (socket says 172.21.15.99:39466) 2024-03-19T01:54:03.296 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.288+0000 7fc02259c640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557992d1faf0 0x557992d245f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:03.296 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.288+0000 7fc021d9b640 1 -- 172.21.15.99:0/2232270627 learned_addr learned my addr 172.21.15.99:0/2232270627 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:54:03.296 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.288+0000 7fc022d9d640 1 --2- 172.21.15.99:0/3377394738 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fc00c0a2920 0x7fc00c0a4de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:03.296 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.288+0000 7fc021d9b640 1 --2- 172.21.15.99:0/2232270627 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557992d24c60 0x557992d270f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:03.296 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.288+0000 7fc0137fe640 1 -- 172.21.15.99:0/3377394738 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fc01c0f0890 con 0x7fc0241022f0 2024-03-19T01:54:03.296 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.289+0000 7fc02259c640 1 -- 172.21.15.99:0/2232270627 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557992d24c60 msgr2=0x557992d270f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:54:03.296 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.289+0000 7fc02259c640 1 --2- 172.21.15.99:0/2232270627 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557992d24c60 0x557992d270f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:03.297 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.289+0000 7fc02259c640 1 -- 172.21.15.99:0/2232270627 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557992d1f100 msgr2=0x557992d1f520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:54:03.297 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.289+0000 7fc02259c640 1 --2- 172.21.15.99:0/2232270627 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557992d1f100 0x557992d1f520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:03.297 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.289+0000 7fc02259c640 1 -- 172.21.15.99:0/2232270627 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x557992c23d20 con 0x557992d1faf0 2024-03-19T01:54:03.297 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.289+0000 7fc021d9b640 1 --2- 172.21.15.99:0/2232270627 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557992d24c60 0x557992d270f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-19T01:54:03.297 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.289+0000 7fc022d9d640 1 --2- 172.21.15.99:0/3377394738 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fc00c0a2920 0x7fc00c0a4de0 secure :-1 s=READY pgs=331 cs=0 l=1 rev1=1 crypto rx=0x7fc01c0ee350 tx=0x7fc01c088040 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:54:03.297 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.289+0000 7fc02259c640 1 --2- 172.21.15.99:0/2232270627 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557992d1faf0 0x557992d245f0 secure :-1 s=READY pgs=112 cs=0 l=1 rev1=1 crypto rx=0x7fc014009bd0 tx=0x7fc014011810 comp rx=0 tx=0).ready entity=mon.1 client_cookie=8e642881e9b0992e server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:54:03.297 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.289+0000 7fc010ff9640 1 -- 172.21.15.99:0/2232270627 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc014004230 con 0x557992d1faf0 2024-03-19T01:54:03.297 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.289+0000 7fc010ff9640 1 -- 172.21.15.99:0/2232270627 <== mon.1 v2:172.21.15.115:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fc014009e40 con 0x557992d1faf0 2024-03-19T01:54:03.297 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.289+0000 7fc010ff9640 1 -- 172.21.15.99:0/2232270627 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc0140197e0 con 0x557992d1faf0 2024-03-19T01:54:03.297 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.290+0000 7fc02b7b6740 1 -- 172.21.15.99:0/2232270627 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557992d1faf0 msgr2=0x557992d245f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:54:03.297 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.290+0000 7fc02b7b6740 1 --2- 172.21.15.99:0/2232270627 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557992d1faf0 0x557992d245f0 secure :-1 s=READY pgs=112 cs=0 l=1 rev1=1 crypto rx=0x7fc014009bd0 tx=0x7fc014011810 comp rx=0 tx=0).stop 2024-03-19T01:54:03.297 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.290+0000 7fc02b7b6740 1 -- 172.21.15.99:0/2232270627 shutdown_connections 2024-03-19T01:54:03.297 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.290+0000 7fc02b7b6740 1 --2- 172.21.15.99:0/2232270627 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557992d24c60 0x557992d270f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:03.297 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.290+0000 7fc02b7b6740 1 --2- 172.21.15.99:0/2232270627 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557992d1faf0 0x557992d245f0 unknown :-1 s=CLOSED pgs=112 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:03.298 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.290+0000 7fc02b7b6740 1 --2- 172.21.15.99:0/2232270627 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557992d1f100 0x557992d1f520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:03.298 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.290+0000 7fc02b7b6740 1 -- 172.21.15.99:0/2232270627 >> 172.21.15.99:0/2232270627 conn(0x557992d1a670 msgr2=0x557992d1aa70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:54:03.298 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.290+0000 7fc02b7b6740 1 -- 172.21.15.99:0/2232270627 shutdown_connections 2024-03-19T01:54:03.298 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.290+0000 7fc02b7b6740 1 -- 172.21.15.99:0/2232270627 wait complete. 2024-03-19T01:54:03.298 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.290+0000 7fc02b7b6740 1 -- start start 2024-03-19T01:54:03.298 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.290+0000 7fc02b7b6740 1 Processor -- start 2024-03-19T01:54:03.298 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.290+0000 7fc02b7b6740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557992d1f100 0x557992d29080 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:03.298 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.290+0000 7fc02b7b6740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557992d1faf0 0x557992d295c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:03.298 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.290+0000 7fc02b7b6740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557992d24c60 0x557992d30600 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:03.298 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.290+0000 7fc02b7b6740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x557992d27c10 con 0x557992d24c60 2024-03-19T01:54:03.298 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.291+0000 7fc02b7b6740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x557992c3fdb0 con 0x557992d1f100 2024-03-19T01:54:03.298 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.291+0000 7fc02b7b6740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x557992d27d90 con 0x557992d1faf0 2024-03-19T01:54:03.298 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.291+0000 7fc02b7b6740 10 client.0 _ll_register_callbacks cb 0x557992c44e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:54:03.298 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.291+0000 7fc021d9b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557992d24c60 0x557992d30600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:03.299 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.291+0000 7fc021d9b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557992d24c60 0x557992d30600 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:39476/0 (socket says 172.21.15.99:39476) 2024-03-19T01:54:03.299 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.291+0000 7fc02259c640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557992d1faf0 0x557992d295c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:03.299 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.291+0000 7fc021d9b640 1 -- 172.21.15.99:0/158973649 learned_addr learned my addr 172.21.15.99:0/158973649 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:54:03.299 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.291+0000 7fc021d9b640 1 --2- 172.21.15.99:0/158973649 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557992d1f100 0x557992d29080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:03.299 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.291+0000 7fc021d9b640 1 -- 172.21.15.99:0/158973649 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557992d1faf0 msgr2=0x557992d295c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:54:03.299 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.291+0000 7fc021d9b640 1 --2- 172.21.15.99:0/158973649 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557992d1faf0 0x557992d295c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:03.299 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.291+0000 7fc021d9b640 1 -- 172.21.15.99:0/158973649 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557992d1f100 msgr2=0x557992d29080 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:54:03.299 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.291+0000 7fc021d9b640 1 --2- 172.21.15.99:0/158973649 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557992d1f100 0x557992d29080 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:03.299 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.291+0000 7fc021d9b640 1 -- 172.21.15.99:0/158973649 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x557992c3ec60 con 0x557992d24c60 2024-03-19T01:54:03.299 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.291+0000 7fc021d9b640 1 --2- 172.21.15.99:0/158973649 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557992d24c60 0x557992d30600 secure :-1 s=READY pgs=598 cs=0 l=1 rev1=1 crypto rx=0x7fc018010ef0 tx=0x7fc018015040 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3e93f82ccec5877f server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:54:03.299 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.291+0000 7fc009ffb640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:54:03.299 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.291+0000 7fc009ffb640 1 -- 172.21.15.99:0/158973649 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc018004030 con 0x557992d24c60 2024-03-19T01:54:03.299 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.291+0000 7fc009ffb640 1 -- 172.21.15.99:0/158973649 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fc0180041d0 con 0x557992d24c60 2024-03-19T01:54:03.299 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.291+0000 7fc02b7b6740 1 -- 172.21.15.99:0/158973649 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x557992c91120 con 0x557992d24c60 2024-03-19T01:54:03.300 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.291+0000 7fc009ffb640 1 -- 172.21.15.99:0/158973649 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=33) ==== 24+0+0 (secure 0 0 0) 0x7fc018029050 con 0x557992d24c60 2024-03-19T01:54:03.300 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.291+0000 7fc02b7b6740 10 client.16392 fetch_fsmap learned FSMap version 33 2024-03-19T01:54:03.300 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.291+0000 7fc02b7b6740 1 -- 172.21.15.99:0/158973649 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=33,monmap=4+}) -- 0x557992998210 con 0x557992d24c60 2024-03-19T01:54:03.300 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.292+0000 7fc009ffb640 1 -- 172.21.15.99:0/158973649 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x557992c91120 con 0x557992d24c60 2024-03-19T01:54:03.300 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.292+0000 7fc009ffb640 1 -- 172.21.15.99:0/158973649 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 33) ==== 8362+0+0 (secure 0 0 0) 0x7fc018020370 con 0x557992d24c60 2024-03-19T01:54:03.300 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.292+0000 7fc02b7b6740 10 client.16392 fetch_fsmap finished waiting for FSMap version 33 2024-03-19T01:54:03.300 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.292+0000 7fc02b7b6740 10 client.16392 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:54:03.300 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.292+0000 7fc02b7b6740 20 client.16392 populate_metadata read hostname 'smithi099' 2024-03-19T01:54:03.300 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.292+0000 7fc02b7b6740 1 --2- 172.21.15.99:0/158973649 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x557992d34eb0 0x557992d37350 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:03.300 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.292+0000 7fc02b7b6740 4 client.16392 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:54:03.300 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.292+0000 7fc02b7b6740 1 -- 172.21.15.99:0/158973649 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x557992c8b470 con 0x557992d34eb0 2024-03-19T01:54:03.300 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.292+0000 7fc02259c640 1 --2- 172.21.15.99:0/158973649 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x557992d34eb0 0x557992d37350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:03.300 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.292+0000 7fc02259c640 1 --2- 172.21.15.99:0/158973649 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x557992d34eb0 0x557992d37350 crc :-1 s=READY pgs=145 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=f50fe034f655bbd4 server_cookie=4c604d265b525592 in_seq=0 out_seq=0 2024-03-19T01:54:03.300 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.292+0000 7fc009ffb640 10 client.16392 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:54:03.300 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.295+0000 7fc009ffb640 1 -- 172.21.15.99:0/158973649 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x557992c8b470 con 0x557992d34eb0 2024-03-19T01:54:03.301 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.295+0000 7fc009ffb640 10 client.16392 handle_command_reply: tid=0 2024-03-19T01:54:03.301 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.295+0000 7fc02b7b6740 1 client.16392 shutdown 2024-03-19T01:54:03.301 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.295+0000 7fc02b7b6740 1 -- 172.21.15.99:0/158973649 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557992d24c60 msgr2=0x557992d30600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:54:03.301 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.295+0000 7fc02b7b6740 1 --2- 172.21.15.99:0/158973649 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557992d24c60 0x557992d30600 secure :-1 s=READY pgs=598 cs=0 l=1 rev1=1 crypto rx=0x7fc018010ef0 tx=0x7fc018015040 comp rx=0 tx=0).stop 2024-03-19T01:54:03.301 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.295+0000 7fc02b7b6740 1 -- 172.21.15.99:0/158973649 shutdown_connections 2024-03-19T01:54:03.301 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.295+0000 7fc02b7b6740 1 --2- 172.21.15.99:0/158973649 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x557992d34eb0 0x557992d37350 crc :-1 s=READY pgs=145 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:03.301 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.295+0000 7fc02b7b6740 1 --2- 172.21.15.99:0/158973649 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557992d24c60 0x557992d30600 unknown :-1 s=CLOSED pgs=598 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:03.301 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.295+0000 7fc02b7b6740 1 --2- 172.21.15.99:0/158973649 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557992d1faf0 0x557992d295c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:03.301 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.295+0000 7fc009ffb640 0 client.16392 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:54:03.301 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.295+0000 7fc02b7b6740 1 --2- 172.21.15.99:0/158973649 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557992d1f100 0x557992d29080 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:03.301 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.295+0000 7fc02b7b6740 1 -- 172.21.15.99:0/158973649 >> 172.21.15.99:0/158973649 conn(0x557992d1a670 msgr2=0x557992d1b8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:54:03.301 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.295+0000 7fc02b7b6740 1 -- 172.21.15.99:0/158973649 shutdown_connections 2024-03-19T01:54:03.301 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.295+0000 7fc02b7b6740 1 -- 172.21.15.99:0/158973649 wait complete. 2024-03-19T01:54:03.301 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.295+0000 7fc02b7b6740 20 client.16392 trim_cache size 0 max 16384 2024-03-19T01:54:03.316 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.312+0000 7fc02b7b6740 1 Processor -- start 2024-03-19T01:54:03.316 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.312+0000 7fc02b7b6740 1 -- start start 2024-03-19T01:54:03.316 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.312+0000 7fc02b7b6740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557992d1f100 0x557992d19ab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:03.316 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.312+0000 7fc02b7b6740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557992d1faf0 0x557992d305c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:03.316 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.312+0000 7fc02b7b6740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557992d24c60 0x557992d20ca0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:03.316 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.312+0000 7fc02b7b6740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fc0241ab680 con 0x557992d24c60 2024-03-19T01:54:03.316 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.312+0000 7fc02b7b6740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fc0241a8e90 con 0x557992d1faf0 2024-03-19T01:54:03.316 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.312+0000 7fc02b7b6740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fc0241a6ce0 con 0x557992d1f100 2024-03-19T01:54:03.316 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.312+0000 7fc021d9b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557992d24c60 0x557992d20ca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:03.316 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.312+0000 7fc021d9b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557992d24c60 0x557992d20ca0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:39478/0 (socket says 172.21.15.99:39478) 2024-03-19T01:54:03.316 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.312+0000 7fc021d9b640 1 -- 172.21.15.99:0/1063301865 learned_addr learned my addr 172.21.15.99:0/1063301865 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:54:03.316 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.312+0000 7fc02259c640 1 --2- 172.21.15.99:0/1063301865 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557992d1faf0 0x557992d305c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:03.317 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.312+0000 7fc021d9b640 1 --2- 172.21.15.99:0/1063301865 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557992d1f100 0x557992d19ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:03.317 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.312+0000 7fc02259c640 1 -- 172.21.15.99:0/1063301865 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557992d1f100 msgr2=0x557992d19ab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:54:03.317 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.312+0000 7fc02259c640 1 --2- 172.21.15.99:0/1063301865 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557992d1f100 0x557992d19ab0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:03.317 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.312+0000 7fc02259c640 1 -- 172.21.15.99:0/1063301865 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557992d24c60 msgr2=0x557992d20ca0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:54:03.317 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.312+0000 7fc02259c640 1 --2- 172.21.15.99:0/1063301865 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557992d24c60 0x557992d20ca0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:03.317 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.312+0000 7fc02259c640 1 -- 172.21.15.99:0/1063301865 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x557992a64620 con 0x557992d1faf0 2024-03-19T01:54:03.317 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.312+0000 7fc021d9b640 1 --2- 172.21.15.99:0/1063301865 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557992d24c60 0x557992d20ca0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:54:03.317 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.312+0000 7fc02259c640 1 --2- 172.21.15.99:0/1063301865 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557992d1faf0 0x557992d305c0 secure :-1 s=READY pgs=160 cs=0 l=1 rev1=1 crypto rx=0x557992d1fef0 tx=0x7fc014009640 comp rx=0 tx=0).ready entity=mon.2 client_cookie=1660975ce560b07d server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:54:03.317 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.313+0000 7fc00bfff640 1 -- 172.21.15.99:0/1063301865 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc01402cc40 con 0x557992d1faf0 2024-03-19T01:54:03.317 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.313+0000 7fc00bfff640 1 -- 172.21.15.99:0/1063301865 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fc01402c7d0 con 0x557992d1faf0 2024-03-19T01:54:03.317 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.313+0000 7fc00bfff640 1 -- 172.21.15.99:0/1063301865 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc0140055a0 con 0x557992d1faf0 2024-03-19T01:54:03.317 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.313+0000 7fc02b7b6740 1 -- 172.21.15.99:0/1063301865 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557992d1faf0 msgr2=0x557992d305c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:54:03.317 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.313+0000 7fc02b7b6740 1 --2- 172.21.15.99:0/1063301865 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557992d1faf0 0x557992d305c0 secure :-1 s=READY pgs=160 cs=0 l=1 rev1=1 crypto rx=0x557992d1fef0 tx=0x7fc014009640 comp rx=0 tx=0).stop 2024-03-19T01:54:03.317 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.313+0000 7fc02b7b6740 1 -- 172.21.15.99:0/1063301865 shutdown_connections 2024-03-19T01:54:03.318 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.313+0000 7fc02b7b6740 1 --2- 172.21.15.99:0/1063301865 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557992d1faf0 0x557992d305c0 unknown :-1 s=CLOSED pgs=160 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:03.318 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.313+0000 7fc02b7b6740 1 --2- 172.21.15.99:0/1063301865 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557992d24c60 0x557992d20ca0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:03.318 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.313+0000 7fc02b7b6740 1 --2- 172.21.15.99:0/1063301865 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557992d1f100 0x557992d19ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:03.318 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.313+0000 7fc02b7b6740 1 -- 172.21.15.99:0/1063301865 >> 172.21.15.99:0/1063301865 conn(0x557992d1a670 msgr2=0x557992d20590 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:54:03.318 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.313+0000 7fc02b7b6740 1 -- 172.21.15.99:0/1063301865 shutdown_connections 2024-03-19T01:54:03.318 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.313+0000 7fc02b7b6740 1 -- 172.21.15.99:0/1063301865 wait complete. 2024-03-19T01:54:03.318 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.314+0000 7fc02b7b6740 1 -- start start 2024-03-19T01:54:03.318 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.314+0000 7fc02b7b6740 1 Processor -- start 2024-03-19T01:54:03.318 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.314+0000 7fc02b7b6740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557992d1f100 0x557992d2e1c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:03.318 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.314+0000 7fc02b7b6740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557992d1faf0 0x557992d2e700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:03.318 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.314+0000 7fc02b7b6740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557992d24c60 0x557992d2f710 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:03.318 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.314+0000 7fc02b7b6740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fc0241a7350 con 0x557992d1f100 2024-03-19T01:54:03.318 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.314+0000 7fc02b7b6740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fc0241a87c0 con 0x557992d1faf0 2024-03-19T01:54:03.318 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.314+0000 7fc02b7b6740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fc0241a7a20 con 0x557992d24c60 2024-03-19T01:54:03.319 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.314+0000 7fc02b7b6740 10 client.0 _ll_register_callbacks cb 0x7fc018010d10 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:54:03.319 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.314+0000 7fc021d9b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557992d1f100 0x557992d2e1c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:03.319 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.314+0000 7fc021d9b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557992d1f100 0x557992d2e1c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:39480/0 (socket says 172.21.15.99:39480) 2024-03-19T01:54:03.319 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.314+0000 7fc021d9b640 1 -- 172.21.15.99:0/1768635182 learned_addr learned my addr 172.21.15.99:0/1768635182 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:54:03.319 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.314+0000 7fc02259c640 1 --2- 172.21.15.99:0/1768635182 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557992d1faf0 0x557992d2e700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:03.319 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.314+0000 7fc021d9b640 1 --2- 172.21.15.99:0/1768635182 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557992d24c60 0x557992d2f710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:03.319 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.314+0000 7fc021d9b640 1 -- 172.21.15.99:0/1768635182 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557992d24c60 msgr2=0x557992d2f710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:54:03.319 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.314+0000 7fc021d9b640 1 --2- 172.21.15.99:0/1768635182 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557992d24c60 0x557992d2f710 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:03.319 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.314+0000 7fc021d9b640 1 -- 172.21.15.99:0/1768635182 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557992d1faf0 msgr2=0x557992d2e700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:54:03.319 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.314+0000 7fc021d9b640 1 --2- 172.21.15.99:0/1768635182 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557992d1faf0 0x557992d2e700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:03.319 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.314+0000 7fc021d9b640 1 -- 172.21.15.99:0/1768635182 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x557992d2fd00 con 0x557992d1f100 2024-03-19T01:54:03.319 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.314+0000 7fc021d9b640 1 --2- 172.21.15.99:0/1768635182 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557992d1f100 0x557992d2e1c0 secure :-1 s=READY pgs=599 cs=0 l=1 rev1=1 crypto rx=0x7fc018010ce0 tx=0x7fc018009950 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1824ab61be4fdcf server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:54:03.319 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.315+0000 7fc009ffb640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:54:03.319 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.315+0000 7fc009ffb640 1 -- 172.21.15.99:0/1768635182 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc018004040 con 0x557992d1f100 2024-03-19T01:54:03.320 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.315+0000 7fc009ffb640 1 -- 172.21.15.99:0/1768635182 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fc0180041e0 con 0x557992d1f100 2024-03-19T01:54:03.320 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.315+0000 7fc02b7b6740 1 -- 172.21.15.99:0/1768635182 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fc01402c7d0 con 0x557992d1f100 2024-03-19T01:54:03.320 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.315+0000 7fc009ffb640 1 -- 172.21.15.99:0/1768635182 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=33) ==== 24+0+0 (secure 0 0 0) 0x7fc0180043e0 con 0x557992d1f100 2024-03-19T01:54:03.320 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.315+0000 7fc02b7b6740 10 client.16398 fetch_fsmap learned FSMap version 33 2024-03-19T01:54:03.320 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.315+0000 7fc02b7b6740 1 -- 172.21.15.99:0/1768635182 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=33,monmap=4+}) -- 0x557992d9b780 con 0x557992d1f100 2024-03-19T01:54:03.320 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.315+0000 7fc009ffb640 1 -- 172.21.15.99:0/1768635182 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fc01402c7d0 con 0x557992d1f100 2024-03-19T01:54:03.320 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.315+0000 7fc009ffb640 1 -- 172.21.15.99:0/1768635182 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 33) ==== 8362+0+0 (secure 0 0 0) 0x7fc018003a40 con 0x557992d1f100 2024-03-19T01:54:03.320 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.315+0000 7fc02b7b6740 10 client.16398 fetch_fsmap finished waiting for FSMap version 33 2024-03-19T01:54:03.320 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.315+0000 7fc02b7b6740 10 client.16398 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:54:03.320 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.315+0000 7fc02b7b6740 20 client.16398 populate_metadata read hostname 'smithi099' 2024-03-19T01:54:03.320 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.315+0000 7fc02b7b6740 1 --2- 172.21.15.99:0/1768635182 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x557992d9c0e0 0x557992d9e500 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:03.320 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.315+0000 7fc02b7b6740 4 client.16398 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:54:03.320 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.315+0000 7fc02b7b6740 1 -- 172.21.15.99:0/1768635182 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x557992c09770 con 0x557992d9c0e0 2024-03-19T01:54:03.320 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.315+0000 7fc02259c640 1 --2- 172.21.15.99:0/1768635182 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x557992d9c0e0 0x557992d9e500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:03.320 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.316+0000 7fc009ffb640 10 client.16398 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:54:03.321 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.316+0000 7fc02259c640 1 --2- 172.21.15.99:0/1768635182 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x557992d9c0e0 0x557992d9e500 crc :-1 s=READY pgs=146 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=77333ee9ae057bc server_cookie=321f00445a1b205 in_seq=0 out_seq=0 2024-03-19T01:54:03.965 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.965+0000 7fc009ffb640 1 -- 172.21.15.99:0/1768635182 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x557992c09770 con 0x557992d9c0e0 2024-03-19T01:54:03.965 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.965+0000 7fc009ffb640 10 client.16398 handle_command_reply: tid=0 2024-03-19T01:54:03.965 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.965+0000 7fc02b7b6740 1 client.16398 shutdown 2024-03-19T01:54:03.966 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.965+0000 7fc02b7b6740 1 -- 172.21.15.99:0/1768635182 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557992d1f100 msgr2=0x557992d2e1c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:54:03.966 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.965+0000 7fc02b7b6740 1 --2- 172.21.15.99:0/1768635182 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557992d1f100 0x557992d2e1c0 secure :-1 s=READY pgs=599 cs=0 l=1 rev1=1 crypto rx=0x7fc018010ce0 tx=0x7fc018009950 comp rx=0 tx=0).stop 2024-03-19T01:54:03.966 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.965+0000 7fc02b7b6740 1 -- 172.21.15.99:0/1768635182 shutdown_connections 2024-03-19T01:54:03.966 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.965+0000 7fc02b7b6740 1 --2- 172.21.15.99:0/1768635182 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x557992d9c0e0 0x557992d9e500 crc :-1 s=READY pgs=146 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:03.966 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.965+0000 7fc02b7b6740 1 --2- 172.21.15.99:0/1768635182 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557992d1faf0 0x557992d2e700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:03.966 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.965+0000 7fc02b7b6740 1 --2- 172.21.15.99:0/1768635182 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557992d24c60 0x557992d2f710 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:03.966 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.965+0000 7fc02b7b6740 1 --2- 172.21.15.99:0/1768635182 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557992d1f100 0x557992d2e1c0 unknown :-1 s=CLOSED pgs=599 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:03.966 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.965+0000 7fc009ffb640 0 client.16398 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:54:03.966 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.965+0000 7fc02b7b6740 1 -- 172.21.15.99:0/1768635182 >> 172.21.15.99:0/1768635182 conn(0x557992d1a670 msgr2=0x557992d20e20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:54:03.966 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.965+0000 7fc02b7b6740 1 -- 172.21.15.99:0/1768635182 shutdown_connections 2024-03-19T01:54:03.966 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.965+0000 7fc02b7b6740 1 -- 172.21.15.99:0/1768635182 wait complete. 2024-03-19T01:54:03.966 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.965+0000 7fc02b7b6740 20 client.16398 trim_cache size 0 max 16384 2024-03-19T01:54:03.967 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:54:03.967 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:54:03.967 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:54:03.967 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:54:03.968 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.966+0000 7fc028bd3640 1 -- 172.21.15.99:0/3377394738 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fc00c0a2920 msgr2=0x7fc00c0a4de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:54:03.968 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.966+0000 7fc028bd3640 1 --2- 172.21.15.99:0/3377394738 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fc00c0a2920 0x7fc00c0a4de0 secure :-1 s=READY pgs=331 cs=0 l=1 rev1=1 crypto rx=0x7fc01c0ee350 tx=0x7fc01c088040 comp rx=0 tx=0).stop 2024-03-19T01:54:03.968 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.966+0000 7fc028bd3640 1 -- 172.21.15.99:0/3377394738 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc0241022f0 msgr2=0x7fc02419edb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:54:03.968 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.966+0000 7fc028bd3640 1 --2- 172.21.15.99:0/3377394738 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc0241022f0 0x7fc02419edb0 secure :-1 s=READY pgs=597 cs=0 l=1 rev1=1 crypto rx=0x7fc01c037040 tx=0x7fc01c032f40 comp rx=0 tx=0).stop 2024-03-19T01:54:03.968 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.966+0000 7fc028bd3640 1 -- 172.21.15.99:0/3377394738 shutdown_connections 2024-03-19T01:54:03.968 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.966+0000 7fc028bd3640 1 --2- 172.21.15.99:0/3377394738 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc0241022f0 0x7fc02419edb0 unknown :-1 s=CLOSED pgs=597 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:03.968 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.966+0000 7fc028bd3640 1 --2- 172.21.15.99:0/3377394738 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fc024101930 0x7fc02419c770 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:03.968 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.966+0000 7fc028bd3640 1 --2- 172.21.15.99:0/3377394738 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fc00c0a2920 0x7fc00c0a4de0 unknown :-1 s=CLOSED pgs=331 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:03.968 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.966+0000 7fc028bd3640 1 --2- 172.21.15.99:0/3377394738 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fc024100740 0x7fc024069a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:03.968 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.966+0000 7fc028bd3640 1 -- 172.21.15.99:0/3377394738 >> 172.21.15.99:0/3377394738 conn(0x7fc0240fbcb0 msgr2=0x7fc024104bd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:54:03.968 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.966+0000 7fc028bd3640 1 -- 172.21.15.99:0/3377394738 shutdown_connections 2024-03-19T01:54:03.969 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:03.966+0000 7fc028bd3640 1 -- 172.21.15.99:0/3377394738 wait complete. 2024-03-19T01:54:04.097 INFO:tasks.workunit.client.0.smithi099.stdout: 10 15514 30.61 MB/sec warmup 33 sec latency 648.587 ms 2024-03-19T01:54:04.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:54:04 smithi115 ceph-mon[67332]: pgmap v740: 97 pgs: 97 active+clean; 271 MiB data, 4.0 GiB used, 1.0 TiB / 1.0 TiB avail; 1.4 MiB/s rd, 4.6 MiB/s wr, 498 op/s 2024-03-19T01:54:04.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:54:04 smithi099 ceph-mon[62951]: pgmap v740: 97 pgs: 97 active+clean; 271 MiB data, 4.0 GiB used, 1.0 TiB / 1.0 TiB avail; 1.4 MiB/s rd, 4.6 MiB/s wr, 498 op/s 2024-03-19T01:54:04.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:54:04 smithi112 ceph-mon[66819]: pgmap v740: 97 pgs: 97 active+clean; 271 MiB data, 4.0 GiB used, 1.0 TiB / 1.0 TiB avail; 1.4 MiB/s rd, 4.6 MiB/s wr, 498 op/s 2024-03-19T01:54:05.098 INFO:tasks.workunit.client.0.smithi099.stdout: 10 15895 30.28 MB/sec warmup 34 sec latency 721.155 ms 2024-03-19T01:54:06.097 INFO:tasks.workunit.client.0.smithi099.stdout: 10 16663 30.29 MB/sec warmup 35 sec latency 67.771 ms 2024-03-19T01:54:06.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:54:06 smithi115 ceph-mon[67332]: pgmap v741: 97 pgs: 97 active+clean; 294 MiB data, 4.1 GiB used, 1.0 TiB / 1.0 TiB avail; 1.4 MiB/s rd, 6.1 MiB/s wr, 383 op/s 2024-03-19T01:54:06.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:54:06 smithi099 ceph-mon[62951]: pgmap v741: 97 pgs: 97 active+clean; 294 MiB data, 4.1 GiB used, 1.0 TiB / 1.0 TiB avail; 1.4 MiB/s rd, 6.1 MiB/s wr, 383 op/s 2024-03-19T01:54:06.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:54:06 smithi112 ceph-mon[66819]: pgmap v741: 97 pgs: 97 active+clean; 294 MiB data, 4.1 GiB used, 1.0 TiB / 1.0 TiB avail; 1.4 MiB/s rd, 6.1 MiB/s wr, 383 op/s 2024-03-19T01:54:07.097 INFO:tasks.workunit.client.0.smithi099.stdout: 10 17222 30.36 MB/sec warmup 36 sec latency 72.989 ms 2024-03-19T01:54:07.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:54:07 smithi099 ceph-mon[62951]: pgmap v742: 97 pgs: 97 active+clean; 403 MiB data, 4.4 GiB used, 1.0 TiB / 1.0 TiB avail; 1.5 MiB/s rd, 15 MiB/s wr, 487 op/s 2024-03-19T01:54:07.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:54:07 smithi112 ceph-mon[66819]: pgmap v742: 97 pgs: 97 active+clean; 403 MiB data, 4.4 GiB used, 1.0 TiB / 1.0 TiB avail; 1.5 MiB/s rd, 15 MiB/s wr, 487 op/s 2024-03-19T01:54:07.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:54:07 smithi115 ceph-mon[67332]: pgmap v742: 97 pgs: 97 active+clean; 403 MiB data, 4.4 GiB used, 1.0 TiB / 1.0 TiB avail; 1.5 MiB/s rd, 15 MiB/s wr, 487 op/s 2024-03-19T01:54:08.098 INFO:tasks.workunit.client.0.smithi099.stdout: 10 17837 30.52 MB/sec warmup 37 sec latency 61.773 ms 2024-03-19T01:54:09.097 INFO:tasks.workunit.client.0.smithi099.stdout: 10 18222 30.48 MB/sec warmup 38 sec latency 326.594 ms 2024-03-19T01:54:10.097 INFO:tasks.workunit.client.0.smithi099.stdout: 10 18630 30.43 MB/sec warmup 39 sec latency 92.721 ms 2024-03-19T01:54:10.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:54:10 smithi115 ceph-mon[67332]: pgmap v743: 97 pgs: 97 active+clean; 441 MiB data, 4.5 GiB used, 1.0 TiB / 1.0 TiB avail; 1.5 MiB/s rd, 19 MiB/s wr, 506 op/s 2024-03-19T01:54:10.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:54:10 smithi099 ceph-mon[62951]: pgmap v743: 97 pgs: 97 active+clean; 441 MiB data, 4.5 GiB used, 1.0 TiB / 1.0 TiB avail; 1.5 MiB/s rd, 19 MiB/s wr, 506 op/s 2024-03-19T01:54:10.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:54:10 smithi112 ceph-mon[66819]: pgmap v743: 97 pgs: 97 active+clean; 441 MiB data, 4.5 GiB used, 1.0 TiB / 1.0 TiB avail; 1.5 MiB/s rd, 19 MiB/s wr, 506 op/s 2024-03-19T01:54:11.098 INFO:tasks.workunit.client.0.smithi099.stdout: 10 19077 30.31 MB/sec warmup 40 sec latency 72.388 ms 2024-03-19T01:54:11.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:54:11 smithi099 ceph-mon[62951]: pgmap v744: 97 pgs: 97 active+clean; 449 MiB data, 4.6 GiB used, 1.0 TiB / 1.0 TiB avail; 2.1 MiB/s rd, 19 MiB/s wr, 531 op/s 2024-03-19T01:54:11.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:54:11 smithi112 ceph-mon[66819]: pgmap v744: 97 pgs: 97 active+clean; 449 MiB data, 4.6 GiB used, 1.0 TiB / 1.0 TiB avail; 2.1 MiB/s rd, 19 MiB/s wr, 531 op/s 2024-03-19T01:54:11.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:54:11 smithi115 ceph-mon[67332]: pgmap v744: 97 pgs: 97 active+clean; 449 MiB data, 4.6 GiB used, 1.0 TiB / 1.0 TiB avail; 2.1 MiB/s rd, 19 MiB/s wr, 531 op/s 2024-03-19T01:54:12.097 INFO:tasks.workunit.client.0.smithi099.stdout: 10 19441 30.01 MB/sec warmup 41 sec latency 94.559 ms 2024-03-19T01:54:13.098 INFO:tasks.workunit.client.0.smithi099.stdout: 10 19799 29.62 MB/sec warmup 42 sec latency 100.301 ms 2024-03-19T01:54:14.098 INFO:tasks.workunit.client.0.smithi099.stdout: 10 20164 29.38 MB/sec warmup 43 sec latency 164.851 ms 2024-03-19T01:54:14.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:54:14 smithi115 ceph-mon[67332]: pgmap v745: 97 pgs: 97 active+clean; 451 MiB data, 4.5 GiB used, 1.0 TiB / 1.0 TiB avail; 3.3 MiB/s rd, 20 MiB/s wr, 611 op/s 2024-03-19T01:54:14.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:54:14 smithi099 ceph-mon[62951]: pgmap v745: 97 pgs: 97 active+clean; 451 MiB data, 4.5 GiB used, 1.0 TiB / 1.0 TiB avail; 3.3 MiB/s rd, 20 MiB/s wr, 611 op/s 2024-03-19T01:54:14.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:54:14 smithi112 ceph-mon[66819]: pgmap v745: 97 pgs: 97 active+clean; 451 MiB data, 4.5 GiB used, 1.0 TiB / 1.0 TiB avail; 3.3 MiB/s rd, 20 MiB/s wr, 611 op/s 2024-03-19T01:54:15.098 INFO:tasks.workunit.client.0.smithi099.stdout: 10 20488 29.13 MB/sec warmup 44 sec latency 101.107 ms 2024-03-19T01:54:16.098 INFO:tasks.workunit.client.0.smithi099.stdout: 10 21102 29.03 MB/sec warmup 45 sec latency 70.488 ms 2024-03-19T01:54:16.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:54:16 smithi115 ceph-mon[67332]: pgmap v746: 97 pgs: 97 active+clean; 449 MiB data, 4.5 GiB used, 1.0 TiB / 1.0 TiB avail; 3.3 MiB/s rd, 20 MiB/s wr, 566 op/s 2024-03-19T01:54:16.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:54:16 smithi099 ceph-mon[62951]: pgmap v746: 97 pgs: 97 active+clean; 449 MiB data, 4.5 GiB used, 1.0 TiB / 1.0 TiB avail; 3.3 MiB/s rd, 20 MiB/s wr, 566 op/s 2024-03-19T01:54:16.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:54:16 smithi112 ceph-mon[66819]: pgmap v746: 97 pgs: 97 active+clean; 449 MiB data, 4.5 GiB used, 1.0 TiB / 1.0 TiB avail; 3.3 MiB/s rd, 20 MiB/s wr, 566 op/s 2024-03-19T01:54:17.065 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.055+0000 7fd5643b9640 1 -- 172.21.15.99:0/2478891056 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd55c101930 msgr2=0x7fd55c101db0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:54:17.065 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.055+0000 7fd5643b9640 1 --2- 172.21.15.99:0/2478891056 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd55c101930 0x7fd55c101db0 secure :-1 s=READY pgs=600 cs=0 l=1 rev1=1 crypto rx=0x7fd54c009900 tx=0x7fd54c02bc30 comp rx=0 tx=0).stop 2024-03-19T01:54:17.065 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.055+0000 7fd5643b9640 1 -- 172.21.15.99:0/2478891056 shutdown_connections 2024-03-19T01:54:17.065 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.055+0000 7fd5643b9640 1 --2- 172.21.15.99:0/2478891056 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd55c1022f0 0x7fd55c108b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:17.065 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.055+0000 7fd5643b9640 1 --2- 172.21.15.99:0/2478891056 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd55c101930 0x7fd55c101db0 unknown :-1 s=CLOSED pgs=600 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:17.065 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.055+0000 7fd5643b9640 1 --2- 172.21.15.99:0/2478891056 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd55c100740 0x7fd55c100b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:17.065 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.055+0000 7fd5643b9640 1 -- 172.21.15.99:0/2478891056 >> 172.21.15.99:0/2478891056 conn(0x7fd55c0fbcb0 msgr2=0x7fd55c0fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:54:17.066 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.055+0000 7fd5643b9640 1 -- 172.21.15.99:0/2478891056 shutdown_connections 2024-03-19T01:54:17.066 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.055+0000 7fd5643b9640 1 -- 172.21.15.99:0/2478891056 wait complete. 2024-03-19T01:54:17.066 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.056+0000 7fd5643b9640 1 Processor -- start 2024-03-19T01:54:17.066 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.056+0000 7fd5643b9640 1 -- start start 2024-03-19T01:54:17.066 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.056+0000 7fd5643b9640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd55c100740 0x7fd55c197aa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:17.066 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.056+0000 7fd5643b9640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd55c101930 0x7fd55c197fe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:17.066 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.056+0000 7fd5643b9640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd55c1022f0 0x7fd55c19f050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:17.066 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.056+0000 7fd5643b9640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fd55c10c3c0 con 0x7fd55c101930 2024-03-19T01:54:17.066 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.056+0000 7fd5643b9640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fd55c10c240 con 0x7fd55c1022f0 2024-03-19T01:54:17.066 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.056+0000 7fd5643b9640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fd55c10c540 con 0x7fd55c100740 2024-03-19T01:54:17.066 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.056+0000 7fd562155640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd55c100740 0x7fd55c197aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:17.066 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.056+0000 7fd562956640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd55c1022f0 0x7fd55c19f050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:17.067 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.056+0000 7fd561954640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd55c101930 0x7fd55c197fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:17.067 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.056+0000 7fd562155640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd55c100740 0x7fd55c197aa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:60114/0 (socket says 172.21.15.99:60114) 2024-03-19T01:54:17.067 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.056+0000 7fd562956640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd55c1022f0 0x7fd55c19f050 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:58110/0 (socket says 172.21.15.99:58110) 2024-03-19T01:54:17.067 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.056+0000 7fd562155640 1 -- 172.21.15.99:0/1120200963 learned_addr learned my addr 172.21.15.99:0/1120200963 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:54:17.067 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.057+0000 7fd561954640 1 -- 172.21.15.99:0/1120200963 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd55c100740 msgr2=0x7fd55c197aa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:54:17.067 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.057+0000 7fd561954640 1 --2- 172.21.15.99:0/1120200963 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd55c100740 0x7fd55c197aa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:17.067 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.057+0000 7fd561954640 1 -- 172.21.15.99:0/1120200963 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd55c1022f0 msgr2=0x7fd55c19f050 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:54:17.067 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.057+0000 7fd561954640 1 --2- 172.21.15.99:0/1120200963 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd55c1022f0 0x7fd55c19f050 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:17.067 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.057+0000 7fd561954640 1 -- 172.21.15.99:0/1120200963 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd55c19f6c0 con 0x7fd55c101930 2024-03-19T01:54:17.067 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.057+0000 7fd562956640 1 --2- 172.21.15.99:0/1120200963 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd55c1022f0 0x7fd55c19f050 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:54:17.067 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.057+0000 7fd561954640 1 --2- 172.21.15.99:0/1120200963 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd55c101930 0x7fd55c197fe0 secure :-1 s=READY pgs=601 cs=0 l=1 rev1=1 crypto rx=0x7fd54c009230 tx=0x7fd54c02c650 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:54:17.067 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.057+0000 7fd5537fe640 1 -- 172.21.15.99:0/1120200963 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd54c041070 con 0x7fd55c101930 2024-03-19T01:54:17.067 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.057+0000 7fd5643b9640 1 -- 172.21.15.99:0/1120200963 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd55c19f950 con 0x7fd55c101930 2024-03-19T01:54:17.068 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.057+0000 7fd5643b9640 1 -- 172.21.15.99:0/1120200963 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fd55c19fe90 con 0x7fd55c101930 2024-03-19T01:54:17.068 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.057+0000 7fd5537fe640 1 -- 172.21.15.99:0/1120200963 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd54c004450 con 0x7fd55c101930 2024-03-19T01:54:17.068 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.057+0000 7fd5537fe640 1 -- 172.21.15.99:0/1120200963 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd54c04b060 con 0x7fd55c101930 2024-03-19T01:54:17.068 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.059+0000 7fd5537fe640 1 -- 172.21.15.99:0/1120200963 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7fd54c044070 con 0x7fd55c101930 2024-03-19T01:54:17.068 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.060+0000 7fd566fa1740 1 Processor -- start 2024-03-19T01:54:17.068 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.060+0000 7fd566fa1740 1 -- start start 2024-03-19T01:54:17.068 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.060+0000 7fd566fa1740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562340378100 0x562340378520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:17.068 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.060+0000 7fd566fa1740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562340378af0 0x56234037d5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:17.068 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.060+0000 7fd566fa1740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56234037dc60 0x5623403800f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:17.068 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.060+0000 7fd566fa1740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x5623402dcbe0 con 0x562340378100 2024-03-19T01:54:17.068 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.060+0000 7fd566fa1740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x562340285960 con 0x56234037dc60 2024-03-19T01:54:17.068 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.060+0000 7fd566fa1740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x5623402d8c80 con 0x562340378af0 2024-03-19T01:54:17.068 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.060+0000 7fd5537fe640 1 --2- 172.21.15.99:0/1120200963 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd5480a28a0 0x7fd5480a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:17.068 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.060+0000 7fd562956640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562340378100 0x562340378520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:17.069 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.060+0000 7fd561954640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56234037dc60 0x5623403800f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:17.069 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.060+0000 7fd562956640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562340378100 0x562340378520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:50778/0 (socket says 172.21.15.99:50778) 2024-03-19T01:54:17.069 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.060+0000 7fd562956640 1 -- 172.21.15.99:0/2749470311 learned_addr learned my addr 172.21.15.99:0/2749470311 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:54:17.069 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.060+0000 7fd562155640 1 --2- 172.21.15.99:0/2749470311 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562340378af0 0x56234037d5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:17.069 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.061+0000 7fd562956640 1 --2- 172.21.15.99:0/1120200963 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd5480a28a0 0x7fd5480a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:17.069 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.061+0000 7fd5537fe640 1 -- 172.21.15.99:0/1120200963 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fd54c0f0000 con 0x7fd55c101930 2024-03-19T01:54:17.069 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.061+0000 7fd562956640 1 -- 172.21.15.99:0/2749470311 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562340378af0 msgr2=0x56234037d5f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:54:17.069 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.061+0000 7fd562956640 1 --2- 172.21.15.99:0/2749470311 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562340378af0 0x56234037d5f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:17.069 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.061+0000 7fd562956640 1 -- 172.21.15.99:0/2749470311 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56234037dc60 msgr2=0x5623403800f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:54:17.069 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.061+0000 7fd562956640 1 --2- 172.21.15.99:0/2749470311 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56234037dc60 0x5623403800f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:17.070 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.061+0000 7fd562956640 1 -- 172.21.15.99:0/2749470311 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x56234027cd20 con 0x562340378100 2024-03-19T01:54:17.070 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.061+0000 7fd562956640 1 --2- 172.21.15.99:0/2749470311 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562340378100 0x562340378520 secure :-1 s=READY pgs=602 cs=0 l=1 rev1=1 crypto rx=0x7fd55400f5c0 tx=0x7fd55400fa80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=fe912ebce0e610e4 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:54:17.070 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.061+0000 7fd562956640 1 --2- 172.21.15.99:0/1120200963 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd5480a28a0 0x7fd5480a4d60 secure :-1 s=READY pgs=332 cs=0 l=1 rev1=1 crypto rx=0x7fd5540099e0 tx=0x7fd55401e000 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:54:17.070 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.061+0000 7fd550bfe640 1 -- 172.21.15.99:0/2749470311 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd55402b070 con 0x562340378100 2024-03-19T01:54:17.070 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.061+0000 7fd550bfe640 1 -- 172.21.15.99:0/2749470311 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd554003e50 con 0x562340378100 2024-03-19T01:54:17.070 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.061+0000 7fd550bfe640 1 -- 172.21.15.99:0/2749470311 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd554022070 con 0x562340378100 2024-03-19T01:54:17.070 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.062+0000 7fd566fa1740 1 -- 172.21.15.99:0/2749470311 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562340378100 msgr2=0x562340378520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:54:17.071 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.062+0000 7fd566fa1740 1 --2- 172.21.15.99:0/2749470311 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562340378100 0x562340378520 secure :-1 s=READY pgs=602 cs=0 l=1 rev1=1 crypto rx=0x7fd55400f5c0 tx=0x7fd55400fa80 comp rx=0 tx=0).stop 2024-03-19T01:54:17.071 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.062+0000 7fd566fa1740 1 -- 172.21.15.99:0/2749470311 shutdown_connections 2024-03-19T01:54:17.071 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.062+0000 7fd566fa1740 1 --2- 172.21.15.99:0/2749470311 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56234037dc60 0x5623403800f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:17.071 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.062+0000 7fd566fa1740 1 --2- 172.21.15.99:0/2749470311 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562340378af0 0x56234037d5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:17.071 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.062+0000 7fd566fa1740 1 --2- 172.21.15.99:0/2749470311 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562340378100 0x562340378520 unknown :-1 s=CLOSED pgs=602 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:17.071 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.062+0000 7fd566fa1740 1 -- 172.21.15.99:0/2749470311 >> 172.21.15.99:0/2749470311 conn(0x562340373670 msgr2=0x562340373a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:54:17.071 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.062+0000 7fd566fa1740 1 -- 172.21.15.99:0/2749470311 shutdown_connections 2024-03-19T01:54:17.071 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.062+0000 7fd566fa1740 1 -- 172.21.15.99:0/2749470311 wait complete. 2024-03-19T01:54:17.072 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.062+0000 7fd566fa1740 1 -- start start 2024-03-19T01:54:17.072 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.062+0000 7fd566fa1740 1 Processor -- start 2024-03-19T01:54:17.072 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.063+0000 7fd566fa1740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562340378100 0x5623403820d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:17.072 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.063+0000 7fd566fa1740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562340378af0 0x562340382610 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:17.072 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.063+0000 7fd566fa1740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56234037dc60 0x562340389650 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:17.072 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.063+0000 7fd566fa1740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x562340380bb0 con 0x562340378100 2024-03-19T01:54:17.072 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.063+0000 7fd566fa1740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x562340298db0 con 0x562340378af0 2024-03-19T01:54:17.073 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.063+0000 7fd566fa1740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x562340380d30 con 0x56234037dc60 2024-03-19T01:54:17.073 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.063+0000 7fd566fa1740 10 client.0 _ll_register_callbacks cb 0x56234029de70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:54:17.073 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.063+0000 7fd562155640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562340378100 0x5623403820d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:17.073 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.063+0000 7fd562155640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562340378100 0x5623403820d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:50780/0 (socket says 172.21.15.99:50780) 2024-03-19T01:54:17.073 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.063+0000 7fd562155640 1 -- 172.21.15.99:0/1878518579 learned_addr learned my addr 172.21.15.99:0/1878518579 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:54:17.073 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.063+0000 7fd561954640 1 --2- 172.21.15.99:0/1878518579 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562340378af0 0x562340382610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:17.074 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.063+0000 7fd562956640 1 --2- 172.21.15.99:0/1878518579 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56234037dc60 0x562340389650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:17.074 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.063+0000 7fd562155640 1 -- 172.21.15.99:0/1878518579 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56234037dc60 msgr2=0x562340389650 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:54:17.074 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.063+0000 7fd562155640 1 --2- 172.21.15.99:0/1878518579 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56234037dc60 0x562340389650 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:17.074 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.063+0000 7fd562155640 1 -- 172.21.15.99:0/1878518579 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562340378af0 msgr2=0x562340382610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:54:17.074 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.063+0000 7fd562155640 1 --2- 172.21.15.99:0/1878518579 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562340378af0 0x562340382610 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:17.074 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.063+0000 7fd562155640 1 -- 172.21.15.99:0/1878518579 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x562340297c60 con 0x562340378100 2024-03-19T01:54:17.075 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.063+0000 7fd561954640 1 --2- 172.21.15.99:0/1878518579 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562340378af0 0x562340382610 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:54:17.075 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.063+0000 7fd562155640 1 --2- 172.21.15.99:0/1878518579 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562340378100 0x5623403820d0 secure :-1 s=READY pgs=603 cs=0 l=1 rev1=1 crypto rx=0x7fd558012700 tx=0x7fd558012b00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=82121bf7dd8e0d01 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:54:17.075 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.063+0000 7fd545ffb640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:54:17.075 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.063+0000 7fd545ffb640 1 -- 172.21.15.99:0/1878518579 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd558004050 con 0x562340378100 2024-03-19T01:54:17.075 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.063+0000 7fd545ffb640 1 -- 172.21.15.99:0/1878518579 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fd5580041f0 con 0x562340378100 2024-03-19T01:54:17.075 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.063+0000 7fd566fa1740 1 -- 172.21.15.99:0/1878518579 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fd554003e50 con 0x562340378100 2024-03-19T01:54:17.080 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.066+0000 7fd545ffb640 1 -- 172.21.15.99:0/1878518579 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=33) ==== 24+0+0 (secure 0 0 0) 0x7fd5580043f0 con 0x562340378100 2024-03-19T01:54:17.080 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.066+0000 7fd566fa1740 10 client.16410 fetch_fsmap learned FSMap version 33 2024-03-19T01:54:17.080 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.066+0000 7fd566fa1740 1 -- 172.21.15.99:0/1878518579 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=33,monmap=4+}) -- 0x56233fff1210 con 0x562340378100 2024-03-19T01:54:17.080 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.067+0000 7fd545ffb640 1 -- 172.21.15.99:0/1878518579 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd554003e50 con 0x562340378100 2024-03-19T01:54:17.080 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.067+0000 7fd545ffb640 1 -- 172.21.15.99:0/1878518579 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 33) ==== 8362+0+0 (secure 0 0 0) 0x7fd55800f890 con 0x562340378100 2024-03-19T01:54:17.080 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.067+0000 7fd566fa1740 10 client.16410 fetch_fsmap finished waiting for FSMap version 33 2024-03-19T01:54:17.081 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.067+0000 7fd566fa1740 10 client.16410 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:54:17.081 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.067+0000 7fd566fa1740 20 client.16410 populate_metadata read hostname 'smithi099' 2024-03-19T01:54:17.081 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.067+0000 7fd566fa1740 1 --2- 172.21.15.99:0/1878518579 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56234038e020 0x562340390470 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:17.081 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.067+0000 7fd566fa1740 4 client.16410 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:54:17.081 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.067+0000 7fd566fa1740 1 -- 172.21.15.99:0/1878518579 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5623402ea120 con 0x56234038e020 2024-03-19T01:54:17.081 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.067+0000 7fd561954640 1 --2- 172.21.15.99:0/1878518579 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56234038e020 0x562340390470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:17.081 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.067+0000 7fd561954640 1 --2- 172.21.15.99:0/1878518579 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56234038e020 0x562340390470 crc :-1 s=READY pgs=147 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=ef0c4c0764799802 server_cookie=7a9cec903e6c1949 in_seq=0 out_seq=0 2024-03-19T01:54:17.081 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.067+0000 7fd545ffb640 10 client.16410 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:54:17.081 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.072+0000 7fd545ffb640 1 -- 172.21.15.99:0/1878518579 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x5623402ea120 con 0x56234038e020 2024-03-19T01:54:17.081 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.072+0000 7fd545ffb640 10 client.16410 handle_command_reply: tid=0 2024-03-19T01:54:17.081 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.072+0000 7fd566fa1740 1 client.16410 shutdown 2024-03-19T01:54:17.081 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.072+0000 7fd566fa1740 1 -- 172.21.15.99:0/1878518579 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562340378100 msgr2=0x5623403820d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:54:17.081 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.072+0000 7fd566fa1740 1 --2- 172.21.15.99:0/1878518579 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562340378100 0x5623403820d0 secure :-1 s=READY pgs=603 cs=0 l=1 rev1=1 crypto rx=0x7fd558012700 tx=0x7fd558012b00 comp rx=0 tx=0).stop 2024-03-19T01:54:17.081 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.073+0000 7fd566fa1740 1 -- 172.21.15.99:0/1878518579 shutdown_connections 2024-03-19T01:54:17.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.073+0000 7fd566fa1740 1 --2- 172.21.15.99:0/1878518579 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56234038e020 0x562340390470 crc :-1 s=READY pgs=147 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:17.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.073+0000 7fd566fa1740 1 --2- 172.21.15.99:0/1878518579 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562340378af0 0x562340382610 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:17.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.073+0000 7fd545ffb640 0 client.16410 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:54:17.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.073+0000 7fd566fa1740 1 --2- 172.21.15.99:0/1878518579 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56234037dc60 0x562340389650 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:17.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.073+0000 7fd566fa1740 1 --2- 172.21.15.99:0/1878518579 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562340378100 0x5623403820d0 unknown :-1 s=CLOSED pgs=603 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:17.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.073+0000 7fd566fa1740 1 -- 172.21.15.99:0/1878518579 >> 172.21.15.99:0/1878518579 conn(0x562340373670 msgr2=0x5623403748c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:54:17.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.073+0000 7fd566fa1740 1 -- 172.21.15.99:0/1878518579 shutdown_connections 2024-03-19T01:54:17.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.073+0000 7fd566fa1740 1 -- 172.21.15.99:0/1878518579 wait complete. 2024-03-19T01:54:17.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.073+0000 7fd566fa1740 20 client.16410 trim_cache size 0 max 16384 2024-03-19T01:54:17.092 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.088+0000 7fd566fa1740 1 Processor -- start 2024-03-19T01:54:17.093 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.088+0000 7fd566fa1740 1 -- start start 2024-03-19T01:54:17.093 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.088+0000 7fd566fa1740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562340378100 0x562340372ab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:17.093 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.088+0000 7fd566fa1740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562340378af0 0x562340389610 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:17.093 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.088+0000 7fd566fa1740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56234037dc60 0x562340376fe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:17.093 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.088+0000 7fd566fa1740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fd540010f30 con 0x56234037dc60 2024-03-19T01:54:17.093 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.088+0000 7fd566fa1740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fd54000e740 con 0x562340378af0 2024-03-19T01:54:17.093 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.088+0000 7fd566fa1740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fd54000c590 con 0x562340378100 2024-03-19T01:54:17.093 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.089+0000 7fd562956640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56234037dc60 0x562340376fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:17.093 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.089+0000 7fd562956640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56234037dc60 0x562340376fe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:50786/0 (socket says 172.21.15.99:50786) 2024-03-19T01:54:17.093 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.089+0000 7fd562956640 1 -- 172.21.15.99:0/3962995808 learned_addr learned my addr 172.21.15.99:0/3962995808 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:54:17.093 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.089+0000 7fd561954640 1 --2- 172.21.15.99:0/3962995808 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562340378af0 0x562340389610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:17.094 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.089+0000 7fd562155640 1 --2- 172.21.15.99:0/3962995808 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562340378100 0x562340372ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:17.094 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.089+0000 7fd562956640 1 -- 172.21.15.99:0/3962995808 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562340378100 msgr2=0x562340372ab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:54:17.094 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.089+0000 7fd562956640 1 --2- 172.21.15.99:0/3962995808 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562340378100 0x562340372ab0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:17.094 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.089+0000 7fd562956640 1 -- 172.21.15.99:0/3962995808 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562340378af0 msgr2=0x562340389610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:54:17.094 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.089+0000 7fd562956640 1 --2- 172.21.15.99:0/3962995808 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562340378af0 0x562340389610 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:17.094 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.089+0000 7fd562956640 1 -- 172.21.15.99:0/3962995808 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5623400bd620 con 0x56234037dc60 2024-03-19T01:54:17.094 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.089+0000 7fd561954640 1 --2- 172.21.15.99:0/3962995808 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562340378af0 0x562340389610 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:54:17.094 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.089+0000 7fd562956640 1 --2- 172.21.15.99:0/3962995808 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56234037dc60 0x562340376fe0 secure :-1 s=READY pgs=604 cs=0 l=1 rev1=1 crypto rx=0x562340379350 tx=0x7fd55400ff80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2e21a36f8c5bff59 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:54:17.094 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.089+0000 7fd547fff640 1 -- 172.21.15.99:0/3962995808 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd554034070 con 0x56234037dc60 2024-03-19T01:54:17.094 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.089+0000 7fd547fff640 1 -- 172.21.15.99:0/3962995808 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd554003e50 con 0x56234037dc60 2024-03-19T01:54:17.094 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.089+0000 7fd547fff640 1 -- 172.21.15.99:0/3962995808 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd55402b040 con 0x56234037dc60 2024-03-19T01:54:17.094 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.090+0000 7fd566fa1740 1 -- 172.21.15.99:0/3962995808 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56234037dc60 msgr2=0x562340376fe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:54:17.094 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.090+0000 7fd566fa1740 1 --2- 172.21.15.99:0/3962995808 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56234037dc60 0x562340376fe0 secure :-1 s=READY pgs=604 cs=0 l=1 rev1=1 crypto rx=0x562340379350 tx=0x7fd55400ff80 comp rx=0 tx=0).stop 2024-03-19T01:54:17.095 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.090+0000 7fd566fa1740 1 -- 172.21.15.99:0/3962995808 shutdown_connections 2024-03-19T01:54:17.095 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.090+0000 7fd566fa1740 1 --2- 172.21.15.99:0/3962995808 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562340378af0 0x562340389610 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:17.095 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.090+0000 7fd566fa1740 1 --2- 172.21.15.99:0/3962995808 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56234037dc60 0x562340376fe0 unknown :-1 s=CLOSED pgs=604 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:17.095 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.090+0000 7fd566fa1740 1 --2- 172.21.15.99:0/3962995808 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562340378100 0x562340372ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:17.095 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.090+0000 7fd566fa1740 1 -- 172.21.15.99:0/3962995808 >> 172.21.15.99:0/3962995808 conn(0x562340373670 msgr2=0x5623403768d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:54:17.095 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.090+0000 7fd566fa1740 1 -- 172.21.15.99:0/3962995808 shutdown_connections 2024-03-19T01:54:17.095 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.090+0000 7fd566fa1740 1 -- 172.21.15.99:0/3962995808 wait complete. 2024-03-19T01:54:17.095 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.090+0000 7fd566fa1740 1 -- start start 2024-03-19T01:54:17.095 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.090+0000 7fd566fa1740 1 Processor -- start 2024-03-19T01:54:17.095 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.091+0000 7fd566fa1740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562340378100 0x562340386d60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:17.095 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.091+0000 7fd566fa1740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562340378af0 0x5623403872a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:17.095 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.091+0000 7fd566fa1740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56234037dc60 0x5623403881f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:17.095 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.091+0000 7fd566fa1740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fd54000cc00 con 0x562340378af0 2024-03-19T01:54:17.095 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.091+0000 7fd566fa1740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fd54000e070 con 0x56234037dc60 2024-03-19T01:54:17.096 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.091+0000 7fd566fa1740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fd54000d2d0 con 0x562340378100 2024-03-19T01:54:17.096 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.091+0000 7fd566fa1740 10 client.0 _ll_register_callbacks cb 0x7fd558009700 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:54:17.096 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.091+0000 7fd561954640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562340378af0 0x5623403872a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:17.096 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.091+0000 7fd562155640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562340378100 0x562340386d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:17.096 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.091+0000 7fd562956640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56234037dc60 0x5623403881f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:17.096 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.091+0000 7fd562155640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562340378100 0x562340386d60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:60150/0 (socket says 172.21.15.99:60150) 2024-03-19T01:54:17.096 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.091+0000 7fd561954640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562340378af0 0x5623403872a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:50788/0 (socket says 172.21.15.99:50788) 2024-03-19T01:54:17.096 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.091+0000 7fd562155640 1 -- 172.21.15.99:0/2206166240 learned_addr learned my addr 172.21.15.99:0/2206166240 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:54:17.096 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.091+0000 7fd561954640 1 -- 172.21.15.99:0/2206166240 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562340378100 msgr2=0x562340386d60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:54:17.096 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.091+0000 7fd561954640 1 --2- 172.21.15.99:0/2206166240 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562340378100 0x562340386d60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:17.096 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.091+0000 7fd561954640 1 -- 172.21.15.99:0/2206166240 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56234037dc60 msgr2=0x5623403881f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:54:17.096 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.091+0000 7fd561954640 1 --2- 172.21.15.99:0/2206166240 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56234037dc60 0x5623403881f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:17.096 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.091+0000 7fd561954640 1 -- 172.21.15.99:0/2206166240 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5623403887e0 con 0x562340378af0 2024-03-19T01:54:17.096 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.091+0000 7fd561954640 1 --2- 172.21.15.99:0/2206166240 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562340378af0 0x5623403872a0 secure :-1 s=READY pgs=605 cs=0 l=1 rev1=1 crypto rx=0x7fd54c0edbb0 tx=0x7fd54c08b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=82e7b718e9aec866 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:54:17.097 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.091+0000 7fd545ffb640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:54:17.097 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.091+0000 7fd545ffb640 1 -- 172.21.15.99:0/2206166240 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd54c0c0b50 con 0x562340378af0 2024-03-19T01:54:17.097 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.091+0000 7fd545ffb640 1 -- 172.21.15.99:0/2206166240 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fd54c0badb0 con 0x562340378af0 2024-03-19T01:54:17.097 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.091+0000 7fd566fa1740 1 -- 172.21.15.99:0/2206166240 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fd554003e50 con 0x562340378af0 2024-03-19T01:54:17.097 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.091+0000 7fd545ffb640 1 -- 172.21.15.99:0/2206166240 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=33) ==== 24+0+0 (secure 0 0 0) 0x7fd54c0c51a0 con 0x562340378af0 2024-03-19T01:54:17.097 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.092+0000 7fd566fa1740 10 client.16416 fetch_fsmap learned FSMap version 33 2024-03-19T01:54:17.097 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.092+0000 7fd566fa1740 1 -- 172.21.15.99:0/2206166240 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=33,monmap=4+}) -- 0x5623403f43b0 con 0x562340378af0 2024-03-19T01:54:17.097 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.092+0000 7fd545ffb640 1 -- 172.21.15.99:0/2206166240 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd554003e50 con 0x562340378af0 2024-03-19T01:54:17.097 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.092+0000 7fd545ffb640 1 -- 172.21.15.99:0/2206166240 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 33) ==== 8362+0+0 (secure 0 0 0) 0x7fd54c04e060 con 0x562340378af0 2024-03-19T01:54:17.097 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.092+0000 7fd566fa1740 10 client.16416 fetch_fsmap finished waiting for FSMap version 33 2024-03-19T01:54:17.097 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.092+0000 7fd566fa1740 10 client.16416 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:54:17.097 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.092+0000 7fd566fa1740 20 client.16416 populate_metadata read hostname 'smithi099' 2024-03-19T01:54:17.097 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.092+0000 7fd566fa1740 1 --2- 172.21.15.99:0/2206166240 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5623403f4d70 0x5623403f7190 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:17.097 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.092+0000 7fd566fa1740 4 client.16416 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:54:17.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.092+0000 7fd566fa1740 1 -- 172.21.15.99:0/2206166240 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x5623402e4470 con 0x5623403f4d70 2024-03-19T01:54:17.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.092+0000 7fd562155640 1 --2- 172.21.15.99:0/2206166240 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5623403f4d70 0x5623403f7190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:17.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.092+0000 7fd562155640 1 --2- 172.21.15.99:0/2206166240 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5623403f4d70 0x5623403f7190 crc :-1 s=READY pgs=148 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=977cb0e19890ec9d server_cookie=643352fb3a4f64e in_seq=0 out_seq=0 2024-03-19T01:54:17.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.092+0000 7fd545ffb640 10 client.16416 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:54:17.098 INFO:tasks.workunit.client.0.smithi099.stdout: 10 21764 28.99 MB/sec warmup 46 sec latency 66.812 ms 2024-03-19T01:54:17.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:54:17 smithi099 ceph-mon[62951]: pgmap v747: 97 pgs: 97 active+clean; 452 MiB data, 4.6 GiB used, 1.0 TiB / 1.0 TiB avail; 4.5 MiB/s rd, 19 MiB/s wr, 617 op/s 2024-03-19T01:54:17.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:54:17 smithi112 ceph-mon[66819]: pgmap v747: 97 pgs: 97 active+clean; 452 MiB data, 4.6 GiB used, 1.0 TiB / 1.0 TiB avail; 4.5 MiB/s rd, 19 MiB/s wr, 617 op/s 2024-03-19T01:54:17.462 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.461+0000 7fd545ffb640 1 -- 172.21.15.99:0/2206166240 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x5623402e4470 con 0x5623403f4d70 2024-03-19T01:54:17.462 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.461+0000 7fd545ffb640 10 client.16416 handle_command_reply: tid=0 2024-03-19T01:54:17.462 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.461+0000 7fd566fa1740 1 client.16416 shutdown 2024-03-19T01:54:17.462 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.461+0000 7fd566fa1740 1 -- 172.21.15.99:0/2206166240 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562340378af0 msgr2=0x5623403872a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:54:17.462 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.461+0000 7fd566fa1740 1 --2- 172.21.15.99:0/2206166240 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562340378af0 0x5623403872a0 secure :-1 s=READY pgs=605 cs=0 l=1 rev1=1 crypto rx=0x7fd54c0edbb0 tx=0x7fd54c08b910 comp rx=0 tx=0).stop 2024-03-19T01:54:17.462 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.461+0000 7fd566fa1740 1 -- 172.21.15.99:0/2206166240 shutdown_connections 2024-03-19T01:54:17.462 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.461+0000 7fd566fa1740 1 --2- 172.21.15.99:0/2206166240 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5623403f4d70 0x5623403f7190 crc :-1 s=READY pgs=148 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:17.462 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.461+0000 7fd566fa1740 1 --2- 172.21.15.99:0/2206166240 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56234037dc60 0x5623403881f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:17.462 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.461+0000 7fd566fa1740 1 --2- 172.21.15.99:0/2206166240 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562340378af0 0x5623403872a0 unknown :-1 s=CLOSED pgs=605 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:17.463 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.461+0000 7fd566fa1740 1 --2- 172.21.15.99:0/2206166240 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562340378100 0x562340386d60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:17.463 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.461+0000 7fd566fa1740 1 -- 172.21.15.99:0/2206166240 >> 172.21.15.99:0/2206166240 conn(0x562340373670 msgr2=0x562340377160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:54:17.463 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.461+0000 7fd545ffb640 0 client.16416 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:54:17.463 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.461+0000 7fd566fa1740 1 -- 172.21.15.99:0/2206166240 shutdown_connections 2024-03-19T01:54:17.463 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.461+0000 7fd566fa1740 1 -- 172.21.15.99:0/2206166240 wait complete. 2024-03-19T01:54:17.463 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.462+0000 7fd566fa1740 20 client.16416 trim_cache size 0 max 16384 2024-03-19T01:54:17.463 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:54:17.463 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:54:17.463 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:54:17.463 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:54:17.464 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.462+0000 7fd5643b9640 1 -- 172.21.15.99:0/1120200963 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd5480a28a0 msgr2=0x7fd5480a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:54:17.464 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.462+0000 7fd5643b9640 1 --2- 172.21.15.99:0/1120200963 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd5480a28a0 0x7fd5480a4d60 secure :-1 s=READY pgs=332 cs=0 l=1 rev1=1 crypto rx=0x7fd5540099e0 tx=0x7fd55401e000 comp rx=0 tx=0).stop 2024-03-19T01:54:17.464 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.462+0000 7fd5643b9640 1 -- 172.21.15.99:0/1120200963 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd55c101930 msgr2=0x7fd55c197fe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:54:17.464 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.462+0000 7fd5643b9640 1 --2- 172.21.15.99:0/1120200963 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd55c101930 0x7fd55c197fe0 secure :-1 s=READY pgs=601 cs=0 l=1 rev1=1 crypto rx=0x7fd54c009230 tx=0x7fd54c02c650 comp rx=0 tx=0).stop 2024-03-19T01:54:17.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.462+0000 7fd5643b9640 1 -- 172.21.15.99:0/1120200963 shutdown_connections 2024-03-19T01:54:17.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.462+0000 7fd5643b9640 1 --2- 172.21.15.99:0/1120200963 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd5480a28a0 0x7fd5480a4d60 unknown :-1 s=CLOSED pgs=332 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:17.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.462+0000 7fd5643b9640 1 --2- 172.21.15.99:0/1120200963 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd55c1022f0 0x7fd55c19f050 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:17.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.462+0000 7fd5643b9640 1 --2- 172.21.15.99:0/1120200963 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd55c101930 0x7fd55c197fe0 unknown :-1 s=CLOSED pgs=601 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:17.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.462+0000 7fd5643b9640 1 --2- 172.21.15.99:0/1120200963 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd55c100740 0x7fd55c197aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:17.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.462+0000 7fd5643b9640 1 -- 172.21.15.99:0/1120200963 >> 172.21.15.99:0/1120200963 conn(0x7fd55c0fbcb0 msgr2=0x7fd55c0fd6d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:54:17.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.462+0000 7fd5643b9640 1 -- 172.21.15.99:0/1120200963 shutdown_connections 2024-03-19T01:54:17.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:17.462+0000 7fd5643b9640 1 -- 172.21.15.99:0/1120200963 wait complete. 2024-03-19T01:54:17.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:54:17 smithi115 ceph-mon[67332]: pgmap v747: 97 pgs: 97 active+clean; 452 MiB data, 4.6 GiB used, 1.0 TiB / 1.0 TiB avail; 4.5 MiB/s rd, 19 MiB/s wr, 617 op/s 2024-03-19T01:54:18.098 INFO:tasks.workunit.client.0.smithi099.stdout: 10 22148 28.97 MB/sec warmup 47 sec latency 300.636 ms 2024-03-19T01:54:19.098 INFO:tasks.workunit.client.0.smithi099.stdout: 10 22569 28.93 MB/sec warmup 48 sec latency 302.285 ms 2024-03-19T01:54:20.098 INFO:tasks.workunit.client.0.smithi099.stdout: 10 22993 28.86 MB/sec warmup 49 sec latency 323.070 ms 2024-03-19T01:54:20.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:54:20 smithi115 ceph-mon[67332]: pgmap v748: 97 pgs: 97 active+clean; 454 MiB data, 4.6 GiB used, 1.0 TiB / 1.0 TiB avail; 4.4 MiB/s rd, 9.9 MiB/s wr, 518 op/s 2024-03-19T01:54:20.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:54:20 smithi099 ceph-mon[62951]: pgmap v748: 97 pgs: 97 active+clean; 454 MiB data, 4.6 GiB used, 1.0 TiB / 1.0 TiB avail; 4.4 MiB/s rd, 9.9 MiB/s wr, 518 op/s 2024-03-19T01:54:20.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:54:20 smithi112 ceph-mon[66819]: pgmap v748: 97 pgs: 97 active+clean; 454 MiB data, 4.6 GiB used, 1.0 TiB / 1.0 TiB avail; 4.4 MiB/s rd, 9.9 MiB/s wr, 518 op/s 2024-03-19T01:54:21.098 INFO:tasks.workunit.client.0.smithi099.stdout: 10 23410 28.78 MB/sec warmup 50 sec latency 160.597 ms 2024-03-19T01:54:22.098 INFO:tasks.workunit.client.0.smithi099.stdout: 10 23922 28.85 MB/sec warmup 51 sec latency 178.615 ms 2024-03-19T01:54:22.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:54:22 smithi115 ceph-mon[67332]: pgmap v749: 97 pgs: 97 active+clean; 451 MiB data, 4.6 GiB used, 1.0 TiB / 1.0 TiB avail; 5.0 MiB/s rd, 6.9 MiB/s wr, 525 op/s 2024-03-19T01:54:22.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:54:22 smithi099 ceph-mon[62951]: pgmap v749: 97 pgs: 97 active+clean; 451 MiB data, 4.6 GiB used, 1.0 TiB / 1.0 TiB avail; 5.0 MiB/s rd, 6.9 MiB/s wr, 525 op/s 2024-03-19T01:54:22.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:54:22 smithi112 ceph-mon[66819]: pgmap v749: 97 pgs: 97 active+clean; 451 MiB data, 4.6 GiB used, 1.0 TiB / 1.0 TiB avail; 5.0 MiB/s rd, 6.9 MiB/s wr, 525 op/s 2024-03-19T01:54:23.098 INFO:tasks.workunit.client.0.smithi099.stdout: 10 24315 28.68 MB/sec warmup 52 sec latency 62.480 ms 2024-03-19T01:54:23.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:54:23 smithi115 ceph-mon[67332]: pgmap v750: 97 pgs: 97 active+clean; 455 MiB data, 4.7 GiB used, 1.0 TiB / 1.0 TiB avail; 5.5 MiB/s rd, 7.4 MiB/s wr, 629 op/s 2024-03-19T01:54:23.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:54:23 smithi099 ceph-mon[62951]: pgmap v750: 97 pgs: 97 active+clean; 455 MiB data, 4.7 GiB used, 1.0 TiB / 1.0 TiB avail; 5.5 MiB/s rd, 7.4 MiB/s wr, 629 op/s 2024-03-19T01:54:23.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:54:23 smithi112 ceph-mon[66819]: pgmap v750: 97 pgs: 97 active+clean; 455 MiB data, 4.7 GiB used, 1.0 TiB / 1.0 TiB avail; 5.5 MiB/s rd, 7.4 MiB/s wr, 629 op/s 2024-03-19T01:54:24.098 INFO:tasks.workunit.client.0.smithi099.stdout: 10 24662 28.48 MB/sec warmup 53 sec latency 160.600 ms 2024-03-19T01:54:25.098 INFO:tasks.workunit.client.0.smithi099.stdout: 10 25022 28.18 MB/sec warmup 54 sec latency 197.082 ms 2024-03-19T01:54:26.098 INFO:tasks.workunit.client.0.smithi099.stdout: 10 25533 28.21 MB/sec warmup 55 sec latency 57.115 ms 2024-03-19T01:54:26.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:54:26 smithi115 ceph-mon[67332]: pgmap v751: 97 pgs: 97 active+clean; 451 MiB data, 4.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.8 MiB/s rd, 5.9 MiB/s wr, 551 op/s 2024-03-19T01:54:26.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:54:26 smithi099 ceph-mon[62951]: pgmap v751: 97 pgs: 97 active+clean; 451 MiB data, 4.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.8 MiB/s rd, 5.9 MiB/s wr, 551 op/s 2024-03-19T01:54:26.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:54:26 smithi112 ceph-mon[66819]: pgmap v751: 97 pgs: 97 active+clean; 451 MiB data, 4.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.8 MiB/s rd, 5.9 MiB/s wr, 551 op/s 2024-03-19T01:54:27.098 INFO:tasks.workunit.client.0.smithi099.stdout: 10 26045 28.13 MB/sec warmup 56 sec latency 133.723 ms 2024-03-19T01:54:28.098 INFO:tasks.workunit.client.0.smithi099.stdout: 10 26704 28.06 MB/sec warmup 57 sec latency 59.976 ms 2024-03-19T01:54:28.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:54:28 smithi115 ceph-mon[67332]: pgmap v752: 97 pgs: 97 active+clean; 455 MiB data, 4.7 GiB used, 1.0 TiB / 1.0 TiB avail; 6.1 MiB/s rd, 7.0 MiB/s wr, 618 op/s 2024-03-19T01:54:28.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:54:28 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:54:28.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:54:28 smithi099 ceph-mon[62951]: pgmap v752: 97 pgs: 97 active+clean; 455 MiB data, 4.7 GiB used, 1.0 TiB / 1.0 TiB avail; 6.1 MiB/s rd, 7.0 MiB/s wr, 618 op/s 2024-03-19T01:54:28.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:54:28 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:54:28.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:54:28 smithi112 ceph-mon[66819]: pgmap v752: 97 pgs: 97 active+clean; 455 MiB data, 4.7 GiB used, 1.0 TiB / 1.0 TiB avail; 6.1 MiB/s rd, 7.0 MiB/s wr, 618 op/s 2024-03-19T01:54:28.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:54:28 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:54:29.098 INFO:tasks.workunit.client.0.smithi099.stdout: 10 27220 28.18 MB/sec warmup 58 sec latency 113.332 ms 2024-03-19T01:54:29.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:54:29 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:54:29.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:54:29 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:54:29.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:54:29 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:54:29.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:54:29 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:54:29.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:54:29 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:54:29.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:54:29 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:54:29.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:54:29 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:54:29.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:54:29 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:54:29.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:54:29 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:54:30.098 INFO:tasks.workunit.client.0.smithi099.stdout: 10 27600 28.17 MB/sec warmup 59 sec latency 301.489 ms 2024-03-19T01:54:30.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:54:30 smithi115 ceph-mon[67332]: pgmap v753: 97 pgs: 97 active+clean; 457 MiB data, 4.7 GiB used, 1.0 TiB / 1.0 TiB avail; 5.4 MiB/s rd, 6.1 MiB/s wr, 563 op/s 2024-03-19T01:54:30.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:54:30 smithi099 ceph-mon[62951]: pgmap v753: 97 pgs: 97 active+clean; 457 MiB data, 4.7 GiB used, 1.0 TiB / 1.0 TiB avail; 5.4 MiB/s rd, 6.1 MiB/s wr, 563 op/s 2024-03-19T01:54:30.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:54:30 smithi112 ceph-mon[66819]: pgmap v753: 97 pgs: 97 active+clean; 457 MiB data, 4.7 GiB used, 1.0 TiB / 1.0 TiB avail; 5.4 MiB/s rd, 6.1 MiB/s wr, 563 op/s 2024-03-19T01:54:30.560 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.558+0000 7fe49d0ac640 1 -- 172.21.15.99:0/1148348976 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe498100740 msgr2=0x7fe498100b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:54:30.560 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.558+0000 7fe49d0ac640 1 --2- 172.21.15.99:0/1148348976 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe498100740 0x7fe498100b60 secure :-1 s=READY pgs=606 cs=0 l=1 rev1=1 crypto rx=0x7fe488009900 tx=0x7fe48802bc30 comp rx=0 tx=0).stop 2024-03-19T01:54:30.560 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.558+0000 7fe49d0ac640 1 -- 172.21.15.99:0/1148348976 shutdown_connections 2024-03-19T01:54:30.560 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.558+0000 7fe49d0ac640 1 --2- 172.21.15.99:0/1148348976 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fe498101930 0x7fe498101db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:30.561 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.558+0000 7fe49d0ac640 1 --2- 172.21.15.99:0/1148348976 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fe4981022f0 0x7fe498108b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:30.561 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.558+0000 7fe49d0ac640 1 --2- 172.21.15.99:0/1148348976 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe498100740 0x7fe498100b60 unknown :-1 s=CLOSED pgs=606 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:30.561 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.558+0000 7fe49d0ac640 1 -- 172.21.15.99:0/1148348976 >> 172.21.15.99:0/1148348976 conn(0x7fe4980fbcb0 msgr2=0x7fe4980fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:54:30.561 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.558+0000 7fe49d0ac640 1 -- 172.21.15.99:0/1148348976 shutdown_connections 2024-03-19T01:54:30.561 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.558+0000 7fe49d0ac640 1 -- 172.21.15.99:0/1148348976 wait complete. 2024-03-19T01:54:30.561 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.558+0000 7fe49d0ac640 1 Processor -- start 2024-03-19T01:54:30.561 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.559+0000 7fe49d0ac640 1 -- start start 2024-03-19T01:54:30.561 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.559+0000 7fe49d0ac640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fe498100740 0x7fe498069a10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:30.561 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.559+0000 7fe49d0ac640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe498101930 0x7fe49819ec50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:30.561 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.559+0000 7fe49d0ac640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fe4981022f0 0x7fe4981a11a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:30.562 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.559+0000 7fe49d0ac640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fe49810c390 con 0x7fe498101930 2024-03-19T01:54:30.562 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.559+0000 7fe49d0ac640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fe49810c210 con 0x7fe498100740 2024-03-19T01:54:30.562 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.559+0000 7fe49d0ac640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fe49810c510 con 0x7fe4981022f0 2024-03-19T01:54:30.562 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.559+0000 7fe48ffff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe498101930 0x7fe49819ec50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:30.562 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.559+0000 7fe48ffff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe498101930 0x7fe49819ec50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:54944/0 (socket says 172.21.15.99:54944) 2024-03-19T01:54:30.562 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.559+0000 7fe48ffff640 1 -- 172.21.15.99:0/1007566149 learned_addr learned my addr 172.21.15.99:0/1007566149 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:54:30.562 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.559+0000 7fe496d9d640 1 --2- 172.21.15.99:0/1007566149 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fe498100740 0x7fe498069a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:30.563 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.559+0000 7fe49759e640 1 --2- 172.21.15.99:0/1007566149 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fe4981022f0 0x7fe4981a11a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:30.563 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.559+0000 7fe48ffff640 1 -- 172.21.15.99:0/1007566149 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fe4981022f0 msgr2=0x7fe4981a11a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:54:30.563 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.559+0000 7fe48ffff640 1 --2- 172.21.15.99:0/1007566149 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fe4981022f0 0x7fe4981a11a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:30.563 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.559+0000 7fe48ffff640 1 -- 172.21.15.99:0/1007566149 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fe498100740 msgr2=0x7fe498069a10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:54:30.563 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.559+0000 7fe48ffff640 1 --2- 172.21.15.99:0/1007566149 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fe498100740 0x7fe498069a10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:30.563 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.559+0000 7fe48ffff640 1 -- 172.21.15.99:0/1007566149 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fe4981a1990 con 0x7fe498101930 2024-03-19T01:54:30.563 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.559+0000 7fe496d9d640 1 --2- 172.21.15.99:0/1007566149 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fe498100740 0x7fe498069a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:54:30.564 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.559+0000 7fe48ffff640 1 --2- 172.21.15.99:0/1007566149 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe498101930 0x7fe49819ec50 secure :-1 s=READY pgs=607 cs=0 l=1 rev1=1 crypto rx=0x7fe48000e9f0 tx=0x7fe48000eeb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:54:30.564 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.559+0000 7fe494d99640 1 -- 172.21.15.99:0/1007566149 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe48000cd80 con 0x7fe498101930 2024-03-19T01:54:30.564 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.559+0000 7fe49d0ac640 1 -- 172.21.15.99:0/1007566149 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fe4981a1c20 con 0x7fe498101930 2024-03-19T01:54:30.564 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.559+0000 7fe494d99640 1 -- 172.21.15.99:0/1007566149 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe480010910 con 0x7fe498101930 2024-03-19T01:54:30.564 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.560+0000 7fe494d99640 1 -- 172.21.15.99:0/1007566149 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe4800188f0 con 0x7fe498101930 2024-03-19T01:54:30.564 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.560+0000 7fe49d0ac640 1 -- 172.21.15.99:0/1007566149 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fe4981a2170 con 0x7fe498101930 2024-03-19T01:54:30.571 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.562+0000 7fe494d99640 1 -- 172.21.15.99:0/1007566149 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7fe480010ac0 con 0x7fe498101930 2024-03-19T01:54:30.571 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.562+0000 7fe49fd75740 1 Processor -- start 2024-03-19T01:54:30.571 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.562+0000 7fe49fd75740 1 -- start start 2024-03-19T01:54:30.571 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.563+0000 7fe49fd75740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a1be282100 0x55a1be282520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:30.571 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.563+0000 7fe49fd75740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a1be282af0 0x55a1be2875f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:30.571 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.563+0000 7fe494d99640 1 --2- 172.21.15.99:0/1007566149 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fe4840a28d0 0x7fe4840a4d90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:30.571 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.563+0000 7fe49fd75740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a1be287c60 0x55a1be28a0f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:30.571 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.563+0000 7fe49fd75740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55a1be1e6be0 con 0x55a1be282af0 2024-03-19T01:54:30.571 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.563+0000 7fe49fd75740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55a1be18f960 con 0x55a1be282100 2024-03-19T01:54:30.572 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.563+0000 7fe49fd75740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55a1be1e2c80 con 0x55a1be287c60 2024-03-19T01:54:30.572 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.563+0000 7fe48ffff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a1be282af0 0x55a1be2875f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:30.572 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.563+0000 7fe49759e640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a1be282100 0x55a1be282520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:30.572 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.563+0000 7fe496d9d640 1 --2- 172.21.15.99:0/1007566149 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fe4840a28d0 0x7fe4840a4d90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:30.572 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.563+0000 7fe49759e640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a1be287c60 0x55a1be28a0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:30.572 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.563+0000 7fe48ffff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a1be282af0 0x55a1be2875f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:54950/0 (socket says 172.21.15.99:54950) 2024-03-19T01:54:30.572 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.563+0000 7fe48ffff640 1 -- 172.21.15.99:0/149968448 learned_addr learned my addr 172.21.15.99:0/149968448 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:54:30.572 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.563+0000 7fe49759e640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a1be287c60 0x55a1be28a0f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:58922/0 (socket says 172.21.15.99:58922) 2024-03-19T01:54:30.572 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.563+0000 7fe494d99640 1 -- 172.21.15.99:0/1007566149 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fe4800cee40 con 0x7fe498101930 2024-03-19T01:54:30.572 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.563+0000 7fe48ffff640 1 -- 172.21.15.99:0/149968448 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a1be287c60 msgr2=0x55a1be28a0f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:54:30.572 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.563+0000 7fe48ffff640 1 --2- 172.21.15.99:0/149968448 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a1be287c60 0x55a1be28a0f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:30.572 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.563+0000 7fe48ffff640 1 -- 172.21.15.99:0/149968448 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a1be282100 msgr2=0x55a1be282520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:54:30.572 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.563+0000 7fe48ffff640 1 --2- 172.21.15.99:0/149968448 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a1be282100 0x55a1be282520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:30.573 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.563+0000 7fe48ffff640 1 -- 172.21.15.99:0/149968448 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55a1be186d20 con 0x55a1be282af0 2024-03-19T01:54:30.573 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.563+0000 7fe49759e640 1 --2- 172.21.15.99:0/149968448 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a1be287c60 0x55a1be28a0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:54:30.573 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.563+0000 7fe496d9d640 1 --2- 172.21.15.99:0/1007566149 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fe4840a28d0 0x7fe4840a4d90 secure :-1 s=READY pgs=333 cs=0 l=1 rev1=1 crypto rx=0x7fe488000c00 tx=0x7fe488005fd0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:54:30.573 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.563+0000 7fe48ffff640 1 --2- 172.21.15.99:0/149968448 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a1be282af0 0x55a1be2875f0 secure :-1 s=READY pgs=608 cs=0 l=1 rev1=1 crypto rx=0x7fe480097cb0 tx=0x7fe48006d590 comp rx=0 tx=0).ready entity=mon.0 client_cookie=214078959a58011f server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:54:30.573 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.563+0000 7fe48d7fa640 1 -- 172.21.15.99:0/149968448 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe48009f8f0 con 0x55a1be282af0 2024-03-19T01:54:30.573 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.563+0000 7fe48d7fa640 1 -- 172.21.15.99:0/149968448 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe4800a3f40 con 0x55a1be282af0 2024-03-19T01:54:30.573 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.563+0000 7fe48d7fa640 1 -- 172.21.15.99:0/149968448 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe4800c93b0 con 0x55a1be282af0 2024-03-19T01:54:30.573 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.564+0000 7fe49fd75740 1 -- 172.21.15.99:0/149968448 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a1be282af0 msgr2=0x55a1be2875f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:54:30.573 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.564+0000 7fe49fd75740 1 --2- 172.21.15.99:0/149968448 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a1be282af0 0x55a1be2875f0 secure :-1 s=READY pgs=608 cs=0 l=1 rev1=1 crypto rx=0x7fe480097cb0 tx=0x7fe48006d590 comp rx=0 tx=0).stop 2024-03-19T01:54:30.573 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.564+0000 7fe49fd75740 1 -- 172.21.15.99:0/149968448 shutdown_connections 2024-03-19T01:54:30.573 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.564+0000 7fe49fd75740 1 --2- 172.21.15.99:0/149968448 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a1be287c60 0x55a1be28a0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:30.573 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.564+0000 7fe49fd75740 1 --2- 172.21.15.99:0/149968448 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a1be282af0 0x55a1be2875f0 unknown :-1 s=CLOSED pgs=608 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:30.573 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.564+0000 7fe49fd75740 1 --2- 172.21.15.99:0/149968448 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a1be282100 0x55a1be282520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:30.573 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.564+0000 7fe49fd75740 1 -- 172.21.15.99:0/149968448 >> 172.21.15.99:0/149968448 conn(0x55a1be27d670 msgr2=0x55a1be27da70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:54:30.574 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.564+0000 7fe49fd75740 1 -- 172.21.15.99:0/149968448 shutdown_connections 2024-03-19T01:54:30.574 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.564+0000 7fe49fd75740 1 -- 172.21.15.99:0/149968448 wait complete. 2024-03-19T01:54:30.574 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.565+0000 7fe49fd75740 1 -- start start 2024-03-19T01:54:30.574 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.565+0000 7fe49fd75740 1 Processor -- start 2024-03-19T01:54:30.574 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.565+0000 7fe49fd75740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a1be282100 0x55a1be28c0e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:30.574 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.565+0000 7fe49fd75740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a1be282af0 0x55a1be28c620 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:30.574 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.565+0000 7fe49fd75740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a1be287c60 0x55a1be293660 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:30.574 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.565+0000 7fe49fd75740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55a1be28abb0 con 0x55a1be282100 2024-03-19T01:54:30.574 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.565+0000 7fe49fd75740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55a1be1a2db0 con 0x55a1be282af0 2024-03-19T01:54:30.574 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.565+0000 7fe49fd75740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55a1be28ad30 con 0x55a1be287c60 2024-03-19T01:54:30.574 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.565+0000 7fe49fd75740 10 client.0 _ll_register_callbacks cb 0x55a1be1a7e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:54:30.575 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.565+0000 7fe496d9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a1be282100 0x55a1be28c0e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:30.575 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.565+0000 7fe48ffff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a1be282af0 0x55a1be28c620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:30.575 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.565+0000 7fe496d9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a1be282100 0x55a1be28c0e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:54962/0 (socket says 172.21.15.99:54962) 2024-03-19T01:54:30.575 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.565+0000 7fe496d9d640 1 -- 172.21.15.99:0/3585613059 learned_addr learned my addr 172.21.15.99:0/3585613059 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:54:30.575 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.565+0000 7fe49759e640 1 --2- 172.21.15.99:0/3585613059 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a1be287c60 0x55a1be293660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:30.575 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.565+0000 7fe496d9d640 1 -- 172.21.15.99:0/3585613059 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a1be287c60 msgr2=0x55a1be293660 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:54:30.575 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.565+0000 7fe496d9d640 1 --2- 172.21.15.99:0/3585613059 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a1be287c60 0x55a1be293660 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:30.575 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.565+0000 7fe496d9d640 1 -- 172.21.15.99:0/3585613059 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a1be282af0 msgr2=0x55a1be28c620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:54:30.575 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.565+0000 7fe496d9d640 1 --2- 172.21.15.99:0/3585613059 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a1be282af0 0x55a1be28c620 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:30.575 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.565+0000 7fe496d9d640 1 -- 172.21.15.99:0/3585613059 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55a1be1a1c60 con 0x55a1be282100 2024-03-19T01:54:30.575 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.565+0000 7fe48ffff640 1 --2- 172.21.15.99:0/3585613059 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a1be282af0 0x55a1be28c620 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:54:30.575 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.565+0000 7fe496d9d640 1 --2- 172.21.15.99:0/3585613059 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a1be282100 0x55a1be28c0e0 secure :-1 s=READY pgs=609 cs=0 l=1 rev1=1 crypto rx=0x7fe488009900 tx=0x7fe488009260 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1c33d5222d0ca20e server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:54:30.575 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.565+0000 7fe47e7fc640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:54:30.576 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.566+0000 7fe47e7fc640 1 -- 172.21.15.99:0/3585613059 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe48804e070 con 0x55a1be282100 2024-03-19T01:54:30.576 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.566+0000 7fe47e7fc640 1 -- 172.21.15.99:0/3585613059 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fe488048440 con 0x55a1be282100 2024-03-19T01:54:30.576 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.566+0000 7fe49fd75740 1 -- 172.21.15.99:0/3585613059 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fe4800a3f40 con 0x55a1be282100 2024-03-19T01:54:30.576 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.566+0000 7fe47e7fc640 1 -- 172.21.15.99:0/3585613059 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=33) ==== 24+0+0 (secure 0 0 0) 0x7fe48805a050 con 0x55a1be282100 2024-03-19T01:54:30.576 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.566+0000 7fe49fd75740 10 client.16428 fetch_fsmap learned FSMap version 33 2024-03-19T01:54:30.576 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.566+0000 7fe49fd75740 1 -- 172.21.15.99:0/3585613059 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=33,monmap=4+}) -- 0x55a1bdefb210 con 0x55a1be282100 2024-03-19T01:54:30.576 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.566+0000 7fe47e7fc640 1 -- 172.21.15.99:0/3585613059 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe4800a3f40 con 0x55a1be282100 2024-03-19T01:54:30.576 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.566+0000 7fe47e7fc640 1 -- 172.21.15.99:0/3585613059 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 33) ==== 8362+0+0 (secure 0 0 0) 0x7fe4880347a0 con 0x55a1be282100 2024-03-19T01:54:30.576 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.566+0000 7fe49fd75740 10 client.16428 fetch_fsmap finished waiting for FSMap version 33 2024-03-19T01:54:30.576 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.566+0000 7fe49fd75740 10 client.16428 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:54:30.576 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.566+0000 7fe49fd75740 20 client.16428 populate_metadata read hostname 'smithi099' 2024-03-19T01:54:30.576 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.566+0000 7fe49fd75740 1 --2- 172.21.15.99:0/3585613059 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55a1be297fb0 0x55a1be29a420 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:30.576 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.566+0000 7fe49fd75740 4 client.16428 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:54:30.577 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.566+0000 7fe49fd75740 1 -- 172.21.15.99:0/3585613059 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55a1be1f4120 con 0x55a1be297fb0 2024-03-19T01:54:30.577 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.566+0000 7fe48ffff640 1 --2- 172.21.15.99:0/3585613059 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55a1be297fb0 0x55a1be29a420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:30.577 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.567+0000 7fe47e7fc640 10 client.16428 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:54:30.577 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.567+0000 7fe48ffff640 1 --2- 172.21.15.99:0/3585613059 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55a1be297fb0 0x55a1be29a420 crc :-1 s=READY pgs=149 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=71dd40cd221f5f4c server_cookie=40bfba1cd3a9aac0 in_seq=0 out_seq=0 2024-03-19T01:54:30.577 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.569+0000 7fe47e7fc640 1 -- 172.21.15.99:0/3585613059 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x55a1be1f4120 con 0x55a1be297fb0 2024-03-19T01:54:30.577 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.569+0000 7fe47e7fc640 10 client.16428 handle_command_reply: tid=0 2024-03-19T01:54:30.577 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.569+0000 7fe49fd75740 1 client.16428 shutdown 2024-03-19T01:54:30.577 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.570+0000 7fe49fd75740 1 -- 172.21.15.99:0/3585613059 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a1be282100 msgr2=0x55a1be28c0e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:54:30.577 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.570+0000 7fe49fd75740 1 --2- 172.21.15.99:0/3585613059 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a1be282100 0x55a1be28c0e0 secure :-1 s=READY pgs=609 cs=0 l=1 rev1=1 crypto rx=0x7fe488009900 tx=0x7fe488009260 comp rx=0 tx=0).stop 2024-03-19T01:54:30.577 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.570+0000 7fe49fd75740 1 -- 172.21.15.99:0/3585613059 shutdown_connections 2024-03-19T01:54:30.577 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.570+0000 7fe49fd75740 1 --2- 172.21.15.99:0/3585613059 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55a1be297fb0 0x55a1be29a420 crc :-1 s=READY pgs=149 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:30.577 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.570+0000 7fe47e7fc640 0 client.16428 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:54:30.578 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.570+0000 7fe49fd75740 1 --2- 172.21.15.99:0/3585613059 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a1be282af0 0x55a1be28c620 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:30.578 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.570+0000 7fe49fd75740 1 --2- 172.21.15.99:0/3585613059 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a1be287c60 0x55a1be293660 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:30.578 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.570+0000 7fe49fd75740 1 --2- 172.21.15.99:0/3585613059 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a1be282100 0x55a1be28c0e0 unknown :-1 s=CLOSED pgs=609 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:30.578 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.570+0000 7fe49fd75740 1 -- 172.21.15.99:0/3585613059 >> 172.21.15.99:0/3585613059 conn(0x55a1be27d670 msgr2=0x55a1be27e8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:54:30.578 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.570+0000 7fe49fd75740 1 -- 172.21.15.99:0/3585613059 shutdown_connections 2024-03-19T01:54:30.578 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.570+0000 7fe49fd75740 1 -- 172.21.15.99:0/3585613059 wait complete. 2024-03-19T01:54:30.578 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.570+0000 7fe49fd75740 20 client.16428 trim_cache size 0 max 16384 2024-03-19T01:54:30.594 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.589+0000 7fe49fd75740 1 Processor -- start 2024-03-19T01:54:30.594 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.589+0000 7fe49fd75740 1 -- start start 2024-03-19T01:54:30.594 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.589+0000 7fe49fd75740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a1be282100 0x55a1be28c580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:30.594 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.589+0000 7fe49fd75740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a1be282af0 0x55a1be293620 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:30.594 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.589+0000 7fe49fd75740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a1be287c60 0x55a1be27c930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:30.595 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.589+0000 7fe49fd75740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fe4981ad980 con 0x55a1be287c60 2024-03-19T01:54:30.595 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.589+0000 7fe49fd75740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fe4981ab190 con 0x55a1be282af0 2024-03-19T01:54:30.595 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.589+0000 7fe49fd75740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fe4981a8fe0 con 0x55a1be282100 2024-03-19T01:54:30.595 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.589+0000 7fe49759e640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a1be287c60 0x55a1be27c930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:30.595 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.589+0000 7fe48ffff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a1be282af0 0x55a1be293620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:30.595 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.589+0000 7fe49759e640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a1be287c60 0x55a1be27c930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:54978/0 (socket says 172.21.15.99:54978) 2024-03-19T01:54:30.595 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.589+0000 7fe49759e640 1 -- 172.21.15.99:0/812465760 learned_addr learned my addr 172.21.15.99:0/812465760 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:54:30.595 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.589+0000 7fe496d9d640 1 --2- 172.21.15.99:0/812465760 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a1be282100 0x55a1be28c580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:30.595 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.589+0000 7fe49759e640 1 -- 172.21.15.99:0/812465760 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a1be282100 msgr2=0x55a1be28c580 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:54:30.596 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.589+0000 7fe49759e640 1 --2- 172.21.15.99:0/812465760 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a1be282100 0x55a1be28c580 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:30.596 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.589+0000 7fe49759e640 1 -- 172.21.15.99:0/812465760 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a1be282af0 msgr2=0x55a1be293620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:54:30.596 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.589+0000 7fe49759e640 1 --2- 172.21.15.99:0/812465760 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a1be282af0 0x55a1be293620 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:30.596 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.589+0000 7fe49759e640 1 -- 172.21.15.99:0/812465760 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55a1bdfc7620 con 0x55a1be287c60 2024-03-19T01:54:30.596 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.589+0000 7fe496d9d640 1 --2- 172.21.15.99:0/812465760 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a1be282100 0x55a1be28c580 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:54:30.596 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.589+0000 7fe48ffff640 1 --2- 172.21.15.99:0/812465760 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a1be282af0 0x55a1be293620 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:54:30.596 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.590+0000 7fe49759e640 1 --2- 172.21.15.99:0/812465760 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a1be287c60 0x55a1be27c930 secure :-1 s=READY pgs=610 cs=0 l=1 rev1=1 crypto rx=0x7fe490010a70 tx=0x7fe490009e70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=223bd792049cac9e server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:54:30.596 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.590+0000 7fe48cff9640 1 -- 172.21.15.99:0/812465760 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe490022020 con 0x55a1be287c60 2024-03-19T01:54:30.596 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.590+0000 7fe48cff9640 1 -- 172.21.15.99:0/812465760 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe49000fbc0 con 0x55a1be287c60 2024-03-19T01:54:30.596 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.590+0000 7fe48cff9640 1 -- 172.21.15.99:0/812465760 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe490019040 con 0x55a1be287c60 2024-03-19T01:54:30.597 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.590+0000 7fe49fd75740 1 -- 172.21.15.99:0/812465760 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a1be287c60 msgr2=0x55a1be27c930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:54:30.597 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.590+0000 7fe49fd75740 1 --2- 172.21.15.99:0/812465760 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a1be287c60 0x55a1be27c930 secure :-1 s=READY pgs=610 cs=0 l=1 rev1=1 crypto rx=0x7fe490010a70 tx=0x7fe490009e70 comp rx=0 tx=0).stop 2024-03-19T01:54:30.597 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.591+0000 7fe49fd75740 1 -- 172.21.15.99:0/812465760 shutdown_connections 2024-03-19T01:54:30.597 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.591+0000 7fe49fd75740 1 --2- 172.21.15.99:0/812465760 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a1be282af0 0x55a1be293620 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:30.597 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.591+0000 7fe49fd75740 1 --2- 172.21.15.99:0/812465760 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a1be287c60 0x55a1be27c930 unknown :-1 s=CLOSED pgs=610 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:30.597 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.591+0000 7fe49fd75740 1 --2- 172.21.15.99:0/812465760 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a1be282100 0x55a1be28c580 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:30.597 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.591+0000 7fe49fd75740 1 -- 172.21.15.99:0/812465760 >> 172.21.15.99:0/812465760 conn(0x55a1be27d670 msgr2=0x55a1be27c220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:54:30.597 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.591+0000 7fe49fd75740 1 -- 172.21.15.99:0/812465760 shutdown_connections 2024-03-19T01:54:30.597 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.591+0000 7fe49fd75740 1 -- 172.21.15.99:0/812465760 wait complete. 2024-03-19T01:54:30.598 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.591+0000 7fe49fd75740 1 -- start start 2024-03-19T01:54:30.598 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.591+0000 7fe49fd75740 1 Processor -- start 2024-03-19T01:54:30.598 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.591+0000 7fe49fd75740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a1be282100 0x55a1be290ec0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:30.598 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.591+0000 7fe49fd75740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a1be282af0 0x55a1be291400 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:30.598 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.591+0000 7fe49fd75740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a1be287c60 0x55a1be292410 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:30.598 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.591+0000 7fe49fd75740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fe4981a9650 con 0x55a1be282af0 2024-03-19T01:54:30.598 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.591+0000 7fe49fd75740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fe4981aaac0 con 0x55a1be282100 2024-03-19T01:54:30.598 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.591+0000 7fe49fd75740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fe4981a9d20 con 0x55a1be287c60 2024-03-19T01:54:30.599 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.591+0000 7fe49fd75740 10 client.0 _ll_register_callbacks cb 0x7fe488041050 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:54:30.599 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.591+0000 7fe48ffff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a1be282af0 0x55a1be291400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:30.599 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.591+0000 7fe48ffff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a1be282af0 0x55a1be291400 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:54984/0 (socket says 172.21.15.99:54984) 2024-03-19T01:54:30.599 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.591+0000 7fe48ffff640 1 -- 172.21.15.99:0/3439468272 learned_addr learned my addr 172.21.15.99:0/3439468272 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:54:30.599 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.591+0000 7fe496d9d640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a1be282100 0x55a1be290ec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:30.599 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.591+0000 7fe49759e640 1 --2- 172.21.15.99:0/3439468272 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a1be287c60 0x55a1be292410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:30.599 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.592+0000 7fe48ffff640 1 -- 172.21.15.99:0/3439468272 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a1be287c60 msgr2=0x55a1be292410 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:54:30.599 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.592+0000 7fe48ffff640 1 --2- 172.21.15.99:0/3439468272 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a1be287c60 0x55a1be292410 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:30.599 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.592+0000 7fe48ffff640 1 -- 172.21.15.99:0/3439468272 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a1be282100 msgr2=0x55a1be290ec0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:54:30.599 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.592+0000 7fe48ffff640 1 --2- 172.21.15.99:0/3439468272 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a1be282100 0x55a1be290ec0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:30.599 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.592+0000 7fe48ffff640 1 -- 172.21.15.99:0/3439468272 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55a1be292a00 con 0x55a1be282af0 2024-03-19T01:54:30.600 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.592+0000 7fe48ffff640 1 --2- 172.21.15.99:0/3439468272 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a1be282af0 0x55a1be291400 secure :-1 s=READY pgs=611 cs=0 l=1 rev1=1 crypto rx=0x7fe4800cc950 tx=0x7fe480086040 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6ed97bc55535918a server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:54:30.600 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.592+0000 7fe47e7fc640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:54:30.600 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.592+0000 7fe47e7fc640 1 -- 172.21.15.99:0/3439468272 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe480097af0 con 0x55a1be282af0 2024-03-19T01:54:30.600 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.592+0000 7fe47e7fc640 1 -- 172.21.15.99:0/3439468272 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fe480097ec0 con 0x55a1be282af0 2024-03-19T01:54:30.600 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.592+0000 7fe49fd75740 1 -- 172.21.15.99:0/3439468272 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fe49000fbc0 con 0x55a1be282af0 2024-03-19T01:54:30.600 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.592+0000 7fe47e7fc640 1 -- 172.21.15.99:0/3439468272 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=33) ==== 24+0+0 (secure 0 0 0) 0x7fe4800a3f40 con 0x55a1be282af0 2024-03-19T01:54:30.600 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.592+0000 7fe49fd75740 10 client.16434 fetch_fsmap learned FSMap version 33 2024-03-19T01:54:30.600 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.592+0000 7fe49fd75740 1 -- 172.21.15.99:0/3439468272 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=33,monmap=4+}) -- 0x55a1be2fe4c0 con 0x55a1be282af0 2024-03-19T01:54:30.600 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.592+0000 7fe47e7fc640 1 -- 172.21.15.99:0/3439468272 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fe49000fbc0 con 0x55a1be282af0 2024-03-19T01:54:30.600 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.592+0000 7fe47e7fc640 1 -- 172.21.15.99:0/3439468272 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 33) ==== 8362+0+0 (secure 0 0 0) 0x7fe4800afb80 con 0x55a1be282af0 2024-03-19T01:54:30.600 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.592+0000 7fe49fd75740 10 client.16434 fetch_fsmap finished waiting for FSMap version 33 2024-03-19T01:54:30.600 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.592+0000 7fe49fd75740 10 client.16434 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:54:30.600 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.592+0000 7fe49fd75740 20 client.16434 populate_metadata read hostname 'smithi099' 2024-03-19T01:54:30.600 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.592+0000 7fe49fd75740 1 --2- 172.21.15.99:0/3439468272 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55a1be297fb0 0x55a1be300b00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:30.601 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.592+0000 7fe49fd75740 4 client.16434 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:54:30.601 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.592+0000 7fe49fd75740 1 -- 172.21.15.99:0/3439468272 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55a1be1ee470 con 0x55a1be297fb0 2024-03-19T01:54:30.601 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.593+0000 7fe496d9d640 1 --2- 172.21.15.99:0/3439468272 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55a1be297fb0 0x55a1be300b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:30.601 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.593+0000 7fe47e7fc640 10 client.16434 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:54:30.601 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:30.593+0000 7fe496d9d640 1 --2- 172.21.15.99:0/3439468272 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55a1be297fb0 0x55a1be300b00 crc :-1 s=READY pgs=150 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=b139d2cbd198160 server_cookie=e04b76f673b3fd50 in_seq=0 out_seq=0 2024-03-19T01:54:31.013 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:31.013+0000 7fe47e7fc640 1 -- 172.21.15.99:0/3439468272 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x55a1be1ee470 con 0x55a1be297fb0 2024-03-19T01:54:31.013 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:31.013+0000 7fe47e7fc640 10 client.16434 handle_command_reply: tid=0 2024-03-19T01:54:31.013 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:31.013+0000 7fe49fd75740 1 client.16434 shutdown 2024-03-19T01:54:31.014 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:31.013+0000 7fe49fd75740 1 -- 172.21.15.99:0/3439468272 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a1be282af0 msgr2=0x55a1be291400 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:54:31.014 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:31.013+0000 7fe49fd75740 1 --2- 172.21.15.99:0/3439468272 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a1be282af0 0x55a1be291400 secure :-1 s=READY pgs=611 cs=0 l=1 rev1=1 crypto rx=0x7fe4800cc950 tx=0x7fe480086040 comp rx=0 tx=0).stop 2024-03-19T01:54:31.014 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:31.013+0000 7fe49fd75740 1 -- 172.21.15.99:0/3439468272 shutdown_connections 2024-03-19T01:54:31.014 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:31.013+0000 7fe49fd75740 1 --2- 172.21.15.99:0/3439468272 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55a1be297fb0 0x55a1be300b00 crc :-1 s=READY pgs=150 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:31.014 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:31.013+0000 7fe49fd75740 1 --2- 172.21.15.99:0/3439468272 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a1be287c60 0x55a1be292410 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:31.014 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:31.013+0000 7fe47e7fc640 0 client.16434 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:54:31.014 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:31.013+0000 7fe49fd75740 1 --2- 172.21.15.99:0/3439468272 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a1be282af0 0x55a1be291400 unknown :-1 s=CLOSED pgs=611 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:31.014 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:31.013+0000 7fe49fd75740 1 --2- 172.21.15.99:0/3439468272 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a1be282100 0x55a1be290ec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:31.014 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:31.013+0000 7fe49fd75740 1 -- 172.21.15.99:0/3439468272 >> 172.21.15.99:0/3439468272 conn(0x55a1be27d670 msgr2=0x55a1be28c400 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:54:31.014 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:31.013+0000 7fe49fd75740 1 -- 172.21.15.99:0/3439468272 shutdown_connections 2024-03-19T01:54:31.014 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:31.013+0000 7fe49fd75740 1 -- 172.21.15.99:0/3439468272 wait complete. 2024-03-19T01:54:31.014 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:31.013+0000 7fe49fd75740 20 client.16434 trim_cache size 0 max 16384 2024-03-19T01:54:31.015 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:54:31.015 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:54:31.015 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:54:31.015 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:54:31.016 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:31.014+0000 7fe49d0ac640 1 -- 172.21.15.99:0/1007566149 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fe4840a28d0 msgr2=0x7fe4840a4d90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:54:31.016 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:31.014+0000 7fe49d0ac640 1 --2- 172.21.15.99:0/1007566149 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fe4840a28d0 0x7fe4840a4d90 secure :-1 s=READY pgs=333 cs=0 l=1 rev1=1 crypto rx=0x7fe488000c00 tx=0x7fe488005fd0 comp rx=0 tx=0).stop 2024-03-19T01:54:31.016 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:31.014+0000 7fe49d0ac640 1 -- 172.21.15.99:0/1007566149 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe498101930 msgr2=0x7fe49819ec50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:54:31.016 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:31.014+0000 7fe49d0ac640 1 --2- 172.21.15.99:0/1007566149 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe498101930 0x7fe49819ec50 secure :-1 s=READY pgs=607 cs=0 l=1 rev1=1 crypto rx=0x7fe48000e9f0 tx=0x7fe48000eeb0 comp rx=0 tx=0).stop 2024-03-19T01:54:31.016 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:31.014+0000 7fe49d0ac640 1 -- 172.21.15.99:0/1007566149 shutdown_connections 2024-03-19T01:54:31.016 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:31.014+0000 7fe49d0ac640 1 --2- 172.21.15.99:0/1007566149 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fe4981022f0 0x7fe4981a11a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:31.016 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:31.014+0000 7fe49d0ac640 1 --2- 172.21.15.99:0/1007566149 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fe498101930 0x7fe49819ec50 unknown :-1 s=CLOSED pgs=607 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:31.016 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:31.014+0000 7fe49d0ac640 1 --2- 172.21.15.99:0/1007566149 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fe4840a28d0 0x7fe4840a4d90 unknown :-1 s=CLOSED pgs=333 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:31.016 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:31.014+0000 7fe49d0ac640 1 --2- 172.21.15.99:0/1007566149 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fe498100740 0x7fe498069a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:31.016 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:31.014+0000 7fe49d0ac640 1 -- 172.21.15.99:0/1007566149 >> 172.21.15.99:0/1007566149 conn(0x7fe4980fbcb0 msgr2=0x7fe498104c30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:54:31.017 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:31.014+0000 7fe49d0ac640 1 -- 172.21.15.99:0/1007566149 shutdown_connections 2024-03-19T01:54:31.017 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:31.014+0000 7fe49d0ac640 1 -- 172.21.15.99:0/1007566149 wait complete. 2024-03-19T01:54:31.098 INFO:tasks.workunit.client.0.smithi099.stdout: 10 27951 28.10 MB/sec warmup 60 sec latency 298.381 ms 2024-03-19T01:54:31.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:54:31 smithi115 ceph-mon[67332]: pgmap v754: 97 pgs: 97 active+clean; 468 MiB data, 4.7 GiB used, 1.0 TiB / 1.0 TiB avail; 5.7 MiB/s rd, 6.5 MiB/s wr, 513 op/s 2024-03-19T01:54:31.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:54:31 smithi099 ceph-mon[62951]: pgmap v754: 97 pgs: 97 active+clean; 468 MiB data, 4.7 GiB used, 1.0 TiB / 1.0 TiB avail; 5.7 MiB/s rd, 6.5 MiB/s wr, 513 op/s 2024-03-19T01:54:31.363 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:54:31 smithi112 ceph-mon[66819]: pgmap v754: 97 pgs: 97 active+clean; 468 MiB data, 4.7 GiB used, 1.0 TiB / 1.0 TiB avail; 5.7 MiB/s rd, 6.5 MiB/s wr, 513 op/s 2024-03-19T01:54:32.098 INFO:tasks.workunit.client.0.smithi099.stdout: 10 28458 28.00 MB/sec warmup 61 sec latency 109.750 ms 2024-03-19T01:54:33.098 INFO:tasks.workunit.client.0.smithi099.stdout: 10 28927 27.95 MB/sec warmup 62 sec latency 55.630 ms 2024-03-19T01:54:34.098 INFO:tasks.workunit.client.0.smithi099.stdout: 10 29363 27.99 MB/sec warmup 63 sec latency 121.472 ms 2024-03-19T01:54:34.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:54:34 smithi115 ceph-mon[67332]: pgmap v755: 97 pgs: 97 active+clean; 467 MiB data, 4.8 GiB used, 1.0 TiB / 1.0 TiB avail; 6.3 MiB/s rd, 7.3 MiB/s wr, 627 op/s 2024-03-19T01:54:34.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:54:34 smithi099 ceph-mon[62951]: pgmap v755: 97 pgs: 97 active+clean; 467 MiB data, 4.8 GiB used, 1.0 TiB / 1.0 TiB avail; 6.3 MiB/s rd, 7.3 MiB/s wr, 627 op/s 2024-03-19T01:54:34.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:54:34 smithi112 ceph-mon[66819]: pgmap v755: 97 pgs: 97 active+clean; 467 MiB data, 4.8 GiB used, 1.0 TiB / 1.0 TiB avail; 6.3 MiB/s rd, 7.3 MiB/s wr, 627 op/s 2024-03-19T01:54:35.099 INFO:tasks.workunit.client.0.smithi099.stdout: 10 29616 27.75 MB/sec warmup 64 sec latency 195.637 ms 2024-03-19T01:54:36.099 INFO:tasks.workunit.client.0.smithi099.stdout: 10 29885 27.44 MB/sec warmup 65 sec latency 140.139 ms 2024-03-19T01:54:36.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:54:36 smithi115 ceph-mon[67332]: pgmap v756: 97 pgs: 97 active+clean; 466 MiB data, 4.8 GiB used, 1.0 TiB / 1.0 TiB avail; 5.4 MiB/s rd, 6.0 MiB/s wr, 508 op/s 2024-03-19T01:54:36.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:54:36 smithi099 ceph-mon[62951]: pgmap v756: 97 pgs: 97 active+clean; 466 MiB data, 4.8 GiB used, 1.0 TiB / 1.0 TiB avail; 5.4 MiB/s rd, 6.0 MiB/s wr, 508 op/s 2024-03-19T01:54:36.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:54:36 smithi112 ceph-mon[66819]: pgmap v756: 97 pgs: 97 active+clean; 466 MiB data, 4.8 GiB used, 1.0 TiB / 1.0 TiB avail; 5.4 MiB/s rd, 6.0 MiB/s wr, 508 op/s 2024-03-19T01:54:37.099 INFO:tasks.workunit.client.0.smithi099.stdout: 10 30435 27.41 MB/sec warmup 66 sec latency 66.383 ms 2024-03-19T01:54:37.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:54:37 smithi115 ceph-mon[67332]: pgmap v757: 97 pgs: 97 active+clean; 476 MiB data, 4.9 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 MiB/s rd, 8.3 MiB/s wr, 598 op/s 2024-03-19T01:54:37.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:54:37 smithi099 ceph-mon[62951]: pgmap v757: 97 pgs: 97 active+clean; 476 MiB data, 4.9 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 MiB/s rd, 8.3 MiB/s wr, 598 op/s 2024-03-19T01:54:37.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:54:37 smithi112 ceph-mon[66819]: pgmap v757: 97 pgs: 97 active+clean; 476 MiB data, 4.9 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 MiB/s rd, 8.3 MiB/s wr, 598 op/s 2024-03-19T01:54:38.099 INFO:tasks.workunit.client.0.smithi099.stdout: 10 31037 27.56 MB/sec warmup 67 sec latency 90.569 ms 2024-03-19T01:54:39.099 INFO:tasks.workunit.client.0.smithi099.stdout: 10 31601 27.43 MB/sec warmup 68 sec latency 50.221 ms 2024-03-19T01:54:40.099 INFO:tasks.workunit.client.0.smithi099.stdout: 10 31894 27.29 MB/sec warmup 69 sec latency 340.422 ms 2024-03-19T01:54:40.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:54:40 smithi115 ceph-mon[67332]: pgmap v758: 97 pgs: 97 active+clean; 486 MiB data, 4.9 GiB used, 1.0 TiB / 1.0 TiB avail; 5.6 MiB/s rd, 8.2 MiB/s wr, 563 op/s 2024-03-19T01:54:40.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:54:40 smithi099 ceph-mon[62951]: pgmap v758: 97 pgs: 97 active+clean; 486 MiB data, 4.9 GiB used, 1.0 TiB / 1.0 TiB avail; 5.6 MiB/s rd, 8.2 MiB/s wr, 563 op/s 2024-03-19T01:54:40.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:54:40 smithi112 ceph-mon[66819]: pgmap v758: 97 pgs: 97 active+clean; 486 MiB data, 4.9 GiB used, 1.0 TiB / 1.0 TiB avail; 5.6 MiB/s rd, 8.2 MiB/s wr, 563 op/s 2024-03-19T01:54:41.099 INFO:tasks.workunit.client.0.smithi099.stdout: 10 32379 27.37 MB/sec warmup 70 sec latency 109.669 ms 2024-03-19T01:54:41.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:54:41 smithi115 ceph-mon[67332]: pgmap v759: 97 pgs: 97 active+clean; 493 MiB data, 4.9 GiB used, 1.0 TiB / 1.0 TiB avail; 6.3 MiB/s rd, 8.7 MiB/s wr, 595 op/s 2024-03-19T01:54:41.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:54:41 smithi099 ceph-mon[62951]: pgmap v759: 97 pgs: 97 active+clean; 493 MiB data, 4.9 GiB used, 1.0 TiB / 1.0 TiB avail; 6.3 MiB/s rd, 8.7 MiB/s wr, 595 op/s 2024-03-19T01:54:41.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:54:41 smithi112 ceph-mon[66819]: pgmap v759: 97 pgs: 97 active+clean; 493 MiB data, 4.9 GiB used, 1.0 TiB / 1.0 TiB avail; 6.3 MiB/s rd, 8.7 MiB/s wr, 595 op/s 2024-03-19T01:54:42.099 INFO:tasks.workunit.client.0.smithi099.stdout: 10 33038 27.61 MB/sec warmup 71 sec latency 49.287 ms 2024-03-19T01:54:43.099 INFO:tasks.workunit.client.0.smithi099.stdout: 10 33479 27.56 MB/sec warmup 72 sec latency 66.261 ms 2024-03-19T01:54:44.099 INFO:tasks.workunit.client.0.smithi099.stdout: 10 33809 27.36 MB/sec warmup 73 sec latency 87.365 ms 2024-03-19T01:54:44.105 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.105+0000 7f0b87c54640 1 -- 172.21.15.99:0/2494440511 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0b80101930 msgr2=0x7f0b80101db0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:54:44.105 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.105+0000 7f0b87c54640 1 --2- 172.21.15.99:0/2494440511 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0b80101930 0x7f0b80101db0 secure :-1 s=READY pgs=612 cs=0 l=1 rev1=1 crypto rx=0x7f0b70009900 tx=0x7f0b7002bc30 comp rx=0 tx=0).stop 2024-03-19T01:54:44.105 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.105+0000 7f0b87c54640 1 -- 172.21.15.99:0/2494440511 shutdown_connections 2024-03-19T01:54:44.106 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.105+0000 7f0b87c54640 1 --2- 172.21.15.99:0/2494440511 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f0b801022f0 0x7f0b80108b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:44.106 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.105+0000 7f0b87c54640 1 --2- 172.21.15.99:0/2494440511 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0b80101930 0x7f0b80101db0 unknown :-1 s=CLOSED pgs=612 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:44.106 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.105+0000 7f0b87c54640 1 --2- 172.21.15.99:0/2494440511 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f0b80100740 0x7f0b80100b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:44.106 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.105+0000 7f0b87c54640 1 -- 172.21.15.99:0/2494440511 >> 172.21.15.99:0/2494440511 conn(0x7f0b800fbcb0 msgr2=0x7f0b800fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:54:44.106 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.105+0000 7f0b87c54640 1 -- 172.21.15.99:0/2494440511 shutdown_connections 2024-03-19T01:54:44.106 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.106+0000 7f0b87c54640 1 -- 172.21.15.99:0/2494440511 wait complete. 2024-03-19T01:54:44.109 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.106+0000 7f0b87c54640 1 Processor -- start 2024-03-19T01:54:44.109 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.106+0000 7f0b87c54640 1 -- start start 2024-03-19T01:54:44.109 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.106+0000 7f0b87c54640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f0b80100740 0x7f0b80197aa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:44.109 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.106+0000 7f0b87c54640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0b80101930 0x7f0b80197fe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:44.109 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.106+0000 7f0b87c54640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f0b801022f0 0x7f0b8019f050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:44.109 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.106+0000 7f0b87c54640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f0b8010c3c0 con 0x7f0b80101930 2024-03-19T01:54:44.109 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.106+0000 7f0b87c54640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f0b8010c240 con 0x7f0b801022f0 2024-03-19T01:54:44.109 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.106+0000 7f0b87c54640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f0b8010c540 con 0x7f0b80100740 2024-03-19T01:54:44.109 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.106+0000 7f0b851ef640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0b80101930 0x7f0b80197fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:44.109 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.106+0000 7f0b851ef640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0b80101930 0x7f0b80197fe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58322/0 (socket says 172.21.15.99:58322) 2024-03-19T01:54:44.110 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.106+0000 7f0b851ef640 1 -- 172.21.15.99:0/1452597660 learned_addr learned my addr 172.21.15.99:0/1452597660 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:54:44.110 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.106+0000 7f0b859f0640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f0b80100740 0x7f0b80197aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:44.110 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.107+0000 7f0b861f1640 1 --2- 172.21.15.99:0/1452597660 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f0b801022f0 0x7f0b8019f050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:44.110 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.107+0000 7f0b851ef640 1 -- 172.21.15.99:0/1452597660 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f0b80100740 msgr2=0x7f0b80197aa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:54:44.110 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.107+0000 7f0b851ef640 1 --2- 172.21.15.99:0/1452597660 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f0b80100740 0x7f0b80197aa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:44.110 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.107+0000 7f0b851ef640 1 -- 172.21.15.99:0/1452597660 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f0b801022f0 msgr2=0x7f0b8019f050 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:54:44.110 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.107+0000 7f0b851ef640 1 --2- 172.21.15.99:0/1452597660 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f0b801022f0 0x7f0b8019f050 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:44.111 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.107+0000 7f0b851ef640 1 -- 172.21.15.99:0/1452597660 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f0b8019f6c0 con 0x7f0b80101930 2024-03-19T01:54:44.111 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.107+0000 7f0b859f0640 1 --2- 172.21.15.99:0/1452597660 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f0b80100740 0x7f0b80197aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:54:44.111 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.107+0000 7f0b851ef640 1 --2- 172.21.15.99:0/1452597660 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0b80101930 0x7f0b80197fe0 secure :-1 s=READY pgs=613 cs=0 l=1 rev1=1 crypto rx=0x7f0b70009230 tx=0x7f0b7002c650 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:54:44.111 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.107+0000 7f0b76ffd640 1 -- 172.21.15.99:0/1452597660 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0b70041070 con 0x7f0b80101930 2024-03-19T01:54:44.111 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.107+0000 7f0b87c54640 1 -- 172.21.15.99:0/1452597660 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f0b8019f950 con 0x7f0b80101930 2024-03-19T01:54:44.111 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.107+0000 7f0b76ffd640 1 -- 172.21.15.99:0/1452597660 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f0b70004450 con 0x7f0b80101930 2024-03-19T01:54:44.111 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.107+0000 7f0b76ffd640 1 -- 172.21.15.99:0/1452597660 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0b70003e20 con 0x7f0b80101930 2024-03-19T01:54:44.111 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.107+0000 7f0b87c54640 1 -- 172.21.15.99:0/1452597660 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f0b8019fe90 con 0x7f0b80101930 2024-03-19T01:54:44.120 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.110+0000 7f0b76ffd640 1 -- 172.21.15.99:0/1452597660 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f0b70044070 con 0x7f0b80101930 2024-03-19T01:54:44.120 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.110+0000 7f0b76ffd640 1 --2- 172.21.15.99:0/1452597660 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f0b6c0a28a0 0x7f0b6c0a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:44.121 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.110+0000 7f0b76ffd640 1 -- 172.21.15.99:0/1452597660 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f0b700f0650 con 0x7f0b80101930 2024-03-19T01:54:44.121 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.110+0000 7f0b859f0640 1 --2- 172.21.15.99:0/1452597660 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f0b6c0a28a0 0x7f0b6c0a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:44.121 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.111+0000 7f0b8a83d740 1 Processor -- start 2024-03-19T01:54:44.121 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.111+0000 7f0b859f0640 1 --2- 172.21.15.99:0/1452597660 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f0b6c0a28a0 0x7f0b6c0a4d60 secure :-1 s=READY pgs=334 cs=0 l=1 rev1=1 crypto rx=0x7f0b80101790 tx=0x7f0b7c005f90 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:54:44.121 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.111+0000 7f0b8a83d740 1 -- start start 2024-03-19T01:54:44.121 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.111+0000 7f0b8a83d740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fed63b1510 0x55fed63b1930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:44.121 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.111+0000 7f0b8a83d740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fed63b1f00 0x55fed63b6a00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:44.121 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.111+0000 7f0b8a83d740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fed63b7070 0x55fed63b9500 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:44.121 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.111+0000 7f0b8a83d740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55fed6311be0 con 0x55fed63b1510 2024-03-19T01:54:44.121 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.111+0000 7f0b8a83d740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55fed62ba960 con 0x55fed63b7070 2024-03-19T01:54:44.121 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.111+0000 7f0b8a83d740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55fed630dc80 con 0x55fed63b1f00 2024-03-19T01:54:44.122 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.111+0000 7f0b859f0640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fed63b1510 0x55fed63b1930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:44.122 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.111+0000 7f0b859f0640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fed63b1510 0x55fed63b1930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58326/0 (socket says 172.21.15.99:58326) 2024-03-19T01:54:44.122 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.111+0000 7f0b859f0640 1 -- 172.21.15.99:0/2586498511 learned_addr learned my addr 172.21.15.99:0/2586498511 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:54:44.122 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.111+0000 7f0b851ef640 1 --2- 172.21.15.99:0/2586498511 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fed63b1f00 0x55fed63b6a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:44.122 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.111+0000 7f0b861f1640 1 --2- 172.21.15.99:0/2586498511 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fed63b7070 0x55fed63b9500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:44.122 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.111+0000 7f0b859f0640 1 -- 172.21.15.99:0/2586498511 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fed63b1f00 msgr2=0x55fed63b6a00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:54:44.122 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.111+0000 7f0b859f0640 1 --2- 172.21.15.99:0/2586498511 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fed63b1f00 0x55fed63b6a00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:44.122 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.111+0000 7f0b859f0640 1 -- 172.21.15.99:0/2586498511 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fed63b7070 msgr2=0x55fed63b9500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:54:44.122 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.111+0000 7f0b859f0640 1 --2- 172.21.15.99:0/2586498511 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fed63b7070 0x55fed63b9500 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:44.122 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.111+0000 7f0b859f0640 1 -- 172.21.15.99:0/2586498511 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55fed62b1d20 con 0x55fed63b1510 2024-03-19T01:54:44.122 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.111+0000 7f0b861f1640 1 --2- 172.21.15.99:0/2586498511 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fed63b7070 0x55fed63b9500 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:54:44.122 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.112+0000 7f0b859f0640 1 --2- 172.21.15.99:0/2586498511 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fed63b1510 0x55fed63b1930 secure :-1 s=READY pgs=614 cs=0 l=1 rev1=1 crypto rx=0x7f0b7c009ba0 tx=0x7f0b7c014000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7cbbc5702453fcc server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:54:44.122 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.112+0000 7f0b6bfff640 1 -- 172.21.15.99:0/2586498511 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0b7c00f480 con 0x55fed63b1510 2024-03-19T01:54:44.123 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.112+0000 7f0b6bfff640 1 -- 172.21.15.99:0/2586498511 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f0b7c00fb00 con 0x55fed63b1510 2024-03-19T01:54:44.123 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.112+0000 7f0b6bfff640 1 -- 172.21.15.99:0/2586498511 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0b7c01cc40 con 0x55fed63b1510 2024-03-19T01:54:44.123 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.112+0000 7f0b8a83d740 1 -- 172.21.15.99:0/2586498511 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fed63b1510 msgr2=0x55fed63b1930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:54:44.123 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.112+0000 7f0b8a83d740 1 --2- 172.21.15.99:0/2586498511 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fed63b1510 0x55fed63b1930 secure :-1 s=READY pgs=614 cs=0 l=1 rev1=1 crypto rx=0x7f0b7c009ba0 tx=0x7f0b7c014000 comp rx=0 tx=0).stop 2024-03-19T01:54:44.123 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.113+0000 7f0b8a83d740 1 -- 172.21.15.99:0/2586498511 shutdown_connections 2024-03-19T01:54:44.123 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.113+0000 7f0b8a83d740 1 --2- 172.21.15.99:0/2586498511 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fed63b7070 0x55fed63b9500 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:44.123 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.113+0000 7f0b8a83d740 1 --2- 172.21.15.99:0/2586498511 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fed63b1f00 0x55fed63b6a00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:44.123 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.113+0000 7f0b8a83d740 1 --2- 172.21.15.99:0/2586498511 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fed63b1510 0x55fed63b1930 unknown :-1 s=CLOSED pgs=614 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:44.123 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.113+0000 7f0b8a83d740 1 -- 172.21.15.99:0/2586498511 >> 172.21.15.99:0/2586498511 conn(0x55fed63a8670 msgr2=0x55fed63a8a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:54:44.123 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.113+0000 7f0b8a83d740 1 -- 172.21.15.99:0/2586498511 shutdown_connections 2024-03-19T01:54:44.123 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.113+0000 7f0b8a83d740 1 -- 172.21.15.99:0/2586498511 wait complete. 2024-03-19T01:54:44.123 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.113+0000 7f0b8a83d740 1 -- start start 2024-03-19T01:54:44.124 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.113+0000 7f0b8a83d740 1 Processor -- start 2024-03-19T01:54:44.124 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.113+0000 7f0b8a83d740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fed63b1510 0x55fed63bb4b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:44.124 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.113+0000 7f0b8a83d740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fed63b1f00 0x55fed63bb9f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:44.124 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.113+0000 7f0b8a83d740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fed63b7070 0x55fed63c2a30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:44.124 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.113+0000 7f0b8a83d740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55fed63ba020 con 0x55fed63b7070 2024-03-19T01:54:44.124 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.113+0000 7f0b8a83d740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55fed62cddb0 con 0x55fed63b1f00 2024-03-19T01:54:44.124 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.113+0000 7f0b8a83d740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55fed63ba1a0 con 0x55fed63b1510 2024-03-19T01:54:44.124 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.114+0000 7f0b8a83d740 10 client.0 _ll_register_callbacks cb 0x55fed62d2e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:54:44.124 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.114+0000 7f0b861f1640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fed63b7070 0x55fed63c2a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:44.124 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.114+0000 7f0b851ef640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fed63b1f00 0x55fed63bb9f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:44.124 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.114+0000 7f0b859f0640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fed63b1510 0x55fed63bb4b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:44.124 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.114+0000 7f0b851ef640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fed63b1f00 0x55fed63bb9f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:48692/0 (socket says 172.21.15.99:48692) 2024-03-19T01:54:44.125 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.114+0000 7f0b861f1640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fed63b7070 0x55fed63c2a30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58334/0 (socket says 172.21.15.99:58334) 2024-03-19T01:54:44.125 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.114+0000 7f0b851ef640 1 -- 172.21.15.99:0/3112713449 learned_addr learned my addr 172.21.15.99:0/3112713449 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:54:44.125 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.114+0000 7f0b851ef640 1 -- 172.21.15.99:0/3112713449 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fed63b1510 msgr2=0x55fed63bb4b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:54:44.125 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.114+0000 7f0b851ef640 1 --2- 172.21.15.99:0/3112713449 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fed63b1510 0x55fed63bb4b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:44.125 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.114+0000 7f0b851ef640 1 -- 172.21.15.99:0/3112713449 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fed63b7070 msgr2=0x55fed63c2a30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:54:44.125 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.114+0000 7f0b851ef640 1 --2- 172.21.15.99:0/3112713449 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fed63b7070 0x55fed63c2a30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:44.125 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.114+0000 7f0b851ef640 1 -- 172.21.15.99:0/3112713449 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55fed62ccc60 con 0x55fed63b1f00 2024-03-19T01:54:44.125 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.114+0000 7f0b859f0640 1 --2- 172.21.15.99:0/3112713449 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fed63b1510 0x55fed63bb4b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:54:44.125 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.114+0000 7f0b861f1640 1 --2- 172.21.15.99:0/3112713449 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fed63b7070 0x55fed63c2a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:54:44.125 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.114+0000 7f0b851ef640 1 --2- 172.21.15.99:0/3112713449 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fed63b1f00 0x55fed63bb9f0 secure :-1 s=READY pgs=161 cs=0 l=1 rev1=1 crypto rx=0x7f0b70000c00 tx=0x7f0b70005e50 comp rx=0 tx=0).ready entity=mon.2 client_cookie=a1cc867c6df306ff server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:54:44.125 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.114+0000 7f0b697fa640 10 client.0 ms_handle_connect on v2:172.21.15.112:3300/0 2024-03-19T01:54:44.125 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.114+0000 7f0b697fa640 1 -- 172.21.15.99:0/3112713449 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0b7004b050 con 0x55fed63b1f00 2024-03-19T01:54:44.125 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.114+0000 7f0b697fa640 1 -- 172.21.15.99:0/3112713449 <== mon.2 v2:172.21.15.112:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f0b700b7bd0 con 0x55fed63b1f00 2024-03-19T01:54:44.126 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.114+0000 7f0b8a83d740 1 -- 172.21.15.99:0/3112713449 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f0b7c00fb00 con 0x55fed63b1f00 2024-03-19T01:54:44.126 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.114+0000 7f0b697fa640 1 -- 172.21.15.99:0/3112713449 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_get_version_reply(handle=1 version=33) ==== 24+0+0 (secure 0 0 0) 0x7f0b700c1100 con 0x55fed63b1f00 2024-03-19T01:54:44.126 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.114+0000 7f0b8a83d740 10 client.26216 fetch_fsmap learned FSMap version 33 2024-03-19T01:54:44.126 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.114+0000 7f0b8a83d740 1 -- 172.21.15.99:0/3112713449 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,fsmap=33,monmap=4+}) -- 0x55fed6026210 con 0x55fed63b1f00 2024-03-19T01:54:44.126 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.115+0000 7f0b697fa640 1 -- 172.21.15.99:0/3112713449 <== mon.2 v2:172.21.15.112:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f0b7c00fb00 con 0x55fed63b1f00 2024-03-19T01:54:44.126 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.115+0000 7f0b697fa640 1 -- 172.21.15.99:0/3112713449 <== mon.2 v2:172.21.15.112:3300/0 5 ==== fsmap(e 33) ==== 8362+0+0 (secure 0 0 0) 0x7f0b700bc4b0 con 0x55fed63b1f00 2024-03-19T01:54:44.126 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.115+0000 7f0b8a83d740 10 client.26216 fetch_fsmap finished waiting for FSMap version 33 2024-03-19T01:54:44.126 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.115+0000 7f0b8a83d740 10 client.26216 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:54:44.126 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.115+0000 7f0b8a83d740 20 client.26216 populate_metadata read hostname 'smithi099' 2024-03-19T01:54:44.126 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.115+0000 7f0b8a83d740 1 --2- 172.21.15.99:0/3112713449 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55fed63c7360 0x55fed63c97a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:44.126 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.115+0000 7f0b8a83d740 4 client.26216 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:54:44.127 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.115+0000 7f0b8a83d740 1 -- 172.21.15.99:0/3112713449 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55fed631f120 con 0x55fed63c7360 2024-03-19T01:54:44.127 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.115+0000 7f0b859f0640 1 --2- 172.21.15.99:0/3112713449 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55fed63c7360 0x55fed63c97a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:44.127 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.115+0000 7f0b859f0640 1 --2- 172.21.15.99:0/3112713449 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55fed63c7360 0x55fed63c97a0 crc :-1 s=READY pgs=151 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=d34fa36855a71ef3 server_cookie=1b2bc932429d5d9f in_seq=0 out_seq=0 2024-03-19T01:54:44.127 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.115+0000 7f0b697fa640 10 client.26216 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:54:44.127 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.117+0000 7f0b697fa640 1 -- 172.21.15.99:0/3112713449 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x55fed631f120 con 0x55fed63c7360 2024-03-19T01:54:44.127 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.117+0000 7f0b697fa640 10 client.26216 handle_command_reply: tid=0 2024-03-19T01:54:44.127 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.117+0000 7f0b8a83d740 1 client.26216 shutdown 2024-03-19T01:54:44.127 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.117+0000 7f0b8a83d740 1 -- 172.21.15.99:0/3112713449 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fed63b1f00 msgr2=0x55fed63bb9f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:54:44.127 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.117+0000 7f0b8a83d740 1 --2- 172.21.15.99:0/3112713449 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fed63b1f00 0x55fed63bb9f0 secure :-1 s=READY pgs=161 cs=0 l=1 rev1=1 crypto rx=0x7f0b70000c00 tx=0x7f0b70005e50 comp rx=0 tx=0).stop 2024-03-19T01:54:44.127 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.117+0000 7f0b8a83d740 1 -- 172.21.15.99:0/3112713449 shutdown_connections 2024-03-19T01:54:44.127 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.117+0000 7f0b8a83d740 1 --2- 172.21.15.99:0/3112713449 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55fed63c7360 0x55fed63c97a0 crc :-1 s=READY pgs=151 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:44.127 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.117+0000 7f0b8a83d740 1 --2- 172.21.15.99:0/3112713449 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fed63b1f00 0x55fed63bb9f0 unknown :-1 s=CLOSED pgs=161 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:44.127 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.117+0000 7f0b697fa640 0 client.26216 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:54:44.127 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.117+0000 7f0b8a83d740 1 --2- 172.21.15.99:0/3112713449 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fed63b7070 0x55fed63c2a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:44.128 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.117+0000 7f0b8a83d740 1 --2- 172.21.15.99:0/3112713449 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fed63b1510 0x55fed63bb4b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:44.128 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.117+0000 7f0b8a83d740 1 -- 172.21.15.99:0/3112713449 >> 172.21.15.99:0/3112713449 conn(0x55fed63a8670 msgr2=0x55fed63a98c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:54:44.128 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.117+0000 7f0b8a83d740 1 -- 172.21.15.99:0/3112713449 shutdown_connections 2024-03-19T01:54:44.128 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.117+0000 7f0b8a83d740 1 -- 172.21.15.99:0/3112713449 wait complete. 2024-03-19T01:54:44.128 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.117+0000 7f0b8a83d740 20 client.26216 trim_cache size 0 max 16384 2024-03-19T01:54:44.135 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.134+0000 7f0b8a83d740 1 Processor -- start 2024-03-19T01:54:44.135 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.134+0000 7f0b8a83d740 1 -- start start 2024-03-19T01:54:44.136 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.134+0000 7f0b8a83d740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fed63b1510 0x55fed63a7220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:44.136 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.134+0000 7f0b8a83d740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fed63b1f00 0x55fed63c29f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:44.136 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.134+0000 7f0b8a83d740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fed63b7070 0x55fed63a7760 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:44.136 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.134+0000 7f0b8a83d740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f0b801a9550 con 0x55fed63b7070 2024-03-19T01:54:44.136 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.134+0000 7f0b8a83d740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f0b801a6d60 con 0x55fed63b1f00 2024-03-19T01:54:44.136 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.134+0000 7f0b8a83d740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f0b801a4bb0 con 0x55fed63b1510 2024-03-19T01:54:44.136 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.134+0000 7f0b861f1640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fed63b7070 0x55fed63a7760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:44.136 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.134+0000 7f0b859f0640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fed63b1510 0x55fed63a7220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:44.136 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.134+0000 7f0b851ef640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fed63b1f00 0x55fed63c29f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:44.136 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.134+0000 7f0b859f0640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fed63b1510 0x55fed63a7220 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:41168/0 (socket says 172.21.15.99:41168) 2024-03-19T01:54:44.136 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.134+0000 7f0b859f0640 1 -- 172.21.15.99:0/2144962933 learned_addr learned my addr 172.21.15.99:0/2144962933 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:54:44.137 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.134+0000 7f0b851ef640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fed63b1f00 0x55fed63c29f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:48698/0 (socket says 172.21.15.99:48698) 2024-03-19T01:54:44.137 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.135+0000 7f0b859f0640 1 -- 172.21.15.99:0/2144962933 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fed63b1f00 msgr2=0x55fed63c29f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:54:44.137 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.135+0000 7f0b859f0640 1 --2- 172.21.15.99:0/2144962933 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fed63b1f00 0x55fed63c29f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:44.137 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.135+0000 7f0b859f0640 1 -- 172.21.15.99:0/2144962933 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fed63b7070 msgr2=0x55fed63a7760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:54:44.137 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.135+0000 7f0b859f0640 1 --2- 172.21.15.99:0/2144962933 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fed63b7070 0x55fed63a7760 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:44.137 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.135+0000 7f0b859f0640 1 -- 172.21.15.99:0/2144962933 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55fed60f2620 con 0x55fed63b1510 2024-03-19T01:54:44.137 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.135+0000 7f0b861f1640 1 --2- 172.21.15.99:0/2144962933 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fed63b7070 0x55fed63a7760 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:54:44.137 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.135+0000 7f0b859f0640 1 --2- 172.21.15.99:0/2144962933 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fed63b1510 0x55fed63a7220 secure :-1 s=READY pgs=113 cs=0 l=1 rev1=1 crypto rx=0x7f0b7c009ba0 tx=0x7f0b7c018000 comp rx=0 tx=0).ready entity=mon.1 client_cookie=2438a93476f76f21 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:54:44.137 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.135+0000 7f0b6b7fe640 1 -- 172.21.15.99:0/2144962933 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0b7c00fdb0 con 0x55fed63b1510 2024-03-19T01:54:44.137 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.135+0000 7f0b6b7fe640 1 -- 172.21.15.99:0/2144962933 <== mon.1 v2:172.21.15.115:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f0b7c005520 con 0x55fed63b1510 2024-03-19T01:54:44.137 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.135+0000 7f0b6b7fe640 1 -- 172.21.15.99:0/2144962933 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0b7c00f480 con 0x55fed63b1510 2024-03-19T01:54:44.141 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.136+0000 7f0b8a83d740 1 -- 172.21.15.99:0/2144962933 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fed63b1510 msgr2=0x55fed63a7220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:54:44.141 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.136+0000 7f0b8a83d740 1 --2- 172.21.15.99:0/2144962933 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fed63b1510 0x55fed63a7220 secure :-1 s=READY pgs=113 cs=0 l=1 rev1=1 crypto rx=0x7f0b7c009ba0 tx=0x7f0b7c018000 comp rx=0 tx=0).stop 2024-03-19T01:54:44.141 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.136+0000 7f0b8a83d740 1 -- 172.21.15.99:0/2144962933 shutdown_connections 2024-03-19T01:54:44.141 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.136+0000 7f0b8a83d740 1 --2- 172.21.15.99:0/2144962933 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fed63b1f00 0x55fed63c29f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:44.141 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.136+0000 7f0b8a83d740 1 --2- 172.21.15.99:0/2144962933 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fed63b7070 0x55fed63a7760 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:44.141 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.136+0000 7f0b8a83d740 1 --2- 172.21.15.99:0/2144962933 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fed63b1510 0x55fed63a7220 unknown :-1 s=CLOSED pgs=113 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:44.141 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.136+0000 7f0b8a83d740 1 -- 172.21.15.99:0/2144962933 >> 172.21.15.99:0/2144962933 conn(0x55fed63a8670 msgr2=0x55fed63ac160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:54:44.141 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.136+0000 7f0b8a83d740 1 -- 172.21.15.99:0/2144962933 shutdown_connections 2024-03-19T01:54:44.142 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.136+0000 7f0b8a83d740 1 -- 172.21.15.99:0/2144962933 wait complete. 2024-03-19T01:54:44.142 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.136+0000 7f0b8a83d740 1 -- start start 2024-03-19T01:54:44.142 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.137+0000 7f0b8a83d740 1 Processor -- start 2024-03-19T01:54:44.142 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.137+0000 7f0b8a83d740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fed63b1510 0x55fed63bfc80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:44.142 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.137+0000 7f0b8a83d740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fed63b1f00 0x55fed63c01c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:44.142 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.137+0000 7f0b8a83d740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fed63b7070 0x55fed63c11d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:44.142 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.137+0000 7f0b8a83d740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f0b801a5220 con 0x55fed63b7070 2024-03-19T01:54:44.142 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.137+0000 7f0b8a83d740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f0b801a6690 con 0x55fed63b1f00 2024-03-19T01:54:44.143 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.137+0000 7f0b8a83d740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f0b801a58f0 con 0x55fed63b1510 2024-03-19T01:54:44.143 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.137+0000 7f0b8a83d740 10 client.0 _ll_register_callbacks cb 0x7f0b700b9320 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:54:44.143 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.137+0000 7f0b861f1640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fed63b7070 0x55fed63c11d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:44.143 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.137+0000 7f0b861f1640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fed63b7070 0x55fed63c11d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58354/0 (socket says 172.21.15.99:58354) 2024-03-19T01:54:44.143 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.137+0000 7f0b861f1640 1 -- 172.21.15.99:0/3761937457 learned_addr learned my addr 172.21.15.99:0/3761937457 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:54:44.143 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.137+0000 7f0b851ef640 1 --2- 172.21.15.99:0/3761937457 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fed63b1f00 0x55fed63c01c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:44.143 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.137+0000 7f0b859f0640 1 --2- 172.21.15.99:0/3761937457 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fed63b1510 0x55fed63bfc80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:44.143 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.137+0000 7f0b851ef640 1 -- 172.21.15.99:0/3761937457 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fed63b1510 msgr2=0x55fed63bfc80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:54:44.143 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.137+0000 7f0b851ef640 1 --2- 172.21.15.99:0/3761937457 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fed63b1510 0x55fed63bfc80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:44.143 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.137+0000 7f0b851ef640 1 -- 172.21.15.99:0/3761937457 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fed63b7070 msgr2=0x55fed63c11d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:54:44.143 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.137+0000 7f0b851ef640 1 --2- 172.21.15.99:0/3761937457 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fed63b7070 0x55fed63c11d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:44.144 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.137+0000 7f0b851ef640 1 -- 172.21.15.99:0/3761937457 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55fed63c17c0 con 0x55fed63b1f00 2024-03-19T01:54:44.144 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.137+0000 7f0b859f0640 1 --2- 172.21.15.99:0/3761937457 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fed63b1510 0x55fed63bfc80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:54:44.144 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.138+0000 7f0b851ef640 1 --2- 172.21.15.99:0/3761937457 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fed63b1f00 0x55fed63c01c0 secure :-1 s=READY pgs=162 cs=0 l=1 rev1=1 crypto rx=0x7f0b70009430 tx=0x7f0b70005c00 comp rx=0 tx=0).ready entity=mon.2 client_cookie=8e58b7830b8f726c server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:54:44.144 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.138+0000 7f0b697fa640 10 client.0 ms_handle_connect on v2:172.21.15.112:3300/0 2024-03-19T01:54:44.144 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.138+0000 7f0b697fa640 1 -- 172.21.15.99:0/3761937457 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0b7004b040 con 0x55fed63b1f00 2024-03-19T01:54:44.144 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.138+0000 7f0b8a83d740 1 -- 172.21.15.99:0/3761937457 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f0b7c005520 con 0x55fed63b1f00 2024-03-19T01:54:44.144 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.138+0000 7f0b697fa640 1 -- 172.21.15.99:0/3761937457 <== mon.2 v2:172.21.15.112:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f0b700bc4b0 con 0x55fed63b1f00 2024-03-19T01:54:44.144 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.138+0000 7f0b697fa640 1 -- 172.21.15.99:0/3761937457 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_get_version_reply(handle=1 version=33) ==== 24+0+0 (secure 0 0 0) 0x7f0b700bc6f0 con 0x55fed63b1f00 2024-03-19T01:54:44.144 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.138+0000 7f0b8a83d740 10 client.26222 fetch_fsmap learned FSMap version 33 2024-03-19T01:54:44.144 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.138+0000 7f0b8a83d740 1 -- 172.21.15.99:0/3761937457 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,fsmap=33,monmap=4+}) -- 0x55fed63c17c0 con 0x55fed63b1f00 2024-03-19T01:54:44.145 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.138+0000 7f0b697fa640 1 -- 172.21.15.99:0/3761937457 <== mon.2 v2:172.21.15.112:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f0b7c005520 con 0x55fed63b1f00 2024-03-19T01:54:44.145 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.138+0000 7f0b697fa640 1 -- 172.21.15.99:0/3761937457 <== mon.2 v2:172.21.15.112:3300/0 5 ==== fsmap(e 33) ==== 8362+0+0 (secure 0 0 0) 0x7f0b700a2020 con 0x55fed63b1f00 2024-03-19T01:54:44.145 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.138+0000 7f0b8a83d740 10 client.26222 fetch_fsmap finished waiting for FSMap version 33 2024-03-19T01:54:44.145 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.138+0000 7f0b8a83d740 10 client.26222 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:54:44.145 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.138+0000 7f0b8a83d740 20 client.26222 populate_metadata read hostname 'smithi099' 2024-03-19T01:54:44.145 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.138+0000 7f0b8a83d740 1 --2- 172.21.15.99:0/3761937457 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55fed63c7360 0x55fed642f8f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:44.145 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.139+0000 7f0b8a83d740 4 client.26222 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:54:44.145 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.139+0000 7f0b8a83d740 1 -- 172.21.15.99:0/3761937457 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55fed6319470 con 0x55fed63c7360 2024-03-19T01:54:44.145 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.139+0000 7f0b859f0640 1 --2- 172.21.15.99:0/3761937457 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55fed63c7360 0x55fed642f8f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:44.146 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.139+0000 7f0b859f0640 1 --2- 172.21.15.99:0/3761937457 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55fed63c7360 0x55fed642f8f0 crc :-1 s=READY pgs=152 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=5c9d2674b01c626b server_cookie=bb48a4664f22c66d in_seq=0 out_seq=0 2024-03-19T01:54:44.146 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.139+0000 7f0b697fa640 10 client.26222 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:54:44.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:54:44 smithi115 ceph-mon[67332]: pgmap v760: 97 pgs: 97 active+clean; 496 MiB data, 4.9 GiB used, 1.0 TiB / 1.0 TiB avail; 7.9 MiB/s rd, 9.5 MiB/s wr, 679 op/s 2024-03-19T01:54:44.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:54:44 smithi099 ceph-mon[62951]: pgmap v760: 97 pgs: 97 active+clean; 496 MiB data, 4.9 GiB used, 1.0 TiB / 1.0 TiB avail; 7.9 MiB/s rd, 9.5 MiB/s wr, 679 op/s 2024-03-19T01:54:44.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:54:44 smithi112 ceph-mon[66819]: pgmap v760: 97 pgs: 97 active+clean; 496 MiB data, 4.9 GiB used, 1.0 TiB / 1.0 TiB avail; 7.9 MiB/s rd, 9.5 MiB/s wr, 679 op/s 2024-03-19T01:54:44.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.665+0000 7f0b697fa640 1 -- 172.21.15.99:0/3761937457 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x55fed6319470 con 0x55fed63c7360 2024-03-19T01:54:44.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.665+0000 7f0b697fa640 10 client.26222 handle_command_reply: tid=0 2024-03-19T01:54:44.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.665+0000 7f0b8a83d740 1 client.26222 shutdown 2024-03-19T01:54:44.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.665+0000 7f0b8a83d740 1 -- 172.21.15.99:0/3761937457 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fed63b1f00 msgr2=0x55fed63c01c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:54:44.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.665+0000 7f0b8a83d740 1 --2- 172.21.15.99:0/3761937457 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fed63b1f00 0x55fed63c01c0 secure :-1 s=READY pgs=162 cs=0 l=1 rev1=1 crypto rx=0x7f0b70009430 tx=0x7f0b70005c00 comp rx=0 tx=0).stop 2024-03-19T01:54:44.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.665+0000 7f0b8a83d740 1 -- 172.21.15.99:0/3761937457 shutdown_connections 2024-03-19T01:54:44.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.665+0000 7f0b8a83d740 1 --2- 172.21.15.99:0/3761937457 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55fed63c7360 0x55fed642f8f0 crc :-1 s=READY pgs=152 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:44.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.665+0000 7f0b8a83d740 1 --2- 172.21.15.99:0/3761937457 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fed63b1f00 0x55fed63c01c0 unknown :-1 s=CLOSED pgs=162 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:44.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.665+0000 7f0b8a83d740 1 --2- 172.21.15.99:0/3761937457 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fed63b7070 0x55fed63c11d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:44.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.665+0000 7f0b8a83d740 1 --2- 172.21.15.99:0/3761937457 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fed63b1510 0x55fed63bfc80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:44.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.665+0000 7f0b8a83d740 1 -- 172.21.15.99:0/3761937457 >> 172.21.15.99:0/3761937457 conn(0x55fed63a8670 msgr2=0x55fed63b2790 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:54:44.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.665+0000 7f0b697fa640 0 client.26222 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:54:44.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.665+0000 7f0b8a83d740 1 -- 172.21.15.99:0/3761937457 shutdown_connections 2024-03-19T01:54:44.667 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.665+0000 7f0b8a83d740 1 -- 172.21.15.99:0/3761937457 wait complete. 2024-03-19T01:54:44.667 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.665+0000 7f0b8a83d740 20 client.26222 trim_cache size 0 max 16384 2024-03-19T01:54:44.667 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:54:44.667 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:54:44.667 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:54:44.667 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:54:44.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.666+0000 7f0b87c54640 1 -- 172.21.15.99:0/1452597660 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f0b6c0a28a0 msgr2=0x7f0b6c0a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:54:44.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.666+0000 7f0b87c54640 1 --2- 172.21.15.99:0/1452597660 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f0b6c0a28a0 0x7f0b6c0a4d60 secure :-1 s=READY pgs=334 cs=0 l=1 rev1=1 crypto rx=0x7f0b80101790 tx=0x7f0b7c005f90 comp rx=0 tx=0).stop 2024-03-19T01:54:44.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.666+0000 7f0b87c54640 1 -- 172.21.15.99:0/1452597660 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0b80101930 msgr2=0x7f0b80197fe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:54:44.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.666+0000 7f0b87c54640 1 --2- 172.21.15.99:0/1452597660 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0b80101930 0x7f0b80197fe0 secure :-1 s=READY pgs=613 cs=0 l=1 rev1=1 crypto rx=0x7f0b70009230 tx=0x7f0b7002c650 comp rx=0 tx=0).stop 2024-03-19T01:54:44.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.666+0000 7f0b87c54640 1 -- 172.21.15.99:0/1452597660 shutdown_connections 2024-03-19T01:54:44.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.666+0000 7f0b87c54640 1 --2- 172.21.15.99:0/1452597660 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f0b6c0a28a0 0x7f0b6c0a4d60 unknown :-1 s=CLOSED pgs=334 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:44.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.666+0000 7f0b87c54640 1 --2- 172.21.15.99:0/1452597660 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f0b801022f0 0x7f0b8019f050 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:44.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.666+0000 7f0b87c54640 1 --2- 172.21.15.99:0/1452597660 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f0b80101930 0x7f0b80197fe0 unknown :-1 s=CLOSED pgs=613 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:44.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.666+0000 7f0b87c54640 1 --2- 172.21.15.99:0/1452597660 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f0b80100740 0x7f0b80197aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:44.669 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.666+0000 7f0b87c54640 1 -- 172.21.15.99:0/1452597660 >> 172.21.15.99:0/1452597660 conn(0x7f0b800fbcb0 msgr2=0x7f0b800fd6d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:54:44.669 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.666+0000 7f0b87c54640 1 -- 172.21.15.99:0/1452597660 shutdown_connections 2024-03-19T01:54:44.669 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:44.666+0000 7f0b87c54640 1 -- 172.21.15.99:0/1452597660 wait complete. 2024-03-19T01:54:45.099 INFO:tasks.workunit.client.0.smithi099.stdout: 10 34014 27.18 MB/sec warmup 74 sec latency 414.078 ms 2024-03-19T01:54:45.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:54:45 smithi115 ceph-mon[67332]: pgmap v761: 97 pgs: 97 active+clean; 495 MiB data, 4.9 GiB used, 1.0 TiB / 1.0 TiB avail; 6.8 MiB/s rd, 7.7 MiB/s wr, 480 op/s 2024-03-19T01:54:45.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:54:45 smithi099 ceph-mon[62951]: pgmap v761: 97 pgs: 97 active+clean; 495 MiB data, 4.9 GiB used, 1.0 TiB / 1.0 TiB avail; 6.8 MiB/s rd, 7.7 MiB/s wr, 480 op/s 2024-03-19T01:54:45.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:54:45 smithi112 ceph-mon[66819]: pgmap v761: 97 pgs: 97 active+clean; 495 MiB data, 4.9 GiB used, 1.0 TiB / 1.0 TiB avail; 6.8 MiB/s rd, 7.7 MiB/s wr, 480 op/s 2024-03-19T01:54:46.099 INFO:tasks.workunit.client.0.smithi099.stdout: 10 34391 27.06 MB/sec warmup 75 sec latency 111.344 ms 2024-03-19T01:54:47.099 INFO:tasks.workunit.client.0.smithi099.stdout: 10 34790 26.99 MB/sec warmup 76 sec latency 112.222 ms 2024-03-19T01:54:47.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:54:47 smithi099 ceph-mon[62951]: pgmap v762: 97 pgs: 97 active+clean; 498 MiB data, 4.9 GiB used, 1.0 TiB / 1.0 TiB avail; 8.4 MiB/s rd, 8.8 MiB/s wr, 637 op/s 2024-03-19T01:54:47.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:54:47 smithi112 ceph-mon[66819]: pgmap v762: 97 pgs: 97 active+clean; 498 MiB data, 4.9 GiB used, 1.0 TiB / 1.0 TiB avail; 8.4 MiB/s rd, 8.8 MiB/s wr, 637 op/s 2024-03-19T01:54:47.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:54:47 smithi115 ceph-mon[67332]: pgmap v762: 97 pgs: 97 active+clean; 498 MiB data, 4.9 GiB used, 1.0 TiB / 1.0 TiB avail; 8.4 MiB/s rd, 8.8 MiB/s wr, 637 op/s 2024-03-19T01:54:48.099 INFO:tasks.workunit.client.0.smithi099.stdout: 10 35307 26.97 MB/sec warmup 77 sec latency 66.240 ms 2024-03-19T01:54:49.099 INFO:tasks.workunit.client.0.smithi099.stdout: 10 35865 26.97 MB/sec warmup 78 sec latency 60.185 ms 2024-03-19T01:54:50.099 INFO:tasks.workunit.client.0.smithi099.stdout: 10 36317 26.83 MB/sec warmup 79 sec latency 276.469 ms 2024-03-19T01:54:50.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:54:50 smithi115 ceph-mon[67332]: pgmap v763: 97 pgs: 97 active+clean; 498 MiB data, 4.9 GiB used, 1.0 TiB / 1.0 TiB avail; 7.6 MiB/s rd, 7.2 MiB/s wr, 641 op/s 2024-03-19T01:54:50.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:54:50 smithi099 ceph-mon[62951]: pgmap v763: 97 pgs: 97 active+clean; 498 MiB data, 4.9 GiB used, 1.0 TiB / 1.0 TiB avail; 7.6 MiB/s rd, 7.2 MiB/s wr, 641 op/s 2024-03-19T01:54:50.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:54:50 smithi112 ceph-mon[66819]: pgmap v763: 97 pgs: 97 active+clean; 498 MiB data, 4.9 GiB used, 1.0 TiB / 1.0 TiB avail; 7.6 MiB/s rd, 7.2 MiB/s wr, 641 op/s 2024-03-19T01:54:51.099 INFO:tasks.workunit.client.0.smithi099.stdout: 10 36763 26.84 MB/sec warmup 80 sec latency 106.075 ms 2024-03-19T01:54:51.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:54:51 smithi115 ceph-mon[67332]: pgmap v764: 97 pgs: 97 active+clean; 500 MiB data, 4.9 GiB used, 1.0 TiB / 1.0 TiB avail; 7.5 MiB/s rd, 6.8 MiB/s wr, 634 op/s 2024-03-19T01:54:51.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:54:51 smithi099 ceph-mon[62951]: pgmap v764: 97 pgs: 97 active+clean; 500 MiB data, 4.9 GiB used, 1.0 TiB / 1.0 TiB avail; 7.5 MiB/s rd, 6.8 MiB/s wr, 634 op/s 2024-03-19T01:54:51.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:54:51 smithi112 ceph-mon[66819]: pgmap v764: 97 pgs: 97 active+clean; 500 MiB data, 4.9 GiB used, 1.0 TiB / 1.0 TiB avail; 7.5 MiB/s rd, 6.8 MiB/s wr, 634 op/s 2024-03-19T01:54:52.099 INFO:tasks.workunit.client.0.smithi099.stdout: 10 37274 26.96 MB/sec warmup 81 sec latency 69.638 ms 2024-03-19T01:54:53.099 INFO:tasks.workunit.client.0.smithi099.stdout: 10 37872 27.10 MB/sec warmup 82 sec latency 61.191 ms 2024-03-19T01:54:53.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:54:53 smithi099 ceph-mon[62951]: pgmap v765: 97 pgs: 97 active+clean; 503 MiB data, 5.0 GiB used, 1.0 TiB / 1.0 TiB avail; 7.5 MiB/s rd, 7.3 MiB/s wr, 651 op/s 2024-03-19T01:54:53.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:54:53 smithi112 ceph-mon[66819]: pgmap v765: 97 pgs: 97 active+clean; 503 MiB data, 5.0 GiB used, 1.0 TiB / 1.0 TiB avail; 7.5 MiB/s rd, 7.3 MiB/s wr, 651 op/s 2024-03-19T01:54:53.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:54:53 smithi115 ceph-mon[67332]: pgmap v765: 97 pgs: 97 active+clean; 503 MiB data, 5.0 GiB used, 1.0 TiB / 1.0 TiB avail; 7.5 MiB/s rd, 7.3 MiB/s wr, 651 op/s 2024-03-19T01:54:54.099 INFO:tasks.workunit.client.0.smithi099.stdout: 10 38330 27.06 MB/sec warmup 83 sec latency 86.954 ms 2024-03-19T01:54:55.099 INFO:tasks.workunit.client.0.smithi099.stdout: 10 38666 26.89 MB/sec warmup 84 sec latency 165.847 ms 2024-03-19T01:54:55.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:54:55 smithi099 ceph-mon[62951]: pgmap v766: 97 pgs: 97 active+clean; 505 MiB data, 5.0 GiB used, 1.0 TiB / 1.0 TiB avail; 5.9 MiB/s rd, 5.9 MiB/s wr, 589 op/s 2024-03-19T01:54:55.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:54:55 smithi112 ceph-mon[66819]: pgmap v766: 97 pgs: 97 active+clean; 505 MiB data, 5.0 GiB used, 1.0 TiB / 1.0 TiB avail; 5.9 MiB/s rd, 5.9 MiB/s wr, 589 op/s 2024-03-19T01:54:55.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:54:55 smithi115 ceph-mon[67332]: pgmap v766: 97 pgs: 97 active+clean; 505 MiB data, 5.0 GiB used, 1.0 TiB / 1.0 TiB avail; 5.9 MiB/s rd, 5.9 MiB/s wr, 589 op/s 2024-03-19T01:54:56.099 INFO:tasks.workunit.client.0.smithi099.stdout: 10 39033 26.81 MB/sec warmup 85 sec latency 129.594 ms 2024-03-19T01:54:57.099 INFO:tasks.workunit.client.0.smithi099.stdout: 10 39457 26.74 MB/sec warmup 86 sec latency 70.170 ms 2024-03-19T01:54:57.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:54:57 smithi099 ceph-mon[62951]: pgmap v767: 97 pgs: 97 active+clean; 506 MiB data, 5.0 GiB used, 1.0 TiB / 1.0 TiB avail; 7.0 MiB/s rd, 6.9 MiB/s wr, 647 op/s 2024-03-19T01:54:57.357 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:54:57 smithi115 ceph-mon[67332]: pgmap v767: 97 pgs: 97 active+clean; 506 MiB data, 5.0 GiB used, 1.0 TiB / 1.0 TiB avail; 7.0 MiB/s rd, 6.9 MiB/s wr, 647 op/s 2024-03-19T01:54:57.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:54:57 smithi112 ceph-mon[66819]: pgmap v767: 97 pgs: 97 active+clean; 506 MiB data, 5.0 GiB used, 1.0 TiB / 1.0 TiB avail; 7.0 MiB/s rd, 6.9 MiB/s wr, 647 op/s 2024-03-19T01:54:57.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.760+0000 7f8b13977640 1 -- 172.21.15.99:0/480507770 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8b0c100740 msgr2=0x7f8b0c100b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:54:57.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.760+0000 7f8b13977640 1 --2- 172.21.15.99:0/480507770 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8b0c100740 0x7f8b0c100b60 secure :-1 s=READY pgs=615 cs=0 l=1 rev1=1 crypto rx=0x7f8b08009900 tx=0x7f8b0802bc30 comp rx=0 tx=0).stop 2024-03-19T01:54:57.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.760+0000 7f8b13977640 1 -- 172.21.15.99:0/480507770 shutdown_connections 2024-03-19T01:54:57.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.760+0000 7f8b13977640 1 --2- 172.21.15.99:0/480507770 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8b0c101930 0x7f8b0c101db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:57.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.760+0000 7f8b13977640 1 --2- 172.21.15.99:0/480507770 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8b0c1022f0 0x7f8b0c108b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:57.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.760+0000 7f8b13977640 1 --2- 172.21.15.99:0/480507770 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8b0c100740 0x7f8b0c100b60 unknown :-1 s=CLOSED pgs=615 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:57.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.760+0000 7f8b13977640 1 -- 172.21.15.99:0/480507770 >> 172.21.15.99:0/480507770 conn(0x7f8b0c0fbcb0 msgr2=0x7f8b0c0fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:54:57.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.760+0000 7f8b13977640 1 -- 172.21.15.99:0/480507770 shutdown_connections 2024-03-19T01:54:57.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.760+0000 7f8b13977640 1 -- 172.21.15.99:0/480507770 wait complete. 2024-03-19T01:54:57.761 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.760+0000 7f8b13977640 1 Processor -- start 2024-03-19T01:54:57.761 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.760+0000 7f8b13977640 1 -- start start 2024-03-19T01:54:57.763 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.761+0000 7f8b13977640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8b0c100740 0x7f8b0c1046d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:57.763 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.761+0000 7f8b13977640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8b0c101930 0x7f8b0c102d10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:57.763 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.761+0000 7f8b13977640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8b0c1022f0 0x7f8b0c103250 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:57.763 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.761+0000 7f8b13977640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f8b0c10c3c0 con 0x7f8b0c1022f0 2024-03-19T01:54:57.763 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.761+0000 7f8b13977640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f8b0c10c240 con 0x7f8b0c101930 2024-03-19T01:54:57.763 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.761+0000 7f8b13977640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f8b0c10c540 con 0x7f8b0c100740 2024-03-19T01:54:57.763 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.761+0000 7f8b11f14640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8b0c1022f0 0x7f8b0c103250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:57.763 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.761+0000 7f8b11713640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8b0c100740 0x7f8b0c1046d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:57.763 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.761+0000 7f8b11f14640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8b0c1022f0 0x7f8b0c103250 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:40852/0 (socket says 172.21.15.99:40852) 2024-03-19T01:54:57.763 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.761+0000 7f8b11f14640 1 -- 172.21.15.99:0/2829400743 learned_addr learned my addr 172.21.15.99:0/2829400743 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:54:57.763 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.761+0000 7f8b10f12640 1 --2- 172.21.15.99:0/2829400743 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8b0c101930 0x7f8b0c102d10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:57.763 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.761+0000 7f8b11f14640 1 -- 172.21.15.99:0/2829400743 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8b0c100740 msgr2=0x7f8b0c1046d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:54:57.763 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.761+0000 7f8b11f14640 1 --2- 172.21.15.99:0/2829400743 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8b0c100740 0x7f8b0c1046d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:57.764 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.761+0000 7f8b11f14640 1 -- 172.21.15.99:0/2829400743 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8b0c101930 msgr2=0x7f8b0c102d10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:54:57.764 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.761+0000 7f8b11f14640 1 --2- 172.21.15.99:0/2829400743 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8b0c101930 0x7f8b0c102d10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:57.764 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.761+0000 7f8b11f14640 1 -- 172.21.15.99:0/2829400743 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f8b0c103ae0 con 0x7f8b0c1022f0 2024-03-19T01:54:57.764 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.761+0000 7f8b11713640 1 --2- 172.21.15.99:0/2829400743 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8b0c100740 0x7f8b0c1046d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:54:57.764 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.761+0000 7f8b10f12640 1 --2- 172.21.15.99:0/2829400743 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8b0c101930 0x7f8b0c102d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:54:57.764 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.761+0000 7f8b11f14640 1 --2- 172.21.15.99:0/2829400743 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8b0c1022f0 0x7f8b0c103250 secure :-1 s=READY pgs=616 cs=0 l=1 rev1=1 crypto rx=0x7f8b0400eb10 tx=0x7f8b0400c2d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:54:57.764 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.762+0000 7f8afe7fc640 1 -- 172.21.15.99:0/2829400743 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8b04004b20 con 0x7f8b0c1022f0 2024-03-19T01:54:57.764 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.762+0000 7f8b13977640 1 -- 172.21.15.99:0/2829400743 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f8b0c1a1bf0 con 0x7f8b0c1022f0 2024-03-19T01:54:57.764 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.762+0000 7f8afe7fc640 1 -- 172.21.15.99:0/2829400743 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f8b04004cc0 con 0x7f8b0c1022f0 2024-03-19T01:54:57.764 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.762+0000 7f8afe7fc640 1 -- 172.21.15.99:0/2829400743 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8b04010760 con 0x7f8b0c1022f0 2024-03-19T01:54:57.764 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.762+0000 7f8b13977640 1 -- 172.21.15.99:0/2829400743 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f8b0c1a2190 con 0x7f8b0c1022f0 2024-03-19T01:54:57.767 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.764+0000 7f8afe7fc640 1 -- 172.21.15.99:0/2829400743 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f8b04005020 con 0x7f8b0c1022f0 2024-03-19T01:54:57.767 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.765+0000 7f8b16566740 1 Processor -- start 2024-03-19T01:54:57.767 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.765+0000 7f8b16566740 1 -- start start 2024-03-19T01:54:57.767 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.765+0000 7f8b16566740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c165b6f100 0x55c165b6f520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:57.767 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.765+0000 7f8b16566740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c165b6faf0 0x55c165b745f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:57.767 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.765+0000 7f8b16566740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c165b74c60 0x55c165b770f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:57.767 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.765+0000 7f8b16566740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55c165ad3be0 con 0x55c165b6f100 2024-03-19T01:54:57.767 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.765+0000 7f8b16566740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55c165a7c960 con 0x55c165b6faf0 2024-03-19T01:54:57.767 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.765+0000 7f8b16566740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55c165acfc80 con 0x55c165b74c60 2024-03-19T01:54:57.767 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.765+0000 7f8b10f12640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c165b6f100 0x55c165b6f520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:57.767 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.765+0000 7f8b10f12640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c165b6f100 0x55c165b6f520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:40860/0 (socket says 172.21.15.99:40860) 2024-03-19T01:54:57.768 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.765+0000 7f8b10f12640 1 -- 172.21.15.99:0/2992685229 learned_addr learned my addr 172.21.15.99:0/2992685229 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:54:57.768 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.765+0000 7f8b11713640 1 --2- 172.21.15.99:0/2992685229 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c165b6faf0 0x55c165b745f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:57.768 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.765+0000 7f8b10f12640 1 --2- 172.21.15.99:0/2992685229 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c165b74c60 0x55c165b770f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:57.768 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.765+0000 7f8afe7fc640 1 --2- 172.21.15.99:0/2829400743 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f8af80a28a0 0x7f8af80a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:57.768 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.765+0000 7f8b11713640 1 -- 172.21.15.99:0/2992685229 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c165b74c60 msgr2=0x55c165b770f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:54:57.768 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.765+0000 7f8b11713640 1 --2- 172.21.15.99:0/2992685229 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c165b74c60 0x55c165b770f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:57.768 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.765+0000 7f8b11f14640 1 --2- 172.21.15.99:0/2829400743 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f8af80a28a0 0x7f8af80a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:57.768 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.765+0000 7f8b11713640 1 -- 172.21.15.99:0/2992685229 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c165b6f100 msgr2=0x55c165b6f520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:54:57.768 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.765+0000 7f8b11713640 1 --2- 172.21.15.99:0/2992685229 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c165b6f100 0x55c165b6f520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:57.768 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.765+0000 7f8b11713640 1 -- 172.21.15.99:0/2992685229 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55c165a73d20 con 0x55c165b6faf0 2024-03-19T01:54:57.768 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.766+0000 7f8b11713640 1 --2- 172.21.15.99:0/2992685229 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c165b6faf0 0x55c165b745f0 secure :-1 s=READY pgs=163 cs=0 l=1 rev1=1 crypto rx=0x7f8b0802c170 tx=0x7f8b0802cd30 comp rx=0 tx=0).ready entity=mon.2 client_cookie=b7a282583dafc0a3 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:54:57.768 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.766+0000 7f8af77fe640 1 -- 172.21.15.99:0/2992685229 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8b08005d50 con 0x55c165b6faf0 2024-03-19T01:54:57.768 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.766+0000 7f8af77fe640 1 -- 172.21.15.99:0/2992685229 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f8b08005520 con 0x55c165b6faf0 2024-03-19T01:54:57.768 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.766+0000 7f8af77fe640 1 -- 172.21.15.99:0/2992685229 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8b08004b80 con 0x55c165b6faf0 2024-03-19T01:54:57.768 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.766+0000 7f8afe7fc640 1 -- 172.21.15.99:0/2829400743 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f8b040cf1a0 con 0x7f8b0c1022f0 2024-03-19T01:54:57.768 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.767+0000 7f8b16566740 1 -- 172.21.15.99:0/2992685229 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c165b6faf0 msgr2=0x55c165b745f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:54:57.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.767+0000 7f8b11f14640 1 --2- 172.21.15.99:0/2829400743 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f8af80a28a0 0x7f8af80a4d60 secure :-1 s=READY pgs=335 cs=0 l=1 rev1=1 crypto rx=0x7f8b040984d0 tx=0x7f8b04097700 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:54:57.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.767+0000 7f8b16566740 1 --2- 172.21.15.99:0/2992685229 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c165b6faf0 0x55c165b745f0 secure :-1 s=READY pgs=163 cs=0 l=1 rev1=1 crypto rx=0x7f8b0802c170 tx=0x7f8b0802cd30 comp rx=0 tx=0).stop 2024-03-19T01:54:57.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.767+0000 7f8b16566740 1 -- 172.21.15.99:0/2992685229 shutdown_connections 2024-03-19T01:54:57.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.767+0000 7f8b16566740 1 --2- 172.21.15.99:0/2992685229 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c165b6faf0 0x55c165b745f0 unknown :-1 s=CLOSED pgs=163 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:57.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.767+0000 7f8b16566740 1 --2- 172.21.15.99:0/2992685229 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c165b74c60 0x55c165b770f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:57.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.767+0000 7f8b16566740 1 --2- 172.21.15.99:0/2992685229 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c165b6f100 0x55c165b6f520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:57.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.767+0000 7f8b16566740 1 -- 172.21.15.99:0/2992685229 >> 172.21.15.99:0/2992685229 conn(0x55c165b6a670 msgr2=0x55c165b6aa70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:54:57.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.767+0000 7f8b16566740 1 -- 172.21.15.99:0/2992685229 shutdown_connections 2024-03-19T01:54:57.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.767+0000 7f8b16566740 1 -- 172.21.15.99:0/2992685229 wait complete. 2024-03-19T01:54:57.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.767+0000 7f8b16566740 1 -- start start 2024-03-19T01:54:57.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.767+0000 7f8b16566740 1 Processor -- start 2024-03-19T01:54:57.772 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.768+0000 7f8b16566740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c165b6f100 0x55c165b790a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:57.772 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.768+0000 7f8b16566740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c165b6faf0 0x55c165b795e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:57.772 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.768+0000 7f8b16566740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c165b74c60 0x55c165b80620 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:57.772 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.768+0000 7f8b16566740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55c165b77bb0 con 0x55c165b6faf0 2024-03-19T01:54:57.772 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.768+0000 7f8b16566740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55c165a8fdb0 con 0x55c165b6f100 2024-03-19T01:54:57.772 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.768+0000 7f8b16566740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55c165b77d30 con 0x55c165b74c60 2024-03-19T01:54:57.772 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.768+0000 7f8b16566740 10 client.0 _ll_register_callbacks cb 0x55c165a94e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:54:57.772 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.768+0000 7f8b11713640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c165b6faf0 0x55c165b795e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:57.772 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.768+0000 7f8b11713640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c165b6faf0 0x55c165b795e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:40876/0 (socket says 172.21.15.99:40876) 2024-03-19T01:54:57.772 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.768+0000 7f8b10f12640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c165b6f100 0x55c165b790a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:57.772 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.768+0000 7f8b11713640 1 -- 172.21.15.99:0/3574358786 learned_addr learned my addr 172.21.15.99:0/3574358786 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:54:57.772 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.768+0000 7f8b10f12640 1 --2- 172.21.15.99:0/3574358786 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c165b74c60 0x55c165b80620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:57.772 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.769+0000 7f8b11713640 1 -- 172.21.15.99:0/3574358786 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c165b74c60 msgr2=0x55c165b80620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:54:57.773 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.769+0000 7f8b11713640 1 --2- 172.21.15.99:0/3574358786 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c165b74c60 0x55c165b80620 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:57.773 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.769+0000 7f8b11713640 1 -- 172.21.15.99:0/3574358786 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c165b6f100 msgr2=0x55c165b790a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:54:57.773 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.769+0000 7f8b11713640 1 --2- 172.21.15.99:0/3574358786 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c165b6f100 0x55c165b790a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:57.773 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.769+0000 7f8b11713640 1 -- 172.21.15.99:0/3574358786 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55c165a8ec60 con 0x55c165b6faf0 2024-03-19T01:54:57.773 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.769+0000 7f8b10f12640 1 --2- 172.21.15.99:0/3574358786 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c165b6f100 0x55c165b790a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:54:57.773 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.769+0000 7f8b11713640 1 --2- 172.21.15.99:0/3574358786 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c165b6faf0 0x55c165b795e0 secure :-1 s=READY pgs=617 cs=0 l=1 rev1=1 crypto rx=0x7f8b0802c7b0 tx=0x7f8b080038a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6e58ff42f05acbcd server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:54:57.773 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.769+0000 7f8af4ff9640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:54:57.773 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.769+0000 7f8af4ff9640 1 -- 172.21.15.99:0/3574358786 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8b08005d50 con 0x55c165b6faf0 2024-03-19T01:54:57.773 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.769+0000 7f8af4ff9640 1 -- 172.21.15.99:0/3574358786 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f8b08034440 con 0x55c165b6faf0 2024-03-19T01:54:57.773 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.769+0000 7f8b16566740 1 -- 172.21.15.99:0/3574358786 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f8b08005520 con 0x55c165b6faf0 2024-03-19T01:54:57.773 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.769+0000 7f8af4ff9640 1 -- 172.21.15.99:0/3574358786 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=33) ==== 24+0+0 (secure 0 0 0) 0x7f8b08004460 con 0x55c165b6faf0 2024-03-19T01:54:57.773 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.769+0000 7f8b16566740 10 client.16464 fetch_fsmap learned FSMap version 33 2024-03-19T01:54:57.773 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.770+0000 7f8b16566740 1 -- 172.21.15.99:0/3574358786 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=33,monmap=4+}) -- 0x55c1657e8210 con 0x55c165b6faf0 2024-03-19T01:54:57.773 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.770+0000 7f8af4ff9640 1 -- 172.21.15.99:0/3574358786 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f8b08005520 con 0x55c165b6faf0 2024-03-19T01:54:57.773 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.770+0000 7f8af4ff9640 1 -- 172.21.15.99:0/3574358786 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 33) ==== 8362+0+0 (secure 0 0 0) 0x7f8b08045420 con 0x55c165b6faf0 2024-03-19T01:54:57.773 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.770+0000 7f8b16566740 10 client.16464 fetch_fsmap finished waiting for FSMap version 33 2024-03-19T01:54:57.773 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.770+0000 7f8b16566740 10 client.16464 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:54:57.774 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.770+0000 7f8b16566740 20 client.16464 populate_metadata read hostname 'smithi099' 2024-03-19T01:54:57.774 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.770+0000 7f8b16566740 1 --2- 172.21.15.99:0/3574358786 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55c165b84e50 0x55c165b872b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:57.774 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.770+0000 7f8b16566740 4 client.16464 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:54:57.774 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.770+0000 7f8b16566740 1 -- 172.21.15.99:0/3574358786 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55c165ae1120 con 0x55c165b84e50 2024-03-19T01:54:57.774 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.770+0000 7f8b10f12640 1 --2- 172.21.15.99:0/3574358786 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55c165b84e50 0x55c165b872b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:57.774 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.771+0000 7f8b10f12640 1 --2- 172.21.15.99:0/3574358786 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55c165b84e50 0x55c165b872b0 crc :-1 s=READY pgs=153 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=ee9b2a21df6d5ab7 server_cookie=6e957539f2324a71 in_seq=0 out_seq=0 2024-03-19T01:54:57.774 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.771+0000 7f8af4ff9640 10 client.16464 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:54:57.776 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.776+0000 7f8af4ff9640 1 -- 172.21.15.99:0/3574358786 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x55c165ae1120 con 0x55c165b84e50 2024-03-19T01:54:57.776 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.776+0000 7f8af4ff9640 10 client.16464 handle_command_reply: tid=0 2024-03-19T01:54:57.776 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.776+0000 7f8b16566740 1 client.16464 shutdown 2024-03-19T01:54:57.776 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.776+0000 7f8b16566740 1 -- 172.21.15.99:0/3574358786 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c165b6faf0 msgr2=0x55c165b795e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:54:57.776 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.776+0000 7f8b16566740 1 --2- 172.21.15.99:0/3574358786 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c165b6faf0 0x55c165b795e0 secure :-1 s=READY pgs=617 cs=0 l=1 rev1=1 crypto rx=0x7f8b0802c7b0 tx=0x7f8b080038a0 comp rx=0 tx=0).stop 2024-03-19T01:54:57.776 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.776+0000 7f8b16566740 1 -- 172.21.15.99:0/3574358786 shutdown_connections 2024-03-19T01:54:57.776 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.776+0000 7f8b16566740 1 --2- 172.21.15.99:0/3574358786 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55c165b84e50 0x55c165b872b0 crc :-1 s=READY pgs=153 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:57.776 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.776+0000 7f8b16566740 1 --2- 172.21.15.99:0/3574358786 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c165b74c60 0x55c165b80620 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:57.777 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.776+0000 7f8b16566740 1 --2- 172.21.15.99:0/3574358786 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c165b6faf0 0x55c165b795e0 unknown :-1 s=CLOSED pgs=617 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:57.777 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.776+0000 7f8af4ff9640 0 client.16464 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:54:57.777 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.776+0000 7f8b16566740 1 --2- 172.21.15.99:0/3574358786 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c165b6f100 0x55c165b790a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:57.777 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.776+0000 7f8b16566740 1 -- 172.21.15.99:0/3574358786 >> 172.21.15.99:0/3574358786 conn(0x55c165b6a670 msgr2=0x55c165b6b8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:54:57.777 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.776+0000 7f8b16566740 1 -- 172.21.15.99:0/3574358786 shutdown_connections 2024-03-19T01:54:57.777 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.776+0000 7f8b16566740 1 -- 172.21.15.99:0/3574358786 wait complete. 2024-03-19T01:54:57.777 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.776+0000 7f8b16566740 20 client.16464 trim_cache size 0 max 16384 2024-03-19T01:54:57.792 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.792+0000 7f8b16566740 1 Processor -- start 2024-03-19T01:54:57.793 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.792+0000 7f8b16566740 1 -- start start 2024-03-19T01:54:57.793 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.792+0000 7f8b16566740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c165b6f100 0x55c165b69ab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:57.793 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.792+0000 7f8b16566740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c165b6faf0 0x55c165b805e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:57.793 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.792+0000 7f8b16566740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c165b74c60 0x55c165b6dfe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:57.793 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.792+0000 7f8b16566740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f8b0c1ad8a0 con 0x55c165b6faf0 2024-03-19T01:54:57.793 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.792+0000 7f8b16566740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f8b0c1ab0b0 con 0x55c165b74c60 2024-03-19T01:54:57.793 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.792+0000 7f8b16566740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f8b0c1a8f00 con 0x55c165b6f100 2024-03-19T01:54:57.793 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.792+0000 7f8b11713640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c165b6faf0 0x55c165b805e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:57.793 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.792+0000 7f8b11713640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c165b6faf0 0x55c165b805e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:40880/0 (socket says 172.21.15.99:40880) 2024-03-19T01:54:57.793 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.792+0000 7f8b11713640 1 -- 172.21.15.99:0/1722492316 learned_addr learned my addr 172.21.15.99:0/1722492316 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:54:57.793 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.792+0000 7f8b10f12640 1 --2- 172.21.15.99:0/1722492316 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c165b6f100 0x55c165b69ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:57.793 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.792+0000 7f8b10f12640 1 --2- 172.21.15.99:0/1722492316 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c165b74c60 0x55c165b6dfe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:57.793 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.792+0000 7f8b11713640 1 -- 172.21.15.99:0/1722492316 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c165b6f100 msgr2=0x55c165b69ab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:54:57.793 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.792+0000 7f8b11713640 1 --2- 172.21.15.99:0/1722492316 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c165b6f100 0x55c165b69ab0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:57.793 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.792+0000 7f8b11713640 1 -- 172.21.15.99:0/1722492316 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c165b74c60 msgr2=0x55c165b6dfe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:54:57.794 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.792+0000 7f8b11713640 1 --2- 172.21.15.99:0/1722492316 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c165b74c60 0x55c165b6dfe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:57.794 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.792+0000 7f8b11713640 1 -- 172.21.15.99:0/1722492316 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55c1658b4620 con 0x55c165b6faf0 2024-03-19T01:54:57.794 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.792+0000 7f8b11713640 1 --2- 172.21.15.99:0/1722492316 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c165b6faf0 0x55c165b805e0 secure :-1 s=READY pgs=618 cs=0 l=1 rev1=1 crypto rx=0x7f8b08009900 tx=0x7f8b08001e70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9a5b77c96cbf269f server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:54:57.794 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.793+0000 7f8af6ffd640 1 -- 172.21.15.99:0/1722492316 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8b08005d50 con 0x55c165b6faf0 2024-03-19T01:54:57.794 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.793+0000 7f8af6ffd640 1 -- 172.21.15.99:0/1722492316 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f8b0804d070 con 0x55c165b6faf0 2024-03-19T01:54:57.794 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.793+0000 7f8af6ffd640 1 -- 172.21.15.99:0/1722492316 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8b08045420 con 0x55c165b6faf0 2024-03-19T01:54:57.797 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.793+0000 7f8b16566740 1 -- 172.21.15.99:0/1722492316 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c165b6faf0 msgr2=0x55c165b805e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:54:57.797 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.793+0000 7f8b16566740 1 --2- 172.21.15.99:0/1722492316 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c165b6faf0 0x55c165b805e0 secure :-1 s=READY pgs=618 cs=0 l=1 rev1=1 crypto rx=0x7f8b08009900 tx=0x7f8b08001e70 comp rx=0 tx=0).stop 2024-03-19T01:54:57.797 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.794+0000 7f8b16566740 1 -- 172.21.15.99:0/1722492316 shutdown_connections 2024-03-19T01:54:57.797 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.794+0000 7f8b16566740 1 --2- 172.21.15.99:0/1722492316 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c165b74c60 0x55c165b6dfe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:57.797 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.794+0000 7f8b16566740 1 --2- 172.21.15.99:0/1722492316 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c165b6faf0 0x55c165b805e0 unknown :-1 s=CLOSED pgs=618 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:57.797 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.794+0000 7f8b16566740 1 --2- 172.21.15.99:0/1722492316 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c165b6f100 0x55c165b69ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:57.797 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.794+0000 7f8b16566740 1 -- 172.21.15.99:0/1722492316 >> 172.21.15.99:0/1722492316 conn(0x55c165b6a670 msgr2=0x55c165b6d8d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:54:57.797 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.794+0000 7f8b16566740 1 -- 172.21.15.99:0/1722492316 shutdown_connections 2024-03-19T01:54:57.797 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.794+0000 7f8b16566740 1 -- 172.21.15.99:0/1722492316 wait complete. 2024-03-19T01:54:57.798 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.794+0000 7f8b16566740 1 -- start start 2024-03-19T01:54:57.798 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.794+0000 7f8b16566740 1 Processor -- start 2024-03-19T01:54:57.798 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.794+0000 7f8b16566740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c165b6f100 0x55c165b7df70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:57.798 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.794+0000 7f8b16566740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c165b6faf0 0x55c165b7e4b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:57.798 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.795+0000 7f8b16566740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c165b74c60 0x55c165b7f4c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:57.798 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.795+0000 7f8b16566740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f8b0c1a9570 con 0x55c165b6f100 2024-03-19T01:54:57.798 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.795+0000 7f8b16566740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f8b0c1aa9e0 con 0x55c165b6faf0 2024-03-19T01:54:57.798 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.795+0000 7f8b16566740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f8b0c1a9c40 con 0x55c165b74c60 2024-03-19T01:54:57.798 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.795+0000 7f8b16566740 10 client.0 _ll_register_callbacks cb 0x7f8b00009ec0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:54:57.798 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.795+0000 7f8b10f12640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c165b6f100 0x55c165b7df70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:57.798 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.795+0000 7f8b11713640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c165b6faf0 0x55c165b7e4b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:57.798 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.795+0000 7f8b10f12640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c165b6f100 0x55c165b7df70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:40896/0 (socket says 172.21.15.99:40896) 2024-03-19T01:54:57.798 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.795+0000 7f8b10f12640 1 -- 172.21.15.99:0/2454215238 learned_addr learned my addr 172.21.15.99:0/2454215238 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:54:57.798 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.795+0000 7f8b10f12640 1 --2- 172.21.15.99:0/2454215238 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c165b74c60 0x55c165b7f4c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:57.799 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.795+0000 7f8b11713640 1 -- 172.21.15.99:0/2454215238 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c165b74c60 msgr2=0x55c165b7f4c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:54:57.799 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.795+0000 7f8b11713640 1 --2- 172.21.15.99:0/2454215238 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c165b74c60 0x55c165b7f4c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:57.799 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.795+0000 7f8b11713640 1 -- 172.21.15.99:0/2454215238 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c165b6f100 msgr2=0x55c165b7df70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:54:57.799 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.795+0000 7f8b11713640 1 --2- 172.21.15.99:0/2454215238 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c165b6f100 0x55c165b7df70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:57.799 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.795+0000 7f8b11713640 1 -- 172.21.15.99:0/2454215238 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55c165b7fab0 con 0x55c165b6faf0 2024-03-19T01:54:57.799 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.795+0000 7f8b11713640 1 --2- 172.21.15.99:0/2454215238 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c165b6faf0 0x55c165b7e4b0 secure :-1 s=READY pgs=164 cs=0 l=1 rev1=1 crypto rx=0x55c165b703a0 tx=0x7f8b08003ed0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=8f21ba91223a7f46 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:54:57.799 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.795+0000 7f8af4ff9640 10 client.0 ms_handle_connect on v2:172.21.15.112:3300/0 2024-03-19T01:54:57.799 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.795+0000 7f8af4ff9640 1 -- 172.21.15.99:0/2454215238 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8b080038a0 con 0x55c165b6faf0 2024-03-19T01:54:57.799 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.796+0000 7f8af4ff9640 1 -- 172.21.15.99:0/2454215238 <== mon.2 v2:172.21.15.112:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f8b08005020 con 0x55c165b6faf0 2024-03-19T01:54:57.799 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.796+0000 7f8b16566740 1 -- 172.21.15.99:0/2454215238 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f8b0804d070 con 0x55c165b6faf0 2024-03-19T01:54:57.799 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.796+0000 7f8af4ff9640 1 -- 172.21.15.99:0/2454215238 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_get_version_reply(handle=1 version=33) ==== 24+0+0 (secure 0 0 0) 0x7f8b08045a80 con 0x55c165b6faf0 2024-03-19T01:54:57.799 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.796+0000 7f8b16566740 10 client.26240 fetch_fsmap learned FSMap version 33 2024-03-19T01:54:57.799 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.796+0000 7f8b16566740 1 -- 172.21.15.99:0/2454215238 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,fsmap=33,monmap=4+}) -- 0x55c165beb230 con 0x55c165b6faf0 2024-03-19T01:54:57.799 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.796+0000 7f8af4ff9640 1 -- 172.21.15.99:0/2454215238 <== mon.2 v2:172.21.15.112:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f8b0804d070 con 0x55c165b6faf0 2024-03-19T01:54:57.799 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.796+0000 7f8af4ff9640 1 -- 172.21.15.99:0/2454215238 <== mon.2 v2:172.21.15.112:3300/0 5 ==== fsmap(e 33) ==== 8362+0+0 (secure 0 0 0) 0x7f8b08064080 con 0x55c165b6faf0 2024-03-19T01:54:57.799 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.796+0000 7f8b16566740 10 client.26240 fetch_fsmap finished waiting for FSMap version 33 2024-03-19T01:54:57.800 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.796+0000 7f8b16566740 10 client.26240 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:54:57.800 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.796+0000 7f8b16566740 20 client.26240 populate_metadata read hostname 'smithi099' 2024-03-19T01:54:57.800 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.796+0000 7f8b16566740 1 --2- 172.21.15.99:0/2454215238 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55c165bebba0 0x55c165bedfc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:54:57.800 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.796+0000 7f8b16566740 4 client.26240 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:54:57.800 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.796+0000 7f8b16566740 1 -- 172.21.15.99:0/2454215238 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55c165adb470 con 0x55c165bebba0 2024-03-19T01:54:57.800 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.796+0000 7f8b10f12640 1 --2- 172.21.15.99:0/2454215238 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55c165bebba0 0x55c165bedfc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:54:57.800 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.797+0000 7f8af4ff9640 10 client.26240 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:54:57.800 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:57.797+0000 7f8b10f12640 1 --2- 172.21.15.99:0/2454215238 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55c165bebba0 0x55c165bedfc0 crc :-1 s=READY pgs=154 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=76554ab459e3b74d server_cookie=79434d34fdf834f3 in_seq=0 out_seq=0 2024-03-19T01:54:58.099 INFO:tasks.workunit.client.0.smithi099.stdout: 10 39847 26.68 MB/sec warmup 87 sec latency 187.779 ms 2024-03-19T01:54:58.206 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:58.205+0000 7f8af4ff9640 1 -- 172.21.15.99:0/2454215238 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x55c165adb470 con 0x55c165bebba0 2024-03-19T01:54:58.206 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:58.205+0000 7f8af4ff9640 10 client.26240 handle_command_reply: tid=0 2024-03-19T01:54:58.206 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:58.205+0000 7f8b16566740 1 client.26240 shutdown 2024-03-19T01:54:58.206 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:58.205+0000 7f8b16566740 1 -- 172.21.15.99:0/2454215238 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c165b6faf0 msgr2=0x55c165b7e4b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:54:58.206 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:58.205+0000 7f8b16566740 1 --2- 172.21.15.99:0/2454215238 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c165b6faf0 0x55c165b7e4b0 secure :-1 s=READY pgs=164 cs=0 l=1 rev1=1 crypto rx=0x55c165b703a0 tx=0x7f8b08003ed0 comp rx=0 tx=0).stop 2024-03-19T01:54:58.206 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:58.205+0000 7f8b16566740 1 -- 172.21.15.99:0/2454215238 shutdown_connections 2024-03-19T01:54:58.206 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:58.205+0000 7f8b16566740 1 --2- 172.21.15.99:0/2454215238 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55c165bebba0 0x55c165bedfc0 crc :-1 s=READY pgs=154 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:58.206 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:58.205+0000 7f8b16566740 1 --2- 172.21.15.99:0/2454215238 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c165b6faf0 0x55c165b7e4b0 unknown :-1 s=CLOSED pgs=164 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:58.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:58.205+0000 7f8af4ff9640 0 client.26240 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:54:58.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:58.205+0000 7f8b16566740 1 --2- 172.21.15.99:0/2454215238 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c165b74c60 0x55c165b7f4c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:58.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:58.205+0000 7f8b16566740 1 --2- 172.21.15.99:0/2454215238 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c165b6f100 0x55c165b7df70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:58.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:58.205+0000 7f8b16566740 1 -- 172.21.15.99:0/2454215238 >> 172.21.15.99:0/2454215238 conn(0x55c165b6a670 msgr2=0x55c165b6e160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:54:58.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:58.206+0000 7f8b16566740 1 -- 172.21.15.99:0/2454215238 shutdown_connections 2024-03-19T01:54:58.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:58.206+0000 7f8b16566740 1 -- 172.21.15.99:0/2454215238 wait complete. 2024-03-19T01:54:58.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:58.206+0000 7f8b16566740 20 client.26240 trim_cache size 0 max 16384 2024-03-19T01:54:58.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:54:58.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:54:58.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:54:58.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:54:58.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:58.206+0000 7f8b13977640 1 -- 172.21.15.99:0/2829400743 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f8af80a28a0 msgr2=0x7f8af80a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:54:58.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:58.206+0000 7f8b13977640 1 --2- 172.21.15.99:0/2829400743 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f8af80a28a0 0x7f8af80a4d60 secure :-1 s=READY pgs=335 cs=0 l=1 rev1=1 crypto rx=0x7f8b040984d0 tx=0x7f8b04097700 comp rx=0 tx=0).stop 2024-03-19T01:54:58.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:58.206+0000 7f8b13977640 1 -- 172.21.15.99:0/2829400743 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8b0c1022f0 msgr2=0x7f8b0c103250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:54:58.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:58.206+0000 7f8b13977640 1 --2- 172.21.15.99:0/2829400743 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8b0c1022f0 0x7f8b0c103250 secure :-1 s=READY pgs=616 cs=0 l=1 rev1=1 crypto rx=0x7f8b0400eb10 tx=0x7f8b0400c2d0 comp rx=0 tx=0).stop 2024-03-19T01:54:58.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:58.206+0000 7f8b13977640 1 -- 172.21.15.99:0/2829400743 shutdown_connections 2024-03-19T01:54:58.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:58.206+0000 7f8b13977640 1 --2- 172.21.15.99:0/2829400743 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f8af80a28a0 0x7f8af80a4d60 unknown :-1 s=CLOSED pgs=335 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:58.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:58.206+0000 7f8b13977640 1 --2- 172.21.15.99:0/2829400743 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8b0c101930 0x7f8b0c102d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:58.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:58.206+0000 7f8b13977640 1 --2- 172.21.15.99:0/2829400743 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8b0c1022f0 0x7f8b0c103250 unknown :-1 s=CLOSED pgs=616 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:58.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:58.206+0000 7f8b13977640 1 --2- 172.21.15.99:0/2829400743 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8b0c100740 0x7f8b0c1046d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:54:58.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:58.206+0000 7f8b13977640 1 -- 172.21.15.99:0/2829400743 >> 172.21.15.99:0/2829400743 conn(0x7f8b0c0fbcb0 msgr2=0x7f8b0c0fd6e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:54:58.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:58.206+0000 7f8b13977640 1 -- 172.21.15.99:0/2829400743 shutdown_connections 2024-03-19T01:54:58.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:54:58.206+0000 7f8b13977640 1 -- 172.21.15.99:0/2829400743 wait complete. 2024-03-19T01:54:59.099 INFO:tasks.workunit.client.0.smithi099.stdout: 10 40288 26.67 MB/sec warmup 88 sec latency 223.879 ms 2024-03-19T01:54:59.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:54:59 smithi099 ceph-mon[62951]: pgmap v768: 97 pgs: 97 active+clean; 506 MiB data, 5.0 GiB used, 1.0 TiB / 1.0 TiB avail; 5.6 MiB/s rd, 6.1 MiB/s wr, 495 op/s 2024-03-19T01:54:59.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:54:59 smithi112 ceph-mon[66819]: pgmap v768: 97 pgs: 97 active+clean; 506 MiB data, 5.0 GiB used, 1.0 TiB / 1.0 TiB avail; 5.6 MiB/s rd, 6.1 MiB/s wr, 495 op/s 2024-03-19T01:54:59.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:54:59 smithi115 ceph-mon[67332]: pgmap v768: 97 pgs: 97 active+clean; 506 MiB data, 5.0 GiB used, 1.0 TiB / 1.0 TiB avail; 5.6 MiB/s rd, 6.1 MiB/s wr, 495 op/s 2024-03-19T01:55:00.099 INFO:tasks.workunit.client.0.smithi099.stdout: 10 40622 26.55 MB/sec warmup 89 sec latency 210.919 ms 2024-03-19T01:55:01.099 INFO:tasks.workunit.client.0.smithi099.stdout: 10 41090 26.50 MB/sec warmup 90 sec latency 99.501 ms 2024-03-19T01:55:01.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:55:01 smithi099 ceph-mon[62951]: pgmap v769: 97 pgs: 97 active+clean; 496 MiB data, 5.1 GiB used, 1.0 TiB / 1.0 TiB avail; 5.9 MiB/s rd, 6.5 MiB/s wr, 502 op/s 2024-03-19T01:55:01.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:55:01 smithi112 ceph-mon[66819]: pgmap v769: 97 pgs: 97 active+clean; 496 MiB data, 5.1 GiB used, 1.0 TiB / 1.0 TiB avail; 5.9 MiB/s rd, 6.5 MiB/s wr, 502 op/s 2024-03-19T01:55:01.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:55:01 smithi115 ceph-mon[67332]: pgmap v769: 97 pgs: 97 active+clean; 496 MiB data, 5.1 GiB used, 1.0 TiB / 1.0 TiB avail; 5.9 MiB/s rd, 6.5 MiB/s wr, 502 op/s 2024-03-19T01:55:02.099 INFO:tasks.workunit.client.0.smithi099.stdout: 10 41639 26.45 MB/sec warmup 91 sec latency 95.588 ms 2024-03-19T01:55:03.100 INFO:tasks.workunit.client.0.smithi099.stdout: 10 42214 26.57 MB/sec warmup 92 sec latency 54.300 ms 2024-03-19T01:55:03.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:55:03 smithi099 ceph-mon[62951]: pgmap v770: 97 pgs: 97 active+clean; 489 MiB data, 5.0 GiB used, 1.0 TiB / 1.0 TiB avail; 5.9 MiB/s rd, 7.0 MiB/s wr, 605 op/s 2024-03-19T01:55:03.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:55:03 smithi112 ceph-mon[66819]: pgmap v770: 97 pgs: 97 active+clean; 489 MiB data, 5.0 GiB used, 1.0 TiB / 1.0 TiB avail; 5.9 MiB/s rd, 7.0 MiB/s wr, 605 op/s 2024-03-19T01:55:03.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:55:03 smithi115 ceph-mon[67332]: pgmap v770: 97 pgs: 97 active+clean; 489 MiB data, 5.0 GiB used, 1.0 TiB / 1.0 TiB avail; 5.9 MiB/s rd, 7.0 MiB/s wr, 605 op/s 2024-03-19T01:55:04.100 INFO:tasks.workunit.client.0.smithi099.stdout: 10 42774 26.73 MB/sec warmup 93 sec latency 51.423 ms 2024-03-19T01:55:05.100 INFO:tasks.workunit.client.0.smithi099.stdout: 10 43187 26.66 MB/sec warmup 94 sec latency 198.506 ms 2024-03-19T01:55:05.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:55:05 smithi099 ceph-mon[62951]: pgmap v771: 97 pgs: 97 active+clean; 492 MiB data, 5.0 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 MiB/s rd, 6.2 MiB/s wr, 564 op/s 2024-03-19T01:55:05.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:55:05 smithi112 ceph-mon[66819]: pgmap v771: 97 pgs: 97 active+clean; 492 MiB data, 5.0 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 MiB/s rd, 6.2 MiB/s wr, 564 op/s 2024-03-19T01:55:05.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:55:05 smithi115 ceph-mon[67332]: pgmap v771: 97 pgs: 97 active+clean; 492 MiB data, 5.0 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 MiB/s rd, 6.2 MiB/s wr, 564 op/s 2024-03-19T01:55:06.100 INFO:tasks.workunit.client.0.smithi099.stdout: 10 43570 26.53 MB/sec warmup 95 sec latency 105.344 ms 2024-03-19T01:55:07.100 INFO:tasks.workunit.client.0.smithi099.stdout: 10 43980 26.55 MB/sec warmup 96 sec latency 120.287 ms 2024-03-19T01:55:07.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:55:07 smithi099 ceph-mon[62951]: pgmap v772: 97 pgs: 97 active+clean; 488 MiB data, 5.1 GiB used, 1.0 TiB / 1.0 TiB avail; 5.5 MiB/s rd, 6.9 MiB/s wr, 583 op/s 2024-03-19T01:55:07.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:55:07 smithi112 ceph-mon[66819]: pgmap v772: 97 pgs: 97 active+clean; 488 MiB data, 5.1 GiB used, 1.0 TiB / 1.0 TiB avail; 5.5 MiB/s rd, 6.9 MiB/s wr, 583 op/s 2024-03-19T01:55:07.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:55:07 smithi115 ceph-mon[67332]: pgmap v772: 97 pgs: 97 active+clean; 488 MiB data, 5.1 GiB used, 1.0 TiB / 1.0 TiB avail; 5.5 MiB/s rd, 6.9 MiB/s wr, 583 op/s 2024-03-19T01:55:08.100 INFO:tasks.workunit.client.0.smithi099.stdout: 10 44415 26.45 MB/sec warmup 97 sec latency 101.780 ms 2024-03-19T01:55:09.100 INFO:tasks.workunit.client.0.smithi099.stdout: 10 44904 26.48 MB/sec warmup 98 sec latency 94.424 ms 2024-03-19T01:55:09.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:55:09 smithi099 ceph-mon[62951]: pgmap v773: 97 pgs: 97 active+clean; 492 MiB data, 5.1 GiB used, 1.0 TiB / 1.0 TiB avail; 4.9 MiB/s rd, 6.5 MiB/s wr, 569 op/s 2024-03-19T01:55:09.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:55:09 smithi112 ceph-mon[66819]: pgmap v773: 97 pgs: 97 active+clean; 492 MiB data, 5.1 GiB used, 1.0 TiB / 1.0 TiB avail; 4.9 MiB/s rd, 6.5 MiB/s wr, 569 op/s 2024-03-19T01:55:09.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:55:09 smithi115 ceph-mon[67332]: pgmap v773: 97 pgs: 97 active+clean; 492 MiB data, 5.1 GiB used, 1.0 TiB / 1.0 TiB avail; 4.9 MiB/s rd, 6.5 MiB/s wr, 569 op/s 2024-03-19T01:55:10.100 INFO:tasks.workunit.client.0.smithi099.stdout: 10 45356 26.41 MB/sec warmup 99 sec latency 131.462 ms 2024-03-19T01:55:11.100 INFO:tasks.workunit.client.0.smithi099.stdout: 10 45827 26.42 MB/sec warmup 100 sec latency 238.595 ms 2024-03-19T01:55:11.308 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.306+0000 7f3a1d977640 1 -- 172.21.15.99:0/536769741 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3a1810db40 msgr2=0x7f3a1810dfc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:55:11.308 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.306+0000 7f3a1d977640 1 --2- 172.21.15.99:0/536769741 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3a1810db40 0x7f3a1810dfc0 secure :-1 s=READY pgs=165 cs=0 l=1 rev1=1 crypto rx=0x7f3a10009be0 tx=0x7f3a1002bef0 comp rx=0 tx=0).stop 2024-03-19T01:55:11.308 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.306+0000 7f3a1d977640 1 -- 172.21.15.99:0/536769741 shutdown_connections 2024-03-19T01:55:11.308 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.306+0000 7f3a1d977640 1 --2- 172.21.15.99:0/536769741 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3a1810db40 0x7f3a1810dfc0 unknown :-1 s=CLOSED pgs=165 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:11.308 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.306+0000 7f3a1d977640 1 --2- 172.21.15.99:0/536769741 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3a1810b190 0x7f3a1810d600 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:11.308 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.306+0000 7f3a1d977640 1 --2- 172.21.15.99:0/536769741 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f3a18108820 0x7f3a1810ac50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:11.308 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.306+0000 7f3a1d977640 1 -- 172.21.15.99:0/536769741 >> 172.21.15.99:0/536769741 conn(0x7f3a1806e860 msgr2=0x7f3a18070ca0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:55:11.308 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.306+0000 7f3a1d977640 1 -- 172.21.15.99:0/536769741 shutdown_connections 2024-03-19T01:55:11.308 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.306+0000 7f3a1d977640 1 -- 172.21.15.99:0/536769741 wait complete. 2024-03-19T01:55:11.308 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.307+0000 7f3a1d977640 1 Processor -- start 2024-03-19T01:55:11.308 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.307+0000 7f3a1d977640 1 -- start start 2024-03-19T01:55:11.309 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.307+0000 7f3a1d977640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f3a18108820 0x7f3a181a6860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:11.309 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.307+0000 7f3a1d977640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3a1810b190 0x7f3a181a6da0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:11.309 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.307+0000 7f3a1d977640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3a1810db40 0x7f3a181addc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:11.309 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.307+0000 7f3a1d977640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f3a1811c2c0 con 0x7f3a1810db40 2024-03-19T01:55:11.309 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.307+0000 7f3a1d977640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f3a1811c140 con 0x7f3a1810b190 2024-03-19T01:55:11.309 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.307+0000 7f3a1d977640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f3a1811c440 con 0x7f3a18108820 2024-03-19T01:55:11.309 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.307+0000 7f3a177fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3a1810db40 0x7f3a181addc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:11.309 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.307+0000 7f3a0ffff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3a1810b190 0x7f3a181a6da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:11.309 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.307+0000 7f3a177fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3a1810db40 0x7f3a181addc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:56548/0 (socket says 172.21.15.99:56548) 2024-03-19T01:55:11.309 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.307+0000 7f3a16ffd640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f3a18108820 0x7f3a181a6860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:11.309 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.307+0000 7f3a177fe640 1 -- 172.21.15.99:0/3776929186 learned_addr learned my addr 172.21.15.99:0/3776929186 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:55:11.309 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.307+0000 7f3a177fe640 1 -- 172.21.15.99:0/3776929186 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f3a18108820 msgr2=0x7f3a181a6860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:55:11.309 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.307+0000 7f3a177fe640 1 --2- 172.21.15.99:0/3776929186 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f3a18108820 0x7f3a181a6860 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:11.309 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.307+0000 7f3a177fe640 1 -- 172.21.15.99:0/3776929186 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3a1810b190 msgr2=0x7f3a181a6da0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:55:11.310 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.307+0000 7f3a177fe640 1 --2- 172.21.15.99:0/3776929186 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3a1810b190 0x7f3a181a6da0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:11.310 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.307+0000 7f3a177fe640 1 -- 172.21.15.99:0/3776929186 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f3a181ae430 con 0x7f3a1810db40 2024-03-19T01:55:11.310 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.307+0000 7f3a0ffff640 1 --2- 172.21.15.99:0/3776929186 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3a1810b190 0x7f3a181a6da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:55:11.310 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.308+0000 7f3a177fe640 1 --2- 172.21.15.99:0/3776929186 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3a1810db40 0x7f3a181addc0 secure :-1 s=READY pgs=619 cs=0 l=1 rev1=1 crypto rx=0x7f3a10036040 tx=0x7f3a100337b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:55:11.310 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.308+0000 7f3a1c975640 1 -- 172.21.15.99:0/3776929186 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3a10043020 con 0x7f3a1810db40 2024-03-19T01:55:11.310 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.308+0000 7f3a1c975640 1 -- 172.21.15.99:0/3776929186 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f3a10003b80 con 0x7f3a1810db40 2024-03-19T01:55:11.310 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.308+0000 7f3a1d977640 1 -- 172.21.15.99:0/3776929186 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f3a181ae6c0 con 0x7f3a1810db40 2024-03-19T01:55:11.310 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.308+0000 7f3a1c975640 1 -- 172.21.15.99:0/3776929186 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3a100347b0 con 0x7f3a1810db40 2024-03-19T01:55:11.310 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.308+0000 7f3a1d977640 1 -- 172.21.15.99:0/3776929186 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f3a181aec10 con 0x7f3a1810db40 2024-03-19T01:55:11.314 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.310+0000 7f3a1c975640 1 -- 172.21.15.99:0/3776929186 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f3a10046020 con 0x7f3a1810db40 2024-03-19T01:55:11.314 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.310+0000 7f3a1c975640 1 --2- 172.21.15.99:0/3776929186 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f3a040a28a0 0x7f3a040a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:11.314 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.311+0000 7f3a16ffd640 1 --2- 172.21.15.99:0/3776929186 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f3a040a28a0 0x7f3a040a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:11.314 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.311+0000 7f3a1c975640 1 -- 172.21.15.99:0/3776929186 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f3a100f1170 con 0x7f3a1810db40 2024-03-19T01:55:11.314 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.311+0000 7f3a2063a740 1 Processor -- start 2024-03-19T01:55:11.314 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.311+0000 7f3a2063a740 1 -- start start 2024-03-19T01:55:11.314 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.311+0000 7f3a16ffd640 1 --2- 172.21.15.99:0/3776929186 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f3a040a28a0 0x7f3a040a4d60 secure :-1 s=READY pgs=336 cs=0 l=1 rev1=1 crypto rx=0x7f3a08005d00 tx=0x7f3a0800b410 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:55:11.314 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.311+0000 7f3a2063a740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55adabef1100 0x55adabef1520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:11.315 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.311+0000 7f3a2063a740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55adabef1af0 0x55adabef65f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:11.315 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.311+0000 7f3a2063a740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55adabef6c60 0x55adabef90f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:11.315 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.311+0000 7f3a2063a740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55adabe55be0 con 0x55adabef6c60 2024-03-19T01:55:11.315 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.311+0000 7f3a2063a740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55adabdfe960 con 0x55adabef1af0 2024-03-19T01:55:11.315 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.311+0000 7f3a2063a740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55adabe51c80 con 0x55adabef1100 2024-03-19T01:55:11.315 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.311+0000 7f3a177fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55adabef6c60 0x55adabef90f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:11.315 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.311+0000 7f3a16ffd640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55adabef1100 0x55adabef1520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:11.315 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.311+0000 7f3a177fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55adabef6c60 0x55adabef90f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:56558/0 (socket says 172.21.15.99:56558) 2024-03-19T01:55:11.315 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.311+0000 7f3a177fe640 1 -- 172.21.15.99:0/2651026645 learned_addr learned my addr 172.21.15.99:0/2651026645 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:55:11.315 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.311+0000 7f3a0ffff640 1 --2- 172.21.15.99:0/2651026645 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55adabef1af0 0x55adabef65f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:11.315 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.311+0000 7f3a177fe640 1 -- 172.21.15.99:0/2651026645 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55adabef1100 msgr2=0x55adabef1520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:55:11.315 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.311+0000 7f3a177fe640 1 --2- 172.21.15.99:0/2651026645 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55adabef1100 0x55adabef1520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:11.315 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.311+0000 7f3a177fe640 1 -- 172.21.15.99:0/2651026645 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55adabef1af0 msgr2=0x55adabef65f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:55:11.315 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.311+0000 7f3a177fe640 1 --2- 172.21.15.99:0/2651026645 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55adabef1af0 0x55adabef65f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:11.316 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.311+0000 7f3a177fe640 1 -- 172.21.15.99:0/2651026645 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55adabdf5d20 con 0x55adabef6c60 2024-03-19T01:55:11.316 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.312+0000 7f3a177fe640 1 --2- 172.21.15.99:0/2651026645 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55adabef6c60 0x55adabef90f0 secure :-1 s=READY pgs=620 cs=0 l=1 rev1=1 crypto rx=0x7f3a100b9f30 tx=0x7f3a1002cac0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f9926c7f11dc2bd8 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:55:11.316 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.312+0000 7f3a0dffb640 1 -- 172.21.15.99:0/2651026645 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3a100d6070 con 0x55adabef6c60 2024-03-19T01:55:11.316 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.312+0000 7f3a0dffb640 1 -- 172.21.15.99:0/2651026645 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f3a100c1c20 con 0x55adabef6c60 2024-03-19T01:55:11.316 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.312+0000 7f3a0dffb640 1 -- 172.21.15.99:0/2651026645 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3a10003e10 con 0x55adabef6c60 2024-03-19T01:55:11.316 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.312+0000 7f3a2063a740 1 -- 172.21.15.99:0/2651026645 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55adabef6c60 msgr2=0x55adabef90f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:55:11.316 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.312+0000 7f3a2063a740 1 --2- 172.21.15.99:0/2651026645 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55adabef6c60 0x55adabef90f0 secure :-1 s=READY pgs=620 cs=0 l=1 rev1=1 crypto rx=0x7f3a100b9f30 tx=0x7f3a1002cac0 comp rx=0 tx=0).stop 2024-03-19T01:55:11.316 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.313+0000 7f3a2063a740 1 -- 172.21.15.99:0/2651026645 shutdown_connections 2024-03-19T01:55:11.316 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.313+0000 7f3a2063a740 1 --2- 172.21.15.99:0/2651026645 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55adabef1af0 0x55adabef65f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:11.316 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.313+0000 7f3a2063a740 1 --2- 172.21.15.99:0/2651026645 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55adabef6c60 0x55adabef90f0 unknown :-1 s=CLOSED pgs=620 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:11.316 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.313+0000 7f3a2063a740 1 --2- 172.21.15.99:0/2651026645 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55adabef1100 0x55adabef1520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:11.316 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.313+0000 7f3a2063a740 1 -- 172.21.15.99:0/2651026645 >> 172.21.15.99:0/2651026645 conn(0x55adabeec670 msgr2=0x55adabeeca70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:55:11.316 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.313+0000 7f3a2063a740 1 -- 172.21.15.99:0/2651026645 shutdown_connections 2024-03-19T01:55:11.316 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.313+0000 7f3a2063a740 1 -- 172.21.15.99:0/2651026645 wait complete. 2024-03-19T01:55:11.316 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.313+0000 7f3a2063a740 1 -- start start 2024-03-19T01:55:11.317 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.313+0000 7f3a2063a740 1 Processor -- start 2024-03-19T01:55:11.317 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.314+0000 7f3a2063a740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55adabef1100 0x55adabefae60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:11.317 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.314+0000 7f3a2063a740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55adabef1af0 0x55adabefb3a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:11.317 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.314+0000 7f3a2063a740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55adabef6c60 0x55adabf023e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:11.317 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.314+0000 7f3a2063a740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55adabef99d0 con 0x55adabef1af0 2024-03-19T01:55:11.317 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.314+0000 7f3a2063a740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55adabe11db0 con 0x55adabef1100 2024-03-19T01:55:11.317 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.314+0000 7f3a2063a740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55adabef9b50 con 0x55adabef6c60 2024-03-19T01:55:11.317 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.314+0000 7f3a2063a740 10 client.0 _ll_register_callbacks cb 0x55adabe16e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:55:11.317 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.314+0000 7f3a0ffff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55adabef1af0 0x55adabefb3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:11.317 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.314+0000 7f3a0ffff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55adabef1af0 0x55adabefb3a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:56568/0 (socket says 172.21.15.99:56568) 2024-03-19T01:55:11.317 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.314+0000 7f3a16ffd640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55adabef1100 0x55adabefae60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:11.317 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.314+0000 7f3a0ffff640 1 -- 172.21.15.99:0/4147186994 learned_addr learned my addr 172.21.15.99:0/4147186994 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:55:11.317 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.314+0000 7f3a177fe640 1 --2- 172.21.15.99:0/4147186994 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55adabef6c60 0x55adabf023e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:11.319 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.314+0000 7f3a0ffff640 1 -- 172.21.15.99:0/4147186994 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55adabef6c60 msgr2=0x55adabf023e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:55:11.319 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.314+0000 7f3a0ffff640 1 --2- 172.21.15.99:0/4147186994 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55adabef6c60 0x55adabf023e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:11.319 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.314+0000 7f3a0ffff640 1 -- 172.21.15.99:0/4147186994 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55adabef1100 msgr2=0x55adabefae60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:55:11.319 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.314+0000 7f3a0ffff640 1 --2- 172.21.15.99:0/4147186994 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55adabef1100 0x55adabefae60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:11.319 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.314+0000 7f3a0ffff640 1 -- 172.21.15.99:0/4147186994 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55adabe10c60 con 0x55adabef1af0 2024-03-19T01:55:11.319 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.315+0000 7f3a0ffff640 1 --2- 172.21.15.99:0/4147186994 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55adabef1af0 0x55adabefb3a0 secure :-1 s=READY pgs=621 cs=0 l=1 rev1=1 crypto rx=0x7f3a00005fd0 tx=0x7f3a0000f790 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4503aef8d59b0ab8 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:55:11.319 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.315+0000 7f39f6ffd640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:55:11.319 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.315+0000 7f39f6ffd640 1 -- 172.21.15.99:0/4147186994 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3a0001f060 con 0x55adabef1af0 2024-03-19T01:55:11.319 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.315+0000 7f39f6ffd640 1 -- 172.21.15.99:0/4147186994 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f3a00010430 con 0x55adabef1af0 2024-03-19T01:55:11.319 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.315+0000 7f3a2063a740 1 -- 172.21.15.99:0/4147186994 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f3a100c1c20 con 0x55adabef1af0 2024-03-19T01:55:11.320 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.315+0000 7f39f6ffd640 1 -- 172.21.15.99:0/4147186994 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=33) ==== 24+0+0 (secure 0 0 0) 0x7f3a00010630 con 0x55adabef1af0 2024-03-19T01:55:11.320 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.316+0000 7f3a2063a740 10 client.16482 fetch_fsmap learned FSMap version 33 2024-03-19T01:55:11.320 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.316+0000 7f3a2063a740 1 -- 172.21.15.99:0/4147186994 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=33,monmap=4+}) -- 0x55adabb6a210 con 0x55adabef1af0 2024-03-19T01:55:11.320 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.316+0000 7f39f6ffd640 1 -- 172.21.15.99:0/4147186994 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f3a100c1c20 con 0x55adabef1af0 2024-03-19T01:55:11.320 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.316+0000 7f39f6ffd640 1 -- 172.21.15.99:0/4147186994 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 33) ==== 8362+0+0 (secure 0 0 0) 0x7f3a00025910 con 0x55adabef1af0 2024-03-19T01:55:11.320 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.316+0000 7f3a2063a740 10 client.16482 fetch_fsmap finished waiting for FSMap version 33 2024-03-19T01:55:11.320 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.316+0000 7f3a2063a740 10 client.16482 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:55:11.320 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.316+0000 7f3a2063a740 20 client.16482 populate_metadata read hostname 'smithi099' 2024-03-19T01:55:11.320 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.316+0000 7f3a2063a740 1 --2- 172.21.15.99:0/4147186994 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55adabf06e90 0x55adabf092b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:11.320 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.316+0000 7f3a2063a740 4 client.16482 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:55:11.320 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.316+0000 7f3a2063a740 1 -- 172.21.15.99:0/4147186994 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55adabe63120 con 0x55adabf06e90 2024-03-19T01:55:11.320 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.317+0000 7f3a16ffd640 1 --2- 172.21.15.99:0/4147186994 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55adabf06e90 0x55adabf092b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:11.320 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.319+0000 7f3a16ffd640 1 --2- 172.21.15.99:0/4147186994 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55adabf06e90 0x55adabf092b0 crc :-1 s=READY pgs=155 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=78cb173ad79af48 server_cookie=f69a3d4a35f6a620 in_seq=0 out_seq=0 2024-03-19T01:55:11.320 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.319+0000 7f39f6ffd640 10 client.16482 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:55:11.326 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.325+0000 7f39f6ffd640 1 -- 172.21.15.99:0/4147186994 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x55adabe63120 con 0x55adabf06e90 2024-03-19T01:55:11.326 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.325+0000 7f39f6ffd640 10 client.16482 handle_command_reply: tid=0 2024-03-19T01:55:11.326 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.326+0000 7f3a2063a740 1 client.16482 shutdown 2024-03-19T01:55:11.326 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.326+0000 7f3a2063a740 1 -- 172.21.15.99:0/4147186994 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55adabef1af0 msgr2=0x55adabefb3a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:55:11.326 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.326+0000 7f3a2063a740 1 --2- 172.21.15.99:0/4147186994 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55adabef1af0 0x55adabefb3a0 secure :-1 s=READY pgs=621 cs=0 l=1 rev1=1 crypto rx=0x7f3a00005fd0 tx=0x7f3a0000f790 comp rx=0 tx=0).stop 2024-03-19T01:55:11.326 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.326+0000 7f3a2063a740 1 -- 172.21.15.99:0/4147186994 shutdown_connections 2024-03-19T01:55:11.326 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.326+0000 7f3a2063a740 1 --2- 172.21.15.99:0/4147186994 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55adabf06e90 0x55adabf092b0 crc :-1 s=READY pgs=155 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:11.326 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.326+0000 7f3a2063a740 1 --2- 172.21.15.99:0/4147186994 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55adabef6c60 0x55adabf023e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:11.327 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.326+0000 7f3a2063a740 1 --2- 172.21.15.99:0/4147186994 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55adabef1af0 0x55adabefb3a0 unknown :-1 s=CLOSED pgs=621 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:11.327 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.326+0000 7f39f6ffd640 0 client.16482 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:55:11.327 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.326+0000 7f3a2063a740 1 --2- 172.21.15.99:0/4147186994 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55adabef1100 0x55adabefae60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:11.327 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.326+0000 7f3a2063a740 1 -- 172.21.15.99:0/4147186994 >> 172.21.15.99:0/4147186994 conn(0x55adabeec670 msgr2=0x55adabeed8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:55:11.327 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.326+0000 7f3a2063a740 1 -- 172.21.15.99:0/4147186994 shutdown_connections 2024-03-19T01:55:11.327 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.326+0000 7f3a2063a740 1 -- 172.21.15.99:0/4147186994 wait complete. 2024-03-19T01:55:11.327 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.326+0000 7f3a2063a740 20 client.16482 trim_cache size 0 max 16384 2024-03-19T01:55:11.348 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.344+0000 7f3a2063a740 1 Processor -- start 2024-03-19T01:55:11.348 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.344+0000 7f3a2063a740 1 -- start start 2024-03-19T01:55:11.348 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.344+0000 7f3a2063a740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55adabef1100 0x55adabeeffe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:11.348 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.344+0000 7f3a2063a740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55adabef1af0 0x55adabf023a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:11.348 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.344+0000 7f3a2063a740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55adabef6c60 0x55adabf0db70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:11.348 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.344+0000 7f3a2063a740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f3a181ba510 con 0x55adabef6c60 2024-03-19T01:55:11.348 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.344+0000 7f3a2063a740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f3a181b7d20 con 0x55adabef1100 2024-03-19T01:55:11.349 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.344+0000 7f3a2063a740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f3a181b5b70 con 0x55adabef1af0 2024-03-19T01:55:11.349 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.344+0000 7f3a177fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55adabef6c60 0x55adabf0db70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:11.349 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.344+0000 7f3a177fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55adabef6c60 0x55adabf0db70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:56582/0 (socket says 172.21.15.99:56582) 2024-03-19T01:55:11.349 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.344+0000 7f3a177fe640 1 -- 172.21.15.99:0/1534884256 learned_addr learned my addr 172.21.15.99:0/1534884256 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:55:11.349 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.344+0000 7f3a16ffd640 1 --2- 172.21.15.99:0/1534884256 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55adabef1100 0x55adabeeffe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:11.349 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.344+0000 7f3a0ffff640 1 --2- 172.21.15.99:0/1534884256 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55adabef1af0 0x55adabf023a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:11.349 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.345+0000 7f3a177fe640 1 -- 172.21.15.99:0/1534884256 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55adabef1af0 msgr2=0x55adabf023a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:55:11.349 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.345+0000 7f3a177fe640 1 --2- 172.21.15.99:0/1534884256 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55adabef1af0 0x55adabf023a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:11.349 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.345+0000 7f3a177fe640 1 -- 172.21.15.99:0/1534884256 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55adabef1100 msgr2=0x55adabeeffe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:55:11.349 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.345+0000 7f3a177fe640 1 --2- 172.21.15.99:0/1534884256 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55adabef1100 0x55adabeeffe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:11.349 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.345+0000 7f3a177fe640 1 -- 172.21.15.99:0/1534884256 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55adabc36620 con 0x55adabef6c60 2024-03-19T01:55:11.349 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.345+0000 7f3a177fe640 1 --2- 172.21.15.99:0/1534884256 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55adabef6c60 0x55adabf0db70 secure :-1 s=READY pgs=622 cs=0 l=1 rev1=1 crypto rx=0x7f3a100ba4a0 tx=0x7f3a100047e0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=aef836e317d8c748 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:55:11.349 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.345+0000 7f3a0d7fa640 1 -- 172.21.15.99:0/1534884256 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3a100da070 con 0x55adabef6c60 2024-03-19T01:55:11.349 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.345+0000 7f3a0d7fa640 1 -- 172.21.15.99:0/1534884256 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f3a100bbb70 con 0x55adabef6c60 2024-03-19T01:55:11.349 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.345+0000 7f3a0d7fa640 1 -- 172.21.15.99:0/1534884256 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3a100d6040 con 0x55adabef6c60 2024-03-19T01:55:11.349 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.346+0000 7f3a2063a740 1 -- 172.21.15.99:0/1534884256 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55adabef6c60 msgr2=0x55adabf0db70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:55:11.350 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.346+0000 7f3a2063a740 1 --2- 172.21.15.99:0/1534884256 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55adabef6c60 0x55adabf0db70 secure :-1 s=READY pgs=622 cs=0 l=1 rev1=1 crypto rx=0x7f3a100ba4a0 tx=0x7f3a100047e0 comp rx=0 tx=0).stop 2024-03-19T01:55:11.350 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.346+0000 7f3a2063a740 1 -- 172.21.15.99:0/1534884256 shutdown_connections 2024-03-19T01:55:11.350 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.346+0000 7f3a2063a740 1 --2- 172.21.15.99:0/1534884256 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55adabef6c60 0x55adabf0db70 unknown :-1 s=CLOSED pgs=622 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:11.350 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.346+0000 7f3a2063a740 1 --2- 172.21.15.99:0/1534884256 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55adabef1af0 0x55adabf023a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:11.350 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.346+0000 7f3a2063a740 1 --2- 172.21.15.99:0/1534884256 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55adabef1100 0x55adabeeffe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:11.350 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.346+0000 7f3a2063a740 1 -- 172.21.15.99:0/1534884256 >> 172.21.15.99:0/1534884256 conn(0x55adabeec670 msgr2=0x55adabeef8d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:55:11.350 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.346+0000 7f3a2063a740 1 -- 172.21.15.99:0/1534884256 shutdown_connections 2024-03-19T01:55:11.350 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.346+0000 7f3a2063a740 1 -- 172.21.15.99:0/1534884256 wait complete. 2024-03-19T01:55:11.350 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.346+0000 7f3a2063a740 1 -- start start 2024-03-19T01:55:11.350 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.346+0000 7f3a2063a740 1 Processor -- start 2024-03-19T01:55:11.350 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.346+0000 7f3a2063a740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55adabef1100 0x55adabf020a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:11.350 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.346+0000 7f3a2063a740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55adabef1af0 0x55adabf00700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:11.350 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.346+0000 7f3a2063a740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55adabef6c60 0x55adabf00c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:11.350 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.346+0000 7f3a2063a740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f3a181b61e0 con 0x55adabef1100 2024-03-19T01:55:11.351 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.346+0000 7f3a2063a740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f3a181b7650 con 0x55adabef6c60 2024-03-19T01:55:11.351 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.346+0000 7f3a2063a740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f3a181b68b0 con 0x55adabef1af0 2024-03-19T01:55:11.351 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.346+0000 7f3a2063a740 10 client.0 _ll_register_callbacks cb 0x7f3a00009c40 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:55:11.351 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.347+0000 7f3a16ffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55adabef1100 0x55adabf020a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:11.351 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.347+0000 7f3a16ffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55adabef1100 0x55adabf020a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:56598/0 (socket says 172.21.15.99:56598) 2024-03-19T01:55:11.351 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.347+0000 7f3a16ffd640 1 -- 172.21.15.99:0/2691509827 learned_addr learned my addr 172.21.15.99:0/2691509827 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:55:11.351 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.347+0000 7f3a177fe640 1 --2- 172.21.15.99:0/2691509827 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55adabef6c60 0x55adabf00c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:11.351 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.347+0000 7f3a16ffd640 1 -- 172.21.15.99:0/2691509827 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55adabef1af0 msgr2=0x55adabf00700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:55:11.351 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.347+0000 7f3a16ffd640 1 --2- 172.21.15.99:0/2691509827 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55adabef1af0 0x55adabf00700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:11.351 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.347+0000 7f3a16ffd640 1 -- 172.21.15.99:0/2691509827 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55adabef6c60 msgr2=0x55adabf00c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:55:11.351 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.347+0000 7f3a16ffd640 1 --2- 172.21.15.99:0/2691509827 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55adabef6c60 0x55adabf00c40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:11.351 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.347+0000 7f3a16ffd640 1 -- 172.21.15.99:0/2691509827 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55adabf01230 con 0x55adabef1100 2024-03-19T01:55:11.351 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.347+0000 7f3a16ffd640 1 --2- 172.21.15.99:0/2691509827 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55adabef1100 0x55adabf020a0 secure :-1 s=READY pgs=623 cs=0 l=1 rev1=1 crypto rx=0x55adabef2350 tx=0x7f3a08004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7bc36df947c63dd1 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:55:11.351 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.347+0000 7f39f6ffd640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:55:11.352 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.347+0000 7f39f6ffd640 1 -- 172.21.15.99:0/2691509827 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3a0802ae60 con 0x55adabef1100 2024-03-19T01:55:11.352 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.347+0000 7f39f6ffd640 1 -- 172.21.15.99:0/2691509827 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f3a08011d90 con 0x55adabef1100 2024-03-19T01:55:11.352 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.347+0000 7f3a2063a740 1 -- 172.21.15.99:0/2691509827 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x55adabe5d470 con 0x55adabef1100 2024-03-19T01:55:11.352 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.347+0000 7f39f6ffd640 1 -- 172.21.15.99:0/2691509827 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=33) ==== 24+0+0 (secure 0 0 0) 0x7f3a08033470 con 0x55adabef1100 2024-03-19T01:55:11.352 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.347+0000 7f3a2063a740 10 client.16488 fetch_fsmap learned FSMap version 33 2024-03-19T01:55:11.352 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.347+0000 7f3a2063a740 1 -- 172.21.15.99:0/2691509827 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=33,monmap=4+}) -- 0x55adabf6e090 con 0x55adabef1100 2024-03-19T01:55:11.352 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.347+0000 7f39f6ffd640 1 -- 172.21.15.99:0/2691509827 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x55adabe5d470 con 0x55adabef1100 2024-03-19T01:55:11.352 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.348+0000 7f39f6ffd640 1 -- 172.21.15.99:0/2691509827 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 33) ==== 8362+0+0 (secure 0 0 0) 0x7f3a0803d680 con 0x55adabef1100 2024-03-19T01:55:11.352 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.348+0000 7f3a2063a740 10 client.16488 fetch_fsmap finished waiting for FSMap version 33 2024-03-19T01:55:11.352 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.348+0000 7f3a2063a740 10 client.16488 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:55:11.352 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.348+0000 7f3a2063a740 20 client.16488 populate_metadata read hostname 'smithi099' 2024-03-19T01:55:11.352 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.348+0000 7f3a2063a740 1 --2- 172.21.15.99:0/2691509827 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55adabf6ea80 0x55adabf70ea0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:11.352 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.348+0000 7f3a2063a740 4 client.16488 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:55:11.352 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.348+0000 7f3a2063a740 1 -- 172.21.15.99:0/2691509827 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55adabddb770 con 0x55adabf6ea80 2024-03-19T01:55:11.352 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.348+0000 7f3a0ffff640 1 --2- 172.21.15.99:0/2691509827 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55adabf6ea80 0x55adabf70ea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:11.352 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.348+0000 7f3a0ffff640 1 --2- 172.21.15.99:0/2691509827 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55adabf6ea80 0x55adabf70ea0 crc :-1 s=READY pgs=156 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=9cdd851833266f08 server_cookie=bbd90bc45dc5c22d in_seq=0 out_seq=0 2024-03-19T01:55:11.353 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.348+0000 7f39f6ffd640 10 client.16488 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:55:11.781 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.781+0000 7f39f6ffd640 1 -- 172.21.15.99:0/2691509827 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x55adabddb770 con 0x55adabf6ea80 2024-03-19T01:55:11.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.781+0000 7f39f6ffd640 10 client.16488 handle_command_reply: tid=0 2024-03-19T01:55:11.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.781+0000 7f3a2063a740 1 client.16488 shutdown 2024-03-19T01:55:11.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.781+0000 7f3a2063a740 1 -- 172.21.15.99:0/2691509827 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55adabef1100 msgr2=0x55adabf020a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:55:11.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.781+0000 7f3a2063a740 1 --2- 172.21.15.99:0/2691509827 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55adabef1100 0x55adabf020a0 secure :-1 s=READY pgs=623 cs=0 l=1 rev1=1 crypto rx=0x55adabef2350 tx=0x7f3a08004730 comp rx=0 tx=0).stop 2024-03-19T01:55:11.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.781+0000 7f3a2063a740 1 -- 172.21.15.99:0/2691509827 shutdown_connections 2024-03-19T01:55:11.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.781+0000 7f3a2063a740 1 --2- 172.21.15.99:0/2691509827 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55adabf6ea80 0x55adabf70ea0 crc :-1 s=READY pgs=156 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:11.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.781+0000 7f3a2063a740 1 --2- 172.21.15.99:0/2691509827 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55adabef6c60 0x55adabf00c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:11.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.781+0000 7f3a2063a740 1 --2- 172.21.15.99:0/2691509827 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55adabef1af0 0x55adabf00700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:11.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.781+0000 7f3a2063a740 1 --2- 172.21.15.99:0/2691509827 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55adabef1100 0x55adabf020a0 unknown :-1 s=CLOSED pgs=623 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:11.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.781+0000 7f3a2063a740 1 -- 172.21.15.99:0/2691509827 >> 172.21.15.99:0/2691509827 conn(0x55adabeec670 msgr2=0x55adabef0160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:55:11.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.781+0000 7f39f6ffd640 0 client.16488 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:55:11.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.781+0000 7f3a2063a740 1 -- 172.21.15.99:0/2691509827 shutdown_connections 2024-03-19T01:55:11.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.781+0000 7f3a2063a740 1 -- 172.21.15.99:0/2691509827 wait complete. 2024-03-19T01:55:11.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.782+0000 7f3a2063a740 20 client.16488 trim_cache size 0 max 16384 2024-03-19T01:55:11.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:55:11.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:55:11.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:55:11.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:55:11.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.782+0000 7f3a1d977640 1 -- 172.21.15.99:0/3776929186 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f3a040a28a0 msgr2=0x7f3a040a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:55:11.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.782+0000 7f3a1d977640 1 --2- 172.21.15.99:0/3776929186 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f3a040a28a0 0x7f3a040a4d60 secure :-1 s=READY pgs=336 cs=0 l=1 rev1=1 crypto rx=0x7f3a08005d00 tx=0x7f3a0800b410 comp rx=0 tx=0).stop 2024-03-19T01:55:11.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.782+0000 7f3a1d977640 1 -- 172.21.15.99:0/3776929186 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3a1810db40 msgr2=0x7f3a181addc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:55:11.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.782+0000 7f3a1d977640 1 --2- 172.21.15.99:0/3776929186 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3a1810db40 0x7f3a181addc0 secure :-1 s=READY pgs=619 cs=0 l=1 rev1=1 crypto rx=0x7f3a10036040 tx=0x7f3a100337b0 comp rx=0 tx=0).stop 2024-03-19T01:55:11.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.782+0000 7f3a1d977640 1 -- 172.21.15.99:0/3776929186 shutdown_connections 2024-03-19T01:55:11.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.782+0000 7f3a1d977640 1 --2- 172.21.15.99:0/3776929186 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f3a040a28a0 0x7f3a040a4d60 unknown :-1 s=CLOSED pgs=336 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:11.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.782+0000 7f3a1d977640 1 --2- 172.21.15.99:0/3776929186 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3a1810b190 0x7f3a181a6da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:11.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.782+0000 7f3a1d977640 1 --2- 172.21.15.99:0/3776929186 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3a1810db40 0x7f3a181addc0 unknown :-1 s=CLOSED pgs=619 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:11.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.782+0000 7f3a1d977640 1 --2- 172.21.15.99:0/3776929186 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f3a18108820 0x7f3a181a6860 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:11.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.782+0000 7f3a1d977640 1 -- 172.21.15.99:0/3776929186 >> 172.21.15.99:0/3776929186 conn(0x7f3a1806e860 msgr2=0x7f3a18109650 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:55:11.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.782+0000 7f3a1d977640 1 -- 172.21.15.99:0/3776929186 shutdown_connections 2024-03-19T01:55:11.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:11.782+0000 7f3a1d977640 1 -- 172.21.15.99:0/3776929186 wait complete. 2024-03-19T01:55:11.806 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:55:11 smithi115 ceph-mon[67332]: pgmap v774: 97 pgs: 97 active+clean; 490 MiB data, 5.1 GiB used, 1.0 TiB / 1.0 TiB avail; 4.8 MiB/s rd, 6.3 MiB/s wr, 544 op/s 2024-03-19T01:55:11.840 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:55:11 smithi099 ceph-mon[62951]: pgmap v774: 97 pgs: 97 active+clean; 490 MiB data, 5.1 GiB used, 1.0 TiB / 1.0 TiB avail; 4.8 MiB/s rd, 6.3 MiB/s wr, 544 op/s 2024-03-19T01:55:11.861 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:55:11 smithi112 ceph-mon[66819]: pgmap v774: 97 pgs: 97 active+clean; 490 MiB data, 5.1 GiB used, 1.0 TiB / 1.0 TiB avail; 4.8 MiB/s rd, 6.3 MiB/s wr, 544 op/s 2024-03-19T01:55:12.100 INFO:tasks.workunit.client.0.smithi099.stdout: 10 46174 26.33 MB/sec warmup 101 sec latency 339.043 ms 2024-03-19T01:55:13.100 INFO:tasks.workunit.client.0.smithi099.stdout: 10 46720 26.33 MB/sec warmup 102 sec latency 52.312 ms 2024-03-19T01:55:13.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:55:13 smithi099 ceph-mon[62951]: pgmap v775: 97 pgs: 97 active+clean; 493 MiB data, 5.1 GiB used, 1.0 TiB / 1.0 TiB avail; 5.2 MiB/s rd, 7.1 MiB/s wr, 613 op/s 2024-03-19T01:55:13.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:55:13 smithi112 ceph-mon[66819]: pgmap v775: 97 pgs: 97 active+clean; 493 MiB data, 5.1 GiB used, 1.0 TiB / 1.0 TiB avail; 5.2 MiB/s rd, 7.1 MiB/s wr, 613 op/s 2024-03-19T01:55:13.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:55:13 smithi115 ceph-mon[67332]: pgmap v775: 97 pgs: 97 active+clean; 493 MiB data, 5.1 GiB used, 1.0 TiB / 1.0 TiB avail; 5.2 MiB/s rd, 7.1 MiB/s wr, 613 op/s 2024-03-19T01:55:14.100 INFO:tasks.workunit.client.0.smithi099.stdout: 10 47268 26.40 MB/sec warmup 103 sec latency 71.814 ms 2024-03-19T01:55:15.100 INFO:tasks.workunit.client.0.smithi099.stdout: 10 47774 26.47 MB/sec warmup 104 sec latency 93.900 ms 2024-03-19T01:55:15.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:55:15 smithi099 ceph-mon[62951]: pgmap v776: 97 pgs: 97 active+clean; 484 MiB data, 5.1 GiB used, 1.0 TiB / 1.0 TiB avail; 4.5 MiB/s rd, 6.0 MiB/s wr, 491 op/s 2024-03-19T01:55:15.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:55:15 smithi112 ceph-mon[66819]: pgmap v776: 97 pgs: 97 active+clean; 484 MiB data, 5.1 GiB used, 1.0 TiB / 1.0 TiB avail; 4.5 MiB/s rd, 6.0 MiB/s wr, 491 op/s 2024-03-19T01:55:15.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:55:15 smithi115 ceph-mon[67332]: pgmap v776: 97 pgs: 97 active+clean; 484 MiB data, 5.1 GiB used, 1.0 TiB / 1.0 TiB avail; 4.5 MiB/s rd, 6.0 MiB/s wr, 491 op/s 2024-03-19T01:55:16.100 INFO:tasks.workunit.client.0.smithi099.stdout: 10 48121 26.42 MB/sec warmup 105 sec latency 226.582 ms 2024-03-19T01:55:17.100 INFO:tasks.workunit.client.0.smithi099.stdout: 10 48458 26.29 MB/sec warmup 106 sec latency 125.284 ms 2024-03-19T01:55:17.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:55:17 smithi099 ceph-mon[62951]: pgmap v777: 97 pgs: 97 active+clean; 490 MiB data, 5.2 GiB used, 1.0 TiB / 1.0 TiB avail; 5.3 MiB/s rd, 6.7 MiB/s wr, 633 op/s 2024-03-19T01:55:17.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:55:17 smithi112 ceph-mon[66819]: pgmap v777: 97 pgs: 97 active+clean; 490 MiB data, 5.2 GiB used, 1.0 TiB / 1.0 TiB avail; 5.3 MiB/s rd, 6.7 MiB/s wr, 633 op/s 2024-03-19T01:55:17.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:55:17 smithi115 ceph-mon[67332]: pgmap v777: 97 pgs: 97 active+clean; 490 MiB data, 5.2 GiB used, 1.0 TiB / 1.0 TiB avail; 5.3 MiB/s rd, 6.7 MiB/s wr, 633 op/s 2024-03-19T01:55:18.100 INFO:tasks.workunit.client.0.smithi099.stdout: 10 48837 26.19 MB/sec warmup 107 sec latency 71.284 ms 2024-03-19T01:55:19.100 INFO:tasks.workunit.client.0.smithi099.stdout: 10 49324 26.22 MB/sec warmup 108 sec latency 62.644 ms 2024-03-19T01:55:19.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:55:19 smithi099 ceph-mon[62951]: pgmap v778: 97 pgs: 97 active+clean; 491 MiB data, 5.2 GiB used, 1.0 TiB / 1.0 TiB avail; 4.8 MiB/s rd, 6.3 MiB/s wr, 594 op/s 2024-03-19T01:55:19.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:55:19 smithi112 ceph-mon[66819]: pgmap v778: 97 pgs: 97 active+clean; 491 MiB data, 5.2 GiB used, 1.0 TiB / 1.0 TiB avail; 4.8 MiB/s rd, 6.3 MiB/s wr, 594 op/s 2024-03-19T01:55:19.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:55:19 smithi115 ceph-mon[67332]: pgmap v778: 97 pgs: 97 active+clean; 491 MiB data, 5.2 GiB used, 1.0 TiB / 1.0 TiB avail; 4.8 MiB/s rd, 6.3 MiB/s wr, 594 op/s 2024-03-19T01:55:20.100 INFO:tasks.workunit.client.0.smithi099.stdout: 10 49818 26.25 MB/sec warmup 109 sec latency 68.418 ms 2024-03-19T01:55:21.100 INFO:tasks.workunit.client.0.smithi099.stdout: 10 50129 26.10 MB/sec warmup 110 sec latency 358.670 ms 2024-03-19T01:55:21.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:55:21 smithi099 ceph-mon[62951]: pgmap v779: 97 pgs: 97 active+clean; 490 MiB data, 5.2 GiB used, 1.0 TiB / 1.0 TiB avail; 4.4 MiB/s rd, 6.0 MiB/s wr, 544 op/s 2024-03-19T01:55:21.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:55:21 smithi112 ceph-mon[66819]: pgmap v779: 97 pgs: 97 active+clean; 490 MiB data, 5.2 GiB used, 1.0 TiB / 1.0 TiB avail; 4.4 MiB/s rd, 6.0 MiB/s wr, 544 op/s 2024-03-19T01:55:21.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:55:21 smithi115 ceph-mon[67332]: pgmap v779: 97 pgs: 97 active+clean; 490 MiB data, 5.2 GiB used, 1.0 TiB / 1.0 TiB avail; 4.4 MiB/s rd, 6.0 MiB/s wr, 544 op/s 2024-03-19T01:55:22.101 INFO:tasks.workunit.client.0.smithi099.stdout: 10 50615 26.15 MB/sec warmup 111 sec latency 100.175 ms 2024-03-19T01:55:23.100 INFO:tasks.workunit.client.0.smithi099.stdout: 10 51117 26.09 MB/sec warmup 112 sec latency 68.193 ms 2024-03-19T01:55:23.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:55:23 smithi099 ceph-mon[62951]: pgmap v780: 97 pgs: 97 active+clean; 499 MiB data, 5.2 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 MiB/s rd, 7.2 MiB/s wr, 653 op/s 2024-03-19T01:55:23.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:55:23 smithi112 ceph-mon[66819]: pgmap v780: 97 pgs: 97 active+clean; 499 MiB data, 5.2 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 MiB/s rd, 7.2 MiB/s wr, 653 op/s 2024-03-19T01:55:23.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:55:23 smithi115 ceph-mon[67332]: pgmap v780: 97 pgs: 97 active+clean; 499 MiB data, 5.2 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 MiB/s rd, 7.2 MiB/s wr, 653 op/s 2024-03-19T01:55:24.100 INFO:tasks.workunit.client.0.smithi099.stdout: 10 51723 26.18 MB/sec warmup 113 sec latency 96.022 ms 2024-03-19T01:55:24.877 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.876+0000 7fdaf473b640 1 -- 172.21.15.99:0/3383720540 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fdaec101cd0 msgr2=0x7fdaec1020f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:55:24.877 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.876+0000 7fdaf473b640 1 --2- 172.21.15.99:0/3383720540 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fdaec101cd0 0x7fdaec1020f0 secure :-1 s=READY pgs=624 cs=0 l=1 rev1=1 crypto rx=0x7fdae8009900 tx=0x7fdae802bc30 comp rx=0 tx=0).stop 2024-03-19T01:55:24.877 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.876+0000 7fdaf473b640 1 -- 172.21.15.99:0/3383720540 shutdown_connections 2024-03-19T01:55:24.877 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.876+0000 7fdaf473b640 1 --2- 172.21.15.99:0/3383720540 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fdaec100a00 0x7fdaec100e80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:24.877 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.876+0000 7fdaf473b640 1 --2- 172.21.15.99:0/3383720540 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fdaec1013c0 0x7fdaec108c60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:24.877 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.876+0000 7fdaf473b640 1 --2- 172.21.15.99:0/3383720540 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fdaec101cd0 0x7fdaec1020f0 unknown :-1 s=CLOSED pgs=624 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:24.877 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.876+0000 7fdaf473b640 1 -- 172.21.15.99:0/3383720540 >> 172.21.15.99:0/3383720540 conn(0x7fdaec0fb470 msgr2=0x7fdaec0fd890 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:55:24.877 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.876+0000 7fdaf473b640 1 -- 172.21.15.99:0/3383720540 shutdown_connections 2024-03-19T01:55:24.877 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.876+0000 7fdaf473b640 1 -- 172.21.15.99:0/3383720540 wait complete. 2024-03-19T01:55:24.877 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.876+0000 7fdaf473b640 1 Processor -- start 2024-03-19T01:55:24.877 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.876+0000 7fdaf473b640 1 -- start start 2024-03-19T01:55:24.877 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.876+0000 7fdaf473b640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fdaec100a00 0x7fdaec100460 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:24.877 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.876+0000 7fdaf473b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fdaec1013c0 0x7fdaec19c8a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:24.878 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.876+0000 7fdaf473b640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fdaec101cd0 0x7fdaec19eec0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:24.878 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.877+0000 7fdaf473b640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fdaec10c410 con 0x7fdaec1013c0 2024-03-19T01:55:24.878 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.877+0000 7fdaf473b640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fdaec10c290 con 0x7fdaec100a00 2024-03-19T01:55:24.878 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.877+0000 7fdaf473b640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fdaec10c590 con 0x7fdaec101cd0 2024-03-19T01:55:24.878 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.877+0000 7fdaf24d7640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fdaec100a00 0x7fdaec100460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:24.878 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.877+0000 7fdaf1cd6640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fdaec1013c0 0x7fdaec19c8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:24.878 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.877+0000 7fdaf24d7640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fdaec100a00 0x7fdaec100460 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:52742/0 (socket says 172.21.15.99:52742) 2024-03-19T01:55:24.878 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.877+0000 7fdaf24d7640 1 -- 172.21.15.99:0/3388376697 learned_addr learned my addr 172.21.15.99:0/3388376697 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:55:24.878 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.877+0000 7fdaf2cd8640 1 --2- 172.21.15.99:0/3388376697 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fdaec101cd0 0x7fdaec19eec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:24.878 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.877+0000 7fdaf1cd6640 1 -- 172.21.15.99:0/3388376697 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fdaec101cd0 msgr2=0x7fdaec19eec0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:55:24.878 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.877+0000 7fdaf1cd6640 1 --2- 172.21.15.99:0/3388376697 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fdaec101cd0 0x7fdaec19eec0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:24.878 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.877+0000 7fdaf1cd6640 1 -- 172.21.15.99:0/3388376697 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fdaec100a00 msgr2=0x7fdaec100460 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:55:24.878 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.877+0000 7fdaf1cd6640 1 --2- 172.21.15.99:0/3388376697 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fdaec100a00 0x7fdaec100460 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:24.880 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.877+0000 7fdaf1cd6640 1 -- 172.21.15.99:0/3388376697 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fdaec19f7d0 con 0x7fdaec1013c0 2024-03-19T01:55:24.880 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.877+0000 7fdaf1cd6640 1 --2- 172.21.15.99:0/3388376697 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fdaec1013c0 0x7fdaec19c8a0 secure :-1 s=READY pgs=625 cs=0 l=1 rev1=1 crypto rx=0x7fdadc00e9f0 tx=0x7fdadc00eeb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:55:24.880 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.877+0000 7fdae37fe640 1 -- 172.21.15.99:0/3388376697 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdadc00ccc0 con 0x7fdaec1013c0 2024-03-19T01:55:24.880 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.877+0000 7fdaf473b640 1 -- 172.21.15.99:0/3388376697 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fdaec19fa60 con 0x7fdaec1013c0 2024-03-19T01:55:24.880 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.877+0000 7fdae37fe640 1 -- 172.21.15.99:0/3388376697 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fdadc010910 con 0x7fdaec1013c0 2024-03-19T01:55:24.880 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.877+0000 7fdae37fe640 1 -- 172.21.15.99:0/3388376697 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdadc018960 con 0x7fdaec1013c0 2024-03-19T01:55:24.880 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.878+0000 7fdaf473b640 1 -- 172.21.15.99:0/3388376697 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fdaec1a0000 con 0x7fdaec1013c0 2024-03-19T01:55:24.880 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.880+0000 7fdae37fe640 1 -- 172.21.15.99:0/3388376697 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7fdadc010ac0 con 0x7fdaec1013c0 2024-03-19T01:55:24.880 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.880+0000 7fdaf7407740 1 Processor -- start 2024-03-19T01:55:24.880 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.880+0000 7fdaf7407740 1 -- start start 2024-03-19T01:55:24.885 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.880+0000 7fdaf7407740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fa959ea100 0x55fa959ea520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:24.885 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.880+0000 7fdaf7407740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fa959eaaf0 0x55fa959ef5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:24.885 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.880+0000 7fdaf7407740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fa959efc60 0x55fa959f20f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:24.885 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.880+0000 7fdaf7407740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55fa9594ebe0 con 0x55fa959efc60 2024-03-19T01:55:24.885 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.880+0000 7fdaf7407740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55fa958f7960 con 0x55fa959eaaf0 2024-03-19T01:55:24.885 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.880+0000 7fdaf7407740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55fa9594ac80 con 0x55fa959ea100 2024-03-19T01:55:24.885 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.880+0000 7fdae37fe640 1 --2- 172.21.15.99:0/3388376697 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fdad80a28a0 0x7fdad80a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:24.885 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.881+0000 7fdaf24d7640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fa959eaaf0 0x55fa959ef5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:24.886 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.881+0000 7fdaf2cd8640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fa959ea100 0x55fa959ea520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:24.886 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.881+0000 7fdaf1cd6640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fa959efc60 0x55fa959f20f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:24.886 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.881+0000 7fdaf24d7640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fa959eaaf0 0x55fa959ef5f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:52754/0 (socket says 172.21.15.99:52754) 2024-03-19T01:55:24.886 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.881+0000 7fdaf24d7640 1 -- 172.21.15.99:0/96177731 learned_addr learned my addr 172.21.15.99:0/96177731 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:55:24.886 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.881+0000 7fdaf2cd8640 1 --2- 172.21.15.99:0/3388376697 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fdad80a28a0 0x7fdad80a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:24.886 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.881+0000 7fdae37fe640 1 -- 172.21.15.99:0/3388376697 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fdadc0cee50 con 0x7fdaec1013c0 2024-03-19T01:55:24.886 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.881+0000 7fdaf24d7640 1 -- 172.21.15.99:0/96177731 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fa959ea100 msgr2=0x55fa959ea520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:55:24.886 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.881+0000 7fdaf24d7640 1 --2- 172.21.15.99:0/96177731 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fa959ea100 0x55fa959ea520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:24.886 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.881+0000 7fdaf24d7640 1 -- 172.21.15.99:0/96177731 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fa959efc60 msgr2=0x55fa959f20f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:55:24.886 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.881+0000 7fdaf24d7640 1 --2- 172.21.15.99:0/96177731 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fa959efc60 0x55fa959f20f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:24.886 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.881+0000 7fdaf24d7640 1 -- 172.21.15.99:0/96177731 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55fa958eed20 con 0x55fa959eaaf0 2024-03-19T01:55:24.886 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.881+0000 7fdaf2cd8640 1 --2- 172.21.15.99:0/3388376697 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fdad80a28a0 0x7fdad80a4d60 secure :-1 s=READY pgs=337 cs=0 l=1 rev1=1 crypto rx=0x55fa959eb350 tx=0x7fdae4009500 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:55:24.886 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.881+0000 7fdaf24d7640 1 --2- 172.21.15.99:0/96177731 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fa959eaaf0 0x55fa959ef5f0 secure :-1 s=READY pgs=166 cs=0 l=1 rev1=1 crypto rx=0x7fdae802c170 tx=0x7fdae8009560 comp rx=0 tx=0).ready entity=mon.2 client_cookie=a2f4d244626047ff server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:55:24.887 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.881+0000 7fdae0ff9640 1 -- 172.21.15.99:0/96177731 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdae8045070 con 0x55fa959eaaf0 2024-03-19T01:55:24.887 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.881+0000 7fdae0ff9640 1 -- 172.21.15.99:0/96177731 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fdae8004040 con 0x55fa959eaaf0 2024-03-19T01:55:24.887 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.881+0000 7fdae0ff9640 1 -- 172.21.15.99:0/96177731 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdae8004cb0 con 0x55fa959eaaf0 2024-03-19T01:55:24.887 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.882+0000 7fdaf7407740 1 -- 172.21.15.99:0/96177731 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fa959eaaf0 msgr2=0x55fa959ef5f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:55:24.887 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.882+0000 7fdaf7407740 1 --2- 172.21.15.99:0/96177731 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fa959eaaf0 0x55fa959ef5f0 secure :-1 s=READY pgs=166 cs=0 l=1 rev1=1 crypto rx=0x7fdae802c170 tx=0x7fdae8009560 comp rx=0 tx=0).stop 2024-03-19T01:55:24.887 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.882+0000 7fdaf7407740 1 -- 172.21.15.99:0/96177731 shutdown_connections 2024-03-19T01:55:24.887 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.882+0000 7fdaf7407740 1 --2- 172.21.15.99:0/96177731 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fa959eaaf0 0x55fa959ef5f0 unknown :-1 s=CLOSED pgs=166 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:24.887 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.882+0000 7fdaf7407740 1 --2- 172.21.15.99:0/96177731 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fa959efc60 0x55fa959f20f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:24.887 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.882+0000 7fdaf7407740 1 --2- 172.21.15.99:0/96177731 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fa959ea100 0x55fa959ea520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:24.887 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.882+0000 7fdaf7407740 1 -- 172.21.15.99:0/96177731 >> 172.21.15.99:0/96177731 conn(0x55fa959e5670 msgr2=0x55fa959e5a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:55:24.887 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.882+0000 7fdaf7407740 1 -- 172.21.15.99:0/96177731 shutdown_connections 2024-03-19T01:55:24.887 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.882+0000 7fdaf7407740 1 -- 172.21.15.99:0/96177731 wait complete. 2024-03-19T01:55:24.887 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.883+0000 7fdaf7407740 1 -- start start 2024-03-19T01:55:24.887 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.883+0000 7fdaf7407740 1 Processor -- start 2024-03-19T01:55:24.887 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.883+0000 7fdaf7407740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fa959ea100 0x55fa959f4050 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:24.888 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.883+0000 7fdaf7407740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fa959eaaf0 0x55fa959f4590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:24.888 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.883+0000 7fdaf7407740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fa959efc60 0x55fa959fb5d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:24.888 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.883+0000 7fdaf7407740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55fa959f2bb0 con 0x55fa959efc60 2024-03-19T01:55:24.888 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.883+0000 7fdaf7407740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55fa9590adb0 con 0x55fa959ea100 2024-03-19T01:55:24.888 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.883+0000 7fdaf7407740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55fa959f2d30 con 0x55fa959eaaf0 2024-03-19T01:55:24.888 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.883+0000 7fdaf7407740 10 client.0 _ll_register_callbacks cb 0x55fa9590fe70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:55:24.888 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.883+0000 7fdaf2cd8640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fa959efc60 0x55fa959fb5d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:24.888 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.883+0000 7fdaf24d7640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fa959ea100 0x55fa959f4050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:24.888 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.883+0000 7fdaf2cd8640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fa959efc60 0x55fa959fb5d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:42310/0 (socket says 172.21.15.99:42310) 2024-03-19T01:55:24.888 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.883+0000 7fdaf2cd8640 1 -- 172.21.15.99:0/256552702 learned_addr learned my addr 172.21.15.99:0/256552702 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:55:24.888 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.883+0000 7fdaf1cd6640 1 --2- 172.21.15.99:0/256552702 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fa959eaaf0 0x55fa959f4590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:24.888 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.883+0000 7fdaf2cd8640 1 -- 172.21.15.99:0/256552702 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fa959eaaf0 msgr2=0x55fa959f4590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:55:24.888 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.883+0000 7fdaf2cd8640 1 --2- 172.21.15.99:0/256552702 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fa959eaaf0 0x55fa959f4590 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:24.888 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.883+0000 7fdaf2cd8640 1 -- 172.21.15.99:0/256552702 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fa959ea100 msgr2=0x55fa959f4050 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:55:24.888 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.883+0000 7fdaf2cd8640 1 --2- 172.21.15.99:0/256552702 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fa959ea100 0x55fa959f4050 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:24.889 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.883+0000 7fdaf2cd8640 1 -- 172.21.15.99:0/256552702 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55fa95909c60 con 0x55fa959efc60 2024-03-19T01:55:24.889 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.884+0000 7fdaf2cd8640 1 --2- 172.21.15.99:0/256552702 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fa959efc60 0x55fa959fb5d0 secure :-1 s=READY pgs=626 cs=0 l=1 rev1=1 crypto rx=0x7fdae4011ee0 tx=0x7fdae4004ab0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=120b1658e3a7aa6 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:55:24.889 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.884+0000 7fdad5ffb640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:55:24.889 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.884+0000 7fdad5ffb640 1 -- 172.21.15.99:0/256552702 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdae401e410 con 0x55fa959efc60 2024-03-19T01:55:24.889 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.884+0000 7fdad5ffb640 1 -- 172.21.15.99:0/256552702 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fdae401e5b0 con 0x55fa959efc60 2024-03-19T01:55:24.889 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.884+0000 7fdaf7407740 1 -- 172.21.15.99:0/256552702 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fdae8004040 con 0x55fa959efc60 2024-03-19T01:55:24.889 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.884+0000 7fdad5ffb640 1 -- 172.21.15.99:0/256552702 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=33) ==== 24+0+0 (secure 0 0 0) 0x7fdae401e7b0 con 0x55fa959efc60 2024-03-19T01:55:24.889 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.884+0000 7fdaf7407740 10 client.16500 fetch_fsmap learned FSMap version 33 2024-03-19T01:55:24.889 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.884+0000 7fdaf7407740 1 -- 172.21.15.99:0/256552702 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=33,monmap=4+}) -- 0x55fa95663210 con 0x55fa959efc60 2024-03-19T01:55:24.889 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.884+0000 7fdad5ffb640 1 -- 172.21.15.99:0/256552702 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fdae8004040 con 0x55fa959efc60 2024-03-19T01:55:24.889 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.884+0000 7fdad5ffb640 1 -- 172.21.15.99:0/256552702 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 33) ==== 8362+0+0 (secure 0 0 0) 0x7fdae401d070 con 0x55fa959efc60 2024-03-19T01:55:24.889 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.884+0000 7fdaf7407740 10 client.16500 fetch_fsmap finished waiting for FSMap version 33 2024-03-19T01:55:24.889 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.884+0000 7fdaf7407740 10 client.16500 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:55:24.889 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.884+0000 7fdaf7407740 20 client.16500 populate_metadata read hostname 'smithi099' 2024-03-19T01:55:24.890 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.884+0000 7fdaf7407740 1 --2- 172.21.15.99:0/256552702 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55fa959fffe0 0x55fa95a02470 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:24.890 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.884+0000 7fdaf7407740 4 client.16500 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:55:24.890 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.884+0000 7fdaf7407740 1 -- 172.21.15.99:0/256552702 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55fa9595c120 con 0x55fa959fffe0 2024-03-19T01:55:24.890 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.885+0000 7fdaf24d7640 1 --2- 172.21.15.99:0/256552702 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55fa959fffe0 0x55fa95a02470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:24.890 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.885+0000 7fdad5ffb640 10 client.16500 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:55:24.890 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.885+0000 7fdaf24d7640 1 --2- 172.21.15.99:0/256552702 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55fa959fffe0 0x55fa95a02470 crc :-1 s=READY pgs=157 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=6807789ee4e3edea server_cookie=731913f69a78a70f in_seq=0 out_seq=0 2024-03-19T01:55:24.891 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.887+0000 7fdad5ffb640 1 -- 172.21.15.99:0/256552702 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x55fa9595c120 con 0x55fa959fffe0 2024-03-19T01:55:24.891 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.887+0000 7fdad5ffb640 10 client.16500 handle_command_reply: tid=0 2024-03-19T01:55:24.891 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.887+0000 7fdaf7407740 1 client.16500 shutdown 2024-03-19T01:55:24.891 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.887+0000 7fdaf7407740 1 -- 172.21.15.99:0/256552702 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fa959efc60 msgr2=0x55fa959fb5d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:55:24.891 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.887+0000 7fdaf7407740 1 --2- 172.21.15.99:0/256552702 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fa959efc60 0x55fa959fb5d0 secure :-1 s=READY pgs=626 cs=0 l=1 rev1=1 crypto rx=0x7fdae4011ee0 tx=0x7fdae4004ab0 comp rx=0 tx=0).stop 2024-03-19T01:55:24.891 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.887+0000 7fdaf7407740 1 -- 172.21.15.99:0/256552702 shutdown_connections 2024-03-19T01:55:24.891 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.887+0000 7fdaf7407740 1 --2- 172.21.15.99:0/256552702 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55fa959fffe0 0x55fa95a02470 crc :-1 s=READY pgs=157 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:24.891 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.887+0000 7fdaf7407740 1 --2- 172.21.15.99:0/256552702 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fa959efc60 0x55fa959fb5d0 unknown :-1 s=CLOSED pgs=626 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:24.891 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.887+0000 7fdad5ffb640 0 client.16500 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:55:24.892 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.887+0000 7fdaf7407740 1 --2- 172.21.15.99:0/256552702 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fa959eaaf0 0x55fa959f4590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:24.892 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.887+0000 7fdaf7407740 1 --2- 172.21.15.99:0/256552702 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fa959ea100 0x55fa959f4050 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:24.892 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.887+0000 7fdaf7407740 1 -- 172.21.15.99:0/256552702 >> 172.21.15.99:0/256552702 conn(0x55fa959e5670 msgr2=0x55fa959e68c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:55:24.892 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.887+0000 7fdaf7407740 1 -- 172.21.15.99:0/256552702 shutdown_connections 2024-03-19T01:55:24.892 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.887+0000 7fdaf7407740 1 -- 172.21.15.99:0/256552702 wait complete. 2024-03-19T01:55:24.892 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.887+0000 7fdaf7407740 20 client.16500 trim_cache size 0 max 16384 2024-03-19T01:55:24.909 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.905+0000 7fdaf7407740 1 Processor -- start 2024-03-19T01:55:24.909 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.905+0000 7fdaf7407740 1 -- start start 2024-03-19T01:55:24.909 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.905+0000 7fdaf7407740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fa959ea100 0x55fa959e4ab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:24.909 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.905+0000 7fdaf7407740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fa959eaaf0 0x55fa959fb590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:24.909 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.905+0000 7fdaf7407740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fa959efc60 0x55fa959ebca0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:24.910 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.905+0000 7fdaf7407740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fdaec1ab7c0 con 0x55fa959ea100 2024-03-19T01:55:24.910 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.905+0000 7fdaf7407740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fdaec1a8fd0 con 0x55fa959eaaf0 2024-03-19T01:55:24.910 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.905+0000 7fdaf7407740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fdaec1a6e20 con 0x55fa959efc60 2024-03-19T01:55:24.910 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.905+0000 7fdaf24d7640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fa959ea100 0x55fa959e4ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:24.910 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.905+0000 7fdaf2cd8640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fa959efc60 0x55fa959ebca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:24.910 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.905+0000 7fdaf24d7640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fa959ea100 0x55fa959e4ab0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:42322/0 (socket says 172.21.15.99:42322) 2024-03-19T01:55:24.910 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.905+0000 7fdaf24d7640 1 -- 172.21.15.99:0/642336037 learned_addr learned my addr 172.21.15.99:0/642336037 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:55:24.910 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.905+0000 7fdaf1cd6640 1 --2- 172.21.15.99:0/642336037 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fa959eaaf0 0x55fa959fb590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:24.910 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.905+0000 7fdaf24d7640 1 -- 172.21.15.99:0/642336037 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fa959efc60 msgr2=0x55fa959ebca0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:55:24.910 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.905+0000 7fdaf24d7640 1 --2- 172.21.15.99:0/642336037 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fa959efc60 0x55fa959ebca0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:24.910 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.905+0000 7fdaf24d7640 1 -- 172.21.15.99:0/642336037 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fa959eaaf0 msgr2=0x55fa959fb590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:55:24.910 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.905+0000 7fdaf24d7640 1 --2- 172.21.15.99:0/642336037 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fa959eaaf0 0x55fa959fb590 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:24.910 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.905+0000 7fdaf24d7640 1 -- 172.21.15.99:0/642336037 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55fa9572f620 con 0x55fa959ea100 2024-03-19T01:55:24.910 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.905+0000 7fdaf2cd8640 1 --2- 172.21.15.99:0/642336037 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fa959efc60 0x55fa959ebca0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:55:24.911 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.906+0000 7fdaf24d7640 1 --2- 172.21.15.99:0/642336037 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fa959ea100 0x55fa959e4ab0 secure :-1 s=READY pgs=627 cs=0 l=1 rev1=1 crypto rx=0x7fdae802c170 tx=0x7fdae802df90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b71ff9b2cc078fc6 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:55:24.911 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.906+0000 7fdad7fff640 1 -- 172.21.15.99:0/642336037 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdae8058070 con 0x55fa959ea100 2024-03-19T01:55:24.911 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.906+0000 7fdad7fff640 1 -- 172.21.15.99:0/642336037 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fdae8004490 con 0x55fa959ea100 2024-03-19T01:55:24.911 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.906+0000 7fdad7fff640 1 -- 172.21.15.99:0/642336037 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdae8045040 con 0x55fa959ea100 2024-03-19T01:55:24.911 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.906+0000 7fdaf7407740 1 -- 172.21.15.99:0/642336037 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fa959ea100 msgr2=0x55fa959e4ab0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:55:24.911 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.906+0000 7fdaf7407740 1 --2- 172.21.15.99:0/642336037 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fa959ea100 0x55fa959e4ab0 secure :-1 s=READY pgs=627 cs=0 l=1 rev1=1 crypto rx=0x7fdae802c170 tx=0x7fdae802df90 comp rx=0 tx=0).stop 2024-03-19T01:55:24.911 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.907+0000 7fdaf7407740 1 -- 172.21.15.99:0/642336037 shutdown_connections 2024-03-19T01:55:24.911 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.907+0000 7fdaf7407740 1 --2- 172.21.15.99:0/642336037 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fa959eaaf0 0x55fa959fb590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:24.911 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.907+0000 7fdaf7407740 1 --2- 172.21.15.99:0/642336037 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fa959efc60 0x55fa959ebca0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:24.911 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.907+0000 7fdaf7407740 1 --2- 172.21.15.99:0/642336037 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fa959ea100 0x55fa959e4ab0 unknown :-1 s=CLOSED pgs=627 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:24.911 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.907+0000 7fdaf7407740 1 -- 172.21.15.99:0/642336037 >> 172.21.15.99:0/642336037 conn(0x55fa959e5670 msgr2=0x55fa959eb590 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:55:24.911 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.907+0000 7fdaf7407740 1 -- 172.21.15.99:0/642336037 shutdown_connections 2024-03-19T01:55:24.911 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.907+0000 7fdaf7407740 1 -- 172.21.15.99:0/642336037 wait complete. 2024-03-19T01:55:24.911 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.907+0000 7fdaf7407740 1 -- start start 2024-03-19T01:55:24.912 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.907+0000 7fdaf7407740 1 Processor -- start 2024-03-19T01:55:24.912 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.907+0000 7fdaf7407740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fa959ea100 0x55fa959f8dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:24.912 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.907+0000 7fdaf7407740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fa959eaaf0 0x55fa959f9310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:24.912 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.907+0000 7fdaf7407740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fa959efc60 0x55fa959fa320 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:24.912 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.907+0000 7fdaf7407740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fdaec1a7490 con 0x55fa959eaaf0 2024-03-19T01:55:24.912 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.907+0000 7fdaf7407740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fdaec1a8900 con 0x55fa959ea100 2024-03-19T01:55:24.912 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.907+0000 7fdaf7407740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fdaec1a7b60 con 0x55fa959efc60 2024-03-19T01:55:24.912 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.907+0000 7fdaf7407740 10 client.0 _ll_register_callbacks cb 0x7fdae4011da0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:55:24.912 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.907+0000 7fdaf1cd6640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fa959eaaf0 0x55fa959f9310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:24.912 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.907+0000 7fdaf1cd6640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fa959eaaf0 0x55fa959f9310 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:42334/0 (socket says 172.21.15.99:42334) 2024-03-19T01:55:24.912 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.907+0000 7fdaf1cd6640 1 -- 172.21.15.99:0/3706916469 learned_addr learned my addr 172.21.15.99:0/3706916469 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:55:24.912 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.907+0000 7fdaf2cd8640 1 --2- 172.21.15.99:0/3706916469 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fa959efc60 0x55fa959fa320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:24.912 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.907+0000 7fdaf24d7640 1 --2- 172.21.15.99:0/3706916469 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fa959ea100 0x55fa959f8dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:24.912 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.907+0000 7fdaf1cd6640 1 -- 172.21.15.99:0/3706916469 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fa959efc60 msgr2=0x55fa959fa320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:55:24.913 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.907+0000 7fdaf1cd6640 1 --2- 172.21.15.99:0/3706916469 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fa959efc60 0x55fa959fa320 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:24.913 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.907+0000 7fdaf1cd6640 1 -- 172.21.15.99:0/3706916469 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fa959ea100 msgr2=0x55fa959f8dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:55:24.913 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.908+0000 7fdaf1cd6640 1 --2- 172.21.15.99:0/3706916469 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fa959ea100 0x55fa959f8dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:24.913 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.908+0000 7fdaf1cd6640 1 -- 172.21.15.99:0/3706916469 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55fa959fa910 con 0x55fa959eaaf0 2024-03-19T01:55:24.913 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.908+0000 7fdaf1cd6640 1 --2- 172.21.15.99:0/3706916469 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fa959eaaf0 0x55fa959f9310 secure :-1 s=READY pgs=628 cs=0 l=1 rev1=1 crypto rx=0x7fdadc00cf10 tx=0x7fdadc06b380 comp rx=0 tx=0).ready entity=mon.0 client_cookie=51f14ddfaa9b3bc1 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:55:24.913 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.908+0000 7fdad5ffb640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:55:24.913 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.908+0000 7fdad5ffb640 1 -- 172.21.15.99:0/3706916469 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdadc06d0e0 con 0x55fa959eaaf0 2024-03-19T01:55:24.913 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.908+0000 7fdad5ffb640 1 -- 172.21.15.99:0/3706916469 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fdadc06d280 con 0x55fa959eaaf0 2024-03-19T01:55:24.913 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.908+0000 7fdaf7407740 1 -- 172.21.15.99:0/3706916469 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fdae8004490 con 0x55fa959eaaf0 2024-03-19T01:55:24.913 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.908+0000 7fdad5ffb640 1 -- 172.21.15.99:0/3706916469 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=33) ==== 24+0+0 (secure 0 0 0) 0x7fdadc0a3ac0 con 0x55fa959eaaf0 2024-03-19T01:55:24.913 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.908+0000 7fdaf7407740 10 client.16506 fetch_fsmap learned FSMap version 33 2024-03-19T01:55:24.913 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.908+0000 7fdaf7407740 1 -- 172.21.15.99:0/3706916469 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=33,monmap=4+}) -- 0x55fa95a665a0 con 0x55fa959eaaf0 2024-03-19T01:55:24.913 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.908+0000 7fdad5ffb640 1 -- 172.21.15.99:0/3706916469 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fdae8004490 con 0x55fa959eaaf0 2024-03-19T01:55:24.913 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.908+0000 7fdad5ffb640 1 -- 172.21.15.99:0/3706916469 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 33) ==== 8362+0+0 (secure 0 0 0) 0x7fdadc066b30 con 0x55fa959eaaf0 2024-03-19T01:55:24.914 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.908+0000 7fdaf7407740 10 client.16506 fetch_fsmap finished waiting for FSMap version 33 2024-03-19T01:55:24.914 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.908+0000 7fdaf7407740 10 client.16506 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:55:24.914 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.908+0000 7fdaf7407740 20 client.16506 populate_metadata read hostname 'smithi099' 2024-03-19T01:55:24.914 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.908+0000 7fdaf7407740 1 --2- 172.21.15.99:0/3706916469 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55fa95a67030 0x55fa95a69450 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:24.914 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.908+0000 7fdaf7407740 4 client.16506 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:55:24.914 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.908+0000 7fdaf7407740 1 -- 172.21.15.99:0/3706916469 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55fa95956470 con 0x55fa95a67030 2024-03-19T01:55:24.914 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.909+0000 7fdaf24d7640 1 --2- 172.21.15.99:0/3706916469 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55fa95a67030 0x55fa95a69450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:24.914 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.909+0000 7fdad5ffb640 10 client.16506 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:55:24.914 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:24.909+0000 7fdaf24d7640 1 --2- 172.21.15.99:0/3706916469 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55fa95a67030 0x55fa95a69450 crc :-1 s=READY pgs=158 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=9203ba141d4e6719 server_cookie=a7a4324e7b376f57 in_seq=0 out_seq=0 2024-03-19T01:55:25.100 INFO:tasks.workunit.client.0.smithi099.stdout: 10 52100 26.17 MB/sec warmup 114 sec latency 175.637 ms 2024-03-19T01:55:25.295 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:25.294+0000 7fdad5ffb640 1 -- 172.21.15.99:0/3706916469 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x55fa95956470 con 0x55fa95a67030 2024-03-19T01:55:25.295 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:25.294+0000 7fdad5ffb640 10 client.16506 handle_command_reply: tid=0 2024-03-19T01:55:25.295 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:25.294+0000 7fdaf7407740 1 client.16506 shutdown 2024-03-19T01:55:25.295 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:25.294+0000 7fdaf7407740 1 -- 172.21.15.99:0/3706916469 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fa959eaaf0 msgr2=0x55fa959f9310 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:55:25.295 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:25.294+0000 7fdaf7407740 1 --2- 172.21.15.99:0/3706916469 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fa959eaaf0 0x55fa959f9310 secure :-1 s=READY pgs=628 cs=0 l=1 rev1=1 crypto rx=0x7fdadc00cf10 tx=0x7fdadc06b380 comp rx=0 tx=0).stop 2024-03-19T01:55:25.295 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:25.294+0000 7fdaf7407740 1 -- 172.21.15.99:0/3706916469 shutdown_connections 2024-03-19T01:55:25.295 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:25.294+0000 7fdaf7407740 1 --2- 172.21.15.99:0/3706916469 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55fa95a67030 0x55fa95a69450 crc :-1 s=READY pgs=158 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:25.295 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:25.294+0000 7fdaf7407740 1 --2- 172.21.15.99:0/3706916469 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55fa959efc60 0x55fa959fa320 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:25.295 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:25.294+0000 7fdaf7407740 1 --2- 172.21.15.99:0/3706916469 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55fa959eaaf0 0x55fa959f9310 unknown :-1 s=CLOSED pgs=628 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:25.295 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:25.294+0000 7fdad5ffb640 0 client.16506 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:55:25.295 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:25.294+0000 7fdaf7407740 1 --2- 172.21.15.99:0/3706916469 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55fa959ea100 0x55fa959f8dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:25.295 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:25.294+0000 7fdaf7407740 1 -- 172.21.15.99:0/3706916469 >> 172.21.15.99:0/3706916469 conn(0x55fa959e5670 msgr2=0x55fa959ebe20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:55:25.295 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:25.295+0000 7fdaf7407740 1 -- 172.21.15.99:0/3706916469 shutdown_connections 2024-03-19T01:55:25.296 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:25.295+0000 7fdaf7407740 1 -- 172.21.15.99:0/3706916469 wait complete. 2024-03-19T01:55:25.296 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:25.295+0000 7fdaf7407740 20 client.16506 trim_cache size 0 max 16384 2024-03-19T01:55:25.296 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:55:25.296 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:55:25.296 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:55:25.296 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:55:25.297 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:25.295+0000 7fdaf473b640 1 -- 172.21.15.99:0/3388376697 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fdad80a28a0 msgr2=0x7fdad80a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:55:25.297 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:25.295+0000 7fdaf473b640 1 --2- 172.21.15.99:0/3388376697 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fdad80a28a0 0x7fdad80a4d60 secure :-1 s=READY pgs=337 cs=0 l=1 rev1=1 crypto rx=0x55fa959eb350 tx=0x7fdae4009500 comp rx=0 tx=0).stop 2024-03-19T01:55:25.297 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:25.295+0000 7fdaf473b640 1 -- 172.21.15.99:0/3388376697 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fdaec1013c0 msgr2=0x7fdaec19c8a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:55:25.297 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:25.295+0000 7fdaf473b640 1 --2- 172.21.15.99:0/3388376697 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fdaec1013c0 0x7fdaec19c8a0 secure :-1 s=READY pgs=625 cs=0 l=1 rev1=1 crypto rx=0x7fdadc00e9f0 tx=0x7fdadc00eeb0 comp rx=0 tx=0).stop 2024-03-19T01:55:25.297 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:25.295+0000 7fdaf473b640 1 -- 172.21.15.99:0/3388376697 shutdown_connections 2024-03-19T01:55:25.297 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:25.295+0000 7fdaf473b640 1 --2- 172.21.15.99:0/3388376697 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fdaec101cd0 0x7fdaec19eec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:25.297 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:25.295+0000 7fdaf473b640 1 --2- 172.21.15.99:0/3388376697 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fdaec1013c0 0x7fdaec19c8a0 unknown :-1 s=CLOSED pgs=625 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:25.297 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:25.295+0000 7fdaf473b640 1 --2- 172.21.15.99:0/3388376697 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fdad80a28a0 0x7fdad80a4d60 unknown :-1 s=CLOSED pgs=337 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:25.297 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:25.295+0000 7fdaf473b640 1 --2- 172.21.15.99:0/3388376697 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fdaec100a00 0x7fdaec100460 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:25.297 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:25.295+0000 7fdaf473b640 1 -- 172.21.15.99:0/3388376697 >> 172.21.15.99:0/3388376697 conn(0x7fdaec0fb470 msgr2=0x7fdaec104cb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:55:25.297 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:25.295+0000 7fdaf473b640 1 -- 172.21.15.99:0/3388376697 shutdown_connections 2024-03-19T01:55:25.297 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:25.296+0000 7fdaf473b640 1 -- 172.21.15.99:0/3388376697 wait complete. 2024-03-19T01:55:25.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:55:25 smithi099 ceph-mon[62951]: pgmap v781: 97 pgs: 97 active+clean; 499 MiB data, 5.2 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 5.4 MiB/s wr, 479 op/s 2024-03-19T01:55:25.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:55:25 smithi112 ceph-mon[66819]: pgmap v781: 97 pgs: 97 active+clean; 499 MiB data, 5.2 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 5.4 MiB/s wr, 479 op/s 2024-03-19T01:55:25.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:55:25 smithi115 ceph-mon[67332]: pgmap v781: 97 pgs: 97 active+clean; 499 MiB data, 5.2 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 5.4 MiB/s wr, 479 op/s 2024-03-19T01:55:26.100 INFO:tasks.workunit.client.0.smithi099.stdout: 10 52441 26.11 MB/sec warmup 115 sec latency 277.931 ms 2024-03-19T01:55:27.100 INFO:tasks.workunit.client.0.smithi099.stdout: 10 52884 26.09 MB/sec warmup 116 sec latency 109.777 ms 2024-03-19T01:55:28.064 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:55:27 smithi112 ceph-mon[66819]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T01:55:28.064 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:55:27 smithi112 ceph-mon[66819]: pgmap v782: 97 pgs: 97 active+clean; 497 MiB data, 5.3 GiB used, 1.0 TiB / 1.0 TiB avail; 5.2 MiB/s rd, 6.3 MiB/s wr, 628 op/s 2024-03-19T01:55:28.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:55:27 smithi099 ceph-mon[62951]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T01:55:28.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:55:27 smithi099 ceph-mon[62951]: pgmap v782: 97 pgs: 97 active+clean; 497 MiB data, 5.3 GiB used, 1.0 TiB / 1.0 TiB avail; 5.2 MiB/s rd, 6.3 MiB/s wr, 628 op/s 2024-03-19T01:55:28.100 INFO:tasks.workunit.client.0.smithi099.stdout: 10 53311 26.06 MB/sec warmup 117 sec latency 65.768 ms 2024-03-19T01:55:28.258 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:55:27 smithi115 ceph-mon[67332]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T01:55:28.258 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:55:27 smithi115 ceph-mon[67332]: pgmap v782: 97 pgs: 97 active+clean; 497 MiB data, 5.3 GiB used, 1.0 TiB / 1.0 TiB avail; 5.2 MiB/s rd, 6.3 MiB/s wr, 628 op/s 2024-03-19T01:55:29.100 INFO:tasks.workunit.client.0.smithi099.stdout: 10 53768 26.02 MB/sec warmup 118 sec latency 89.297 ms 2024-03-19T01:55:29.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:55:29 smithi099 ceph-mon[62951]: pgmap v783: 97 pgs: 97 active+clean; 495 MiB data, 5.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.5 MiB/s rd, 5.9 MiB/s wr, 527 op/s 2024-03-19T01:55:29.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:55:29 smithi112 ceph-mon[66819]: pgmap v783: 97 pgs: 97 active+clean; 495 MiB data, 5.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.5 MiB/s rd, 5.9 MiB/s wr, 527 op/s 2024-03-19T01:55:29.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:55:29 smithi115 ceph-mon[67332]: pgmap v783: 97 pgs: 97 active+clean; 495 MiB data, 5.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.5 MiB/s rd, 5.9 MiB/s wr, 527 op/s 2024-03-19T01:55:30.101 INFO:tasks.workunit.client.0.smithi099.stdout: 10 54233 26.02 MB/sec warmup 119 sec latency 83.673 ms 2024-03-19T01:55:30.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:55:30 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:55:30.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:55:30 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:55:30.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:55:30 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:55:31.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:55:31 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:55:31.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:55:31 smithi099 ceph-mon[62951]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T01:55:31.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:55:31 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:55:31.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:55:31 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:55:31.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:55:31 smithi099 ceph-mon[62951]: pgmap v784: 97 pgs: 97 active+clean; 495 MiB data, 5.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.5 MiB/s rd, 6.0 MiB/s wr, 550 op/s 2024-03-19T01:55:31.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:55:31 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:55:31.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:55:31 smithi112 ceph-mon[66819]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T01:55:31.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:55:31 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:55:31.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:55:31 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:55:31.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:55:31 smithi112 ceph-mon[66819]: pgmap v784: 97 pgs: 97 active+clean; 495 MiB data, 5.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.5 MiB/s rd, 6.0 MiB/s wr, 550 op/s 2024-03-19T01:55:31.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:55:31 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:55:31.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:55:31 smithi115 ceph-mon[67332]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T01:55:31.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:55:31 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:55:31.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:55:31 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:55:31.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:55:31 smithi115 ceph-mon[67332]: pgmap v784: 97 pgs: 97 active+clean; 495 MiB data, 5.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.5 MiB/s rd, 6.0 MiB/s wr, 550 op/s 2024-03-19T01:55:32.101 INFO:tasks.workunit.client.0.smithi099.stdout: 10 55119 25.05 MB/sec execute 1 sec latency 219.364 ms 2024-03-19T01:55:33.101 INFO:tasks.workunit.client.0.smithi099.stdout: 10 55568 25.19 MB/sec execute 2 sec latency 75.990 ms 2024-03-19T01:55:33.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:55:33 smithi099 ceph-mon[62951]: pgmap v785: 97 pgs: 97 active+clean; 498 MiB data, 5.3 GiB used, 1.0 TiB / 1.0 TiB avail; 5.4 MiB/s rd, 6.7 MiB/s wr, 651 op/s 2024-03-19T01:55:33.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:55:33 smithi112 ceph-mon[66819]: pgmap v785: 97 pgs: 97 active+clean; 498 MiB data, 5.3 GiB used, 1.0 TiB / 1.0 TiB avail; 5.4 MiB/s rd, 6.7 MiB/s wr, 651 op/s 2024-03-19T01:55:33.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:55:33 smithi115 ceph-mon[67332]: pgmap v785: 97 pgs: 97 active+clean; 498 MiB data, 5.3 GiB used, 1.0 TiB / 1.0 TiB avail; 5.4 MiB/s rd, 6.7 MiB/s wr, 651 op/s 2024-03-19T01:55:34.101 INFO:tasks.workunit.client.0.smithi099.stdout: 10 56109 24.91 MB/sec execute 3 sec latency 79.272 ms 2024-03-19T01:55:35.101 INFO:tasks.workunit.client.0.smithi099.stdout: 10 56588 26.33 MB/sec execute 4 sec latency 97.992 ms 2024-03-19T01:55:35.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:55:35 smithi099 ceph-mon[62951]: pgmap v786: 97 pgs: 97 active+clean; 502 MiB data, 5.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.5 MiB/s rd, 5.5 MiB/s wr, 542 op/s 2024-03-19T01:55:35.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:55:35 smithi112 ceph-mon[66819]: pgmap v786: 97 pgs: 97 active+clean; 502 MiB data, 5.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.5 MiB/s rd, 5.5 MiB/s wr, 542 op/s 2024-03-19T01:55:35.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:55:35 smithi115 ceph-mon[67332]: pgmap v786: 97 pgs: 97 active+clean; 502 MiB data, 5.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.5 MiB/s rd, 5.5 MiB/s wr, 542 op/s 2024-03-19T01:55:36.101 INFO:tasks.workunit.client.0.smithi099.stdout: 10 56941 25.90 MB/sec execute 5 sec latency 312.650 ms 2024-03-19T01:55:37.101 INFO:tasks.workunit.client.0.smithi099.stdout: 10 57315 25.13 MB/sec execute 6 sec latency 124.060 ms 2024-03-19T01:55:37.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:55:37 smithi099 ceph-mon[62951]: pgmap v787: 97 pgs: 97 active+clean; 508 MiB data, 5.4 GiB used, 1.0 TiB / 1.0 TiB avail; 5.6 MiB/s rd, 6.7 MiB/s wr, 616 op/s 2024-03-19T01:55:37.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:55:37 smithi112 ceph-mon[66819]: pgmap v787: 97 pgs: 97 active+clean; 508 MiB data, 5.4 GiB used, 1.0 TiB / 1.0 TiB avail; 5.6 MiB/s rd, 6.7 MiB/s wr, 616 op/s 2024-03-19T01:55:37.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:55:37 smithi115 ceph-mon[67332]: pgmap v787: 97 pgs: 97 active+clean; 508 MiB data, 5.4 GiB used, 1.0 TiB / 1.0 TiB avail; 5.6 MiB/s rd, 6.7 MiB/s wr, 616 op/s 2024-03-19T01:55:38.101 INFO:tasks.workunit.client.0.smithi099.stdout: 10 57710 24.48 MB/sec execute 7 sec latency 91.479 ms 2024-03-19T01:55:38.398 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.398+0000 7fd34fe19640 1 -- 172.21.15.99:0/6982516 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd348102b60 msgr2=0x7fd348102fe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:55:38.398 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.398+0000 7fd34fe19640 1 --2- 172.21.15.99:0/6982516 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd348102b60 0x7fd348102fe0 secure :-1 s=READY pgs=167 cs=0 l=1 rev1=1 crypto rx=0x7fd338009930 tx=0x7fd33802bc30 comp rx=0 tx=0).stop 2024-03-19T01:55:38.398 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.398+0000 7fd34fe19640 1 -- 172.21.15.99:0/6982516 shutdown_connections 2024-03-19T01:55:38.398 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.398+0000 7fd34fe19640 1 --2- 172.21.15.99:0/6982516 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd348102b60 0x7fd348102fe0 unknown :-1 s=CLOSED pgs=167 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:38.398 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.398+0000 7fd34fe19640 1 --2- 172.21.15.99:0/6982516 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd348103520 0x7fd348109db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:38.398 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.398+0000 7fd34fe19640 1 --2- 172.21.15.99:0/6982516 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd3480ffaf0 0x7fd3480fff10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:38.398 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.398+0000 7fd34fe19640 1 -- 172.21.15.99:0/6982516 >> 172.21.15.99:0/6982516 conn(0x7fd3480fb440 msgr2=0x7fd3480fd860 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:55:38.399 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.398+0000 7fd34fe19640 1 -- 172.21.15.99:0/6982516 shutdown_connections 2024-03-19T01:55:38.399 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.398+0000 7fd34fe19640 1 -- 172.21.15.99:0/6982516 wait complete. 2024-03-19T01:55:38.401 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.399+0000 7fd34fe19640 1 Processor -- start 2024-03-19T01:55:38.401 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.399+0000 7fd34fe19640 1 -- start start 2024-03-19T01:55:38.401 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.399+0000 7fd34fe19640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd3480ffaf0 0x7fd3481024d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:38.401 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.399+0000 7fd34fe19640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd348102b60 0x7fd348100b10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:38.401 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.399+0000 7fd34fe19640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd348103520 0x7fd348101050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:38.401 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.399+0000 7fd34fe19640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fd34810d4a0 con 0x7fd3480ffaf0 2024-03-19T01:55:38.401 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.399+0000 7fd34fe19640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fd34810d320 con 0x7fd348103520 2024-03-19T01:55:38.401 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.399+0000 7fd34fe19640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fd34810d620 con 0x7fd348102b60 2024-03-19T01:55:38.401 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.399+0000 7fd34dbb5640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd3480ffaf0 0x7fd3481024d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:38.402 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.399+0000 7fd34d3b4640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd348102b60 0x7fd348100b10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:38.402 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.399+0000 7fd34d3b4640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd348102b60 0x7fd348100b10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:48164/0 (socket says 172.21.15.99:48164) 2024-03-19T01:55:38.402 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.399+0000 7fd34e3b6640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd348103520 0x7fd348101050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:38.402 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.399+0000 7fd34dbb5640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd3480ffaf0 0x7fd3481024d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:50750/0 (socket says 172.21.15.99:50750) 2024-03-19T01:55:38.402 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.399+0000 7fd34dbb5640 1 -- 172.21.15.99:0/2924937074 learned_addr learned my addr 172.21.15.99:0/2924937074 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:55:38.402 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.400+0000 7fd34dbb5640 1 -- 172.21.15.99:0/2924937074 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd348102b60 msgr2=0x7fd348100b10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:55:38.402 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.400+0000 7fd34dbb5640 1 --2- 172.21.15.99:0/2924937074 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd348102b60 0x7fd348100b10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:38.402 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.400+0000 7fd34dbb5640 1 -- 172.21.15.99:0/2924937074 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd348103520 msgr2=0x7fd348101050 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:55:38.402 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.400+0000 7fd34dbb5640 1 --2- 172.21.15.99:0/2924937074 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd348103520 0x7fd348101050 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:38.402 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.400+0000 7fd34dbb5640 1 -- 172.21.15.99:0/2924937074 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd3481018e0 con 0x7fd3480ffaf0 2024-03-19T01:55:38.402 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.400+0000 7fd34dbb5640 1 --2- 172.21.15.99:0/2924937074 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd3480ffaf0 0x7fd3481024d0 secure :-1 s=READY pgs=629 cs=0 l=1 rev1=1 crypto rx=0x7fd34400d950 tx=0x7fd34400de10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:55:38.403 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.401+0000 7fd33effd640 1 -- 172.21.15.99:0/2924937074 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd344017070 con 0x7fd3480ffaf0 2024-03-19T01:55:38.403 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.401+0000 7fd33effd640 1 -- 172.21.15.99:0/2924937074 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd344009c30 con 0x7fd3480ffaf0 2024-03-19T01:55:38.403 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.401+0000 7fd33effd640 1 -- 172.21.15.99:0/2924937074 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd344017070 con 0x7fd3480ffaf0 2024-03-19T01:55:38.403 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.401+0000 7fd34fe19640 1 -- 172.21.15.99:0/2924937074 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd3481a1c00 con 0x7fd3480ffaf0 2024-03-19T01:55:38.403 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.401+0000 7fd34fe19640 1 -- 172.21.15.99:0/2924937074 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fd3481a20d0 con 0x7fd3480ffaf0 2024-03-19T01:55:38.404 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.404+0000 7fd33effd640 1 -- 172.21.15.99:0/2924937074 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7fd34401a070 con 0x7fd3480ffaf0 2024-03-19T01:55:38.404 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.405+0000 7fd352a07740 1 Processor -- start 2024-03-19T01:55:38.404 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.405+0000 7fd352a07740 1 -- start start 2024-03-19T01:55:38.407 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.405+0000 7fd352a07740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x555b1c3bb100 0x555b1c3bb520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:38.407 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.405+0000 7fd352a07740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x555b1c3bbaf0 0x555b1c3c05f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:38.407 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.406+0000 7fd352a07740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x555b1c3c0c60 0x555b1c3c30f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:38.407 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.406+0000 7fd33effd640 1 --2- 172.21.15.99:0/2924937074 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd3340a2ca0 0x7fd3340a5160 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:38.407 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.406+0000 7fd34dbb5640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x555b1c3bbaf0 0x555b1c3c05f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:38.407 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.406+0000 7fd34dbb5640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x555b1c3bbaf0 0x555b1c3c05f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:50752/0 (socket says 172.21.15.99:50752) 2024-03-19T01:55:38.407 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.406+0000 7fd34e3b6640 1 --2- 172.21.15.99:0/2924937074 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd3340a2ca0 0x7fd3340a5160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:38.408 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.406+0000 7fd34dbb5640 1 -- 172.21.15.99:0/1718866251 learned_addr learned my addr 172.21.15.99:0/1718866251 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:55:38.408 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.406+0000 7fd33effd640 1 -- 172.21.15.99:0/2924937074 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fd344009a30 con 0x7fd3480ffaf0 2024-03-19T01:55:38.408 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.406+0000 7fd352a07740 1 -- 172.21.15.99:0/1718866251 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x555b1c31fbe0 con 0x555b1c3bbaf0 2024-03-19T01:55:38.408 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.406+0000 7fd352a07740 1 -- 172.21.15.99:0/1718866251 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x555b1c2c8960 con 0x555b1c3c0c60 2024-03-19T01:55:38.408 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.406+0000 7fd34e3b6640 1 --2- 172.21.15.99:0/1718866251 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x555b1c3bb100 0x555b1c3bb520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:38.408 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.406+0000 7fd352a07740 1 -- 172.21.15.99:0/1718866251 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x555b1c31bc80 con 0x555b1c3bb100 2024-03-19T01:55:38.408 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.406+0000 7fd34d3b4640 1 --2- 172.21.15.99:0/1718866251 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x555b1c3c0c60 0x555b1c3c30f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:38.408 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.406+0000 7fd34dbb5640 1 -- 172.21.15.99:0/1718866251 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x555b1c3bb100 msgr2=0x555b1c3bb520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:55:38.408 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.406+0000 7fd34dbb5640 1 --2- 172.21.15.99:0/1718866251 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x555b1c3bb100 0x555b1c3bb520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:38.408 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.406+0000 7fd34dbb5640 1 -- 172.21.15.99:0/1718866251 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x555b1c3c0c60 msgr2=0x555b1c3c30f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:55:38.408 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.406+0000 7fd34dbb5640 1 --2- 172.21.15.99:0/1718866251 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x555b1c3c0c60 0x555b1c3c30f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:38.408 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.406+0000 7fd34dbb5640 1 -- 172.21.15.99:0/1718866251 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x555b1c2bfd20 con 0x555b1c3bbaf0 2024-03-19T01:55:38.408 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.406+0000 7fd34d3b4640 1 --2- 172.21.15.99:0/1718866251 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x555b1c3c0c60 0x555b1c3c30f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-19T01:55:38.409 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.407+0000 7fd34e3b6640 1 --2- 172.21.15.99:0/2924937074 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd3340a2ca0 0x7fd3340a5160 secure :-1 s=READY pgs=338 cs=0 l=1 rev1=1 crypto rx=0x7fd340005fd0 tx=0x7fd340009f00 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:55:38.409 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.407+0000 7fd34dbb5640 1 --2- 172.21.15.99:0/1718866251 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x555b1c3bbaf0 0x555b1c3c05f0 secure :-1 s=READY pgs=630 cs=0 l=1 rev1=1 crypto rx=0x7fd3440cb9c0 tx=0x7fd34406d640 comp rx=0 tx=0).ready entity=mon.0 client_cookie=474343fb053c6740 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:55:38.409 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.407+0000 7fd333fff640 1 -- 172.21.15.99:0/1718866251 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd344021070 con 0x555b1c3bbaf0 2024-03-19T01:55:38.409 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.407+0000 7fd333fff640 1 -- 172.21.15.99:0/1718866251 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd34409a910 con 0x555b1c3bbaf0 2024-03-19T01:55:38.409 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.407+0000 7fd333fff640 1 -- 172.21.15.99:0/1718866251 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd344021070 con 0x555b1c3bbaf0 2024-03-19T01:55:38.411 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.408+0000 7fd352a07740 1 -- 172.21.15.99:0/1718866251 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x555b1c3bbaf0 msgr2=0x555b1c3c05f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:55:38.411 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.408+0000 7fd352a07740 1 --2- 172.21.15.99:0/1718866251 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x555b1c3bbaf0 0x555b1c3c05f0 secure :-1 s=READY pgs=630 cs=0 l=1 rev1=1 crypto rx=0x7fd3440cb9c0 tx=0x7fd34406d640 comp rx=0 tx=0).stop 2024-03-19T01:55:38.411 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.408+0000 7fd352a07740 1 -- 172.21.15.99:0/1718866251 shutdown_connections 2024-03-19T01:55:38.412 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.408+0000 7fd352a07740 1 --2- 172.21.15.99:0/1718866251 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x555b1c3c0c60 0x555b1c3c30f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:38.412 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.408+0000 7fd352a07740 1 --2- 172.21.15.99:0/1718866251 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x555b1c3bbaf0 0x555b1c3c05f0 unknown :-1 s=CLOSED pgs=630 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:38.412 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.408+0000 7fd352a07740 1 --2- 172.21.15.99:0/1718866251 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x555b1c3bb100 0x555b1c3bb520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:38.412 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.408+0000 7fd352a07740 1 -- 172.21.15.99:0/1718866251 >> 172.21.15.99:0/1718866251 conn(0x555b1c3b6670 msgr2=0x555b1c3b6a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:55:38.412 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.408+0000 7fd352a07740 1 -- 172.21.15.99:0/1718866251 shutdown_connections 2024-03-19T01:55:38.412 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.408+0000 7fd352a07740 1 -- 172.21.15.99:0/1718866251 wait complete. 2024-03-19T01:55:38.412 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.409+0000 7fd352a07740 1 -- start start 2024-03-19T01:55:38.412 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.409+0000 7fd352a07740 1 Processor -- start 2024-03-19T01:55:38.412 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.410+0000 7fd352a07740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x555b1c3bb100 0x555b1c3cd7d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:38.412 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.410+0000 7fd352a07740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x555b1c3bbaf0 0x555b1c3cdd10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:38.412 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.410+0000 7fd352a07740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x555b1c3c0c60 0x555b1c3d4d50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:38.412 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.410+0000 7fd352a07740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x555b1c3c3b50 con 0x555b1c3bbaf0 2024-03-19T01:55:38.412 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.410+0000 7fd352a07740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x555b1c2dbdb0 con 0x555b1c3c0c60 2024-03-19T01:55:38.412 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.410+0000 7fd352a07740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x555b1c3c3cd0 con 0x555b1c3bb100 2024-03-19T01:55:38.413 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.410+0000 7fd352a07740 10 client.0 _ll_register_callbacks cb 0x555b1c2e0e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:55:38.413 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.410+0000 7fd34e3b6640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x555b1c3c0c60 0x555b1c3d4d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:38.413 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.410+0000 7fd34e3b6640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x555b1c3c0c60 0x555b1c3d4d50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:55100/0 (socket says 172.21.15.99:55100) 2024-03-19T01:55:38.413 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.410+0000 7fd34e3b6640 1 -- 172.21.15.99:0/231469971 learned_addr learned my addr 172.21.15.99:0/231469971 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:55:38.413 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.411+0000 7fd34d3b4640 1 --2- 172.21.15.99:0/231469971 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x555b1c3bbaf0 0x555b1c3cdd10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:38.413 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.411+0000 7fd34dbb5640 1 --2- 172.21.15.99:0/231469971 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x555b1c3bb100 0x555b1c3cd7d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:38.413 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.411+0000 7fd34dbb5640 1 -- 172.21.15.99:0/231469971 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x555b1c3c0c60 msgr2=0x555b1c3d4d50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:55:38.413 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.411+0000 7fd34dbb5640 1 --2- 172.21.15.99:0/231469971 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x555b1c3c0c60 0x555b1c3d4d50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:38.413 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.411+0000 7fd34dbb5640 1 -- 172.21.15.99:0/231469971 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x555b1c3bbaf0 msgr2=0x555b1c3cdd10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:55:38.413 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.411+0000 7fd34dbb5640 1 --2- 172.21.15.99:0/231469971 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x555b1c3bbaf0 0x555b1c3cdd10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:38.413 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.411+0000 7fd34dbb5640 1 -- 172.21.15.99:0/231469971 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({osdmap=0}) -- 0x555b1c2dac60 con 0x555b1c3bb100 2024-03-19T01:55:38.413 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.411+0000 7fd34dbb5640 1 --2- 172.21.15.99:0/231469971 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x555b1c3bb100 0x555b1c3cd7d0 secure :-1 s=READY pgs=114 cs=0 l=1 rev1=1 crypto rx=0x7fd3440cb9c0 tx=0x7fd34409eaf0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=87fb17784b9987b6 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:55:38.413 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.411+0000 7fd3317fa640 10 client.0 ms_handle_connect on v2:172.21.15.115:3300/0 2024-03-19T01:55:38.414 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.411+0000 7fd3317fa640 1 -- 172.21.15.99:0/231469971 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd344014030 con 0x555b1c3bb100 2024-03-19T01:55:38.414 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.411+0000 7fd352a07740 1 -- 172.21.15.99:0/231469971 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fd34409a910 con 0x555b1c3bb100 2024-03-19T01:55:38.414 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.412+0000 7fd3317fa640 1 -- 172.21.15.99:0/231469971 <== mon.1 v2:172.21.15.115:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fd3440852c0 con 0x555b1c3bb100 2024-03-19T01:55:38.414 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.412+0000 7fd3317fa640 1 -- 172.21.15.99:0/231469971 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_get_version_reply(handle=1 version=35) ==== 24+0+0 (secure 0 0 0) 0x7fd3440b8050 con 0x555b1c3bb100 2024-03-19T01:55:38.414 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.412+0000 7fd352a07740 10 client.26335 fetch_fsmap learned FSMap version 35 2024-03-19T01:55:38.414 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.412+0000 7fd352a07740 1 -- 172.21.15.99:0/231469971 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,fsmap=35,monmap=4+}) -- 0x555b1c034210 con 0x555b1c3bb100 2024-03-19T01:55:38.415 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.412+0000 7fd3317fa640 1 -- 172.21.15.99:0/231469971 <== mon.1 v2:172.21.15.115:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd34409a910 con 0x555b1c3bb100 2024-03-19T01:55:38.415 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.412+0000 7fd3317fa640 1 -- 172.21.15.99:0/231469971 <== mon.1 v2:172.21.15.115:3300/0 5 ==== fsmap(e 35) ==== 8354+0+0 (secure 0 0 0) 0x7fd3440997d0 con 0x555b1c3bb100 2024-03-19T01:55:38.415 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.412+0000 7fd352a07740 10 client.26335 fetch_fsmap finished waiting for FSMap version 35 2024-03-19T01:55:38.415 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.412+0000 7fd352a07740 10 client.26335 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:55:38.415 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.413+0000 7fd352a07740 20 client.26335 populate_metadata read hostname 'smithi099' 2024-03-19T01:55:38.415 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.413+0000 7fd352a07740 1 --2- 172.21.15.99:0/231469971 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x555b1c3d9290 0x555b1c3db680 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:38.415 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.413+0000 7fd352a07740 4 client.26335 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:55:38.416 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.413+0000 7fd352a07740 1 -- 172.21.15.99:0/231469971 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x555b1c32d120 con 0x555b1c3d9290 2024-03-19T01:55:38.416 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.413+0000 7fd34d3b4640 1 --2- 172.21.15.99:0/231469971 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x555b1c3d9290 0x555b1c3db680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:38.416 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.414+0000 7fd34d3b4640 1 --2- 172.21.15.99:0/231469971 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x555b1c3d9290 0x555b1c3db680 crc :-1 s=READY pgs=159 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=81aecb026e6b0524 server_cookie=53a34acb3c7b1db7 in_seq=0 out_seq=0 2024-03-19T01:55:38.416 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.414+0000 7fd3317fa640 10 client.26335 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:55:38.424 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.424+0000 7fd3317fa640 1 -- 172.21.15.99:0/231469971 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x555b1c32d120 con 0x555b1c3d9290 2024-03-19T01:55:38.424 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.424+0000 7fd3317fa640 10 client.26335 handle_command_reply: tid=0 2024-03-19T01:55:38.424 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.425+0000 7fd352a07740 1 client.26335 shutdown 2024-03-19T01:55:38.425 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.425+0000 7fd352a07740 1 -- 172.21.15.99:0/231469971 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x555b1c3bb100 msgr2=0x555b1c3cd7d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:55:38.425 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.425+0000 7fd352a07740 1 --2- 172.21.15.99:0/231469971 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x555b1c3bb100 0x555b1c3cd7d0 secure :-1 s=READY pgs=114 cs=0 l=1 rev1=1 crypto rx=0x7fd3440cb9c0 tx=0x7fd34409eaf0 comp rx=0 tx=0).stop 2024-03-19T01:55:38.426 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.425+0000 7fd352a07740 1 -- 172.21.15.99:0/231469971 shutdown_connections 2024-03-19T01:55:38.426 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.426+0000 7fd352a07740 1 --2- 172.21.15.99:0/231469971 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x555b1c3d9290 0x555b1c3db680 crc :-1 s=READY pgs=159 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:38.426 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.426+0000 7fd352a07740 1 --2- 172.21.15.99:0/231469971 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x555b1c3c0c60 0x555b1c3d4d50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:38.426 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.426+0000 7fd352a07740 1 --2- 172.21.15.99:0/231469971 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x555b1c3bbaf0 0x555b1c3cdd10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:38.426 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.426+0000 7fd352a07740 1 --2- 172.21.15.99:0/231469971 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x555b1c3bb100 0x555b1c3cd7d0 unknown :-1 s=CLOSED pgs=114 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:38.426 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.426+0000 7fd352a07740 1 -- 172.21.15.99:0/231469971 >> 172.21.15.99:0/231469971 conn(0x555b1c3b6670 msgr2=0x555b1c3b78c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:55:38.426 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.426+0000 7fd3317fa640 0 client.26335 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:55:38.426 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.426+0000 7fd352a07740 1 -- 172.21.15.99:0/231469971 shutdown_connections 2024-03-19T01:55:38.426 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.426+0000 7fd352a07740 1 -- 172.21.15.99:0/231469971 wait complete. 2024-03-19T01:55:38.426 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.426+0000 7fd352a07740 20 client.26335 trim_cache size 0 max 16384 2024-03-19T01:55:38.451 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.450+0000 7fd352a07740 1 Processor -- start 2024-03-19T01:55:38.451 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.450+0000 7fd352a07740 1 -- start start 2024-03-19T01:55:38.451 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.450+0000 7fd352a07740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x555b1c3bb100 0x555b1c3bcc70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:38.451 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.450+0000 7fd352a07740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x555b1c3bbaf0 0x555b1c3ccdf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:38.451 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.450+0000 7fd352a07740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x555b1c3c0c60 0x555b1c3cd330 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:38.451 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.450+0000 7fd352a07740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fd3481a95b0 con 0x555b1c3c0c60 2024-03-19T01:55:38.451 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.450+0000 7fd352a07740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fd3481a6df0 con 0x555b1c3bbaf0 2024-03-19T01:55:38.451 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.450+0000 7fd352a07740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fd3481a4c40 con 0x555b1c3bb100 2024-03-19T01:55:38.451 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.450+0000 7fd34e3b6640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x555b1c3c0c60 0x555b1c3cd330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:38.452 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.450+0000 7fd34e3b6640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x555b1c3c0c60 0x555b1c3cd330 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:50770/0 (socket says 172.21.15.99:50770) 2024-03-19T01:55:38.452 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.450+0000 7fd34d3b4640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x555b1c3bbaf0 0x555b1c3ccdf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:38.452 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.450+0000 7fd34e3b6640 1 -- 172.21.15.99:0/2562078592 learned_addr learned my addr 172.21.15.99:0/2562078592 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:55:38.452 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.450+0000 7fd34dbb5640 1 --2- 172.21.15.99:0/2562078592 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x555b1c3bb100 0x555b1c3bcc70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:38.452 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.450+0000 7fd34e3b6640 1 -- 172.21.15.99:0/2562078592 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x555b1c3bb100 msgr2=0x555b1c3bcc70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:55:38.452 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.450+0000 7fd34e3b6640 1 --2- 172.21.15.99:0/2562078592 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x555b1c3bb100 0x555b1c3bcc70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:38.452 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.450+0000 7fd34e3b6640 1 -- 172.21.15.99:0/2562078592 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x555b1c3bbaf0 msgr2=0x555b1c3ccdf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:55:38.452 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.450+0000 7fd34e3b6640 1 --2- 172.21.15.99:0/2562078592 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x555b1c3bbaf0 0x555b1c3ccdf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:38.452 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.450+0000 7fd34e3b6640 1 -- 172.21.15.99:0/2562078592 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x555b1c3db9f0 con 0x555b1c3c0c60 2024-03-19T01:55:38.452 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.450+0000 7fd34d3b4640 1 --2- 172.21.15.99:0/2562078592 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x555b1c3bbaf0 0x555b1c3ccdf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:55:38.452 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.451+0000 7fd34e3b6640 1 --2- 172.21.15.99:0/2562078592 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x555b1c3c0c60 0x555b1c3cd330 secure :-1 s=READY pgs=631 cs=0 l=1 rev1=1 crypto rx=0x7fd340009f70 tx=0x7fd34001f000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6c8e38a6a81b4b39 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:55:38.452 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.451+0000 7fd3337fe640 1 -- 172.21.15.99:0/2562078592 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd340011c50 con 0x555b1c3c0c60 2024-03-19T01:55:38.452 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.451+0000 7fd3337fe640 1 -- 172.21.15.99:0/2562078592 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd340024040 con 0x555b1c3c0c60 2024-03-19T01:55:38.452 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.451+0000 7fd3337fe640 1 -- 172.21.15.99:0/2562078592 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd340005960 con 0x555b1c3c0c60 2024-03-19T01:55:38.456 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.452+0000 7fd352a07740 1 -- 172.21.15.99:0/2562078592 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x555b1c3c0c60 msgr2=0x555b1c3cd330 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:55:38.456 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.452+0000 7fd352a07740 1 --2- 172.21.15.99:0/2562078592 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x555b1c3c0c60 0x555b1c3cd330 secure :-1 s=READY pgs=631 cs=0 l=1 rev1=1 crypto rx=0x7fd340009f70 tx=0x7fd34001f000 comp rx=0 tx=0).stop 2024-03-19T01:55:38.456 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.452+0000 7fd352a07740 1 -- 172.21.15.99:0/2562078592 shutdown_connections 2024-03-19T01:55:38.457 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.452+0000 7fd352a07740 1 --2- 172.21.15.99:0/2562078592 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x555b1c3bbaf0 0x555b1c3ccdf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:38.457 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.452+0000 7fd352a07740 1 --2- 172.21.15.99:0/2562078592 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x555b1c3c0c60 0x555b1c3cd330 unknown :-1 s=CLOSED pgs=631 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:38.457 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.452+0000 7fd352a07740 1 --2- 172.21.15.99:0/2562078592 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x555b1c3bb100 0x555b1c3bcc70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:38.457 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.452+0000 7fd352a07740 1 -- 172.21.15.99:0/2562078592 >> 172.21.15.99:0/2562078592 conn(0x555b1c3b6670 msgr2=0x555b1c3bc560 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:55:38.457 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.452+0000 7fd352a07740 1 -- 172.21.15.99:0/2562078592 shutdown_connections 2024-03-19T01:55:38.457 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.452+0000 7fd352a07740 1 -- 172.21.15.99:0/2562078592 wait complete. 2024-03-19T01:55:38.457 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.452+0000 7fd352a07740 1 -- start start 2024-03-19T01:55:38.457 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.452+0000 7fd352a07740 1 Processor -- start 2024-03-19T01:55:38.457 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.453+0000 7fd352a07740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x555b1c3bb100 0x555b1c3d2650 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:38.457 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.453+0000 7fd352a07740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x555b1c3bbaf0 0x555b1c3d2b90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:38.457 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.453+0000 7fd352a07740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x555b1c3c0c60 0x555b1c3d3ba0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:38.457 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.453+0000 7fd34dbb5640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x555b1c3bb100 0x555b1c3d2650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:38.457 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.453+0000 7fd34e3b6640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x555b1c3c0c60 0x555b1c3d3ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:38.457 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.453+0000 7fd34dbb5640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x555b1c3bb100 0x555b1c3d2650 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:55126/0 (socket says 172.21.15.99:55126) 2024-03-19T01:55:38.457 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.453+0000 7fd34dbb5640 1 -- 172.21.15.99:0/3571431204 learned_addr learned my addr 172.21.15.99:0/3571431204 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:55:38.458 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.453+0000 7fd352a07740 1 -- 172.21.15.99:0/3571431204 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fd3481a52b0 con 0x555b1c3c0c60 2024-03-19T01:55:38.458 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.453+0000 7fd34d3b4640 1 --2- 172.21.15.99:0/3571431204 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x555b1c3bbaf0 0x555b1c3d2b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:38.458 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.453+0000 7fd352a07740 1 -- 172.21.15.99:0/3571431204 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fd3481a6720 con 0x555b1c3bb100 2024-03-19T01:55:38.458 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.453+0000 7fd352a07740 1 -- 172.21.15.99:0/3571431204 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fd3481a5980 con 0x555b1c3bbaf0 2024-03-19T01:55:38.458 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.453+0000 7fd352a07740 10 client.0 _ll_register_callbacks cb 0x7fd3400117a0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:55:38.458 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.453+0000 7fd34e3b6640 1 -- 172.21.15.99:0/3571431204 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x555b1c3bbaf0 msgr2=0x555b1c3d2b90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:55:38.458 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.453+0000 7fd34e3b6640 1 --2- 172.21.15.99:0/3571431204 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x555b1c3bbaf0 0x555b1c3d2b90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:38.458 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.453+0000 7fd34e3b6640 1 -- 172.21.15.99:0/3571431204 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x555b1c3bb100 msgr2=0x555b1c3d2650 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:55:38.458 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.453+0000 7fd34e3b6640 1 --2- 172.21.15.99:0/3571431204 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x555b1c3bb100 0x555b1c3d2650 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:38.458 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.453+0000 7fd34e3b6640 1 -- 172.21.15.99:0/3571431204 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x555b1c100620 con 0x555b1c3c0c60 2024-03-19T01:55:38.458 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.453+0000 7fd34dbb5640 1 --2- 172.21.15.99:0/3571431204 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x555b1c3bb100 0x555b1c3d2650 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:55:38.458 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.453+0000 7fd34d3b4640 1 --2- 172.21.15.99:0/3571431204 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x555b1c3bbaf0 0x555b1c3d2b90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-19T01:55:38.459 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.453+0000 7fd34e3b6640 1 --2- 172.21.15.99:0/3571431204 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x555b1c3c0c60 0x555b1c3d3ba0 secure :-1 s=READY pgs=632 cs=0 l=1 rev1=1 crypto rx=0x7fd34000fcc0 tx=0x7fd340009d30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=13171aa2d74f715a server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:55:38.459 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.453+0000 7fd3317fa640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:55:38.459 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.454+0000 7fd3317fa640 1 -- 172.21.15.99:0/3571431204 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd340037020 con 0x555b1c3c0c60 2024-03-19T01:55:38.459 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.454+0000 7fd3317fa640 1 -- 172.21.15.99:0/3571431204 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fd340005870 con 0x555b1c3c0c60 2024-03-19T01:55:38.459 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.454+0000 7fd352a07740 1 -- 172.21.15.99:0/3571431204 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fd340024040 con 0x555b1c3c0c60 2024-03-19T01:55:38.459 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.454+0000 7fd3317fa640 1 -- 172.21.15.99:0/3571431204 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=35) ==== 24+0+0 (secure 0 0 0) 0x7fd340042050 con 0x555b1c3c0c60 2024-03-19T01:55:38.459 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.454+0000 7fd352a07740 10 client.16524 fetch_fsmap learned FSMap version 35 2024-03-19T01:55:38.459 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.454+0000 7fd352a07740 1 -- 172.21.15.99:0/3571431204 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=35,monmap=4+}) -- 0x555b1c43f8c0 con 0x555b1c3c0c60 2024-03-19T01:55:38.459 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.454+0000 7fd3317fa640 1 -- 172.21.15.99:0/3571431204 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd340024040 con 0x555b1c3c0c60 2024-03-19T01:55:38.459 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.455+0000 7fd3317fa640 1 -- 172.21.15.99:0/3571431204 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 35) ==== 8354+0+0 (secure 0 0 0) 0x7fd340027070 con 0x555b1c3c0c60 2024-03-19T01:55:38.459 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.455+0000 7fd352a07740 10 client.16524 fetch_fsmap finished waiting for FSMap version 35 2024-03-19T01:55:38.459 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.455+0000 7fd352a07740 10 client.16524 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:55:38.459 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.455+0000 7fd352a07740 20 client.16524 populate_metadata read hostname 'smithi099' 2024-03-19T01:55:38.459 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.455+0000 7fd352a07740 1 --2- 172.21.15.99:0/3571431204 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x555b1c440310 0x555b1c442730 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:38.460 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.455+0000 7fd352a07740 4 client.16524 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:55:38.460 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.455+0000 7fd352a07740 1 -- 172.21.15.99:0/3571431204 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x555b1c327470 con 0x555b1c440310 2024-03-19T01:55:38.460 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.455+0000 7fd34dbb5640 1 --2- 172.21.15.99:0/3571431204 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x555b1c440310 0x555b1c442730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:38.460 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.455+0000 7fd3317fa640 10 client.16524 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:55:38.460 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.455+0000 7fd34dbb5640 1 --2- 172.21.15.99:0/3571431204 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x555b1c440310 0x555b1c442730 crc :-1 s=READY pgs=160 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=93a05e793cb5de98 server_cookie=e3c576ab62d18a33 in_seq=0 out_seq=0 2024-03-19T01:55:38.883 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.879+0000 7fd3317fa640 1 -- 172.21.15.99:0/3571431204 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x555b1c327470 con 0x555b1c440310 2024-03-19T01:55:38.883 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.879+0000 7fd3317fa640 10 client.16524 handle_command_reply: tid=0 2024-03-19T01:55:38.883 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.879+0000 7fd352a07740 1 client.16524 shutdown 2024-03-19T01:55:38.883 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.879+0000 7fd352a07740 1 -- 172.21.15.99:0/3571431204 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x555b1c3c0c60 msgr2=0x555b1c3d3ba0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:55:38.883 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.879+0000 7fd352a07740 1 --2- 172.21.15.99:0/3571431204 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x555b1c3c0c60 0x555b1c3d3ba0 secure :-1 s=READY pgs=632 cs=0 l=1 rev1=1 crypto rx=0x7fd34000fcc0 tx=0x7fd340009d30 comp rx=0 tx=0).stop 2024-03-19T01:55:38.883 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.879+0000 7fd352a07740 1 -- 172.21.15.99:0/3571431204 shutdown_connections 2024-03-19T01:55:38.883 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.879+0000 7fd352a07740 1 --2- 172.21.15.99:0/3571431204 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x555b1c440310 0x555b1c442730 crc :-1 s=READY pgs=160 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:38.883 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.879+0000 7fd3317fa640 0 client.16524 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:55:38.883 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.879+0000 7fd352a07740 1 --2- 172.21.15.99:0/3571431204 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x555b1c3c0c60 0x555b1c3d3ba0 unknown :-1 s=CLOSED pgs=632 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:38.883 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.879+0000 7fd352a07740 1 --2- 172.21.15.99:0/3571431204 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x555b1c3bbaf0 0x555b1c3d2b90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:38.883 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.879+0000 7fd352a07740 1 --2- 172.21.15.99:0/3571431204 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x555b1c3bb100 0x555b1c3d2650 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:38.883 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.879+0000 7fd352a07740 1 -- 172.21.15.99:0/3571431204 >> 172.21.15.99:0/3571431204 conn(0x555b1c3b6670 msgr2=0x555b1c3ba160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:55:38.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.880+0000 7fd352a07740 1 -- 172.21.15.99:0/3571431204 shutdown_connections 2024-03-19T01:55:38.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.880+0000 7fd352a07740 1 -- 172.21.15.99:0/3571431204 wait complete. 2024-03-19T01:55:38.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.880+0000 7fd352a07740 20 client.16524 trim_cache size 0 max 16384 2024-03-19T01:55:38.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:55:38.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:55:38.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:55:38.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:55:38.885 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.881+0000 7fd34fe19640 1 -- 172.21.15.99:0/2924937074 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd3340a2ca0 msgr2=0x7fd3340a5160 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:55:38.885 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.881+0000 7fd34fe19640 1 --2- 172.21.15.99:0/2924937074 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd3340a2ca0 0x7fd3340a5160 secure :-1 s=READY pgs=338 cs=0 l=1 rev1=1 crypto rx=0x7fd340005fd0 tx=0x7fd340009f00 comp rx=0 tx=0).stop 2024-03-19T01:55:38.885 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.881+0000 7fd34fe19640 1 -- 172.21.15.99:0/2924937074 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd3480ffaf0 msgr2=0x7fd3481024d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:55:38.885 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.881+0000 7fd34fe19640 1 --2- 172.21.15.99:0/2924937074 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd3480ffaf0 0x7fd3481024d0 secure :-1 s=READY pgs=629 cs=0 l=1 rev1=1 crypto rx=0x7fd34400d950 tx=0x7fd34400de10 comp rx=0 tx=0).stop 2024-03-19T01:55:38.885 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.881+0000 7fd34fe19640 1 -- 172.21.15.99:0/2924937074 shutdown_connections 2024-03-19T01:55:38.885 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.881+0000 7fd34fe19640 1 --2- 172.21.15.99:0/2924937074 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd3340a2ca0 0x7fd3340a5160 unknown :-1 s=CLOSED pgs=338 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:38.885 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.881+0000 7fd34fe19640 1 --2- 172.21.15.99:0/2924937074 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd348103520 0x7fd348101050 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:38.885 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.881+0000 7fd34fe19640 1 --2- 172.21.15.99:0/2924937074 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd348102b60 0x7fd348100b10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:38.885 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.881+0000 7fd34fe19640 1 --2- 172.21.15.99:0/2924937074 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd3480ffaf0 0x7fd3481024d0 unknown :-1 s=CLOSED pgs=629 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:38.885 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.881+0000 7fd34fe19640 1 -- 172.21.15.99:0/2924937074 >> 172.21.15.99:0/2924937074 conn(0x7fd3480fb440 msgr2=0x7fd348105e60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:55:38.886 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.881+0000 7fd34fe19640 1 -- 172.21.15.99:0/2924937074 shutdown_connections 2024-03-19T01:55:38.886 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:38.881+0000 7fd34fe19640 1 -- 172.21.15.99:0/2924937074 wait complete. 2024-03-19T01:55:39.101 INFO:tasks.workunit.client.0.smithi099.stdout: 10 57968 22.61 MB/sec execute 8 sec latency 281.663 ms 2024-03-19T01:55:39.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:55:39 smithi099 ceph-mon[62951]: pgmap v788: 97 pgs: 97 active+clean; 510 MiB data, 5.4 GiB used, 1.0 TiB / 1.0 TiB avail; 5.0 MiB/s rd, 6.5 MiB/s wr, 495 op/s 2024-03-19T01:55:39.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:55:39 smithi112 ceph-mon[66819]: pgmap v788: 97 pgs: 97 active+clean; 510 MiB data, 5.4 GiB used, 1.0 TiB / 1.0 TiB avail; 5.0 MiB/s rd, 6.5 MiB/s wr, 495 op/s 2024-03-19T01:55:39.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:55:39 smithi115 ceph-mon[67332]: pgmap v788: 97 pgs: 97 active+clean; 510 MiB data, 5.4 GiB used, 1.0 TiB / 1.0 TiB avail; 5.0 MiB/s rd, 6.5 MiB/s wr, 495 op/s 2024-03-19T01:55:40.101 INFO:tasks.workunit.client.0.smithi099.stdout: 10 58345 21.98 MB/sec execute 9 sec latency 129.304 ms 2024-03-19T01:55:41.101 INFO:tasks.workunit.client.0.smithi099.stdout: 10 58713 21.62 MB/sec execute 10 sec latency 230.476 ms 2024-03-19T01:55:41.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:55:41 smithi099 ceph-mon[62951]: pgmap v789: 97 pgs: 97 active+clean; 508 MiB data, 5.4 GiB used, 1.0 TiB / 1.0 TiB avail; 4.8 MiB/s rd, 6.0 MiB/s wr, 473 op/s 2024-03-19T01:55:41.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:55:41 smithi112 ceph-mon[66819]: pgmap v789: 97 pgs: 97 active+clean; 508 MiB data, 5.4 GiB used, 1.0 TiB / 1.0 TiB avail; 4.8 MiB/s rd, 6.0 MiB/s wr, 473 op/s 2024-03-19T01:55:41.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:55:41 smithi115 ceph-mon[67332]: pgmap v789: 97 pgs: 97 active+clean; 508 MiB data, 5.4 GiB used, 1.0 TiB / 1.0 TiB avail; 4.8 MiB/s rd, 6.0 MiB/s wr, 473 op/s 2024-03-19T01:55:42.101 INFO:tasks.workunit.client.0.smithi099.stdout: 10 59120 22.31 MB/sec execute 11 sec latency 99.768 ms 2024-03-19T01:55:43.101 INFO:tasks.workunit.client.0.smithi099.stdout: 10 59572 22.51 MB/sec execute 12 sec latency 61.155 ms 2024-03-19T01:55:43.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:55:43 smithi099 ceph-mon[62951]: pgmap v790: 97 pgs: 97 active+clean; 508 MiB data, 5.4 GiB used, 1.0 TiB / 1.0 TiB avail; 5.4 MiB/s rd, 6.7 MiB/s wr, 619 op/s 2024-03-19T01:55:43.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:55:43 smithi112 ceph-mon[66819]: pgmap v790: 97 pgs: 97 active+clean; 508 MiB data, 5.4 GiB used, 1.0 TiB / 1.0 TiB avail; 5.4 MiB/s rd, 6.7 MiB/s wr, 619 op/s 2024-03-19T01:55:43.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:55:43 smithi115 ceph-mon[67332]: pgmap v790: 97 pgs: 97 active+clean; 508 MiB data, 5.4 GiB used, 1.0 TiB / 1.0 TiB avail; 5.4 MiB/s rd, 6.7 MiB/s wr, 619 op/s 2024-03-19T01:55:44.101 INFO:tasks.workunit.client.0.smithi099.stdout: 10 60126 22.82 MB/sec execute 13 sec latency 98.630 ms 2024-03-19T01:55:45.101 INFO:tasks.workunit.client.0.smithi099.stdout: 10 60532 22.38 MB/sec execute 14 sec latency 68.779 ms 2024-03-19T01:55:45.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:55:45 smithi099 ceph-mon[62951]: pgmap v791: 97 pgs: 97 active+clean; 505 MiB data, 5.4 GiB used, 1.0 TiB / 1.0 TiB avail; 4.4 MiB/s rd, 5.5 MiB/s wr, 515 op/s 2024-03-19T01:55:45.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:55:45 smithi112 ceph-mon[66819]: pgmap v791: 97 pgs: 97 active+clean; 505 MiB data, 5.4 GiB used, 1.0 TiB / 1.0 TiB avail; 4.4 MiB/s rd, 5.5 MiB/s wr, 515 op/s 2024-03-19T01:55:45.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:55:45 smithi115 ceph-mon[67332]: pgmap v791: 97 pgs: 97 active+clean; 505 MiB data, 5.4 GiB used, 1.0 TiB / 1.0 TiB avail; 4.4 MiB/s rd, 5.5 MiB/s wr, 515 op/s 2024-03-19T01:55:46.101 INFO:tasks.workunit.client.0.smithi099.stdout: 10 60917 22.54 MB/sec execute 15 sec latency 271.672 ms 2024-03-19T01:55:47.101 INFO:tasks.workunit.client.0.smithi099.stdout: 10 61364 22.94 MB/sec execute 16 sec latency 141.996 ms 2024-03-19T01:55:47.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:55:47 smithi099 ceph-mon[62951]: pgmap v792: 97 pgs: 97 active+clean; 513 MiB data, 5.4 GiB used, 1.0 TiB / 1.0 TiB avail; 5.6 MiB/s rd, 7.0 MiB/s wr, 610 op/s 2024-03-19T01:55:47.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:55:47 smithi112 ceph-mon[66819]: pgmap v792: 97 pgs: 97 active+clean; 513 MiB data, 5.4 GiB used, 1.0 TiB / 1.0 TiB avail; 5.6 MiB/s rd, 7.0 MiB/s wr, 610 op/s 2024-03-19T01:55:47.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:55:47 smithi115 ceph-mon[67332]: pgmap v792: 97 pgs: 97 active+clean; 513 MiB data, 5.4 GiB used, 1.0 TiB / 1.0 TiB avail; 5.6 MiB/s rd, 7.0 MiB/s wr, 610 op/s 2024-03-19T01:55:48.101 INFO:tasks.workunit.client.0.smithi099.stdout: 10 61859 23.12 MB/sec execute 17 sec latency 84.169 ms 2024-03-19T01:55:49.101 INFO:tasks.workunit.client.0.smithi099.stdout: 10 62291 23.13 MB/sec execute 18 sec latency 60.136 ms 2024-03-19T01:55:49.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:55:49 smithi099 ceph-mon[62951]: pgmap v793: 97 pgs: 97 active+clean; 513 MiB data, 5.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.8 MiB/s rd, 6.5 MiB/s wr, 549 op/s 2024-03-19T01:55:49.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:55:49 smithi112 ceph-mon[66819]: pgmap v793: 97 pgs: 97 active+clean; 513 MiB data, 5.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.8 MiB/s rd, 6.5 MiB/s wr, 549 op/s 2024-03-19T01:55:49.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:55:49 smithi115 ceph-mon[67332]: pgmap v793: 97 pgs: 97 active+clean; 513 MiB data, 5.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.8 MiB/s rd, 6.5 MiB/s wr, 549 op/s 2024-03-19T01:55:50.101 INFO:tasks.workunit.client.0.smithi099.stdout: 10 62674 22.99 MB/sec execute 19 sec latency 133.266 ms 2024-03-19T01:55:51.101 INFO:tasks.workunit.client.0.smithi099.stdout: 10 62994 22.44 MB/sec execute 20 sec latency 193.023 ms 2024-03-19T01:55:51.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:55:51 smithi099 ceph-mon[62951]: pgmap v794: 97 pgs: 97 active+clean; 518 MiB data, 5.5 GiB used, 1.0 TiB / 1.0 TiB avail; 5.2 MiB/s rd, 6.9 MiB/s wr, 568 op/s 2024-03-19T01:55:51.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:55:51 smithi112 ceph-mon[66819]: pgmap v794: 97 pgs: 97 active+clean; 518 MiB data, 5.5 GiB used, 1.0 TiB / 1.0 TiB avail; 5.2 MiB/s rd, 6.9 MiB/s wr, 568 op/s 2024-03-19T01:55:51.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:55:51 smithi115 ceph-mon[67332]: pgmap v794: 97 pgs: 97 active+clean; 518 MiB data, 5.5 GiB used, 1.0 TiB / 1.0 TiB avail; 5.2 MiB/s rd, 6.9 MiB/s wr, 568 op/s 2024-03-19T01:55:51.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.979+0000 7f9ae33c6640 1 -- 172.21.15.99:0/1770819247 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9adc102a60 msgr2=0x7f9adc102ee0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:55:51.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.979+0000 7f9ae33c6640 1 --2- 172.21.15.99:0/1770819247 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9adc102a60 0x7f9adc102ee0 secure :-1 s=READY pgs=633 cs=0 l=1 rev1=1 crypto rx=0x7f9acc009900 tx=0x7f9acc02bc30 comp rx=0 tx=0).stop 2024-03-19T01:55:51.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.979+0000 7f9ae33c6640 1 -- 172.21.15.99:0/1770819247 shutdown_connections 2024-03-19T01:55:51.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.979+0000 7f9ae33c6640 1 --2- 172.21.15.99:0/1770819247 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9adc103420 0x7f9adc109cb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:51.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.979+0000 7f9ae33c6640 1 --2- 172.21.15.99:0/1770819247 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9adc102a60 0x7f9adc102ee0 unknown :-1 s=CLOSED pgs=633 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:51.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.979+0000 7f9ae33c6640 1 --2- 172.21.15.99:0/1770819247 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9adc101870 0x7f9adc101c90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:51.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.979+0000 7f9ae33c6640 1 -- 172.21.15.99:0/1770819247 >> 172.21.15.99:0/1770819247 conn(0x7f9adc0fcde0 msgr2=0x7f9adc0ff220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:55:51.980 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.979+0000 7f9ae33c6640 1 -- 172.21.15.99:0/1770819247 shutdown_connections 2024-03-19T01:55:51.980 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.979+0000 7f9ae33c6640 1 -- 172.21.15.99:0/1770819247 wait complete. 2024-03-19T01:55:51.981 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.980+0000 7f9ae33c6640 1 Processor -- start 2024-03-19T01:55:51.981 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.980+0000 7f9ae33c6640 1 -- start start 2024-03-19T01:55:51.981 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.980+0000 7f9ae33c6640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9adc101870 0x7f9adc197b50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:51.982 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.980+0000 7f9ae33c6640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9adc102a60 0x7f9adc198090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:51.982 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.980+0000 7f9ae33c6640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9adc103420 0x7f9adc19f100 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:51.982 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.980+0000 7f9ae33c6640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f9adc10d4f0 con 0x7f9adc102a60 2024-03-19T01:55:51.982 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.980+0000 7f9ae33c6640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f9adc10d370 con 0x7f9adc101870 2024-03-19T01:55:51.982 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.980+0000 7f9ae33c6640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f9adc10d670 con 0x7f9adc103420 2024-03-19T01:55:51.982 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.980+0000 7f9ae0961640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9adc102a60 0x7f9adc198090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:51.982 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.980+0000 7f9ae1162640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9adc101870 0x7f9adc197b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:51.982 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.980+0000 7f9ae0961640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9adc102a60 0x7f9adc198090 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:54146/0 (socket says 172.21.15.99:54146) 2024-03-19T01:55:51.982 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.980+0000 7f9ae0961640 1 -- 172.21.15.99:0/2410653272 learned_addr learned my addr 172.21.15.99:0/2410653272 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:55:51.982 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.980+0000 7f9ae1963640 1 --2- 172.21.15.99:0/2410653272 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9adc103420 0x7f9adc19f100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:51.982 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.980+0000 7f9ae1162640 1 -- 172.21.15.99:0/2410653272 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9adc103420 msgr2=0x7f9adc19f100 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:55:51.982 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.980+0000 7f9ae1162640 1 --2- 172.21.15.99:0/2410653272 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9adc103420 0x7f9adc19f100 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:51.982 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.980+0000 7f9ae1162640 1 -- 172.21.15.99:0/2410653272 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9adc102a60 msgr2=0x7f9adc198090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:55:51.983 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.980+0000 7f9ae1162640 1 --2- 172.21.15.99:0/2410653272 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9adc102a60 0x7f9adc198090 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:51.983 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.981+0000 7f9ae1162640 1 -- 172.21.15.99:0/2410653272 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f9adc19f770 con 0x7f9adc101870 2024-03-19T01:55:51.983 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.981+0000 7f9ae1162640 1 --2- 172.21.15.99:0/2410653272 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9adc101870 0x7f9adc197b50 secure :-1 s=READY pgs=168 cs=0 l=1 rev1=1 crypto rx=0x7f9ad800e9c0 tx=0x7f9ad800ee80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:55:51.983 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.981+0000 7f9ad27fc640 1 -- 172.21.15.99:0/2410653272 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9ad800ccc0 con 0x7f9adc101870 2024-03-19T01:55:51.983 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.981+0000 7f9ad27fc640 1 -- 172.21.15.99:0/2410653272 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f9ad80035f0 con 0x7f9adc101870 2024-03-19T01:55:51.983 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.981+0000 7f9ae33c6640 1 -- 172.21.15.99:0/2410653272 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f9adc073ab0 con 0x7f9adc101870 2024-03-19T01:55:51.983 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.981+0000 7f9ad27fc640 1 -- 172.21.15.99:0/2410653272 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9ad800ce60 con 0x7f9adc101870 2024-03-19T01:55:51.983 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.981+0000 7f9ae33c6640 1 -- 172.21.15.99:0/2410653272 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f9adc073fb0 con 0x7f9adc101870 2024-03-19T01:55:51.988 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.983+0000 7f9ad27fc640 1 -- 172.21.15.99:0/2410653272 <== mon.2 v2:172.21.15.112:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f9ad80037a0 con 0x7f9adc101870 2024-03-19T01:55:51.988 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.984+0000 7f9ae5fb4740 1 Processor -- start 2024-03-19T01:55:51.988 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.984+0000 7f9ae5fb4740 1 -- start start 2024-03-19T01:55:51.988 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.984+0000 7f9ae5fb4740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5624bf77d100 0x5624bf77d520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:51.988 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.984+0000 7f9ae5fb4740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5624bf77daf0 0x5624bf7825f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:51.989 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.984+0000 7f9ae5fb4740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5624bf782c60 0x5624bf7850f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:51.989 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.984+0000 7f9ae5fb4740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x5624bf6e1be0 con 0x5624bf77daf0 2024-03-19T01:55:51.989 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.984+0000 7f9ae5fb4740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x5624bf68a960 con 0x5624bf77d100 2024-03-19T01:55:51.989 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.984+0000 7f9ae5fb4740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x5624bf6ddc80 con 0x5624bf782c60 2024-03-19T01:55:51.989 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.984+0000 7f9ad27fc640 1 --2- 172.21.15.99:0/2410653272 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f9ac80a28a0 0x7f9ac80a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:51.989 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.984+0000 7f9ae1162640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5624bf77daf0 0x5624bf7825f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:51.989 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.984+0000 7f9ae1963640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5624bf77d100 0x5624bf77d520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:51.989 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.984+0000 7f9ae1162640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5624bf77daf0 0x5624bf7825f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:54152/0 (socket says 172.21.15.99:54152) 2024-03-19T01:55:51.989 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.984+0000 7f9ae0961640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5624bf782c60 0x5624bf7850f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:51.989 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.984+0000 7f9ae1162640 1 -- 172.21.15.99:0/2785153626 learned_addr learned my addr 172.21.15.99:0/2785153626 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:55:51.989 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.984+0000 7f9ae1963640 1 --2- 172.21.15.99:0/2410653272 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f9ac80a28a0 0x7f9ac80a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:51.989 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.984+0000 7f9ad27fc640 1 -- 172.21.15.99:0/2410653272 <== mon.2 v2:172.21.15.112:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f9ad80ceff0 con 0x7f9adc101870 2024-03-19T01:55:51.989 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.984+0000 7f9ae1162640 1 -- 172.21.15.99:0/2785153626 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5624bf782c60 msgr2=0x5624bf7850f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:55:51.989 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.984+0000 7f9ae1162640 1 --2- 172.21.15.99:0/2785153626 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5624bf782c60 0x5624bf7850f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:51.990 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.984+0000 7f9ae1162640 1 -- 172.21.15.99:0/2785153626 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5624bf77d100 msgr2=0x5624bf77d520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:55:51.990 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.984+0000 7f9ae1162640 1 --2- 172.21.15.99:0/2785153626 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5624bf77d100 0x5624bf77d520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:51.990 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.984+0000 7f9ae1162640 1 -- 172.21.15.99:0/2785153626 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5624bf681d20 con 0x5624bf77daf0 2024-03-19T01:55:51.990 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.984+0000 7f9ae1963640 1 --2- 172.21.15.99:0/2785153626 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5624bf77d100 0x5624bf77d520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:55:51.990 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.985+0000 7f9ae1162640 1 --2- 172.21.15.99:0/2785153626 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5624bf77daf0 0x5624bf7825f0 secure :-1 s=READY pgs=634 cs=0 l=1 rev1=1 crypto rx=0x7f9ad80a3b80 tx=0x7f9ad809b7e0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=69c85a529a240819 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:55:51.990 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.985+0000 7f9ae1963640 1 --2- 172.21.15.99:0/2410653272 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f9ac80a28a0 0x7f9ac80a4d60 secure :-1 s=READY pgs=339 cs=0 l=1 rev1=1 crypto rx=0x5624bf77e350 tx=0x7f9ad4009500 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:55:51.990 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.985+0000 7f9ac77fe640 1 -- 172.21.15.99:0/2785153626 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9ad809c010 con 0x5624bf77daf0 2024-03-19T01:55:51.990 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.985+0000 7f9ac77fe640 1 -- 172.21.15.99:0/2785153626 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f9ad809b910 con 0x5624bf77daf0 2024-03-19T01:55:51.990 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.985+0000 7f9ac77fe640 1 -- 172.21.15.99:0/2785153626 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9ad80d10d0 con 0x5624bf77daf0 2024-03-19T01:55:51.990 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.985+0000 7f9ae5fb4740 1 -- 172.21.15.99:0/2785153626 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5624bf77daf0 msgr2=0x5624bf7825f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:55:51.990 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.985+0000 7f9ae5fb4740 1 --2- 172.21.15.99:0/2785153626 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5624bf77daf0 0x5624bf7825f0 secure :-1 s=READY pgs=634 cs=0 l=1 rev1=1 crypto rx=0x7f9ad80a3b80 tx=0x7f9ad809b7e0 comp rx=0 tx=0).stop 2024-03-19T01:55:51.990 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.985+0000 7f9ae5fb4740 1 -- 172.21.15.99:0/2785153626 shutdown_connections 2024-03-19T01:55:51.990 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.985+0000 7f9ae5fb4740 1 --2- 172.21.15.99:0/2785153626 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5624bf782c60 0x5624bf7850f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:51.991 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.986+0000 7f9ae5fb4740 1 --2- 172.21.15.99:0/2785153626 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5624bf77daf0 0x5624bf7825f0 unknown :-1 s=CLOSED pgs=634 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:51.991 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.986+0000 7f9ae5fb4740 1 --2- 172.21.15.99:0/2785153626 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5624bf77d100 0x5624bf77d520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:51.991 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.986+0000 7f9ae5fb4740 1 -- 172.21.15.99:0/2785153626 >> 172.21.15.99:0/2785153626 conn(0x5624bf778670 msgr2=0x5624bf778a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:55:51.991 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.986+0000 7f9ae5fb4740 1 -- 172.21.15.99:0/2785153626 shutdown_connections 2024-03-19T01:55:51.991 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.986+0000 7f9ae5fb4740 1 -- 172.21.15.99:0/2785153626 wait complete. 2024-03-19T01:55:51.991 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.986+0000 7f9ae5fb4740 1 -- start start 2024-03-19T01:55:51.991 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.986+0000 7f9ae5fb4740 1 Processor -- start 2024-03-19T01:55:51.993 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.986+0000 7f9ae5fb4740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5624bf77d100 0x5624bf787010 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:51.993 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.986+0000 7f9ae5fb4740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5624bf77daf0 0x5624bf787550 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:51.994 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.986+0000 7f9ae5fb4740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5624bf782c60 0x5624bf78e590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:51.994 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.986+0000 7f9ae5fb4740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x5624bf785b50 con 0x5624bf77d100 2024-03-19T01:55:51.994 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.986+0000 7f9ae5fb4740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x5624bf69ddb0 con 0x5624bf77daf0 2024-03-19T01:55:51.994 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.986+0000 7f9ae5fb4740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x5624bf785cd0 con 0x5624bf782c60 2024-03-19T01:55:51.994 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.986+0000 7f9ae5fb4740 10 client.0 _ll_register_callbacks cb 0x5624bf6a2e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:55:51.994 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.986+0000 7f9ae1162640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5624bf77d100 0x5624bf787010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:51.994 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.986+0000 7f9ae1162640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5624bf77d100 0x5624bf787010 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:54164/0 (socket says 172.21.15.99:54164) 2024-03-19T01:55:51.994 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.986+0000 7f9ae1162640 1 -- 172.21.15.99:0/1914594417 learned_addr learned my addr 172.21.15.99:0/1914594417 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:55:51.994 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.986+0000 7f9ae0961640 1 --2- 172.21.15.99:0/1914594417 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5624bf77daf0 0x5624bf787550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:51.994 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.986+0000 7f9ae1963640 1 --2- 172.21.15.99:0/1914594417 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5624bf782c60 0x5624bf78e590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:51.994 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.987+0000 7f9ae1162640 1 -- 172.21.15.99:0/1914594417 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5624bf782c60 msgr2=0x5624bf78e590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:55:51.994 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.987+0000 7f9ae1162640 1 --2- 172.21.15.99:0/1914594417 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5624bf782c60 0x5624bf78e590 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:51.994 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.987+0000 7f9ae1162640 1 -- 172.21.15.99:0/1914594417 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5624bf77daf0 msgr2=0x5624bf787550 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:55:51.994 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.987+0000 7f9ae1162640 1 --2- 172.21.15.99:0/1914594417 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5624bf77daf0 0x5624bf787550 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:51.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.987+0000 7f9ae1162640 1 -- 172.21.15.99:0/1914594417 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5624bf69cc60 con 0x5624bf77d100 2024-03-19T01:55:51.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.987+0000 7f9ae1162640 1 --2- 172.21.15.99:0/1914594417 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5624bf77d100 0x5624bf787010 secure :-1 s=READY pgs=635 cs=0 l=1 rev1=1 crypto rx=0x7f9ad8097d70 tx=0x7f9ad8097bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=29a725d492aea34 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:55:51.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.987+0000 7f9ac4ff9640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:55:51.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.987+0000 7f9ac4ff9640 1 -- 172.21.15.99:0/1914594417 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9ad80862b0 con 0x5624bf77d100 2024-03-19T01:55:51.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.987+0000 7f9ac4ff9640 1 -- 172.21.15.99:0/1914594417 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f9ad80bbb90 con 0x5624bf77d100 2024-03-19T01:55:51.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.987+0000 7f9ae5fb4740 1 -- 172.21.15.99:0/1914594417 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f9ad809b910 con 0x5624bf77d100 2024-03-19T01:55:51.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.987+0000 7f9ac4ff9640 1 -- 172.21.15.99:0/1914594417 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=35) ==== 24+0+0 (secure 0 0 0) 0x7f9ad809fa60 con 0x5624bf77d100 2024-03-19T01:55:51.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.987+0000 7f9ae5fb4740 10 client.16536 fetch_fsmap learned FSMap version 35 2024-03-19T01:55:51.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.987+0000 7f9ae5fb4740 1 -- 172.21.15.99:0/1914594417 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=35,monmap=4+}) -- 0x5624bf3f6210 con 0x5624bf77d100 2024-03-19T01:55:51.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.987+0000 7f9ac4ff9640 1 -- 172.21.15.99:0/1914594417 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f9ad809b910 con 0x5624bf77d100 2024-03-19T01:55:51.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.987+0000 7f9ac4ff9640 1 -- 172.21.15.99:0/1914594417 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 35) ==== 8354+0+0 (secure 0 0 0) 0x7f9ad80977f0 con 0x5624bf77d100 2024-03-19T01:55:51.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.988+0000 7f9ae5fb4740 10 client.16536 fetch_fsmap finished waiting for FSMap version 35 2024-03-19T01:55:51.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.988+0000 7f9ae5fb4740 10 client.16536 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:55:51.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.988+0000 7f9ae5fb4740 20 client.16536 populate_metadata read hostname 'smithi099' 2024-03-19T01:55:51.996 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.988+0000 7f9ae5fb4740 1 --2- 172.21.15.99:0/1914594417 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5624bf792a80 0x5624bf794e70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:51.996 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.988+0000 7f9ae5fb4740 4 client.16536 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:55:51.996 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.988+0000 7f9ae5fb4740 1 -- 172.21.15.99:0/1914594417 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5624bf6ef120 con 0x5624bf792a80 2024-03-19T01:55:51.996 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.988+0000 7f9ae0961640 1 --2- 172.21.15.99:0/1914594417 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5624bf792a80 0x5624bf794e70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:51.996 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.988+0000 7f9ae0961640 1 --2- 172.21.15.99:0/1914594417 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5624bf792a80 0x5624bf794e70 crc :-1 s=READY pgs=161 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=b0ee7180290411ad server_cookie=3266a28416d53d27 in_seq=0 out_seq=0 2024-03-19T01:55:51.996 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.988+0000 7f9ac4ff9640 10 client.16536 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:55:51.997 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.990+0000 7f9ac4ff9640 1 -- 172.21.15.99:0/1914594417 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x5624bf6ef120 con 0x5624bf792a80 2024-03-19T01:55:51.997 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.990+0000 7f9ac4ff9640 10 client.16536 handle_command_reply: tid=0 2024-03-19T01:55:51.997 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.990+0000 7f9ae5fb4740 1 client.16536 shutdown 2024-03-19T01:55:51.997 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.990+0000 7f9ae5fb4740 1 -- 172.21.15.99:0/1914594417 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5624bf77d100 msgr2=0x5624bf787010 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:55:51.997 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.990+0000 7f9ae5fb4740 1 --2- 172.21.15.99:0/1914594417 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5624bf77d100 0x5624bf787010 secure :-1 s=READY pgs=635 cs=0 l=1 rev1=1 crypto rx=0x7f9ad8097d70 tx=0x7f9ad8097bb0 comp rx=0 tx=0).stop 2024-03-19T01:55:51.997 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.990+0000 7f9ae5fb4740 1 -- 172.21.15.99:0/1914594417 shutdown_connections 2024-03-19T01:55:51.997 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.990+0000 7f9ae5fb4740 1 --2- 172.21.15.99:0/1914594417 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5624bf792a80 0x5624bf794e70 crc :-1 s=READY pgs=161 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:51.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.990+0000 7f9ae5fb4740 1 --2- 172.21.15.99:0/1914594417 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5624bf77daf0 0x5624bf787550 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:51.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.990+0000 7f9ac4ff9640 0 client.16536 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:55:51.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.990+0000 7f9ae5fb4740 1 --2- 172.21.15.99:0/1914594417 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5624bf782c60 0x5624bf78e590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:51.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.990+0000 7f9ae5fb4740 1 --2- 172.21.15.99:0/1914594417 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5624bf77d100 0x5624bf787010 unknown :-1 s=CLOSED pgs=635 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:51.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.990+0000 7f9ae5fb4740 1 -- 172.21.15.99:0/1914594417 >> 172.21.15.99:0/1914594417 conn(0x5624bf778670 msgr2=0x5624bf7798c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:55:51.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.990+0000 7f9ae5fb4740 1 -- 172.21.15.99:0/1914594417 shutdown_connections 2024-03-19T01:55:51.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.990+0000 7f9ae5fb4740 1 -- 172.21.15.99:0/1914594417 wait complete. 2024-03-19T01:55:51.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:51.990+0000 7f9ae5fb4740 20 client.16536 trim_cache size 0 max 16384 2024-03-19T01:55:52.007 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.006+0000 7f9ae5fb4740 1 Processor -- start 2024-03-19T01:55:52.008 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.006+0000 7f9ae5fb4740 1 -- start start 2024-03-19T01:55:52.008 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.006+0000 7f9ae5fb4740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5624bf77d100 0x5624bf7874b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:52.008 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.006+0000 7f9ae5fb4740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5624bf77daf0 0x5624bf78e550 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:52.008 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.006+0000 7f9ae5fb4740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5624bf782c60 0x5624bf77bfe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:52.008 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.006+0000 7f9ae5fb4740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f9adc1ada10 con 0x5624bf77d100 2024-03-19T01:55:52.008 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.006+0000 7f9ae5fb4740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f9adc1ab250 con 0x5624bf782c60 2024-03-19T01:55:52.008 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.006+0000 7f9ae5fb4740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f9adc1a90a0 con 0x5624bf77daf0 2024-03-19T01:55:52.008 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.006+0000 7f9ae1162640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5624bf77d100 0x5624bf7874b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:52.008 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.006+0000 7f9ae1162640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5624bf77d100 0x5624bf7874b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:54168/0 (socket says 172.21.15.99:54168) 2024-03-19T01:55:52.008 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.006+0000 7f9ae1162640 1 -- 172.21.15.99:0/2138930164 learned_addr learned my addr 172.21.15.99:0/2138930164 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:55:52.008 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.006+0000 7f9ae0961640 1 --2- 172.21.15.99:0/2138930164 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5624bf77daf0 0x5624bf78e550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:52.008 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.006+0000 7f9ae1963640 1 --2- 172.21.15.99:0/2138930164 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5624bf782c60 0x5624bf77bfe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:52.009 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.006+0000 7f9ae1162640 1 -- 172.21.15.99:0/2138930164 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5624bf77daf0 msgr2=0x5624bf78e550 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:55:52.009 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.006+0000 7f9ae1162640 1 --2- 172.21.15.99:0/2138930164 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5624bf77daf0 0x5624bf78e550 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:52.009 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.006+0000 7f9ae1162640 1 -- 172.21.15.99:0/2138930164 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5624bf782c60 msgr2=0x5624bf77bfe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:55:52.009 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.006+0000 7f9ae1162640 1 --2- 172.21.15.99:0/2138930164 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5624bf782c60 0x5624bf77bfe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:52.009 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.006+0000 7f9ae1162640 1 -- 172.21.15.99:0/2138930164 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5624bf4c2620 con 0x5624bf77d100 2024-03-19T01:55:52.009 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.006+0000 7f9ae0961640 1 --2- 172.21.15.99:0/2138930164 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5624bf77daf0 0x5624bf78e550 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:55:52.009 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.007+0000 7f9ae1162640 1 --2- 172.21.15.99:0/2138930164 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5624bf77d100 0x5624bf7874b0 secure :-1 s=READY pgs=636 cs=0 l=1 rev1=1 crypto rx=0x7f9ad809a8d0 tx=0x7f9ad809aa90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3baaaf9e9735e177 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:55:52.009 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.007+0000 7f9ac6ffd640 1 -- 172.21.15.99:0/2138930164 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9ad80ba1f0 con 0x5624bf77d100 2024-03-19T01:55:52.009 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.007+0000 7f9ac6ffd640 1 -- 172.21.15.99:0/2138930164 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f9ad809c010 con 0x5624bf77d100 2024-03-19T01:55:52.009 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.007+0000 7f9ac6ffd640 1 -- 172.21.15.99:0/2138930164 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9ad809afe0 con 0x5624bf77d100 2024-03-19T01:55:52.009 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.008+0000 7f9ae5fb4740 1 -- 172.21.15.99:0/2138930164 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5624bf77d100 msgr2=0x5624bf7874b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:55:52.009 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.008+0000 7f9ae5fb4740 1 --2- 172.21.15.99:0/2138930164 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5624bf77d100 0x5624bf7874b0 secure :-1 s=READY pgs=636 cs=0 l=1 rev1=1 crypto rx=0x7f9ad809a8d0 tx=0x7f9ad809aa90 comp rx=0 tx=0).stop 2024-03-19T01:55:52.009 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.008+0000 7f9ae5fb4740 1 -- 172.21.15.99:0/2138930164 shutdown_connections 2024-03-19T01:55:52.009 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.008+0000 7f9ae5fb4740 1 --2- 172.21.15.99:0/2138930164 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5624bf782c60 0x5624bf77bfe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:52.009 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.008+0000 7f9ae5fb4740 1 --2- 172.21.15.99:0/2138930164 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5624bf77daf0 0x5624bf78e550 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:52.010 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.008+0000 7f9ae5fb4740 1 --2- 172.21.15.99:0/2138930164 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5624bf77d100 0x5624bf7874b0 unknown :-1 s=CLOSED pgs=636 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:52.010 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.008+0000 7f9ae5fb4740 1 -- 172.21.15.99:0/2138930164 >> 172.21.15.99:0/2138930164 conn(0x5624bf778670 msgr2=0x5624bf77b8d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:55:52.010 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.008+0000 7f9ae5fb4740 1 -- 172.21.15.99:0/2138930164 shutdown_connections 2024-03-19T01:55:52.010 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.008+0000 7f9ae5fb4740 1 -- 172.21.15.99:0/2138930164 wait complete. 2024-03-19T01:55:52.010 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.008+0000 7f9ae5fb4740 1 -- start start 2024-03-19T01:55:52.010 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.008+0000 7f9ae5fb4740 1 Processor -- start 2024-03-19T01:55:52.012 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.008+0000 7f9ae5fb4740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5624bf77d100 0x5624bf78c040 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:52.013 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.008+0000 7f9ae5fb4740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5624bf77daf0 0x5624bf78c580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:52.013 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.008+0000 7f9ae5fb4740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5624bf782c60 0x5624bf78d590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:52.013 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.008+0000 7f9ae5fb4740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f9adc1a9710 con 0x5624bf782c60 2024-03-19T01:55:52.013 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.008+0000 7f9ae5fb4740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f9adc1aab80 con 0x5624bf77d100 2024-03-19T01:55:52.013 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.008+0000 7f9ae5fb4740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f9adc1a9de0 con 0x5624bf77daf0 2024-03-19T01:55:52.013 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.008+0000 7f9ae5fb4740 10 client.0 _ll_register_callbacks cb 0x7f9ad8005e30 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:55:52.013 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.009+0000 7f9ae1963640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5624bf782c60 0x5624bf78d590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:52.013 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.009+0000 7f9ae1963640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5624bf782c60 0x5624bf78d590 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:54178/0 (socket says 172.21.15.99:54178) 2024-03-19T01:55:52.013 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.009+0000 7f9ae1963640 1 -- 172.21.15.99:0/3028460295 learned_addr learned my addr 172.21.15.99:0/3028460295 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:55:52.013 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.009+0000 7f9ae0961640 1 --2- 172.21.15.99:0/3028460295 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5624bf77daf0 0x5624bf78c580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:52.013 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.009+0000 7f9ae1162640 1 --2- 172.21.15.99:0/3028460295 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5624bf77d100 0x5624bf78c040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:52.013 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.009+0000 7f9ae1963640 1 -- 172.21.15.99:0/3028460295 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5624bf77daf0 msgr2=0x5624bf78c580 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:55:52.013 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.009+0000 7f9ae1963640 1 --2- 172.21.15.99:0/3028460295 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5624bf77daf0 0x5624bf78c580 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:52.014 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.009+0000 7f9ae1963640 1 -- 172.21.15.99:0/3028460295 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5624bf77d100 msgr2=0x5624bf78c040 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:55:52.014 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.009+0000 7f9ae1963640 1 --2- 172.21.15.99:0/3028460295 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5624bf77d100 0x5624bf78c040 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:52.014 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.009+0000 7f9ae1963640 1 -- 172.21.15.99:0/3028460295 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5624bf78db80 con 0x5624bf782c60 2024-03-19T01:55:52.014 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.009+0000 7f9ae1162640 1 --2- 172.21.15.99:0/3028460295 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5624bf77d100 0x5624bf78c040 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:55:52.014 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.009+0000 7f9ae1963640 1 --2- 172.21.15.99:0/3028460295 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5624bf782c60 0x5624bf78d590 secure :-1 s=READY pgs=637 cs=0 l=1 rev1=1 crypto rx=0x7f9ad4003ab0 tx=0x7f9ad4004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8a3f8ce67b8651 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:55:52.014 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.009+0000 7f9ac4ff9640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:55:52.014 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.009+0000 7f9ac4ff9640 1 -- 172.21.15.99:0/3028460295 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9ad4004b20 con 0x5624bf782c60 2024-03-19T01:55:52.014 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.009+0000 7f9ac4ff9640 1 -- 172.21.15.99:0/3028460295 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f9ad4004cc0 con 0x5624bf782c60 2024-03-19T01:55:52.014 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.009+0000 7f9ae5fb4740 1 -- 172.21.15.99:0/3028460295 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f9ad809c010 con 0x5624bf782c60 2024-03-19T01:55:52.014 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.009+0000 7f9ac4ff9640 1 -- 172.21.15.99:0/3028460295 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=35) ==== 24+0+0 (secure 0 0 0) 0x7f9ad4005020 con 0x5624bf782c60 2024-03-19T01:55:52.014 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.009+0000 7f9ae5fb4740 10 client.16542 fetch_fsmap learned FSMap version 35 2024-03-19T01:55:52.014 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.009+0000 7f9ae5fb4740 1 -- 172.21.15.99:0/3028460295 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=35,monmap=4+}) -- 0x5624bf7f9080 con 0x5624bf782c60 2024-03-19T01:55:52.015 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.010+0000 7f9ac4ff9640 1 -- 172.21.15.99:0/3028460295 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f9ad809c010 con 0x5624bf782c60 2024-03-19T01:55:52.015 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.010+0000 7f9ac4ff9640 1 -- 172.21.15.99:0/3028460295 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 35) ==== 8354+0+0 (secure 0 0 0) 0x7f9ad402a070 con 0x5624bf782c60 2024-03-19T01:55:52.015 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.010+0000 7f9ae5fb4740 10 client.16542 fetch_fsmap finished waiting for FSMap version 35 2024-03-19T01:55:52.015 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.010+0000 7f9ae5fb4740 10 client.16542 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:55:52.015 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.010+0000 7f9ae5fb4740 20 client.16542 populate_metadata read hostname 'smithi099' 2024-03-19T01:55:52.015 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.010+0000 7f9ae5fb4740 1 --2- 172.21.15.99:0/3028460295 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5624bf7f9b00 0x5624bf7fbf20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:55:52.015 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.010+0000 7f9ae5fb4740 4 client.16542 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:55:52.015 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.010+0000 7f9ae5fb4740 1 -- 172.21.15.99:0/3028460295 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x5624bf6e9470 con 0x5624bf7f9b00 2024-03-19T01:55:52.015 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.010+0000 7f9ae1162640 1 --2- 172.21.15.99:0/3028460295 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5624bf7f9b00 0x5624bf7fbf20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:55:52.015 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.010+0000 7f9ae1162640 1 --2- 172.21.15.99:0/3028460295 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5624bf7f9b00 0x5624bf7fbf20 crc :-1 s=READY pgs=162 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=61b7fd6465db1762 server_cookie=a1596b67e5d1b51c in_seq=0 out_seq=0 2024-03-19T01:55:52.015 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.010+0000 7f9ac4ff9640 10 client.16542 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:55:52.102 INFO:tasks.workunit.client.0.smithi099.stdout: 10 63329 22.33 MB/sec execute 21 sec latency 274.681 ms 2024-03-19T01:55:52.453 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.452+0000 7f9ac4ff9640 1 -- 172.21.15.99:0/3028460295 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x5624bf6e9470 con 0x5624bf7f9b00 2024-03-19T01:55:52.453 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.452+0000 7f9ac4ff9640 10 client.16542 handle_command_reply: tid=0 2024-03-19T01:55:52.453 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.452+0000 7f9ae5fb4740 1 client.16542 shutdown 2024-03-19T01:55:52.453 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.453+0000 7f9ae5fb4740 1 -- 172.21.15.99:0/3028460295 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5624bf782c60 msgr2=0x5624bf78d590 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:55:52.453 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.453+0000 7f9ae5fb4740 1 --2- 172.21.15.99:0/3028460295 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5624bf782c60 0x5624bf78d590 secure :-1 s=READY pgs=637 cs=0 l=1 rev1=1 crypto rx=0x7f9ad4003ab0 tx=0x7f9ad4004a40 comp rx=0 tx=0).stop 2024-03-19T01:55:52.454 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.453+0000 7f9ae5fb4740 1 -- 172.21.15.99:0/3028460295 shutdown_connections 2024-03-19T01:55:52.454 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.453+0000 7f9ae5fb4740 1 --2- 172.21.15.99:0/3028460295 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5624bf7f9b00 0x5624bf7fbf20 crc :-1 s=READY pgs=162 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:52.454 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.453+0000 7f9ae5fb4740 1 --2- 172.21.15.99:0/3028460295 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5624bf782c60 0x5624bf78d590 unknown :-1 s=CLOSED pgs=637 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:52.454 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.453+0000 7f9ae5fb4740 1 --2- 172.21.15.99:0/3028460295 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5624bf77daf0 0x5624bf78c580 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:52.454 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.453+0000 7f9ae5fb4740 1 --2- 172.21.15.99:0/3028460295 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5624bf77d100 0x5624bf78c040 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:52.454 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.453+0000 7f9ae5fb4740 1 -- 172.21.15.99:0/3028460295 >> 172.21.15.99:0/3028460295 conn(0x5624bf778670 msgr2=0x5624bf77c160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:55:52.454 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.453+0000 7f9ac4ff9640 0 client.16542 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:55:52.454 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.453+0000 7f9ae5fb4740 1 -- 172.21.15.99:0/3028460295 shutdown_connections 2024-03-19T01:55:52.454 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.453+0000 7f9ae5fb4740 1 -- 172.21.15.99:0/3028460295 wait complete. 2024-03-19T01:55:52.454 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.453+0000 7f9ae5fb4740 20 client.16542 trim_cache size 0 max 16384 2024-03-19T01:55:52.455 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:55:52.455 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:55:52.455 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:55:52.455 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:55:52.456 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.454+0000 7f9ae33c6640 1 -- 172.21.15.99:0/2410653272 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f9ac80a28a0 msgr2=0x7f9ac80a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:55:52.456 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.454+0000 7f9ae33c6640 1 --2- 172.21.15.99:0/2410653272 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f9ac80a28a0 0x7f9ac80a4d60 secure :-1 s=READY pgs=339 cs=0 l=1 rev1=1 crypto rx=0x5624bf77e350 tx=0x7f9ad4009500 comp rx=0 tx=0).stop 2024-03-19T01:55:52.456 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.454+0000 7f9ae33c6640 1 -- 172.21.15.99:0/2410653272 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9adc101870 msgr2=0x7f9adc197b50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:55:52.456 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.454+0000 7f9ae33c6640 1 --2- 172.21.15.99:0/2410653272 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9adc101870 0x7f9adc197b50 secure :-1 s=READY pgs=168 cs=0 l=1 rev1=1 crypto rx=0x7f9ad800e9c0 tx=0x7f9ad800ee80 comp rx=0 tx=0).stop 2024-03-19T01:55:52.456 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.454+0000 7f9ae33c6640 1 -- 172.21.15.99:0/2410653272 shutdown_connections 2024-03-19T01:55:52.456 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.454+0000 7f9ae33c6640 1 --2- 172.21.15.99:0/2410653272 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9adc103420 0x7f9adc19f100 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:52.456 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.454+0000 7f9ae33c6640 1 --2- 172.21.15.99:0/2410653272 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9adc102a60 0x7f9adc198090 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:52.456 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.454+0000 7f9ae33c6640 1 --2- 172.21.15.99:0/2410653272 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f9ac80a28a0 0x7f9ac80a4d60 unknown :-1 s=CLOSED pgs=339 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:52.456 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.454+0000 7f9ae33c6640 1 --2- 172.21.15.99:0/2410653272 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9adc101870 0x7f9adc197b50 unknown :-1 s=CLOSED pgs=168 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:55:52.456 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.454+0000 7f9ae33c6640 1 -- 172.21.15.99:0/2410653272 >> 172.21.15.99:0/2410653272 conn(0x7f9adc0fcde0 msgr2=0x7f9adc105d00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:55:52.456 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.454+0000 7f9ae33c6640 1 -- 172.21.15.99:0/2410653272 shutdown_connections 2024-03-19T01:55:52.456 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:55:52.454+0000 7f9ae33c6640 1 -- 172.21.15.99:0/2410653272 wait complete. 2024-03-19T01:55:53.102 INFO:tasks.workunit.client.0.smithi099.stdout: 10 63646 21.82 MB/sec execute 22 sec latency 246.268 ms 2024-03-19T01:55:53.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:55:53 smithi099 ceph-mon[62951]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T01:55:53.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:55:53 smithi099 ceph-mon[62951]: pgmap v795: 97 pgs: 97 active+clean; 519 MiB data, 5.5 GiB used, 1.0 TiB / 1.0 TiB avail; 5.7 MiB/s rd, 7.5 MiB/s wr, 590 op/s 2024-03-19T01:55:53.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:55:53 smithi112 ceph-mon[66819]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T01:55:53.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:55:53 smithi112 ceph-mon[66819]: pgmap v795: 97 pgs: 97 active+clean; 519 MiB data, 5.5 GiB used, 1.0 TiB / 1.0 TiB avail; 5.7 MiB/s rd, 7.5 MiB/s wr, 590 op/s 2024-03-19T01:55:53.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:55:53 smithi115 ceph-mon[67332]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T01:55:53.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:55:53 smithi115 ceph-mon[67332]: pgmap v795: 97 pgs: 97 active+clean; 519 MiB data, 5.5 GiB used, 1.0 TiB / 1.0 TiB avail; 5.7 MiB/s rd, 7.5 MiB/s wr, 590 op/s 2024-03-19T01:55:54.102 INFO:tasks.workunit.client.0.smithi099.stdout: 10 64238 22.52 MB/sec execute 23 sec latency 53.683 ms 2024-03-19T01:55:55.104 INFO:tasks.workunit.client.0.smithi099.stdout: 10 64760 22.88 MB/sec execute 24 sec latency 102.228 ms 2024-03-19T01:55:55.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:55:55 smithi099 ceph-mon[62951]: pgmap v796: 97 pgs: 97 active+clean; 518 MiB data, 5.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.7 MiB/s rd, 6.4 MiB/s wr, 434 op/s 2024-03-19T01:55:55.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:55:55 smithi112 ceph-mon[66819]: pgmap v796: 97 pgs: 97 active+clean; 518 MiB data, 5.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.7 MiB/s rd, 6.4 MiB/s wr, 434 op/s 2024-03-19T01:55:55.568 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:55:55 smithi115 ceph-mon[67332]: pgmap v796: 97 pgs: 97 active+clean; 518 MiB data, 5.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.7 MiB/s rd, 6.4 MiB/s wr, 434 op/s 2024-03-19T01:55:56.102 INFO:tasks.workunit.client.0.smithi099.stdout: 10 65185 22.69 MB/sec execute 25 sec latency 126.567 ms 2024-03-19T01:55:57.102 INFO:tasks.workunit.client.0.smithi099.stdout: 10 65418 22.26 MB/sec execute 26 sec latency 259.399 ms 2024-03-19T01:55:57.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:55:57 smithi115 ceph-mon[67332]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T01:55:57.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:55:57 smithi115 ceph-mon[67332]: pgmap v797: 97 pgs: 97 active+clean; 517 MiB data, 5.5 GiB used, 1.0 TiB / 1.0 TiB avail; 5.8 MiB/s rd, 7.8 MiB/s wr, 622 op/s 2024-03-19T01:55:57.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:55:57 smithi099 ceph-mon[62951]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T01:55:57.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:55:57 smithi099 ceph-mon[62951]: pgmap v797: 97 pgs: 97 active+clean; 517 MiB data, 5.5 GiB used, 1.0 TiB / 1.0 TiB avail; 5.8 MiB/s rd, 7.8 MiB/s wr, 622 op/s 2024-03-19T01:55:57.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:55:57 smithi112 ceph-mon[66819]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T01:55:57.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:55:57 smithi112 ceph-mon[66819]: pgmap v797: 97 pgs: 97 active+clean; 517 MiB data, 5.5 GiB used, 1.0 TiB / 1.0 TiB avail; 5.8 MiB/s rd, 7.8 MiB/s wr, 622 op/s 2024-03-19T01:55:58.102 INFO:tasks.workunit.client.0.smithi099.stdout: 10 65983 22.66 MB/sec execute 27 sec latency 76.803 ms 2024-03-19T01:55:59.102 INFO:tasks.workunit.client.0.smithi099.stdout: 10 66502 23.09 MB/sec execute 28 sec latency 76.397 ms 2024-03-19T01:55:59.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:55:59 smithi099 ceph-mon[62951]: pgmap v798: 97 pgs: 97 active+clean; 516 MiB data, 5.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.4 MiB/s rd, 6.3 MiB/s wr, 529 op/s 2024-03-19T01:55:59.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:55:59 smithi112 ceph-mon[66819]: pgmap v798: 97 pgs: 97 active+clean; 516 MiB data, 5.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.4 MiB/s rd, 6.3 MiB/s wr, 529 op/s 2024-03-19T01:55:59.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:55:59 smithi115 ceph-mon[67332]: pgmap v798: 97 pgs: 97 active+clean; 516 MiB data, 5.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.4 MiB/s rd, 6.3 MiB/s wr, 529 op/s 2024-03-19T01:56:00.102 INFO:tasks.workunit.client.0.smithi099.stdout: 10 67024 23.42 MB/sec execute 29 sec latency 58.762 ms 2024-03-19T01:56:01.102 INFO:tasks.workunit.client.0.smithi099.stdout: 10 67348 22.83 MB/sec execute 30 sec latency 61.740 ms 2024-03-19T01:56:01.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:56:01 smithi099 ceph-mon[62951]: pgmap v799: 97 pgs: 97 active+clean; 522 MiB data, 5.6 GiB used, 1.0 TiB / 1.0 TiB avail; 4.8 MiB/s rd, 6.5 MiB/s wr, 583 op/s 2024-03-19T01:56:01.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:56:01 smithi112 ceph-mon[66819]: pgmap v799: 97 pgs: 97 active+clean; 522 MiB data, 5.6 GiB used, 1.0 TiB / 1.0 TiB avail; 4.8 MiB/s rd, 6.5 MiB/s wr, 583 op/s 2024-03-19T01:56:01.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:56:01 smithi115 ceph-mon[67332]: pgmap v799: 97 pgs: 97 active+clean; 522 MiB data, 5.6 GiB used, 1.0 TiB / 1.0 TiB avail; 4.8 MiB/s rd, 6.5 MiB/s wr, 583 op/s 2024-03-19T01:56:02.102 INFO:tasks.workunit.client.0.smithi099.stdout: 10 67626 22.51 MB/sec execute 31 sec latency 321.559 ms 2024-03-19T01:56:03.102 INFO:tasks.workunit.client.0.smithi099.stdout: 10 68051 22.42 MB/sec execute 32 sec latency 86.537 ms 2024-03-19T01:56:03.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:56:03 smithi099 ceph-mon[62951]: pgmap v800: 97 pgs: 97 active+clean; 527 MiB data, 5.6 GiB used, 1.0 TiB / 1.0 TiB avail; 4.7 MiB/s rd, 6.7 MiB/s wr, 591 op/s 2024-03-19T01:56:03.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:56:03 smithi112 ceph-mon[66819]: pgmap v800: 97 pgs: 97 active+clean; 527 MiB data, 5.6 GiB used, 1.0 TiB / 1.0 TiB avail; 4.7 MiB/s rd, 6.7 MiB/s wr, 591 op/s 2024-03-19T01:56:03.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:56:03 smithi115 ceph-mon[67332]: pgmap v800: 97 pgs: 97 active+clean; 527 MiB data, 5.6 GiB used, 1.0 TiB / 1.0 TiB avail; 4.7 MiB/s rd, 6.7 MiB/s wr, 591 op/s 2024-03-19T01:56:04.102 INFO:tasks.workunit.client.0.smithi099.stdout: 10 68611 22.71 MB/sec execute 33 sec latency 77.992 ms 2024-03-19T01:56:05.103 INFO:tasks.workunit.client.0.smithi099.stdout: 10 69206 23.13 MB/sec execute 34 sec latency 136.123 ms 2024-03-19T01:56:05.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:56:05 smithi099 ceph-mon[62951]: pgmap v801: 97 pgs: 97 active+clean; 526 MiB data, 5.6 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 5.7 MiB/s wr, 505 op/s 2024-03-19T01:56:05.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:56:05 smithi112 ceph-mon[66819]: pgmap v801: 97 pgs: 97 active+clean; 526 MiB data, 5.6 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 5.7 MiB/s wr, 505 op/s 2024-03-19T01:56:05.548 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.547+0000 7f568db3b640 1 -- 172.21.15.99:0/2156031418 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f56881033e0 msgr2=0x7f5688109c70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:56:05.549 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.547+0000 7f568db3b640 1 --2- 172.21.15.99:0/2156031418 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f56881033e0 0x7f5688109c70 secure :-1 s=READY pgs=638 cs=0 l=1 rev1=1 crypto rx=0x7f5680009ca0 tx=0x7f568002c0f0 comp rx=0 tx=0).stop 2024-03-19T01:56:05.549 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.548+0000 7f568db3b640 1 -- 172.21.15.99:0/2156031418 shutdown_connections 2024-03-19T01:56:05.549 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.548+0000 7f568db3b640 1 --2- 172.21.15.99:0/2156031418 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f56881033e0 0x7f5688109c70 unknown :-1 s=CLOSED pgs=638 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:05.549 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.548+0000 7f568db3b640 1 --2- 172.21.15.99:0/2156031418 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5688102a20 0x7f5688102ea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:05.549 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.548+0000 7f568db3b640 1 --2- 172.21.15.99:0/2156031418 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5688101830 0x7f5688101c50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:05.549 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.548+0000 7f568db3b640 1 -- 172.21.15.99:0/2156031418 >> 172.21.15.99:0/2156031418 conn(0x7f56880fcda0 msgr2=0x7f56880ff1e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:56:05.549 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.548+0000 7f568db3b640 1 -- 172.21.15.99:0/2156031418 shutdown_connections 2024-03-19T01:56:05.549 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.548+0000 7f568db3b640 1 -- 172.21.15.99:0/2156031418 wait complete. 2024-03-19T01:56:05.549 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.548+0000 7f568db3b640 1 Processor -- start 2024-03-19T01:56:05.551 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.548+0000 7f568db3b640 1 -- start start 2024-03-19T01:56:05.551 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.549+0000 7f568db3b640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5688101830 0x7f56880755d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:05.552 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.549+0000 7f568db3b640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5688102a20 0x7f5688073c10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:05.552 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.549+0000 7f568db3b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f56881033e0 0x7f5688074150 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:05.552 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.549+0000 7f568db3b640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f568810d4b0 con 0x7f56881033e0 2024-03-19T01:56:05.552 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.549+0000 7f568db3b640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f568810d330 con 0x7f5688102a20 2024-03-19T01:56:05.552 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.549+0000 7f568db3b640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f568810d630 con 0x7f5688101830 2024-03-19T01:56:05.552 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.549+0000 7f5687fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f56881033e0 0x7f5688074150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:05.552 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.549+0000 7f5686ffd640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5688102a20 0x7f5688073c10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:05.552 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.549+0000 7f5687fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f56881033e0 0x7f5688074150 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:55740/0 (socket says 172.21.15.99:55740) 2024-03-19T01:56:05.552 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.549+0000 7f5687fff640 1 -- 172.21.15.99:0/1512797317 learned_addr learned my addr 172.21.15.99:0/1512797317 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:56:05.552 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.549+0000 7f56877fe640 1 --2- 172.21.15.99:0/1512797317 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5688101830 0x7f56880755d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:05.552 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.549+0000 7f5687fff640 1 -- 172.21.15.99:0/1512797317 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5688101830 msgr2=0x7f56880755d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:56:05.552 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.549+0000 7f5687fff640 1 --2- 172.21.15.99:0/1512797317 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5688101830 0x7f56880755d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:05.552 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.549+0000 7f5687fff640 1 -- 172.21.15.99:0/1512797317 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5688102a20 msgr2=0x7f5688073c10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:56:05.553 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.549+0000 7f5687fff640 1 --2- 172.21.15.99:0/1512797317 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5688102a20 0x7f5688073c10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:05.553 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.549+0000 7f5687fff640 1 -- 172.21.15.99:0/1512797317 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f56880749e0 con 0x7f56881033e0 2024-03-19T01:56:05.553 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.549+0000 7f5686ffd640 1 --2- 172.21.15.99:0/1512797317 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5688102a20 0x7f5688073c10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:56:05.553 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.549+0000 7f5687fff640 1 --2- 172.21.15.99:0/1512797317 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f56881033e0 0x7f5688074150 secure :-1 s=READY pgs=639 cs=0 l=1 rev1=1 crypto rx=0x7f5680037040 tx=0x7f5680032f40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:56:05.553 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.549+0000 7f5684ff9640 1 -- 172.21.15.99:0/1512797317 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5680041020 con 0x7f56881033e0 2024-03-19T01:56:05.553 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.549+0000 7f568db3b640 1 -- 172.21.15.99:0/1512797317 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f56881a1af0 con 0x7f56881033e0 2024-03-19T01:56:05.553 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.549+0000 7f5684ff9640 1 -- 172.21.15.99:0/1512797317 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f5680034cd0 con 0x7f56881033e0 2024-03-19T01:56:05.553 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.549+0000 7f5684ff9640 1 -- 172.21.15.99:0/1512797317 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5680039040 con 0x7f56881033e0 2024-03-19T01:56:05.553 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.550+0000 7f568db3b640 1 -- 172.21.15.99:0/1512797317 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f56881a2090 con 0x7f56881033e0 2024-03-19T01:56:05.558 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.552+0000 7f5684ff9640 1 -- 172.21.15.99:0/1512797317 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f5680044020 con 0x7f56881033e0 2024-03-19T01:56:05.559 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.552+0000 7f5690804740 1 Processor -- start 2024-03-19T01:56:05.559 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.553+0000 7f5690804740 1 -- start start 2024-03-19T01:56:05.559 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.553+0000 7f5690804740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5635ef019100 0x5635ef019520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:05.559 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.553+0000 7f5690804740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5635ef019af0 0x5635ef01e5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:05.559 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.553+0000 7f5690804740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5635ef01ec60 0x5635ef0210f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:05.559 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.553+0000 7f5690804740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x5635eef7dbe0 con 0x5635ef01ec60 2024-03-19T01:56:05.559 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.553+0000 7f5690804740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x5635eef26960 con 0x5635ef019af0 2024-03-19T01:56:05.559 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.553+0000 7f5690804740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x5635eef79c80 con 0x5635ef019100 2024-03-19T01:56:05.559 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.553+0000 7f5686ffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5635ef01ec60 0x5635ef0210f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:05.559 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.553+0000 7f56877fe640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5635ef019af0 0x5635ef01e5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:05.559 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.553+0000 7f5686ffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5635ef01ec60 0x5635ef0210f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:55752/0 (socket says 172.21.15.99:55752) 2024-03-19T01:56:05.559 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.553+0000 7f5684ff9640 1 --2- 172.21.15.99:0/1512797317 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f56700a28a0 0x7f56700a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:05.559 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.553+0000 7f5686ffd640 1 -- 172.21.15.99:0/131928635 learned_addr learned my addr 172.21.15.99:0/131928635 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:56:05.559 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.553+0000 7f5686ffd640 1 --2- 172.21.15.99:0/131928635 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5635ef019100 0x5635ef019520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:05.560 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.553+0000 7f5687fff640 1 --2- 172.21.15.99:0/1512797317 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f56700a28a0 0x7f56700a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:05.560 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.553+0000 7f56877fe640 1 -- 172.21.15.99:0/131928635 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5635ef019100 msgr2=0x5635ef019520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:56:05.560 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.553+0000 7f56877fe640 1 --2- 172.21.15.99:0/131928635 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5635ef019100 0x5635ef019520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:05.560 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.553+0000 7f56877fe640 1 -- 172.21.15.99:0/131928635 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5635ef01ec60 msgr2=0x5635ef0210f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:56:05.560 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.553+0000 7f56877fe640 1 --2- 172.21.15.99:0/131928635 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5635ef01ec60 0x5635ef0210f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:05.560 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.553+0000 7f56877fe640 1 -- 172.21.15.99:0/131928635 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5635eef1dd20 con 0x5635ef019af0 2024-03-19T01:56:05.560 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.553+0000 7f5686ffd640 1 --2- 172.21.15.99:0/131928635 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5635ef01ec60 0x5635ef0210f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:56:05.560 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.553+0000 7f5684ff9640 1 -- 172.21.15.99:0/1512797317 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f56800f08c0 con 0x7f56881033e0 2024-03-19T01:56:05.560 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.554+0000 7f56877fe640 1 --2- 172.21.15.99:0/131928635 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5635ef019af0 0x5635ef01e5f0 secure :-1 s=READY pgs=169 cs=0 l=1 rev1=1 crypto rx=0x7f5678009bf0 tx=0x7f5678011810 comp rx=0 tx=0).ready entity=mon.2 client_cookie=b30b4281844c42c7 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:56:05.560 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.554+0000 7f5687fff640 1 --2- 172.21.15.99:0/1512797317 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f56700a28a0 0x7f56700a4d60 secure :-1 s=READY pgs=340 cs=0 l=1 rev1=1 crypto rx=0x7f56800c5490 tx=0x7f568002e040 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:56:05.560 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.554+0000 7f566dffb640 1 -- 172.21.15.99:0/131928635 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5678004050 con 0x5635ef019af0 2024-03-19T01:56:05.560 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.554+0000 7f566dffb640 1 -- 172.21.15.99:0/131928635 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f5678005500 con 0x5635ef019af0 2024-03-19T01:56:05.560 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.554+0000 7f566dffb640 1 -- 172.21.15.99:0/131928635 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5678004d10 con 0x5635ef019af0 2024-03-19T01:56:05.560 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.555+0000 7f5690804740 1 -- 172.21.15.99:0/131928635 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5635ef019af0 msgr2=0x5635ef01e5f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:56:05.560 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.555+0000 7f5690804740 1 --2- 172.21.15.99:0/131928635 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5635ef019af0 0x5635ef01e5f0 secure :-1 s=READY pgs=169 cs=0 l=1 rev1=1 crypto rx=0x7f5678009bf0 tx=0x7f5678011810 comp rx=0 tx=0).stop 2024-03-19T01:56:05.561 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.555+0000 7f5690804740 1 -- 172.21.15.99:0/131928635 shutdown_connections 2024-03-19T01:56:05.561 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.555+0000 7f5690804740 1 --2- 172.21.15.99:0/131928635 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5635ef019af0 0x5635ef01e5f0 unknown :-1 s=CLOSED pgs=169 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:05.561 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.555+0000 7f5690804740 1 --2- 172.21.15.99:0/131928635 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5635ef01ec60 0x5635ef0210f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:05.561 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.555+0000 7f5690804740 1 --2- 172.21.15.99:0/131928635 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5635ef019100 0x5635ef019520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:05.561 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.555+0000 7f5690804740 1 -- 172.21.15.99:0/131928635 >> 172.21.15.99:0/131928635 conn(0x5635ef014670 msgr2=0x5635ef014a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:56:05.561 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.555+0000 7f5690804740 1 -- 172.21.15.99:0/131928635 shutdown_connections 2024-03-19T01:56:05.561 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.555+0000 7f5690804740 1 -- 172.21.15.99:0/131928635 wait complete. 2024-03-19T01:56:05.561 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.555+0000 7f5690804740 1 -- start start 2024-03-19T01:56:05.561 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.555+0000 7f5690804740 1 Processor -- start 2024-03-19T01:56:05.561 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.555+0000 7f5690804740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5635ef019100 0x5635ef0230e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:05.561 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.555+0000 7f5690804740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5635ef019af0 0x5635ef023620 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:05.561 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.555+0000 7f5690804740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5635ef01ec60 0x5635ef02a660 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:05.561 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.555+0000 7f5690804740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x5635ef021c10 con 0x5635ef01ec60 2024-03-19T01:56:05.561 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.555+0000 7f5690804740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x5635eef39db0 con 0x5635ef019af0 2024-03-19T01:56:05.561 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.555+0000 7f5690804740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x5635ef021d90 con 0x5635ef019100 2024-03-19T01:56:05.562 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.555+0000 7f5690804740 10 client.0 _ll_register_callbacks cb 0x5635eef3ee70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:56:05.562 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.556+0000 7f56877fe640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5635ef019af0 0x5635ef023620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:05.562 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.556+0000 7f5686ffd640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5635ef019100 0x5635ef0230e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:05.562 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.556+0000 7f5686ffd640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5635ef019100 0x5635ef0230e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:38972/0 (socket says 172.21.15.99:38972) 2024-03-19T01:56:05.562 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.556+0000 7f56877fe640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5635ef019af0 0x5635ef023620 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:38272/0 (socket says 172.21.15.99:38272) 2024-03-19T01:56:05.562 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.556+0000 7f5686ffd640 1 -- 172.21.15.99:0/2031019378 learned_addr learned my addr 172.21.15.99:0/2031019378 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:56:05.562 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.556+0000 7f5686ffd640 1 --2- 172.21.15.99:0/2031019378 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5635ef01ec60 0x5635ef02a660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:05.562 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.556+0000 7f5686ffd640 1 -- 172.21.15.99:0/2031019378 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5635ef019af0 msgr2=0x5635ef023620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:56:05.562 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.556+0000 7f5686ffd640 1 --2- 172.21.15.99:0/2031019378 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5635ef019af0 0x5635ef023620 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:05.562 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.556+0000 7f5686ffd640 1 -- 172.21.15.99:0/2031019378 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5635ef01ec60 msgr2=0x5635ef02a660 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:56:05.562 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.556+0000 7f5686ffd640 1 --2- 172.21.15.99:0/2031019378 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5635ef01ec60 0x5635ef02a660 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:05.562 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.556+0000 7f5686ffd640 1 -- 172.21.15.99:0/2031019378 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5635eef38c60 con 0x5635ef019100 2024-03-19T01:56:05.562 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.556+0000 7f56877fe640 1 --2- 172.21.15.99:0/2031019378 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5635ef019af0 0x5635ef023620 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:56:05.562 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.556+0000 7f5686ffd640 1 --2- 172.21.15.99:0/2031019378 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5635ef019100 0x5635ef0230e0 secure :-1 s=READY pgs=115 cs=0 l=1 rev1=1 crypto rx=0x7f567c01e410 tx=0x7f567c020040 comp rx=0 tx=0).ready entity=mon.1 client_cookie=d89bb47c4a61502a server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:56:05.562 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.556+0000 7f5666ffd640 10 client.0 ms_handle_connect on v2:172.21.15.115:3300/0 2024-03-19T01:56:05.563 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.556+0000 7f5666ffd640 1 -- 172.21.15.99:0/2031019378 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f567c010d00 con 0x5635ef019100 2024-03-19T01:56:05.563 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.556+0000 7f5690804740 1 -- 172.21.15.99:0/2031019378 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f5678005500 con 0x5635ef019100 2024-03-19T01:56:05.563 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.556+0000 7f5666ffd640 1 -- 172.21.15.99:0/2031019378 <== mon.1 v2:172.21.15.115:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f567c02b450 con 0x5635ef019100 2024-03-19T01:56:05.563 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.556+0000 7f5666ffd640 1 -- 172.21.15.99:0/2031019378 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7f567c005020 con 0x5635ef019100 2024-03-19T01:56:05.563 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.556+0000 7f5690804740 10 client.26371 fetch_fsmap learned FSMap version 37 2024-03-19T01:56:05.563 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.556+0000 7f5690804740 1 -- 172.21.15.99:0/2031019378 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x5635eec92210 con 0x5635ef019100 2024-03-19T01:56:05.563 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.557+0000 7f5666ffd640 1 -- 172.21.15.99:0/2031019378 <== mon.1 v2:172.21.15.115:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f5678005500 con 0x5635ef019100 2024-03-19T01:56:05.563 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.557+0000 7f5666ffd640 1 -- 172.21.15.99:0/2031019378 <== mon.1 v2:172.21.15.115:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7f567c02d700 con 0x5635ef019100 2024-03-19T01:56:05.563 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.557+0000 7f5690804740 10 client.26371 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T01:56:05.563 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.557+0000 7f5690804740 10 client.26371 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:56:05.563 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.557+0000 7f5690804740 20 client.26371 populate_metadata read hostname 'smithi099' 2024-03-19T01:56:05.563 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.557+0000 7f5690804740 1 --2- 172.21.15.99:0/2031019378 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5635ef02eef0 0x5635ef031360 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:05.563 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.557+0000 7f5690804740 4 client.26371 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:56:05.563 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.557+0000 7f5690804740 1 -- 172.21.15.99:0/2031019378 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5635eef8b120 con 0x5635ef02eef0 2024-03-19T01:56:05.563 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.557+0000 7f56877fe640 1 --2- 172.21.15.99:0/2031019378 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5635ef02eef0 0x5635ef031360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:05.564 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.558+0000 7f56877fe640 1 --2- 172.21.15.99:0/2031019378 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5635ef02eef0 0x5635ef031360 crc :-1 s=READY pgs=163 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=7c6b466edc12467 server_cookie=2cbb6068d0804a89 in_seq=0 out_seq=0 2024-03-19T01:56:05.564 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.558+0000 7f5666ffd640 10 client.26371 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:56:05.564 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:56:05 smithi115 ceph-mon[67332]: pgmap v801: 97 pgs: 97 active+clean; 526 MiB data, 5.6 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 5.7 MiB/s wr, 505 op/s 2024-03-19T01:56:05.565 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.560+0000 7f5666ffd640 1 -- 172.21.15.99:0/2031019378 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x5635eef8b120 con 0x5635ef02eef0 2024-03-19T01:56:05.565 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.560+0000 7f5666ffd640 10 client.26371 handle_command_reply: tid=0 2024-03-19T01:56:05.566 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.561+0000 7f5690804740 1 client.26371 shutdown 2024-03-19T01:56:05.566 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.561+0000 7f5690804740 1 -- 172.21.15.99:0/2031019378 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5635ef019100 msgr2=0x5635ef0230e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:56:05.566 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.561+0000 7f5690804740 1 --2- 172.21.15.99:0/2031019378 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5635ef019100 0x5635ef0230e0 secure :-1 s=READY pgs=115 cs=0 l=1 rev1=1 crypto rx=0x7f567c01e410 tx=0x7f567c020040 comp rx=0 tx=0).stop 2024-03-19T01:56:05.566 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.561+0000 7f5690804740 1 -- 172.21.15.99:0/2031019378 shutdown_connections 2024-03-19T01:56:05.566 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.561+0000 7f5690804740 1 --2- 172.21.15.99:0/2031019378 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5635ef02eef0 0x5635ef031360 crc :-1 s=READY pgs=163 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:05.566 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.561+0000 7f5690804740 1 --2- 172.21.15.99:0/2031019378 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5635ef019af0 0x5635ef023620 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:05.566 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.561+0000 7f5690804740 1 --2- 172.21.15.99:0/2031019378 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5635ef01ec60 0x5635ef02a660 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:05.566 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.561+0000 7f5666ffd640 0 client.26371 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:56:05.566 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.561+0000 7f5690804740 1 --2- 172.21.15.99:0/2031019378 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5635ef019100 0x5635ef0230e0 unknown :-1 s=CLOSED pgs=115 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:05.566 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.561+0000 7f5690804740 1 -- 172.21.15.99:0/2031019378 >> 172.21.15.99:0/2031019378 conn(0x5635ef014670 msgr2=0x5635ef0158c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:56:05.566 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.561+0000 7f5690804740 1 -- 172.21.15.99:0/2031019378 shutdown_connections 2024-03-19T01:56:05.566 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.561+0000 7f5690804740 1 -- 172.21.15.99:0/2031019378 wait complete. 2024-03-19T01:56:05.566 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.561+0000 7f5690804740 20 client.26371 trim_cache size 0 max 16384 2024-03-19T01:56:05.580 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.578+0000 7f5690804740 1 Processor -- start 2024-03-19T01:56:05.580 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.578+0000 7f5690804740 1 -- start start 2024-03-19T01:56:05.580 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.578+0000 7f5690804740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5635ef019100 0x5635ef022700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:05.580 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.578+0000 7f5690804740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5635ef019af0 0x5635ef02a620 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:05.580 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.578+0000 7f5690804740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5635ef01ec60 0x5635ef022c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:05.580 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.578+0000 7f5690804740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f56881a9410 con 0x5635ef01ec60 2024-03-19T01:56:05.581 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.578+0000 7f5690804740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f56881a6ce0 con 0x5635ef019100 2024-03-19T01:56:05.581 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.578+0000 7f5690804740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f56881a4b30 con 0x5635ef019af0 2024-03-19T01:56:05.581 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.578+0000 7f5686ffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5635ef01ec60 0x5635ef022c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:05.581 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.578+0000 7f56877fe640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5635ef019af0 0x5635ef02a620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:05.581 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.578+0000 7f5686ffd640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5635ef019100 0x5635ef022700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:05.581 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.578+0000 7f56877fe640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5635ef019af0 0x5635ef02a620 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:38974/0 (socket says 172.21.15.99:38974) 2024-03-19T01:56:05.581 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.578+0000 7f56877fe640 1 -- 172.21.15.99:0/4170397706 learned_addr learned my addr 172.21.15.99:0/4170397706 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:56:05.581 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.579+0000 7f5686ffd640 1 -- 172.21.15.99:0/4170397706 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5635ef019af0 msgr2=0x5635ef02a620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:56:05.581 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.579+0000 7f5686ffd640 1 --2- 172.21.15.99:0/4170397706 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5635ef019af0 0x5635ef02a620 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:05.581 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.579+0000 7f5686ffd640 1 -- 172.21.15.99:0/4170397706 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5635ef01ec60 msgr2=0x5635ef022c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:56:05.581 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.579+0000 7f5686ffd640 1 --2- 172.21.15.99:0/4170397706 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5635ef01ec60 0x5635ef022c40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:05.581 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.579+0000 7f5686ffd640 1 -- 172.21.15.99:0/4170397706 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5635eed5e620 con 0x5635ef019100 2024-03-19T01:56:05.581 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.579+0000 7f5686ffd640 1 --2- 172.21.15.99:0/4170397706 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5635ef019100 0x5635ef022700 secure :-1 s=READY pgs=170 cs=0 l=1 rev1=1 crypto rx=0x7f567c02db00 tx=0x7f567c000bc0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=cde1c3cc0a7760a4 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:56:05.581 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.579+0000 7f566d7fa640 1 -- 172.21.15.99:0/4170397706 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f567c010b00 con 0x5635ef019100 2024-03-19T01:56:05.582 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.579+0000 7f566d7fa640 1 -- 172.21.15.99:0/4170397706 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f567c039020 con 0x5635ef019100 2024-03-19T01:56:05.582 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.579+0000 7f566d7fa640 1 -- 172.21.15.99:0/4170397706 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f567c004ce0 con 0x5635ef019100 2024-03-19T01:56:05.585 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.580+0000 7f5690804740 1 -- 172.21.15.99:0/4170397706 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5635ef019100 msgr2=0x5635ef022700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:56:05.585 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.580+0000 7f5690804740 1 --2- 172.21.15.99:0/4170397706 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5635ef019100 0x5635ef022700 secure :-1 s=READY pgs=170 cs=0 l=1 rev1=1 crypto rx=0x7f567c02db00 tx=0x7f567c000bc0 comp rx=0 tx=0).stop 2024-03-19T01:56:05.585 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.580+0000 7f5690804740 1 -- 172.21.15.99:0/4170397706 shutdown_connections 2024-03-19T01:56:05.585 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.580+0000 7f5690804740 1 --2- 172.21.15.99:0/4170397706 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5635ef01ec60 0x5635ef022c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:05.585 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.580+0000 7f5690804740 1 --2- 172.21.15.99:0/4170397706 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5635ef019af0 0x5635ef02a620 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:05.585 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.580+0000 7f5690804740 1 --2- 172.21.15.99:0/4170397706 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5635ef019100 0x5635ef022700 unknown :-1 s=CLOSED pgs=170 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:05.585 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.580+0000 7f5690804740 1 -- 172.21.15.99:0/4170397706 >> 172.21.15.99:0/4170397706 conn(0x5635ef014670 msgr2=0x5635ef01ae20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:56:05.585 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.580+0000 7f5690804740 1 -- 172.21.15.99:0/4170397706 shutdown_connections 2024-03-19T01:56:05.585 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.580+0000 7f5690804740 1 -- 172.21.15.99:0/4170397706 wait complete. 2024-03-19T01:56:05.586 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.580+0000 7f5690804740 1 -- start start 2024-03-19T01:56:05.586 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.580+0000 7f5690804740 1 Processor -- start 2024-03-19T01:56:05.586 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.581+0000 7f5690804740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5635ef019100 0x5635ef028260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:05.586 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.581+0000 7f5690804740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5635ef019af0 0x5635ef0287a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:05.586 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.581+0000 7f5690804740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5635ef01ec60 0x5635ef0297b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:05.586 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.581+0000 7f5690804740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f56881a51a0 con 0x5635ef019100 2024-03-19T01:56:05.586 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.581+0000 7f5690804740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f56881a6610 con 0x5635ef01ec60 2024-03-19T01:56:05.586 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.581+0000 7f5690804740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f56881a5870 con 0x5635ef019af0 2024-03-19T01:56:05.586 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.581+0000 7f5690804740 10 client.0 _ll_register_callbacks cb 0x7f567c01e050 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:56:05.586 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.581+0000 7f5686ffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5635ef019100 0x5635ef028260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:05.586 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.581+0000 7f5686ffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5635ef019100 0x5635ef028260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:55782/0 (socket says 172.21.15.99:55782) 2024-03-19T01:56:05.586 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.581+0000 7f5686ffd640 1 -- 172.21.15.99:0/2649720589 learned_addr learned my addr 172.21.15.99:0/2649720589 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:56:05.586 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.581+0000 7f56877fe640 1 --2- 172.21.15.99:0/2649720589 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5635ef019af0 0x5635ef0287a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:05.586 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.581+0000 7f5686ffd640 1 --2- 172.21.15.99:0/2649720589 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5635ef01ec60 0x5635ef0297b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:05.586 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.581+0000 7f56877fe640 1 -- 172.21.15.99:0/2649720589 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5635ef01ec60 msgr2=0x5635ef0297b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:56:05.587 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.581+0000 7f56877fe640 1 --2- 172.21.15.99:0/2649720589 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5635ef01ec60 0x5635ef0297b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:05.587 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.581+0000 7f56877fe640 1 -- 172.21.15.99:0/2649720589 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5635ef019100 msgr2=0x5635ef028260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:56:05.587 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.581+0000 7f56877fe640 1 --2- 172.21.15.99:0/2649720589 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5635ef019100 0x5635ef028260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:05.587 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.581+0000 7f56877fe640 1 -- 172.21.15.99:0/2649720589 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5635ef029da0 con 0x5635ef019af0 2024-03-19T01:56:05.587 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.581+0000 7f5686ffd640 1 --2- 172.21.15.99:0/2649720589 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5635ef019100 0x5635ef028260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:56:05.587 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.581+0000 7f56877fe640 1 --2- 172.21.15.99:0/2649720589 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5635ef019af0 0x5635ef0287a0 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7f5678009b40 tx=0x7f5678009310 comp rx=0 tx=0).ready entity=mon.1 client_cookie=8ca6fee11cb52bdb server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:56:05.587 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.581+0000 7f5666ffd640 10 client.0 ms_handle_connect on v2:172.21.15.115:3300/0 2024-03-19T01:56:05.587 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.581+0000 7f5666ffd640 1 -- 172.21.15.99:0/2649720589 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5678034060 con 0x5635ef019af0 2024-03-19T01:56:05.587 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.581+0000 7f5690804740 1 -- 172.21.15.99:0/2649720589 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f567c039020 con 0x5635ef019af0 2024-03-19T01:56:05.587 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.582+0000 7f5666ffd640 1 -- 172.21.15.99:0/2649720589 <== mon.1 v2:172.21.15.115:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f567802e2d0 con 0x5635ef019af0 2024-03-19T01:56:05.587 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.582+0000 7f5666ffd640 1 -- 172.21.15.99:0/2649720589 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7f567802e4d0 con 0x5635ef019af0 2024-03-19T01:56:05.587 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.582+0000 7f5690804740 10 client.26377 fetch_fsmap learned FSMap version 37 2024-03-19T01:56:05.587 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.582+0000 7f5690804740 1 -- 172.21.15.99:0/2649720589 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x5635ef0958b0 con 0x5635ef019af0 2024-03-19T01:56:05.587 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.582+0000 7f5666ffd640 1 -- 172.21.15.99:0/2649720589 <== mon.1 v2:172.21.15.115:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f567c039020 con 0x5635ef019af0 2024-03-19T01:56:05.587 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.582+0000 7f5666ffd640 1 -- 172.21.15.99:0/2649720589 <== mon.1 v2:172.21.15.115:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7f5678003710 con 0x5635ef019af0 2024-03-19T01:56:05.587 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.582+0000 7f5690804740 10 client.26377 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T01:56:05.588 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.582+0000 7f5690804740 10 client.26377 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:56:05.588 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.582+0000 7f5690804740 20 client.26377 populate_metadata read hostname 'smithi099' 2024-03-19T01:56:05.588 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.582+0000 7f5690804740 1 --2- 172.21.15.99:0/2649720589 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5635ef02eef0 0x5635ef0980b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:05.588 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.582+0000 7f5690804740 4 client.26377 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:56:05.588 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.582+0000 7f5690804740 1 -- 172.21.15.99:0/2649720589 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x5635eef85470 con 0x5635ef02eef0 2024-03-19T01:56:05.588 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.582+0000 7f5686ffd640 1 --2- 172.21.15.99:0/2649720589 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5635ef02eef0 0x5635ef0980b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:05.588 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.583+0000 7f5686ffd640 1 --2- 172.21.15.99:0/2649720589 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5635ef02eef0 0x5635ef0980b0 crc :-1 s=READY pgs=164 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=b64774dd4177e8b1 server_cookie=5d2eaa7250b742c7 in_seq=0 out_seq=0 2024-03-19T01:56:05.588 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.583+0000 7f5666ffd640 10 client.26377 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:56:05.949 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.947+0000 7f5666ffd640 1 -- 172.21.15.99:0/2649720589 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x5635eef85470 con 0x5635ef02eef0 2024-03-19T01:56:05.949 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.947+0000 7f5666ffd640 10 client.26377 handle_command_reply: tid=0 2024-03-19T01:56:05.949 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.947+0000 7f5690804740 1 client.26377 shutdown 2024-03-19T01:56:05.949 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.948+0000 7f5690804740 1 -- 172.21.15.99:0/2649720589 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5635ef019af0 msgr2=0x5635ef0287a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:56:05.949 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.948+0000 7f5690804740 1 --2- 172.21.15.99:0/2649720589 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5635ef019af0 0x5635ef0287a0 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7f5678009b40 tx=0x7f5678009310 comp rx=0 tx=0).stop 2024-03-19T01:56:05.949 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.948+0000 7f5690804740 1 -- 172.21.15.99:0/2649720589 shutdown_connections 2024-03-19T01:56:05.949 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.948+0000 7f5690804740 1 --2- 172.21.15.99:0/2649720589 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5635ef02eef0 0x5635ef0980b0 crc :-1 s=READY pgs=164 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:05.949 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.948+0000 7f5690804740 1 --2- 172.21.15.99:0/2649720589 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5635ef01ec60 0x5635ef0297b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:05.949 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.948+0000 7f5666ffd640 0 client.26377 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:56:05.950 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.948+0000 7f5690804740 1 --2- 172.21.15.99:0/2649720589 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5635ef019af0 0x5635ef0287a0 unknown :-1 s=CLOSED pgs=116 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:05.950 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.948+0000 7f5690804740 1 --2- 172.21.15.99:0/2649720589 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5635ef019100 0x5635ef028260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:05.950 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.948+0000 7f5690804740 1 -- 172.21.15.99:0/2649720589 >> 172.21.15.99:0/2649720589 conn(0x5635ef014670 msgr2=0x5635ef0178d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:56:05.950 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.948+0000 7f5690804740 1 -- 172.21.15.99:0/2649720589 shutdown_connections 2024-03-19T01:56:05.950 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.948+0000 7f5690804740 1 -- 172.21.15.99:0/2649720589 wait complete. 2024-03-19T01:56:05.950 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.948+0000 7f5690804740 20 client.26377 trim_cache size 0 max 16384 2024-03-19T01:56:05.950 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:56:05.950 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:56:05.950 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:56:05.951 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:56:05.951 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.948+0000 7f568db3b640 1 -- 172.21.15.99:0/1512797317 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f56700a28a0 msgr2=0x7f56700a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:56:05.951 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.948+0000 7f568db3b640 1 --2- 172.21.15.99:0/1512797317 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f56700a28a0 0x7f56700a4d60 secure :-1 s=READY pgs=340 cs=0 l=1 rev1=1 crypto rx=0x7f56800c5490 tx=0x7f568002e040 comp rx=0 tx=0).stop 2024-03-19T01:56:05.952 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.948+0000 7f568db3b640 1 -- 172.21.15.99:0/1512797317 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f56881033e0 msgr2=0x7f5688074150 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:56:05.952 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.948+0000 7f568db3b640 1 --2- 172.21.15.99:0/1512797317 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f56881033e0 0x7f5688074150 secure :-1 s=READY pgs=639 cs=0 l=1 rev1=1 crypto rx=0x7f5680037040 tx=0x7f5680032f40 comp rx=0 tx=0).stop 2024-03-19T01:56:05.952 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.948+0000 7f568db3b640 1 -- 172.21.15.99:0/1512797317 shutdown_connections 2024-03-19T01:56:05.952 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.948+0000 7f568db3b640 1 --2- 172.21.15.99:0/1512797317 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f56700a28a0 0x7f56700a4d60 unknown :-1 s=CLOSED pgs=340 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:05.952 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.948+0000 7f568db3b640 1 --2- 172.21.15.99:0/1512797317 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5688102a20 0x7f5688073c10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:05.952 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.948+0000 7f568db3b640 1 --2- 172.21.15.99:0/1512797317 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f56881033e0 0x7f5688074150 unknown :-1 s=CLOSED pgs=639 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:05.952 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.948+0000 7f568db3b640 1 --2- 172.21.15.99:0/1512797317 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5688101830 0x7f56880755d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:05.952 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.948+0000 7f568db3b640 1 -- 172.21.15.99:0/1512797317 >> 172.21.15.99:0/1512797317 conn(0x7f56880fcda0 msgr2=0x7f5688105c50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:56:05.952 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.949+0000 7f568db3b640 1 -- 172.21.15.99:0/1512797317 shutdown_connections 2024-03-19T01:56:05.952 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:05.949+0000 7f568db3b640 1 -- 172.21.15.99:0/1512797317 wait complete. 2024-03-19T01:56:06.103 INFO:tasks.workunit.client.0.smithi099.stdout: 10 69570 23.16 MB/sec execute 35 sec latency 227.889 ms 2024-03-19T01:56:07.103 INFO:tasks.workunit.client.0.smithi099.stdout: 10 69909 22.81 MB/sec execute 36 sec latency 225.624 ms 2024-03-19T01:56:07.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:56:07 smithi099 ceph-mon[62951]: pgmap v802: 97 pgs: 97 active+clean; 532 MiB data, 5.7 GiB used, 1.0 TiB / 1.0 TiB avail; 5.2 MiB/s rd, 6.9 MiB/s wr, 648 op/s 2024-03-19T01:56:07.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:56:07 smithi112 ceph-mon[66819]: pgmap v802: 97 pgs: 97 active+clean; 532 MiB data, 5.7 GiB used, 1.0 TiB / 1.0 TiB avail; 5.2 MiB/s rd, 6.9 MiB/s wr, 648 op/s 2024-03-19T01:56:07.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:56:07 smithi115 ceph-mon[67332]: pgmap v802: 97 pgs: 97 active+clean; 532 MiB data, 5.7 GiB used, 1.0 TiB / 1.0 TiB avail; 5.2 MiB/s rd, 6.9 MiB/s wr, 648 op/s 2024-03-19T01:56:08.103 INFO:tasks.workunit.client.0.smithi099.stdout: 10 70304 22.84 MB/sec execute 37 sec latency 93.280 ms 2024-03-19T01:56:09.103 INFO:tasks.workunit.client.0.smithi099.stdout: 10 70817 23.08 MB/sec execute 38 sec latency 84.959 ms 2024-03-19T01:56:09.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:56:09 smithi099 ceph-mon[62951]: pgmap v803: 97 pgs: 97 active+clean; 532 MiB data, 5.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.3 MiB/s rd, 5.8 MiB/s wr, 521 op/s 2024-03-19T01:56:09.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:56:09 smithi112 ceph-mon[66819]: pgmap v803: 97 pgs: 97 active+clean; 532 MiB data, 5.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.3 MiB/s rd, 5.8 MiB/s wr, 521 op/s 2024-03-19T01:56:09.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:56:09 smithi115 ceph-mon[67332]: pgmap v803: 97 pgs: 97 active+clean; 532 MiB data, 5.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.3 MiB/s rd, 5.8 MiB/s wr, 521 op/s 2024-03-19T01:56:10.103 INFO:tasks.workunit.client.0.smithi099.stdout: 10 71274 23.11 MB/sec execute 39 sec latency 59.563 ms 2024-03-19T01:56:11.103 INFO:tasks.workunit.client.0.smithi099.stdout: 10 71773 23.24 MB/sec execute 40 sec latency 52.534 ms 2024-03-19T01:56:11.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:56:11 smithi099 ceph-mon[62951]: pgmap v804: 97 pgs: 97 active+clean; 520 MiB data, 5.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.4 MiB/s rd, 6.1 MiB/s wr, 552 op/s 2024-03-19T01:56:11.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:56:11 smithi112 ceph-mon[66819]: pgmap v804: 97 pgs: 97 active+clean; 520 MiB data, 5.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.4 MiB/s rd, 6.1 MiB/s wr, 552 op/s 2024-03-19T01:56:11.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:56:11 smithi115 ceph-mon[67332]: pgmap v804: 97 pgs: 97 active+clean; 520 MiB data, 5.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.4 MiB/s rd, 6.1 MiB/s wr, 552 op/s 2024-03-19T01:56:12.103 INFO:tasks.workunit.client.0.smithi099.stdout: 10 72059 23.01 MB/sec execute 41 sec latency 283.775 ms 2024-03-19T01:56:13.103 INFO:tasks.workunit.client.0.smithi099.stdout: 10 72412 22.81 MB/sec execute 42 sec latency 152.283 ms 2024-03-19T01:56:13.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:56:13 smithi099 ceph-mon[62951]: pgmap v805: 97 pgs: 97 active+clean; 524 MiB data, 5.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.8 MiB/s rd, 6.2 MiB/s wr, 589 op/s 2024-03-19T01:56:13.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:56:13 smithi112 ceph-mon[66819]: pgmap v805: 97 pgs: 97 active+clean; 524 MiB data, 5.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.8 MiB/s rd, 6.2 MiB/s wr, 589 op/s 2024-03-19T01:56:13.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:56:13 smithi115 ceph-mon[67332]: pgmap v805: 97 pgs: 97 active+clean; 524 MiB data, 5.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.8 MiB/s rd, 6.2 MiB/s wr, 589 op/s 2024-03-19T01:56:14.103 INFO:tasks.workunit.client.0.smithi099.stdout: 10 72896 22.81 MB/sec execute 43 sec latency 66.042 ms 2024-03-19T01:56:15.103 INFO:tasks.workunit.client.0.smithi099.stdout: 10 73381 22.82 MB/sec execute 44 sec latency 124.391 ms 2024-03-19T01:56:15.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:56:15 smithi099 ceph-mon[62951]: pgmap v806: 97 pgs: 97 active+clean; 528 MiB data, 5.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.4 MiB/s rd, 5.1 MiB/s wr, 534 op/s 2024-03-19T01:56:15.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:56:15 smithi112 ceph-mon[66819]: pgmap v806: 97 pgs: 97 active+clean; 528 MiB data, 5.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.4 MiB/s rd, 5.1 MiB/s wr, 534 op/s 2024-03-19T01:56:15.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:56:15 smithi115 ceph-mon[67332]: pgmap v806: 97 pgs: 97 active+clean; 528 MiB data, 5.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.4 MiB/s rd, 5.1 MiB/s wr, 534 op/s 2024-03-19T01:56:16.103 INFO:tasks.workunit.client.0.smithi099.stdout: 10 74058 23.23 MB/sec execute 45 sec latency 52.431 ms 2024-03-19T01:56:17.103 INFO:tasks.workunit.client.0.smithi099.stdout: 10 74453 23.40 MB/sec execute 46 sec latency 291.125 ms 2024-03-19T01:56:17.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:56:17 smithi099 ceph-mon[62951]: pgmap v807: 97 pgs: 97 active+clean; 535 MiB data, 5.7 GiB used, 1.0 TiB / 1.0 TiB avail; 5.2 MiB/s rd, 6.7 MiB/s wr, 630 op/s 2024-03-19T01:56:17.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:56:17 smithi112 ceph-mon[66819]: pgmap v807: 97 pgs: 97 active+clean; 535 MiB data, 5.7 GiB used, 1.0 TiB / 1.0 TiB avail; 5.2 MiB/s rd, 6.7 MiB/s wr, 630 op/s 2024-03-19T01:56:17.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:56:17 smithi115 ceph-mon[67332]: pgmap v807: 97 pgs: 97 active+clean; 535 MiB data, 5.7 GiB used, 1.0 TiB / 1.0 TiB avail; 5.2 MiB/s rd, 6.7 MiB/s wr, 630 op/s 2024-03-19T01:56:18.103 INFO:tasks.workunit.client.0.smithi099.stdout: 10 74857 23.28 MB/sec execute 47 sec latency 154.320 ms 2024-03-19T01:56:19.049 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.046+0000 7fa36b59e640 1 -- 172.21.15.99:0/3173973837 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa36c102120 msgr2=0x7fa36c1089b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:56:19.049 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.046+0000 7fa36b59e640 1 --2- 172.21.15.99:0/3173973837 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa36c102120 0x7fa36c1089b0 secure :-1 s=READY pgs=640 cs=0 l=1 rev1=1 crypto rx=0x7fa364009ca0 tx=0x7fa36402c0f0 comp rx=0 tx=0).stop 2024-03-19T01:56:19.049 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.046+0000 7fa36b59e640 1 -- 172.21.15.99:0/3173973837 shutdown_connections 2024-03-19T01:56:19.049 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.046+0000 7fa36b59e640 1 --2- 172.21.15.99:0/3173973837 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa36c102120 0x7fa36c1089b0 unknown :-1 s=CLOSED pgs=640 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:19.049 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.046+0000 7fa36b59e640 1 --2- 172.21.15.99:0/3173973837 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa36c101760 0x7fa36c101be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:19.050 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.046+0000 7fa36b59e640 1 --2- 172.21.15.99:0/3173973837 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa36c100570 0x7fa36c100990 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:19.050 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.046+0000 7fa36b59e640 1 -- 172.21.15.99:0/3173973837 >> 172.21.15.99:0/3173973837 conn(0x7fa36c0fbb00 msgr2=0x7fa36c0fdf20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:56:19.050 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.046+0000 7fa36b59e640 1 -- 172.21.15.99:0/3173973837 shutdown_connections 2024-03-19T01:56:19.050 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.046+0000 7fa36b59e640 1 -- 172.21.15.99:0/3173973837 wait complete. 2024-03-19T01:56:19.050 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.046+0000 7fa36b59e640 1 Processor -- start 2024-03-19T01:56:19.050 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.047+0000 7fa36b59e640 1 -- start start 2024-03-19T01:56:19.050 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.047+0000 7fa36b59e640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa36c100570 0x7fa36c197900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:19.050 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.047+0000 7fa36b59e640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa36c101760 0x7fa36c197e40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:19.050 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.047+0000 7fa36b59e640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa36c102120 0x7fa36c19eeb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:19.050 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.047+0000 7fa36b59e640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fa36c10c200 con 0x7fa36c100570 2024-03-19T01:56:19.050 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.047+0000 7fa36b59e640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fa36c10c080 con 0x7fa36c102120 2024-03-19T01:56:19.051 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.047+0000 7fa36b59e640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fa36c10c380 con 0x7fa36c101760 2024-03-19T01:56:19.051 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.047+0000 7fa36a59c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa36c100570 0x7fa36c197900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:19.051 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.047+0000 7fa36a59c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa36c100570 0x7fa36c197900 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:45688/0 (socket says 172.21.15.99:45688) 2024-03-19T01:56:19.051 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.047+0000 7fa36a59c640 1 -- 172.21.15.99:0/3602891330 learned_addr learned my addr 172.21.15.99:0/3602891330 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:56:19.051 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.047+0000 7fa36ad9d640 1 --2- 172.21.15.99:0/3602891330 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa36c102120 0x7fa36c19eeb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:19.051 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.047+0000 7fa363fff640 1 --2- 172.21.15.99:0/3602891330 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa36c101760 0x7fa36c197e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:19.051 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.047+0000 7fa36a59c640 1 -- 172.21.15.99:0/3602891330 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa36c101760 msgr2=0x7fa36c197e40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:56:19.051 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.047+0000 7fa36a59c640 1 --2- 172.21.15.99:0/3602891330 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa36c101760 0x7fa36c197e40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:19.052 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.047+0000 7fa36a59c640 1 -- 172.21.15.99:0/3602891330 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa36c102120 msgr2=0x7fa36c19eeb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:56:19.052 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.047+0000 7fa36a59c640 1 --2- 172.21.15.99:0/3602891330 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa36c102120 0x7fa36c19eeb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:19.052 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.047+0000 7fa36a59c640 1 -- 172.21.15.99:0/3602891330 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa36c19f520 con 0x7fa36c100570 2024-03-19T01:56:19.052 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.047+0000 7fa36a59c640 1 --2- 172.21.15.99:0/3602891330 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa36c100570 0x7fa36c197900 secure :-1 s=READY pgs=641 cs=0 l=1 rev1=1 crypto rx=0x7fa35c00e9c0 tx=0x7fa35c00ee80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:56:19.052 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.048+0000 7fa3637fe640 1 -- 172.21.15.99:0/3602891330 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa35c00cd80 con 0x7fa36c100570 2024-03-19T01:56:19.052 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.048+0000 7fa36b59e640 1 -- 172.21.15.99:0/3602891330 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa36c19f810 con 0x7fa36c100570 2024-03-19T01:56:19.052 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.048+0000 7fa3637fe640 1 -- 172.21.15.99:0/3602891330 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa35c0035f0 con 0x7fa36c100570 2024-03-19T01:56:19.052 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.048+0000 7fa3637fe640 1 -- 172.21.15.99:0/3602891330 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa35c010690 con 0x7fa36c100570 2024-03-19T01:56:19.052 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.048+0000 7fa36b59e640 1 -- 172.21.15.99:0/3602891330 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fa36c19fd60 con 0x7fa36c100570 2024-03-19T01:56:19.060 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.050+0000 7fa3739fa740 1 Processor -- start 2024-03-19T01:56:19.060 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.050+0000 7fa3739fa740 1 -- start start 2024-03-19T01:56:19.060 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.050+0000 7fa3637fe640 1 -- 172.21.15.99:0/3602891330 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7fa35c010830 con 0x7fa36c100570 2024-03-19T01:56:19.060 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.051+0000 7fa3739fa740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56003248d2e0 0x56003248d700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:19.060 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.051+0000 7fa3739fa740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56003248dcd0 0x5600324927d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:19.060 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.051+0000 7fa3739fa740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560032492e40 0x5600324952d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:19.060 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.051+0000 7fa3739fa740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x5600323f1be0 con 0x56003248dcd0 2024-03-19T01:56:19.060 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.051+0000 7fa3739fa740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x56003239a960 con 0x560032492e40 2024-03-19T01:56:19.060 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.051+0000 7fa3739fa740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x5600323edc80 con 0x56003248d2e0 2024-03-19T01:56:19.060 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.051+0000 7fa36a59c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56003248dcd0 0x5600324927d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:19.061 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.051+0000 7fa36ad9d640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56003248d2e0 0x56003248d700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:19.061 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.051+0000 7fa36a59c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56003248dcd0 0x5600324927d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:45698/0 (socket says 172.21.15.99:45698) 2024-03-19T01:56:19.061 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.051+0000 7fa36a59c640 1 -- 172.21.15.99:0/2629836651 learned_addr learned my addr 172.21.15.99:0/2629836651 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:56:19.061 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.051+0000 7fa363fff640 1 --2- 172.21.15.99:0/2629836651 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560032492e40 0x5600324952d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:19.061 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.051+0000 7fa36a59c640 1 -- 172.21.15.99:0/2629836651 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56003248d2e0 msgr2=0x56003248d700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:56:19.061 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.051+0000 7fa36a59c640 1 --2- 172.21.15.99:0/2629836651 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56003248d2e0 0x56003248d700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:19.061 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.051+0000 7fa36a59c640 1 -- 172.21.15.99:0/2629836651 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560032492e40 msgr2=0x5600324952d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:56:19.061 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.051+0000 7fa36a59c640 1 --2- 172.21.15.99:0/2629836651 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560032492e40 0x5600324952d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:19.061 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.051+0000 7fa36a59c640 1 -- 172.21.15.99:0/2629836651 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x560032391d20 con 0x56003248dcd0 2024-03-19T01:56:19.061 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.051+0000 7fa36a59c640 1 --2- 172.21.15.99:0/2629836651 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56003248dcd0 0x5600324927d0 secure :-1 s=READY pgs=642 cs=0 l=1 rev1=1 crypto rx=0x7fa35c0d2e10 tx=0x7fa35c0d74b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=fb7b33aa3d1102e1 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:56:19.061 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.051+0000 7fa3637fe640 1 --2- 172.21.15.99:0/3602891330 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa3580a27f0 0x7fa3580a4cb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:19.061 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.051+0000 7fa360ff9640 1 -- 172.21.15.99:0/2629836651 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa35c0e2070 con 0x56003248dcd0 2024-03-19T01:56:19.062 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.052+0000 7fa36ad9d640 1 --2- 172.21.15.99:0/3602891330 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa3580a27f0 0x7fa3580a4cb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:19.062 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.052+0000 7fa3637fe640 1 -- 172.21.15.99:0/3602891330 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fa35c0cee90 con 0x7fa36c100570 2024-03-19T01:56:19.062 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.052+0000 7fa360ff9640 1 -- 172.21.15.99:0/2629836651 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa35c09bef0 con 0x56003248dcd0 2024-03-19T01:56:19.062 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.052+0000 7fa360ff9640 1 -- 172.21.15.99:0/2629836651 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa35c09b5c0 con 0x56003248dcd0 2024-03-19T01:56:19.062 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.052+0000 7fa3739fa740 1 -- 172.21.15.99:0/2629836651 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56003248dcd0 msgr2=0x5600324927d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:56:19.062 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.052+0000 7fa3739fa740 1 --2- 172.21.15.99:0/2629836651 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56003248dcd0 0x5600324927d0 secure :-1 s=READY pgs=642 cs=0 l=1 rev1=1 crypto rx=0x7fa35c0d2e10 tx=0x7fa35c0d74b0 comp rx=0 tx=0).stop 2024-03-19T01:56:19.062 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.052+0000 7fa36ad9d640 1 --2- 172.21.15.99:0/3602891330 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa3580a27f0 0x7fa3580a4cb0 secure :-1 s=READY pgs=341 cs=0 l=1 rev1=1 crypto rx=0x7fa364005e30 tx=0x7fa364032590 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:56:19.062 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.052+0000 7fa3739fa740 1 -- 172.21.15.99:0/2629836651 shutdown_connections 2024-03-19T01:56:19.062 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.052+0000 7fa3739fa740 1 --2- 172.21.15.99:0/2629836651 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560032492e40 0x5600324952d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:19.062 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.052+0000 7fa3739fa740 1 --2- 172.21.15.99:0/2629836651 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56003248dcd0 0x5600324927d0 unknown :-1 s=CLOSED pgs=642 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:19.062 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.052+0000 7fa3739fa740 1 --2- 172.21.15.99:0/2629836651 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56003248d2e0 0x56003248d700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:19.063 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.052+0000 7fa3739fa740 1 -- 172.21.15.99:0/2629836651 >> 172.21.15.99:0/2629836651 conn(0x560032488850 msgr2=0x560032488c50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:56:19.063 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.052+0000 7fa3739fa740 1 -- 172.21.15.99:0/2629836651 shutdown_connections 2024-03-19T01:56:19.063 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.053+0000 7fa3739fa740 1 -- 172.21.15.99:0/2629836651 wait complete. 2024-03-19T01:56:19.063 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.053+0000 7fa3739fa740 1 -- start start 2024-03-19T01:56:19.063 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.053+0000 7fa3739fa740 1 Processor -- start 2024-03-19T01:56:19.063 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.053+0000 7fa3739fa740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56003248d2e0 0x5600324972e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:19.063 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.053+0000 7fa3739fa740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56003248dcd0 0x560032497820 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:19.063 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.053+0000 7fa3739fa740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560032492e40 0x56003249e860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:19.063 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.053+0000 7fa3739fa740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x560032495df0 con 0x56003248d2e0 2024-03-19T01:56:19.063 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.053+0000 7fa3739fa740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x5600323addb0 con 0x56003248dcd0 2024-03-19T01:56:19.063 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.053+0000 7fa3739fa740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x560032495f70 con 0x560032492e40 2024-03-19T01:56:19.063 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.053+0000 7fa3739fa740 10 client.0 _ll_register_callbacks cb 0x5600323b2e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:56:19.064 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.053+0000 7fa36a59c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56003248d2e0 0x5600324972e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:19.064 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.053+0000 7fa36a59c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56003248d2e0 0x5600324972e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:45714/0 (socket says 172.21.15.99:45714) 2024-03-19T01:56:19.064 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.053+0000 7fa36a59c640 1 -- 172.21.15.99:0/1428253417 learned_addr learned my addr 172.21.15.99:0/1428253417 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:56:19.064 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.053+0000 7fa363fff640 1 --2- 172.21.15.99:0/1428253417 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56003248dcd0 0x560032497820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:19.064 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.053+0000 7fa36ad9d640 1 --2- 172.21.15.99:0/1428253417 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560032492e40 0x56003249e860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:19.064 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.053+0000 7fa36a59c640 1 -- 172.21.15.99:0/1428253417 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560032492e40 msgr2=0x56003249e860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:56:19.064 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.053+0000 7fa36a59c640 1 --2- 172.21.15.99:0/1428253417 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560032492e40 0x56003249e860 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:19.064 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.053+0000 7fa36a59c640 1 -- 172.21.15.99:0/1428253417 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56003248dcd0 msgr2=0x560032497820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:56:19.064 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.053+0000 7fa36a59c640 1 --2- 172.21.15.99:0/1428253417 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56003248dcd0 0x560032497820 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:19.064 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.053+0000 7fa36a59c640 1 -- 172.21.15.99:0/1428253417 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5600323acc60 con 0x56003248d2e0 2024-03-19T01:56:19.064 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.053+0000 7fa36ad9d640 1 --2- 172.21.15.99:0/1428253417 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560032492e40 0x56003249e860 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-19T01:56:19.065 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.054+0000 7fa36a59c640 1 --2- 172.21.15.99:0/1428253417 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56003248d2e0 0x5600324972e0 secure :-1 s=READY pgs=643 cs=0 l=1 rev1=1 crypto rx=0x7fa35c0d2e10 tx=0x7fa35c003e30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=28ac144bd0212d57 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:56:19.065 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.054+0000 7fa351ffb640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:56:19.065 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.054+0000 7fa351ffb640 1 -- 172.21.15.99:0/1428253417 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa35c09b150 con 0x56003248d2e0 2024-03-19T01:56:19.065 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.054+0000 7fa351ffb640 1 -- 172.21.15.99:0/1428253417 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fa35c003b30 con 0x56003248d2e0 2024-03-19T01:56:19.065 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.054+0000 7fa3739fa740 1 -- 172.21.15.99:0/1428253417 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fa35c09bef0 con 0x56003248d2e0 2024-03-19T01:56:19.065 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.054+0000 7fa351ffb640 1 -- 172.21.15.99:0/1428253417 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7fa35c09b330 con 0x56003248d2e0 2024-03-19T01:56:19.065 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.054+0000 7fa3739fa740 10 client.16572 fetch_fsmap learned FSMap version 37 2024-03-19T01:56:19.065 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.054+0000 7fa3739fa740 1 -- 172.21.15.99:0/1428253417 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x560032106210 con 0x56003248d2e0 2024-03-19T01:56:19.065 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.054+0000 7fa351ffb640 1 -- 172.21.15.99:0/1428253417 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa35c09bef0 con 0x56003248d2e0 2024-03-19T01:56:19.065 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.054+0000 7fa351ffb640 1 -- 172.21.15.99:0/1428253417 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7fa35c0bb040 con 0x56003248d2e0 2024-03-19T01:56:19.065 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.054+0000 7fa3739fa740 10 client.16572 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T01:56:19.065 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.054+0000 7fa3739fa740 10 client.16572 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:56:19.066 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.054+0000 7fa3739fa740 20 client.16572 populate_metadata read hostname 'smithi099' 2024-03-19T01:56:19.066 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.054+0000 7fa3739fa740 1 --2- 172.21.15.99:0/1428253417 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5600324a2d50 0x5600324a5140 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:19.066 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.054+0000 7fa3739fa740 4 client.16572 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:56:19.066 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.054+0000 7fa3739fa740 1 -- 172.21.15.99:0/1428253417 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5600323ff120 con 0x5600324a2d50 2024-03-19T01:56:19.066 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.054+0000 7fa363fff640 1 --2- 172.21.15.99:0/1428253417 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5600324a2d50 0x5600324a5140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:19.066 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.055+0000 7fa363fff640 1 --2- 172.21.15.99:0/1428253417 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5600324a2d50 0x5600324a5140 crc :-1 s=READY pgs=165 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=d49ce86e404f2605 server_cookie=a6c4f91f806dd9dc in_seq=0 out_seq=0 2024-03-19T01:56:19.066 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.055+0000 7fa351ffb640 10 client.16572 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:56:19.066 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.058+0000 7fa351ffb640 1 -- 172.21.15.99:0/1428253417 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x5600323ff120 con 0x5600324a2d50 2024-03-19T01:56:19.066 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.058+0000 7fa351ffb640 10 client.16572 handle_command_reply: tid=0 2024-03-19T01:56:19.066 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.058+0000 7fa3739fa740 1 client.16572 shutdown 2024-03-19T01:56:19.066 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.059+0000 7fa3739fa740 1 -- 172.21.15.99:0/1428253417 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56003248d2e0 msgr2=0x5600324972e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:56:19.066 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.059+0000 7fa3739fa740 1 --2- 172.21.15.99:0/1428253417 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56003248d2e0 0x5600324972e0 secure :-1 s=READY pgs=643 cs=0 l=1 rev1=1 crypto rx=0x7fa35c0d2e10 tx=0x7fa35c003e30 comp rx=0 tx=0).stop 2024-03-19T01:56:19.067 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.059+0000 7fa3739fa740 1 -- 172.21.15.99:0/1428253417 shutdown_connections 2024-03-19T01:56:19.067 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.059+0000 7fa3739fa740 1 --2- 172.21.15.99:0/1428253417 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5600324a2d50 0x5600324a5140 crc :-1 s=READY pgs=165 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:19.067 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.059+0000 7fa3739fa740 1 --2- 172.21.15.99:0/1428253417 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56003248dcd0 0x560032497820 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:19.067 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.059+0000 7fa3739fa740 1 --2- 172.21.15.99:0/1428253417 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560032492e40 0x56003249e860 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:19.067 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.059+0000 7fa3739fa740 1 --2- 172.21.15.99:0/1428253417 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56003248d2e0 0x5600324972e0 unknown :-1 s=CLOSED pgs=643 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:19.067 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.059+0000 7fa351ffb640 0 client.16572 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:56:19.067 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.059+0000 7fa3739fa740 1 -- 172.21.15.99:0/1428253417 >> 172.21.15.99:0/1428253417 conn(0x560032488850 msgr2=0x560032487ab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:56:19.067 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.059+0000 7fa3739fa740 1 -- 172.21.15.99:0/1428253417 shutdown_connections 2024-03-19T01:56:19.067 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.059+0000 7fa3739fa740 1 -- 172.21.15.99:0/1428253417 wait complete. 2024-03-19T01:56:19.067 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.059+0000 7fa3739fa740 20 client.16572 trim_cache size 0 max 16384 2024-03-19T01:56:19.080 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.075+0000 7fa3739fa740 1 Processor -- start 2024-03-19T01:56:19.080 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.075+0000 7fa3739fa740 1 -- start start 2024-03-19T01:56:19.080 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.075+0000 7fa3739fa740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56003248d2e0 0x560032497780 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:19.080 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.075+0000 7fa3739fa740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56003248dcd0 0x56003249e7f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:19.080 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.075+0000 7fa3739fa740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560032492e40 0x56003248ee50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:19.080 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.075+0000 7fa3739fa740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fa36c1ab5c0 con 0x56003248dcd0 2024-03-19T01:56:19.080 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.075+0000 7fa3739fa740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fa36c1a8e90 con 0x56003248d2e0 2024-03-19T01:56:19.080 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.075+0000 7fa3739fa740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fa36c1a6ce0 con 0x560032492e40 2024-03-19T01:56:19.080 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.075+0000 7fa363fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56003248dcd0 0x56003249e7f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:19.081 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.075+0000 7fa363fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56003248dcd0 0x56003249e7f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:45716/0 (socket says 172.21.15.99:45716) 2024-03-19T01:56:19.081 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.075+0000 7fa363fff640 1 -- 172.21.15.99:0/2135376812 learned_addr learned my addr 172.21.15.99:0/2135376812 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:56:19.081 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.075+0000 7fa36a59c640 1 --2- 172.21.15.99:0/2135376812 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56003248d2e0 0x560032497780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:19.081 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.075+0000 7fa36ad9d640 1 --2- 172.21.15.99:0/2135376812 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560032492e40 0x56003248ee50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:19.081 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.076+0000 7fa363fff640 1 -- 172.21.15.99:0/2135376812 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560032492e40 msgr2=0x56003248ee50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:56:19.081 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.076+0000 7fa363fff640 1 --2- 172.21.15.99:0/2135376812 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560032492e40 0x56003248ee50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:19.081 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.076+0000 7fa363fff640 1 -- 172.21.15.99:0/2135376812 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56003248d2e0 msgr2=0x560032497780 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:56:19.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.076+0000 7fa363fff640 1 --2- 172.21.15.99:0/2135376812 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56003248d2e0 0x560032497780 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:19.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.076+0000 7fa363fff640 1 -- 172.21.15.99:0/2135376812 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5600321d2620 con 0x56003248dcd0 2024-03-19T01:56:19.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.076+0000 7fa363fff640 1 --2- 172.21.15.99:0/2135376812 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56003248dcd0 0x56003249e7f0 secure :-1 s=READY pgs=644 cs=0 l=1 rev1=1 crypto rx=0x5600324a57d0 tx=0x7fa354005f90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6bf5846a57bc7d45 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:56:19.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.076+0000 7fa353fff640 1 -- 172.21.15.99:0/2135376812 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa354024070 con 0x56003248dcd0 2024-03-19T01:56:19.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.076+0000 7fa353fff640 1 -- 172.21.15.99:0/2135376812 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa354004250 con 0x56003248dcd0 2024-03-19T01:56:19.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.076+0000 7fa353fff640 1 -- 172.21.15.99:0/2135376812 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa354003b50 con 0x56003248dcd0 2024-03-19T01:56:19.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.077+0000 7fa3739fa740 1 -- 172.21.15.99:0/2135376812 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56003248dcd0 msgr2=0x56003249e7f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:56:19.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.077+0000 7fa3739fa740 1 --2- 172.21.15.99:0/2135376812 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56003248dcd0 0x56003249e7f0 secure :-1 s=READY pgs=644 cs=0 l=1 rev1=1 crypto rx=0x5600324a57d0 tx=0x7fa354005f90 comp rx=0 tx=0).stop 2024-03-19T01:56:19.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.077+0000 7fa3739fa740 1 -- 172.21.15.99:0/2135376812 shutdown_connections 2024-03-19T01:56:19.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.077+0000 7fa3739fa740 1 --2- 172.21.15.99:0/2135376812 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560032492e40 0x56003248ee50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:19.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.077+0000 7fa3739fa740 1 --2- 172.21.15.99:0/2135376812 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56003248dcd0 0x56003249e7f0 unknown :-1 s=CLOSED pgs=644 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:19.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.077+0000 7fa3739fa740 1 --2- 172.21.15.99:0/2135376812 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56003248d2e0 0x560032497780 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:19.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.077+0000 7fa3739fa740 1 -- 172.21.15.99:0/2135376812 >> 172.21.15.99:0/2135376812 conn(0x560032488850 msgr2=0x56003248e740 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:56:19.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.077+0000 7fa3739fa740 1 -- 172.21.15.99:0/2135376812 shutdown_connections 2024-03-19T01:56:19.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.077+0000 7fa3739fa740 1 -- 172.21.15.99:0/2135376812 wait complete. 2024-03-19T01:56:19.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.077+0000 7fa3739fa740 1 -- start start 2024-03-19T01:56:19.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.077+0000 7fa3739fa740 1 Processor -- start 2024-03-19T01:56:19.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.077+0000 7fa3739fa740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56003248d2e0 0x56003249c680 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:19.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.077+0000 7fa3739fa740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56003248dcd0 0x56003249cbc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:19.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.077+0000 7fa3739fa740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560032492e40 0x56003249dbd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:19.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.077+0000 7fa3739fa740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fa36c1a7350 con 0x560032492e40 2024-03-19T01:56:19.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.077+0000 7fa3739fa740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fa36c1a87c0 con 0x56003248dcd0 2024-03-19T01:56:19.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.077+0000 7fa3739fa740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fa36c1a7a20 con 0x56003248d2e0 2024-03-19T01:56:19.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.077+0000 7fa3739fa740 10 client.0 _ll_register_callbacks cb 0x7fa35c0e0070 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:56:19.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.077+0000 7fa36ad9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560032492e40 0x56003249dbd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:19.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.077+0000 7fa36ad9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560032492e40 0x56003249dbd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:45728/0 (socket says 172.21.15.99:45728) 2024-03-19T01:56:19.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.077+0000 7fa36ad9d640 1 -- 172.21.15.99:0/1658387657 learned_addr learned my addr 172.21.15.99:0/1658387657 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:56:19.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.078+0000 7fa36a59c640 1 --2- 172.21.15.99:0/1658387657 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56003248d2e0 0x56003249c680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:19.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.078+0000 7fa363fff640 1 --2- 172.21.15.99:0/1658387657 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56003248dcd0 0x56003249cbc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:19.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.078+0000 7fa36ad9d640 1 -- 172.21.15.99:0/1658387657 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56003248d2e0 msgr2=0x56003249c680 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:56:19.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.078+0000 7fa36ad9d640 1 --2- 172.21.15.99:0/1658387657 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56003248d2e0 0x56003249c680 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:19.086 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.078+0000 7fa36ad9d640 1 -- 172.21.15.99:0/1658387657 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56003248dcd0 msgr2=0x56003249cbc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:56:19.086 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.078+0000 7fa36ad9d640 1 --2- 172.21.15.99:0/1658387657 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56003248dcd0 0x56003249cbc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:19.086 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.078+0000 7fa36ad9d640 1 -- 172.21.15.99:0/1658387657 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x56003249e1c0 con 0x560032492e40 2024-03-19T01:56:19.086 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.078+0000 7fa36ad9d640 1 --2- 172.21.15.99:0/1658387657 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560032492e40 0x56003249dbd0 secure :-1 s=READY pgs=645 cs=0 l=1 rev1=1 crypto rx=0x7fa364009ca0 tx=0x7fa3640095c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7a47cdc0170cc0d2 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:56:19.086 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.078+0000 7fa351ffb640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:56:19.086 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.078+0000 7fa351ffb640 1 -- 172.21.15.99:0/1658387657 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa364052070 con 0x560032492e40 2024-03-19T01:56:19.086 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.078+0000 7fa351ffb640 1 -- 172.21.15.99:0/1658387657 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fa364003b70 con 0x560032492e40 2024-03-19T01:56:19.086 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.078+0000 7fa3739fa740 1 -- 172.21.15.99:0/1658387657 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fa354004250 con 0x560032492e40 2024-03-19T01:56:19.088 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.081+0000 7fa351ffb640 1 -- 172.21.15.99:0/1658387657 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7fa364039040 con 0x560032492e40 2024-03-19T01:56:19.088 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.081+0000 7fa3739fa740 10 client.16578 fetch_fsmap learned FSMap version 37 2024-03-19T01:56:19.088 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.082+0000 7fa3739fa740 1 -- 172.21.15.99:0/1658387657 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x5600325096b0 con 0x560032492e40 2024-03-19T01:56:19.088 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.082+0000 7fa351ffb640 1 -- 172.21.15.99:0/1658387657 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa354004250 con 0x560032492e40 2024-03-19T01:56:19.088 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.082+0000 7fa351ffb640 1 -- 172.21.15.99:0/1658387657 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7fa364034540 con 0x560032492e40 2024-03-19T01:56:19.088 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.082+0000 7fa3739fa740 10 client.16578 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T01:56:19.088 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.082+0000 7fa3739fa740 10 client.16578 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:56:19.088 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.082+0000 7fa3739fa740 20 client.16578 populate_metadata read hostname 'smithi099' 2024-03-19T01:56:19.088 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.082+0000 7fa3739fa740 1 --2- 172.21.15.99:0/1658387657 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56003250a110 0x56003250c530 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:19.088 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.082+0000 7fa3739fa740 4 client.16578 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:56:19.089 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.082+0000 7fa3739fa740 1 -- 172.21.15.99:0/1658387657 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x5600323f9470 con 0x56003250a110 2024-03-19T01:56:19.089 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.082+0000 7fa36a59c640 1 --2- 172.21.15.99:0/1658387657 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56003250a110 0x56003250c530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:19.089 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.083+0000 7fa351ffb640 10 client.16578 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:56:19.089 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.083+0000 7fa36a59c640 1 --2- 172.21.15.99:0/1658387657 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56003250a110 0x56003250c530 crc :-1 s=READY pgs=166 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=8a8db10c8e90ecc7 server_cookie=dabe92bbc0300f36 in_seq=0 out_seq=0 2024-03-19T01:56:19.103 INFO:tasks.workunit.client.0.smithi099.stdout: 10 75285 23.31 MB/sec execute 48 sec latency 70.990 ms 2024-03-19T01:56:19.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:56:19 smithi099 ceph-mon[62951]: pgmap v808: 97 pgs: 97 active+clean; 537 MiB data, 5.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 5.9 MiB/s wr, 478 op/s 2024-03-19T01:56:19.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:56:19 smithi112 ceph-mon[66819]: pgmap v808: 97 pgs: 97 active+clean; 537 MiB data, 5.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 5.9 MiB/s wr, 478 op/s 2024-03-19T01:56:19.531 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.530+0000 7fa351ffb640 1 -- 172.21.15.99:0/1658387657 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x5600323f9470 con 0x56003250a110 2024-03-19T01:56:19.531 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.530+0000 7fa351ffb640 10 client.16578 handle_command_reply: tid=0 2024-03-19T01:56:19.531 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.530+0000 7fa3739fa740 1 client.16578 shutdown 2024-03-19T01:56:19.531 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.530+0000 7fa3739fa740 1 -- 172.21.15.99:0/1658387657 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560032492e40 msgr2=0x56003249dbd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:56:19.531 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.530+0000 7fa3739fa740 1 --2- 172.21.15.99:0/1658387657 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560032492e40 0x56003249dbd0 secure :-1 s=READY pgs=645 cs=0 l=1 rev1=1 crypto rx=0x7fa364009ca0 tx=0x7fa3640095c0 comp rx=0 tx=0).stop 2024-03-19T01:56:19.531 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.530+0000 7fa3739fa740 1 -- 172.21.15.99:0/1658387657 shutdown_connections 2024-03-19T01:56:19.531 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.530+0000 7fa3739fa740 1 --2- 172.21.15.99:0/1658387657 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56003250a110 0x56003250c530 crc :-1 s=READY pgs=166 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:19.531 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.530+0000 7fa3739fa740 1 --2- 172.21.15.99:0/1658387657 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56003248dcd0 0x56003249cbc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:19.531 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.530+0000 7fa3739fa740 1 --2- 172.21.15.99:0/1658387657 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560032492e40 0x56003249dbd0 unknown :-1 s=CLOSED pgs=645 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:19.532 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.530+0000 7fa3739fa740 1 --2- 172.21.15.99:0/1658387657 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56003248d2e0 0x56003249c680 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:19.532 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.530+0000 7fa351ffb640 0 client.16578 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:56:19.532 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.531+0000 7fa3739fa740 1 -- 172.21.15.99:0/1658387657 >> 172.21.15.99:0/1658387657 conn(0x560032488850 msgr2=0x560032497600 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:56:19.532 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.531+0000 7fa3739fa740 1 -- 172.21.15.99:0/1658387657 shutdown_connections 2024-03-19T01:56:19.532 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.531+0000 7fa3739fa740 1 -- 172.21.15.99:0/1658387657 wait complete. 2024-03-19T01:56:19.532 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.531+0000 7fa3739fa740 20 client.16578 trim_cache size 0 max 16384 2024-03-19T01:56:19.532 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:56:19.532 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:56:19.532 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:56:19.532 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:56:19.533 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.531+0000 7fa36b59e640 1 -- 172.21.15.99:0/3602891330 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa3580a27f0 msgr2=0x7fa3580a4cb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:56:19.533 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.531+0000 7fa36b59e640 1 --2- 172.21.15.99:0/3602891330 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa3580a27f0 0x7fa3580a4cb0 secure :-1 s=READY pgs=341 cs=0 l=1 rev1=1 crypto rx=0x7fa364005e30 tx=0x7fa364032590 comp rx=0 tx=0).stop 2024-03-19T01:56:19.533 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.531+0000 7fa36b59e640 1 -- 172.21.15.99:0/3602891330 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa36c100570 msgr2=0x7fa36c197900 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:56:19.533 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.531+0000 7fa36b59e640 1 --2- 172.21.15.99:0/3602891330 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa36c100570 0x7fa36c197900 secure :-1 s=READY pgs=641 cs=0 l=1 rev1=1 crypto rx=0x7fa35c00e9c0 tx=0x7fa35c00ee80 comp rx=0 tx=0).stop 2024-03-19T01:56:19.533 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.531+0000 7fa36b59e640 1 -- 172.21.15.99:0/3602891330 shutdown_connections 2024-03-19T01:56:19.533 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.531+0000 7fa36b59e640 1 --2- 172.21.15.99:0/3602891330 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa3580a27f0 0x7fa3580a4cb0 unknown :-1 s=CLOSED pgs=341 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:19.534 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.531+0000 7fa36b59e640 1 --2- 172.21.15.99:0/3602891330 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa36c102120 0x7fa36c19eeb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:19.534 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.531+0000 7fa36b59e640 1 --2- 172.21.15.99:0/3602891330 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa36c101760 0x7fa36c197e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:19.534 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.531+0000 7fa36b59e640 1 --2- 172.21.15.99:0/3602891330 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa36c100570 0x7fa36c197900 unknown :-1 s=CLOSED pgs=641 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:19.534 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.531+0000 7fa36b59e640 1 -- 172.21.15.99:0/3602891330 >> 172.21.15.99:0/3602891330 conn(0x7fa36c0fbb00 msgr2=0x7fa36c068ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:56:19.534 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.531+0000 7fa36b59e640 1 -- 172.21.15.99:0/3602891330 shutdown_connections 2024-03-19T01:56:19.534 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:19.531+0000 7fa36b59e640 1 -- 172.21.15.99:0/3602891330 wait complete. 2024-03-19T01:56:19.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:56:19 smithi115 ceph-mon[67332]: pgmap v808: 97 pgs: 97 active+clean; 537 MiB data, 5.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 5.9 MiB/s wr, 478 op/s 2024-03-19T01:56:20.103 INFO:tasks.workunit.client.0.smithi099.stdout: 10 75654 23.13 MB/sec execute 49 sec latency 340.642 ms 2024-03-19T01:56:21.103 INFO:tasks.workunit.client.0.smithi099.stdout: 10 76107 23.22 MB/sec execute 50 sec latency 75.366 ms 2024-03-19T01:56:21.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:56:21 smithi099 ceph-mon[62951]: pgmap v809: 97 pgs: 97 active+clean; 536 MiB data, 5.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 6.0 MiB/s wr, 474 op/s 2024-03-19T01:56:21.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:56:21 smithi112 ceph-mon[66819]: pgmap v809: 97 pgs: 97 active+clean; 536 MiB data, 5.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 6.0 MiB/s wr, 474 op/s 2024-03-19T01:56:21.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:56:21 smithi115 ceph-mon[67332]: pgmap v809: 97 pgs: 97 active+clean; 536 MiB data, 5.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 6.0 MiB/s wr, 474 op/s 2024-03-19T01:56:22.103 INFO:tasks.workunit.client.0.smithi099.stdout: 10 76484 23.32 MB/sec execute 51 sec latency 252.593 ms 2024-03-19T01:56:23.103 INFO:tasks.workunit.client.0.smithi099.stdout: 10 76810 23.06 MB/sec execute 52 sec latency 161.490 ms 2024-03-19T01:56:23.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:56:23 smithi099 ceph-mon[62951]: pgmap v810: 97 pgs: 97 active+clean; 534 MiB data, 5.7 GiB used, 1.0 TiB / 1.0 TiB avail; 5.0 MiB/s rd, 6.6 MiB/s wr, 622 op/s 2024-03-19T01:56:23.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:56:23 smithi112 ceph-mon[66819]: pgmap v810: 97 pgs: 97 active+clean; 534 MiB data, 5.7 GiB used, 1.0 TiB / 1.0 TiB avail; 5.0 MiB/s rd, 6.6 MiB/s wr, 622 op/s 2024-03-19T01:56:23.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:56:23 smithi115 ceph-mon[67332]: pgmap v810: 97 pgs: 97 active+clean; 534 MiB data, 5.7 GiB used, 1.0 TiB / 1.0 TiB avail; 5.0 MiB/s rd, 6.6 MiB/s wr, 622 op/s 2024-03-19T01:56:24.103 INFO:tasks.workunit.client.0.smithi099.stdout: 10 77299 22.99 MB/sec execute 53 sec latency 71.402 ms 2024-03-19T01:56:25.103 INFO:tasks.workunit.client.0.smithi099.stdout: 10 77826 22.97 MB/sec execute 54 sec latency 106.092 ms 2024-03-19T01:56:25.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:56:25 smithi099 ceph-mon[62951]: pgmap v811: 97 pgs: 97 active+clean; 531 MiB data, 5.7 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 5.7 MiB/s wr, 531 op/s 2024-03-19T01:56:25.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:56:25 smithi112 ceph-mon[66819]: pgmap v811: 97 pgs: 97 active+clean; 531 MiB data, 5.7 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 5.7 MiB/s wr, 531 op/s 2024-03-19T01:56:25.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:56:25 smithi115 ceph-mon[67332]: pgmap v811: 97 pgs: 97 active+clean; 531 MiB data, 5.7 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 5.7 MiB/s wr, 531 op/s 2024-03-19T01:56:26.105 INFO:tasks.workunit.client.0.smithi099.stdout: 10 78346 23.16 MB/sec execute 55 sec latency 68.162 ms 2024-03-19T01:56:27.103 INFO:tasks.workunit.client.0.smithi099.stdout: 10 78785 23.26 MB/sec execute 56 sec latency 269.075 ms 2024-03-19T01:56:27.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:56:27 smithi099 ceph-mon[62951]: pgmap v812: 97 pgs: 97 active+clean; 535 MiB data, 5.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.4 MiB/s rd, 7.1 MiB/s wr, 608 op/s 2024-03-19T01:56:27.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:56:27 smithi112 ceph-mon[66819]: pgmap v812: 97 pgs: 97 active+clean; 535 MiB data, 5.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.4 MiB/s rd, 7.1 MiB/s wr, 608 op/s 2024-03-19T01:56:27.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:56:27 smithi115 ceph-mon[67332]: pgmap v812: 97 pgs: 97 active+clean; 535 MiB data, 5.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.4 MiB/s rd, 7.1 MiB/s wr, 608 op/s 2024-03-19T01:56:28.103 INFO:tasks.workunit.client.0.smithi099.stdout: 10 79254 23.46 MB/sec execute 57 sec latency 92.686 ms 2024-03-19T01:56:29.103 INFO:tasks.workunit.client.0.smithi099.stdout: 10 79707 23.33 MB/sec execute 58 sec latency 62.533 ms 2024-03-19T01:56:29.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:56:29 smithi099 ceph-mon[62951]: pgmap v813: 97 pgs: 97 active+clean; 528 MiB data, 5.7 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 5.9 MiB/s wr, 544 op/s 2024-03-19T01:56:29.357 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:56:29 smithi115 ceph-mon[67332]: pgmap v813: 97 pgs: 97 active+clean; 528 MiB data, 5.7 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 5.9 MiB/s wr, 544 op/s 2024-03-19T01:56:29.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:56:29 smithi112 ceph-mon[66819]: pgmap v813: 97 pgs: 97 active+clean; 528 MiB data, 5.7 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 5.9 MiB/s wr, 544 op/s 2024-03-19T01:56:30.104 INFO:tasks.workunit.client.0.smithi099.stdout: 10 80137 23.41 MB/sec execute 59 sec latency 74.847 ms 2024-03-19T01:56:30.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:56:30 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:56:30.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:56:30 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:56:30.806 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:56:30 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:56:31.103 INFO:tasks.workunit.client.0.smithi099.stdout: 10 80524 23.28 MB/sec execute 60 sec latency 75.405 ms 2024-03-19T01:56:31.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:56:31 smithi112 ceph-mon[66819]: pgmap v814: 97 pgs: 97 active+clean; 524 MiB data, 5.7 GiB used, 1.0 TiB / 1.0 TiB avail; 3.9 MiB/s rd, 5.6 MiB/s wr, 539 op/s 2024-03-19T01:56:31.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:56:31 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:56:31.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:56:31 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:56:31.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:56:31 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:56:31.806 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:56:31 smithi115 ceph-mon[67332]: pgmap v814: 97 pgs: 97 active+clean; 524 MiB data, 5.7 GiB used, 1.0 TiB / 1.0 TiB avail; 3.9 MiB/s rd, 5.6 MiB/s wr, 539 op/s 2024-03-19T01:56:31.806 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:56:31 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:56:31.806 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:56:31 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:56:31.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:56:31 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:56:31.840 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:56:31 smithi099 ceph-mon[62951]: pgmap v814: 97 pgs: 97 active+clean; 524 MiB data, 5.7 GiB used, 1.0 TiB / 1.0 TiB avail; 3.9 MiB/s rd, 5.6 MiB/s wr, 539 op/s 2024-03-19T01:56:31.840 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:56:31 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:56:31.840 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:56:31 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:56:31.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:56:31 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:56:32.104 INFO:tasks.workunit.client.0.smithi099.stdout: 10 80914 23.28 MB/sec execute 61 sec latency 140.160 ms 2024-03-19T01:56:32.647 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.647+0000 7fa3f40de640 1 -- 172.21.15.99:0/2269909517 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa3ec101930 msgr2=0x7fa3ec101db0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:56:32.647 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.647+0000 7fa3f40de640 1 --2- 172.21.15.99:0/2269909517 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa3ec101930 0x7fa3ec101db0 secure :-1 s=READY pgs=646 cs=0 l=1 rev1=1 crypto rx=0x7fa3e8009860 tx=0x7fa3e802bb70 comp rx=0 tx=0).stop 2024-03-19T01:56:32.647 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.647+0000 7fa3f40de640 1 -- 172.21.15.99:0/2269909517 shutdown_connections 2024-03-19T01:56:32.647 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.647+0000 7fa3f40de640 1 --2- 172.21.15.99:0/2269909517 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa3ec1022f0 0x7fa3ec108b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:32.647 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.647+0000 7fa3f40de640 1 --2- 172.21.15.99:0/2269909517 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa3ec101930 0x7fa3ec101db0 unknown :-1 s=CLOSED pgs=646 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:32.647 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.647+0000 7fa3f40de640 1 --2- 172.21.15.99:0/2269909517 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa3ec100740 0x7fa3ec100b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:32.647 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.647+0000 7fa3f40de640 1 -- 172.21.15.99:0/2269909517 >> 172.21.15.99:0/2269909517 conn(0x7fa3ec0fbcb0 msgr2=0x7fa3ec0fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:56:32.648 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.648+0000 7fa3f40de640 1 -- 172.21.15.99:0/2269909517 shutdown_connections 2024-03-19T01:56:32.648 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.648+0000 7fa3f40de640 1 -- 172.21.15.99:0/2269909517 wait complete. 2024-03-19T01:56:32.649 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.649+0000 7fa3f40de640 1 Processor -- start 2024-03-19T01:56:32.649 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.649+0000 7fa3f40de640 1 -- start start 2024-03-19T01:56:32.649 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.649+0000 7fa3f40de640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa3ec100740 0x7fa3ec0690c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:32.650 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.649+0000 7fa3f40de640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa3ec101930 0x7fa3ec069600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:32.650 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.650+0000 7fa3f1e7a640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa3ec100740 0x7fa3ec0690c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:32.650 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.650+0000 7fa3f40de640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa3ec1022f0 0x7fa3ec1a9bf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:32.650 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.650+0000 7fa3f40de640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fa3ec06bb40 con 0x7fa3ec100740 2024-03-19T01:56:32.650 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.650+0000 7fa3f40de640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fa3ec06b9c0 con 0x7fa3ec1022f0 2024-03-19T01:56:32.650 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.650+0000 7fa3f40de640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fa3ec06bcc0 con 0x7fa3ec101930 2024-03-19T01:56:32.650 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.650+0000 7fa3f1e7a640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa3ec100740 0x7fa3ec0690c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:59842/0 (socket says 172.21.15.99:59842) 2024-03-19T01:56:32.650 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.650+0000 7fa3f1e7a640 1 -- 172.21.15.99:0/3033804524 learned_addr learned my addr 172.21.15.99:0/3033804524 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:56:32.650 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.650+0000 7fa3f267b640 1 --2- 172.21.15.99:0/3033804524 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa3ec1022f0 0x7fa3ec1a9bf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:32.650 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.650+0000 7fa3f1679640 1 --2- 172.21.15.99:0/3033804524 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa3ec101930 0x7fa3ec069600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:32.651 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.650+0000 7fa3f1e7a640 1 -- 172.21.15.99:0/3033804524 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa3ec101930 msgr2=0x7fa3ec069600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:56:32.651 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.650+0000 7fa3f1e7a640 1 --2- 172.21.15.99:0/3033804524 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa3ec101930 0x7fa3ec069600 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:32.651 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.650+0000 7fa3f1e7a640 1 -- 172.21.15.99:0/3033804524 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa3ec1022f0 msgr2=0x7fa3ec1a9bf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:56:32.652 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.650+0000 7fa3f1e7a640 1 --2- 172.21.15.99:0/3033804524 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa3ec1022f0 0x7fa3ec1a9bf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:32.652 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.650+0000 7fa3f1e7a640 1 -- 172.21.15.99:0/3033804524 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa3ec0737f0 con 0x7fa3ec100740 2024-03-19T01:56:32.652 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.650+0000 7fa3f1679640 1 --2- 172.21.15.99:0/3033804524 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa3ec101930 0x7fa3ec069600 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:56:32.652 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.651+0000 7fa3f1e7a640 1 --2- 172.21.15.99:0/3033804524 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa3ec100740 0x7fa3ec0690c0 secure :-1 s=READY pgs=647 cs=0 l=1 rev1=1 crypto rx=0x7fa3e400c3a0 tx=0x7fa3e400c860 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:56:32.652 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.651+0000 7fa3e2ffd640 1 -- 172.21.15.99:0/3033804524 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa3e4003970 con 0x7fa3ec100740 2024-03-19T01:56:32.652 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.651+0000 7fa3e2ffd640 1 -- 172.21.15.99:0/3033804524 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa3e4004020 con 0x7fa3ec100740 2024-03-19T01:56:32.652 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.651+0000 7fa3e2ffd640 1 -- 172.21.15.99:0/3033804524 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa3e400b9f0 con 0x7fa3ec100740 2024-03-19T01:56:32.652 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.651+0000 7fa3f40de640 1 -- 172.21.15.99:0/3033804524 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa3ec1aa1f0 con 0x7fa3ec100740 2024-03-19T01:56:32.652 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.651+0000 7fa3f40de640 1 -- 172.21.15.99:0/3033804524 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fa3ec1aa6c0 con 0x7fa3ec100740 2024-03-19T01:56:32.654 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.655+0000 7fa3e2ffd640 1 -- 172.21.15.99:0/3033804524 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7fa3e401d020 con 0x7fa3ec100740 2024-03-19T01:56:32.655 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.655+0000 7fa3f6d9c740 1 Processor -- start 2024-03-19T01:56:32.655 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.655+0000 7fa3f6d9c740 1 -- start start 2024-03-19T01:56:32.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.656+0000 7fa3f6d9c740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563097e8e910 0x563097e8ed30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:32.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.656+0000 7fa3f6d9c740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563097e8f300 0x563097e93e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:32.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.656+0000 7fa3f6d9c740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563097e94470 0x563097e96900 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:32.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.656+0000 7fa3f6d9c740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x563097deabe0 con 0x563097e8f300 2024-03-19T01:56:32.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.656+0000 7fa3f6d9c740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x563097d93960 con 0x563097e8e910 2024-03-19T01:56:32.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.656+0000 7fa3f6d9c740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x563097de6c80 con 0x563097e94470 2024-03-19T01:56:32.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.656+0000 7fa3f267b640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563097e8e910 0x563097e8ed30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:32.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.656+0000 7fa3f1679640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563097e94470 0x563097e96900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:32.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.656+0000 7fa3f267b640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563097e8e910 0x563097e8ed30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:56922/0 (socket says 172.21.15.99:56922) 2024-03-19T01:56:32.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.656+0000 7fa3f267b640 1 -- 172.21.15.99:0/2190914722 learned_addr learned my addr 172.21.15.99:0/2190914722 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:56:32.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.656+0000 7fa3f1e7a640 1 --2- 172.21.15.99:0/2190914722 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563097e8f300 0x563097e93e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:32.657 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.656+0000 7fa3f1679640 1 -- 172.21.15.99:0/2190914722 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563097e8e910 msgr2=0x563097e8ed30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:56:32.657 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.656+0000 7fa3f1679640 1 --2- 172.21.15.99:0/2190914722 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563097e8e910 0x563097e8ed30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:32.658 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.656+0000 7fa3f1679640 1 -- 172.21.15.99:0/2190914722 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563097e8f300 msgr2=0x563097e93e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:56:32.658 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.656+0000 7fa3f1679640 1 --2- 172.21.15.99:0/2190914722 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563097e8f300 0x563097e93e00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:32.658 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.656+0000 7fa3f1679640 1 -- 172.21.15.99:0/2190914722 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x563097d8ad20 con 0x563097e94470 2024-03-19T01:56:32.658 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.657+0000 7fa3e2ffd640 1 --2- 172.21.15.99:0/3033804524 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa3d80a2920 0x7fa3d80a4de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:32.658 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.657+0000 7fa3f1e7a640 1 --2- 172.21.15.99:0/2190914722 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563097e8f300 0x563097e93e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-19T01:56:32.658 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.657+0000 7fa3e2ffd640 1 -- 172.21.15.99:0/3033804524 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fa3e40cead0 con 0x7fa3ec100740 2024-03-19T01:56:32.658 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.657+0000 7fa3f267b640 1 --2- 172.21.15.99:0/3033804524 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa3d80a2920 0x7fa3d80a4de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:32.658 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.657+0000 7fa3f1679640 1 --2- 172.21.15.99:0/2190914722 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563097e94470 0x563097e96900 secure :-1 s=READY pgs=117 cs=0 l=1 rev1=1 crypto rx=0x7fa3e802c0b0 tx=0x7fa3e8035f00 comp rx=0 tx=0).ready entity=mon.1 client_cookie=ea8700d86078cac9 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:56:32.658 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.657+0000 7fa3d7fff640 1 -- 172.21.15.99:0/2190914722 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa3e8039040 con 0x563097e94470 2024-03-19T01:56:32.658 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.658+0000 7fa3d7fff640 1 -- 172.21.15.99:0/2190914722 <== mon.1 v2:172.21.15.115:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa3e8034640 con 0x563097e94470 2024-03-19T01:56:32.659 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.658+0000 7fa3d7fff640 1 -- 172.21.15.99:0/2190914722 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa3e802cb60 con 0x563097e94470 2024-03-19T01:56:32.659 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.659+0000 7fa3f6d9c740 1 -- 172.21.15.99:0/2190914722 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563097e94470 msgr2=0x563097e96900 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:56:32.659 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.659+0000 7fa3f6d9c740 1 --2- 172.21.15.99:0/2190914722 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563097e94470 0x563097e96900 secure :-1 s=READY pgs=117 cs=0 l=1 rev1=1 crypto rx=0x7fa3e802c0b0 tx=0x7fa3e8035f00 comp rx=0 tx=0).stop 2024-03-19T01:56:32.659 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.659+0000 7fa3f267b640 1 --2- 172.21.15.99:0/3033804524 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa3d80a2920 0x7fa3d80a4de0 secure :-1 s=READY pgs=342 cs=0 l=1 rev1=1 crypto rx=0x563097e8fb60 tx=0x7fa3dc005f90 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:56:32.659 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.659+0000 7fa3f6d9c740 1 -- 172.21.15.99:0/2190914722 shutdown_connections 2024-03-19T01:56:32.659 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.659+0000 7fa3f6d9c740 1 --2- 172.21.15.99:0/2190914722 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563097e94470 0x563097e96900 unknown :-1 s=CLOSED pgs=117 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:32.659 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.659+0000 7fa3f6d9c740 1 --2- 172.21.15.99:0/2190914722 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563097e8f300 0x563097e93e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:32.659 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.659+0000 7fa3f6d9c740 1 --2- 172.21.15.99:0/2190914722 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563097e8e910 0x563097e8ed30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:32.659 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.659+0000 7fa3f6d9c740 1 -- 172.21.15.99:0/2190914722 >> 172.21.15.99:0/2190914722 conn(0x563097e89e80 msgr2=0x563097e8a280 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:56:32.660 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.661+0000 7fa3f6d9c740 1 -- 172.21.15.99:0/2190914722 shutdown_connections 2024-03-19T01:56:32.660 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.661+0000 7fa3f6d9c740 1 -- 172.21.15.99:0/2190914722 wait complete. 2024-03-19T01:56:32.661 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.661+0000 7fa3f6d9c740 1 -- start start 2024-03-19T01:56:32.661 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.661+0000 7fa3f6d9c740 1 Processor -- start 2024-03-19T01:56:32.662 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.662+0000 7fa3f6d9c740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563097e8e910 0x563097e98860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:32.662 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.662+0000 7fa3f6d9c740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563097e8f300 0x563097e98da0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:32.662 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.662+0000 7fa3f6d9c740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563097e94470 0x563097e9fde0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:32.662 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.662+0000 7fa3f6d9c740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x563097e97460 con 0x563097e94470 2024-03-19T01:56:32.662 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.662+0000 7fa3f6d9c740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x563097da6db0 con 0x563097e8e910 2024-03-19T01:56:32.662 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.662+0000 7fa3f6d9c740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x563097e975e0 con 0x563097e8f300 2024-03-19T01:56:32.662 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.662+0000 7fa3f6d9c740 10 client.0 _ll_register_callbacks cb 0x563097dabe70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:56:32.662 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.662+0000 7fa3f1679640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563097e8f300 0x563097e98da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:32.662 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.662+0000 7fa3f1679640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563097e8f300 0x563097e98da0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:51298/0 (socket says 172.21.15.99:51298) 2024-03-19T01:56:32.662 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.662+0000 7fa3f1679640 1 -- 172.21.15.99:0/4048049034 learned_addr learned my addr 172.21.15.99:0/4048049034 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:56:32.663 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.663+0000 7fa3f1679640 1 -- 172.21.15.99:0/4048049034 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563097e8e910 msgr2=0x563097e98860 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-03-19T01:56:32.663 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.663+0000 7fa3f1679640 1 --2- 172.21.15.99:0/4048049034 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563097e8e910 0x563097e98860 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:32.664 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.663+0000 7fa3f1679640 1 -- 172.21.15.99:0/4048049034 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563097e94470 msgr2=0x563097e9fde0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:56:32.664 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.663+0000 7fa3f1679640 1 --2- 172.21.15.99:0/4048049034 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563097e94470 0x563097e9fde0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:32.664 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.663+0000 7fa3f1679640 1 -- 172.21.15.99:0/4048049034 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({osdmap=0}) -- 0x563097da5c60 con 0x563097e8f300 2024-03-19T01:56:32.664 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.663+0000 7fa3f1679640 1 --2- 172.21.15.99:0/4048049034 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563097e8f300 0x563097e98da0 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 crypto rx=0x7fa3e8005e00 tx=0x7fa3e802c2f0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=2ac706afb5f46c68 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:56:32.664 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.663+0000 7fa3d57fa640 10 client.0 ms_handle_connect on v2:172.21.15.115:3300/0 2024-03-19T01:56:32.664 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.663+0000 7fa3d57fa640 1 -- 172.21.15.99:0/4048049034 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa3e804c070 con 0x563097e8f300 2024-03-19T01:56:32.664 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.663+0000 7fa3f6d9c740 1 -- 172.21.15.99:0/4048049034 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fa3e8034640 con 0x563097e8f300 2024-03-19T01:56:32.664 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.664+0000 7fa3d57fa640 1 -- 172.21.15.99:0/4048049034 <== mon.1 v2:172.21.15.115:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fa3e8004250 con 0x563097e8f300 2024-03-19T01:56:32.664 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.664+0000 7fa3d57fa640 1 -- 172.21.15.99:0/4048049034 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7fa3e8039040 con 0x563097e8f300 2024-03-19T01:56:32.664 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.664+0000 7fa3f6d9c740 10 client.26407 fetch_fsmap learned FSMap version 37 2024-03-19T01:56:32.664 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.664+0000 7fa3f6d9c740 1 -- 172.21.15.99:0/4048049034 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x563097aff210 con 0x563097e8f300 2024-03-19T01:56:32.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.665+0000 7fa3d57fa640 1 -- 172.21.15.99:0/4048049034 <== mon.1 v2:172.21.15.115:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa3e8034640 con 0x563097e8f300 2024-03-19T01:56:32.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.665+0000 7fa3d57fa640 1 -- 172.21.15.99:0/4048049034 <== mon.1 v2:172.21.15.115:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7fa3e8003790 con 0x563097e8f300 2024-03-19T01:56:32.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.665+0000 7fa3f6d9c740 10 client.26407 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T01:56:32.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.665+0000 7fa3f6d9c740 10 client.26407 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:56:32.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.665+0000 7fa3f6d9c740 20 client.26407 populate_metadata read hostname 'smithi099' 2024-03-19T01:56:32.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.665+0000 7fa3f6d9c740 1 --2- 172.21.15.99:0/4048049034 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x563097ea4240 0x563097ea66b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:32.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.665+0000 7fa3f6d9c740 4 client.26407 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:56:32.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.665+0000 7fa3f6d9c740 1 -- 172.21.15.99:0/4048049034 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x563097df8120 con 0x563097ea4240 2024-03-19T01:56:32.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.665+0000 7fa3f1e7a640 1 --2- 172.21.15.99:0/4048049034 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x563097ea4240 0x563097ea66b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:32.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.666+0000 7fa3d57fa640 10 client.26407 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:56:32.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.666+0000 7fa3f1e7a640 1 --2- 172.21.15.99:0/4048049034 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x563097ea4240 0x563097ea66b0 crc :-1 s=READY pgs=167 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=92eb382b55bee152 server_cookie=df29b18157e60955 in_seq=0 out_seq=0 2024-03-19T01:56:32.667 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.668+0000 7fa3d57fa640 1 -- 172.21.15.99:0/4048049034 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x563097df8120 con 0x563097ea4240 2024-03-19T01:56:32.667 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.668+0000 7fa3d57fa640 10 client.26407 handle_command_reply: tid=0 2024-03-19T01:56:32.667 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.668+0000 7fa3f6d9c740 1 client.26407 shutdown 2024-03-19T01:56:32.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.668+0000 7fa3f6d9c740 1 -- 172.21.15.99:0/4048049034 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563097e8f300 msgr2=0x563097e98da0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:56:32.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.668+0000 7fa3f6d9c740 1 --2- 172.21.15.99:0/4048049034 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563097e8f300 0x563097e98da0 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 crypto rx=0x7fa3e8005e00 tx=0x7fa3e802c2f0 comp rx=0 tx=0).stop 2024-03-19T01:56:32.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.668+0000 7fa3f6d9c740 1 -- 172.21.15.99:0/4048049034 shutdown_connections 2024-03-19T01:56:32.669 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.668+0000 7fa3f6d9c740 1 --2- 172.21.15.99:0/4048049034 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x563097ea4240 0x563097ea66b0 crc :-1 s=READY pgs=167 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:32.669 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.668+0000 7fa3f6d9c740 1 --2- 172.21.15.99:0/4048049034 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563097e94470 0x563097e9fde0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:32.669 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.668+0000 7fa3f6d9c740 1 --2- 172.21.15.99:0/4048049034 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563097e8f300 0x563097e98da0 unknown :-1 s=CLOSED pgs=118 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:32.669 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.668+0000 7fa3f6d9c740 1 --2- 172.21.15.99:0/4048049034 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563097e8e910 0x563097e98860 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:32.669 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.668+0000 7fa3f6d9c740 1 -- 172.21.15.99:0/4048049034 >> 172.21.15.99:0/4048049034 conn(0x563097e89e80 msgr2=0x563097e8b0d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:56:32.669 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.668+0000 7fa3d57fa640 0 client.26407 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:56:32.669 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.668+0000 7fa3f6d9c740 1 -- 172.21.15.99:0/4048049034 shutdown_connections 2024-03-19T01:56:32.669 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.669+0000 7fa3f6d9c740 1 -- 172.21.15.99:0/4048049034 wait complete. 2024-03-19T01:56:32.669 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.669+0000 7fa3f6d9c740 20 client.26407 trim_cache size 0 max 16384 2024-03-19T01:56:32.694 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.693+0000 7fa3f6d9c740 1 Processor -- start 2024-03-19T01:56:32.695 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.693+0000 7fa3f6d9c740 1 -- start start 2024-03-19T01:56:32.695 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.694+0000 7fa3f6d9c740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563097e8e910 0x563097e8fda0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:32.695 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.694+0000 7fa3f6d9c740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563097e8f300 0x563097e9fda0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:32.695 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.694+0000 7fa3f6d9c740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563097e94470 0x563097e902e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:32.695 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.694+0000 7fa3f6d9c740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fa3ec1b1ac0 con 0x563097e94470 2024-03-19T01:56:32.695 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.694+0000 7fa3f6d9c740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fa3ec1af390 con 0x563097e8f300 2024-03-19T01:56:32.695 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.694+0000 7fa3f6d9c740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fa3ec1ad1e0 con 0x563097e8e910 2024-03-19T01:56:32.695 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.694+0000 7fa3f1679640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563097e8f300 0x563097e9fda0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:32.695 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.694+0000 7fa3f1679640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563097e8f300 0x563097e9fda0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:56954/0 (socket says 172.21.15.99:56954) 2024-03-19T01:56:32.695 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.694+0000 7fa3f1679640 1 -- 172.21.15.99:0/3284789165 learned_addr learned my addr 172.21.15.99:0/3284789165 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:56:32.695 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.694+0000 7fa3f267b640 1 --2- 172.21.15.99:0/3284789165 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563097e94470 0x563097e902e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:32.695 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.694+0000 7fa3f1679640 1 -- 172.21.15.99:0/3284789165 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563097e8e910 msgr2=0x563097e8fda0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:56:32.695 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.694+0000 7fa3f1679640 1 --2- 172.21.15.99:0/3284789165 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563097e8e910 0x563097e8fda0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:32.696 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.694+0000 7fa3f1679640 1 -- 172.21.15.99:0/3284789165 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563097e94470 msgr2=0x563097e902e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:56:32.696 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.694+0000 7fa3f1679640 1 --2- 172.21.15.99:0/3284789165 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563097e94470 0x563097e902e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:32.696 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.694+0000 7fa3f1679640 1 -- 172.21.15.99:0/3284789165 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x563097ea6a20 con 0x563097e8f300 2024-03-19T01:56:32.696 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.694+0000 7fa3f267b640 1 --2- 172.21.15.99:0/3284789165 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563097e94470 0x563097e902e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-19T01:56:32.696 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.695+0000 7fa3f1679640 1 --2- 172.21.15.99:0/3284789165 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563097e8f300 0x563097e9fda0 secure :-1 s=READY pgs=171 cs=0 l=1 rev1=1 crypto rx=0x7fa3e802cd70 tx=0x7fa3e8009450 comp rx=0 tx=0).ready entity=mon.2 client_cookie=14a19b86294cdd8c server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:56:32.696 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.695+0000 7fa3d77fe640 1 -- 172.21.15.99:0/3284789165 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa3e804c030 con 0x563097e8f300 2024-03-19T01:56:32.696 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.695+0000 7fa3d77fe640 1 -- 172.21.15.99:0/3284789165 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa3e805f020 con 0x563097e8f300 2024-03-19T01:56:32.696 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.695+0000 7fa3d77fe640 1 -- 172.21.15.99:0/3284789165 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa3e8004250 con 0x563097e8f300 2024-03-19T01:56:32.699 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.695+0000 7fa3f6d9c740 1 -- 172.21.15.99:0/3284789165 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563097e8f300 msgr2=0x563097e9fda0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:56:32.699 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.695+0000 7fa3f6d9c740 1 --2- 172.21.15.99:0/3284789165 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563097e8f300 0x563097e9fda0 secure :-1 s=READY pgs=171 cs=0 l=1 rev1=1 crypto rx=0x7fa3e802cd70 tx=0x7fa3e8009450 comp rx=0 tx=0).stop 2024-03-19T01:56:32.699 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.696+0000 7fa3f6d9c740 1 -- 172.21.15.99:0/3284789165 shutdown_connections 2024-03-19T01:56:32.699 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.696+0000 7fa3f6d9c740 1 --2- 172.21.15.99:0/3284789165 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563097e8f300 0x563097e9fda0 unknown :-1 s=CLOSED pgs=171 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:32.699 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.696+0000 7fa3f6d9c740 1 --2- 172.21.15.99:0/3284789165 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563097e94470 0x563097e902e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:32.700 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.696+0000 7fa3f6d9c740 1 --2- 172.21.15.99:0/3284789165 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563097e8e910 0x563097e8fda0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:32.700 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.696+0000 7fa3f6d9c740 1 -- 172.21.15.99:0/3284789165 >> 172.21.15.99:0/3284789165 conn(0x563097e89e80 msgr2=0x563097e93800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:56:32.700 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.696+0000 7fa3f6d9c740 1 -- 172.21.15.99:0/3284789165 shutdown_connections 2024-03-19T01:56:32.700 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.696+0000 7fa3f6d9c740 1 -- 172.21.15.99:0/3284789165 wait complete. 2024-03-19T01:56:32.700 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.696+0000 7fa3f6d9c740 1 -- start start 2024-03-19T01:56:32.700 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.696+0000 7fa3f6d9c740 1 Processor -- start 2024-03-19T01:56:32.700 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.696+0000 7fa3f6d9c740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563097e8e910 0x563097e9d270 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:32.700 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.696+0000 7fa3f6d9c740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563097e8f300 0x563097e9d7b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:32.700 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.696+0000 7fa3f6d9c740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563097e94470 0x563097e9e7c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:32.700 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.697+0000 7fa3f6d9c740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fa3ec1ad850 con 0x563097e94470 2024-03-19T01:56:32.700 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.697+0000 7fa3f6d9c740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fa3ec1aecc0 con 0x563097e8e910 2024-03-19T01:56:32.700 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.697+0000 7fa3f1e7a640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563097e8e910 0x563097e9d270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:32.700 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.697+0000 7fa3f6d9c740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fa3ec1adf20 con 0x563097e8f300 2024-03-19T01:56:32.700 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.697+0000 7fa3f1e7a640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563097e8e910 0x563097e9d270 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:56966/0 (socket says 172.21.15.99:56966) 2024-03-19T01:56:32.701 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.697+0000 7fa3f1e7a640 1 -- 172.21.15.99:0/2304984199 learned_addr learned my addr 172.21.15.99:0/2304984199 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:56:32.701 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.697+0000 7fa3f6d9c740 10 client.0 _ll_register_callbacks cb 0x7fa3dc009780 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:56:32.701 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.697+0000 7fa3f267b640 1 --2- 172.21.15.99:0/2304984199 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563097e94470 0x563097e9e7c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:32.701 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.697+0000 7fa3f1679640 1 --2- 172.21.15.99:0/2304984199 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563097e8f300 0x563097e9d7b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:32.701 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.697+0000 7fa3f1e7a640 1 -- 172.21.15.99:0/2304984199 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563097e8f300 msgr2=0x563097e9d7b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:56:32.701 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.697+0000 7fa3f1e7a640 1 --2- 172.21.15.99:0/2304984199 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563097e8f300 0x563097e9d7b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:32.701 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.697+0000 7fa3f1e7a640 1 -- 172.21.15.99:0/2304984199 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563097e94470 msgr2=0x563097e9e7c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:56:32.701 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.697+0000 7fa3f1e7a640 1 --2- 172.21.15.99:0/2304984199 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563097e94470 0x563097e9e7c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:32.701 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.697+0000 7fa3f1e7a640 1 -- 172.21.15.99:0/2304984199 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x563097bcb620 con 0x563097e8e910 2024-03-19T01:56:32.701 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.697+0000 7fa3f1e7a640 1 --2- 172.21.15.99:0/2304984199 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563097e8e910 0x563097e9d270 secure :-1 s=READY pgs=172 cs=0 l=1 rev1=1 crypto rx=0x7fa3e4097890 tx=0x7fa3e409aeb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=2ae13d262766ce44 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:56:32.701 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.697+0000 7fa3d57fa640 10 client.0 ms_handle_connect on v2:172.21.15.112:3300/0 2024-03-19T01:56:32.701 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.697+0000 7fa3d57fa640 1 -- 172.21.15.99:0/2304984199 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa3e40a3bd0 con 0x563097e8e910 2024-03-19T01:56:32.701 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.697+0000 7fa3d57fa640 1 -- 172.21.15.99:0/2304984199 <== mon.2 v2:172.21.15.112:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fa3e40044d0 con 0x563097e8e910 2024-03-19T01:56:32.702 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.697+0000 7fa3f6d9c740 1 -- 172.21.15.99:0/2304984199 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fa3e805f020 con 0x563097e8e910 2024-03-19T01:56:32.702 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.697+0000 7fa3d57fa640 1 -- 172.21.15.99:0/2304984199 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7fa3e409b430 con 0x563097e8e910 2024-03-19T01:56:32.702 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.697+0000 7fa3f6d9c740 10 client.26363 fetch_fsmap learned FSMap version 37 2024-03-19T01:56:32.702 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.697+0000 7fa3f6d9c740 1 -- 172.21.15.99:0/2304984199 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x563097e9ff60 con 0x563097e8e910 2024-03-19T01:56:32.702 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.698+0000 7fa3d57fa640 1 -- 172.21.15.99:0/2304984199 <== mon.2 v2:172.21.15.112:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa3e805f020 con 0x563097e8e910 2024-03-19T01:56:32.702 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.698+0000 7fa3d57fa640 1 -- 172.21.15.99:0/2304984199 <== mon.2 v2:172.21.15.112:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7fa3e400b040 con 0x563097e8e910 2024-03-19T01:56:32.702 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.698+0000 7fa3f6d9c740 10 client.26363 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T01:56:32.702 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.698+0000 7fa3f6d9c740 10 client.26363 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:56:32.702 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.698+0000 7fa3f6d9c740 20 client.26363 populate_metadata read hostname 'smithi099' 2024-03-19T01:56:32.702 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.698+0000 7fa3f6d9c740 1 --2- 172.21.15.99:0/2304984199 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x563097f0ab90 0x563097f0cfb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:32.702 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.698+0000 7fa3f6d9c740 4 client.26363 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:56:32.702 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.698+0000 7fa3f6d9c740 1 -- 172.21.15.99:0/2304984199 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x563097df2470 con 0x563097f0ab90 2024-03-19T01:56:32.702 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.698+0000 7fa3f1679640 1 --2- 172.21.15.99:0/2304984199 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x563097f0ab90 0x563097f0cfb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:32.703 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.699+0000 7fa3f1679640 1 --2- 172.21.15.99:0/2304984199 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x563097f0ab90 0x563097f0cfb0 crc :-1 s=READY pgs=168 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=f5fc8fc4d8972c67 server_cookie=d4c6897399bd40a1 in_seq=0 out_seq=0 2024-03-19T01:56:32.703 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:32.699+0000 7fa3d57fa640 10 client.26363 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:56:33.104 INFO:tasks.workunit.client.0.smithi099.stdout: 10 81274 23.22 MB/sec execute 62 sec latency 174.039 ms 2024-03-19T01:56:33.293 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:33.292+0000 7fa3d57fa640 1 -- 172.21.15.99:0/2304984199 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x563097df2470 con 0x563097f0ab90 2024-03-19T01:56:33.293 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:33.292+0000 7fa3d57fa640 10 client.26363 handle_command_reply: tid=0 2024-03-19T01:56:33.293 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:33.292+0000 7fa3f6d9c740 1 client.26363 shutdown 2024-03-19T01:56:33.293 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:33.292+0000 7fa3f6d9c740 1 -- 172.21.15.99:0/2304984199 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563097e8e910 msgr2=0x563097e9d270 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:56:33.293 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:33.292+0000 7fa3f6d9c740 1 --2- 172.21.15.99:0/2304984199 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563097e8e910 0x563097e9d270 secure :-1 s=READY pgs=172 cs=0 l=1 rev1=1 crypto rx=0x7fa3e4097890 tx=0x7fa3e409aeb0 comp rx=0 tx=0).stop 2024-03-19T01:56:33.293 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:33.292+0000 7fa3f6d9c740 1 -- 172.21.15.99:0/2304984199 shutdown_connections 2024-03-19T01:56:33.293 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:33.292+0000 7fa3f6d9c740 1 --2- 172.21.15.99:0/2304984199 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x563097f0ab90 0x563097f0cfb0 crc :-1 s=READY pgs=168 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:33.293 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:33.292+0000 7fa3f6d9c740 1 --2- 172.21.15.99:0/2304984199 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563097e94470 0x563097e9e7c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:33.293 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:33.292+0000 7fa3f6d9c740 1 --2- 172.21.15.99:0/2304984199 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563097e8f300 0x563097e9d7b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:33.293 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:33.293+0000 7fa3f6d9c740 1 --2- 172.21.15.99:0/2304984199 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563097e8e910 0x563097e9d270 unknown :-1 s=CLOSED pgs=172 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:33.293 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:33.293+0000 7fa3d57fa640 0 client.26363 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:56:33.294 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:33.293+0000 7fa3f6d9c740 1 -- 172.21.15.99:0/2304984199 >> 172.21.15.99:0/2304984199 conn(0x563097e89e80 msgr2=0x563097e8fda0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:56:33.294 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:33.293+0000 7fa3f6d9c740 1 -- 172.21.15.99:0/2304984199 shutdown_connections 2024-03-19T01:56:33.294 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:33.293+0000 7fa3f6d9c740 1 -- 172.21.15.99:0/2304984199 wait complete. 2024-03-19T01:56:33.294 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:33.293+0000 7fa3f6d9c740 20 client.26363 trim_cache size 0 max 16384 2024-03-19T01:56:33.294 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:56:33.294 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:56:33.294 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:56:33.294 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:56:33.295 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:33.293+0000 7fa3f40de640 1 -- 172.21.15.99:0/3033804524 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa3d80a2920 msgr2=0x7fa3d80a4de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:56:33.295 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:33.293+0000 7fa3f40de640 1 --2- 172.21.15.99:0/3033804524 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa3d80a2920 0x7fa3d80a4de0 secure :-1 s=READY pgs=342 cs=0 l=1 rev1=1 crypto rx=0x563097e8fb60 tx=0x7fa3dc005f90 comp rx=0 tx=0).stop 2024-03-19T01:56:33.295 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:33.293+0000 7fa3f40de640 1 -- 172.21.15.99:0/3033804524 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa3ec100740 msgr2=0x7fa3ec0690c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:56:33.296 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:33.293+0000 7fa3f40de640 1 --2- 172.21.15.99:0/3033804524 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa3ec100740 0x7fa3ec0690c0 secure :-1 s=READY pgs=647 cs=0 l=1 rev1=1 crypto rx=0x7fa3e400c3a0 tx=0x7fa3e400c860 comp rx=0 tx=0).stop 2024-03-19T01:56:33.296 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:33.293+0000 7fa3f40de640 1 -- 172.21.15.99:0/3033804524 shutdown_connections 2024-03-19T01:56:33.296 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:33.293+0000 7fa3f40de640 1 --2- 172.21.15.99:0/3033804524 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa3d80a2920 0x7fa3d80a4de0 unknown :-1 s=CLOSED pgs=342 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:33.296 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:33.293+0000 7fa3f40de640 1 --2- 172.21.15.99:0/3033804524 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa3ec1022f0 0x7fa3ec1a9bf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:33.296 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:33.293+0000 7fa3f40de640 1 --2- 172.21.15.99:0/3033804524 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa3ec101930 0x7fa3ec069600 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:33.296 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:33.293+0000 7fa3f40de640 1 --2- 172.21.15.99:0/3033804524 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa3ec100740 0x7fa3ec0690c0 unknown :-1 s=CLOSED pgs=647 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:33.296 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:33.293+0000 7fa3f40de640 1 -- 172.21.15.99:0/3033804524 >> 172.21.15.99:0/3033804524 conn(0x7fa3ec0fbcb0 msgr2=0x7fa3ec104b60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:56:33.296 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:33.293+0000 7fa3f40de640 1 -- 172.21.15.99:0/3033804524 shutdown_connections 2024-03-19T01:56:33.296 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:33.293+0000 7fa3f40de640 1 -- 172.21.15.99:0/3033804524 wait complete. 2024-03-19T01:56:33.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:56:33 smithi099 ceph-mon[62951]: pgmap v815: 97 pgs: 97 active+clean; 521 MiB data, 5.8 GiB used, 1.0 TiB / 1.0 TiB avail; 4.8 MiB/s rd, 6.5 MiB/s wr, 587 op/s 2024-03-19T01:56:33.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:56:33 smithi112 ceph-mon[66819]: pgmap v815: 97 pgs: 97 active+clean; 521 MiB data, 5.8 GiB used, 1.0 TiB / 1.0 TiB avail; 4.8 MiB/s rd, 6.5 MiB/s wr, 587 op/s 2024-03-19T01:56:33.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:56:33 smithi115 ceph-mon[67332]: pgmap v815: 97 pgs: 97 active+clean; 521 MiB data, 5.8 GiB used, 1.0 TiB / 1.0 TiB avail; 4.8 MiB/s rd, 6.5 MiB/s wr, 587 op/s 2024-03-19T01:56:34.104 INFO:tasks.workunit.client.0.smithi099.stdout: 10 81604 23.13 MB/sec execute 63 sec latency 274.224 ms 2024-03-19T01:56:35.104 INFO:tasks.workunit.client.0.smithi099.stdout: 10 82040 23.15 MB/sec execute 64 sec latency 70.328 ms 2024-03-19T01:56:35.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:56:35 smithi099 ceph-mon[62951]: pgmap v816: 97 pgs: 97 active+clean; 524 MiB data, 5.8 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 5.8 MiB/s wr, 436 op/s 2024-03-19T01:56:35.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:56:35 smithi112 ceph-mon[66819]: pgmap v816: 97 pgs: 97 active+clean; 524 MiB data, 5.8 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 5.8 MiB/s wr, 436 op/s 2024-03-19T01:56:35.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:56:35 smithi115 ceph-mon[67332]: pgmap v816: 97 pgs: 97 active+clean; 524 MiB data, 5.8 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 5.8 MiB/s wr, 436 op/s 2024-03-19T01:56:36.104 INFO:tasks.workunit.client.0.smithi099.stdout: 10 82574 23.04 MB/sec execute 65 sec latency 91.695 ms 2024-03-19T01:56:37.104 INFO:tasks.workunit.client.0.smithi099.stdout: 10 83021 22.98 MB/sec execute 66 sec latency 108.924 ms 2024-03-19T01:56:37.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:56:37 smithi099 ceph-mon[62951]: pgmap v817: 97 pgs: 97 active+clean; 516 MiB data, 5.8 GiB used, 1.0 TiB / 1.0 TiB avail; 4.7 MiB/s rd, 6.9 MiB/s wr, 596 op/s 2024-03-19T01:56:37.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:56:37 smithi112 ceph-mon[66819]: pgmap v817: 97 pgs: 97 active+clean; 516 MiB data, 5.8 GiB used, 1.0 TiB / 1.0 TiB avail; 4.7 MiB/s rd, 6.9 MiB/s wr, 596 op/s 2024-03-19T01:56:37.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:56:37 smithi115 ceph-mon[67332]: pgmap v817: 97 pgs: 97 active+clean; 516 MiB data, 5.8 GiB used, 1.0 TiB / 1.0 TiB avail; 4.7 MiB/s rd, 6.9 MiB/s wr, 596 op/s 2024-03-19T01:56:38.104 INFO:tasks.workunit.client.0.smithi099.stdout: 10 83458 23.13 MB/sec execute 67 sec latency 303.563 ms 2024-03-19T01:56:39.104 INFO:tasks.workunit.client.0.smithi099.stdout: 10 84014 23.42 MB/sec execute 68 sec latency 49.331 ms 2024-03-19T01:56:39.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:56:39 smithi099 ceph-mon[62951]: pgmap v818: 97 pgs: 97 active+clean; 517 MiB data, 5.8 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 5.7 MiB/s wr, 546 op/s 2024-03-19T01:56:39.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:56:39 smithi112 ceph-mon[66819]: pgmap v818: 97 pgs: 97 active+clean; 517 MiB data, 5.8 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 5.7 MiB/s wr, 546 op/s 2024-03-19T01:56:39.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:56:39 smithi115 ceph-mon[67332]: pgmap v818: 97 pgs: 97 active+clean; 517 MiB data, 5.8 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 5.7 MiB/s wr, 546 op/s 2024-03-19T01:56:40.104 INFO:tasks.workunit.client.0.smithi099.stdout: 10 84440 23.41 MB/sec execute 69 sec latency 79.851 ms 2024-03-19T01:56:41.104 INFO:tasks.workunit.client.0.smithi099.stdout: 10 84794 23.26 MB/sec execute 70 sec latency 75.858 ms 2024-03-19T01:56:41.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:56:41 smithi099 ceph-mon[62951]: pgmap v819: 97 pgs: 97 active+clean; 517 MiB data, 5.8 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 5.8 MiB/s wr, 544 op/s 2024-03-19T01:56:41.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:56:41 smithi112 ceph-mon[66819]: pgmap v819: 97 pgs: 97 active+clean; 517 MiB data, 5.8 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 5.8 MiB/s wr, 544 op/s 2024-03-19T01:56:41.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:56:41 smithi115 ceph-mon[67332]: pgmap v819: 97 pgs: 97 active+clean; 517 MiB data, 5.8 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 5.8 MiB/s wr, 544 op/s 2024-03-19T01:56:42.104 INFO:tasks.workunit.client.0.smithi099.stdout: 10 85236 23.23 MB/sec execute 71 sec latency 146.386 ms 2024-03-19T01:56:43.104 INFO:tasks.workunit.client.0.smithi099.stdout: 10 85460 23.14 MB/sec execute 72 sec latency 240.343 ms 2024-03-19T01:56:43.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:56:43 smithi099 ceph-mon[62951]: pgmap v820: 97 pgs: 97 active+clean; 524 MiB data, 5.9 GiB used, 1.0 TiB / 1.0 TiB avail; 4.8 MiB/s rd, 6.8 MiB/s wr, 619 op/s 2024-03-19T01:56:43.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:56:43 smithi112 ceph-mon[66819]: pgmap v820: 97 pgs: 97 active+clean; 524 MiB data, 5.9 GiB used, 1.0 TiB / 1.0 TiB avail; 4.8 MiB/s rd, 6.8 MiB/s wr, 619 op/s 2024-03-19T01:56:43.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:56:43 smithi115 ceph-mon[67332]: pgmap v820: 97 pgs: 97 active+clean; 524 MiB data, 5.9 GiB used, 1.0 TiB / 1.0 TiB avail; 4.8 MiB/s rd, 6.8 MiB/s wr, 619 op/s 2024-03-19T01:56:44.104 INFO:tasks.workunit.client.0.smithi099.stdout: 10 85943 23.26 MB/sec execute 73 sec latency 64.253 ms 2024-03-19T01:56:45.104 INFO:tasks.workunit.client.0.smithi099.stdout: 10 86281 23.11 MB/sec execute 74 sec latency 60.821 ms 2024-03-19T01:56:45.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:56:45 smithi099 ceph-mon[62951]: pgmap v821: 97 pgs: 97 active+clean; 523 MiB data, 5.9 GiB used, 1.0 TiB / 1.0 TiB avail; 3.6 MiB/s rd, 5.5 MiB/s wr, 506 op/s 2024-03-19T01:56:45.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:56:45 smithi112 ceph-mon[66819]: pgmap v821: 97 pgs: 97 active+clean; 523 MiB data, 5.9 GiB used, 1.0 TiB / 1.0 TiB avail; 3.6 MiB/s rd, 5.5 MiB/s wr, 506 op/s 2024-03-19T01:56:45.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:56:45 smithi115 ceph-mon[67332]: pgmap v821: 97 pgs: 97 active+clean; 523 MiB data, 5.9 GiB used, 1.0 TiB / 1.0 TiB avail; 3.6 MiB/s rd, 5.5 MiB/s wr, 506 op/s 2024-03-19T01:56:46.104 INFO:tasks.workunit.client.0.smithi099.stdout: 10 86695 23.06 MB/sec execute 75 sec latency 71.015 ms 2024-03-19T01:56:46.390 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.388+0000 7f5d47cde640 1 -- 172.21.15.99:0/1297954349 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5d40102300 msgr2=0x7f5d401046f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:56:46.390 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.388+0000 7f5d47cde640 1 --2- 172.21.15.99:0/1297954349 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5d40102300 0x7f5d401046f0 secure :-1 s=READY pgs=648 cs=0 l=1 rev1=1 crypto rx=0x7f5d3c009900 tx=0x7f5d3c02bc30 comp rx=0 tx=0).stop 2024-03-19T01:56:46.390 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.388+0000 7f5d47cde640 1 -- 172.21.15.99:0/1297954349 shutdown_connections 2024-03-19T01:56:46.390 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.388+0000 7f5d47cde640 1 --2- 172.21.15.99:0/1297954349 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5d40107560 0x7f5d40109970 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:46.390 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.388+0000 7f5d47cde640 1 --2- 172.21.15.99:0/1297954349 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5d40104c30 0x7f5d40107020 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:46.390 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.388+0000 7f5d47cde640 1 --2- 172.21.15.99:0/1297954349 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5d40102300 0x7f5d401046f0 unknown :-1 s=CLOSED pgs=648 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:46.390 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.388+0000 7f5d47cde640 1 -- 172.21.15.99:0/1297954349 >> 172.21.15.99:0/1297954349 conn(0x7f5d400fbcb0 msgr2=0x7f5d400fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:56:46.390 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.388+0000 7f5d47cde640 1 -- 172.21.15.99:0/1297954349 shutdown_connections 2024-03-19T01:56:46.390 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.388+0000 7f5d47cde640 1 -- 172.21.15.99:0/1297954349 wait complete. 2024-03-19T01:56:46.391 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.388+0000 7f5d47cde640 1 Processor -- start 2024-03-19T01:56:46.391 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.389+0000 7f5d47cde640 1 -- start start 2024-03-19T01:56:46.391 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.389+0000 7f5d47cde640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5d40102300 0x7f5d40197b00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:46.391 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.389+0000 7f5d47cde640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5d40104c30 0x7f5d40198040 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:46.391 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.389+0000 7f5d47cde640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5d40107560 0x7f5d4019f0b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:46.391 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.389+0000 7f5d47cde640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f5d4010d4e0 con 0x7f5d40102300 2024-03-19T01:56:46.391 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.389+0000 7f5d47cde640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f5d4010d360 con 0x7f5d40107560 2024-03-19T01:56:46.391 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.389+0000 7f5d47cde640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f5d4010d660 con 0x7f5d40104c30 2024-03-19T01:56:46.391 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.389+0000 7f5d45a7a640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5d40102300 0x7f5d40197b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:46.391 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.389+0000 7f5d45a7a640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5d40102300 0x7f5d40197b00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:57674/0 (socket says 172.21.15.99:57674) 2024-03-19T01:56:46.391 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.389+0000 7f5d45a7a640 1 -- 172.21.15.99:0/3300591474 learned_addr learned my addr 172.21.15.99:0/3300591474 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:56:46.391 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.389+0000 7f5d4627b640 1 --2- 172.21.15.99:0/3300591474 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5d40107560 0x7f5d4019f0b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:46.391 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.389+0000 7f5d45279640 1 --2- 172.21.15.99:0/3300591474 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5d40104c30 0x7f5d40198040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:46.391 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.389+0000 7f5d45a7a640 1 -- 172.21.15.99:0/3300591474 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5d40104c30 msgr2=0x7f5d40198040 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:56:46.392 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.389+0000 7f5d45a7a640 1 --2- 172.21.15.99:0/3300591474 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5d40104c30 0x7f5d40198040 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:46.392 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.389+0000 7f5d45a7a640 1 -- 172.21.15.99:0/3300591474 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5d40107560 msgr2=0x7f5d4019f0b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:56:46.392 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.389+0000 7f5d45a7a640 1 --2- 172.21.15.99:0/3300591474 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5d40107560 0x7f5d4019f0b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:46.392 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.389+0000 7f5d45a7a640 1 -- 172.21.15.99:0/3300591474 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f5d4019f720 con 0x7f5d40102300 2024-03-19T01:56:46.392 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.389+0000 7f5d45a7a640 1 --2- 172.21.15.99:0/3300591474 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5d40102300 0x7f5d40197b00 secure :-1 s=READY pgs=649 cs=0 l=1 rev1=1 crypto rx=0x7f5d3c009230 tx=0x7f5d3c02c650 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:56:46.392 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.389+0000 7f5d36ffd640 1 -- 172.21.15.99:0/3300591474 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5d3c041070 con 0x7f5d40102300 2024-03-19T01:56:46.392 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.390+0000 7f5d36ffd640 1 -- 172.21.15.99:0/3300591474 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f5d3c004450 con 0x7f5d40102300 2024-03-19T01:56:46.392 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.390+0000 7f5d47cde640 1 -- 172.21.15.99:0/3300591474 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f5d4019f9b0 con 0x7f5d40102300 2024-03-19T01:56:46.392 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.390+0000 7f5d36ffd640 1 -- 172.21.15.99:0/3300591474 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5d3c003e20 con 0x7f5d40102300 2024-03-19T01:56:46.392 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.390+0000 7f5d47cde640 1 -- 172.21.15.99:0/3300591474 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f5d4019fef0 con 0x7f5d40102300 2024-03-19T01:56:46.397 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.392+0000 7f5d36ffd640 1 -- 172.21.15.99:0/3300591474 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f5d3c044070 con 0x7f5d40102300 2024-03-19T01:56:46.398 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.392+0000 7f5d4a99b740 1 Processor -- start 2024-03-19T01:56:46.398 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.392+0000 7f5d4a99b740 1 -- start start 2024-03-19T01:56:46.398 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.393+0000 7f5d4a99b740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55986b968100 0x55986b968520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:46.398 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.393+0000 7f5d4a99b740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55986b968af0 0x55986b96d5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:46.398 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.393+0000 7f5d4a99b740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55986b96dc60 0x55986b9700f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:46.398 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.393+0000 7f5d4a99b740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55986b8ccbe0 con 0x55986b968af0 2024-03-19T01:56:46.398 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.393+0000 7f5d4a99b740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55986b875960 con 0x55986b968100 2024-03-19T01:56:46.398 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.393+0000 7f5d4a99b740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55986b8c8c80 con 0x55986b96dc60 2024-03-19T01:56:46.398 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.393+0000 7f5d36ffd640 1 --2- 172.21.15.99:0/3300591474 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f5d2c0a2920 0x7f5d2c0a4de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:46.398 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.393+0000 7f5d4627b640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55986b968100 0x55986b968520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:46.398 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.393+0000 7f5d4627b640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55986b968100 0x55986b968520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:60848/0 (socket says 172.21.15.99:60848) 2024-03-19T01:56:46.398 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.393+0000 7f5d45279640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55986b96dc60 0x55986b9700f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:46.398 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.393+0000 7f5d4627b640 1 -- 172.21.15.99:0/107367522 learned_addr learned my addr 172.21.15.99:0/107367522 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:56:46.399 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.393+0000 7f5d45a7a640 1 --2- 172.21.15.99:0/107367522 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55986b968af0 0x55986b96d5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:46.399 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.393+0000 7f5d4627b640 1 --2- 172.21.15.99:0/3300591474 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f5d2c0a2920 0x7f5d2c0a4de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:46.399 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.393+0000 7f5d36ffd640 1 -- 172.21.15.99:0/3300591474 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f5d3c0f05b0 con 0x7f5d40102300 2024-03-19T01:56:46.399 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.393+0000 7f5d45279640 1 -- 172.21.15.99:0/107367522 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55986b968100 msgr2=0x55986b968520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:56:46.399 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.393+0000 7f5d45279640 1 --2- 172.21.15.99:0/107367522 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55986b968100 0x55986b968520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:46.399 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.393+0000 7f5d45279640 1 -- 172.21.15.99:0/107367522 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55986b968af0 msgr2=0x55986b96d5f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:56:46.399 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.393+0000 7f5d45279640 1 --2- 172.21.15.99:0/107367522 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55986b968af0 0x55986b96d5f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:46.399 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.393+0000 7f5d45279640 1 -- 172.21.15.99:0/107367522 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55986b86cd20 con 0x55986b96dc60 2024-03-19T01:56:46.399 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.393+0000 7f5d4627b640 1 --2- 172.21.15.99:0/107367522 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55986b968100 0x55986b968520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:56:46.399 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.393+0000 7f5d45279640 1 --2- 172.21.15.99:0/107367522 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55986b96dc60 0x55986b9700f0 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7f5d400699c0 tx=0x7f5d30009c10 comp rx=0 tx=0).ready entity=mon.1 client_cookie=14737f0b8365f274 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:56:46.399 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.394+0000 7f5d4627b640 1 --2- 172.21.15.99:0/3300591474 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f5d2c0a2920 0x7f5d2c0a4de0 secure :-1 s=READY pgs=343 cs=0 l=1 rev1=1 crypto rx=0x55986b969350 tx=0x7f5d38004ab0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:56:46.399 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.394+0000 7f5d2bfff640 1 -- 172.21.15.99:0/107367522 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5d30004050 con 0x55986b96dc60 2024-03-19T01:56:46.399 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.394+0000 7f5d2bfff640 1 -- 172.21.15.99:0/107367522 <== mon.1 v2:172.21.15.115:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f5d3000ee10 con 0x55986b96dc60 2024-03-19T01:56:46.399 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.394+0000 7f5d2bfff640 1 -- 172.21.15.99:0/107367522 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5d30003c20 con 0x55986b96dc60 2024-03-19T01:56:46.400 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.394+0000 7f5d4a99b740 1 -- 172.21.15.99:0/107367522 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55986b96dc60 msgr2=0x55986b9700f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:56:46.400 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.394+0000 7f5d4a99b740 1 --2- 172.21.15.99:0/107367522 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55986b96dc60 0x55986b9700f0 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7f5d400699c0 tx=0x7f5d30009c10 comp rx=0 tx=0).stop 2024-03-19T01:56:46.400 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.395+0000 7f5d4a99b740 1 -- 172.21.15.99:0/107367522 shutdown_connections 2024-03-19T01:56:46.400 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.395+0000 7f5d4a99b740 1 --2- 172.21.15.99:0/107367522 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55986b96dc60 0x55986b9700f0 unknown :-1 s=CLOSED pgs=119 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:46.400 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.395+0000 7f5d4a99b740 1 --2- 172.21.15.99:0/107367522 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55986b968af0 0x55986b96d5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:46.400 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.395+0000 7f5d4a99b740 1 --2- 172.21.15.99:0/107367522 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55986b968100 0x55986b968520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:46.400 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.395+0000 7f5d4a99b740 1 -- 172.21.15.99:0/107367522 >> 172.21.15.99:0/107367522 conn(0x55986b963670 msgr2=0x55986b963a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:56:46.400 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.395+0000 7f5d4a99b740 1 -- 172.21.15.99:0/107367522 shutdown_connections 2024-03-19T01:56:46.400 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.395+0000 7f5d4a99b740 1 -- 172.21.15.99:0/107367522 wait complete. 2024-03-19T01:56:46.400 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.395+0000 7f5d4a99b740 1 -- start start 2024-03-19T01:56:46.400 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.395+0000 7f5d4a99b740 1 Processor -- start 2024-03-19T01:56:46.400 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.395+0000 7f5d4a99b740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55986b968100 0x55986b96ae10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:46.400 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.395+0000 7f5d4a99b740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55986b968af0 0x55986b96b350 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:46.400 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.395+0000 7f5d4a99b740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55986b96dc60 0x55986b96c360 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:46.401 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.395+0000 7f5d4a99b740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55986b970b50 con 0x55986b968100 2024-03-19T01:56:46.401 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.395+0000 7f5d4a99b740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55986b888db0 con 0x55986b968af0 2024-03-19T01:56:46.401 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.395+0000 7f5d4a99b740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55986b970cd0 con 0x55986b96dc60 2024-03-19T01:56:46.401 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.395+0000 7f5d4a99b740 10 client.0 _ll_register_callbacks cb 0x55986b88de70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:56:46.401 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.396+0000 7f5d45279640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55986b968af0 0x55986b96b350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:46.401 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.396+0000 7f5d45a7a640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55986b968100 0x55986b96ae10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:46.401 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.396+0000 7f5d4627b640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55986b96dc60 0x55986b96c360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:46.401 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.396+0000 7f5d45279640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55986b968af0 0x55986b96b350 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:60862/0 (socket says 172.21.15.99:60862) 2024-03-19T01:56:46.401 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.396+0000 7f5d45279640 1 -- 172.21.15.99:0/496547893 learned_addr learned my addr 172.21.15.99:0/496547893 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:56:46.401 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.396+0000 7f5d45a7a640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55986b968100 0x55986b96ae10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:57694/0 (socket says 172.21.15.99:57694) 2024-03-19T01:56:46.401 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.396+0000 7f5d45a7a640 1 -- 172.21.15.99:0/496547893 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55986b96dc60 msgr2=0x55986b96c360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:56:46.401 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.396+0000 7f5d45a7a640 1 --2- 172.21.15.99:0/496547893 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55986b96dc60 0x55986b96c360 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:46.401 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.396+0000 7f5d45a7a640 1 -- 172.21.15.99:0/496547893 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55986b968af0 msgr2=0x55986b96b350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:56:46.402 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.396+0000 7f5d45a7a640 1 --2- 172.21.15.99:0/496547893 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55986b968af0 0x55986b96b350 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:46.402 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.396+0000 7f5d45a7a640 1 -- 172.21.15.99:0/496547893 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55986b887c60 con 0x55986b968100 2024-03-19T01:56:46.402 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.396+0000 7f5d45a7a640 1 --2- 172.21.15.99:0/496547893 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55986b968100 0x55986b96ae10 secure :-1 s=READY pgs=650 cs=0 l=1 rev1=1 crypto rx=0x7f5d3c009430 tx=0x7f5d3c02bc30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=eff4f10eb42be1f6 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:56:46.402 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.396+0000 7f5d297fa640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:56:46.402 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.396+0000 7f5d297fa640 1 -- 172.21.15.99:0/496547893 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5d3c04b070 con 0x55986b968100 2024-03-19T01:56:46.402 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.396+0000 7f5d297fa640 1 -- 172.21.15.99:0/496547893 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f5d3c0c5220 con 0x55986b968100 2024-03-19T01:56:46.402 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.396+0000 7f5d4a99b740 1 -- 172.21.15.99:0/496547893 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f5d3000ee10 con 0x55986b968100 2024-03-19T01:56:46.402 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.396+0000 7f5d297fa640 1 -- 172.21.15.99:0/496547893 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7f5d3c0c1060 con 0x55986b968100 2024-03-19T01:56:46.402 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.396+0000 7f5d4a99b740 10 client.16602 fetch_fsmap learned FSMap version 37 2024-03-19T01:56:46.402 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.396+0000 7f5d4a99b740 1 -- 172.21.15.99:0/496547893 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x55986b5e1210 con 0x55986b968100 2024-03-19T01:56:46.402 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.396+0000 7f5d297fa640 1 -- 172.21.15.99:0/496547893 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f5d3000ee10 con 0x55986b968100 2024-03-19T01:56:46.402 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.397+0000 7f5d297fa640 1 -- 172.21.15.99:0/496547893 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7f5d3c0d2020 con 0x55986b968100 2024-03-19T01:56:46.402 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.397+0000 7f5d4a99b740 10 client.16602 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T01:56:46.402 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.397+0000 7f5d4a99b740 10 client.16602 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:56:46.403 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.397+0000 7f5d4a99b740 20 client.16602 populate_metadata read hostname 'smithi099' 2024-03-19T01:56:46.403 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.397+0000 7f5d4a99b740 1 --2- 172.21.15.99:0/496547893 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55986b97def0 0x55986b980360 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:46.403 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.397+0000 7f5d4a99b740 4 client.16602 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:56:46.403 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.397+0000 7f5d4a99b740 1 -- 172.21.15.99:0/496547893 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55986b8da120 con 0x55986b97def0 2024-03-19T01:56:46.403 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.397+0000 7f5d45279640 1 --2- 172.21.15.99:0/496547893 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55986b97def0 0x55986b980360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:46.403 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.397+0000 7f5d297fa640 10 client.16602 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:56:46.403 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.397+0000 7f5d45279640 1 --2- 172.21.15.99:0/496547893 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55986b97def0 0x55986b980360 crc :-1 s=READY pgs=169 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=55adab7ba2f522ae server_cookie=5691376955566064 in_seq=0 out_seq=0 2024-03-19T01:56:46.404 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.400+0000 7f5d297fa640 1 -- 172.21.15.99:0/496547893 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x55986b8da120 con 0x55986b97def0 2024-03-19T01:56:46.404 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.400+0000 7f5d297fa640 10 client.16602 handle_command_reply: tid=0 2024-03-19T01:56:46.404 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.400+0000 7f5d4a99b740 1 client.16602 shutdown 2024-03-19T01:56:46.404 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.400+0000 7f5d4a99b740 1 -- 172.21.15.99:0/496547893 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55986b968100 msgr2=0x55986b96ae10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:56:46.404 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.400+0000 7f5d4a99b740 1 --2- 172.21.15.99:0/496547893 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55986b968100 0x55986b96ae10 secure :-1 s=READY pgs=650 cs=0 l=1 rev1=1 crypto rx=0x7f5d3c009430 tx=0x7f5d3c02bc30 comp rx=0 tx=0).stop 2024-03-19T01:56:46.405 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.401+0000 7f5d4a99b740 1 -- 172.21.15.99:0/496547893 shutdown_connections 2024-03-19T01:56:46.405 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.401+0000 7f5d4a99b740 1 --2- 172.21.15.99:0/496547893 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55986b97def0 0x55986b980360 crc :-1 s=READY pgs=169 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:46.405 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.401+0000 7f5d4a99b740 1 --2- 172.21.15.99:0/496547893 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55986b968af0 0x55986b96b350 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:46.405 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.401+0000 7f5d297fa640 0 client.16602 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:56:46.405 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.401+0000 7f5d4a99b740 1 --2- 172.21.15.99:0/496547893 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55986b96dc60 0x55986b96c360 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:46.405 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.401+0000 7f5d4a99b740 1 --2- 172.21.15.99:0/496547893 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55986b968100 0x55986b96ae10 unknown :-1 s=CLOSED pgs=650 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:46.405 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.401+0000 7f5d4a99b740 1 -- 172.21.15.99:0/496547893 >> 172.21.15.99:0/496547893 conn(0x55986b963670 msgr2=0x55986b9648c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:56:46.405 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.401+0000 7f5d4a99b740 1 -- 172.21.15.99:0/496547893 shutdown_connections 2024-03-19T01:56:46.405 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.401+0000 7f5d4a99b740 1 -- 172.21.15.99:0/496547893 wait complete. 2024-03-19T01:56:46.405 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.401+0000 7f5d4a99b740 20 client.16602 trim_cache size 0 max 16384 2024-03-19T01:56:46.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.415+0000 7f5d4a99b740 1 Processor -- start 2024-03-19T01:56:46.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.415+0000 7f5d4a99b740 1 -- start start 2024-03-19T01:56:46.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.416+0000 7f5d4a99b740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55986b968100 0x55986b96aa20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:46.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.416+0000 7f5d4a99b740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55986b968af0 0x55986b96af60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:46.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.416+0000 7f5d4a99b740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55986b96dc60 0x55986b971070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:46.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.416+0000 7f5d4a99b740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f5d401a93b0 con 0x55986b968af0 2024-03-19T01:56:46.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.416+0000 7f5d4a99b740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f5d401a6c80 con 0x55986b96dc60 2024-03-19T01:56:46.418 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.416+0000 7f5d4a99b740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f5d401a4ad0 con 0x55986b968100 2024-03-19T01:56:46.418 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.416+0000 7f5d45279640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55986b968af0 0x55986b96af60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:46.418 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.416+0000 7f5d45279640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55986b968af0 0x55986b96af60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:57704/0 (socket says 172.21.15.99:57704) 2024-03-19T01:56:46.418 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.416+0000 7f5d45279640 1 -- 172.21.15.99:0/3025190111 learned_addr learned my addr 172.21.15.99:0/3025190111 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:56:46.418 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.416+0000 7f5d45a7a640 1 --2- 172.21.15.99:0/3025190111 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55986b968100 0x55986b96aa20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:46.418 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.416+0000 7f5d4627b640 1 --2- 172.21.15.99:0/3025190111 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55986b96dc60 0x55986b971070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:46.418 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.416+0000 7f5d45279640 1 -- 172.21.15.99:0/3025190111 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55986b968100 msgr2=0x55986b96aa20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:56:46.418 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.416+0000 7f5d45279640 1 --2- 172.21.15.99:0/3025190111 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55986b968100 0x55986b96aa20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:46.418 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.416+0000 7f5d45279640 1 -- 172.21.15.99:0/3025190111 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55986b96dc60 msgr2=0x55986b971070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:56:46.418 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.416+0000 7f5d45279640 1 --2- 172.21.15.99:0/3025190111 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55986b96dc60 0x55986b971070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:46.418 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.416+0000 7f5d45279640 1 -- 172.21.15.99:0/3025190111 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55986b6ad620 con 0x55986b968af0 2024-03-19T01:56:46.418 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.416+0000 7f5d45279640 1 --2- 172.21.15.99:0/3025190111 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55986b968af0 0x55986b96af60 secure :-1 s=READY pgs=651 cs=0 l=1 rev1=1 crypto rx=0x55986b96c2f0 tx=0x7f5d30005d30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e7c50e9bd5738b52 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:56:46.418 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.416+0000 7f5d2b7fe640 1 -- 172.21.15.99:0/3025190111 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5d30017bd0 con 0x55986b968af0 2024-03-19T01:56:46.419 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.416+0000 7f5d2b7fe640 1 -- 172.21.15.99:0/3025190111 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f5d30036070 con 0x55986b968af0 2024-03-19T01:56:46.419 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.416+0000 7f5d2b7fe640 1 -- 172.21.15.99:0/3025190111 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5d30005040 con 0x55986b968af0 2024-03-19T01:56:46.419 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.417+0000 7f5d4a99b740 1 -- 172.21.15.99:0/3025190111 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55986b968af0 msgr2=0x55986b96af60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:56:46.419 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.417+0000 7f5d4a99b740 1 --2- 172.21.15.99:0/3025190111 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55986b968af0 0x55986b96af60 secure :-1 s=READY pgs=651 cs=0 l=1 rev1=1 crypto rx=0x55986b96c2f0 tx=0x7f5d30005d30 comp rx=0 tx=0).stop 2024-03-19T01:56:46.419 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.417+0000 7f5d4a99b740 1 -- 172.21.15.99:0/3025190111 shutdown_connections 2024-03-19T01:56:46.419 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.417+0000 7f5d4a99b740 1 --2- 172.21.15.99:0/3025190111 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55986b96dc60 0x55986b971070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:46.419 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.417+0000 7f5d4a99b740 1 --2- 172.21.15.99:0/3025190111 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55986b968af0 0x55986b96af60 unknown :-1 s=CLOSED pgs=651 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:46.419 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.417+0000 7f5d4a99b740 1 --2- 172.21.15.99:0/3025190111 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55986b968100 0x55986b96aa20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:46.419 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.417+0000 7f5d4a99b740 1 -- 172.21.15.99:0/3025190111 >> 172.21.15.99:0/3025190111 conn(0x55986b963670 msgr2=0x55986b963a50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:56:46.419 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.417+0000 7f5d4a99b740 1 -- 172.21.15.99:0/3025190111 shutdown_connections 2024-03-19T01:56:46.419 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.417+0000 7f5d4a99b740 1 -- 172.21.15.99:0/3025190111 wait complete. 2024-03-19T01:56:46.419 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.417+0000 7f5d4a99b740 1 -- start start 2024-03-19T01:56:46.419 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.417+0000 7f5d4a99b740 1 Processor -- start 2024-03-19T01:56:46.422 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.418+0000 7f5d4a99b740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55986b968100 0x55986b977f50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:46.422 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.418+0000 7f5d4a99b740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55986b968af0 0x55986b978490 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:46.422 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.418+0000 7f5d4a99b740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55986b96dc60 0x55986b979400 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:46.422 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.418+0000 7f5d4a99b740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f5d401a5140 con 0x55986b968af0 2024-03-19T01:56:46.422 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.418+0000 7f5d4a99b740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f5d401a65b0 con 0x55986b96dc60 2024-03-19T01:56:46.422 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.418+0000 7f5d4a99b740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f5d401a5810 con 0x55986b968100 2024-03-19T01:56:46.422 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.418+0000 7f5d4a99b740 10 client.0 _ll_register_callbacks cb 0x7f5d3c009560 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:56:46.422 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.418+0000 7f5d45279640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55986b968af0 0x55986b978490 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:46.423 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.418+0000 7f5d45279640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55986b968af0 0x55986b978490 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:57714/0 (socket says 172.21.15.99:57714) 2024-03-19T01:56:46.423 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.418+0000 7f5d45279640 1 -- 172.21.15.99:0/2743211769 learned_addr learned my addr 172.21.15.99:0/2743211769 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:56:46.423 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.418+0000 7f5d45a7a640 1 --2- 172.21.15.99:0/2743211769 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55986b968100 0x55986b977f50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:46.423 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.418+0000 7f5d4627b640 1 --2- 172.21.15.99:0/2743211769 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55986b96dc60 0x55986b979400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:46.423 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.418+0000 7f5d45279640 1 -- 172.21.15.99:0/2743211769 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55986b968100 msgr2=0x55986b977f50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:56:46.423 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.418+0000 7f5d45279640 1 --2- 172.21.15.99:0/2743211769 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55986b968100 0x55986b977f50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:46.423 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.418+0000 7f5d45279640 1 -- 172.21.15.99:0/2743211769 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55986b96dc60 msgr2=0x55986b979400 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:56:46.423 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.418+0000 7f5d45279640 1 --2- 172.21.15.99:0/2743211769 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55986b96dc60 0x55986b979400 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:46.423 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.418+0000 7f5d45279640 1 -- 172.21.15.99:0/2743211769 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55986b9799f0 con 0x55986b968af0 2024-03-19T01:56:46.423 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.418+0000 7f5d45279640 1 --2- 172.21.15.99:0/2743211769 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55986b968af0 0x55986b978490 secure :-1 s=READY pgs=652 cs=0 l=1 rev1=1 crypto rx=0x7f5d30009710 tx=0x7f5d30017bd0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d9461a45adb8d998 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:56:46.423 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.418+0000 7f5d297fa640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:56:46.423 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.418+0000 7f5d297fa640 1 -- 172.21.15.99:0/2743211769 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5d3000eb00 con 0x55986b968af0 2024-03-19T01:56:46.423 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.418+0000 7f5d297fa640 1 -- 172.21.15.99:0/2743211769 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f5d3001d040 con 0x55986b968af0 2024-03-19T01:56:46.423 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.419+0000 7f5d4a99b740 1 -- 172.21.15.99:0/2743211769 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f5d30036070 con 0x55986b968af0 2024-03-19T01:56:46.424 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.419+0000 7f5d297fa640 1 -- 172.21.15.99:0/2743211769 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7f5d30031220 con 0x55986b968af0 2024-03-19T01:56:46.424 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.419+0000 7f5d4a99b740 10 client.16608 fetch_fsmap learned FSMap version 37 2024-03-19T01:56:46.424 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.419+0000 7f5d4a99b740 1 -- 172.21.15.99:0/2743211769 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x55986b9e42a0 con 0x55986b968af0 2024-03-19T01:56:46.424 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.419+0000 7f5d297fa640 1 -- 172.21.15.99:0/2743211769 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f5d30036070 con 0x55986b968af0 2024-03-19T01:56:46.424 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.419+0000 7f5d297fa640 1 -- 172.21.15.99:0/2743211769 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7f5d30003730 con 0x55986b968af0 2024-03-19T01:56:46.424 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.419+0000 7f5d4a99b740 10 client.16608 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T01:56:46.424 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.419+0000 7f5d4a99b740 10 client.16608 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:56:46.424 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.419+0000 7f5d4a99b740 20 client.16608 populate_metadata read hostname 'smithi099' 2024-03-19T01:56:46.424 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.419+0000 7f5d4a99b740 1 --2- 172.21.15.99:0/2743211769 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55986b9e4a70 0x55986b9e6e90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:46.424 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.419+0000 7f5d4a99b740 4 client.16608 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:56:46.424 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.419+0000 7f5d4a99b740 1 -- 172.21.15.99:0/2743211769 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55986b8d4470 con 0x55986b9e4a70 2024-03-19T01:56:46.424 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.419+0000 7f5d45a7a640 1 --2- 172.21.15.99:0/2743211769 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55986b9e4a70 0x55986b9e6e90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:46.424 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.420+0000 7f5d297fa640 10 client.16608 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:56:46.424 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.420+0000 7f5d45a7a640 1 --2- 172.21.15.99:0/2743211769 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55986b9e4a70 0x55986b9e6e90 crc :-1 s=READY pgs=170 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=9051244f3d0b4b69 server_cookie=f570748cfe29a5a7 in_seq=0 out_seq=0 2024-03-19T01:56:46.880 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.880+0000 7f5d297fa640 1 -- 172.21.15.99:0/2743211769 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x55986b8d4470 con 0x55986b9e4a70 2024-03-19T01:56:46.880 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.880+0000 7f5d297fa640 10 client.16608 handle_command_reply: tid=0 2024-03-19T01:56:46.881 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.880+0000 7f5d4a99b740 1 client.16608 shutdown 2024-03-19T01:56:46.881 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.880+0000 7f5d4a99b740 1 -- 172.21.15.99:0/2743211769 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55986b968af0 msgr2=0x55986b978490 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:56:46.881 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.880+0000 7f5d4a99b740 1 --2- 172.21.15.99:0/2743211769 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55986b968af0 0x55986b978490 secure :-1 s=READY pgs=652 cs=0 l=1 rev1=1 crypto rx=0x7f5d30009710 tx=0x7f5d30017bd0 comp rx=0 tx=0).stop 2024-03-19T01:56:46.881 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.880+0000 7f5d4a99b740 1 -- 172.21.15.99:0/2743211769 shutdown_connections 2024-03-19T01:56:46.881 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.880+0000 7f5d4a99b740 1 --2- 172.21.15.99:0/2743211769 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55986b9e4a70 0x55986b9e6e90 crc :-1 s=READY pgs=170 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:46.881 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.880+0000 7f5d4a99b740 1 --2- 172.21.15.99:0/2743211769 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55986b96dc60 0x55986b979400 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:46.881 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.880+0000 7f5d297fa640 0 client.16608 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:56:46.881 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.880+0000 7f5d4a99b740 1 --2- 172.21.15.99:0/2743211769 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55986b968af0 0x55986b978490 unknown :-1 s=CLOSED pgs=652 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:46.881 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.880+0000 7f5d4a99b740 1 --2- 172.21.15.99:0/2743211769 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55986b968100 0x55986b977f50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:46.881 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.880+0000 7f5d4a99b740 1 -- 172.21.15.99:0/2743211769 >> 172.21.15.99:0/2743211769 conn(0x55986b963670 msgr2=0x55986b96a9d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:56:46.881 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.880+0000 7f5d4a99b740 1 -- 172.21.15.99:0/2743211769 shutdown_connections 2024-03-19T01:56:46.881 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.880+0000 7f5d4a99b740 1 -- 172.21.15.99:0/2743211769 wait complete. 2024-03-19T01:56:46.882 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.880+0000 7f5d4a99b740 20 client.16608 trim_cache size 0 max 16384 2024-03-19T01:56:46.882 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:56:46.882 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:56:46.882 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:56:46.882 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:56:46.883 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.881+0000 7f5d47cde640 1 -- 172.21.15.99:0/3300591474 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f5d2c0a2920 msgr2=0x7f5d2c0a4de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:56:46.883 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.881+0000 7f5d47cde640 1 --2- 172.21.15.99:0/3300591474 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f5d2c0a2920 0x7f5d2c0a4de0 secure :-1 s=READY pgs=343 cs=0 l=1 rev1=1 crypto rx=0x55986b969350 tx=0x7f5d38004ab0 comp rx=0 tx=0).stop 2024-03-19T01:56:46.883 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.881+0000 7f5d47cde640 1 -- 172.21.15.99:0/3300591474 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5d40102300 msgr2=0x7f5d40197b00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:56:46.883 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.881+0000 7f5d47cde640 1 --2- 172.21.15.99:0/3300591474 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5d40102300 0x7f5d40197b00 secure :-1 s=READY pgs=649 cs=0 l=1 rev1=1 crypto rx=0x7f5d3c009230 tx=0x7f5d3c02c650 comp rx=0 tx=0).stop 2024-03-19T01:56:46.883 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.881+0000 7f5d47cde640 1 -- 172.21.15.99:0/3300591474 shutdown_connections 2024-03-19T01:56:46.883 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.881+0000 7f5d47cde640 1 --2- 172.21.15.99:0/3300591474 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f5d2c0a2920 0x7f5d2c0a4de0 unknown :-1 s=CLOSED pgs=343 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:46.883 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.881+0000 7f5d47cde640 1 --2- 172.21.15.99:0/3300591474 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f5d40107560 0x7f5d4019f0b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:46.883 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.881+0000 7f5d47cde640 1 --2- 172.21.15.99:0/3300591474 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f5d40104c30 0x7f5d40198040 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:46.883 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.881+0000 7f5d47cde640 1 --2- 172.21.15.99:0/3300591474 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f5d40102300 0x7f5d40197b00 unknown :-1 s=CLOSED pgs=649 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:46.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.881+0000 7f5d47cde640 1 -- 172.21.15.99:0/3300591474 >> 172.21.15.99:0/3300591474 conn(0x7f5d400fbcb0 msgr2=0x7f5d40102c40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:56:46.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.881+0000 7f5d47cde640 1 -- 172.21.15.99:0/3300591474 shutdown_connections 2024-03-19T01:56:46.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:46.881+0000 7f5d47cde640 1 -- 172.21.15.99:0/3300591474 wait complete. 2024-03-19T01:56:47.104 INFO:tasks.workunit.client.0.smithi099.stdout: 10 86969 22.92 MB/sec execute 76 sec latency 317.604 ms 2024-03-19T01:56:47.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:56:47 smithi099 ceph-mon[62951]: pgmap v822: 97 pgs: 97 active+clean; 531 MiB data, 5.9 GiB used, 1.0 TiB / 1.0 TiB avail; 4.6 MiB/s rd, 6.7 MiB/s wr, 598 op/s 2024-03-19T01:56:47.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:56:47 smithi112 ceph-mon[66819]: pgmap v822: 97 pgs: 97 active+clean; 531 MiB data, 5.9 GiB used, 1.0 TiB / 1.0 TiB avail; 4.6 MiB/s rd, 6.7 MiB/s wr, 598 op/s 2024-03-19T01:56:47.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:56:47 smithi115 ceph-mon[67332]: pgmap v822: 97 pgs: 97 active+clean; 531 MiB data, 5.9 GiB used, 1.0 TiB / 1.0 TiB avail; 4.6 MiB/s rd, 6.7 MiB/s wr, 598 op/s 2024-03-19T01:56:48.104 INFO:tasks.workunit.client.0.smithi099.stdout: 10 87270 22.74 MB/sec execute 77 sec latency 270.759 ms 2024-03-19T01:56:49.104 INFO:tasks.workunit.client.0.smithi099.stdout: 10 87853 22.87 MB/sec execute 78 sec latency 91.010 ms 2024-03-19T01:56:49.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:56:49 smithi099 ceph-mon[62951]: pgmap v823: 97 pgs: 97 active+clean; 530 MiB data, 5.9 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 5.9 MiB/s wr, 484 op/s 2024-03-19T01:56:49.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:56:49 smithi112 ceph-mon[66819]: pgmap v823: 97 pgs: 97 active+clean; 530 MiB data, 5.9 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 5.9 MiB/s wr, 484 op/s 2024-03-19T01:56:49.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:56:49 smithi115 ceph-mon[67332]: pgmap v823: 97 pgs: 97 active+clean; 530 MiB data, 5.9 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 5.9 MiB/s wr, 484 op/s 2024-03-19T01:56:50.104 INFO:tasks.workunit.client.0.smithi099.stdout: 10 88444 23.12 MB/sec execute 79 sec latency 55.292 ms 2024-03-19T01:56:51.104 INFO:tasks.workunit.client.0.smithi099.stdout: 10 88952 23.22 MB/sec execute 80 sec latency 61.974 ms 2024-03-19T01:56:51.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:56:51 smithi099 ceph-mon[62951]: pgmap v824: 97 pgs: 97 active+clean; 524 MiB data, 5.9 GiB used, 1.0 TiB / 1.0 TiB avail; 4.3 MiB/s rd, 6.1 MiB/s wr, 515 op/s 2024-03-19T01:56:51.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:56:51 smithi112 ceph-mon[66819]: pgmap v824: 97 pgs: 97 active+clean; 524 MiB data, 5.9 GiB used, 1.0 TiB / 1.0 TiB avail; 4.3 MiB/s rd, 6.1 MiB/s wr, 515 op/s 2024-03-19T01:56:51.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:56:51 smithi115 ceph-mon[67332]: pgmap v824: 97 pgs: 97 active+clean; 524 MiB data, 5.9 GiB used, 1.0 TiB / 1.0 TiB avail; 4.3 MiB/s rd, 6.1 MiB/s wr, 515 op/s 2024-03-19T01:56:52.104 INFO:tasks.workunit.client.0.smithi099.stdout: 10 89366 23.16 MB/sec execute 81 sec latency 93.487 ms 2024-03-19T01:56:53.104 INFO:tasks.workunit.client.0.smithi099.stdout: 10 89649 23.07 MB/sec execute 82 sec latency 247.344 ms 2024-03-19T01:56:53.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:56:53 smithi099 ceph-mon[62951]: pgmap v825: 97 pgs: 97 active+clean; 525 MiB data, 6.0 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 MiB/s rd, 7.1 MiB/s wr, 619 op/s 2024-03-19T01:56:53.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:56:53 smithi112 ceph-mon[66819]: pgmap v825: 97 pgs: 97 active+clean; 525 MiB data, 6.0 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 MiB/s rd, 7.1 MiB/s wr, 619 op/s 2024-03-19T01:56:53.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:56:53 smithi115 ceph-mon[67332]: pgmap v825: 97 pgs: 97 active+clean; 525 MiB data, 6.0 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 MiB/s rd, 7.1 MiB/s wr, 619 op/s 2024-03-19T01:56:54.104 INFO:tasks.workunit.client.0.smithi099.stdout: 10 90088 23.05 MB/sec execute 83 sec latency 72.856 ms 2024-03-19T01:56:55.104 INFO:tasks.workunit.client.0.smithi099.stdout: 10 90603 23.17 MB/sec execute 84 sec latency 87.010 ms 2024-03-19T01:56:55.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:56:55 smithi099 ceph-mon[62951]: pgmap v826: 97 pgs: 97 active+clean; 525 MiB data, 6.0 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 5.8 MiB/s wr, 526 op/s 2024-03-19T01:56:55.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:56:55 smithi112 ceph-mon[66819]: pgmap v826: 97 pgs: 97 active+clean; 525 MiB data, 6.0 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 5.8 MiB/s wr, 526 op/s 2024-03-19T01:56:55.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:56:55 smithi115 ceph-mon[67332]: pgmap v826: 97 pgs: 97 active+clean; 525 MiB data, 6.0 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 5.8 MiB/s wr, 526 op/s 2024-03-19T01:56:56.104 INFO:tasks.workunit.client.0.smithi099.stdout: 10 91032 23.09 MB/sec execute 85 sec latency 71.821 ms 2024-03-19T01:56:57.106 INFO:tasks.workunit.client.0.smithi099.stdout: 10 91450 23.07 MB/sec execute 86 sec latency 148.199 ms 2024-03-19T01:56:57.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:56:57 smithi099 ceph-mon[62951]: pgmap v827: 97 pgs: 97 active+clean; 530 MiB data, 6.0 GiB used, 1.0 TiB / 1.0 TiB avail; 5.2 MiB/s rd, 7.0 MiB/s wr, 604 op/s 2024-03-19T01:56:57.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:56:57 smithi112 ceph-mon[66819]: pgmap v827: 97 pgs: 97 active+clean; 530 MiB data, 6.0 GiB used, 1.0 TiB / 1.0 TiB avail; 5.2 MiB/s rd, 7.0 MiB/s wr, 604 op/s 2024-03-19T01:56:57.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:56:57 smithi115 ceph-mon[67332]: pgmap v827: 97 pgs: 97 active+clean; 530 MiB data, 6.0 GiB used, 1.0 TiB / 1.0 TiB avail; 5.2 MiB/s rd, 7.0 MiB/s wr, 604 op/s 2024-03-19T01:56:58.105 INFO:tasks.workunit.client.0.smithi099.stdout: 10 91753 23.07 MB/sec execute 87 sec latency 350.073 ms 2024-03-19T01:56:59.104 INFO:tasks.workunit.client.0.smithi099.stdout: 10 92287 22.94 MB/sec execute 88 sec latency 138.994 ms 2024-03-19T01:56:59.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:56:59 smithi099 ceph-mon[62951]: pgmap v828: 97 pgs: 97 active+clean; 535 MiB data, 6.0 GiB used, 1.0 TiB / 1.0 TiB avail; 4.4 MiB/s rd, 6.2 MiB/s wr, 512 op/s 2024-03-19T01:56:59.357 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:56:59 smithi115 ceph-mon[67332]: pgmap v828: 97 pgs: 97 active+clean; 535 MiB data, 6.0 GiB used, 1.0 TiB / 1.0 TiB avail; 4.4 MiB/s rd, 6.2 MiB/s wr, 512 op/s 2024-03-19T01:56:59.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:56:59 smithi112 ceph-mon[66819]: pgmap v828: 97 pgs: 97 active+clean; 535 MiB data, 6.0 GiB used, 1.0 TiB / 1.0 TiB avail; 4.4 MiB/s rd, 6.2 MiB/s wr, 512 op/s 2024-03-19T01:56:59.983 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.981+0000 7f8fe4412640 1 -- 172.21.15.99:0/757315529 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8fdc1023a0 msgr2=0x7f8fdc104790 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:56:59.983 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.981+0000 7f8fe4412640 1 --2- 172.21.15.99:0/757315529 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8fdc1023a0 0x7f8fdc104790 secure :-1 s=READY pgs=653 cs=0 l=1 rev1=1 crypto rx=0x7f8fd8009900 tx=0x7f8fd802bc30 comp rx=0 tx=0).stop 2024-03-19T01:56:59.983 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.981+0000 7f8fe4412640 1 -- 172.21.15.99:0/757315529 shutdown_connections 2024-03-19T01:56:59.983 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.981+0000 7f8fe4412640 1 --2- 172.21.15.99:0/757315529 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8fdc104cd0 0x7f8fdc1070c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:59.983 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.981+0000 7f8fe4412640 1 --2- 172.21.15.99:0/757315529 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8fdc107600 0x7f8fdc1099f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:59.984 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.981+0000 7f8fe4412640 1 --2- 172.21.15.99:0/757315529 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8fdc1023a0 0x7f8fdc104790 unknown :-1 s=CLOSED pgs=653 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:59.984 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.981+0000 7f8fe4412640 1 -- 172.21.15.99:0/757315529 >> 172.21.15.99:0/757315529 conn(0x7f8fdc0fbcf0 msgr2=0x7f8fdc0fe130 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:56:59.984 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.981+0000 7f8fe4412640 1 -- 172.21.15.99:0/757315529 shutdown_connections 2024-03-19T01:56:59.984 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.981+0000 7f8fe4412640 1 -- 172.21.15.99:0/757315529 wait complete. 2024-03-19T01:56:59.984 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.982+0000 7f8fe4412640 1 Processor -- start 2024-03-19T01:56:59.984 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.982+0000 7f8fe4412640 1 -- start start 2024-03-19T01:56:59.984 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.982+0000 7f8fe4412640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8fdc1023a0 0x7f8fdc197ad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:59.984 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.982+0000 7f8fe4412640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8fdc104cd0 0x7f8fdc198010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:59.984 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.982+0000 7f8fe4412640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8fdc107600 0x7f8fdc19f080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:59.984 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.982+0000 7f8fe4412640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f8fdc10d510 con 0x7f8fdc104cd0 2024-03-19T01:56:59.984 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.982+0000 7f8fe4412640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f8fdc10d390 con 0x7f8fdc1023a0 2024-03-19T01:56:59.984 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.982+0000 7f8fe4412640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f8fdc10d690 con 0x7f8fdc107600 2024-03-19T01:56:59.984 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.982+0000 7f8fe21ae640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8fdc1023a0 0x7f8fdc197ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:59.985 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.982+0000 7f8fe19ad640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8fdc104cd0 0x7f8fdc198010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:59.985 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.982+0000 7f8fe21ae640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8fdc1023a0 0x7f8fdc197ad0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:58760/0 (socket says 172.21.15.99:58760) 2024-03-19T01:56:59.985 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.982+0000 7f8fe19ad640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8fdc104cd0 0x7f8fdc198010 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:34018/0 (socket says 172.21.15.99:34018) 2024-03-19T01:56:59.985 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.982+0000 7f8fe21ae640 1 -- 172.21.15.99:0/3827108752 learned_addr learned my addr 172.21.15.99:0/3827108752 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:56:59.985 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.982+0000 7f8fe29af640 1 --2- 172.21.15.99:0/3827108752 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8fdc107600 0x7f8fdc19f080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:59.985 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.982+0000 7f8fe19ad640 1 -- 172.21.15.99:0/3827108752 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8fdc107600 msgr2=0x7f8fdc19f080 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:56:59.985 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.982+0000 7f8fe19ad640 1 --2- 172.21.15.99:0/3827108752 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8fdc107600 0x7f8fdc19f080 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:59.985 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.982+0000 7f8fe19ad640 1 -- 172.21.15.99:0/3827108752 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8fdc1023a0 msgr2=0x7f8fdc197ad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:56:59.985 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.982+0000 7f8fe19ad640 1 --2- 172.21.15.99:0/3827108752 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8fdc1023a0 0x7f8fdc197ad0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:59.985 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.982+0000 7f8fe19ad640 1 -- 172.21.15.99:0/3827108752 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f8fdc19f6f0 con 0x7f8fdc104cd0 2024-03-19T01:56:59.985 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.982+0000 7f8fe21ae640 1 --2- 172.21.15.99:0/3827108752 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8fdc1023a0 0x7f8fdc197ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:56:59.985 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.983+0000 7f8fe19ad640 1 --2- 172.21.15.99:0/3827108752 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8fdc104cd0 0x7f8fdc198010 secure :-1 s=READY pgs=654 cs=0 l=1 rev1=1 crypto rx=0x7f8fcc00e9c0 tx=0x7f8fcc00ee80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:56:59.985 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.983+0000 7f8fd37fe640 1 -- 172.21.15.99:0/3827108752 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8fcc00ccc0 con 0x7f8fdc104cd0 2024-03-19T01:56:59.985 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.983+0000 7f8fe4412640 1 -- 172.21.15.99:0/3827108752 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f8fdc19f9e0 con 0x7f8fdc104cd0 2024-03-19T01:56:59.986 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.983+0000 7f8fd37fe640 1 -- 172.21.15.99:0/3827108752 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f8fcc010910 con 0x7f8fdc104cd0 2024-03-19T01:56:59.986 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.983+0000 7f8fe4412640 1 -- 172.21.15.99:0/3827108752 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f8fdc19ff80 con 0x7f8fdc104cd0 2024-03-19T01:56:59.986 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.983+0000 7f8fd37fe640 1 -- 172.21.15.99:0/3827108752 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8fcc018880 con 0x7f8fdc104cd0 2024-03-19T01:56:59.988 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.985+0000 7f8fd37fe640 1 -- 172.21.15.99:0/3827108752 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f8fcc010ac0 con 0x7f8fdc104cd0 2024-03-19T01:56:59.988 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.986+0000 7f8fe6fff740 1 Processor -- start 2024-03-19T01:56:59.988 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.986+0000 7f8fd37fe640 1 --2- 172.21.15.99:0/3827108752 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f8fc80a6cb0 0x7f8fc80a9170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:59.988 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.986+0000 7f8fe6fff740 1 -- start start 2024-03-19T01:56:59.988 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.986+0000 7f8fe6fff740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556b4671f100 0x556b4671f520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:59.988 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.987+0000 7f8fe6fff740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556b4671faf0 0x556b467245f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:59.989 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.987+0000 7f8fe6fff740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556b46724c60 0x556b467270f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:59.989 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.987+0000 7f8fe6fff740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x556b46683be0 con 0x556b46724c60 2024-03-19T01:56:59.989 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.987+0000 7f8fe6fff740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x556b4662c960 con 0x556b4671f100 2024-03-19T01:56:59.989 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.987+0000 7f8fe6fff740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x556b4667fc80 con 0x556b4671faf0 2024-03-19T01:56:59.989 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.987+0000 7f8fd37fe640 1 -- 172.21.15.99:0/3827108752 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f8fcc0cee00 con 0x7f8fdc104cd0 2024-03-19T01:56:59.989 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.987+0000 7f8fe29af640 1 --2- 172.21.15.99:0/3827108752 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f8fc80a6cb0 0x7f8fc80a9170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:59.989 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.987+0000 7f8fe21ae640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556b4671f100 0x556b4671f520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:59.989 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.987+0000 7f8fe21ae640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556b4671f100 0x556b4671f520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:58768/0 (socket says 172.21.15.99:58768) 2024-03-19T01:56:59.989 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.987+0000 7f8fe21ae640 1 -- 172.21.15.99:0/1998990585 learned_addr learned my addr 172.21.15.99:0/1998990585 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:56:59.989 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.987+0000 7f8fe29af640 1 --2- 172.21.15.99:0/1998990585 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556b46724c60 0x556b467270f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:59.989 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.987+0000 7f8fe21ae640 1 -- 172.21.15.99:0/1998990585 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556b4671faf0 msgr2=0x556b467245f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:56:59.989 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.987+0000 7f8fe21ae640 1 --2- 172.21.15.99:0/1998990585 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556b4671faf0 0x556b467245f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:59.989 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.987+0000 7f8fe21ae640 1 -- 172.21.15.99:0/1998990585 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556b46724c60 msgr2=0x556b467270f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:56:59.989 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.987+0000 7f8fe21ae640 1 --2- 172.21.15.99:0/1998990585 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556b46724c60 0x556b467270f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:59.990 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.987+0000 7f8fe21ae640 1 -- 172.21.15.99:0/1998990585 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x556b46623d20 con 0x556b4671f100 2024-03-19T01:56:59.990 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.987+0000 7f8fe29af640 1 --2- 172.21.15.99:0/3827108752 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f8fc80a6cb0 0x7f8fc80a9170 secure :-1 s=READY pgs=344 cs=0 l=1 rev1=1 crypto rx=0x7f8fd4005fd0 tx=0x7f8fd4013b60 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:56:59.990 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.987+0000 7f8fe21ae640 1 --2- 172.21.15.99:0/1998990585 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556b4671f100 0x556b4671f520 secure :-1 s=READY pgs=173 cs=0 l=1 rev1=1 crypto rx=0x7f8fd8009230 tx=0x7f8fd802cd30 comp rx=0 tx=0).ready entity=mon.2 client_cookie=7c51c391c6b805f8 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:56:59.990 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.988+0000 7f8fd0bfe640 1 -- 172.21.15.99:0/1998990585 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8fd8042070 con 0x556b4671f100 2024-03-19T01:56:59.990 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.988+0000 7f8fd0bfe640 1 -- 172.21.15.99:0/1998990585 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f8fd8004ba0 con 0x556b4671f100 2024-03-19T01:56:59.990 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.988+0000 7f8fd0bfe640 1 -- 172.21.15.99:0/1998990585 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8fd8005350 con 0x556b4671f100 2024-03-19T01:56:59.990 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.988+0000 7f8fe6fff740 1 -- 172.21.15.99:0/1998990585 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556b4671f100 msgr2=0x556b4671f520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:56:59.990 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.988+0000 7f8fe6fff740 1 --2- 172.21.15.99:0/1998990585 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556b4671f100 0x556b4671f520 secure :-1 s=READY pgs=173 cs=0 l=1 rev1=1 crypto rx=0x7f8fd8009230 tx=0x7f8fd802cd30 comp rx=0 tx=0).stop 2024-03-19T01:56:59.990 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.988+0000 7f8fe6fff740 1 -- 172.21.15.99:0/1998990585 shutdown_connections 2024-03-19T01:56:59.990 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.988+0000 7f8fe6fff740 1 --2- 172.21.15.99:0/1998990585 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556b46724c60 0x556b467270f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:59.990 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.988+0000 7f8fe6fff740 1 --2- 172.21.15.99:0/1998990585 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556b4671faf0 0x556b467245f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:59.990 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.988+0000 7f8fe6fff740 1 --2- 172.21.15.99:0/1998990585 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556b4671f100 0x556b4671f520 unknown :-1 s=CLOSED pgs=173 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:59.990 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.988+0000 7f8fe6fff740 1 -- 172.21.15.99:0/1998990585 >> 172.21.15.99:0/1998990585 conn(0x556b4671a670 msgr2=0x556b4671aa70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:56:59.990 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.989+0000 7f8fe6fff740 1 -- 172.21.15.99:0/1998990585 shutdown_connections 2024-03-19T01:56:59.990 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.989+0000 7f8fe6fff740 1 -- 172.21.15.99:0/1998990585 wait complete. 2024-03-19T01:56:59.993 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.989+0000 7f8fe6fff740 1 -- start start 2024-03-19T01:56:59.993 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.989+0000 7f8fe6fff740 1 Processor -- start 2024-03-19T01:56:59.993 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.989+0000 7f8fe6fff740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556b4671f100 0x556b46729160 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:59.993 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.989+0000 7f8fe6fff740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556b4671faf0 0x556b467296a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:59.993 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.989+0000 7f8fe6fff740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556b46724c60 0x556b467306e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:59.993 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.989+0000 7f8fe6fff740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x556b46727bb0 con 0x556b46724c60 2024-03-19T01:56:59.993 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.989+0000 7f8fe6fff740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x556b4663fdb0 con 0x556b4671faf0 2024-03-19T01:56:59.993 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.989+0000 7f8fe6fff740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x556b46727d30 con 0x556b4671f100 2024-03-19T01:56:59.994 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.989+0000 7f8fe6fff740 10 client.0 _ll_register_callbacks cb 0x556b46644e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:56:59.994 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.990+0000 7f8fe19ad640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556b4671faf0 0x556b467296a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:59.994 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.990+0000 7f8fe29af640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556b46724c60 0x556b467306e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:59.994 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.990+0000 7f8fe29af640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556b46724c60 0x556b467306e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:34044/0 (socket says 172.21.15.99:34044) 2024-03-19T01:56:59.994 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.990+0000 7f8fe19ad640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556b4671faf0 0x556b467296a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:58784/0 (socket says 172.21.15.99:58784) 2024-03-19T01:56:59.994 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.990+0000 7f8fe29af640 1 -- 172.21.15.99:0/2415156136 learned_addr learned my addr 172.21.15.99:0/2415156136 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:56:59.994 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.990+0000 7f8fe21ae640 1 --2- 172.21.15.99:0/2415156136 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556b4671f100 0x556b46729160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:59.994 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.990+0000 7f8fe19ad640 1 -- 172.21.15.99:0/2415156136 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556b4671f100 msgr2=0x556b46729160 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:56:59.994 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.990+0000 7f8fe19ad640 1 --2- 172.21.15.99:0/2415156136 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556b4671f100 0x556b46729160 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:59.994 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.990+0000 7f8fe19ad640 1 -- 172.21.15.99:0/2415156136 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556b46724c60 msgr2=0x556b467306e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:56:59.994 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.990+0000 7f8fe19ad640 1 --2- 172.21.15.99:0/2415156136 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556b46724c60 0x556b467306e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:59.994 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.990+0000 7f8fe19ad640 1 -- 172.21.15.99:0/2415156136 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x556b4663ec60 con 0x556b4671faf0 2024-03-19T01:56:59.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.990+0000 7f8fe21ae640 1 --2- 172.21.15.99:0/2415156136 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556b4671f100 0x556b46729160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-19T01:56:59.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.990+0000 7f8fe29af640 1 --2- 172.21.15.99:0/2415156136 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556b46724c60 0x556b467306e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:56:59.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.990+0000 7f8fe19ad640 1 --2- 172.21.15.99:0/2415156136 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556b4671faf0 0x556b467296a0 secure :-1 s=READY pgs=174 cs=0 l=1 rev1=1 crypto rx=0x556b46720560 tx=0x7f8fcc09b230 comp rx=0 tx=0).ready entity=mon.2 client_cookie=d737d128462d9c75 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:56:59.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.990+0000 7f8fc5ffb640 10 client.0 ms_handle_connect on v2:172.21.15.112:3300/0 2024-03-19T01:56:59.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.990+0000 7f8fc5ffb640 1 -- 172.21.15.99:0/2415156136 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8fcc0afb40 con 0x556b4671faf0 2024-03-19T01:56:59.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.990+0000 7f8fc5ffb640 1 -- 172.21.15.99:0/2415156136 <== mon.2 v2:172.21.15.112:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f8fcc09ac60 con 0x556b4671faf0 2024-03-19T01:56:59.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.990+0000 7f8fe6fff740 1 -- 172.21.15.99:0/2415156136 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f8fd8004ba0 con 0x556b4671faf0 2024-03-19T01:56:59.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.991+0000 7f8fc5ffb640 1 -- 172.21.15.99:0/2415156136 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7f8fcc09ae60 con 0x556b4671faf0 2024-03-19T01:56:59.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.991+0000 7f8fe6fff740 10 client.26393 fetch_fsmap learned FSMap version 37 2024-03-19T01:56:59.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.991+0000 7f8fe6fff740 1 -- 172.21.15.99:0/2415156136 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x556b46398210 con 0x556b4671faf0 2024-03-19T01:56:59.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.991+0000 7f8fc5ffb640 1 -- 172.21.15.99:0/2415156136 <== mon.2 v2:172.21.15.112:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f8fd8004ba0 con 0x556b4671faf0 2024-03-19T01:56:59.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.991+0000 7f8fc5ffb640 1 -- 172.21.15.99:0/2415156136 <== mon.2 v2:172.21.15.112:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7f8fcc0afce0 con 0x556b4671faf0 2024-03-19T01:56:59.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.991+0000 7f8fe6fff740 10 client.26393 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T01:56:59.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.991+0000 7f8fe6fff740 10 client.26393 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:56:59.996 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.991+0000 7f8fe6fff740 20 client.26393 populate_metadata read hostname 'smithi099' 2024-03-19T01:56:59.996 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.991+0000 7f8fe6fff740 1 --2- 172.21.15.99:0/2415156136 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x556b46734fb0 0x556b467373a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:56:59.996 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.991+0000 7f8fe6fff740 4 client.26393 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:56:59.996 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.991+0000 7f8fe6fff740 1 -- 172.21.15.99:0/2415156136 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x556b46691120 con 0x556b46734fb0 2024-03-19T01:56:59.996 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.991+0000 7f8fe21ae640 1 --2- 172.21.15.99:0/2415156136 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x556b46734fb0 0x556b467373a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:56:59.996 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.992+0000 7f8fe21ae640 1 --2- 172.21.15.99:0/2415156136 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x556b46734fb0 0x556b467373a0 crc :-1 s=READY pgs=171 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=adcdc65189f9c787 server_cookie=18d347d2cad4171c in_seq=0 out_seq=0 2024-03-19T01:56:59.996 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.992+0000 7f8fc5ffb640 10 client.26393 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:56:59.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.998+0000 7f8fc5ffb640 1 -- 172.21.15.99:0/2415156136 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x556b46691120 con 0x556b46734fb0 2024-03-19T01:56:59.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.998+0000 7f8fc5ffb640 10 client.26393 handle_command_reply: tid=0 2024-03-19T01:56:59.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.999+0000 7f8fe6fff740 1 client.26393 shutdown 2024-03-19T01:56:59.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.999+0000 7f8fe6fff740 1 -- 172.21.15.99:0/2415156136 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556b4671faf0 msgr2=0x556b467296a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:56:59.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.999+0000 7f8fe6fff740 1 --2- 172.21.15.99:0/2415156136 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556b4671faf0 0x556b467296a0 secure :-1 s=READY pgs=174 cs=0 l=1 rev1=1 crypto rx=0x556b46720560 tx=0x7f8fcc09b230 comp rx=0 tx=0).stop 2024-03-19T01:56:59.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.999+0000 7f8fe6fff740 1 -- 172.21.15.99:0/2415156136 shutdown_connections 2024-03-19T01:56:59.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.999+0000 7f8fe6fff740 1 --2- 172.21.15.99:0/2415156136 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x556b46734fb0 0x556b467373a0 crc :-1 s=READY pgs=171 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:59.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.999+0000 7f8fe6fff740 1 --2- 172.21.15.99:0/2415156136 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556b4671faf0 0x556b467296a0 unknown :-1 s=CLOSED pgs=174 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:56:59.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.999+0000 7f8fe6fff740 1 --2- 172.21.15.99:0/2415156136 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556b46724c60 0x556b467306e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:00.000 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.999+0000 7f8fc5ffb640 0 client.26393 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:57:00.000 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.999+0000 7f8fe6fff740 1 --2- 172.21.15.99:0/2415156136 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556b4671f100 0x556b46729160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:00.000 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.999+0000 7f8fe6fff740 1 -- 172.21.15.99:0/2415156136 >> 172.21.15.99:0/2415156136 conn(0x556b4671a670 msgr2=0x556b4671b8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:57:00.000 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.999+0000 7f8fe6fff740 1 -- 172.21.15.99:0/2415156136 shutdown_connections 2024-03-19T01:57:00.000 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.999+0000 7f8fe6fff740 1 -- 172.21.15.99:0/2415156136 wait complete. 2024-03-19T01:57:00.000 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:56:59.999+0000 7f8fe6fff740 20 client.26393 trim_cache size 0 max 16384 2024-03-19T01:57:00.018 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.015+0000 7f8fe6fff740 1 Processor -- start 2024-03-19T01:57:00.018 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.015+0000 7f8fe6fff740 1 -- start start 2024-03-19T01:57:00.018 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.015+0000 7f8fe6fff740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556b4671f100 0x556b46724620 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:00.018 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.015+0000 7f8fe6fff740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556b4671faf0 0x556b467306a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:00.019 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.015+0000 7f8fe6fff740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556b46724c60 0x556b46719930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:00.019 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.015+0000 7f8fe6fff740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f8fdc1a9470 con 0x556b4671f100 2024-03-19T01:57:00.019 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.015+0000 7f8fe6fff740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f8fdc1a6d40 con 0x556b4671faf0 2024-03-19T01:57:00.019 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.015+0000 7f8fe6fff740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f8fdc1a4b90 con 0x556b46724c60 2024-03-19T01:57:00.019 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.015+0000 7f8fe21ae640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556b4671f100 0x556b46724620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:00.019 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.015+0000 7f8fe21ae640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556b4671f100 0x556b46724620 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:34056/0 (socket says 172.21.15.99:34056) 2024-03-19T01:57:00.019 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.015+0000 7f8fe21ae640 1 -- 172.21.15.99:0/2156834304 learned_addr learned my addr 172.21.15.99:0/2156834304 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:57:00.019 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.015+0000 7f8fe19ad640 1 --2- 172.21.15.99:0/2156834304 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556b4671faf0 0x556b467306a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:00.019 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.015+0000 7f8fe29af640 1 --2- 172.21.15.99:0/2156834304 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556b46724c60 0x556b46719930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:00.019 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.015+0000 7f8fe21ae640 1 -- 172.21.15.99:0/2156834304 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556b46724c60 msgr2=0x556b46719930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:57:00.019 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.015+0000 7f8fe21ae640 1 --2- 172.21.15.99:0/2156834304 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556b46724c60 0x556b46719930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:00.019 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.015+0000 7f8fe21ae640 1 -- 172.21.15.99:0/2156834304 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556b4671faf0 msgr2=0x556b467306a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:57:00.019 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.015+0000 7f8fe21ae640 1 --2- 172.21.15.99:0/2156834304 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556b4671faf0 0x556b467306a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:00.020 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.015+0000 7f8fe21ae640 1 -- 172.21.15.99:0/2156834304 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x556b46464620 con 0x556b4671f100 2024-03-19T01:57:00.020 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.016+0000 7f8fe19ad640 1 --2- 172.21.15.99:0/2156834304 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556b4671faf0 0x556b467306a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:57:00.020 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.016+0000 7f8fe21ae640 1 --2- 172.21.15.99:0/2156834304 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556b4671f100 0x556b46724620 secure :-1 s=READY pgs=655 cs=0 l=1 rev1=1 crypto rx=0x7f8fd802cfd0 tx=0x7f8fd802bc30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c79c8b474e88f4c3 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:57:00.020 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.016+0000 7f8fc7fff640 1 -- 172.21.15.99:0/2156834304 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8fd8042040 con 0x556b4671f100 2024-03-19T01:57:00.020 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.016+0000 7f8fc7fff640 1 -- 172.21.15.99:0/2156834304 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f8fd8053070 con 0x556b4671f100 2024-03-19T01:57:00.020 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.016+0000 7f8fc7fff640 1 -- 172.21.15.99:0/2156834304 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8fd8004040 con 0x556b4671f100 2024-03-19T01:57:00.020 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.017+0000 7f8fe6fff740 1 -- 172.21.15.99:0/2156834304 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556b4671f100 msgr2=0x556b46724620 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:57:00.020 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.017+0000 7f8fe6fff740 1 --2- 172.21.15.99:0/2156834304 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556b4671f100 0x556b46724620 secure :-1 s=READY pgs=655 cs=0 l=1 rev1=1 crypto rx=0x7f8fd802cfd0 tx=0x7f8fd802bc30 comp rx=0 tx=0).stop 2024-03-19T01:57:00.020 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.017+0000 7f8fe6fff740 1 -- 172.21.15.99:0/2156834304 shutdown_connections 2024-03-19T01:57:00.020 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.017+0000 7f8fe6fff740 1 --2- 172.21.15.99:0/2156834304 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556b4671faf0 0x556b467306a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:00.020 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.017+0000 7f8fe6fff740 1 --2- 172.21.15.99:0/2156834304 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556b46724c60 0x556b46719930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:00.020 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.017+0000 7f8fe6fff740 1 --2- 172.21.15.99:0/2156834304 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556b4671f100 0x556b46724620 unknown :-1 s=CLOSED pgs=655 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:00.020 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.017+0000 7f8fe6fff740 1 -- 172.21.15.99:0/2156834304 >> 172.21.15.99:0/2156834304 conn(0x556b4671a670 msgr2=0x556b46719220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:57:00.021 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.017+0000 7f8fe6fff740 1 -- 172.21.15.99:0/2156834304 shutdown_connections 2024-03-19T01:57:00.021 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.017+0000 7f8fe6fff740 1 -- 172.21.15.99:0/2156834304 wait complete. 2024-03-19T01:57:00.021 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.017+0000 7f8fe6fff740 1 -- start start 2024-03-19T01:57:00.021 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.017+0000 7f8fe6fff740 1 Processor -- start 2024-03-19T01:57:00.021 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.017+0000 7f8fe6fff740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556b4671f100 0x556b4672d4e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:00.021 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.017+0000 7f8fe6fff740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556b4671faf0 0x556b4672da20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:00.021 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.017+0000 7f8fe6fff740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556b46724c60 0x556b46799260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:00.021 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.017+0000 7f8fe6fff740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f8fdc1a5200 con 0x556b46724c60 2024-03-19T01:57:00.021 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.017+0000 7f8fe6fff740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f8fdc1a6670 con 0x556b4671faf0 2024-03-19T01:57:00.021 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.017+0000 7f8fe6fff740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f8fdc1a58d0 con 0x556b4671f100 2024-03-19T01:57:00.021 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.017+0000 7f8fe6fff740 10 client.0 _ll_register_callbacks cb 0x7f8fcc00ce60 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:57:00.021 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.017+0000 7f8fe29af640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556b46724c60 0x556b46799260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:00.021 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.017+0000 7f8fe29af640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556b46724c60 0x556b46799260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:34066/0 (socket says 172.21.15.99:34066) 2024-03-19T01:57:00.022 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.017+0000 7f8fe29af640 1 -- 172.21.15.99:0/3996768602 learned_addr learned my addr 172.21.15.99:0/3996768602 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:57:00.022 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.017+0000 7f8fe21ae640 1 --2- 172.21.15.99:0/3996768602 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556b4671f100 0x556b4672d4e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:00.022 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.017+0000 7f8fe19ad640 1 --2- 172.21.15.99:0/3996768602 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556b4671faf0 0x556b4672da20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:00.022 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.018+0000 7f8fe29af640 1 -- 172.21.15.99:0/3996768602 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556b4671f100 msgr2=0x556b4672d4e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:57:00.022 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.018+0000 7f8fe29af640 1 --2- 172.21.15.99:0/3996768602 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556b4671f100 0x556b4672d4e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:00.022 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.018+0000 7f8fe29af640 1 -- 172.21.15.99:0/3996768602 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556b4671faf0 msgr2=0x556b4672da20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:57:00.022 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.018+0000 7f8fe29af640 1 --2- 172.21.15.99:0/3996768602 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556b4671faf0 0x556b4672da20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:00.022 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.018+0000 7f8fe29af640 1 -- 172.21.15.99:0/3996768602 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x556b467997a0 con 0x556b46724c60 2024-03-19T01:57:00.022 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.018+0000 7f8fe29af640 1 --2- 172.21.15.99:0/3996768602 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556b46724c60 0x556b46799260 secure :-1 s=READY pgs=656 cs=0 l=1 rev1=1 crypto rx=0x7f8fd4003d40 tx=0x7f8fd4004ab0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=491c28658ad0ef6e server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:57:00.022 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.018+0000 7f8fc5ffb640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:57:00.022 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.018+0000 7f8fc5ffb640 1 -- 172.21.15.99:0/3996768602 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8fd40046d0 con 0x556b46724c60 2024-03-19T01:57:00.022 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.018+0000 7f8fe6fff740 1 -- 172.21.15.99:0/3996768602 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f8fd8053070 con 0x556b46724c60 2024-03-19T01:57:00.024 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.019+0000 7f8fc5ffb640 1 -- 172.21.15.99:0/3996768602 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f8fd4010430 con 0x556b46724c60 2024-03-19T01:57:00.024 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.019+0000 7f8fc5ffb640 1 -- 172.21.15.99:0/3996768602 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7f8fd4010630 con 0x556b46724c60 2024-03-19T01:57:00.024 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.019+0000 7f8fe6fff740 10 client.16626 fetch_fsmap learned FSMap version 37 2024-03-19T01:57:00.024 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.019+0000 7f8fe6fff740 1 -- 172.21.15.99:0/3996768602 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x556b4679a9b0 con 0x556b46724c60 2024-03-19T01:57:00.024 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.020+0000 7f8fc5ffb640 1 -- 172.21.15.99:0/3996768602 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f8fd8053070 con 0x556b46724c60 2024-03-19T01:57:00.024 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.020+0000 7f8fc5ffb640 1 -- 172.21.15.99:0/3996768602 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7f8fd4029070 con 0x556b46724c60 2024-03-19T01:57:00.024 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.020+0000 7f8fe6fff740 10 client.16626 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T01:57:00.024 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.020+0000 7f8fe6fff740 10 client.16626 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:57:00.024 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.020+0000 7f8fe6fff740 20 client.16626 populate_metadata read hostname 'smithi099' 2024-03-19T01:57:00.024 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.020+0000 7f8fe6fff740 1 --2- 172.21.15.99:0/3996768602 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x556b4679b370 0x556b4679d790 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:00.024 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.020+0000 7f8fe6fff740 4 client.16626 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:57:00.025 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.020+0000 7f8fe6fff740 1 -- 172.21.15.99:0/3996768602 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x556b4668b470 con 0x556b4679b370 2024-03-19T01:57:00.025 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.020+0000 7f8fe21ae640 1 --2- 172.21.15.99:0/3996768602 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x556b4679b370 0x556b4679d790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:00.025 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.021+0000 7f8fc5ffb640 10 client.16626 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:57:00.025 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.021+0000 7f8fe21ae640 1 --2- 172.21.15.99:0/3996768602 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x556b4679b370 0x556b4679d790 crc :-1 s=READY pgs=172 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=687a5d94d17999 server_cookie=3a4c91bb88b5d1cc in_seq=0 out_seq=0 2024-03-19T01:57:00.105 INFO:tasks.workunit.client.0.smithi099.stdout: 10 92831 23.05 MB/sec execute 89 sec latency 110.772 ms 2024-03-19T01:57:00.496 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.494+0000 7f8fc5ffb640 1 -- 172.21.15.99:0/3996768602 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x556b4668b470 con 0x556b4679b370 2024-03-19T01:57:00.497 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.494+0000 7f8fc5ffb640 10 client.16626 handle_command_reply: tid=0 2024-03-19T01:57:00.497 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.494+0000 7f8fe6fff740 1 client.16626 shutdown 2024-03-19T01:57:00.497 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.495+0000 7f8fe6fff740 1 -- 172.21.15.99:0/3996768602 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556b46724c60 msgr2=0x556b46799260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:57:00.497 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.495+0000 7f8fe6fff740 1 --2- 172.21.15.99:0/3996768602 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556b46724c60 0x556b46799260 secure :-1 s=READY pgs=656 cs=0 l=1 rev1=1 crypto rx=0x7f8fd4003d40 tx=0x7f8fd4004ab0 comp rx=0 tx=0).stop 2024-03-19T01:57:00.497 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.495+0000 7f8fe6fff740 1 -- 172.21.15.99:0/3996768602 shutdown_connections 2024-03-19T01:57:00.497 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.495+0000 7f8fe6fff740 1 --2- 172.21.15.99:0/3996768602 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x556b4679b370 0x556b4679d790 crc :-1 s=READY pgs=172 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:00.498 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.495+0000 7f8fe6fff740 1 --2- 172.21.15.99:0/3996768602 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556b4671faf0 0x556b4672da20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:00.498 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.495+0000 7f8fc5ffb640 0 client.16626 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:57:00.498 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.495+0000 7f8fe6fff740 1 --2- 172.21.15.99:0/3996768602 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556b46724c60 0x556b46799260 unknown :-1 s=CLOSED pgs=656 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:00.498 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.495+0000 7f8fe6fff740 1 --2- 172.21.15.99:0/3996768602 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556b4671f100 0x556b4672d4e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:00.498 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.495+0000 7f8fe6fff740 1 -- 172.21.15.99:0/3996768602 >> 172.21.15.99:0/3996768602 conn(0x556b4671a670 msgr2=0x556b46719220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:57:00.498 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.495+0000 7f8fe6fff740 1 -- 172.21.15.99:0/3996768602 shutdown_connections 2024-03-19T01:57:00.498 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.495+0000 7f8fe6fff740 1 -- 172.21.15.99:0/3996768602 wait complete. 2024-03-19T01:57:00.498 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.495+0000 7f8fe6fff740 20 client.16626 trim_cache size 0 max 16384 2024-03-19T01:57:00.498 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:57:00.499 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:57:00.499 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:57:00.499 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:57:00.499 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.496+0000 7f8fe4412640 1 -- 172.21.15.99:0/3827108752 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f8fc80a6cb0 msgr2=0x7f8fc80a9170 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:57:00.499 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.496+0000 7f8fe4412640 1 --2- 172.21.15.99:0/3827108752 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f8fc80a6cb0 0x7f8fc80a9170 secure :-1 s=READY pgs=344 cs=0 l=1 rev1=1 crypto rx=0x7f8fd4005fd0 tx=0x7f8fd4013b60 comp rx=0 tx=0).stop 2024-03-19T01:57:00.499 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.496+0000 7f8fe4412640 1 -- 172.21.15.99:0/3827108752 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8fdc104cd0 msgr2=0x7f8fdc198010 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:57:00.500 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.496+0000 7f8fe4412640 1 --2- 172.21.15.99:0/3827108752 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8fdc104cd0 0x7f8fdc198010 secure :-1 s=READY pgs=654 cs=0 l=1 rev1=1 crypto rx=0x7f8fcc00e9c0 tx=0x7f8fcc00ee80 comp rx=0 tx=0).stop 2024-03-19T01:57:00.500 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.496+0000 7f8fe4412640 1 -- 172.21.15.99:0/3827108752 shutdown_connections 2024-03-19T01:57:00.500 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.496+0000 7f8fe4412640 1 --2- 172.21.15.99:0/3827108752 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8fdc107600 0x7f8fdc19f080 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:00.500 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.496+0000 7f8fe4412640 1 --2- 172.21.15.99:0/3827108752 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8fdc104cd0 0x7f8fdc198010 unknown :-1 s=CLOSED pgs=654 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:00.500 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.496+0000 7f8fe4412640 1 --2- 172.21.15.99:0/3827108752 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f8fc80a6cb0 0x7f8fc80a9170 unknown :-1 s=CLOSED pgs=344 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:00.500 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.496+0000 7f8fe4412640 1 --2- 172.21.15.99:0/3827108752 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8fdc1023a0 0x7f8fdc197ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:00.500 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.496+0000 7f8fe4412640 1 -- 172.21.15.99:0/3827108752 >> 172.21.15.99:0/3827108752 conn(0x7f8fdc0fbcf0 msgr2=0x7f8fdc102e80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:57:00.500 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.496+0000 7f8fe4412640 1 -- 172.21.15.99:0/3827108752 shutdown_connections 2024-03-19T01:57:00.500 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:00.496+0000 7f8fe4412640 1 -- 172.21.15.99:0/3827108752 wait complete. 2024-03-19T01:57:01.105 INFO:tasks.workunit.client.0.smithi099.stdout: 10 93209 23.14 MB/sec execute 90 sec latency 282.429 ms 2024-03-19T01:57:01.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:57:01 smithi099 ceph-mon[62951]: pgmap v829: 97 pgs: 97 active+clean; 530 MiB data, 6.0 GiB used, 1.0 TiB / 1.0 TiB avail; 4.8 MiB/s rd, 6.5 MiB/s wr, 524 op/s 2024-03-19T01:57:01.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:57:01 smithi112 ceph-mon[66819]: pgmap v829: 97 pgs: 97 active+clean; 530 MiB data, 6.0 GiB used, 1.0 TiB / 1.0 TiB avail; 4.8 MiB/s rd, 6.5 MiB/s wr, 524 op/s 2024-03-19T01:57:01.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:57:01 smithi115 ceph-mon[67332]: pgmap v829: 97 pgs: 97 active+clean; 530 MiB data, 6.0 GiB used, 1.0 TiB / 1.0 TiB avail; 4.8 MiB/s rd, 6.5 MiB/s wr, 524 op/s 2024-03-19T01:57:02.105 INFO:tasks.workunit.client.0.smithi099.stdout: 10 93760 23.25 MB/sec execute 91 sec latency 50.374 ms 2024-03-19T01:57:03.105 INFO:tasks.workunit.client.0.smithi099.stdout: 10 94063 23.20 MB/sec execute 92 sec latency 259.044 ms 2024-03-19T01:57:03.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:57:03 smithi099 ceph-mon[62951]: pgmap v830: 97 pgs: 97 active+clean; 528 MiB data, 6.0 GiB used, 1.0 TiB / 1.0 TiB avail; 4.9 MiB/s rd, 7.4 MiB/s wr, 623 op/s 2024-03-19T01:57:03.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:57:03 smithi112 ceph-mon[66819]: pgmap v830: 97 pgs: 97 active+clean; 528 MiB data, 6.0 GiB used, 1.0 TiB / 1.0 TiB avail; 4.9 MiB/s rd, 7.4 MiB/s wr, 623 op/s 2024-03-19T01:57:03.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:57:03 smithi115 ceph-mon[67332]: pgmap v830: 97 pgs: 97 active+clean; 528 MiB data, 6.0 GiB used, 1.0 TiB / 1.0 TiB avail; 4.9 MiB/s rd, 7.4 MiB/s wr, 623 op/s 2024-03-19T01:57:04.105 INFO:tasks.workunit.client.0.smithi099.stdout: 10 94407 23.09 MB/sec execute 93 sec latency 97.738 ms 2024-03-19T01:57:05.105 INFO:tasks.workunit.client.0.smithi099.stdout: 10 94880 23.05 MB/sec execute 94 sec latency 70.770 ms 2024-03-19T01:57:05.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:57:05 smithi099 ceph-mon[62951]: pgmap v831: 97 pgs: 97 active+clean; 528 MiB data, 6.0 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 6.2 MiB/s wr, 518 op/s 2024-03-19T01:57:05.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:57:05 smithi112 ceph-mon[66819]: pgmap v831: 97 pgs: 97 active+clean; 528 MiB data, 6.0 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 6.2 MiB/s wr, 518 op/s 2024-03-19T01:57:05.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:57:05 smithi115 ceph-mon[67332]: pgmap v831: 97 pgs: 97 active+clean; 528 MiB data, 6.0 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 6.2 MiB/s wr, 518 op/s 2024-03-19T01:57:06.105 INFO:tasks.workunit.client.0.smithi099.stdout: 10 95427 23.25 MB/sec execute 95 sec latency 55.368 ms 2024-03-19T01:57:07.105 INFO:tasks.workunit.client.0.smithi099.stdout: 10 95822 23.17 MB/sec execute 96 sec latency 86.743 ms 2024-03-19T01:57:07.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:57:07 smithi099 ceph-mon[62951]: pgmap v832: 97 pgs: 97 active+clean; 532 MiB data, 6.0 GiB used, 1.0 TiB / 1.0 TiB avail; 4.6 MiB/s rd, 7.3 MiB/s wr, 607 op/s 2024-03-19T01:57:07.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:57:07 smithi112 ceph-mon[66819]: pgmap v832: 97 pgs: 97 active+clean; 532 MiB data, 6.0 GiB used, 1.0 TiB / 1.0 TiB avail; 4.6 MiB/s rd, 7.3 MiB/s wr, 607 op/s 2024-03-19T01:57:07.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:57:07 smithi115 ceph-mon[67332]: pgmap v832: 97 pgs: 97 active+clean; 532 MiB data, 6.0 GiB used, 1.0 TiB / 1.0 TiB avail; 4.6 MiB/s rd, 7.3 MiB/s wr, 607 op/s 2024-03-19T01:57:08.105 INFO:tasks.workunit.client.0.smithi099.stdout: 10 96154 23.08 MB/sec execute 97 sec latency 290.049 ms 2024-03-19T01:57:09.105 INFO:tasks.workunit.client.0.smithi099.stdout: 10 96560 23.11 MB/sec execute 98 sec latency 89.029 ms 2024-03-19T01:57:09.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:57:09 smithi099 ceph-mon[62951]: pgmap v833: 97 pgs: 97 active+clean; 534 MiB data, 6.1 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 6.5 MiB/s wr, 579 op/s 2024-03-19T01:57:09.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:57:09 smithi112 ceph-mon[66819]: pgmap v833: 97 pgs: 97 active+clean; 534 MiB data, 6.1 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 6.5 MiB/s wr, 579 op/s 2024-03-19T01:57:09.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:57:09 smithi115 ceph-mon[67332]: pgmap v833: 97 pgs: 97 active+clean; 534 MiB data, 6.1 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 6.5 MiB/s wr, 579 op/s 2024-03-19T01:57:10.105 INFO:tasks.workunit.client.0.smithi099.stdout: 10 97141 23.06 MB/sec execute 99 sec latency 85.818 ms 2024-03-19T01:57:11.105 INFO:tasks.workunit.client.0.smithi099.stdout: 10 97739 23.25 MB/sec execute 100 sec latency 74.469 ms 2024-03-19T01:57:11.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:57:11 smithi099 ceph-mon[62951]: pgmap v834: 97 pgs: 97 active+clean; 535 MiB data, 6.1 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 6.2 MiB/s wr, 535 op/s 2024-03-19T01:57:11.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:57:11 smithi112 ceph-mon[66819]: pgmap v834: 97 pgs: 97 active+clean; 535 MiB data, 6.1 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 6.2 MiB/s wr, 535 op/s 2024-03-19T01:57:11.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:57:11 smithi115 ceph-mon[67332]: pgmap v834: 97 pgs: 97 active+clean; 535 MiB data, 6.1 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 6.2 MiB/s wr, 535 op/s 2024-03-19T01:57:12.105 INFO:tasks.workunit.client.0.smithi099.stdout: 10 98139 23.22 MB/sec execute 101 sec latency 64.230 ms 2024-03-19T01:57:13.105 INFO:tasks.workunit.client.0.smithi099.stdout: 10 98557 23.28 MB/sec execute 102 sec latency 244.040 ms 2024-03-19T01:57:13.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:57:13 smithi099 ceph-mon[62951]: pgmap v835: 97 pgs: 97 active+clean; 539 MiB data, 6.1 GiB used, 1.0 TiB / 1.0 TiB avail; 4.3 MiB/s rd, 6.3 MiB/s wr, 563 op/s 2024-03-19T01:57:13.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:57:13 smithi112 ceph-mon[66819]: pgmap v835: 97 pgs: 97 active+clean; 539 MiB data, 6.1 GiB used, 1.0 TiB / 1.0 TiB avail; 4.3 MiB/s rd, 6.3 MiB/s wr, 563 op/s 2024-03-19T01:57:13.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:57:13 smithi115 ceph-mon[67332]: pgmap v835: 97 pgs: 97 active+clean; 539 MiB data, 6.1 GiB used, 1.0 TiB / 1.0 TiB avail; 4.3 MiB/s rd, 6.3 MiB/s wr, 563 op/s 2024-03-19T01:57:13.589 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.587+0000 7f939b122640 1 -- 172.21.15.99:0/4191730462 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f93941022f0 msgr2=0x7f9394108b80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:57:13.589 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.588+0000 7f939b122640 1 --2- 172.21.15.99:0/4191730462 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f93941022f0 0x7f9394108b80 secure :-1 s=READY pgs=657 cs=0 l=1 rev1=1 crypto rx=0x7f938c009ca0 tx=0x7f938c02c0f0 comp rx=0 tx=0).stop 2024-03-19T01:57:13.590 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.588+0000 7f939b122640 1 -- 172.21.15.99:0/4191730462 shutdown_connections 2024-03-19T01:57:13.590 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.588+0000 7f939b122640 1 --2- 172.21.15.99:0/4191730462 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f93941022f0 0x7f9394108b80 unknown :-1 s=CLOSED pgs=657 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:13.590 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.588+0000 7f939b122640 1 --2- 172.21.15.99:0/4191730462 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9394101930 0x7f9394101db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:13.590 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.588+0000 7f939b122640 1 --2- 172.21.15.99:0/4191730462 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9394100740 0x7f9394100b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:13.590 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.588+0000 7f939b122640 1 -- 172.21.15.99:0/4191730462 >> 172.21.15.99:0/4191730462 conn(0x7f93940fbcb0 msgr2=0x7f93940fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:57:13.590 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.588+0000 7f939b122640 1 -- 172.21.15.99:0/4191730462 shutdown_connections 2024-03-19T01:57:13.590 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.588+0000 7f939b122640 1 -- 172.21.15.99:0/4191730462 wait complete. 2024-03-19T01:57:13.590 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.588+0000 7f939b122640 1 Processor -- start 2024-03-19T01:57:13.590 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.588+0000 7f939b122640 1 -- start start 2024-03-19T01:57:13.591 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.588+0000 7f939b122640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9394100740 0x7f9394197ac0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:13.591 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.588+0000 7f939b122640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9394101930 0x7f9394198000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:13.591 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.588+0000 7f939b122640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f93941022f0 0x7f939419f070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:13.591 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.588+0000 7f939b122640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f939410c3c0 con 0x7f9394100740 2024-03-19T01:57:13.591 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.588+0000 7f939b122640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f939410c240 con 0x7f9394101930 2024-03-19T01:57:13.591 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.588+0000 7f939b122640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f939410c540 con 0x7f93941022f0 2024-03-19T01:57:13.591 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.589+0000 7f9398ebe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9394100740 0x7f9394197ac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:13.591 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.589+0000 7f9398ebe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9394100740 0x7f9394197ac0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:50254/0 (socket says 172.21.15.99:50254) 2024-03-19T01:57:13.591 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.589+0000 7f9398ebe640 1 -- 172.21.15.99:0/439289974 learned_addr learned my addr 172.21.15.99:0/439289974 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:57:13.591 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.589+0000 7f9393fff640 1 --2- 172.21.15.99:0/439289974 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9394101930 0x7f9394198000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:13.591 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.589+0000 7f93996bf640 1 --2- 172.21.15.99:0/439289974 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f93941022f0 0x7f939419f070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:13.591 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.589+0000 7f9398ebe640 1 -- 172.21.15.99:0/439289974 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f93941022f0 msgr2=0x7f939419f070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:57:13.592 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.589+0000 7f9398ebe640 1 --2- 172.21.15.99:0/439289974 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f93941022f0 0x7f939419f070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:13.592 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.589+0000 7f9398ebe640 1 -- 172.21.15.99:0/439289974 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9394101930 msgr2=0x7f9394198000 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:57:13.592 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.589+0000 7f9398ebe640 1 --2- 172.21.15.99:0/439289974 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9394101930 0x7f9394198000 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:13.592 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.589+0000 7f9398ebe640 1 -- 172.21.15.99:0/439289974 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f939419f6e0 con 0x7f9394100740 2024-03-19T01:57:13.592 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.589+0000 7f9398ebe640 1 --2- 172.21.15.99:0/439289974 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9394100740 0x7f9394197ac0 secure :-1 s=READY pgs=658 cs=0 l=1 rev1=1 crypto rx=0x7f938800e9c0 tx=0x7f938800ee80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:57:13.592 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.589+0000 7f9391ffb640 1 -- 172.21.15.99:0/439289974 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f938800cd80 con 0x7f9394100740 2024-03-19T01:57:13.592 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.589+0000 7f939b122640 1 -- 172.21.15.99:0/439289974 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f939419f9d0 con 0x7f9394100740 2024-03-19T01:57:13.592 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.589+0000 7f9391ffb640 1 -- 172.21.15.99:0/439289974 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f93880035f0 con 0x7f9394100740 2024-03-19T01:57:13.592 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.589+0000 7f9391ffb640 1 -- 172.21.15.99:0/439289974 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9388010690 con 0x7f9394100740 2024-03-19T01:57:13.592 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.590+0000 7f939b122640 1 -- 172.21.15.99:0/439289974 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f939419ff20 con 0x7f9394100740 2024-03-19T01:57:13.597 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.591+0000 7f9391ffb640 1 -- 172.21.15.99:0/439289974 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f93880037a0 con 0x7f9394100740 2024-03-19T01:57:13.598 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.592+0000 7f939dded740 1 Processor -- start 2024-03-19T01:57:13.598 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.592+0000 7f939dded740 1 -- start start 2024-03-19T01:57:13.598 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.592+0000 7f9391ffb640 1 --2- 172.21.15.99:0/439289974 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f93840a28a0 0x7f93840a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:13.598 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.592+0000 7f939dded740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5627260ea100 0x5627260ea520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:13.598 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.592+0000 7f939dded740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5627260eaaf0 0x5627260ef5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:13.598 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.592+0000 7f939dded740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5627260efc60 0x5627260f20f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:13.598 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.592+0000 7f939dded740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x56272604ebe0 con 0x5627260efc60 2024-03-19T01:57:13.598 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.592+0000 7f939dded740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x562725ff7960 con 0x5627260ea100 2024-03-19T01:57:13.598 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.592+0000 7f939dded740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x56272604ac80 con 0x5627260eaaf0 2024-03-19T01:57:13.598 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.592+0000 7f93996bf640 1 --2- 172.21.15.99:0/439289974 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f93840a28a0 0x7f93840a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:13.598 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.592+0000 7f9398ebe640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5627260ea100 0x5627260ea520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:13.598 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.592+0000 7f9391ffb640 1 -- 172.21.15.99:0/439289974 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f93880ceeb0 con 0x7f9394100740 2024-03-19T01:57:13.598 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.592+0000 7f9393fff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5627260eaaf0 0x5627260ef5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:13.598 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.592+0000 7f9398ebe640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5627260ea100 0x5627260ea520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:55340/0 (socket says 172.21.15.99:55340) 2024-03-19T01:57:13.599 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.592+0000 7f9393fff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5627260eaaf0 0x5627260ef5f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:34140/0 (socket says 172.21.15.99:34140) 2024-03-19T01:57:13.599 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.592+0000 7f9398ebe640 1 -- 172.21.15.99:0/3796901559 learned_addr learned my addr 172.21.15.99:0/3796901559 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:57:13.599 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.592+0000 7f93996bf640 1 --2- 172.21.15.99:0/3796901559 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5627260efc60 0x5627260f20f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:13.599 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.593+0000 7f9398ebe640 1 -- 172.21.15.99:0/3796901559 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5627260eaaf0 msgr2=0x5627260ef5f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:57:13.599 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.593+0000 7f9398ebe640 1 --2- 172.21.15.99:0/3796901559 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5627260eaaf0 0x5627260ef5f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:13.599 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.593+0000 7f9398ebe640 1 -- 172.21.15.99:0/3796901559 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5627260efc60 msgr2=0x5627260f20f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:57:13.599 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.593+0000 7f9398ebe640 1 --2- 172.21.15.99:0/3796901559 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5627260efc60 0x5627260f20f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:13.599 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.593+0000 7f9398ebe640 1 -- 172.21.15.99:0/3796901559 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x562725feed20 con 0x5627260ea100 2024-03-19T01:57:13.599 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.593+0000 7f93996bf640 1 --2- 172.21.15.99:0/439289974 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f93840a28a0 0x7f93840a4d60 secure :-1 s=READY pgs=345 cs=0 l=1 rev1=1 crypto rx=0x7f938c005e30 tx=0x7f938c009800 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:57:13.599 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.593+0000 7f9398ebe640 1 --2- 172.21.15.99:0/3796901559 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5627260ea100 0x5627260ea520 secure :-1 s=READY pgs=175 cs=0 l=1 rev1=1 crypto rx=0x7f9388098180 tx=0x7f938809b110 comp rx=0 tx=0).ready entity=mon.2 client_cookie=a436128c2321b619 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:57:13.599 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.593+0000 7f937effd640 1 -- 172.21.15.99:0/3796901559 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f938809f900 con 0x5627260ea100 2024-03-19T01:57:13.599 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.593+0000 7f937effd640 1 -- 172.21.15.99:0/3796901559 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f93880a3f50 con 0x5627260ea100 2024-03-19T01:57:13.599 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.593+0000 7f937effd640 1 -- 172.21.15.99:0/3796901559 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9388097e20 con 0x5627260ea100 2024-03-19T01:57:13.599 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.594+0000 7f939dded740 1 -- 172.21.15.99:0/3796901559 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5627260ea100 msgr2=0x5627260ea520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:57:13.600 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.594+0000 7f939dded740 1 --2- 172.21.15.99:0/3796901559 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5627260ea100 0x5627260ea520 secure :-1 s=READY pgs=175 cs=0 l=1 rev1=1 crypto rx=0x7f9388098180 tx=0x7f938809b110 comp rx=0 tx=0).stop 2024-03-19T01:57:13.600 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.594+0000 7f939dded740 1 -- 172.21.15.99:0/3796901559 shutdown_connections 2024-03-19T01:57:13.600 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.594+0000 7f939dded740 1 --2- 172.21.15.99:0/3796901559 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5627260efc60 0x5627260f20f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:13.600 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.594+0000 7f939dded740 1 --2- 172.21.15.99:0/3796901559 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5627260eaaf0 0x5627260ef5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:13.600 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.594+0000 7f939dded740 1 --2- 172.21.15.99:0/3796901559 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5627260ea100 0x5627260ea520 unknown :-1 s=CLOSED pgs=175 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:13.600 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.594+0000 7f939dded740 1 -- 172.21.15.99:0/3796901559 >> 172.21.15.99:0/3796901559 conn(0x5627260e5670 msgr2=0x5627260e5a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:57:13.600 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.594+0000 7f939dded740 1 -- 172.21.15.99:0/3796901559 shutdown_connections 2024-03-19T01:57:13.600 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.594+0000 7f939dded740 1 -- 172.21.15.99:0/3796901559 wait complete. 2024-03-19T01:57:13.600 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.594+0000 7f939dded740 1 -- start start 2024-03-19T01:57:13.600 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.594+0000 7f939dded740 1 Processor -- start 2024-03-19T01:57:13.600 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.595+0000 7f939dded740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5627260ea100 0x5627260f40d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:13.600 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.595+0000 7f939dded740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5627260eaaf0 0x5627260f4610 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:13.600 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.595+0000 7f939dded740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5627260efc60 0x5627260fb650 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:13.600 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.595+0000 7f939dded740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x5627260f2bb0 con 0x5627260eaaf0 2024-03-19T01:57:13.600 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.595+0000 7f939dded740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x56272600adb0 con 0x5627260efc60 2024-03-19T01:57:13.601 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.595+0000 7f939dded740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x5627260f2d30 con 0x5627260ea100 2024-03-19T01:57:13.601 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.595+0000 7f939dded740 10 client.0 _ll_register_callbacks cb 0x56272600fe70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:57:13.601 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.595+0000 7f9393fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5627260eaaf0 0x5627260f4610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:13.601 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.595+0000 7f9398ebe640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5627260ea100 0x5627260f40d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:13.601 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.595+0000 7f93996bf640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5627260efc60 0x5627260fb650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:13.601 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.595+0000 7f9393fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5627260eaaf0 0x5627260f4610 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:50264/0 (socket says 172.21.15.99:50264) 2024-03-19T01:57:13.601 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.595+0000 7f9393fff640 1 -- 172.21.15.99:0/1640151581 learned_addr learned my addr 172.21.15.99:0/1640151581 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:57:13.601 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.595+0000 7f93996bf640 1 --2- 172.21.15.99:0/1640151581 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5627260efc60 0x5627260fb650 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:55342/0 (socket says 172.21.15.99:55342) 2024-03-19T01:57:13.601 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.595+0000 7f9398ebe640 1 -- 172.21.15.99:0/1640151581 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5627260efc60 msgr2=0x5627260fb650 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:57:13.601 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.595+0000 7f9398ebe640 1 --2- 172.21.15.99:0/1640151581 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5627260efc60 0x5627260fb650 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:13.601 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.595+0000 7f9398ebe640 1 -- 172.21.15.99:0/1640151581 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5627260eaaf0 msgr2=0x5627260f4610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:57:13.601 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.595+0000 7f9398ebe640 1 --2- 172.21.15.99:0/1640151581 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5627260eaaf0 0x5627260f4610 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:13.601 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.595+0000 7f9398ebe640 1 -- 172.21.15.99:0/1640151581 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({osdmap=0}) -- 0x562726009c60 con 0x5627260ea100 2024-03-19T01:57:13.601 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.595+0000 7f93996bf640 1 --2- 172.21.15.99:0/1640151581 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5627260efc60 0x5627260fb650 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:57:13.602 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.595+0000 7f9393fff640 1 --2- 172.21.15.99:0/1640151581 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5627260eaaf0 0x5627260f4610 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:57:13.602 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.595+0000 7f9398ebe640 1 --2- 172.21.15.99:0/1640151581 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5627260ea100 0x5627260f40d0 secure :-1 s=READY pgs=120 cs=0 l=1 rev1=1 crypto rx=0x7f938809b4b0 tx=0x7f938809b240 comp rx=0 tx=0).ready entity=mon.1 client_cookie=e961bfbdfb636873 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:57:13.602 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.595+0000 7f9373fff640 10 client.0 ms_handle_connect on v2:172.21.15.115:3300/0 2024-03-19T01:57:13.602 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.595+0000 7f9373fff640 1 -- 172.21.15.99:0/1640151581 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f93880b3020 con 0x5627260ea100 2024-03-19T01:57:13.602 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.596+0000 7f939dded740 1 -- 172.21.15.99:0/1640151581 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f93880a3f50 con 0x5627260ea100 2024-03-19T01:57:13.602 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.596+0000 7f9373fff640 1 -- 172.21.15.99:0/1640151581 <== mon.1 v2:172.21.15.115:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f93880afb90 con 0x5627260ea100 2024-03-19T01:57:13.602 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.596+0000 7f9373fff640 1 -- 172.21.15.99:0/1640151581 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7f93880afd90 con 0x5627260ea100 2024-03-19T01:57:13.602 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.596+0000 7f939dded740 10 client.26455 fetch_fsmap learned FSMap version 37 2024-03-19T01:57:13.602 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.596+0000 7f939dded740 1 -- 172.21.15.99:0/1640151581 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x562725d63210 con 0x5627260ea100 2024-03-19T01:57:13.602 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.596+0000 7f9373fff640 1 -- 172.21.15.99:0/1640151581 <== mon.1 v2:172.21.15.115:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f93880a3f50 con 0x5627260ea100 2024-03-19T01:57:13.602 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.596+0000 7f9373fff640 1 -- 172.21.15.99:0/1640151581 <== mon.1 v2:172.21.15.115:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7f938809a670 con 0x5627260ea100 2024-03-19T01:57:13.602 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.596+0000 7f939dded740 10 client.26455 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T01:57:13.602 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.596+0000 7f939dded740 10 client.26455 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:57:13.602 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.596+0000 7f939dded740 20 client.26455 populate_metadata read hostname 'smithi099' 2024-03-19T01:57:13.602 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.596+0000 7f939dded740 1 --2- 172.21.15.99:0/1640151581 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5627260ffb40 0x562726101f30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:13.603 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.596+0000 7f939dded740 4 client.26455 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:57:13.603 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.596+0000 7f939dded740 1 -- 172.21.15.99:0/1640151581 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x56272605c120 con 0x5627260ffb40 2024-03-19T01:57:13.603 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.596+0000 7f9393fff640 1 --2- 172.21.15.99:0/1640151581 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5627260ffb40 0x562726101f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:13.603 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.597+0000 7f9393fff640 1 --2- 172.21.15.99:0/1640151581 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5627260ffb40 0x562726101f30 crc :-1 s=READY pgs=173 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=44ceec4446b864d5 server_cookie=a69ec442e855bc78 in_seq=0 out_seq=0 2024-03-19T01:57:13.603 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.597+0000 7f9373fff640 10 client.26455 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:57:13.604 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.601+0000 7f9373fff640 1 -- 172.21.15.99:0/1640151581 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x56272605c120 con 0x5627260ffb40 2024-03-19T01:57:13.604 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.601+0000 7f9373fff640 10 client.26455 handle_command_reply: tid=0 2024-03-19T01:57:13.604 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.601+0000 7f939dded740 1 client.26455 shutdown 2024-03-19T01:57:13.604 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.601+0000 7f939dded740 1 -- 172.21.15.99:0/1640151581 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5627260ea100 msgr2=0x5627260f40d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:57:13.604 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.601+0000 7f939dded740 1 --2- 172.21.15.99:0/1640151581 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5627260ea100 0x5627260f40d0 secure :-1 s=READY pgs=120 cs=0 l=1 rev1=1 crypto rx=0x7f938809b4b0 tx=0x7f938809b240 comp rx=0 tx=0).stop 2024-03-19T01:57:13.604 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.601+0000 7f939dded740 1 -- 172.21.15.99:0/1640151581 shutdown_connections 2024-03-19T01:57:13.604 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.601+0000 7f939dded740 1 --2- 172.21.15.99:0/1640151581 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5627260ffb40 0x562726101f30 crc :-1 s=READY pgs=173 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:13.604 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.601+0000 7f939dded740 1 --2- 172.21.15.99:0/1640151581 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5627260efc60 0x5627260fb650 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:13.605 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.601+0000 7f9373fff640 0 client.26455 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:57:13.605 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.601+0000 7f939dded740 1 --2- 172.21.15.99:0/1640151581 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5627260eaaf0 0x5627260f4610 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:13.605 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.601+0000 7f939dded740 1 --2- 172.21.15.99:0/1640151581 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5627260ea100 0x5627260f40d0 unknown :-1 s=CLOSED pgs=120 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:13.605 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.601+0000 7f939dded740 1 -- 172.21.15.99:0/1640151581 >> 172.21.15.99:0/1640151581 conn(0x5627260e5670 msgr2=0x5627260e68c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:57:13.605 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.601+0000 7f939dded740 1 -- 172.21.15.99:0/1640151581 shutdown_connections 2024-03-19T01:57:13.605 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.601+0000 7f939dded740 1 -- 172.21.15.99:0/1640151581 wait complete. 2024-03-19T01:57:13.605 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.601+0000 7f939dded740 20 client.26455 trim_cache size 0 max 16384 2024-03-19T01:57:13.620 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.616+0000 7f939dded740 1 Processor -- start 2024-03-19T01:57:13.620 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.616+0000 7f939dded740 1 -- start start 2024-03-19T01:57:13.620 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.616+0000 7f939dded740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5627260ea100 0x5627260e4890 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:13.620 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.616+0000 7f939dded740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5627260eaaf0 0x5627260f4570 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:13.620 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.616+0000 7f939dded740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5627260efc60 0x5627260e8fe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:13.621 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.616+0000 7f939dded740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f93941ab700 con 0x5627260eaaf0 2024-03-19T01:57:13.621 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.617+0000 7f939dded740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f93941a8fd0 con 0x5627260ea100 2024-03-19T01:57:13.621 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.617+0000 7f939dded740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f93941a6e20 con 0x5627260efc60 2024-03-19T01:57:13.621 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.617+0000 7f9393fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5627260eaaf0 0x5627260f4570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:13.621 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.617+0000 7f9393fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5627260eaaf0 0x5627260f4570 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:50270/0 (socket says 172.21.15.99:50270) 2024-03-19T01:57:13.621 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.617+0000 7f9393fff640 1 -- 172.21.15.99:0/642070766 learned_addr learned my addr 172.21.15.99:0/642070766 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:57:13.621 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.617+0000 7f9398ebe640 1 --2- 172.21.15.99:0/642070766 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5627260ea100 0x5627260e4890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:13.621 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.617+0000 7f93996bf640 1 --2- 172.21.15.99:0/642070766 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5627260efc60 0x5627260e8fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:13.621 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.617+0000 7f9393fff640 1 -- 172.21.15.99:0/642070766 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5627260efc60 msgr2=0x5627260e8fe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:57:13.621 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.617+0000 7f9393fff640 1 --2- 172.21.15.99:0/642070766 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5627260efc60 0x5627260e8fe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:13.621 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.617+0000 7f9393fff640 1 -- 172.21.15.99:0/642070766 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5627260ea100 msgr2=0x5627260e4890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:57:13.621 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.617+0000 7f9393fff640 1 --2- 172.21.15.99:0/642070766 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5627260ea100 0x5627260e4890 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:13.621 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.617+0000 7f9393fff640 1 -- 172.21.15.99:0/642070766 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x562725e2f620 con 0x5627260eaaf0 2024-03-19T01:57:13.621 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.617+0000 7f9393fff640 1 --2- 172.21.15.99:0/642070766 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5627260eaaf0 0x5627260f4570 secure :-1 s=READY pgs=659 cs=0 l=1 rev1=1 crypto rx=0x5627261026b0 tx=0x7f9380005f90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=779125aad63d2752 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:57:13.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.617+0000 7f937e7fc640 1 -- 172.21.15.99:0/642070766 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9380005220 con 0x5627260eaaf0 2024-03-19T01:57:13.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.617+0000 7f937e7fc640 1 -- 172.21.15.99:0/642070766 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f9380004c80 con 0x5627260eaaf0 2024-03-19T01:57:13.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.617+0000 7f937e7fc640 1 -- 172.21.15.99:0/642070766 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9380020670 con 0x5627260eaaf0 2024-03-19T01:57:13.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.618+0000 7f939dded740 1 -- 172.21.15.99:0/642070766 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5627260eaaf0 msgr2=0x5627260f4570 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:57:13.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.618+0000 7f939dded740 1 --2- 172.21.15.99:0/642070766 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5627260eaaf0 0x5627260f4570 secure :-1 s=READY pgs=659 cs=0 l=1 rev1=1 crypto rx=0x5627261026b0 tx=0x7f9380005f90 comp rx=0 tx=0).stop 2024-03-19T01:57:13.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.618+0000 7f939dded740 1 -- 172.21.15.99:0/642070766 shutdown_connections 2024-03-19T01:57:13.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.618+0000 7f939dded740 1 --2- 172.21.15.99:0/642070766 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5627260efc60 0x5627260e8fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:13.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.618+0000 7f939dded740 1 --2- 172.21.15.99:0/642070766 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5627260eaaf0 0x5627260f4570 unknown :-1 s=CLOSED pgs=659 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:13.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.618+0000 7f939dded740 1 --2- 172.21.15.99:0/642070766 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5627260ea100 0x5627260e4890 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:13.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.618+0000 7f939dded740 1 -- 172.21.15.99:0/642070766 >> 172.21.15.99:0/642070766 conn(0x5627260e5670 msgr2=0x5627260e88d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:57:13.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.618+0000 7f939dded740 1 -- 172.21.15.99:0/642070766 shutdown_connections 2024-03-19T01:57:13.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.618+0000 7f939dded740 1 -- 172.21.15.99:0/642070766 wait complete. 2024-03-19T01:57:13.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.618+0000 7f939dded740 1 -- start start 2024-03-19T01:57:13.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.618+0000 7f939dded740 1 Processor -- start 2024-03-19T01:57:13.623 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.619+0000 7f939dded740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5627260ea100 0x5627260f9180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:13.623 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.619+0000 7f939dded740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5627260eaaf0 0x5627260f96c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:13.623 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.619+0000 7f939dded740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5627260efc60 0x5627260fa6d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:13.623 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.619+0000 7f939dded740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f93941a7490 con 0x5627260efc60 2024-03-19T01:57:13.623 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.619+0000 7f939dded740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f93941a8900 con 0x5627260eaaf0 2024-03-19T01:57:13.623 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.619+0000 7f939dded740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f93941a7b60 con 0x5627260ea100 2024-03-19T01:57:13.623 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.619+0000 7f939dded740 10 client.0 _ll_register_callbacks cb 0x7f9388005e30 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:57:13.623 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.619+0000 7f93996bf640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5627260efc60 0x5627260fa6d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:13.623 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.619+0000 7f9393fff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5627260eaaf0 0x5627260f96c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:13.623 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.619+0000 7f93996bf640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5627260efc60 0x5627260fa6d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:50282/0 (socket says 172.21.15.99:50282) 2024-03-19T01:57:13.623 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.619+0000 7f93996bf640 1 -- 172.21.15.99:0/2561030152 learned_addr learned my addr 172.21.15.99:0/2561030152 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:57:13.623 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.619+0000 7f9393fff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5627260eaaf0 0x5627260f96c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:55374/0 (socket says 172.21.15.99:55374) 2024-03-19T01:57:13.623 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.619+0000 7f9398ebe640 1 --2- 172.21.15.99:0/2561030152 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5627260ea100 0x5627260f9180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:13.623 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.619+0000 7f93996bf640 1 -- 172.21.15.99:0/2561030152 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5627260ea100 msgr2=0x5627260f9180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:57:13.623 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.619+0000 7f93996bf640 1 --2- 172.21.15.99:0/2561030152 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5627260ea100 0x5627260f9180 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:13.624 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.619+0000 7f93996bf640 1 -- 172.21.15.99:0/2561030152 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5627260eaaf0 msgr2=0x5627260f96c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:57:13.624 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.619+0000 7f93996bf640 1 --2- 172.21.15.99:0/2561030152 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5627260eaaf0 0x5627260f96c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:13.624 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.619+0000 7f93996bf640 1 -- 172.21.15.99:0/2561030152 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5627260facc0 con 0x5627260efc60 2024-03-19T01:57:13.624 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.619+0000 7f9393fff640 1 --2- 172.21.15.99:0/2561030152 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5627260eaaf0 0x5627260f96c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:57:13.624 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.619+0000 7f93996bf640 1 --2- 172.21.15.99:0/2561030152 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5627260efc60 0x5627260fa6d0 secure :-1 s=READY pgs=660 cs=0 l=1 rev1=1 crypto rx=0x7f938c0324c0 tx=0x7f938c032590 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e00ea901c4bc56e5 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:57:13.624 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.619+0000 7f9373fff640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:57:13.624 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.619+0000 7f9373fff640 1 -- 172.21.15.99:0/2561030152 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f938c052020 con 0x5627260efc60 2024-03-19T01:57:13.624 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.619+0000 7f9373fff640 1 -- 172.21.15.99:0/2561030152 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f938c004250 con 0x5627260efc60 2024-03-19T01:57:13.624 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.619+0000 7f939dded740 1 -- 172.21.15.99:0/2561030152 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f9380004c80 con 0x5627260efc60 2024-03-19T01:57:13.624 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.620+0000 7f9373fff640 1 -- 172.21.15.99:0/2561030152 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7f938c05d050 con 0x5627260efc60 2024-03-19T01:57:13.624 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.620+0000 7f939dded740 10 client.16644 fetch_fsmap learned FSMap version 37 2024-03-19T01:57:13.624 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.620+0000 7f939dded740 1 -- 172.21.15.99:0/2561030152 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x562726166500 con 0x5627260efc60 2024-03-19T01:57:13.624 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.620+0000 7f9373fff640 1 -- 172.21.15.99:0/2561030152 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f9380004c80 con 0x5627260efc60 2024-03-19T01:57:13.624 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.620+0000 7f9373fff640 1 -- 172.21.15.99:0/2561030152 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7f938c03f040 con 0x5627260efc60 2024-03-19T01:57:13.625 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.620+0000 7f939dded740 10 client.16644 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T01:57:13.625 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.620+0000 7f939dded740 10 client.16644 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:57:13.625 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.620+0000 7f939dded740 20 client.16644 populate_metadata read hostname 'smithi099' 2024-03-19T01:57:13.625 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.620+0000 7f939dded740 1 --2- 172.21.15.99:0/2561030152 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x562726166f80 0x5627261693a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:13.625 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.620+0000 7f939dded740 4 client.16644 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:57:13.625 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.620+0000 7f939dded740 1 -- 172.21.15.99:0/2561030152 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x562726056470 con 0x562726166f80 2024-03-19T01:57:13.625 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.620+0000 7f9398ebe640 1 --2- 172.21.15.99:0/2561030152 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x562726166f80 0x5627261693a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:13.625 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.620+0000 7f9373fff640 10 client.16644 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:57:13.625 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:13.620+0000 7f9398ebe640 1 --2- 172.21.15.99:0/2561030152 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x562726166f80 0x5627261693a0 crc :-1 s=READY pgs=174 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=5bb38e80e3d498f3 server_cookie=d51b3f5b0b4f4c63 in_seq=0 out_seq=0 2024-03-19T01:57:14.105 INFO:tasks.workunit.client.0.smithi099.stdout: 10 98815 23.17 MB/sec execute 103 sec latency 417.170 ms 2024-03-19T01:57:14.239 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:14.239+0000 7f9373fff640 1 -- 172.21.15.99:0/2561030152 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x562726056470 con 0x562726166f80 2024-03-19T01:57:14.240 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:14.239+0000 7f9373fff640 10 client.16644 handle_command_reply: tid=0 2024-03-19T01:57:14.240 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:14.239+0000 7f939dded740 1 client.16644 shutdown 2024-03-19T01:57:14.240 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:14.239+0000 7f939dded740 1 -- 172.21.15.99:0/2561030152 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5627260efc60 msgr2=0x5627260fa6d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:57:14.240 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:14.239+0000 7f939dded740 1 --2- 172.21.15.99:0/2561030152 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5627260efc60 0x5627260fa6d0 secure :-1 s=READY pgs=660 cs=0 l=1 rev1=1 crypto rx=0x7f938c0324c0 tx=0x7f938c032590 comp rx=0 tx=0).stop 2024-03-19T01:57:14.240 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:14.239+0000 7f939dded740 1 -- 172.21.15.99:0/2561030152 shutdown_connections 2024-03-19T01:57:14.240 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:14.239+0000 7f939dded740 1 --2- 172.21.15.99:0/2561030152 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x562726166f80 0x5627261693a0 crc :-1 s=READY pgs=174 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:14.240 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:14.239+0000 7f939dded740 1 --2- 172.21.15.99:0/2561030152 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5627260eaaf0 0x5627260f96c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:14.240 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:14.240+0000 7f939dded740 1 --2- 172.21.15.99:0/2561030152 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5627260efc60 0x5627260fa6d0 unknown :-1 s=CLOSED pgs=660 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:14.240 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:14.240+0000 7f939dded740 1 --2- 172.21.15.99:0/2561030152 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5627260ea100 0x5627260f9180 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:14.240 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:14.240+0000 7f939dded740 1 -- 172.21.15.99:0/2561030152 >> 172.21.15.99:0/2561030152 conn(0x5627260e5670 msgr2=0x5627260e9160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:57:14.240 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:14.240+0000 7f9373fff640 0 client.16644 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:57:14.241 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:14.240+0000 7f939dded740 1 -- 172.21.15.99:0/2561030152 shutdown_connections 2024-03-19T01:57:14.241 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:14.240+0000 7f939dded740 1 -- 172.21.15.99:0/2561030152 wait complete. 2024-03-19T01:57:14.241 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:14.240+0000 7f939dded740 20 client.16644 trim_cache size 0 max 16384 2024-03-19T01:57:14.241 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:57:14.241 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:57:14.241 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:57:14.241 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:57:14.242 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:14.241+0000 7f939b122640 1 -- 172.21.15.99:0/439289974 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f93840a28a0 msgr2=0x7f93840a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:57:14.242 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:14.241+0000 7f939b122640 1 --2- 172.21.15.99:0/439289974 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f93840a28a0 0x7f93840a4d60 secure :-1 s=READY pgs=345 cs=0 l=1 rev1=1 crypto rx=0x7f938c005e30 tx=0x7f938c009800 comp rx=0 tx=0).stop 2024-03-19T01:57:14.242 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:14.241+0000 7f939b122640 1 -- 172.21.15.99:0/439289974 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9394100740 msgr2=0x7f9394197ac0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:57:14.242 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:14.241+0000 7f939b122640 1 --2- 172.21.15.99:0/439289974 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9394100740 0x7f9394197ac0 secure :-1 s=READY pgs=658 cs=0 l=1 rev1=1 crypto rx=0x7f938800e9c0 tx=0x7f938800ee80 comp rx=0 tx=0).stop 2024-03-19T01:57:14.242 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:14.241+0000 7f939b122640 1 -- 172.21.15.99:0/439289974 shutdown_connections 2024-03-19T01:57:14.242 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:14.241+0000 7f939b122640 1 --2- 172.21.15.99:0/439289974 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f93840a28a0 0x7f93840a4d60 unknown :-1 s=CLOSED pgs=345 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:14.243 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:14.241+0000 7f939b122640 1 --2- 172.21.15.99:0/439289974 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9394101930 0x7f9394198000 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:14.243 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:14.241+0000 7f939b122640 1 --2- 172.21.15.99:0/439289974 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f93941022f0 0x7f939419f070 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:14.243 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:14.241+0000 7f939b122640 1 --2- 172.21.15.99:0/439289974 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9394100740 0x7f9394197ac0 unknown :-1 s=CLOSED pgs=658 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:14.243 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:14.241+0000 7f939b122640 1 -- 172.21.15.99:0/439289974 >> 172.21.15.99:0/439289974 conn(0x7f93940fbcb0 msgr2=0x7f93940fd6c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:57:14.243 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:14.241+0000 7f939b122640 1 -- 172.21.15.99:0/439289974 shutdown_connections 2024-03-19T01:57:14.243 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:14.241+0000 7f939b122640 1 -- 172.21.15.99:0/439289974 wait complete. 2024-03-19T01:57:15.105 INFO:tasks.workunit.client.0.smithi099.stdout: 10 99201 23.12 MB/sec execute 104 sec latency 312.959 ms 2024-03-19T01:57:15.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:57:15 smithi099 ceph-mon[62951]: pgmap v836: 97 pgs: 97 active+clean; 541 MiB data, 6.1 GiB used, 1.0 TiB / 1.0 TiB avail; 3.9 MiB/s rd, 4.9 MiB/s wr, 423 op/s 2024-03-19T01:57:15.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:57:15 smithi112 ceph-mon[66819]: pgmap v836: 97 pgs: 97 active+clean; 541 MiB data, 6.1 GiB used, 1.0 TiB / 1.0 TiB avail; 3.9 MiB/s rd, 4.9 MiB/s wr, 423 op/s 2024-03-19T01:57:15.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:57:15 smithi115 ceph-mon[67332]: pgmap v836: 97 pgs: 97 active+clean; 541 MiB data, 6.1 GiB used, 1.0 TiB / 1.0 TiB avail; 3.9 MiB/s rd, 4.9 MiB/s wr, 423 op/s 2024-03-19T01:57:16.105 INFO:tasks.workunit.client.0.smithi099.stdout: 10 99686 23.16 MB/sec execute 105 sec latency 90.609 ms 2024-03-19T01:57:17.105 INFO:tasks.workunit.client.0.smithi099.stdout: 10 100165 23.19 MB/sec execute 106 sec latency 84.870 ms 2024-03-19T01:57:17.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:57:17 smithi099 ceph-mon[62951]: pgmap v837: 97 pgs: 97 active+clean; 543 MiB data, 6.1 GiB used, 1.0 TiB / 1.0 TiB avail; 5.0 MiB/s rd, 6.4 MiB/s wr, 587 op/s 2024-03-19T01:57:17.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:57:17 smithi112 ceph-mon[66819]: pgmap v837: 97 pgs: 97 active+clean; 543 MiB data, 6.1 GiB used, 1.0 TiB / 1.0 TiB avail; 5.0 MiB/s rd, 6.4 MiB/s wr, 587 op/s 2024-03-19T01:57:17.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:57:17 smithi115 ceph-mon[67332]: pgmap v837: 97 pgs: 97 active+clean; 543 MiB data, 6.1 GiB used, 1.0 TiB / 1.0 TiB avail; 5.0 MiB/s rd, 6.4 MiB/s wr, 587 op/s 2024-03-19T01:57:18.105 INFO:tasks.workunit.client.0.smithi099.stdout: 10 100509 23.12 MB/sec execute 107 sec latency 157.424 ms 2024-03-19T01:57:19.105 INFO:tasks.workunit.client.0.smithi099.stdout: 10 100828 23.05 MB/sec execute 108 sec latency 331.259 ms 2024-03-19T01:57:19.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:57:19 smithi099 ceph-mon[62951]: pgmap v838: 97 pgs: 97 active+clean; 541 MiB data, 6.1 GiB used, 1.0 TiB / 1.0 TiB avail; 4.6 MiB/s rd, 5.8 MiB/s wr, 586 op/s 2024-03-19T01:57:19.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:57:19 smithi112 ceph-mon[66819]: pgmap v838: 97 pgs: 97 active+clean; 541 MiB data, 6.1 GiB used, 1.0 TiB / 1.0 TiB avail; 4.6 MiB/s rd, 5.8 MiB/s wr, 586 op/s 2024-03-19T01:57:19.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:57:19 smithi115 ceph-mon[67332]: pgmap v838: 97 pgs: 97 active+clean; 541 MiB data, 6.1 GiB used, 1.0 TiB / 1.0 TiB avail; 4.6 MiB/s rd, 5.8 MiB/s wr, 586 op/s 2024-03-19T01:57:20.105 INFO:tasks.workunit.client.0.smithi099.stdout: 10 101255 23.09 MB/sec execute 109 sec latency 113.308 ms 2024-03-19T01:57:21.105 INFO:tasks.workunit.client.0.smithi099.stdout: 10 101797 23.09 MB/sec execute 110 sec latency 67.497 ms 2024-03-19T01:57:21.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:57:21 smithi099 ceph-mon[62951]: pgmap v839: 97 pgs: 97 active+clean; 541 MiB data, 6.1 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 5.4 MiB/s wr, 552 op/s 2024-03-19T01:57:21.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:57:21 smithi112 ceph-mon[66819]: pgmap v839: 97 pgs: 97 active+clean; 541 MiB data, 6.1 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 5.4 MiB/s wr, 552 op/s 2024-03-19T01:57:21.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:57:21 smithi115 ceph-mon[67332]: pgmap v839: 97 pgs: 97 active+clean; 541 MiB data, 6.1 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 5.4 MiB/s wr, 552 op/s 2024-03-19T01:57:22.105 INFO:tasks.workunit.client.0.smithi099.stdout: 10 102414 23.17 MB/sec execute 111 sec latency 73.825 ms 2024-03-19T01:57:23.106 INFO:tasks.workunit.client.0.smithi099.stdout: 10 102910 23.22 MB/sec execute 112 sec latency 66.707 ms 2024-03-19T01:57:23.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:57:23 smithi099 ceph-mon[62951]: pgmap v840: 97 pgs: 97 active+clean; 548 MiB data, 6.2 GiB used, 1.0 TiB / 1.0 TiB avail; 4.8 MiB/s rd, 6.0 MiB/s wr, 646 op/s 2024-03-19T01:57:23.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:57:23 smithi112 ceph-mon[66819]: pgmap v840: 97 pgs: 97 active+clean; 548 MiB data, 6.2 GiB used, 1.0 TiB / 1.0 TiB avail; 4.8 MiB/s rd, 6.0 MiB/s wr, 646 op/s 2024-03-19T01:57:23.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:57:23 smithi115 ceph-mon[67332]: pgmap v840: 97 pgs: 97 active+clean; 548 MiB data, 6.2 GiB used, 1.0 TiB / 1.0 TiB avail; 4.8 MiB/s rd, 6.0 MiB/s wr, 646 op/s 2024-03-19T01:57:24.105 INFO:tasks.workunit.client.0.smithi099.stdout: 10 103243 23.21 MB/sec execute 113 sec latency 222.327 ms 2024-03-19T01:57:25.106 INFO:tasks.workunit.client.0.smithi099.stdout: 10 103692 23.23 MB/sec execute 114 sec latency 73.070 ms 2024-03-19T01:57:25.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:57:25 smithi099 ceph-mon[62951]: pgmap v841: 97 pgs: 97 active+clean; 548 MiB data, 6.2 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 5.3 MiB/s wr, 551 op/s 2024-03-19T01:57:25.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:57:25 smithi112 ceph-mon[66819]: pgmap v841: 97 pgs: 97 active+clean; 548 MiB data, 6.2 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 5.3 MiB/s wr, 551 op/s 2024-03-19T01:57:25.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:57:25 smithi115 ceph-mon[67332]: pgmap v841: 97 pgs: 97 active+clean; 548 MiB data, 6.2 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 5.3 MiB/s wr, 551 op/s 2024-03-19T01:57:26.106 INFO:tasks.workunit.client.0.smithi099.stdout: 10 104322 23.26 MB/sec execute 115 sec latency 54.254 ms 2024-03-19T01:57:27.106 INFO:tasks.workunit.client.0.smithi099.stdout: 10 104806 23.34 MB/sec execute 116 sec latency 80.893 ms 2024-03-19T01:57:27.336 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.334+0000 7f370e1df640 1 -- 172.21.15.99:0/73312024 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3708101900 msgr2=0x7f3708101d80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:57:27.336 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.334+0000 7f370e1df640 1 --2- 172.21.15.99:0/73312024 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3708101900 0x7f3708101d80 secure :-1 s=READY pgs=661 cs=0 l=1 rev1=1 crypto rx=0x7f36fc009900 tx=0x7f36fc02bc30 comp rx=0 tx=0).stop 2024-03-19T01:57:27.336 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.334+0000 7f370e1df640 1 -- 172.21.15.99:0/73312024 shutdown_connections 2024-03-19T01:57:27.336 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.334+0000 7f370e1df640 1 --2- 172.21.15.99:0/73312024 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f37081022c0 0x7f3708108b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:27.336 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.334+0000 7f370e1df640 1 --2- 172.21.15.99:0/73312024 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3708101900 0x7f3708101d80 unknown :-1 s=CLOSED pgs=661 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:27.336 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.334+0000 7f370e1df640 1 --2- 172.21.15.99:0/73312024 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f3708100710 0x7f3708100b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:27.336 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.334+0000 7f370e1df640 1 -- 172.21.15.99:0/73312024 >> 172.21.15.99:0/73312024 conn(0x7f37080fbc80 msgr2=0x7f37080fe0c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:57:27.336 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.334+0000 7f370e1df640 1 -- 172.21.15.99:0/73312024 shutdown_connections 2024-03-19T01:57:27.336 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.334+0000 7f370e1df640 1 -- 172.21.15.99:0/73312024 wait complete. 2024-03-19T01:57:27.336 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.334+0000 7f370e1df640 1 Processor -- start 2024-03-19T01:57:27.337 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.335+0000 7f370e1df640 1 -- start start 2024-03-19T01:57:27.337 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.335+0000 7f370e1df640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3708100710 0x7f3708196770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:27.337 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.335+0000 7f370e1df640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3708101900 0x7f3708196cb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:27.337 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.335+0000 7f370e1df640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f37081022c0 0x7f370819dd20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:27.337 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.335+0000 7f370e1df640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f370810c370 con 0x7f3708100710 2024-03-19T01:57:27.337 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.335+0000 7f370e1df640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f370810c1f0 con 0x7f3708101900 2024-03-19T01:57:27.337 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.335+0000 7f370e1df640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f370810c4f0 con 0x7f37081022c0 2024-03-19T01:57:27.337 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.335+0000 7f37077fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3708100710 0x7f3708196770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:27.337 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.335+0000 7f37077fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3708100710 0x7f3708196770 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:55918/0 (socket says 172.21.15.99:55918) 2024-03-19T01:57:27.337 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.335+0000 7f37077fe640 1 -- 172.21.15.99:0/1535111746 learned_addr learned my addr 172.21.15.99:0/1535111746 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:57:27.337 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.335+0000 7f3706ffd640 1 --2- 172.21.15.99:0/1535111746 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3708101900 0x7f3708196cb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:27.337 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.335+0000 7f37077fe640 1 -- 172.21.15.99:0/1535111746 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f37081022c0 msgr2=0x7f370819dd20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:57:27.337 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.335+0000 7f37077fe640 1 --2- 172.21.15.99:0/1535111746 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f37081022c0 0x7f370819dd20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:27.337 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.335+0000 7f37077fe640 1 -- 172.21.15.99:0/1535111746 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3708101900 msgr2=0x7f3708196cb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:57:27.337 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.335+0000 7f37077fe640 1 --2- 172.21.15.99:0/1535111746 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3708101900 0x7f3708196cb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:27.338 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.335+0000 7f37077fe640 1 -- 172.21.15.99:0/1535111746 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f370819e390 con 0x7f3708100710 2024-03-19T01:57:27.338 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.335+0000 7f37077fe640 1 --2- 172.21.15.99:0/1535111746 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3708100710 0x7f3708196770 secure :-1 s=READY pgs=662 cs=0 l=1 rev1=1 crypto rx=0x7f36f800e9c0 tx=0x7f36f800ee80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:57:27.338 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.336+0000 7f3704ff9640 1 -- 172.21.15.99:0/1535111746 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f36f800cd80 con 0x7f3708100710 2024-03-19T01:57:27.338 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.336+0000 7f370e1df640 1 -- 172.21.15.99:0/1535111746 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f370819e680 con 0x7f3708100710 2024-03-19T01:57:27.338 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.336+0000 7f370e1df640 1 -- 172.21.15.99:0/1535111746 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f370819ec20 con 0x7f3708100710 2024-03-19T01:57:27.338 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.336+0000 7f3704ff9640 1 -- 172.21.15.99:0/1535111746 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f36f80035f0 con 0x7f3708100710 2024-03-19T01:57:27.338 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.336+0000 7f3704ff9640 1 -- 172.21.15.99:0/1535111746 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f36f8010690 con 0x7f3708100710 2024-03-19T01:57:27.340 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.339+0000 7f3704ff9640 1 -- 172.21.15.99:0/1535111746 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f36f80108f0 con 0x7f3708100710 2024-03-19T01:57:27.340 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.339+0000 7f3710dc8740 1 Processor -- start 2024-03-19T01:57:27.341 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.339+0000 7f3710dc8740 1 -- start start 2024-03-19T01:57:27.341 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.339+0000 7f3710dc8740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e08abfe310 0x55e08abfe730 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:27.341 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.339+0000 7f3710dc8740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e08abfed00 0x55e08ac03800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:27.341 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.339+0000 7f3710dc8740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e08ac03e70 0x55e08ac06300 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:27.341 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.339+0000 7f3710dc8740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55e08ab60be0 con 0x55e08ac03e70 2024-03-19T01:57:27.341 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.339+0000 7f3710dc8740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55e08ab09960 con 0x55e08abfe310 2024-03-19T01:57:27.341 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.339+0000 7f3710dc8740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55e08ab5cc80 con 0x55e08abfed00 2024-03-19T01:57:27.341 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.339+0000 7f3707fff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e08abfe310 0x55e08abfe730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:27.341 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.339+0000 7f3706ffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e08ac03e70 0x55e08ac06300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:27.341 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.339+0000 7f3707fff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e08abfe310 0x55e08abfe730 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:42410/0 (socket says 172.21.15.99:42410) 2024-03-19T01:57:27.341 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.339+0000 7f3706ffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e08ac03e70 0x55e08ac06300 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:55922/0 (socket says 172.21.15.99:55922) 2024-03-19T01:57:27.341 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.339+0000 7f3707fff640 1 -- 172.21.15.99:0/1773868073 learned_addr learned my addr 172.21.15.99:0/1773868073 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:57:27.341 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.339+0000 7f37077fe640 1 --2- 172.21.15.99:0/1773868073 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e08abfed00 0x55e08ac03800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:27.342 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.339+0000 7f3704ff9640 1 --2- 172.21.15.99:0/1535111746 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f36f00a28a0 0x7f36f00a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:27.342 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.339+0000 7f3707fff640 1 -- 172.21.15.99:0/1773868073 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e08abfed00 msgr2=0x55e08ac03800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:57:27.342 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.339+0000 7f3707fff640 1 --2- 172.21.15.99:0/1773868073 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e08abfed00 0x55e08ac03800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:27.342 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.339+0000 7f3707fff640 1 -- 172.21.15.99:0/1773868073 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e08ac03e70 msgr2=0x55e08ac06300 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:57:27.342 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.339+0000 7f3707fff640 1 --2- 172.21.15.99:0/1773868073 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e08ac03e70 0x55e08ac06300 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:27.342 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.339+0000 7f3707fff640 1 -- 172.21.15.99:0/1773868073 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55e08ab00d20 con 0x55e08abfe310 2024-03-19T01:57:27.342 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.340+0000 7f3707fff640 1 --2- 172.21.15.99:0/1535111746 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f36f00a28a0 0x7f36f00a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:27.342 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.340+0000 7f3704ff9640 1 -- 172.21.15.99:0/1535111746 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f36f80ceef0 con 0x7f3708100710 2024-03-19T01:57:27.342 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.340+0000 7f3707fff640 1 --2- 172.21.15.99:0/1773868073 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e08abfe310 0x55e08abfe730 secure :-1 s=READY pgs=176 cs=0 l=1 rev1=1 crypto rx=0x7f3700009fd0 tx=0x7f370000e830 comp rx=0 tx=0).ready entity=mon.2 client_cookie=5f77a7865351a8e7 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:57:27.342 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.340+0000 7f36edffb640 1 -- 172.21.15.99:0/1773868073 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3700020070 con 0x55e08abfe310 2024-03-19T01:57:27.342 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.340+0000 7f36edffb640 1 -- 172.21.15.99:0/1773868073 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f370001ae20 con 0x55e08abfe310 2024-03-19T01:57:27.342 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.340+0000 7f36edffb640 1 -- 172.21.15.99:0/1773868073 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3700019d10 con 0x55e08abfe310 2024-03-19T01:57:27.343 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:57:27 smithi099 ceph-mon[62951]: pgmap v842: 97 pgs: 97 active+clean; 554 MiB data, 6.2 GiB used, 1.0 TiB / 1.0 TiB avail; 4.9 MiB/s rd, 6.7 MiB/s wr, 614 op/s 2024-03-19T01:57:27.346 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.341+0000 7f3710dc8740 1 -- 172.21.15.99:0/1773868073 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e08abfe310 msgr2=0x55e08abfe730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:57:27.346 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.341+0000 7f3707fff640 1 --2- 172.21.15.99:0/1535111746 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f36f00a28a0 0x7f36f00a4d60 secure :-1 s=READY pgs=346 cs=0 l=1 rev1=1 crypto rx=0x7f370001d000 tx=0x7f370001e000 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:57:27.346 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.341+0000 7f3710dc8740 1 --2- 172.21.15.99:0/1773868073 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e08abfe310 0x55e08abfe730 secure :-1 s=READY pgs=176 cs=0 l=1 rev1=1 crypto rx=0x7f3700009fd0 tx=0x7f370000e830 comp rx=0 tx=0).stop 2024-03-19T01:57:27.346 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.341+0000 7f3710dc8740 1 -- 172.21.15.99:0/1773868073 shutdown_connections 2024-03-19T01:57:27.346 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.341+0000 7f3710dc8740 1 --2- 172.21.15.99:0/1773868073 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e08ac03e70 0x55e08ac06300 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:27.347 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.341+0000 7f3710dc8740 1 --2- 172.21.15.99:0/1773868073 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e08abfed00 0x55e08ac03800 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:27.347 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.341+0000 7f3710dc8740 1 --2- 172.21.15.99:0/1773868073 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e08abfe310 0x55e08abfe730 unknown :-1 s=CLOSED pgs=176 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:27.347 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.341+0000 7f3710dc8740 1 -- 172.21.15.99:0/1773868073 >> 172.21.15.99:0/1773868073 conn(0x55e08abf9880 msgr2=0x55e08abf9c80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:57:27.347 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.341+0000 7f3710dc8740 1 -- 172.21.15.99:0/1773868073 shutdown_connections 2024-03-19T01:57:27.347 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.341+0000 7f3710dc8740 1 -- 172.21.15.99:0/1773868073 wait complete. 2024-03-19T01:57:27.347 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.341+0000 7f3710dc8740 1 -- start start 2024-03-19T01:57:27.347 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.342+0000 7f3710dc8740 1 Processor -- start 2024-03-19T01:57:27.347 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.342+0000 7f3710dc8740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e08abfe310 0x55e08ac08280 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:27.347 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.342+0000 7f3710dc8740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e08abfed00 0x55e08ac087c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:27.347 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.342+0000 7f3710dc8740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e08ac03e70 0x55e08ac0f800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:27.347 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.342+0000 7f3710dc8740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55e08ac06e20 con 0x55e08abfed00 2024-03-19T01:57:27.347 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.342+0000 7f3710dc8740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55e08ab1cdb0 con 0x55e08ac03e70 2024-03-19T01:57:27.347 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.342+0000 7f3710dc8740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55e08ac06fa0 con 0x55e08abfe310 2024-03-19T01:57:27.347 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.342+0000 7f3710dc8740 10 client.0 _ll_register_callbacks cb 0x55e08ab21e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:57:27.348 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.342+0000 7f3707fff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e08ac03e70 0x55e08ac0f800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:27.348 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.342+0000 7f3706ffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e08abfed00 0x55e08ac087c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:27.348 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.342+0000 7f3707fff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e08ac03e70 0x55e08ac0f800 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:42420/0 (socket says 172.21.15.99:42420) 2024-03-19T01:57:27.348 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.342+0000 7f3706ffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e08abfed00 0x55e08ac087c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:55926/0 (socket says 172.21.15.99:55926) 2024-03-19T01:57:27.348 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.342+0000 7f3707fff640 1 -- 172.21.15.99:0/637836558 learned_addr learned my addr 172.21.15.99:0/637836558 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:57:27.348 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.342+0000 7f37077fe640 1 --2- 172.21.15.99:0/637836558 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e08abfe310 0x55e08ac08280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:27.348 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.342+0000 7f3707fff640 1 -- 172.21.15.99:0/637836558 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e08abfe310 msgr2=0x55e08ac08280 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:57:27.348 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.342+0000 7f3707fff640 1 --2- 172.21.15.99:0/637836558 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e08abfe310 0x55e08ac08280 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:27.348 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.342+0000 7f3707fff640 1 -- 172.21.15.99:0/637836558 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e08abfed00 msgr2=0x55e08ac087c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:57:27.348 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.342+0000 7f3707fff640 1 --2- 172.21.15.99:0/637836558 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e08abfed00 0x55e08ac087c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:27.348 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.342+0000 7f3707fff640 1 -- 172.21.15.99:0/637836558 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55e08ab1bc60 con 0x55e08ac03e70 2024-03-19T01:57:27.348 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.342+0000 7f37077fe640 1 --2- 172.21.15.99:0/637836558 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e08abfe310 0x55e08ac08280 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-19T01:57:27.348 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.342+0000 7f3706ffd640 1 --2- 172.21.15.99:0/637836558 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e08abfed00 0x55e08ac087c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-19T01:57:27.348 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.343+0000 7f3707fff640 1 --2- 172.21.15.99:0/637836558 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e08ac03e70 0x55e08ac0f800 secure :-1 s=READY pgs=177 cs=0 l=1 rev1=1 crypto rx=0x7f37000049a0 tx=0x7f3700009320 comp rx=0 tx=0).ready entity=mon.2 client_cookie=2a3f6911a9190231 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:57:27.348 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.343+0000 7f36e6ffd640 10 client.0 ms_handle_connect on v2:172.21.15.112:3300/0 2024-03-19T01:57:27.349 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.343+0000 7f36e6ffd640 1 -- 172.21.15.99:0/637836558 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3700020070 con 0x55e08ac03e70 2024-03-19T01:57:27.349 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.343+0000 7f36e6ffd640 1 -- 172.21.15.99:0/637836558 <== mon.2 v2:172.21.15.112:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f370001a940 con 0x55e08ac03e70 2024-03-19T01:57:27.349 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.343+0000 7f3710dc8740 1 -- 172.21.15.99:0/637836558 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f370001ae20 con 0x55e08ac03e70 2024-03-19T01:57:27.349 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.343+0000 7f36e6ffd640 1 -- 172.21.15.99:0/637836558 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7f3700009bd0 con 0x55e08ac03e70 2024-03-19T01:57:27.349 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.343+0000 7f3710dc8740 10 client.26429 fetch_fsmap learned FSMap version 37 2024-03-19T01:57:27.349 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.343+0000 7f3710dc8740 1 -- 172.21.15.99:0/637836558 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x55e08a875210 con 0x55e08ac03e70 2024-03-19T01:57:27.349 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.343+0000 7f36e6ffd640 1 -- 172.21.15.99:0/637836558 <== mon.2 v2:172.21.15.112:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f370001ae20 con 0x55e08ac03e70 2024-03-19T01:57:27.349 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.343+0000 7f36e6ffd640 1 -- 172.21.15.99:0/637836558 <== mon.2 v2:172.21.15.112:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7f3700024b10 con 0x55e08ac03e70 2024-03-19T01:57:27.349 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.343+0000 7f3710dc8740 10 client.26429 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T01:57:27.349 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.343+0000 7f3710dc8740 10 client.26429 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:57:27.349 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.343+0000 7f3710dc8740 20 client.26429 populate_metadata read hostname 'smithi099' 2024-03-19T01:57:27.349 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.344+0000 7f3710dc8740 1 --2- 172.21.15.99:0/637836558 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e08ac16480 0x55e08ac18870 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:27.350 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.344+0000 7f3710dc8740 4 client.26429 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:57:27.350 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.344+0000 7f3710dc8740 1 -- 172.21.15.99:0/637836558 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55e08ab6e120 con 0x55e08ac16480 2024-03-19T01:57:27.350 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.344+0000 7f37077fe640 1 --2- 172.21.15.99:0/637836558 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e08ac16480 0x55e08ac18870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:27.350 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.344+0000 7f36e6ffd640 10 client.26429 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:57:27.350 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.344+0000 7f37077fe640 1 --2- 172.21.15.99:0/637836558 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e08ac16480 0x55e08ac18870 crc :-1 s=READY pgs=175 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=76fe08ba63e7b7b6 server_cookie=a153749ed6cf83fc in_seq=0 out_seq=0 2024-03-19T01:57:27.351 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.350+0000 7f36e6ffd640 1 -- 172.21.15.99:0/637836558 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x55e08ab6e120 con 0x55e08ac16480 2024-03-19T01:57:27.351 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.350+0000 7f36e6ffd640 10 client.26429 handle_command_reply: tid=0 2024-03-19T01:57:27.351 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.350+0000 7f3710dc8740 1 client.26429 shutdown 2024-03-19T01:57:27.351 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.351+0000 7f3710dc8740 1 -- 172.21.15.99:0/637836558 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e08ac03e70 msgr2=0x55e08ac0f800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:57:27.352 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.351+0000 7f3710dc8740 1 --2- 172.21.15.99:0/637836558 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e08ac03e70 0x55e08ac0f800 secure :-1 s=READY pgs=177 cs=0 l=1 rev1=1 crypto rx=0x7f37000049a0 tx=0x7f3700009320 comp rx=0 tx=0).stop 2024-03-19T01:57:27.352 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.351+0000 7f3710dc8740 1 -- 172.21.15.99:0/637836558 shutdown_connections 2024-03-19T01:57:27.352 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.351+0000 7f3710dc8740 1 --2- 172.21.15.99:0/637836558 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e08ac16480 0x55e08ac18870 crc :-1 s=READY pgs=175 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:27.352 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.351+0000 7f3710dc8740 1 --2- 172.21.15.99:0/637836558 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e08ac03e70 0x55e08ac0f800 unknown :-1 s=CLOSED pgs=177 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:27.352 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.351+0000 7f36e6ffd640 0 client.26429 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:57:27.352 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.351+0000 7f3710dc8740 1 --2- 172.21.15.99:0/637836558 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e08abfed00 0x55e08ac087c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:27.352 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.351+0000 7f3710dc8740 1 --2- 172.21.15.99:0/637836558 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e08abfe310 0x55e08ac08280 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:27.352 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.351+0000 7f3710dc8740 1 -- 172.21.15.99:0/637836558 >> 172.21.15.99:0/637836558 conn(0x55e08abf9880 msgr2=0x55e08abfaad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:57:27.353 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.351+0000 7f3710dc8740 1 -- 172.21.15.99:0/637836558 shutdown_connections 2024-03-19T01:57:27.353 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.351+0000 7f3710dc8740 1 -- 172.21.15.99:0/637836558 wait complete. 2024-03-19T01:57:27.353 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.351+0000 7f3710dc8740 20 client.26429 trim_cache size 0 max 16384 2024-03-19T01:57:27.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:57:27 smithi112 ceph-mon[66819]: pgmap v842: 97 pgs: 97 active+clean; 554 MiB data, 6.2 GiB used, 1.0 TiB / 1.0 TiB avail; 4.9 MiB/s rd, 6.7 MiB/s wr, 614 op/s 2024-03-19T01:57:27.370 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.366+0000 7f3710dc8740 1 Processor -- start 2024-03-19T01:57:27.370 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.366+0000 7f3710dc8740 1 -- start start 2024-03-19T01:57:27.370 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.366+0000 7f3710dc8740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e08abfe310 0x55e08abf6930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:27.371 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.366+0000 7f3710dc8740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e08abfed00 0x55e08ac0f7c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:27.371 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.367+0000 7f3710dc8740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e08ac03e70 0x55e08abff560 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:27.371 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.367+0000 7f3710dc8740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f36f4010e70 con 0x55e08abfed00 2024-03-19T01:57:27.371 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.367+0000 7f3710dc8740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f36f400e740 con 0x55e08abfe310 2024-03-19T01:57:27.371 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.367+0000 7f3710dc8740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f36f400c590 con 0x55e08ac03e70 2024-03-19T01:57:27.371 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.367+0000 7f3706ffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e08abfed00 0x55e08ac0f7c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:27.371 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.367+0000 7f37077fe640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e08abfe310 0x55e08abf6930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:27.371 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.367+0000 7f3706ffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e08abfed00 0x55e08ac0f7c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:55928/0 (socket says 172.21.15.99:55928) 2024-03-19T01:57:27.371 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.367+0000 7f3706ffd640 1 -- 172.21.15.99:0/1296441158 learned_addr learned my addr 172.21.15.99:0/1296441158 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:57:27.371 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.367+0000 7f3707fff640 1 --2- 172.21.15.99:0/1296441158 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e08ac03e70 0x55e08abff560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:27.371 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.367+0000 7f3706ffd640 1 -- 172.21.15.99:0/1296441158 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e08ac03e70 msgr2=0x55e08abff560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:57:27.371 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.367+0000 7f3706ffd640 1 --2- 172.21.15.99:0/1296441158 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e08ac03e70 0x55e08abff560 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:27.371 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.367+0000 7f3706ffd640 1 -- 172.21.15.99:0/1296441158 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e08abfe310 msgr2=0x55e08abf6930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:57:27.372 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.367+0000 7f3706ffd640 1 --2- 172.21.15.99:0/1296441158 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e08abfe310 0x55e08abf6930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:27.372 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.367+0000 7f3706ffd640 1 -- 172.21.15.99:0/1296441158 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55e08a941620 con 0x55e08abfed00 2024-03-19T01:57:27.372 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.367+0000 7f37077fe640 1 --2- 172.21.15.99:0/1296441158 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e08abfe310 0x55e08abf6930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:57:27.372 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.367+0000 7f3706ffd640 1 --2- 172.21.15.99:0/1296441158 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e08abfed00 0x55e08ac0f7c0 secure :-1 s=READY pgs=663 cs=0 l=1 rev1=1 crypto rx=0x7f36fc009900 tx=0x7f36fc009670 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3b9f43c13bb65ae4 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:57:27.372 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.367+0000 7f36ed7fa640 1 -- 172.21.15.99:0/1296441158 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f36fc02cd00 con 0x55e08abfed00 2024-03-19T01:57:27.372 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.367+0000 7f36ed7fa640 1 -- 172.21.15.99:0/1296441158 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f36fc0046c0 con 0x55e08abfed00 2024-03-19T01:57:27.372 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.367+0000 7f36ed7fa640 1 -- 172.21.15.99:0/1296441158 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f36fc0053c0 con 0x55e08abfed00 2024-03-19T01:57:27.372 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.368+0000 7f3710dc8740 1 -- 172.21.15.99:0/1296441158 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e08abfed00 msgr2=0x55e08ac0f7c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:57:27.372 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.368+0000 7f3710dc8740 1 --2- 172.21.15.99:0/1296441158 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e08abfed00 0x55e08ac0f7c0 secure :-1 s=READY pgs=663 cs=0 l=1 rev1=1 crypto rx=0x7f36fc009900 tx=0x7f36fc009670 comp rx=0 tx=0).stop 2024-03-19T01:57:27.372 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.368+0000 7f3710dc8740 1 -- 172.21.15.99:0/1296441158 shutdown_connections 2024-03-19T01:57:27.372 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.368+0000 7f3710dc8740 1 --2- 172.21.15.99:0/1296441158 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e08ac03e70 0x55e08abff560 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:27.372 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.368+0000 7f3710dc8740 1 --2- 172.21.15.99:0/1296441158 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e08abfed00 0x55e08ac0f7c0 unknown :-1 s=CLOSED pgs=663 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:27.372 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.368+0000 7f3710dc8740 1 --2- 172.21.15.99:0/1296441158 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e08abfe310 0x55e08abf6930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:27.372 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.368+0000 7f3710dc8740 1 -- 172.21.15.99:0/1296441158 >> 172.21.15.99:0/1296441158 conn(0x55e08abf9880 msgr2=0x55e08abf6220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:57:27.373 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.368+0000 7f3710dc8740 1 -- 172.21.15.99:0/1296441158 shutdown_connections 2024-03-19T01:57:27.373 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.368+0000 7f3710dc8740 1 -- 172.21.15.99:0/1296441158 wait complete. 2024-03-19T01:57:27.373 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.369+0000 7f3710dc8740 1 -- start start 2024-03-19T01:57:27.373 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.369+0000 7f3710dc8740 1 Processor -- start 2024-03-19T01:57:27.373 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.369+0000 7f3710dc8740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e08abfe310 0x55e08ac0d230 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:27.373 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.369+0000 7f3710dc8740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e08abfed00 0x55e08ac0d770 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:27.373 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.369+0000 7f3710dc8740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e08ac03e70 0x55e08ac0e780 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:27.373 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.369+0000 7f3710dc8740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f36f400cc00 con 0x55e08ac03e70 2024-03-19T01:57:27.373 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.369+0000 7f3710dc8740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f36f400e070 con 0x55e08abfed00 2024-03-19T01:57:27.373 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.369+0000 7f3710dc8740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f36f400d2d0 con 0x55e08abfe310 2024-03-19T01:57:27.373 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.369+0000 7f3710dc8740 10 client.0 _ll_register_callbacks cb 0x7f370001e070 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:57:27.373 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.369+0000 7f3706ffd640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e08abfed00 0x55e08ac0d770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:27.373 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.369+0000 7f37077fe640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e08abfe310 0x55e08ac0d230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:27.373 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.369+0000 7f3706ffd640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e08abfed00 0x55e08ac0d770 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:42428/0 (socket says 172.21.15.99:42428) 2024-03-19T01:57:27.373 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.369+0000 7f3706ffd640 1 -- 172.21.15.99:0/733234148 learned_addr learned my addr 172.21.15.99:0/733234148 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:57:27.374 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.369+0000 7f3707fff640 1 --2- 172.21.15.99:0/733234148 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e08ac03e70 0x55e08ac0e780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:27.374 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.369+0000 7f3706ffd640 1 -- 172.21.15.99:0/733234148 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e08abfe310 msgr2=0x55e08ac0d230 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:57:27.374 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.369+0000 7f3706ffd640 1 --2- 172.21.15.99:0/733234148 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e08abfe310 0x55e08ac0d230 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:27.374 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.369+0000 7f3706ffd640 1 -- 172.21.15.99:0/733234148 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e08ac03e70 msgr2=0x55e08ac0e780 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:57:27.374 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.369+0000 7f3706ffd640 1 --2- 172.21.15.99:0/733234148 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e08ac03e70 0x55e08ac0e780 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:27.374 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.369+0000 7f3706ffd640 1 -- 172.21.15.99:0/733234148 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55e08ac0ed70 con 0x55e08abfed00 2024-03-19T01:57:27.374 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.370+0000 7f3706ffd640 1 --2- 172.21.15.99:0/733234148 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e08abfed00 0x55e08ac0d770 secure :-1 s=READY pgs=178 cs=0 l=1 rev1=1 crypto rx=0x7f36fc02c170 tx=0x7f36fc004aa0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=9dbd99211e2cd588 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:57:27.374 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.370+0000 7f36e6ffd640 10 client.0 ms_handle_connect on v2:172.21.15.112:3300/0 2024-03-19T01:57:27.374 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.370+0000 7f36e6ffd640 1 -- 172.21.15.99:0/733234148 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f36fc005510 con 0x55e08abfed00 2024-03-19T01:57:27.374 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.370+0000 7f3710dc8740 1 -- 172.21.15.99:0/733234148 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f36fc0046c0 con 0x55e08abfed00 2024-03-19T01:57:27.374 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.370+0000 7f36e6ffd640 1 -- 172.21.15.99:0/733234148 <== mon.2 v2:172.21.15.112:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f36fc047480 con 0x55e08abfed00 2024-03-19T01:57:27.374 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.370+0000 7f36e6ffd640 1 -- 172.21.15.99:0/733234148 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7f36fc047680 con 0x55e08abfed00 2024-03-19T01:57:27.374 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.370+0000 7f3710dc8740 10 client.26435 fetch_fsmap learned FSMap version 37 2024-03-19T01:57:27.374 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.370+0000 7f3710dc8740 1 -- 172.21.15.99:0/733234148 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x55e08ac7c800 con 0x55e08abfed00 2024-03-19T01:57:27.375 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.370+0000 7f36e6ffd640 1 -- 172.21.15.99:0/733234148 <== mon.2 v2:172.21.15.112:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f36fc0046c0 con 0x55e08abfed00 2024-03-19T01:57:27.375 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.370+0000 7f36e6ffd640 1 -- 172.21.15.99:0/733234148 <== mon.2 v2:172.21.15.112:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7f36fc0347d0 con 0x55e08abfed00 2024-03-19T01:57:27.375 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.370+0000 7f3710dc8740 10 client.26435 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T01:57:27.375 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.370+0000 7f3710dc8740 10 client.26435 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:57:27.375 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.371+0000 7f3710dc8740 20 client.26435 populate_metadata read hostname 'smithi099' 2024-03-19T01:57:27.375 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.371+0000 7f3710dc8740 1 --2- 172.21.15.99:0/733234148 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e08ac7d160 0x55e08ac7f580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:27.375 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.371+0000 7f3710dc8740 4 client.26435 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:57:27.375 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.371+0000 7f3710dc8740 1 -- 172.21.15.99:0/733234148 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55e08ab68470 con 0x55e08ac7d160 2024-03-19T01:57:27.375 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.371+0000 7f37077fe640 1 --2- 172.21.15.99:0/733234148 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e08ac7d160 0x55e08ac7f580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:27.376 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.372+0000 7f36e6ffd640 10 client.26435 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:57:27.376 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.372+0000 7f37077fe640 1 --2- 172.21.15.99:0/733234148 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e08ac7d160 0x55e08ac7f580 crc :-1 s=READY pgs=176 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=a252bc85c26582c8 server_cookie=1c2aa262f56bf829 in_seq=0 out_seq=0 2024-03-19T01:57:27.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:57:27 smithi115 ceph-mon[67332]: pgmap v842: 97 pgs: 97 active+clean; 554 MiB data, 6.2 GiB used, 1.0 TiB / 1.0 TiB avail; 4.9 MiB/s rd, 6.7 MiB/s wr, 614 op/s 2024-03-19T01:57:27.795 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.794+0000 7f36e6ffd640 1 -- 172.21.15.99:0/733234148 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x55e08ab68470 con 0x55e08ac7d160 2024-03-19T01:57:27.798 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.794+0000 7f36e6ffd640 10 client.26435 handle_command_reply: tid=0 2024-03-19T01:57:27.798 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.794+0000 7f3710dc8740 1 client.26435 shutdown 2024-03-19T01:57:27.798 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.795+0000 7f3710dc8740 1 -- 172.21.15.99:0/733234148 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e08abfed00 msgr2=0x55e08ac0d770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:57:27.798 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.795+0000 7f3710dc8740 1 --2- 172.21.15.99:0/733234148 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e08abfed00 0x55e08ac0d770 secure :-1 s=READY pgs=178 cs=0 l=1 rev1=1 crypto rx=0x7f36fc02c170 tx=0x7f36fc004aa0 comp rx=0 tx=0).stop 2024-03-19T01:57:27.799 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.795+0000 7f3710dc8740 1 -- 172.21.15.99:0/733234148 shutdown_connections 2024-03-19T01:57:27.799 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.795+0000 7f3710dc8740 1 --2- 172.21.15.99:0/733234148 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e08ac7d160 0x55e08ac7f580 crc :-1 s=READY pgs=176 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:27.799 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.795+0000 7f3710dc8740 1 --2- 172.21.15.99:0/733234148 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e08abfed00 0x55e08ac0d770 unknown :-1 s=CLOSED pgs=178 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:27.799 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.795+0000 7f36e6ffd640 0 client.26435 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:57:27.799 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.795+0000 7f3710dc8740 1 --2- 172.21.15.99:0/733234148 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e08ac03e70 0x55e08ac0e780 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:27.799 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.795+0000 7f3710dc8740 1 --2- 172.21.15.99:0/733234148 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e08abfe310 0x55e08ac0d230 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:27.799 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.795+0000 7f3710dc8740 1 -- 172.21.15.99:0/733234148 >> 172.21.15.99:0/733234148 conn(0x55e08abf9880 msgr2=0x55e08abfcae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:57:27.800 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.795+0000 7f3710dc8740 1 -- 172.21.15.99:0/733234148 shutdown_connections 2024-03-19T01:57:27.800 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.795+0000 7f3710dc8740 1 -- 172.21.15.99:0/733234148 wait complete. 2024-03-19T01:57:27.800 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.795+0000 7f3710dc8740 20 client.26435 trim_cache size 0 max 16384 2024-03-19T01:57:27.800 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:57:27.800 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:57:27.800 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:57:27.800 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:57:27.801 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.796+0000 7f370e1df640 1 -- 172.21.15.99:0/1535111746 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f36f00a28a0 msgr2=0x7f36f00a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:57:27.802 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.796+0000 7f370e1df640 1 --2- 172.21.15.99:0/1535111746 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f36f00a28a0 0x7f36f00a4d60 secure :-1 s=READY pgs=346 cs=0 l=1 rev1=1 crypto rx=0x7f370001d000 tx=0x7f370001e000 comp rx=0 tx=0).stop 2024-03-19T01:57:27.802 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.796+0000 7f370e1df640 1 -- 172.21.15.99:0/1535111746 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3708100710 msgr2=0x7f3708196770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:57:27.802 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.796+0000 7f370e1df640 1 --2- 172.21.15.99:0/1535111746 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3708100710 0x7f3708196770 secure :-1 s=READY pgs=662 cs=0 l=1 rev1=1 crypto rx=0x7f36f800e9c0 tx=0x7f36f800ee80 comp rx=0 tx=0).stop 2024-03-19T01:57:27.802 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.796+0000 7f370e1df640 1 -- 172.21.15.99:0/1535111746 shutdown_connections 2024-03-19T01:57:27.802 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.796+0000 7f370e1df640 1 --2- 172.21.15.99:0/1535111746 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f36f00a28a0 0x7f36f00a4d60 unknown :-1 s=CLOSED pgs=346 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:27.802 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.796+0000 7f370e1df640 1 --2- 172.21.15.99:0/1535111746 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3708101900 0x7f3708196cb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:27.802 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.796+0000 7f370e1df640 1 --2- 172.21.15.99:0/1535111746 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f37081022c0 0x7f370819dd20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:27.802 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.796+0000 7f370e1df640 1 --2- 172.21.15.99:0/1535111746 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3708100710 0x7f3708196770 unknown :-1 s=CLOSED pgs=662 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:27.802 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.796+0000 7f370e1df640 1 -- 172.21.15.99:0/1535111746 >> 172.21.15.99:0/1535111746 conn(0x7f37080fbc80 msgr2=0x7f3708104b30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:57:27.802 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.796+0000 7f370e1df640 1 -- 172.21.15.99:0/1535111746 shutdown_connections 2024-03-19T01:57:27.802 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:27.796+0000 7f370e1df640 1 -- 172.21.15.99:0/1535111746 wait complete. 2024-03-19T01:57:28.106 INFO:tasks.workunit.client.0.smithi099.stdout: 10 105093 23.32 MB/sec execute 117 sec latency 292.958 ms 2024-03-19T01:57:29.106 INFO:tasks.workunit.client.0.smithi099.stdout: 10 105362 23.22 MB/sec execute 118 sec latency 150.748 ms 2024-03-19T01:57:29.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:57:29 smithi099 ceph-mon[62951]: pgmap v843: 97 pgs: 97 active+clean; 555 MiB data, 6.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 5.7 MiB/s wr, 484 op/s 2024-03-19T01:57:29.357 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:57:29 smithi115 ceph-mon[67332]: pgmap v843: 97 pgs: 97 active+clean; 555 MiB data, 6.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 5.7 MiB/s wr, 484 op/s 2024-03-19T01:57:29.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:57:29 smithi112 ceph-mon[66819]: pgmap v843: 97 pgs: 97 active+clean; 555 MiB data, 6.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 5.7 MiB/s wr, 484 op/s 2024-03-19T01:57:30.106 INFO:tasks.workunit.client.0.smithi099.stdout: 10 105743 23.19 MB/sec execute 119 sec latency 83.858 ms 2024-03-19T01:57:31.106 INFO:tasks.workunit.client.0.smithi099.stdout: 10 106119 23.14 MB/sec execute 120 sec latency 68.835 ms 2024-03-19T01:57:31.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:57:31 smithi099 ceph-mon[62951]: pgmap v844: 97 pgs: 97 active+clean; 551 MiB data, 6.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 5.8 MiB/s wr, 467 op/s 2024-03-19T01:57:31.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:57:31 smithi112 ceph-mon[66819]: pgmap v844: 97 pgs: 97 active+clean; 551 MiB data, 6.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 5.8 MiB/s wr, 467 op/s 2024-03-19T01:57:31.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:57:31 smithi115 ceph-mon[67332]: pgmap v844: 97 pgs: 97 active+clean; 551 MiB data, 6.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 5.8 MiB/s wr, 467 op/s 2024-03-19T01:57:32.106 INFO:tasks.workunit.client.0.smithi099.stdout: 10 106639 23.17 MB/sec execute 121 sec latency 98.069 ms 2024-03-19T01:57:32.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:57:32 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:57:32.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:57:32 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:57:32.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:57:32 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:57:33.106 INFO:tasks.workunit.client.0.smithi099.stdout: 10 107164 23.20 MB/sec execute 122 sec latency 56.336 ms 2024-03-19T01:57:33.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:57:33 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:57:33.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:57:33 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:57:33.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:57:33 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:57:33.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:57:33 smithi112 ceph-mon[66819]: pgmap v845: 97 pgs: 97 active+clean; 548 MiB data, 6.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.8 MiB/s rd, 6.9 MiB/s wr, 622 op/s 2024-03-19T01:57:33.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:57:33 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:57:33.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:57:33 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:57:33.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:57:33 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:57:33.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:57:33 smithi115 ceph-mon[67332]: pgmap v845: 97 pgs: 97 active+clean; 548 MiB data, 6.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.8 MiB/s rd, 6.9 MiB/s wr, 622 op/s 2024-03-19T01:57:33.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:57:33 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:57:33.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:57:33 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:57:33.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:57:33 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:57:33.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:57:33 smithi099 ceph-mon[62951]: pgmap v845: 97 pgs: 97 active+clean; 548 MiB data, 6.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.8 MiB/s rd, 6.9 MiB/s wr, 622 op/s 2024-03-19T01:57:34.106 INFO:tasks.workunit.client.0.smithi099.stdout: 10 107484 23.16 MB/sec execute 123 sec latency 259.148 ms 2024-03-19T01:57:35.106 INFO:tasks.workunit.client.0.smithi099.stdout: 10 107933 23.13 MB/sec execute 124 sec latency 119.084 ms 2024-03-19T01:57:35.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:57:35 smithi099 ceph-mon[62951]: pgmap v846: 97 pgs: 97 active+clean; 553 MiB data, 6.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 6.5 MiB/s wr, 561 op/s 2024-03-19T01:57:35.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:57:35 smithi112 ceph-mon[66819]: pgmap v846: 97 pgs: 97 active+clean; 553 MiB data, 6.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 6.5 MiB/s wr, 561 op/s 2024-03-19T01:57:35.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:57:35 smithi115 ceph-mon[67332]: pgmap v846: 97 pgs: 97 active+clean; 553 MiB data, 6.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 6.5 MiB/s wr, 561 op/s 2024-03-19T01:57:36.106 INFO:tasks.workunit.client.0.smithi099.stdout: 10 108429 23.22 MB/sec execute 125 sec latency 55.670 ms 2024-03-19T01:57:37.106 INFO:tasks.workunit.client.0.smithi099.stdout: 10 108871 23.21 MB/sec execute 126 sec latency 189.256 ms 2024-03-19T01:57:37.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:57:37 smithi099 ceph-mon[62951]: pgmap v847: 97 pgs: 97 active+clean; 556 MiB data, 6.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.6 MiB/s rd, 7.6 MiB/s wr, 621 op/s 2024-03-19T01:57:37.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:57:37 smithi112 ceph-mon[66819]: pgmap v847: 97 pgs: 97 active+clean; 556 MiB data, 6.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.6 MiB/s rd, 7.6 MiB/s wr, 621 op/s 2024-03-19T01:57:37.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:57:37 smithi115 ceph-mon[67332]: pgmap v847: 97 pgs: 97 active+clean; 556 MiB data, 6.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.6 MiB/s rd, 7.6 MiB/s wr, 621 op/s 2024-03-19T01:57:38.106 INFO:tasks.workunit.client.0.smithi099.stdout: 10 109381 23.21 MB/sec execute 127 sec latency 79.624 ms 2024-03-19T01:57:39.106 INFO:tasks.workunit.client.0.smithi099.stdout: 10 109789 23.27 MB/sec execute 128 sec latency 236.332 ms 2024-03-19T01:57:39.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:57:39 smithi099 ceph-mon[62951]: pgmap v848: 97 pgs: 97 active+clean; 559 MiB data, 6.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 6.8 MiB/s wr, 565 op/s 2024-03-19T01:57:39.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:57:39 smithi112 ceph-mon[66819]: pgmap v848: 97 pgs: 97 active+clean; 559 MiB data, 6.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 6.8 MiB/s wr, 565 op/s 2024-03-19T01:57:39.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:57:39 smithi115 ceph-mon[67332]: pgmap v848: 97 pgs: 97 active+clean; 559 MiB data, 6.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 6.8 MiB/s wr, 565 op/s 2024-03-19T01:57:40.106 INFO:tasks.workunit.client.0.smithi099.stdout: 10 110181 23.29 MB/sec execute 129 sec latency 90.633 ms 2024-03-19T01:57:40.887 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.885+0000 7f10b65ff640 1 -- 172.21.15.99:0/2102466224 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f10b0104c70 msgr2=0x7f10b01050f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:57:40.887 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.885+0000 7f10b65ff640 1 --2- 172.21.15.99:0/2102466224 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f10b0104c70 0x7f10b01050f0 secure :-1 s=READY pgs=664 cs=0 l=1 rev1=1 crypto rx=0x7f10a4009900 tx=0x7f10a402bc30 comp rx=0 tx=0).stop 2024-03-19T01:57:40.887 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.885+0000 7f10b65ff640 1 -- 172.21.15.99:0/2102466224 shutdown_connections 2024-03-19T01:57:40.887 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.885+0000 7f10b65ff640 1 --2- 172.21.15.99:0/2102466224 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f10b0105630 0x7f10b010bec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:40.887 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.885+0000 7f10b65ff640 1 --2- 172.21.15.99:0/2102466224 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f10b0104c70 0x7f10b01050f0 unknown :-1 s=CLOSED pgs=664 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:40.887 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.885+0000 7f10b65ff640 1 --2- 172.21.15.99:0/2102466224 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f10b0103a80 0x7f10b0103ea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:40.887 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.885+0000 7f10b65ff640 1 -- 172.21.15.99:0/2102466224 >> 172.21.15.99:0/2102466224 conn(0x7f10b00ff010 msgr2=0x7f10b0101430 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:57:40.887 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.885+0000 7f10b65ff640 1 -- 172.21.15.99:0/2102466224 shutdown_connections 2024-03-19T01:57:40.887 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.886+0000 7f10b65ff640 1 -- 172.21.15.99:0/2102466224 wait complete. 2024-03-19T01:57:40.887 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.886+0000 7f10b65ff640 1 Processor -- start 2024-03-19T01:57:40.887 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.886+0000 7f10b65ff640 1 -- start start 2024-03-19T01:57:40.889 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.886+0000 7f10b65ff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f10b0103a80 0x7f10b0199c80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:40.889 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.886+0000 7f10b65ff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f10b0104c70 0x7f10b019a1c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:40.889 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.886+0000 7f10b65ff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f10b0105630 0x7f10b01a1220 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:40.889 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.886+0000 7f10b65ff640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f10b010f700 con 0x7f10b0104c70 2024-03-19T01:57:40.889 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.886+0000 7f10b65ff640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f10b010f580 con 0x7f10b0105630 2024-03-19T01:57:40.890 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.886+0000 7f10b65ff640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f10b010f880 con 0x7f10b0103a80 2024-03-19T01:57:40.890 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.887+0000 7f10af7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f10b0104c70 0x7f10b019a1c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:40.890 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.887+0000 7f10af7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f10b0104c70 0x7f10b019a1c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:55722/0 (socket says 172.21.15.99:55722) 2024-03-19T01:57:40.890 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.887+0000 7f10af7fe640 1 -- 172.21.15.99:0/548929188 learned_addr learned my addr 172.21.15.99:0/548929188 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:57:40.890 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.887+0000 7f10affff640 1 --2- 172.21.15.99:0/548929188 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f10b0103a80 0x7f10b0199c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:40.890 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.887+0000 7f10b4b9c640 1 --2- 172.21.15.99:0/548929188 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f10b0105630 0x7f10b01a1220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:40.890 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.887+0000 7f10af7fe640 1 -- 172.21.15.99:0/548929188 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f10b0103a80 msgr2=0x7f10b0199c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:57:40.890 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.887+0000 7f10af7fe640 1 --2- 172.21.15.99:0/548929188 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f10b0103a80 0x7f10b0199c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:40.890 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.887+0000 7f10af7fe640 1 -- 172.21.15.99:0/548929188 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f10b0105630 msgr2=0x7f10b01a1220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:57:40.890 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.887+0000 7f10af7fe640 1 --2- 172.21.15.99:0/548929188 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f10b0105630 0x7f10b01a1220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:40.890 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.887+0000 7f10af7fe640 1 -- 172.21.15.99:0/548929188 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f10b01a18c0 con 0x7f10b0104c70 2024-03-19T01:57:40.890 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.887+0000 7f10af7fe640 1 --2- 172.21.15.99:0/548929188 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f10b0104c70 0x7f10b019a1c0 secure :-1 s=READY pgs=665 cs=0 l=1 rev1=1 crypto rx=0x7f10a4009230 tx=0x7f10a402c650 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:57:40.890 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.887+0000 7f10ad7fa640 1 -- 172.21.15.99:0/548929188 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f10a4041070 con 0x7f10b0104c70 2024-03-19T01:57:40.890 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.887+0000 7f10b65ff640 1 -- 172.21.15.99:0/548929188 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f10b01a1b50 con 0x7f10b0104c70 2024-03-19T01:57:40.891 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.887+0000 7f10ad7fa640 1 -- 172.21.15.99:0/548929188 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f10a4004450 con 0x7f10b0104c70 2024-03-19T01:57:40.891 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.887+0000 7f10ad7fa640 1 -- 172.21.15.99:0/548929188 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f10a4003e20 con 0x7f10b0104c70 2024-03-19T01:57:40.891 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.887+0000 7f10b65ff640 1 -- 172.21.15.99:0/548929188 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f10b01a2040 con 0x7f10b0104c70 2024-03-19T01:57:40.898 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.889+0000 7f10ad7fa640 1 -- 172.21.15.99:0/548929188 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f10a4044070 con 0x7f10b0104c70 2024-03-19T01:57:40.898 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.890+0000 7f10b91e2740 1 Processor -- start 2024-03-19T01:57:40.898 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.890+0000 7f10b91e2740 1 -- start start 2024-03-19T01:57:40.898 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.890+0000 7f10b91e2740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561008759100 0x561008759520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:40.898 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.890+0000 7f10b91e2740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561008759af0 0x56100875e5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:40.898 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.890+0000 7f10b91e2740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56100875ec60 0x5610087610f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:40.898 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.890+0000 7f10b91e2740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x5610086bdbe0 con 0x561008759100 2024-03-19T01:57:40.898 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.890+0000 7f10b91e2740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x561008666960 con 0x561008759af0 2024-03-19T01:57:40.899 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.890+0000 7f10b91e2740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x5610086b9c80 con 0x56100875ec60 2024-03-19T01:57:40.899 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.890+0000 7f10b4b9c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561008759100 0x561008759520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:40.899 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.890+0000 7f10b4b9c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561008759100 0x561008759520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:55728/0 (socket says 172.21.15.99:55728) 2024-03-19T01:57:40.899 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.890+0000 7f10b4b9c640 1 -- 172.21.15.99:0/2247673345 learned_addr learned my addr 172.21.15.99:0/2247673345 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:57:40.899 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.890+0000 7f10af7fe640 1 --2- 172.21.15.99:0/2247673345 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56100875ec60 0x5610087610f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:40.899 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.890+0000 7f10affff640 1 --2- 172.21.15.99:0/2247673345 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561008759af0 0x56100875e5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:40.899 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.890+0000 7f10ad7fa640 1 --2- 172.21.15.99:0/548929188 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f10980a28a0 0x7f10980a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:40.899 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.890+0000 7f10b4b9c640 1 -- 172.21.15.99:0/2247673345 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56100875ec60 msgr2=0x5610087610f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:57:40.899 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.890+0000 7f10b4b9c640 1 --2- 172.21.15.99:0/2247673345 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56100875ec60 0x5610087610f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:40.899 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.890+0000 7f10b4b9c640 1 -- 172.21.15.99:0/2247673345 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561008759af0 msgr2=0x56100875e5f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:57:40.899 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.890+0000 7f10b4b9c640 1 --2- 172.21.15.99:0/2247673345 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561008759af0 0x56100875e5f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:40.899 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.890+0000 7f10b4b9c640 1 -- 172.21.15.99:0/2247673345 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x56100865dd20 con 0x561008759100 2024-03-19T01:57:40.899 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.890+0000 7f10affff640 1 --2- 172.21.15.99:0/2247673345 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561008759af0 0x56100875e5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:57:40.899 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.890+0000 7f10ad7fa640 1 -- 172.21.15.99:0/548929188 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f10a40f0680 con 0x7f10b0104c70 2024-03-19T01:57:40.900 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.890+0000 7f10b4b9c640 1 --2- 172.21.15.99:0/548929188 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f10980a28a0 0x7f10980a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:40.900 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.891+0000 7f10b4b9c640 1 --2- 172.21.15.99:0/2247673345 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561008759100 0x561008759520 secure :-1 s=READY pgs=666 cs=0 l=1 rev1=1 crypto rx=0x7f10a8011590 tx=0x7f10a8011a50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1bcf07e6fae3dbf3 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:57:40.900 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.891+0000 7f10967fc640 1 -- 172.21.15.99:0/2247673345 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f10a8017050 con 0x561008759100 2024-03-19T01:57:40.900 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.891+0000 7f10967fc640 1 -- 172.21.15.99:0/2247673345 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f10a800fd20 con 0x561008759100 2024-03-19T01:57:40.900 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.891+0000 7f10967fc640 1 -- 172.21.15.99:0/2247673345 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f10a801ba70 con 0x561008759100 2024-03-19T01:57:40.900 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.891+0000 7f10b4b9c640 1 --2- 172.21.15.99:0/548929188 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f10980a28a0 0x7f10980a4d60 secure :-1 s=READY pgs=347 cs=0 l=1 rev1=1 crypto rx=0x7f10a8022000 tx=0x7f10a8023000 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:57:40.900 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.891+0000 7f10b91e2740 1 -- 172.21.15.99:0/2247673345 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561008759100 msgr2=0x561008759520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:57:40.900 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.891+0000 7f10b91e2740 1 --2- 172.21.15.99:0/2247673345 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561008759100 0x561008759520 secure :-1 s=READY pgs=666 cs=0 l=1 rev1=1 crypto rx=0x7f10a8011590 tx=0x7f10a8011a50 comp rx=0 tx=0).stop 2024-03-19T01:57:40.900 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.892+0000 7f10b91e2740 1 -- 172.21.15.99:0/2247673345 shutdown_connections 2024-03-19T01:57:40.900 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.892+0000 7f10b91e2740 1 --2- 172.21.15.99:0/2247673345 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561008759af0 0x56100875e5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:40.900 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.892+0000 7f10b91e2740 1 --2- 172.21.15.99:0/2247673345 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56100875ec60 0x5610087610f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:40.900 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.892+0000 7f10b91e2740 1 --2- 172.21.15.99:0/2247673345 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561008759100 0x561008759520 unknown :-1 s=CLOSED pgs=666 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:40.900 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.892+0000 7f10b91e2740 1 -- 172.21.15.99:0/2247673345 >> 172.21.15.99:0/2247673345 conn(0x561008754670 msgr2=0x561008754a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:57:40.900 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.892+0000 7f10b91e2740 1 -- 172.21.15.99:0/2247673345 shutdown_connections 2024-03-19T01:57:40.900 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.892+0000 7f10b91e2740 1 -- 172.21.15.99:0/2247673345 wait complete. 2024-03-19T01:57:40.901 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.892+0000 7f10b91e2740 1 -- start start 2024-03-19T01:57:40.901 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.892+0000 7f10b91e2740 1 Processor -- start 2024-03-19T01:57:40.901 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.892+0000 7f10b91e2740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561008759100 0x561008763110 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:40.901 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.892+0000 7f10b91e2740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561008759af0 0x561008763650 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:40.901 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.892+0000 7f10b91e2740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56100875ec60 0x56100876a690 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:40.901 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.892+0000 7f10b91e2740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x561008761c10 con 0x56100875ec60 2024-03-19T01:57:40.901 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.892+0000 7f10b91e2740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x561008679db0 con 0x561008759100 2024-03-19T01:57:40.901 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.892+0000 7f10b91e2740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x561008761d90 con 0x561008759af0 2024-03-19T01:57:40.901 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.892+0000 7f10b91e2740 10 client.0 _ll_register_callbacks cb 0x56100867ee70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:57:40.901 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.892+0000 7f10b4b9c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56100875ec60 0x56100876a690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:40.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.892+0000 7f10b4b9c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56100875ec60 0x56100876a690 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:55730/0 (socket says 172.21.15.99:55730) 2024-03-19T01:57:40.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.892+0000 7f10b4b9c640 1 -- 172.21.15.99:0/1034786163 learned_addr learned my addr 172.21.15.99:0/1034786163 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:57:40.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.892+0000 7f10af7fe640 1 --2- 172.21.15.99:0/1034786163 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561008759af0 0x561008763650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:40.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.892+0000 7f10affff640 1 --2- 172.21.15.99:0/1034786163 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561008759100 0x561008763110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:40.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.893+0000 7f10b4b9c640 1 -- 172.21.15.99:0/1034786163 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561008759af0 msgr2=0x561008763650 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:57:40.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.893+0000 7f10b4b9c640 1 --2- 172.21.15.99:0/1034786163 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561008759af0 0x561008763650 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:40.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.893+0000 7f10b4b9c640 1 -- 172.21.15.99:0/1034786163 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561008759100 msgr2=0x561008763110 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:57:40.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.893+0000 7f10b4b9c640 1 --2- 172.21.15.99:0/1034786163 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561008759100 0x561008763110 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:40.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.893+0000 7f10b4b9c640 1 -- 172.21.15.99:0/1034786163 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x561008678c60 con 0x56100875ec60 2024-03-19T01:57:40.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.893+0000 7f10affff640 1 --2- 172.21.15.99:0/1034786163 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561008759100 0x561008763110 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:57:40.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.893+0000 7f10b4b9c640 1 --2- 172.21.15.99:0/1034786163 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56100875ec60 0x56100876a690 secure :-1 s=READY pgs=667 cs=0 l=1 rev1=1 crypto rx=0x7f10a8005c10 tx=0x7f10a800fef0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=cdfd16cc8dd6a065 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:57:40.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.893+0000 7f108f7fe640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:57:40.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.893+0000 7f108f7fe640 1 -- 172.21.15.99:0/1034786163 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f10a8004bf0 con 0x56100875ec60 2024-03-19T01:57:40.902 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.893+0000 7f10b91e2740 1 -- 172.21.15.99:0/1034786163 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f10a800fd20 con 0x56100875ec60 2024-03-19T01:57:40.903 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.893+0000 7f108f7fe640 1 -- 172.21.15.99:0/1034786163 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f10a8004d90 con 0x56100875ec60 2024-03-19T01:57:40.903 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.893+0000 7f108f7fe640 1 -- 172.21.15.99:0/1034786163 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7f10a8017020 con 0x56100875ec60 2024-03-19T01:57:40.903 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.893+0000 7f10b91e2740 10 client.16674 fetch_fsmap learned FSMap version 37 2024-03-19T01:57:40.903 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.893+0000 7f10b91e2740 1 -- 172.21.15.99:0/1034786163 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x5610083d2210 con 0x56100875ec60 2024-03-19T01:57:40.903 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.893+0000 7f108f7fe640 1 -- 172.21.15.99:0/1034786163 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f10a800fd20 con 0x56100875ec60 2024-03-19T01:57:40.903 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.893+0000 7f108f7fe640 1 -- 172.21.15.99:0/1034786163 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7f10a801a070 con 0x56100875ec60 2024-03-19T01:57:40.903 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.894+0000 7f10b91e2740 10 client.16674 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T01:57:40.903 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.894+0000 7f10b91e2740 10 client.16674 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:57:40.903 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.894+0000 7f10b91e2740 20 client.16674 populate_metadata read hostname 'smithi099' 2024-03-19T01:57:40.903 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.894+0000 7f10b91e2740 1 --2- 172.21.15.99:0/1034786163 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56100876edc0 0x5610087711f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:40.903 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.894+0000 7f10b91e2740 4 client.16674 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:57:40.903 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.894+0000 7f10b91e2740 1 -- 172.21.15.99:0/1034786163 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5610086cb120 con 0x56100876edc0 2024-03-19T01:57:40.903 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.894+0000 7f10affff640 1 --2- 172.21.15.99:0/1034786163 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56100876edc0 0x5610087711f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:40.903 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.894+0000 7f108f7fe640 10 client.16674 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:57:40.904 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.894+0000 7f10affff640 1 --2- 172.21.15.99:0/1034786163 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56100876edc0 0x5610087711f0 crc :-1 s=READY pgs=177 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=ef1cb95f512d3b83 server_cookie=ffa765178f225950 in_seq=0 out_seq=0 2024-03-19T01:57:40.904 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.895+0000 7f108f7fe640 1 -- 172.21.15.99:0/1034786163 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x5610086cb120 con 0x56100876edc0 2024-03-19T01:57:40.904 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.895+0000 7f108f7fe640 10 client.16674 handle_command_reply: tid=0 2024-03-19T01:57:40.904 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.895+0000 7f10b91e2740 1 client.16674 shutdown 2024-03-19T01:57:40.904 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.896+0000 7f10b91e2740 1 -- 172.21.15.99:0/1034786163 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56100875ec60 msgr2=0x56100876a690 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:57:40.904 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.896+0000 7f10b91e2740 1 --2- 172.21.15.99:0/1034786163 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56100875ec60 0x56100876a690 secure :-1 s=READY pgs=667 cs=0 l=1 rev1=1 crypto rx=0x7f10a8005c10 tx=0x7f10a800fef0 comp rx=0 tx=0).stop 2024-03-19T01:57:40.904 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.896+0000 7f10b91e2740 1 -- 172.21.15.99:0/1034786163 shutdown_connections 2024-03-19T01:57:40.904 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.896+0000 7f10b91e2740 1 --2- 172.21.15.99:0/1034786163 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56100876edc0 0x5610087711f0 crc :-1 s=READY pgs=177 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:40.904 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.896+0000 7f10b91e2740 1 --2- 172.21.15.99:0/1034786163 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56100875ec60 0x56100876a690 unknown :-1 s=CLOSED pgs=667 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:40.904 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.896+0000 7f10b91e2740 1 --2- 172.21.15.99:0/1034786163 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561008759af0 0x561008763650 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:40.904 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.896+0000 7f108f7fe640 0 client.16674 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:57:40.904 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.896+0000 7f10b91e2740 1 --2- 172.21.15.99:0/1034786163 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561008759100 0x561008763110 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:40.904 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.896+0000 7f10b91e2740 1 -- 172.21.15.99:0/1034786163 >> 172.21.15.99:0/1034786163 conn(0x561008754670 msgr2=0x5610087558c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:57:40.904 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.896+0000 7f10b91e2740 1 -- 172.21.15.99:0/1034786163 shutdown_connections 2024-03-19T01:57:40.904 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.896+0000 7f10b91e2740 1 -- 172.21.15.99:0/1034786163 wait complete. 2024-03-19T01:57:40.905 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.896+0000 7f10b91e2740 20 client.16674 trim_cache size 0 max 16384 2024-03-19T01:57:40.913 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.911+0000 7f10b91e2740 1 Processor -- start 2024-03-19T01:57:40.913 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.911+0000 7f10b91e2740 1 -- start start 2024-03-19T01:57:40.913 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.911+0000 7f10b91e2740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561008759100 0x561008757fe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:40.914 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.911+0000 7f10b91e2740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561008759af0 0x56100876a650 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:40.914 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.911+0000 7f10b91e2740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56100875ec60 0x561008775af0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:40.914 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.912+0000 7f10b91e2740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f10b01ad7f0 con 0x56100875ec60 2024-03-19T01:57:40.914 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.912+0000 7f10b91e2740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f10b01ab0c0 con 0x561008759af0 2024-03-19T01:57:40.914 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.912+0000 7f10b91e2740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f10b01a8f10 con 0x561008759100 2024-03-19T01:57:40.914 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.912+0000 7f10b4b9c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56100875ec60 0x561008775af0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:40.914 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.912+0000 7f10b4b9c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56100875ec60 0x561008775af0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:55732/0 (socket says 172.21.15.99:55732) 2024-03-19T01:57:40.914 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.912+0000 7f10b4b9c640 1 -- 172.21.15.99:0/3356399355 learned_addr learned my addr 172.21.15.99:0/3356399355 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:57:40.914 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.912+0000 7f10af7fe640 1 --2- 172.21.15.99:0/3356399355 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561008759af0 0x56100876a650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:40.914 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.912+0000 7f10affff640 1 --2- 172.21.15.99:0/3356399355 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561008759100 0x561008757fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:40.914 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.912+0000 7f10b4b9c640 1 -- 172.21.15.99:0/3356399355 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561008759100 msgr2=0x561008757fe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:57:40.914 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.912+0000 7f10b4b9c640 1 --2- 172.21.15.99:0/3356399355 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561008759100 0x561008757fe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:40.914 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.912+0000 7f10b4b9c640 1 -- 172.21.15.99:0/3356399355 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561008759af0 msgr2=0x56100876a650 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:57:40.914 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.912+0000 7f10b4b9c640 1 --2- 172.21.15.99:0/3356399355 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561008759af0 0x56100876a650 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:40.915 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.912+0000 7f10b4b9c640 1 -- 172.21.15.99:0/3356399355 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x56100849e620 con 0x56100875ec60 2024-03-19T01:57:40.915 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.912+0000 7f10b4b9c640 1 --2- 172.21.15.99:0/3356399355 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56100875ec60 0x561008775af0 secure :-1 s=READY pgs=668 cs=0 l=1 rev1=1 crypto rx=0x7f10a8009730 tx=0x7f10a801bf90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5d28238f77ce1c22 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:57:40.915 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.912+0000 7f1095ffb640 1 -- 172.21.15.99:0/3356399355 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f10a803f070 con 0x56100875ec60 2024-03-19T01:57:40.915 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.912+0000 7f1095ffb640 1 -- 172.21.15.99:0/3356399355 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f10a8003e50 con 0x56100875ec60 2024-03-19T01:57:40.915 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.912+0000 7f1095ffb640 1 -- 172.21.15.99:0/3356399355 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f10a801b580 con 0x56100875ec60 2024-03-19T01:57:40.918 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.913+0000 7f10b91e2740 1 -- 172.21.15.99:0/3356399355 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56100875ec60 msgr2=0x561008775af0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:57:40.918 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.913+0000 7f10b91e2740 1 --2- 172.21.15.99:0/3356399355 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56100875ec60 0x561008775af0 secure :-1 s=READY pgs=668 cs=0 l=1 rev1=1 crypto rx=0x7f10a8009730 tx=0x7f10a801bf90 comp rx=0 tx=0).stop 2024-03-19T01:57:40.918 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.913+0000 7f10b91e2740 1 -- 172.21.15.99:0/3356399355 shutdown_connections 2024-03-19T01:57:40.919 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.913+0000 7f10b91e2740 1 --2- 172.21.15.99:0/3356399355 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x561008759af0 0x56100876a650 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:40.919 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.913+0000 7f10b91e2740 1 --2- 172.21.15.99:0/3356399355 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56100875ec60 0x561008775af0 unknown :-1 s=CLOSED pgs=668 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:40.919 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.913+0000 7f10b91e2740 1 --2- 172.21.15.99:0/3356399355 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561008759100 0x561008757fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:40.919 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.913+0000 7f10b91e2740 1 -- 172.21.15.99:0/3356399355 >> 172.21.15.99:0/3356399355 conn(0x561008754670 msgr2=0x5610087578d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:57:40.919 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.913+0000 7f10b91e2740 1 -- 172.21.15.99:0/3356399355 shutdown_connections 2024-03-19T01:57:40.919 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.913+0000 7f10b91e2740 1 -- 172.21.15.99:0/3356399355 wait complete. 2024-03-19T01:57:40.919 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.913+0000 7f10b91e2740 1 -- start start 2024-03-19T01:57:40.919 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.914+0000 7f10b91e2740 1 Processor -- start 2024-03-19T01:57:40.919 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.914+0000 7f10b91e2740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561008759100 0x561008768180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:40.919 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.914+0000 7f10b91e2740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561008759af0 0x5610087686c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:40.919 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.914+0000 7f10b91e2740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56100875ec60 0x5610087696d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:40.919 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.914+0000 7f10b91e2740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f10b01a9580 con 0x561008759100 2024-03-19T01:57:40.919 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.914+0000 7f10b91e2740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f10b01aa9f0 con 0x56100875ec60 2024-03-19T01:57:40.919 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.914+0000 7f10b91e2740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f10b01a9c50 con 0x561008759af0 2024-03-19T01:57:40.920 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.914+0000 7f10b91e2740 10 client.0 _ll_register_callbacks cb 0x7f10a00097a0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:57:40.920 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.914+0000 7f10affff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561008759100 0x561008768180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:40.920 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.914+0000 7f10affff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561008759100 0x561008768180 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:55738/0 (socket says 172.21.15.99:55738) 2024-03-19T01:57:40.920 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.914+0000 7f10affff640 1 -- 172.21.15.99:0/2062210421 learned_addr learned my addr 172.21.15.99:0/2062210421 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:57:40.920 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.914+0000 7f10b4b9c640 1 --2- 172.21.15.99:0/2062210421 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56100875ec60 0x5610087696d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:40.920 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.914+0000 7f10af7fe640 1 --2- 172.21.15.99:0/2062210421 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561008759af0 0x5610087686c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:40.920 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.914+0000 7f10affff640 1 -- 172.21.15.99:0/2062210421 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561008759af0 msgr2=0x5610087686c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:57:40.920 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.914+0000 7f10affff640 1 --2- 172.21.15.99:0/2062210421 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561008759af0 0x5610087686c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:40.920 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.914+0000 7f10affff640 1 -- 172.21.15.99:0/2062210421 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56100875ec60 msgr2=0x5610087696d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:57:40.920 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.914+0000 7f10affff640 1 --2- 172.21.15.99:0/2062210421 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56100875ec60 0x5610087696d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:40.920 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.914+0000 7f10affff640 1 -- 172.21.15.99:0/2062210421 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x561008769cc0 con 0x561008759100 2024-03-19T01:57:40.920 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.914+0000 7f10affff640 1 --2- 172.21.15.99:0/2062210421 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561008759100 0x561008768180 secure :-1 s=READY pgs=669 cs=0 l=1 rev1=1 crypto rx=0x7f10a0000c00 tx=0x7f10a0012d10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e5daa5cad40e49f1 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:57:40.920 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.914+0000 7f108f7fe640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:57:40.920 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.915+0000 7f108f7fe640 1 -- 172.21.15.99:0/2062210421 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f10a0020a50 con 0x561008759100 2024-03-19T01:57:40.921 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.915+0000 7f108f7fe640 1 -- 172.21.15.99:0/2062210421 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f10a0020bf0 con 0x561008759100 2024-03-19T01:57:40.921 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.915+0000 7f10b91e2740 1 -- 172.21.15.99:0/2062210421 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f10a8003e50 con 0x561008759100 2024-03-19T01:57:40.921 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.915+0000 7f108f7fe640 1 -- 172.21.15.99:0/2062210421 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7f10a0020df0 con 0x561008759100 2024-03-19T01:57:40.921 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.915+0000 7f10b91e2740 10 client.16680 fetch_fsmap learned FSMap version 37 2024-03-19T01:57:40.921 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.915+0000 7f10b91e2740 1 -- 172.21.15.99:0/2062210421 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x5610087d55f0 con 0x561008759100 2024-03-19T01:57:40.921 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.915+0000 7f108f7fe640 1 -- 172.21.15.99:0/2062210421 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f10a8003e50 con 0x561008759100 2024-03-19T01:57:40.921 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.915+0000 7f108f7fe640 1 -- 172.21.15.99:0/2062210421 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7f10a0004bf0 con 0x561008759100 2024-03-19T01:57:40.921 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.915+0000 7f10b91e2740 10 client.16680 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T01:57:40.921 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.915+0000 7f10b91e2740 10 client.16680 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:57:40.921 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.915+0000 7f10b91e2740 20 client.16680 populate_metadata read hostname 'smithi099' 2024-03-19T01:57:40.921 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.915+0000 7f10b91e2740 1 --2- 172.21.15.99:0/2062210421 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5610087d5f80 0x5610087d83a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:40.921 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.915+0000 7f10b91e2740 4 client.16680 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:57:40.921 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.915+0000 7f10b91e2740 1 -- 172.21.15.99:0/2062210421 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x5610086c5470 con 0x5610087d5f80 2024-03-19T01:57:40.921 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.915+0000 7f10af7fe640 1 --2- 172.21.15.99:0/2062210421 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5610087d5f80 0x5610087d83a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:40.922 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.916+0000 7f10af7fe640 1 --2- 172.21.15.99:0/2062210421 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5610087d5f80 0x5610087d83a0 crc :-1 s=READY pgs=178 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=133121dfb0897e71 server_cookie=86176e61c04421c5 in_seq=0 out_seq=0 2024-03-19T01:57:40.922 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:40.916+0000 7f108f7fe640 10 client.16680 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:57:41.106 INFO:tasks.workunit.client.0.smithi099.stdout: 10 110523 23.24 MB/sec execute 130 sec latency 201.516 ms 2024-03-19T01:57:41.408 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:41.407+0000 7f108f7fe640 1 -- 172.21.15.99:0/2062210421 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x5610086c5470 con 0x5610087d5f80 2024-03-19T01:57:41.408 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:41.407+0000 7f108f7fe640 10 client.16680 handle_command_reply: tid=0 2024-03-19T01:57:41.408 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:41.407+0000 7f10b91e2740 1 client.16680 shutdown 2024-03-19T01:57:41.408 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:41.407+0000 7f10b91e2740 1 -- 172.21.15.99:0/2062210421 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561008759100 msgr2=0x561008768180 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:57:41.408 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:41.407+0000 7f10b91e2740 1 --2- 172.21.15.99:0/2062210421 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561008759100 0x561008768180 secure :-1 s=READY pgs=669 cs=0 l=1 rev1=1 crypto rx=0x7f10a0000c00 tx=0x7f10a0012d10 comp rx=0 tx=0).stop 2024-03-19T01:57:41.409 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:41.407+0000 7f10b91e2740 1 -- 172.21.15.99:0/2062210421 shutdown_connections 2024-03-19T01:57:41.409 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:41.407+0000 7f10b91e2740 1 --2- 172.21.15.99:0/2062210421 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5610087d5f80 0x5610087d83a0 crc :-1 s=READY pgs=178 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:41.409 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:41.407+0000 7f10b91e2740 1 --2- 172.21.15.99:0/2062210421 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56100875ec60 0x5610087696d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:41.409 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:41.407+0000 7f108f7fe640 0 client.16680 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:57:41.409 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:41.407+0000 7f10b91e2740 1 --2- 172.21.15.99:0/2062210421 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x561008759af0 0x5610087686c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:41.409 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:41.407+0000 7f10b91e2740 1 --2- 172.21.15.99:0/2062210421 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x561008759100 0x561008768180 unknown :-1 s=CLOSED pgs=669 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:41.409 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:41.407+0000 7f10b91e2740 1 -- 172.21.15.99:0/2062210421 >> 172.21.15.99:0/2062210421 conn(0x561008754670 msgr2=0x561008758160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:57:41.409 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:41.407+0000 7f10b91e2740 1 -- 172.21.15.99:0/2062210421 shutdown_connections 2024-03-19T01:57:41.409 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:41.407+0000 7f10b91e2740 1 -- 172.21.15.99:0/2062210421 wait complete. 2024-03-19T01:57:41.409 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:41.407+0000 7f10b91e2740 20 client.16680 trim_cache size 0 max 16384 2024-03-19T01:57:41.409 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:57:41.410 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:57:41.410 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:57:41.410 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:57:41.410 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:41.408+0000 7f10b65ff640 1 -- 172.21.15.99:0/548929188 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f10980a28a0 msgr2=0x7f10980a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:57:41.411 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:41.408+0000 7f10b65ff640 1 --2- 172.21.15.99:0/548929188 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f10980a28a0 0x7f10980a4d60 secure :-1 s=READY pgs=347 cs=0 l=1 rev1=1 crypto rx=0x7f10a8022000 tx=0x7f10a8023000 comp rx=0 tx=0).stop 2024-03-19T01:57:41.411 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:41.408+0000 7f10b65ff640 1 -- 172.21.15.99:0/548929188 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f10b0104c70 msgr2=0x7f10b019a1c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:57:41.411 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:41.408+0000 7f10b65ff640 1 --2- 172.21.15.99:0/548929188 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f10b0104c70 0x7f10b019a1c0 secure :-1 s=READY pgs=665 cs=0 l=1 rev1=1 crypto rx=0x7f10a4009230 tx=0x7f10a402c650 comp rx=0 tx=0).stop 2024-03-19T01:57:41.411 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:41.408+0000 7f10b65ff640 1 -- 172.21.15.99:0/548929188 shutdown_connections 2024-03-19T01:57:41.411 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:41.408+0000 7f10b65ff640 1 --2- 172.21.15.99:0/548929188 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f10980a28a0 0x7f10980a4d60 unknown :-1 s=CLOSED pgs=347 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:41.411 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:41.408+0000 7f10b65ff640 1 --2- 172.21.15.99:0/548929188 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f10b0105630 0x7f10b01a1220 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:41.411 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:41.408+0000 7f10b65ff640 1 --2- 172.21.15.99:0/548929188 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f10b0104c70 0x7f10b019a1c0 unknown :-1 s=CLOSED pgs=665 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:41.411 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:41.408+0000 7f10b65ff640 1 --2- 172.21.15.99:0/548929188 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f10b0103a80 0x7f10b0199c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:41.411 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:41.408+0000 7f10b65ff640 1 -- 172.21.15.99:0/548929188 >> 172.21.15.99:0/548929188 conn(0x7f10b00ff010 msgr2=0x7f10b0107f10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:57:41.411 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:41.408+0000 7f10b65ff640 1 -- 172.21.15.99:0/548929188 shutdown_connections 2024-03-19T01:57:41.411 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:41.408+0000 7f10b65ff640 1 -- 172.21.15.99:0/548929188 wait complete. 2024-03-19T01:57:41.806 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:57:41 smithi115 ceph-mon[67332]: pgmap v849: 97 pgs: 97 active+clean; 560 MiB data, 6.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 6.6 MiB/s wr, 536 op/s 2024-03-19T01:57:41.840 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:57:41 smithi099 ceph-mon[62951]: pgmap v849: 97 pgs: 97 active+clean; 560 MiB data, 6.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 6.6 MiB/s wr, 536 op/s 2024-03-19T01:57:41.861 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:57:41 smithi112 ceph-mon[66819]: pgmap v849: 97 pgs: 97 active+clean; 560 MiB data, 6.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 6.6 MiB/s wr, 536 op/s 2024-03-19T01:57:42.106 INFO:tasks.workunit.client.0.smithi099.stdout: 10 110742 23.14 MB/sec execute 131 sec latency 368.393 ms 2024-03-19T01:57:43.106 INFO:tasks.workunit.client.0.smithi099.stdout: 10 111162 23.09 MB/sec execute 132 sec latency 95.816 ms 2024-03-19T01:57:43.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:57:43 smithi112 ceph-mon[66819]: pgmap v850: 97 pgs: 97 active+clean; 557 MiB data, 6.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.8 MiB/s rd, 7.1 MiB/s wr, 618 op/s 2024-03-19T01:57:43.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:57:43 smithi115 ceph-mon[67332]: pgmap v850: 97 pgs: 97 active+clean; 557 MiB data, 6.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.8 MiB/s rd, 7.1 MiB/s wr, 618 op/s 2024-03-19T01:57:43.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:57:43 smithi099 ceph-mon[62951]: pgmap v850: 97 pgs: 97 active+clean; 557 MiB data, 6.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.8 MiB/s rd, 7.1 MiB/s wr, 618 op/s 2024-03-19T01:57:44.106 INFO:tasks.workunit.client.0.smithi099.stdout: 10 111500 23.02 MB/sec execute 133 sec latency 318.116 ms 2024-03-19T01:57:45.106 INFO:tasks.workunit.client.0.smithi099.stdout: 10 112020 23.12 MB/sec execute 134 sec latency 103.631 ms 2024-03-19T01:57:45.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:57:45 smithi112 ceph-mon[66819]: pgmap v851: 97 pgs: 97 active+clean; 551 MiB data, 6.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 6.1 MiB/s wr, 505 op/s 2024-03-19T01:57:45.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:57:45 smithi115 ceph-mon[67332]: pgmap v851: 97 pgs: 97 active+clean; 551 MiB data, 6.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 6.1 MiB/s wr, 505 op/s 2024-03-19T01:57:45.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:57:45 smithi099 ceph-mon[62951]: pgmap v851: 97 pgs: 97 active+clean; 551 MiB data, 6.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 6.1 MiB/s wr, 505 op/s 2024-03-19T01:57:46.106 INFO:tasks.workunit.client.0.smithi099.stdout: 10 112433 23.07 MB/sec execute 135 sec latency 61.691 ms 2024-03-19T01:57:47.106 INFO:tasks.workunit.client.0.smithi099.stdout: 10 112841 23.06 MB/sec execute 136 sec latency 89.690 ms 2024-03-19T01:57:47.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:57:47 smithi112 ceph-mon[66819]: pgmap v852: 97 pgs: 97 active+clean; 556 MiB data, 6.4 GiB used, 1.0 TiB / 1.0 TiB avail; 4.6 MiB/s rd, 6.9 MiB/s wr, 582 op/s 2024-03-19T01:57:47.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:57:47 smithi115 ceph-mon[67332]: pgmap v852: 97 pgs: 97 active+clean; 556 MiB data, 6.4 GiB used, 1.0 TiB / 1.0 TiB avail; 4.6 MiB/s rd, 6.9 MiB/s wr, 582 op/s 2024-03-19T01:57:47.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:57:47 smithi099 ceph-mon[62951]: pgmap v852: 97 pgs: 97 active+clean; 556 MiB data, 6.4 GiB used, 1.0 TiB / 1.0 TiB avail; 4.6 MiB/s rd, 6.9 MiB/s wr, 582 op/s 2024-03-19T01:57:48.106 INFO:tasks.workunit.client.0.smithi099.stdout: 10 113295 23.05 MB/sec execute 137 sec latency 82.003 ms 2024-03-19T01:57:49.106 INFO:tasks.workunit.client.0.smithi099.stdout: 10 113734 23.05 MB/sec execute 138 sec latency 174.017 ms 2024-03-19T01:57:49.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:57:49 smithi112 ceph-mon[66819]: pgmap v853: 97 pgs: 97 active+clean; 557 MiB data, 6.4 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 6.0 MiB/s wr, 548 op/s 2024-03-19T01:57:49.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:57:49 smithi115 ceph-mon[67332]: pgmap v853: 97 pgs: 97 active+clean; 557 MiB data, 6.4 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 6.0 MiB/s wr, 548 op/s 2024-03-19T01:57:49.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:57:49 smithi099 ceph-mon[62951]: pgmap v853: 97 pgs: 97 active+clean; 557 MiB data, 6.4 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 6.0 MiB/s wr, 548 op/s 2024-03-19T01:57:50.106 INFO:tasks.workunit.client.0.smithi099.stdout: 10 114247 23.13 MB/sec execute 139 sec latency 102.212 ms 2024-03-19T01:57:51.106 INFO:tasks.workunit.client.0.smithi099.stdout: 10 114686 23.17 MB/sec execute 140 sec latency 58.589 ms 2024-03-19T01:57:51.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:57:51 smithi112 ceph-mon[66819]: pgmap v854: 97 pgs: 97 active+clean; 563 MiB data, 6.4 GiB used, 1.0 TiB / 1.0 TiB avail; 4.7 MiB/s rd, 6.3 MiB/s wr, 574 op/s 2024-03-19T01:57:51.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:57:51 smithi115 ceph-mon[67332]: pgmap v854: 97 pgs: 97 active+clean; 563 MiB data, 6.4 GiB used, 1.0 TiB / 1.0 TiB avail; 4.7 MiB/s rd, 6.3 MiB/s wr, 574 op/s 2024-03-19T01:57:51.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:57:51 smithi099 ceph-mon[62951]: pgmap v854: 97 pgs: 97 active+clean; 563 MiB data, 6.4 GiB used, 1.0 TiB / 1.0 TiB avail; 4.7 MiB/s rd, 6.3 MiB/s wr, 574 op/s 2024-03-19T01:57:52.106 INFO:tasks.workunit.client.0.smithi099.stdout: 10 115142 23.19 MB/sec execute 141 sec latency 91.595 ms 2024-03-19T01:57:53.106 INFO:tasks.workunit.client.0.smithi099.stdout: 10 115522 23.14 MB/sec execute 142 sec latency 68.257 ms 2024-03-19T01:57:53.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:57:53 smithi112 ceph-mon[66819]: pgmap v855: 97 pgs: 97 active+clean; 563 MiB data, 6.5 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 MiB/s rd, 6.5 MiB/s wr, 582 op/s 2024-03-19T01:57:53.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:57:53 smithi115 ceph-mon[67332]: pgmap v855: 97 pgs: 97 active+clean; 563 MiB data, 6.5 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 MiB/s rd, 6.5 MiB/s wr, 582 op/s 2024-03-19T01:57:53.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:57:53 smithi099 ceph-mon[62951]: pgmap v855: 97 pgs: 97 active+clean; 563 MiB data, 6.5 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 MiB/s rd, 6.5 MiB/s wr, 582 op/s 2024-03-19T01:57:54.106 INFO:tasks.workunit.client.0.smithi099.stdout: 10 115804 23.06 MB/sec execute 143 sec latency 285.706 ms 2024-03-19T01:57:54.512 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.509+0000 7f1a8ee68640 1 -- 172.21.15.99:0/1274948527 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1a88101900 msgr2=0x7f1a88101d80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:57:54.512 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.509+0000 7f1a8ee68640 1 --2- 172.21.15.99:0/1274948527 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1a88101900 0x7f1a88101d80 secure :-1 s=READY pgs=179 cs=0 l=1 rev1=1 crypto rx=0x7f1a74009900 tx=0x7f1a7402bc30 comp rx=0 tx=0).stop 2024-03-19T01:57:54.512 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.509+0000 7f1a8ee68640 1 -- 172.21.15.99:0/1274948527 shutdown_connections 2024-03-19T01:57:54.512 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.509+0000 7f1a8ee68640 1 --2- 172.21.15.99:0/1274948527 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1a88101900 0x7f1a88101d80 unknown :-1 s=CLOSED pgs=179 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:54.512 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.509+0000 7f1a8ee68640 1 --2- 172.21.15.99:0/1274948527 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1a881022c0 0x7f1a88108b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:54.512 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.509+0000 7f1a8ee68640 1 --2- 172.21.15.99:0/1274948527 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1a88100710 0x7f1a88100b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:54.512 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.509+0000 7f1a8ee68640 1 -- 172.21.15.99:0/1274948527 >> 172.21.15.99:0/1274948527 conn(0x7f1a880fbc80 msgr2=0x7f1a880fe0c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:57:54.512 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.509+0000 7f1a8ee68640 1 -- 172.21.15.99:0/1274948527 shutdown_connections 2024-03-19T01:57:54.513 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.509+0000 7f1a8ee68640 1 -- 172.21.15.99:0/1274948527 wait complete. 2024-03-19T01:57:54.513 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.510+0000 7f1a8ee68640 1 Processor -- start 2024-03-19T01:57:54.513 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.510+0000 7f1a8ee68640 1 -- start start 2024-03-19T01:57:54.513 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.510+0000 7f1a8ee68640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1a88100710 0x7f1a88069a10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:54.513 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.510+0000 7f1a8ee68640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1a88101900 0x7f1a8819c7c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:54.513 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.510+0000 7f1a8ee68640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1a881022c0 0x7f1a8819ede0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:54.513 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.510+0000 7f1a8ee68640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f1a8810c390 con 0x7f1a88100710 2024-03-19T01:57:54.513 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.510+0000 7f1a8ee68640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f1a8810c210 con 0x7f1a881022c0 2024-03-19T01:57:54.513 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.510+0000 7f1a8ee68640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f1a8810c510 con 0x7f1a88101900 2024-03-19T01:57:54.513 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.510+0000 7f1a8cc04640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1a88100710 0x7f1a88069a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:54.513 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.510+0000 7f1a8cc04640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1a88100710 0x7f1a88069a10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:60346/0 (socket says 172.21.15.99:60346) 2024-03-19T01:57:54.513 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.510+0000 7f1a8cc04640 1 -- 172.21.15.99:0/3753568221 learned_addr learned my addr 172.21.15.99:0/3753568221 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:57:54.513 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.510+0000 7f1a87fff640 1 --2- 172.21.15.99:0/3753568221 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1a88101900 0x7f1a8819c7c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:54.513 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.510+0000 7f1a8d405640 1 --2- 172.21.15.99:0/3753568221 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1a881022c0 0x7f1a8819ede0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:54.514 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.510+0000 7f1a8cc04640 1 -- 172.21.15.99:0/3753568221 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1a88101900 msgr2=0x7f1a8819c7c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:57:54.514 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.510+0000 7f1a8cc04640 1 --2- 172.21.15.99:0/3753568221 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1a88101900 0x7f1a8819c7c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:54.514 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.510+0000 7f1a8cc04640 1 -- 172.21.15.99:0/3753568221 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1a881022c0 msgr2=0x7f1a8819ede0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:57:54.514 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.510+0000 7f1a8cc04640 1 --2- 172.21.15.99:0/3753568221 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1a881022c0 0x7f1a8819ede0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:54.514 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.510+0000 7f1a8cc04640 1 -- 172.21.15.99:0/3753568221 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f1a8819f6f0 con 0x7f1a88100710 2024-03-19T01:57:54.514 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.510+0000 7f1a87fff640 1 --2- 172.21.15.99:0/3753568221 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1a88101900 0x7f1a8819c7c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:57:54.514 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.510+0000 7f1a8cc04640 1 --2- 172.21.15.99:0/3753568221 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1a88100710 0x7f1a88069a10 secure :-1 s=READY pgs=670 cs=0 l=1 rev1=1 crypto rx=0x7f1a7c00e9c0 tx=0x7f1a7c00ee80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:57:54.514 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.511+0000 7f1a85ffb640 1 -- 172.21.15.99:0/3753568221 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1a7c00cd80 con 0x7f1a88100710 2024-03-19T01:57:54.514 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.511+0000 7f1a85ffb640 1 -- 172.21.15.99:0/3753568221 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f1a7c0035f0 con 0x7f1a88100710 2024-03-19T01:57:54.514 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.511+0000 7f1a8ee68640 1 -- 172.21.15.99:0/3753568221 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f1a8819f980 con 0x7f1a88100710 2024-03-19T01:57:54.514 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.511+0000 7f1a85ffb640 1 -- 172.21.15.99:0/3753568221 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1a7c010690 con 0x7f1a88100710 2024-03-19T01:57:54.514 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.511+0000 7f1a8ee68640 1 -- 172.21.15.99:0/3753568221 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f1a8819fea0 con 0x7f1a88100710 2024-03-19T01:57:54.518 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.513+0000 7f1a85ffb640 1 -- 172.21.15.99:0/3753568221 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f1a7c0037a0 con 0x7f1a88100710 2024-03-19T01:57:54.518 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.514+0000 7f1a91a50740 1 Processor -- start 2024-03-19T01:57:54.518 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.514+0000 7f1a91a50740 1 -- start start 2024-03-19T01:57:54.518 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.514+0000 7f1a91a50740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5603aee31100 0x5603aee31520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:54.519 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.514+0000 7f1a91a50740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5603aee31af0 0x5603aee365f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:54.519 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.514+0000 7f1a85ffb640 1 --2- 172.21.15.99:0/3753568221 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f1a780a28a0 0x7f1a780a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:54.519 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.514+0000 7f1a91a50740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5603aee36c60 0x5603aee390f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:54.519 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.514+0000 7f1a91a50740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x5603aed95be0 con 0x5603aee36c60 2024-03-19T01:57:54.519 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.514+0000 7f1a91a50740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x5603aed3e960 con 0x5603aee31af0 2024-03-19T01:57:54.519 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.514+0000 7f1a91a50740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x5603aed91c80 con 0x5603aee31100 2024-03-19T01:57:54.519 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.514+0000 7f1a8d405640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5603aee36c60 0x5603aee390f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:54.519 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.514+0000 7f1a8d405640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5603aee36c60 0x5603aee390f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:60350/0 (socket says 172.21.15.99:60350) 2024-03-19T01:57:54.519 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.514+0000 7f1a8d405640 1 -- 172.21.15.99:0/3632948548 learned_addr learned my addr 172.21.15.99:0/3632948548 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:57:54.519 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.514+0000 7f1a8d405640 1 --2- 172.21.15.99:0/3632948548 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5603aee31100 0x5603aee31520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:54.519 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.514+0000 7f1a8cc04640 1 --2- 172.21.15.99:0/3753568221 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f1a780a28a0 0x7f1a780a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:54.519 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.514+0000 7f1a87fff640 1 --2- 172.21.15.99:0/3632948548 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5603aee31af0 0x5603aee365f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:54.519 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.514+0000 7f1a85ffb640 1 -- 172.21.15.99:0/3753568221 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f1a7c0ceeb0 con 0x7f1a88100710 2024-03-19T01:57:54.520 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.514+0000 7f1a8d405640 1 -- 172.21.15.99:0/3632948548 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5603aee31100 msgr2=0x5603aee31520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:57:54.520 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.514+0000 7f1a8d405640 1 --2- 172.21.15.99:0/3632948548 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5603aee31100 0x5603aee31520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:54.520 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.514+0000 7f1a8d405640 1 -- 172.21.15.99:0/3632948548 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5603aee31af0 msgr2=0x5603aee365f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:57:54.520 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.514+0000 7f1a8d405640 1 --2- 172.21.15.99:0/3632948548 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5603aee31af0 0x5603aee365f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:54.520 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.514+0000 7f1a8d405640 1 -- 172.21.15.99:0/3632948548 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5603aed35d20 con 0x5603aee36c60 2024-03-19T01:57:54.520 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.515+0000 7f1a8d405640 1 --2- 172.21.15.99:0/3632948548 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5603aee36c60 0x5603aee390f0 secure :-1 s=READY pgs=671 cs=0 l=1 rev1=1 crypto rx=0x7f1a80009fd0 tx=0x7f1a80009b80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=26e4ebcee2d78062 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:57:54.520 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.515+0000 7f1a8cc04640 1 --2- 172.21.15.99:0/3753568221 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f1a780a28a0 0x7f1a780a4d60 secure :-1 s=READY pgs=348 cs=0 l=1 rev1=1 crypto rx=0x7f1a7c097c10 tx=0x7f1a7c0c6200 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:57:54.520 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.515+0000 7f1a72ffd640 1 -- 172.21.15.99:0/3632948548 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1a80011070 con 0x5603aee36c60 2024-03-19T01:57:54.520 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.515+0000 7f1a72ffd640 1 -- 172.21.15.99:0/3632948548 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f1a80003e50 con 0x5603aee36c60 2024-03-19T01:57:54.520 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.515+0000 7f1a72ffd640 1 -- 172.21.15.99:0/3632948548 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1a800042b0 con 0x5603aee36c60 2024-03-19T01:57:54.520 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.515+0000 7f1a91a50740 1 -- 172.21.15.99:0/3632948548 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5603aee36c60 msgr2=0x5603aee390f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:57:54.520 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.515+0000 7f1a91a50740 1 --2- 172.21.15.99:0/3632948548 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5603aee36c60 0x5603aee390f0 secure :-1 s=READY pgs=671 cs=0 l=1 rev1=1 crypto rx=0x7f1a80009fd0 tx=0x7f1a80009b80 comp rx=0 tx=0).stop 2024-03-19T01:57:54.520 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.515+0000 7f1a91a50740 1 -- 172.21.15.99:0/3632948548 shutdown_connections 2024-03-19T01:57:54.520 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.515+0000 7f1a91a50740 1 --2- 172.21.15.99:0/3632948548 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5603aee31af0 0x5603aee365f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:54.521 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.515+0000 7f1a91a50740 1 --2- 172.21.15.99:0/3632948548 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5603aee36c60 0x5603aee390f0 unknown :-1 s=CLOSED pgs=671 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:54.521 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.515+0000 7f1a91a50740 1 --2- 172.21.15.99:0/3632948548 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5603aee31100 0x5603aee31520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:54.521 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.515+0000 7f1a91a50740 1 -- 172.21.15.99:0/3632948548 >> 172.21.15.99:0/3632948548 conn(0x5603aee2c670 msgr2=0x5603aee2ca70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:57:54.521 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.516+0000 7f1a91a50740 1 -- 172.21.15.99:0/3632948548 shutdown_connections 2024-03-19T01:57:54.521 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.516+0000 7f1a91a50740 1 -- 172.21.15.99:0/3632948548 wait complete. 2024-03-19T01:57:54.521 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.516+0000 7f1a91a50740 1 -- start start 2024-03-19T01:57:54.521 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.516+0000 7f1a91a50740 1 Processor -- start 2024-03-19T01:57:54.521 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.516+0000 7f1a91a50740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5603aee31100 0x5603aee3b070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:54.521 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.516+0000 7f1a91a50740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5603aee31af0 0x5603aee3b5b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:54.521 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.516+0000 7f1a91a50740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5603aee36c60 0x5603aee425f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:54.521 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.516+0000 7f1a91a50740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x5603aee39c10 con 0x5603aee31af0 2024-03-19T01:57:54.522 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.516+0000 7f1a91a50740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x5603aed51db0 con 0x5603aee31100 2024-03-19T01:57:54.522 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.516+0000 7f1a91a50740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x5603aee39d90 con 0x5603aee36c60 2024-03-19T01:57:54.522 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.516+0000 7f1a91a50740 10 client.0 _ll_register_callbacks cb 0x5603aed56e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:57:54.522 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.516+0000 7f1a87fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5603aee31af0 0x5603aee3b5b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:54.522 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.517+0000 7f1a87fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5603aee31af0 0x5603aee3b5b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:60364/0 (socket says 172.21.15.99:60364) 2024-03-19T01:57:54.522 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.517+0000 7f1a87fff640 1 -- 172.21.15.99:0/229988730 learned_addr learned my addr 172.21.15.99:0/229988730 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:57:54.522 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.516+0000 7f1a8d405640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5603aee31100 0x5603aee3b070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:54.522 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.517+0000 7f1a8d405640 1 --2- 172.21.15.99:0/229988730 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5603aee36c60 0x5603aee425f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:54.522 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.517+0000 7f1a87fff640 1 -- 172.21.15.99:0/229988730 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5603aee36c60 msgr2=0x5603aee425f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:57:54.522 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.517+0000 7f1a87fff640 1 --2- 172.21.15.99:0/229988730 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5603aee36c60 0x5603aee425f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:54.523 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.517+0000 7f1a87fff640 1 -- 172.21.15.99:0/229988730 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5603aee31100 msgr2=0x5603aee3b070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:57:54.523 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.517+0000 7f1a87fff640 1 --2- 172.21.15.99:0/229988730 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5603aee31100 0x5603aee3b070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:54.523 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.517+0000 7f1a87fff640 1 -- 172.21.15.99:0/229988730 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5603aed50c60 con 0x5603aee31af0 2024-03-19T01:57:54.523 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.517+0000 7f1a8d405640 1 --2- 172.21.15.99:0/229988730 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5603aee31100 0x5603aee3b070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:57:54.523 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.517+0000 7f1a87fff640 1 --2- 172.21.15.99:0/229988730 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5603aee31af0 0x5603aee3b5b0 secure :-1 s=READY pgs=672 cs=0 l=1 rev1=1 crypto rx=0x7f1a74036040 tx=0x7f1a74004270 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2dc0502ea29fece server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:57:54.523 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.517+0000 7f1a67fff640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:57:54.523 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.518+0000 7f1a67fff640 1 -- 172.21.15.99:0/229988730 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1a74051060 con 0x5603aee31af0 2024-03-19T01:57:54.523 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.518+0000 7f1a67fff640 1 -- 172.21.15.99:0/229988730 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f1a74005230 con 0x5603aee31af0 2024-03-19T01:57:54.523 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.518+0000 7f1a91a50740 1 -- 172.21.15.99:0/229988730 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f1a80003e50 con 0x5603aee31af0 2024-03-19T01:57:54.523 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.518+0000 7f1a67fff640 1 -- 172.21.15.99:0/229988730 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7f1a74005d50 con 0x5603aee31af0 2024-03-19T01:57:54.523 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.518+0000 7f1a91a50740 10 client.16692 fetch_fsmap learned FSMap version 37 2024-03-19T01:57:54.523 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.518+0000 7f1a91a50740 1 -- 172.21.15.99:0/229988730 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x5603aeaaa210 con 0x5603aee31af0 2024-03-19T01:57:54.525 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.518+0000 7f1a67fff640 1 -- 172.21.15.99:0/229988730 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f1a80003e50 con 0x5603aee31af0 2024-03-19T01:57:54.525 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.518+0000 7f1a67fff640 1 -- 172.21.15.99:0/229988730 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7f1a740035f0 con 0x5603aee31af0 2024-03-19T01:57:54.525 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.519+0000 7f1a91a50740 10 client.16692 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T01:57:54.525 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.519+0000 7f1a91a50740 10 client.16692 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:57:54.525 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.519+0000 7f1a91a50740 20 client.16692 populate_metadata read hostname 'smithi099' 2024-03-19T01:57:54.525 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.519+0000 7f1a91a50740 1 --2- 172.21.15.99:0/229988730 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5603aee44150 0x5603aee465a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:54.525 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.519+0000 7f1a91a50740 4 client.16692 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:57:54.525 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.519+0000 7f1a91a50740 1 -- 172.21.15.99:0/229988730 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5603aeda3120 con 0x5603aee44150 2024-03-19T01:57:54.525 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.519+0000 7f1a8d405640 1 --2- 172.21.15.99:0/229988730 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5603aee44150 0x5603aee465a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:54.525 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.519+0000 7f1a67fff640 10 client.16692 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:57:54.525 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.520+0000 7f1a8d405640 1 --2- 172.21.15.99:0/229988730 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5603aee44150 0x5603aee465a0 crc :-1 s=READY pgs=179 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=ce2eb729813e927a server_cookie=1495ca2599bdf61b in_seq=0 out_seq=0 2024-03-19T01:57:54.525 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.521+0000 7f1a67fff640 1 -- 172.21.15.99:0/229988730 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x5603aeda3120 con 0x5603aee44150 2024-03-19T01:57:54.526 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.521+0000 7f1a67fff640 10 client.16692 handle_command_reply: tid=0 2024-03-19T01:57:54.526 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.521+0000 7f1a91a50740 1 client.16692 shutdown 2024-03-19T01:57:54.526 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.521+0000 7f1a91a50740 1 -- 172.21.15.99:0/229988730 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5603aee31af0 msgr2=0x5603aee3b5b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:57:54.526 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.521+0000 7f1a91a50740 1 --2- 172.21.15.99:0/229988730 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5603aee31af0 0x5603aee3b5b0 secure :-1 s=READY pgs=672 cs=0 l=1 rev1=1 crypto rx=0x7f1a74036040 tx=0x7f1a74004270 comp rx=0 tx=0).stop 2024-03-19T01:57:54.526 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.521+0000 7f1a91a50740 1 -- 172.21.15.99:0/229988730 shutdown_connections 2024-03-19T01:57:54.526 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.521+0000 7f1a91a50740 1 --2- 172.21.15.99:0/229988730 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5603aee44150 0x5603aee465a0 crc :-1 s=READY pgs=179 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:54.526 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.521+0000 7f1a91a50740 1 --2- 172.21.15.99:0/229988730 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5603aee36c60 0x5603aee425f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:54.526 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.521+0000 7f1a91a50740 1 --2- 172.21.15.99:0/229988730 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5603aee31af0 0x5603aee3b5b0 unknown :-1 s=CLOSED pgs=672 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:54.526 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.521+0000 7f1a91a50740 1 --2- 172.21.15.99:0/229988730 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5603aee31100 0x5603aee3b070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:54.526 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.521+0000 7f1a67fff640 0 client.16692 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:57:54.526 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.521+0000 7f1a91a50740 1 -- 172.21.15.99:0/229988730 >> 172.21.15.99:0/229988730 conn(0x5603aee2c670 msgr2=0x5603aee2d8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:57:54.526 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.521+0000 7f1a91a50740 1 -- 172.21.15.99:0/229988730 shutdown_connections 2024-03-19T01:57:54.526 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.521+0000 7f1a91a50740 1 -- 172.21.15.99:0/229988730 wait complete. 2024-03-19T01:57:54.526 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.521+0000 7f1a91a50740 20 client.16692 trim_cache size 0 max 16384 2024-03-19T01:57:54.543 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.538+0000 7f1a91a50740 1 Processor -- start 2024-03-19T01:57:54.543 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.538+0000 7f1a91a50740 1 -- start start 2024-03-19T01:57:54.543 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.538+0000 7f1a91a50740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5603aee31100 0x5603aee2b220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:54.543 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.538+0000 7f1a91a50740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5603aee31af0 0x5603aee2b760 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:54.543 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.538+0000 7f1a91a50740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5603aee36c60 0x5603aee321e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:54.544 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.538+0000 7f1a91a50740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f1a881a9500 con 0x5603aee31af0 2024-03-19T01:57:54.544 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.538+0000 7f1a91a50740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f1a881a6dd0 con 0x5603aee31100 2024-03-19T01:57:54.544 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.538+0000 7f1a91a50740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f1a881a4c20 con 0x5603aee36c60 2024-03-19T01:57:54.544 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.538+0000 7f1a87fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5603aee31af0 0x5603aee2b760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:54.544 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.538+0000 7f1a87fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5603aee31af0 0x5603aee2b760 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:60374/0 (socket says 172.21.15.99:60374) 2024-03-19T01:57:54.544 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.538+0000 7f1a87fff640 1 -- 172.21.15.99:0/2504095939 learned_addr learned my addr 172.21.15.99:0/2504095939 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:57:54.544 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.538+0000 7f1a8d405640 1 --2- 172.21.15.99:0/2504095939 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5603aee31100 0x5603aee2b220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:54.544 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.538+0000 7f1a8d405640 1 --2- 172.21.15.99:0/2504095939 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5603aee36c60 0x5603aee321e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:54.544 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.539+0000 7f1a8d405640 1 -- 172.21.15.99:0/2504095939 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5603aee36c60 msgr2=0x5603aee321e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:57:54.544 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.539+0000 7f1a8d405640 1 --2- 172.21.15.99:0/2504095939 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5603aee36c60 0x5603aee321e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:54.544 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.539+0000 7f1a8d405640 1 -- 172.21.15.99:0/2504095939 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5603aee31af0 msgr2=0x5603aee2b760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:57:54.544 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.539+0000 7f1a8d405640 1 --2- 172.21.15.99:0/2504095939 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5603aee31af0 0x5603aee2b760 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:54.544 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.539+0000 7f1a8d405640 1 -- 172.21.15.99:0/2504095939 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5603aeb76620 con 0x5603aee31100 2024-03-19T01:57:54.544 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.539+0000 7f1a87fff640 1 --2- 172.21.15.99:0/2504095939 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5603aee31af0 0x5603aee2b760 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:57:54.545 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.539+0000 7f1a8d405640 1 --2- 172.21.15.99:0/2504095939 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5603aee31100 0x5603aee2b220 secure :-1 s=READY pgs=180 cs=0 l=1 rev1=1 crypto rx=0x7f1a80005fd0 tx=0x7f1a80009f40 comp rx=0 tx=0).ready entity=mon.2 client_cookie=2a68df27ec863f8 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:57:54.545 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.539+0000 7f1a727fc640 1 -- 172.21.15.99:0/2504095939 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1a8000f910 con 0x5603aee31100 2024-03-19T01:57:54.545 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.539+0000 7f1a727fc640 1 -- 172.21.15.99:0/2504095939 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f1a8000fbd0 con 0x5603aee31100 2024-03-19T01:57:54.545 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.539+0000 7f1a727fc640 1 -- 172.21.15.99:0/2504095939 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1a800050f0 con 0x5603aee31100 2024-03-19T01:57:54.545 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.540+0000 7f1a91a50740 1 -- 172.21.15.99:0/2504095939 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5603aee31100 msgr2=0x5603aee2b220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:57:54.545 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.540+0000 7f1a91a50740 1 --2- 172.21.15.99:0/2504095939 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5603aee31100 0x5603aee2b220 secure :-1 s=READY pgs=180 cs=0 l=1 rev1=1 crypto rx=0x7f1a80005fd0 tx=0x7f1a80009f40 comp rx=0 tx=0).stop 2024-03-19T01:57:54.545 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.540+0000 7f1a91a50740 1 -- 172.21.15.99:0/2504095939 shutdown_connections 2024-03-19T01:57:54.545 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.540+0000 7f1a91a50740 1 --2- 172.21.15.99:0/2504095939 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5603aee36c60 0x5603aee321e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:54.545 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.540+0000 7f1a91a50740 1 --2- 172.21.15.99:0/2504095939 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5603aee31af0 0x5603aee2b760 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:54.545 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.540+0000 7f1a91a50740 1 --2- 172.21.15.99:0/2504095939 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5603aee31100 0x5603aee2b220 unknown :-1 s=CLOSED pgs=180 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:54.545 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.540+0000 7f1a91a50740 1 -- 172.21.15.99:0/2504095939 >> 172.21.15.99:0/2504095939 conn(0x5603aee2c670 msgr2=0x5603aee2ca50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:57:54.545 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.540+0000 7f1a91a50740 1 -- 172.21.15.99:0/2504095939 shutdown_connections 2024-03-19T01:57:54.545 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.540+0000 7f1a91a50740 1 -- 172.21.15.99:0/2504095939 wait complete. 2024-03-19T01:57:54.545 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.540+0000 7f1a91a50740 1 -- start start 2024-03-19T01:57:54.545 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.540+0000 7f1a91a50740 1 Processor -- start 2024-03-19T01:57:54.546 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.541+0000 7f1a91a50740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5603aee31100 0x5603aee4e2d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:54.546 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.541+0000 7f1a91a50740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5603aee31af0 0x5603aee4e810 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:54.546 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.541+0000 7f1a91a50740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5603aee36c60 0x5603aeeab450 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:54.546 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.541+0000 7f1a91a50740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f1a881a5290 con 0x5603aee31af0 2024-03-19T01:57:54.546 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.541+0000 7f1a91a50740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f1a881a6700 con 0x5603aee36c60 2024-03-19T01:57:54.546 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.541+0000 7f1a91a50740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f1a881a5960 con 0x5603aee31100 2024-03-19T01:57:54.546 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.541+0000 7f1a91a50740 10 client.0 _ll_register_callbacks cb 0x7f1a74009780 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:57:54.546 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.541+0000 7f1a87fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5603aee31af0 0x5603aee4e810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:54.546 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.541+0000 7f1a87fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5603aee31af0 0x5603aee4e810 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:60386/0 (socket says 172.21.15.99:60386) 2024-03-19T01:57:54.546 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.541+0000 7f1a87fff640 1 -- 172.21.15.99:0/587067082 learned_addr learned my addr 172.21.15.99:0/587067082 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:57:54.546 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.541+0000 7f1a8d405640 1 --2- 172.21.15.99:0/587067082 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5603aee31100 0x5603aee4e2d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:54.546 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.541+0000 7f1a8d405640 1 --2- 172.21.15.99:0/587067082 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5603aee36c60 0x5603aeeab450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:54.546 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.541+0000 7f1a87fff640 1 -- 172.21.15.99:0/587067082 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5603aee31100 msgr2=0x5603aee4e2d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:57:54.547 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.541+0000 7f1a87fff640 1 --2- 172.21.15.99:0/587067082 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5603aee31100 0x5603aee4e2d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:54.547 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.541+0000 7f1a87fff640 1 -- 172.21.15.99:0/587067082 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5603aee36c60 msgr2=0x5603aeeab450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:57:54.547 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.541+0000 7f1a87fff640 1 --2- 172.21.15.99:0/587067082 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5603aee36c60 0x5603aeeab450 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:54.547 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.541+0000 7f1a87fff640 1 -- 172.21.15.99:0/587067082 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5603aeeaba40 con 0x5603aee31af0 2024-03-19T01:57:54.547 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.541+0000 7f1a87fff640 1 --2- 172.21.15.99:0/587067082 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5603aee31af0 0x5603aee4e810 secure :-1 s=READY pgs=673 cs=0 l=1 rev1=1 crypto rx=0x7f1a74000c00 tx=0x7f1a74003eb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d7f77ce612773a80 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:57:54.547 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.541+0000 7f1a67fff640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:57:54.547 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.541+0000 7f1a67fff640 1 -- 172.21.15.99:0/587067082 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1a74051030 con 0x5603aee31af0 2024-03-19T01:57:54.547 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.541+0000 7f1a67fff640 1 -- 172.21.15.99:0/587067082 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f1a740046c0 con 0x5603aee31af0 2024-03-19T01:57:54.547 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.541+0000 7f1a91a50740 1 -- 172.21.15.99:0/587067082 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f1a8000fbd0 con 0x5603aee31af0 2024-03-19T01:57:54.547 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.541+0000 7f1a67fff640 1 -- 172.21.15.99:0/587067082 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7f1a74003800 con 0x5603aee31af0 2024-03-19T01:57:54.547 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.541+0000 7f1a91a50740 10 client.16698 fetch_fsmap learned FSMap version 37 2024-03-19T01:57:54.547 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.541+0000 7f1a91a50740 1 -- 172.21.15.99:0/587067082 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x5603aeeacdc0 con 0x5603aee31af0 2024-03-19T01:57:54.548 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.542+0000 7f1a67fff640 1 -- 172.21.15.99:0/587067082 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f1a8000fbd0 con 0x5603aee31af0 2024-03-19T01:57:54.548 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.542+0000 7f1a67fff640 1 -- 172.21.15.99:0/587067082 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7f1a7405e020 con 0x5603aee31af0 2024-03-19T01:57:54.548 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.542+0000 7f1a91a50740 10 client.16698 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T01:57:54.548 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.542+0000 7f1a91a50740 10 client.16698 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:57:54.548 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.542+0000 7f1a91a50740 20 client.16698 populate_metadata read hostname 'smithi099' 2024-03-19T01:57:54.548 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.542+0000 7f1a91a50740 1 --2- 172.21.15.99:0/587067082 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5603aeead720 0x5603aeeafb40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:57:54.548 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.542+0000 7f1a91a50740 4 client.16698 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:57:54.548 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.542+0000 7f1a91a50740 1 -- 172.21.15.99:0/587067082 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x5603aed9d470 con 0x5603aeead720 2024-03-19T01:57:54.548 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.542+0000 7f1a8d405640 1 --2- 172.21.15.99:0/587067082 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5603aeead720 0x5603aeeafb40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:57:54.548 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.542+0000 7f1a67fff640 10 client.16698 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:57:54.548 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:54.542+0000 7f1a8d405640 1 --2- 172.21.15.99:0/587067082 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5603aeead720 0x5603aeeafb40 crc :-1 s=READY pgs=180 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=a71cb41efe7aa691 server_cookie=e781c3bed2b0d02b in_seq=0 out_seq=0 2024-03-19T01:57:55.065 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:55.064+0000 7f1a67fff640 1 -- 172.21.15.99:0/587067082 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x5603aed9d470 con 0x5603aeead720 2024-03-19T01:57:55.065 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:55.064+0000 7f1a67fff640 10 client.16698 handle_command_reply: tid=0 2024-03-19T01:57:55.065 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:55.064+0000 7f1a91a50740 1 client.16698 shutdown 2024-03-19T01:57:55.065 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:55.065+0000 7f1a91a50740 1 -- 172.21.15.99:0/587067082 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5603aee31af0 msgr2=0x5603aee4e810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:57:55.065 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:55.065+0000 7f1a91a50740 1 --2- 172.21.15.99:0/587067082 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5603aee31af0 0x5603aee4e810 secure :-1 s=READY pgs=673 cs=0 l=1 rev1=1 crypto rx=0x7f1a74000c00 tx=0x7f1a74003eb0 comp rx=0 tx=0).stop 2024-03-19T01:57:55.066 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:55.065+0000 7f1a91a50740 1 -- 172.21.15.99:0/587067082 shutdown_connections 2024-03-19T01:57:55.066 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:55.065+0000 7f1a91a50740 1 --2- 172.21.15.99:0/587067082 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5603aeead720 0x5603aeeafb40 crc :-1 s=READY pgs=180 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:55.066 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:55.065+0000 7f1a91a50740 1 --2- 172.21.15.99:0/587067082 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5603aee36c60 0x5603aeeab450 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:55.066 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:55.065+0000 7f1a91a50740 1 --2- 172.21.15.99:0/587067082 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5603aee31af0 0x5603aee4e810 unknown :-1 s=CLOSED pgs=673 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:55.066 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:55.065+0000 7f1a67fff640 0 client.16698 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:57:55.066 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:55.065+0000 7f1a91a50740 1 --2- 172.21.15.99:0/587067082 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5603aee31100 0x5603aee4e2d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:55.066 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:55.065+0000 7f1a91a50740 1 -- 172.21.15.99:0/587067082 >> 172.21.15.99:0/587067082 conn(0x5603aee2c670 msgr2=0x5603aee2b220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:57:55.066 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:55.065+0000 7f1a91a50740 1 -- 172.21.15.99:0/587067082 shutdown_connections 2024-03-19T01:57:55.066 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:55.065+0000 7f1a91a50740 1 -- 172.21.15.99:0/587067082 wait complete. 2024-03-19T01:57:55.066 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:55.065+0000 7f1a91a50740 20 client.16698 trim_cache size 0 max 16384 2024-03-19T01:57:55.066 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:57:55.067 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:57:55.067 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:57:55.067 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:57:55.067 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:55.066+0000 7f1a8ee68640 1 -- 172.21.15.99:0/3753568221 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f1a780a28a0 msgr2=0x7f1a780a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:57:55.068 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:55.066+0000 7f1a8ee68640 1 --2- 172.21.15.99:0/3753568221 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f1a780a28a0 0x7f1a780a4d60 secure :-1 s=READY pgs=348 cs=0 l=1 rev1=1 crypto rx=0x7f1a7c097c10 tx=0x7f1a7c0c6200 comp rx=0 tx=0).stop 2024-03-19T01:57:55.068 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:55.066+0000 7f1a8ee68640 1 -- 172.21.15.99:0/3753568221 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1a88100710 msgr2=0x7f1a88069a10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:57:55.068 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:55.066+0000 7f1a8ee68640 1 --2- 172.21.15.99:0/3753568221 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1a88100710 0x7f1a88069a10 secure :-1 s=READY pgs=670 cs=0 l=1 rev1=1 crypto rx=0x7f1a7c00e9c0 tx=0x7f1a7c00ee80 comp rx=0 tx=0).stop 2024-03-19T01:57:55.068 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:55.066+0000 7f1a8ee68640 1 -- 172.21.15.99:0/3753568221 shutdown_connections 2024-03-19T01:57:55.068 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:55.066+0000 7f1a8ee68640 1 --2- 172.21.15.99:0/3753568221 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f1a780a28a0 0x7f1a780a4d60 unknown :-1 s=CLOSED pgs=348 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:55.068 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:55.066+0000 7f1a8ee68640 1 --2- 172.21.15.99:0/3753568221 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1a881022c0 0x7f1a8819ede0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:55.068 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:55.066+0000 7f1a8ee68640 1 --2- 172.21.15.99:0/3753568221 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1a88101900 0x7f1a8819c7c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:55.068 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:55.066+0000 7f1a8ee68640 1 --2- 172.21.15.99:0/3753568221 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1a88100710 0x7f1a88069a10 unknown :-1 s=CLOSED pgs=670 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:57:55.068 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:55.066+0000 7f1a8ee68640 1 -- 172.21.15.99:0/3753568221 >> 172.21.15.99:0/3753568221 conn(0x7f1a880fbc80 msgr2=0x7f1a88104b30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:57:55.068 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:55.066+0000 7f1a8ee68640 1 -- 172.21.15.99:0/3753568221 shutdown_connections 2024-03-19T01:57:55.068 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:57:55.066+0000 7f1a8ee68640 1 -- 172.21.15.99:0/3753568221 wait complete. 2024-03-19T01:57:55.107 INFO:tasks.workunit.client.0.smithi099.stdout: 10 116091 23.04 MB/sec execute 144 sec latency 325.386 ms 2024-03-19T01:57:55.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:57:55 smithi112 ceph-mon[66819]: pgmap v856: 97 pgs: 97 active+clean; 568 MiB data, 6.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.6 MiB/s rd, 6.0 MiB/s wr, 483 op/s 2024-03-19T01:57:55.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:57:55 smithi115 ceph-mon[67332]: pgmap v856: 97 pgs: 97 active+clean; 568 MiB data, 6.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.6 MiB/s rd, 6.0 MiB/s wr, 483 op/s 2024-03-19T01:57:55.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:57:55 smithi099 ceph-mon[62951]: pgmap v856: 97 pgs: 97 active+clean; 568 MiB data, 6.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.6 MiB/s rd, 6.0 MiB/s wr, 483 op/s 2024-03-19T01:57:56.106 INFO:tasks.workunit.client.0.smithi099.stdout: 10 116488 23.02 MB/sec execute 145 sec latency 129.847 ms 2024-03-19T01:57:57.107 INFO:tasks.workunit.client.0.smithi099.stdout: 10 116964 22.97 MB/sec execute 146 sec latency 75.621 ms 2024-03-19T01:57:57.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:57:57 smithi112 ceph-mon[66819]: pgmap v857: 97 pgs: 97 active+clean; 562 MiB data, 6.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.9 MiB/s rd, 6.7 MiB/s wr, 557 op/s 2024-03-19T01:57:57.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:57:57 smithi115 ceph-mon[67332]: pgmap v857: 97 pgs: 97 active+clean; 562 MiB data, 6.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.9 MiB/s rd, 6.7 MiB/s wr, 557 op/s 2024-03-19T01:57:57.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:57:57 smithi099 ceph-mon[62951]: pgmap v857: 97 pgs: 97 active+clean; 562 MiB data, 6.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.9 MiB/s rd, 6.7 MiB/s wr, 557 op/s 2024-03-19T01:57:58.107 INFO:tasks.workunit.client.0.smithi099.stdout: 10 117480 23.02 MB/sec execute 147 sec latency 65.953 ms 2024-03-19T01:57:59.107 INFO:tasks.workunit.client.0.smithi099.stdout: 10 117923 23.03 MB/sec execute 148 sec latency 150.686 ms 2024-03-19T01:57:59.356 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:57:59 smithi115 ceph-mon[67332]: pgmap v858: 97 pgs: 97 active+clean; 564 MiB data, 6.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 5.5 MiB/s wr, 497 op/s 2024-03-19T01:57:59.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:57:59 smithi112 ceph-mon[66819]: pgmap v858: 97 pgs: 97 active+clean; 564 MiB data, 6.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 5.5 MiB/s wr, 497 op/s 2024-03-19T01:57:59.593 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:57:59 smithi099 ceph-mon[62951]: pgmap v858: 97 pgs: 97 active+clean; 564 MiB data, 6.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 5.5 MiB/s wr, 497 op/s 2024-03-19T01:58:00.107 INFO:tasks.workunit.client.0.smithi099.stdout: 10 118420 23.06 MB/sec execute 149 sec latency 244.911 ms 2024-03-19T01:58:01.107 INFO:tasks.workunit.client.0.smithi099.stdout: 10 118913 23.08 MB/sec execute 150 sec latency 104.284 ms 2024-03-19T01:58:01.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:58:01 smithi112 ceph-mon[66819]: pgmap v859: 97 pgs: 97 active+clean; 568 MiB data, 6.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.9 MiB/s rd, 6.0 MiB/s wr, 565 op/s 2024-03-19T01:58:01.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:58:01 smithi115 ceph-mon[67332]: pgmap v859: 97 pgs: 97 active+clean; 568 MiB data, 6.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.9 MiB/s rd, 6.0 MiB/s wr, 565 op/s 2024-03-19T01:58:01.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:58:01 smithi099 ceph-mon[62951]: pgmap v859: 97 pgs: 97 active+clean; 568 MiB data, 6.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.9 MiB/s rd, 6.0 MiB/s wr, 565 op/s 2024-03-19T01:58:02.107 INFO:tasks.workunit.client.0.smithi099.stdout: 10 119422 23.14 MB/sec execute 151 sec latency 85.103 ms 2024-03-19T01:58:03.107 INFO:tasks.workunit.client.0.smithi099.stdout: 10 119896 23.20 MB/sec execute 152 sec latency 108.275 ms 2024-03-19T01:58:03.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:58:03 smithi112 ceph-mon[66819]: pgmap v860: 97 pgs: 97 active+clean; 571 MiB data, 6.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.7 MiB/s rd, 6.4 MiB/s wr, 562 op/s 2024-03-19T01:58:03.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:58:03 smithi115 ceph-mon[67332]: pgmap v860: 97 pgs: 97 active+clean; 571 MiB data, 6.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.7 MiB/s rd, 6.4 MiB/s wr, 562 op/s 2024-03-19T01:58:03.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:58:03 smithi099 ceph-mon[62951]: pgmap v860: 97 pgs: 97 active+clean; 571 MiB data, 6.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.7 MiB/s rd, 6.4 MiB/s wr, 562 op/s 2024-03-19T01:58:04.107 INFO:tasks.workunit.client.0.smithi099.stdout: 10 120298 23.18 MB/sec execute 153 sec latency 76.987 ms 2024-03-19T01:58:05.107 INFO:tasks.workunit.client.0.smithi099.stdout: 10 120582 23.12 MB/sec execute 154 sec latency 292.269 ms 2024-03-19T01:58:05.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:58:05 smithi112 ceph-mon[66819]: pgmap v861: 97 pgs: 97 active+clean; 567 MiB data, 6.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 6.3 MiB/s wr, 570 op/s 2024-03-19T01:58:05.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:58:05 smithi115 ceph-mon[67332]: pgmap v861: 97 pgs: 97 active+clean; 567 MiB data, 6.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 6.3 MiB/s wr, 570 op/s 2024-03-19T01:58:05.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:58:05 smithi099 ceph-mon[62951]: pgmap v861: 97 pgs: 97 active+clean; 567 MiB data, 6.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 6.3 MiB/s wr, 570 op/s 2024-03-19T01:58:06.107 INFO:tasks.workunit.client.0.smithi099.stdout: 10 120932 23.09 MB/sec execute 155 sec latency 82.823 ms 2024-03-19T01:58:07.107 INFO:tasks.workunit.client.0.smithi099.stdout: 10 121432 23.07 MB/sec execute 156 sec latency 62.955 ms 2024-03-19T01:58:07.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:58:07 smithi112 ceph-mon[66819]: pgmap v862: 97 pgs: 97 active+clean; 555 MiB data, 6.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.3 MiB/s rd, 6.5 MiB/s wr, 552 op/s 2024-03-19T01:58:07.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:58:07 smithi115 ceph-mon[67332]: pgmap v862: 97 pgs: 97 active+clean; 555 MiB data, 6.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.3 MiB/s rd, 6.5 MiB/s wr, 552 op/s 2024-03-19T01:58:07.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:58:07 smithi099 ceph-mon[62951]: pgmap v862: 97 pgs: 97 active+clean; 555 MiB data, 6.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.3 MiB/s rd, 6.5 MiB/s wr, 552 op/s 2024-03-19T01:58:08.107 INFO:tasks.workunit.client.0.smithi099.stdout: 10 121966 23.11 MB/sec execute 157 sec latency 76.957 ms 2024-03-19T01:58:08.177 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.177+0000 7fb23f0d1640 1 -- 172.21.15.99:0/900920860 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb238102a60 msgr2=0x7fb238102ee0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:58:08.178 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.177+0000 7fb23f0d1640 1 --2- 172.21.15.99:0/900920860 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb238102a60 0x7fb238102ee0 secure :-1 s=READY pgs=674 cs=0 l=1 rev1=1 crypto rx=0x7fb224009900 tx=0x7fb22402bc30 comp rx=0 tx=0).stop 2024-03-19T01:58:08.178 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.177+0000 7fb23f0d1640 1 -- 172.21.15.99:0/900920860 shutdown_connections 2024-03-19T01:58:08.178 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.177+0000 7fb23f0d1640 1 --2- 172.21.15.99:0/900920860 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb238103420 0x7fb238109cb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:08.178 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.178+0000 7fb23f0d1640 1 --2- 172.21.15.99:0/900920860 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb238102a60 0x7fb238102ee0 unknown :-1 s=CLOSED pgs=674 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:08.178 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.178+0000 7fb23f0d1640 1 --2- 172.21.15.99:0/900920860 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb238101870 0x7fb238101c90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:08.178 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.178+0000 7fb23f0d1640 1 -- 172.21.15.99:0/900920860 >> 172.21.15.99:0/900920860 conn(0x7fb2380fcde0 msgr2=0x7fb2380ff220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:58:08.178 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.178+0000 7fb23f0d1640 1 -- 172.21.15.99:0/900920860 shutdown_connections 2024-03-19T01:58:08.178 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.178+0000 7fb23f0d1640 1 -- 172.21.15.99:0/900920860 wait complete. 2024-03-19T01:58:08.180 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.178+0000 7fb23f0d1640 1 Processor -- start 2024-03-19T01:58:08.181 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.178+0000 7fb23f0d1640 1 -- start start 2024-03-19T01:58:08.181 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.179+0000 7fb23f0d1640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb238101870 0x7fb238197ac0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:08.181 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.179+0000 7fb23f0d1640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb238102a60 0x7fb238198000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:08.181 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.179+0000 7fb23f0d1640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb238103420 0x7fb23819f070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:08.181 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.179+0000 7fb23f0d1640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fb23810d4d0 con 0x7fb238101870 2024-03-19T01:58:08.181 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.179+0000 7fb23f0d1640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fb23810d350 con 0x7fb238102a60 2024-03-19T01:58:08.181 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.179+0000 7fb23f0d1640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fb23810d650 con 0x7fb238103420 2024-03-19T01:58:08.181 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.179+0000 7fb237fff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb238102a60 0x7fb238198000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:08.181 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.179+0000 7fb23ce6d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb238101870 0x7fb238197ac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:08.181 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.179+0000 7fb237fff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb238102a60 0x7fb238198000 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:44206/0 (socket says 172.21.15.99:44206) 2024-03-19T01:58:08.181 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.179+0000 7fb237fff640 1 -- 172.21.15.99:0/769758192 learned_addr learned my addr 172.21.15.99:0/769758192 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:58:08.181 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.179+0000 7fb23d66e640 1 --2- 172.21.15.99:0/769758192 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb238103420 0x7fb23819f070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:08.181 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.179+0000 7fb237fff640 1 -- 172.21.15.99:0/769758192 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb238103420 msgr2=0x7fb23819f070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:58:08.182 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.179+0000 7fb237fff640 1 --2- 172.21.15.99:0/769758192 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb238103420 0x7fb23819f070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:08.182 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.179+0000 7fb237fff640 1 -- 172.21.15.99:0/769758192 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb238101870 msgr2=0x7fb238197ac0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:58:08.182 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.179+0000 7fb237fff640 1 --2- 172.21.15.99:0/769758192 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb238101870 0x7fb238197ac0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:08.182 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.179+0000 7fb237fff640 1 -- 172.21.15.99:0/769758192 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb23819f6e0 con 0x7fb238102a60 2024-03-19T01:58:08.182 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.179+0000 7fb23ce6d640 1 --2- 172.21.15.99:0/769758192 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb238101870 0x7fb238197ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:58:08.182 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.179+0000 7fb237fff640 1 --2- 172.21.15.99:0/769758192 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb238102a60 0x7fb238198000 secure :-1 s=READY pgs=181 cs=0 l=1 rev1=1 crypto rx=0x7fb224009230 tx=0x7fb22402c650 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:58:08.182 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.180+0000 7fb235ffb640 1 -- 172.21.15.99:0/769758192 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb224041070 con 0x7fb238102a60 2024-03-19T01:58:08.182 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.180+0000 7fb235ffb640 1 -- 172.21.15.99:0/769758192 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fb224004450 con 0x7fb238102a60 2024-03-19T01:58:08.182 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.180+0000 7fb23f0d1640 1 -- 172.21.15.99:0/769758192 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb23819f970 con 0x7fb238102a60 2024-03-19T01:58:08.182 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.180+0000 7fb235ffb640 1 -- 172.21.15.99:0/769758192 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb224003db0 con 0x7fb238102a60 2024-03-19T01:58:08.182 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.180+0000 7fb23f0d1640 1 -- 172.21.15.99:0/769758192 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fb23819fe60 con 0x7fb238102a60 2024-03-19T01:58:08.184 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.183+0000 7fb235ffb640 1 -- 172.21.15.99:0/769758192 <== mon.2 v2:172.21.15.112:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7fb224044070 con 0x7fb238102a60 2024-03-19T01:58:08.184 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.183+0000 7fb241d99740 1 Processor -- start 2024-03-19T01:58:08.185 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.183+0000 7fb241d99740 1 -- start start 2024-03-19T01:58:08.185 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.183+0000 7fb241d99740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564e2e5e6100 0x564e2e5e6520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:08.185 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.183+0000 7fb235ffb640 1 --2- 172.21.15.99:0/769758192 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb2280a28a0 0x7fb2280a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:08.185 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.183+0000 7fb241d99740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564e2e5e6af0 0x564e2e5eb5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:08.185 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.183+0000 7fb241d99740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564e2e5ebc60 0x564e2e5ee0f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:08.185 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.183+0000 7fb241d99740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x564e2e54abe0 con 0x564e2e5e6af0 2024-03-19T01:58:08.185 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.183+0000 7fb241d99740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x564e2e4f3960 con 0x564e2e5ebc60 2024-03-19T01:58:08.185 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.183+0000 7fb241d99740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x564e2e546c80 con 0x564e2e5e6100 2024-03-19T01:58:08.185 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.184+0000 7fb23d66e640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564e2e5e6100 0x564e2e5e6520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:08.185 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.184+0000 7fb235ffb640 1 -- 172.21.15.99:0/769758192 <== mon.2 v2:172.21.15.112:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fb2240f0670 con 0x7fb238102a60 2024-03-19T01:58:08.185 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.184+0000 7fb23d66e640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564e2e5e6100 0x564e2e5e6520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:55118/0 (socket says 172.21.15.99:55118) 2024-03-19T01:58:08.185 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.184+0000 7fb23d66e640 1 -- 172.21.15.99:0/1245142428 learned_addr learned my addr 172.21.15.99:0/1245142428 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:58:08.185 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.184+0000 7fb23d66e640 1 --2- 172.21.15.99:0/1245142428 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564e2e5ebc60 0x564e2e5ee0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:08.186 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.184+0000 7fb23ce6d640 1 --2- 172.21.15.99:0/769758192 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb2280a28a0 0x7fb2280a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:08.186 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.184+0000 7fb237fff640 1 --2- 172.21.15.99:0/1245142428 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564e2e5e6af0 0x564e2e5eb5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:08.186 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.184+0000 7fb23d66e640 1 -- 172.21.15.99:0/1245142428 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564e2e5ebc60 msgr2=0x564e2e5ee0f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:58:08.186 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.184+0000 7fb23d66e640 1 --2- 172.21.15.99:0/1245142428 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564e2e5ebc60 0x564e2e5ee0f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:08.186 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.184+0000 7fb23d66e640 1 -- 172.21.15.99:0/1245142428 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564e2e5e6af0 msgr2=0x564e2e5eb5f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:58:08.186 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.184+0000 7fb23d66e640 1 --2- 172.21.15.99:0/1245142428 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564e2e5e6af0 0x564e2e5eb5f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:08.186 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.184+0000 7fb23d66e640 1 -- 172.21.15.99:0/1245142428 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x564e2e4ead20 con 0x564e2e5e6100 2024-03-19T01:58:08.186 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.184+0000 7fb23d66e640 1 --2- 172.21.15.99:0/1245142428 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564e2e5e6100 0x564e2e5e6520 secure :-1 s=READY pgs=121 cs=0 l=1 rev1=1 crypto rx=0x7fb230009b80 tx=0x7fb230009af0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=2a07f68439737c91 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:58:08.186 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.184+0000 7fb222ffd640 1 -- 172.21.15.99:0/1245142428 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb23000fc20 con 0x564e2e5e6100 2024-03-19T01:58:08.186 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.185+0000 7fb222ffd640 1 -- 172.21.15.99:0/1245142428 <== mon.1 v2:172.21.15.115:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fb23000fdc0 con 0x564e2e5e6100 2024-03-19T01:58:08.186 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.185+0000 7fb222ffd640 1 -- 172.21.15.99:0/1245142428 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb23000e3b0 con 0x564e2e5e6100 2024-03-19T01:58:08.189 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.185+0000 7fb241d99740 1 -- 172.21.15.99:0/1245142428 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564e2e5e6100 msgr2=0x564e2e5e6520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:58:08.189 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.185+0000 7fb23ce6d640 1 --2- 172.21.15.99:0/769758192 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb2280a28a0 0x7fb2280a4d60 secure :-1 s=READY pgs=349 cs=0 l=1 rev1=1 crypto rx=0x7fb2381028c0 tx=0x7fb22c005f90 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:58:08.189 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.185+0000 7fb241d99740 1 --2- 172.21.15.99:0/1245142428 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564e2e5e6100 0x564e2e5e6520 secure :-1 s=READY pgs=121 cs=0 l=1 rev1=1 crypto rx=0x7fb230009b80 tx=0x7fb230009af0 comp rx=0 tx=0).stop 2024-03-19T01:58:08.189 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.186+0000 7fb241d99740 1 -- 172.21.15.99:0/1245142428 shutdown_connections 2024-03-19T01:58:08.190 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.186+0000 7fb241d99740 1 --2- 172.21.15.99:0/1245142428 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564e2e5ebc60 0x564e2e5ee0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:08.190 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.186+0000 7fb241d99740 1 --2- 172.21.15.99:0/1245142428 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564e2e5e6af0 0x564e2e5eb5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:08.190 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.186+0000 7fb241d99740 1 --2- 172.21.15.99:0/1245142428 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564e2e5e6100 0x564e2e5e6520 unknown :-1 s=CLOSED pgs=121 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:08.190 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.186+0000 7fb241d99740 1 -- 172.21.15.99:0/1245142428 >> 172.21.15.99:0/1245142428 conn(0x564e2e5e1670 msgr2=0x564e2e5e1a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:58:08.190 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.186+0000 7fb241d99740 1 -- 172.21.15.99:0/1245142428 shutdown_connections 2024-03-19T01:58:08.190 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.186+0000 7fb241d99740 1 -- 172.21.15.99:0/1245142428 wait complete. 2024-03-19T01:58:08.190 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.186+0000 7fb241d99740 1 -- start start 2024-03-19T01:58:08.190 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.186+0000 7fb241d99740 1 Processor -- start 2024-03-19T01:58:08.190 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.187+0000 7fb241d99740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564e2e5e6100 0x564e2e5f8910 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:08.190 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.187+0000 7fb241d99740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564e2e5e6af0 0x564e2e5f8e50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:08.190 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.187+0000 7fb241d99740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564e2e5ebc60 0x564e2e5ffe90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:08.190 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.187+0000 7fb237fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564e2e5e6af0 0x564e2e5f8e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:08.191 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.187+0000 7fb237fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564e2e5e6af0 0x564e2e5f8e50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:38830/0 (socket says 172.21.15.99:38830) 2024-03-19T01:58:08.191 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.187+0000 7fb23ce6d640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564e2e5e6100 0x564e2e5f8910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:08.191 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.187+0000 7fb23ce6d640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564e2e5e6100 0x564e2e5f8910 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:44226/0 (socket says 172.21.15.99:44226) 2024-03-19T01:58:08.191 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.187+0000 7fb237fff640 1 -- 172.21.15.99:0/2096878892 learned_addr learned my addr 172.21.15.99:0/2096878892 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:58:08.191 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.187+0000 7fb241d99740 1 -- 172.21.15.99:0/2096878892 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x564e2e5eec60 con 0x564e2e5e6af0 2024-03-19T01:58:08.191 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.187+0000 7fb241d99740 1 -- 172.21.15.99:0/2096878892 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x564e2e506db0 con 0x564e2e5e6100 2024-03-19T01:58:08.191 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.187+0000 7fb241d99740 1 -- 172.21.15.99:0/2096878892 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x564e2e5eede0 con 0x564e2e5ebc60 2024-03-19T01:58:08.191 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.187+0000 7fb23d66e640 1 --2- 172.21.15.99:0/2096878892 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564e2e5ebc60 0x564e2e5ffe90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:08.191 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.187+0000 7fb241d99740 10 client.0 _ll_register_callbacks cb 0x564e2e50be70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:58:08.191 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.188+0000 7fb23ce6d640 1 -- 172.21.15.99:0/2096878892 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564e2e5ebc60 msgr2=0x564e2e5ffe90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:58:08.191 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.188+0000 7fb23ce6d640 1 --2- 172.21.15.99:0/2096878892 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564e2e5ebc60 0x564e2e5ffe90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:08.191 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.188+0000 7fb23ce6d640 1 -- 172.21.15.99:0/2096878892 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564e2e5e6af0 msgr2=0x564e2e5f8e50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:58:08.191 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.188+0000 7fb23ce6d640 1 --2- 172.21.15.99:0/2096878892 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564e2e5e6af0 0x564e2e5f8e50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:08.191 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.188+0000 7fb23ce6d640 1 -- 172.21.15.99:0/2096878892 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x564e2e505c60 con 0x564e2e5e6100 2024-03-19T01:58:08.192 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.188+0000 7fb241d99740 1 -- 172.21.15.99:0/2096878892 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fb23000fdc0 con 0x564e2e5e6100 2024-03-19T01:58:08.192 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.188+0000 7fb237fff640 1 --2- 172.21.15.99:0/2096878892 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564e2e5e6af0 0x564e2e5f8e50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:58:08.192 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.188+0000 7fb23ce6d640 1 --2- 172.21.15.99:0/2096878892 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564e2e5e6100 0x564e2e5f8910 secure :-1 s=READY pgs=182 cs=0 l=1 rev1=1 crypto rx=0x564e2e5e25e0 tx=0x7fb22c004730 comp rx=0 tx=0).ready entity=mon.2 client_cookie=ba821bcd7dd8fe67 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:58:08.192 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.188+0000 7fb217fff640 10 client.26483 ms_handle_connect on v2:172.21.15.112:3300/0 2024-03-19T01:58:08.192 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.188+0000 7fb217fff640 1 -- 172.21.15.99:0/2096878892 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb22c005220 con 0x564e2e5e6100 2024-03-19T01:58:08.192 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.188+0000 7fb217fff640 1 -- 172.21.15.99:0/2096878892 <== mon.2 v2:172.21.15.112:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fb22c0053c0 con 0x564e2e5e6100 2024-03-19T01:58:08.192 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.189+0000 7fb217fff640 1 -- 172.21.15.99:0/2096878892 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7fb22c005620 con 0x564e2e5e6100 2024-03-19T01:58:08.192 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.189+0000 7fb241d99740 10 client.26483 fetch_fsmap learned FSMap version 37 2024-03-19T01:58:08.192 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.189+0000 7fb241d99740 1 -- 172.21.15.99:0/2096878892 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({fsmap=37}) -- 0x564e2e25f210 con 0x564e2e5e6100 2024-03-19T01:58:08.192 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.189+0000 7fb217fff640 1 -- 172.21.15.99:0/2096878892 <== mon.2 v2:172.21.15.112:3300/0 4 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7fb22c0047a0 con 0x564e2e5e6100 2024-03-19T01:58:08.192 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.189+0000 7fb241d99740 10 client.26483 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T01:58:08.192 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.189+0000 7fb241d99740 10 client.26483 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:58:08.192 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.189+0000 7fb241d99740 20 client.26483 populate_metadata read hostname 'smithi099' 2024-03-19T01:58:08.192 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.189+0000 7fb241d99740 1 --2- 172.21.15.99:0/2096878892 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x564e2e604610 0x564e2e606a60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:08.193 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.189+0000 7fb241d99740 4 client.26483 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:58:08.193 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.189+0000 7fb241d99740 1 -- 172.21.15.99:0/2096878892 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x564e2e558120 con 0x564e2e604610 2024-03-19T01:58:08.193 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.189+0000 7fb237fff640 1 --2- 172.21.15.99:0/2096878892 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x564e2e604610 0x564e2e606a60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:08.193 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.189+0000 7fb217fff640 10 client.26483 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:58:08.193 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.189+0000 7fb237fff640 1 --2- 172.21.15.99:0/2096878892 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x564e2e604610 0x564e2e606a60 crc :-1 s=READY pgs=181 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=bc76868ea120e0d8 server_cookie=8c90281866327c4f in_seq=0 out_seq=0 2024-03-19T01:58:08.194 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.191+0000 7fb217fff640 1 -- 172.21.15.99:0/2096878892 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x564e2e558120 con 0x564e2e604610 2024-03-19T01:58:08.194 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.191+0000 7fb217fff640 10 client.26483 handle_command_reply: tid=0 2024-03-19T01:58:08.194 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.191+0000 7fb241d99740 1 client.26483 shutdown 2024-03-19T01:58:08.194 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.191+0000 7fb241d99740 1 -- 172.21.15.99:0/2096878892 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564e2e5e6100 msgr2=0x564e2e5f8910 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:58:08.194 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.191+0000 7fb241d99740 1 --2- 172.21.15.99:0/2096878892 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564e2e5e6100 0x564e2e5f8910 secure :-1 s=READY pgs=182 cs=0 l=1 rev1=1 crypto rx=0x564e2e5e25e0 tx=0x7fb22c004730 comp rx=0 tx=0).stop 2024-03-19T01:58:08.194 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.191+0000 7fb241d99740 1 -- 172.21.15.99:0/2096878892 shutdown_connections 2024-03-19T01:58:08.194 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.191+0000 7fb241d99740 1 --2- 172.21.15.99:0/2096878892 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x564e2e604610 0x564e2e606a60 crc :-1 s=READY pgs=181 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:08.194 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.191+0000 7fb217fff640 0 client.26483 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:58:08.195 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.191+0000 7fb241d99740 1 --2- 172.21.15.99:0/2096878892 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564e2e5ebc60 0x564e2e5ffe90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:08.195 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.191+0000 7fb241d99740 1 --2- 172.21.15.99:0/2096878892 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564e2e5e6af0 0x564e2e5f8e50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:08.195 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.191+0000 7fb241d99740 1 --2- 172.21.15.99:0/2096878892 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564e2e5e6100 0x564e2e5f8910 unknown :-1 s=CLOSED pgs=182 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:08.195 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.191+0000 7fb241d99740 1 -- 172.21.15.99:0/2096878892 >> 172.21.15.99:0/2096878892 conn(0x564e2e5e1670 msgr2=0x564e2e5e28c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:58:08.195 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.192+0000 7fb241d99740 1 -- 172.21.15.99:0/2096878892 shutdown_connections 2024-03-19T01:58:08.195 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.192+0000 7fb241d99740 1 -- 172.21.15.99:0/2096878892 wait complete. 2024-03-19T01:58:08.195 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.192+0000 7fb241d99740 20 client.26483 trim_cache size 0 max 16384 2024-03-19T01:58:08.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.209+0000 7fb241d99740 1 Processor -- start 2024-03-19T01:58:08.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.210+0000 7fb241d99740 1 -- start start 2024-03-19T01:58:08.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.210+0000 7fb241d99740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564e2e5e6100 0x564e2e5e0220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:08.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.210+0000 7fb241d99740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564e2e5e6af0 0x564e2e5ffe50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:08.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.210+0000 7fb241d99740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564e2e5ebc60 0x564e2e5e0760 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:08.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.210+0000 7fb241d99740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fb2381a9460 con 0x564e2e5ebc60 2024-03-19T01:58:08.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.210+0000 7fb241d99740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fb2381a6d30 con 0x564e2e5e6af0 2024-03-19T01:58:08.211 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.210+0000 7fb241d99740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fb2381a4b80 con 0x564e2e5e6100 2024-03-19T01:58:08.211 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.211+0000 7fb237fff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564e2e5e6af0 0x564e2e5ffe50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:08.211 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.211+0000 7fb237fff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564e2e5e6af0 0x564e2e5ffe50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:44242/0 (socket says 172.21.15.99:44242) 2024-03-19T01:58:08.211 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.211+0000 7fb237fff640 1 -- 172.21.15.99:0/986864193 learned_addr learned my addr 172.21.15.99:0/986864193 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:58:08.211 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.212+0000 7fb237fff640 1 -- 172.21.15.99:0/986864193 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564e2e5e6100 msgr2=0x564e2e5e0220 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-03-19T01:58:08.211 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.212+0000 7fb237fff640 1 --2- 172.21.15.99:0/986864193 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564e2e5e6100 0x564e2e5e0220 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:08.212 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.212+0000 7fb237fff640 1 -- 172.21.15.99:0/986864193 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564e2e5ebc60 msgr2=0x564e2e5e0760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:58:08.212 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.212+0000 7fb237fff640 1 --2- 172.21.15.99:0/986864193 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564e2e5ebc60 0x564e2e5e0760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:08.212 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.212+0000 7fb237fff640 1 -- 172.21.15.99:0/986864193 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x564e2e32b620 con 0x564e2e5e6af0 2024-03-19T01:58:08.212 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.212+0000 7fb237fff640 1 --2- 172.21.15.99:0/986864193 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564e2e5e6af0 0x564e2e5ffe50 secure :-1 s=READY pgs=183 cs=0 l=1 rev1=1 crypto rx=0x7fb224009430 tx=0x7fb224005e50 comp rx=0 tx=0).ready entity=mon.2 client_cookie=8fe855f27d264ec1 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:58:08.212 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.213+0000 7fb2227fc640 1 -- 172.21.15.99:0/986864193 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb22404b040 con 0x564e2e5e6af0 2024-03-19T01:58:08.212 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.213+0000 7fb2227fc640 1 -- 172.21.15.99:0/986864193 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fb2240bd670 con 0x564e2e5e6af0 2024-03-19T01:58:08.212 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.213+0000 7fb2227fc640 1 -- 172.21.15.99:0/986864193 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb2240bcf70 con 0x564e2e5e6af0 2024-03-19T01:58:08.214 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.214+0000 7fb241d99740 1 -- 172.21.15.99:0/986864193 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564e2e5e6af0 msgr2=0x564e2e5ffe50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:58:08.214 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.214+0000 7fb241d99740 1 --2- 172.21.15.99:0/986864193 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564e2e5e6af0 0x564e2e5ffe50 secure :-1 s=READY pgs=183 cs=0 l=1 rev1=1 crypto rx=0x7fb224009430 tx=0x7fb224005e50 comp rx=0 tx=0).stop 2024-03-19T01:58:08.214 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.214+0000 7fb241d99740 1 -- 172.21.15.99:0/986864193 shutdown_connections 2024-03-19T01:58:08.214 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.214+0000 7fb241d99740 1 --2- 172.21.15.99:0/986864193 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564e2e5e6af0 0x564e2e5ffe50 unknown :-1 s=CLOSED pgs=183 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:08.214 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.214+0000 7fb241d99740 1 --2- 172.21.15.99:0/986864193 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564e2e5ebc60 0x564e2e5e0760 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:08.214 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.214+0000 7fb241d99740 1 --2- 172.21.15.99:0/986864193 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564e2e5e6100 0x564e2e5e0220 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:08.214 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.214+0000 7fb241d99740 1 -- 172.21.15.99:0/986864193 >> 172.21.15.99:0/986864193 conn(0x564e2e5e1670 msgr2=0x564e2e5e5160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:58:08.214 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.214+0000 7fb241d99740 1 -- 172.21.15.99:0/986864193 shutdown_connections 2024-03-19T01:58:08.214 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.214+0000 7fb241d99740 1 -- 172.21.15.99:0/986864193 wait complete. 2024-03-19T01:58:08.214 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.214+0000 7fb241d99740 1 -- start start 2024-03-19T01:58:08.215 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.214+0000 7fb241d99740 1 Processor -- start 2024-03-19T01:58:08.216 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.215+0000 7fb241d99740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564e2e5e6100 0x564e2e5fd9f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:08.216 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.215+0000 7fb241d99740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564e2e5e6af0 0x564e2e5fdf30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:08.217 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.215+0000 7fb241d99740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564e2e5ebc60 0x564e2e5fed70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:08.217 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.215+0000 7fb241d99740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fb2381a51f0 con 0x564e2e5ebc60 2024-03-19T01:58:08.217 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.215+0000 7fb241d99740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fb2381a6660 con 0x564e2e5e6100 2024-03-19T01:58:08.217 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.215+0000 7fb241d99740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fb2381a58c0 con 0x564e2e5e6af0 2024-03-19T01:58:08.217 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.215+0000 7fb23ce6d640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564e2e5e6100 0x564e2e5fd9f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:08.217 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.215+0000 7fb23d66e640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564e2e5ebc60 0x564e2e5fed70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:08.217 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.215+0000 7fb23ce6d640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564e2e5e6100 0x564e2e5fd9f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:44246/0 (socket says 172.21.15.99:44246) 2024-03-19T01:58:08.217 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.215+0000 7fb23ce6d640 1 -- 172.21.15.99:0/3729062075 learned_addr learned my addr 172.21.15.99:0/3729062075 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:58:08.217 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.215+0000 7fb241d99740 10 client.0 _ll_register_callbacks cb 0x7fb22c009db0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:58:08.217 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.216+0000 7fb23d66e640 1 -- 172.21.15.99:0/3729062075 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564e2e5e6af0 msgr2=0x564e2e5fdf30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:58:08.217 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.216+0000 7fb23d66e640 1 --2- 172.21.15.99:0/3729062075 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564e2e5e6af0 0x564e2e5fdf30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:08.217 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.216+0000 7fb23d66e640 1 -- 172.21.15.99:0/3729062075 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564e2e5e6100 msgr2=0x564e2e5fd9f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:58:08.217 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.216+0000 7fb23d66e640 1 --2- 172.21.15.99:0/3729062075 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564e2e5e6100 0x564e2e5fd9f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:08.218 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.216+0000 7fb23d66e640 1 -- 172.21.15.99:0/3729062075 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x564e2e5ff360 con 0x564e2e5ebc60 2024-03-19T01:58:08.218 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.216+0000 7fb23d66e640 1 --2- 172.21.15.99:0/3729062075 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564e2e5ebc60 0x564e2e5fed70 secure :-1 s=READY pgs=675 cs=0 l=1 rev1=1 crypto rx=0x564e2e600450 tx=0x7fb230011510 comp rx=0 tx=0).ready entity=mon.0 client_cookie=124241352e86950 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:58:08.218 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.216+0000 7fb217fff640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:58:08.218 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.216+0000 7fb217fff640 1 -- 172.21.15.99:0/3729062075 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb23002e060 con 0x564e2e5ebc60 2024-03-19T01:58:08.218 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.216+0000 7fb241d99740 1 -- 172.21.15.99:0/3729062075 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fb2240bd670 con 0x564e2e5ebc60 2024-03-19T01:58:08.218 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.216+0000 7fb217fff640 1 -- 172.21.15.99:0/3729062075 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fb230005040 con 0x564e2e5ebc60 2024-03-19T01:58:08.218 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.217+0000 7fb217fff640 1 -- 172.21.15.99:0/3729062075 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7fb23000fc20 con 0x564e2e5ebc60 2024-03-19T01:58:08.218 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.217+0000 7fb241d99740 10 client.16713 fetch_fsmap learned FSMap version 37 2024-03-19T01:58:08.218 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.217+0000 7fb241d99740 1 -- 172.21.15.99:0/3729062075 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x564e2e66ac20 con 0x564e2e5ebc60 2024-03-19T01:58:08.219 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.218+0000 7fb217fff640 1 -- 172.21.15.99:0/3729062075 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fb230004e10 con 0x564e2e5ebc60 2024-03-19T01:58:08.219 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.218+0000 7fb217fff640 1 -- 172.21.15.99:0/3729062075 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7fb230028070 con 0x564e2e5ebc60 2024-03-19T01:58:08.219 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.218+0000 7fb241d99740 10 client.16713 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T01:58:08.219 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.218+0000 7fb241d99740 10 client.16713 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:58:08.219 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.218+0000 7fb241d99740 20 client.16713 populate_metadata read hostname 'smithi099' 2024-03-19T01:58:08.220 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.218+0000 7fb241d99740 1 --2- 172.21.15.99:0/3729062075 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x564e2e66b320 0x564e2e66d710 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:08.220 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.218+0000 7fb241d99740 4 client.16713 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:58:08.220 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.218+0000 7fb241d99740 1 -- 172.21.15.99:0/3729062075 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x7fb2240bd670 con 0x564e2e66b320 2024-03-19T01:58:08.220 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.218+0000 7fb23ce6d640 1 --2- 172.21.15.99:0/3729062075 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x564e2e66b320 0x564e2e66d710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:08.220 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.220+0000 7fb23ce6d640 1 --2- 172.21.15.99:0/3729062075 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x564e2e66b320 0x564e2e66d710 crc :-1 s=READY pgs=182 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=7b1595a09fd06a0a server_cookie=575ea1b82e1d890d in_seq=0 out_seq=0 2024-03-19T01:58:08.220 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.220+0000 7fb217fff640 10 client.16713 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:58:08.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.664+0000 7fb217fff640 1 -- 172.21.15.99:0/3729062075 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x7fb2240bd670 con 0x564e2e66b320 2024-03-19T01:58:08.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.664+0000 7fb217fff640 10 client.16713 handle_command_reply: tid=0 2024-03-19T01:58:08.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.664+0000 7fb241d99740 1 client.16713 shutdown 2024-03-19T01:58:08.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.664+0000 7fb241d99740 1 -- 172.21.15.99:0/3729062075 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564e2e5ebc60 msgr2=0x564e2e5fed70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:58:08.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.664+0000 7fb241d99740 1 --2- 172.21.15.99:0/3729062075 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564e2e5ebc60 0x564e2e5fed70 secure :-1 s=READY pgs=675 cs=0 l=1 rev1=1 crypto rx=0x564e2e600450 tx=0x7fb230011510 comp rx=0 tx=0).stop 2024-03-19T01:58:08.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.664+0000 7fb241d99740 1 -- 172.21.15.99:0/3729062075 shutdown_connections 2024-03-19T01:58:08.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.664+0000 7fb241d99740 1 --2- 172.21.15.99:0/3729062075 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x564e2e66b320 0x564e2e66d710 crc :-1 s=READY pgs=182 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:08.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.664+0000 7fb241d99740 1 --2- 172.21.15.99:0/3729062075 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564e2e5ebc60 0x564e2e5fed70 unknown :-1 s=CLOSED pgs=675 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:08.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.664+0000 7fb241d99740 1 --2- 172.21.15.99:0/3729062075 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564e2e5e6af0 0x564e2e5fdf30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:08.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.665+0000 7fb241d99740 1 --2- 172.21.15.99:0/3729062075 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564e2e5e6100 0x564e2e5fd9f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:08.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.665+0000 7fb217fff640 0 client.16713 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:58:08.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.665+0000 7fb241d99740 1 -- 172.21.15.99:0/3729062075 >> 172.21.15.99:0/3729062075 conn(0x564e2e5e1670 msgr2=0x564e2e5e74f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:58:08.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.665+0000 7fb241d99740 1 -- 172.21.15.99:0/3729062075 shutdown_connections 2024-03-19T01:58:08.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.665+0000 7fb241d99740 1 -- 172.21.15.99:0/3729062075 wait complete. 2024-03-19T01:58:08.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.665+0000 7fb241d99740 20 client.16713 trim_cache size 0 max 16384 2024-03-19T01:58:08.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:58:08.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:58:08.666 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:58:08.667 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:58:08.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.666+0000 7fb23f0d1640 1 -- 172.21.15.99:0/769758192 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb2280a28a0 msgr2=0x7fb2280a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:58:08.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.666+0000 7fb23f0d1640 1 --2- 172.21.15.99:0/769758192 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb2280a28a0 0x7fb2280a4d60 secure :-1 s=READY pgs=349 cs=0 l=1 rev1=1 crypto rx=0x7fb2381028c0 tx=0x7fb22c005f90 comp rx=0 tx=0).stop 2024-03-19T01:58:08.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.666+0000 7fb23f0d1640 1 -- 172.21.15.99:0/769758192 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb238102a60 msgr2=0x7fb238198000 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:58:08.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.666+0000 7fb23f0d1640 1 --2- 172.21.15.99:0/769758192 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb238102a60 0x7fb238198000 secure :-1 s=READY pgs=181 cs=0 l=1 rev1=1 crypto rx=0x7fb224009230 tx=0x7fb22402c650 comp rx=0 tx=0).stop 2024-03-19T01:58:08.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.666+0000 7fb23f0d1640 1 -- 172.21.15.99:0/769758192 shutdown_connections 2024-03-19T01:58:08.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.666+0000 7fb23f0d1640 1 --2- 172.21.15.99:0/769758192 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb2280a28a0 0x7fb2280a4d60 unknown :-1 s=CLOSED pgs=349 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:08.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.667+0000 7fb23f0d1640 1 --2- 172.21.15.99:0/769758192 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb238102a60 0x7fb238198000 unknown :-1 s=CLOSED pgs=181 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:08.668 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.667+0000 7fb23f0d1640 1 --2- 172.21.15.99:0/769758192 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb238103420 0x7fb23819f070 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:08.669 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.667+0000 7fb23f0d1640 1 --2- 172.21.15.99:0/769758192 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb238101870 0x7fb238197ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:08.669 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.667+0000 7fb23f0d1640 1 -- 172.21.15.99:0/769758192 >> 172.21.15.99:0/769758192 conn(0x7fb2380fcde0 msgr2=0x7fb238105d60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:58:08.669 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.667+0000 7fb23f0d1640 1 -- 172.21.15.99:0/769758192 shutdown_connections 2024-03-19T01:58:08.669 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:08.667+0000 7fb23f0d1640 1 -- 172.21.15.99:0/769758192 wait complete. 2024-03-19T01:58:09.107 INFO:tasks.workunit.client.0.smithi099.stdout: 10 122212 23.05 MB/sec execute 158 sec latency 278.004 ms 2024-03-19T01:58:09.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:58:09 smithi112 ceph-mon[66819]: pgmap v863: 97 pgs: 97 active+clean; 555 MiB data, 6.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.3 MiB/s rd, 6.1 MiB/s wr, 487 op/s 2024-03-19T01:58:09.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:58:09 smithi115 ceph-mon[67332]: pgmap v863: 97 pgs: 97 active+clean; 555 MiB data, 6.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.3 MiB/s rd, 6.1 MiB/s wr, 487 op/s 2024-03-19T01:58:09.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:58:09 smithi099 ceph-mon[62951]: pgmap v863: 97 pgs: 97 active+clean; 555 MiB data, 6.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.3 MiB/s rd, 6.1 MiB/s wr, 487 op/s 2024-03-19T01:58:10.107 INFO:tasks.workunit.client.0.smithi099.stdout: 10 122508 22.97 MB/sec execute 159 sec latency 219.101 ms 2024-03-19T01:58:11.107 INFO:tasks.workunit.client.0.smithi099.stdout: 10 123036 23.05 MB/sec execute 160 sec latency 113.613 ms 2024-03-19T01:58:11.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:58:11 smithi112 ceph-mon[66819]: pgmap v864: 97 pgs: 97 active+clean; 546 MiB data, 6.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 6.5 MiB/s wr, 507 op/s 2024-03-19T01:58:11.391 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:58:11 smithi099 ceph-mon[62951]: pgmap v864: 97 pgs: 97 active+clean; 546 MiB data, 6.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 6.5 MiB/s wr, 507 op/s 2024-03-19T01:58:11.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:58:11 smithi115 ceph-mon[67332]: pgmap v864: 97 pgs: 97 active+clean; 546 MiB data, 6.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 6.5 MiB/s wr, 507 op/s 2024-03-19T01:58:12.107 INFO:tasks.workunit.client.0.smithi099.stdout: 10 123665 23.10 MB/sec execute 161 sec latency 81.206 ms 2024-03-19T01:58:13.107 INFO:tasks.workunit.client.0.smithi099.stdout: 10 124204 23.18 MB/sec execute 162 sec latency 59.136 ms 2024-03-19T01:58:13.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:58:13 smithi112 ceph-mon[66819]: pgmap v865: 97 pgs: 97 active+clean; 552 MiB data, 6.6 GiB used, 1.0 TiB / 1.0 TiB avail; 4.3 MiB/s rd, 7.6 MiB/s wr, 610 op/s 2024-03-19T01:58:13.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:58:13 smithi115 ceph-mon[67332]: pgmap v865: 97 pgs: 97 active+clean; 552 MiB data, 6.6 GiB used, 1.0 TiB / 1.0 TiB avail; 4.3 MiB/s rd, 7.6 MiB/s wr, 610 op/s 2024-03-19T01:58:13.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:58:13 smithi099 ceph-mon[62951]: pgmap v865: 97 pgs: 97 active+clean; 552 MiB data, 6.6 GiB used, 1.0 TiB / 1.0 TiB avail; 4.3 MiB/s rd, 7.6 MiB/s wr, 610 op/s 2024-03-19T01:58:14.107 INFO:tasks.workunit.client.0.smithi099.stdout: 10 124659 23.23 MB/sec execute 163 sec latency 66.776 ms 2024-03-19T01:58:15.107 INFO:tasks.workunit.client.0.smithi099.stdout: 10 125027 23.18 MB/sec execute 164 sec latency 227.018 ms 2024-03-19T01:58:15.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:58:15 smithi112 ceph-mon[66819]: pgmap v866: 97 pgs: 97 active+clean; 551 MiB data, 6.6 GiB used, 1.0 TiB / 1.0 TiB avail; 3.4 MiB/s rd, 5.9 MiB/s wr, 534 op/s 2024-03-19T01:58:15.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:58:15 smithi115 ceph-mon[67332]: pgmap v866: 97 pgs: 97 active+clean; 551 MiB data, 6.6 GiB used, 1.0 TiB / 1.0 TiB avail; 3.4 MiB/s rd, 5.9 MiB/s wr, 534 op/s 2024-03-19T01:58:15.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:58:15 smithi099 ceph-mon[62951]: pgmap v866: 97 pgs: 97 active+clean; 551 MiB data, 6.6 GiB used, 1.0 TiB / 1.0 TiB avail; 3.4 MiB/s rd, 5.9 MiB/s wr, 534 op/s 2024-03-19T01:58:16.107 INFO:tasks.workunit.client.0.smithi099.stdout: 10 125408 23.15 MB/sec execute 165 sec latency 122.619 ms 2024-03-19T01:58:17.107 INFO:tasks.workunit.client.0.smithi099.stdout: 10 125770 23.16 MB/sec execute 166 sec latency 76.090 ms 2024-03-19T01:58:17.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:58:17 smithi112 ceph-mon[66819]: pgmap v867: 97 pgs: 97 active+clean; 557 MiB data, 6.6 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 6.7 MiB/s wr, 582 op/s 2024-03-19T01:58:17.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:58:17 smithi115 ceph-mon[67332]: pgmap v867: 97 pgs: 97 active+clean; 557 MiB data, 6.6 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 6.7 MiB/s wr, 582 op/s 2024-03-19T01:58:17.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:58:17 smithi099 ceph-mon[62951]: pgmap v867: 97 pgs: 97 active+clean; 557 MiB data, 6.6 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 6.7 MiB/s wr, 582 op/s 2024-03-19T01:58:18.107 INFO:tasks.workunit.client.0.smithi099.stdout: 10 126209 23.13 MB/sec execute 167 sec latency 64.501 ms 2024-03-19T01:58:19.107 INFO:tasks.workunit.client.0.smithi099.stdout: 10 126669 23.10 MB/sec execute 168 sec latency 79.961 ms 2024-03-19T01:58:19.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:58:19 smithi112 ceph-mon[66819]: pgmap v868: 97 pgs: 97 active+clean; 558 MiB data, 6.6 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 6.2 MiB/s wr, 575 op/s 2024-03-19T01:58:19.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:58:19 smithi115 ceph-mon[67332]: pgmap v868: 97 pgs: 97 active+clean; 558 MiB data, 6.6 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 6.2 MiB/s wr, 575 op/s 2024-03-19T01:58:19.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:58:19 smithi099 ceph-mon[62951]: pgmap v868: 97 pgs: 97 active+clean; 558 MiB data, 6.6 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 6.2 MiB/s wr, 575 op/s 2024-03-19T01:58:20.107 INFO:tasks.workunit.client.0.smithi099.stdout: 10 127026 23.06 MB/sec execute 169 sec latency 302.266 ms 2024-03-19T01:58:21.107 INFO:tasks.workunit.client.0.smithi099.stdout: 10 127515 23.09 MB/sec execute 170 sec latency 82.534 ms 2024-03-19T01:58:21.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:58:21 smithi112 ceph-mon[66819]: pgmap v869: 97 pgs: 97 active+clean; 560 MiB data, 6.7 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 6.1 MiB/s wr, 532 op/s 2024-03-19T01:58:21.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:58:21 smithi115 ceph-mon[67332]: pgmap v869: 97 pgs: 97 active+clean; 560 MiB data, 6.7 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 6.1 MiB/s wr, 532 op/s 2024-03-19T01:58:21.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:58:21 smithi099 ceph-mon[62951]: pgmap v869: 97 pgs: 97 active+clean; 560 MiB data, 6.7 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 6.1 MiB/s wr, 532 op/s 2024-03-19T01:58:21.777 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.777+0000 7f04db7d4640 1 -- 172.21.15.99:0/1557565038 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f04d41013b0 msgr2=0x7f04d4101830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:58:21.777 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.777+0000 7f04db7d4640 1 --2- 172.21.15.99:0/1557565038 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f04d41013b0 0x7f04d4101830 secure :-1 s=READY pgs=184 cs=0 l=1 rev1=1 crypto rx=0x7f04cc009be0 tx=0x7f04cc02bef0 comp rx=0 tx=0).stop 2024-03-19T01:58:21.777 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.777+0000 7f04db7d4640 1 -- 172.21.15.99:0/1557565038 shutdown_connections 2024-03-19T01:58:21.777 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.777+0000 7f04db7d4640 1 --2- 172.21.15.99:0/1557565038 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f04d41013b0 0x7f04d4101830 unknown :-1 s=CLOSED pgs=184 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:21.777 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.777+0000 7f04db7d4640 1 --2- 172.21.15.99:0/1557565038 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f04d4100a10 0x7f04d4100e70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:21.777 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.777+0000 7f04db7d4640 1 --2- 172.21.15.99:0/1557565038 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f04d41000b0 0x7f04d41004d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:21.777 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.777+0000 7f04db7d4640 1 -- 172.21.15.99:0/1557565038 >> 172.21.15.99:0/1557565038 conn(0x7f04d40fbcb0 msgr2=0x7f04d40fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:58:21.777 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.777+0000 7f04db7d4640 1 -- 172.21.15.99:0/1557565038 shutdown_connections 2024-03-19T01:58:21.777 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.777+0000 7f04db7d4640 1 -- 172.21.15.99:0/1557565038 wait complete. 2024-03-19T01:58:21.779 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.778+0000 7f04db7d4640 1 Processor -- start 2024-03-19T01:58:21.779 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.778+0000 7f04db7d4640 1 -- start start 2024-03-19T01:58:21.779 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.778+0000 7f04db7d4640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f04d41000b0 0x7f04d4199c40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:21.779 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.778+0000 7f04db7d4640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f04d4100a10 0x7f04d419a180 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:21.779 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.778+0000 7f04db7d4640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f04d41013b0 0x7f04d41a11b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:21.779 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.778+0000 7f04db7d4640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f04d410f690 con 0x7f04d41013b0 2024-03-19T01:58:21.779 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.778+0000 7f04db7d4640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f04d410f510 con 0x7f04d4100a10 2024-03-19T01:58:21.779 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.778+0000 7f04db7d4640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f04d410f810 con 0x7f04d41000b0 2024-03-19T01:58:21.779 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.778+0000 7f04d9570640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f04d41000b0 0x7f04d4199c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:21.779 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.778+0000 7f04d9570640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f04d41000b0 0x7f04d4199c40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:54290/0 (socket says 172.21.15.99:54290) 2024-03-19T01:58:21.779 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.778+0000 7f04d9570640 1 -- 172.21.15.99:0/1764492727 learned_addr learned my addr 172.21.15.99:0/1764492727 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:58:21.779 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.778+0000 7f04d8d6f640 1 --2- 172.21.15.99:0/1764492727 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f04d4100a10 0x7f04d419a180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:21.779 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.778+0000 7f04d9d71640 1 --2- 172.21.15.99:0/1764492727 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f04d41013b0 0x7f04d41a11b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:21.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.778+0000 7f04d9570640 1 -- 172.21.15.99:0/1764492727 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f04d4100a10 msgr2=0x7f04d419a180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:58:21.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.778+0000 7f04d9570640 1 --2- 172.21.15.99:0/1764492727 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f04d4100a10 0x7f04d419a180 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:21.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.778+0000 7f04d9570640 1 -- 172.21.15.99:0/1764492727 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f04d41013b0 msgr2=0x7f04d41a11b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:58:21.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.778+0000 7f04d9570640 1 --2- 172.21.15.99:0/1764492727 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f04d41013b0 0x7f04d41a11b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:21.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.778+0000 7f04d9570640 1 -- 172.21.15.99:0/1764492727 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f04d41a1850 con 0x7f04d41000b0 2024-03-19T01:58:21.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.779+0000 7f04d9570640 1 --2- 172.21.15.99:0/1764492727 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f04d41000b0 0x7f04d4199c40 secure :-1 s=READY pgs=122 cs=0 l=1 rev1=1 crypto rx=0x7f04d000e9c0 tx=0x7f04d000ee80 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:58:21.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.779+0000 7f04ca7fc640 1 -- 172.21.15.99:0/1764492727 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f04d000ccc0 con 0x7f04d41000b0 2024-03-19T01:58:21.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.779+0000 7f04db7d4640 1 -- 172.21.15.99:0/1764492727 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f04d406b2a0 con 0x7f04d41000b0 2024-03-19T01:58:21.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.779+0000 7f04ca7fc640 1 -- 172.21.15.99:0/1764492727 <== mon.1 v2:172.21.15.115:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f04d0005500 con 0x7f04d41000b0 2024-03-19T01:58:21.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.779+0000 7f04ca7fc640 1 -- 172.21.15.99:0/1764492727 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f04d0010430 con 0x7f04d41000b0 2024-03-19T01:58:21.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.779+0000 7f04db7d4640 1 -- 172.21.15.99:0/1764492727 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f04d406b7e0 con 0x7f04d41000b0 2024-03-19T01:58:21.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.782+0000 7f04ca7fc640 1 -- 172.21.15.99:0/1764492727 <== mon.1 v2:172.21.15.115:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f04d00046c0 con 0x7f04d41000b0 2024-03-19T01:58:21.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.783+0000 7f04de3c3740 1 Processor -- start 2024-03-19T01:58:21.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.783+0000 7f04de3c3740 1 -- start start 2024-03-19T01:58:21.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.783+0000 7f04de3c3740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5576576ad910 0x5576576add30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:21.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.783+0000 7f04de3c3740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5576576ae300 0x5576576b2e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:21.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.783+0000 7f04de3c3740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5576576b3470 0x5576576b5900 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:21.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.783+0000 7f04de3c3740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x557657609be0 con 0x5576576ad910 2024-03-19T01:58:21.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.783+0000 7f04de3c3740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x5576575b2960 con 0x5576576ae300 2024-03-19T01:58:21.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.783+0000 7f04de3c3740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x557657605c80 con 0x5576576b3470 2024-03-19T01:58:21.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.784+0000 7f04d9d71640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5576576ad910 0x5576576add30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:21.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.784+0000 7f04d9d71640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5576576ad910 0x5576576add30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:33976/0 (socket says 172.21.15.99:33976) 2024-03-19T01:58:21.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.784+0000 7f04d9d71640 1 -- 172.21.15.99:0/1992698340 learned_addr learned my addr 172.21.15.99:0/1992698340 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:58:21.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.784+0000 7f04d9570640 1 --2- 172.21.15.99:0/1992698340 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5576576ae300 0x5576576b2e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:21.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.784+0000 7f04d9d71640 1 -- 172.21.15.99:0/1992698340 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5576576b3470 msgr2=0x5576576b5900 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-03-19T01:58:21.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.784+0000 7f04d9d71640 1 --2- 172.21.15.99:0/1992698340 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5576576b3470 0x5576576b5900 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:21.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.784+0000 7f04d9d71640 1 -- 172.21.15.99:0/1992698340 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5576576ae300 msgr2=0x5576576b2e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:58:21.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.784+0000 7f04d9d71640 1 --2- 172.21.15.99:0/1992698340 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5576576ae300 0x5576576b2e00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:21.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.784+0000 7f04d9d71640 1 -- 172.21.15.99:0/1992698340 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5576575a9d20 con 0x5576576ad910 2024-03-19T01:58:21.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.785+0000 7f04d9d71640 1 --2- 172.21.15.99:0/1992698340 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5576576ad910 0x5576576add30 secure :-1 s=READY pgs=676 cs=0 l=1 rev1=1 crypto rx=0x7f04cc009be0 tx=0x7f04cc009310 comp rx=0 tx=0).ready entity=mon.0 client_cookie=290cd865f0736ce4 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:58:21.786 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.785+0000 7f04ca7fc640 1 --2- 172.21.15.99:0/1764492727 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f04c00a2920 0x7f04c00a4de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:21.786 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.785+0000 7f04ca7fc640 1 -- 172.21.15.99:0/1764492727 <== mon.1 v2:172.21.15.115:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f04d00cedc0 con 0x7f04d41000b0 2024-03-19T01:58:21.786 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.785+0000 7f04bf7fe640 1 -- 172.21.15.99:0/1992698340 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f04cc043030 con 0x5576576ad910 2024-03-19T01:58:21.786 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.785+0000 7f04bf7fe640 1 -- 172.21.15.99:0/1992698340 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f04cc033b90 con 0x5576576ad910 2024-03-19T01:58:21.786 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.785+0000 7f04bf7fe640 1 -- 172.21.15.99:0/1992698340 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f04cc02c890 con 0x5576576ad910 2024-03-19T01:58:21.786 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.785+0000 7f04d9570640 1 --2- 172.21.15.99:0/1764492727 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f04c00a2920 0x7f04c00a4de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:21.787 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.786+0000 7f04de3c3740 1 -- 172.21.15.99:0/1992698340 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5576576ad910 msgr2=0x5576576add30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:58:21.787 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.786+0000 7f04d9570640 1 --2- 172.21.15.99:0/1764492727 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f04c00a2920 0x7f04c00a4de0 secure :-1 s=READY pgs=350 cs=0 l=1 rev1=1 crypto rx=0x7f04d0097a90 tx=0x7f04d0005880 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:58:21.787 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.786+0000 7f04de3c3740 1 --2- 172.21.15.99:0/1992698340 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5576576ad910 0x5576576add30 secure :-1 s=READY pgs=676 cs=0 l=1 rev1=1 crypto rx=0x7f04cc009be0 tx=0x7f04cc009310 comp rx=0 tx=0).stop 2024-03-19T01:58:21.787 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.786+0000 7f04de3c3740 1 -- 172.21.15.99:0/1992698340 shutdown_connections 2024-03-19T01:58:21.787 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.786+0000 7f04de3c3740 1 --2- 172.21.15.99:0/1992698340 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5576576ae300 0x5576576b2e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:21.787 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.786+0000 7f04de3c3740 1 --2- 172.21.15.99:0/1992698340 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5576576b3470 0x5576576b5900 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:21.787 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.786+0000 7f04de3c3740 1 --2- 172.21.15.99:0/1992698340 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5576576ad910 0x5576576add30 unknown :-1 s=CLOSED pgs=676 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:21.787 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.786+0000 7f04de3c3740 1 -- 172.21.15.99:0/1992698340 >> 172.21.15.99:0/1992698340 conn(0x5576576a0670 msgr2=0x5576576a0a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:58:21.787 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.787+0000 7f04de3c3740 1 -- 172.21.15.99:0/1992698340 shutdown_connections 2024-03-19T01:58:21.787 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.787+0000 7f04de3c3740 1 -- 172.21.15.99:0/1992698340 wait complete. 2024-03-19T01:58:21.791 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.787+0000 7f04de3c3740 1 -- start start 2024-03-19T01:58:21.791 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.787+0000 7f04de3c3740 1 Processor -- start 2024-03-19T01:58:21.791 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.788+0000 7f04de3c3740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5576576ad910 0x5576576b78e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:21.791 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.788+0000 7f04de3c3740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5576576ae300 0x5576576b7e20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:21.791 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.788+0000 7f04de3c3740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5576576b3470 0x5576576bee60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:21.791 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.788+0000 7f04de3c3740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x5576576b63c0 con 0x5576576ad910 2024-03-19T01:58:21.791 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.788+0000 7f04de3c3740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x5576575c5db0 con 0x5576576b3470 2024-03-19T01:58:21.791 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.788+0000 7f04de3c3740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x5576576b6540 con 0x5576576ae300 2024-03-19T01:58:21.791 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.788+0000 7f04de3c3740 10 client.0 _ll_register_callbacks cb 0x5576575cae70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:58:21.791 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.788+0000 7f04d8d6f640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5576576ae300 0x5576576b7e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:21.791 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.788+0000 7f04d8d6f640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5576576ae300 0x5576576b7e20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:54298/0 (socket says 172.21.15.99:54298) 2024-03-19T01:58:21.792 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.788+0000 7f04d8d6f640 1 -- 172.21.15.99:0/2075310820 learned_addr learned my addr 172.21.15.99:0/2075310820 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:58:21.792 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.788+0000 7f04d9d71640 1 --2- 172.21.15.99:0/2075310820 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5576576b3470 0x5576576bee60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:21.792 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.788+0000 7f04d9d71640 1 --2- 172.21.15.99:0/2075310820 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5576576ad910 0x5576576b78e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:21.792 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.788+0000 7f04d8d6f640 1 -- 172.21.15.99:0/2075310820 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5576576b3470 msgr2=0x5576576bee60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:58:21.792 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.788+0000 7f04d8d6f640 1 --2- 172.21.15.99:0/2075310820 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5576576b3470 0x5576576bee60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:21.792 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.788+0000 7f04d8d6f640 1 -- 172.21.15.99:0/2075310820 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5576576ad910 msgr2=0x5576576b78e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:58:21.792 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.788+0000 7f04d8d6f640 1 --2- 172.21.15.99:0/2075310820 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5576576ad910 0x5576576b78e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:21.792 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.788+0000 7f04d8d6f640 1 -- 172.21.15.99:0/2075310820 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5576575c4c60 con 0x5576576ae300 2024-03-19T01:58:21.792 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.789+0000 7f04d8d6f640 1 --2- 172.21.15.99:0/2075310820 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5576576ae300 0x5576576b7e20 secure :-1 s=READY pgs=123 cs=0 l=1 rev1=1 crypto rx=0x7f04c4022700 tx=0x7f04c4022bc0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=600f4c35186cd119 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:58:21.792 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.789+0000 7f04bcff9640 10 client.0 ms_handle_connect on v2:172.21.15.115:3300/0 2024-03-19T01:58:21.792 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.789+0000 7f04bcff9640 1 -- 172.21.15.99:0/2075310820 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f04c4020990 con 0x5576576ae300 2024-03-19T01:58:21.792 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.789+0000 7f04de3c3740 1 -- 172.21.15.99:0/2075310820 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f04cc033b90 con 0x5576576ae300 2024-03-19T01:58:21.792 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.789+0000 7f04bcff9640 1 -- 172.21.15.99:0/2075310820 <== mon.1 v2:172.21.15.115:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f04c4020b30 con 0x5576576ae300 2024-03-19T01:58:21.793 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.789+0000 7f04bcff9640 1 -- 172.21.15.99:0/2075310820 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7f04c4020d30 con 0x5576576ae300 2024-03-19T01:58:21.793 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.789+0000 7f04de3c3740 10 client.26533 fetch_fsmap learned FSMap version 37 2024-03-19T01:58:21.793 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.789+0000 7f04de3c3740 1 -- 172.21.15.99:0/2075310820 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x55765731e210 con 0x5576576ae300 2024-03-19T01:58:21.793 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.790+0000 7f04bcff9640 1 -- 172.21.15.99:0/2075310820 <== mon.1 v2:172.21.15.115:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f04cc033b90 con 0x5576576ae300 2024-03-19T01:58:21.793 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.790+0000 7f04bcff9640 1 -- 172.21.15.99:0/2075310820 <== mon.1 v2:172.21.15.115:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7f04c4029070 con 0x5576576ae300 2024-03-19T01:58:21.793 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.790+0000 7f04de3c3740 10 client.26533 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T01:58:21.793 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.790+0000 7f04de3c3740 10 client.26533 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:58:21.793 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.790+0000 7f04de3c3740 20 client.26533 populate_metadata read hostname 'smithi099' 2024-03-19T01:58:21.793 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.790+0000 7f04de3c3740 1 --2- 172.21.15.99:0/2075310820 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5576576c3770 0x5576576c5bd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:21.793 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.790+0000 7f04de3c3740 4 client.26533 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:58:21.793 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.790+0000 7f04de3c3740 1 -- 172.21.15.99:0/2075310820 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x557657617120 con 0x5576576c3770 2024-03-19T01:58:21.793 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.791+0000 7f04d9d71640 1 --2- 172.21.15.99:0/2075310820 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5576576c3770 0x5576576c5bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:21.794 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.793+0000 7f04bcff9640 10 client.26533 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:58:21.794 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.793+0000 7f04d9d71640 1 --2- 172.21.15.99:0/2075310820 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5576576c3770 0x5576576c5bd0 crc :-1 s=READY pgs=183 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=7fc512cdc798a4da server_cookie=72363da5c53b8162 in_seq=0 out_seq=0 2024-03-19T01:58:21.800 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.799+0000 7f04bcff9640 1 -- 172.21.15.99:0/2075310820 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x557657617120 con 0x5576576c3770 2024-03-19T01:58:21.800 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.799+0000 7f04bcff9640 10 client.26533 handle_command_reply: tid=0 2024-03-19T01:58:21.800 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.799+0000 7f04de3c3740 1 client.26533 shutdown 2024-03-19T01:58:21.800 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.799+0000 7f04de3c3740 1 -- 172.21.15.99:0/2075310820 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5576576ae300 msgr2=0x5576576b7e20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:58:21.800 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.799+0000 7f04de3c3740 1 --2- 172.21.15.99:0/2075310820 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5576576ae300 0x5576576b7e20 secure :-1 s=READY pgs=123 cs=0 l=1 rev1=1 crypto rx=0x7f04c4022700 tx=0x7f04c4022bc0 comp rx=0 tx=0).stop 2024-03-19T01:58:21.800 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.800+0000 7f04de3c3740 1 -- 172.21.15.99:0/2075310820 shutdown_connections 2024-03-19T01:58:21.800 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.800+0000 7f04de3c3740 1 --2- 172.21.15.99:0/2075310820 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5576576c3770 0x5576576c5bd0 crc :-1 s=READY pgs=183 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:21.800 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.800+0000 7f04de3c3740 1 --2- 172.21.15.99:0/2075310820 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5576576b3470 0x5576576bee60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:21.800 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.800+0000 7f04de3c3740 1 --2- 172.21.15.99:0/2075310820 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5576576ae300 0x5576576b7e20 unknown :-1 s=CLOSED pgs=123 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:21.800 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.800+0000 7f04de3c3740 1 --2- 172.21.15.99:0/2075310820 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5576576ad910 0x5576576b78e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:21.800 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.800+0000 7f04bcff9640 0 client.26533 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:58:21.800 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.800+0000 7f04de3c3740 1 -- 172.21.15.99:0/2075310820 >> 172.21.15.99:0/2075310820 conn(0x5576576a0670 msgr2=0x5576576a18c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:58:21.801 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.800+0000 7f04de3c3740 1 -- 172.21.15.99:0/2075310820 shutdown_connections 2024-03-19T01:58:21.801 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.800+0000 7f04de3c3740 1 -- 172.21.15.99:0/2075310820 wait complete. 2024-03-19T01:58:21.801 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.800+0000 7f04de3c3740 20 client.26533 trim_cache size 0 max 16384 2024-03-19T01:58:21.820 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.819+0000 7f04de3c3740 1 Processor -- start 2024-03-19T01:58:21.820 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.819+0000 7f04de3c3740 1 -- start start 2024-03-19T01:58:21.820 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.819+0000 7f04de3c3740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5576576ad910 0x5576576aeda0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:21.820 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.819+0000 7f04de3c3740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5576576ae300 0x5576576bee20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:21.820 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.819+0000 7f04de3c3740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5576576b3470 0x5576576af2e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:21.820 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.819+0000 7f04de3c3740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f04d4072c90 con 0x5576576b3470 2024-03-19T01:58:21.820 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.819+0000 7f04de3c3740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f04d4070560 con 0x5576576ad910 2024-03-19T01:58:21.820 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.819+0000 7f04de3c3740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f04d406e3b0 con 0x5576576ae300 2024-03-19T01:58:21.820 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.819+0000 7f04d8d6f640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5576576ae300 0x5576576bee20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:21.820 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.819+0000 7f04d9d71640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5576576ad910 0x5576576aeda0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:21.820 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.819+0000 7f04d8d6f640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5576576ae300 0x5576576bee20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:54306/0 (socket says 172.21.15.99:54306) 2024-03-19T01:58:21.820 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.819+0000 7f04d8d6f640 1 -- 172.21.15.99:0/2892663170 learned_addr learned my addr 172.21.15.99:0/2892663170 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:58:21.820 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.819+0000 7f04d9d71640 1 --2- 172.21.15.99:0/2892663170 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5576576ad910 0x5576576aeda0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:47248/0 (socket says 172.21.15.99:47248) 2024-03-19T01:58:21.821 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.819+0000 7f04d9d71640 1 --2- 172.21.15.99:0/2892663170 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5576576b3470 0x5576576af2e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:21.821 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.819+0000 7f04d9d71640 1 -- 172.21.15.99:0/2892663170 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5576576ae300 msgr2=0x5576576bee20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:58:21.821 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.819+0000 7f04d9d71640 1 --2- 172.21.15.99:0/2892663170 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5576576ae300 0x5576576bee20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:21.821 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.819+0000 7f04d9d71640 1 -- 172.21.15.99:0/2892663170 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5576576b3470 msgr2=0x5576576af2e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:58:21.821 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.819+0000 7f04d9d71640 1 --2- 172.21.15.99:0/2892663170 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5576576b3470 0x5576576af2e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:21.821 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.819+0000 7f04d9d71640 1 -- 172.21.15.99:0/2892663170 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5576573ea620 con 0x5576576ad910 2024-03-19T01:58:21.821 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.820+0000 7f04d9d71640 1 --2- 172.21.15.99:0/2892663170 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5576576ad910 0x5576576aeda0 secure :-1 s=READY pgs=185 cs=0 l=1 rev1=1 crypto rx=0x7f04cc003c80 tx=0x7f04cc04c040 comp rx=0 tx=0).ready entity=mon.2 client_cookie=dcbcdbfe732e45fa server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:58:21.821 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.820+0000 7f04beffd640 1 -- 172.21.15.99:0/2892663170 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f04cc043040 con 0x5576576ad910 2024-03-19T01:58:21.821 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.820+0000 7f04beffd640 1 -- 172.21.15.99:0/2892663170 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f04cc044040 con 0x5576576ad910 2024-03-19T01:58:21.821 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.820+0000 7f04beffd640 1 -- 172.21.15.99:0/2892663170 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f04cc02cc10 con 0x5576576ad910 2024-03-19T01:58:21.824 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.821+0000 7f04de3c3740 1 -- 172.21.15.99:0/2892663170 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5576576ad910 msgr2=0x5576576aeda0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:58:21.825 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.821+0000 7f04de3c3740 1 --2- 172.21.15.99:0/2892663170 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5576576ad910 0x5576576aeda0 secure :-1 s=READY pgs=185 cs=0 l=1 rev1=1 crypto rx=0x7f04cc003c80 tx=0x7f04cc04c040 comp rx=0 tx=0).stop 2024-03-19T01:58:21.825 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.821+0000 7f04de3c3740 1 -- 172.21.15.99:0/2892663170 shutdown_connections 2024-03-19T01:58:21.825 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.821+0000 7f04de3c3740 1 --2- 172.21.15.99:0/2892663170 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5576576b3470 0x5576576af2e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:21.825 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.821+0000 7f04de3c3740 1 --2- 172.21.15.99:0/2892663170 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5576576ae300 0x5576576bee20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:21.825 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.821+0000 7f04de3c3740 1 --2- 172.21.15.99:0/2892663170 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5576576ad910 0x5576576aeda0 unknown :-1 s=CLOSED pgs=185 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:21.825 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.821+0000 7f04de3c3740 1 -- 172.21.15.99:0/2892663170 >> 172.21.15.99:0/2892663170 conn(0x5576576a0670 msgr2=0x55765769f220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:58:21.825 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.821+0000 7f04de3c3740 1 -- 172.21.15.99:0/2892663170 shutdown_connections 2024-03-19T01:58:21.825 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.821+0000 7f04de3c3740 1 -- 172.21.15.99:0/2892663170 wait complete. 2024-03-19T01:58:21.825 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.821+0000 7f04de3c3740 1 -- start start 2024-03-19T01:58:21.825 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.821+0000 7f04de3c3740 1 Processor -- start 2024-03-19T01:58:21.825 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.822+0000 7f04de3c3740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5576576ad910 0x5576576bc810 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:21.825 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.822+0000 7f04de3c3740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5576576ae300 0x5576576bcd50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:21.825 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.822+0000 7f04de3c3740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5576576b3470 0x5576576bdd60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:21.826 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.822+0000 7f04de3c3740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f04d406ea20 con 0x5576576ad910 2024-03-19T01:58:21.826 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.822+0000 7f04de3c3740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f04d406fe90 con 0x5576576b3470 2024-03-19T01:58:21.826 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.822+0000 7f04de3c3740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f04d406f0f0 con 0x5576576ae300 2024-03-19T01:58:21.826 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.822+0000 7f04de3c3740 10 client.0 _ll_register_callbacks cb 0x7f04c40097a0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:58:21.826 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.822+0000 7f04d9d71640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5576576ad910 0x5576576bc810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:21.826 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.822+0000 7f04d8d6f640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5576576ae300 0x5576576bcd50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:21.826 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.822+0000 7f04d8d6f640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5576576ae300 0x5576576bcd50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:54318/0 (socket says 172.21.15.99:54318) 2024-03-19T01:58:21.826 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.822+0000 7f04d8d6f640 1 -- 172.21.15.99:0/2797613296 learned_addr learned my addr 172.21.15.99:0/2797613296 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:58:21.826 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.822+0000 7f04d9d71640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5576576ad910 0x5576576bc810 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:33988/0 (socket says 172.21.15.99:33988) 2024-03-19T01:58:21.826 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.822+0000 7f04d9d71640 1 --2- 172.21.15.99:0/2797613296 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5576576b3470 0x5576576bdd60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:21.826 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.823+0000 7f04d8d6f640 1 -- 172.21.15.99:0/2797613296 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5576576b3470 msgr2=0x5576576bdd60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:58:21.826 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.823+0000 7f04d8d6f640 1 --2- 172.21.15.99:0/2797613296 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5576576b3470 0x5576576bdd60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:21.826 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.823+0000 7f04d8d6f640 1 -- 172.21.15.99:0/2797613296 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5576576ad910 msgr2=0x5576576bc810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:58:21.826 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.823+0000 7f04d8d6f640 1 --2- 172.21.15.99:0/2797613296 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5576576ad910 0x5576576bc810 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:21.827 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.823+0000 7f04d8d6f640 1 -- 172.21.15.99:0/2797613296 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5576576be350 con 0x5576576ae300 2024-03-19T01:58:21.827 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.823+0000 7f04d9d71640 1 --2- 172.21.15.99:0/2797613296 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5576576ad910 0x5576576bc810 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:58:21.827 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.823+0000 7f04d8d6f640 1 --2- 172.21.15.99:0/2797613296 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5576576ae300 0x5576576bcd50 secure :-1 s=READY pgs=124 cs=0 l=1 rev1=1 crypto rx=0x7f04c4035770 tx=0x7f04c4009270 comp rx=0 tx=0).ready entity=mon.1 client_cookie=af7205418fa8ca96 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:58:21.827 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.823+0000 7f04bcff9640 10 client.0 ms_handle_connect on v2:172.21.15.115:3300/0 2024-03-19T01:58:21.827 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.823+0000 7f04bcff9640 1 -- 172.21.15.99:0/2797613296 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f04c4005360 con 0x5576576ae300 2024-03-19T01:58:21.827 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.823+0000 7f04bcff9640 1 -- 172.21.15.99:0/2797613296 <== mon.1 v2:172.21.15.115:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f04c4005500 con 0x5576576ae300 2024-03-19T01:58:21.827 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.823+0000 7f04de3c3740 1 -- 172.21.15.99:0/2797613296 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f04cc044040 con 0x5576576ae300 2024-03-19T01:58:21.827 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.823+0000 7f04bcff9640 1 -- 172.21.15.99:0/2797613296 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7f04c4035910 con 0x5576576ae300 2024-03-19T01:58:21.827 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.823+0000 7f04de3c3740 10 client.26539 fetch_fsmap learned FSMap version 37 2024-03-19T01:58:21.827 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.823+0000 7f04de3c3740 1 -- 172.21.15.99:0/2797613296 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x557657729e60 con 0x5576576ae300 2024-03-19T01:58:21.827 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.824+0000 7f04bcff9640 1 -- 172.21.15.99:0/2797613296 <== mon.1 v2:172.21.15.115:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f04cc044040 con 0x5576576ae300 2024-03-19T01:58:21.827 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.824+0000 7f04bcff9640 1 -- 172.21.15.99:0/2797613296 <== mon.1 v2:172.21.15.115:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7f04c403d030 con 0x5576576ae300 2024-03-19T01:58:21.827 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.824+0000 7f04de3c3740 10 client.26539 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T01:58:21.827 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.824+0000 7f04de3c3740 10 client.26539 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:58:21.828 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.824+0000 7f04de3c3740 20 client.26539 populate_metadata read hostname 'smithi099' 2024-03-19T01:58:21.828 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.824+0000 7f04de3c3740 1 --2- 172.21.15.99:0/2797613296 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55765772a7e0 0x55765772cc00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:21.828 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.824+0000 7f04de3c3740 4 client.26539 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:58:21.828 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.824+0000 7f04de3c3740 1 -- 172.21.15.99:0/2797613296 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x557657611470 con 0x55765772a7e0 2024-03-19T01:58:21.828 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.824+0000 7f04d9d71640 1 --2- 172.21.15.99:0/2797613296 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55765772a7e0 0x55765772cc00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:21.828 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.825+0000 7f04bcff9640 10 client.26539 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:58:21.828 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:21.825+0000 7f04d9d71640 1 --2- 172.21.15.99:0/2797613296 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55765772a7e0 0x55765772cc00 crc :-1 s=READY pgs=184 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=65fe4ac69bc6e1be server_cookie=db278bd315ff16ed in_seq=0 out_seq=0 2024-03-19T01:58:22.107 INFO:tasks.workunit.client.0.smithi099.stdout: 10 127952 23.10 MB/sec execute 171 sec latency 265.423 ms 2024-03-19T01:58:22.255 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:22.254+0000 7f04bcff9640 1 -- 172.21.15.99:0/2797613296 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x557657611470 con 0x55765772a7e0 2024-03-19T01:58:22.255 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:22.254+0000 7f04bcff9640 10 client.26539 handle_command_reply: tid=0 2024-03-19T01:58:22.255 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:22.254+0000 7f04de3c3740 1 client.26539 shutdown 2024-03-19T01:58:22.255 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:22.255+0000 7f04de3c3740 1 -- 172.21.15.99:0/2797613296 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5576576ae300 msgr2=0x5576576bcd50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:58:22.255 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:22.255+0000 7f04de3c3740 1 --2- 172.21.15.99:0/2797613296 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5576576ae300 0x5576576bcd50 secure :-1 s=READY pgs=124 cs=0 l=1 rev1=1 crypto rx=0x7f04c4035770 tx=0x7f04c4009270 comp rx=0 tx=0).stop 2024-03-19T01:58:22.255 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:22.255+0000 7f04de3c3740 1 -- 172.21.15.99:0/2797613296 shutdown_connections 2024-03-19T01:58:22.255 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:22.255+0000 7f04de3c3740 1 --2- 172.21.15.99:0/2797613296 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55765772a7e0 0x55765772cc00 crc :-1 s=READY pgs=184 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:22.256 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:22.255+0000 7f04bcff9640 0 client.26539 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:58:22.256 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:22.255+0000 7f04de3c3740 1 --2- 172.21.15.99:0/2797613296 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5576576b3470 0x5576576bdd60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:22.256 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:22.255+0000 7f04de3c3740 1 --2- 172.21.15.99:0/2797613296 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5576576ae300 0x5576576bcd50 unknown :-1 s=CLOSED pgs=124 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:22.256 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:22.255+0000 7f04de3c3740 1 --2- 172.21.15.99:0/2797613296 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5576576ad910 0x5576576bc810 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:22.256 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:22.255+0000 7f04de3c3740 1 -- 172.21.15.99:0/2797613296 >> 172.21.15.99:0/2797613296 conn(0x5576576a0670 msgr2=0x5576576a38d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:58:22.256 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:22.255+0000 7f04de3c3740 1 -- 172.21.15.99:0/2797613296 shutdown_connections 2024-03-19T01:58:22.256 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:22.255+0000 7f04de3c3740 1 -- 172.21.15.99:0/2797613296 wait complete. 2024-03-19T01:58:22.256 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:22.255+0000 7f04de3c3740 20 client.26539 trim_cache size 0 max 16384 2024-03-19T01:58:22.256 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:58:22.256 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:58:22.256 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:58:22.257 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:58:22.257 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:22.256+0000 7f04db7d4640 1 -- 172.21.15.99:0/1764492727 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f04c00a2920 msgr2=0x7f04c00a4de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:58:22.257 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:22.256+0000 7f04db7d4640 1 --2- 172.21.15.99:0/1764492727 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f04c00a2920 0x7f04c00a4de0 secure :-1 s=READY pgs=350 cs=0 l=1 rev1=1 crypto rx=0x7f04d0097a90 tx=0x7f04d0005880 comp rx=0 tx=0).stop 2024-03-19T01:58:22.257 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:22.256+0000 7f04db7d4640 1 -- 172.21.15.99:0/1764492727 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f04d41000b0 msgr2=0x7f04d4199c40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:58:22.258 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:22.256+0000 7f04db7d4640 1 --2- 172.21.15.99:0/1764492727 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f04d41000b0 0x7f04d4199c40 secure :-1 s=READY pgs=122 cs=0 l=1 rev1=1 crypto rx=0x7f04d000e9c0 tx=0x7f04d000ee80 comp rx=0 tx=0).stop 2024-03-19T01:58:22.258 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:22.256+0000 7f04db7d4640 1 -- 172.21.15.99:0/1764492727 shutdown_connections 2024-03-19T01:58:22.258 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:22.256+0000 7f04db7d4640 1 --2- 172.21.15.99:0/1764492727 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f04c00a2920 0x7f04c00a4de0 unknown :-1 s=CLOSED pgs=350 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:22.258 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:22.256+0000 7f04db7d4640 1 --2- 172.21.15.99:0/1764492727 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f04d4100a10 0x7f04d419a180 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:22.258 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:22.256+0000 7f04db7d4640 1 --2- 172.21.15.99:0/1764492727 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f04d41013b0 0x7f04d41a11b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:22.258 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:22.256+0000 7f04db7d4640 1 --2- 172.21.15.99:0/1764492727 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f04d41000b0 0x7f04d4199c40 unknown :-1 s=CLOSED pgs=122 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:22.258 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:22.256+0000 7f04db7d4640 1 -- 172.21.15.99:0/1764492727 >> 172.21.15.99:0/1764492727 conn(0x7f04d40fbcb0 msgr2=0x7f04d40fd7b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:58:22.258 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:22.256+0000 7f04db7d4640 1 -- 172.21.15.99:0/1764492727 shutdown_connections 2024-03-19T01:58:22.258 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:22.256+0000 7f04db7d4640 1 -- 172.21.15.99:0/1764492727 wait complete. 2024-03-19T01:58:23.107 INFO:tasks.workunit.client.0.smithi099.stdout: 10 128331 23.09 MB/sec execute 172 sec latency 295.058 ms 2024-03-19T01:58:23.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:58:23 smithi112 ceph-mon[66819]: pgmap v870: 97 pgs: 97 active+clean; 567 MiB data, 6.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.7 MiB/s rd, 6.8 MiB/s wr, 598 op/s 2024-03-19T01:58:23.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:58:23 smithi115 ceph-mon[67332]: pgmap v870: 97 pgs: 97 active+clean; 567 MiB data, 6.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.7 MiB/s rd, 6.8 MiB/s wr, 598 op/s 2024-03-19T01:58:23.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:58:23 smithi099 ceph-mon[62951]: pgmap v870: 97 pgs: 97 active+clean; 567 MiB data, 6.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.7 MiB/s rd, 6.8 MiB/s wr, 598 op/s 2024-03-19T01:58:24.107 INFO:tasks.workunit.client.0.smithi099.stdout: 10 128870 23.12 MB/sec execute 173 sec latency 61.124 ms 2024-03-19T01:58:25.108 INFO:tasks.workunit.client.0.smithi099.stdout: 10 129292 23.16 MB/sec execute 174 sec latency 194.656 ms 2024-03-19T01:58:25.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:58:25 smithi112 ceph-mon[66819]: pgmap v871: 97 pgs: 97 active+clean; 568 MiB data, 6.7 GiB used, 1.0 TiB / 1.0 TiB avail; 3.9 MiB/s rd, 5.0 MiB/s wr, 440 op/s 2024-03-19T01:58:25.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:58:25 smithi115 ceph-mon[67332]: pgmap v871: 97 pgs: 97 active+clean; 568 MiB data, 6.7 GiB used, 1.0 TiB / 1.0 TiB avail; 3.9 MiB/s rd, 5.0 MiB/s wr, 440 op/s 2024-03-19T01:58:25.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:58:25 smithi099 ceph-mon[62951]: pgmap v871: 97 pgs: 97 active+clean; 568 MiB data, 6.7 GiB used, 1.0 TiB / 1.0 TiB avail; 3.9 MiB/s rd, 5.0 MiB/s wr, 440 op/s 2024-03-19T01:58:26.108 INFO:tasks.workunit.client.0.smithi099.stdout: 10 129716 23.16 MB/sec execute 175 sec latency 89.926 ms 2024-03-19T01:58:27.108 INFO:tasks.workunit.client.0.smithi099.stdout: 10 130032 23.12 MB/sec execute 176 sec latency 122.046 ms 2024-03-19T01:58:27.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:58:27 smithi112 ceph-mon[66819]: pgmap v872: 97 pgs: 97 active+clean; 559 MiB data, 6.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.5 MiB/s rd, 6.1 MiB/s wr, 559 op/s 2024-03-19T01:58:27.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:58:27 smithi115 ceph-mon[67332]: pgmap v872: 97 pgs: 97 active+clean; 559 MiB data, 6.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.5 MiB/s rd, 6.1 MiB/s wr, 559 op/s 2024-03-19T01:58:27.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:58:27 smithi099 ceph-mon[62951]: pgmap v872: 97 pgs: 97 active+clean; 559 MiB data, 6.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.5 MiB/s rd, 6.1 MiB/s wr, 559 op/s 2024-03-19T01:58:28.108 INFO:tasks.workunit.client.0.smithi099.stdout: 10 130370 23.09 MB/sec execute 177 sec latency 117.665 ms 2024-03-19T01:58:29.108 INFO:tasks.workunit.client.0.smithi099.stdout: 10 130779 23.09 MB/sec execute 178 sec latency 72.072 ms 2024-03-19T01:58:29.358 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:58:29 smithi115 ceph-mon[67332]: pgmap v873: 97 pgs: 97 active+clean; 559 MiB data, 6.7 GiB used, 1.0 TiB / 1.0 TiB avail; 3.9 MiB/s rd, 5.6 MiB/s wr, 516 op/s 2024-03-19T01:58:29.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:58:29 smithi112 ceph-mon[66819]: pgmap v873: 97 pgs: 97 active+clean; 559 MiB data, 6.7 GiB used, 1.0 TiB / 1.0 TiB avail; 3.9 MiB/s rd, 5.6 MiB/s wr, 516 op/s 2024-03-19T01:58:29.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:58:29 smithi099 ceph-mon[62951]: pgmap v873: 97 pgs: 97 active+clean; 559 MiB data, 6.7 GiB used, 1.0 TiB / 1.0 TiB avail; 3.9 MiB/s rd, 5.6 MiB/s wr, 516 op/s 2024-03-19T01:58:30.108 INFO:tasks.workunit.client.0.smithi099.stdout: 10 131151 23.03 MB/sec execute 179 sec latency 239.026 ms 2024-03-19T01:58:31.108 INFO:tasks.workunit.client.0.smithi099.stdout: 10 131656 23.06 MB/sec execute 180 sec latency 110.353 ms 2024-03-19T01:58:31.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:58:31 smithi112 ceph-mon[66819]: pgmap v874: 97 pgs: 97 active+clean; 567 MiB data, 6.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 6.4 MiB/s wr, 565 op/s 2024-03-19T01:58:31.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:58:31 smithi115 ceph-mon[67332]: pgmap v874: 97 pgs: 97 active+clean; 567 MiB data, 6.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 6.4 MiB/s wr, 565 op/s 2024-03-19T01:58:31.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:58:31 smithi099 ceph-mon[62951]: pgmap v874: 97 pgs: 97 active+clean; 567 MiB data, 6.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 6.4 MiB/s wr, 565 op/s 2024-03-19T01:58:32.108 INFO:tasks.workunit.client.0.smithi099.stdout: 10 132106 23.06 MB/sec execute 181 sec latency 96.752 ms 2024-03-19T01:58:32.806 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:58:32 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:58:32.840 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:58:32 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:58:32.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:58:32 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:58:33.108 INFO:tasks.workunit.client.0.smithi099.stdout: 10 132592 23.07 MB/sec execute 182 sec latency 102.595 ms 2024-03-19T01:58:33.806 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:58:33 smithi115 ceph-mon[67332]: pgmap v875: 97 pgs: 97 active+clean; 568 MiB data, 6.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.7 MiB/s rd, 6.9 MiB/s wr, 609 op/s 2024-03-19T01:58:33.806 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:58:33 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:58:33.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:58:33 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:58:33.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:58:33 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:58:33.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:58:33 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:58:33.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:58:33 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:58:33.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:58:33 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.10", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:58:33.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:58:33 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:58:33.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:58:33 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.9", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:58:33.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:58:33 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:58:33.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:58:33 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:58:33.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:58:33 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:58:33.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:58:33 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.11", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:58:33.808 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:58:33 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:58:33.808 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:58:33 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:58:33.808 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:58:33 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:58:33.840 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:58:33 smithi099 ceph-mon[62951]: pgmap v875: 97 pgs: 97 active+clean; 568 MiB data, 6.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.7 MiB/s rd, 6.9 MiB/s wr, 609 op/s 2024-03-19T01:58:33.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:58:33 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:58:33.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:58:33 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:58:33.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:58:33 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:58:33.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:58:33 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:58:33.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:58:33 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:58:33.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:58:33 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.10", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:58:33.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:58:33 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:58:33.842 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:58:33 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.9", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:58:33.842 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:58:33 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:58:33.842 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:58:33 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:58:33.842 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:58:33 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:58:33.842 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:58:33 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.11", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:58:33.842 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:58:33 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:58:33.842 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:58:33 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:58:33.842 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:58:33 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:58:33.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:58:33 smithi112 ceph-mon[66819]: pgmap v875: 97 pgs: 97 active+clean; 568 MiB data, 6.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.7 MiB/s rd, 6.9 MiB/s wr, 609 op/s 2024-03-19T01:58:33.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:58:33 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:58:33.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:58:33 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:58:33.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:58:33 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:58:33.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:58:33 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:58:33.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:58:33 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:58:33.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:58:33 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.10", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:58:33.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:58:33 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:58:33.863 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:58:33 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.9", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:58:33.864 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:58:33 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:58:33.864 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:58:33 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:58:33.864 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:58:33 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:58:33.864 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:58:33 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.11", "name": "osd_memory_target"}]: dispatch 2024-03-19T01:58:33.864 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:58:33 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:58:33.864 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:58:33 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:58:33.865 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:58:33 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:58:34.108 INFO:tasks.workunit.client.0.smithi099.stdout: 10 133144 23.13 MB/sec execute 183 sec latency 72.510 ms 2024-03-19T01:58:34.806 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:58:34 smithi115 ceph-mon[67332]: Adjusting osd_memory_target on smithi112 to 174.1M 2024-03-19T01:58:34.806 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:58:34 smithi115 ceph-mon[67332]: Unable to set osd_memory_target on smithi112 to 182611968: error parsing value: Value '182611968' is below minimum 939524096 2024-03-19T01:58:34.806 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:58:34 smithi115 ceph-mon[67332]: Adjusting osd_memory_target on smithi099 to 176.2M 2024-03-19T01:58:34.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:58:34 smithi115 ceph-mon[67332]: Unable to set osd_memory_target on smithi099 to 184763801: error parsing value: Value '184763801' is below minimum 939524096 2024-03-19T01:58:34.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:58:34 smithi115 ceph-mon[67332]: Adjusting osd_memory_target on smithi115 to 174.1M 2024-03-19T01:58:34.807 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:58:34 smithi115 ceph-mon[67332]: Unable to set osd_memory_target on smithi115 to 182618419: error parsing value: Value '182618419' is below minimum 939524096 2024-03-19T01:58:34.840 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:58:34 smithi099 ceph-mon[62951]: Adjusting osd_memory_target on smithi112 to 174.1M 2024-03-19T01:58:34.840 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:58:34 smithi099 ceph-mon[62951]: Unable to set osd_memory_target on smithi112 to 182611968: error parsing value: Value '182611968' is below minimum 939524096 2024-03-19T01:58:34.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:58:34 smithi099 ceph-mon[62951]: Adjusting osd_memory_target on smithi099 to 176.2M 2024-03-19T01:58:34.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:58:34 smithi099 ceph-mon[62951]: Unable to set osd_memory_target on smithi099 to 184763801: error parsing value: Value '184763801' is below minimum 939524096 2024-03-19T01:58:34.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:58:34 smithi099 ceph-mon[62951]: Adjusting osd_memory_target on smithi115 to 174.1M 2024-03-19T01:58:34.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:58:34 smithi099 ceph-mon[62951]: Unable to set osd_memory_target on smithi115 to 182618419: error parsing value: Value '182618419' is below minimum 939524096 2024-03-19T01:58:34.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:58:34 smithi112 ceph-mon[66819]: Adjusting osd_memory_target on smithi112 to 174.1M 2024-03-19T01:58:34.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:58:34 smithi112 ceph-mon[66819]: Unable to set osd_memory_target on smithi112 to 182611968: error parsing value: Value '182611968' is below minimum 939524096 2024-03-19T01:58:34.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:58:34 smithi112 ceph-mon[66819]: Adjusting osd_memory_target on smithi099 to 176.2M 2024-03-19T01:58:34.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:58:34 smithi112 ceph-mon[66819]: Unable to set osd_memory_target on smithi099 to 184763801: error parsing value: Value '184763801' is below minimum 939524096 2024-03-19T01:58:34.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:58:34 smithi112 ceph-mon[66819]: Adjusting osd_memory_target on smithi115 to 174.1M 2024-03-19T01:58:34.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:58:34 smithi112 ceph-mon[66819]: Unable to set osd_memory_target on smithi115 to 182618419: error parsing value: Value '182618419' is below minimum 939524096 2024-03-19T01:58:35.108 INFO:tasks.workunit.client.0.smithi099.stdout: 10 133544 23.11 MB/sec execute 184 sec latency 290.449 ms 2024-03-19T01:58:35.366 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.365+0000 7fa7d3f60640 1 -- 172.21.15.99:0/625431890 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa7cc107e80 msgr2=0x7fa7cc1082a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:58:35.366 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.365+0000 7fa7d3f60640 1 --2- 172.21.15.99:0/625431890 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa7cc107e80 0x7fa7cc1082a0 secure :-1 s=READY pgs=186 cs=0 l=1 rev1=1 crypto rx=0x7fa7bc009900 tx=0x7fa7bc02bc80 comp rx=0 tx=0).stop 2024-03-19T01:58:35.366 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.366+0000 7fa7d3f60640 1 -- 172.21.15.99:0/625431890 shutdown_connections 2024-03-19T01:58:35.366 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.366+0000 7fa7d3f60640 1 --2- 172.21.15.99:0/625431890 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa7cc109990 0x7fa7cc1102b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:35.366 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.366+0000 7fa7d3f60640 1 --2- 172.21.15.99:0/625431890 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa7cc108fd0 0x7fa7cc109450 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:35.366 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.366+0000 7fa7d3f60640 1 --2- 172.21.15.99:0/625431890 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa7cc107e80 0x7fa7cc1082a0 unknown :-1 s=CLOSED pgs=186 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:35.366 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.366+0000 7fa7d3f60640 1 -- 172.21.15.99:0/625431890 >> 172.21.15.99:0/625431890 conn(0x7fa7cc076e20 msgr2=0x7fa7cc079260 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:58:35.366 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.366+0000 7fa7d3f60640 1 -- 172.21.15.99:0/625431890 shutdown_connections 2024-03-19T01:58:35.367 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.366+0000 7fa7d3f60640 1 -- 172.21.15.99:0/625431890 wait complete. 2024-03-19T01:58:35.367 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.367+0000 7fa7d3f60640 1 Processor -- start 2024-03-19T01:58:35.368 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.367+0000 7fa7d3f60640 1 -- start start 2024-03-19T01:58:35.368 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.367+0000 7fa7d3f60640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa7cc107e80 0x7fa7cc19e210 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:35.368 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.367+0000 7fa7d3f60640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa7cc108fd0 0x7fa7cc19e750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:35.368 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.367+0000 7fa7d3f60640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa7cc109990 0x7fa7cc1a57c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:35.368 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.367+0000 7fa7d3f60640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fa7cc113c90 con 0x7fa7cc109990 2024-03-19T01:58:35.368 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.367+0000 7fa7d3f60640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fa7cc113b10 con 0x7fa7cc107e80 2024-03-19T01:58:35.368 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.367+0000 7fa7d3f60640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fa7cc113e10 con 0x7fa7cc108fd0 2024-03-19T01:58:35.368 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.367+0000 7fa7d24fd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa7cc109990 0x7fa7cc1a57c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:35.368 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.367+0000 7fa7d24fd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa7cc109990 0x7fa7cc1a57c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:34028/0 (socket says 172.21.15.99:34028) 2024-03-19T01:58:35.368 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.367+0000 7fa7d24fd640 1 -- 172.21.15.99:0/60490931 learned_addr learned my addr 172.21.15.99:0/60490931 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:58:35.368 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.368+0000 7fa7d24fd640 1 -- 172.21.15.99:0/60490931 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa7cc108fd0 msgr2=0x7fa7cc19e750 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:58:35.368 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.368+0000 7fa7d24fd640 1 --2- 172.21.15.99:0/60490931 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa7cc108fd0 0x7fa7cc19e750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:35.369 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.368+0000 7fa7d24fd640 1 -- 172.21.15.99:0/60490931 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa7cc107e80 msgr2=0x7fa7cc19e210 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-03-19T01:58:35.369 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.368+0000 7fa7d24fd640 1 --2- 172.21.15.99:0/60490931 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa7cc107e80 0x7fa7cc19e210 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:35.369 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.368+0000 7fa7d24fd640 1 -- 172.21.15.99:0/60490931 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa7cc1a5e30 con 0x7fa7cc109990 2024-03-19T01:58:35.369 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.368+0000 7fa7d24fd640 1 --2- 172.21.15.99:0/60490931 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa7cc109990 0x7fa7cc1a57c0 secure :-1 s=READY pgs=677 cs=0 l=1 rev1=1 crypto rx=0x7fa7c400b970 tx=0x7fa7c400be30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:58:35.369 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.368+0000 7fa7c2ffd640 1 -- 172.21.15.99:0/60490931 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa7c4019070 con 0x7fa7cc109990 2024-03-19T01:58:35.370 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.368+0000 7fa7c2ffd640 1 -- 172.21.15.99:0/60490931 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa7c400cb40 con 0x7fa7cc109990 2024-03-19T01:58:35.370 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.368+0000 7fa7c2ffd640 1 -- 172.21.15.99:0/60490931 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa7c40076e0 con 0x7fa7cc109990 2024-03-19T01:58:35.370 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.368+0000 7fa7d3f60640 1 -- 172.21.15.99:0/60490931 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa7cc1a6120 con 0x7fa7cc109990 2024-03-19T01:58:35.370 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.369+0000 7fa7d3f60640 1 -- 172.21.15.99:0/60490931 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fa7cc1a6620 con 0x7fa7cc109990 2024-03-19T01:58:35.371 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.372+0000 7fa7d6c20740 1 Processor -- start 2024-03-19T01:58:35.373 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.374+0000 7fa7c2ffd640 1 -- 172.21.15.99:0/60490931 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7fa7c401c070 con 0x7fa7cc109990 2024-03-19T01:58:35.376 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.374+0000 7fa7d6c20740 1 -- start start 2024-03-19T01:58:35.376 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.374+0000 7fa7d6c20740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562e2d134100 0x562e2d134520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:35.376 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.374+0000 7fa7d6c20740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562e2d134af0 0x562e2d1395f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:35.376 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.374+0000 7fa7d6c20740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562e2d139c60 0x562e2d13c0f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:35.376 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.374+0000 7fa7d6c20740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x562e2d098be0 con 0x562e2d139c60 2024-03-19T01:58:35.376 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.374+0000 7fa7d6c20740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x562e2d041960 con 0x562e2d134100 2024-03-19T01:58:35.376 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.374+0000 7fa7d6c20740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x562e2d094c80 con 0x562e2d134af0 2024-03-19T01:58:35.376 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.375+0000 7fa7d1cfc640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562e2d134af0 0x562e2d1395f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:35.376 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.375+0000 7fa7d1cfc640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562e2d134af0 0x562e2d1395f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:35916/0 (socket says 172.21.15.99:35916) 2024-03-19T01:58:35.376 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.375+0000 7fa7d1cfc640 1 -- 172.21.15.99:0/3371381032 learned_addr learned my addr 172.21.15.99:0/3371381032 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:58:35.377 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.375+0000 7fa7d14fb640 1 --2- 172.21.15.99:0/3371381032 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562e2d134100 0x562e2d134520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:35.377 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.375+0000 7fa7d14fb640 1 --2- 172.21.15.99:0/3371381032 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562e2d139c60 0x562e2d13c0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:35.377 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.375+0000 7fa7d1cfc640 1 -- 172.21.15.99:0/3371381032 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562e2d134100 msgr2=0x562e2d134520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:58:35.377 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.375+0000 7fa7d1cfc640 1 --2- 172.21.15.99:0/3371381032 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562e2d134100 0x562e2d134520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:35.377 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.375+0000 7fa7d1cfc640 1 -- 172.21.15.99:0/3371381032 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562e2d139c60 msgr2=0x562e2d13c0f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:58:35.377 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.375+0000 7fa7d1cfc640 1 --2- 172.21.15.99:0/3371381032 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562e2d139c60 0x562e2d13c0f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:35.377 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.375+0000 7fa7d1cfc640 1 -- 172.21.15.99:0/3371381032 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x562e2d038d20 con 0x562e2d134af0 2024-03-19T01:58:35.377 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.375+0000 7fa7c2ffd640 1 --2- 172.21.15.99:0/60490931 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa7b80a2a90 0x7fa7b80a4f50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:35.377 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.376+0000 7fa7c2ffd640 1 -- 172.21.15.99:0/60490931 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fa7c40ce030 con 0x7fa7cc109990 2024-03-19T01:58:35.377 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.376+0000 7fa7d14fb640 1 --2- 172.21.15.99:0/60490931 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa7b80a2a90 0x7fa7b80a4f50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:35.377 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.376+0000 7fa7d1cfc640 1 --2- 172.21.15.99:0/3371381032 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562e2d134af0 0x562e2d1395f0 secure :-1 s=READY pgs=125 cs=0 l=1 rev1=1 crypto rx=0x7fa7bc000c00 tx=0x7fa7bc02c720 comp rx=0 tx=0).ready entity=mon.1 client_cookie=90daf80dec3d3e2d server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:58:35.378 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.376+0000 7fa7b7fff640 1 -- 172.21.15.99:0/3371381032 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa7bc041070 con 0x562e2d134af0 2024-03-19T01:58:35.378 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.376+0000 7fa7b7fff640 1 -- 172.21.15.99:0/3371381032 <== mon.1 v2:172.21.15.115:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa7bc02cbd0 con 0x562e2d134af0 2024-03-19T01:58:35.378 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.376+0000 7fa7b7fff640 1 -- 172.21.15.99:0/3371381032 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa7bc004920 con 0x562e2d134af0 2024-03-19T01:58:35.378 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.376+0000 7fa7d14fb640 1 --2- 172.21.15.99:0/60490931 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa7b80a2a90 0x7fa7b80a4f50 secure :-1 s=READY pgs=351 cs=0 l=1 rev1=1 crypto rx=0x562e2d135350 tx=0x7fa7c8001140 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:58:35.381 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.377+0000 7fa7d6c20740 1 -- 172.21.15.99:0/3371381032 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562e2d134af0 msgr2=0x562e2d1395f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:58:35.381 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.377+0000 7fa7d6c20740 1 --2- 172.21.15.99:0/3371381032 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562e2d134af0 0x562e2d1395f0 secure :-1 s=READY pgs=125 cs=0 l=1 rev1=1 crypto rx=0x7fa7bc000c00 tx=0x7fa7bc02c720 comp rx=0 tx=0).stop 2024-03-19T01:58:35.381 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.377+0000 7fa7d6c20740 1 -- 172.21.15.99:0/3371381032 shutdown_connections 2024-03-19T01:58:35.381 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.377+0000 7fa7d6c20740 1 --2- 172.21.15.99:0/3371381032 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562e2d139c60 0x562e2d13c0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:35.382 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.377+0000 7fa7d6c20740 1 --2- 172.21.15.99:0/3371381032 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562e2d134af0 0x562e2d1395f0 unknown :-1 s=CLOSED pgs=125 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:35.383 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.377+0000 7fa7d6c20740 1 --2- 172.21.15.99:0/3371381032 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562e2d134100 0x562e2d134520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:35.383 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.377+0000 7fa7d6c20740 1 -- 172.21.15.99:0/3371381032 >> 172.21.15.99:0/3371381032 conn(0x562e2d12f670 msgr2=0x562e2d12fa70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:58:35.383 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.377+0000 7fa7d6c20740 1 -- 172.21.15.99:0/3371381032 shutdown_connections 2024-03-19T01:58:35.383 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.377+0000 7fa7d6c20740 1 -- 172.21.15.99:0/3371381032 wait complete. 2024-03-19T01:58:35.383 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.378+0000 7fa7d6c20740 1 -- start start 2024-03-19T01:58:35.383 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.378+0000 7fa7d6c20740 1 Processor -- start 2024-03-19T01:58:35.383 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.378+0000 7fa7d6c20740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562e2d134100 0x562e2d13e100 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:35.383 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.378+0000 7fa7d6c20740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562e2d134af0 0x562e2d13e640 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:35.383 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.378+0000 7fa7d6c20740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562e2d139c60 0x562e2d145680 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:35.383 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.378+0000 7fa7d6c20740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x562e2d13cbb0 con 0x562e2d134100 2024-03-19T01:58:35.383 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.378+0000 7fa7d6c20740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x562e2d054db0 con 0x562e2d134af0 2024-03-19T01:58:35.384 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.378+0000 7fa7d6c20740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x562e2d13cd30 con 0x562e2d139c60 2024-03-19T01:58:35.384 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.378+0000 7fa7d6c20740 10 client.0 _ll_register_callbacks cb 0x562e2d059e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:58:35.384 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.378+0000 7fa7d1cfc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562e2d134100 0x562e2d13e100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:35.384 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.378+0000 7fa7d24fd640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562e2d139c60 0x562e2d145680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:35.384 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.378+0000 7fa7d14fb640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562e2d134af0 0x562e2d13e640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:35.384 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.378+0000 7fa7d24fd640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562e2d139c60 0x562e2d145680 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:35932/0 (socket says 172.21.15.99:35932) 2024-03-19T01:58:35.384 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.378+0000 7fa7d24fd640 1 -- 172.21.15.99:0/1893524498 learned_addr learned my addr 172.21.15.99:0/1893524498 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:58:35.384 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.378+0000 7fa7d14fb640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562e2d134af0 0x562e2d13e640 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:51748/0 (socket says 172.21.15.99:51748) 2024-03-19T01:58:35.384 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.379+0000 7fa7d1cfc640 1 -- 172.21.15.99:0/1893524498 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562e2d139c60 msgr2=0x562e2d145680 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:58:35.384 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.379+0000 7fa7d1cfc640 1 --2- 172.21.15.99:0/1893524498 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562e2d139c60 0x562e2d145680 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:35.384 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.379+0000 7fa7d1cfc640 1 -- 172.21.15.99:0/1893524498 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562e2d134af0 msgr2=0x562e2d13e640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:58:35.384 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.379+0000 7fa7d1cfc640 1 --2- 172.21.15.99:0/1893524498 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562e2d134af0 0x562e2d13e640 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:35.384 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.379+0000 7fa7d1cfc640 1 -- 172.21.15.99:0/1893524498 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x562e2d053c60 con 0x562e2d134100 2024-03-19T01:58:35.384 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.379+0000 7fa7d1cfc640 1 --2- 172.21.15.99:0/1893524498 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562e2d134100 0x562e2d13e100 secure :-1 s=READY pgs=678 cs=0 l=1 rev1=1 crypto rx=0x562e2d1305e0 tx=0x7fa7bc0048f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3c715cc97b40b00b server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:58:35.385 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.379+0000 7fa7b57fa640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:58:35.385 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.379+0000 7fa7b57fa640 1 -- 172.21.15.99:0/1893524498 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa7bc041040 con 0x562e2d134100 2024-03-19T01:58:35.385 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.379+0000 7fa7d6c20740 1 -- 172.21.15.99:0/1893524498 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fa7bc02cbd0 con 0x562e2d134100 2024-03-19T01:58:35.385 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.379+0000 7fa7b57fa640 1 -- 172.21.15.99:0/1893524498 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fa7bc004c70 con 0x562e2d134100 2024-03-19T01:58:35.385 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.379+0000 7fa7b57fa640 1 -- 172.21.15.99:0/1893524498 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7fa7bc02cd80 con 0x562e2d134100 2024-03-19T01:58:35.385 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.379+0000 7fa7d6c20740 10 client.16740 fetch_fsmap learned FSMap version 37 2024-03-19T01:58:35.385 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.379+0000 7fa7d6c20740 1 -- 172.21.15.99:0/1893524498 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x562e2cdad210 con 0x562e2d134100 2024-03-19T01:58:35.385 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.380+0000 7fa7b57fa640 1 -- 172.21.15.99:0/1893524498 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa7bc02cbd0 con 0x562e2d134100 2024-03-19T01:58:35.385 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.380+0000 7fa7b57fa640 1 -- 172.21.15.99:0/1893524498 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7fa7bc0345e0 con 0x562e2d134100 2024-03-19T01:58:35.385 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.380+0000 7fa7d6c20740 10 client.16740 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T01:58:35.385 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.380+0000 7fa7d6c20740 10 client.16740 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:58:35.385 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.380+0000 7fa7d6c20740 20 client.16740 populate_metadata read hostname 'smithi099' 2024-03-19T01:58:35.385 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.380+0000 7fa7d6c20740 1 --2- 172.21.15.99:0/1893524498 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x562e2d149fb0 0x562e2d14c450 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:35.386 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.380+0000 7fa7d6c20740 4 client.16740 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:58:35.386 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.380+0000 7fa7d6c20740 1 -- 172.21.15.99:0/1893524498 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x562e2d0a6120 con 0x562e2d149fb0 2024-03-19T01:58:35.386 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.380+0000 7fa7d14fb640 1 --2- 172.21.15.99:0/1893524498 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x562e2d149fb0 0x562e2d14c450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:35.386 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.381+0000 7fa7b57fa640 10 client.16740 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:58:35.386 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.381+0000 7fa7d14fb640 1 --2- 172.21.15.99:0/1893524498 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x562e2d149fb0 0x562e2d14c450 crc :-1 s=READY pgs=185 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=cc60cdfd1d676443 server_cookie=63d6a3dfb1e86bc2 in_seq=0 out_seq=0 2024-03-19T01:58:35.387 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.383+0000 7fa7b57fa640 1 -- 172.21.15.99:0/1893524498 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x562e2d0a6120 con 0x562e2d149fb0 2024-03-19T01:58:35.387 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.383+0000 7fa7b57fa640 10 client.16740 handle_command_reply: tid=0 2024-03-19T01:58:35.387 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.383+0000 7fa7d6c20740 1 client.16740 shutdown 2024-03-19T01:58:35.387 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.383+0000 7fa7d6c20740 1 -- 172.21.15.99:0/1893524498 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562e2d134100 msgr2=0x562e2d13e100 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:58:35.387 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.383+0000 7fa7d6c20740 1 --2- 172.21.15.99:0/1893524498 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562e2d134100 0x562e2d13e100 secure :-1 s=READY pgs=678 cs=0 l=1 rev1=1 crypto rx=0x562e2d1305e0 tx=0x7fa7bc0048f0 comp rx=0 tx=0).stop 2024-03-19T01:58:35.387 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.384+0000 7fa7d6c20740 1 -- 172.21.15.99:0/1893524498 shutdown_connections 2024-03-19T01:58:35.388 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.384+0000 7fa7d6c20740 1 --2- 172.21.15.99:0/1893524498 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x562e2d149fb0 0x562e2d14c450 crc :-1 s=READY pgs=185 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:35.388 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.384+0000 7fa7d6c20740 1 --2- 172.21.15.99:0/1893524498 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562e2d134af0 0x562e2d13e640 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:35.388 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.384+0000 7fa7d6c20740 1 --2- 172.21.15.99:0/1893524498 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562e2d139c60 0x562e2d145680 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:35.388 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.384+0000 7fa7d6c20740 1 --2- 172.21.15.99:0/1893524498 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562e2d134100 0x562e2d13e100 unknown :-1 s=CLOSED pgs=678 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:35.388 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.384+0000 7fa7d6c20740 1 -- 172.21.15.99:0/1893524498 >> 172.21.15.99:0/1893524498 conn(0x562e2d12f670 msgr2=0x562e2d1308c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:58:35.388 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.384+0000 7fa7b57fa640 0 client.16740 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:58:35.388 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.384+0000 7fa7d6c20740 1 -- 172.21.15.99:0/1893524498 shutdown_connections 2024-03-19T01:58:35.388 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.384+0000 7fa7d6c20740 1 -- 172.21.15.99:0/1893524498 wait complete. 2024-03-19T01:58:35.388 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.384+0000 7fa7d6c20740 20 client.16740 trim_cache size 0 max 16384 2024-03-19T01:58:35.402 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.402+0000 7fa7d6c20740 1 Processor -- start 2024-03-19T01:58:35.403 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.402+0000 7fa7d6c20740 1 -- start start 2024-03-19T01:58:35.403 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.402+0000 7fa7d6c20740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562e2d134100 0x562e2d13d720 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:35.403 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.402+0000 7fa7d6c20740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562e2d134af0 0x562e2d13dc60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:35.403 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.402+0000 7fa7d6c20740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562e2d139c60 0x562e2d13e1a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:35.403 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.402+0000 7fa7d14fb640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562e2d134af0 0x562e2d13dc60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:35.403 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.402+0000 7fa7d1cfc640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562e2d134100 0x562e2d13d720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:35.403 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.402+0000 7fa7d6c20740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fa7b0010e70 con 0x562e2d134af0 2024-03-19T01:58:35.403 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.402+0000 7fa7d14fb640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562e2d134af0 0x562e2d13dc60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:34052/0 (socket says 172.21.15.99:34052) 2024-03-19T01:58:35.403 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.402+0000 7fa7d6c20740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fa7b000e740 con 0x562e2d134100 2024-03-19T01:58:35.403 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.402+0000 7fa7d6c20740 1 -- 172.21.15.99:0/1754502828 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fa7b000c590 con 0x562e2d139c60 2024-03-19T01:58:35.403 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.402+0000 7fa7d1cfc640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562e2d134100 0x562e2d13d720 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:51758/0 (socket says 172.21.15.99:51758) 2024-03-19T01:58:35.403 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.402+0000 7fa7d14fb640 1 -- 172.21.15.99:0/1754502828 learned_addr learned my addr 172.21.15.99:0/1754502828 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:58:35.403 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.402+0000 7fa7d24fd640 1 --2- 172.21.15.99:0/1754502828 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562e2d139c60 0x562e2d13e1a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:35.403 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.402+0000 7fa7d1cfc640 1 -- 172.21.15.99:0/1754502828 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562e2d139c60 msgr2=0x562e2d13e1a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:58:35.404 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.402+0000 7fa7d1cfc640 1 --2- 172.21.15.99:0/1754502828 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562e2d139c60 0x562e2d13e1a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:35.404 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.402+0000 7fa7d1cfc640 1 -- 172.21.15.99:0/1754502828 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562e2d134af0 msgr2=0x562e2d13dc60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:58:35.404 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.402+0000 7fa7d1cfc640 1 --2- 172.21.15.99:0/1754502828 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562e2d134af0 0x562e2d13dc60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:35.404 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.402+0000 7fa7d1cfc640 1 -- 172.21.15.99:0/1754502828 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x562e2ce79620 con 0x562e2d134100 2024-03-19T01:58:35.404 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.403+0000 7fa7d1cfc640 1 --2- 172.21.15.99:0/1754502828 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562e2d134100 0x562e2d13d720 secure :-1 s=READY pgs=187 cs=0 l=1 rev1=1 crypto rx=0x7fa7bc0096c0 tx=0x7fa7bc005c00 comp rx=0 tx=0).ready entity=mon.2 client_cookie=e18ff8195f963bbb server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:58:35.404 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.403+0000 7fa7b77fe640 1 -- 172.21.15.99:0/1754502828 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa7bc005400 con 0x562e2d134100 2024-03-19T01:58:35.404 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.403+0000 7fa7b77fe640 1 -- 172.21.15.99:0/1754502828 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa7bc054030 con 0x562e2d134100 2024-03-19T01:58:35.404 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.403+0000 7fa7b77fe640 1 -- 172.21.15.99:0/1754502828 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa7bc046450 con 0x562e2d134100 2024-03-19T01:58:35.408 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.404+0000 7fa7d6c20740 1 -- 172.21.15.99:0/1754502828 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562e2d134100 msgr2=0x562e2d13d720 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:58:35.408 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.404+0000 7fa7d6c20740 1 --2- 172.21.15.99:0/1754502828 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562e2d134100 0x562e2d13d720 secure :-1 s=READY pgs=187 cs=0 l=1 rev1=1 crypto rx=0x7fa7bc0096c0 tx=0x7fa7bc005c00 comp rx=0 tx=0).stop 2024-03-19T01:58:35.408 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.404+0000 7fa7d6c20740 1 -- 172.21.15.99:0/1754502828 shutdown_connections 2024-03-19T01:58:35.408 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.404+0000 7fa7d6c20740 1 --2- 172.21.15.99:0/1754502828 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562e2d139c60 0x562e2d13e1a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:35.408 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.404+0000 7fa7d6c20740 1 --2- 172.21.15.99:0/1754502828 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562e2d134af0 0x562e2d13dc60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:35.408 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.404+0000 7fa7d6c20740 1 --2- 172.21.15.99:0/1754502828 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562e2d134100 0x562e2d13d720 unknown :-1 s=CLOSED pgs=187 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:35.409 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.404+0000 7fa7d6c20740 1 -- 172.21.15.99:0/1754502828 >> 172.21.15.99:0/1754502828 conn(0x562e2d12f670 msgr2=0x562e2d133160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:58:35.409 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.404+0000 7fa7d6c20740 1 -- 172.21.15.99:0/1754502828 shutdown_connections 2024-03-19T01:58:35.409 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.404+0000 7fa7d6c20740 1 -- 172.21.15.99:0/1754502828 wait complete. 2024-03-19T01:58:35.409 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.405+0000 7fa7d6c20740 1 -- start start 2024-03-19T01:58:35.409 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.405+0000 7fa7d6c20740 1 Processor -- start 2024-03-19T01:58:35.409 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.405+0000 7fa7d6c20740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562e2d134100 0x562e2d1426c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:35.409 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.405+0000 7fa7d1cfc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562e2d134100 0x562e2d1426c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:35.409 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.405+0000 7fa7d1cfc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562e2d134100 0x562e2d1426c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:34062/0 (socket says 172.21.15.99:34062) 2024-03-19T01:58:35.409 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.405+0000 7fa7d6c20740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562e2d134af0 0x562e2d142c00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:35.409 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.405+0000 7fa7d1cfc640 1 -- 172.21.15.99:0/2810964050 learned_addr learned my addr 172.21.15.99:0/2810964050 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:58:35.409 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.405+0000 7fa7d6c20740 1 --2- 172.21.15.99:0/2810964050 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562e2d139c60 0x562e2d143b50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:35.409 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.405+0000 7fa7d6c20740 1 -- 172.21.15.99:0/2810964050 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fa7b000cc00 con 0x562e2d134100 2024-03-19T01:58:35.409 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.405+0000 7fa7d6c20740 1 -- 172.21.15.99:0/2810964050 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fa7b000e070 con 0x562e2d139c60 2024-03-19T01:58:35.409 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.405+0000 7fa7d6c20740 1 -- 172.21.15.99:0/2810964050 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fa7b000d2d0 con 0x562e2d134af0 2024-03-19T01:58:35.410 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.406+0000 7fa7d6c20740 10 client.0 _ll_register_callbacks cb 0x7fa7c80019a0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:58:35.410 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.406+0000 7fa7d24fd640 1 --2- 172.21.15.99:0/2810964050 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562e2d139c60 0x562e2d143b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:35.410 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.406+0000 7fa7d14fb640 1 --2- 172.21.15.99:0/2810964050 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562e2d134af0 0x562e2d142c00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:35.410 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.406+0000 7fa7d1cfc640 1 -- 172.21.15.99:0/2810964050 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562e2d134af0 msgr2=0x562e2d142c00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:58:35.410 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.406+0000 7fa7d1cfc640 1 --2- 172.21.15.99:0/2810964050 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562e2d134af0 0x562e2d142c00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:35.410 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.406+0000 7fa7d1cfc640 1 -- 172.21.15.99:0/2810964050 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562e2d139c60 msgr2=0x562e2d143b50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:58:35.410 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.406+0000 7fa7d1cfc640 1 --2- 172.21.15.99:0/2810964050 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562e2d139c60 0x562e2d143b50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:35.410 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.406+0000 7fa7d1cfc640 1 -- 172.21.15.99:0/2810964050 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x562e2d144140 con 0x562e2d134100 2024-03-19T01:58:35.410 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.406+0000 7fa7d24fd640 1 --2- 172.21.15.99:0/2810964050 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562e2d139c60 0x562e2d143b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:58:35.410 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.406+0000 7fa7d14fb640 1 --2- 172.21.15.99:0/2810964050 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562e2d134af0 0x562e2d142c00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-19T01:58:35.410 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.406+0000 7fa7d1cfc640 1 --2- 172.21.15.99:0/2810964050 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562e2d134100 0x562e2d1426c0 secure :-1 s=READY pgs=679 cs=0 l=1 rev1=1 crypto rx=0x7fa7bc009900 tx=0x7fa7bc005fd0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=29de6362e72511dd server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:58:35.410 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.406+0000 7fa7b57fa640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:58:35.411 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.406+0000 7fa7b57fa640 1 -- 172.21.15.99:0/2810964050 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa7bc034440 con 0x562e2d134100 2024-03-19T01:58:35.411 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.406+0000 7fa7b57fa640 1 -- 172.21.15.99:0/2810964050 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fa7bc0345e0 con 0x562e2d134100 2024-03-19T01:58:35.411 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.406+0000 7fa7d6c20740 1 -- 172.21.15.99:0/2810964050 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fa7bc054030 con 0x562e2d134100 2024-03-19T01:58:35.411 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.406+0000 7fa7b57fa640 1 -- 172.21.15.99:0/2810964050 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7fa7bc068050 con 0x562e2d134100 2024-03-19T01:58:35.411 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.407+0000 7fa7d6c20740 10 client.16746 fetch_fsmap learned FSMap version 37 2024-03-19T01:58:35.411 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.407+0000 7fa7d6c20740 1 -- 172.21.15.99:0/2810964050 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x562e2d145430 con 0x562e2d134100 2024-03-19T01:58:35.411 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.407+0000 7fa7b57fa640 1 -- 172.21.15.99:0/2810964050 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa7bc054030 con 0x562e2d134100 2024-03-19T01:58:35.411 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.407+0000 7fa7b57fa640 1 -- 172.21.15.99:0/2810964050 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7fa7bc046910 con 0x562e2d134100 2024-03-19T01:58:35.411 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.407+0000 7fa7d6c20740 10 client.16746 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T01:58:35.411 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.407+0000 7fa7d6c20740 10 client.16746 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:58:35.411 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.407+0000 7fa7d6c20740 20 client.16746 populate_metadata read hostname 'smithi099' 2024-03-19T01:58:35.411 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.407+0000 7fa7d6c20740 1 --2- 172.21.15.99:0/2810964050 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x562e2d1b0880 0x562e2d1b2ca0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:35.411 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.407+0000 7fa7d6c20740 4 client.16746 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:58:35.411 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.407+0000 7fa7d6c20740 1 -- 172.21.15.99:0/2810964050 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x562e2d0a0470 con 0x562e2d1b0880 2024-03-19T01:58:35.412 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.407+0000 7fa7d14fb640 1 --2- 172.21.15.99:0/2810964050 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x562e2d1b0880 0x562e2d1b2ca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:35.412 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.408+0000 7fa7d14fb640 1 --2- 172.21.15.99:0/2810964050 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x562e2d1b0880 0x562e2d1b2ca0 crc :-1 s=READY pgs=186 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=93dac47a2fc99c78 server_cookie=90b012bbec7e400 in_seq=0 out_seq=0 2024-03-19T01:58:35.412 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.408+0000 7fa7b57fa640 10 client.16746 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:58:35.806 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:58:35 smithi115 ceph-mon[67332]: pgmap v876: 97 pgs: 97 active+clean; 573 MiB data, 6.8 GiB used, 1.0 TiB / 1.0 TiB avail; 3.7 MiB/s rd, 5.7 MiB/s wr, 468 op/s 2024-03-19T01:58:35.840 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:58:35 smithi099 ceph-mon[62951]: pgmap v876: 97 pgs: 97 active+clean; 573 MiB data, 6.8 GiB used, 1.0 TiB / 1.0 TiB avail; 3.7 MiB/s rd, 5.7 MiB/s wr, 468 op/s 2024-03-19T01:58:35.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:58:35 smithi112 ceph-mon[66819]: pgmap v876: 97 pgs: 97 active+clean; 573 MiB data, 6.8 GiB used, 1.0 TiB / 1.0 TiB avail; 3.7 MiB/s rd, 5.7 MiB/s wr, 468 op/s 2024-03-19T01:58:35.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.997+0000 7fa7b57fa640 1 -- 172.21.15.99:0/2810964050 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x562e2d0a0470 con 0x562e2d1b0880 2024-03-19T01:58:35.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.997+0000 7fa7b57fa640 10 client.16746 handle_command_reply: tid=0 2024-03-19T01:58:35.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.997+0000 7fa7d6c20740 1 client.16746 shutdown 2024-03-19T01:58:35.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.997+0000 7fa7d6c20740 1 -- 172.21.15.99:0/2810964050 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562e2d134100 msgr2=0x562e2d1426c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:58:35.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.997+0000 7fa7d6c20740 1 --2- 172.21.15.99:0/2810964050 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562e2d134100 0x562e2d1426c0 secure :-1 s=READY pgs=679 cs=0 l=1 rev1=1 crypto rx=0x7fa7bc009900 tx=0x7fa7bc005fd0 comp rx=0 tx=0).stop 2024-03-19T01:58:35.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.997+0000 7fa7d6c20740 1 -- 172.21.15.99:0/2810964050 shutdown_connections 2024-03-19T01:58:35.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.997+0000 7fa7d6c20740 1 --2- 172.21.15.99:0/2810964050 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x562e2d1b0880 0x562e2d1b2ca0 crc :-1 s=READY pgs=186 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:35.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.997+0000 7fa7d6c20740 1 --2- 172.21.15.99:0/2810964050 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562e2d139c60 0x562e2d143b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:35.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.997+0000 7fa7b57fa640 0 client.16746 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:58:35.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.997+0000 7fa7d6c20740 1 --2- 172.21.15.99:0/2810964050 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562e2d134af0 0x562e2d142c00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:35.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.997+0000 7fa7d6c20740 1 --2- 172.21.15.99:0/2810964050 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562e2d134100 0x562e2d1426c0 unknown :-1 s=CLOSED pgs=679 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:35.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.997+0000 7fa7d6c20740 1 -- 172.21.15.99:0/2810964050 >> 172.21.15.99:0/2810964050 conn(0x562e2d12f670 msgr2=0x562e2d1354d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:58:35.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.997+0000 7fa7d6c20740 1 -- 172.21.15.99:0/2810964050 shutdown_connections 2024-03-19T01:58:36.000 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.997+0000 7fa7d6c20740 1 -- 172.21.15.99:0/2810964050 wait complete. 2024-03-19T01:58:36.000 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.998+0000 7fa7d6c20740 20 client.16746 trim_cache size 0 max 16384 2024-03-19T01:58:36.000 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:58:36.000 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:58:36.000 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:58:36.000 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:58:36.002 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.998+0000 7fa7d3f60640 1 -- 172.21.15.99:0/60490931 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa7b80a2a90 msgr2=0x7fa7b80a4f50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:58:36.002 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.998+0000 7fa7d3f60640 1 --2- 172.21.15.99:0/60490931 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa7b80a2a90 0x7fa7b80a4f50 secure :-1 s=READY pgs=351 cs=0 l=1 rev1=1 crypto rx=0x562e2d135350 tx=0x7fa7c8001140 comp rx=0 tx=0).stop 2024-03-19T01:58:36.002 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.998+0000 7fa7d3f60640 1 -- 172.21.15.99:0/60490931 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa7cc109990 msgr2=0x7fa7cc1a57c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:58:36.002 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.998+0000 7fa7d3f60640 1 --2- 172.21.15.99:0/60490931 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa7cc109990 0x7fa7cc1a57c0 secure :-1 s=READY pgs=677 cs=0 l=1 rev1=1 crypto rx=0x7fa7c400b970 tx=0x7fa7c400be30 comp rx=0 tx=0).stop 2024-03-19T01:58:36.003 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.998+0000 7fa7d3f60640 1 -- 172.21.15.99:0/60490931 shutdown_connections 2024-03-19T01:58:36.003 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.998+0000 7fa7d3f60640 1 --2- 172.21.15.99:0/60490931 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa7cc109990 0x7fa7cc1a57c0 unknown :-1 s=CLOSED pgs=677 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:36.003 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.998+0000 7fa7d3f60640 1 --2- 172.21.15.99:0/60490931 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa7cc108fd0 0x7fa7cc19e750 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:36.003 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.998+0000 7fa7d3f60640 1 --2- 172.21.15.99:0/60490931 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa7b80a2a90 0x7fa7b80a4f50 unknown :-1 s=CLOSED pgs=351 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:36.003 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.998+0000 7fa7d3f60640 1 --2- 172.21.15.99:0/60490931 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa7cc107e80 0x7fa7cc19e210 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:36.003 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.998+0000 7fa7d3f60640 1 -- 172.21.15.99:0/60490931 >> 172.21.15.99:0/60490931 conn(0x7fa7cc076e20 msgr2=0x7fa7cc10c300 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:58:36.004 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.998+0000 7fa7d3f60640 1 -- 172.21.15.99:0/60490931 shutdown_connections 2024-03-19T01:58:36.004 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:35.998+0000 7fa7d3f60640 1 -- 172.21.15.99:0/60490931 wait complete. 2024-03-19T01:58:36.108 INFO:tasks.workunit.client.0.smithi099.stdout: 10 133825 23.09 MB/sec execute 185 sec latency 326.764 ms 2024-03-19T01:58:37.108 INFO:tasks.workunit.client.0.smithi099.stdout: 10 134305 23.14 MB/sec execute 186 sec latency 123.874 ms 2024-03-19T01:58:37.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:58:37 smithi112 ceph-mon[66819]: pgmap v877: 97 pgs: 97 active+clean; 575 MiB data, 6.8 GiB used, 1.0 TiB / 1.0 TiB avail; 4.4 MiB/s rd, 6.7 MiB/s wr, 550 op/s 2024-03-19T01:58:37.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:58:37 smithi115 ceph-mon[67332]: pgmap v877: 97 pgs: 97 active+clean; 575 MiB data, 6.8 GiB used, 1.0 TiB / 1.0 TiB avail; 4.4 MiB/s rd, 6.7 MiB/s wr, 550 op/s 2024-03-19T01:58:37.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:58:37 smithi099 ceph-mon[62951]: pgmap v877: 97 pgs: 97 active+clean; 575 MiB data, 6.8 GiB used, 1.0 TiB / 1.0 TiB avail; 4.4 MiB/s rd, 6.7 MiB/s wr, 550 op/s 2024-03-19T01:58:38.108 INFO:tasks.workunit.client.0.smithi099.stdout: 10 134687 23.11 MB/sec execute 187 sec latency 72.868 ms 2024-03-19T01:58:39.108 INFO:tasks.workunit.client.0.smithi099.stdout: 10 135088 23.08 MB/sec execute 188 sec latency 115.431 ms 2024-03-19T01:58:39.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:58:39 smithi112 ceph-mon[66819]: pgmap v878: 97 pgs: 97 active+clean; 575 MiB data, 6.8 GiB used, 1.0 TiB / 1.0 TiB avail; 4.3 MiB/s rd, 6.2 MiB/s wr, 527 op/s 2024-03-19T01:58:39.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:58:39 smithi115 ceph-mon[67332]: pgmap v878: 97 pgs: 97 active+clean; 575 MiB data, 6.8 GiB used, 1.0 TiB / 1.0 TiB avail; 4.3 MiB/s rd, 6.2 MiB/s wr, 527 op/s 2024-03-19T01:58:39.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:58:39 smithi099 ceph-mon[62951]: pgmap v878: 97 pgs: 97 active+clean; 575 MiB data, 6.8 GiB used, 1.0 TiB / 1.0 TiB avail; 4.3 MiB/s rd, 6.2 MiB/s wr, 527 op/s 2024-03-19T01:58:40.108 INFO:tasks.workunit.client.0.smithi099.stdout: 10 135386 23.05 MB/sec execute 189 sec latency 204.956 ms 2024-03-19T01:58:41.108 INFO:tasks.workunit.client.0.smithi099.stdout: 10 135813 23.05 MB/sec execute 190 sec latency 172.595 ms 2024-03-19T01:58:41.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:58:41 smithi112 ceph-mon[66819]: pgmap v879: 97 pgs: 97 active+clean; 569 MiB data, 6.8 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 6.0 MiB/s wr, 544 op/s 2024-03-19T01:58:41.391 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:58:41 smithi099 ceph-mon[62951]: pgmap v879: 97 pgs: 97 active+clean; 569 MiB data, 6.8 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 6.0 MiB/s wr, 544 op/s 2024-03-19T01:58:41.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:58:41 smithi115 ceph-mon[67332]: pgmap v879: 97 pgs: 97 active+clean; 569 MiB data, 6.8 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 6.0 MiB/s wr, 544 op/s 2024-03-19T01:58:42.108 INFO:tasks.workunit.client.0.smithi099.stdout: 10 136314 23.05 MB/sec execute 191 sec latency 61.116 ms 2024-03-19T01:58:43.108 INFO:tasks.workunit.client.0.smithi099.stdout: 10 136874 23.06 MB/sec execute 192 sec latency 95.169 ms 2024-03-19T01:58:43.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:58:43 smithi112 ceph-mon[66819]: pgmap v880: 97 pgs: 97 active+clean; 576 MiB data, 6.8 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 6.5 MiB/s wr, 579 op/s 2024-03-19T01:58:43.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:58:43 smithi115 ceph-mon[67332]: pgmap v880: 97 pgs: 97 active+clean; 576 MiB data, 6.8 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 6.5 MiB/s wr, 579 op/s 2024-03-19T01:58:43.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:58:43 smithi099 ceph-mon[62951]: pgmap v880: 97 pgs: 97 active+clean; 576 MiB data, 6.8 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 6.5 MiB/s wr, 579 op/s 2024-03-19T01:58:44.108 INFO:tasks.workunit.client.0.smithi099.stdout: 10 137428 23.08 MB/sec execute 193 sec latency 99.901 ms 2024-03-19T01:58:45.108 INFO:tasks.workunit.client.0.smithi099.stdout: 10 137950 23.17 MB/sec execute 194 sec latency 127.135 ms 2024-03-19T01:58:45.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:58:45 smithi112 ceph-mon[66819]: pgmap v881: 97 pgs: 97 active+clean; 577 MiB data, 6.8 GiB used, 1.0 TiB / 1.0 TiB avail; 3.5 MiB/s rd, 5.6 MiB/s wr, 532 op/s 2024-03-19T01:58:45.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:58:45 smithi115 ceph-mon[67332]: pgmap v881: 97 pgs: 97 active+clean; 577 MiB data, 6.8 GiB used, 1.0 TiB / 1.0 TiB avail; 3.5 MiB/s rd, 5.6 MiB/s wr, 532 op/s 2024-03-19T01:58:45.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:58:45 smithi099 ceph-mon[62951]: pgmap v881: 97 pgs: 97 active+clean; 577 MiB data, 6.8 GiB used, 1.0 TiB / 1.0 TiB avail; 3.5 MiB/s rd, 5.6 MiB/s wr, 532 op/s 2024-03-19T01:58:46.108 INFO:tasks.workunit.client.0.smithi099.stdout: 10 138270 23.17 MB/sec execute 195 sec latency 280.852 ms 2024-03-19T01:58:47.108 INFO:tasks.workunit.client.0.smithi099.stdout: 10 138750 23.15 MB/sec execute 196 sec latency 75.765 ms 2024-03-19T01:58:47.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:58:47 smithi112 ceph-mon[66819]: pgmap v882: 97 pgs: 97 active+clean; 583 MiB data, 6.8 GiB used, 1.0 TiB / 1.0 TiB avail; 3.9 MiB/s rd, 6.8 MiB/s wr, 571 op/s 2024-03-19T01:58:47.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:58:47 smithi115 ceph-mon[67332]: pgmap v882: 97 pgs: 97 active+clean; 583 MiB data, 6.8 GiB used, 1.0 TiB / 1.0 TiB avail; 3.9 MiB/s rd, 6.8 MiB/s wr, 571 op/s 2024-03-19T01:58:47.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:58:47 smithi099 ceph-mon[62951]: pgmap v882: 97 pgs: 97 active+clean; 583 MiB data, 6.8 GiB used, 1.0 TiB / 1.0 TiB avail; 3.9 MiB/s rd, 6.8 MiB/s wr, 571 op/s 2024-03-19T01:58:48.108 INFO:tasks.workunit.client.0.smithi099.stdout: 10 139230 23.16 MB/sec execute 197 sec latency 61.151 ms 2024-03-19T01:58:49.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.098+0000 7f664cfda640 1 -- 172.21.15.99:0/3657794101 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f66481033d0 msgr2=0x7f6648109c60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:58:49.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.098+0000 7f664cfda640 1 --2- 172.21.15.99:0/3657794101 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f66481033d0 0x7f6648109c60 secure :-1 s=READY pgs=680 cs=0 l=1 rev1=1 crypto rx=0x7f6640009ca0 tx=0x7f664002c0f0 comp rx=0 tx=0).stop 2024-03-19T01:58:49.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.099+0000 7f664cfda640 1 -- 172.21.15.99:0/3657794101 shutdown_connections 2024-03-19T01:58:49.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.099+0000 7f664cfda640 1 --2- 172.21.15.99:0/3657794101 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f66481033d0 0x7f6648109c60 unknown :-1 s=CLOSED pgs=680 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:49.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.099+0000 7f664cfda640 1 --2- 172.21.15.99:0/3657794101 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f6648102a10 0x7f6648102e90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:49.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.099+0000 7f664cfda640 1 --2- 172.21.15.99:0/3657794101 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6648101820 0x7f6648101c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:49.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.099+0000 7f664cfda640 1 -- 172.21.15.99:0/3657794101 >> 172.21.15.99:0/3657794101 conn(0x7f66480fcd90 msgr2=0x7f66480ff1d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:58:49.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.099+0000 7f664cfda640 1 -- 172.21.15.99:0/3657794101 shutdown_connections 2024-03-19T01:58:49.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.099+0000 7f664cfda640 1 -- 172.21.15.99:0/3657794101 wait complete. 2024-03-19T01:58:49.101 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.100+0000 7f664cfda640 1 Processor -- start 2024-03-19T01:58:49.102 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.100+0000 7f664cfda640 1 -- start start 2024-03-19T01:58:49.102 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.100+0000 7f664cfda640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6648101820 0x7f6648197a30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:49.102 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.100+0000 7f664cfda640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6648102a10 0x7f6648197f70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:49.102 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.100+0000 7f664cfda640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f66481033d0 0x7f664819efe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:49.102 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.100+0000 7f664cfda640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f664810d4a0 con 0x7f6648101820 2024-03-19T01:58:49.102 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.100+0000 7f664cfda640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f664810d320 con 0x7f6648102a10 2024-03-19T01:58:49.102 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.100+0000 7f664cfda640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f664810d620 con 0x7f66481033d0 2024-03-19T01:58:49.102 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.100+0000 7f664659c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6648101820 0x7f6648197a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:49.102 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.100+0000 7f664659c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6648101820 0x7f6648197a30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:33780/0 (socket says 172.21.15.99:33780) 2024-03-19T01:58:49.102 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.100+0000 7f664659c640 1 -- 172.21.15.99:0/2404133201 learned_addr learned my addr 172.21.15.99:0/2404133201 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:58:49.102 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.100+0000 7f663ffff640 1 --2- 172.21.15.99:0/2404133201 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6648102a10 0x7f6648197f70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:49.102 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.100+0000 7f6646d9d640 1 --2- 172.21.15.99:0/2404133201 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f66481033d0 0x7f664819efe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:49.102 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.100+0000 7f664659c640 1 -- 172.21.15.99:0/2404133201 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f66481033d0 msgr2=0x7f664819efe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:58:49.103 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.100+0000 7f664659c640 1 --2- 172.21.15.99:0/2404133201 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f66481033d0 0x7f664819efe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:49.103 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.100+0000 7f664659c640 1 -- 172.21.15.99:0/2404133201 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6648102a10 msgr2=0x7f6648197f70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:58:49.103 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.100+0000 7f664659c640 1 --2- 172.21.15.99:0/2404133201 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6648102a10 0x7f6648197f70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:49.103 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.100+0000 7f664659c640 1 -- 172.21.15.99:0/2404133201 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f664819f650 con 0x7f6648101820 2024-03-19T01:58:49.103 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.100+0000 7f664659c640 1 --2- 172.21.15.99:0/2404133201 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6648101820 0x7f6648197a30 secure :-1 s=READY pgs=681 cs=0 l=1 rev1=1 crypto rx=0x7f663800e9c0 tx=0x7f663800ee80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:58:49.103 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.102+0000 7f663f7fe640 1 -- 172.21.15.99:0/2404133201 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f663800cd80 con 0x7f6648101820 2024-03-19T01:58:49.103 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.102+0000 7f663f7fe640 1 -- 172.21.15.99:0/2404133201 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f66380035f0 con 0x7f6648101820 2024-03-19T01:58:49.103 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.102+0000 7f664cfda640 1 -- 172.21.15.99:0/2404133201 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f664819f940 con 0x7f6648101820 2024-03-19T01:58:49.103 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.102+0000 7f663f7fe640 1 -- 172.21.15.99:0/2404133201 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6638010690 con 0x7f6648101820 2024-03-19T01:58:49.103 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.102+0000 7f664cfda640 1 -- 172.21.15.99:0/2404133201 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f664819fe90 con 0x7f6648101820 2024-03-19T01:58:49.106 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.104+0000 7f663f7fe640 1 -- 172.21.15.99:0/2404133201 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f66380037a0 con 0x7f6648101820 2024-03-19T01:58:49.107 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.104+0000 7f663f7fe640 1 --2- 172.21.15.99:0/2404133201 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f66340a28a0 0x7f66340a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:49.107 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.104+0000 7f664fbbd740 1 Processor -- start 2024-03-19T01:58:49.107 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.104+0000 7f663f7fe640 1 -- 172.21.15.99:0/2404133201 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f66380cee50 con 0x7f6648101820 2024-03-19T01:58:49.107 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.104+0000 7f663ffff640 1 --2- 172.21.15.99:0/2404133201 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f66340a28a0 0x7f66340a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:49.107 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.105+0000 7f664fbbd740 1 -- start start 2024-03-19T01:58:49.107 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.105+0000 7f664fbbd740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557cee3be100 0x557cee3be520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:49.107 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.105+0000 7f664fbbd740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557cee3beaf0 0x557cee3c35f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:49.107 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.105+0000 7f664fbbd740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557cee3c3c60 0x557cee3c60f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:49.107 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.105+0000 7f664fbbd740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x557cee322be0 con 0x557cee3c3c60 2024-03-19T01:58:49.107 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.105+0000 7f664fbbd740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x557cee2cb960 con 0x557cee3beaf0 2024-03-19T01:58:49.107 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.105+0000 7f664fbbd740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x557cee31ec80 con 0x557cee3be100 2024-03-19T01:58:49.107 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.105+0000 7f6646d9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557cee3c3c60 0x557cee3c60f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:49.108 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.105+0000 7f664659c640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557cee3be100 0x557cee3be520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:49.108 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.105+0000 7f6646d9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557cee3c3c60 0x557cee3c60f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:33784/0 (socket says 172.21.15.99:33784) 2024-03-19T01:58:49.108 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.105+0000 7f6646d9d640 1 -- 172.21.15.99:0/2466791670 learned_addr learned my addr 172.21.15.99:0/2466791670 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:58:49.108 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.105+0000 7f663ffff640 1 --2- 172.21.15.99:0/2466791670 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557cee3beaf0 0x557cee3c35f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:49.108 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.105+0000 7f663ffff640 1 --2- 172.21.15.99:0/2404133201 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f66340a28a0 0x7f66340a4d60 secure :-1 s=READY pgs=352 cs=0 l=1 rev1=1 crypto rx=0x7f6648198f40 tx=0x7f6630005f90 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:58:49.108 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.105+0000 7f6646d9d640 1 -- 172.21.15.99:0/2466791670 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557cee3be100 msgr2=0x557cee3be520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:58:49.108 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.105+0000 7f6646d9d640 1 --2- 172.21.15.99:0/2466791670 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557cee3be100 0x557cee3be520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:49.108 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.105+0000 7f6646d9d640 1 -- 172.21.15.99:0/2466791670 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557cee3beaf0 msgr2=0x557cee3c35f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:58:49.108 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.105+0000 7f6646d9d640 1 --2- 172.21.15.99:0/2466791670 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557cee3beaf0 0x557cee3c35f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:49.108 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.105+0000 7f6646d9d640 1 -- 172.21.15.99:0/2466791670 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x557cee2c2d20 con 0x557cee3c3c60 2024-03-19T01:58:49.108 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.105+0000 7f664659c640 1 --2- 172.21.15.99:0/2466791670 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557cee3be100 0x557cee3be520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:58:49.108 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.106+0000 7f6646d9d640 1 --2- 172.21.15.99:0/2466791670 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557cee3c3c60 0x557cee3c60f0 secure :-1 s=READY pgs=682 cs=0 l=1 rev1=1 crypto rx=0x7f6640009ca0 tx=0x7f66400323c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=afd53176ebf2f2f3 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:58:49.108 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.106+0000 7f663cff9640 1 -- 172.21.15.99:0/2466791670 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6640043070 con 0x557cee3c3c60 2024-03-19T01:58:49.108 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.106+0000 7f663cff9640 1 -- 172.21.15.99:0/2466791670 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f6640032c90 con 0x557cee3c3c60 2024-03-19T01:58:49.108 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.106+0000 7f663cff9640 1 -- 172.21.15.99:0/2466791670 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6640039040 con 0x557cee3c3c60 2024-03-19T01:58:49.109 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.106+0000 7f664fbbd740 1 -- 172.21.15.99:0/2466791670 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557cee3c3c60 msgr2=0x557cee3c60f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:58:49.109 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.106+0000 7f664fbbd740 1 --2- 172.21.15.99:0/2466791670 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557cee3c3c60 0x557cee3c60f0 secure :-1 s=READY pgs=682 cs=0 l=1 rev1=1 crypto rx=0x7f6640009ca0 tx=0x7f66400323c0 comp rx=0 tx=0).stop 2024-03-19T01:58:49.109 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.107+0000 7f664fbbd740 1 -- 172.21.15.99:0/2466791670 shutdown_connections 2024-03-19T01:58:49.109 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.107+0000 7f664fbbd740 1 --2- 172.21.15.99:0/2466791670 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557cee3beaf0 0x557cee3c35f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:49.109 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.107+0000 7f664fbbd740 1 --2- 172.21.15.99:0/2466791670 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557cee3c3c60 0x557cee3c60f0 unknown :-1 s=CLOSED pgs=682 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:49.109 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.107+0000 7f664fbbd740 1 --2- 172.21.15.99:0/2466791670 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557cee3be100 0x557cee3be520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:49.109 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.107+0000 7f664fbbd740 1 -- 172.21.15.99:0/2466791670 >> 172.21.15.99:0/2466791670 conn(0x557cee3b9670 msgr2=0x557cee3b9a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:58:49.109 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.107+0000 7f664fbbd740 1 -- 172.21.15.99:0/2466791670 shutdown_connections 2024-03-19T01:58:49.109 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.107+0000 7f664fbbd740 1 -- 172.21.15.99:0/2466791670 wait complete. 2024-03-19T01:58:49.109 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.107+0000 7f664fbbd740 1 -- start start 2024-03-19T01:58:49.109 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.107+0000 7f664fbbd740 1 Processor -- start 2024-03-19T01:58:49.112 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.107+0000 7f664fbbd740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557cee3be100 0x557cee3c8050 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:49.113 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.107+0000 7f664fbbd740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557cee3beaf0 0x557cee3c8590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:49.113 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.107+0000 7f664fbbd740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557cee3c3c60 0x557cee3cf5d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:49.113 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.107+0000 7f664fbbd740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x557cee3c6bb0 con 0x557cee3c3c60 2024-03-19T01:58:49.113 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.107+0000 7f664fbbd740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x557cee2dedb0 con 0x557cee3beaf0 2024-03-19T01:58:49.113 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.107+0000 7f664fbbd740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x557cee3c6d30 con 0x557cee3be100 2024-03-19T01:58:49.113 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.107+0000 7f664fbbd740 10 client.0 _ll_register_callbacks cb 0x557cee2e3e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:58:49.113 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.107+0000 7f6646d9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557cee3c3c60 0x557cee3cf5d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:49.113 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.107+0000 7f663ffff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557cee3beaf0 0x557cee3c8590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:49.114 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.107+0000 7f6646d9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557cee3c3c60 0x557cee3cf5d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:33788/0 (socket says 172.21.15.99:33788) 2024-03-19T01:58:49.114 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.107+0000 7f6646d9d640 1 -- 172.21.15.99:0/139032712 learned_addr learned my addr 172.21.15.99:0/139032712 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:58:49.114 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.108+0000 7f664659c640 1 --2- 172.21.15.99:0/139032712 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557cee3be100 0x557cee3c8050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:49.114 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.107+0000 7f663ffff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557cee3beaf0 0x557cee3c8590 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:33972/0 (socket says 172.21.15.99:33972) 2024-03-19T01:58:49.114 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.108+0000 7f6646d9d640 1 -- 172.21.15.99:0/139032712 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557cee3be100 msgr2=0x557cee3c8050 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:58:49.114 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.108+0000 7f6646d9d640 1 --2- 172.21.15.99:0/139032712 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557cee3be100 0x557cee3c8050 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:49.114 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.108+0000 7f6646d9d640 1 -- 172.21.15.99:0/139032712 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557cee3beaf0 msgr2=0x557cee3c8590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:58:49.114 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.108+0000 7f6646d9d640 1 --2- 172.21.15.99:0/139032712 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557cee3beaf0 0x557cee3c8590 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:49.114 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.108+0000 7f6646d9d640 1 -- 172.21.15.99:0/139032712 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x557cee2ddc60 con 0x557cee3c3c60 2024-03-19T01:58:49.114 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.108+0000 7f663ffff640 1 --2- 172.21.15.99:0/139032712 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557cee3beaf0 0x557cee3c8590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:58:49.115 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.108+0000 7f664659c640 1 --2- 172.21.15.99:0/139032712 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557cee3be100 0x557cee3c8050 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:58:49.115 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.108+0000 7f6646d9d640 1 --2- 172.21.15.99:0/139032712 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557cee3c3c60 0x557cee3cf5d0 secure :-1 s=READY pgs=683 cs=0 l=1 rev1=1 crypto rx=0x7f664002c700 tx=0x7f6640005bd0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=fc0b5108a5c0dc46 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:58:49.115 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.108+0000 7f662dffb640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:58:49.115 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.108+0000 7f662dffb640 1 -- 172.21.15.99:0/139032712 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f664004c070 con 0x557cee3c3c60 2024-03-19T01:58:49.115 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.108+0000 7f662dffb640 1 -- 172.21.15.99:0/139032712 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f6640048650 con 0x557cee3c3c60 2024-03-19T01:58:49.115 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.108+0000 7f664fbbd740 1 -- 172.21.15.99:0/139032712 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f6640032c90 con 0x557cee3c3c60 2024-03-19T01:58:49.115 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.108+0000 7f662dffb640 1 -- 172.21.15.99:0/139032712 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7f6640058050 con 0x557cee3c3c60 2024-03-19T01:58:49.115 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.108+0000 7f664fbbd740 10 client.16758 fetch_fsmap learned FSMap version 37 2024-03-19T01:58:49.115 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.109+0000 7f664fbbd740 1 -- 172.21.15.99:0/139032712 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x557cee037210 con 0x557cee3c3c60 2024-03-19T01:58:49.115 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.109+0000 7f662dffb640 1 -- 172.21.15.99:0/139032712 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f6640032c90 con 0x557cee3c3c60 2024-03-19T01:58:49.116 INFO:tasks.workunit.client.0.smithi099.stdout: 10 139628 23.16 MB/sec execute 198 sec latency 80.961 ms 2024-03-19T01:58:49.118 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.109+0000 7f662dffb640 1 -- 172.21.15.99:0/139032712 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7f66400347b0 con 0x557cee3c3c60 2024-03-19T01:58:49.118 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.109+0000 7f664fbbd740 10 client.16758 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T01:58:49.118 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.109+0000 7f664fbbd740 10 client.16758 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:58:49.118 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.109+0000 7f664fbbd740 20 client.16758 populate_metadata read hostname 'smithi099' 2024-03-19T01:58:49.118 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.109+0000 7f664fbbd740 1 --2- 172.21.15.99:0/139032712 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x557cee3d3ac0 0x557cee3d5eb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:49.118 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.109+0000 7f664fbbd740 4 client.16758 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:58:49.119 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.109+0000 7f664fbbd740 1 -- 172.21.15.99:0/139032712 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x557cee330120 con 0x557cee3d3ac0 2024-03-19T01:58:49.119 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.109+0000 7f664659c640 1 --2- 172.21.15.99:0/139032712 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x557cee3d3ac0 0x557cee3d5eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:49.119 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.110+0000 7f664659c640 1 --2- 172.21.15.99:0/139032712 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x557cee3d3ac0 0x557cee3d5eb0 crc :-1 s=READY pgs=187 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=ddd3079c1be8bcc4 server_cookie=63aa6821828ceca9 in_seq=0 out_seq=0 2024-03-19T01:58:49.119 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.110+0000 7f662dffb640 10 client.16758 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:58:49.119 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.112+0000 7f662dffb640 1 -- 172.21.15.99:0/139032712 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x557cee330120 con 0x557cee3d3ac0 2024-03-19T01:58:49.119 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.112+0000 7f662dffb640 10 client.16758 handle_command_reply: tid=0 2024-03-19T01:58:49.119 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.112+0000 7f664fbbd740 1 client.16758 shutdown 2024-03-19T01:58:49.119 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.112+0000 7f664fbbd740 1 -- 172.21.15.99:0/139032712 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557cee3c3c60 msgr2=0x557cee3cf5d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:58:49.119 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.112+0000 7f664fbbd740 1 --2- 172.21.15.99:0/139032712 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557cee3c3c60 0x557cee3cf5d0 secure :-1 s=READY pgs=683 cs=0 l=1 rev1=1 crypto rx=0x7f664002c700 tx=0x7f6640005bd0 comp rx=0 tx=0).stop 2024-03-19T01:58:49.120 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.112+0000 7f664fbbd740 1 -- 172.21.15.99:0/139032712 shutdown_connections 2024-03-19T01:58:49.120 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.112+0000 7f664fbbd740 1 --2- 172.21.15.99:0/139032712 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x557cee3d3ac0 0x557cee3d5eb0 crc :-1 s=READY pgs=187 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:49.120 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.112+0000 7f664fbbd740 1 --2- 172.21.15.99:0/139032712 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557cee3beaf0 0x557cee3c8590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:49.120 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.112+0000 7f664fbbd740 1 --2- 172.21.15.99:0/139032712 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557cee3c3c60 0x557cee3cf5d0 unknown :-1 s=CLOSED pgs=683 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:49.120 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.112+0000 7f664fbbd740 1 --2- 172.21.15.99:0/139032712 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557cee3be100 0x557cee3c8050 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:49.120 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.112+0000 7f664fbbd740 1 -- 172.21.15.99:0/139032712 >> 172.21.15.99:0/139032712 conn(0x557cee3b9670 msgr2=0x557cee3ba8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:58:49.120 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.112+0000 7f662dffb640 0 client.16758 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:58:49.120 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.112+0000 7f664fbbd740 1 -- 172.21.15.99:0/139032712 shutdown_connections 2024-03-19T01:58:49.120 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.112+0000 7f664fbbd740 1 -- 172.21.15.99:0/139032712 wait complete. 2024-03-19T01:58:49.120 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.112+0000 7f664fbbd740 20 client.16758 trim_cache size 0 max 16384 2024-03-19T01:58:49.133 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.128+0000 7f664fbbd740 1 Processor -- start 2024-03-19T01:58:49.133 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.128+0000 7f664fbbd740 1 -- start start 2024-03-19T01:58:49.133 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.128+0000 7f664fbbd740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557cee3b9670 0x557cee3bfca0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:49.133 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.128+0000 7f664fbbd740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557cee3be100 0x557cee3da7e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:49.133 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.128+0000 7f664fbbd740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557cee3beaf0 0x557cee3dad20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:49.133 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.128+0000 7f664fbbd740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f66481ab5b0 con 0x557cee3beaf0 2024-03-19T01:58:49.133 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.128+0000 7f664fbbd740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f66481a8e80 con 0x557cee3b9670 2024-03-19T01:58:49.133 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.128+0000 7f664fbbd740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f66481a6cd0 con 0x557cee3be100 2024-03-19T01:58:49.133 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.128+0000 7f6646d9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557cee3beaf0 0x557cee3dad20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:49.133 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.128+0000 7f6646d9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557cee3beaf0 0x557cee3dad20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:33804/0 (socket says 172.21.15.99:33804) 2024-03-19T01:58:49.133 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.128+0000 7f6646d9d640 1 -- 172.21.15.99:0/1267779945 learned_addr learned my addr 172.21.15.99:0/1267779945 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:58:49.134 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.128+0000 7f663ffff640 1 --2- 172.21.15.99:0/1267779945 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557cee3be100 0x557cee3da7e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:49.134 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.128+0000 7f664659c640 1 --2- 172.21.15.99:0/1267779945 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557cee3b9670 0x557cee3bfca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:49.134 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.128+0000 7f6646d9d640 1 -- 172.21.15.99:0/1267779945 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557cee3be100 msgr2=0x557cee3da7e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:58:49.134 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.128+0000 7f6646d9d640 1 --2- 172.21.15.99:0/1267779945 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557cee3be100 0x557cee3da7e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:49.134 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.128+0000 7f6646d9d640 1 -- 172.21.15.99:0/1267779945 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557cee3b9670 msgr2=0x557cee3bfca0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:58:49.134 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.128+0000 7f6646d9d640 1 --2- 172.21.15.99:0/1267779945 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557cee3b9670 0x557cee3bfca0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:49.134 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.128+0000 7f6646d9d640 1 -- 172.21.15.99:0/1267779945 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x557cee103620 con 0x557cee3beaf0 2024-03-19T01:58:49.134 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.128+0000 7f6646d9d640 1 --2- 172.21.15.99:0/1267779945 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557cee3beaf0 0x557cee3dad20 secure :-1 s=READY pgs=684 cs=0 l=1 rev1=1 crypto rx=0x7f664002c6d0 tx=0x7f664002c890 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c527093f77078593 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:58:49.134 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.129+0000 7f662ffff640 1 -- 172.21.15.99:0/1267779945 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6640055070 con 0x557cee3beaf0 2024-03-19T01:58:49.134 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.129+0000 7f662ffff640 1 -- 172.21.15.99:0/1267779945 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f6640034920 con 0x557cee3beaf0 2024-03-19T01:58:49.135 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.129+0000 7f662ffff640 1 -- 172.21.15.99:0/1267779945 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f664004c040 con 0x557cee3beaf0 2024-03-19T01:58:49.135 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.129+0000 7f664fbbd740 1 -- 172.21.15.99:0/1267779945 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557cee3beaf0 msgr2=0x557cee3dad20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:58:49.135 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.129+0000 7f664fbbd740 1 --2- 172.21.15.99:0/1267779945 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557cee3beaf0 0x557cee3dad20 secure :-1 s=READY pgs=684 cs=0 l=1 rev1=1 crypto rx=0x7f664002c6d0 tx=0x7f664002c890 comp rx=0 tx=0).stop 2024-03-19T01:58:49.135 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.129+0000 7f664fbbd740 1 -- 172.21.15.99:0/1267779945 shutdown_connections 2024-03-19T01:58:49.135 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.129+0000 7f664fbbd740 1 --2- 172.21.15.99:0/1267779945 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557cee3beaf0 0x557cee3dad20 unknown :-1 s=CLOSED pgs=684 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:49.135 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.129+0000 7f664fbbd740 1 --2- 172.21.15.99:0/1267779945 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557cee3be100 0x557cee3da7e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:49.135 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.129+0000 7f664fbbd740 1 --2- 172.21.15.99:0/1267779945 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557cee3b9670 0x557cee3bfca0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:49.135 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.129+0000 7f664fbbd740 1 -- 172.21.15.99:0/1267779945 >> 172.21.15.99:0/1267779945 conn(0x557cee3c7c10 msgr2=0x557cee3bf590 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:58:49.135 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.129+0000 7f664fbbd740 1 -- 172.21.15.99:0/1267779945 shutdown_connections 2024-03-19T01:58:49.135 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.129+0000 7f664fbbd740 1 -- 172.21.15.99:0/1267779945 wait complete. 2024-03-19T01:58:49.135 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.130+0000 7f664fbbd740 1 -- start start 2024-03-19T01:58:49.135 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.130+0000 7f664fbbd740 1 Processor -- start 2024-03-19T01:58:49.135 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.130+0000 7f664fbbd740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557cee3b9670 0x557cee435910 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:49.135 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.130+0000 7f664fbbd740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557cee3be100 0x557cee435e50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:49.136 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.130+0000 7f664fbbd740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557cee3beaf0 0x557cee43ce90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:49.136 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.130+0000 7f664fbbd740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f66481a7340 con 0x557cee3b9670 2024-03-19T01:58:49.136 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.130+0000 7f664fbbd740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f66481a87b0 con 0x557cee3beaf0 2024-03-19T01:58:49.136 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.130+0000 7f664fbbd740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f66481a7a10 con 0x557cee3be100 2024-03-19T01:58:49.136 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.130+0000 7f664fbbd740 10 client.0 _ll_register_callbacks cb 0x7f66300099c0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:58:49.136 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.130+0000 7f664659c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557cee3b9670 0x557cee435910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:49.136 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.130+0000 7f664659c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557cee3b9670 0x557cee435910 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:33812/0 (socket says 172.21.15.99:33812) 2024-03-19T01:58:49.136 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.130+0000 7f664659c640 1 -- 172.21.15.99:0/650506857 learned_addr learned my addr 172.21.15.99:0/650506857 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:58:49.136 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.130+0000 7f663ffff640 1 --2- 172.21.15.99:0/650506857 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557cee3be100 0x557cee435e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:49.136 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.130+0000 7f6646d9d640 1 --2- 172.21.15.99:0/650506857 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557cee3beaf0 0x557cee43ce90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:49.136 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.130+0000 7f664659c640 1 -- 172.21.15.99:0/650506857 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557cee3be100 msgr2=0x557cee435e50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:58:49.136 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.130+0000 7f664659c640 1 --2- 172.21.15.99:0/650506857 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557cee3be100 0x557cee435e50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:49.136 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.130+0000 7f664659c640 1 -- 172.21.15.99:0/650506857 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557cee3beaf0 msgr2=0x557cee43ce90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:58:49.136 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.130+0000 7f664659c640 1 --2- 172.21.15.99:0/650506857 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557cee3beaf0 0x557cee43ce90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:49.137 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.130+0000 7f664659c640 1 -- 172.21.15.99:0/650506857 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x557cee43d480 con 0x557cee3b9670 2024-03-19T01:58:49.137 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.130+0000 7f664659c640 1 --2- 172.21.15.99:0/650506857 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557cee3b9670 0x557cee435910 secure :-1 s=READY pgs=685 cs=0 l=1 rev1=1 crypto rx=0x7f66380ccc80 tx=0x7f6638086040 comp rx=0 tx=0).ready entity=mon.0 client_cookie=28475d86a97d88fe server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:58:49.137 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.130+0000 7f662dffb640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:58:49.137 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.131+0000 7f662dffb640 1 -- 172.21.15.99:0/650506857 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f66380b8030 con 0x557cee3b9670 2024-03-19T01:58:49.137 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.131+0000 7f662dffb640 1 -- 172.21.15.99:0/650506857 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f6638099ab0 con 0x557cee3b9670 2024-03-19T01:58:49.137 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.131+0000 7f664fbbd740 1 -- 172.21.15.99:0/650506857 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f6640034920 con 0x557cee3b9670 2024-03-19T01:58:49.137 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.131+0000 7f662dffb640 1 -- 172.21.15.99:0/650506857 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7f663809f900 con 0x557cee3b9670 2024-03-19T01:58:49.137 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.131+0000 7f664fbbd740 10 client.16764 fetch_fsmap learned FSMap version 37 2024-03-19T01:58:49.137 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.131+0000 7f664fbbd740 1 -- 172.21.15.99:0/650506857 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x557cee43e800 con 0x557cee3b9670 2024-03-19T01:58:49.137 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.131+0000 7f662dffb640 1 -- 172.21.15.99:0/650506857 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f6640034920 con 0x557cee3b9670 2024-03-19T01:58:49.137 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.131+0000 7f662dffb640 1 -- 172.21.15.99:0/650506857 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7f66380afcb0 con 0x557cee3b9670 2024-03-19T01:58:49.137 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.131+0000 7f664fbbd740 10 client.16764 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T01:58:49.137 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.131+0000 7f664fbbd740 10 client.16764 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:58:49.137 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.131+0000 7f664fbbd740 20 client.16764 populate_metadata read hostname 'smithi099' 2024-03-19T01:58:49.138 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.131+0000 7f664fbbd740 1 --2- 172.21.15.99:0/650506857 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x557cee3c3c60 0x557cee441160 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:58:49.138 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.131+0000 7f664fbbd740 4 client.16764 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:58:49.138 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.131+0000 7f664fbbd740 1 -- 172.21.15.99:0/650506857 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x557cee32a470 con 0x557cee3c3c60 2024-03-19T01:58:49.138 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.131+0000 7f663ffff640 1 --2- 172.21.15.99:0/650506857 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x557cee3c3c60 0x557cee441160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:58:49.138 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.132+0000 7f663ffff640 1 --2- 172.21.15.99:0/650506857 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x557cee3c3c60 0x557cee441160 crc :-1 s=READY pgs=188 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=5c2278d145cb862a server_cookie=367548d4c22cc7bb in_seq=0 out_seq=0 2024-03-19T01:58:49.138 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.132+0000 7f662dffb640 10 client.16764 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:58:49.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:58:49 smithi112 ceph-mon[66819]: pgmap v883: 97 pgs: 97 active+clean; 588 MiB data, 6.9 GiB used, 1.0 TiB / 1.0 TiB avail; 3.6 MiB/s rd, 6.5 MiB/s wr, 508 op/s 2024-03-19T01:58:49.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:58:49 smithi115 ceph-mon[67332]: pgmap v883: 97 pgs: 97 active+clean; 588 MiB data, 6.9 GiB used, 1.0 TiB / 1.0 TiB avail; 3.6 MiB/s rd, 6.5 MiB/s wr, 508 op/s 2024-03-19T01:58:49.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:58:49 smithi099 ceph-mon[62951]: pgmap v883: 97 pgs: 97 active+clean; 588 MiB data, 6.9 GiB used, 1.0 TiB / 1.0 TiB avail; 3.6 MiB/s rd, 6.5 MiB/s wr, 508 op/s 2024-03-19T01:58:49.646 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.646+0000 7f662dffb640 1 -- 172.21.15.99:0/650506857 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x557cee32a470 con 0x557cee3c3c60 2024-03-19T01:58:49.646 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.646+0000 7f662dffb640 10 client.16764 handle_command_reply: tid=0 2024-03-19T01:58:49.647 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.646+0000 7f664fbbd740 1 client.16764 shutdown 2024-03-19T01:58:49.647 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.646+0000 7f664fbbd740 1 -- 172.21.15.99:0/650506857 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557cee3b9670 msgr2=0x557cee435910 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:58:49.647 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.646+0000 7f664fbbd740 1 --2- 172.21.15.99:0/650506857 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557cee3b9670 0x557cee435910 secure :-1 s=READY pgs=685 cs=0 l=1 rev1=1 crypto rx=0x7f66380ccc80 tx=0x7f6638086040 comp rx=0 tx=0).stop 2024-03-19T01:58:49.647 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.646+0000 7f664fbbd740 1 -- 172.21.15.99:0/650506857 shutdown_connections 2024-03-19T01:58:49.647 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.646+0000 7f664fbbd740 1 --2- 172.21.15.99:0/650506857 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x557cee3c3c60 0x557cee441160 crc :-1 s=READY pgs=188 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:49.647 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.646+0000 7f662dffb640 0 client.16764 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:58:49.647 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.646+0000 7f664fbbd740 1 --2- 172.21.15.99:0/650506857 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x557cee3beaf0 0x557cee43ce90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:49.647 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.646+0000 7f664fbbd740 1 --2- 172.21.15.99:0/650506857 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x557cee3be100 0x557cee435e50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:49.647 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.646+0000 7f664fbbd740 1 --2- 172.21.15.99:0/650506857 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x557cee3b9670 0x557cee435910 unknown :-1 s=CLOSED pgs=685 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:49.648 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.646+0000 7f664fbbd740 1 -- 172.21.15.99:0/650506857 >> 172.21.15.99:0/650506857 conn(0x557cee3c7c10 msgr2=0x557cee3bf590 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:58:49.648 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.646+0000 7f664fbbd740 1 -- 172.21.15.99:0/650506857 shutdown_connections 2024-03-19T01:58:49.648 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.646+0000 7f664fbbd740 1 -- 172.21.15.99:0/650506857 wait complete. 2024-03-19T01:58:49.648 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.646+0000 7f664fbbd740 20 client.16764 trim_cache size 0 max 16384 2024-03-19T01:58:49.648 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:58:49.648 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:58:49.648 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:58:49.648 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:58:49.649 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.647+0000 7f664cfda640 1 -- 172.21.15.99:0/2404133201 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f66340a28a0 msgr2=0x7f66340a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:58:49.650 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.647+0000 7f664cfda640 1 --2- 172.21.15.99:0/2404133201 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f66340a28a0 0x7f66340a4d60 secure :-1 s=READY pgs=352 cs=0 l=1 rev1=1 crypto rx=0x7f6648198f40 tx=0x7f6630005f90 comp rx=0 tx=0).stop 2024-03-19T01:58:49.650 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.647+0000 7f664cfda640 1 -- 172.21.15.99:0/2404133201 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6648101820 msgr2=0x7f6648197a30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:58:49.650 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.647+0000 7f664cfda640 1 --2- 172.21.15.99:0/2404133201 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6648101820 0x7f6648197a30 secure :-1 s=READY pgs=681 cs=0 l=1 rev1=1 crypto rx=0x7f663800e9c0 tx=0x7f663800ee80 comp rx=0 tx=0).stop 2024-03-19T01:58:49.650 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.647+0000 7f664cfda640 1 -- 172.21.15.99:0/2404133201 shutdown_connections 2024-03-19T01:58:49.650 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.647+0000 7f664cfda640 1 --2- 172.21.15.99:0/2404133201 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f66340a28a0 0x7f66340a4d60 unknown :-1 s=CLOSED pgs=352 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:49.650 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.647+0000 7f664cfda640 1 --2- 172.21.15.99:0/2404133201 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6648102a10 0x7f6648197f70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:49.650 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.647+0000 7f664cfda640 1 --2- 172.21.15.99:0/2404133201 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f66481033d0 0x7f664819efe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:49.650 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.647+0000 7f664cfda640 1 --2- 172.21.15.99:0/2404133201 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6648101820 0x7f6648197a30 unknown :-1 s=CLOSED pgs=681 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:58:49.650 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.647+0000 7f664cfda640 1 -- 172.21.15.99:0/2404133201 >> 172.21.15.99:0/2404133201 conn(0x7f66480fcd90 msgr2=0x7f6648105cb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:58:49.650 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.647+0000 7f664cfda640 1 -- 172.21.15.99:0/2404133201 shutdown_connections 2024-03-19T01:58:49.650 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:58:49.647+0000 7f664cfda640 1 -- 172.21.15.99:0/2404133201 wait complete. 2024-03-19T01:58:50.109 INFO:tasks.workunit.client.0.smithi099.stdout: 10 139874 23.12 MB/sec execute 199 sec latency 328.532 ms 2024-03-19T01:58:51.109 INFO:tasks.workunit.client.0.smithi099.stdout: 10 140099 23.06 MB/sec execute 200 sec latency 298.153 ms 2024-03-19T01:58:51.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:58:51 smithi112 ceph-mon[66819]: pgmap v884: 97 pgs: 97 active+clean; 583 MiB data, 6.9 GiB used, 1.0 TiB / 1.0 TiB avail; 3.2 MiB/s rd, 6.5 MiB/s wr, 501 op/s 2024-03-19T01:58:51.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:58:51 smithi115 ceph-mon[67332]: pgmap v884: 97 pgs: 97 active+clean; 583 MiB data, 6.9 GiB used, 1.0 TiB / 1.0 TiB avail; 3.2 MiB/s rd, 6.5 MiB/s wr, 501 op/s 2024-03-19T01:58:51.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:58:51 smithi099 ceph-mon[62951]: pgmap v884: 97 pgs: 97 active+clean; 583 MiB data, 6.9 GiB used, 1.0 TiB / 1.0 TiB avail; 3.2 MiB/s rd, 6.5 MiB/s wr, 501 op/s 2024-03-19T01:58:52.109 INFO:tasks.workunit.client.0.smithi099.stdout: 10 140495 23.04 MB/sec execute 201 sec latency 118.756 ms 2024-03-19T01:58:53.109 INFO:tasks.workunit.client.0.smithi099.stdout: 10 141064 23.04 MB/sec execute 202 sec latency 61.260 ms 2024-03-19T01:58:53.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:58:53 smithi112 ceph-mon[66819]: pgmap v885: 97 pgs: 97 active+clean; 576 MiB data, 6.9 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 7.1 MiB/s wr, 604 op/s 2024-03-19T01:58:53.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:58:53 smithi115 ceph-mon[67332]: pgmap v885: 97 pgs: 97 active+clean; 576 MiB data, 6.9 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 7.1 MiB/s wr, 604 op/s 2024-03-19T01:58:53.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:58:53 smithi099 ceph-mon[62951]: pgmap v885: 97 pgs: 97 active+clean; 576 MiB data, 6.9 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 7.1 MiB/s wr, 604 op/s 2024-03-19T01:58:54.109 INFO:tasks.workunit.client.0.smithi099.stdout: 10 141649 23.07 MB/sec execute 203 sec latency 64.649 ms 2024-03-19T01:58:55.109 INFO:tasks.workunit.client.0.smithi099.stdout: 10 142173 23.14 MB/sec execute 204 sec latency 85.446 ms 2024-03-19T01:58:55.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:58:55 smithi112 ceph-mon[66819]: pgmap v886: 97 pgs: 97 active+clean; 578 MiB data, 6.8 GiB used, 1.0 TiB / 1.0 TiB avail; 3.1 MiB/s rd, 5.7 MiB/s wr, 528 op/s 2024-03-19T01:58:55.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:58:55 smithi115 ceph-mon[67332]: pgmap v886: 97 pgs: 97 active+clean; 578 MiB data, 6.8 GiB used, 1.0 TiB / 1.0 TiB avail; 3.1 MiB/s rd, 5.7 MiB/s wr, 528 op/s 2024-03-19T01:58:55.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:58:55 smithi099 ceph-mon[62951]: pgmap v886: 97 pgs: 97 active+clean; 578 MiB data, 6.8 GiB used, 1.0 TiB / 1.0 TiB avail; 3.1 MiB/s rd, 5.7 MiB/s wr, 528 op/s 2024-03-19T01:58:56.109 INFO:tasks.workunit.client.0.smithi099.stdout: 10 142443 23.08 MB/sec execute 205 sec latency 328.764 ms 2024-03-19T01:58:57.109 INFO:tasks.workunit.client.0.smithi099.stdout: 10 142933 23.15 MB/sec execute 206 sec latency 90.958 ms 2024-03-19T01:58:57.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:58:57 smithi112 ceph-mon[66819]: pgmap v887: 97 pgs: 97 active+clean; 588 MiB data, 6.9 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 7.2 MiB/s wr, 605 op/s 2024-03-19T01:58:57.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:58:57 smithi115 ceph-mon[67332]: pgmap v887: 97 pgs: 97 active+clean; 588 MiB data, 6.9 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 7.2 MiB/s wr, 605 op/s 2024-03-19T01:58:57.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:58:57 smithi099 ceph-mon[62951]: pgmap v887: 97 pgs: 97 active+clean; 588 MiB data, 6.9 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 7.2 MiB/s wr, 605 op/s 2024-03-19T01:58:58.109 INFO:tasks.workunit.client.0.smithi099.stdout: 10 143430 23.14 MB/sec execute 207 sec latency 84.760 ms 2024-03-19T01:58:59.109 INFO:tasks.workunit.client.0.smithi099.stdout: 10 143857 23.14 MB/sec execute 208 sec latency 79.340 ms 2024-03-19T01:58:59.357 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:58:59 smithi115 ceph-mon[67332]: pgmap v888: 97 pgs: 97 active+clean; 588 MiB data, 6.9 GiB used, 1.0 TiB / 1.0 TiB avail; 3.9 MiB/s rd, 6.1 MiB/s wr, 571 op/s 2024-03-19T01:58:59.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:58:59 smithi112 ceph-mon[66819]: pgmap v888: 97 pgs: 97 active+clean; 588 MiB data, 6.9 GiB used, 1.0 TiB / 1.0 TiB avail; 3.9 MiB/s rd, 6.1 MiB/s wr, 571 op/s 2024-03-19T01:58:59.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:58:59 smithi099 ceph-mon[62951]: pgmap v888: 97 pgs: 97 active+clean; 588 MiB data, 6.9 GiB used, 1.0 TiB / 1.0 TiB avail; 3.9 MiB/s rd, 6.1 MiB/s wr, 571 op/s 2024-03-19T01:59:00.109 INFO:tasks.workunit.client.0.smithi099.stdout: 10 144311 23.17 MB/sec execute 209 sec latency 73.318 ms 2024-03-19T01:59:01.109 INFO:tasks.workunit.client.0.smithi099.stdout: 10 144596 23.11 MB/sec execute 210 sec latency 174.703 ms 2024-03-19T01:59:01.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:59:01 smithi112 ceph-mon[66819]: pgmap v889: 97 pgs: 97 active+clean; 592 MiB data, 6.9 GiB used, 1.0 TiB / 1.0 TiB avail; 3.9 MiB/s rd, 6.0 MiB/s wr, 559 op/s 2024-03-19T01:59:01.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:59:01 smithi115 ceph-mon[67332]: pgmap v889: 97 pgs: 97 active+clean; 592 MiB data, 6.9 GiB used, 1.0 TiB / 1.0 TiB avail; 3.9 MiB/s rd, 6.0 MiB/s wr, 559 op/s 2024-03-19T01:59:01.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:59:01 smithi099 ceph-mon[62951]: pgmap v889: 97 pgs: 97 active+clean; 592 MiB data, 6.9 GiB used, 1.0 TiB / 1.0 TiB avail; 3.9 MiB/s rd, 6.0 MiB/s wr, 559 op/s 2024-03-19T01:59:02.109 INFO:tasks.workunit.client.0.smithi099.stdout: 10 144950 23.08 MB/sec execute 211 sec latency 93.971 ms 2024-03-19T01:59:02.744 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.741+0000 7f04c0769640 1 -- 172.21.15.99:0/543275510 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f04b8102360 msgr2=0x7f04b8104750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:59:02.744 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.741+0000 7f04c0769640 1 --2- 172.21.15.99:0/543275510 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f04b8102360 0x7f04b8104750 secure :-1 s=READY pgs=686 cs=0 l=1 rev1=1 crypto rx=0x7f04b0009ca0 tx=0x7f04b002c0f0 comp rx=0 tx=0).stop 2024-03-19T01:59:02.744 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.741+0000 7f04c0769640 1 -- 172.21.15.99:0/543275510 shutdown_connections 2024-03-19T01:59:02.744 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.741+0000 7f04c0769640 1 --2- 172.21.15.99:0/543275510 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f04b81075c0 0x7f04b81099b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:02.744 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.741+0000 7f04c0769640 1 --2- 172.21.15.99:0/543275510 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f04b8104c90 0x7f04b8107080 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:02.744 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.741+0000 7f04c0769640 1 --2- 172.21.15.99:0/543275510 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f04b8102360 0x7f04b8104750 unknown :-1 s=CLOSED pgs=686 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:02.744 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.741+0000 7f04c0769640 1 -- 172.21.15.99:0/543275510 >> 172.21.15.99:0/543275510 conn(0x7f04b80fbcb0 msgr2=0x7f04b80fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:59:02.744 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.742+0000 7f04c0769640 1 -- 172.21.15.99:0/543275510 shutdown_connections 2024-03-19T01:59:02.744 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.742+0000 7f04c0769640 1 -- 172.21.15.99:0/543275510 wait complete. 2024-03-19T01:59:02.744 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.742+0000 7f04c0769640 1 Processor -- start 2024-03-19T01:59:02.744 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.742+0000 7f04c0769640 1 -- start start 2024-03-19T01:59:02.745 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.742+0000 7f04c0769640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f04b8102360 0x7f04b8197b30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:02.745 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.742+0000 7f04c0769640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f04b8104c90 0x7f04b8198070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:02.745 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.742+0000 7f04c0769640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f04b81075c0 0x7f04b819f0e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:02.745 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.742+0000 7f04c0769640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f04b810d4e0 con 0x7f04b81075c0 2024-03-19T01:59:02.745 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.742+0000 7f04c0769640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f04b810d360 con 0x7f04b8104c90 2024-03-19T01:59:02.745 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.742+0000 7f04c0769640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f04b810d660 con 0x7f04b8102360 2024-03-19T01:59:02.745 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.743+0000 7f04bed06640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f04b81075c0 0x7f04b819f0e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:02.745 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.743+0000 7f04bdd04640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f04b8104c90 0x7f04b8198070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:02.745 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.743+0000 7f04bed06640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f04b81075c0 0x7f04b819f0e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:41502/0 (socket says 172.21.15.99:41502) 2024-03-19T01:59:02.745 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.743+0000 7f04bed06640 1 -- 172.21.15.99:0/420362377 learned_addr learned my addr 172.21.15.99:0/420362377 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:59:02.745 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.743+0000 7f04be505640 1 --2- 172.21.15.99:0/420362377 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f04b8102360 0x7f04b8197b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:02.745 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.743+0000 7f04bed06640 1 -- 172.21.15.99:0/420362377 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f04b8102360 msgr2=0x7f04b8197b30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:59:02.745 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.743+0000 7f04bed06640 1 --2- 172.21.15.99:0/420362377 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f04b8102360 0x7f04b8197b30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:02.745 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.743+0000 7f04bed06640 1 -- 172.21.15.99:0/420362377 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f04b8104c90 msgr2=0x7f04b8198070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:59:02.746 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.743+0000 7f04bed06640 1 --2- 172.21.15.99:0/420362377 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f04b8104c90 0x7f04b8198070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:02.746 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.743+0000 7f04bed06640 1 -- 172.21.15.99:0/420362377 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f04b819f750 con 0x7f04b81075c0 2024-03-19T01:59:02.746 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.743+0000 7f04bdd04640 1 --2- 172.21.15.99:0/420362377 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f04b8104c90 0x7f04b8198070 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:59:02.746 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.743+0000 7f04bed06640 1 --2- 172.21.15.99:0/420362377 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f04b81075c0 0x7f04b819f0e0 secure :-1 s=READY pgs=687 cs=0 l=1 rev1=1 crypto rx=0x7f04a800e9b0 tx=0x7f04a800ee70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:59:02.746 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.743+0000 7f04b77fe640 1 -- 172.21.15.99:0/420362377 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f04a800cd80 con 0x7f04b81075c0 2024-03-19T01:59:02.746 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.743+0000 7f04c0769640 1 -- 172.21.15.99:0/420362377 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f04b819fa40 con 0x7f04b81075c0 2024-03-19T01:59:02.746 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.743+0000 7f04c0769640 1 -- 172.21.15.99:0/420362377 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f04b819ff90 con 0x7f04b81075c0 2024-03-19T01:59:02.746 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.743+0000 7f04b77fe640 1 -- 172.21.15.99:0/420362377 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f04a80035f0 con 0x7f04b81075c0 2024-03-19T01:59:02.746 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.743+0000 7f04b77fe640 1 -- 172.21.15.99:0/420362377 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f04a8010670 con 0x7f04b81075c0 2024-03-19T01:59:02.753 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.746+0000 7f04b77fe640 1 -- 172.21.15.99:0/420362377 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f04a80037a0 con 0x7f04b81075c0 2024-03-19T01:59:02.753 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.746+0000 7f04b77fe640 1 --2- 172.21.15.99:0/420362377 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f04a00a2920 0x7f04a00a4de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:02.753 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.746+0000 7f04be505640 1 --2- 172.21.15.99:0/420362377 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f04a00a2920 0x7f04a00a4de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:02.753 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.746+0000 7f04b77fe640 1 -- 172.21.15.99:0/420362377 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f04a80ced20 con 0x7f04b81075c0 2024-03-19T01:59:02.754 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.747+0000 7f04c3432740 1 Processor -- start 2024-03-19T01:59:02.754 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.747+0000 7f04be505640 1 --2- 172.21.15.99:0/420362377 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f04a00a2920 0x7f04a00a4de0 secure :-1 s=READY pgs=353 cs=0 l=1 rev1=1 crypto rx=0x7f04b0006010 tx=0x7f04b0032590 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:59:02.754 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.747+0000 7f04c3432740 1 -- start start 2024-03-19T01:59:02.754 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.747+0000 7f04c3432740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a9c0218100 0x55a9c0218520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:02.754 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.747+0000 7f04c3432740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a9c0218af0 0x55a9c021d5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:02.754 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.747+0000 7f04c3432740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a9c021dc60 0x55a9c02200f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:02.754 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.747+0000 7f04c3432740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55a9c017cbe0 con 0x55a9c0218af0 2024-03-19T01:59:02.754 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.747+0000 7f04c3432740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55a9c0125960 con 0x55a9c0218100 2024-03-19T01:59:02.754 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.747+0000 7f04c3432740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55a9c0178c80 con 0x55a9c021dc60 2024-03-19T01:59:02.754 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.747+0000 7f04be505640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a9c0218100 0x55a9c0218520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:02.754 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.747+0000 7f04be505640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a9c0218100 0x55a9c0218520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:52378/0 (socket says 172.21.15.99:52378) 2024-03-19T01:59:02.754 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.747+0000 7f04be505640 1 -- 172.21.15.99:0/3785279044 learned_addr learned my addr 172.21.15.99:0/3785279044 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:59:02.754 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.747+0000 7f04bdd04640 1 --2- 172.21.15.99:0/3785279044 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a9c0218af0 0x55a9c021d5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:02.755 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.747+0000 7f04bed06640 1 --2- 172.21.15.99:0/3785279044 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a9c021dc60 0x55a9c02200f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:02.755 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.748+0000 7f04be505640 1 -- 172.21.15.99:0/3785279044 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a9c021dc60 msgr2=0x55a9c02200f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:59:02.755 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.748+0000 7f04be505640 1 --2- 172.21.15.99:0/3785279044 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a9c021dc60 0x55a9c02200f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:02.755 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.748+0000 7f04be505640 1 -- 172.21.15.99:0/3785279044 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a9c0218af0 msgr2=0x55a9c021d5f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:59:02.755 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.748+0000 7f04be505640 1 --2- 172.21.15.99:0/3785279044 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a9c0218af0 0x55a9c021d5f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:02.755 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.748+0000 7f04be505640 1 -- 172.21.15.99:0/3785279044 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55a9c011cd20 con 0x55a9c0218100 2024-03-19T01:59:02.755 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.748+0000 7f04be505640 1 --2- 172.21.15.99:0/3785279044 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a9c0218100 0x55a9c0218520 secure :-1 s=READY pgs=188 cs=0 l=1 rev1=1 crypto rx=0x7f04b0005e30 tx=0x7f04b003e000 comp rx=0 tx=0).ready entity=mon.2 client_cookie=3d9258e2afca9d13 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:59:02.755 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.748+0000 7f04b4ff9640 1 -- 172.21.15.99:0/3785279044 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f04b0050070 con 0x55a9c0218100 2024-03-19T01:59:02.755 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.748+0000 7f04b4ff9640 1 -- 172.21.15.99:0/3785279044 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f04b0003e50 con 0x55a9c0218100 2024-03-19T01:59:02.755 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.748+0000 7f04b4ff9640 1 -- 172.21.15.99:0/3785279044 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f04b0044070 con 0x55a9c0218100 2024-03-19T01:59:02.755 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.749+0000 7f04c3432740 1 -- 172.21.15.99:0/3785279044 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a9c0218100 msgr2=0x55a9c0218520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:59:02.755 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.749+0000 7f04c3432740 1 --2- 172.21.15.99:0/3785279044 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a9c0218100 0x55a9c0218520 secure :-1 s=READY pgs=188 cs=0 l=1 rev1=1 crypto rx=0x7f04b0005e30 tx=0x7f04b003e000 comp rx=0 tx=0).stop 2024-03-19T01:59:02.755 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.749+0000 7f04c3432740 1 -- 172.21.15.99:0/3785279044 shutdown_connections 2024-03-19T01:59:02.755 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.749+0000 7f04c3432740 1 --2- 172.21.15.99:0/3785279044 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a9c021dc60 0x55a9c02200f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:02.756 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.749+0000 7f04c3432740 1 --2- 172.21.15.99:0/3785279044 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a9c0218af0 0x55a9c021d5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:02.756 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.749+0000 7f04c3432740 1 --2- 172.21.15.99:0/3785279044 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a9c0218100 0x55a9c0218520 unknown :-1 s=CLOSED pgs=188 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:02.756 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.749+0000 7f04c3432740 1 -- 172.21.15.99:0/3785279044 >> 172.21.15.99:0/3785279044 conn(0x55a9c0213670 msgr2=0x55a9c0213a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:59:02.756 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.749+0000 7f04c3432740 1 -- 172.21.15.99:0/3785279044 shutdown_connections 2024-03-19T01:59:02.756 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.749+0000 7f04c3432740 1 -- 172.21.15.99:0/3785279044 wait complete. 2024-03-19T01:59:02.756 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.749+0000 7f04c3432740 1 -- start start 2024-03-19T01:59:02.756 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.749+0000 7f04c3432740 1 Processor -- start 2024-03-19T01:59:02.756 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.749+0000 7f04c3432740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a9c0218100 0x55a9c02220d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:02.756 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.749+0000 7f04c3432740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a9c0218af0 0x55a9c0222610 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:02.756 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.749+0000 7f04c3432740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a9c021dc60 0x55a9c0229650 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:02.756 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.749+0000 7f04c3432740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55a9c0220bb0 con 0x55a9c0218af0 2024-03-19T01:59:02.756 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.749+0000 7f04c3432740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55a9c0138db0 con 0x55a9c021dc60 2024-03-19T01:59:02.756 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.749+0000 7f04c3432740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55a9c0220d30 con 0x55a9c0218100 2024-03-19T01:59:02.757 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.749+0000 7f04c3432740 10 client.0 _ll_register_callbacks cb 0x55a9c013de70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:59:02.757 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.749+0000 7f04bdd04640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a9c0218af0 0x55a9c0222610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:02.757 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.750+0000 7f04bdd04640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a9c0218af0 0x55a9c0222610 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:41530/0 (socket says 172.21.15.99:41530) 2024-03-19T01:59:02.757 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.750+0000 7f04bed06640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a9c021dc60 0x55a9c0229650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:02.757 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.750+0000 7f04be505640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a9c0218100 0x55a9c02220d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:02.757 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.750+0000 7f04bdd04640 1 -- 172.21.15.99:0/1325654686 learned_addr learned my addr 172.21.15.99:0/1325654686 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:59:02.757 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.750+0000 7f04bdd04640 1 -- 172.21.15.99:0/1325654686 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a9c0218100 msgr2=0x55a9c02220d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:59:02.757 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.750+0000 7f04bdd04640 1 --2- 172.21.15.99:0/1325654686 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a9c0218100 0x55a9c02220d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:02.757 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.750+0000 7f04bdd04640 1 -- 172.21.15.99:0/1325654686 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a9c021dc60 msgr2=0x55a9c0229650 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:59:02.757 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.750+0000 7f04bdd04640 1 --2- 172.21.15.99:0/1325654686 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a9c021dc60 0x55a9c0229650 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:02.757 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.750+0000 7f04bdd04640 1 -- 172.21.15.99:0/1325654686 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55a9c0137c60 con 0x55a9c0218af0 2024-03-19T01:59:02.757 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.750+0000 7f04bdd04640 1 --2- 172.21.15.99:0/1325654686 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a9c0218af0 0x55a9c0222610 secure :-1 s=READY pgs=688 cs=0 l=1 rev1=1 crypto rx=0x7f04ac012cf0 tx=0x7f04ac00f790 comp rx=0 tx=0).ready entity=mon.0 client_cookie=119386ce12a0b53a server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:59:02.757 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.750+0000 7f049dffb640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:59:02.757 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.750+0000 7f049dffb640 1 -- 172.21.15.99:0/1325654686 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f04ac012110 con 0x55a9c0218af0 2024-03-19T01:59:02.758 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.750+0000 7f049dffb640 1 -- 172.21.15.99:0/1325654686 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f04ac004050 con 0x55a9c0218af0 2024-03-19T01:59:02.758 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.750+0000 7f04c3432740 1 -- 172.21.15.99:0/1325654686 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f04b0003e50 con 0x55a9c0218af0 2024-03-19T01:59:02.758 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.750+0000 7f049dffb640 1 -- 172.21.15.99:0/1325654686 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7f04ac0122b0 con 0x55a9c0218af0 2024-03-19T01:59:02.758 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.750+0000 7f04c3432740 10 client.16776 fetch_fsmap learned FSMap version 37 2024-03-19T01:59:02.758 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.750+0000 7f04c3432740 1 -- 172.21.15.99:0/1325654686 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x55a9bfe91210 con 0x55a9c0218af0 2024-03-19T01:59:02.758 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.750+0000 7f049dffb640 1 -- 172.21.15.99:0/1325654686 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f04b0003e50 con 0x55a9c0218af0 2024-03-19T01:59:02.758 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.751+0000 7f049dffb640 1 -- 172.21.15.99:0/1325654686 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7f04ac010430 con 0x55a9c0218af0 2024-03-19T01:59:02.758 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.751+0000 7f04c3432740 10 client.16776 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T01:59:02.758 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.751+0000 7f04c3432740 10 client.16776 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:59:02.758 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.751+0000 7f04c3432740 20 client.16776 populate_metadata read hostname 'smithi099' 2024-03-19T01:59:02.758 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.751+0000 7f04c3432740 1 --2- 172.21.15.99:0/1325654686 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55a9c022e000 0x55a9c0230460 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:02.758 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.751+0000 7f04c3432740 4 client.16776 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:59:02.758 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.751+0000 7f04c3432740 1 -- 172.21.15.99:0/1325654686 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55a9c018a120 con 0x55a9c022e000 2024-03-19T01:59:02.759 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.751+0000 7f04be505640 1 --2- 172.21.15.99:0/1325654686 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55a9c022e000 0x55a9c0230460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:02.759 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.751+0000 7f049dffb640 10 client.16776 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:59:02.759 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.751+0000 7f04be505640 1 --2- 172.21.15.99:0/1325654686 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55a9c022e000 0x55a9c0230460 crc :-1 s=READY pgs=189 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=b60540c64dad14c7 server_cookie=b07aa13943f5c956 in_seq=0 out_seq=0 2024-03-19T01:59:02.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.757+0000 7f049dffb640 1 -- 172.21.15.99:0/1325654686 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x55a9c018a120 con 0x55a9c022e000 2024-03-19T01:59:02.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.757+0000 7f049dffb640 10 client.16776 handle_command_reply: tid=0 2024-03-19T01:59:02.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.757+0000 7f04c3432740 1 client.16776 shutdown 2024-03-19T01:59:02.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.758+0000 7f04c3432740 1 -- 172.21.15.99:0/1325654686 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a9c0218af0 msgr2=0x55a9c0222610 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:59:02.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.758+0000 7f04c3432740 1 --2- 172.21.15.99:0/1325654686 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a9c0218af0 0x55a9c0222610 secure :-1 s=READY pgs=688 cs=0 l=1 rev1=1 crypto rx=0x7f04ac012cf0 tx=0x7f04ac00f790 comp rx=0 tx=0).stop 2024-03-19T01:59:02.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.758+0000 7f04c3432740 1 -- 172.21.15.99:0/1325654686 shutdown_connections 2024-03-19T01:59:02.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.758+0000 7f04c3432740 1 --2- 172.21.15.99:0/1325654686 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55a9c022e000 0x55a9c0230460 crc :-1 s=READY pgs=189 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:02.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.758+0000 7f049dffb640 0 client.16776 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:59:02.761 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.758+0000 7f04c3432740 1 --2- 172.21.15.99:0/1325654686 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a9c021dc60 0x55a9c0229650 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:02.761 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.758+0000 7f04c3432740 1 --2- 172.21.15.99:0/1325654686 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a9c0218af0 0x55a9c0222610 unknown :-1 s=CLOSED pgs=688 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:02.761 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.758+0000 7f04c3432740 1 --2- 172.21.15.99:0/1325654686 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a9c0218100 0x55a9c02220d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:02.761 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.758+0000 7f04c3432740 1 -- 172.21.15.99:0/1325654686 >> 172.21.15.99:0/1325654686 conn(0x55a9c0213670 msgr2=0x55a9c02148c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:59:02.761 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.758+0000 7f04c3432740 1 -- 172.21.15.99:0/1325654686 shutdown_connections 2024-03-19T01:59:02.761 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.758+0000 7f04c3432740 1 -- 172.21.15.99:0/1325654686 wait complete. 2024-03-19T01:59:02.761 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.758+0000 7f04c3432740 20 client.16776 trim_cache size 0 max 16384 2024-03-19T01:59:02.778 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.774+0000 7f04c3432740 1 Processor -- start 2024-03-19T01:59:02.778 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.774+0000 7f04c3432740 1 -- start start 2024-03-19T01:59:02.778 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.775+0000 7f04c3432740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a9c0218100 0x55a9c0212ab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:02.779 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.775+0000 7f04c3432740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a9c0218af0 0x55a9c0216fe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:02.779 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.775+0000 7f04c3432740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a9c021dc60 0x55a9c0234d60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:02.779 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.775+0000 7f04c3432740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f04b81ab6d0 con 0x55a9c0218af0 2024-03-19T01:59:02.779 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.775+0000 7f04c3432740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f04b81a8fa0 con 0x55a9c0218100 2024-03-19T01:59:02.779 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.775+0000 7f04c3432740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f04b81a6df0 con 0x55a9c021dc60 2024-03-19T01:59:02.779 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.775+0000 7f04bdd04640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a9c0218af0 0x55a9c0216fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:02.779 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.775+0000 7f04bdd04640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a9c0218af0 0x55a9c0216fe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:41532/0 (socket says 172.21.15.99:41532) 2024-03-19T01:59:02.779 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.775+0000 7f04bdd04640 1 -- 172.21.15.99:0/1338258048 learned_addr learned my addr 172.21.15.99:0/1338258048 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:59:02.779 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.775+0000 7f04bed06640 1 --2- 172.21.15.99:0/1338258048 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a9c021dc60 0x55a9c0234d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:02.779 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.775+0000 7f04be505640 1 --2- 172.21.15.99:0/1338258048 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a9c0218100 0x55a9c0212ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:02.779 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.775+0000 7f04bdd04640 1 -- 172.21.15.99:0/1338258048 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a9c021dc60 msgr2=0x55a9c0234d60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:59:02.779 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.775+0000 7f04bdd04640 1 --2- 172.21.15.99:0/1338258048 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a9c021dc60 0x55a9c0234d60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:02.779 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.775+0000 7f04bdd04640 1 -- 172.21.15.99:0/1338258048 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a9c0218100 msgr2=0x55a9c0212ab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:59:02.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.775+0000 7f04bdd04640 1 --2- 172.21.15.99:0/1338258048 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a9c0218100 0x55a9c0212ab0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:02.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.775+0000 7f04bdd04640 1 -- 172.21.15.99:0/1338258048 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55a9bff5d620 con 0x55a9c0218af0 2024-03-19T01:59:02.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.775+0000 7f04bdd04640 1 --2- 172.21.15.99:0/1338258048 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a9c0218af0 0x55a9c0216fe0 secure :-1 s=READY pgs=689 cs=0 l=1 rev1=1 crypto rx=0x7f04ac0108a0 tx=0x7f04ac012910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5ebd60c34b436aca server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:59:02.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.775+0000 7f049ffff640 1 -- 172.21.15.99:0/1338258048 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f04ac00fe80 con 0x55a9c0218af0 2024-03-19T01:59:02.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.775+0000 7f049ffff640 1 -- 172.21.15.99:0/1338258048 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f04ac0055f0 con 0x55a9c0218af0 2024-03-19T01:59:02.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.775+0000 7f049ffff640 1 -- 172.21.15.99:0/1338258048 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f04ac003920 con 0x55a9c0218af0 2024-03-19T01:59:02.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.776+0000 7f04c3432740 1 -- 172.21.15.99:0/1338258048 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a9c0218af0 msgr2=0x55a9c0216fe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:59:02.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.776+0000 7f04c3432740 1 --2- 172.21.15.99:0/1338258048 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a9c0218af0 0x55a9c0216fe0 secure :-1 s=READY pgs=689 cs=0 l=1 rev1=1 crypto rx=0x7f04ac0108a0 tx=0x7f04ac012910 comp rx=0 tx=0).stop 2024-03-19T01:59:02.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.776+0000 7f04c3432740 1 -- 172.21.15.99:0/1338258048 shutdown_connections 2024-03-19T01:59:02.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.776+0000 7f04c3432740 1 --2- 172.21.15.99:0/1338258048 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a9c021dc60 0x55a9c0234d60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:02.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.776+0000 7f04c3432740 1 --2- 172.21.15.99:0/1338258048 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a9c0218af0 0x55a9c0216fe0 unknown :-1 s=CLOSED pgs=689 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:02.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.776+0000 7f04c3432740 1 --2- 172.21.15.99:0/1338258048 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a9c0218100 0x55a9c0212ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:02.780 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.776+0000 7f04c3432740 1 -- 172.21.15.99:0/1338258048 >> 172.21.15.99:0/1338258048 conn(0x55a9c0213670 msgr2=0x55a9c02168d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:59:02.781 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.776+0000 7f04c3432740 1 -- 172.21.15.99:0/1338258048 shutdown_connections 2024-03-19T01:59:02.781 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.776+0000 7f04c3432740 1 -- 172.21.15.99:0/1338258048 wait complete. 2024-03-19T01:59:02.781 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.776+0000 7f04c3432740 1 -- start start 2024-03-19T01:59:02.781 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.776+0000 7f04c3432740 1 Processor -- start 2024-03-19T01:59:02.781 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.777+0000 7f04c3432740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a9c0218100 0x55a9c02900d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:02.781 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.777+0000 7f04c3432740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a9c0218af0 0x55a9c0290610 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:02.781 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.777+0000 7f04c3432740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a9c021dc60 0x55a9c0297650 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:02.781 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.777+0000 7f04c3432740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f04b81a7460 con 0x55a9c0218100 2024-03-19T01:59:02.781 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.777+0000 7f04c3432740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f04b81a88d0 con 0x55a9c021dc60 2024-03-19T01:59:02.781 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.777+0000 7f04c3432740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f04b81a7b30 con 0x55a9c0218af0 2024-03-19T01:59:02.781 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.777+0000 7f04c3432740 10 client.0 _ll_register_callbacks cb 0x7f04ac009cc0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:59:02.781 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.777+0000 7f04be505640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a9c0218100 0x55a9c02900d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:02.781 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.777+0000 7f04bed06640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a9c021dc60 0x55a9c0297650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:02.781 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.777+0000 7f04bed06640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a9c021dc60 0x55a9c0297650 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:52402/0 (socket says 172.21.15.99:52402) 2024-03-19T01:59:02.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.777+0000 7f04bed06640 1 -- 172.21.15.99:0/4210489020 learned_addr learned my addr 172.21.15.99:0/4210489020 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:59:02.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.777+0000 7f04bdd04640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a9c0218af0 0x55a9c0290610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:02.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.777+0000 7f04be505640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a9c0218100 0x55a9c02900d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:41540/0 (socket says 172.21.15.99:41540) 2024-03-19T01:59:02.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.777+0000 7f04bed06640 1 -- 172.21.15.99:0/4210489020 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a9c0218af0 msgr2=0x55a9c0290610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:59:02.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.777+0000 7f04bed06640 1 --2- 172.21.15.99:0/4210489020 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a9c0218af0 0x55a9c0290610 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:02.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.777+0000 7f04bed06640 1 -- 172.21.15.99:0/4210489020 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a9c0218100 msgr2=0x55a9c02900d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:59:02.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.777+0000 7f04bed06640 1 --2- 172.21.15.99:0/4210489020 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a9c0218100 0x55a9c02900d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:02.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.777+0000 7f04bed06640 1 -- 172.21.15.99:0/4210489020 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55a9c0297c40 con 0x55a9c021dc60 2024-03-19T01:59:02.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.777+0000 7f04be505640 1 --2- 172.21.15.99:0/4210489020 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a9c0218100 0x55a9c02900d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:59:02.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.777+0000 7f04bed06640 1 --2- 172.21.15.99:0/4210489020 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a9c021dc60 0x55a9c0297650 secure :-1 s=READY pgs=189 cs=0 l=1 rev1=1 crypto rx=0x7f04a809b380 tx=0x7f04a809be40 comp rx=0 tx=0).ready entity=mon.2 client_cookie=bede309a5f1bed22 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:59:02.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.777+0000 7f049dffb640 10 client.0 ms_handle_connect on v2:172.21.15.112:3300/0 2024-03-19T01:59:02.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.777+0000 7f049dffb640 1 -- 172.21.15.99:0/4210489020 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f04a80b8030 con 0x55a9c021dc60 2024-03-19T01:59:02.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.777+0000 7f04c3432740 1 -- 172.21.15.99:0/4210489020 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f04ac0055f0 con 0x55a9c021dc60 2024-03-19T01:59:02.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.777+0000 7f049dffb640 1 -- 172.21.15.99:0/4210489020 <== mon.2 v2:172.21.15.112:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f04a8003b10 con 0x55a9c021dc60 2024-03-19T01:59:02.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.778+0000 7f049dffb640 1 -- 172.21.15.99:0/4210489020 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7f04a80a3e20 con 0x55a9c021dc60 2024-03-19T01:59:02.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.778+0000 7f04c3432740 10 client.26558 fetch_fsmap learned FSMap version 37 2024-03-19T01:59:02.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.778+0000 7f04c3432740 1 -- 172.21.15.99:0/4210489020 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x55a9c0299010 con 0x55a9c021dc60 2024-03-19T01:59:02.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.778+0000 7f049dffb640 1 -- 172.21.15.99:0/4210489020 <== mon.2 v2:172.21.15.112:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f04ac0055f0 con 0x55a9c021dc60 2024-03-19T01:59:02.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.778+0000 7f049dffb640 1 -- 172.21.15.99:0/4210489020 <== mon.2 v2:172.21.15.112:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7f04a8004000 con 0x55a9c021dc60 2024-03-19T01:59:02.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.778+0000 7f04c3432740 10 client.26558 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T01:59:02.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.778+0000 7f04c3432740 10 client.26558 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:59:02.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.778+0000 7f04c3432740 20 client.26558 populate_metadata read hostname 'smithi099' 2024-03-19T01:59:02.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.778+0000 7f04c3432740 1 --2- 172.21.15.99:0/4210489020 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55a9c022e000 0x55a9c029ba00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:02.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.778+0000 7f04c3432740 4 client.26558 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:59:02.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.778+0000 7f04c3432740 1 -- 172.21.15.99:0/4210489020 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55a9c0184470 con 0x55a9c022e000 2024-03-19T01:59:02.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.778+0000 7f04be505640 1 --2- 172.21.15.99:0/4210489020 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55a9c022e000 0x55a9c029ba00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:02.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.779+0000 7f04be505640 1 --2- 172.21.15.99:0/4210489020 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55a9c022e000 0x55a9c029ba00 crc :-1 s=READY pgs=190 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=8b4bd8eb13ef4f5b server_cookie=9497b5ae95d533e7 in_seq=0 out_seq=0 2024-03-19T01:59:02.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:02.779+0000 7f049dffb640 10 client.26558 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:59:03.109 INFO:tasks.workunit.client.0.smithi099.stdout: 10 145250 23.06 MB/sec execute 212 sec latency 310.606 ms 2024-03-19T01:59:03.228 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:03.227+0000 7f049dffb640 1 -- 172.21.15.99:0/4210489020 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x55a9c0184470 con 0x55a9c022e000 2024-03-19T01:59:03.228 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:03.227+0000 7f049dffb640 10 client.26558 handle_command_reply: tid=0 2024-03-19T01:59:03.228 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:03.227+0000 7f04c3432740 1 client.26558 shutdown 2024-03-19T01:59:03.228 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:03.227+0000 7f04c3432740 1 -- 172.21.15.99:0/4210489020 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a9c021dc60 msgr2=0x55a9c0297650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:59:03.228 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:03.227+0000 7f04c3432740 1 --2- 172.21.15.99:0/4210489020 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a9c021dc60 0x55a9c0297650 secure :-1 s=READY pgs=189 cs=0 l=1 rev1=1 crypto rx=0x7f04a809b380 tx=0x7f04a809be40 comp rx=0 tx=0).stop 2024-03-19T01:59:03.228 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:03.228+0000 7f04c3432740 1 -- 172.21.15.99:0/4210489020 shutdown_connections 2024-03-19T01:59:03.228 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:03.228+0000 7f04c3432740 1 --2- 172.21.15.99:0/4210489020 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55a9c022e000 0x55a9c029ba00 crc :-1 s=READY pgs=190 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:03.228 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:03.228+0000 7f04c3432740 1 --2- 172.21.15.99:0/4210489020 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a9c021dc60 0x55a9c0297650 unknown :-1 s=CLOSED pgs=189 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:03.228 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:03.228+0000 7f04c3432740 1 --2- 172.21.15.99:0/4210489020 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a9c0218af0 0x55a9c0290610 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:03.228 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:03.228+0000 7f04c3432740 1 --2- 172.21.15.99:0/4210489020 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a9c0218100 0x55a9c02900d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:03.229 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:03.228+0000 7f049dffb640 0 client.26558 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:59:03.229 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:03.228+0000 7f04c3432740 1 -- 172.21.15.99:0/4210489020 >> 172.21.15.99:0/4210489020 conn(0x55a9c0213670 msgr2=0x55a9c0217160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:59:03.229 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:03.228+0000 7f04c3432740 1 -- 172.21.15.99:0/4210489020 shutdown_connections 2024-03-19T01:59:03.229 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:03.228+0000 7f04c3432740 1 -- 172.21.15.99:0/4210489020 wait complete. 2024-03-19T01:59:03.229 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:03.228+0000 7f04c3432740 20 client.26558 trim_cache size 0 max 16384 2024-03-19T01:59:03.229 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:59:03.229 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:59:03.229 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:59:03.229 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:59:03.230 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:03.228+0000 7f04c0769640 1 -- 172.21.15.99:0/420362377 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f04a00a2920 msgr2=0x7f04a00a4de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:59:03.230 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:03.228+0000 7f04c0769640 1 --2- 172.21.15.99:0/420362377 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f04a00a2920 0x7f04a00a4de0 secure :-1 s=READY pgs=353 cs=0 l=1 rev1=1 crypto rx=0x7f04b0006010 tx=0x7f04b0032590 comp rx=0 tx=0).stop 2024-03-19T01:59:03.230 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:03.228+0000 7f04c0769640 1 -- 172.21.15.99:0/420362377 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f04b81075c0 msgr2=0x7f04b819f0e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:59:03.230 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:03.228+0000 7f04c0769640 1 --2- 172.21.15.99:0/420362377 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f04b81075c0 0x7f04b819f0e0 secure :-1 s=READY pgs=687 cs=0 l=1 rev1=1 crypto rx=0x7f04a800e9b0 tx=0x7f04a800ee70 comp rx=0 tx=0).stop 2024-03-19T01:59:03.230 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:03.228+0000 7f04c0769640 1 -- 172.21.15.99:0/420362377 shutdown_connections 2024-03-19T01:59:03.231 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:03.228+0000 7f04c0769640 1 --2- 172.21.15.99:0/420362377 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f04a00a2920 0x7f04a00a4de0 unknown :-1 s=CLOSED pgs=353 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:03.231 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:03.228+0000 7f04c0769640 1 --2- 172.21.15.99:0/420362377 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f04b8104c90 0x7f04b8198070 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:03.231 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:03.228+0000 7f04c0769640 1 --2- 172.21.15.99:0/420362377 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f04b81075c0 0x7f04b819f0e0 unknown :-1 s=CLOSED pgs=687 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:03.231 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:03.228+0000 7f04c0769640 1 --2- 172.21.15.99:0/420362377 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f04b8102360 0x7f04b8197b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:03.231 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:03.228+0000 7f04c0769640 1 -- 172.21.15.99:0/420362377 >> 172.21.15.99:0/420362377 conn(0x7f04b80fbcb0 msgr2=0x7f04b8102de0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:59:03.231 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:03.228+0000 7f04c0769640 1 -- 172.21.15.99:0/420362377 shutdown_connections 2024-03-19T01:59:03.231 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:03.228+0000 7f04c0769640 1 -- 172.21.15.99:0/420362377 wait complete. 2024-03-19T01:59:03.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:59:03 smithi112 ceph-mon[66819]: pgmap v890: 97 pgs: 97 active+clean; 593 MiB data, 6.9 GiB used, 1.0 TiB / 1.0 TiB avail; 4.9 MiB/s rd, 6.5 MiB/s wr, 571 op/s 2024-03-19T01:59:03.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:59:03 smithi115 ceph-mon[67332]: pgmap v890: 97 pgs: 97 active+clean; 593 MiB data, 6.9 GiB used, 1.0 TiB / 1.0 TiB avail; 4.9 MiB/s rd, 6.5 MiB/s wr, 571 op/s 2024-03-19T01:59:03.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:59:03 smithi099 ceph-mon[62951]: pgmap v890: 97 pgs: 97 active+clean; 593 MiB data, 6.9 GiB used, 1.0 TiB / 1.0 TiB avail; 4.9 MiB/s rd, 6.5 MiB/s wr, 571 op/s 2024-03-19T01:59:04.109 INFO:tasks.workunit.client.0.smithi099.stdout: 10 145687 23.03 MB/sec execute 213 sec latency 370.734 ms 2024-03-19T01:59:05.109 INFO:tasks.workunit.client.0.smithi099.stdout: 10 146248 23.04 MB/sec execute 214 sec latency 51.241 ms 2024-03-19T01:59:05.364 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:59:05 smithi112 ceph-mon[66819]: pgmap v891: 97 pgs: 97 active+clean; 592 MiB data, 7.0 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 5.4 MiB/s wr, 426 op/s 2024-03-19T01:59:05.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:59:05 smithi115 ceph-mon[67332]: pgmap v891: 97 pgs: 97 active+clean; 592 MiB data, 7.0 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 5.4 MiB/s wr, 426 op/s 2024-03-19T01:59:05.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:59:05 smithi099 ceph-mon[62951]: pgmap v891: 97 pgs: 97 active+clean; 592 MiB data, 7.0 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 5.4 MiB/s wr, 426 op/s 2024-03-19T01:59:06.109 INFO:tasks.workunit.client.0.smithi099.stdout: 10 146604 23.04 MB/sec execute 215 sec latency 285.686 ms 2024-03-19T01:59:07.109 INFO:tasks.workunit.client.0.smithi099.stdout: 10 147117 23.07 MB/sec execute 216 sec latency 81.105 ms 2024-03-19T01:59:07.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:59:07 smithi112 ceph-mon[66819]: pgmap v892: 97 pgs: 97 active+clean; 587 MiB data, 7.0 GiB used, 1.0 TiB / 1.0 TiB avail; 5.3 MiB/s rd, 6.7 MiB/s wr, 567 op/s 2024-03-19T01:59:07.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:59:07 smithi115 ceph-mon[67332]: pgmap v892: 97 pgs: 97 active+clean; 587 MiB data, 7.0 GiB used, 1.0 TiB / 1.0 TiB avail; 5.3 MiB/s rd, 6.7 MiB/s wr, 567 op/s 2024-03-19T01:59:07.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:59:07 smithi099 ceph-mon[62951]: pgmap v892: 97 pgs: 97 active+clean; 587 MiB data, 7.0 GiB used, 1.0 TiB / 1.0 TiB avail; 5.3 MiB/s rd, 6.7 MiB/s wr, 567 op/s 2024-03-19T01:59:08.109 INFO:tasks.workunit.client.0.smithi099.stdout: 10 147656 23.13 MB/sec execute 217 sec latency 55.860 ms 2024-03-19T01:59:09.109 INFO:tasks.workunit.client.0.smithi099.stdout: 10 148071 23.14 MB/sec execute 218 sec latency 85.393 ms 2024-03-19T01:59:09.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:59:09 smithi112 ceph-mon[66819]: pgmap v893: 97 pgs: 97 active+clean; 588 MiB data, 7.0 GiB used, 1.0 TiB / 1.0 TiB avail; 4.5 MiB/s rd, 5.5 MiB/s wr, 529 op/s 2024-03-19T01:59:09.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:59:09 smithi115 ceph-mon[67332]: pgmap v893: 97 pgs: 97 active+clean; 588 MiB data, 7.0 GiB used, 1.0 TiB / 1.0 TiB avail; 4.5 MiB/s rd, 5.5 MiB/s wr, 529 op/s 2024-03-19T01:59:09.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:59:09 smithi099 ceph-mon[62951]: pgmap v893: 97 pgs: 97 active+clean; 588 MiB data, 7.0 GiB used, 1.0 TiB / 1.0 TiB avail; 4.5 MiB/s rd, 5.5 MiB/s wr, 529 op/s 2024-03-19T01:59:10.109 INFO:tasks.workunit.client.0.smithi099.stdout: 10 148566 23.13 MB/sec execute 219 sec latency 63.213 ms 2024-03-19T01:59:11.109 INFO:tasks.workunit.client.0.smithi099.stdout: 10 148935 23.12 MB/sec execute 220 sec latency 284.519 ms 2024-03-19T01:59:11.806 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:59:11 smithi115 ceph-mon[67332]: pgmap v894: 97 pgs: 97 active+clean; 590 MiB data, 7.0 GiB used, 1.0 TiB / 1.0 TiB avail; 4.8 MiB/s rd, 6.0 MiB/s wr, 572 op/s 2024-03-19T01:59:11.840 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:59:11 smithi099 ceph-mon[62951]: pgmap v894: 97 pgs: 97 active+clean; 590 MiB data, 7.0 GiB used, 1.0 TiB / 1.0 TiB avail; 4.8 MiB/s rd, 6.0 MiB/s wr, 572 op/s 2024-03-19T01:59:11.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:59:11 smithi112 ceph-mon[66819]: pgmap v894: 97 pgs: 97 active+clean; 590 MiB data, 7.0 GiB used, 1.0 TiB / 1.0 TiB avail; 4.8 MiB/s rd, 6.0 MiB/s wr, 572 op/s 2024-03-19T01:59:12.109 INFO:tasks.workunit.client.0.smithi099.stdout: 10 149296 23.12 MB/sec execute 221 sec latency 138.864 ms 2024-03-19T01:59:13.109 INFO:tasks.workunit.client.0.smithi099.stdout: 10 149715 23.10 MB/sec execute 222 sec latency 55.627 ms 2024-03-19T01:59:13.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:59:13 smithi112 ceph-mon[66819]: pgmap v895: 97 pgs: 97 active+clean; 592 MiB data, 7.0 GiB used, 1.0 TiB / 1.0 TiB avail; 5.2 MiB/s rd, 6.2 MiB/s wr, 601 op/s 2024-03-19T01:59:13.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:59:13 smithi115 ceph-mon[67332]: pgmap v895: 97 pgs: 97 active+clean; 592 MiB data, 7.0 GiB used, 1.0 TiB / 1.0 TiB avail; 5.2 MiB/s rd, 6.2 MiB/s wr, 601 op/s 2024-03-19T01:59:13.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:59:13 smithi099 ceph-mon[62951]: pgmap v895: 97 pgs: 97 active+clean; 592 MiB data, 7.0 GiB used, 1.0 TiB / 1.0 TiB avail; 5.2 MiB/s rd, 6.2 MiB/s wr, 601 op/s 2024-03-19T01:59:14.109 INFO:tasks.workunit.client.0.smithi099.stdout: 10 150027 23.07 MB/sec execute 223 sec latency 164.684 ms 2024-03-19T01:59:15.109 INFO:tasks.workunit.client.0.smithi099.stdout: 10 150556 23.06 MB/sec execute 224 sec latency 64.839 ms 2024-03-19T01:59:15.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:59:15 smithi112 ceph-mon[66819]: pgmap v896: 97 pgs: 97 active+clean; 595 MiB data, 7.1 GiB used, 1.0 TiB / 1.0 TiB avail; 4.4 MiB/s rd, 5.4 MiB/s wr, 527 op/s 2024-03-19T01:59:15.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:59:15 smithi115 ceph-mon[67332]: pgmap v896: 97 pgs: 97 active+clean; 595 MiB data, 7.1 GiB used, 1.0 TiB / 1.0 TiB avail; 4.4 MiB/s rd, 5.4 MiB/s wr, 527 op/s 2024-03-19T01:59:15.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:59:15 smithi099 ceph-mon[62951]: pgmap v896: 97 pgs: 97 active+clean; 595 MiB data, 7.1 GiB used, 1.0 TiB / 1.0 TiB avail; 4.4 MiB/s rd, 5.4 MiB/s wr, 527 op/s 2024-03-19T01:59:16.109 INFO:tasks.workunit.client.0.smithi099.stdout: 10 150980 23.04 MB/sec execute 225 sec latency 396.071 ms 2024-03-19T01:59:16.329 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.328+0000 7f76c9df4640 1 -- 172.21.15.99:0/3076537883 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f76c4102330 msgr2=0x7f76c4108bc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:59:16.329 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.328+0000 7f76c9df4640 1 --2- 172.21.15.99:0/3076537883 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f76c4102330 0x7f76c4108bc0 secure :-1 s=READY pgs=690 cs=0 l=1 rev1=1 crypto rx=0x7f76bc009ca0 tx=0x7f76bc02c0f0 comp rx=0 tx=0).stop 2024-03-19T01:59:16.329 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.328+0000 7f76c9df4640 1 -- 172.21.15.99:0/3076537883 shutdown_connections 2024-03-19T01:59:16.329 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.328+0000 7f76c9df4640 1 --2- 172.21.15.99:0/3076537883 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f76c4102330 0x7f76c4108bc0 unknown :-1 s=CLOSED pgs=690 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:16.329 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.328+0000 7f76c9df4640 1 --2- 172.21.15.99:0/3076537883 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f76c4101970 0x7f76c4101df0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:16.329 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.328+0000 7f76c9df4640 1 --2- 172.21.15.99:0/3076537883 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f76c4100780 0x7f76c4100ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:16.329 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.328+0000 7f76c9df4640 1 -- 172.21.15.99:0/3076537883 >> 172.21.15.99:0/3076537883 conn(0x7f76c40fbcf0 msgr2=0x7f76c40fe130 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:59:16.329 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.328+0000 7f76c9df4640 1 -- 172.21.15.99:0/3076537883 shutdown_connections 2024-03-19T01:59:16.329 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.328+0000 7f76c9df4640 1 -- 172.21.15.99:0/3076537883 wait complete. 2024-03-19T01:59:16.330 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.329+0000 7f76c9df4640 1 Processor -- start 2024-03-19T01:59:16.330 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.329+0000 7f76c9df4640 1 -- start start 2024-03-19T01:59:16.331 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.330+0000 7f76c9df4640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f76c4100780 0x7f76c4197a80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:16.331 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.330+0000 7f76c9df4640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f76c4101970 0x7f76c4197fc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:16.331 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.330+0000 7f76c9df4640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f76c4102330 0x7f76c419f030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:16.332 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.330+0000 7f76c9df4640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f76c410c400 con 0x7f76c4100780 2024-03-19T01:59:16.332 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.330+0000 7f76c9df4640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f76c410c280 con 0x7f76c4102330 2024-03-19T01:59:16.332 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.330+0000 7f76c9df4640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f76c410c580 con 0x7f76c4101970 2024-03-19T01:59:16.332 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.330+0000 7f76c37fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f76c4100780 0x7f76c4197a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:16.332 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.330+0000 7f76c37fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f76c4100780 0x7f76c4197a80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:48476/0 (socket says 172.21.15.99:48476) 2024-03-19T01:59:16.332 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.330+0000 7f76c37fe640 1 -- 172.21.15.99:0/86335691 learned_addr learned my addr 172.21.15.99:0/86335691 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:59:16.332 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.330+0000 7f76c3fff640 1 --2- 172.21.15.99:0/86335691 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f76c4102330 0x7f76c419f030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:16.332 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.330+0000 7f76c2ffd640 1 --2- 172.21.15.99:0/86335691 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f76c4101970 0x7f76c4197fc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:16.332 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.330+0000 7f76c37fe640 1 -- 172.21.15.99:0/86335691 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f76c4101970 msgr2=0x7f76c4197fc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:59:16.332 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.330+0000 7f76c37fe640 1 --2- 172.21.15.99:0/86335691 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f76c4101970 0x7f76c4197fc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:16.332 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.330+0000 7f76c37fe640 1 -- 172.21.15.99:0/86335691 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f76c4102330 msgr2=0x7f76c419f030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:59:16.332 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.330+0000 7f76c37fe640 1 --2- 172.21.15.99:0/86335691 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f76c4102330 0x7f76c419f030 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:16.332 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.330+0000 7f76c37fe640 1 -- 172.21.15.99:0/86335691 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f76c419f6a0 con 0x7f76c4100780 2024-03-19T01:59:16.332 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.330+0000 7f76c2ffd640 1 --2- 172.21.15.99:0/86335691 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f76c4101970 0x7f76c4197fc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-19T01:59:16.332 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.330+0000 7f76c3fff640 1 --2- 172.21.15.99:0/86335691 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f76c4102330 0x7f76c419f030 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:59:16.333 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.330+0000 7f76c37fe640 1 --2- 172.21.15.99:0/86335691 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f76c4100780 0x7f76c4197a80 secure :-1 s=READY pgs=691 cs=0 l=1 rev1=1 crypto rx=0x7f76b400e9c0 tx=0x7f76b400ee80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:59:16.333 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.330+0000 7f76c0ff9640 1 -- 172.21.15.99:0/86335691 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f76b400cde0 con 0x7f76c4100780 2024-03-19T01:59:16.333 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.330+0000 7f76c0ff9640 1 -- 172.21.15.99:0/86335691 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f76b4010910 con 0x7f76c4100780 2024-03-19T01:59:16.333 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.330+0000 7f76c9df4640 1 -- 172.21.15.99:0/86335691 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f76c419f990 con 0x7f76c4100780 2024-03-19T01:59:16.333 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.330+0000 7f76c0ff9640 1 -- 172.21.15.99:0/86335691 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f76b4018a00 con 0x7f76c4100780 2024-03-19T01:59:16.333 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.330+0000 7f76c9df4640 1 -- 172.21.15.99:0/86335691 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f76c419fee0 con 0x7f76c4100780 2024-03-19T01:59:16.339 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.333+0000 7f76c0ff9640 1 -- 172.21.15.99:0/86335691 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f76b401f020 con 0x7f76c4100780 2024-03-19T01:59:16.339 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.333+0000 7f76c0ff9640 1 --2- 172.21.15.99:0/86335691 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f76ac0a28a0 0x7f76ac0a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:16.339 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.333+0000 7f76cc9e6740 1 Processor -- start 2024-03-19T01:59:16.340 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.333+0000 7f76cc9e6740 1 -- start start 2024-03-19T01:59:16.340 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.333+0000 7f76c2ffd640 1 --2- 172.21.15.99:0/86335691 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f76ac0a28a0 0x7f76ac0a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:16.340 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.333+0000 7f76c0ff9640 1 -- 172.21.15.99:0/86335691 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f76b40ced30 con 0x7f76c4100780 2024-03-19T01:59:16.340 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.333+0000 7f76cc9e6740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5613a0b0e100 0x5613a0b0e520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:16.340 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.333+0000 7f76cc9e6740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5613a0b0eaf0 0x5613a0b135f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:16.340 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.333+0000 7f76cc9e6740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5613a0b13c60 0x5613a0b160f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:16.340 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.333+0000 7f76cc9e6740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x5613a0a72be0 con 0x5613a0b13c60 2024-03-19T01:59:16.340 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.333+0000 7f76cc9e6740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x5613a0a1b960 con 0x5613a0b0eaf0 2024-03-19T01:59:16.340 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.333+0000 7f76cc9e6740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x5613a0a6ec80 con 0x5613a0b0e100 2024-03-19T01:59:16.340 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.333+0000 7f76c2ffd640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5613a0b0eaf0 0x5613a0b135f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:16.340 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.333+0000 7f76c3fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5613a0b13c60 0x5613a0b160f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:16.340 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.333+0000 7f76c37fe640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5613a0b0e100 0x5613a0b0e520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:16.340 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.333+0000 7f76c2ffd640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5613a0b0eaf0 0x5613a0b135f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:41786/0 (socket says 172.21.15.99:41786) 2024-03-19T01:59:16.341 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.333+0000 7f76c3fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5613a0b13c60 0x5613a0b160f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:48478/0 (socket says 172.21.15.99:48478) 2024-03-19T01:59:16.341 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.333+0000 7f76c2ffd640 1 -- 172.21.15.99:0/2799183967 learned_addr learned my addr 172.21.15.99:0/2799183967 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:59:16.341 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.334+0000 7f76c2ffd640 1 --2- 172.21.15.99:0/86335691 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f76ac0a28a0 0x7f76ac0a4d60 secure :-1 s=READY pgs=354 cs=0 l=1 rev1=1 crypto rx=0x7f76b8015670 tx=0x7f76b8015a70 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:59:16.341 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.334+0000 7f76c3fff640 1 -- 172.21.15.99:0/2799183967 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5613a0b0e100 msgr2=0x5613a0b0e520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:59:16.341 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.334+0000 7f76c3fff640 1 --2- 172.21.15.99:0/2799183967 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5613a0b0e100 0x5613a0b0e520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:16.341 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.334+0000 7f76c3fff640 1 -- 172.21.15.99:0/2799183967 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5613a0b0eaf0 msgr2=0x5613a0b135f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:59:16.341 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.334+0000 7f76c3fff640 1 --2- 172.21.15.99:0/2799183967 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5613a0b0eaf0 0x5613a0b135f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:16.341 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.334+0000 7f76c3fff640 1 -- 172.21.15.99:0/2799183967 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5613a0a12d20 con 0x5613a0b13c60 2024-03-19T01:59:16.341 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.334+0000 7f76c2ffd640 1 --2- 172.21.15.99:0/2799183967 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5613a0b0eaf0 0x5613a0b135f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:59:16.341 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.334+0000 7f76c3fff640 1 --2- 172.21.15.99:0/2799183967 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5613a0b13c60 0x5613a0b160f0 secure :-1 s=READY pgs=692 cs=0 l=1 rev1=1 crypto rx=0x7f76bc02c630 tx=0x7f76bc005ef0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=be9ce1b060d95f3b server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:59:16.341 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.334+0000 7f76a9ffb640 1 -- 172.21.15.99:0/2799183967 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f76bc043070 con 0x5613a0b13c60 2024-03-19T01:59:16.341 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.334+0000 7f76a9ffb640 1 -- 172.21.15.99:0/2799183967 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f76bc02cca0 con 0x5613a0b13c60 2024-03-19T01:59:16.341 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.334+0000 7f76a9ffb640 1 -- 172.21.15.99:0/2799183967 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f76bc039040 con 0x5613a0b13c60 2024-03-19T01:59:16.341 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.335+0000 7f76cc9e6740 1 -- 172.21.15.99:0/2799183967 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5613a0b13c60 msgr2=0x5613a0b160f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:59:16.341 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.335+0000 7f76cc9e6740 1 --2- 172.21.15.99:0/2799183967 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5613a0b13c60 0x5613a0b160f0 secure :-1 s=READY pgs=692 cs=0 l=1 rev1=1 crypto rx=0x7f76bc02c630 tx=0x7f76bc005ef0 comp rx=0 tx=0).stop 2024-03-19T01:59:16.342 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.335+0000 7f76cc9e6740 1 -- 172.21.15.99:0/2799183967 shutdown_connections 2024-03-19T01:59:16.342 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.335+0000 7f76cc9e6740 1 --2- 172.21.15.99:0/2799183967 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5613a0b0eaf0 0x5613a0b135f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:16.342 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.335+0000 7f76cc9e6740 1 --2- 172.21.15.99:0/2799183967 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5613a0b13c60 0x5613a0b160f0 unknown :-1 s=CLOSED pgs=692 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:16.342 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.335+0000 7f76cc9e6740 1 --2- 172.21.15.99:0/2799183967 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5613a0b0e100 0x5613a0b0e520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:16.342 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.335+0000 7f76cc9e6740 1 -- 172.21.15.99:0/2799183967 >> 172.21.15.99:0/2799183967 conn(0x5613a0b09670 msgr2=0x5613a0b09a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:59:16.342 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.335+0000 7f76cc9e6740 1 -- 172.21.15.99:0/2799183967 shutdown_connections 2024-03-19T01:59:16.342 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.335+0000 7f76cc9e6740 1 -- 172.21.15.99:0/2799183967 wait complete. 2024-03-19T01:59:16.342 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.335+0000 7f76cc9e6740 1 -- start start 2024-03-19T01:59:16.342 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.335+0000 7f76cc9e6740 1 Processor -- start 2024-03-19T01:59:16.342 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.336+0000 7f76cc9e6740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5613a0b0e100 0x5613a0b17ff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:16.342 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.336+0000 7f76cc9e6740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5613a0b0eaf0 0x5613a0b18530 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:16.342 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.336+0000 7f76cc9e6740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5613a0b13c60 0x5613a0b1f570 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:16.342 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.336+0000 7f76cc9e6740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x5613a0b16bb0 con 0x5613a0b13c60 2024-03-19T01:59:16.342 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.336+0000 7f76cc9e6740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x5613a0a2edb0 con 0x5613a0b0eaf0 2024-03-19T01:59:16.343 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.336+0000 7f76cc9e6740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x5613a0b16d30 con 0x5613a0b0e100 2024-03-19T01:59:16.343 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.336+0000 7f76cc9e6740 10 client.0 _ll_register_callbacks cb 0x5613a0a33e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:59:16.343 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.336+0000 7f76c3fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5613a0b13c60 0x5613a0b1f570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:16.343 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.336+0000 7f76c3fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5613a0b13c60 0x5613a0b1f570 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:48482/0 (socket says 172.21.15.99:48482) 2024-03-19T01:59:16.343 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.336+0000 7f76c2ffd640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5613a0b0eaf0 0x5613a0b18530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:16.343 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.336+0000 7f76c3fff640 1 -- 172.21.15.99:0/4201654228 learned_addr learned my addr 172.21.15.99:0/4201654228 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:59:16.343 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.336+0000 7f76c37fe640 1 --2- 172.21.15.99:0/4201654228 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5613a0b0e100 0x5613a0b17ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:16.343 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.336+0000 7f76c3fff640 1 -- 172.21.15.99:0/4201654228 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5613a0b0e100 msgr2=0x5613a0b17ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:59:16.343 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.336+0000 7f76c3fff640 1 --2- 172.21.15.99:0/4201654228 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5613a0b0e100 0x5613a0b17ff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:16.343 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.336+0000 7f76c3fff640 1 -- 172.21.15.99:0/4201654228 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5613a0b0eaf0 msgr2=0x5613a0b18530 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:59:16.343 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.336+0000 7f76c3fff640 1 --2- 172.21.15.99:0/4201654228 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5613a0b0eaf0 0x5613a0b18530 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:16.343 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.336+0000 7f76c3fff640 1 -- 172.21.15.99:0/4201654228 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5613a0a2dc60 con 0x5613a0b13c60 2024-03-19T01:59:16.343 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.336+0000 7f76c2ffd640 1 --2- 172.21.15.99:0/4201654228 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5613a0b0eaf0 0x5613a0b18530 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:59:16.343 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.336+0000 7f76c3fff640 1 --2- 172.21.15.99:0/4201654228 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5613a0b13c60 0x5613a0b1f570 secure :-1 s=READY pgs=693 cs=0 l=1 rev1=1 crypto rx=0x7f76bc037040 tx=0x7f76bc0093e0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2e7444c8eccf700d server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:59:16.344 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.336+0000 7f76a2ffd640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:59:16.344 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.336+0000 7f76a2ffd640 1 -- 172.21.15.99:0/4201654228 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f76bc04c030 con 0x5613a0b13c60 2024-03-19T01:59:16.344 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.336+0000 7f76a2ffd640 1 -- 172.21.15.99:0/4201654228 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f76bc0328a0 con 0x5613a0b13c60 2024-03-19T01:59:16.344 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.336+0000 7f76cc9e6740 1 -- 172.21.15.99:0/4201654228 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f76bc02cca0 con 0x5613a0b13c60 2024-03-19T01:59:16.344 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.336+0000 7f76a2ffd640 1 -- 172.21.15.99:0/4201654228 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7f76bc039070 con 0x5613a0b13c60 2024-03-19T01:59:16.344 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.337+0000 7f76cc9e6740 10 client.16794 fetch_fsmap learned FSMap version 37 2024-03-19T01:59:16.344 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.337+0000 7f76cc9e6740 1 -- 172.21.15.99:0/4201654228 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x5613a0787210 con 0x5613a0b13c60 2024-03-19T01:59:16.344 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.337+0000 7f76a2ffd640 1 -- 172.21.15.99:0/4201654228 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f76bc02cca0 con 0x5613a0b13c60 2024-03-19T01:59:16.344 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.337+0000 7f76a2ffd640 1 -- 172.21.15.99:0/4201654228 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7f76bc048a10 con 0x5613a0b13c60 2024-03-19T01:59:16.344 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.337+0000 7f76cc9e6740 10 client.16794 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T01:59:16.344 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.337+0000 7f76cc9e6740 10 client.16794 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:59:16.344 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.337+0000 7f76cc9e6740 20 client.16794 populate_metadata read hostname 'smithi099' 2024-03-19T01:59:16.344 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.337+0000 7f76cc9e6740 1 --2- 172.21.15.99:0/4201654228 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5613a0b23a60 0x5613a0b25e50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:16.344 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.337+0000 7f76cc9e6740 4 client.16794 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:59:16.344 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.337+0000 7f76cc9e6740 1 -- 172.21.15.99:0/4201654228 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5613a0a80120 con 0x5613a0b23a60 2024-03-19T01:59:16.344 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.337+0000 7f76c37fe640 1 --2- 172.21.15.99:0/4201654228 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5613a0b23a60 0x5613a0b25e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:16.345 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.337+0000 7f76a2ffd640 10 client.16794 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:59:16.345 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.337+0000 7f76c37fe640 1 --2- 172.21.15.99:0/4201654228 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5613a0b23a60 0x5613a0b25e50 crc :-1 s=READY pgs=191 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=15f879627ff0d629 server_cookie=75319db2440bad8e in_seq=0 out_seq=0 2024-03-19T01:59:16.345 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.339+0000 7f76a2ffd640 1 -- 172.21.15.99:0/4201654228 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x5613a0a80120 con 0x5613a0b23a60 2024-03-19T01:59:16.345 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.339+0000 7f76a2ffd640 10 client.16794 handle_command_reply: tid=0 2024-03-19T01:59:16.345 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.339+0000 7f76cc9e6740 1 client.16794 shutdown 2024-03-19T01:59:16.345 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.339+0000 7f76cc9e6740 1 -- 172.21.15.99:0/4201654228 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5613a0b13c60 msgr2=0x5613a0b1f570 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:59:16.345 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.339+0000 7f76cc9e6740 1 --2- 172.21.15.99:0/4201654228 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5613a0b13c60 0x5613a0b1f570 secure :-1 s=READY pgs=693 cs=0 l=1 rev1=1 crypto rx=0x7f76bc037040 tx=0x7f76bc0093e0 comp rx=0 tx=0).stop 2024-03-19T01:59:16.345 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.339+0000 7f76cc9e6740 1 -- 172.21.15.99:0/4201654228 shutdown_connections 2024-03-19T01:59:16.345 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.339+0000 7f76cc9e6740 1 --2- 172.21.15.99:0/4201654228 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5613a0b23a60 0x5613a0b25e50 crc :-1 s=READY pgs=191 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:16.345 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.339+0000 7f76cc9e6740 1 --2- 172.21.15.99:0/4201654228 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5613a0b0eaf0 0x5613a0b18530 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:16.345 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.339+0000 7f76a2ffd640 0 client.16794 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:59:16.345 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.339+0000 7f76cc9e6740 1 --2- 172.21.15.99:0/4201654228 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5613a0b13c60 0x5613a0b1f570 unknown :-1 s=CLOSED pgs=693 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:16.345 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.339+0000 7f76cc9e6740 1 --2- 172.21.15.99:0/4201654228 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5613a0b0e100 0x5613a0b17ff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:16.345 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.339+0000 7f76cc9e6740 1 -- 172.21.15.99:0/4201654228 >> 172.21.15.99:0/4201654228 conn(0x5613a0b09670 msgr2=0x5613a0b0a8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:59:16.345 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.339+0000 7f76cc9e6740 1 -- 172.21.15.99:0/4201654228 shutdown_connections 2024-03-19T01:59:16.346 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.339+0000 7f76cc9e6740 1 -- 172.21.15.99:0/4201654228 wait complete. 2024-03-19T01:59:16.346 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.339+0000 7f76cc9e6740 20 client.16794 trim_cache size 0 max 16384 2024-03-19T01:59:16.356 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.355+0000 7f76cc9e6740 1 Processor -- start 2024-03-19T01:59:16.356 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.355+0000 7f76cc9e6740 1 -- start start 2024-03-19T01:59:16.356 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.356+0000 7f76cc9e6740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5613a0b0e100 0x5613a0b0f590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:16.356 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.356+0000 7f76cc9e6740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5613a0b0eaf0 0x5613a0b1f530 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:16.356 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.356+0000 7f76cc9e6740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5613a0b13c60 0x5613a0b0fad0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:16.357 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.356+0000 7f76cc9e6740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f76c41ab660 con 0x5613a0b0eaf0 2024-03-19T01:59:16.357 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.356+0000 7f76cc9e6740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f76c41a8f30 con 0x5613a0b13c60 2024-03-19T01:59:16.357 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.356+0000 7f76cc9e6740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f76c41a6d80 con 0x5613a0b0e100 2024-03-19T01:59:16.357 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.356+0000 7f76c2ffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5613a0b0eaf0 0x5613a0b1f530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:16.357 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.356+0000 7f76c2ffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5613a0b0eaf0 0x5613a0b1f530 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:48494/0 (socket says 172.21.15.99:48494) 2024-03-19T01:59:16.357 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.356+0000 7f76c2ffd640 1 -- 172.21.15.99:0/834093512 learned_addr learned my addr 172.21.15.99:0/834093512 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:59:16.357 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.356+0000 7f76c3fff640 1 --2- 172.21.15.99:0/834093512 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5613a0b13c60 0x5613a0b0fad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:16.357 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.356+0000 7f76c37fe640 1 --2- 172.21.15.99:0/834093512 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5613a0b0e100 0x5613a0b0f590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:16.357 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.356+0000 7f76c2ffd640 1 -- 172.21.15.99:0/834093512 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5613a0b0e100 msgr2=0x5613a0b0f590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:59:16.357 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.356+0000 7f76c2ffd640 1 --2- 172.21.15.99:0/834093512 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5613a0b0e100 0x5613a0b0f590 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:16.357 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.356+0000 7f76c2ffd640 1 -- 172.21.15.99:0/834093512 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5613a0b13c60 msgr2=0x5613a0b0fad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:59:16.357 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.356+0000 7f76c2ffd640 1 --2- 172.21.15.99:0/834093512 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5613a0b13c60 0x5613a0b0fad0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:16.357 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.356+0000 7f76c2ffd640 1 -- 172.21.15.99:0/834093512 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5613a0853620 con 0x5613a0b0eaf0 2024-03-19T01:59:16.358 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.356+0000 7f76c3fff640 1 --2- 172.21.15.99:0/834093512 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5613a0b13c60 0x5613a0b0fad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:59:16.358 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.356+0000 7f76c2ffd640 1 --2- 172.21.15.99:0/834093512 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5613a0b0eaf0 0x5613a0b1f530 secure :-1 s=READY pgs=694 cs=0 l=1 rev1=1 crypto rx=0x7f76b8011780 tx=0x7f76b801e000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=324966ae98c3b0d2 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:59:16.358 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.357+0000 7f76a97fa640 1 -- 172.21.15.99:0/834093512 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f76b80125d0 con 0x5613a0b0eaf0 2024-03-19T01:59:16.358 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.357+0000 7f76a97fa640 1 -- 172.21.15.99:0/834093512 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f76b8012c50 con 0x5613a0b0eaf0 2024-03-19T01:59:16.358 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.357+0000 7f76a97fa640 1 -- 172.21.15.99:0/834093512 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f76b8021dd0 con 0x5613a0b0eaf0 2024-03-19T01:59:16.361 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.358+0000 7f76cc9e6740 1 -- 172.21.15.99:0/834093512 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5613a0b0eaf0 msgr2=0x5613a0b1f530 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:59:16.361 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.358+0000 7f76cc9e6740 1 --2- 172.21.15.99:0/834093512 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5613a0b0eaf0 0x5613a0b1f530 secure :-1 s=READY pgs=694 cs=0 l=1 rev1=1 crypto rx=0x7f76b8011780 tx=0x7f76b801e000 comp rx=0 tx=0).stop 2024-03-19T01:59:16.361 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.358+0000 7f76cc9e6740 1 -- 172.21.15.99:0/834093512 shutdown_connections 2024-03-19T01:59:16.361 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.358+0000 7f76cc9e6740 1 --2- 172.21.15.99:0/834093512 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5613a0b13c60 0x5613a0b0fad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:16.361 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.358+0000 7f76cc9e6740 1 --2- 172.21.15.99:0/834093512 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5613a0b0eaf0 0x5613a0b1f530 unknown :-1 s=CLOSED pgs=694 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:16.361 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.358+0000 7f76cc9e6740 1 --2- 172.21.15.99:0/834093512 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5613a0b0e100 0x5613a0b0f590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:16.361 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.358+0000 7f76cc9e6740 1 -- 172.21.15.99:0/834093512 >> 172.21.15.99:0/834093512 conn(0x5613a0b09670 msgr2=0x5613a0b17ff0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:59:16.361 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.358+0000 7f76cc9e6740 1 -- 172.21.15.99:0/834093512 shutdown_connections 2024-03-19T01:59:16.361 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.358+0000 7f76cc9e6740 1 -- 172.21.15.99:0/834093512 wait complete. 2024-03-19T01:59:16.361 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.358+0000 7f76cc9e6740 1 -- start start 2024-03-19T01:59:16.361 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.358+0000 7f76cc9e6740 1 Processor -- start 2024-03-19T01:59:16.361 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.359+0000 7f76cc9e6740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5613a0b0e100 0x5613a0b853d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:16.362 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.359+0000 7f76cc9e6740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5613a0b0eaf0 0x5613a0b85910 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:16.362 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.359+0000 7f76cc9e6740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5613a0b13c60 0x5613a0b8c950 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:16.362 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.359+0000 7f76cc9e6740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f76c41a73f0 con 0x5613a0b0e100 2024-03-19T01:59:16.362 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.359+0000 7f76cc9e6740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f76c41a8860 con 0x5613a0b0eaf0 2024-03-19T01:59:16.362 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.359+0000 7f76cc9e6740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f76c41a7ac0 con 0x5613a0b13c60 2024-03-19T01:59:16.362 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.359+0000 7f76c37fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5613a0b0e100 0x5613a0b853d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:16.362 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.359+0000 7f76cc9e6740 10 client.0 _ll_register_callbacks cb 0x7f76b40972f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:59:16.362 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.359+0000 7f76c2ffd640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5613a0b0eaf0 0x5613a0b85910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:16.362 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.359+0000 7f76c37fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5613a0b0e100 0x5613a0b853d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:48504/0 (socket says 172.21.15.99:48504) 2024-03-19T01:59:16.362 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.359+0000 7f76c37fe640 1 -- 172.21.15.99:0/3406573847 learned_addr learned my addr 172.21.15.99:0/3406573847 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:59:16.362 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.359+0000 7f76c3fff640 1 --2- 172.21.15.99:0/3406573847 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5613a0b13c60 0x5613a0b8c950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:16.362 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.359+0000 7f76c2ffd640 1 -- 172.21.15.99:0/3406573847 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5613a0b13c60 msgr2=0x5613a0b8c950 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:59:16.362 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.359+0000 7f76c2ffd640 1 --2- 172.21.15.99:0/3406573847 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5613a0b13c60 0x5613a0b8c950 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:16.363 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.359+0000 7f76c2ffd640 1 -- 172.21.15.99:0/3406573847 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5613a0b0e100 msgr2=0x5613a0b853d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:59:16.363 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.359+0000 7f76c2ffd640 1 --2- 172.21.15.99:0/3406573847 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5613a0b0e100 0x5613a0b853d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:16.363 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.359+0000 7f76c2ffd640 1 -- 172.21.15.99:0/3406573847 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5613a0b8cf40 con 0x5613a0b0eaf0 2024-03-19T01:59:16.363 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.359+0000 7f76c37fe640 1 --2- 172.21.15.99:0/3406573847 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5613a0b0e100 0x5613a0b853d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:59:16.363 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.359+0000 7f76c2ffd640 1 --2- 172.21.15.99:0/3406573847 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5613a0b0eaf0 0x5613a0b85910 secure :-1 s=READY pgs=190 cs=0 l=1 rev1=1 crypto rx=0x7f76b802c040 tx=0x7f76b8005e30 comp rx=0 tx=0).ready entity=mon.2 client_cookie=afe96bf14bb889f1 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:59:16.363 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.359+0000 7f76a2ffd640 10 client.0 ms_handle_connect on v2:172.21.15.112:3300/0 2024-03-19T01:59:16.363 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.359+0000 7f76a2ffd640 1 -- 172.21.15.99:0/3406573847 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f76b8038060 con 0x5613a0b0eaf0 2024-03-19T01:59:16.363 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.360+0000 7f76a2ffd640 1 -- 172.21.15.99:0/3406573847 <== mon.2 v2:172.21.15.112:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f76b80125d0 con 0x5613a0b0eaf0 2024-03-19T01:59:16.363 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.360+0000 7f76cc9e6740 1 -- 172.21.15.99:0/3406573847 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f76b8012c50 con 0x5613a0b0eaf0 2024-03-19T01:59:16.363 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.360+0000 7f76a2ffd640 1 -- 172.21.15.99:0/3406573847 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7f76b8012e00 con 0x5613a0b0eaf0 2024-03-19T01:59:16.363 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.360+0000 7f76cc9e6740 10 client.26576 fetch_fsmap learned FSMap version 37 2024-03-19T01:59:16.363 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.360+0000 7f76cc9e6740 1 -- 172.21.15.99:0/3406573847 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x5613a0b8e260 con 0x5613a0b0eaf0 2024-03-19T01:59:16.363 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.360+0000 7f76a2ffd640 1 -- 172.21.15.99:0/3406573847 <== mon.2 v2:172.21.15.112:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f76b8005520 con 0x5613a0b0eaf0 2024-03-19T01:59:16.363 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.360+0000 7f76a2ffd640 1 -- 172.21.15.99:0/3406573847 <== mon.2 v2:172.21.15.112:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7f76b8004750 con 0x5613a0b0eaf0 2024-03-19T01:59:16.364 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.360+0000 7f76cc9e6740 10 client.26576 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T01:59:16.364 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.360+0000 7f76cc9e6740 10 client.26576 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:59:16.364 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.360+0000 7f76cc9e6740 20 client.26576 populate_metadata read hostname 'smithi099' 2024-03-19T01:59:16.364 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.360+0000 7f76cc9e6740 1 --2- 172.21.15.99:0/3406573847 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5613a0b23a60 0x5613a0b90bd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:16.364 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.360+0000 7f76cc9e6740 4 client.26576 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:59:16.364 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.360+0000 7f76cc9e6740 1 -- 172.21.15.99:0/3406573847 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x7f76b8012c50 con 0x5613a0b23a60 2024-03-19T01:59:16.364 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.360+0000 7f76c37fe640 1 --2- 172.21.15.99:0/3406573847 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5613a0b23a60 0x5613a0b90bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:16.364 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.361+0000 7f76c37fe640 1 --2- 172.21.15.99:0/3406573847 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5613a0b23a60 0x5613a0b90bd0 crc :-1 s=READY pgs=192 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=e836a1439453e857 server_cookie=21f4abe504b08553 in_seq=0 out_seq=0 2024-03-19T01:59:16.364 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:16.361+0000 7f76a2ffd640 10 client.26576 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:59:17.022 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:17.021+0000 7f76a2ffd640 1 -- 172.21.15.99:0/3406573847 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x7f76b8012c50 con 0x5613a0b23a60 2024-03-19T01:59:17.022 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:17.021+0000 7f76a2ffd640 10 client.26576 handle_command_reply: tid=0 2024-03-19T01:59:17.022 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:17.021+0000 7f76cc9e6740 1 client.26576 shutdown 2024-03-19T01:59:17.022 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:17.022+0000 7f76cc9e6740 1 -- 172.21.15.99:0/3406573847 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5613a0b0eaf0 msgr2=0x5613a0b85910 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:59:17.022 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:17.022+0000 7f76cc9e6740 1 --2- 172.21.15.99:0/3406573847 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5613a0b0eaf0 0x5613a0b85910 secure :-1 s=READY pgs=190 cs=0 l=1 rev1=1 crypto rx=0x7f76b802c040 tx=0x7f76b8005e30 comp rx=0 tx=0).stop 2024-03-19T01:59:17.022 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:17.022+0000 7f76cc9e6740 1 -- 172.21.15.99:0/3406573847 shutdown_connections 2024-03-19T01:59:17.022 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:17.022+0000 7f76cc9e6740 1 --2- 172.21.15.99:0/3406573847 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5613a0b23a60 0x5613a0b90bd0 crc :-1 s=READY pgs=192 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:17.023 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:17.022+0000 7f76cc9e6740 1 --2- 172.21.15.99:0/3406573847 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5613a0b0eaf0 0x5613a0b85910 unknown :-1 s=CLOSED pgs=190 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:17.023 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:17.022+0000 7f76a2ffd640 0 client.26576 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:59:17.023 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:17.022+0000 7f76cc9e6740 1 --2- 172.21.15.99:0/3406573847 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5613a0b13c60 0x5613a0b8c950 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:17.023 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:17.022+0000 7f76cc9e6740 1 --2- 172.21.15.99:0/3406573847 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5613a0b0e100 0x5613a0b853d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:17.023 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:17.022+0000 7f76cc9e6740 1 -- 172.21.15.99:0/3406573847 >> 172.21.15.99:0/3406573847 conn(0x5613a0b09670 msgr2=0x5613a0b0f560 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:59:17.023 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:17.022+0000 7f76cc9e6740 1 -- 172.21.15.99:0/3406573847 shutdown_connections 2024-03-19T01:59:17.023 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:17.022+0000 7f76cc9e6740 1 -- 172.21.15.99:0/3406573847 wait complete. 2024-03-19T01:59:17.023 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:17.022+0000 7f76cc9e6740 20 client.26576 trim_cache size 0 max 16384 2024-03-19T01:59:17.023 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:59:17.023 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:59:17.024 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:59:17.024 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:59:17.024 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:17.022+0000 7f76c9df4640 1 -- 172.21.15.99:0/86335691 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f76ac0a28a0 msgr2=0x7f76ac0a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:59:17.024 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:17.022+0000 7f76c9df4640 1 --2- 172.21.15.99:0/86335691 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f76ac0a28a0 0x7f76ac0a4d60 secure :-1 s=READY pgs=354 cs=0 l=1 rev1=1 crypto rx=0x7f76b8015670 tx=0x7f76b8015a70 comp rx=0 tx=0).stop 2024-03-19T01:59:17.025 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:17.022+0000 7f76c9df4640 1 -- 172.21.15.99:0/86335691 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f76c4100780 msgr2=0x7f76c4197a80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:59:17.025 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:17.022+0000 7f76c9df4640 1 --2- 172.21.15.99:0/86335691 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f76c4100780 0x7f76c4197a80 secure :-1 s=READY pgs=691 cs=0 l=1 rev1=1 crypto rx=0x7f76b400e9c0 tx=0x7f76b400ee80 comp rx=0 tx=0).stop 2024-03-19T01:59:17.025 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:17.023+0000 7f76c9df4640 1 -- 172.21.15.99:0/86335691 shutdown_connections 2024-03-19T01:59:17.025 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:17.023+0000 7f76c9df4640 1 --2- 172.21.15.99:0/86335691 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f76ac0a28a0 0x7f76ac0a4d60 unknown :-1 s=CLOSED pgs=354 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:17.025 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:17.023+0000 7f76c9df4640 1 --2- 172.21.15.99:0/86335691 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f76c4102330 0x7f76c419f030 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:17.025 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:17.023+0000 7f76c9df4640 1 --2- 172.21.15.99:0/86335691 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f76c4101970 0x7f76c4197fc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:17.025 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:17.023+0000 7f76c9df4640 1 --2- 172.21.15.99:0/86335691 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f76c4100780 0x7f76c4197a80 unknown :-1 s=CLOSED pgs=691 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:17.025 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:17.023+0000 7f76c9df4640 1 -- 172.21.15.99:0/86335691 >> 172.21.15.99:0/86335691 conn(0x7f76c40fbcf0 msgr2=0x7f76c40fd6d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:59:17.025 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:17.023+0000 7f76c9df4640 1 -- 172.21.15.99:0/86335691 shutdown_connections 2024-03-19T01:59:17.025 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:17.023+0000 7f76c9df4640 1 -- 172.21.15.99:0/86335691 wait complete. 2024-03-19T01:59:17.109 INFO:tasks.workunit.client.0.smithi099.stdout: 10 151306 23.04 MB/sec execute 226 sec latency 404.565 ms 2024-03-19T01:59:17.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:59:17 smithi112 ceph-mon[66819]: pgmap v897: 97 pgs: 97 active+clean; 599 MiB data, 7.1 GiB used, 1.0 TiB / 1.0 TiB avail; 4.5 MiB/s rd, 6.7 MiB/s wr, 567 op/s 2024-03-19T01:59:17.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:59:17 smithi115 ceph-mon[67332]: pgmap v897: 97 pgs: 97 active+clean; 599 MiB data, 7.1 GiB used, 1.0 TiB / 1.0 TiB avail; 4.5 MiB/s rd, 6.7 MiB/s wr, 567 op/s 2024-03-19T01:59:17.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:59:17 smithi099 ceph-mon[62951]: pgmap v897: 97 pgs: 97 active+clean; 599 MiB data, 7.1 GiB used, 1.0 TiB / 1.0 TiB avail; 4.5 MiB/s rd, 6.7 MiB/s wr, 567 op/s 2024-03-19T01:59:18.109 INFO:tasks.workunit.client.0.smithi099.stdout: 10 151751 23.07 MB/sec execute 227 sec latency 198.743 ms 2024-03-19T01:59:19.109 INFO:tasks.workunit.client.0.smithi099.stdout: 10 152142 23.06 MB/sec execute 228 sec latency 106.278 ms 2024-03-19T01:59:19.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:59:19 smithi112 ceph-mon[66819]: pgmap v898: 97 pgs: 97 active+clean; 601 MiB data, 7.1 GiB used, 1.0 TiB / 1.0 TiB avail; 3.3 MiB/s rd, 5.8 MiB/s wr, 456 op/s 2024-03-19T01:59:19.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:59:19 smithi115 ceph-mon[67332]: pgmap v898: 97 pgs: 97 active+clean; 601 MiB data, 7.1 GiB used, 1.0 TiB / 1.0 TiB avail; 3.3 MiB/s rd, 5.8 MiB/s wr, 456 op/s 2024-03-19T01:59:19.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:59:19 smithi099 ceph-mon[62951]: pgmap v898: 97 pgs: 97 active+clean; 601 MiB data, 7.1 GiB used, 1.0 TiB / 1.0 TiB avail; 3.3 MiB/s rd, 5.8 MiB/s wr, 456 op/s 2024-03-19T01:59:20.110 INFO:tasks.workunit.client.0.smithi099.stdout: 10 152622 23.08 MB/sec execute 229 sec latency 102.497 ms 2024-03-19T01:59:21.110 INFO:tasks.workunit.client.0.smithi099.stdout: 10 152926 23.05 MB/sec execute 230 sec latency 219.068 ms 2024-03-19T01:59:21.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:59:21 smithi112 ceph-mon[66819]: pgmap v899: 97 pgs: 97 active+clean; 599 MiB data, 7.1 GiB used, 1.0 TiB / 1.0 TiB avail; 3.7 MiB/s rd, 6.8 MiB/s wr, 573 op/s 2024-03-19T01:59:21.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:59:21 smithi115 ceph-mon[67332]: pgmap v899: 97 pgs: 97 active+clean; 599 MiB data, 7.1 GiB used, 1.0 TiB / 1.0 TiB avail; 3.7 MiB/s rd, 6.8 MiB/s wr, 573 op/s 2024-03-19T01:59:21.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:59:21 smithi099 ceph-mon[62951]: pgmap v899: 97 pgs: 97 active+clean; 599 MiB data, 7.1 GiB used, 1.0 TiB / 1.0 TiB avail; 3.7 MiB/s rd, 6.8 MiB/s wr, 573 op/s 2024-03-19T01:59:22.110 INFO:tasks.workunit.client.0.smithi099.stdout: 10 153344 23.05 MB/sec execute 231 sec latency 206.954 ms 2024-03-19T01:59:23.110 INFO:tasks.workunit.client.0.smithi099.stdout: 10 153799 23.08 MB/sec execute 232 sec latency 64.824 ms 2024-03-19T01:59:23.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:59:23 smithi112 ceph-mon[66819]: pgmap v900: 97 pgs: 97 active+clean; 595 MiB data, 7.1 GiB used, 1.0 TiB / 1.0 TiB avail; 3.7 MiB/s rd, 6.9 MiB/s wr, 576 op/s 2024-03-19T01:59:23.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:59:23 smithi115 ceph-mon[67332]: pgmap v900: 97 pgs: 97 active+clean; 595 MiB data, 7.1 GiB used, 1.0 TiB / 1.0 TiB avail; 3.7 MiB/s rd, 6.9 MiB/s wr, 576 op/s 2024-03-19T01:59:23.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:59:23 smithi099 ceph-mon[62951]: pgmap v900: 97 pgs: 97 active+clean; 595 MiB data, 7.1 GiB used, 1.0 TiB / 1.0 TiB avail; 3.7 MiB/s rd, 6.9 MiB/s wr, 576 op/s 2024-03-19T01:59:24.110 INFO:tasks.workunit.client.0.smithi099.stdout: 10 154105 23.03 MB/sec execute 233 sec latency 109.610 ms 2024-03-19T01:59:25.110 INFO:tasks.workunit.client.0.smithi099.stdout: 10 154534 23.02 MB/sec execute 234 sec latency 65.104 ms 2024-03-19T01:59:25.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:59:25 smithi112 ceph-mon[66819]: pgmap v901: 97 pgs: 97 active+clean; 595 MiB data, 7.1 GiB used, 1.0 TiB / 1.0 TiB avail; 2.9 MiB/s rd, 6.0 MiB/s wr, 500 op/s 2024-03-19T01:59:25.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:59:25 smithi115 ceph-mon[67332]: pgmap v901: 97 pgs: 97 active+clean; 595 MiB data, 7.1 GiB used, 1.0 TiB / 1.0 TiB avail; 2.9 MiB/s rd, 6.0 MiB/s wr, 500 op/s 2024-03-19T01:59:25.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:59:25 smithi099 ceph-mon[62951]: pgmap v901: 97 pgs: 97 active+clean; 595 MiB data, 7.1 GiB used, 1.0 TiB / 1.0 TiB avail; 2.9 MiB/s rd, 6.0 MiB/s wr, 500 op/s 2024-03-19T01:59:26.110 INFO:tasks.workunit.client.0.smithi099.stdout: 10 154931 22.99 MB/sec execute 235 sec latency 159.713 ms 2024-03-19T01:59:27.110 INFO:tasks.workunit.client.0.smithi099.stdout: 10 155256 22.98 MB/sec execute 236 sec latency 351.634 ms 2024-03-19T01:59:27.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:59:27 smithi112 ceph-mon[66819]: pgmap v902: 97 pgs: 97 active+clean; 604 MiB data, 7.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 7.2 MiB/s wr, 582 op/s 2024-03-19T01:59:27.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:59:27 smithi115 ceph-mon[67332]: pgmap v902: 97 pgs: 97 active+clean; 604 MiB data, 7.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 7.2 MiB/s wr, 582 op/s 2024-03-19T01:59:27.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:59:27 smithi099 ceph-mon[62951]: pgmap v902: 97 pgs: 97 active+clean; 604 MiB data, 7.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 7.2 MiB/s wr, 582 op/s 2024-03-19T01:59:28.110 INFO:tasks.workunit.client.0.smithi099.stdout: 10 155829 23.00 MB/sec execute 237 sec latency 123.619 ms 2024-03-19T01:59:29.110 INFO:tasks.workunit.client.0.smithi099.stdout: 10 156319 23.01 MB/sec execute 238 sec latency 61.174 ms 2024-03-19T01:59:29.356 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:59:29 smithi115 ceph-mon[67332]: pgmap v903: 97 pgs: 97 active+clean; 605 MiB data, 7.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.7 MiB/s rd, 6.2 MiB/s wr, 520 op/s 2024-03-19T01:59:29.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:59:29 smithi112 ceph-mon[66819]: pgmap v903: 97 pgs: 97 active+clean; 605 MiB data, 7.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.7 MiB/s rd, 6.2 MiB/s wr, 520 op/s 2024-03-19T01:59:29.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:59:29 smithi099 ceph-mon[62951]: pgmap v903: 97 pgs: 97 active+clean; 605 MiB data, 7.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.7 MiB/s rd, 6.2 MiB/s wr, 520 op/s 2024-03-19T01:59:30.110 INFO:tasks.workunit.client.0.smithi099.stdout: 10 156825 23.06 MB/sec execute 239 sec latency 84.741 ms 2024-03-19T01:59:30.139 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.123+0000 7f99d0c27640 1 -- 172.21.15.99:0/925638671 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f99cc1000d0 msgr2=0x7f99cc102500 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:59:30.139 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.123+0000 7f99d0c27640 1 --2- 172.21.15.99:0/925638671 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f99cc1000d0 0x7f99cc102500 secure :-1 s=READY pgs=695 cs=0 l=1 rev1=1 crypto rx=0x7f99bc009900 tx=0x7f99bc02bc30 comp rx=0 tx=0).stop 2024-03-19T01:59:30.139 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.123+0000 7f99d0c27640 1 -- 172.21.15.99:0/925638671 shutdown_connections 2024-03-19T01:59:30.140 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.123+0000 7f99d0c27640 1 --2- 172.21.15.99:0/925638671 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f99cc102a40 0x7f99cc104eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:30.140 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.123+0000 7f99d0c27640 1 --2- 172.21.15.99:0/925638671 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f99cc1053f0 0x7f99cc105870 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:30.140 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.123+0000 7f99d0c27640 1 --2- 172.21.15.99:0/925638671 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f99cc1000d0 0x7f99cc102500 unknown :-1 s=CLOSED pgs=695 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:30.140 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.123+0000 7f99d0c27640 1 -- 172.21.15.99:0/925638671 >> 172.21.15.99:0/925638671 conn(0x7f99cc0fbcb0 msgr2=0x7f99cc0fe110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:59:30.140 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.123+0000 7f99d0c27640 1 -- 172.21.15.99:0/925638671 shutdown_connections 2024-03-19T01:59:30.140 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.123+0000 7f99d0c27640 1 -- 172.21.15.99:0/925638671 wait complete. 2024-03-19T01:59:30.141 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.123+0000 7f99d0c27640 1 Processor -- start 2024-03-19T01:59:30.141 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.123+0000 7f99d0c27640 1 -- start start 2024-03-19T01:59:30.141 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.124+0000 7f99d0c27640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f99cc1000d0 0x7f99cc19e0c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:30.141 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.124+0000 7f99d0c27640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f99cc102a40 0x7f99cc19e600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:30.141 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.124+0000 7f99d0c27640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f99cc1053f0 0x7f99cc1a5670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:30.141 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.124+0000 7f99d0c27640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f99cc113ad0 con 0x7f99cc102a40 2024-03-19T01:59:30.142 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.124+0000 7f99d0c27640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f99cc113950 con 0x7f99cc1000d0 2024-03-19T01:59:30.142 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.124+0000 7f99d0c27640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f99cc113c50 con 0x7f99cc1053f0 2024-03-19T01:59:30.142 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.124+0000 7f99c9d9b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f99cc102a40 0x7f99cc19e600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:30.142 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.124+0000 7f99c9d9b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f99cc102a40 0x7f99cc19e600 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:54820/0 (socket says 172.21.15.99:54820) 2024-03-19T01:59:30.142 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.124+0000 7f99c9d9b640 1 -- 172.21.15.99:0/3561951347 learned_addr learned my addr 172.21.15.99:0/3561951347 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:59:30.142 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.124+0000 7f99ca59c640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f99cc1000d0 0x7f99cc19e0c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:30.143 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.124+0000 7f99cad9d640 1 --2- 172.21.15.99:0/3561951347 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f99cc1053f0 0x7f99cc1a5670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:30.143 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.124+0000 7f99ca59c640 1 -- 172.21.15.99:0/3561951347 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f99cc1053f0 msgr2=0x7f99cc1a5670 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:59:30.143 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.124+0000 7f99ca59c640 1 --2- 172.21.15.99:0/3561951347 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f99cc1053f0 0x7f99cc1a5670 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:30.143 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.124+0000 7f99ca59c640 1 -- 172.21.15.99:0/3561951347 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f99cc102a40 msgr2=0x7f99cc19e600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:59:30.143 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.124+0000 7f99ca59c640 1 --2- 172.21.15.99:0/3561951347 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f99cc102a40 0x7f99cc19e600 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:30.143 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.124+0000 7f99ca59c640 1 -- 172.21.15.99:0/3561951347 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f99cc1a5ce0 con 0x7f99cc1000d0 2024-03-19T01:59:30.143 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.124+0000 7f99ca59c640 1 --2- 172.21.15.99:0/3561951347 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f99cc1000d0 0x7f99cc19e0c0 secure :-1 s=READY pgs=191 cs=0 l=1 rev1=1 crypto rx=0x7f99bc009230 tx=0x7f99bc02c650 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:59:30.143 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.125+0000 7f99bb7fe640 1 -- 172.21.15.99:0/3561951347 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f99bc041070 con 0x7f99cc1000d0 2024-03-19T01:59:30.143 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.125+0000 7f99bb7fe640 1 -- 172.21.15.99:0/3561951347 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f99bc004450 con 0x7f99cc1000d0 2024-03-19T01:59:30.143 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.125+0000 7f99d0c27640 1 -- 172.21.15.99:0/3561951347 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f99cc1a5f70 con 0x7f99cc1000d0 2024-03-19T01:59:30.143 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.125+0000 7f99bb7fe640 1 -- 172.21.15.99:0/3561951347 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f99bc003df0 con 0x7f99cc1000d0 2024-03-19T01:59:30.143 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.125+0000 7f99d0c27640 1 -- 172.21.15.99:0/3561951347 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f99cc1a6380 con 0x7f99cc1000d0 2024-03-19T01:59:30.143 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.127+0000 7f99bb7fe640 1 -- 172.21.15.99:0/3561951347 <== mon.2 v2:172.21.15.112:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f99bc044070 con 0x7f99cc1000d0 2024-03-19T01:59:30.144 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.127+0000 7f99d380e740 1 Processor -- start 2024-03-19T01:59:30.144 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.127+0000 7f99d380e740 1 -- start start 2024-03-19T01:59:30.144 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.127+0000 7f99d380e740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55765c1a2100 0x55765c1a2520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:30.144 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.127+0000 7f99d380e740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55765c1a2af0 0x55765c1a75f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:30.144 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.127+0000 7f99d380e740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55765c1a7c60 0x55765c1aa0f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:30.144 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.127+0000 7f99d380e740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55765c106be0 con 0x55765c1a2100 2024-03-19T01:59:30.144 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.127+0000 7f99d380e740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55765c0af960 con 0x55765c1a7c60 2024-03-19T01:59:30.144 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.127+0000 7f99d380e740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55765c102c80 con 0x55765c1a2af0 2024-03-19T01:59:30.144 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.128+0000 7f99cad9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55765c1a2100 0x55765c1a2520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:30.144 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.128+0000 7f99cad9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55765c1a2100 0x55765c1a2520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:54828/0 (socket says 172.21.15.99:54828) 2024-03-19T01:59:30.144 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.128+0000 7f99cad9d640 1 -- 172.21.15.99:0/2892148172 learned_addr learned my addr 172.21.15.99:0/2892148172 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:59:30.144 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.128+0000 7f99ca59c640 1 --2- 172.21.15.99:0/2892148172 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55765c1a2af0 0x55765c1a75f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:30.144 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.128+0000 7f99c9d9b640 1 --2- 172.21.15.99:0/2892148172 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55765c1a7c60 0x55765c1aa0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:30.144 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.128+0000 7f99bb7fe640 1 --2- 172.21.15.99:0/3561951347 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f99b40a28a0 0x7f99b40a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:30.144 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.128+0000 7f99cad9d640 1 -- 172.21.15.99:0/2892148172 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55765c1a2af0 msgr2=0x55765c1a75f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:59:30.145 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.128+0000 7f99cad9d640 1 --2- 172.21.15.99:0/2892148172 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55765c1a2af0 0x55765c1a75f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:30.145 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.128+0000 7f99cad9d640 1 -- 172.21.15.99:0/2892148172 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55765c1a7c60 msgr2=0x55765c1aa0f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:59:30.145 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.128+0000 7f99cad9d640 1 --2- 172.21.15.99:0/2892148172 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55765c1a7c60 0x55765c1aa0f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:30.145 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.128+0000 7f99cad9d640 1 -- 172.21.15.99:0/2892148172 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55765c0a6d20 con 0x55765c1a2100 2024-03-19T01:59:30.145 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.128+0000 7f99cad9d640 1 --2- 172.21.15.99:0/3561951347 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f99b40a28a0 0x7f99b40a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:30.145 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.128+0000 7f99bb7fe640 1 -- 172.21.15.99:0/3561951347 <== mon.2 v2:172.21.15.112:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f99bc0f04d0 con 0x7f99cc1000d0 2024-03-19T01:59:30.145 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.128+0000 7f99cad9d640 1 --2- 172.21.15.99:0/2892148172 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55765c1a2100 0x55765c1a2520 secure :-1 s=READY pgs=696 cs=0 l=1 rev1=1 crypto rx=0x7f99c4011590 tx=0x7f99c4011a50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=779eb7754bc3eb69 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:59:30.145 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.128+0000 7f99cad9d640 1 --2- 172.21.15.99:0/3561951347 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f99b40a28a0 0x7f99b40a4d60 secure :-1 s=READY pgs=355 cs=0 l=1 rev1=1 crypto rx=0x7f99c4022000 tx=0x7f99c4023000 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:59:30.145 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.128+0000 7f99b8ff9640 1 -- 172.21.15.99:0/2892148172 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f99c4017050 con 0x55765c1a2100 2024-03-19T01:59:30.145 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.128+0000 7f99b8ff9640 1 -- 172.21.15.99:0/2892148172 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f99c400fde0 con 0x55765c1a2100 2024-03-19T01:59:30.145 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.128+0000 7f99b8ff9640 1 -- 172.21.15.99:0/2892148172 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f99c401bc40 con 0x55765c1a2100 2024-03-19T01:59:30.145 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.129+0000 7f99d380e740 1 -- 172.21.15.99:0/2892148172 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55765c1a2100 msgr2=0x55765c1a2520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:59:30.146 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.129+0000 7f99d380e740 1 --2- 172.21.15.99:0/2892148172 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55765c1a2100 0x55765c1a2520 secure :-1 s=READY pgs=696 cs=0 l=1 rev1=1 crypto rx=0x7f99c4011590 tx=0x7f99c4011a50 comp rx=0 tx=0).stop 2024-03-19T01:59:30.146 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.129+0000 7f99d380e740 1 -- 172.21.15.99:0/2892148172 shutdown_connections 2024-03-19T01:59:30.146 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.129+0000 7f99d380e740 1 --2- 172.21.15.99:0/2892148172 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55765c1a7c60 0x55765c1aa0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:30.146 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.129+0000 7f99d380e740 1 --2- 172.21.15.99:0/2892148172 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55765c1a2af0 0x55765c1a75f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:30.146 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.129+0000 7f99d380e740 1 --2- 172.21.15.99:0/2892148172 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55765c1a2100 0x55765c1a2520 unknown :-1 s=CLOSED pgs=696 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:30.146 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.129+0000 7f99d380e740 1 -- 172.21.15.99:0/2892148172 >> 172.21.15.99:0/2892148172 conn(0x55765c19d670 msgr2=0x55765c19da70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:59:30.147 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.129+0000 7f99d380e740 1 -- 172.21.15.99:0/2892148172 shutdown_connections 2024-03-19T01:59:30.147 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.129+0000 7f99d380e740 1 -- 172.21.15.99:0/2892148172 wait complete. 2024-03-19T01:59:30.147 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.130+0000 7f99d380e740 1 -- start start 2024-03-19T01:59:30.147 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.130+0000 7f99d380e740 1 Processor -- start 2024-03-19T01:59:30.147 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.130+0000 7f99d380e740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55765c1a2100 0x55765c1ac160 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:30.147 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.130+0000 7f99d380e740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55765c1a2af0 0x55765c1ac6a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:30.147 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.130+0000 7f99d380e740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55765c1a7c60 0x55765c1b36e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:30.147 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.130+0000 7f99d380e740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55765c1aac10 con 0x55765c1a2af0 2024-03-19T01:59:30.147 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.130+0000 7f99d380e740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55765c0c2db0 con 0x55765c1a2100 2024-03-19T01:59:30.147 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.130+0000 7f99d380e740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55765c1aad90 con 0x55765c1a7c60 2024-03-19T01:59:30.147 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.130+0000 7f99d380e740 10 client.0 _ll_register_callbacks cb 0x55765c0c7e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:59:30.147 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.130+0000 7f99ca59c640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55765c1a2100 0x55765c1ac160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:30.148 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.130+0000 7f99c9d9b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55765c1a2af0 0x55765c1ac6a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:30.148 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.130+0000 7f99ca59c640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55765c1a2100 0x55765c1ac160 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:34362/0 (socket says 172.21.15.99:34362) 2024-03-19T01:59:30.148 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.130+0000 7f99ca59c640 1 -- 172.21.15.99:0/4128871497 learned_addr learned my addr 172.21.15.99:0/4128871497 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:59:30.148 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.130+0000 7f99cad9d640 1 --2- 172.21.15.99:0/4128871497 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55765c1a7c60 0x55765c1b36e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:30.148 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.130+0000 7f99ca59c640 1 -- 172.21.15.99:0/4128871497 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55765c1a7c60 msgr2=0x55765c1b36e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:59:30.148 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.130+0000 7f99ca59c640 1 --2- 172.21.15.99:0/4128871497 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55765c1a7c60 0x55765c1b36e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:30.148 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.130+0000 7f99ca59c640 1 -- 172.21.15.99:0/4128871497 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55765c1a2af0 msgr2=0x55765c1ac6a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:59:30.148 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.130+0000 7f99ca59c640 1 --2- 172.21.15.99:0/4128871497 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55765c1a2af0 0x55765c1ac6a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:30.148 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.130+0000 7f99ca59c640 1 -- 172.21.15.99:0/4128871497 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55765c0c1c60 con 0x55765c1a2100 2024-03-19T01:59:30.148 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.131+0000 7f99ca59c640 1 --2- 172.21.15.99:0/4128871497 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55765c1a2100 0x55765c1ac160 secure :-1 s=READY pgs=192 cs=0 l=1 rev1=1 crypto rx=0x7f99bc006010 tx=0x7f99bc005e50 comp rx=0 tx=0).ready entity=mon.2 client_cookie=1a62017647146c78 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:59:30.148 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.131+0000 7f99b1ffb640 10 client.0 ms_handle_connect on v2:172.21.15.112:3300/0 2024-03-19T01:59:30.148 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.131+0000 7f99b1ffb640 1 -- 172.21.15.99:0/4128871497 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f99bc04b040 con 0x55765c1a2100 2024-03-19T01:59:30.148 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.131+0000 7f99b1ffb640 1 -- 172.21.15.99:0/4128871497 <== mon.2 v2:172.21.15.112:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f99bc0f00d0 con 0x55765c1a2100 2024-03-19T01:59:30.148 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.131+0000 7f99d380e740 1 -- 172.21.15.99:0/4128871497 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f99c400fde0 con 0x55765c1a2100 2024-03-19T01:59:30.149 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.131+0000 7f99b1ffb640 1 -- 172.21.15.99:0/4128871497 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7f99bc0bccb0 con 0x55765c1a2100 2024-03-19T01:59:30.149 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.131+0000 7f99d380e740 10 client.26588 fetch_fsmap learned FSMap version 37 2024-03-19T01:59:30.149 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.131+0000 7f99d380e740 1 -- 172.21.15.99:0/4128871497 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x55765be1b210 con 0x55765c1a2100 2024-03-19T01:59:30.149 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.131+0000 7f99b1ffb640 1 -- 172.21.15.99:0/4128871497 <== mon.2 v2:172.21.15.112:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f99c400fde0 con 0x55765c1a2100 2024-03-19T01:59:30.149 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.131+0000 7f99b1ffb640 1 -- 172.21.15.99:0/4128871497 <== mon.2 v2:172.21.15.112:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7f99bc0d5070 con 0x55765c1a2100 2024-03-19T01:59:30.149 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.131+0000 7f99d380e740 10 client.26588 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T01:59:30.149 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.131+0000 7f99d380e740 10 client.26588 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:59:30.149 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.131+0000 7f99d380e740 20 client.26588 populate_metadata read hostname 'smithi099' 2024-03-19T01:59:30.149 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.131+0000 7f99d380e740 1 --2- 172.21.15.99:0/4128871497 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55765c1b7f60 0x55765c1ba390 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:30.149 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.131+0000 7f99d380e740 4 client.26588 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:59:30.149 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.131+0000 7f99d380e740 1 -- 172.21.15.99:0/4128871497 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55765c114120 con 0x55765c1b7f60 2024-03-19T01:59:30.149 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.132+0000 7f99c9d9b640 1 --2- 172.21.15.99:0/4128871497 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55765c1b7f60 0x55765c1ba390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:30.149 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.132+0000 7f99b1ffb640 10 client.26588 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:59:30.149 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.132+0000 7f99c9d9b640 1 --2- 172.21.15.99:0/4128871497 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55765c1b7f60 0x55765c1ba390 crc :-1 s=READY pgs=193 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=70ac3f4bb62faa24 server_cookie=5ee18f8f59ab55e7 in_seq=0 out_seq=0 2024-03-19T01:59:30.149 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.134+0000 7f99b1ffb640 1 -- 172.21.15.99:0/4128871497 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x55765c114120 con 0x55765c1b7f60 2024-03-19T01:59:30.150 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.134+0000 7f99b1ffb640 10 client.26588 handle_command_reply: tid=0 2024-03-19T01:59:30.150 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.134+0000 7f99d380e740 1 client.26588 shutdown 2024-03-19T01:59:30.150 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.134+0000 7f99d380e740 1 -- 172.21.15.99:0/4128871497 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55765c1a2100 msgr2=0x55765c1ac160 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:59:30.150 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.134+0000 7f99d380e740 1 --2- 172.21.15.99:0/4128871497 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55765c1a2100 0x55765c1ac160 secure :-1 s=READY pgs=192 cs=0 l=1 rev1=1 crypto rx=0x7f99bc006010 tx=0x7f99bc005e50 comp rx=0 tx=0).stop 2024-03-19T01:59:30.150 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.134+0000 7f99d380e740 1 -- 172.21.15.99:0/4128871497 shutdown_connections 2024-03-19T01:59:30.150 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.134+0000 7f99d380e740 1 --2- 172.21.15.99:0/4128871497 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55765c1b7f60 0x55765c1ba390 crc :-1 s=READY pgs=193 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:30.150 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.134+0000 7f99d380e740 1 --2- 172.21.15.99:0/4128871497 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55765c1a7c60 0x55765c1b36e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:30.150 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.134+0000 7f99d380e740 1 --2- 172.21.15.99:0/4128871497 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55765c1a2af0 0x55765c1ac6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:30.150 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.134+0000 7f99d380e740 1 --2- 172.21.15.99:0/4128871497 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55765c1a2100 0x55765c1ac160 unknown :-1 s=CLOSED pgs=192 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:30.150 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.134+0000 7f99b1ffb640 0 client.26588 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:59:30.150 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.134+0000 7f99d380e740 1 -- 172.21.15.99:0/4128871497 >> 172.21.15.99:0/4128871497 conn(0x55765c19d670 msgr2=0x55765c19e8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:59:30.150 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.134+0000 7f99d380e740 1 -- 172.21.15.99:0/4128871497 shutdown_connections 2024-03-19T01:59:30.150 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.134+0000 7f99d380e740 1 -- 172.21.15.99:0/4128871497 wait complete. 2024-03-19T01:59:30.150 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.134+0000 7f99d380e740 20 client.26588 trim_cache size 0 max 16384 2024-03-19T01:59:30.155 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.150+0000 7f99d380e740 1 Processor -- start 2024-03-19T01:59:30.155 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.150+0000 7f99d380e740 1 -- start start 2024-03-19T01:59:30.155 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.150+0000 7f99d380e740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55765c1a2100 0x55765c1ac480 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:30.155 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.150+0000 7f99d380e740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55765c1a2af0 0x55765c1b3670 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:30.155 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.150+0000 7f99d380e740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55765c1a7c60 0x55765c1a34f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:30.155 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.150+0000 7f99d380e740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f99cc1b1c80 con 0x55765c1a2100 2024-03-19T01:59:30.155 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.150+0000 7f99d380e740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f99cc1af550 con 0x55765c1a7c60 2024-03-19T01:59:30.156 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.150+0000 7f99d380e740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f99cc1ad3a0 con 0x55765c1a2af0 2024-03-19T01:59:30.156 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.150+0000 7f99ca59c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55765c1a2100 0x55765c1ac480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:30.156 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.150+0000 7f99c9d9b640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55765c1a2af0 0x55765c1b3670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:30.156 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.150+0000 7f99ca59c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55765c1a2100 0x55765c1ac480 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:54846/0 (socket says 172.21.15.99:54846) 2024-03-19T01:59:30.156 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.150+0000 7f99ca59c640 1 -- 172.21.15.99:0/984609089 learned_addr learned my addr 172.21.15.99:0/984609089 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:59:30.156 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.150+0000 7f99cad9d640 1 --2- 172.21.15.99:0/984609089 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55765c1a7c60 0x55765c1a34f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:30.156 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.151+0000 7f99ca59c640 1 -- 172.21.15.99:0/984609089 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55765c1a2af0 msgr2=0x55765c1b3670 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:59:30.156 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.151+0000 7f99ca59c640 1 --2- 172.21.15.99:0/984609089 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55765c1a2af0 0x55765c1b3670 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:30.156 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.151+0000 7f99ca59c640 1 -- 172.21.15.99:0/984609089 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55765c1a7c60 msgr2=0x55765c1a34f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:59:30.156 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.151+0000 7f99ca59c640 1 --2- 172.21.15.99:0/984609089 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55765c1a7c60 0x55765c1a34f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:30.156 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.151+0000 7f99ca59c640 1 -- 172.21.15.99:0/984609089 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55765bee7620 con 0x55765c1a2100 2024-03-19T01:59:30.156 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.151+0000 7f99ca59c640 1 --2- 172.21.15.99:0/984609089 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55765c1a2100 0x55765c1ac480 secure :-1 s=READY pgs=697 cs=0 l=1 rev1=1 crypto rx=0x7f99bc0ee300 tx=0x7f99bc02bc30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=51653899d33a849b server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:59:30.156 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.151+0000 7f99b3fff640 1 -- 172.21.15.99:0/984609089 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f99bc04b070 con 0x55765c1a2100 2024-03-19T01:59:30.156 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.151+0000 7f99b3fff640 1 -- 172.21.15.99:0/984609089 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f99bc004c80 con 0x55765c1a2100 2024-03-19T01:59:30.156 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.151+0000 7f99b3fff640 1 -- 172.21.15.99:0/984609089 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f99bc0bc840 con 0x55765c1a2100 2024-03-19T01:59:30.157 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.152+0000 7f99d380e740 1 -- 172.21.15.99:0/984609089 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55765c1a2100 msgr2=0x55765c1ac480 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:59:30.157 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.152+0000 7f99d380e740 1 --2- 172.21.15.99:0/984609089 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55765c1a2100 0x55765c1ac480 secure :-1 s=READY pgs=697 cs=0 l=1 rev1=1 crypto rx=0x7f99bc0ee300 tx=0x7f99bc02bc30 comp rx=0 tx=0).stop 2024-03-19T01:59:30.157 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.152+0000 7f99d380e740 1 -- 172.21.15.99:0/984609089 shutdown_connections 2024-03-19T01:59:30.157 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.152+0000 7f99d380e740 1 --2- 172.21.15.99:0/984609089 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55765c1a7c60 0x55765c1a34f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:30.157 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.152+0000 7f99d380e740 1 --2- 172.21.15.99:0/984609089 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55765c1a2af0 0x55765c1b3670 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:30.157 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.152+0000 7f99d380e740 1 --2- 172.21.15.99:0/984609089 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55765c1a2100 0x55765c1ac480 unknown :-1 s=CLOSED pgs=697 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:30.157 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.152+0000 7f99d380e740 1 -- 172.21.15.99:0/984609089 >> 172.21.15.99:0/984609089 conn(0x55765c19d670 msgr2=0x55765c1abd70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:59:30.157 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.152+0000 7f99d380e740 1 -- 172.21.15.99:0/984609089 shutdown_connections 2024-03-19T01:59:30.157 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.152+0000 7f99d380e740 1 -- 172.21.15.99:0/984609089 wait complete. 2024-03-19T01:59:30.157 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.152+0000 7f99d380e740 1 -- start start 2024-03-19T01:59:30.157 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.152+0000 7f99d380e740 1 Processor -- start 2024-03-19T01:59:30.157 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.153+0000 7f99d380e740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55765c1a2100 0x55765c1b1140 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:30.157 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.153+0000 7f99d380e740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55765c1a2af0 0x55765c1b1680 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:30.157 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.153+0000 7f99d380e740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55765c1a7c60 0x55765c1b2690 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:30.157 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.153+0000 7f99d380e740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f99cc1ada10 con 0x55765c1a2100 2024-03-19T01:59:30.158 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.153+0000 7f99d380e740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f99cc1aee80 con 0x55765c1a7c60 2024-03-19T01:59:30.158 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.153+0000 7f99d380e740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f99cc1ae0e0 con 0x55765c1a2af0 2024-03-19T01:59:30.158 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.153+0000 7f99d380e740 10 client.0 _ll_register_callbacks cb 0x7f99bc0f3ca0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:59:30.158 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.153+0000 7f99ca59c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55765c1a2100 0x55765c1b1140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:30.158 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.153+0000 7f99ca59c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55765c1a2100 0x55765c1b1140 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:54856/0 (socket says 172.21.15.99:54856) 2024-03-19T01:59:30.158 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.153+0000 7f99ca59c640 1 -- 172.21.15.99:0/1317709855 learned_addr learned my addr 172.21.15.99:0/1317709855 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:59:30.158 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.153+0000 7f99cad9d640 1 --2- 172.21.15.99:0/1317709855 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55765c1a7c60 0x55765c1b2690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:30.158 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.153+0000 7f99c9d9b640 1 --2- 172.21.15.99:0/1317709855 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55765c1a2af0 0x55765c1b1680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:30.158 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.153+0000 7f99ca59c640 1 -- 172.21.15.99:0/1317709855 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55765c1a2af0 msgr2=0x55765c1b1680 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:59:30.158 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.153+0000 7f99ca59c640 1 --2- 172.21.15.99:0/1317709855 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55765c1a2af0 0x55765c1b1680 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:30.158 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.153+0000 7f99ca59c640 1 -- 172.21.15.99:0/1317709855 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55765c1a7c60 msgr2=0x55765c1b2690 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:59:30.158 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.153+0000 7f99ca59c640 1 --2- 172.21.15.99:0/1317709855 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55765c1a7c60 0x55765c1b2690 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:30.158 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.153+0000 7f99ca59c640 1 -- 172.21.15.99:0/1317709855 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55765c1b2c80 con 0x55765c1a2100 2024-03-19T01:59:30.159 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.153+0000 7f99cad9d640 1 --2- 172.21.15.99:0/1317709855 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55765c1a7c60 0x55765c1b2690 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:59:30.159 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.153+0000 7f99ca59c640 1 --2- 172.21.15.99:0/1317709855 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55765c1a2100 0x55765c1b1140 secure :-1 s=READY pgs=698 cs=0 l=1 rev1=1 crypto rx=0x7f99bc0f3d50 tx=0x7f99bc000bc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=bec3921ab84eddac server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:59:30.159 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.153+0000 7f99b1ffb640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:59:30.159 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.153+0000 7f99b1ffb640 1 -- 172.21.15.99:0/1317709855 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f99bc04b050 con 0x55765c1a2100 2024-03-19T01:59:30.159 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.153+0000 7f99b1ffb640 1 -- 172.21.15.99:0/1317709855 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f99bc0bbfb0 con 0x55765c1a2100 2024-03-19T01:59:30.159 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.153+0000 7f99d380e740 1 -- 172.21.15.99:0/1317709855 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f99bc004c80 con 0x55765c1a2100 2024-03-19T01:59:30.159 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.153+0000 7f99b1ffb640 1 -- 172.21.15.99:0/1317709855 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7f99bc0bca20 con 0x55765c1a2100 2024-03-19T01:59:30.159 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.154+0000 7f99d380e740 10 client.16818 fetch_fsmap learned FSMap version 37 2024-03-19T01:59:30.159 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.154+0000 7f99d380e740 1 -- 172.21.15.99:0/1317709855 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x55765c1b2c80 con 0x55765c1a2100 2024-03-19T01:59:30.159 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.154+0000 7f99b1ffb640 1 -- 172.21.15.99:0/1317709855 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f99bc004c80 con 0x55765c1a2100 2024-03-19T01:59:30.159 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.154+0000 7f99b1ffb640 1 -- 172.21.15.99:0/1317709855 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7f99bc0d5070 con 0x55765c1a2100 2024-03-19T01:59:30.159 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.154+0000 7f99d380e740 10 client.16818 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T01:59:30.159 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.154+0000 7f99d380e740 10 client.16818 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:59:30.159 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.154+0000 7f99d380e740 20 client.16818 populate_metadata read hostname 'smithi099' 2024-03-19T01:59:30.159 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.154+0000 7f99d380e740 1 --2- 172.21.15.99:0/1317709855 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55765c21ea90 0x55765c220eb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:30.160 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.154+0000 7f99d380e740 4 client.16818 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:59:30.160 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.154+0000 7f99d380e740 1 -- 172.21.15.99:0/1317709855 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55765c10e470 con 0x55765c21ea90 2024-03-19T01:59:30.160 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.154+0000 7f99c9d9b640 1 --2- 172.21.15.99:0/1317709855 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55765c21ea90 0x55765c220eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:30.160 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.155+0000 7f99b1ffb640 10 client.16818 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:59:30.160 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.155+0000 7f99c9d9b640 1 --2- 172.21.15.99:0/1317709855 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55765c21ea90 0x55765c220eb0 crc :-1 s=READY pgs=194 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=75c5ba75b9940929 server_cookie=725c469ac6cf775b in_seq=0 out_seq=0 2024-03-19T01:59:30.662 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.661+0000 7f99b1ffb640 1 -- 172.21.15.99:0/1317709855 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x55765c10e470 con 0x55765c21ea90 2024-03-19T01:59:30.662 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.661+0000 7f99b1ffb640 10 client.16818 handle_command_reply: tid=0 2024-03-19T01:59:30.662 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.661+0000 7f99d380e740 1 client.16818 shutdown 2024-03-19T01:59:30.662 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.662+0000 7f99d380e740 1 -- 172.21.15.99:0/1317709855 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55765c1a2100 msgr2=0x55765c1b1140 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:59:30.662 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.662+0000 7f99d380e740 1 --2- 172.21.15.99:0/1317709855 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55765c1a2100 0x55765c1b1140 secure :-1 s=READY pgs=698 cs=0 l=1 rev1=1 crypto rx=0x7f99bc0f3d50 tx=0x7f99bc000bc0 comp rx=0 tx=0).stop 2024-03-19T01:59:30.662 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.662+0000 7f99d380e740 1 -- 172.21.15.99:0/1317709855 shutdown_connections 2024-03-19T01:59:30.663 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.662+0000 7f99d380e740 1 --2- 172.21.15.99:0/1317709855 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55765c21ea90 0x55765c220eb0 crc :-1 s=READY pgs=194 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:30.663 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.662+0000 7f99b1ffb640 0 client.16818 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:59:30.663 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.662+0000 7f99d380e740 1 --2- 172.21.15.99:0/1317709855 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55765c1a7c60 0x55765c1b2690 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:30.663 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.662+0000 7f99d380e740 1 --2- 172.21.15.99:0/1317709855 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55765c1a2af0 0x55765c1b1680 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:30.663 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.662+0000 7f99d380e740 1 --2- 172.21.15.99:0/1317709855 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55765c1a2100 0x55765c1b1140 unknown :-1 s=CLOSED pgs=698 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:30.663 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.662+0000 7f99d380e740 1 -- 172.21.15.99:0/1317709855 >> 172.21.15.99:0/1317709855 conn(0x55765c19d670 msgr2=0x55765c1a08d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:59:30.663 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.662+0000 7f99d380e740 1 -- 172.21.15.99:0/1317709855 shutdown_connections 2024-03-19T01:59:30.663 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.662+0000 7f99d380e740 1 -- 172.21.15.99:0/1317709855 wait complete. 2024-03-19T01:59:30.663 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.662+0000 7f99d380e740 20 client.16818 trim_cache size 0 max 16384 2024-03-19T01:59:30.664 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:59:30.664 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:59:30.664 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:59:30.664 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:59:30.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.662+0000 7f99d0c27640 1 -- 172.21.15.99:0/3561951347 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f99b40a28a0 msgr2=0x7f99b40a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:59:30.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.662+0000 7f99d0c27640 1 --2- 172.21.15.99:0/3561951347 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f99b40a28a0 0x7f99b40a4d60 secure :-1 s=READY pgs=355 cs=0 l=1 rev1=1 crypto rx=0x7f99c4022000 tx=0x7f99c4023000 comp rx=0 tx=0).stop 2024-03-19T01:59:30.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.662+0000 7f99d0c27640 1 -- 172.21.15.99:0/3561951347 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f99cc1000d0 msgr2=0x7f99cc19e0c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:59:30.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.662+0000 7f99d0c27640 1 --2- 172.21.15.99:0/3561951347 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f99cc1000d0 0x7f99cc19e0c0 secure :-1 s=READY pgs=191 cs=0 l=1 rev1=1 crypto rx=0x7f99bc009230 tx=0x7f99bc02c650 comp rx=0 tx=0).stop 2024-03-19T01:59:30.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.662+0000 7f99d0c27640 1 -- 172.21.15.99:0/3561951347 shutdown_connections 2024-03-19T01:59:30.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.662+0000 7f99d0c27640 1 --2- 172.21.15.99:0/3561951347 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f99cc1053f0 0x7f99cc1a5670 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:30.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.662+0000 7f99d0c27640 1 --2- 172.21.15.99:0/3561951347 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f99cc102a40 0x7f99cc19e600 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:30.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.662+0000 7f99d0c27640 1 --2- 172.21.15.99:0/3561951347 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f99b40a28a0 0x7f99b40a4d60 unknown :-1 s=CLOSED pgs=355 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:30.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.662+0000 7f99d0c27640 1 --2- 172.21.15.99:0/3561951347 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f99cc1000d0 0x7f99cc19e0c0 unknown :-1 s=CLOSED pgs=191 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:30.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.662+0000 7f99d0c27640 1 -- 172.21.15.99:0/3561951347 >> 172.21.15.99:0/3561951347 conn(0x7f99cc0fbcb0 msgr2=0x7f99cc103780 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:59:30.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.663+0000 7f99d0c27640 1 -- 172.21.15.99:0/3561951347 shutdown_connections 2024-03-19T01:59:30.665 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:30.663+0000 7f99d0c27640 1 -- 172.21.15.99:0/3561951347 wait complete. 2024-03-19T01:59:31.110 INFO:tasks.workunit.client.0.smithi099.stdout: 10 157140 23.03 MB/sec execute 240 sec latency 325.963 ms 2024-03-19T01:59:31.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:59:31 smithi112 ceph-mon[66819]: pgmap v904: 97 pgs: 97 active+clean; 610 MiB data, 7.2 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 6.7 MiB/s wr, 576 op/s 2024-03-19T01:59:31.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:59:31 smithi115 ceph-mon[67332]: pgmap v904: 97 pgs: 97 active+clean; 610 MiB data, 7.2 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 6.7 MiB/s wr, 576 op/s 2024-03-19T01:59:31.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:59:31 smithi099 ceph-mon[62951]: pgmap v904: 97 pgs: 97 active+clean; 610 MiB data, 7.2 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 6.7 MiB/s wr, 576 op/s 2024-03-19T01:59:32.110 INFO:tasks.workunit.client.0.smithi099.stdout: 10 157435 23.01 MB/sec execute 241 sec latency 244.789 ms 2024-03-19T01:59:33.110 INFO:tasks.workunit.client.0.smithi099.stdout: 10 157787 23.01 MB/sec execute 242 sec latency 96.671 ms 2024-03-19T01:59:33.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:59:33 smithi112 ceph-mon[66819]: pgmap v905: 97 pgs: 97 active+clean; 604 MiB data, 7.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.7 MiB/s rd, 6.5 MiB/s wr, 521 op/s 2024-03-19T01:59:33.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:59:33 smithi115 ceph-mon[67332]: pgmap v905: 97 pgs: 97 active+clean; 604 MiB data, 7.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.7 MiB/s rd, 6.5 MiB/s wr, 521 op/s 2024-03-19T01:59:33.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:59:33 smithi099 ceph-mon[62951]: pgmap v905: 97 pgs: 97 active+clean; 604 MiB data, 7.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.7 MiB/s rd, 6.5 MiB/s wr, 521 op/s 2024-03-19T01:59:34.110 INFO:tasks.workunit.client.0.smithi099.stdout: 10 158168 23.00 MB/sec execute 243 sec latency 93.289 ms 2024-03-19T01:59:34.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:59:34 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:59:34.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:59:34 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:59:34.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:59:34 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T01:59:35.110 INFO:tasks.workunit.client.0.smithi099.stdout: 10 158556 22.97 MB/sec execute 244 sec latency 144.837 ms 2024-03-19T01:59:35.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:59:35 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:59:35.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:59:35 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:59:35.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:59:35 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:59:35.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:59:35 smithi112 ceph-mon[66819]: pgmap v906: 97 pgs: 97 active+clean; 602 MiB data, 7.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.2 MiB/s rd, 5.7 MiB/s wr, 496 op/s 2024-03-19T01:59:35.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:59:35 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:59:35.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:59:35 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:59:35.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:59:35 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:59:35.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:59:35 smithi115 ceph-mon[67332]: pgmap v906: 97 pgs: 97 active+clean; 602 MiB data, 7.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.2 MiB/s rd, 5.7 MiB/s wr, 496 op/s 2024-03-19T01:59:35.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:59:35 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T01:59:35.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:59:35 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T01:59:35.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:59:35 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T01:59:35.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:59:35 smithi099 ceph-mon[62951]: pgmap v906: 97 pgs: 97 active+clean; 602 MiB data, 7.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.2 MiB/s rd, 5.7 MiB/s wr, 496 op/s 2024-03-19T01:59:36.110 INFO:tasks.workunit.client.0.smithi099.stdout: 10 158959 22.95 MB/sec execute 245 sec latency 74.921 ms 2024-03-19T01:59:37.110 INFO:tasks.workunit.client.0.smithi099.stdout: 10 159263 22.94 MB/sec execute 246 sec latency 293.294 ms 2024-03-19T01:59:37.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:59:37 smithi112 ceph-mon[66819]: pgmap v907: 97 pgs: 97 active+clean; 599 MiB data, 7.2 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 6.7 MiB/s wr, 606 op/s 2024-03-19T01:59:37.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:59:37 smithi115 ceph-mon[67332]: pgmap v907: 97 pgs: 97 active+clean; 599 MiB data, 7.2 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 6.7 MiB/s wr, 606 op/s 2024-03-19T01:59:37.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:59:37 smithi099 ceph-mon[62951]: pgmap v907: 97 pgs: 97 active+clean; 599 MiB data, 7.2 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 6.7 MiB/s wr, 606 op/s 2024-03-19T01:59:38.110 INFO:tasks.workunit.client.0.smithi099.stdout: 10 159673 22.92 MB/sec execute 247 sec latency 157.386 ms 2024-03-19T01:59:39.110 INFO:tasks.workunit.client.0.smithi099.stdout: 10 160180 22.94 MB/sec execute 248 sec latency 67.534 ms 2024-03-19T01:59:39.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:59:39 smithi112 ceph-mon[66819]: pgmap v908: 97 pgs: 97 active+clean; 587 MiB data, 7.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.1 MiB/s rd, 5.4 MiB/s wr, 507 op/s 2024-03-19T01:59:39.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:59:39 smithi115 ceph-mon[67332]: pgmap v908: 97 pgs: 97 active+clean; 587 MiB data, 7.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.1 MiB/s rd, 5.4 MiB/s wr, 507 op/s 2024-03-19T01:59:39.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:59:39 smithi099 ceph-mon[62951]: pgmap v908: 97 pgs: 97 active+clean; 587 MiB data, 7.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.1 MiB/s rd, 5.4 MiB/s wr, 507 op/s 2024-03-19T01:59:40.110 INFO:tasks.workunit.client.0.smithi099.stdout: 10 160801 22.96 MB/sec execute 249 sec latency 84.424 ms 2024-03-19T01:59:41.110 INFO:tasks.workunit.client.0.smithi099.stdout: 10 161361 23.02 MB/sec execute 250 sec latency 55.741 ms 2024-03-19T01:59:41.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:59:41 smithi112 ceph-mon[66819]: pgmap v909: 97 pgs: 97 active+clean; 584 MiB data, 7.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.5 MiB/s rd, 5.7 MiB/s wr, 539 op/s 2024-03-19T01:59:41.391 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:59:41 smithi099 ceph-mon[62951]: pgmap v909: 97 pgs: 97 active+clean; 584 MiB data, 7.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.5 MiB/s rd, 5.7 MiB/s wr, 539 op/s 2024-03-19T01:59:41.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:59:41 smithi115 ceph-mon[67332]: pgmap v909: 97 pgs: 97 active+clean; 584 MiB data, 7.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.5 MiB/s rd, 5.7 MiB/s wr, 539 op/s 2024-03-19T01:59:42.110 INFO:tasks.workunit.client.0.smithi099.stdout: 10 161738 23.02 MB/sec execute 251 sec latency 269.712 ms 2024-03-19T01:59:43.110 INFO:tasks.workunit.client.0.smithi099.stdout: 10 162159 23.02 MB/sec execute 252 sec latency 70.186 ms 2024-03-19T01:59:43.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:59:43 smithi112 ceph-mon[66819]: pgmap v910: 97 pgs: 97 active+clean; 592 MiB data, 7.2 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 6.1 MiB/s wr, 535 op/s 2024-03-19T01:59:43.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:59:43 smithi115 ceph-mon[67332]: pgmap v910: 97 pgs: 97 active+clean; 592 MiB data, 7.2 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 6.1 MiB/s wr, 535 op/s 2024-03-19T01:59:43.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:59:43 smithi099 ceph-mon[62951]: pgmap v910: 97 pgs: 97 active+clean; 592 MiB data, 7.2 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 6.1 MiB/s wr, 535 op/s 2024-03-19T01:59:43.759 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.757+0000 7fc0e484b640 1 -- 172.21.15.99:0/3266438808 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc0e0101970 msgr2=0x7fc0e0101df0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:59:43.759 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.757+0000 7fc0e484b640 1 --2- 172.21.15.99:0/3266438808 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc0e0101970 0x7fc0e0101df0 secure :-1 s=READY pgs=699 cs=0 l=1 rev1=1 crypto rx=0x7fc0c8009900 tx=0x7fc0c802bc30 comp rx=0 tx=0).stop 2024-03-19T01:59:43.759 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.757+0000 7fc0e484b640 1 -- 172.21.15.99:0/3266438808 shutdown_connections 2024-03-19T01:59:43.759 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.757+0000 7fc0e484b640 1 --2- 172.21.15.99:0/3266438808 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fc0e0102330 0x7fc0e0108bc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:43.759 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.757+0000 7fc0e484b640 1 --2- 172.21.15.99:0/3266438808 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc0e0101970 0x7fc0e0101df0 unknown :-1 s=CLOSED pgs=699 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:43.759 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.757+0000 7fc0e484b640 1 --2- 172.21.15.99:0/3266438808 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fc0e0100780 0x7fc0e0100ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:43.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.757+0000 7fc0e484b640 1 -- 172.21.15.99:0/3266438808 >> 172.21.15.99:0/3266438808 conn(0x7fc0e00fbcf0 msgr2=0x7fc0e00fe130 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:59:43.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.757+0000 7fc0e484b640 1 -- 172.21.15.99:0/3266438808 shutdown_connections 2024-03-19T01:59:43.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.757+0000 7fc0e484b640 1 -- 172.21.15.99:0/3266438808 wait complete. 2024-03-19T01:59:43.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.758+0000 7fc0e484b640 1 Processor -- start 2024-03-19T01:59:43.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.758+0000 7fc0e484b640 1 -- start start 2024-03-19T01:59:43.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.758+0000 7fc0e484b640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fc0e0100780 0x7fc0e006a5c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:43.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.758+0000 7fc0e484b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc0e0101970 0x7fc0e019c830 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:43.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.758+0000 7fc0e484b640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fc0e0102330 0x7fc0e019ee50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:43.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.758+0000 7fc0e484b640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fc0e010c400 con 0x7fc0e0101970 2024-03-19T01:59:43.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.758+0000 7fc0e484b640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fc0e010c280 con 0x7fc0e0100780 2024-03-19T01:59:43.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.758+0000 7fc0e484b640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fc0e010c580 con 0x7fc0e0102330 2024-03-19T01:59:43.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.758+0000 7fc0d7fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc0e0101970 0x7fc0e019c830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:43.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.758+0000 7fc0de59c640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fc0e0100780 0x7fc0e006a5c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:43.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.758+0000 7fc0d7fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc0e0101970 0x7fc0e019c830 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:33156/0 (socket says 172.21.15.99:33156) 2024-03-19T01:59:43.761 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.758+0000 7fc0de59c640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fc0e0100780 0x7fc0e006a5c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:49946/0 (socket says 172.21.15.99:49946) 2024-03-19T01:59:43.761 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.758+0000 7fc0d7fff640 1 -- 172.21.15.99:0/1369110451 learned_addr learned my addr 172.21.15.99:0/1369110451 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:59:43.761 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.758+0000 7fc0ded9d640 1 --2- 172.21.15.99:0/1369110451 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fc0e0102330 0x7fc0e019ee50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:43.761 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.758+0000 7fc0de59c640 1 -- 172.21.15.99:0/1369110451 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fc0e0102330 msgr2=0x7fc0e019ee50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:59:43.761 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.758+0000 7fc0de59c640 1 --2- 172.21.15.99:0/1369110451 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fc0e0102330 0x7fc0e019ee50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:43.761 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.758+0000 7fc0de59c640 1 -- 172.21.15.99:0/1369110451 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc0e0101970 msgr2=0x7fc0e019c830 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:59:43.761 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.758+0000 7fc0de59c640 1 --2- 172.21.15.99:0/1369110451 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc0e0101970 0x7fc0e019c830 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:43.761 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.758+0000 7fc0de59c640 1 -- 172.21.15.99:0/1369110451 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fc0e019f760 con 0x7fc0e0100780 2024-03-19T01:59:43.761 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.759+0000 7fc0de59c640 1 --2- 172.21.15.99:0/1369110451 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fc0e0100780 0x7fc0e006a5c0 secure :-1 s=READY pgs=193 cs=0 l=1 rev1=1 crypto rx=0x7fc0d000e9c0 tx=0x7fc0d000ee80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:59:43.761 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.759+0000 7fc0d77fe640 1 -- 172.21.15.99:0/1369110451 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc0d000ccc0 con 0x7fc0e0100780 2024-03-19T01:59:43.761 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.759+0000 7fc0d77fe640 1 -- 172.21.15.99:0/1369110451 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fc0d00035f0 con 0x7fc0e0100780 2024-03-19T01:59:43.761 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.759+0000 7fc0e484b640 1 -- 172.21.15.99:0/1369110451 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fc0e019f9f0 con 0x7fc0e0100780 2024-03-19T01:59:43.761 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.759+0000 7fc0d77fe640 1 -- 172.21.15.99:0/1369110451 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc0d000ce60 con 0x7fc0e0100780 2024-03-19T01:59:43.761 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.759+0000 7fc0e484b640 1 -- 172.21.15.99:0/1369110451 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fc0e019fec0 con 0x7fc0e0100780 2024-03-19T01:59:43.767 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.761+0000 7fc0d77fe640 1 -- 172.21.15.99:0/1369110451 <== mon.2 v2:172.21.15.112:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7fc0d00037a0 con 0x7fc0e0100780 2024-03-19T01:59:43.767 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.762+0000 7fc0e7434740 1 Processor -- start 2024-03-19T01:59:43.767 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.762+0000 7fc0e7434740 1 -- start start 2024-03-19T01:59:43.767 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.762+0000 7fc0d77fe640 1 --2- 172.21.15.99:0/1369110451 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fc0cc0a28a0 0x7fc0cc0a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:43.767 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.762+0000 7fc0e7434740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560ccfbe6510 0x560ccfbe6930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:43.767 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.762+0000 7fc0e7434740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560ccfbe6f00 0x560ccfbeba00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:43.767 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.762+0000 7fc0e7434740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560ccfbec070 0x560ccfbee500 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:43.767 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.762+0000 7fc0e7434740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x560ccfb46be0 con 0x560ccfbe6510 2024-03-19T01:59:43.767 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.762+0000 7fc0e7434740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x560ccfaef960 con 0x560ccfbe6f00 2024-03-19T01:59:43.767 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.762+0000 7fc0e7434740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x560ccfb42c80 con 0x560ccfbec070 2024-03-19T01:59:43.767 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.762+0000 7fc0d7fff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560ccfbe6f00 0x560ccfbeba00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:43.767 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.762+0000 7fc0ded9d640 1 --2- 172.21.15.99:0/1369110451 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fc0cc0a28a0 0x7fc0cc0a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:43.767 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.762+0000 7fc0de59c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560ccfbe6510 0x560ccfbe6930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:43.768 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.762+0000 7fc0d7fff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560ccfbe6f00 0x560ccfbeba00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:49958/0 (socket says 172.21.15.99:49958) 2024-03-19T01:59:43.768 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.762+0000 7fc0d7fff640 1 -- 172.21.15.99:0/2038317198 learned_addr learned my addr 172.21.15.99:0/2038317198 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:59:43.768 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.762+0000 7fc0ded9d640 1 --2- 172.21.15.99:0/2038317198 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560ccfbec070 0x560ccfbee500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:43.768 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.762+0000 7fc0d77fe640 1 -- 172.21.15.99:0/1369110451 <== mon.2 v2:172.21.15.112:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fc0d00cefb0 con 0x7fc0e0100780 2024-03-19T01:59:43.768 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.762+0000 7fc0d7fff640 1 -- 172.21.15.99:0/2038317198 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560ccfbec070 msgr2=0x560ccfbee500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:59:43.768 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.762+0000 7fc0d7fff640 1 --2- 172.21.15.99:0/2038317198 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560ccfbec070 0x560ccfbee500 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:43.768 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.762+0000 7fc0d7fff640 1 -- 172.21.15.99:0/2038317198 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560ccfbe6510 msgr2=0x560ccfbe6930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:59:43.768 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.762+0000 7fc0d7fff640 1 --2- 172.21.15.99:0/2038317198 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560ccfbe6510 0x560ccfbe6930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:43.768 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.762+0000 7fc0d7fff640 1 -- 172.21.15.99:0/2038317198 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x560ccfae6d20 con 0x560ccfbe6f00 2024-03-19T01:59:43.768 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.763+0000 7fc0ded9d640 1 --2- 172.21.15.99:0/1369110451 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fc0cc0a28a0 0x7fc0cc0a4d60 secure :-1 s=READY pgs=356 cs=0 l=1 rev1=1 crypto rx=0x7fc0d8005fd0 tx=0x7fc0d8009f10 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:59:43.768 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.763+0000 7fc0d7fff640 1 --2- 172.21.15.99:0/2038317198 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560ccfbe6f00 0x560ccfbeba00 secure :-1 s=READY pgs=194 cs=0 l=1 rev1=1 crypto rx=0x7fc0c8009230 tx=0x7fc0c8009560 comp rx=0 tx=0).ready entity=mon.2 client_cookie=78b0d1b2de6cfac9 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:59:43.768 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.763+0000 7fc0d4ff9640 1 -- 172.21.15.99:0/2038317198 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc0c8042070 con 0x560ccfbe6f00 2024-03-19T01:59:43.768 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.763+0000 7fc0d4ff9640 1 -- 172.21.15.99:0/2038317198 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fc0c802cb80 con 0x560ccfbe6f00 2024-03-19T01:59:43.768 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.763+0000 7fc0d4ff9640 1 -- 172.21.15.99:0/2038317198 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc0c8004ed0 con 0x560ccfbe6f00 2024-03-19T01:59:43.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.764+0000 7fc0e7434740 1 -- 172.21.15.99:0/2038317198 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560ccfbe6f00 msgr2=0x560ccfbeba00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:59:43.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.764+0000 7fc0e7434740 1 --2- 172.21.15.99:0/2038317198 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560ccfbe6f00 0x560ccfbeba00 secure :-1 s=READY pgs=194 cs=0 l=1 rev1=1 crypto rx=0x7fc0c8009230 tx=0x7fc0c8009560 comp rx=0 tx=0).stop 2024-03-19T01:59:43.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.764+0000 7fc0e7434740 1 -- 172.21.15.99:0/2038317198 shutdown_connections 2024-03-19T01:59:43.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.764+0000 7fc0e7434740 1 --2- 172.21.15.99:0/2038317198 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560ccfbe6f00 0x560ccfbeba00 unknown :-1 s=CLOSED pgs=194 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:43.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.764+0000 7fc0e7434740 1 --2- 172.21.15.99:0/2038317198 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560ccfbec070 0x560ccfbee500 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:43.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.764+0000 7fc0e7434740 1 --2- 172.21.15.99:0/2038317198 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560ccfbe6510 0x560ccfbe6930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:43.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.764+0000 7fc0e7434740 1 -- 172.21.15.99:0/2038317198 >> 172.21.15.99:0/2038317198 conn(0x560ccfbdd670 msgr2=0x560ccfbdda70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:59:43.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.764+0000 7fc0e7434740 1 -- 172.21.15.99:0/2038317198 shutdown_connections 2024-03-19T01:59:43.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.764+0000 7fc0e7434740 1 -- 172.21.15.99:0/2038317198 wait complete. 2024-03-19T01:59:43.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.764+0000 7fc0e7434740 1 -- start start 2024-03-19T01:59:43.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.764+0000 7fc0e7434740 1 Processor -- start 2024-03-19T01:59:43.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.764+0000 7fc0e7434740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560ccfbe6510 0x560ccfbf04e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:43.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.764+0000 7fc0e7434740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560ccfbe6f00 0x560ccfbf0a20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:43.769 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.764+0000 7fc0e7434740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560ccfbec070 0x560ccfbf7a60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:43.770 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.764+0000 7fc0e7434740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x560ccfbeefc0 con 0x560ccfbe6f00 2024-03-19T01:59:43.770 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.764+0000 7fc0e7434740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x560ccfb02db0 con 0x560ccfbec070 2024-03-19T01:59:43.770 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.764+0000 7fc0e7434740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x560ccfbef140 con 0x560ccfbe6510 2024-03-19T01:59:43.770 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.764+0000 7fc0e7434740 10 client.0 _ll_register_callbacks cb 0x560ccfb07e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:59:43.770 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.764+0000 7fc0d7fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560ccfbe6f00 0x560ccfbf0a20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:43.770 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.764+0000 7fc0ded9d640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560ccfbec070 0x560ccfbf7a60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:43.770 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.764+0000 7fc0de59c640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560ccfbe6510 0x560ccfbf04e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:43.770 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.764+0000 7fc0ded9d640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560ccfbec070 0x560ccfbf7a60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:49962/0 (socket says 172.21.15.99:49962) 2024-03-19T01:59:43.770 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.764+0000 7fc0ded9d640 1 -- 172.21.15.99:0/1185395571 learned_addr learned my addr 172.21.15.99:0/1185395571 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:59:43.770 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.765+0000 7fc0ded9d640 1 -- 172.21.15.99:0/1185395571 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560ccfbe6510 msgr2=0x560ccfbf04e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:59:43.770 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.765+0000 7fc0ded9d640 1 --2- 172.21.15.99:0/1185395571 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560ccfbe6510 0x560ccfbf04e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:43.770 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.765+0000 7fc0ded9d640 1 -- 172.21.15.99:0/1185395571 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560ccfbe6f00 msgr2=0x560ccfbf0a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:59:43.771 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.765+0000 7fc0ded9d640 1 --2- 172.21.15.99:0/1185395571 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560ccfbe6f00 0x560ccfbf0a20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:43.771 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.765+0000 7fc0ded9d640 1 -- 172.21.15.99:0/1185395571 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x560ccfb01c60 con 0x560ccfbec070 2024-03-19T01:59:43.771 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.765+0000 7fc0d7fff640 1 --2- 172.21.15.99:0/1185395571 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560ccfbe6f00 0x560ccfbf0a20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:59:43.771 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.765+0000 7fc0ded9d640 1 --2- 172.21.15.99:0/1185395571 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560ccfbec070 0x560ccfbf7a60 secure :-1 s=READY pgs=195 cs=0 l=1 rev1=1 crypto rx=0x7fc0d801e4f0 tx=0x7fc0d801d000 comp rx=0 tx=0).ready entity=mon.2 client_cookie=279db283a5ef9072 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:59:43.771 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.765+0000 7fc0c5ffb640 10 client.0 ms_handle_connect on v2:172.21.15.112:3300/0 2024-03-19T01:59:43.771 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.765+0000 7fc0c5ffb640 1 -- 172.21.15.99:0/1185395571 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc0d8024070 con 0x560ccfbec070 2024-03-19T01:59:43.771 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.765+0000 7fc0c5ffb640 1 -- 172.21.15.99:0/1185395571 <== mon.2 v2:172.21.15.112:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fc0d80206c0 con 0x560ccfbec070 2024-03-19T01:59:43.771 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.765+0000 7fc0e7434740 1 -- 172.21.15.99:0/1185395571 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fc0c802cb80 con 0x560ccfbec070 2024-03-19T01:59:43.771 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.765+0000 7fc0c5ffb640 1 -- 172.21.15.99:0/1185395571 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7fc0d8004b20 con 0x560ccfbec070 2024-03-19T01:59:43.771 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.765+0000 7fc0e7434740 10 client.26606 fetch_fsmap learned FSMap version 37 2024-03-19T01:59:43.771 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.765+0000 7fc0e7434740 1 -- 172.21.15.99:0/1185395571 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x560ccf85b210 con 0x560ccfbec070 2024-03-19T01:59:43.771 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.765+0000 7fc0c5ffb640 1 -- 172.21.15.99:0/1185395571 <== mon.2 v2:172.21.15.112:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fc0c802cb80 con 0x560ccfbec070 2024-03-19T01:59:43.771 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.765+0000 7fc0c5ffb640 1 -- 172.21.15.99:0/1185395571 <== mon.2 v2:172.21.15.112:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7fc0d80051e0 con 0x560ccfbec070 2024-03-19T01:59:43.771 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.766+0000 7fc0e7434740 10 client.26606 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T01:59:43.772 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.766+0000 7fc0e7434740 10 client.26606 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:59:43.772 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.766+0000 7fc0e7434740 20 client.26606 populate_metadata read hostname 'smithi099' 2024-03-19T01:59:43.772 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.766+0000 7fc0e7434740 1 --2- 172.21.15.99:0/1185395571 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x560ccfbfc3c0 0x560ccfbfe810 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:43.772 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.766+0000 7fc0e7434740 4 client.26606 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:59:43.772 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.766+0000 7fc0e7434740 1 -- 172.21.15.99:0/1185395571 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x560ccfb54120 con 0x560ccfbfc3c0 2024-03-19T01:59:43.772 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.766+0000 7fc0de59c640 1 --2- 172.21.15.99:0/1185395571 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x560ccfbfc3c0 0x560ccfbfe810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:43.772 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.766+0000 7fc0c5ffb640 10 client.26606 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:59:43.772 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.766+0000 7fc0de59c640 1 --2- 172.21.15.99:0/1185395571 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x560ccfbfc3c0 0x560ccfbfe810 crc :-1 s=READY pgs=195 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=2815eea6cf76dc0b server_cookie=e069633f4c43a589 in_seq=0 out_seq=0 2024-03-19T01:59:43.773 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.768+0000 7fc0c5ffb640 1 -- 172.21.15.99:0/1185395571 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x560ccfb54120 con 0x560ccfbfc3c0 2024-03-19T01:59:43.773 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.768+0000 7fc0c5ffb640 10 client.26606 handle_command_reply: tid=0 2024-03-19T01:59:43.774 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.768+0000 7fc0e7434740 1 client.26606 shutdown 2024-03-19T01:59:43.774 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.769+0000 7fc0e7434740 1 -- 172.21.15.99:0/1185395571 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560ccfbec070 msgr2=0x560ccfbf7a60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:59:43.774 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.769+0000 7fc0e7434740 1 --2- 172.21.15.99:0/1185395571 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560ccfbec070 0x560ccfbf7a60 secure :-1 s=READY pgs=195 cs=0 l=1 rev1=1 crypto rx=0x7fc0d801e4f0 tx=0x7fc0d801d000 comp rx=0 tx=0).stop 2024-03-19T01:59:43.774 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.769+0000 7fc0e7434740 1 -- 172.21.15.99:0/1185395571 shutdown_connections 2024-03-19T01:59:43.774 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.769+0000 7fc0e7434740 1 --2- 172.21.15.99:0/1185395571 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x560ccfbfc3c0 0x560ccfbfe810 crc :-1 s=READY pgs=195 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:43.774 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.769+0000 7fc0e7434740 1 --2- 172.21.15.99:0/1185395571 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560ccfbec070 0x560ccfbf7a60 unknown :-1 s=CLOSED pgs=195 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:43.774 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.769+0000 7fc0c5ffb640 0 client.26606 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:59:43.774 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.769+0000 7fc0e7434740 1 --2- 172.21.15.99:0/1185395571 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560ccfbe6f00 0x560ccfbf0a20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:43.774 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.769+0000 7fc0e7434740 1 --2- 172.21.15.99:0/1185395571 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560ccfbe6510 0x560ccfbf04e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:43.774 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.769+0000 7fc0e7434740 1 -- 172.21.15.99:0/1185395571 >> 172.21.15.99:0/1185395571 conn(0x560ccfbdd670 msgr2=0x560ccfbde8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:59:43.774 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.769+0000 7fc0e7434740 1 -- 172.21.15.99:0/1185395571 shutdown_connections 2024-03-19T01:59:43.774 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.769+0000 7fc0e7434740 1 -- 172.21.15.99:0/1185395571 wait complete. 2024-03-19T01:59:43.774 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.769+0000 7fc0e7434740 20 client.26606 trim_cache size 0 max 16384 2024-03-19T01:59:43.786 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.786+0000 7fc0e7434740 1 Processor -- start 2024-03-19T01:59:43.786 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.786+0000 7fc0e7434740 1 -- start start 2024-03-19T01:59:43.786 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.786+0000 7fc0e7434740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560ccfbe6510 0x560ccfbdc220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:43.786 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.786+0000 7fc0e7434740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560ccfbe6f00 0x560ccfbf7a20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:43.786 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.786+0000 7fc0e7434740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560ccfbec070 0x560ccfbdc760 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:43.787 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.786+0000 7fc0e7434740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fc0e01a9380 con 0x560ccfbec070 2024-03-19T01:59:43.787 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.786+0000 7fc0e7434740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fc0e01a6c50 con 0x560ccfbe6f00 2024-03-19T01:59:43.787 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.786+0000 7fc0e7434740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fc0e01a4aa0 con 0x560ccfbe6510 2024-03-19T01:59:43.787 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.786+0000 7fc0ded9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560ccfbec070 0x560ccfbdc760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:43.787 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.786+0000 7fc0ded9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560ccfbec070 0x560ccfbdc760 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:33182/0 (socket says 172.21.15.99:33182) 2024-03-19T01:59:43.787 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.786+0000 7fc0ded9d640 1 -- 172.21.15.99:0/187270075 learned_addr learned my addr 172.21.15.99:0/187270075 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:59:43.787 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.786+0000 7fc0de59c640 1 --2- 172.21.15.99:0/187270075 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560ccfbe6510 0x560ccfbdc220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:43.787 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.786+0000 7fc0d7fff640 1 --2- 172.21.15.99:0/187270075 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560ccfbe6f00 0x560ccfbf7a20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:43.787 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.786+0000 7fc0ded9d640 1 -- 172.21.15.99:0/187270075 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560ccfbe6510 msgr2=0x560ccfbdc220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:59:43.787 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.786+0000 7fc0ded9d640 1 --2- 172.21.15.99:0/187270075 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560ccfbe6510 0x560ccfbdc220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:43.787 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.786+0000 7fc0ded9d640 1 -- 172.21.15.99:0/187270075 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560ccfbe6f00 msgr2=0x560ccfbf7a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:59:43.787 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.786+0000 7fc0ded9d640 1 --2- 172.21.15.99:0/187270075 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560ccfbe6f00 0x560ccfbf7a20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:43.787 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.786+0000 7fc0ded9d640 1 -- 172.21.15.99:0/187270075 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x560ccfbd8ee0 con 0x560ccfbec070 2024-03-19T01:59:43.788 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.787+0000 7fc0ded9d640 1 --2- 172.21.15.99:0/187270075 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560ccfbec070 0x560ccfbdc760 secure :-1 s=READY pgs=700 cs=0 l=1 rev1=1 crypto rx=0x7fc0d8005670 tx=0x7fc0d8004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=32ffeaaab30fcc65 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:59:43.788 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.787+0000 7fc0c7fff640 1 -- 172.21.15.99:0/187270075 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc0d8024040 con 0x560ccfbec070 2024-03-19T01:59:43.788 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.787+0000 7fc0c7fff640 1 -- 172.21.15.99:0/187270075 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fc0d8011950 con 0x560ccfbec070 2024-03-19T01:59:43.788 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.787+0000 7fc0c7fff640 1 -- 172.21.15.99:0/187270075 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc0d80058e0 con 0x560ccfbec070 2024-03-19T01:59:43.791 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.787+0000 7fc0e7434740 1 -- 172.21.15.99:0/187270075 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560ccfbec070 msgr2=0x560ccfbdc760 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:59:43.791 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.787+0000 7fc0e7434740 1 --2- 172.21.15.99:0/187270075 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560ccfbec070 0x560ccfbdc760 secure :-1 s=READY pgs=700 cs=0 l=1 rev1=1 crypto rx=0x7fc0d8005670 tx=0x7fc0d8004a40 comp rx=0 tx=0).stop 2024-03-19T01:59:43.791 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.788+0000 7fc0e7434740 1 -- 172.21.15.99:0/187270075 shutdown_connections 2024-03-19T01:59:43.791 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.788+0000 7fc0e7434740 1 --2- 172.21.15.99:0/187270075 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560ccfbe6f00 0x560ccfbf7a20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:43.791 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.788+0000 7fc0e7434740 1 --2- 172.21.15.99:0/187270075 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560ccfbec070 0x560ccfbdc760 unknown :-1 s=CLOSED pgs=700 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:43.791 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.788+0000 7fc0e7434740 1 --2- 172.21.15.99:0/187270075 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560ccfbe6510 0x560ccfbdc220 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:43.792 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.788+0000 7fc0e7434740 1 -- 172.21.15.99:0/187270075 >> 172.21.15.99:0/187270075 conn(0x560ccfbdd670 msgr2=0x560ccfbf04e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:59:43.792 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.788+0000 7fc0e7434740 1 -- 172.21.15.99:0/187270075 shutdown_connections 2024-03-19T01:59:43.792 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.788+0000 7fc0e7434740 1 -- 172.21.15.99:0/187270075 wait complete. 2024-03-19T01:59:43.792 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.788+0000 7fc0e7434740 1 -- start start 2024-03-19T01:59:43.792 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.788+0000 7fc0e7434740 1 Processor -- start 2024-03-19T01:59:43.792 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.788+0000 7fc0e7434740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560ccfbe6510 0x560ccfbf50e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:43.792 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.788+0000 7fc0e7434740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560ccfbe6f00 0x560ccfbf5620 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:43.792 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.788+0000 7fc0e7434740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560ccfbec070 0x560ccfbf65c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:43.792 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.788+0000 7fc0e7434740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fc0e01a5110 con 0x560ccfbe6510 2024-03-19T01:59:43.792 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.788+0000 7fc0e7434740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fc0e01a6580 con 0x560ccfbe6f00 2024-03-19T01:59:43.792 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.788+0000 7fc0e7434740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fc0e01a57e0 con 0x560ccfbec070 2024-03-19T01:59:43.792 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.788+0000 7fc0e7434740 10 client.0 _ll_register_callbacks cb 0x7fc0d8009a90 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:59:43.792 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.788+0000 7fc0de59c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560ccfbe6510 0x560ccfbf50e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:43.792 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.788+0000 7fc0de59c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560ccfbe6510 0x560ccfbf50e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:33198/0 (socket says 172.21.15.99:33198) 2024-03-19T01:59:43.792 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.788+0000 7fc0de59c640 1 -- 172.21.15.99:0/99223406 learned_addr learned my addr 172.21.15.99:0/99223406 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:59:43.792 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.788+0000 7fc0d7fff640 1 --2- 172.21.15.99:0/99223406 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560ccfbe6f00 0x560ccfbf5620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:43.793 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.788+0000 7fc0ded9d640 1 --2- 172.21.15.99:0/99223406 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560ccfbec070 0x560ccfbf65c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:43.793 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.789+0000 7fc0de59c640 1 -- 172.21.15.99:0/99223406 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560ccfbec070 msgr2=0x560ccfbf65c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:59:43.793 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.789+0000 7fc0de59c640 1 --2- 172.21.15.99:0/99223406 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560ccfbec070 0x560ccfbf65c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:43.793 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.789+0000 7fc0de59c640 1 -- 172.21.15.99:0/99223406 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560ccfbe6f00 msgr2=0x560ccfbf5620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:59:43.793 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.789+0000 7fc0de59c640 1 --2- 172.21.15.99:0/99223406 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560ccfbe6f00 0x560ccfbf5620 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:43.793 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.789+0000 7fc0de59c640 1 -- 172.21.15.99:0/99223406 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x560ccf927620 con 0x560ccfbe6510 2024-03-19T01:59:43.793 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.789+0000 7fc0de59c640 1 --2- 172.21.15.99:0/99223406 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560ccfbe6510 0x560ccfbf50e0 secure :-1 s=READY pgs=701 cs=0 l=1 rev1=1 crypto rx=0x7fc0d00cca70 tx=0x7fc0d001def0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=edc79b17e013e2fa server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:59:43.793 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.789+0000 7fc0c5ffb640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:59:43.793 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.789+0000 7fc0c5ffb640 1 -- 172.21.15.99:0/99223406 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc0d009fa60 con 0x560ccfbe6510 2024-03-19T01:59:43.793 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.789+0000 7fc0c5ffb640 1 -- 172.21.15.99:0/99223406 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fc0d0003b10 con 0x560ccfbe6510 2024-03-19T01:59:43.793 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.789+0000 7fc0e7434740 1 -- 172.21.15.99:0/99223406 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fc0d8011950 con 0x560ccfbe6510 2024-03-19T01:59:43.793 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.789+0000 7fc0c5ffb640 1 -- 172.21.15.99:0/99223406 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7fc0d00a40b0 con 0x560ccfbe6510 2024-03-19T01:59:43.793 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.789+0000 7fc0e7434740 10 client.16836 fetch_fsmap learned FSMap version 37 2024-03-19T01:59:43.793 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.789+0000 7fc0e7434740 1 -- 172.21.15.99:0/99223406 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x560ccfbf7d60 con 0x560ccfbe6510 2024-03-19T01:59:43.793 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.789+0000 7fc0c5ffb640 1 -- 172.21.15.99:0/99223406 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fc0d8011950 con 0x560ccfbe6510 2024-03-19T01:59:43.794 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.789+0000 7fc0c5ffb640 1 -- 172.21.15.99:0/99223406 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7fc0d0086070 con 0x560ccfbe6510 2024-03-19T01:59:43.794 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.789+0000 7fc0e7434740 10 client.16836 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T01:59:43.794 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.789+0000 7fc0e7434740 10 client.16836 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:59:43.794 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.789+0000 7fc0e7434740 20 client.16836 populate_metadata read hostname 'smithi099' 2024-03-19T01:59:43.794 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.789+0000 7fc0e7434740 1 --2- 172.21.15.99:0/99223406 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x560ccfc62ea0 0x560ccfc652c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:43.794 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.789+0000 7fc0e7434740 4 client.16836 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:59:43.794 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.789+0000 7fc0e7434740 1 -- 172.21.15.99:0/99223406 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x560ccfb4e470 con 0x560ccfc62ea0 2024-03-19T01:59:43.794 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.790+0000 7fc0d7fff640 1 --2- 172.21.15.99:0/99223406 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x560ccfc62ea0 0x560ccfc652c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:43.794 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.790+0000 7fc0c5ffb640 10 client.16836 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:59:43.794 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:43.790+0000 7fc0d7fff640 1 --2- 172.21.15.99:0/99223406 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x560ccfc62ea0 0x560ccfc652c0 crc :-1 s=READY pgs=196 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=61767b90cd7c557c server_cookie=178fbb3a5c5b683b in_seq=0 out_seq=0 2024-03-19T01:59:44.110 INFO:tasks.workunit.client.0.smithi099.stdout: 10 162481 22.99 MB/sec execute 253 sec latency 261.597 ms 2024-03-19T01:59:44.452 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:44.451+0000 7fc0c5ffb640 1 -- 172.21.15.99:0/99223406 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x560ccfb4e470 con 0x560ccfc62ea0 2024-03-19T01:59:44.452 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:44.451+0000 7fc0c5ffb640 10 client.16836 handle_command_reply: tid=0 2024-03-19T01:59:44.452 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:44.451+0000 7fc0e7434740 1 client.16836 shutdown 2024-03-19T01:59:44.452 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:44.452+0000 7fc0e7434740 1 -- 172.21.15.99:0/99223406 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560ccfbe6510 msgr2=0x560ccfbf50e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:59:44.452 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:44.452+0000 7fc0e7434740 1 --2- 172.21.15.99:0/99223406 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560ccfbe6510 0x560ccfbf50e0 secure :-1 s=READY pgs=701 cs=0 l=1 rev1=1 crypto rx=0x7fc0d00cca70 tx=0x7fc0d001def0 comp rx=0 tx=0).stop 2024-03-19T01:59:44.452 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:44.452+0000 7fc0e7434740 1 -- 172.21.15.99:0/99223406 shutdown_connections 2024-03-19T01:59:44.452 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:44.452+0000 7fc0e7434740 1 --2- 172.21.15.99:0/99223406 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x560ccfc62ea0 0x560ccfc652c0 crc :-1 s=READY pgs=196 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:44.452 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:44.452+0000 7fc0c5ffb640 0 client.16836 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:59:44.453 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:44.452+0000 7fc0e7434740 1 --2- 172.21.15.99:0/99223406 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560ccfbe6f00 0x560ccfbf5620 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:44.453 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:44.452+0000 7fc0e7434740 1 --2- 172.21.15.99:0/99223406 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560ccfbec070 0x560ccfbf65c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:44.453 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:44.452+0000 7fc0e7434740 1 --2- 172.21.15.99:0/99223406 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560ccfbe6510 0x560ccfbf50e0 unknown :-1 s=CLOSED pgs=701 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:44.453 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:44.452+0000 7fc0e7434740 1 -- 172.21.15.99:0/99223406 >> 172.21.15.99:0/99223406 conn(0x560ccfbdd670 msgr2=0x560ccfbdc220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:59:44.453 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:44.452+0000 7fc0e7434740 1 -- 172.21.15.99:0/99223406 shutdown_connections 2024-03-19T01:59:44.453 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:44.452+0000 7fc0e7434740 1 -- 172.21.15.99:0/99223406 wait complete. 2024-03-19T01:59:44.453 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:44.452+0000 7fc0e7434740 20 client.16836 trim_cache size 0 max 16384 2024-03-19T01:59:44.453 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:59:44.453 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:59:44.453 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:59:44.454 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:59:44.454 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:44.452+0000 7fc0e484b640 1 -- 172.21.15.99:0/1369110451 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fc0cc0a28a0 msgr2=0x7fc0cc0a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:59:44.454 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:44.452+0000 7fc0e484b640 1 --2- 172.21.15.99:0/1369110451 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fc0cc0a28a0 0x7fc0cc0a4d60 secure :-1 s=READY pgs=356 cs=0 l=1 rev1=1 crypto rx=0x7fc0d8005fd0 tx=0x7fc0d8009f10 comp rx=0 tx=0).stop 2024-03-19T01:59:44.455 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:44.452+0000 7fc0e484b640 1 -- 172.21.15.99:0/1369110451 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fc0e0100780 msgr2=0x7fc0e006a5c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:59:44.455 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:44.452+0000 7fc0e484b640 1 --2- 172.21.15.99:0/1369110451 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fc0e0100780 0x7fc0e006a5c0 secure :-1 s=READY pgs=193 cs=0 l=1 rev1=1 crypto rx=0x7fc0d000e9c0 tx=0x7fc0d000ee80 comp rx=0 tx=0).stop 2024-03-19T01:59:44.455 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:44.453+0000 7fc0e484b640 1 -- 172.21.15.99:0/1369110451 shutdown_connections 2024-03-19T01:59:44.455 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:44.453+0000 7fc0e484b640 1 --2- 172.21.15.99:0/1369110451 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fc0e0102330 0x7fc0e019ee50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:44.455 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:44.453+0000 7fc0e484b640 1 --2- 172.21.15.99:0/1369110451 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fc0e0101970 0x7fc0e019c830 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:44.455 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:44.453+0000 7fc0e484b640 1 --2- 172.21.15.99:0/1369110451 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fc0cc0a28a0 0x7fc0cc0a4d60 unknown :-1 s=CLOSED pgs=356 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:44.455 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:44.453+0000 7fc0e484b640 1 --2- 172.21.15.99:0/1369110451 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fc0e0100780 0x7fc0e006a5c0 unknown :-1 s=CLOSED pgs=193 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:44.455 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:44.453+0000 7fc0e484b640 1 -- 172.21.15.99:0/1369110451 >> 172.21.15.99:0/1369110451 conn(0x7fc0e00fbcf0 msgr2=0x7fc0e0104c70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:59:44.455 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:44.453+0000 7fc0e484b640 1 -- 172.21.15.99:0/1369110451 shutdown_connections 2024-03-19T01:59:44.455 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:44.453+0000 7fc0e484b640 1 -- 172.21.15.99:0/1369110451 wait complete. 2024-03-19T01:59:45.110 INFO:tasks.workunit.client.0.smithi099.stdout: 10 162733 22.96 MB/sec execute 254 sec latency 468.154 ms 2024-03-19T01:59:45.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:59:45 smithi112 ceph-mon[66819]: pgmap v911: 97 pgs: 97 active+clean; 592 MiB data, 7.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 5.3 MiB/s wr, 443 op/s 2024-03-19T01:59:45.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:59:45 smithi115 ceph-mon[67332]: pgmap v911: 97 pgs: 97 active+clean; 592 MiB data, 7.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 5.3 MiB/s wr, 443 op/s 2024-03-19T01:59:45.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:59:45 smithi099 ceph-mon[62951]: pgmap v911: 97 pgs: 97 active+clean; 592 MiB data, 7.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 5.3 MiB/s wr, 443 op/s 2024-03-19T01:59:46.110 INFO:tasks.workunit.client.0.smithi099.stdout: 10 163289 23.00 MB/sec execute 255 sec latency 53.535 ms 2024-03-19T01:59:47.110 INFO:tasks.workunit.client.0.smithi099.stdout: 10 163506 22.95 MB/sec execute 256 sec latency 235.600 ms 2024-03-19T01:59:47.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:59:47 smithi112 ceph-mon[66819]: pgmap v912: 97 pgs: 97 active+clean; 589 MiB data, 7.2 GiB used, 1.0 TiB / 1.0 TiB avail; 4.7 MiB/s rd, 6.5 MiB/s wr, 590 op/s 2024-03-19T01:59:47.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:59:47 smithi115 ceph-mon[67332]: pgmap v912: 97 pgs: 97 active+clean; 589 MiB data, 7.2 GiB used, 1.0 TiB / 1.0 TiB avail; 4.7 MiB/s rd, 6.5 MiB/s wr, 590 op/s 2024-03-19T01:59:47.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:59:47 smithi099 ceph-mon[62951]: pgmap v912: 97 pgs: 97 active+clean; 589 MiB data, 7.2 GiB used, 1.0 TiB / 1.0 TiB avail; 4.7 MiB/s rd, 6.5 MiB/s wr, 590 op/s 2024-03-19T01:59:48.110 INFO:tasks.workunit.client.0.smithi099.stdout: 10 163893 22.92 MB/sec execute 257 sec latency 189.102 ms 2024-03-19T01:59:49.110 INFO:tasks.workunit.client.0.smithi099.stdout: 10 164333 22.90 MB/sec execute 258 sec latency 70.160 ms 2024-03-19T01:59:49.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:59:49 smithi112 ceph-mon[66819]: pgmap v913: 97 pgs: 97 active+clean; 589 MiB data, 7.3 GiB used, 1.0 TiB / 1.0 TiB avail; 3.7 MiB/s rd, 5.5 MiB/s wr, 516 op/s 2024-03-19T01:59:49.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:59:49 smithi115 ceph-mon[67332]: pgmap v913: 97 pgs: 97 active+clean; 589 MiB data, 7.3 GiB used, 1.0 TiB / 1.0 TiB avail; 3.7 MiB/s rd, 5.5 MiB/s wr, 516 op/s 2024-03-19T01:59:49.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:59:49 smithi099 ceph-mon[62951]: pgmap v913: 97 pgs: 97 active+clean; 589 MiB data, 7.3 GiB used, 1.0 TiB / 1.0 TiB avail; 3.7 MiB/s rd, 5.5 MiB/s wr, 516 op/s 2024-03-19T01:59:50.111 INFO:tasks.workunit.client.0.smithi099.stdout: 10 164793 22.92 MB/sec execute 259 sec latency 64.286 ms 2024-03-19T01:59:51.111 INFO:tasks.workunit.client.0.smithi099.stdout: 10 165252 22.94 MB/sec execute 260 sec latency 78.136 ms 2024-03-19T01:59:51.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:59:51 smithi112 ceph-mon[66819]: pgmap v914: 97 pgs: 97 active+clean; 590 MiB data, 7.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 6.0 MiB/s wr, 576 op/s 2024-03-19T01:59:51.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:59:51 smithi115 ceph-mon[67332]: pgmap v914: 97 pgs: 97 active+clean; 590 MiB data, 7.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 6.0 MiB/s wr, 576 op/s 2024-03-19T01:59:51.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:59:51 smithi099 ceph-mon[62951]: pgmap v914: 97 pgs: 97 active+clean; 590 MiB data, 7.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 6.0 MiB/s wr, 576 op/s 2024-03-19T01:59:52.110 INFO:tasks.workunit.client.0.smithi099.stdout: 10 165659 22.91 MB/sec execute 261 sec latency 312.169 ms 2024-03-19T01:59:53.111 INFO:tasks.workunit.client.0.smithi099.stdout: 10 166086 22.92 MB/sec execute 262 sec latency 66.646 ms 2024-03-19T01:59:53.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:59:53 smithi112 ceph-mon[66819]: pgmap v915: 97 pgs: 97 active+clean; 591 MiB data, 7.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 6.6 MiB/s wr, 595 op/s 2024-03-19T01:59:53.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:59:53 smithi115 ceph-mon[67332]: pgmap v915: 97 pgs: 97 active+clean; 591 MiB data, 7.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 6.6 MiB/s wr, 595 op/s 2024-03-19T01:59:53.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:59:53 smithi099 ceph-mon[62951]: pgmap v915: 97 pgs: 97 active+clean; 591 MiB data, 7.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 6.6 MiB/s wr, 595 op/s 2024-03-19T01:59:54.111 INFO:tasks.workunit.client.0.smithi099.stdout: 10 166641 22.95 MB/sec execute 263 sec latency 74.573 ms 2024-03-19T01:59:55.111 INFO:tasks.workunit.client.0.smithi099.stdout: 10 167075 23.00 MB/sec execute 264 sec latency 120.970 ms 2024-03-19T01:59:55.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:59:55 smithi112 ceph-mon[66819]: pgmap v916: 97 pgs: 97 active+clean; 592 MiB data, 7.3 GiB used, 1.0 TiB / 1.0 TiB avail; 3.0 MiB/s rd, 5.2 MiB/s wr, 477 op/s 2024-03-19T01:59:55.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:59:55 smithi115 ceph-mon[67332]: pgmap v916: 97 pgs: 97 active+clean; 592 MiB data, 7.3 GiB used, 1.0 TiB / 1.0 TiB avail; 3.0 MiB/s rd, 5.2 MiB/s wr, 477 op/s 2024-03-19T01:59:55.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:59:55 smithi099 ceph-mon[62951]: pgmap v916: 97 pgs: 97 active+clean; 592 MiB data, 7.3 GiB used, 1.0 TiB / 1.0 TiB avail; 3.0 MiB/s rd, 5.2 MiB/s wr, 477 op/s 2024-03-19T01:59:56.111 INFO:tasks.workunit.client.0.smithi099.stdout: 10 167411 22.96 MB/sec execute 265 sec latency 73.955 ms 2024-03-19T01:59:57.111 INFO:tasks.workunit.client.0.smithi099.stdout: 10 167680 22.94 MB/sec execute 266 sec latency 295.210 ms 2024-03-19T01:59:57.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:59:57 smithi112 ceph-mon[66819]: pgmap v917: 97 pgs: 97 active+clean; 599 MiB data, 7.4 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 6.5 MiB/s wr, 566 op/s 2024-03-19T01:59:57.545 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.544+0000 7f832e652640 1 -- 172.21.15.99:0/40847402 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f83281022f0 msgr2=0x7f8328108b80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:59:57.546 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.544+0000 7f832e652640 1 --2- 172.21.15.99:0/40847402 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f83281022f0 0x7f8328108b80 secure :-1 s=READY pgs=702 cs=0 l=1 rev1=1 crypto rx=0x7f8320009ca0 tx=0x7f832002c0f0 comp rx=0 tx=0).stop 2024-03-19T01:59:57.546 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.544+0000 7f832e652640 1 -- 172.21.15.99:0/40847402 shutdown_connections 2024-03-19T01:59:57.546 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.544+0000 7f832e652640 1 --2- 172.21.15.99:0/40847402 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8328101930 0x7f8328101db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:57.546 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.544+0000 7f832e652640 1 --2- 172.21.15.99:0/40847402 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f83281022f0 0x7f8328108b80 unknown :-1 s=CLOSED pgs=702 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:57.546 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.544+0000 7f832e652640 1 --2- 172.21.15.99:0/40847402 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8328100740 0x7f8328100b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:57.546 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.544+0000 7f832e652640 1 -- 172.21.15.99:0/40847402 >> 172.21.15.99:0/40847402 conn(0x7f83280fbcb0 msgr2=0x7f83280fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:59:57.546 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.544+0000 7f832e652640 1 -- 172.21.15.99:0/40847402 shutdown_connections 2024-03-19T01:59:57.546 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.544+0000 7f832e652640 1 -- 172.21.15.99:0/40847402 wait complete. 2024-03-19T01:59:57.546 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.544+0000 7f832e652640 1 Processor -- start 2024-03-19T01:59:57.546 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.544+0000 7f832e652640 1 -- start start 2024-03-19T01:59:57.546 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.545+0000 7f832e652640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8328100740 0x7f8328197a40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:57.547 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.545+0000 7f832e652640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8328101930 0x7f8328197f80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:57.547 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.545+0000 7f832e652640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f83281022f0 0x7f832819eff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:57.547 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.545+0000 7f832e652640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f832810d4e0 con 0x7f8328101930 2024-03-19T01:59:57.547 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.545+0000 7f832e652640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f832810d360 con 0x7f83281022f0 2024-03-19T01:59:57.547 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.545+0000 7f832e652640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f832810d660 con 0x7f8328100740 2024-03-19T01:59:57.547 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.545+0000 7f83277fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8328101930 0x7f8328197f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:57.547 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.545+0000 7f83277fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8328101930 0x7f8328197f80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:33654/0 (socket says 172.21.15.99:33654) 2024-03-19T01:59:57.547 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.545+0000 7f832cbef640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f83281022f0 0x7f832819eff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:57.547 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.545+0000 7f83277fe640 1 -- 172.21.15.99:0/1274391368 learned_addr learned my addr 172.21.15.99:0/1274391368 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:59:57.547 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.545+0000 7f8327fff640 1 --2- 172.21.15.99:0/1274391368 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8328100740 0x7f8328197a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:57.547 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.545+0000 7f83277fe640 1 -- 172.21.15.99:0/1274391368 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8328100740 msgr2=0x7f8328197a40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:59:57.547 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.545+0000 7f83277fe640 1 --2- 172.21.15.99:0/1274391368 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8328100740 0x7f8328197a40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:57.547 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.545+0000 7f83277fe640 1 -- 172.21.15.99:0/1274391368 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f83281022f0 msgr2=0x7f832819eff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:59:57.548 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.545+0000 7f83277fe640 1 --2- 172.21.15.99:0/1274391368 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f83281022f0 0x7f832819eff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:57.548 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.545+0000 7f83277fe640 1 -- 172.21.15.99:0/1274391368 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f832819f660 con 0x7f8328101930 2024-03-19T01:59:57.548 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.545+0000 7f832cbef640 1 --2- 172.21.15.99:0/1274391368 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f83281022f0 0x7f832819eff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:59:57.548 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.545+0000 7f83277fe640 1 --2- 172.21.15.99:0/1274391368 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8328101930 0x7f8328197f80 secure :-1 s=READY pgs=703 cs=0 l=1 rev1=1 crypto rx=0x7f831800e9f0 tx=0x7f831800eeb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:59:57.548 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.545+0000 7f83257fa640 1 -- 172.21.15.99:0/1274391368 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f831800cd80 con 0x7f8328101930 2024-03-19T01:59:57.548 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.545+0000 7f832e652640 1 -- 172.21.15.99:0/1274391368 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f832819f950 con 0x7f8328101930 2024-03-19T01:59:57.548 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.545+0000 7f83257fa640 1 -- 172.21.15.99:0/1274391368 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f8318010910 con 0x7f8328101930 2024-03-19T01:59:57.548 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.545+0000 7f83257fa640 1 -- 172.21.15.99:0/1274391368 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8318018990 con 0x7f8328101930 2024-03-19T01:59:57.548 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.546+0000 7f832e652640 1 -- 172.21.15.99:0/1274391368 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f832819fea0 con 0x7f8328101930 2024-03-19T01:59:57.554 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.548+0000 7f83257fa640 1 -- 172.21.15.99:0/1274391368 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f8318010ac0 con 0x7f8328101930 2024-03-19T01:59:57.554 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.548+0000 7f833123f740 1 Processor -- start 2024-03-19T01:59:57.554 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.548+0000 7f833123f740 1 -- start start 2024-03-19T01:59:57.554 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.548+0000 7f83257fa640 1 --2- 172.21.15.99:0/1274391368 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f83100a28a0 0x7f83100a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:57.554 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.548+0000 7f833123f740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e742158100 0x55e742158520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:57.554 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.548+0000 7f833123f740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e742158af0 0x55e74215d5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:57.554 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.548+0000 7f833123f740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e74215dc60 0x55e7421600f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:57.554 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.548+0000 7f833123f740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55e7420bcbe0 con 0x55e74215dc60 2024-03-19T01:59:57.554 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.548+0000 7f833123f740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55e742065960 con 0x55e742158100 2024-03-19T01:59:57.555 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.548+0000 7f833123f740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55e7420b8c80 con 0x55e742158af0 2024-03-19T01:59:57.555 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.549+0000 7f8327fff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e742158100 0x55e742158520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:57.555 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.549+0000 7f832cbef640 1 --2- 172.21.15.99:0/1274391368 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f83100a28a0 0x7f83100a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:57.555 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.549+0000 7f83257fa640 1 -- 172.21.15.99:0/1274391368 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f83180cedd0 con 0x7f8328101930 2024-03-19T01:59:57.555 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.549+0000 7f8327fff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e742158100 0x55e742158520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:42712/0 (socket says 172.21.15.99:42712) 2024-03-19T01:59:57.555 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.549+0000 7f8327fff640 1 -- 172.21.15.99:0/1414767966 learned_addr learned my addr 172.21.15.99:0/1414767966 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:59:57.555 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.549+0000 7f832cbef640 1 --2- 172.21.15.99:0/1414767966 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e74215dc60 0x55e7421600f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:57.555 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.549+0000 7f83277fe640 1 --2- 172.21.15.99:0/1414767966 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e742158af0 0x55e74215d5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:57.555 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.549+0000 7f8327fff640 1 -- 172.21.15.99:0/1414767966 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e742158af0 msgr2=0x55e74215d5f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:59:57.555 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.549+0000 7f8327fff640 1 --2- 172.21.15.99:0/1414767966 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e742158af0 0x55e74215d5f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:57.555 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.549+0000 7f8327fff640 1 -- 172.21.15.99:0/1414767966 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e74215dc60 msgr2=0x55e7421600f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:59:57.555 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.549+0000 7f8327fff640 1 --2- 172.21.15.99:0/1414767966 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e74215dc60 0x55e7421600f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:57.556 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.549+0000 7f8327fff640 1 -- 172.21.15.99:0/1414767966 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55e74205cd20 con 0x55e742158100 2024-03-19T01:59:57.556 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.549+0000 7f832cbef640 1 --2- 172.21.15.99:0/1274391368 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f83100a28a0 0x7f83100a4d60 secure :-1 s=READY pgs=357 cs=0 l=1 rev1=1 crypto rx=0x7f8320005e30 tx=0x7f832002e040 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:59:57.556 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.549+0000 7f8327fff640 1 --2- 172.21.15.99:0/1414767966 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e742158100 0x55e742158520 secure :-1 s=READY pgs=196 cs=0 l=1 rev1=1 crypto rx=0x7f831c009ca0 tx=0x7f831c011810 comp rx=0 tx=0).ready entity=mon.2 client_cookie=71bc495b39f8c3dc server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:59:57.556 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.549+0000 7f830e7fc640 1 -- 172.21.15.99:0/1414767966 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f831c004220 con 0x55e742158100 2024-03-19T01:59:57.556 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.549+0000 7f830e7fc640 1 -- 172.21.15.99:0/1414767966 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f831c0043c0 con 0x55e742158100 2024-03-19T01:59:57.556 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.549+0000 7f830e7fc640 1 -- 172.21.15.99:0/1414767966 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f831c019890 con 0x55e742158100 2024-03-19T01:59:57.556 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.550+0000 7f833123f740 1 -- 172.21.15.99:0/1414767966 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e742158100 msgr2=0x55e742158520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:59:57.556 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.550+0000 7f833123f740 1 --2- 172.21.15.99:0/1414767966 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e742158100 0x55e742158520 secure :-1 s=READY pgs=196 cs=0 l=1 rev1=1 crypto rx=0x7f831c009ca0 tx=0x7f831c011810 comp rx=0 tx=0).stop 2024-03-19T01:59:57.556 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.550+0000 7f833123f740 1 -- 172.21.15.99:0/1414767966 shutdown_connections 2024-03-19T01:59:57.556 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.550+0000 7f833123f740 1 --2- 172.21.15.99:0/1414767966 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e74215dc60 0x55e7421600f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:57.556 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.550+0000 7f833123f740 1 --2- 172.21.15.99:0/1414767966 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e742158af0 0x55e74215d5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:57.556 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.550+0000 7f833123f740 1 --2- 172.21.15.99:0/1414767966 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e742158100 0x55e742158520 unknown :-1 s=CLOSED pgs=196 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:57.556 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.550+0000 7f833123f740 1 -- 172.21.15.99:0/1414767966 >> 172.21.15.99:0/1414767966 conn(0x55e742153670 msgr2=0x55e742153a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:59:57.557 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.550+0000 7f833123f740 1 -- 172.21.15.99:0/1414767966 shutdown_connections 2024-03-19T01:59:57.557 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.550+0000 7f833123f740 1 -- 172.21.15.99:0/1414767966 wait complete. 2024-03-19T01:59:57.557 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.550+0000 7f833123f740 1 -- start start 2024-03-19T01:59:57.557 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.551+0000 7f833123f740 1 Processor -- start 2024-03-19T01:59:57.557 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.551+0000 7f833123f740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e742158100 0x55e74216a830 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:57.557 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.551+0000 7f833123f740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e742158af0 0x55e74216ad70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:57.557 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.551+0000 7f833123f740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e74215dc60 0x55e742171db0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:57.557 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.551+0000 7f833123f740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55e742160c10 con 0x55e74215dc60 2024-03-19T01:59:57.557 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.551+0000 7f833123f740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55e742078db0 con 0x55e742158af0 2024-03-19T01:59:57.557 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.551+0000 7f833123f740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55e742160d90 con 0x55e742158100 2024-03-19T01:59:57.557 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.551+0000 7f833123f740 10 client.0 _ll_register_callbacks cb 0x55e74207de70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:59:57.557 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.551+0000 7f832cbef640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e74215dc60 0x55e742171db0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:57.557 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.551+0000 7f832cbef640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e74215dc60 0x55e742171db0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:33668/0 (socket says 172.21.15.99:33668) 2024-03-19T01:59:57.557 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.551+0000 7f83277fe640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e742158af0 0x55e74216ad70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:57.558 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.551+0000 7f832cbef640 1 -- 172.21.15.99:0/2726082692 learned_addr learned my addr 172.21.15.99:0/2726082692 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:59:57.558 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.551+0000 7f8327fff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e742158100 0x55e74216a830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:57.558 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.551+0000 7f832cbef640 1 -- 172.21.15.99:0/2726082692 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e742158100 msgr2=0x55e74216a830 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:59:57.558 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.551+0000 7f832cbef640 1 --2- 172.21.15.99:0/2726082692 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e742158100 0x55e74216a830 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:57.558 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.551+0000 7f832cbef640 1 -- 172.21.15.99:0/2726082692 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e742158af0 msgr2=0x55e74216ad70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:59:57.558 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.551+0000 7f832cbef640 1 --2- 172.21.15.99:0/2726082692 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e742158af0 0x55e74216ad70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:57.558 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.551+0000 7f832cbef640 1 -- 172.21.15.99:0/2726082692 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55e742077c60 con 0x55e74215dc60 2024-03-19T01:59:57.558 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.551+0000 7f83277fe640 1 --2- 172.21.15.99:0/2726082692 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e742158af0 0x55e74216ad70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:59:57.558 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.552+0000 7f832cbef640 1 --2- 172.21.15.99:0/2726082692 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e74215dc60 0x55e742171db0 secure :-1 s=READY pgs=704 cs=0 l=1 rev1=1 crypto rx=0x7f8320036000 tx=0x7f832002c0f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c667239d210cfe62 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:59:57.558 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.552+0000 7f83077fe640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T01:59:57.558 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.552+0000 7f83077fe640 1 -- 172.21.15.99:0/2726082692 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8320053060 con 0x55e74215dc60 2024-03-19T01:59:57.558 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.552+0000 7f83077fe640 1 -- 172.21.15.99:0/2726082692 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f8320034a00 con 0x55e74215dc60 2024-03-19T01:59:57.558 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.552+0000 7f833123f740 1 -- 172.21.15.99:0/2726082692 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f831c0043c0 con 0x55e74215dc60 2024-03-19T01:59:57.559 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.552+0000 7f83077fe640 1 -- 172.21.15.99:0/2726082692 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7f8320039040 con 0x55e74215dc60 2024-03-19T01:59:57.559 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.552+0000 7f833123f740 10 client.16848 fetch_fsmap learned FSMap version 37 2024-03-19T01:59:57.559 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.552+0000 7f833123f740 1 -- 172.21.15.99:0/2726082692 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x55e741dd1210 con 0x55e74215dc60 2024-03-19T01:59:57.559 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.552+0000 7f83077fe640 1 -- 172.21.15.99:0/2726082692 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f831c0043c0 con 0x55e74215dc60 2024-03-19T01:59:57.559 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.552+0000 7f83077fe640 1 -- 172.21.15.99:0/2726082692 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7f832004e470 con 0x55e74215dc60 2024-03-19T01:59:57.559 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.552+0000 7f833123f740 10 client.16848 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T01:59:57.559 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.552+0000 7f833123f740 10 client.16848 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:59:57.559 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.552+0000 7f833123f740 20 client.16848 populate_metadata read hostname 'smithi099' 2024-03-19T01:59:57.559 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.552+0000 7f833123f740 1 --2- 172.21.15.99:0/2726082692 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e742176700 0x55e742178b30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:57.559 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.552+0000 7f833123f740 4 client.16848 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T01:59:57.559 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.552+0000 7f833123f740 1 -- 172.21.15.99:0/2726082692 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55e7420ca120 con 0x55e742176700 2024-03-19T01:59:57.559 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.552+0000 7f8327fff640 1 --2- 172.21.15.99:0/2726082692 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e742176700 0x55e742178b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:57.559 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.553+0000 7f83077fe640 10 client.16848 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:59:57.559 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.553+0000 7f8327fff640 1 --2- 172.21.15.99:0/2726082692 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e742176700 0x55e742178b30 crc :-1 s=READY pgs=197 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=79b11e484fb39f87 server_cookie=84acc33641d7633e in_seq=0 out_seq=0 2024-03-19T01:59:57.560 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:59:57 smithi115 ceph-mon[67332]: pgmap v917: 97 pgs: 97 active+clean; 599 MiB data, 7.4 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 6.5 MiB/s wr, 566 op/s 2024-03-19T01:59:57.561 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.555+0000 7f83077fe640 1 -- 172.21.15.99:0/2726082692 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x55e7420ca120 con 0x55e742176700 2024-03-19T01:59:57.561 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.555+0000 7f83077fe640 10 client.16848 handle_command_reply: tid=0 2024-03-19T01:59:57.561 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.555+0000 7f833123f740 1 client.16848 shutdown 2024-03-19T01:59:57.561 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.555+0000 7f833123f740 1 -- 172.21.15.99:0/2726082692 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e74215dc60 msgr2=0x55e742171db0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:59:57.561 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.555+0000 7f833123f740 1 --2- 172.21.15.99:0/2726082692 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e74215dc60 0x55e742171db0 secure :-1 s=READY pgs=704 cs=0 l=1 rev1=1 crypto rx=0x7f8320036000 tx=0x7f832002c0f0 comp rx=0 tx=0).stop 2024-03-19T01:59:57.561 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.555+0000 7f833123f740 1 -- 172.21.15.99:0/2726082692 shutdown_connections 2024-03-19T01:59:57.562 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.555+0000 7f833123f740 1 --2- 172.21.15.99:0/2726082692 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e742176700 0x55e742178b30 crc :-1 s=READY pgs=197 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:57.562 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.555+0000 7f833123f740 1 --2- 172.21.15.99:0/2726082692 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e742158af0 0x55e74216ad70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:57.562 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.555+0000 7f833123f740 1 --2- 172.21.15.99:0/2726082692 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e74215dc60 0x55e742171db0 unknown :-1 s=CLOSED pgs=704 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:57.562 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.555+0000 7f83077fe640 0 client.16848 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:59:57.562 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.555+0000 7f833123f740 1 --2- 172.21.15.99:0/2726082692 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e742158100 0x55e74216a830 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:57.562 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.555+0000 7f833123f740 1 -- 172.21.15.99:0/2726082692 >> 172.21.15.99:0/2726082692 conn(0x55e742153670 msgr2=0x55e7421548c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:59:57.562 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.555+0000 7f833123f740 1 -- 172.21.15.99:0/2726082692 shutdown_connections 2024-03-19T01:59:57.562 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.555+0000 7f833123f740 1 -- 172.21.15.99:0/2726082692 wait complete. 2024-03-19T01:59:57.562 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.555+0000 7f833123f740 20 client.16848 trim_cache size 0 max 16384 2024-03-19T01:59:57.573 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.572+0000 7f833123f740 1 Processor -- start 2024-03-19T01:59:57.573 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.572+0000 7f833123f740 1 -- start start 2024-03-19T01:59:57.573 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.572+0000 7f833123f740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e742158100 0x55e742152930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:57.573 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.572+0000 7f833123f740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e742158af0 0x55e742171d70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:57.573 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.572+0000 7f833123f740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e74215dc60 0x55e742159380 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:57.574 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.572+0000 7f833123f740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f83281a9420 con 0x55e74215dc60 2024-03-19T01:59:57.574 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.572+0000 7f833123f740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f83281a6cf0 con 0x55e742158af0 2024-03-19T01:59:57.574 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.572+0000 7f833123f740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f83281a4b40 con 0x55e742158100 2024-03-19T01:59:57.574 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.572+0000 7f832cbef640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e74215dc60 0x55e742159380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:57.574 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.572+0000 7f83277fe640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e742158af0 0x55e742171d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:57.574 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.572+0000 7f83277fe640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e742158af0 0x55e742171d70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:42730/0 (socket says 172.21.15.99:42730) 2024-03-19T01:59:57.574 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.572+0000 7f832cbef640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e74215dc60 0x55e742159380 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:33682/0 (socket says 172.21.15.99:33682) 2024-03-19T01:59:57.574 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.572+0000 7f83277fe640 1 -- 172.21.15.99:0/516446244 learned_addr learned my addr 172.21.15.99:0/516446244 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:59:57.574 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.572+0000 7f8327fff640 1 --2- 172.21.15.99:0/516446244 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e742158100 0x55e742152930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:57.574 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.572+0000 7f83277fe640 1 -- 172.21.15.99:0/516446244 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e742158100 msgr2=0x55e742152930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:59:57.574 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.572+0000 7f83277fe640 1 --2- 172.21.15.99:0/516446244 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e742158100 0x55e742152930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:57.574 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.572+0000 7f83277fe640 1 -- 172.21.15.99:0/516446244 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e74215dc60 msgr2=0x55e742159380 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:59:57.574 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.572+0000 7f83277fe640 1 --2- 172.21.15.99:0/516446244 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e74215dc60 0x55e742159380 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:57.575 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.572+0000 7f83277fe640 1 -- 172.21.15.99:0/516446244 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55e741e9d620 con 0x55e742158af0 2024-03-19T01:59:57.575 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.572+0000 7f832cbef640 1 --2- 172.21.15.99:0/516446244 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e74215dc60 0x55e742159380 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T01:59:57.575 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.573+0000 7f83277fe640 1 --2- 172.21.15.99:0/516446244 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e742158af0 0x55e742171d70 secure :-1 s=READY pgs=197 cs=0 l=1 rev1=1 crypto rx=0x7f831800cfd0 tx=0x7f83180973e0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=3b67f12ddc770188 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:59:57.575 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.573+0000 7f830dffb640 1 -- 172.21.15.99:0/516446244 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8318099880 con 0x55e742158af0 2024-03-19T01:59:57.575 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.573+0000 7f830dffb640 1 -- 172.21.15.99:0/516446244 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f8318099ae0 con 0x55e742158af0 2024-03-19T01:59:57.575 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.573+0000 7f830dffb640 1 -- 172.21.15.99:0/516446244 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8318018d90 con 0x55e742158af0 2024-03-19T01:59:57.575 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.573+0000 7f833123f740 1 -- 172.21.15.99:0/516446244 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e742158af0 msgr2=0x55e742171d70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:59:57.575 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.573+0000 7f833123f740 1 --2- 172.21.15.99:0/516446244 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e742158af0 0x55e742171d70 secure :-1 s=READY pgs=197 cs=0 l=1 rev1=1 crypto rx=0x7f831800cfd0 tx=0x7f83180973e0 comp rx=0 tx=0).stop 2024-03-19T01:59:57.575 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.573+0000 7f833123f740 1 -- 172.21.15.99:0/516446244 shutdown_connections 2024-03-19T01:59:57.575 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.573+0000 7f833123f740 1 --2- 172.21.15.99:0/516446244 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e742158af0 0x55e742171d70 unknown :-1 s=CLOSED pgs=197 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:57.575 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.573+0000 7f833123f740 1 --2- 172.21.15.99:0/516446244 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e74215dc60 0x55e742159380 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:57.575 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.573+0000 7f833123f740 1 --2- 172.21.15.99:0/516446244 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e742158100 0x55e742152930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:57.575 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.573+0000 7f833123f740 1 -- 172.21.15.99:0/516446244 >> 172.21.15.99:0/516446244 conn(0x55e742153670 msgr2=0x55e742152220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:59:57.576 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.573+0000 7f833123f740 1 -- 172.21.15.99:0/516446244 shutdown_connections 2024-03-19T01:59:57.576 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.574+0000 7f833123f740 1 -- 172.21.15.99:0/516446244 wait complete. 2024-03-19T01:59:57.576 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.574+0000 7f833123f740 1 -- start start 2024-03-19T01:59:57.578 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.574+0000 7f833123f740 1 Processor -- start 2024-03-19T01:59:57.579 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.574+0000 7f833123f740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e742158100 0x55e74216f8f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:57.579 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.574+0000 7f833123f740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e742158af0 0x55e74216fe30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:57.579 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.574+0000 7f833123f740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e74215dc60 0x55e742170e40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:57.579 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.574+0000 7f833123f740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f83281a51b0 con 0x55e74215dc60 2024-03-19T01:59:57.579 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.574+0000 7f833123f740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f83281a6620 con 0x55e742158100 2024-03-19T01:59:57.579 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.574+0000 7f833123f740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f83281a5880 con 0x55e742158af0 2024-03-19T01:59:57.579 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.574+0000 7f833123f740 10 client.0 _ll_register_callbacks cb 0x7f831c005e30 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T01:59:57.579 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.574+0000 7f8327fff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e742158100 0x55e74216f8f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:57.579 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.574+0000 7f832cbef640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e74215dc60 0x55e742170e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:57.579 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.574+0000 7f83277fe640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e742158af0 0x55e74216fe30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:57.579 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.574+0000 7f8327fff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e742158100 0x55e74216f8f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:42734/0 (socket says 172.21.15.99:42734) 2024-03-19T01:59:57.579 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.574+0000 7f8327fff640 1 -- 172.21.15.99:0/81564715 learned_addr learned my addr 172.21.15.99:0/81564715 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T01:59:57.579 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.574+0000 7f832cbef640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e74215dc60 0x55e742170e40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:33688/0 (socket says 172.21.15.99:33688) 2024-03-19T01:59:57.580 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.575+0000 7f8327fff640 1 -- 172.21.15.99:0/81564715 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e742158af0 msgr2=0x55e74216fe30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:59:57.580 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.575+0000 7f8327fff640 1 --2- 172.21.15.99:0/81564715 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e742158af0 0x55e74216fe30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:57.580 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.575+0000 7f8327fff640 1 -- 172.21.15.99:0/81564715 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e74215dc60 msgr2=0x55e742170e40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T01:59:57.580 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.575+0000 7f8327fff640 1 --2- 172.21.15.99:0/81564715 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e74215dc60 0x55e742170e40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:57.580 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.575+0000 7f8327fff640 1 -- 172.21.15.99:0/81564715 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55e742171430 con 0x55e742158100 2024-03-19T01:59:57.580 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.575+0000 7f8327fff640 1 --2- 172.21.15.99:0/81564715 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e742158100 0x55e74216f8f0 secure :-1 s=READY pgs=198 cs=0 l=1 rev1=1 crypto rx=0x7f831c004150 tx=0x7f831c0097e0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=63c5e6271bfc2adc server_cookie=0 in_seq=0 out_seq=0 2024-03-19T01:59:57.580 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.575+0000 7f83077fe640 10 client.0 ms_handle_connect on v2:172.21.15.112:3300/0 2024-03-19T01:59:57.580 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.575+0000 7f83077fe640 1 -- 172.21.15.99:0/81564715 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f831c005500 con 0x55e742158100 2024-03-19T01:59:57.580 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.575+0000 7f833123f740 1 -- 172.21.15.99:0/81564715 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f8318099ae0 con 0x55e742158100 2024-03-19T01:59:57.580 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.575+0000 7f83077fe640 1 -- 172.21.15.99:0/81564715 <== mon.2 v2:172.21.15.112:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f831c02ba30 con 0x55e742158100 2024-03-19T01:59:57.580 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.575+0000 7f83077fe640 1 -- 172.21.15.99:0/81564715 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7f831c03b050 con 0x55e742158100 2024-03-19T01:59:57.580 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.575+0000 7f833123f740 10 client.26630 fetch_fsmap learned FSMap version 37 2024-03-19T01:59:57.580 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.575+0000 7f833123f740 1 -- 172.21.15.99:0/81564715 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x55e7421dcf20 con 0x55e742158100 2024-03-19T01:59:57.581 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.575+0000 7f83077fe640 1 -- 172.21.15.99:0/81564715 <== mon.2 v2:172.21.15.112:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f8318099ae0 con 0x55e742158100 2024-03-19T01:59:57.581 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.576+0000 7f83077fe640 1 -- 172.21.15.99:0/81564715 <== mon.2 v2:172.21.15.112:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7f831c003710 con 0x55e742158100 2024-03-19T01:59:57.581 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.576+0000 7f833123f740 10 client.26630 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T01:59:57.581 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.576+0000 7f833123f740 10 client.26630 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T01:59:57.581 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.576+0000 7f833123f740 20 client.26630 populate_metadata read hostname 'smithi099' 2024-03-19T01:59:57.581 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.576+0000 7f833123f740 1 --2- 172.21.15.99:0/81564715 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e7421dd6d0 0x55e7421dfaf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T01:59:57.581 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.576+0000 7f833123f740 4 client.26630 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T01:59:57.581 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.576+0000 7f833123f740 1 -- 172.21.15.99:0/81564715 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55e7420c4470 con 0x55e7421dd6d0 2024-03-19T01:59:57.581 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.576+0000 7f83277fe640 1 --2- 172.21.15.99:0/81564715 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e7421dd6d0 0x55e7421dfaf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T01:59:57.581 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.576+0000 7f83077fe640 10 client.26630 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T01:59:57.581 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:57.576+0000 7f83277fe640 1 --2- 172.21.15.99:0/81564715 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e7421dd6d0 0x55e7421dfaf0 crc :-1 s=READY pgs=198 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=89fb1641defa8e28 server_cookie=8bf6924d22260c5e in_seq=0 out_seq=0 2024-03-19T01:59:57.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:59:57 smithi099 ceph-mon[62951]: pgmap v917: 97 pgs: 97 active+clean; 599 MiB data, 7.4 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 6.5 MiB/s wr, 566 op/s 2024-03-19T01:59:58.106 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:58.105+0000 7f83077fe640 1 -- 172.21.15.99:0/81564715 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x55e7420c4470 con 0x55e7421dd6d0 2024-03-19T01:59:58.106 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:58.105+0000 7f83077fe640 10 client.26630 handle_command_reply: tid=0 2024-03-19T01:59:58.107 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:58.105+0000 7f833123f740 1 client.26630 shutdown 2024-03-19T01:59:58.107 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:58.105+0000 7f833123f740 1 -- 172.21.15.99:0/81564715 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e742158100 msgr2=0x55e74216f8f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:59:58.107 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:58.105+0000 7f833123f740 1 --2- 172.21.15.99:0/81564715 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e742158100 0x55e74216f8f0 secure :-1 s=READY pgs=198 cs=0 l=1 rev1=1 crypto rx=0x7f831c004150 tx=0x7f831c0097e0 comp rx=0 tx=0).stop 2024-03-19T01:59:58.107 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:58.105+0000 7f833123f740 1 -- 172.21.15.99:0/81564715 shutdown_connections 2024-03-19T01:59:58.107 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:58.105+0000 7f833123f740 1 --2- 172.21.15.99:0/81564715 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e7421dd6d0 0x55e7421dfaf0 crc :-1 s=READY pgs=198 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:58.107 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:58.105+0000 7f833123f740 1 --2- 172.21.15.99:0/81564715 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e74215dc60 0x55e742170e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:58.107 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:58.105+0000 7f83077fe640 0 client.26630 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T01:59:58.107 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:58.105+0000 7f833123f740 1 --2- 172.21.15.99:0/81564715 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e742158af0 0x55e74216fe30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:58.107 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:58.105+0000 7f833123f740 1 --2- 172.21.15.99:0/81564715 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e742158100 0x55e74216f8f0 unknown :-1 s=CLOSED pgs=198 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:58.107 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:58.105+0000 7f833123f740 1 -- 172.21.15.99:0/81564715 >> 172.21.15.99:0/81564715 conn(0x55e742153670 msgr2=0x55e74215d360 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:59:58.107 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:58.105+0000 7f833123f740 1 -- 172.21.15.99:0/81564715 shutdown_connections 2024-03-19T01:59:58.107 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:58.105+0000 7f833123f740 1 -- 172.21.15.99:0/81564715 wait complete. 2024-03-19T01:59:58.107 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:58.105+0000 7f833123f740 20 client.26630 trim_cache size 0 max 16384 2024-03-19T01:59:58.108 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T01:59:58.108 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T01:59:58.108 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T01:59:58.108 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T01:59:58.109 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:58.106+0000 7f832e652640 1 -- 172.21.15.99:0/1274391368 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f83100a28a0 msgr2=0x7f83100a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:59:58.109 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:58.106+0000 7f832e652640 1 --2- 172.21.15.99:0/1274391368 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f83100a28a0 0x7f83100a4d60 secure :-1 s=READY pgs=357 cs=0 l=1 rev1=1 crypto rx=0x7f8320005e30 tx=0x7f832002e040 comp rx=0 tx=0).stop 2024-03-19T01:59:58.109 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:58.106+0000 7f832e652640 1 -- 172.21.15.99:0/1274391368 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8328101930 msgr2=0x7f8328197f80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T01:59:58.109 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:58.106+0000 7f832e652640 1 --2- 172.21.15.99:0/1274391368 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8328101930 0x7f8328197f80 secure :-1 s=READY pgs=703 cs=0 l=1 rev1=1 crypto rx=0x7f831800e9f0 tx=0x7f831800eeb0 comp rx=0 tx=0).stop 2024-03-19T01:59:58.109 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:58.106+0000 7f832e652640 1 -- 172.21.15.99:0/1274391368 shutdown_connections 2024-03-19T01:59:58.109 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:58.106+0000 7f832e652640 1 --2- 172.21.15.99:0/1274391368 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f83100a28a0 0x7f83100a4d60 unknown :-1 s=CLOSED pgs=357 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:58.109 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:58.106+0000 7f832e652640 1 --2- 172.21.15.99:0/1274391368 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f83281022f0 0x7f832819eff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:58.109 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:58.106+0000 7f832e652640 1 --2- 172.21.15.99:0/1274391368 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8328101930 0x7f8328197f80 unknown :-1 s=CLOSED pgs=703 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:58.109 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:58.106+0000 7f832e652640 1 --2- 172.21.15.99:0/1274391368 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8328100740 0x7f8328197a40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T01:59:58.109 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:58.106+0000 7f832e652640 1 -- 172.21.15.99:0/1274391368 >> 172.21.15.99:0/1274391368 conn(0x7f83280fbcb0 msgr2=0x7f8328104bd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T01:59:58.109 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:58.106+0000 7f832e652640 1 -- 172.21.15.99:0/1274391368 shutdown_connections 2024-03-19T01:59:58.110 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T01:59:58.106+0000 7f832e652640 1 -- 172.21.15.99:0/1274391368 wait complete. 2024-03-19T01:59:58.110 INFO:tasks.workunit.client.0.smithi099.stdout: 10 167912 22.91 MB/sec execute 267 sec latency 333.673 ms 2024-03-19T01:59:59.111 INFO:tasks.workunit.client.0.smithi099.stdout: 10 168273 22.85 MB/sec execute 268 sec latency 341.631 ms 2024-03-19T01:59:59.357 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 01:59:59 smithi115 ceph-mon[67332]: pgmap v918: 97 pgs: 97 active+clean; 600 MiB data, 7.4 GiB used, 1.0 TiB / 1.0 TiB avail; 3.4 MiB/s rd, 5.5 MiB/s wr, 396 op/s 2024-03-19T01:59:59.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 01:59:59 smithi112 ceph-mon[66819]: pgmap v918: 97 pgs: 97 active+clean; 600 MiB data, 7.4 GiB used, 1.0 TiB / 1.0 TiB avail; 3.4 MiB/s rd, 5.5 MiB/s wr, 396 op/s 2024-03-19T01:59:59.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 01:59:59 smithi099 ceph-mon[62951]: pgmap v918: 97 pgs: 97 active+clean; 600 MiB data, 7.4 GiB used, 1.0 TiB / 1.0 TiB avail; 3.4 MiB/s rd, 5.5 MiB/s wr, 396 op/s 2024-03-19T02:00:00.111 INFO:tasks.workunit.client.0.smithi099.stdout: 10 168705 22.85 MB/sec execute 269 sec latency 82.742 ms 2024-03-19T02:00:00.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:00:00 smithi112 ceph-mon[66819]: overall HEALTH_OK 2024-03-19T02:00:00.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:00:00 smithi115 ceph-mon[67332]: overall HEALTH_OK 2024-03-19T02:00:00.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:00:00 smithi099 ceph-mon[62951]: overall HEALTH_OK 2024-03-19T02:00:01.111 INFO:tasks.workunit.client.0.smithi099.stdout: 10 169119 22.87 MB/sec execute 270 sec latency 110.600 ms 2024-03-19T02:00:01.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:00:01 smithi115 ceph-mon[67332]: pgmap v919: 97 pgs: 97 active+clean; 596 MiB data, 7.4 GiB used, 1.0 TiB / 1.0 TiB avail; 4.3 MiB/s rd, 6.2 MiB/s wr, 498 op/s 2024-03-19T02:00:01.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:00:01 smithi099 ceph-mon[62951]: pgmap v919: 97 pgs: 97 active+clean; 596 MiB data, 7.4 GiB used, 1.0 TiB / 1.0 TiB avail; 4.3 MiB/s rd, 6.2 MiB/s wr, 498 op/s 2024-03-19T02:00:01.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:00:01 smithi112 ceph-mon[66819]: pgmap v919: 97 pgs: 97 active+clean; 596 MiB data, 7.4 GiB used, 1.0 TiB / 1.0 TiB avail; 4.3 MiB/s rd, 6.2 MiB/s wr, 498 op/s 2024-03-19T02:00:02.111 INFO:tasks.workunit.client.0.smithi099.stdout: 10 169406 22.83 MB/sec execute 271 sec latency 321.709 ms 2024-03-19T02:00:03.111 INFO:tasks.workunit.client.0.smithi099.stdout: 10 169871 22.85 MB/sec execute 272 sec latency 399.712 ms 2024-03-19T02:00:03.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:00:03 smithi112 ceph-mon[66819]: pgmap v920: 97 pgs: 97 active+clean; 594 MiB data, 7.4 GiB used, 1.0 TiB / 1.0 TiB avail; 4.7 MiB/s rd, 6.5 MiB/s wr, 546 op/s 2024-03-19T02:00:03.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:00:03 smithi115 ceph-mon[67332]: pgmap v920: 97 pgs: 97 active+clean; 594 MiB data, 7.4 GiB used, 1.0 TiB / 1.0 TiB avail; 4.7 MiB/s rd, 6.5 MiB/s wr, 546 op/s 2024-03-19T02:00:03.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:00:03 smithi099 ceph-mon[62951]: pgmap v920: 97 pgs: 97 active+clean; 594 MiB data, 7.4 GiB used, 1.0 TiB / 1.0 TiB avail; 4.7 MiB/s rd, 6.5 MiB/s wr, 546 op/s 2024-03-19T02:00:04.111 INFO:tasks.workunit.client.0.smithi099.stdout: 10 170433 22.85 MB/sec execute 273 sec latency 116.689 ms 2024-03-19T02:00:05.111 INFO:tasks.workunit.client.0.smithi099.stdout: 10 170967 22.88 MB/sec execute 274 sec latency 61.010 ms 2024-03-19T02:00:05.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:00:05 smithi112 ceph-mon[66819]: pgmap v921: 97 pgs: 97 active+clean; 594 MiB data, 7.4 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 5.5 MiB/s wr, 488 op/s 2024-03-19T02:00:05.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:00:05 smithi115 ceph-mon[67332]: pgmap v921: 97 pgs: 97 active+clean; 594 MiB data, 7.4 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 5.5 MiB/s wr, 488 op/s 2024-03-19T02:00:05.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:00:05 smithi099 ceph-mon[62951]: pgmap v921: 97 pgs: 97 active+clean; 594 MiB data, 7.4 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 5.5 MiB/s wr, 488 op/s 2024-03-19T02:00:06.111 INFO:tasks.workunit.client.0.smithi099.stdout: 10 171429 22.91 MB/sec execute 275 sec latency 93.061 ms 2024-03-19T02:00:07.111 INFO:tasks.workunit.client.0.smithi099.stdout: 10 171797 22.90 MB/sec execute 276 sec latency 245.287 ms 2024-03-19T02:00:07.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:00:07 smithi112 ceph-mon[66819]: pgmap v922: 97 pgs: 97 active+clean; 603 MiB data, 7.4 GiB used, 1.0 TiB / 1.0 TiB avail; 5.4 MiB/s rd, 7.0 MiB/s wr, 614 op/s 2024-03-19T02:00:07.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:00:07 smithi115 ceph-mon[67332]: pgmap v922: 97 pgs: 97 active+clean; 603 MiB data, 7.4 GiB used, 1.0 TiB / 1.0 TiB avail; 5.4 MiB/s rd, 7.0 MiB/s wr, 614 op/s 2024-03-19T02:00:07.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:00:07 smithi099 ceph-mon[62951]: pgmap v922: 97 pgs: 97 active+clean; 603 MiB data, 7.4 GiB used, 1.0 TiB / 1.0 TiB avail; 5.4 MiB/s rd, 7.0 MiB/s wr, 614 op/s 2024-03-19T02:00:08.111 INFO:tasks.workunit.client.0.smithi099.stdout: 10 172154 22.90 MB/sec execute 277 sec latency 253.471 ms 2024-03-19T02:00:09.111 INFO:tasks.workunit.client.0.smithi099.stdout: 10 172530 22.90 MB/sec execute 278 sec latency 114.503 ms 2024-03-19T02:00:09.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:00:09 smithi112 ceph-mon[66819]: pgmap v923: 97 pgs: 97 active+clean; 600 MiB data, 7.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.5 MiB/s rd, 5.5 MiB/s wr, 506 op/s 2024-03-19T02:00:09.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:00:09 smithi115 ceph-mon[67332]: pgmap v923: 97 pgs: 97 active+clean; 600 MiB data, 7.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.5 MiB/s rd, 5.5 MiB/s wr, 506 op/s 2024-03-19T02:00:09.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:00:09 smithi099 ceph-mon[62951]: pgmap v923: 97 pgs: 97 active+clean; 600 MiB data, 7.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.5 MiB/s rd, 5.5 MiB/s wr, 506 op/s 2024-03-19T02:00:10.111 INFO:tasks.workunit.client.0.smithi099.stdout: 10 173000 22.89 MB/sec execute 279 sec latency 86.187 ms 2024-03-19T02:00:11.111 INFO:tasks.workunit.client.0.smithi099.stdout: 10 173326 22.85 MB/sec execute 280 sec latency 103.786 ms 2024-03-19T02:00:11.205 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.203+0000 7fa3628f8640 1 -- 172.21.15.99:0/1776256245 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa35c102a70 msgr2=0x7fa35c104ee0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:00:11.205 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.203+0000 7fa3628f8640 1 --2- 172.21.15.99:0/1776256245 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa35c102a70 0x7fa35c104ee0 secure :-1 s=READY pgs=705 cs=0 l=1 rev1=1 crypto rx=0x7fa350009900 tx=0x7fa35002bc30 comp rx=0 tx=0).stop 2024-03-19T02:00:11.205 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.203+0000 7fa3628f8640 1 -- 172.21.15.99:0/1776256245 shutdown_connections 2024-03-19T02:00:11.206 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.203+0000 7fa3628f8640 1 --2- 172.21.15.99:0/1776256245 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa35c105420 0x7fa35c1058a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:11.206 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.203+0000 7fa3628f8640 1 --2- 172.21.15.99:0/1776256245 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa35c102a70 0x7fa35c104ee0 unknown :-1 s=CLOSED pgs=705 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:11.206 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.203+0000 7fa3628f8640 1 --2- 172.21.15.99:0/1776256245 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa35c100100 0x7fa35c102530 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:11.206 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.203+0000 7fa3628f8640 1 -- 172.21.15.99:0/1776256245 >> 172.21.15.99:0/1776256245 conn(0x7fa35c0fbd00 msgr2=0x7fa35c0fe140 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:00:11.206 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.203+0000 7fa3628f8640 1 -- 172.21.15.99:0/1776256245 shutdown_connections 2024-03-19T02:00:11.206 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.204+0000 7fa3628f8640 1 -- 172.21.15.99:0/1776256245 wait complete. 2024-03-19T02:00:11.206 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.204+0000 7fa3628f8640 1 Processor -- start 2024-03-19T02:00:11.206 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.204+0000 7fa3628f8640 1 -- start start 2024-03-19T02:00:11.206 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.204+0000 7fa3628f8640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa35c100100 0x7fa35c19e070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:11.206 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.204+0000 7fa3628f8640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa35c102a70 0x7fa35c19e5b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:11.206 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.204+0000 7fa3628f8640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa35c105420 0x7fa35c1a5620 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:11.206 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.204+0000 7fa3628f8640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fa35c113af0 con 0x7fa35c102a70 2024-03-19T02:00:11.206 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.204+0000 7fa3628f8640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fa35c113970 con 0x7fa35c100100 2024-03-19T02:00:11.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.204+0000 7fa3628f8640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fa35c113c70 con 0x7fa35c105420 2024-03-19T02:00:11.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.204+0000 7fa35b7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa35c102a70 0x7fa35c19e5b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:11.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.205+0000 7fa35bfff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa35c100100 0x7fa35c19e070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:11.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.205+0000 7fa35b7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa35c102a70 0x7fa35c19e5b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:52280/0 (socket says 172.21.15.99:52280) 2024-03-19T02:00:11.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.205+0000 7fa35b7fe640 1 -- 172.21.15.99:0/3229207631 learned_addr learned my addr 172.21.15.99:0/3229207631 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:00:11.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.205+0000 7fa360e95640 1 --2- 172.21.15.99:0/3229207631 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa35c105420 0x7fa35c1a5620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:11.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.205+0000 7fa35bfff640 1 -- 172.21.15.99:0/3229207631 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa35c105420 msgr2=0x7fa35c1a5620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:00:11.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.205+0000 7fa35bfff640 1 --2- 172.21.15.99:0/3229207631 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa35c105420 0x7fa35c1a5620 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:11.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.205+0000 7fa35bfff640 1 -- 172.21.15.99:0/3229207631 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa35c102a70 msgr2=0x7fa35c19e5b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:00:11.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.205+0000 7fa35bfff640 1 --2- 172.21.15.99:0/3229207631 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa35c102a70 0x7fa35c19e5b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:11.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.205+0000 7fa35bfff640 1 -- 172.21.15.99:0/3229207631 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa35c1a5c90 con 0x7fa35c100100 2024-03-19T02:00:11.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.205+0000 7fa35bfff640 1 --2- 172.21.15.99:0/3229207631 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa35c100100 0x7fa35c19e070 secure :-1 s=READY pgs=199 cs=0 l=1 rev1=1 crypto rx=0x7fa34c00e9f0 tx=0x7fa34c00eeb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:00:11.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.205+0000 7fa3597fa640 1 -- 172.21.15.99:0/3229207631 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa34c00ccc0 con 0x7fa35c100100 2024-03-19T02:00:11.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.205+0000 7fa3628f8640 1 -- 172.21.15.99:0/3229207631 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa35c1a5f80 con 0x7fa35c100100 2024-03-19T02:00:11.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.205+0000 7fa3597fa640 1 -- 172.21.15.99:0/3229207631 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa34c010910 con 0x7fa35c100100 2024-03-19T02:00:11.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.205+0000 7fa3597fa640 1 -- 172.21.15.99:0/3229207631 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa34c018960 con 0x7fa35c100100 2024-03-19T02:00:11.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.205+0000 7fa3628f8640 1 -- 172.21.15.99:0/3229207631 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fa35c1a63f0 con 0x7fa35c100100 2024-03-19T02:00:11.213 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.208+0000 7fa3655c2740 1 Processor -- start 2024-03-19T02:00:11.214 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.208+0000 7fa3597fa640 1 -- 172.21.15.99:0/3229207631 <== mon.2 v2:172.21.15.112:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7fa34c010ac0 con 0x7fa35c100100 2024-03-19T02:00:11.214 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.208+0000 7fa3655c2740 1 -- start start 2024-03-19T02:00:11.214 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.208+0000 7fa3655c2740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e06ae70100 0x55e06ae70520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:11.214 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.208+0000 7fa3655c2740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e06ae70af0 0x55e06ae755f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:11.214 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.208+0000 7fa3655c2740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e06ae75c60 0x55e06ae780f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:11.214 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.208+0000 7fa3655c2740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55e06add4be0 con 0x55e06ae70af0 2024-03-19T02:00:11.214 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.208+0000 7fa3655c2740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55e06ad7d960 con 0x55e06ae70100 2024-03-19T02:00:11.214 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.208+0000 7fa3655c2740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55e06add0c80 con 0x55e06ae75c60 2024-03-19T02:00:11.214 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.208+0000 7fa35bfff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e06ae70af0 0x55e06ae755f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:11.214 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.208+0000 7fa35bfff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e06ae70af0 0x55e06ae755f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:52296/0 (socket says 172.21.15.99:52296) 2024-03-19T02:00:11.214 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.208+0000 7fa35bfff640 1 -- 172.21.15.99:0/246172732 learned_addr learned my addr 172.21.15.99:0/246172732 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:00:11.214 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.208+0000 7fa360e95640 1 --2- 172.21.15.99:0/246172732 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e06ae70100 0x55e06ae70520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:11.215 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.208+0000 7fa35b7fe640 1 --2- 172.21.15.99:0/246172732 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e06ae75c60 0x55e06ae780f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:11.215 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.208+0000 7fa35bfff640 1 -- 172.21.15.99:0/246172732 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e06ae75c60 msgr2=0x55e06ae780f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:00:11.215 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.208+0000 7fa35bfff640 1 --2- 172.21.15.99:0/246172732 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e06ae75c60 0x55e06ae780f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:11.215 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.208+0000 7fa35bfff640 1 -- 172.21.15.99:0/246172732 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e06ae70100 msgr2=0x55e06ae70520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:00:11.215 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.208+0000 7fa35bfff640 1 --2- 172.21.15.99:0/246172732 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e06ae70100 0x55e06ae70520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:11.215 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.208+0000 7fa35bfff640 1 -- 172.21.15.99:0/246172732 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55e06ad74d20 con 0x55e06ae70af0 2024-03-19T02:00:11.215 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.208+0000 7fa3597fa640 1 --2- 172.21.15.99:0/3229207631 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa3440a28a0 0x7fa3440a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:11.215 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.208+0000 7fa35bfff640 1 --2- 172.21.15.99:0/246172732 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e06ae70af0 0x55e06ae755f0 secure :-1 s=READY pgs=706 cs=0 l=1 rev1=1 crypto rx=0x7fa34c0d2f60 tx=0x7fa34c0d2f90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=96990f6f5b26eac0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:00:11.216 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.208+0000 7fa360e95640 1 --2- 172.21.15.99:0/3229207631 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa3440a28a0 0x7fa3440a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:11.216 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.209+0000 7fa3597fa640 1 -- 172.21.15.99:0/3229207631 <== mon.2 v2:172.21.15.112:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fa34c0cee50 con 0x7fa35c100100 2024-03-19T02:00:11.216 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.209+0000 7fa3427fc640 1 -- 172.21.15.99:0/246172732 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa34c097b00 con 0x55e06ae70af0 2024-03-19T02:00:11.216 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.209+0000 7fa3427fc640 1 -- 172.21.15.99:0/246172732 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa34c09b860 con 0x55e06ae70af0 2024-03-19T02:00:11.216 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.209+0000 7fa3427fc640 1 -- 172.21.15.99:0/246172732 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa34c09b1c0 con 0x55e06ae70af0 2024-03-19T02:00:11.216 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.209+0000 7fa3655c2740 1 -- 172.21.15.99:0/246172732 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e06ae70af0 msgr2=0x55e06ae755f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:00:11.216 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.209+0000 7fa360e95640 1 --2- 172.21.15.99:0/3229207631 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa3440a28a0 0x7fa3440a4d60 secure :-1 s=READY pgs=358 cs=0 l=1 rev1=1 crypto rx=0x55e06ae71350 tx=0x7fa354009500 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:00:11.216 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.209+0000 7fa3655c2740 1 --2- 172.21.15.99:0/246172732 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e06ae70af0 0x55e06ae755f0 secure :-1 s=READY pgs=706 cs=0 l=1 rev1=1 crypto rx=0x7fa34c0d2f60 tx=0x7fa34c0d2f90 comp rx=0 tx=0).stop 2024-03-19T02:00:11.216 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.209+0000 7fa3655c2740 1 -- 172.21.15.99:0/246172732 shutdown_connections 2024-03-19T02:00:11.216 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.209+0000 7fa3655c2740 1 --2- 172.21.15.99:0/246172732 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e06ae75c60 0x55e06ae780f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:11.216 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.209+0000 7fa3655c2740 1 --2- 172.21.15.99:0/246172732 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e06ae70af0 0x55e06ae755f0 unknown :-1 s=CLOSED pgs=706 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:11.217 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.209+0000 7fa3655c2740 1 --2- 172.21.15.99:0/246172732 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e06ae70100 0x55e06ae70520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:11.217 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.209+0000 7fa3655c2740 1 -- 172.21.15.99:0/246172732 >> 172.21.15.99:0/246172732 conn(0x55e06ae6b670 msgr2=0x55e06ae6ba70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:00:11.217 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.210+0000 7fa3655c2740 1 -- 172.21.15.99:0/246172732 shutdown_connections 2024-03-19T02:00:11.217 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.210+0000 7fa3655c2740 1 -- 172.21.15.99:0/246172732 wait complete. 2024-03-19T02:00:11.217 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.210+0000 7fa3655c2740 1 -- start start 2024-03-19T02:00:11.217 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.210+0000 7fa3655c2740 1 Processor -- start 2024-03-19T02:00:11.217 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.210+0000 7fa3655c2740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e06ae70100 0x55e06ae7a0d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:11.217 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.210+0000 7fa3655c2740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e06ae70af0 0x55e06ae7a610 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:11.217 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.210+0000 7fa3655c2740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e06ae75c60 0x55e06ae81650 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:11.217 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.210+0000 7fa3655c2740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55e06ae78c10 con 0x55e06ae70100 2024-03-19T02:00:11.217 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.210+0000 7fa3655c2740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55e06ad90db0 con 0x55e06ae75c60 2024-03-19T02:00:11.217 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.210+0000 7fa3655c2740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55e06ae78d90 con 0x55e06ae70af0 2024-03-19T02:00:11.217 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.210+0000 7fa3655c2740 10 client.0 _ll_register_callbacks cb 0x55e06ad95e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T02:00:11.217 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.210+0000 7fa35bfff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e06ae70100 0x55e06ae7a0d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:11.218 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.210+0000 7fa35bfff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e06ae70100 0x55e06ae7a0d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:52308/0 (socket says 172.21.15.99:52308) 2024-03-19T02:00:11.218 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.210+0000 7fa35bfff640 1 -- 172.21.15.99:0/2485160176 learned_addr learned my addr 172.21.15.99:0/2485160176 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:00:11.218 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.210+0000 7fa360e95640 1 --2- 172.21.15.99:0/2485160176 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e06ae75c60 0x55e06ae81650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:11.218 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.210+0000 7fa35b7fe640 1 --2- 172.21.15.99:0/2485160176 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e06ae70af0 0x55e06ae7a610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:11.218 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.211+0000 7fa35bfff640 1 -- 172.21.15.99:0/2485160176 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e06ae70af0 msgr2=0x55e06ae7a610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:00:11.218 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.211+0000 7fa35bfff640 1 --2- 172.21.15.99:0/2485160176 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e06ae70af0 0x55e06ae7a610 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:11.219 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.211+0000 7fa35bfff640 1 -- 172.21.15.99:0/2485160176 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e06ae75c60 msgr2=0x55e06ae81650 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:00:11.219 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.211+0000 7fa35bfff640 1 --2- 172.21.15.99:0/2485160176 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e06ae75c60 0x55e06ae81650 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:11.219 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.211+0000 7fa35bfff640 1 -- 172.21.15.99:0/2485160176 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55e06ad8fc60 con 0x55e06ae70100 2024-03-19T02:00:11.219 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.211+0000 7fa360e95640 1 --2- 172.21.15.99:0/2485160176 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e06ae75c60 0x55e06ae81650 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:00:11.219 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.211+0000 7fa35bfff640 1 --2- 172.21.15.99:0/2485160176 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e06ae70100 0x55e06ae7a0d0 secure :-1 s=READY pgs=707 cs=0 l=1 rev1=1 crypto rx=0x7fa34c0ccc80 tx=0x7fa34c00cf60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=905e2c64ebf2e3d3 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:00:11.219 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.211+0000 7fa33b7fe640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T02:00:11.219 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.211+0000 7fa33b7fe640 1 -- 172.21.15.99:0/2485160176 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa34c09b3a0 con 0x55e06ae70100 2024-03-19T02:00:11.219 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.211+0000 7fa33b7fe640 1 -- 172.21.15.99:0/2485160176 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fa34c099b60 con 0x55e06ae70100 2024-03-19T02:00:11.219 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.211+0000 7fa3655c2740 1 -- 172.21.15.99:0/2485160176 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fa34c09b860 con 0x55e06ae70100 2024-03-19T02:00:11.219 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.211+0000 7fa33b7fe640 1 -- 172.21.15.99:0/2485160176 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7fa34c09b630 con 0x55e06ae70100 2024-03-19T02:00:11.219 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.211+0000 7fa3655c2740 10 client.16866 fetch_fsmap learned FSMap version 37 2024-03-19T02:00:11.219 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.211+0000 7fa3655c2740 1 -- 172.21.15.99:0/2485160176 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x55e06aae9210 con 0x55e06ae70100 2024-03-19T02:00:11.219 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.211+0000 7fa33b7fe640 1 -- 172.21.15.99:0/2485160176 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa34c09b860 con 0x55e06ae70100 2024-03-19T02:00:11.219 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.211+0000 7fa33b7fe640 1 -- 172.21.15.99:0/2485160176 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7fa34c09bac0 con 0x55e06ae70100 2024-03-19T02:00:11.220 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.211+0000 7fa3655c2740 10 client.16866 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T02:00:11.220 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.211+0000 7fa3655c2740 10 client.16866 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T02:00:11.220 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.212+0000 7fa3655c2740 20 client.16866 populate_metadata read hostname 'smithi099' 2024-03-19T02:00:11.220 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.212+0000 7fa3655c2740 1 --2- 172.21.15.99:0/2485160176 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e06ae85b40 0x55e06ae87f30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:11.220 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.212+0000 7fa3655c2740 4 client.16866 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T02:00:11.220 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.212+0000 7fa3655c2740 1 -- 172.21.15.99:0/2485160176 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55e06ade2120 con 0x55e06ae85b40 2024-03-19T02:00:11.220 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.212+0000 7fa35b7fe640 1 --2- 172.21.15.99:0/2485160176 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e06ae85b40 0x55e06ae87f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:11.220 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.212+0000 7fa35b7fe640 1 --2- 172.21.15.99:0/2485160176 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e06ae85b40 0x55e06ae87f30 crc :-1 s=READY pgs=199 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=f137b75fbcf4eb51 server_cookie=3f9d617dec626cf9 in_seq=0 out_seq=0 2024-03-19T02:00:11.220 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.212+0000 7fa33b7fe640 10 client.16866 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T02:00:11.221 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.215+0000 7fa33b7fe640 1 -- 172.21.15.99:0/2485160176 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x55e06ade2120 con 0x55e06ae85b40 2024-03-19T02:00:11.221 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.215+0000 7fa33b7fe640 10 client.16866 handle_command_reply: tid=0 2024-03-19T02:00:11.221 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.215+0000 7fa3655c2740 1 client.16866 shutdown 2024-03-19T02:00:11.221 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.215+0000 7fa3655c2740 1 -- 172.21.15.99:0/2485160176 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e06ae70100 msgr2=0x55e06ae7a0d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:00:11.221 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.215+0000 7fa3655c2740 1 --2- 172.21.15.99:0/2485160176 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e06ae70100 0x55e06ae7a0d0 secure :-1 s=READY pgs=707 cs=0 l=1 rev1=1 crypto rx=0x7fa34c0ccc80 tx=0x7fa34c00cf60 comp rx=0 tx=0).stop 2024-03-19T02:00:11.221 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.215+0000 7fa3655c2740 1 -- 172.21.15.99:0/2485160176 shutdown_connections 2024-03-19T02:00:11.222 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.215+0000 7fa3655c2740 1 --2- 172.21.15.99:0/2485160176 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e06ae85b40 0x55e06ae87f30 crc :-1 s=READY pgs=199 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:11.222 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.215+0000 7fa3655c2740 1 --2- 172.21.15.99:0/2485160176 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e06ae75c60 0x55e06ae81650 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:11.222 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.215+0000 7fa3655c2740 1 --2- 172.21.15.99:0/2485160176 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e06ae70af0 0x55e06ae7a610 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:11.222 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.215+0000 7fa33b7fe640 0 client.16866 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T02:00:11.222 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.215+0000 7fa3655c2740 1 --2- 172.21.15.99:0/2485160176 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e06ae70100 0x55e06ae7a0d0 unknown :-1 s=CLOSED pgs=707 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:11.222 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.215+0000 7fa3655c2740 1 -- 172.21.15.99:0/2485160176 >> 172.21.15.99:0/2485160176 conn(0x55e06ae6b670 msgr2=0x55e06ae6c8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:00:11.222 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.215+0000 7fa3655c2740 1 -- 172.21.15.99:0/2485160176 shutdown_connections 2024-03-19T02:00:11.222 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.215+0000 7fa3655c2740 1 -- 172.21.15.99:0/2485160176 wait complete. 2024-03-19T02:00:11.222 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.216+0000 7fa3655c2740 20 client.16866 trim_cache size 0 max 16384 2024-03-19T02:00:11.235 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.231+0000 7fa3655c2740 1 Processor -- start 2024-03-19T02:00:11.235 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.231+0000 7fa3655c2740 1 -- start start 2024-03-19T02:00:11.235 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.231+0000 7fa3655c2740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e06ae70100 0x55e06ae79ce0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:11.235 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.231+0000 7fa3655c2740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e06ae70af0 0x55e06ae81610 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:11.235 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.231+0000 7fa3655c2740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e06ae75c60 0x55e06ae7a220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:11.235 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.231+0000 7fa3655c2740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fa35c1b1bf0 con 0x55e06ae70af0 2024-03-19T02:00:11.236 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.231+0000 7fa3655c2740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fa35c1af4c0 con 0x55e06ae75c60 2024-03-19T02:00:11.236 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.231+0000 7fa3655c2740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fa35c1ad310 con 0x55e06ae70100 2024-03-19T02:00:11.236 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.231+0000 7fa35b7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e06ae70af0 0x55e06ae81610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:11.236 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.231+0000 7fa35b7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e06ae70af0 0x55e06ae81610 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:52312/0 (socket says 172.21.15.99:52312) 2024-03-19T02:00:11.236 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.231+0000 7fa35b7fe640 1 -- 172.21.15.99:0/4159014972 learned_addr learned my addr 172.21.15.99:0/4159014972 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:00:11.236 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.231+0000 7fa360e95640 1 --2- 172.21.15.99:0/4159014972 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e06ae75c60 0x55e06ae7a220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:11.236 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.231+0000 7fa35b7fe640 1 -- 172.21.15.99:0/4159014972 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e06ae70100 msgr2=0x55e06ae79ce0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:00:11.236 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.231+0000 7fa35b7fe640 1 --2- 172.21.15.99:0/4159014972 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e06ae70100 0x55e06ae79ce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:11.236 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.231+0000 7fa35b7fe640 1 -- 172.21.15.99:0/4159014972 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e06ae75c60 msgr2=0x55e06ae7a220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:00:11.236 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.231+0000 7fa35b7fe640 1 --2- 172.21.15.99:0/4159014972 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e06ae75c60 0x55e06ae7a220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:11.236 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.231+0000 7fa35b7fe640 1 -- 172.21.15.99:0/4159014972 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55e06abb5620 con 0x55e06ae70af0 2024-03-19T02:00:11.236 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.231+0000 7fa360e95640 1 --2- 172.21.15.99:0/4159014972 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e06ae75c60 0x55e06ae7a220 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:00:11.236 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.232+0000 7fa35b7fe640 1 --2- 172.21.15.99:0/4159014972 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e06ae70af0 0x55e06ae81610 secure :-1 s=READY pgs=708 cs=0 l=1 rev1=1 crypto rx=0x7fa350009430 tx=0x7fa35002c880 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d54717ea87cd548b server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:00:11.237 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.232+0000 7fa341ffb640 1 -- 172.21.15.99:0/4159014972 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa35004c070 con 0x55e06ae70af0 2024-03-19T02:00:11.237 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.232+0000 7fa341ffb640 1 -- 172.21.15.99:0/4159014972 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa35002cd80 con 0x55e06ae70af0 2024-03-19T02:00:11.237 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.232+0000 7fa341ffb640 1 -- 172.21.15.99:0/4159014972 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa3500046c0 con 0x55e06ae70af0 2024-03-19T02:00:11.237 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.232+0000 7fa3655c2740 1 -- 172.21.15.99:0/4159014972 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e06ae70af0 msgr2=0x55e06ae81610 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:00:11.237 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.232+0000 7fa3655c2740 1 --2- 172.21.15.99:0/4159014972 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e06ae70af0 0x55e06ae81610 secure :-1 s=READY pgs=708 cs=0 l=1 rev1=1 crypto rx=0x7fa350009430 tx=0x7fa35002c880 comp rx=0 tx=0).stop 2024-03-19T02:00:11.237 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.233+0000 7fa3655c2740 1 -- 172.21.15.99:0/4159014972 shutdown_connections 2024-03-19T02:00:11.237 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.233+0000 7fa3655c2740 1 --2- 172.21.15.99:0/4159014972 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e06ae75c60 0x55e06ae7a220 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:11.237 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.233+0000 7fa3655c2740 1 --2- 172.21.15.99:0/4159014972 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e06ae70af0 0x55e06ae81610 unknown :-1 s=CLOSED pgs=708 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:11.237 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.233+0000 7fa3655c2740 1 --2- 172.21.15.99:0/4159014972 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e06ae70100 0x55e06ae79ce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:11.237 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.233+0000 7fa3655c2740 1 -- 172.21.15.99:0/4159014972 >> 172.21.15.99:0/4159014972 conn(0x55e06ae6b670 msgr2=0x55e06ae6a220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:00:11.238 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.233+0000 7fa3655c2740 1 -- 172.21.15.99:0/4159014972 shutdown_connections 2024-03-19T02:00:11.238 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.233+0000 7fa3655c2740 1 -- 172.21.15.99:0/4159014972 wait complete. 2024-03-19T02:00:11.238 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.233+0000 7fa3655c2740 1 -- start start 2024-03-19T02:00:11.238 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.233+0000 7fa3655c2740 1 Processor -- start 2024-03-19T02:00:11.238 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.233+0000 7fa3655c2740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e06ae70100 0x55e06ae7eaa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:11.238 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.233+0000 7fa3655c2740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e06ae70af0 0x55e06ae7efe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:11.238 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.233+0000 7fa3655c2740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e06ae75c60 0x55e06ae7fff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:11.238 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.233+0000 7fa3655c2740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fa35c1ad980 con 0x55e06ae70af0 2024-03-19T02:00:11.238 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.233+0000 7fa3655c2740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fa35c1aedf0 con 0x55e06ae75c60 2024-03-19T02:00:11.238 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.233+0000 7fa3655c2740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fa35c1ae050 con 0x55e06ae70100 2024-03-19T02:00:11.238 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.233+0000 7fa3655c2740 10 client.0 _ll_register_callbacks cb 0x7fa354009b30 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T02:00:11.238 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.233+0000 7fa35b7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e06ae70af0 0x55e06ae7efe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:11.238 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.233+0000 7fa35b7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e06ae70af0 0x55e06ae7efe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:52320/0 (socket says 172.21.15.99:52320) 2024-03-19T02:00:11.239 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.233+0000 7fa35b7fe640 1 -- 172.21.15.99:0/1808915545 learned_addr learned my addr 172.21.15.99:0/1808915545 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:00:11.239 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.234+0000 7fa360e95640 1 --2- 172.21.15.99:0/1808915545 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e06ae75c60 0x55e06ae7fff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:11.239 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.234+0000 7fa35bfff640 1 --2- 172.21.15.99:0/1808915545 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e06ae70100 0x55e06ae7eaa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:11.239 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.234+0000 7fa35b7fe640 1 -- 172.21.15.99:0/1808915545 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e06ae70100 msgr2=0x55e06ae7eaa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:00:11.239 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.234+0000 7fa35b7fe640 1 --2- 172.21.15.99:0/1808915545 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e06ae70100 0x55e06ae7eaa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:11.239 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.234+0000 7fa35b7fe640 1 -- 172.21.15.99:0/1808915545 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e06ae75c60 msgr2=0x55e06ae7fff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:00:11.239 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.234+0000 7fa35b7fe640 1 --2- 172.21.15.99:0/1808915545 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e06ae75c60 0x55e06ae7fff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:11.239 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.234+0000 7fa35b7fe640 1 -- 172.21.15.99:0/1808915545 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55e06ae805e0 con 0x55e06ae70af0 2024-03-19T02:00:11.239 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.234+0000 7fa35b7fe640 1 --2- 172.21.15.99:0/1808915545 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e06ae70af0 0x55e06ae7efe0 secure :-1 s=READY pgs=709 cs=0 l=1 rev1=1 crypto rx=0x7fa350009230 tx=0x7fa35002cce0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b68110d8ccf23c1 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:00:11.239 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.234+0000 7fa33b7fe640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T02:00:11.240 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.234+0000 7fa33b7fe640 1 -- 172.21.15.99:0/1808915545 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa350055020 con 0x55e06ae70af0 2024-03-19T02:00:11.240 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.234+0000 7fa33b7fe640 1 -- 172.21.15.99:0/1808915545 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fa3500483a0 con 0x55e06ae70af0 2024-03-19T02:00:11.240 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.234+0000 7fa3655c2740 1 -- 172.21.15.99:0/1808915545 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fa35002cd80 con 0x55e06ae70af0 2024-03-19T02:00:11.240 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.234+0000 7fa33b7fe640 1 -- 172.21.15.99:0/1808915545 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7fa350060050 con 0x55e06ae70af0 2024-03-19T02:00:11.240 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.234+0000 7fa3655c2740 10 client.16872 fetch_fsmap learned FSMap version 37 2024-03-19T02:00:11.240 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.234+0000 7fa3655c2740 1 -- 172.21.15.99:0/1808915545 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x55e06ae81920 con 0x55e06ae70af0 2024-03-19T02:00:11.240 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.234+0000 7fa33b7fe640 1 -- 172.21.15.99:0/1808915545 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fa35002cd80 con 0x55e06ae70af0 2024-03-19T02:00:11.240 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.235+0000 7fa33b7fe640 1 -- 172.21.15.99:0/1808915545 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7fa350004ed0 con 0x55e06ae70af0 2024-03-19T02:00:11.240 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.235+0000 7fa3655c2740 10 client.16872 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T02:00:11.240 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.235+0000 7fa3655c2740 10 client.16872 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T02:00:11.240 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.235+0000 7fa3655c2740 20 client.16872 populate_metadata read hostname 'smithi099' 2024-03-19T02:00:11.240 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.235+0000 7fa3655c2740 1 --2- 172.21.15.99:0/1808915545 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e06aeec400 0x55e06aeee820 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:11.240 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.235+0000 7fa3655c2740 4 client.16872 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T02:00:11.240 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.235+0000 7fa3655c2740 1 -- 172.21.15.99:0/1808915545 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55e06addc470 con 0x55e06aeec400 2024-03-19T02:00:11.240 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.235+0000 7fa35bfff640 1 --2- 172.21.15.99:0/1808915545 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e06aeec400 0x55e06aeee820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:11.241 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.235+0000 7fa35bfff640 1 --2- 172.21.15.99:0/1808915545 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e06aeec400 0x55e06aeee820 crc :-1 s=READY pgs=200 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=f07bdb27e4f69a60 server_cookie=2bc34aae15ab507c in_seq=0 out_seq=0 2024-03-19T02:00:11.241 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.235+0000 7fa33b7fe640 10 client.16872 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T02:00:11.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:00:11 smithi112 ceph-mon[66819]: pgmap v924: 97 pgs: 97 active+clean; 604 MiB data, 7.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.7 MiB/s rd, 6.0 MiB/s wr, 546 op/s 2024-03-19T02:00:11.394 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:00:11 smithi099 ceph-mon[62951]: pgmap v924: 97 pgs: 97 active+clean; 604 MiB data, 7.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.7 MiB/s rd, 6.0 MiB/s wr, 546 op/s 2024-03-19T02:00:11.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:00:11 smithi115 ceph-mon[67332]: pgmap v924: 97 pgs: 97 active+clean; 604 MiB data, 7.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.7 MiB/s rd, 6.0 MiB/s wr, 546 op/s 2024-03-19T02:00:11.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.782+0000 7fa33b7fe640 1 -- 172.21.15.99:0/1808915545 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x55e06addc470 con 0x55e06aeec400 2024-03-19T02:00:11.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.782+0000 7fa33b7fe640 10 client.16872 handle_command_reply: tid=0 2024-03-19T02:00:11.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.782+0000 7fa3655c2740 1 client.16872 shutdown 2024-03-19T02:00:11.782 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.782+0000 7fa3655c2740 1 -- 172.21.15.99:0/1808915545 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e06ae70af0 msgr2=0x55e06ae7efe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:00:11.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.782+0000 7fa3655c2740 1 --2- 172.21.15.99:0/1808915545 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e06ae70af0 0x55e06ae7efe0 secure :-1 s=READY pgs=709 cs=0 l=1 rev1=1 crypto rx=0x7fa350009230 tx=0x7fa35002cce0 comp rx=0 tx=0).stop 2024-03-19T02:00:11.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.782+0000 7fa3655c2740 1 -- 172.21.15.99:0/1808915545 shutdown_connections 2024-03-19T02:00:11.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.782+0000 7fa3655c2740 1 --2- 172.21.15.99:0/1808915545 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55e06aeec400 0x55e06aeee820 crc :-1 s=READY pgs=200 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:11.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.782+0000 7fa3655c2740 1 --2- 172.21.15.99:0/1808915545 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55e06ae75c60 0x55e06ae7fff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:11.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.782+0000 7fa33b7fe640 0 client.16872 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T02:00:11.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.782+0000 7fa3655c2740 1 --2- 172.21.15.99:0/1808915545 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55e06ae70af0 0x55e06ae7efe0 unknown :-1 s=CLOSED pgs=709 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:11.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.782+0000 7fa3655c2740 1 --2- 172.21.15.99:0/1808915545 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55e06ae70100 0x55e06ae7eaa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:11.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.782+0000 7fa3655c2740 1 -- 172.21.15.99:0/1808915545 >> 172.21.15.99:0/1808915545 conn(0x55e06ae6b670 msgr2=0x55e06ae79c70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:00:11.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.782+0000 7fa3655c2740 1 -- 172.21.15.99:0/1808915545 shutdown_connections 2024-03-19T02:00:11.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.782+0000 7fa3655c2740 1 -- 172.21.15.99:0/1808915545 wait complete. 2024-03-19T02:00:11.783 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.782+0000 7fa3655c2740 20 client.16872 trim_cache size 0 max 16384 2024-03-19T02:00:11.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T02:00:11.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T02:00:11.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T02:00:11.784 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T02:00:11.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.783+0000 7fa3628f8640 1 -- 172.21.15.99:0/3229207631 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa3440a28a0 msgr2=0x7fa3440a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:00:11.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.783+0000 7fa3628f8640 1 --2- 172.21.15.99:0/3229207631 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa3440a28a0 0x7fa3440a4d60 secure :-1 s=READY pgs=358 cs=0 l=1 rev1=1 crypto rx=0x55e06ae71350 tx=0x7fa354009500 comp rx=0 tx=0).stop 2024-03-19T02:00:11.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.783+0000 7fa3628f8640 1 -- 172.21.15.99:0/3229207631 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa35c100100 msgr2=0x7fa35c19e070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:00:11.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.783+0000 7fa3628f8640 1 --2- 172.21.15.99:0/3229207631 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa35c100100 0x7fa35c19e070 secure :-1 s=READY pgs=199 cs=0 l=1 rev1=1 crypto rx=0x7fa34c00e9f0 tx=0x7fa34c00eeb0 comp rx=0 tx=0).stop 2024-03-19T02:00:11.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.783+0000 7fa3628f8640 1 -- 172.21.15.99:0/3229207631 shutdown_connections 2024-03-19T02:00:11.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.783+0000 7fa3628f8640 1 --2- 172.21.15.99:0/3229207631 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fa35c105420 0x7fa35c1a5620 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:11.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.783+0000 7fa3628f8640 1 --2- 172.21.15.99:0/3229207631 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fa35c102a70 0x7fa35c19e5b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:11.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.783+0000 7fa3628f8640 1 --2- 172.21.15.99:0/3229207631 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fa3440a28a0 0x7fa3440a4d60 unknown :-1 s=CLOSED pgs=358 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:11.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.783+0000 7fa3628f8640 1 --2- 172.21.15.99:0/3229207631 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fa35c100100 0x7fa35c19e070 unknown :-1 s=CLOSED pgs=199 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:11.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.783+0000 7fa3628f8640 1 -- 172.21.15.99:0/3229207631 >> 172.21.15.99:0/3229207631 conn(0x7fa35c0fbd00 msgr2=0x7fa35c100db0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:00:11.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.783+0000 7fa3628f8640 1 -- 172.21.15.99:0/3229207631 shutdown_connections 2024-03-19T02:00:11.785 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:11.783+0000 7fa3628f8640 1 -- 172.21.15.99:0/3229207631 wait complete. 2024-03-19T02:00:12.111 INFO:tasks.workunit.client.0.smithi099.stdout: 10 173530 22.81 MB/sec execute 281 sec latency 412.618 ms 2024-03-19T02:00:13.111 INFO:tasks.workunit.client.0.smithi099.stdout: 10 173794 22.77 MB/sec execute 282 sec latency 373.125 ms 2024-03-19T02:00:13.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:00:13 smithi112 ceph-mon[66819]: pgmap v925: 97 pgs: 97 active+clean; 602 MiB data, 7.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.3 MiB/s rd, 5.8 MiB/s wr, 511 op/s 2024-03-19T02:00:13.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:00:13 smithi115 ceph-mon[67332]: pgmap v925: 97 pgs: 97 active+clean; 602 MiB data, 7.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.3 MiB/s rd, 5.8 MiB/s wr, 511 op/s 2024-03-19T02:00:13.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:00:13 smithi099 ceph-mon[62951]: pgmap v925: 97 pgs: 97 active+clean; 602 MiB data, 7.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.3 MiB/s rd, 5.8 MiB/s wr, 511 op/s 2024-03-19T02:00:14.111 INFO:tasks.workunit.client.0.smithi099.stdout: 10 174162 22.75 MB/sec execute 283 sec latency 113.138 ms 2024-03-19T02:00:15.111 INFO:tasks.workunit.client.0.smithi099.stdout: 10 174718 22.82 MB/sec execute 284 sec latency 75.455 ms 2024-03-19T02:00:15.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:00:15 smithi112 ceph-mon[66819]: pgmap v926: 97 pgs: 97 active+clean; 607 MiB data, 7.5 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 5.4 MiB/s wr, 461 op/s 2024-03-19T02:00:15.558 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:00:15 smithi115 ceph-mon[67332]: pgmap v926: 97 pgs: 97 active+clean; 607 MiB data, 7.5 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 5.4 MiB/s wr, 461 op/s 2024-03-19T02:00:15.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:00:15 smithi099 ceph-mon[62951]: pgmap v926: 97 pgs: 97 active+clean; 607 MiB data, 7.5 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 5.4 MiB/s wr, 461 op/s 2024-03-19T02:00:16.111 INFO:tasks.workunit.client.0.smithi099.stdout: 10 175244 22.80 MB/sec execute 285 sec latency 80.878 ms 2024-03-19T02:00:17.111 INFO:tasks.workunit.client.0.smithi099.stdout: 10 175700 22.80 MB/sec execute 286 sec latency 69.552 ms 2024-03-19T02:00:17.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:00:17 smithi112 ceph-mon[66819]: pgmap v927: 97 pgs: 97 active+clean; 601 MiB data, 7.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.5 MiB/s rd, 6.6 MiB/s wr, 593 op/s 2024-03-19T02:00:17.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:00:17 smithi115 ceph-mon[67332]: pgmap v927: 97 pgs: 97 active+clean; 601 MiB data, 7.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.5 MiB/s rd, 6.6 MiB/s wr, 593 op/s 2024-03-19T02:00:17.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:00:17 smithi099 ceph-mon[62951]: pgmap v927: 97 pgs: 97 active+clean; 601 MiB data, 7.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.5 MiB/s rd, 6.6 MiB/s wr, 593 op/s 2024-03-19T02:00:18.112 INFO:tasks.workunit.client.0.smithi099.stdout: 10 176118 22.81 MB/sec execute 287 sec latency 286.111 ms 2024-03-19T02:00:19.112 INFO:tasks.workunit.client.0.smithi099.stdout: 10 176566 22.85 MB/sec execute 288 sec latency 86.452 ms 2024-03-19T02:00:19.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:00:19 smithi115 ceph-mon[67332]: pgmap v928: 97 pgs: 97 active+clean; 601 MiB data, 7.5 GiB used, 1.0 TiB / 1.0 TiB avail; 3.7 MiB/s rd, 5.1 MiB/s wr, 474 op/s 2024-03-19T02:00:19.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:00:19 smithi099 ceph-mon[62951]: pgmap v928: 97 pgs: 97 active+clean; 601 MiB data, 7.5 GiB used, 1.0 TiB / 1.0 TiB avail; 3.7 MiB/s rd, 5.1 MiB/s wr, 474 op/s 2024-03-19T02:00:19.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:00:19 smithi112 ceph-mon[66819]: pgmap v928: 97 pgs: 97 active+clean; 601 MiB data, 7.5 GiB used, 1.0 TiB / 1.0 TiB avail; 3.7 MiB/s rd, 5.1 MiB/s wr, 474 op/s 2024-03-19T02:00:20.112 INFO:tasks.workunit.client.0.smithi099.stdout: 10 177002 22.86 MB/sec execute 289 sec latency 64.482 ms 2024-03-19T02:00:21.112 INFO:tasks.workunit.client.0.smithi099.stdout: 10 177393 22.85 MB/sec execute 290 sec latency 69.137 ms 2024-03-19T02:00:21.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:00:21 smithi112 ceph-mon[66819]: pgmap v929: 97 pgs: 97 active+clean; 606 MiB data, 7.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 5.7 MiB/s wr, 542 op/s 2024-03-19T02:00:21.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:00:21 smithi115 ceph-mon[67332]: pgmap v929: 97 pgs: 97 active+clean; 606 MiB data, 7.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 5.7 MiB/s wr, 542 op/s 2024-03-19T02:00:21.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:00:21 smithi099 ceph-mon[62951]: pgmap v929: 97 pgs: 97 active+clean; 606 MiB data, 7.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 5.7 MiB/s wr, 542 op/s 2024-03-19T02:00:22.112 INFO:tasks.workunit.client.0.smithi099.stdout: 10 177795 22.82 MB/sec execute 291 sec latency 84.468 ms 2024-03-19T02:00:23.112 INFO:tasks.workunit.client.0.smithi099.stdout: 10 178102 22.79 MB/sec execute 292 sec latency 253.558 ms 2024-03-19T02:00:23.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:00:23 smithi115 ceph-mon[67332]: pgmap v930: 97 pgs: 97 active+clean; 611 MiB data, 7.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.4 MiB/s rd, 6.1 MiB/s wr, 552 op/s 2024-03-19T02:00:23.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:00:23 smithi099 ceph-mon[62951]: pgmap v930: 97 pgs: 97 active+clean; 611 MiB data, 7.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.4 MiB/s rd, 6.1 MiB/s wr, 552 op/s 2024-03-19T02:00:23.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:00:23 smithi112 ceph-mon[66819]: pgmap v930: 97 pgs: 97 active+clean; 611 MiB data, 7.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.4 MiB/s rd, 6.1 MiB/s wr, 552 op/s 2024-03-19T02:00:24.112 INFO:tasks.workunit.client.0.smithi099.stdout: 10 178559 22.80 MB/sec execute 293 sec latency 145.975 ms 2024-03-19T02:00:24.880 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.879+0000 7fca19d10640 1 -- 172.21.15.99:0/1253543592 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fca14101930 msgr2=0x7fca14101db0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:00:24.881 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.879+0000 7fca19d10640 1 --2- 172.21.15.99:0/1253543592 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fca14101930 0x7fca14101db0 secure :-1 s=READY pgs=710 cs=0 l=1 rev1=1 crypto rx=0x7fca08009900 tx=0x7fca0802bc30 comp rx=0 tx=0).stop 2024-03-19T02:00:24.881 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.879+0000 7fca19d10640 1 -- 172.21.15.99:0/1253543592 shutdown_connections 2024-03-19T02:00:24.881 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.879+0000 7fca19d10640 1 --2- 172.21.15.99:0/1253543592 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fca141022f0 0x7fca14108b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:24.881 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.879+0000 7fca19d10640 1 --2- 172.21.15.99:0/1253543592 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fca14101930 0x7fca14101db0 unknown :-1 s=CLOSED pgs=710 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:24.881 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.879+0000 7fca19d10640 1 --2- 172.21.15.99:0/1253543592 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fca14100740 0x7fca14100b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:24.881 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.879+0000 7fca19d10640 1 -- 172.21.15.99:0/1253543592 >> 172.21.15.99:0/1253543592 conn(0x7fca140fbcb0 msgr2=0x7fca140fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:00:24.881 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.879+0000 7fca19d10640 1 -- 172.21.15.99:0/1253543592 shutdown_connections 2024-03-19T02:00:24.881 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.879+0000 7fca19d10640 1 -- 172.21.15.99:0/1253543592 wait complete. 2024-03-19T02:00:24.881 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.880+0000 7fca19d10640 1 Processor -- start 2024-03-19T02:00:24.881 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.880+0000 7fca19d10640 1 -- start start 2024-03-19T02:00:24.881 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.880+0000 7fca19d10640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fca14100740 0x7fca141046d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:24.881 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.880+0000 7fca19d10640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fca14101930 0x7fca14102d10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:24.882 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.880+0000 7fca19d10640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fca141022f0 0x7fca14103250 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:24.882 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.880+0000 7fca19d10640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fca1410c3a0 con 0x7fca14101930 2024-03-19T02:00:24.882 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.881+0000 7fca19d10640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fca1410c220 con 0x7fca141022f0 2024-03-19T02:00:24.882 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.881+0000 7fca19d10640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fca1410c520 con 0x7fca14100740 2024-03-19T02:00:24.882 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.881+0000 7fca13fff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fca141022f0 0x7fca14103250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:24.882 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.881+0000 7fca137fe640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fca14100740 0x7fca141046d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:24.882 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.881+0000 7fca137fe640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fca14100740 0x7fca141046d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:45418/0 (socket says 172.21.15.99:45418) 2024-03-19T02:00:24.882 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.881+0000 7fca12ffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fca14101930 0x7fca14102d10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:24.882 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.881+0000 7fca137fe640 1 -- 172.21.15.99:0/3021554020 learned_addr learned my addr 172.21.15.99:0/3021554020 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:00:24.883 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.881+0000 7fca137fe640 1 -- 172.21.15.99:0/3021554020 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fca141022f0 msgr2=0x7fca14103250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:00:24.883 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.881+0000 7fca137fe640 1 --2- 172.21.15.99:0/3021554020 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fca141022f0 0x7fca14103250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:24.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.881+0000 7fca137fe640 1 -- 172.21.15.99:0/3021554020 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fca14101930 msgr2=0x7fca14102d10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:00:24.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.881+0000 7fca137fe640 1 --2- 172.21.15.99:0/3021554020 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fca14101930 0x7fca14102d10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:24.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.881+0000 7fca137fe640 1 -- 172.21.15.99:0/3021554020 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fca14103ae0 con 0x7fca14100740 2024-03-19T02:00:24.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.882+0000 7fca137fe640 1 --2- 172.21.15.99:0/3021554020 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fca14100740 0x7fca141046d0 secure :-1 s=READY pgs=126 cs=0 l=1 rev1=1 crypto rx=0x7fca0400e9c0 tx=0x7fca0400ee80 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:00:24.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.882+0000 7fca10ff9640 1 -- 172.21.15.99:0/3021554020 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fca0400cd80 con 0x7fca14100740 2024-03-19T02:00:24.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.882+0000 7fca10ff9640 1 -- 172.21.15.99:0/3021554020 <== mon.1 v2:172.21.15.115:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fca040035f0 con 0x7fca14100740 2024-03-19T02:00:24.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.882+0000 7fca19d10640 1 -- 172.21.15.99:0/3021554020 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fca141a1c00 con 0x7fca14100740 2024-03-19T02:00:24.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.882+0000 7fca10ff9640 1 -- 172.21.15.99:0/3021554020 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fca04010690 con 0x7fca14100740 2024-03-19T02:00:24.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.882+0000 7fca19d10640 1 -- 172.21.15.99:0/3021554020 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fca141a2170 con 0x7fca14100740 2024-03-19T02:00:24.890 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.885+0000 7fca1c9db740 1 Processor -- start 2024-03-19T02:00:24.890 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.885+0000 7fca1c9db740 1 -- start start 2024-03-19T02:00:24.890 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.885+0000 7fca1c9db740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c416918100 0x55c416918520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:24.890 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.885+0000 7fca1c9db740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c416918af0 0x55c41691d5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:24.890 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.885+0000 7fca1c9db740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c41691dc60 0x55c4169200f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:24.890 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.885+0000 7fca1c9db740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55c41687cbe0 con 0x55c41691dc60 2024-03-19T02:00:24.890 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.885+0000 7fca1c9db740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55c416825960 con 0x55c416918af0 2024-03-19T02:00:24.890 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.885+0000 7fca1c9db740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55c416878c80 con 0x55c416918100 2024-03-19T02:00:24.890 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.885+0000 7fca10ff9640 1 -- 172.21.15.99:0/3021554020 <== mon.1 v2:172.21.15.115:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7fca040037a0 con 0x7fca14100740 2024-03-19T02:00:24.890 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.885+0000 7fca13fff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c416918100 0x55c416918520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:24.890 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.885+0000 7fca12ffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c41691dc60 0x55c4169200f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:24.890 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.885+0000 7fca13fff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c416918100 0x55c416918520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:45424/0 (socket says 172.21.15.99:45424) 2024-03-19T02:00:24.890 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.885+0000 7fca13fff640 1 -- 172.21.15.99:0/3097864039 learned_addr learned my addr 172.21.15.99:0/3097864039 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:00:24.891 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.885+0000 7fca137fe640 1 --2- 172.21.15.99:0/3097864039 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c416918af0 0x55c41691d5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:24.891 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.885+0000 7fca12ffd640 1 -- 172.21.15.99:0/3097864039 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c416918100 msgr2=0x55c416918520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:00:24.891 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.885+0000 7fca12ffd640 1 --2- 172.21.15.99:0/3097864039 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c416918100 0x55c416918520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:24.891 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.885+0000 7fca12ffd640 1 -- 172.21.15.99:0/3097864039 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c416918af0 msgr2=0x55c41691d5f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:00:24.891 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.885+0000 7fca12ffd640 1 --2- 172.21.15.99:0/3097864039 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c416918af0 0x55c41691d5f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:24.891 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.885+0000 7fca12ffd640 1 -- 172.21.15.99:0/3097864039 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55c41681cd20 con 0x55c41691dc60 2024-03-19T02:00:24.891 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.886+0000 7fca12ffd640 1 --2- 172.21.15.99:0/3097864039 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c41691dc60 0x55c4169200f0 secure :-1 s=READY pgs=711 cs=0 l=1 rev1=1 crypto rx=0x7fca08009230 tx=0x7fca08009560 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b1b17e8c3112fbed server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:00:24.891 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.886+0000 7fca10ff9640 1 --2- 172.21.15.99:0/3021554020 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fc9fc0a2920 0x7fc9fc0a4de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:24.891 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.886+0000 7fc9f9ffb640 1 -- 172.21.15.99:0/3097864039 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fca0802cb80 con 0x55c41691dc60 2024-03-19T02:00:24.891 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.886+0000 7fca10ff9640 1 -- 172.21.15.99:0/3021554020 <== mon.1 v2:172.21.15.115:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fca040cee10 con 0x7fca14100740 2024-03-19T02:00:24.891 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.886+0000 7fca13fff640 1 --2- 172.21.15.99:0/3021554020 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fc9fc0a2920 0x7fc9fc0a4de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:24.891 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.886+0000 7fc9f9ffb640 1 -- 172.21.15.99:0/3097864039 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fca08005040 con 0x55c41691dc60 2024-03-19T02:00:24.892 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.886+0000 7fc9f9ffb640 1 -- 172.21.15.99:0/3097864039 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fca08004900 con 0x55c41691dc60 2024-03-19T02:00:24.892 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.887+0000 7fca1c9db740 1 -- 172.21.15.99:0/3097864039 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c41691dc60 msgr2=0x55c4169200f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:00:24.892 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.887+0000 7fca13fff640 1 --2- 172.21.15.99:0/3021554020 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fc9fc0a2920 0x7fc9fc0a4de0 secure :-1 s=READY pgs=359 cs=0 l=1 rev1=1 crypto rx=0x55c416919350 tx=0x7fca0c009500 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:00:24.892 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.887+0000 7fca1c9db740 1 --2- 172.21.15.99:0/3097864039 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c41691dc60 0x55c4169200f0 secure :-1 s=READY pgs=711 cs=0 l=1 rev1=1 crypto rx=0x7fca08009230 tx=0x7fca08009560 comp rx=0 tx=0).stop 2024-03-19T02:00:24.892 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.887+0000 7fca1c9db740 1 -- 172.21.15.99:0/3097864039 shutdown_connections 2024-03-19T02:00:24.892 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.887+0000 7fca1c9db740 1 --2- 172.21.15.99:0/3097864039 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c416918af0 0x55c41691d5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:24.892 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.887+0000 7fca1c9db740 1 --2- 172.21.15.99:0/3097864039 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c41691dc60 0x55c4169200f0 unknown :-1 s=CLOSED pgs=711 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:24.892 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.887+0000 7fca1c9db740 1 --2- 172.21.15.99:0/3097864039 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c416918100 0x55c416918520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:24.892 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.887+0000 7fca1c9db740 1 -- 172.21.15.99:0/3097864039 >> 172.21.15.99:0/3097864039 conn(0x55c416913670 msgr2=0x55c416913a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:00:24.892 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.887+0000 7fca1c9db740 1 -- 172.21.15.99:0/3097864039 shutdown_connections 2024-03-19T02:00:24.892 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.887+0000 7fca1c9db740 1 -- 172.21.15.99:0/3097864039 wait complete. 2024-03-19T02:00:24.892 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.887+0000 7fca1c9db740 1 -- start start 2024-03-19T02:00:24.892 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.887+0000 7fca1c9db740 1 Processor -- start 2024-03-19T02:00:24.892 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.888+0000 7fca1c9db740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c416918100 0x55c4169220d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:24.892 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.888+0000 7fca1c9db740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c416918af0 0x55c416922610 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:24.893 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.888+0000 7fca1c9db740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c41691dc60 0x55c416929650 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:24.893 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.888+0000 7fca1c9db740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55c416920bb0 con 0x55c416918100 2024-03-19T02:00:24.893 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.888+0000 7fca1c9db740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55c416838db0 con 0x55c416918af0 2024-03-19T02:00:24.893 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.888+0000 7fca1c9db740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55c416920d30 con 0x55c41691dc60 2024-03-19T02:00:24.893 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.888+0000 7fca1c9db740 10 client.0 _ll_register_callbacks cb 0x55c41683de70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T02:00:24.893 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.888+0000 7fca137fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c416918100 0x55c4169220d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:24.893 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.888+0000 7fca137fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c416918100 0x55c4169220d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:56588/0 (socket says 172.21.15.99:56588) 2024-03-19T02:00:24.893 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.888+0000 7fca137fe640 1 -- 172.21.15.99:0/2229487494 learned_addr learned my addr 172.21.15.99:0/2229487494 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:00:24.893 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.888+0000 7fca12ffd640 1 --2- 172.21.15.99:0/2229487494 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c416918af0 0x55c416922610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:24.893 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.888+0000 7fca13fff640 1 --2- 172.21.15.99:0/2229487494 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c41691dc60 0x55c416929650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:24.893 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.888+0000 7fca12ffd640 1 -- 172.21.15.99:0/2229487494 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c41691dc60 msgr2=0x55c416929650 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:00:24.893 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.888+0000 7fca12ffd640 1 --2- 172.21.15.99:0/2229487494 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c41691dc60 0x55c416929650 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:24.893 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.888+0000 7fca12ffd640 1 -- 172.21.15.99:0/2229487494 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c416918100 msgr2=0x55c4169220d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:00:24.893 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.888+0000 7fca12ffd640 1 --2- 172.21.15.99:0/2229487494 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c416918100 0x55c4169220d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:24.893 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.888+0000 7fca12ffd640 1 -- 172.21.15.99:0/2229487494 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55c416837c60 con 0x55c416918af0 2024-03-19T02:00:24.894 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.888+0000 7fca137fe640 1 --2- 172.21.15.99:0/2229487494 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c416918100 0x55c4169220d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:00:24.894 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.888+0000 7fca12ffd640 1 --2- 172.21.15.99:0/2229487494 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c416918af0 0x55c416922610 secure :-1 s=READY pgs=200 cs=0 l=1 rev1=1 crypto rx=0x7fca08009900 tx=0x7fca08004260 comp rx=0 tx=0).ready entity=mon.2 client_cookie=1affd9543a2b6d46 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:00:24.894 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.888+0000 7fc9f2ffd640 10 client.0 ms_handle_connect on v2:172.21.15.112:3300/0 2024-03-19T02:00:24.894 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.888+0000 7fc9f2ffd640 1 -- 172.21.15.99:0/2229487494 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fca08005400 con 0x55c416918af0 2024-03-19T02:00:24.894 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.888+0000 7fc9f2ffd640 1 -- 172.21.15.99:0/2229487494 <== mon.2 v2:172.21.15.112:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fca080055a0 con 0x55c416918af0 2024-03-19T02:00:24.894 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.889+0000 7fca1c9db740 1 -- 172.21.15.99:0/2229487494 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fca08005040 con 0x55c416918af0 2024-03-19T02:00:24.894 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.889+0000 7fc9f2ffd640 1 -- 172.21.15.99:0/2229487494 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7fca080046c0 con 0x55c416918af0 2024-03-19T02:00:24.894 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.889+0000 7fca1c9db740 10 client.26660 fetch_fsmap learned FSMap version 37 2024-03-19T02:00:24.894 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.889+0000 7fca1c9db740 1 -- 172.21.15.99:0/2229487494 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x55c416591210 con 0x55c416918af0 2024-03-19T02:00:24.894 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.889+0000 7fc9f2ffd640 1 -- 172.21.15.99:0/2229487494 <== mon.2 v2:172.21.15.112:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fca08005040 con 0x55c416918af0 2024-03-19T02:00:24.894 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.889+0000 7fc9f2ffd640 1 -- 172.21.15.99:0/2229487494 <== mon.2 v2:172.21.15.112:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7fca08003b90 con 0x55c416918af0 2024-03-19T02:00:24.894 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.889+0000 7fca1c9db740 10 client.26660 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T02:00:24.894 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.889+0000 7fca1c9db740 10 client.26660 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T02:00:24.894 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.889+0000 7fca1c9db740 20 client.26660 populate_metadata read hostname 'smithi099' 2024-03-19T02:00:24.895 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.889+0000 7fca1c9db740 1 --2- 172.21.15.99:0/2229487494 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55c41692db40 0x55c41692ff30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:24.895 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.889+0000 7fca1c9db740 4 client.26660 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T02:00:24.895 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.889+0000 7fca1c9db740 1 -- 172.21.15.99:0/2229487494 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55c41688a120 con 0x55c41692db40 2024-03-19T02:00:24.895 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.889+0000 7fca137fe640 1 --2- 172.21.15.99:0/2229487494 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55c41692db40 0x55c41692ff30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:24.895 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.890+0000 7fc9f2ffd640 10 client.26660 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T02:00:24.895 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.890+0000 7fca137fe640 1 --2- 172.21.15.99:0/2229487494 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55c41692db40 0x55c41692ff30 crc :-1 s=READY pgs=201 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=83c87d2bb003c3cb server_cookie=72e226a46f271723 in_seq=0 out_seq=0 2024-03-19T02:00:24.896 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.891+0000 7fc9f2ffd640 1 -- 172.21.15.99:0/2229487494 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x55c41688a120 con 0x55c41692db40 2024-03-19T02:00:24.896 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.891+0000 7fc9f2ffd640 10 client.26660 handle_command_reply: tid=0 2024-03-19T02:00:24.896 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.891+0000 7fca1c9db740 1 client.26660 shutdown 2024-03-19T02:00:24.896 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.892+0000 7fca1c9db740 1 -- 172.21.15.99:0/2229487494 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c416918af0 msgr2=0x55c416922610 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:00:24.896 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.892+0000 7fca1c9db740 1 --2- 172.21.15.99:0/2229487494 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c416918af0 0x55c416922610 secure :-1 s=READY pgs=200 cs=0 l=1 rev1=1 crypto rx=0x7fca08009900 tx=0x7fca08004260 comp rx=0 tx=0).stop 2024-03-19T02:00:24.896 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.892+0000 7fca1c9db740 1 -- 172.21.15.99:0/2229487494 shutdown_connections 2024-03-19T02:00:24.896 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.892+0000 7fca1c9db740 1 --2- 172.21.15.99:0/2229487494 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55c41692db40 0x55c41692ff30 crc :-1 s=READY pgs=201 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:24.896 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.892+0000 7fca1c9db740 1 --2- 172.21.15.99:0/2229487494 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c416918af0 0x55c416922610 unknown :-1 s=CLOSED pgs=200 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:24.896 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.892+0000 7fca1c9db740 1 --2- 172.21.15.99:0/2229487494 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c41691dc60 0x55c416929650 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:24.896 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.892+0000 7fc9f2ffd640 0 client.26660 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T02:00:24.897 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.892+0000 7fca1c9db740 1 --2- 172.21.15.99:0/2229487494 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c416918100 0x55c4169220d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:24.897 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.892+0000 7fca1c9db740 1 -- 172.21.15.99:0/2229487494 >> 172.21.15.99:0/2229487494 conn(0x55c416913670 msgr2=0x55c4169148c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:00:24.897 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.892+0000 7fca1c9db740 1 -- 172.21.15.99:0/2229487494 shutdown_connections 2024-03-19T02:00:24.897 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.892+0000 7fca1c9db740 1 -- 172.21.15.99:0/2229487494 wait complete. 2024-03-19T02:00:24.897 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.892+0000 7fca1c9db740 20 client.26660 trim_cache size 0 max 16384 2024-03-19T02:00:24.909 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.907+0000 7fca1c9db740 1 Processor -- start 2024-03-19T02:00:24.909 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.908+0000 7fca1c9db740 1 -- start start 2024-03-19T02:00:24.909 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.908+0000 7fca1c9db740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c416918100 0x55c416912220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:24.909 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.908+0000 7fca1c9db740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c416918af0 0x55c416929610 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:24.909 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.908+0000 7fca1c9db740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c41691dc60 0x55c416912760 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:24.909 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.908+0000 7fca1c9db740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fca141ad800 con 0x55c416918af0 2024-03-19T02:00:24.909 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.908+0000 7fca1c9db740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fca141ab0d0 con 0x55c416918100 2024-03-19T02:00:24.910 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.908+0000 7fca1c9db740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fca141a8f20 con 0x55c41691dc60 2024-03-19T02:00:24.910 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.908+0000 7fca12ffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c416918af0 0x55c416929610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:24.910 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.908+0000 7fca12ffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c416918af0 0x55c416929610 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:56600/0 (socket says 172.21.15.99:56600) 2024-03-19T02:00:24.910 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.908+0000 7fca13fff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c41691dc60 0x55c416912760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:24.910 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.908+0000 7fca12ffd640 1 -- 172.21.15.99:0/1455164958 learned_addr learned my addr 172.21.15.99:0/1455164958 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:00:24.910 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.908+0000 7fca137fe640 1 --2- 172.21.15.99:0/1455164958 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c416918100 0x55c416912220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:24.910 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.908+0000 7fca12ffd640 1 -- 172.21.15.99:0/1455164958 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c41691dc60 msgr2=0x55c416912760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:00:24.910 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.908+0000 7fca12ffd640 1 --2- 172.21.15.99:0/1455164958 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c41691dc60 0x55c416912760 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:24.910 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.908+0000 7fca12ffd640 1 -- 172.21.15.99:0/1455164958 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c416918100 msgr2=0x55c416912220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:00:24.910 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.908+0000 7fca12ffd640 1 --2- 172.21.15.99:0/1455164958 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c416918100 0x55c416912220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:24.910 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.908+0000 7fca12ffd640 1 -- 172.21.15.99:0/1455164958 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55c41665d620 con 0x55c416918af0 2024-03-19T02:00:24.910 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.908+0000 7fca12ffd640 1 --2- 172.21.15.99:0/1455164958 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c416918af0 0x55c416929610 secure :-1 s=READY pgs=712 cs=0 l=1 rev1=1 crypto rx=0x7fca0802c170 tx=0x7fca08005b50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=60db9613dd4707bb server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:00:24.911 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.908+0000 7fc9f97fa640 1 -- 172.21.15.99:0/1455164958 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fca0805c030 con 0x55c416918af0 2024-03-19T02:00:24.911 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.908+0000 7fc9f97fa640 1 -- 172.21.15.99:0/1455164958 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fca0802cc90 con 0x55c416918af0 2024-03-19T02:00:24.911 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.909+0000 7fc9f97fa640 1 -- 172.21.15.99:0/1455164958 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fca080055f0 con 0x55c416918af0 2024-03-19T02:00:24.911 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.909+0000 7fca1c9db740 1 -- 172.21.15.99:0/1455164958 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c416918af0 msgr2=0x55c416929610 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:00:24.911 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.909+0000 7fca1c9db740 1 --2- 172.21.15.99:0/1455164958 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c416918af0 0x55c416929610 secure :-1 s=READY pgs=712 cs=0 l=1 rev1=1 crypto rx=0x7fca0802c170 tx=0x7fca08005b50 comp rx=0 tx=0).stop 2024-03-19T02:00:24.911 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.909+0000 7fca1c9db740 1 -- 172.21.15.99:0/1455164958 shutdown_connections 2024-03-19T02:00:24.911 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.909+0000 7fca1c9db740 1 --2- 172.21.15.99:0/1455164958 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c41691dc60 0x55c416912760 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:24.911 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.909+0000 7fca1c9db740 1 --2- 172.21.15.99:0/1455164958 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c416918af0 0x55c416929610 unknown :-1 s=CLOSED pgs=712 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:24.911 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.909+0000 7fca1c9db740 1 --2- 172.21.15.99:0/1455164958 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c416918100 0x55c416912220 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:24.911 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.909+0000 7fca1c9db740 1 -- 172.21.15.99:0/1455164958 >> 172.21.15.99:0/1455164958 conn(0x55c416913670 msgr2=0x55c416913a50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:00:24.911 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.909+0000 7fca1c9db740 1 -- 172.21.15.99:0/1455164958 shutdown_connections 2024-03-19T02:00:24.911 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.909+0000 7fca1c9db740 1 -- 172.21.15.99:0/1455164958 wait complete. 2024-03-19T02:00:24.911 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.910+0000 7fca1c9db740 1 -- start start 2024-03-19T02:00:24.914 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.910+0000 7fca1c9db740 1 Processor -- start 2024-03-19T02:00:24.914 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.910+0000 7fca1c9db740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c416918100 0x55c416926990 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:24.915 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.910+0000 7fca1c9db740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c416918af0 0x55c416926ed0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:24.915 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.910+0000 7fca1c9db740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c41691dc60 0x55c416927ee0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:24.915 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.910+0000 7fca1c9db740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fca141a9590 con 0x55c41691dc60 2024-03-19T02:00:24.915 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.910+0000 7fca1c9db740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fca141aaa00 con 0x55c416918100 2024-03-19T02:00:24.915 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.910+0000 7fca1c9db740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fca141a9c60 con 0x55c416918af0 2024-03-19T02:00:24.915 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.910+0000 7fca1c9db740 10 client.0 _ll_register_callbacks cb 0x7fca0c009b30 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T02:00:24.915 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.910+0000 7fca13fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c41691dc60 0x55c416927ee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:24.915 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.910+0000 7fca137fe640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c416918100 0x55c416926990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:24.915 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.910+0000 7fca13fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c41691dc60 0x55c416927ee0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:56616/0 (socket says 172.21.15.99:56616) 2024-03-19T02:00:24.915 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.910+0000 7fca13fff640 1 -- 172.21.15.99:0/3060811352 learned_addr learned my addr 172.21.15.99:0/3060811352 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:00:24.915 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.910+0000 7fca12ffd640 1 --2- 172.21.15.99:0/3060811352 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c416918af0 0x55c416926ed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:24.915 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.910+0000 7fca13fff640 1 -- 172.21.15.99:0/3060811352 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c416918af0 msgr2=0x55c416926ed0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:00:24.915 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.910+0000 7fca13fff640 1 --2- 172.21.15.99:0/3060811352 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c416918af0 0x55c416926ed0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:24.916 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.910+0000 7fca13fff640 1 -- 172.21.15.99:0/3060811352 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c416918100 msgr2=0x55c416926990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:00:24.916 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.910+0000 7fca13fff640 1 --2- 172.21.15.99:0/3060811352 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c416918100 0x55c416926990 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:24.916 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.910+0000 7fca13fff640 1 -- 172.21.15.99:0/3060811352 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55c4169284d0 con 0x55c41691dc60 2024-03-19T02:00:24.916 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.910+0000 7fca137fe640 1 --2- 172.21.15.99:0/3060811352 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c416918100 0x55c416926990 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:00:24.916 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.911+0000 7fca13fff640 1 --2- 172.21.15.99:0/3060811352 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c41691dc60 0x55c416927ee0 secure :-1 s=READY pgs=713 cs=0 l=1 rev1=1 crypto rx=0x7fca0c00fba0 tx=0x7fca0c004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=50076afdede6401c server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:00:24.916 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.911+0000 7fc9f2ffd640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T02:00:24.916 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.911+0000 7fc9f2ffd640 1 -- 172.21.15.99:0/3060811352 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fca0c0109b0 con 0x55c41691dc60 2024-03-19T02:00:24.916 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.911+0000 7fc9f2ffd640 1 -- 172.21.15.99:0/3060811352 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fca0c010b50 con 0x55c41691dc60 2024-03-19T02:00:24.916 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.911+0000 7fca1c9db740 1 -- 172.21.15.99:0/3060811352 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fca0802cc90 con 0x55c41691dc60 2024-03-19T02:00:24.916 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.911+0000 7fc9f2ffd640 1 -- 172.21.15.99:0/3060811352 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7fca0c010d50 con 0x55c41691dc60 2024-03-19T02:00:24.916 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.911+0000 7fca1c9db740 10 client.16890 fetch_fsmap learned FSMap version 37 2024-03-19T02:00:24.916 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.911+0000 7fca1c9db740 1 -- 172.21.15.99:0/3060811352 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x55c416929850 con 0x55c41691dc60 2024-03-19T02:00:24.916 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.911+0000 7fc9f2ffd640 1 -- 172.21.15.99:0/3060811352 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fca0802cc90 con 0x55c41691dc60 2024-03-19T02:00:24.917 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.911+0000 7fc9f2ffd640 1 -- 172.21.15.99:0/3060811352 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7fca0c004410 con 0x55c41691dc60 2024-03-19T02:00:24.917 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.911+0000 7fca1c9db740 10 client.16890 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T02:00:24.917 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.911+0000 7fca1c9db740 10 client.16890 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T02:00:24.917 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.911+0000 7fca1c9db740 20 client.16890 populate_metadata read hostname 'smithi099' 2024-03-19T02:00:24.917 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.911+0000 7fca1c9db740 1 --2- 172.21.15.99:0/3060811352 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55c416994520 0x55c416996940 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:24.917 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.911+0000 7fca1c9db740 4 client.16890 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T02:00:24.917 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.911+0000 7fca1c9db740 1 -- 172.21.15.99:0/3060811352 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55c416884470 con 0x55c416994520 2024-03-19T02:00:24.917 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.912+0000 7fca137fe640 1 --2- 172.21.15.99:0/3060811352 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55c416994520 0x55c416996940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:24.917 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.912+0000 7fc9f2ffd640 10 client.16890 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T02:00:24.917 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:24.912+0000 7fca137fe640 1 --2- 172.21.15.99:0/3060811352 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55c416994520 0x55c416996940 crc :-1 s=READY pgs=202 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=8069e22b4d5a75b server_cookie=ebda6efb73f28b43 in_seq=0 out_seq=0 2024-03-19T02:00:25.112 INFO:tasks.workunit.client.0.smithi099.stdout: 10 178921 22.78 MB/sec execute 294 sec latency 182.600 ms 2024-03-19T02:00:25.391 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:25.391+0000 7fc9f2ffd640 1 -- 172.21.15.99:0/3060811352 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x55c416884470 con 0x55c416994520 2024-03-19T02:00:25.392 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:25.391+0000 7fc9f2ffd640 10 client.16890 handle_command_reply: tid=0 2024-03-19T02:00:25.392 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:25.391+0000 7fca1c9db740 1 client.16890 shutdown 2024-03-19T02:00:25.392 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:25.391+0000 7fca1c9db740 1 -- 172.21.15.99:0/3060811352 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c41691dc60 msgr2=0x55c416927ee0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:00:25.392 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:25.391+0000 7fca1c9db740 1 --2- 172.21.15.99:0/3060811352 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c41691dc60 0x55c416927ee0 secure :-1 s=READY pgs=713 cs=0 l=1 rev1=1 crypto rx=0x7fca0c00fba0 tx=0x7fca0c004a40 comp rx=0 tx=0).stop 2024-03-19T02:00:25.392 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:25.391+0000 7fca1c9db740 1 -- 172.21.15.99:0/3060811352 shutdown_connections 2024-03-19T02:00:25.392 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:25.391+0000 7fca1c9db740 1 --2- 172.21.15.99:0/3060811352 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55c416994520 0x55c416996940 crc :-1 s=READY pgs=202 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:25.392 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:25.391+0000 7fc9f2ffd640 0 client.16890 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T02:00:25.392 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:25.391+0000 7fca1c9db740 1 --2- 172.21.15.99:0/3060811352 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c41691dc60 0x55c416927ee0 unknown :-1 s=CLOSED pgs=713 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:25.392 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:25.391+0000 7fca1c9db740 1 --2- 172.21.15.99:0/3060811352 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c416918af0 0x55c416926ed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:25.392 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:25.391+0000 7fca1c9db740 1 --2- 172.21.15.99:0/3060811352 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c416918100 0x55c416926990 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:25.392 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:25.391+0000 7fca1c9db740 1 -- 172.21.15.99:0/3060811352 >> 172.21.15.99:0/3060811352 conn(0x55c416913670 msgr2=0x55c416912220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:00:25.392 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:25.391+0000 7fca1c9db740 1 -- 172.21.15.99:0/3060811352 shutdown_connections 2024-03-19T02:00:25.392 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:25.391+0000 7fca1c9db740 1 -- 172.21.15.99:0/3060811352 wait complete. 2024-03-19T02:00:25.393 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:25.391+0000 7fca1c9db740 20 client.16890 trim_cache size 0 max 16384 2024-03-19T02:00:25.393 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T02:00:25.393 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T02:00:25.393 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T02:00:25.393 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T02:00:25.394 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:25.392+0000 7fca19d10640 1 -- 172.21.15.99:0/3021554020 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fc9fc0a2920 msgr2=0x7fc9fc0a4de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:00:25.394 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:25.392+0000 7fca19d10640 1 --2- 172.21.15.99:0/3021554020 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fc9fc0a2920 0x7fc9fc0a4de0 secure :-1 s=READY pgs=359 cs=0 l=1 rev1=1 crypto rx=0x55c416919350 tx=0x7fca0c009500 comp rx=0 tx=0).stop 2024-03-19T02:00:25.394 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:25.392+0000 7fca19d10640 1 -- 172.21.15.99:0/3021554020 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fca14100740 msgr2=0x7fca141046d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:00:25.394 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:25.392+0000 7fca19d10640 1 --2- 172.21.15.99:0/3021554020 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fca14100740 0x7fca141046d0 secure :-1 s=READY pgs=126 cs=0 l=1 rev1=1 crypto rx=0x7fca0400e9c0 tx=0x7fca0400ee80 comp rx=0 tx=0).stop 2024-03-19T02:00:25.394 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:25.392+0000 7fca19d10640 1 -- 172.21.15.99:0/3021554020 shutdown_connections 2024-03-19T02:00:25.394 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:25.392+0000 7fca19d10640 1 --2- 172.21.15.99:0/3021554020 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fc9fc0a2920 0x7fc9fc0a4de0 unknown :-1 s=CLOSED pgs=359 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:25.394 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:25.392+0000 7fca19d10640 1 --2- 172.21.15.99:0/3021554020 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fca141022f0 0x7fca14103250 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:25.394 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:25.392+0000 7fca19d10640 1 --2- 172.21.15.99:0/3021554020 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fca14101930 0x7fca14102d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:25.394 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:25.392+0000 7fca19d10640 1 --2- 172.21.15.99:0/3021554020 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fca14100740 0x7fca141046d0 unknown :-1 s=CLOSED pgs=126 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:25.394 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:25.392+0000 7fca19d10640 1 -- 172.21.15.99:0/3021554020 >> 172.21.15.99:0/3021554020 conn(0x7fca140fbcb0 msgr2=0x7fca140fd6f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:00:25.394 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:25.392+0000 7fca19d10640 1 -- 172.21.15.99:0/3021554020 shutdown_connections 2024-03-19T02:00:25.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:25.392+0000 7fca19d10640 1 -- 172.21.15.99:0/3021554020 wait complete. 2024-03-19T02:00:25.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:00:25 smithi115 ceph-mon[67332]: pgmap v931: 97 pgs: 97 active+clean; 613 MiB data, 7.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 5.5 MiB/s wr, 469 op/s 2024-03-19T02:00:25.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:00:25 smithi099 ceph-mon[62951]: pgmap v931: 97 pgs: 97 active+clean; 613 MiB data, 7.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 5.5 MiB/s wr, 469 op/s 2024-03-19T02:00:25.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:00:25 smithi112 ceph-mon[66819]: pgmap v931: 97 pgs: 97 active+clean; 613 MiB data, 7.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 5.5 MiB/s wr, 469 op/s 2024-03-19T02:00:26.112 INFO:tasks.workunit.client.0.smithi099.stdout: 10 179279 22.78 MB/sec execute 295 sec latency 394.516 ms 2024-03-19T02:00:27.112 INFO:tasks.workunit.client.0.smithi099.stdout: 10 179905 22.80 MB/sec execute 296 sec latency 68.684 ms 2024-03-19T02:00:27.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:00:27 smithi112 ceph-mon[66819]: pgmap v932: 97 pgs: 97 active+clean; 610 MiB data, 7.6 GiB used, 1.0 TiB / 1.0 TiB avail; 4.5 MiB/s rd, 6.1 MiB/s wr, 580 op/s 2024-03-19T02:00:27.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:00:27 smithi115 ceph-mon[67332]: pgmap v932: 97 pgs: 97 active+clean; 610 MiB data, 7.6 GiB used, 1.0 TiB / 1.0 TiB avail; 4.5 MiB/s rd, 6.1 MiB/s wr, 580 op/s 2024-03-19T02:00:27.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:00:27 smithi099 ceph-mon[62951]: pgmap v932: 97 pgs: 97 active+clean; 610 MiB data, 7.6 GiB used, 1.0 TiB / 1.0 TiB avail; 4.5 MiB/s rd, 6.1 MiB/s wr, 580 op/s 2024-03-19T02:00:28.112 INFO:tasks.workunit.client.0.smithi099.stdout: 10 180332 22.78 MB/sec execute 297 sec latency 248.815 ms 2024-03-19T02:00:29.112 INFO:tasks.workunit.client.0.smithi099.stdout: 10 180815 22.83 MB/sec execute 298 sec latency 102.509 ms 2024-03-19T02:00:29.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:00:29 smithi112 ceph-mon[66819]: pgmap v933: 97 pgs: 97 active+clean; 608 MiB data, 7.6 GiB used, 1.0 TiB / 1.0 TiB avail; 3.9 MiB/s rd, 5.1 MiB/s wr, 490 op/s 2024-03-19T02:00:29.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:00:29 smithi115 ceph-mon[67332]: pgmap v933: 97 pgs: 97 active+clean; 608 MiB data, 7.6 GiB used, 1.0 TiB / 1.0 TiB avail; 3.9 MiB/s rd, 5.1 MiB/s wr, 490 op/s 2024-03-19T02:00:29.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:00:29 smithi099 ceph-mon[62951]: pgmap v933: 97 pgs: 97 active+clean; 608 MiB data, 7.6 GiB used, 1.0 TiB / 1.0 TiB avail; 3.9 MiB/s rd, 5.1 MiB/s wr, 490 op/s 2024-03-19T02:00:30.112 INFO:tasks.workunit.client.0.smithi099.stdout: 10 181307 22.88 MB/sec execute 299 sec latency 56.343 ms 2024-03-19T02:00:31.112 INFO:tasks.workunit.client.0.smithi099.stdout: 10 181721 22.87 MB/sec execute 300 sec latency 78.293 ms 2024-03-19T02:00:31.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:00:31 smithi112 ceph-mon[66819]: pgmap v934: 97 pgs: 97 active+clean; 606 MiB data, 7.6 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 6.0 MiB/s wr, 552 op/s 2024-03-19T02:00:31.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:00:31 smithi115 ceph-mon[67332]: pgmap v934: 97 pgs: 97 active+clean; 606 MiB data, 7.6 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 6.0 MiB/s wr, 552 op/s 2024-03-19T02:00:31.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:00:31 smithi099 ceph-mon[62951]: pgmap v934: 97 pgs: 97 active+clean; 606 MiB data, 7.6 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 6.0 MiB/s wr, 552 op/s 2024-03-19T02:00:32.112 INFO:tasks.workunit.client.0.smithi099.stdout: 10 182123 22.85 MB/sec execute 301 sec latency 88.618 ms 2024-03-19T02:00:33.112 INFO:tasks.workunit.client.0.smithi099.stdout: 10 182387 22.83 MB/sec execute 302 sec latency 247.954 ms 2024-03-19T02:00:33.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:00:33 smithi115 ceph-mon[67332]: pgmap v935: 97 pgs: 97 active+clean; 611 MiB data, 7.6 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 6.7 MiB/s wr, 575 op/s 2024-03-19T02:00:33.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:00:33 smithi099 ceph-mon[62951]: pgmap v935: 97 pgs: 97 active+clean; 611 MiB data, 7.6 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 6.7 MiB/s wr, 575 op/s 2024-03-19T02:00:33.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:00:33 smithi112 ceph-mon[66819]: pgmap v935: 97 pgs: 97 active+clean; 611 MiB data, 7.6 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 6.7 MiB/s wr, 575 op/s 2024-03-19T02:00:34.112 INFO:tasks.workunit.client.0.smithi099.stdout: 10 182752 22.81 MB/sec execute 303 sec latency 145.475 ms 2024-03-19T02:00:34.842 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:00:34 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T02:00:34.861 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:00:34 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T02:00:35.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:00:34 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T02:00:35.112 INFO:tasks.workunit.client.0.smithi099.stdout: 10 183140 22.80 MB/sec execute 304 sec latency 84.644 ms 2024-03-19T02:00:35.840 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:00:35 smithi099 ceph-mon[62951]: pgmap v936: 97 pgs: 97 active+clean; 613 MiB data, 7.6 GiB used, 1.0 TiB / 1.0 TiB avail; 3.6 MiB/s rd, 5.9 MiB/s wr, 535 op/s 2024-03-19T02:00:35.840 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:00:35 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T02:00:35.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:00:35 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T02:00:35.841 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:00:35 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:00:35.861 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:00:35 smithi112 ceph-mon[66819]: pgmap v936: 97 pgs: 97 active+clean; 613 MiB data, 7.6 GiB used, 1.0 TiB / 1.0 TiB avail; 3.6 MiB/s rd, 5.9 MiB/s wr, 535 op/s 2024-03-19T02:00:35.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:00:35 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T02:00:35.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:00:35 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T02:00:35.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:00:35 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:00:36.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:00:35 smithi115 ceph-mon[67332]: pgmap v936: 97 pgs: 97 active+clean; 613 MiB data, 7.6 GiB used, 1.0 TiB / 1.0 TiB avail; 3.6 MiB/s rd, 5.9 MiB/s wr, 535 op/s 2024-03-19T02:00:36.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:00:35 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T02:00:36.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:00:35 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T02:00:36.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:00:35 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:00:36.112 INFO:tasks.workunit.client.0.smithi099.stdout: 10 183619 22.79 MB/sec execute 305 sec latency 78.736 ms 2024-03-19T02:00:37.112 INFO:tasks.workunit.client.0.smithi099.stdout: 10 184116 22.80 MB/sec execute 306 sec latency 67.886 ms 2024-03-19T02:00:37.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:00:37 smithi115 ceph-mon[67332]: pgmap v937: 97 pgs: 97 active+clean; 616 MiB data, 7.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 6.9 MiB/s wr, 577 op/s 2024-03-19T02:00:37.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:00:37 smithi099 ceph-mon[62951]: pgmap v937: 97 pgs: 97 active+clean; 616 MiB data, 7.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 6.9 MiB/s wr, 577 op/s 2024-03-19T02:00:37.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:00:37 smithi112 ceph-mon[66819]: pgmap v937: 97 pgs: 97 active+clean; 616 MiB data, 7.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 6.9 MiB/s wr, 577 op/s 2024-03-19T02:00:38.112 INFO:tasks.workunit.client.0.smithi099.stdout: 10 184558 22.81 MB/sec execute 307 sec latency 183.047 ms 2024-03-19T02:00:38.508 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.506+0000 7f6914297640 1 -- 172.21.15.99:0/583018538 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f690c0ffb00 msgr2=0x7f690c0fff20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:00:38.508 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.506+0000 7f6914297640 1 --2- 172.21.15.99:0/583018538 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f690c0ffb00 0x7f690c0fff20 secure :-1 s=READY pgs=201 cs=0 l=1 rev1=1 crypto rx=0x7f6908009860 tx=0x7f690802bbd0 comp rx=0 tx=0).stop 2024-03-19T02:00:38.508 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.507+0000 7f6914297640 1 -- 172.21.15.99:0/583018538 shutdown_connections 2024-03-19T02:00:38.508 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.507+0000 7f6914297640 1 --2- 172.21.15.99:0/583018538 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f690c103530 0x7f690c109dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:38.509 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.507+0000 7f6914297640 1 --2- 172.21.15.99:0/583018538 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f690c102b70 0x7f690c102ff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:38.509 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.507+0000 7f6914297640 1 --2- 172.21.15.99:0/583018538 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f690c0ffb00 0x7f690c0fff20 unknown :-1 s=CLOSED pgs=201 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:38.509 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.507+0000 7f6914297640 1 -- 172.21.15.99:0/583018538 >> 172.21.15.99:0/583018538 conn(0x7f690c0fb470 msgr2=0x7f690c0fd890 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:00:38.509 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.507+0000 7f6914297640 1 -- 172.21.15.99:0/583018538 shutdown_connections 2024-03-19T02:00:38.509 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.507+0000 7f6914297640 1 -- 172.21.15.99:0/583018538 wait complete. 2024-03-19T02:00:38.509 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.507+0000 7f6914297640 1 Processor -- start 2024-03-19T02:00:38.509 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.507+0000 7f6914297640 1 -- start start 2024-03-19T02:00:38.509 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.507+0000 7f6914297640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f690c0ffb00 0x7f690c102490 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:38.509 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.508+0000 7f6914297640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f690c102b70 0x7f690c100b20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:38.509 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.508+0000 7f6914297640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f690c103530 0x7f690c101060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:38.509 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.508+0000 7f6914297640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f690c10d4d0 con 0x7f690c103530 2024-03-19T02:00:38.509 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.508+0000 7f6914297640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f690c10d350 con 0x7f690c102b70 2024-03-19T02:00:38.509 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.508+0000 7f6914297640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f690c10d650 con 0x7f690c0ffb00 2024-03-19T02:00:38.509 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.508+0000 7f6912834640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f690c103530 0x7f690c101060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:38.510 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.508+0000 7f6912834640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f690c103530 0x7f690c101060 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:41898/0 (socket says 172.21.15.99:41898) 2024-03-19T02:00:38.510 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.508+0000 7f6912834640 1 -- 172.21.15.99:0/3269907704 learned_addr learned my addr 172.21.15.99:0/3269907704 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:00:38.510 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.508+0000 7f6912033640 1 --2- 172.21.15.99:0/3269907704 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f690c0ffb00 0x7f690c102490 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:38.510 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.508+0000 7f6911832640 1 --2- 172.21.15.99:0/3269907704 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f690c102b70 0x7f690c100b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:38.510 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.508+0000 7f6912834640 1 -- 172.21.15.99:0/3269907704 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f690c0ffb00 msgr2=0x7f690c102490 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:00:38.510 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.508+0000 7f6912834640 1 --2- 172.21.15.99:0/3269907704 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f690c0ffb00 0x7f690c102490 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:38.510 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.508+0000 7f6912834640 1 -- 172.21.15.99:0/3269907704 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f690c102b70 msgr2=0x7f690c100b20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:00:38.510 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.508+0000 7f6912834640 1 --2- 172.21.15.99:0/3269907704 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f690c102b70 0x7f690c100b20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:38.510 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.508+0000 7f6912834640 1 -- 172.21.15.99:0/3269907704 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f690c101920 con 0x7f690c103530 2024-03-19T02:00:38.510 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.508+0000 7f6912834640 1 --2- 172.21.15.99:0/3269907704 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f690c103530 0x7f690c101060 secure :-1 s=READY pgs=714 cs=0 l=1 rev1=1 crypto rx=0x7f6904009fd0 tx=0x7f690400c800 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:00:38.510 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.508+0000 7f69037fe640 1 -- 172.21.15.99:0/3269907704 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6904004b20 con 0x7f690c103530 2024-03-19T02:00:38.510 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.508+0000 7f69037fe640 1 -- 172.21.15.99:0/3269907704 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f6904003e50 con 0x7f690c103530 2024-03-19T02:00:38.510 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.508+0000 7f6914297640 1 -- 172.21.15.99:0/3269907704 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f690c1a1c60 con 0x7f690c103530 2024-03-19T02:00:38.511 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.508+0000 7f69037fe640 1 -- 172.21.15.99:0/3269907704 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f690400b630 con 0x7f690c103530 2024-03-19T02:00:38.511 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.508+0000 7f6914297640 1 -- 172.21.15.99:0/3269907704 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f690c1a2130 con 0x7f690c103530 2024-03-19T02:00:38.514 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.511+0000 7f69037fe640 1 -- 172.21.15.99:0/3269907704 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f690401d020 con 0x7f690c103530 2024-03-19T02:00:38.514 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.511+0000 7f6916b22740 1 Processor -- start 2024-03-19T02:00:38.514 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.511+0000 7f6916b22740 1 -- start start 2024-03-19T02:00:38.514 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.512+0000 7f6916b22740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55f2e8b05100 0x55f2e8b05520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:38.514 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.512+0000 7f6916b22740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55f2e8b05af0 0x55f2e8b0a5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:38.514 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.512+0000 7f6916b22740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55f2e8b0ac60 0x55f2e8b0d0f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:38.514 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.512+0000 7f6916b22740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55f2e8a69be0 con 0x55f2e8b05af0 2024-03-19T02:00:38.514 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.512+0000 7f6916b22740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55f2e8a12960 con 0x55f2e8b0ac60 2024-03-19T02:00:38.514 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.512+0000 7f6916b22740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55f2e8a65c80 con 0x55f2e8b05100 2024-03-19T02:00:38.515 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.512+0000 7f69037fe640 1 --2- 172.21.15.99:0/3269907704 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f68f80a28a0 0x7f68f80a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:38.515 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.512+0000 7f6912033640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55f2e8b05af0 0x55f2e8b0a5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:38.515 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.512+0000 7f6912033640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55f2e8b05af0 0x55f2e8b0a5f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:41912/0 (socket says 172.21.15.99:41912) 2024-03-19T02:00:38.515 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.512+0000 7f6912033640 1 -- 172.21.15.99:0/1799895958 learned_addr learned my addr 172.21.15.99:0/1799895958 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:00:38.515 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.512+0000 7f6911832640 1 --2- 172.21.15.99:0/1799895958 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55f2e8b0ac60 0x55f2e8b0d0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:38.515 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.512+0000 7f6912834640 1 --2- 172.21.15.99:0/3269907704 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f68f80a28a0 0x7f68f80a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:38.515 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.512+0000 7f6911832640 1 --2- 172.21.15.99:0/1799895958 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55f2e8b05100 0x55f2e8b05520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:38.515 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.512+0000 7f69037fe640 1 -- 172.21.15.99:0/3269907704 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f69040cea70 con 0x7f690c103530 2024-03-19T02:00:38.516 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.512+0000 7f6912033640 1 -- 172.21.15.99:0/1799895958 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55f2e8b05100 msgr2=0x55f2e8b05520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:00:38.516 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.512+0000 7f6912033640 1 --2- 172.21.15.99:0/1799895958 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55f2e8b05100 0x55f2e8b05520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:38.516 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.512+0000 7f6912033640 1 -- 172.21.15.99:0/1799895958 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55f2e8b0ac60 msgr2=0x55f2e8b0d0f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:00:38.516 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.512+0000 7f6912033640 1 --2- 172.21.15.99:0/1799895958 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55f2e8b0ac60 0x55f2e8b0d0f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:38.516 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.512+0000 7f6912033640 1 -- 172.21.15.99:0/1799895958 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55f2e8a09d20 con 0x55f2e8b05af0 2024-03-19T02:00:38.516 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.512+0000 7f6911832640 1 --2- 172.21.15.99:0/1799895958 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55f2e8b0ac60 0x55f2e8b0d0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:00:38.516 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.512+0000 7f6912033640 1 --2- 172.21.15.99:0/1799895958 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55f2e8b05af0 0x55f2e8b0a5f0 secure :-1 s=READY pgs=715 cs=0 l=1 rev1=1 crypto rx=0x7f69080057c0 tx=0x7f6908005b20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c60eb459c68d4a04 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:00:38.516 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.512+0000 7f6912834640 1 --2- 172.21.15.99:0/3269907704 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f68f80a28a0 0x7f68f80a4d60 secure :-1 s=READY pgs=360 cs=0 l=1 rev1=1 crypto rx=0x7f690409b360 tx=0x7f6904019b40 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:00:38.516 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.513+0000 7f6900bfe640 1 -- 172.21.15.99:0/1799895958 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6908039040 con 0x55f2e8b05af0 2024-03-19T02:00:38.516 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.513+0000 7f6900bfe640 1 -- 172.21.15.99:0/1799895958 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f6908034db0 con 0x55f2e8b05af0 2024-03-19T02:00:38.516 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.513+0000 7f6900bfe640 1 -- 172.21.15.99:0/1799895958 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f690802c840 con 0x55f2e8b05af0 2024-03-19T02:00:38.516 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.513+0000 7f6916b22740 1 -- 172.21.15.99:0/1799895958 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55f2e8b05af0 msgr2=0x55f2e8b0a5f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:00:38.516 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.513+0000 7f6916b22740 1 --2- 172.21.15.99:0/1799895958 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55f2e8b05af0 0x55f2e8b0a5f0 secure :-1 s=READY pgs=715 cs=0 l=1 rev1=1 crypto rx=0x7f69080057c0 tx=0x7f6908005b20 comp rx=0 tx=0).stop 2024-03-19T02:00:38.517 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.513+0000 7f6916b22740 1 -- 172.21.15.99:0/1799895958 shutdown_connections 2024-03-19T02:00:38.517 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.513+0000 7f6916b22740 1 --2- 172.21.15.99:0/1799895958 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55f2e8b0ac60 0x55f2e8b0d0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:38.517 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.514+0000 7f6916b22740 1 --2- 172.21.15.99:0/1799895958 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55f2e8b05af0 0x55f2e8b0a5f0 unknown :-1 s=CLOSED pgs=715 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:38.517 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.514+0000 7f6916b22740 1 --2- 172.21.15.99:0/1799895958 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55f2e8b05100 0x55f2e8b05520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:38.517 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.514+0000 7f6916b22740 1 -- 172.21.15.99:0/1799895958 >> 172.21.15.99:0/1799895958 conn(0x55f2e8b00670 msgr2=0x55f2e8b00a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:00:38.517 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.514+0000 7f6916b22740 1 -- 172.21.15.99:0/1799895958 shutdown_connections 2024-03-19T02:00:38.517 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.514+0000 7f6916b22740 1 -- 172.21.15.99:0/1799895958 wait complete. 2024-03-19T02:00:38.517 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.514+0000 7f6916b22740 1 -- start start 2024-03-19T02:00:38.517 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.514+0000 7f6916b22740 1 Processor -- start 2024-03-19T02:00:38.520 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.515+0000 7f6916b22740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55f2e8b05100 0x55f2e8b0f070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:38.520 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.515+0000 7f6911832640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55f2e8b05100 0x55f2e8b0f070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:38.520 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.515+0000 7f6911832640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55f2e8b05100 0x55f2e8b0f070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:41924/0 (socket says 172.21.15.99:41924) 2024-03-19T02:00:38.520 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.515+0000 7f6916b22740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55f2e8b05af0 0x55f2e8b0f5b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:38.520 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.515+0000 7f6911832640 1 -- 172.21.15.99:0/1516284028 learned_addr learned my addr 172.21.15.99:0/1516284028 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:00:38.520 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.515+0000 7f6916b22740 1 --2- 172.21.15.99:0/1516284028 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55f2e8b0ac60 0x55f2e8b165f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:38.520 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.515+0000 7f6916b22740 1 -- 172.21.15.99:0/1516284028 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55f2e8b0dbb0 con 0x55f2e8b05100 2024-03-19T02:00:38.520 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.515+0000 7f6916b22740 1 -- 172.21.15.99:0/1516284028 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55f2e8a25db0 con 0x55f2e8b05af0 2024-03-19T02:00:38.520 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.515+0000 7f6916b22740 1 -- 172.21.15.99:0/1516284028 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55f2e8b0dd30 con 0x55f2e8b0ac60 2024-03-19T02:00:38.521 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.515+0000 7f6912033640 1 --2- 172.21.15.99:0/1516284028 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55f2e8b05af0 0x55f2e8b0f5b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:38.521 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.515+0000 7f6916b22740 10 client.0 _ll_register_callbacks cb 0x55f2e8a2ae70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T02:00:38.521 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.515+0000 7f6911832640 1 -- 172.21.15.99:0/1516284028 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55f2e8b0ac60 msgr2=0x55f2e8b165f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:00:38.521 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.515+0000 7f6911832640 1 --2- 172.21.15.99:0/1516284028 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55f2e8b0ac60 0x55f2e8b165f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:38.521 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.515+0000 7f6911832640 1 -- 172.21.15.99:0/1516284028 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55f2e8b05af0 msgr2=0x55f2e8b0f5b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:00:38.521 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.515+0000 7f6911832640 1 --2- 172.21.15.99:0/1516284028 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55f2e8b05af0 0x55f2e8b0f5b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:38.521 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.515+0000 7f6911832640 1 -- 172.21.15.99:0/1516284028 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55f2e8a24c60 con 0x55f2e8b05100 2024-03-19T02:00:38.521 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.516+0000 7f6911832640 1 --2- 172.21.15.99:0/1516284028 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55f2e8b05100 0x55f2e8b0f070 secure :-1 s=READY pgs=716 cs=0 l=1 rev1=1 crypto rx=0x7f68fc005880 tx=0x7f68fc003a80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e8bec11763f6982b server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:00:38.521 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.516+0000 7f68f5ffb640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T02:00:38.521 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.516+0000 7f68f5ffb640 1 -- 172.21.15.99:0/1516284028 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f68fc012950 con 0x55f2e8b05100 2024-03-19T02:00:38.521 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.516+0000 7f68f5ffb640 1 -- 172.21.15.99:0/1516284028 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f68fc012af0 con 0x55f2e8b05100 2024-03-19T02:00:38.521 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.516+0000 7f6916b22740 1 -- 172.21.15.99:0/1516284028 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f6908034db0 con 0x55f2e8b05100 2024-03-19T02:00:38.521 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.516+0000 7f68f5ffb640 1 -- 172.21.15.99:0/1516284028 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7f68fc012cf0 con 0x55f2e8b05100 2024-03-19T02:00:38.522 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.516+0000 7f6916b22740 10 client.16902 fetch_fsmap learned FSMap version 37 2024-03-19T02:00:38.522 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.516+0000 7f6916b22740 1 -- 172.21.15.99:0/1516284028 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x55f2e877e210 con 0x55f2e8b05100 2024-03-19T02:00:38.522 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.517+0000 7f68f5ffb640 1 -- 172.21.15.99:0/1516284028 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f6908034db0 con 0x55f2e8b05100 2024-03-19T02:00:38.522 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.517+0000 7f68f5ffb640 1 -- 172.21.15.99:0/1516284028 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7f68fc029490 con 0x55f2e8b05100 2024-03-19T02:00:38.522 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.517+0000 7f6916b22740 10 client.16902 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T02:00:38.522 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.517+0000 7f6916b22740 10 client.16902 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T02:00:38.522 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.517+0000 7f6916b22740 20 client.16902 populate_metadata read hostname 'smithi099' 2024-03-19T02:00:38.522 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.517+0000 7f6916b22740 1 --2- 172.21.15.99:0/1516284028 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55f2e8b1ae80 0x55f2e8b1d2f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:38.522 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.517+0000 7f6916b22740 4 client.16902 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T02:00:38.522 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.517+0000 7f6916b22740 1 -- 172.21.15.99:0/1516284028 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55f2e8a77120 con 0x55f2e8b1ae80 2024-03-19T02:00:38.522 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.518+0000 7f6912033640 1 --2- 172.21.15.99:0/1516284028 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55f2e8b1ae80 0x55f2e8b1d2f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:38.522 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.519+0000 7f68f5ffb640 10 client.16902 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T02:00:38.522 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.519+0000 7f6912033640 1 --2- 172.21.15.99:0/1516284028 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55f2e8b1ae80 0x55f2e8b1d2f0 crc :-1 s=READY pgs=203 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=bcaa357a2293f3a0 server_cookie=6b1ebc38b13507f4 in_seq=0 out_seq=0 2024-03-19T02:00:38.524 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.522+0000 7f68f5ffb640 1 -- 172.21.15.99:0/1516284028 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x55f2e8a77120 con 0x55f2e8b1ae80 2024-03-19T02:00:38.524 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.522+0000 7f68f5ffb640 10 client.16902 handle_command_reply: tid=0 2024-03-19T02:00:38.524 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.522+0000 7f6916b22740 1 client.16902 shutdown 2024-03-19T02:00:38.524 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.522+0000 7f6916b22740 1 -- 172.21.15.99:0/1516284028 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55f2e8b05100 msgr2=0x55f2e8b0f070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:00:38.524 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.522+0000 7f6916b22740 1 --2- 172.21.15.99:0/1516284028 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55f2e8b05100 0x55f2e8b0f070 secure :-1 s=READY pgs=716 cs=0 l=1 rev1=1 crypto rx=0x7f68fc005880 tx=0x7f68fc003a80 comp rx=0 tx=0).stop 2024-03-19T02:00:38.524 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.522+0000 7f6916b22740 1 -- 172.21.15.99:0/1516284028 shutdown_connections 2024-03-19T02:00:38.524 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.522+0000 7f6916b22740 1 --2- 172.21.15.99:0/1516284028 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55f2e8b1ae80 0x55f2e8b1d2f0 crc :-1 s=READY pgs=203 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:38.525 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.522+0000 7f6916b22740 1 --2- 172.21.15.99:0/1516284028 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55f2e8b05af0 0x55f2e8b0f5b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:38.525 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.522+0000 7f68f5ffb640 0 client.16902 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T02:00:38.525 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.522+0000 7f6916b22740 1 --2- 172.21.15.99:0/1516284028 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55f2e8b0ac60 0x55f2e8b165f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:38.525 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.522+0000 7f6916b22740 1 --2- 172.21.15.99:0/1516284028 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55f2e8b05100 0x55f2e8b0f070 unknown :-1 s=CLOSED pgs=716 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:38.525 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.522+0000 7f6916b22740 1 -- 172.21.15.99:0/1516284028 >> 172.21.15.99:0/1516284028 conn(0x55f2e8b00670 msgr2=0x55f2e8b018c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:00:38.525 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.522+0000 7f6916b22740 1 -- 172.21.15.99:0/1516284028 shutdown_connections 2024-03-19T02:00:38.525 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.522+0000 7f6916b22740 1 -- 172.21.15.99:0/1516284028 wait complete. 2024-03-19T02:00:38.525 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.522+0000 7f6916b22740 20 client.16902 trim_cache size 0 max 16384 2024-03-19T02:00:38.545 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.541+0000 7f6916b22740 1 Processor -- start 2024-03-19T02:00:38.546 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.541+0000 7f6916b22740 1 -- start start 2024-03-19T02:00:38.546 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.541+0000 7f6916b22740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55f2e8b05100 0x55f2e8affab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:38.546 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.541+0000 7f6916b22740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55f2e8b05af0 0x55f2e8b165b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:38.546 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.541+0000 7f6916b22740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55f2e8b0ac60 0x55f2e8b06ca0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:38.546 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.541+0000 7f6916b22740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f690c1ad890 con 0x55f2e8b0ac60 2024-03-19T02:00:38.546 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.541+0000 7f6916b22740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f690c1ab160 con 0x55f2e8b05af0 2024-03-19T02:00:38.546 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.541+0000 7f6916b22740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f690c1a8fb0 con 0x55f2e8b05100 2024-03-19T02:00:38.546 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.542+0000 7f6912033640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55f2e8b05af0 0x55f2e8b165b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:38.546 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.542+0000 7f6911832640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55f2e8b0ac60 0x55f2e8b06ca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:38.546 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.542+0000 7f6912033640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55f2e8b05af0 0x55f2e8b165b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:34982/0 (socket says 172.21.15.99:34982) 2024-03-19T02:00:38.546 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.542+0000 7f6912033640 1 -- 172.21.15.99:0/535382318 learned_addr learned my addr 172.21.15.99:0/535382318 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:00:38.546 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.542+0000 7f6911832640 1 --2- 172.21.15.99:0/535382318 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55f2e8b05100 0x55f2e8affab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:38.547 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.542+0000 7f6911832640 1 -- 172.21.15.99:0/535382318 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55f2e8b05100 msgr2=0x55f2e8affab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:00:38.547 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.542+0000 7f6911832640 1 --2- 172.21.15.99:0/535382318 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55f2e8b05100 0x55f2e8affab0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:38.547 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.542+0000 7f6911832640 1 -- 172.21.15.99:0/535382318 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55f2e8b05af0 msgr2=0x55f2e8b165b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:00:38.547 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.542+0000 7f6911832640 1 --2- 172.21.15.99:0/535382318 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55f2e8b05af0 0x55f2e8b165b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:38.547 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.542+0000 7f6911832640 1 -- 172.21.15.99:0/535382318 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55f2e884a620 con 0x55f2e8b0ac60 2024-03-19T02:00:38.547 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.542+0000 7f6912033640 1 --2- 172.21.15.99:0/535382318 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55f2e8b05af0 0x55f2e8b165b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:00:38.547 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.542+0000 7f6911832640 1 --2- 172.21.15.99:0/535382318 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55f2e8b0ac60 0x55f2e8b06ca0 secure :-1 s=READY pgs=717 cs=0 l=1 rev1=1 crypto rx=0x7f68fc0097a0 tx=0x7f68fc014040 comp rx=0 tx=0).ready entity=mon.0 client_cookie=32abfd64a86263d3 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:00:38.547 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.542+0000 7f68f7fff640 1 -- 172.21.15.99:0/535382318 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f68fc000e60 con 0x55f2e8b0ac60 2024-03-19T02:00:38.547 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.542+0000 7f68f7fff640 1 -- 172.21.15.99:0/535382318 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f68fc025030 con 0x55f2e8b0ac60 2024-03-19T02:00:38.547 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.542+0000 7f68f7fff640 1 -- 172.21.15.99:0/535382318 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f68fc029d80 con 0x55f2e8b0ac60 2024-03-19T02:00:38.547 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.543+0000 7f6916b22740 1 -- 172.21.15.99:0/535382318 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55f2e8b0ac60 msgr2=0x55f2e8b06ca0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:00:38.547 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.543+0000 7f6916b22740 1 --2- 172.21.15.99:0/535382318 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55f2e8b0ac60 0x55f2e8b06ca0 secure :-1 s=READY pgs=717 cs=0 l=1 rev1=1 crypto rx=0x7f68fc0097a0 tx=0x7f68fc014040 comp rx=0 tx=0).stop 2024-03-19T02:00:38.547 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.543+0000 7f6916b22740 1 -- 172.21.15.99:0/535382318 shutdown_connections 2024-03-19T02:00:38.548 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.543+0000 7f6916b22740 1 --2- 172.21.15.99:0/535382318 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55f2e8b05af0 0x55f2e8b165b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:38.548 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.543+0000 7f6916b22740 1 --2- 172.21.15.99:0/535382318 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55f2e8b0ac60 0x55f2e8b06ca0 unknown :-1 s=CLOSED pgs=717 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:38.548 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.543+0000 7f6916b22740 1 --2- 172.21.15.99:0/535382318 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55f2e8b05100 0x55f2e8affab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:38.548 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.543+0000 7f6916b22740 1 -- 172.21.15.99:0/535382318 >> 172.21.15.99:0/535382318 conn(0x55f2e8b00670 msgr2=0x55f2e8b06590 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:00:38.548 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.543+0000 7f6916b22740 1 -- 172.21.15.99:0/535382318 shutdown_connections 2024-03-19T02:00:38.548 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.543+0000 7f6916b22740 1 -- 172.21.15.99:0/535382318 wait complete. 2024-03-19T02:00:38.548 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.544+0000 7f6916b22740 1 -- start start 2024-03-19T02:00:38.548 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.544+0000 7f6916b22740 1 Processor -- start 2024-03-19T02:00:38.548 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.544+0000 7f6916b22740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55f2e8b05100 0x55f2e8b13a10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:38.548 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.544+0000 7f6916b22740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55f2e8b05af0 0x55f2e8b13f50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:38.548 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.544+0000 7f6916b22740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55f2e8b0ac60 0x55f2e8b14f60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:38.548 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.544+0000 7f6916b22740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f690c1a9620 con 0x55f2e8b0ac60 2024-03-19T02:00:38.548 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.544+0000 7f6916b22740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f690c1aaa90 con 0x55f2e8b05af0 2024-03-19T02:00:38.548 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.544+0000 7f6916b22740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f690c1a9cf0 con 0x55f2e8b05100 2024-03-19T02:00:38.548 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.544+0000 7f6916b22740 10 client.0 _ll_register_callbacks cb 0x7f68fc009cc0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T02:00:38.549 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.544+0000 7f6911832640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55f2e8b0ac60 0x55f2e8b14f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:38.549 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.544+0000 7f6912033640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55f2e8b05af0 0x55f2e8b13f50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:38.549 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.544+0000 7f6912033640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55f2e8b05af0 0x55f2e8b13f50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:34998/0 (socket says 172.21.15.99:34998) 2024-03-19T02:00:38.549 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.544+0000 7f6912033640 1 -- 172.21.15.99:0/2912592998 learned_addr learned my addr 172.21.15.99:0/2912592998 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:00:38.549 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.544+0000 7f6911832640 1 --2- 172.21.15.99:0/2912592998 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55f2e8b05100 0x55f2e8b13a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:38.549 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.544+0000 7f6911832640 1 -- 172.21.15.99:0/2912592998 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55f2e8b05100 msgr2=0x55f2e8b13a10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:00:38.549 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.544+0000 7f6911832640 1 --2- 172.21.15.99:0/2912592998 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55f2e8b05100 0x55f2e8b13a10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:38.549 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.544+0000 7f6911832640 1 -- 172.21.15.99:0/2912592998 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55f2e8b05af0 msgr2=0x55f2e8b13f50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:00:38.549 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.544+0000 7f6911832640 1 --2- 172.21.15.99:0/2912592998 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55f2e8b05af0 0x55f2e8b13f50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:38.549 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.544+0000 7f6911832640 1 -- 172.21.15.99:0/2912592998 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55f2e8b15550 con 0x55f2e8b0ac60 2024-03-19T02:00:38.549 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.544+0000 7f6912033640 1 --2- 172.21.15.99:0/2912592998 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55f2e8b05af0 0x55f2e8b13f50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:00:38.549 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.545+0000 7f6911832640 1 --2- 172.21.15.99:0/2912592998 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55f2e8b0ac60 0x55f2e8b14f60 secure :-1 s=READY pgs=718 cs=0 l=1 rev1=1 crypto rx=0x7f68fc009b00 tx=0x7f68fc004590 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8cda7ca384f20dc9 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:00:38.549 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.545+0000 7f68f5ffb640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T02:00:38.550 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.545+0000 7f68f5ffb640 1 -- 172.21.15.99:0/2912592998 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f68fc029d20 con 0x55f2e8b0ac60 2024-03-19T02:00:38.550 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.545+0000 7f68f5ffb640 1 -- 172.21.15.99:0/2912592998 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f68fc0293a0 con 0x55f2e8b0ac60 2024-03-19T02:00:38.550 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.545+0000 7f6916b22740 1 -- 172.21.15.99:0/2912592998 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f68fc025030 con 0x55f2e8b0ac60 2024-03-19T02:00:38.550 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.545+0000 7f68f5ffb640 1 -- 172.21.15.99:0/2912592998 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7f68fc0295a0 con 0x55f2e8b0ac60 2024-03-19T02:00:38.550 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.545+0000 7f6916b22740 10 client.16908 fetch_fsmap learned FSMap version 37 2024-03-19T02:00:38.550 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.545+0000 7f6916b22740 1 -- 172.21.15.99:0/2912592998 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x55f2e8b16860 con 0x55f2e8b0ac60 2024-03-19T02:00:38.550 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.545+0000 7f68f5ffb640 1 -- 172.21.15.99:0/2912592998 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f68fc025030 con 0x55f2e8b0ac60 2024-03-19T02:00:38.550 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.545+0000 7f68f5ffb640 1 -- 172.21.15.99:0/2912592998 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7f68fc039040 con 0x55f2e8b0ac60 2024-03-19T02:00:38.550 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.545+0000 7f6916b22740 10 client.16908 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T02:00:38.550 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.545+0000 7f6916b22740 10 client.16908 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T02:00:38.550 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.545+0000 7f6916b22740 20 client.16908 populate_metadata read hostname 'smithi099' 2024-03-19T02:00:38.550 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.545+0000 7f6916b22740 1 --2- 172.21.15.99:0/2912592998 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55f2e8b1ae80 0x55f2e8b83890 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:38.550 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.545+0000 7f6916b22740 4 client.16908 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T02:00:38.551 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.545+0000 7f6916b22740 1 -- 172.21.15.99:0/2912592998 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55f2e8a71470 con 0x55f2e8b1ae80 2024-03-19T02:00:38.551 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.546+0000 7f6912033640 1 --2- 172.21.15.99:0/2912592998 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55f2e8b1ae80 0x55f2e8b83890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:38.551 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.547+0000 7f6912033640 1 --2- 172.21.15.99:0/2912592998 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55f2e8b1ae80 0x55f2e8b83890 crc :-1 s=READY pgs=204 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=ec8c55bb30733dd4 server_cookie=b2a2f5c286e05626 in_seq=0 out_seq=0 2024-03-19T02:00:38.551 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:38.547+0000 7f68f5ffb640 10 client.16908 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T02:00:39.112 INFO:tasks.workunit.client.0.smithi099.stdout: 10 184763 22.76 MB/sec execute 308 sec latency 279.489 ms 2024-03-19T02:00:39.207 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:39.207+0000 7f68f5ffb640 1 -- 172.21.15.99:0/2912592998 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x55f2e8a71470 con 0x55f2e8b1ae80 2024-03-19T02:00:39.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:39.207+0000 7f68f5ffb640 10 client.16908 handle_command_reply: tid=0 2024-03-19T02:00:39.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:39.207+0000 7f6916b22740 1 client.16908 shutdown 2024-03-19T02:00:39.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:39.207+0000 7f6916b22740 1 -- 172.21.15.99:0/2912592998 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55f2e8b0ac60 msgr2=0x55f2e8b14f60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:00:39.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:39.207+0000 7f6916b22740 1 --2- 172.21.15.99:0/2912592998 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55f2e8b0ac60 0x55f2e8b14f60 secure :-1 s=READY pgs=718 cs=0 l=1 rev1=1 crypto rx=0x7f68fc009b00 tx=0x7f68fc004590 comp rx=0 tx=0).stop 2024-03-19T02:00:39.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:39.207+0000 7f6916b22740 1 -- 172.21.15.99:0/2912592998 shutdown_connections 2024-03-19T02:00:39.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:39.207+0000 7f6916b22740 1 --2- 172.21.15.99:0/2912592998 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55f2e8b1ae80 0x55f2e8b83890 crc :-1 s=READY pgs=204 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:39.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:39.207+0000 7f68f5ffb640 0 client.16908 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T02:00:39.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:39.207+0000 7f6916b22740 1 --2- 172.21.15.99:0/2912592998 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55f2e8b05af0 0x55f2e8b13f50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:39.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:39.207+0000 7f6916b22740 1 --2- 172.21.15.99:0/2912592998 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55f2e8b0ac60 0x55f2e8b14f60 unknown :-1 s=CLOSED pgs=718 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:39.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:39.207+0000 7f6916b22740 1 --2- 172.21.15.99:0/2912592998 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55f2e8b05100 0x55f2e8b13a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:39.208 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:39.207+0000 7f6916b22740 1 -- 172.21.15.99:0/2912592998 >> 172.21.15.99:0/2912592998 conn(0x55f2e8b00670 msgr2=0x55f2e8b06e20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:00:39.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:39.207+0000 7f6916b22740 1 -- 172.21.15.99:0/2912592998 shutdown_connections 2024-03-19T02:00:39.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:39.208+0000 7f6916b22740 1 -- 172.21.15.99:0/2912592998 wait complete. 2024-03-19T02:00:39.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:39.208+0000 7f6916b22740 20 client.16908 trim_cache size 0 max 16384 2024-03-19T02:00:39.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T02:00:39.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T02:00:39.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T02:00:39.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T02:00:39.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:39.208+0000 7f6914297640 1 -- 172.21.15.99:0/3269907704 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f68f80a28a0 msgr2=0x7f68f80a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:00:39.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:39.208+0000 7f6914297640 1 --2- 172.21.15.99:0/3269907704 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f68f80a28a0 0x7f68f80a4d60 secure :-1 s=READY pgs=360 cs=0 l=1 rev1=1 crypto rx=0x7f690409b360 tx=0x7f6904019b40 comp rx=0 tx=0).stop 2024-03-19T02:00:39.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:39.208+0000 7f6914297640 1 -- 172.21.15.99:0/3269907704 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f690c103530 msgr2=0x7f690c101060 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:00:39.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:39.208+0000 7f6914297640 1 --2- 172.21.15.99:0/3269907704 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f690c103530 0x7f690c101060 secure :-1 s=READY pgs=714 cs=0 l=1 rev1=1 crypto rx=0x7f6904009fd0 tx=0x7f690400c800 comp rx=0 tx=0).stop 2024-03-19T02:00:39.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:39.208+0000 7f6914297640 1 -- 172.21.15.99:0/3269907704 shutdown_connections 2024-03-19T02:00:39.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:39.208+0000 7f6914297640 1 --2- 172.21.15.99:0/3269907704 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f68f80a28a0 0x7f68f80a4d60 unknown :-1 s=CLOSED pgs=360 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:39.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:39.208+0000 7f6914297640 1 --2- 172.21.15.99:0/3269907704 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f690c102b70 0x7f690c100b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:39.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:39.208+0000 7f6914297640 1 --2- 172.21.15.99:0/3269907704 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f690c103530 0x7f690c101060 unknown :-1 s=CLOSED pgs=714 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:39.211 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:39.208+0000 7f6914297640 1 --2- 172.21.15.99:0/3269907704 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f690c0ffb00 0x7f690c102490 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:39.211 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:39.208+0000 7f6914297640 1 -- 172.21.15.99:0/3269907704 >> 172.21.15.99:0/3269907704 conn(0x7f690c0fb470 msgr2=0x7f690c105e70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:00:39.211 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:39.209+0000 7f6914297640 1 -- 172.21.15.99:0/3269907704 shutdown_connections 2024-03-19T02:00:39.211 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:39.209+0000 7f6914297640 1 -- 172.21.15.99:0/3269907704 wait complete. 2024-03-19T02:00:39.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:00:39 smithi115 ceph-mon[67332]: pgmap v938: 97 pgs: 97 active+clean; 618 MiB data, 7.7 GiB used, 1.0 TiB / 1.0 TiB avail; 3.2 MiB/s rd, 5.7 MiB/s wr, 406 op/s 2024-03-19T02:00:39.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:00:39 smithi099 ceph-mon[62951]: pgmap v938: 97 pgs: 97 active+clean; 618 MiB data, 7.7 GiB used, 1.0 TiB / 1.0 TiB avail; 3.2 MiB/s rd, 5.7 MiB/s wr, 406 op/s 2024-03-19T02:00:39.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:00:39 smithi112 ceph-mon[66819]: pgmap v938: 97 pgs: 97 active+clean; 618 MiB data, 7.7 GiB used, 1.0 TiB / 1.0 TiB avail; 3.2 MiB/s rd, 5.7 MiB/s wr, 406 op/s 2024-03-19T02:00:40.112 INFO:tasks.workunit.client.0.smithi099.stdout: 10 185292 22.78 MB/sec execute 309 sec latency 319.499 ms 2024-03-19T02:00:41.112 INFO:tasks.workunit.client.0.smithi099.stdout: 10 185796 22.80 MB/sec execute 310 sec latency 57.114 ms 2024-03-19T02:00:41.391 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:00:41 smithi099 ceph-mon[62951]: pgmap v939: 97 pgs: 97 active+clean; 619 MiB data, 7.7 GiB used, 1.0 TiB / 1.0 TiB avail; 3.6 MiB/s rd, 6.5 MiB/s wr, 506 op/s 2024-03-19T02:00:41.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:00:41 smithi115 ceph-mon[67332]: pgmap v939: 97 pgs: 97 active+clean; 619 MiB data, 7.7 GiB used, 1.0 TiB / 1.0 TiB avail; 3.6 MiB/s rd, 6.5 MiB/s wr, 506 op/s 2024-03-19T02:00:41.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:00:41 smithi112 ceph-mon[66819]: pgmap v939: 97 pgs: 97 active+clean; 619 MiB data, 7.7 GiB used, 1.0 TiB / 1.0 TiB avail; 3.6 MiB/s rd, 6.5 MiB/s wr, 506 op/s 2024-03-19T02:00:42.113 INFO:tasks.workunit.client.0.smithi099.stdout: 10 186325 22.85 MB/sec execute 311 sec latency 56.647 ms 2024-03-19T02:00:43.113 INFO:tasks.workunit.client.0.smithi099.stdout: 10 186735 22.86 MB/sec execute 312 sec latency 188.036 ms 2024-03-19T02:00:43.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:00:43 smithi115 ceph-mon[67332]: pgmap v940: 97 pgs: 97 active+clean; 610 MiB data, 7.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 6.9 MiB/s wr, 583 op/s 2024-03-19T02:00:43.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:00:43 smithi099 ceph-mon[62951]: pgmap v940: 97 pgs: 97 active+clean; 610 MiB data, 7.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 6.9 MiB/s wr, 583 op/s 2024-03-19T02:00:43.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:00:43 smithi112 ceph-mon[66819]: pgmap v940: 97 pgs: 97 active+clean; 610 MiB data, 7.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 6.9 MiB/s wr, 583 op/s 2024-03-19T02:00:44.112 INFO:tasks.workunit.client.0.smithi099.stdout: 10 186955 22.82 MB/sec execute 313 sec latency 169.468 ms 2024-03-19T02:00:45.112 INFO:tasks.workunit.client.0.smithi099.stdout: 10 187301 22.80 MB/sec execute 314 sec latency 90.265 ms 2024-03-19T02:00:45.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:00:45 smithi115 ceph-mon[67332]: pgmap v941: 97 pgs: 97 active+clean; 612 MiB data, 7.7 GiB used, 1.0 TiB / 1.0 TiB avail; 3.3 MiB/s rd, 5.6 MiB/s wr, 511 op/s 2024-03-19T02:00:45.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:00:45 smithi099 ceph-mon[62951]: pgmap v941: 97 pgs: 97 active+clean; 612 MiB data, 7.7 GiB used, 1.0 TiB / 1.0 TiB avail; 3.3 MiB/s rd, 5.6 MiB/s wr, 511 op/s 2024-03-19T02:00:45.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:00:45 smithi112 ceph-mon[66819]: pgmap v941: 97 pgs: 97 active+clean; 612 MiB data, 7.7 GiB used, 1.0 TiB / 1.0 TiB avail; 3.3 MiB/s rd, 5.6 MiB/s wr, 511 op/s 2024-03-19T02:00:46.112 INFO:tasks.workunit.client.0.smithi099.stdout: 10 187731 22.78 MB/sec execute 315 sec latency 76.538 ms 2024-03-19T02:00:47.113 INFO:tasks.workunit.client.0.smithi099.stdout: 10 188098 22.76 MB/sec execute 316 sec latency 139.507 ms 2024-03-19T02:00:47.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:00:47 smithi115 ceph-mon[67332]: pgmap v942: 97 pgs: 97 active+clean; 613 MiB data, 7.7 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 6.1 MiB/s wr, 534 op/s 2024-03-19T02:00:47.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:00:47 smithi099 ceph-mon[62951]: pgmap v942: 97 pgs: 97 active+clean; 613 MiB data, 7.7 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 6.1 MiB/s wr, 534 op/s 2024-03-19T02:00:47.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:00:47 smithi112 ceph-mon[66819]: pgmap v942: 97 pgs: 97 active+clean; 613 MiB data, 7.7 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 6.1 MiB/s wr, 534 op/s 2024-03-19T02:00:48.113 INFO:tasks.workunit.client.0.smithi099.stdout: 10 188556 22.79 MB/sec execute 317 sec latency 230.107 ms 2024-03-19T02:00:49.113 INFO:tasks.workunit.client.0.smithi099.stdout: 10 188917 22.77 MB/sec execute 318 sec latency 278.906 ms 2024-03-19T02:00:49.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:00:49 smithi115 ceph-mon[67332]: pgmap v943: 97 pgs: 97 active+clean; 617 MiB data, 7.7 GiB used, 1.0 TiB / 1.0 TiB avail; 3.5 MiB/s rd, 5.4 MiB/s wr, 503 op/s 2024-03-19T02:00:49.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:00:49 smithi099 ceph-mon[62951]: pgmap v943: 97 pgs: 97 active+clean; 617 MiB data, 7.7 GiB used, 1.0 TiB / 1.0 TiB avail; 3.5 MiB/s rd, 5.4 MiB/s wr, 503 op/s 2024-03-19T02:00:49.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:00:49 smithi112 ceph-mon[66819]: pgmap v943: 97 pgs: 97 active+clean; 617 MiB data, 7.7 GiB used, 1.0 TiB / 1.0 TiB avail; 3.5 MiB/s rd, 5.4 MiB/s wr, 503 op/s 2024-03-19T02:00:50.113 INFO:tasks.workunit.client.0.smithi099.stdout: 10 189397 22.75 MB/sec execute 319 sec latency 77.885 ms 2024-03-19T02:00:51.113 INFO:tasks.workunit.client.0.smithi099.stdout: 10 189852 22.75 MB/sec execute 320 sec latency 77.657 ms 2024-03-19T02:00:51.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:00:51 smithi115 ceph-mon[67332]: pgmap v944: 97 pgs: 97 active+clean; 620 MiB data, 7.8 GiB used, 1.0 TiB / 1.0 TiB avail; 3.9 MiB/s rd, 6.0 MiB/s wr, 563 op/s 2024-03-19T02:00:51.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:00:51 smithi099 ceph-mon[62951]: pgmap v944: 97 pgs: 97 active+clean; 620 MiB data, 7.8 GiB used, 1.0 TiB / 1.0 TiB avail; 3.9 MiB/s rd, 6.0 MiB/s wr, 563 op/s 2024-03-19T02:00:51.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:00:51 smithi112 ceph-mon[66819]: pgmap v944: 97 pgs: 97 active+clean; 620 MiB data, 7.8 GiB used, 1.0 TiB / 1.0 TiB avail; 3.9 MiB/s rd, 6.0 MiB/s wr, 563 op/s 2024-03-19T02:00:52.113 INFO:tasks.workunit.client.0.smithi099.stdout: 10 190386 22.80 MB/sec execute 321 sec latency 73.598 ms 2024-03-19T02:00:52.322 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.320+0000 7f4e8ea61640 1 -- 172.21.15.99:0/3865200968 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4e88100760 msgr2=0x7f4e88100b80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:00:52.322 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.320+0000 7f4e8ea61640 1 --2- 172.21.15.99:0/3865200968 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4e88100760 0x7f4e88100b80 secure :-1 s=READY pgs=202 cs=0 l=1 rev1=1 crypto rx=0x7f4e78009860 tx=0x7f4e7802bb70 comp rx=0 tx=0).stop 2024-03-19T02:00:52.322 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.320+0000 7f4e8ea61640 1 -- 172.21.15.99:0/3865200968 shutdown_connections 2024-03-19T02:00:52.322 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.320+0000 7f4e8ea61640 1 --2- 172.21.15.99:0/3865200968 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4e88102310 0x7f4e88108ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:52.322 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.320+0000 7f4e8ea61640 1 --2- 172.21.15.99:0/3865200968 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4e88101950 0x7f4e88101dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:52.322 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.320+0000 7f4e8ea61640 1 --2- 172.21.15.99:0/3865200968 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4e88100760 0x7f4e88100b80 unknown :-1 s=CLOSED pgs=202 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:52.322 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.320+0000 7f4e8ea61640 1 -- 172.21.15.99:0/3865200968 >> 172.21.15.99:0/3865200968 conn(0x7f4e880fbcf0 msgr2=0x7f4e880fe110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:00:52.322 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.320+0000 7f4e8ea61640 1 -- 172.21.15.99:0/3865200968 shutdown_connections 2024-03-19T02:00:52.323 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.320+0000 7f4e8ea61640 1 -- 172.21.15.99:0/3865200968 wait complete. 2024-03-19T02:00:52.323 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.320+0000 7f4e8ea61640 1 Processor -- start 2024-03-19T02:00:52.323 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.320+0000 7f4e8ea61640 1 -- start start 2024-03-19T02:00:52.323 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.320+0000 7f4e8ea61640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4e88100760 0x7f4e88197a70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:52.323 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.320+0000 7f4e8ea61640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4e88101950 0x7f4e88197fb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:52.323 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.320+0000 7f4e8ea61640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4e88102310 0x7f4e8819f020 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:52.323 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.320+0000 7f4e8ea61640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f4e8810c3e0 con 0x7f4e88102310 2024-03-19T02:00:52.323 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.320+0000 7f4e8ea61640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f4e8810c260 con 0x7f4e88100760 2024-03-19T02:00:52.323 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.320+0000 7f4e8ea61640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f4e8810c560 con 0x7f4e88101950 2024-03-19T02:00:52.323 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.321+0000 7f4e8cffe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4e88102310 0x7f4e8819f020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:52.323 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.321+0000 7f4e87fff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4e88100760 0x7f4e88197a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:52.323 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.321+0000 7f4e7ffff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4e88101950 0x7f4e88197fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:52.323 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.321+0000 7f4e87fff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4e88100760 0x7f4e88197a70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:47940/0 (socket says 172.21.15.99:47940) 2024-03-19T02:00:52.323 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.321+0000 7f4e87fff640 1 -- 172.21.15.99:0/1089586710 learned_addr learned my addr 172.21.15.99:0/1089586710 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:00:52.324 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.321+0000 7f4e8cffe640 1 -- 172.21.15.99:0/1089586710 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4e88101950 msgr2=0x7f4e88197fb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:00:52.324 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.321+0000 7f4e8cffe640 1 --2- 172.21.15.99:0/1089586710 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4e88101950 0x7f4e88197fb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:52.324 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.321+0000 7f4e8cffe640 1 -- 172.21.15.99:0/1089586710 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4e88100760 msgr2=0x7f4e88197a70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:00:52.324 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.321+0000 7f4e8cffe640 1 --2- 172.21.15.99:0/1089586710 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4e88100760 0x7f4e88197a70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:52.324 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.321+0000 7f4e8cffe640 1 -- 172.21.15.99:0/1089586710 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f4e8819f690 con 0x7f4e88102310 2024-03-19T02:00:52.324 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.321+0000 7f4e87fff640 1 --2- 172.21.15.99:0/1089586710 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4e88100760 0x7f4e88197a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:00:52.324 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.321+0000 7f4e8cffe640 1 --2- 172.21.15.99:0/1089586710 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4e88102310 0x7f4e8819f020 secure :-1 s=READY pgs=719 cs=0 l=1 rev1=1 crypto rx=0x7f4e80009fd0 tx=0x7f4e8000ef10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:00:52.324 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.321+0000 7f4e85ffb640 1 -- 172.21.15.99:0/1089586710 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4e8000ce20 con 0x7f4e88102310 2024-03-19T02:00:52.324 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.321+0000 7f4e85ffb640 1 -- 172.21.15.99:0/1089586710 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f4e80005020 con 0x7f4e88102310 2024-03-19T02:00:52.324 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.321+0000 7f4e85ffb640 1 -- 172.21.15.99:0/1089586710 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4e80010720 con 0x7f4e88102310 2024-03-19T02:00:52.325 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.321+0000 7f4e8ea61640 1 -- 172.21.15.99:0/1089586710 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f4e8819f980 con 0x7f4e88102310 2024-03-19T02:00:52.325 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.321+0000 7f4e8ea61640 1 -- 172.21.15.99:0/1089586710 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f4e8819fe80 con 0x7f4e88102310 2024-03-19T02:00:52.331 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.324+0000 7f4e85ffb640 1 -- 172.21.15.99:0/1089586710 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f4e80005540 con 0x7f4e88102310 2024-03-19T02:00:52.331 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.325+0000 7f4e91651740 1 Processor -- start 2024-03-19T02:00:52.331 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.325+0000 7f4e91651740 1 -- start start 2024-03-19T02:00:52.331 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.325+0000 7f4e91651740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55613567f100 0x55613567f520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:52.331 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.325+0000 7f4e91651740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55613567faf0 0x5561356845f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:52.331 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.325+0000 7f4e91651740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556135684c60 0x5561356870f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:52.331 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.325+0000 7f4e91651740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x5561355e3be0 con 0x55613567f100 2024-03-19T02:00:52.331 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.325+0000 7f4e91651740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55613558c960 con 0x556135684c60 2024-03-19T02:00:52.331 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.325+0000 7f4e91651740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x5561355dfc80 con 0x55613567faf0 2024-03-19T02:00:52.331 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.325+0000 7f4e7ffff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55613567f100 0x55613567f520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:52.331 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.325+0000 7f4e87fff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55613567faf0 0x5561356845f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:52.332 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.325+0000 7f4e87fff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55613567faf0 0x5561356845f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:49544/0 (socket says 172.21.15.99:49544) 2024-03-19T02:00:52.332 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.325+0000 7f4e87fff640 1 -- 172.21.15.99:0/848097598 learned_addr learned my addr 172.21.15.99:0/848097598 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:00:52.332 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.325+0000 7f4e7ffff640 1 --2- 172.21.15.99:0/848097598 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556135684c60 0x5561356870f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:52.332 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.325+0000 7f4e85ffb640 1 --2- 172.21.15.99:0/1089586710 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f4e740a28a0 0x7f4e740a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:52.332 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.325+0000 7f4e7ffff640 1 -- 172.21.15.99:0/848097598 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55613567faf0 msgr2=0x5561356845f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:00:52.332 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.325+0000 7f4e7ffff640 1 --2- 172.21.15.99:0/848097598 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55613567faf0 0x5561356845f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:52.332 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.325+0000 7f4e7ffff640 1 -- 172.21.15.99:0/848097598 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55613567f100 msgr2=0x55613567f520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:00:52.332 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.325+0000 7f4e7ffff640 1 --2- 172.21.15.99:0/848097598 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55613567f100 0x55613567f520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:52.332 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.325+0000 7f4e8cffe640 1 --2- 172.21.15.99:0/1089586710 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f4e740a28a0 0x7f4e740a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:52.332 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.325+0000 7f4e7ffff640 1 -- 172.21.15.99:0/848097598 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x556135583d20 con 0x556135684c60 2024-03-19T02:00:52.332 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.326+0000 7f4e85ffb640 1 -- 172.21.15.99:0/1089586710 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f4e800cf010 con 0x7f4e88102310 2024-03-19T02:00:52.332 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.326+0000 7f4e7ffff640 1 --2- 172.21.15.99:0/848097598 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556135684c60 0x5561356870f0 secure :-1 s=READY pgs=203 cs=0 l=1 rev1=1 crypto rx=0x7f4e700096c0 tx=0x7f4e70009260 comp rx=0 tx=0).ready entity=mon.2 client_cookie=48e06ef369a7fe3d server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:00:52.332 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.326+0000 7f4e7e7fc640 1 -- 172.21.15.99:0/848097598 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4e70016070 con 0x556135684c60 2024-03-19T02:00:52.333 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.326+0000 7f4e7e7fc640 1 -- 172.21.15.99:0/848097598 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f4e700047a0 con 0x556135684c60 2024-03-19T02:00:52.333 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.326+0000 7f4e7e7fc640 1 -- 172.21.15.99:0/848097598 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4e70019040 con 0x556135684c60 2024-03-19T02:00:52.333 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.327+0000 7f4e91651740 1 -- 172.21.15.99:0/848097598 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556135684c60 msgr2=0x5561356870f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:00:52.333 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.327+0000 7f4e91651740 1 --2- 172.21.15.99:0/848097598 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556135684c60 0x5561356870f0 secure :-1 s=READY pgs=203 cs=0 l=1 rev1=1 crypto rx=0x7f4e700096c0 tx=0x7f4e70009260 comp rx=0 tx=0).stop 2024-03-19T02:00:52.333 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.327+0000 7f4e8cffe640 1 --2- 172.21.15.99:0/1089586710 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f4e740a28a0 0x7f4e740a4d60 secure :-1 s=READY pgs=361 cs=0 l=1 rev1=1 crypto rx=0x7f4e8000c760 tx=0x7f4e80005940 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:00:52.333 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.327+0000 7f4e91651740 1 -- 172.21.15.99:0/848097598 shutdown_connections 2024-03-19T02:00:52.333 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.327+0000 7f4e91651740 1 --2- 172.21.15.99:0/848097598 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556135684c60 0x5561356870f0 unknown :-1 s=CLOSED pgs=203 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:52.333 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.327+0000 7f4e91651740 1 --2- 172.21.15.99:0/848097598 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55613567faf0 0x5561356845f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:52.333 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.327+0000 7f4e91651740 1 --2- 172.21.15.99:0/848097598 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55613567f100 0x55613567f520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:52.333 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.327+0000 7f4e91651740 1 -- 172.21.15.99:0/848097598 >> 172.21.15.99:0/848097598 conn(0x55613567a670 msgr2=0x55613567aa70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:00:52.333 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.327+0000 7f4e91651740 1 -- 172.21.15.99:0/848097598 shutdown_connections 2024-03-19T02:00:52.333 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.327+0000 7f4e91651740 1 -- 172.21.15.99:0/848097598 wait complete. 2024-03-19T02:00:52.334 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.327+0000 7f4e91651740 1 -- start start 2024-03-19T02:00:52.334 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.328+0000 7f4e91651740 1 Processor -- start 2024-03-19T02:00:52.334 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.328+0000 7f4e91651740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55613567f100 0x556135689140 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:52.334 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.328+0000 7f4e91651740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55613567faf0 0x556135689680 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:52.334 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.328+0000 7f4e91651740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556135684c60 0x5561356906c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:52.334 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.328+0000 7f4e91651740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x556135687c10 con 0x55613567f100 2024-03-19T02:00:52.334 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.328+0000 7f4e91651740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55613559fdb0 con 0x55613567faf0 2024-03-19T02:00:52.334 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.328+0000 7f4e91651740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x556135687d90 con 0x556135684c60 2024-03-19T02:00:52.334 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.328+0000 7f4e91651740 10 client.0 _ll_register_callbacks cb 0x5561355a4e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T02:00:52.334 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.328+0000 7f4e7ffff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55613567f100 0x556135689140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:52.334 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.328+0000 7f4e87fff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55613567faf0 0x556135689680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:52.334 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.328+0000 7f4e7ffff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55613567f100 0x556135689140 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:33218/0 (socket says 172.21.15.99:33218) 2024-03-19T02:00:52.334 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.328+0000 7f4e7ffff640 1 -- 172.21.15.99:0/549738292 learned_addr learned my addr 172.21.15.99:0/549738292 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:00:52.334 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.328+0000 7f4e7ffff640 1 --2- 172.21.15.99:0/549738292 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556135684c60 0x5561356906c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:52.335 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.329+0000 7f4e87fff640 1 -- 172.21.15.99:0/549738292 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556135684c60 msgr2=0x5561356906c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:00:52.335 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.329+0000 7f4e87fff640 1 --2- 172.21.15.99:0/549738292 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556135684c60 0x5561356906c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:52.335 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.329+0000 7f4e87fff640 1 -- 172.21.15.99:0/549738292 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55613567f100 msgr2=0x556135689140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:00:52.335 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.329+0000 7f4e87fff640 1 --2- 172.21.15.99:0/549738292 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55613567f100 0x556135689140 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:52.335 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.329+0000 7f4e87fff640 1 -- 172.21.15.99:0/549738292 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55613559ec60 con 0x55613567faf0 2024-03-19T02:00:52.335 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.329+0000 7f4e7ffff640 1 --2- 172.21.15.99:0/549738292 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556135684c60 0x5561356906c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-19T02:00:52.335 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.329+0000 7f4e87fff640 1 --2- 172.21.15.99:0/549738292 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55613567faf0 0x556135689680 secure :-1 s=READY pgs=204 cs=0 l=1 rev1=1 crypto rx=0x55613567fef0 tx=0x7f4e78032340 comp rx=0 tx=0).ready entity=mon.2 client_cookie=b867a77d58ae6d1f server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:00:52.335 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.329+0000 7f4e677fe640 10 client.0 ms_handle_connect on v2:172.21.15.112:3300/0 2024-03-19T02:00:52.335 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.329+0000 7f4e677fe640 1 -- 172.21.15.99:0/549738292 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4e78046070 con 0x55613567faf0 2024-03-19T02:00:52.335 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.329+0000 7f4e677fe640 1 -- 172.21.15.99:0/549738292 <== mon.2 v2:172.21.15.112:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f4e78004040 con 0x55613567faf0 2024-03-19T02:00:52.335 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.329+0000 7f4e91651740 1 -- 172.21.15.99:0/549738292 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f4e700047a0 con 0x55613567faf0 2024-03-19T02:00:52.335 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.329+0000 7f4e677fe640 1 -- 172.21.15.99:0/549738292 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7f4e78052050 con 0x55613567faf0 2024-03-19T02:00:52.335 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.329+0000 7f4e91651740 10 client.26696 fetch_fsmap learned FSMap version 37 2024-03-19T02:00:52.335 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.329+0000 7f4e91651740 1 -- 172.21.15.99:0/549738292 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x5561352f8210 con 0x55613567faf0 2024-03-19T02:00:52.336 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.329+0000 7f4e677fe640 1 -- 172.21.15.99:0/549738292 <== mon.2 v2:172.21.15.112:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f4e700047a0 con 0x55613567faf0 2024-03-19T02:00:52.336 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.330+0000 7f4e677fe640 1 -- 172.21.15.99:0/549738292 <== mon.2 v2:172.21.15.112:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7f4e780326d0 con 0x55613567faf0 2024-03-19T02:00:52.336 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.330+0000 7f4e91651740 10 client.26696 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T02:00:52.336 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.330+0000 7f4e91651740 10 client.26696 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T02:00:52.336 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.330+0000 7f4e91651740 20 client.26696 populate_metadata read hostname 'smithi099' 2024-03-19T02:00:52.336 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.330+0000 7f4e91651740 1 --2- 172.21.15.99:0/549738292 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x556135694ef0 0x556135697300 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:52.336 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.330+0000 7f4e91651740 4 client.26696 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T02:00:52.336 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.330+0000 7f4e91651740 1 -- 172.21.15.99:0/549738292 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5561355f1120 con 0x556135694ef0 2024-03-19T02:00:52.336 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.330+0000 7f4e7ffff640 1 --2- 172.21.15.99:0/549738292 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x556135694ef0 0x556135697300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:52.336 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.330+0000 7f4e677fe640 10 client.26696 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T02:00:52.336 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.330+0000 7f4e7ffff640 1 --2- 172.21.15.99:0/549738292 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x556135694ef0 0x556135697300 crc :-1 s=READY pgs=205 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=1189be268e1efe45 server_cookie=1123c1ccc0823f7e in_seq=0 out_seq=0 2024-03-19T02:00:52.337 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.333+0000 7f4e677fe640 1 -- 172.21.15.99:0/549738292 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x5561355f1120 con 0x556135694ef0 2024-03-19T02:00:52.337 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.333+0000 7f4e677fe640 10 client.26696 handle_command_reply: tid=0 2024-03-19T02:00:52.337 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.333+0000 7f4e91651740 1 client.26696 shutdown 2024-03-19T02:00:52.337 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.333+0000 7f4e91651740 1 -- 172.21.15.99:0/549738292 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55613567faf0 msgr2=0x556135689680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:00:52.338 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.333+0000 7f4e91651740 1 --2- 172.21.15.99:0/549738292 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55613567faf0 0x556135689680 secure :-1 s=READY pgs=204 cs=0 l=1 rev1=1 crypto rx=0x55613567fef0 tx=0x7f4e78032340 comp rx=0 tx=0).stop 2024-03-19T02:00:52.338 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.333+0000 7f4e91651740 1 -- 172.21.15.99:0/549738292 shutdown_connections 2024-03-19T02:00:52.338 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.333+0000 7f4e91651740 1 --2- 172.21.15.99:0/549738292 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x556135694ef0 0x556135697300 crc :-1 s=READY pgs=205 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:52.338 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.333+0000 7f4e677fe640 0 client.26696 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T02:00:52.338 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.333+0000 7f4e91651740 1 --2- 172.21.15.99:0/549738292 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55613567faf0 0x556135689680 unknown :-1 s=CLOSED pgs=204 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:52.338 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.333+0000 7f4e91651740 1 --2- 172.21.15.99:0/549738292 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556135684c60 0x5561356906c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:52.338 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.333+0000 7f4e91651740 1 --2- 172.21.15.99:0/549738292 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55613567f100 0x556135689140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:52.338 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.333+0000 7f4e91651740 1 -- 172.21.15.99:0/549738292 >> 172.21.15.99:0/549738292 conn(0x55613567a670 msgr2=0x55613567b8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:00:52.338 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.333+0000 7f4e91651740 1 -- 172.21.15.99:0/549738292 shutdown_connections 2024-03-19T02:00:52.338 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.333+0000 7f4e91651740 1 -- 172.21.15.99:0/549738292 wait complete. 2024-03-19T02:00:52.338 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.334+0000 7f4e91651740 20 client.26696 trim_cache size 0 max 16384 2024-03-19T02:00:52.354 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.354+0000 7f4e91651740 1 Processor -- start 2024-03-19T02:00:52.354 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.354+0000 7f4e91651740 1 -- start start 2024-03-19T02:00:52.354 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.354+0000 7f4e91651740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55613567f100 0x556135679220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:52.354 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.354+0000 7f4e91651740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55613567faf0 0x556135690680 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:52.354 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.354+0000 7f4e91651740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556135684c60 0x556135679760 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:52.354 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.354+0000 7f4e87fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55613567faf0 0x556135690680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:52.354 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.354+0000 7f4e87fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55613567faf0 0x556135690680 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:33230/0 (socket says 172.21.15.99:33230) 2024-03-19T02:00:52.355 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.354+0000 7f4e87fff640 1 -- 172.21.15.99:0/1438348229 learned_addr learned my addr 172.21.15.99:0/1438348229 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:00:52.355 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.354+0000 7f4e91651740 1 -- 172.21.15.99:0/1438348229 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f4e881ab620 con 0x55613567faf0 2024-03-19T02:00:52.355 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.354+0000 7f4e91651740 1 -- 172.21.15.99:0/1438348229 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f4e881a8ef0 con 0x556135684c60 2024-03-19T02:00:52.355 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.354+0000 7f4e91651740 1 -- 172.21.15.99:0/1438348229 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f4e881a6d40 con 0x55613567f100 2024-03-19T02:00:52.355 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.354+0000 7f4e7ffff640 1 --2- 172.21.15.99:0/1438348229 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55613567f100 0x556135679220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:52.355 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.354+0000 7f4e7ffff640 1 --2- 172.21.15.99:0/1438348229 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556135684c60 0x556135679760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:52.355 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.354+0000 7f4e87fff640 1 -- 172.21.15.99:0/1438348229 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55613567f100 msgr2=0x556135679220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:00:52.355 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.354+0000 7f4e87fff640 1 --2- 172.21.15.99:0/1438348229 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55613567f100 0x556135679220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:52.355 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.355+0000 7f4e87fff640 1 -- 172.21.15.99:0/1438348229 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556135684c60 msgr2=0x556135679760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:00:52.355 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.355+0000 7f4e87fff640 1 --2- 172.21.15.99:0/1438348229 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556135684c60 0x556135679760 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:52.355 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.355+0000 7f4e87fff640 1 -- 172.21.15.99:0/1438348229 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5561353c4620 con 0x55613567faf0 2024-03-19T02:00:52.357 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.355+0000 7f4e87fff640 1 --2- 172.21.15.99:0/1438348229 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55613567faf0 0x556135690680 secure :-1 s=READY pgs=720 cs=0 l=1 rev1=1 crypto rx=0x7f4e7802c0b0 tx=0x7f4e7802bb70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5508b0da87044c7b server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:00:52.357 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.355+0000 7f4e7dffb640 1 -- 172.21.15.99:0/1438348229 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4e7805c020 con 0x55613567faf0 2024-03-19T02:00:52.357 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.355+0000 7f4e7dffb640 1 -- 172.21.15.99:0/1438348229 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f4e780323b0 con 0x55613567faf0 2024-03-19T02:00:52.357 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.355+0000 7f4e7dffb640 1 -- 172.21.15.99:0/1438348229 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4e78046040 con 0x55613567faf0 2024-03-19T02:00:52.357 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.356+0000 7f4e91651740 1 -- 172.21.15.99:0/1438348229 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55613567faf0 msgr2=0x556135690680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:00:52.357 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.356+0000 7f4e91651740 1 --2- 172.21.15.99:0/1438348229 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55613567faf0 0x556135690680 secure :-1 s=READY pgs=720 cs=0 l=1 rev1=1 crypto rx=0x7f4e7802c0b0 tx=0x7f4e7802bb70 comp rx=0 tx=0).stop 2024-03-19T02:00:52.357 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.356+0000 7f4e91651740 1 -- 172.21.15.99:0/1438348229 shutdown_connections 2024-03-19T02:00:52.357 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.356+0000 7f4e91651740 1 --2- 172.21.15.99:0/1438348229 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556135684c60 0x556135679760 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:52.357 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.356+0000 7f4e91651740 1 --2- 172.21.15.99:0/1438348229 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55613567faf0 0x556135690680 unknown :-1 s=CLOSED pgs=720 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:52.357 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.356+0000 7f4e91651740 1 --2- 172.21.15.99:0/1438348229 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55613567f100 0x556135679220 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:52.357 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.356+0000 7f4e91651740 1 -- 172.21.15.99:0/1438348229 >> 172.21.15.99:0/1438348229 conn(0x55613567a670 msgr2=0x556135689140 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:00:52.358 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.356+0000 7f4e91651740 1 -- 172.21.15.99:0/1438348229 shutdown_connections 2024-03-19T02:00:52.358 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.356+0000 7f4e91651740 1 -- 172.21.15.99:0/1438348229 wait complete. 2024-03-19T02:00:52.358 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.357+0000 7f4e91651740 1 -- start start 2024-03-19T02:00:52.358 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.357+0000 7f4e91651740 1 Processor -- start 2024-03-19T02:00:52.361 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.357+0000 7f4e91651740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55613567f100 0x55613568dbb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:52.361 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.358+0000 7f4e91651740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55613567faf0 0x55613568e0f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:52.361 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.358+0000 7f4e91651740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556135684c60 0x55613568f100 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:52.361 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.358+0000 7f4e91651740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f4e881a73b0 con 0x55613567f100 2024-03-19T02:00:52.361 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.358+0000 7f4e91651740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f4e881a8820 con 0x556135684c60 2024-03-19T02:00:52.361 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.358+0000 7f4e91651740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f4e881a7a80 con 0x55613567faf0 2024-03-19T02:00:52.361 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.358+0000 7f4e91651740 10 client.0 _ll_register_callbacks cb 0x7f4e800093a0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T02:00:52.361 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.358+0000 7f4e7ffff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556135684c60 0x55613568f100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:52.361 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.358+0000 7f4e7ffff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55613567f100 0x55613568dbb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:52.362 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.358+0000 7f4e7ffff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55613567f100 0x55613568dbb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:33246/0 (socket says 172.21.15.99:33246) 2024-03-19T02:00:52.362 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.358+0000 7f4e7ffff640 1 -- 172.21.15.99:0/157504546 learned_addr learned my addr 172.21.15.99:0/157504546 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:00:52.362 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.358+0000 7f4e87fff640 1 --2- 172.21.15.99:0/157504546 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55613567faf0 0x55613568e0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:52.362 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.358+0000 7f4e7ffff640 1 -- 172.21.15.99:0/157504546 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55613567faf0 msgr2=0x55613568e0f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:00:52.362 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.358+0000 7f4e7ffff640 1 --2- 172.21.15.99:0/157504546 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55613567faf0 0x55613568e0f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:52.362 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.358+0000 7f4e7ffff640 1 -- 172.21.15.99:0/157504546 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55613567f100 msgr2=0x55613568dbb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:00:52.362 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.358+0000 7f4e7ffff640 1 --2- 172.21.15.99:0/157504546 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55613567f100 0x55613568dbb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:52.362 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.358+0000 7f4e7ffff640 1 -- 172.21.15.99:0/157504546 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55613568f6f0 con 0x556135684c60 2024-03-19T02:00:52.362 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.358+0000 7f4e7ffff640 1 --2- 172.21.15.99:0/157504546 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556135684c60 0x55613568f100 secure :-1 s=READY pgs=205 cs=0 l=1 rev1=1 crypto rx=0x7f4e70010930 tx=0x7f4e78034ad0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=703f0c2eb5770c52 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:00:52.362 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.359+0000 7f4e677fe640 10 client.0 ms_handle_connect on v2:172.21.15.112:3300/0 2024-03-19T02:00:52.362 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.359+0000 7f4e677fe640 1 -- 172.21.15.99:0/157504546 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4e70029040 con 0x556135684c60 2024-03-19T02:00:52.362 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.359+0000 7f4e677fe640 1 -- 172.21.15.99:0/157504546 <== mon.2 v2:172.21.15.112:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f4e70004020 con 0x556135684c60 2024-03-19T02:00:52.362 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.359+0000 7f4e91651740 1 -- 172.21.15.99:0/157504546 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f4e780323b0 con 0x556135684c60 2024-03-19T02:00:52.363 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.359+0000 7f4e677fe640 1 -- 172.21.15.99:0/157504546 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7f4e70043050 con 0x556135684c60 2024-03-19T02:00:52.363 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.359+0000 7f4e91651740 10 client.26702 fetch_fsmap learned FSMap version 37 2024-03-19T02:00:52.363 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.359+0000 7f4e91651740 1 -- 172.21.15.99:0/157504546 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x556135690a70 con 0x556135684c60 2024-03-19T02:00:52.363 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.360+0000 7f4e677fe640 1 -- 172.21.15.99:0/157504546 <== mon.2 v2:172.21.15.112:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f4e780323b0 con 0x556135684c60 2024-03-19T02:00:52.363 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.360+0000 7f4e677fe640 1 -- 172.21.15.99:0/157504546 <== mon.2 v2:172.21.15.112:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7f4e7000f820 con 0x556135684c60 2024-03-19T02:00:52.363 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.360+0000 7f4e91651740 10 client.26702 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T02:00:52.363 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.360+0000 7f4e91651740 10 client.26702 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T02:00:52.363 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.360+0000 7f4e91651740 20 client.26702 populate_metadata read hostname 'smithi099' 2024-03-19T02:00:52.363 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.360+0000 7f4e91651740 1 --2- 172.21.15.99:0/157504546 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5561356fb870 0x5561356fdc60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:00:52.363 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.360+0000 7f4e91651740 4 client.26702 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T02:00:52.363 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.360+0000 7f4e91651740 1 -- 172.21.15.99:0/157504546 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x5561355eb470 con 0x5561356fb870 2024-03-19T02:00:52.363 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.361+0000 7f4e87fff640 1 --2- 172.21.15.99:0/157504546 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5561356fb870 0x5561356fdc60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:00:52.363 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.361+0000 7f4e87fff640 1 --2- 172.21.15.99:0/157504546 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5561356fb870 0x5561356fdc60 crc :-1 s=READY pgs=206 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=129daebd91ffd006 server_cookie=745f42cc62b9b43a in_seq=0 out_seq=0 2024-03-19T02:00:52.363 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.361+0000 7f4e677fe640 10 client.26702 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T02:00:52.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.864+0000 7f4e677fe640 1 -- 172.21.15.99:0/157504546 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x5561355eb470 con 0x5561356fb870 2024-03-19T02:00:52.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.864+0000 7f4e677fe640 10 client.26702 handle_command_reply: tid=0 2024-03-19T02:00:52.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.864+0000 7f4e91651740 1 client.26702 shutdown 2024-03-19T02:00:52.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.864+0000 7f4e91651740 1 -- 172.21.15.99:0/157504546 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556135684c60 msgr2=0x55613568f100 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:00:52.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.864+0000 7f4e91651740 1 --2- 172.21.15.99:0/157504546 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556135684c60 0x55613568f100 secure :-1 s=READY pgs=205 cs=0 l=1 rev1=1 crypto rx=0x7f4e70010930 tx=0x7f4e78034ad0 comp rx=0 tx=0).stop 2024-03-19T02:00:52.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.864+0000 7f4e91651740 1 -- 172.21.15.99:0/157504546 shutdown_connections 2024-03-19T02:00:52.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.864+0000 7f4e91651740 1 --2- 172.21.15.99:0/157504546 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5561356fb870 0x5561356fdc60 crc :-1 s=READY pgs=206 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:52.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.864+0000 7f4e91651740 1 --2- 172.21.15.99:0/157504546 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556135684c60 0x55613568f100 unknown :-1 s=CLOSED pgs=205 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:52.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.864+0000 7f4e91651740 1 --2- 172.21.15.99:0/157504546 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55613567faf0 0x55613568e0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:52.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.864+0000 7f4e677fe640 0 client.26702 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T02:00:52.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.864+0000 7f4e91651740 1 --2- 172.21.15.99:0/157504546 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55613567f100 0x55613568dbb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:52.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.864+0000 7f4e91651740 1 -- 172.21.15.99:0/157504546 >> 172.21.15.99:0/157504546 conn(0x55613567a670 msgr2=0x556135683d10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:00:52.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.864+0000 7f4e91651740 1 -- 172.21.15.99:0/157504546 shutdown_connections 2024-03-19T02:00:52.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.865+0000 7f4e91651740 1 -- 172.21.15.99:0/157504546 wait complete. 2024-03-19T02:00:52.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.865+0000 7f4e91651740 20 client.26702 trim_cache size 0 max 16384 2024-03-19T02:00:52.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T02:00:52.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T02:00:52.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T02:00:52.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T02:00:52.867 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.865+0000 7f4e8ea61640 1 -- 172.21.15.99:0/1089586710 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f4e740a28a0 msgr2=0x7f4e740a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:00:52.867 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.865+0000 7f4e8ea61640 1 --2- 172.21.15.99:0/1089586710 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f4e740a28a0 0x7f4e740a4d60 secure :-1 s=READY pgs=361 cs=0 l=1 rev1=1 crypto rx=0x7f4e8000c760 tx=0x7f4e80005940 comp rx=0 tx=0).stop 2024-03-19T02:00:52.867 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.865+0000 7f4e8ea61640 1 -- 172.21.15.99:0/1089586710 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4e88102310 msgr2=0x7f4e8819f020 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:00:52.868 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.865+0000 7f4e8ea61640 1 --2- 172.21.15.99:0/1089586710 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4e88102310 0x7f4e8819f020 secure :-1 s=READY pgs=719 cs=0 l=1 rev1=1 crypto rx=0x7f4e80009fd0 tx=0x7f4e8000ef10 comp rx=0 tx=0).stop 2024-03-19T02:00:52.868 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.865+0000 7f4e8ea61640 1 -- 172.21.15.99:0/1089586710 shutdown_connections 2024-03-19T02:00:52.868 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.865+0000 7f4e8ea61640 1 --2- 172.21.15.99:0/1089586710 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4e88102310 0x7f4e8819f020 unknown :-1 s=CLOSED pgs=719 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:52.868 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.865+0000 7f4e8ea61640 1 --2- 172.21.15.99:0/1089586710 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4e88101950 0x7f4e88197fb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:52.868 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.865+0000 7f4e8ea61640 1 --2- 172.21.15.99:0/1089586710 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f4e740a28a0 0x7f4e740a4d60 unknown :-1 s=CLOSED pgs=361 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:52.868 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.865+0000 7f4e8ea61640 1 --2- 172.21.15.99:0/1089586710 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4e88100760 0x7f4e88197a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:00:52.868 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.865+0000 7f4e8ea61640 1 -- 172.21.15.99:0/1089586710 >> 172.21.15.99:0/1089586710 conn(0x7f4e880fbcf0 msgr2=0x7f4e880fd690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:00:52.868 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.865+0000 7f4e8ea61640 1 -- 172.21.15.99:0/1089586710 shutdown_connections 2024-03-19T02:00:52.868 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:00:52.865+0000 7f4e8ea61640 1 -- 172.21.15.99:0/1089586710 wait complete. 2024-03-19T02:00:53.113 INFO:tasks.workunit.client.0.smithi099.stdout: 10 190731 22.79 MB/sec execute 322 sec latency 341.835 ms 2024-03-19T02:00:53.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:00:53 smithi115 ceph-mon[67332]: pgmap v945: 97 pgs: 97 active+clean; 624 MiB data, 7.8 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 6.4 MiB/s wr, 483 op/s 2024-03-19T02:00:53.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:00:53 smithi099 ceph-mon[62951]: pgmap v945: 97 pgs: 97 active+clean; 624 MiB data, 7.8 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 6.4 MiB/s wr, 483 op/s 2024-03-19T02:00:53.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:00:53 smithi112 ceph-mon[66819]: pgmap v945: 97 pgs: 97 active+clean; 624 MiB data, 7.8 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 6.4 MiB/s wr, 483 op/s 2024-03-19T02:00:54.113 INFO:tasks.workunit.client.0.smithi099.stdout: 10 191097 22.80 MB/sec execute 323 sec latency 285.725 ms 2024-03-19T02:00:55.113 INFO:tasks.workunit.client.0.smithi099.stdout: 10 191469 22.81 MB/sec execute 324 sec latency 91.236 ms 2024-03-19T02:00:55.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:00:55 smithi115 ceph-mon[67332]: pgmap v946: 97 pgs: 97 active+clean; 624 MiB data, 7.8 GiB used, 1.0 TiB / 1.0 TiB avail; 3.6 MiB/s rd, 5.2 MiB/s wr, 410 op/s 2024-03-19T02:00:55.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:00:55 smithi099 ceph-mon[62951]: pgmap v946: 97 pgs: 97 active+clean; 624 MiB data, 7.8 GiB used, 1.0 TiB / 1.0 TiB avail; 3.6 MiB/s rd, 5.2 MiB/s wr, 410 op/s 2024-03-19T02:00:55.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:00:55 smithi112 ceph-mon[66819]: pgmap v946: 97 pgs: 97 active+clean; 624 MiB data, 7.8 GiB used, 1.0 TiB / 1.0 TiB avail; 3.6 MiB/s rd, 5.2 MiB/s wr, 410 op/s 2024-03-19T02:00:56.113 INFO:tasks.workunit.client.0.smithi099.stdout: 10 191805 22.78 MB/sec execute 325 sec latency 66.315 ms 2024-03-19T02:00:57.113 INFO:tasks.workunit.client.0.smithi099.stdout: 10 192088 22.74 MB/sec execute 326 sec latency 89.668 ms 2024-03-19T02:00:57.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:00:57 smithi115 ceph-mon[67332]: pgmap v947: 97 pgs: 97 active+clean; 625 MiB data, 7.8 GiB used, 1.0 TiB / 1.0 TiB avail; 4.4 MiB/s rd, 6.2 MiB/s wr, 544 op/s 2024-03-19T02:00:57.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:00:57 smithi099 ceph-mon[62951]: pgmap v947: 97 pgs: 97 active+clean; 625 MiB data, 7.8 GiB used, 1.0 TiB / 1.0 TiB avail; 4.4 MiB/s rd, 6.2 MiB/s wr, 544 op/s 2024-03-19T02:00:57.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:00:57 smithi112 ceph-mon[66819]: pgmap v947: 97 pgs: 97 active+clean; 625 MiB data, 7.8 GiB used, 1.0 TiB / 1.0 TiB avail; 4.4 MiB/s rd, 6.2 MiB/s wr, 544 op/s 2024-03-19T02:00:58.113 INFO:tasks.workunit.client.0.smithi099.stdout: 10 192415 22.72 MB/sec execute 327 sec latency 96.833 ms 2024-03-19T02:00:59.113 INFO:tasks.workunit.client.0.smithi099.stdout: 10 192714 22.70 MB/sec execute 328 sec latency 373.538 ms 2024-03-19T02:00:59.413 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:00:59 smithi112 ceph-mon[66819]: pgmap v948: 97 pgs: 97 active+clean; 621 MiB data, 7.8 GiB used, 1.0 TiB / 1.0 TiB avail; 3.7 MiB/s rd, 5.6 MiB/s wr, 535 op/s 2024-03-19T02:00:59.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:00:59 smithi115 ceph-mon[67332]: pgmap v948: 97 pgs: 97 active+clean; 621 MiB data, 7.8 GiB used, 1.0 TiB / 1.0 TiB avail; 3.7 MiB/s rd, 5.6 MiB/s wr, 535 op/s 2024-03-19T02:00:59.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:00:59 smithi099 ceph-mon[62951]: pgmap v948: 97 pgs: 97 active+clean; 621 MiB data, 7.8 GiB used, 1.0 TiB / 1.0 TiB avail; 3.7 MiB/s rd, 5.6 MiB/s wr, 535 op/s 2024-03-19T02:01:00.113 INFO:tasks.workunit.client.0.smithi099.stdout: 10 193184 22.69 MB/sec execute 329 sec latency 89.403 ms 2024-03-19T02:01:01.113 INFO:tasks.workunit.client.0.smithi099.stdout: 10 193778 22.71 MB/sec execute 330 sec latency 75.349 ms 2024-03-19T02:01:01.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:01:01 smithi099 ceph-mon[62951]: pgmap v949: 97 pgs: 97 active+clean; 622 MiB data, 7.8 GiB used, 1.0 TiB / 1.0 TiB avail; 3.9 MiB/s rd, 5.9 MiB/s wr, 546 op/s 2024-03-19T02:01:01.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:01:01 smithi112 ceph-mon[66819]: pgmap v949: 97 pgs: 97 active+clean; 622 MiB data, 7.8 GiB used, 1.0 TiB / 1.0 TiB avail; 3.9 MiB/s rd, 5.9 MiB/s wr, 546 op/s 2024-03-19T02:01:01.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:01:01 smithi115 ceph-mon[67332]: pgmap v949: 97 pgs: 97 active+clean; 622 MiB data, 7.8 GiB used, 1.0 TiB / 1.0 TiB avail; 3.9 MiB/s rd, 5.9 MiB/s wr, 546 op/s 2024-03-19T02:01:02.113 INFO:tasks.workunit.client.0.smithi099.stdout: 10 194326 22.73 MB/sec execute 331 sec latency 96.690 ms 2024-03-19T02:01:03.113 INFO:tasks.workunit.client.0.smithi099.stdout: 10 194870 22.78 MB/sec execute 332 sec latency 60.035 ms 2024-03-19T02:01:03.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:01:03 smithi115 ceph-mon[67332]: pgmap v950: 97 pgs: 97 active+clean; 619 MiB data, 7.8 GiB used, 1.0 TiB / 1.0 TiB avail; 4.5 MiB/s rd, 6.2 MiB/s wr, 571 op/s 2024-03-19T02:01:03.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:01:03 smithi099 ceph-mon[62951]: pgmap v950: 97 pgs: 97 active+clean; 619 MiB data, 7.8 GiB used, 1.0 TiB / 1.0 TiB avail; 4.5 MiB/s rd, 6.2 MiB/s wr, 571 op/s 2024-03-19T02:01:03.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:01:03 smithi112 ceph-mon[66819]: pgmap v950: 97 pgs: 97 active+clean; 619 MiB data, 7.8 GiB used, 1.0 TiB / 1.0 TiB avail; 4.5 MiB/s rd, 6.2 MiB/s wr, 571 op/s 2024-03-19T02:01:04.113 INFO:tasks.workunit.client.0.smithi099.stdout: 10 195218 22.76 MB/sec execute 333 sec latency 259.215 ms 2024-03-19T02:01:05.113 INFO:tasks.workunit.client.0.smithi099.stdout: 10 195645 22.78 MB/sec execute 334 sec latency 94.987 ms 2024-03-19T02:01:05.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:01:05 smithi115 ceph-mon[67332]: pgmap v951: 97 pgs: 97 active+clean; 611 MiB data, 7.8 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 5.1 MiB/s wr, 517 op/s 2024-03-19T02:01:05.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:01:05 smithi099 ceph-mon[62951]: pgmap v951: 97 pgs: 97 active+clean; 611 MiB data, 7.8 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 5.1 MiB/s wr, 517 op/s 2024-03-19T02:01:05.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:01:05 smithi112 ceph-mon[66819]: pgmap v951: 97 pgs: 97 active+clean; 611 MiB data, 7.8 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 5.1 MiB/s wr, 517 op/s 2024-03-19T02:01:05.959 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.957+0000 7f851c6c6640 1 -- 172.21.15.99:0/1987332653 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f85141022f0 msgr2=0x7f8514108b80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:01:05.960 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.957+0000 7f851c6c6640 1 --2- 172.21.15.99:0/1987332653 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f85141022f0 0x7f8514108b80 secure :-1 s=READY pgs=721 cs=0 l=1 rev1=1 crypto rx=0x7f850c009ca0 tx=0x7f850c02c0f0 comp rx=0 tx=0).stop 2024-03-19T02:01:05.960 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.957+0000 7f851c6c6640 1 -- 172.21.15.99:0/1987332653 shutdown_connections 2024-03-19T02:01:05.960 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.957+0000 7f851c6c6640 1 --2- 172.21.15.99:0/1987332653 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f85141022f0 0x7f8514108b80 unknown :-1 s=CLOSED pgs=721 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:05.960 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.957+0000 7f851c6c6640 1 --2- 172.21.15.99:0/1987332653 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8514101930 0x7f8514101db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:05.960 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.957+0000 7f851c6c6640 1 --2- 172.21.15.99:0/1987332653 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8514100740 0x7f8514100b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:05.960 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.957+0000 7f851c6c6640 1 -- 172.21.15.99:0/1987332653 >> 172.21.15.99:0/1987332653 conn(0x7f85140fbcb0 msgr2=0x7f85140fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:01:05.960 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.957+0000 7f851c6c6640 1 -- 172.21.15.99:0/1987332653 shutdown_connections 2024-03-19T02:01:05.960 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.957+0000 7f851c6c6640 1 -- 172.21.15.99:0/1987332653 wait complete. 2024-03-19T02:01:05.960 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.958+0000 7f851c6c6640 1 Processor -- start 2024-03-19T02:01:05.960 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.958+0000 7f851c6c6640 1 -- start start 2024-03-19T02:01:05.960 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.958+0000 7f851c6c6640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8514100740 0x7f8514197a40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:05.960 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.958+0000 7f851c6c6640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8514101930 0x7f8514197f80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:05.960 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.958+0000 7f851c6c6640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f85141022f0 0x7f851419eff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:05.961 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.958+0000 7f851c6c6640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f851410d490 con 0x7f85141022f0 2024-03-19T02:01:05.961 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.958+0000 7f851c6c6640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f851410d310 con 0x7f8514100740 2024-03-19T02:01:05.961 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.958+0000 7f851c6c6640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f851410d610 con 0x7f8514101930 2024-03-19T02:01:05.961 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.958+0000 7f851a462640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8514100740 0x7f8514197a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:05.961 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.958+0000 7f851ac63640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f85141022f0 0x7f851419eff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:05.961 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.958+0000 7f851a462640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8514100740 0x7f8514197a40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:43140/0 (socket says 172.21.15.99:43140) 2024-03-19T02:01:05.961 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.958+0000 7f851a462640 1 -- 172.21.15.99:0/2721112241 learned_addr learned my addr 172.21.15.99:0/2721112241 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:01:05.961 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.958+0000 7f8519c61640 1 --2- 172.21.15.99:0/2721112241 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8514101930 0x7f8514197f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:05.961 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.959+0000 7f851ac63640 1 -- 172.21.15.99:0/2721112241 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8514101930 msgr2=0x7f8514197f80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:01:05.961 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.959+0000 7f851ac63640 1 --2- 172.21.15.99:0/2721112241 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8514101930 0x7f8514197f80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:05.961 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.959+0000 7f851ac63640 1 -- 172.21.15.99:0/2721112241 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8514100740 msgr2=0x7f8514197a40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:01:05.961 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.959+0000 7f851ac63640 1 --2- 172.21.15.99:0/2721112241 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8514100740 0x7f8514197a40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:05.961 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.959+0000 7f851ac63640 1 -- 172.21.15.99:0/2721112241 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f851419f660 con 0x7f85141022f0 2024-03-19T02:01:05.962 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.959+0000 7f851ac63640 1 --2- 172.21.15.99:0/2721112241 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f85141022f0 0x7f851419eff0 secure :-1 s=READY pgs=722 cs=0 l=1 rev1=1 crypto rx=0x7f850c02c630 tx=0x7f850c032f20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:01:05.962 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.959+0000 7f850b7fe640 1 -- 172.21.15.99:0/2721112241 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f850c041020 con 0x7f85141022f0 2024-03-19T02:01:05.962 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.959+0000 7f851c6c6640 1 -- 172.21.15.99:0/2721112241 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f851419f8f0 con 0x7f85141022f0 2024-03-19T02:01:05.962 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.959+0000 7f850b7fe640 1 -- 172.21.15.99:0/2721112241 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f850c034cd0 con 0x7f85141022f0 2024-03-19T02:01:05.962 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.959+0000 7f850b7fe640 1 -- 172.21.15.99:0/2721112241 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f850c039040 con 0x7f85141022f0 2024-03-19T02:01:05.962 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.959+0000 7f851c6c6640 1 -- 172.21.15.99:0/2721112241 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f851419fe00 con 0x7f85141022f0 2024-03-19T02:01:05.967 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.961+0000 7f850b7fe640 1 -- 172.21.15.99:0/2721112241 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f850c044020 con 0x7f85141022f0 2024-03-19T02:01:05.967 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.961+0000 7f850b7fe640 1 --2- 172.21.15.99:0/2721112241 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f85000a28a0 0x7f85000a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:05.968 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.961+0000 7f851f390740 1 Processor -- start 2024-03-19T02:01:05.968 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.962+0000 7f850b7fe640 1 -- 172.21.15.99:0/2721112241 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f850c0f0860 con 0x7f85141022f0 2024-03-19T02:01:05.968 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.962+0000 7f851a462640 1 --2- 172.21.15.99:0/2721112241 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f85000a28a0 0x7f85000a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:05.968 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.962+0000 7f851f390740 1 -- start start 2024-03-19T02:01:05.968 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.962+0000 7f851f390740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x558a6f830100 0x558a6f830520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:05.968 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.962+0000 7f851f390740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x558a6f830af0 0x558a6f8355f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:05.968 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.962+0000 7f851f390740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x558a6f835c60 0x558a6f8380f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:05.968 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.962+0000 7f851f390740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x558a6f794be0 con 0x558a6f830100 2024-03-19T02:01:05.968 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.962+0000 7f851f390740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x558a6f73d960 con 0x558a6f835c60 2024-03-19T02:01:05.968 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.962+0000 7f851f390740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x558a6f790c80 con 0x558a6f830af0 2024-03-19T02:01:05.968 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.962+0000 7f851a462640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x558a6f830100 0x558a6f830520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:05.968 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.962+0000 7f851a462640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x558a6f830100 0x558a6f830520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:53482/0 (socket says 172.21.15.99:53482) 2024-03-19T02:01:05.968 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.962+0000 7f8519c61640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x558a6f830af0 0x558a6f8355f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:05.969 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.962+0000 7f851a462640 1 -- 172.21.15.99:0/24865607 learned_addr learned my addr 172.21.15.99:0/24865607 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:01:05.969 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.962+0000 7f851ac63640 1 --2- 172.21.15.99:0/24865607 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x558a6f835c60 0x558a6f8380f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:05.969 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.962+0000 7f851a462640 1 --2- 172.21.15.99:0/2721112241 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f85000a28a0 0x7f85000a4d60 secure :-1 s=READY pgs=362 cs=0 l=1 rev1=1 crypto rx=0x7f85100058c0 tx=0x7f8510011810 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:01:05.969 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.962+0000 7f851ac63640 1 -- 172.21.15.99:0/24865607 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x558a6f830af0 msgr2=0x558a6f8355f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:01:05.969 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.962+0000 7f851ac63640 1 --2- 172.21.15.99:0/24865607 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x558a6f830af0 0x558a6f8355f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:05.969 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.962+0000 7f851ac63640 1 -- 172.21.15.99:0/24865607 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x558a6f830100 msgr2=0x558a6f830520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:01:05.969 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.962+0000 7f851ac63640 1 --2- 172.21.15.99:0/24865607 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x558a6f830100 0x558a6f830520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:05.969 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.962+0000 7f851ac63640 1 -- 172.21.15.99:0/24865607 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x558a6f734d20 con 0x558a6f835c60 2024-03-19T02:01:05.969 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.963+0000 7f851ac63640 1 --2- 172.21.15.99:0/24865607 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x558a6f835c60 0x558a6f8380f0 secure :-1 s=READY pgs=206 cs=0 l=1 rev1=1 crypto rx=0x7f850c0ee690 tx=0x7f850c0095c0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=78519a8689ab420a server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:01:05.969 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.963+0000 7f8508ff9640 1 -- 172.21.15.99:0/24865607 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f850c0c5bb0 con 0x558a6f835c60 2024-03-19T02:01:05.969 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.963+0000 7f8508ff9640 1 -- 172.21.15.99:0/24865607 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f850c0098d0 con 0x558a6f835c60 2024-03-19T02:01:05.969 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.963+0000 7f8508ff9640 1 -- 172.21.15.99:0/24865607 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f850c003970 con 0x558a6f835c60 2024-03-19T02:01:05.969 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.964+0000 7f851f390740 1 -- 172.21.15.99:0/24865607 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x558a6f835c60 msgr2=0x558a6f8380f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:01:05.970 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.964+0000 7f851f390740 1 --2- 172.21.15.99:0/24865607 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x558a6f835c60 0x558a6f8380f0 secure :-1 s=READY pgs=206 cs=0 l=1 rev1=1 crypto rx=0x7f850c0ee690 tx=0x7f850c0095c0 comp rx=0 tx=0).stop 2024-03-19T02:01:05.970 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.964+0000 7f851f390740 1 -- 172.21.15.99:0/24865607 shutdown_connections 2024-03-19T02:01:05.970 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.964+0000 7f851f390740 1 --2- 172.21.15.99:0/24865607 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x558a6f835c60 0x558a6f8380f0 unknown :-1 s=CLOSED pgs=206 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:05.970 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.964+0000 7f851f390740 1 --2- 172.21.15.99:0/24865607 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x558a6f830af0 0x558a6f8355f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:05.970 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.964+0000 7f851f390740 1 --2- 172.21.15.99:0/24865607 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x558a6f830100 0x558a6f830520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:05.970 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.964+0000 7f851f390740 1 -- 172.21.15.99:0/24865607 >> 172.21.15.99:0/24865607 conn(0x558a6f82b670 msgr2=0x558a6f82ba70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:01:05.970 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.964+0000 7f851f390740 1 -- 172.21.15.99:0/24865607 shutdown_connections 2024-03-19T02:01:05.970 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.964+0000 7f851f390740 1 -- 172.21.15.99:0/24865607 wait complete. 2024-03-19T02:01:05.970 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.964+0000 7f851f390740 1 -- start start 2024-03-19T02:01:05.970 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.964+0000 7f851f390740 1 Processor -- start 2024-03-19T02:01:05.970 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.964+0000 7f851f390740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x558a6f830100 0x558a6f839da0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:05.970 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.964+0000 7f851f390740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x558a6f830af0 0x558a6f83a2e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:05.970 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.964+0000 7f851f390740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x558a6f835c60 0x558a6f841320 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:05.971 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.964+0000 7f851f390740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x558a6f8389d0 con 0x558a6f835c60 2024-03-19T02:01:05.971 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.964+0000 7f851f390740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x558a6f750db0 con 0x558a6f830af0 2024-03-19T02:01:05.971 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.964+0000 7f851f390740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x558a6f838b50 con 0x558a6f830100 2024-03-19T02:01:05.971 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.964+0000 7f851f390740 10 client.0 _ll_register_callbacks cb 0x558a6f755e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T02:01:05.971 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.964+0000 7f851ac63640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x558a6f835c60 0x558a6f841320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:05.971 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.965+0000 7f851a462640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x558a6f830100 0x558a6f839da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:05.971 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.965+0000 7f851a462640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x558a6f830100 0x558a6f839da0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:57826/0 (socket says 172.21.15.99:57826) 2024-03-19T02:01:05.971 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.965+0000 7f851a462640 1 -- 172.21.15.99:0/3076821920 learned_addr learned my addr 172.21.15.99:0/3076821920 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:01:05.971 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.965+0000 7f8519c61640 1 --2- 172.21.15.99:0/3076821920 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x558a6f830af0 0x558a6f83a2e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:05.971 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.965+0000 7f851a462640 1 -- 172.21.15.99:0/3076821920 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x558a6f830af0 msgr2=0x558a6f83a2e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:01:05.971 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.965+0000 7f851a462640 1 --2- 172.21.15.99:0/3076821920 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x558a6f830af0 0x558a6f83a2e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:05.971 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.965+0000 7f851a462640 1 -- 172.21.15.99:0/3076821920 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x558a6f835c60 msgr2=0x558a6f841320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:01:05.971 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.965+0000 7f851a462640 1 --2- 172.21.15.99:0/3076821920 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x558a6f835c60 0x558a6f841320 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:05.971 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.965+0000 7f851a462640 1 -- 172.21.15.99:0/3076821920 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({osdmap=0}) -- 0x558a6f74fc60 con 0x558a6f830100 2024-03-19T02:01:05.972 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.965+0000 7f851ac63640 1 --2- 172.21.15.99:0/3076821920 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x558a6f835c60 0x558a6f841320 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:01:05.972 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.965+0000 7f8519c61640 1 --2- 172.21.15.99:0/3076821920 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x558a6f830af0 0x558a6f83a2e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:01:05.972 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.965+0000 7f851a462640 1 --2- 172.21.15.99:0/3076821920 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x558a6f830100 0x558a6f839da0 secure :-1 s=READY pgs=127 cs=0 l=1 rev1=1 crypto rx=0x7f851001d9c0 tx=0x7f8510016000 comp rx=0 tx=0).ready entity=mon.1 client_cookie=b0be7bfc10848d54 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:01:05.972 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.965+0000 7f84fdffb640 10 client.0 ms_handle_connect on v2:172.21.15.115:3300/0 2024-03-19T02:01:05.972 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.965+0000 7f84fdffb640 1 -- 172.21.15.99:0/3076821920 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8510017040 con 0x558a6f830100 2024-03-19T02:01:05.972 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.965+0000 7f851f390740 1 -- 172.21.15.99:0/3076821920 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x558a6f7a2120 con 0x558a6f830100 2024-03-19T02:01:05.972 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.965+0000 7f84fdffb640 1 -- 172.21.15.99:0/3076821920 <== mon.1 v2:172.21.15.115:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f8510005020 con 0x558a6f830100 2024-03-19T02:01:05.972 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.965+0000 7f84fdffb640 1 -- 172.21.15.99:0/3076821920 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7f8510005220 con 0x558a6f830100 2024-03-19T02:01:05.972 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.965+0000 7f851f390740 10 client.26737 fetch_fsmap learned FSMap version 37 2024-03-19T02:01:05.972 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.965+0000 7f851f390740 1 -- 172.21.15.99:0/3076821920 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x558a6f4a9210 con 0x558a6f830100 2024-03-19T02:01:05.972 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.966+0000 7f84fdffb640 1 -- 172.21.15.99:0/3076821920 <== mon.1 v2:172.21.15.115:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x558a6f7a2120 con 0x558a6f830100 2024-03-19T02:01:05.972 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.966+0000 7f84fdffb640 1 -- 172.21.15.99:0/3076821920 <== mon.1 v2:172.21.15.115:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7f851000f8d0 con 0x558a6f830100 2024-03-19T02:01:05.972 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.966+0000 7f851f390740 10 client.26737 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T02:01:05.972 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.966+0000 7f851f390740 10 client.26737 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T02:01:05.973 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.966+0000 7f851f390740 20 client.26737 populate_metadata read hostname 'smithi099' 2024-03-19T02:01:05.973 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.966+0000 7f851f390740 1 --2- 172.21.15.99:0/3076821920 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x558a6f845c20 0x558a6f848070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:05.973 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.966+0000 7f851f390740 4 client.26737 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T02:01:05.973 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.966+0000 7f851f390740 1 -- 172.21.15.99:0/3076821920 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x558a6f79c470 con 0x558a6f845c20 2024-03-19T02:01:05.973 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.966+0000 7f8519c61640 1 --2- 172.21.15.99:0/3076821920 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x558a6f845c20 0x558a6f848070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:05.973 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.966+0000 7f8519c61640 1 --2- 172.21.15.99:0/3076821920 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x558a6f845c20 0x558a6f848070 crc :-1 s=READY pgs=207 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=a67a99b7dc1c0d7f server_cookie=f4c0b4e2dc483fcf in_seq=0 out_seq=0 2024-03-19T02:01:05.973 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.966+0000 7f84fdffb640 10 client.26737 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T02:01:05.974 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.970+0000 7f84fdffb640 1 -- 172.21.15.99:0/3076821920 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x558a6f79c470 con 0x558a6f845c20 2024-03-19T02:01:05.975 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.970+0000 7f84fdffb640 10 client.26737 handle_command_reply: tid=0 2024-03-19T02:01:05.975 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.970+0000 7f851f390740 1 client.26737 shutdown 2024-03-19T02:01:05.975 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.970+0000 7f851f390740 1 -- 172.21.15.99:0/3076821920 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x558a6f830100 msgr2=0x558a6f839da0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:01:05.975 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.970+0000 7f851f390740 1 --2- 172.21.15.99:0/3076821920 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x558a6f830100 0x558a6f839da0 secure :-1 s=READY pgs=127 cs=0 l=1 rev1=1 crypto rx=0x7f851001d9c0 tx=0x7f8510016000 comp rx=0 tx=0).stop 2024-03-19T02:01:05.975 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.970+0000 7f851f390740 1 -- 172.21.15.99:0/3076821920 shutdown_connections 2024-03-19T02:01:05.975 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.970+0000 7f851f390740 1 --2- 172.21.15.99:0/3076821920 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x558a6f845c20 0x558a6f848070 crc :-1 s=READY pgs=207 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:05.975 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.970+0000 7f851f390740 1 --2- 172.21.15.99:0/3076821920 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x558a6f830af0 0x558a6f83a2e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:05.975 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.970+0000 7f851f390740 1 --2- 172.21.15.99:0/3076821920 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x558a6f835c60 0x558a6f841320 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:05.975 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.970+0000 7f851f390740 1 --2- 172.21.15.99:0/3076821920 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x558a6f830100 0x558a6f839da0 unknown :-1 s=CLOSED pgs=127 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:05.975 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.970+0000 7f84fdffb640 0 client.26737 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T02:01:05.975 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.970+0000 7f851f390740 1 -- 172.21.15.99:0/3076821920 >> 172.21.15.99:0/3076821920 conn(0x558a6f82b670 msgr2=0x558a6f82c8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:01:05.975 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.970+0000 7f851f390740 1 -- 172.21.15.99:0/3076821920 shutdown_connections 2024-03-19T02:01:05.975 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.970+0000 7f851f390740 1 -- 172.21.15.99:0/3076821920 wait complete. 2024-03-19T02:01:05.975 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.970+0000 7f851f390740 20 client.26737 trim_cache size 0 max 16384 2024-03-19T02:01:05.994 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.986+0000 7f851f390740 1 Processor -- start 2024-03-19T02:01:05.994 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.986+0000 7f851f390740 1 -- start start 2024-03-19T02:01:05.994 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.986+0000 7f851f390740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x558a6f830100 0x558a6f83a240 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:05.994 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.986+0000 7f851f390740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x558a6f830af0 0x558a6f82a930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:05.994 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.986+0000 7f851f390740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x558a6f835c60 0x558a6f831590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:05.994 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.986+0000 7f851f390740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f85141ab620 con 0x558a6f835c60 2024-03-19T02:01:05.994 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.986+0000 7f851f390740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f85141a8ef0 con 0x558a6f830af0 2024-03-19T02:01:05.994 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.986+0000 7f851f390740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f85141a6d40 con 0x558a6f830100 2024-03-19T02:01:05.994 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.986+0000 7f851ac63640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x558a6f835c60 0x558a6f831590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:05.994 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.986+0000 7f8519c61640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x558a6f830af0 0x558a6f82a930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:05.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.986+0000 7f851ac63640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x558a6f835c60 0x558a6f831590 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:53504/0 (socket says 172.21.15.99:53504) 2024-03-19T02:01:05.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.986+0000 7f851ac63640 1 -- 172.21.15.99:0/4051619184 learned_addr learned my addr 172.21.15.99:0/4051619184 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:01:05.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.986+0000 7f851a462640 1 --2- 172.21.15.99:0/4051619184 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x558a6f830100 0x558a6f83a240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:05.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.986+0000 7f851ac63640 1 -- 172.21.15.99:0/4051619184 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x558a6f830100 msgr2=0x558a6f83a240 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:01:05.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.986+0000 7f851ac63640 1 --2- 172.21.15.99:0/4051619184 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x558a6f830100 0x558a6f83a240 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:05.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.986+0000 7f851ac63640 1 -- 172.21.15.99:0/4051619184 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x558a6f830af0 msgr2=0x558a6f82a930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:01:05.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.986+0000 7f851ac63640 1 --2- 172.21.15.99:0/4051619184 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x558a6f830af0 0x558a6f82a930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:05.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.986+0000 7f851ac63640 1 -- 172.21.15.99:0/4051619184 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x558a6f575620 con 0x558a6f835c60 2024-03-19T02:01:05.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.986+0000 7f8519c61640 1 --2- 172.21.15.99:0/4051619184 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x558a6f830af0 0x558a6f82a930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:01:05.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.986+0000 7f851ac63640 1 --2- 172.21.15.99:0/4051619184 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x558a6f835c60 0x558a6f831590 secure :-1 s=READY pgs=723 cs=0 l=1 rev1=1 crypto rx=0x7f850c0c5430 tx=0x7f850c0c5960 comp rx=0 tx=0).ready entity=mon.0 client_cookie=40810405aa5b42da server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:01:05.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.987+0000 7f84fffff640 1 -- 172.21.15.99:0/4051619184 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f850c0c5bb0 con 0x558a6f835c60 2024-03-19T02:01:05.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.987+0000 7f84fffff640 1 -- 172.21.15.99:0/4051619184 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f850c0bd250 con 0x558a6f835c60 2024-03-19T02:01:05.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.987+0000 7f84fffff640 1 -- 172.21.15.99:0/4051619184 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f850c0b98c0 con 0x558a6f835c60 2024-03-19T02:01:05.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.987+0000 7f851f390740 1 -- 172.21.15.99:0/4051619184 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x558a6f835c60 msgr2=0x558a6f831590 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:01:05.996 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.987+0000 7f851f390740 1 --2- 172.21.15.99:0/4051619184 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x558a6f835c60 0x558a6f831590 secure :-1 s=READY pgs=723 cs=0 l=1 rev1=1 crypto rx=0x7f850c0c5430 tx=0x7f850c0c5960 comp rx=0 tx=0).stop 2024-03-19T02:01:05.996 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.987+0000 7f851f390740 1 -- 172.21.15.99:0/4051619184 shutdown_connections 2024-03-19T02:01:05.996 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.987+0000 7f851f390740 1 --2- 172.21.15.99:0/4051619184 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x558a6f830af0 0x558a6f82a930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:05.996 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.987+0000 7f851f390740 1 --2- 172.21.15.99:0/4051619184 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x558a6f835c60 0x558a6f831590 unknown :-1 s=CLOSED pgs=723 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:05.996 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.987+0000 7f851f390740 1 --2- 172.21.15.99:0/4051619184 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x558a6f830100 0x558a6f83a240 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:05.996 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.987+0000 7f851f390740 1 -- 172.21.15.99:0/4051619184 >> 172.21.15.99:0/4051619184 conn(0x558a6f82b670 msgr2=0x558a6f82a220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:01:05.996 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.987+0000 7f851f390740 1 -- 172.21.15.99:0/4051619184 shutdown_connections 2024-03-19T02:01:05.996 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.987+0000 7f851f390740 1 -- 172.21.15.99:0/4051619184 wait complete. 2024-03-19T02:01:05.996 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.988+0000 7f851f390740 1 -- start start 2024-03-19T02:01:05.996 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.988+0000 7f851f390740 1 Processor -- start 2024-03-19T02:01:05.996 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.988+0000 7f851f390740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x558a6f830100 0x558a6f8a7070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:05.996 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.988+0000 7f851f390740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x558a6f830af0 0x558a6f8a75b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:05.996 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.988+0000 7f851f390740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x558a6f835c60 0x558a6f8ae5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:05.996 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.988+0000 7f851f390740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f85141a73b0 con 0x558a6f830100 2024-03-19T02:01:05.996 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.988+0000 7f851f390740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f85141a8820 con 0x558a6f830af0 2024-03-19T02:01:05.997 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.988+0000 7f851f390740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f85141a7a80 con 0x558a6f835c60 2024-03-19T02:01:05.997 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.988+0000 7f851f390740 10 client.0 _ll_register_callbacks cb 0x7f851001d8d0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T02:01:05.997 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.988+0000 7f851a462640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x558a6f830100 0x558a6f8a7070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:05.997 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.988+0000 7f851a462640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x558a6f830100 0x558a6f8a7070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:53512/0 (socket says 172.21.15.99:53512) 2024-03-19T02:01:05.997 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.988+0000 7f851a462640 1 -- 172.21.15.99:0/230909514 learned_addr learned my addr 172.21.15.99:0/230909514 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:01:05.997 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.988+0000 7f851ac63640 1 --2- 172.21.15.99:0/230909514 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x558a6f835c60 0x558a6f8ae5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:05.997 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.988+0000 7f8519c61640 1 --2- 172.21.15.99:0/230909514 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x558a6f830af0 0x558a6f8a75b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:05.997 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.988+0000 7f851a462640 1 -- 172.21.15.99:0/230909514 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x558a6f835c60 msgr2=0x558a6f8ae5f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:01:05.997 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.988+0000 7f851a462640 1 --2- 172.21.15.99:0/230909514 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x558a6f835c60 0x558a6f8ae5f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:05.997 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.988+0000 7f851a462640 1 -- 172.21.15.99:0/230909514 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x558a6f830af0 msgr2=0x558a6f8a75b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:01:05.997 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.988+0000 7f851a462640 1 --2- 172.21.15.99:0/230909514 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x558a6f830af0 0x558a6f8a75b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:05.997 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.988+0000 7f851a462640 1 -- 172.21.15.99:0/230909514 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x558a6f8aebe0 con 0x558a6f830100 2024-03-19T02:01:05.997 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.988+0000 7f851a462640 1 --2- 172.21.15.99:0/230909514 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x558a6f830100 0x558a6f8a7070 secure :-1 s=READY pgs=724 cs=0 l=1 rev1=1 crypto rx=0x7f851000fd90 tx=0x7f85100265b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2bc212b1f5ab2da3 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:01:05.997 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.988+0000 7f84fdffb640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T02:01:05.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.989+0000 7f84fdffb640 1 -- 172.21.15.99:0/230909514 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f851003e060 con 0x558a6f830100 2024-03-19T02:01:05.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.989+0000 7f84fdffb640 1 -- 172.21.15.99:0/230909514 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f8510009040 con 0x558a6f830100 2024-03-19T02:01:05.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.989+0000 7f851f390740 1 -- 172.21.15.99:0/230909514 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x558a6f71a770 con 0x558a6f830100 2024-03-19T02:01:05.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.989+0000 7f84fdffb640 1 -- 172.21.15.99:0/230909514 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7f8510035070 con 0x558a6f830100 2024-03-19T02:01:05.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.989+0000 7f851f390740 10 client.16944 fetch_fsmap learned FSMap version 37 2024-03-19T02:01:05.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.989+0000 7f851f390740 1 -- 172.21.15.99:0/230909514 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x558a6f8aff60 con 0x558a6f830100 2024-03-19T02:01:05.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.989+0000 7f84fdffb640 1 -- 172.21.15.99:0/230909514 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x558a6f71a770 con 0x558a6f830100 2024-03-19T02:01:05.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.989+0000 7f84fdffb640 1 -- 172.21.15.99:0/230909514 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7f8510026c60 con 0x558a6f830100 2024-03-19T02:01:05.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.989+0000 7f851f390740 10 client.16944 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T02:01:05.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.989+0000 7f851f390740 10 client.16944 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T02:01:05.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.989+0000 7f851f390740 20 client.16944 populate_metadata read hostname 'smithi099' 2024-03-19T02:01:05.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.989+0000 7f851f390740 1 --2- 172.21.15.99:0/230909514 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x558a6f8b0990 0x558a6f8b2db0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:05.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.989+0000 7f851f390740 4 client.16944 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T02:01:05.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.989+0000 7f851f390740 1 -- 172.21.15.99:0/230909514 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x558a6f766700 con 0x558a6f8b0990 2024-03-19T02:01:05.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.989+0000 7f8519c61640 1 --2- 172.21.15.99:0/230909514 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x558a6f8b0990 0x558a6f8b2db0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:05.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.990+0000 7f84fdffb640 10 client.16944 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T02:01:05.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:05.990+0000 7f8519c61640 1 --2- 172.21.15.99:0/230909514 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x558a6f8b0990 0x558a6f8b2db0 crc :-1 s=READY pgs=208 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=ef60d20e3d7da9b9 server_cookie=edffc449ad642002 in_seq=0 out_seq=0 2024-03-19T02:01:06.113 INFO:tasks.workunit.client.0.smithi099.stdout: 10 196042 22.79 MB/sec execute 335 sec latency 93.613 ms 2024-03-19T02:01:06.612 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:06.611+0000 7f84fdffb640 1 -- 172.21.15.99:0/230909514 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x558a6f766700 con 0x558a6f8b0990 2024-03-19T02:01:06.612 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:06.611+0000 7f84fdffb640 10 client.16944 handle_command_reply: tid=0 2024-03-19T02:01:06.612 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:06.611+0000 7f851f390740 1 client.16944 shutdown 2024-03-19T02:01:06.612 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:06.611+0000 7f851f390740 1 -- 172.21.15.99:0/230909514 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x558a6f830100 msgr2=0x558a6f8a7070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:01:06.612 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:06.611+0000 7f851f390740 1 --2- 172.21.15.99:0/230909514 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x558a6f830100 0x558a6f8a7070 secure :-1 s=READY pgs=724 cs=0 l=1 rev1=1 crypto rx=0x7f851000fd90 tx=0x7f85100265b0 comp rx=0 tx=0).stop 2024-03-19T02:01:06.612 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:06.612+0000 7f851f390740 1 -- 172.21.15.99:0/230909514 shutdown_connections 2024-03-19T02:01:06.612 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:06.612+0000 7f851f390740 1 --2- 172.21.15.99:0/230909514 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x558a6f8b0990 0x558a6f8b2db0 crc :-1 s=READY pgs=208 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:06.612 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:06.612+0000 7f84fdffb640 0 client.16944 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T02:01:06.612 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:06.612+0000 7f851f390740 1 --2- 172.21.15.99:0/230909514 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x558a6f830af0 0x558a6f8a75b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:06.612 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:06.612+0000 7f851f390740 1 --2- 172.21.15.99:0/230909514 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x558a6f835c60 0x558a6f8ae5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:06.613 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:06.612+0000 7f851f390740 1 --2- 172.21.15.99:0/230909514 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x558a6f830100 0x558a6f8a7070 unknown :-1 s=CLOSED pgs=724 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:06.613 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:06.612+0000 7f851f390740 1 -- 172.21.15.99:0/230909514 >> 172.21.15.99:0/230909514 conn(0x558a6f82b670 msgr2=0x558a6f831590 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:01:06.613 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:06.612+0000 7f851f390740 1 -- 172.21.15.99:0/230909514 shutdown_connections 2024-03-19T02:01:06.613 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:06.612+0000 7f851f390740 1 -- 172.21.15.99:0/230909514 wait complete. 2024-03-19T02:01:06.613 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:06.612+0000 7f851f390740 20 client.16944 trim_cache size 0 max 16384 2024-03-19T02:01:06.613 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T02:01:06.613 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T02:01:06.613 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T02:01:06.613 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T02:01:06.614 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:06.612+0000 7f851c6c6640 1 -- 172.21.15.99:0/2721112241 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f85000a28a0 msgr2=0x7f85000a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:01:06.614 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:06.612+0000 7f851c6c6640 1 --2- 172.21.15.99:0/2721112241 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f85000a28a0 0x7f85000a4d60 secure :-1 s=READY pgs=362 cs=0 l=1 rev1=1 crypto rx=0x7f85100058c0 tx=0x7f8510011810 comp rx=0 tx=0).stop 2024-03-19T02:01:06.614 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:06.612+0000 7f851c6c6640 1 -- 172.21.15.99:0/2721112241 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f85141022f0 msgr2=0x7f851419eff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:01:06.614 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:06.612+0000 7f851c6c6640 1 --2- 172.21.15.99:0/2721112241 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f85141022f0 0x7f851419eff0 secure :-1 s=READY pgs=722 cs=0 l=1 rev1=1 crypto rx=0x7f850c02c630 tx=0x7f850c032f20 comp rx=0 tx=0).stop 2024-03-19T02:01:06.614 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:06.612+0000 7f851c6c6640 1 -- 172.21.15.99:0/2721112241 shutdown_connections 2024-03-19T02:01:06.614 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:06.612+0000 7f851c6c6640 1 --2- 172.21.15.99:0/2721112241 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f85141022f0 0x7f851419eff0 unknown :-1 s=CLOSED pgs=722 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:06.615 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:06.613+0000 7f851c6c6640 1 --2- 172.21.15.99:0/2721112241 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8514101930 0x7f8514197f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:06.615 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:06.613+0000 7f851c6c6640 1 --2- 172.21.15.99:0/2721112241 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f85000a28a0 0x7f85000a4d60 unknown :-1 s=CLOSED pgs=362 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:06.615 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:06.613+0000 7f851c6c6640 1 --2- 172.21.15.99:0/2721112241 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8514100740 0x7f8514197a40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:06.615 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:06.613+0000 7f851c6c6640 1 -- 172.21.15.99:0/2721112241 >> 172.21.15.99:0/2721112241 conn(0x7f85140fbcb0 msgr2=0x7f8514104bd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:01:06.615 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:06.613+0000 7f851c6c6640 1 -- 172.21.15.99:0/2721112241 shutdown_connections 2024-03-19T02:01:06.615 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:06.613+0000 7f851c6c6640 1 -- 172.21.15.99:0/2721112241 wait complete. 2024-03-19T02:01:07.113 INFO:tasks.workunit.client.0.smithi099.stdout: 10 196257 22.75 MB/sec execute 336 sec latency 420.454 ms 2024-03-19T02:01:07.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:01:07 smithi115 ceph-mon[67332]: pgmap v952: 97 pgs: 97 active+clean; 604 MiB data, 7.8 GiB used, 1.0 TiB / 1.0 TiB avail; 4.4 MiB/s rd, 6.2 MiB/s wr, 581 op/s 2024-03-19T02:01:07.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:01:07 smithi099 ceph-mon[62951]: pgmap v952: 97 pgs: 97 active+clean; 604 MiB data, 7.8 GiB used, 1.0 TiB / 1.0 TiB avail; 4.4 MiB/s rd, 6.2 MiB/s wr, 581 op/s 2024-03-19T02:01:07.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:01:07 smithi112 ceph-mon[66819]: pgmap v952: 97 pgs: 97 active+clean; 604 MiB data, 7.8 GiB used, 1.0 TiB / 1.0 TiB avail; 4.4 MiB/s rd, 6.2 MiB/s wr, 581 op/s 2024-03-19T02:01:08.113 INFO:tasks.workunit.client.0.smithi099.stdout: 10 196593 22.73 MB/sec execute 337 sec latency 107.403 ms 2024-03-19T02:01:09.113 INFO:tasks.workunit.client.0.smithi099.stdout: 10 196854 22.70 MB/sec execute 338 sec latency 292.420 ms 2024-03-19T02:01:09.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:01:09 smithi115 ceph-mon[67332]: pgmap v953: 97 pgs: 97 active+clean; 605 MiB data, 7.9 GiB used, 1.0 TiB / 1.0 TiB avail; 3.7 MiB/s rd, 5.8 MiB/s wr, 485 op/s 2024-03-19T02:01:09.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:01:09 smithi099 ceph-mon[62951]: pgmap v953: 97 pgs: 97 active+clean; 605 MiB data, 7.9 GiB used, 1.0 TiB / 1.0 TiB avail; 3.7 MiB/s rd, 5.8 MiB/s wr, 485 op/s 2024-03-19T02:01:09.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:01:09 smithi112 ceph-mon[66819]: pgmap v953: 97 pgs: 97 active+clean; 605 MiB data, 7.9 GiB used, 1.0 TiB / 1.0 TiB avail; 3.7 MiB/s rd, 5.8 MiB/s wr, 485 op/s 2024-03-19T02:01:10.113 INFO:tasks.workunit.client.0.smithi099.stdout: 10 197085 22.66 MB/sec execute 339 sec latency 131.220 ms 2024-03-19T02:01:11.113 INFO:tasks.workunit.client.0.smithi099.stdout: 10 197525 22.66 MB/sec execute 340 sec latency 134.356 ms 2024-03-19T02:01:11.391 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:01:11 smithi099 ceph-mon[62951]: pgmap v954: 97 pgs: 97 active+clean; 609 MiB data, 7.9 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 6.2 MiB/s wr, 561 op/s 2024-03-19T02:01:11.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:01:11 smithi115 ceph-mon[67332]: pgmap v954: 97 pgs: 97 active+clean; 609 MiB data, 7.9 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 6.2 MiB/s wr, 561 op/s 2024-03-19T02:01:11.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:01:11 smithi112 ceph-mon[66819]: pgmap v954: 97 pgs: 97 active+clean; 609 MiB data, 7.9 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 6.2 MiB/s wr, 561 op/s 2024-03-19T02:01:12.113 INFO:tasks.workunit.client.0.smithi099.stdout: 10 198053 22.67 MB/sec execute 341 sec latency 285.514 ms 2024-03-19T02:01:13.113 INFO:tasks.workunit.client.0.smithi099.stdout: 10 198661 22.69 MB/sec execute 342 sec latency 68.857 ms 2024-03-19T02:01:13.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:01:13 smithi115 ceph-mon[67332]: pgmap v955: 97 pgs: 97 active+clean; 605 MiB data, 7.9 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 6.6 MiB/s wr, 609 op/s 2024-03-19T02:01:13.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:01:13 smithi099 ceph-mon[62951]: pgmap v955: 97 pgs: 97 active+clean; 605 MiB data, 7.9 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 6.6 MiB/s wr, 609 op/s 2024-03-19T02:01:13.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:01:13 smithi112 ceph-mon[66819]: pgmap v955: 97 pgs: 97 active+clean; 605 MiB data, 7.9 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 6.6 MiB/s wr, 609 op/s 2024-03-19T02:01:14.114 INFO:tasks.workunit.client.0.smithi099.stdout: 10 199023 22.68 MB/sec execute 343 sec latency 278.776 ms 2024-03-19T02:01:15.113 INFO:tasks.workunit.client.0.smithi099.stdout: 10 199447 22.70 MB/sec execute 344 sec latency 87.898 ms 2024-03-19T02:01:15.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:01:15 smithi115 ceph-mon[67332]: pgmap v956: 97 pgs: 97 active+clean; 605 MiB data, 7.9 GiB used, 1.0 TiB / 1.0 TiB avail; 3.7 MiB/s rd, 6.1 MiB/s wr, 532 op/s 2024-03-19T02:01:15.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:01:15 smithi099 ceph-mon[62951]: pgmap v956: 97 pgs: 97 active+clean; 605 MiB data, 7.9 GiB used, 1.0 TiB / 1.0 TiB avail; 3.7 MiB/s rd, 6.1 MiB/s wr, 532 op/s 2024-03-19T02:01:15.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:01:15 smithi112 ceph-mon[66819]: pgmap v956: 97 pgs: 97 active+clean; 605 MiB data, 7.9 GiB used, 1.0 TiB / 1.0 TiB avail; 3.7 MiB/s rd, 6.1 MiB/s wr, 532 op/s 2024-03-19T02:01:16.114 INFO:tasks.workunit.client.0.smithi099.stdout: 10 199949 22.68 MB/sec execute 345 sec latency 104.777 ms 2024-03-19T02:01:17.114 INFO:tasks.workunit.client.0.smithi099.stdout: 10 200467 22.74 MB/sec execute 346 sec latency 53.271 ms 2024-03-19T02:01:17.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:01:17 smithi115 ceph-mon[67332]: pgmap v957: 97 pgs: 97 active+clean; 610 MiB data, 7.9 GiB used, 1.0 TiB / 1.0 TiB avail; 4.5 MiB/s rd, 7.4 MiB/s wr, 620 op/s 2024-03-19T02:01:17.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:01:17 smithi099 ceph-mon[62951]: pgmap v957: 97 pgs: 97 active+clean; 610 MiB data, 7.9 GiB used, 1.0 TiB / 1.0 TiB avail; 4.5 MiB/s rd, 7.4 MiB/s wr, 620 op/s 2024-03-19T02:01:17.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:01:17 smithi112 ceph-mon[66819]: pgmap v957: 97 pgs: 97 active+clean; 610 MiB data, 7.9 GiB used, 1.0 TiB / 1.0 TiB avail; 4.5 MiB/s rd, 7.4 MiB/s wr, 620 op/s 2024-03-19T02:01:18.114 INFO:tasks.workunit.client.0.smithi099.stdout: 10 200903 22.76 MB/sec execute 347 sec latency 96.160 ms 2024-03-19T02:01:19.114 INFO:tasks.workunit.client.0.smithi099.stdout: 10 201200 22.74 MB/sec execute 348 sec latency 269.074 ms 2024-03-19T02:01:19.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:01:19 smithi115 ceph-mon[67332]: pgmap v958: 97 pgs: 97 active+clean; 613 MiB data, 7.9 GiB used, 1.0 TiB / 1.0 TiB avail; 4.3 MiB/s rd, 6.4 MiB/s wr, 521 op/s 2024-03-19T02:01:19.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:01:19 smithi099 ceph-mon[62951]: pgmap v958: 97 pgs: 97 active+clean; 613 MiB data, 7.9 GiB used, 1.0 TiB / 1.0 TiB avail; 4.3 MiB/s rd, 6.4 MiB/s wr, 521 op/s 2024-03-19T02:01:19.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:01:19 smithi112 ceph-mon[66819]: pgmap v958: 97 pgs: 97 active+clean; 613 MiB data, 7.9 GiB used, 1.0 TiB / 1.0 TiB avail; 4.3 MiB/s rd, 6.4 MiB/s wr, 521 op/s 2024-03-19T02:01:19.717 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.715+0000 7f21d5ea1640 1 -- 172.21.15.99:0/4014611980 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f21d00ffe70 msgr2=0x7f21d0100290 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:01:19.717 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.715+0000 7f21d5ea1640 1 --2- 172.21.15.99:0/4014611980 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f21d00ffe70 0x7f21d0100290 secure :-1 s=READY pgs=725 cs=0 l=1 rev1=1 crypto rx=0x7f21c0009900 tx=0x7f21c002bc30 comp rx=0 tx=0).stop 2024-03-19T02:01:19.717 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.715+0000 7f21d5ea1640 1 -- 172.21.15.99:0/4014611980 shutdown_connections 2024-03-19T02:01:19.717 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.715+0000 7f21d5ea1640 1 --2- 172.21.15.99:0/4014611980 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f21d0101050 0x7f21d01014d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:19.717 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.715+0000 7f21d5ea1640 1 --2- 172.21.15.99:0/4014611980 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f21d0101a10 0x7f21d0108330 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:19.717 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.715+0000 7f21d5ea1640 1 --2- 172.21.15.99:0/4014611980 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f21d00ffe70 0x7f21d0100290 unknown :-1 s=CLOSED pgs=725 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:19.717 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.715+0000 7f21d5ea1640 1 -- 172.21.15.99:0/4014611980 >> 172.21.15.99:0/4014611980 conn(0x7f21d00fb470 msgr2=0x7f21d00fd890 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:01:19.717 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.715+0000 7f21d5ea1640 1 -- 172.21.15.99:0/4014611980 shutdown_connections 2024-03-19T02:01:19.718 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.715+0000 7f21d5ea1640 1 -- 172.21.15.99:0/4014611980 wait complete. 2024-03-19T02:01:19.718 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.715+0000 7f21d5ea1640 1 Processor -- start 2024-03-19T02:01:19.718 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.716+0000 7f21d5ea1640 1 -- start start 2024-03-19T02:01:19.718 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.716+0000 7f21d5ea1640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f21d00ffe70 0x7f21d0197b00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:19.719 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.716+0000 7f21d5ea1640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f21d0101050 0x7f21d0198040 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:19.719 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.716+0000 7f21d5ea1640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f21d0101a10 0x7f21d019f0b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:19.719 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.716+0000 7f21d5ea1640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f21d010c3f0 con 0x7f21d0101050 2024-03-19T02:01:19.719 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.716+0000 7f21d5ea1640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f21d010c270 con 0x7f21d00ffe70 2024-03-19T02:01:19.719 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.716+0000 7f21d5ea1640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f21d010c570 con 0x7f21d0101a10 2024-03-19T02:01:19.719 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.716+0000 7f21ceffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f21d0101050 0x7f21d0198040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:19.719 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.716+0000 7f21ceffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f21d0101050 0x7f21d0198040 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:49386/0 (socket says 172.21.15.99:49386) 2024-03-19T02:01:19.719 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.716+0000 7f21cf7fe640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f21d00ffe70 0x7f21d0197b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:19.719 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.716+0000 7f21ceffd640 1 -- 172.21.15.99:0/4026296038 learned_addr learned my addr 172.21.15.99:0/4026296038 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:01:19.719 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.716+0000 7f21cffff640 1 --2- 172.21.15.99:0/4026296038 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f21d0101a10 0x7f21d019f0b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:19.719 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.716+0000 7f21cf7fe640 1 -- 172.21.15.99:0/4026296038 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f21d0101a10 msgr2=0x7f21d019f0b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:01:19.719 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.716+0000 7f21cf7fe640 1 --2- 172.21.15.99:0/4026296038 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f21d0101a10 0x7f21d019f0b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:19.719 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.716+0000 7f21cf7fe640 1 -- 172.21.15.99:0/4026296038 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f21d0101050 msgr2=0x7f21d0198040 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:01:19.719 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.716+0000 7f21cf7fe640 1 --2- 172.21.15.99:0/4026296038 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f21d0101050 0x7f21d0198040 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:19.720 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.716+0000 7f21cf7fe640 1 -- 172.21.15.99:0/4026296038 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f21d019f720 con 0x7f21d00ffe70 2024-03-19T02:01:19.720 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.716+0000 7f21ceffd640 1 --2- 172.21.15.99:0/4026296038 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f21d0101050 0x7f21d0198040 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:01:19.720 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.716+0000 7f21cf7fe640 1 --2- 172.21.15.99:0/4026296038 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f21d00ffe70 0x7f21d0197b00 secure :-1 s=READY pgs=207 cs=0 l=1 rev1=1 crypto rx=0x7f21c002c170 tx=0x7f21c002c650 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:01:19.720 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.717+0000 7f21ccff9640 1 -- 172.21.15.99:0/4026296038 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f21c0041070 con 0x7f21d00ffe70 2024-03-19T02:01:19.720 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.717+0000 7f21d5ea1640 1 -- 172.21.15.99:0/4026296038 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f21d019f9b0 con 0x7f21d00ffe70 2024-03-19T02:01:19.720 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.717+0000 7f21ccff9640 1 -- 172.21.15.99:0/4026296038 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f21c0004450 con 0x7f21d00ffe70 2024-03-19T02:01:19.720 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.717+0000 7f21ccff9640 1 -- 172.21.15.99:0/4026296038 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f21c0003e50 con 0x7f21d00ffe70 2024-03-19T02:01:19.720 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.717+0000 7f21d5ea1640 1 -- 172.21.15.99:0/4026296038 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f21d019fe70 con 0x7f21d00ffe70 2024-03-19T02:01:19.725 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.719+0000 7f21ccff9640 1 -- 172.21.15.99:0/4026296038 <== mon.2 v2:172.21.15.112:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f21c0044070 con 0x7f21d00ffe70 2024-03-19T02:01:19.725 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.719+0000 7f21d8b6f740 1 Processor -- start 2024-03-19T02:01:19.725 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.719+0000 7f21d8b6f740 1 -- start start 2024-03-19T02:01:19.725 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.719+0000 7f21d8b6f740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5613da676100 0x5613da676520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:19.726 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.719+0000 7f21d8b6f740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5613da676af0 0x5613da67b5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:19.726 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.719+0000 7f21d8b6f740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5613da67bc60 0x5613da67e0f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:19.726 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.719+0000 7f21ccff9640 1 --2- 172.21.15.99:0/4026296038 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f21b80a2a40 0x7f21b80a4f00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:19.726 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.719+0000 7f21d8b6f740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x5613da5dabe0 con 0x5613da676100 2024-03-19T02:01:19.726 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.719+0000 7f21d8b6f740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x5613da583960 con 0x5613da676af0 2024-03-19T02:01:19.726 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.719+0000 7f21d8b6f740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x5613da5d6c80 con 0x5613da67bc60 2024-03-19T02:01:19.726 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.720+0000 7f21cffff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5613da676100 0x5613da676520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:19.726 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.720+0000 7f21ceffd640 1 --2- 172.21.15.99:0/4026296038 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f21b80a2a40 0x7f21b80a4f00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:19.726 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.720+0000 7f21cf7fe640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5613da676af0 0x5613da67b5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:19.726 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.720+0000 7f21cffff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5613da676100 0x5613da676520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:49396/0 (socket says 172.21.15.99:49396) 2024-03-19T02:01:19.726 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.720+0000 7f21cffff640 1 -- 172.21.15.99:0/3676894407 learned_addr learned my addr 172.21.15.99:0/3676894407 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:01:19.726 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.720+0000 7f21cffff640 1 --2- 172.21.15.99:0/3676894407 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5613da67bc60 0x5613da67e0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:19.726 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.720+0000 7f21ccff9640 1 -- 172.21.15.99:0/4026296038 <== mon.2 v2:172.21.15.112:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f21c00f05a0 con 0x7f21d00ffe70 2024-03-19T02:01:19.727 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.720+0000 7f21cf7fe640 1 -- 172.21.15.99:0/3676894407 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5613da67bc60 msgr2=0x5613da67e0f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:01:19.727 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.720+0000 7f21cf7fe640 1 --2- 172.21.15.99:0/3676894407 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5613da67bc60 0x5613da67e0f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:19.727 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.720+0000 7f21cf7fe640 1 -- 172.21.15.99:0/3676894407 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5613da676100 msgr2=0x5613da676520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:01:19.727 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.720+0000 7f21cf7fe640 1 --2- 172.21.15.99:0/3676894407 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5613da676100 0x5613da676520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:19.727 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.720+0000 7f21cf7fe640 1 -- 172.21.15.99:0/3676894407 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5613da57ad20 con 0x5613da676af0 2024-03-19T02:01:19.727 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.720+0000 7f21cffff640 1 --2- 172.21.15.99:0/3676894407 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5613da67bc60 0x5613da67e0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:01:19.727 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.720+0000 7f21ceffd640 1 --2- 172.21.15.99:0/4026296038 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f21b80a2a40 0x7f21b80a4f00 secure :-1 s=READY pgs=363 cs=0 l=1 rev1=1 crypto rx=0x7f21c40095f0 tx=0x7f21c4005f90 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:01:19.727 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.720+0000 7f21cf7fe640 1 --2- 172.21.15.99:0/3676894407 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5613da676af0 0x5613da67b5f0 secure :-1 s=READY pgs=208 cs=0 l=1 rev1=1 crypto rx=0x7f21c002c7b0 tx=0x7f21c004b590 comp rx=0 tx=0).ready entity=mon.2 client_cookie=b5325d45a64c4da4 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:01:19.727 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.720+0000 7f21b5ffb640 1 -- 172.21.15.99:0/3676894407 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f21c004b070 con 0x5613da676af0 2024-03-19T02:01:19.727 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.720+0000 7f21b5ffb640 1 -- 172.21.15.99:0/3676894407 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f21c00bcf50 con 0x5613da676af0 2024-03-19T02:01:19.727 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.720+0000 7f21b5ffb640 1 -- 172.21.15.99:0/3676894407 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f21c00ee110 con 0x5613da676af0 2024-03-19T02:01:19.727 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.721+0000 7f21d8b6f740 1 -- 172.21.15.99:0/3676894407 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5613da676af0 msgr2=0x5613da67b5f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:01:19.727 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.721+0000 7f21d8b6f740 1 --2- 172.21.15.99:0/3676894407 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5613da676af0 0x5613da67b5f0 secure :-1 s=READY pgs=208 cs=0 l=1 rev1=1 crypto rx=0x7f21c002c7b0 tx=0x7f21c004b590 comp rx=0 tx=0).stop 2024-03-19T02:01:19.728 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.721+0000 7f21d8b6f740 1 -- 172.21.15.99:0/3676894407 shutdown_connections 2024-03-19T02:01:19.728 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.721+0000 7f21d8b6f740 1 --2- 172.21.15.99:0/3676894407 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5613da676af0 0x5613da67b5f0 unknown :-1 s=CLOSED pgs=208 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:19.728 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.721+0000 7f21d8b6f740 1 --2- 172.21.15.99:0/3676894407 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5613da67bc60 0x5613da67e0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:19.728 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.721+0000 7f21d8b6f740 1 --2- 172.21.15.99:0/3676894407 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5613da676100 0x5613da676520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:19.728 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.721+0000 7f21d8b6f740 1 -- 172.21.15.99:0/3676894407 >> 172.21.15.99:0/3676894407 conn(0x5613da671670 msgr2=0x5613da671a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:01:19.728 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.721+0000 7f21d8b6f740 1 -- 172.21.15.99:0/3676894407 shutdown_connections 2024-03-19T02:01:19.728 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.721+0000 7f21d8b6f740 1 -- 172.21.15.99:0/3676894407 wait complete. 2024-03-19T02:01:19.728 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.721+0000 7f21d8b6f740 1 -- start start 2024-03-19T02:01:19.728 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.722+0000 7f21d8b6f740 1 Processor -- start 2024-03-19T02:01:19.728 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.722+0000 7f21d8b6f740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5613da676100 0x5613da67fe50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:19.728 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.722+0000 7f21d8b6f740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5613da676af0 0x5613da680390 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:19.728 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.722+0000 7f21d8b6f740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5613da67bc60 0x5613da6873d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:19.729 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.722+0000 7f21d8b6f740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x5613da67e9d0 con 0x5613da67bc60 2024-03-19T02:01:19.729 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.722+0000 7f21d8b6f740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x5613da596db0 con 0x5613da676100 2024-03-19T02:01:19.729 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.722+0000 7f21d8b6f740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x5613da67eb50 con 0x5613da676af0 2024-03-19T02:01:19.729 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.722+0000 7f21d8b6f740 10 client.0 _ll_register_callbacks cb 0x5613da59be70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T02:01:19.729 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.722+0000 7f21cf7fe640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5613da676100 0x5613da67fe50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:19.729 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.722+0000 7f21ceffd640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5613da676af0 0x5613da680390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:19.729 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.722+0000 7f21cffff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5613da67bc60 0x5613da6873d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:19.729 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.722+0000 7f21cffff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5613da67bc60 0x5613da6873d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:49410/0 (socket says 172.21.15.99:49410) 2024-03-19T02:01:19.729 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.722+0000 7f21cf7fe640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5613da676100 0x5613da67fe50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:49486/0 (socket says 172.21.15.99:49486) 2024-03-19T02:01:19.729 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.722+0000 7f21ceffd640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5613da676af0 0x5613da680390 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:49126/0 (socket says 172.21.15.99:49126) 2024-03-19T02:01:19.729 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.722+0000 7f21cffff640 1 -- 172.21.15.99:0/1646458739 learned_addr learned my addr 172.21.15.99:0/1646458739 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:01:19.729 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.722+0000 7f21cffff640 1 -- 172.21.15.99:0/1646458739 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5613da676af0 msgr2=0x5613da680390 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:01:19.729 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.722+0000 7f21cffff640 1 --2- 172.21.15.99:0/1646458739 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5613da676af0 0x5613da680390 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:19.730 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.722+0000 7f21cffff640 1 -- 172.21.15.99:0/1646458739 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5613da676100 msgr2=0x5613da67fe50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:01:19.730 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.722+0000 7f21cffff640 1 --2- 172.21.15.99:0/1646458739 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5613da676100 0x5613da67fe50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:19.730 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.722+0000 7f21cffff640 1 -- 172.21.15.99:0/1646458739 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5613da595c60 con 0x5613da67bc60 2024-03-19T02:01:19.730 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.722+0000 7f21ceffd640 1 --2- 172.21.15.99:0/1646458739 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5613da676af0 0x5613da680390 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:01:19.730 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.722+0000 7f21cf7fe640 1 --2- 172.21.15.99:0/1646458739 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5613da676100 0x5613da67fe50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:01:19.730 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.723+0000 7f21cffff640 1 --2- 172.21.15.99:0/1646458739 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5613da67bc60 0x5613da6873d0 secure :-1 s=READY pgs=726 cs=0 l=1 rev1=1 crypto rx=0x7f21c8011e30 tx=0x7f21c8009f50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=fa8f1e52a9e71024 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:01:19.730 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.723+0000 7f21aeffd640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T02:01:19.730 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.723+0000 7f21aeffd640 1 -- 172.21.15.99:0/1646458739 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f21c8005b50 con 0x5613da67bc60 2024-03-19T02:01:19.730 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.723+0000 7f21d8b6f740 1 -- 172.21.15.99:0/1646458739 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f21c00bcf50 con 0x5613da67bc60 2024-03-19T02:01:19.730 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.723+0000 7f21aeffd640 1 -- 172.21.15.99:0/1646458739 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f21c8003b70 con 0x5613da67bc60 2024-03-19T02:01:19.730 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.723+0000 7f21aeffd640 1 -- 172.21.15.99:0/1646458739 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7f21c8003d70 con 0x5613da67bc60 2024-03-19T02:01:19.730 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.723+0000 7f21d8b6f740 10 client.16956 fetch_fsmap learned FSMap version 37 2024-03-19T02:01:19.730 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.723+0000 7f21d8b6f740 1 -- 172.21.15.99:0/1646458739 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x5613da2ef210 con 0x5613da67bc60 2024-03-19T02:01:19.731 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.723+0000 7f21aeffd640 1 -- 172.21.15.99:0/1646458739 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f21c00bcf50 con 0x5613da67bc60 2024-03-19T02:01:19.731 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.723+0000 7f21aeffd640 1 -- 172.21.15.99:0/1646458739 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7f21c801d070 con 0x5613da67bc60 2024-03-19T02:01:19.731 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.724+0000 7f21d8b6f740 10 client.16956 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T02:01:19.731 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.724+0000 7f21d8b6f740 10 client.16956 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T02:01:19.731 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.724+0000 7f21d8b6f740 20 client.16956 populate_metadata read hostname 'smithi099' 2024-03-19T02:01:19.731 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.724+0000 7f21d8b6f740 1 --2- 172.21.15.99:0/1646458739 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5613da68bcd0 0x5613da68e0e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:19.731 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.724+0000 7f21d8b6f740 4 client.16956 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T02:01:19.731 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.724+0000 7f21d8b6f740 1 -- 172.21.15.99:0/1646458739 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5613da5e8120 con 0x5613da68bcd0 2024-03-19T02:01:19.731 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.724+0000 7f21cf7fe640 1 --2- 172.21.15.99:0/1646458739 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5613da68bcd0 0x5613da68e0e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:19.731 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.724+0000 7f21aeffd640 10 client.16956 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T02:01:19.731 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.724+0000 7f21cf7fe640 1 --2- 172.21.15.99:0/1646458739 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5613da68bcd0 0x5613da68e0e0 crc :-1 s=READY pgs=209 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=142f30e4fba9a78d server_cookie=fe00d339d8579372 in_seq=0 out_seq=0 2024-03-19T02:01:19.732 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.728+0000 7f21aeffd640 1 -- 172.21.15.99:0/1646458739 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x5613da5e8120 con 0x5613da68bcd0 2024-03-19T02:01:19.732 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.728+0000 7f21aeffd640 10 client.16956 handle_command_reply: tid=0 2024-03-19T02:01:19.733 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.728+0000 7f21d8b6f740 1 client.16956 shutdown 2024-03-19T02:01:19.733 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.728+0000 7f21d8b6f740 1 -- 172.21.15.99:0/1646458739 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5613da67bc60 msgr2=0x5613da6873d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:01:19.733 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.728+0000 7f21d8b6f740 1 --2- 172.21.15.99:0/1646458739 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5613da67bc60 0x5613da6873d0 secure :-1 s=READY pgs=726 cs=0 l=1 rev1=1 crypto rx=0x7f21c8011e30 tx=0x7f21c8009f50 comp rx=0 tx=0).stop 2024-03-19T02:01:19.733 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.728+0000 7f21d8b6f740 1 -- 172.21.15.99:0/1646458739 shutdown_connections 2024-03-19T02:01:19.733 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.728+0000 7f21d8b6f740 1 --2- 172.21.15.99:0/1646458739 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5613da68bcd0 0x5613da68e0e0 crc :-1 s=READY pgs=209 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:19.733 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.728+0000 7f21d8b6f740 1 --2- 172.21.15.99:0/1646458739 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5613da67bc60 0x5613da6873d0 unknown :-1 s=CLOSED pgs=726 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:19.733 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.728+0000 7f21d8b6f740 1 --2- 172.21.15.99:0/1646458739 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5613da676af0 0x5613da680390 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:19.733 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.728+0000 7f21aeffd640 0 client.16956 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T02:01:19.733 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.728+0000 7f21d8b6f740 1 --2- 172.21.15.99:0/1646458739 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5613da676100 0x5613da67fe50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:19.733 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.728+0000 7f21d8b6f740 1 -- 172.21.15.99:0/1646458739 >> 172.21.15.99:0/1646458739 conn(0x5613da671670 msgr2=0x5613da6728c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:01:19.733 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.728+0000 7f21d8b6f740 1 -- 172.21.15.99:0/1646458739 shutdown_connections 2024-03-19T02:01:19.733 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.728+0000 7f21d8b6f740 1 -- 172.21.15.99:0/1646458739 wait complete. 2024-03-19T02:01:19.733 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.728+0000 7f21d8b6f740 20 client.16956 trim_cache size 0 max 16384 2024-03-19T02:01:19.749 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.748+0000 7f21d8b6f740 1 Processor -- start 2024-03-19T02:01:19.749 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.748+0000 7f21d8b6f740 1 -- start start 2024-03-19T02:01:19.750 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.749+0000 7f21d8b6f740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5613da676100 0x5613da670ab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:19.750 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.749+0000 7f21d8b6f740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5613da676af0 0x5613da687390 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:19.750 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.749+0000 7f21d8b6f740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5613da67bc60 0x5613da674fe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:19.750 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.749+0000 7f21d8b6f740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f21d01a9450 con 0x5613da676100 2024-03-19T02:01:19.750 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.749+0000 7f21d8b6f740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f21d01a6d20 con 0x5613da676af0 2024-03-19T02:01:19.750 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.749+0000 7f21d8b6f740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f21d01a4b70 con 0x5613da67bc60 2024-03-19T02:01:19.750 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.749+0000 7f21cf7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5613da676100 0x5613da670ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:19.750 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.749+0000 7f21cffff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5613da67bc60 0x5613da674fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:19.750 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.749+0000 7f21cf7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5613da676100 0x5613da670ab0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:49418/0 (socket says 172.21.15.99:49418) 2024-03-19T02:01:19.750 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.749+0000 7f21ceffd640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5613da676af0 0x5613da687390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:19.750 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.749+0000 7f21cf7fe640 1 -- 172.21.15.99:0/348912887 learned_addr learned my addr 172.21.15.99:0/348912887 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:01:19.750 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.749+0000 7f21cf7fe640 1 -- 172.21.15.99:0/348912887 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5613da67bc60 msgr2=0x5613da674fe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:01:19.750 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.749+0000 7f21cf7fe640 1 --2- 172.21.15.99:0/348912887 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5613da67bc60 0x5613da674fe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:19.751 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.749+0000 7f21cf7fe640 1 -- 172.21.15.99:0/348912887 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5613da676af0 msgr2=0x5613da687390 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:01:19.751 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.749+0000 7f21cf7fe640 1 --2- 172.21.15.99:0/348912887 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5613da676af0 0x5613da687390 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:19.751 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.749+0000 7f21cf7fe640 1 -- 172.21.15.99:0/348912887 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5613da3bb620 con 0x5613da676100 2024-03-19T02:01:19.751 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.749+0000 7f21cffff640 1 --2- 172.21.15.99:0/348912887 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5613da67bc60 0x5613da674fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:01:19.751 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.749+0000 7f21cf7fe640 1 --2- 172.21.15.99:0/348912887 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5613da676100 0x5613da670ab0 secure :-1 s=READY pgs=727 cs=0 l=1 rev1=1 crypto rx=0x7f21c00b93b0 tx=0x7f21c00b8ba0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=784e09b0b58623d0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:01:19.751 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.749+0000 7f21b57fa640 1 -- 172.21.15.99:0/348912887 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f21c004b040 con 0x5613da676100 2024-03-19T02:01:19.751 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.750+0000 7f21b57fa640 1 -- 172.21.15.99:0/348912887 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f21c00ee080 con 0x5613da676100 2024-03-19T02:01:19.751 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.750+0000 7f21b57fa640 1 -- 172.21.15.99:0/348912887 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f21c00b9730 con 0x5613da676100 2024-03-19T02:01:19.755 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.750+0000 7f21d8b6f740 1 -- 172.21.15.99:0/348912887 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5613da676100 msgr2=0x5613da670ab0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:01:19.755 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.750+0000 7f21d8b6f740 1 --2- 172.21.15.99:0/348912887 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5613da676100 0x5613da670ab0 secure :-1 s=READY pgs=727 cs=0 l=1 rev1=1 crypto rx=0x7f21c00b93b0 tx=0x7f21c00b8ba0 comp rx=0 tx=0).stop 2024-03-19T02:01:19.755 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.750+0000 7f21d8b6f740 1 -- 172.21.15.99:0/348912887 shutdown_connections 2024-03-19T02:01:19.755 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.750+0000 7f21d8b6f740 1 --2- 172.21.15.99:0/348912887 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5613da676af0 0x5613da687390 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:19.755 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.750+0000 7f21d8b6f740 1 --2- 172.21.15.99:0/348912887 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5613da67bc60 0x5613da674fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:19.756 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.750+0000 7f21d8b6f740 1 --2- 172.21.15.99:0/348912887 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5613da676100 0x5613da670ab0 unknown :-1 s=CLOSED pgs=727 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:19.756 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.750+0000 7f21d8b6f740 1 -- 172.21.15.99:0/348912887 >> 172.21.15.99:0/348912887 conn(0x5613da671670 msgr2=0x5613da6748d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:01:19.756 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.750+0000 7f21d8b6f740 1 -- 172.21.15.99:0/348912887 shutdown_connections 2024-03-19T02:01:19.756 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.750+0000 7f21d8b6f740 1 -- 172.21.15.99:0/348912887 wait complete. 2024-03-19T02:01:19.756 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.751+0000 7f21d8b6f740 1 -- start start 2024-03-19T02:01:19.756 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.751+0000 7f21d8b6f740 1 Processor -- start 2024-03-19T02:01:19.756 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.751+0000 7f21d8b6f740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5613da676100 0x5613da684760 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:19.756 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.751+0000 7f21d8b6f740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5613da676af0 0x5613da684ca0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:19.756 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.751+0000 7f21d8b6f740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5613da67bc60 0x5613da685cb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:19.756 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.751+0000 7f21d8b6f740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f21d01a51e0 con 0x5613da676100 2024-03-19T02:01:19.756 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.751+0000 7f21cf7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5613da676100 0x5613da684760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:19.756 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.751+0000 7f21cf7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5613da676100 0x5613da684760 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:49432/0 (socket says 172.21.15.99:49432) 2024-03-19T02:01:19.756 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.751+0000 7f21cf7fe640 1 -- 172.21.15.99:0/1238715821 learned_addr learned my addr 172.21.15.99:0/1238715821 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:01:19.756 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.751+0000 7f21cffff640 1 --2- 172.21.15.99:0/1238715821 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5613da67bc60 0x5613da685cb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:19.757 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.751+0000 7f21d8b6f740 1 -- 172.21.15.99:0/1238715821 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f21d01a6650 con 0x5613da67bc60 2024-03-19T02:01:19.757 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.751+0000 7f21d8b6f740 1 -- 172.21.15.99:0/1238715821 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f21d01a58b0 con 0x5613da676af0 2024-03-19T02:01:19.757 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.751+0000 7f21ceffd640 1 --2- 172.21.15.99:0/1238715821 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5613da676af0 0x5613da684ca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:19.757 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.751+0000 7f21d8b6f740 10 client.0 _ll_register_callbacks cb 0x7f21c8011cf0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T02:01:19.757 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.752+0000 7f21cf7fe640 1 -- 172.21.15.99:0/1238715821 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5613da676af0 msgr2=0x5613da684ca0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:01:19.757 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.752+0000 7f21cf7fe640 1 --2- 172.21.15.99:0/1238715821 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5613da676af0 0x5613da684ca0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:19.757 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.752+0000 7f21cf7fe640 1 -- 172.21.15.99:0/1238715821 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5613da67bc60 msgr2=0x5613da685cb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:01:19.757 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.752+0000 7f21cf7fe640 1 --2- 172.21.15.99:0/1238715821 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5613da67bc60 0x5613da685cb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:19.757 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.752+0000 7f21cf7fe640 1 -- 172.21.15.99:0/1238715821 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5613da6862a0 con 0x5613da676100 2024-03-19T02:01:19.757 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.752+0000 7f21cf7fe640 1 --2- 172.21.15.99:0/1238715821 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5613da676100 0x5613da684760 secure :-1 s=READY pgs=728 cs=0 l=1 rev1=1 crypto rx=0x7f21c00bb200 tx=0x7f21c00ee230 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e7c6f29dcdac79e8 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:01:19.757 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.752+0000 7f21aeffd640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T02:01:19.757 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.752+0000 7f21aeffd640 1 -- 172.21.15.99:0/1238715821 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f21c00da060 con 0x5613da676100 2024-03-19T02:01:19.757 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.752+0000 7f21aeffd640 1 -- 172.21.15.99:0/1238715821 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f21c00bd3c0 con 0x5613da676100 2024-03-19T02:01:19.757 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.752+0000 7f21d8b6f740 1 -- 172.21.15.99:0/1238715821 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f21c00ee080 con 0x5613da676100 2024-03-19T02:01:19.758 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.752+0000 7f21aeffd640 1 -- 172.21.15.99:0/1238715821 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7f21c004b070 con 0x5613da676100 2024-03-19T02:01:19.758 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.752+0000 7f21d8b6f740 10 client.16962 fetch_fsmap learned FSMap version 37 2024-03-19T02:01:19.758 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.752+0000 7f21d8b6f740 1 -- 172.21.15.99:0/1238715821 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x5613da687590 con 0x5613da676100 2024-03-19T02:01:19.758 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.752+0000 7f21aeffd640 1 -- 172.21.15.99:0/1238715821 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f21c00ee080 con 0x5613da676100 2024-03-19T02:01:19.758 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.753+0000 7f21aeffd640 1 -- 172.21.15.99:0/1238715821 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7f21c00f00f0 con 0x5613da676100 2024-03-19T02:01:19.758 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.753+0000 7f21d8b6f740 10 client.16962 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T02:01:19.758 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.753+0000 7f21d8b6f740 10 client.16962 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T02:01:19.758 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.753+0000 7f21d8b6f740 20 client.16962 populate_metadata read hostname 'smithi099' 2024-03-19T02:01:19.758 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.753+0000 7f21d8b6f740 1 --2- 172.21.15.99:0/1238715821 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5613da6f23b0 0x5613da6f47a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:19.758 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.753+0000 7f21d8b6f740 4 client.16962 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T02:01:19.758 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.753+0000 7f21d8b6f740 1 -- 172.21.15.99:0/1238715821 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x5613da5e2470 con 0x5613da6f23b0 2024-03-19T02:01:19.758 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.753+0000 7f21ceffd640 1 --2- 172.21.15.99:0/1238715821 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5613da6f23b0 0x5613da6f47a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:19.758 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.753+0000 7f21ceffd640 1 --2- 172.21.15.99:0/1238715821 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5613da6f23b0 0x5613da6f47a0 crc :-1 s=READY pgs=210 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=1f8491a0f2d7dd85 server_cookie=e7e8436a63a6bc4d in_seq=0 out_seq=0 2024-03-19T02:01:19.759 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:19.753+0000 7f21aeffd640 10 client.16962 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T02:01:20.114 INFO:tasks.workunit.client.0.smithi099.stdout: 10 201425 22.70 MB/sec execute 349 sec latency 224.668 ms 2024-03-19T02:01:20.333 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:20.332+0000 7f21aeffd640 1 -- 172.21.15.99:0/1238715821 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x5613da5e2470 con 0x5613da6f23b0 2024-03-19T02:01:20.333 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:20.332+0000 7f21aeffd640 10 client.16962 handle_command_reply: tid=0 2024-03-19T02:01:20.333 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:20.332+0000 7f21d8b6f740 1 client.16962 shutdown 2024-03-19T02:01:20.333 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:20.333+0000 7f21d8b6f740 1 -- 172.21.15.99:0/1238715821 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5613da676100 msgr2=0x5613da684760 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:01:20.333 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:20.333+0000 7f21d8b6f740 1 --2- 172.21.15.99:0/1238715821 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5613da676100 0x5613da684760 secure :-1 s=READY pgs=728 cs=0 l=1 rev1=1 crypto rx=0x7f21c00bb200 tx=0x7f21c00ee230 comp rx=0 tx=0).stop 2024-03-19T02:01:20.333 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:20.333+0000 7f21d8b6f740 1 -- 172.21.15.99:0/1238715821 shutdown_connections 2024-03-19T02:01:20.334 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:20.333+0000 7f21d8b6f740 1 --2- 172.21.15.99:0/1238715821 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5613da6f23b0 0x5613da6f47a0 crc :-1 s=READY pgs=210 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:20.334 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:20.333+0000 7f21d8b6f740 1 --2- 172.21.15.99:0/1238715821 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5613da67bc60 0x5613da685cb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:20.334 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:20.333+0000 7f21aeffd640 0 client.16962 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T02:01:20.334 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:20.333+0000 7f21d8b6f740 1 --2- 172.21.15.99:0/1238715821 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5613da676af0 0x5613da684ca0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:20.334 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:20.333+0000 7f21d8b6f740 1 --2- 172.21.15.99:0/1238715821 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5613da676100 0x5613da684760 unknown :-1 s=CLOSED pgs=728 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:20.334 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:20.333+0000 7f21d8b6f740 1 -- 172.21.15.99:0/1238715821 >> 172.21.15.99:0/1238715821 conn(0x5613da671670 msgr2=0x5613da675160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:01:20.334 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:20.333+0000 7f21d8b6f740 1 -- 172.21.15.99:0/1238715821 shutdown_connections 2024-03-19T02:01:20.334 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:20.333+0000 7f21d8b6f740 1 -- 172.21.15.99:0/1238715821 wait complete. 2024-03-19T02:01:20.334 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:20.333+0000 7f21d8b6f740 20 client.16962 trim_cache size 0 max 16384 2024-03-19T02:01:20.334 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T02:01:20.335 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T02:01:20.335 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T02:01:20.335 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T02:01:20.335 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:20.333+0000 7f21d5ea1640 1 -- 172.21.15.99:0/4026296038 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f21b80a2a40 msgr2=0x7f21b80a4f00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:01:20.336 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:20.333+0000 7f21d5ea1640 1 --2- 172.21.15.99:0/4026296038 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f21b80a2a40 0x7f21b80a4f00 secure :-1 s=READY pgs=363 cs=0 l=1 rev1=1 crypto rx=0x7f21c40095f0 tx=0x7f21c4005f90 comp rx=0 tx=0).stop 2024-03-19T02:01:20.336 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:20.333+0000 7f21d5ea1640 1 -- 172.21.15.99:0/4026296038 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f21d00ffe70 msgr2=0x7f21d0197b00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:01:20.336 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:20.333+0000 7f21d5ea1640 1 --2- 172.21.15.99:0/4026296038 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f21d00ffe70 0x7f21d0197b00 secure :-1 s=READY pgs=207 cs=0 l=1 rev1=1 crypto rx=0x7f21c002c170 tx=0x7f21c002c650 comp rx=0 tx=0).stop 2024-03-19T02:01:20.336 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:20.333+0000 7f21d5ea1640 1 -- 172.21.15.99:0/4026296038 shutdown_connections 2024-03-19T02:01:20.336 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:20.333+0000 7f21d5ea1640 1 --2- 172.21.15.99:0/4026296038 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f21d0101a10 0x7f21d019f0b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:20.336 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:20.334+0000 7f21d5ea1640 1 --2- 172.21.15.99:0/4026296038 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f21d0101050 0x7f21d0198040 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:20.336 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:20.334+0000 7f21d5ea1640 1 --2- 172.21.15.99:0/4026296038 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f21b80a2a40 0x7f21b80a4f00 unknown :-1 s=CLOSED pgs=363 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:20.336 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:20.334+0000 7f21d5ea1640 1 --2- 172.21.15.99:0/4026296038 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f21d00ffe70 0x7f21d0197b00 unknown :-1 s=CLOSED pgs=207 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:20.336 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:20.334+0000 7f21d5ea1640 1 -- 172.21.15.99:0/4026296038 >> 172.21.15.99:0/4026296038 conn(0x7f21d00fb470 msgr2=0x7f21d0108cf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:01:20.336 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:20.334+0000 7f21d5ea1640 1 -- 172.21.15.99:0/4026296038 shutdown_connections 2024-03-19T02:01:20.336 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:20.334+0000 7f21d5ea1640 1 -- 172.21.15.99:0/4026296038 wait complete. 2024-03-19T02:01:21.114 INFO:tasks.workunit.client.0.smithi099.stdout: 10 201722 22.67 MB/sec execute 350 sec latency 354.931 ms 2024-03-19T02:01:21.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:01:21 smithi115 ceph-mon[67332]: pgmap v959: 97 pgs: 97 active+clean; 610 MiB data, 8.0 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 6.2 MiB/s wr, 537 op/s 2024-03-19T02:01:21.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:01:21 smithi099 ceph-mon[62951]: pgmap v959: 97 pgs: 97 active+clean; 610 MiB data, 8.0 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 6.2 MiB/s wr, 537 op/s 2024-03-19T02:01:21.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:01:21 smithi112 ceph-mon[66819]: pgmap v959: 97 pgs: 97 active+clean; 610 MiB data, 8.0 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 6.2 MiB/s wr, 537 op/s 2024-03-19T02:01:22.114 INFO:tasks.workunit.client.0.smithi099.stdout: 10 202051 22.64 MB/sec execute 351 sec latency 91.999 ms 2024-03-19T02:01:23.114 INFO:tasks.workunit.client.0.smithi099.stdout: 10 202584 22.66 MB/sec execute 352 sec latency 62.778 ms 2024-03-19T02:01:23.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:01:23 smithi115 ceph-mon[67332]: pgmap v960: 97 pgs: 97 active+clean; 604 MiB data, 8.0 GiB used, 1.0 TiB / 1.0 TiB avail; 4.5 MiB/s rd, 6.5 MiB/s wr, 565 op/s 2024-03-19T02:01:23.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:01:23 smithi099 ceph-mon[62951]: pgmap v960: 97 pgs: 97 active+clean; 604 MiB data, 8.0 GiB used, 1.0 TiB / 1.0 TiB avail; 4.5 MiB/s rd, 6.5 MiB/s wr, 565 op/s 2024-03-19T02:01:23.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:01:23 smithi112 ceph-mon[66819]: pgmap v960: 97 pgs: 97 active+clean; 604 MiB data, 8.0 GiB used, 1.0 TiB / 1.0 TiB avail; 4.5 MiB/s rd, 6.5 MiB/s wr, 565 op/s 2024-03-19T02:01:24.114 INFO:tasks.workunit.client.0.smithi099.stdout: 10 202882 22.63 MB/sec execute 353 sec latency 347.106 ms 2024-03-19T02:01:25.114 INFO:tasks.workunit.client.0.smithi099.stdout: 10 203339 22.64 MB/sec execute 354 sec latency 215.360 ms 2024-03-19T02:01:25.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:01:25 smithi115 ceph-mon[67332]: pgmap v961: 97 pgs: 97 active+clean; 605 MiB data, 8.0 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 5.3 MiB/s wr, 500 op/s 2024-03-19T02:01:25.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:01:25 smithi099 ceph-mon[62951]: pgmap v961: 97 pgs: 97 active+clean; 605 MiB data, 8.0 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 5.3 MiB/s wr, 500 op/s 2024-03-19T02:01:25.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:01:25 smithi112 ceph-mon[66819]: pgmap v961: 97 pgs: 97 active+clean; 605 MiB data, 8.0 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 5.3 MiB/s wr, 500 op/s 2024-03-19T02:01:26.114 INFO:tasks.workunit.client.0.smithi099.stdout: 10 203941 22.66 MB/sec execute 355 sec latency 71.738 ms 2024-03-19T02:01:27.114 INFO:tasks.workunit.client.0.smithi099.stdout: 10 204390 22.67 MB/sec execute 356 sec latency 89.021 ms 2024-03-19T02:01:27.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:01:27 smithi115 ceph-mon[67332]: pgmap v962: 97 pgs: 97 active+clean; 610 MiB data, 8.0 GiB used, 1.0 TiB / 1.0 TiB avail; 4.6 MiB/s rd, 6.3 MiB/s wr, 588 op/s 2024-03-19T02:01:27.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:01:27 smithi099 ceph-mon[62951]: pgmap v962: 97 pgs: 97 active+clean; 610 MiB data, 8.0 GiB used, 1.0 TiB / 1.0 TiB avail; 4.6 MiB/s rd, 6.3 MiB/s wr, 588 op/s 2024-03-19T02:01:27.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:01:27 smithi112 ceph-mon[66819]: pgmap v962: 97 pgs: 97 active+clean; 610 MiB data, 8.0 GiB used, 1.0 TiB / 1.0 TiB avail; 4.6 MiB/s rd, 6.3 MiB/s wr, 588 op/s 2024-03-19T02:01:28.114 INFO:tasks.workunit.client.0.smithi099.stdout: 10 204872 22.68 MB/sec execute 357 sec latency 66.017 ms 2024-03-19T02:01:29.114 INFO:tasks.workunit.client.0.smithi099.stdout: 10 205354 22.72 MB/sec execute 358 sec latency 179.669 ms 2024-03-19T02:01:29.413 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:01:29 smithi112 ceph-mon[66819]: pgmap v963: 97 pgs: 97 active+clean; 611 MiB data, 7.9 GiB used, 1.0 TiB / 1.0 TiB avail; 3.9 MiB/s rd, 5.2 MiB/s wr, 513 op/s 2024-03-19T02:01:29.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:01:29 smithi115 ceph-mon[67332]: pgmap v963: 97 pgs: 97 active+clean; 611 MiB data, 7.9 GiB used, 1.0 TiB / 1.0 TiB avail; 3.9 MiB/s rd, 5.2 MiB/s wr, 513 op/s 2024-03-19T02:01:29.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:01:29 smithi099 ceph-mon[62951]: pgmap v963: 97 pgs: 97 active+clean; 611 MiB data, 7.9 GiB used, 1.0 TiB / 1.0 TiB avail; 3.9 MiB/s rd, 5.2 MiB/s wr, 513 op/s 2024-03-19T02:01:30.114 INFO:tasks.workunit.client.0.smithi099.stdout: 10 205763 22.71 MB/sec execute 359 sec latency 106.079 ms 2024-03-19T02:01:31.114 INFO:tasks.workunit.client.0.smithi099.stdout: 10 206208 22.72 MB/sec execute 360 sec latency 81.649 ms 2024-03-19T02:01:31.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:01:31 smithi115 ceph-mon[67332]: pgmap v964: 97 pgs: 97 active+clean; 617 MiB data, 8.0 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 5.6 MiB/s wr, 554 op/s 2024-03-19T02:01:31.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:01:31 smithi099 ceph-mon[62951]: pgmap v964: 97 pgs: 97 active+clean; 617 MiB data, 8.0 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 5.6 MiB/s wr, 554 op/s 2024-03-19T02:01:31.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:01:31 smithi112 ceph-mon[66819]: pgmap v964: 97 pgs: 97 active+clean; 617 MiB data, 8.0 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 5.6 MiB/s wr, 554 op/s 2024-03-19T02:01:32.114 INFO:tasks.workunit.client.0.smithi099.stdout: 10 206485 22.70 MB/sec execute 361 sec latency 70.574 ms 2024-03-19T02:01:33.114 INFO:tasks.workunit.client.0.smithi099.stdout: 10 206833 22.66 MB/sec execute 362 sec latency 73.410 ms 2024-03-19T02:01:33.443 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.441+0000 7f9f501f1640 1 -- 172.21.15.99:0/1785648219 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9f48100760 msgr2=0x7f9f48100b80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:01:33.443 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.441+0000 7f9f501f1640 1 --2- 172.21.15.99:0/1785648219 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9f48100760 0x7f9f48100b80 secure :-1 s=READY pgs=729 cs=0 l=1 rev1=1 crypto rx=0x7f9f44009900 tx=0x7f9f4402bc30 comp rx=0 tx=0).stop 2024-03-19T02:01:33.443 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.442+0000 7f9f501f1640 1 -- 172.21.15.99:0/1785648219 shutdown_connections 2024-03-19T02:01:33.443 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.442+0000 7f9f501f1640 1 --2- 172.21.15.99:0/1785648219 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9f48101950 0x7f9f48101dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:33.443 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.442+0000 7f9f501f1640 1 --2- 172.21.15.99:0/1785648219 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9f48102310 0x7f9f48108ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:33.444 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.442+0000 7f9f501f1640 1 --2- 172.21.15.99:0/1785648219 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9f48100760 0x7f9f48100b80 unknown :-1 s=CLOSED pgs=729 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:33.444 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.442+0000 7f9f501f1640 1 -- 172.21.15.99:0/1785648219 >> 172.21.15.99:0/1785648219 conn(0x7f9f480fbcf0 msgr2=0x7f9f480fe110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:01:33.444 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.442+0000 7f9f501f1640 1 -- 172.21.15.99:0/1785648219 shutdown_connections 2024-03-19T02:01:33.444 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.442+0000 7f9f501f1640 1 -- 172.21.15.99:0/1785648219 wait complete. 2024-03-19T02:01:33.446 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.443+0000 7f9f501f1640 1 Processor -- start 2024-03-19T02:01:33.446 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.443+0000 7f9f501f1640 1 -- start start 2024-03-19T02:01:33.446 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.443+0000 7f9f501f1640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9f48100760 0x7f9f48197930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:33.446 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.443+0000 7f9f501f1640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9f48101950 0x7f9f48197e70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:33.446 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.443+0000 7f9f501f1640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9f48102310 0x7f9f4819ee40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:33.446 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.443+0000 7f9f501f1640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f9f4810c400 con 0x7f9f48102310 2024-03-19T02:01:33.447 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.443+0000 7f9f501f1640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f9f4810c280 con 0x7f9f48101950 2024-03-19T02:01:33.447 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.443+0000 7f9f501f1640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f9f4810c580 con 0x7f9f48100760 2024-03-19T02:01:33.447 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.444+0000 7f9f4e78e640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9f48102310 0x7f9f4819ee40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:33.447 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.444+0000 7f9f4d78c640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9f48101950 0x7f9f48197e70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:33.447 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.444+0000 7f9f4d78c640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9f48101950 0x7f9f48197e70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:53074/0 (socket says 172.21.15.99:53074) 2024-03-19T02:01:33.447 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.444+0000 7f9f4d78c640 1 -- 172.21.15.99:0/3352341523 learned_addr learned my addr 172.21.15.99:0/3352341523 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:01:33.447 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.444+0000 7f9f4df8d640 1 --2- 172.21.15.99:0/3352341523 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9f48100760 0x7f9f48197930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:33.447 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.444+0000 7f9f4e78e640 1 -- 172.21.15.99:0/3352341523 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9f48100760 msgr2=0x7f9f48197930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:01:33.447 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.444+0000 7f9f4e78e640 1 --2- 172.21.15.99:0/3352341523 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9f48100760 0x7f9f48197930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:33.447 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.444+0000 7f9f4e78e640 1 -- 172.21.15.99:0/3352341523 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9f48101950 msgr2=0x7f9f48197e70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:01:33.447 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.444+0000 7f9f4e78e640 1 --2- 172.21.15.99:0/3352341523 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9f48101950 0x7f9f48197e70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:33.447 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.444+0000 7f9f4e78e640 1 -- 172.21.15.99:0/3352341523 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f9f4819f4b0 con 0x7f9f48102310 2024-03-19T02:01:33.447 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.444+0000 7f9f4d78c640 1 --2- 172.21.15.99:0/3352341523 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9f48101950 0x7f9f48197e70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:01:33.447 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.444+0000 7f9f4e78e640 1 --2- 172.21.15.99:0/3352341523 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9f48102310 0x7f9f4819ee40 secure :-1 s=READY pgs=730 cs=0 l=1 rev1=1 crypto rx=0x7f9f4000cab0 tx=0x7f9f4000cf70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:01:33.448 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.445+0000 7f9f3effd640 1 -- 172.21.15.99:0/3352341523 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9f4000ae20 con 0x7f9f48102310 2024-03-19T02:01:33.448 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.445+0000 7f9f501f1640 1 -- 172.21.15.99:0/3352341523 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f9f4819f7a0 con 0x7f9f48102310 2024-03-19T02:01:33.448 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.445+0000 7f9f501f1640 1 -- 172.21.15.99:0/3352341523 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f9f4819fcf0 con 0x7f9f48102310 2024-03-19T02:01:33.448 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.445+0000 7f9f3effd640 1 -- 172.21.15.99:0/3352341523 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f9f4000f930 con 0x7f9f48102310 2024-03-19T02:01:33.448 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.445+0000 7f9f3effd640 1 -- 172.21.15.99:0/3352341523 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9f40018b30 con 0x7f9f48102310 2024-03-19T02:01:33.452 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.448+0000 7f9f52dd8740 1 Processor -- start 2024-03-19T02:01:33.452 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.448+0000 7f9f3effd640 1 -- 172.21.15.99:0/3352341523 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f9f400213f0 con 0x7f9f48102310 2024-03-19T02:01:33.452 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.448+0000 7f9f52dd8740 1 -- start start 2024-03-19T02:01:33.452 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.448+0000 7f9f52dd8740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ab32fa5100 0x55ab32fa5520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:33.452 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.449+0000 7f9f52dd8740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ab32fa5af0 0x55ab32faa5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:33.452 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.449+0000 7f9f52dd8740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ab32faac60 0x55ab32fad0f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:33.452 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.449+0000 7f9f52dd8740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55ab32f09be0 con 0x55ab32faac60 2024-03-19T02:01:33.452 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.449+0000 7f9f52dd8740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55ab32eb2960 con 0x55ab32fa5af0 2024-03-19T02:01:33.452 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.449+0000 7f9f52dd8740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55ab32f05c80 con 0x55ab32fa5100 2024-03-19T02:01:33.452 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.449+0000 7f9f4df8d640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ab32fa5af0 0x55ab32faa5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:33.452 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.449+0000 7f9f4df8d640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ab32fa5af0 0x55ab32faa5f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:53080/0 (socket says 172.21.15.99:53080) 2024-03-19T02:01:33.452 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.449+0000 7f9f4df8d640 1 -- 172.21.15.99:0/3920181912 learned_addr learned my addr 172.21.15.99:0/3920181912 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:01:33.453 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.449+0000 7f9f4d78c640 1 --2- 172.21.15.99:0/3920181912 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ab32faac60 0x55ab32fad0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:33.453 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.449+0000 7f9f4df8d640 1 -- 172.21.15.99:0/3920181912 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ab32fa5100 msgr2=0x55ab32fa5520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:01:33.453 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.449+0000 7f9f4df8d640 1 --2- 172.21.15.99:0/3920181912 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ab32fa5100 0x55ab32fa5520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:33.453 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.449+0000 7f9f4df8d640 1 -- 172.21.15.99:0/3920181912 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ab32faac60 msgr2=0x55ab32fad0f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:01:33.453 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.449+0000 7f9f4df8d640 1 --2- 172.21.15.99:0/3920181912 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ab32faac60 0x55ab32fad0f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:33.453 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.449+0000 7f9f3effd640 1 --2- 172.21.15.99:0/3352341523 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f9f340a6db0 0x7f9f340a9270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:33.453 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.449+0000 7f9f4df8d640 1 -- 172.21.15.99:0/3920181912 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55ab32ea9d20 con 0x55ab32fa5af0 2024-03-19T02:01:33.453 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.449+0000 7f9f3effd640 1 -- 172.21.15.99:0/3352341523 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f9f400cefd0 con 0x7f9f48102310 2024-03-19T02:01:33.453 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.450+0000 7f9f4df8d640 1 --2- 172.21.15.99:0/3920181912 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ab32fa5af0 0x55ab32faa5f0 secure :-1 s=READY pgs=209 cs=0 l=1 rev1=1 crypto rx=0x7f9f481017b0 tx=0x7f9f44000bc0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=7b204aff9db8748b server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:01:33.453 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.450+0000 7f9f4d78c640 1 --2- 172.21.15.99:0/3352341523 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f9f340a6db0 0x7f9f340a9270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:33.453 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.450+0000 7f9f33fff640 1 -- 172.21.15.99:0/3920181912 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9f44045070 con 0x55ab32fa5af0 2024-03-19T02:01:33.453 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.450+0000 7f9f33fff640 1 -- 172.21.15.99:0/3920181912 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f9f4402cb00 con 0x55ab32fa5af0 2024-03-19T02:01:33.453 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.450+0000 7f9f33fff640 1 -- 172.21.15.99:0/3920181912 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9f440051e0 con 0x55ab32fa5af0 2024-03-19T02:01:33.453 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.451+0000 7f9f52dd8740 1 -- 172.21.15.99:0/3920181912 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ab32fa5af0 msgr2=0x55ab32faa5f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:01:33.453 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.451+0000 7f9f4d78c640 1 --2- 172.21.15.99:0/3352341523 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f9f340a6db0 0x7f9f340a9270 secure :-1 s=READY pgs=364 cs=0 l=1 rev1=1 crypto rx=0x7f9f38009fd0 tx=0x7f9f38009c60 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:01:33.454 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.451+0000 7f9f52dd8740 1 --2- 172.21.15.99:0/3920181912 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ab32fa5af0 0x55ab32faa5f0 secure :-1 s=READY pgs=209 cs=0 l=1 rev1=1 crypto rx=0x7f9f481017b0 tx=0x7f9f44000bc0 comp rx=0 tx=0).stop 2024-03-19T02:01:33.454 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.451+0000 7f9f52dd8740 1 -- 172.21.15.99:0/3920181912 shutdown_connections 2024-03-19T02:01:33.454 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.451+0000 7f9f52dd8740 1 --2- 172.21.15.99:0/3920181912 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ab32fa5af0 0x55ab32faa5f0 unknown :-1 s=CLOSED pgs=209 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:33.454 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.451+0000 7f9f52dd8740 1 --2- 172.21.15.99:0/3920181912 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ab32faac60 0x55ab32fad0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:33.454 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.451+0000 7f9f52dd8740 1 --2- 172.21.15.99:0/3920181912 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ab32fa5100 0x55ab32fa5520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:33.454 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.451+0000 7f9f52dd8740 1 -- 172.21.15.99:0/3920181912 >> 172.21.15.99:0/3920181912 conn(0x55ab32fa0670 msgr2=0x55ab32fa0a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:01:33.454 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.451+0000 7f9f52dd8740 1 -- 172.21.15.99:0/3920181912 shutdown_connections 2024-03-19T02:01:33.454 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.451+0000 7f9f52dd8740 1 -- 172.21.15.99:0/3920181912 wait complete. 2024-03-19T02:01:33.454 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.451+0000 7f9f52dd8740 1 -- start start 2024-03-19T02:01:33.454 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.451+0000 7f9f52dd8740 1 Processor -- start 2024-03-19T02:01:33.456 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.452+0000 7f9f52dd8740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ab32fa5100 0x55ab32faf0a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:33.456 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.452+0000 7f9f52dd8740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ab32fa5af0 0x55ab32faf5e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:33.456 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.452+0000 7f9f4d78c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ab32fa5af0 0x55ab32faf5e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:33.457 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.452+0000 7f9f4d78c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ab32fa5af0 0x55ab32faf5e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:39154/0 (socket says 172.21.15.99:39154) 2024-03-19T02:01:33.457 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.452+0000 7f9f4d78c640 1 -- 172.21.15.99:0/1315629073 learned_addr learned my addr 172.21.15.99:0/1315629073 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:01:33.457 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.452+0000 7f9f52dd8740 1 --2- 172.21.15.99:0/1315629073 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ab32faac60 0x55ab32fb6620 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:33.457 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.452+0000 7f9f52dd8740 1 -- 172.21.15.99:0/1315629073 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55ab32fadbb0 con 0x55ab32fa5af0 2024-03-19T02:01:33.457 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.452+0000 7f9f52dd8740 1 -- 172.21.15.99:0/1315629073 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55ab32ec5db0 con 0x55ab32faac60 2024-03-19T02:01:33.457 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.452+0000 7f9f52dd8740 1 -- 172.21.15.99:0/1315629073 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55ab32fadd30 con 0x55ab32fa5100 2024-03-19T02:01:33.457 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.452+0000 7f9f4e78e640 1 --2- 172.21.15.99:0/1315629073 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ab32faac60 0x55ab32fb6620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:33.457 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.453+0000 7f9f4d78c640 1 -- 172.21.15.99:0/1315629073 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ab32fa5100 msgr2=0x55ab32faf0a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:01:33.457 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.453+0000 7f9f4d78c640 1 --2- 172.21.15.99:0/1315629073 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ab32fa5100 0x55ab32faf0a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:33.457 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.453+0000 7f9f4d78c640 1 -- 172.21.15.99:0/1315629073 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ab32faac60 msgr2=0x55ab32fb6620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:01:33.457 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.453+0000 7f9f4d78c640 1 --2- 172.21.15.99:0/1315629073 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ab32faac60 0x55ab32fb6620 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:33.457 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.453+0000 7f9f4d78c640 1 -- 172.21.15.99:0/1315629073 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55ab32ec4c60 con 0x55ab32fa5af0 2024-03-19T02:01:33.457 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.453+0000 7f9f4d78c640 1 --2- 172.21.15.99:0/1315629073 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ab32fa5af0 0x55ab32faf5e0 secure :-1 s=READY pgs=731 cs=0 l=1 rev1=1 crypto rx=0x7f9f38004a80 tx=0x7f9f380046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=eb19c57aa5848833 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:01:33.457 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.453+0000 7f9f317fa640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T02:01:33.457 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.453+0000 7f9f52dd8740 10 client.0 _ll_register_callbacks cb 0x55ab32ecae70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T02:01:33.458 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.453+0000 7f9f52dd8740 1 -- 172.21.15.99:0/1315629073 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f9f4402cb00 con 0x55ab32fa5af0 2024-03-19T02:01:33.458 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.453+0000 7f9f317fa640 1 -- 172.21.15.99:0/1315629073 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9f3800fde0 con 0x55ab32fa5af0 2024-03-19T02:01:33.458 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.453+0000 7f9f317fa640 1 -- 172.21.15.99:0/1315629073 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f9f38029440 con 0x55ab32fa5af0 2024-03-19T02:01:33.458 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.453+0000 7f9f317fa640 1 -- 172.21.15.99:0/1315629073 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7f9f38029640 con 0x55ab32fa5af0 2024-03-19T02:01:33.458 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.453+0000 7f9f52dd8740 10 client.16974 fetch_fsmap learned FSMap version 37 2024-03-19T02:01:33.458 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.453+0000 7f9f52dd8740 1 -- 172.21.15.99:0/1315629073 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({fsmap=37}) -- 0x55ab32c1e210 con 0x55ab32fa5af0 2024-03-19T02:01:33.458 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.454+0000 7f9f317fa640 1 -- 172.21.15.99:0/1315629073 <== mon.0 v2:172.21.15.99:3300/0 4 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7f9f38027070 con 0x55ab32fa5af0 2024-03-19T02:01:33.458 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.454+0000 7f9f52dd8740 10 client.16974 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T02:01:33.458 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.454+0000 7f9f52dd8740 10 client.16974 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T02:01:33.458 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.454+0000 7f9f52dd8740 20 client.16974 populate_metadata read hostname 'smithi099' 2024-03-19T02:01:33.458 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.454+0000 7f9f52dd8740 1 --2- 172.21.15.99:0/1315629073 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55ab32fbaeb0 0x55ab32fbd340 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:33.458 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.454+0000 7f9f52dd8740 4 client.16974 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T02:01:33.458 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.454+0000 7f9f52dd8740 1 -- 172.21.15.99:0/1315629073 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55ab32f17120 con 0x55ab32fbaeb0 2024-03-19T02:01:33.459 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.456+0000 7f9f4df8d640 1 --2- 172.21.15.99:0/1315629073 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55ab32fbaeb0 0x55ab32fbd340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:33.460 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.459+0000 7f9f4df8d640 1 --2- 172.21.15.99:0/1315629073 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55ab32fbaeb0 0x55ab32fbd340 crc :-1 s=READY pgs=211 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=fc97a9c89fbcc456 server_cookie=c57b63b8b9b597e3 in_seq=0 out_seq=0 2024-03-19T02:01:33.460 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.459+0000 7f9f317fa640 10 client.16974 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T02:01:33.464 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.463+0000 7f9f317fa640 1 -- 172.21.15.99:0/1315629073 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x55ab32f17120 con 0x55ab32fbaeb0 2024-03-19T02:01:33.464 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.463+0000 7f9f317fa640 10 client.16974 handle_command_reply: tid=0 2024-03-19T02:01:33.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.463+0000 7f9f52dd8740 1 client.16974 shutdown 2024-03-19T02:01:33.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.463+0000 7f9f52dd8740 1 -- 172.21.15.99:0/1315629073 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ab32fa5af0 msgr2=0x55ab32faf5e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:01:33.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.463+0000 7f9f52dd8740 1 --2- 172.21.15.99:0/1315629073 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ab32fa5af0 0x55ab32faf5e0 secure :-1 s=READY pgs=731 cs=0 l=1 rev1=1 crypto rx=0x7f9f38004a80 tx=0x7f9f380046c0 comp rx=0 tx=0).stop 2024-03-19T02:01:33.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.464+0000 7f9f52dd8740 1 -- 172.21.15.99:0/1315629073 shutdown_connections 2024-03-19T02:01:33.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.464+0000 7f9f52dd8740 1 --2- 172.21.15.99:0/1315629073 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55ab32fbaeb0 0x55ab32fbd340 crc :-1 s=READY pgs=211 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:33.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.464+0000 7f9f52dd8740 1 --2- 172.21.15.99:0/1315629073 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ab32faac60 0x55ab32fb6620 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:33.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.464+0000 7f9f52dd8740 1 --2- 172.21.15.99:0/1315629073 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ab32fa5af0 0x55ab32faf5e0 unknown :-1 s=CLOSED pgs=731 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:33.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.464+0000 7f9f52dd8740 1 --2- 172.21.15.99:0/1315629073 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ab32fa5100 0x55ab32faf0a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:33.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.464+0000 7f9f52dd8740 1 -- 172.21.15.99:0/1315629073 >> 172.21.15.99:0/1315629073 conn(0x55ab32fa0670 msgr2=0x55ab32fa18c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:01:33.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.464+0000 7f9f317fa640 0 client.16974 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T02:01:33.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.464+0000 7f9f52dd8740 1 -- 172.21.15.99:0/1315629073 shutdown_connections 2024-03-19T02:01:33.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.464+0000 7f9f52dd8740 1 -- 172.21.15.99:0/1315629073 wait complete. 2024-03-19T02:01:33.465 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.464+0000 7f9f52dd8740 20 client.16974 trim_cache size 0 max 16384 2024-03-19T02:01:33.498 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.490+0000 7f9f52dd8740 1 Processor -- start 2024-03-19T02:01:33.498 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.490+0000 7f9f52dd8740 1 -- start start 2024-03-19T02:01:33.498 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.490+0000 7f9f52dd8740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ab32fa5100 0x55ab32fae370 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:33.498 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.490+0000 7f9f52dd8740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ab32fa5af0 0x55ab32fb65e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:33.499 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.490+0000 7f9f52dd8740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ab32faac60 0x55ab32fae8b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:33.499 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.490+0000 7f9f52dd8740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f9f2c010db0 con 0x55ab32faac60 2024-03-19T02:01:33.499 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.490+0000 7f9f52dd8740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f9f2c00e680 con 0x55ab32fa5100 2024-03-19T02:01:33.499 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.490+0000 7f9f52dd8740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f9f2c00c4d0 con 0x55ab32fa5af0 2024-03-19T02:01:33.499 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.491+0000 7f9f4e78e640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ab32faac60 0x55ab32fae8b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:33.499 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.491+0000 7f9f4e78e640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ab32faac60 0x55ab32fae8b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:39156/0 (socket says 172.21.15.99:39156) 2024-03-19T02:01:33.499 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.491+0000 7f9f4e78e640 1 -- 172.21.15.99:0/2323807621 learned_addr learned my addr 172.21.15.99:0/2323807621 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:01:33.499 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.491+0000 7f9f4df8d640 1 --2- 172.21.15.99:0/2323807621 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ab32fa5100 0x55ab32fae370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:33.499 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.491+0000 7f9f4e78e640 1 -- 172.21.15.99:0/2323807621 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ab32fa5af0 msgr2=0x55ab32fb65e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:01:33.499 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.491+0000 7f9f4e78e640 1 --2- 172.21.15.99:0/2323807621 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ab32fa5af0 0x55ab32fb65e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:33.499 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.491+0000 7f9f4e78e640 1 -- 172.21.15.99:0/2323807621 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ab32fa5100 msgr2=0x55ab32fae370 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:01:33.499 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.491+0000 7f9f4e78e640 1 --2- 172.21.15.99:0/2323807621 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ab32fa5100 0x55ab32fae370 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:33.499 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.491+0000 7f9f4e78e640 1 -- 172.21.15.99:0/2323807621 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55ab32fb73e0 con 0x55ab32faac60 2024-03-19T02:01:33.499 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.491+0000 7f9f4e78e640 1 --2- 172.21.15.99:0/2323807621 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ab32faac60 0x55ab32fae8b0 secure :-1 s=READY pgs=732 cs=0 l=1 rev1=1 crypto rx=0x7f9f40004af0 tx=0x7f9f40097bf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2209428093b5a0db server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:01:33.500 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.491+0000 7f9f337fe640 1 -- 172.21.15.99:0/2323807621 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9f400b4020 con 0x55ab32faac60 2024-03-19T02:01:33.500 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.491+0000 7f9f337fe640 1 -- 172.21.15.99:0/2323807621 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f9f4009fa20 con 0x55ab32faac60 2024-03-19T02:01:33.500 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.491+0000 7f9f337fe640 1 -- 172.21.15.99:0/2323807621 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9f40084e60 con 0x55ab32faac60 2024-03-19T02:01:33.500 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.492+0000 7f9f52dd8740 1 -- 172.21.15.99:0/2323807621 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ab32faac60 msgr2=0x55ab32fae8b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:01:33.500 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.492+0000 7f9f52dd8740 1 --2- 172.21.15.99:0/2323807621 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ab32faac60 0x55ab32fae8b0 secure :-1 s=READY pgs=732 cs=0 l=1 rev1=1 crypto rx=0x7f9f40004af0 tx=0x7f9f40097bf0 comp rx=0 tx=0).stop 2024-03-19T02:01:33.500 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.492+0000 7f9f52dd8740 1 -- 172.21.15.99:0/2323807621 shutdown_connections 2024-03-19T02:01:33.500 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.492+0000 7f9f52dd8740 1 --2- 172.21.15.99:0/2323807621 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ab32faac60 0x55ab32fae8b0 unknown :-1 s=CLOSED pgs=732 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:33.500 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.492+0000 7f9f52dd8740 1 --2- 172.21.15.99:0/2323807621 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ab32fa5af0 0x55ab32fb65e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:33.500 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.492+0000 7f9f52dd8740 1 --2- 172.21.15.99:0/2323807621 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ab32fa5100 0x55ab32fae370 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:33.500 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.492+0000 7f9f52dd8740 1 -- 172.21.15.99:0/2323807621 >> 172.21.15.99:0/2323807621 conn(0x55ab32fa0670 msgr2=0x55ab32fa6590 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:01:33.500 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.492+0000 7f9f52dd8740 1 -- 172.21.15.99:0/2323807621 shutdown_connections 2024-03-19T02:01:33.500 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.492+0000 7f9f52dd8740 1 -- 172.21.15.99:0/2323807621 wait complete. 2024-03-19T02:01:33.500 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.493+0000 7f9f52dd8740 1 -- start start 2024-03-19T02:01:33.500 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.493+0000 7f9f52dd8740 1 Processor -- start 2024-03-19T02:01:33.501 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.493+0000 7f9f52dd8740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ab32fa5100 0x55ab32fb3cb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:33.501 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.493+0000 7f9f52dd8740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ab32fa5af0 0x55ab32fb41f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:33.501 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.493+0000 7f9f52dd8740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ab32faac60 0x55ab32fb5200 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:33.501 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.493+0000 7f9f52dd8740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f9f2c00cb40 con 0x55ab32fa5af0 2024-03-19T02:01:33.501 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.493+0000 7f9f52dd8740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f9f2c00dfb0 con 0x55ab32faac60 2024-03-19T02:01:33.501 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.493+0000 7f9f52dd8740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f9f2c00d210 con 0x55ab32fa5100 2024-03-19T02:01:33.501 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.493+0000 7f9f52dd8740 10 client.0 _ll_register_callbacks cb 0x7f9f38010430 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T02:01:33.501 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.493+0000 7f9f4d78c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ab32fa5af0 0x55ab32fb41f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:33.501 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.493+0000 7f9f4d78c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ab32fa5af0 0x55ab32fb41f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:39166/0 (socket says 172.21.15.99:39166) 2024-03-19T02:01:33.501 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.493+0000 7f9f4e78e640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ab32faac60 0x55ab32fb5200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:33.501 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.493+0000 7f9f4d78c640 1 -- 172.21.15.99:0/4071073681 learned_addr learned my addr 172.21.15.99:0/4071073681 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:01:33.501 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.494+0000 7f9f4d78c640 1 -- 172.21.15.99:0/4071073681 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ab32fa5100 msgr2=0x55ab32fb3cb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:01:33.501 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.494+0000 7f9f4d78c640 1 --2- 172.21.15.99:0/4071073681 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ab32fa5100 0x55ab32fb3cb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:33.501 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.494+0000 7f9f4d78c640 1 -- 172.21.15.99:0/4071073681 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ab32faac60 msgr2=0x55ab32fb5200 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:01:33.502 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.494+0000 7f9f4d78c640 1 --2- 172.21.15.99:0/4071073681 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ab32faac60 0x55ab32fb5200 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:33.502 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.494+0000 7f9f4d78c640 1 -- 172.21.15.99:0/4071073681 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55ab32cea620 con 0x55ab32fa5af0 2024-03-19T02:01:33.502 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.494+0000 7f9f4d78c640 1 --2- 172.21.15.99:0/4071073681 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ab32fa5af0 0x55ab32fb41f0 secure :-1 s=READY pgs=733 cs=0 l=1 rev1=1 crypto rx=0x7f9f38010680 tx=0x7f9f38005f90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b2cb863b5488044a server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:01:33.502 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.494+0000 7f9f317fa640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T02:01:33.502 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.494+0000 7f9f317fa640 1 -- 172.21.15.99:0/4071073681 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9f38037070 con 0x55ab32fa5af0 2024-03-19T02:01:33.502 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.494+0000 7f9f317fa640 1 -- 172.21.15.99:0/4071073681 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f9f3800fc40 con 0x55ab32fa5af0 2024-03-19T02:01:33.502 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.494+0000 7f9f52dd8740 1 -- 172.21.15.99:0/4071073681 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f9f4009fa20 con 0x55ab32fa5af0 2024-03-19T02:01:33.502 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.494+0000 7f9f317fa640 1 -- 172.21.15.99:0/4071073681 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7f9f38043050 con 0x55ab32fa5af0 2024-03-19T02:01:33.502 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.494+0000 7f9f52dd8740 10 client.16980 fetch_fsmap learned FSMap version 37 2024-03-19T02:01:33.502 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.494+0000 7f9f52dd8740 1 -- 172.21.15.99:0/4071073681 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x55ab33021280 con 0x55ab32fa5af0 2024-03-19T02:01:33.502 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.494+0000 7f9f317fa640 1 -- 172.21.15.99:0/4071073681 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f9f4009fa20 con 0x55ab32fa5af0 2024-03-19T02:01:33.502 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.494+0000 7f9f317fa640 1 -- 172.21.15.99:0/4071073681 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7f9f38027040 con 0x55ab32fa5af0 2024-03-19T02:01:33.502 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.495+0000 7f9f52dd8740 10 client.16980 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T02:01:33.502 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.495+0000 7f9f52dd8740 10 client.16980 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T02:01:33.503 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.495+0000 7f9f52dd8740 20 client.16980 populate_metadata read hostname 'smithi099' 2024-03-19T02:01:33.503 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.495+0000 7f9f52dd8740 1 --2- 172.21.15.99:0/4071073681 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55ab33021b10 0x55ab33023f30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:33.503 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.495+0000 7f9f52dd8740 4 client.16980 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T02:01:33.503 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.495+0000 7f9f52dd8740 1 -- 172.21.15.99:0/4071073681 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55ab32f11470 con 0x55ab33021b10 2024-03-19T02:01:33.503 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.495+0000 7f9f4df8d640 1 --2- 172.21.15.99:0/4071073681 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55ab33021b10 0x55ab33023f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:33.503 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.496+0000 7f9f4df8d640 1 --2- 172.21.15.99:0/4071073681 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55ab33021b10 0x55ab33023f30 crc :-1 s=READY pgs=212 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=453c36505333f0bd server_cookie=fe49497e81ced109 in_seq=0 out_seq=0 2024-03-19T02:01:33.503 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:33.496+0000 7f9f317fa640 10 client.16980 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T02:01:33.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:01:33 smithi115 ceph-mon[67332]: pgmap v965: 97 pgs: 97 active+clean; 620 MiB data, 8.0 GiB used, 1.0 TiB / 1.0 TiB avail; 4.8 MiB/s rd, 6.0 MiB/s wr, 539 op/s 2024-03-19T02:01:33.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:01:33 smithi099 ceph-mon[62951]: pgmap v965: 97 pgs: 97 active+clean; 620 MiB data, 8.0 GiB used, 1.0 TiB / 1.0 TiB avail; 4.8 MiB/s rd, 6.0 MiB/s wr, 539 op/s 2024-03-19T02:01:33.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:01:33 smithi112 ceph-mon[66819]: pgmap v965: 97 pgs: 97 active+clean; 620 MiB data, 8.0 GiB used, 1.0 TiB / 1.0 TiB avail; 4.8 MiB/s rd, 6.0 MiB/s wr, 539 op/s 2024-03-19T02:01:34.114 INFO:tasks.workunit.client.0.smithi099.stdout: 10 207028 22.62 MB/sec execute 363 sec latency 433.517 ms 2024-03-19T02:01:34.281 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:34.279+0000 7f9f317fa640 1 -- 172.21.15.99:0/4071073681 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x55ab32f11470 con 0x55ab33021b10 2024-03-19T02:01:34.281 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:34.279+0000 7f9f317fa640 10 client.16980 handle_command_reply: tid=0 2024-03-19T02:01:34.281 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:34.279+0000 7f9f52dd8740 1 client.16980 shutdown 2024-03-19T02:01:34.282 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:34.280+0000 7f9f52dd8740 1 -- 172.21.15.99:0/4071073681 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ab32fa5af0 msgr2=0x55ab32fb41f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:01:34.282 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:34.280+0000 7f9f52dd8740 1 --2- 172.21.15.99:0/4071073681 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ab32fa5af0 0x55ab32fb41f0 secure :-1 s=READY pgs=733 cs=0 l=1 rev1=1 crypto rx=0x7f9f38010680 tx=0x7f9f38005f90 comp rx=0 tx=0).stop 2024-03-19T02:01:34.282 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:34.280+0000 7f9f52dd8740 1 -- 172.21.15.99:0/4071073681 shutdown_connections 2024-03-19T02:01:34.282 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:34.280+0000 7f9f52dd8740 1 --2- 172.21.15.99:0/4071073681 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55ab33021b10 0x55ab33023f30 crc :-1 s=READY pgs=212 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:34.282 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:34.280+0000 7f9f52dd8740 1 --2- 172.21.15.99:0/4071073681 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ab32faac60 0x55ab32fb5200 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:34.282 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:34.280+0000 7f9f317fa640 0 client.16980 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T02:01:34.282 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:34.280+0000 7f9f52dd8740 1 --2- 172.21.15.99:0/4071073681 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ab32fa5af0 0x55ab32fb41f0 unknown :-1 s=CLOSED pgs=733 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:34.282 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:34.280+0000 7f9f52dd8740 1 --2- 172.21.15.99:0/4071073681 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ab32fa5100 0x55ab32fb3cb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:34.282 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:34.280+0000 7f9f52dd8740 1 -- 172.21.15.99:0/4071073681 >> 172.21.15.99:0/4071073681 conn(0x55ab32fa0670 msgr2=0x55ab32fa38d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:01:34.282 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:34.280+0000 7f9f52dd8740 1 -- 172.21.15.99:0/4071073681 shutdown_connections 2024-03-19T02:01:34.282 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:34.280+0000 7f9f52dd8740 1 -- 172.21.15.99:0/4071073681 wait complete. 2024-03-19T02:01:34.282 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:34.280+0000 7f9f52dd8740 20 client.16980 trim_cache size 0 max 16384 2024-03-19T02:01:34.283 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T02:01:34.283 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T02:01:34.283 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T02:01:34.283 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T02:01:34.284 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:34.281+0000 7f9f501f1640 1 -- 172.21.15.99:0/3352341523 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f9f340a6db0 msgr2=0x7f9f340a9270 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:01:34.284 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:34.281+0000 7f9f501f1640 1 --2- 172.21.15.99:0/3352341523 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f9f340a6db0 0x7f9f340a9270 secure :-1 s=READY pgs=364 cs=0 l=1 rev1=1 crypto rx=0x7f9f38009fd0 tx=0x7f9f38009c60 comp rx=0 tx=0).stop 2024-03-19T02:01:34.284 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:34.281+0000 7f9f501f1640 1 -- 172.21.15.99:0/3352341523 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9f48102310 msgr2=0x7f9f4819ee40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:01:34.284 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:34.281+0000 7f9f501f1640 1 --2- 172.21.15.99:0/3352341523 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9f48102310 0x7f9f4819ee40 secure :-1 s=READY pgs=730 cs=0 l=1 rev1=1 crypto rx=0x7f9f4000cab0 tx=0x7f9f4000cf70 comp rx=0 tx=0).stop 2024-03-19T02:01:34.284 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:34.281+0000 7f9f501f1640 1 -- 172.21.15.99:0/3352341523 shutdown_connections 2024-03-19T02:01:34.284 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:34.281+0000 7f9f501f1640 1 --2- 172.21.15.99:0/3352341523 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f9f340a6db0 0x7f9f340a9270 unknown :-1 s=CLOSED pgs=364 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:34.284 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:34.281+0000 7f9f501f1640 1 --2- 172.21.15.99:0/3352341523 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9f48101950 0x7f9f48197e70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:34.284 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:34.281+0000 7f9f501f1640 1 --2- 172.21.15.99:0/3352341523 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9f48102310 0x7f9f4819ee40 unknown :-1 s=CLOSED pgs=730 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:34.284 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:34.281+0000 7f9f501f1640 1 --2- 172.21.15.99:0/3352341523 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9f48100760 0x7f9f48197930 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:34.284 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:34.281+0000 7f9f501f1640 1 -- 172.21.15.99:0/3352341523 >> 172.21.15.99:0/3352341523 conn(0x7f9f480fbcf0 msgr2=0x7f9f480ff0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:01:34.284 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:34.281+0000 7f9f501f1640 1 -- 172.21.15.99:0/3352341523 shutdown_connections 2024-03-19T02:01:34.284 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:34.281+0000 7f9f501f1640 1 -- 172.21.15.99:0/3352341523 wait complete. 2024-03-19T02:01:35.114 INFO:tasks.workunit.client.0.smithi099.stdout: 10 207393 22.61 MB/sec execute 364 sec latency 611.553 ms 2024-03-19T02:01:35.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:01:35 smithi115 ceph-mon[67332]: pgmap v966: 97 pgs: 97 active+clean; 621 MiB data, 8.0 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 5.1 MiB/s wr, 395 op/s 2024-03-19T02:01:35.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:01:35 smithi099 ceph-mon[62951]: pgmap v966: 97 pgs: 97 active+clean; 621 MiB data, 8.0 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 5.1 MiB/s wr, 395 op/s 2024-03-19T02:01:35.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:01:35 smithi112 ceph-mon[66819]: pgmap v966: 97 pgs: 97 active+clean; 621 MiB data, 8.0 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 5.1 MiB/s wr, 395 op/s 2024-03-19T02:01:36.114 INFO:tasks.workunit.client.0.smithi099.stdout: 10 207777 22.62 MB/sec execute 365 sec latency 168.024 ms 2024-03-19T02:01:36.479 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:01:36 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T02:01:36.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:01:36 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T02:01:36.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:01:36 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T02:01:37.114 INFO:tasks.workunit.client.0.smithi099.stdout: 10 208296 22.64 MB/sec execute 366 sec latency 63.730 ms 2024-03-19T02:01:37.416 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:01:37 smithi115 ceph-mon[67332]: pgmap v967: 97 pgs: 97 active+clean; 620 MiB data, 8.0 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 MiB/s rd, 7.1 MiB/s wr, 581 op/s 2024-03-19T02:01:37.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:01:37 smithi099 ceph-mon[62951]: pgmap v967: 97 pgs: 97 active+clean; 620 MiB data, 8.0 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 MiB/s rd, 7.1 MiB/s wr, 581 op/s 2024-03-19T02:01:37.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:01:37 smithi112 ceph-mon[66819]: pgmap v967: 97 pgs: 97 active+clean; 620 MiB data, 8.0 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 MiB/s rd, 7.1 MiB/s wr, 581 op/s 2024-03-19T02:01:38.114 INFO:tasks.workunit.client.0.smithi099.stdout: 10 208821 22.64 MB/sec execute 367 sec latency 64.549 ms 2024-03-19T02:01:39.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:01:38 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:01:39.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:01:38 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:01:39.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:01:38 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:01:39.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:01:38 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:01:39.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:01:38 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:01:39.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:01:38 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:01:39.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:01:38 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:01:39.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:01:38 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:01:39.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:01:38 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:01:39.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:01:38 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:01:39.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:01:38 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:01:39.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:01:38 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:01:39.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:01:38 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:01:39.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:01:38 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:01:39.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:01:38 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:01:39.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:01:38 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:01:39.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:01:38 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:01:39.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:01:38 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:01:39.114 INFO:tasks.workunit.client.0.smithi099.stdout: 10 209185 22.62 MB/sec execute 368 sec latency 119.275 ms 2024-03-19T02:01:40.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:01:39 smithi115 ceph-mon[67332]: pgmap v968: 97 pgs: 97 active+clean; 622 MiB data, 8.0 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 6.1 MiB/s wr, 543 op/s 2024-03-19T02:01:40.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:01:39 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T02:01:40.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:01:39 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T02:01:40.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:01:39 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:01:40.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:01:39 smithi099 ceph-mon[62951]: pgmap v968: 97 pgs: 97 active+clean; 622 MiB data, 8.0 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 6.1 MiB/s wr, 543 op/s 2024-03-19T02:01:40.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:01:39 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T02:01:40.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:01:39 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T02:01:40.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:01:39 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:01:40.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:01:39 smithi112 ceph-mon[66819]: pgmap v968: 97 pgs: 97 active+clean; 622 MiB data, 8.0 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 6.1 MiB/s wr, 543 op/s 2024-03-19T02:01:40.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:01:39 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T02:01:40.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:01:39 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T02:01:40.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:01:39 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:01:40.114 INFO:tasks.workunit.client.0.smithi099.stdout: 10 209581 22.62 MB/sec execute 369 sec latency 242.456 ms 2024-03-19T02:01:41.114 INFO:tasks.workunit.client.0.smithi099.stdout: 10 210187 22.68 MB/sec execute 370 sec latency 76.293 ms 2024-03-19T02:01:41.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:01:41 smithi115 ceph-mon[67332]: pgmap v969: 97 pgs: 97 active+clean; 620 MiB data, 8.0 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 6.3 MiB/s wr, 564 op/s 2024-03-19T02:01:41.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:01:41 smithi099 ceph-mon[62951]: pgmap v969: 97 pgs: 97 active+clean; 620 MiB data, 8.0 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 6.3 MiB/s wr, 564 op/s 2024-03-19T02:01:41.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:01:41 smithi112 ceph-mon[66819]: pgmap v969: 97 pgs: 97 active+clean; 620 MiB data, 8.0 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 6.3 MiB/s wr, 564 op/s 2024-03-19T02:01:42.114 INFO:tasks.workunit.client.0.smithi099.stdout: 10 210602 22.69 MB/sec execute 371 sec latency 80.652 ms 2024-03-19T02:01:43.115 INFO:tasks.workunit.client.0.smithi099.stdout: 10 210970 22.69 MB/sec execute 372 sec latency 61.190 ms 2024-03-19T02:01:43.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:01:43 smithi099 ceph-mon[62951]: pgmap v970: 97 pgs: 97 active+clean; 624 MiB data, 8.0 GiB used, 1.0 TiB / 1.0 TiB avail; 3.9 MiB/s rd, 6.5 MiB/s wr, 543 op/s 2024-03-19T02:01:43.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:01:43 smithi115 ceph-mon[67332]: pgmap v970: 97 pgs: 97 active+clean; 624 MiB data, 8.0 GiB used, 1.0 TiB / 1.0 TiB avail; 3.9 MiB/s rd, 6.5 MiB/s wr, 543 op/s 2024-03-19T02:01:43.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:01:43 smithi112 ceph-mon[66819]: pgmap v970: 97 pgs: 97 active+clean; 624 MiB data, 8.0 GiB used, 1.0 TiB / 1.0 TiB avail; 3.9 MiB/s rd, 6.5 MiB/s wr, 543 op/s 2024-03-19T02:01:44.115 INFO:tasks.workunit.client.0.smithi099.stdout: 10 211268 22.66 MB/sec execute 373 sec latency 110.466 ms 2024-03-19T02:01:45.115 INFO:tasks.workunit.client.0.smithi099.stdout: 10 211449 22.61 MB/sec execute 374 sec latency 347.331 ms 2024-03-19T02:01:45.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:01:45 smithi115 ceph-mon[67332]: pgmap v971: 97 pgs: 97 active+clean; 626 MiB data, 8.0 GiB used, 1.0 TiB / 1.0 TiB avail; 3.0 MiB/s rd, 5.6 MiB/s wr, 496 op/s 2024-03-19T02:01:45.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:01:45 smithi099 ceph-mon[62951]: pgmap v971: 97 pgs: 97 active+clean; 626 MiB data, 8.0 GiB used, 1.0 TiB / 1.0 TiB avail; 3.0 MiB/s rd, 5.6 MiB/s wr, 496 op/s 2024-03-19T02:01:45.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:01:45 smithi112 ceph-mon[66819]: pgmap v971: 97 pgs: 97 active+clean; 626 MiB data, 8.0 GiB used, 1.0 TiB / 1.0 TiB avail; 3.0 MiB/s rd, 5.6 MiB/s wr, 496 op/s 2024-03-19T02:01:46.115 INFO:tasks.workunit.client.0.smithi099.stdout: 10 211741 22.58 MB/sec execute 375 sec latency 187.428 ms 2024-03-19T02:01:47.115 INFO:tasks.workunit.client.0.smithi099.stdout: 10 212205 22.58 MB/sec execute 376 sec latency 71.249 ms 2024-03-19T02:01:47.383 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.381+0000 7efe6379e640 1 -- 172.21.15.99:0/1252092765 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7efe5c102300 msgr2=0x7efe5c108b90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:01:47.383 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.381+0000 7efe6379e640 1 --2- 172.21.15.99:0/1252092765 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7efe5c102300 0x7efe5c108b90 secure :-1 s=READY pgs=734 cs=0 l=1 rev1=1 crypto rx=0x7efe54009ca0 tx=0x7efe5402c0f0 comp rx=0 tx=0).stop 2024-03-19T02:01:47.383 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.381+0000 7efe6379e640 1 -- 172.21.15.99:0/1252092765 shutdown_connections 2024-03-19T02:01:47.383 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.381+0000 7efe6379e640 1 --2- 172.21.15.99:0/1252092765 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7efe5c101940 0x7efe5c101dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:47.383 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.381+0000 7efe6379e640 1 --2- 172.21.15.99:0/1252092765 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7efe5c102300 0x7efe5c108b90 unknown :-1 s=CLOSED pgs=734 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:47.383 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.381+0000 7efe6379e640 1 --2- 172.21.15.99:0/1252092765 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7efe5c100750 0x7efe5c100b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:47.383 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.381+0000 7efe6379e640 1 -- 172.21.15.99:0/1252092765 >> 172.21.15.99:0/1252092765 conn(0x7efe5c0fbce0 msgr2=0x7efe5c0fe100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:01:47.383 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.381+0000 7efe6379e640 1 -- 172.21.15.99:0/1252092765 shutdown_connections 2024-03-19T02:01:47.384 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.381+0000 7efe6379e640 1 -- 172.21.15.99:0/1252092765 wait complete. 2024-03-19T02:01:47.384 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.381+0000 7efe6379e640 1 Processor -- start 2024-03-19T02:01:47.384 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.381+0000 7efe6379e640 1 -- start start 2024-03-19T02:01:47.384 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.381+0000 7efe6379e640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7efe5c100750 0x7efe5c1969c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:47.384 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.381+0000 7efe6379e640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7efe5c101940 0x7efe5c196f00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:47.384 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.381+0000 7efe6379e640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7efe5c102300 0x7efe5c19df70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:47.384 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.381+0000 7efe6379e640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7efe5c10c3d0 con 0x7efe5c102300 2024-03-19T02:01:47.384 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.382+0000 7efe6379e640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7efe5c10c250 con 0x7efe5c100750 2024-03-19T02:01:47.384 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.382+0000 7efe6379e640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7efe5c10c550 con 0x7efe5c101940 2024-03-19T02:01:47.385 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.382+0000 7efe6153a640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7efe5c100750 0x7efe5c1969c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:47.385 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.382+0000 7efe61d3b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7efe5c102300 0x7efe5c19df70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:47.385 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.382+0000 7efe6153a640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7efe5c100750 0x7efe5c1969c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:56166/0 (socket says 172.21.15.99:56166) 2024-03-19T02:01:47.385 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.382+0000 7efe61d3b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7efe5c102300 0x7efe5c19df70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:33260/0 (socket says 172.21.15.99:33260) 2024-03-19T02:01:47.385 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.382+0000 7efe6153a640 1 -- 172.21.15.99:0/3301585420 learned_addr learned my addr 172.21.15.99:0/3301585420 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:01:47.385 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.382+0000 7efe60d39640 1 --2- 172.21.15.99:0/3301585420 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7efe5c101940 0x7efe5c196f00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:47.385 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.382+0000 7efe6153a640 1 -- 172.21.15.99:0/3301585420 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7efe5c101940 msgr2=0x7efe5c196f00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:01:47.385 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.382+0000 7efe6153a640 1 --2- 172.21.15.99:0/3301585420 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7efe5c101940 0x7efe5c196f00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:47.385 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.382+0000 7efe6153a640 1 -- 172.21.15.99:0/3301585420 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7efe5c102300 msgr2=0x7efe5c19df70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:01:47.385 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.382+0000 7efe6153a640 1 --2- 172.21.15.99:0/3301585420 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7efe5c102300 0x7efe5c19df70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:47.385 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.382+0000 7efe6153a640 1 -- 172.21.15.99:0/3301585420 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7efe5c19e5e0 con 0x7efe5c100750 2024-03-19T02:01:47.385 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.382+0000 7efe6153a640 1 --2- 172.21.15.99:0/3301585420 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7efe5c100750 0x7efe5c1969c0 secure :-1 s=READY pgs=210 cs=0 l=1 rev1=1 crypto rx=0x7efe5800e9c0 tx=0x7efe5800ee80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:01:47.385 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.382+0000 7efe527fc640 1 -- 172.21.15.99:0/3301585420 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efe5800cd80 con 0x7efe5c100750 2024-03-19T02:01:47.386 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.382+0000 7efe527fc640 1 -- 172.21.15.99:0/3301585420 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7efe580035f0 con 0x7efe5c100750 2024-03-19T02:01:47.386 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.382+0000 7efe6379e640 1 -- 172.21.15.99:0/3301585420 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7efe5c19e8d0 con 0x7efe5c100750 2024-03-19T02:01:47.386 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.382+0000 7efe527fc640 1 -- 172.21.15.99:0/3301585420 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efe58010690 con 0x7efe5c100750 2024-03-19T02:01:47.386 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.382+0000 7efe6379e640 1 -- 172.21.15.99:0/3301585420 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7efe5c19edd0 con 0x7efe5c100750 2024-03-19T02:01:47.391 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.385+0000 7efe527fc640 1 -- 172.21.15.99:0/3301585420 <== mon.2 v2:172.21.15.112:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7efe580037a0 con 0x7efe5c100750 2024-03-19T02:01:47.392 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.385+0000 7efe66388740 1 Processor -- start 2024-03-19T02:01:47.392 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.385+0000 7efe66388740 1 -- start start 2024-03-19T02:01:47.392 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.385+0000 7efe66388740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d49cdf7310 0x55d49cdf7730 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:47.392 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.385+0000 7efe66388740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d49cdf7d00 0x55d49cdfc800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:47.392 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.385+0000 7efe66388740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d49cdfce70 0x55d49cdff300 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:47.392 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.385+0000 7efe66388740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55d49cd59be0 con 0x55d49cdfce70 2024-03-19T02:01:47.392 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.385+0000 7efe66388740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55d49cd02960 con 0x55d49cdf7d00 2024-03-19T02:01:47.392 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.385+0000 7efe66388740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55d49cd55c80 con 0x55d49cdf7310 2024-03-19T02:01:47.392 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.385+0000 7efe6153a640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d49cdf7d00 0x55d49cdfc800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:47.392 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.385+0000 7efe60d39640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d49cdfce70 0x55d49cdff300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:47.393 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.385+0000 7efe61d3b640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d49cdf7310 0x55d49cdf7730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:47.393 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.385+0000 7efe6153a640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d49cdf7d00 0x55d49cdfc800 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:56180/0 (socket says 172.21.15.99:56180) 2024-03-19T02:01:47.393 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.385+0000 7efe60d39640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d49cdfce70 0x55d49cdff300 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:33268/0 (socket says 172.21.15.99:33268) 2024-03-19T02:01:47.393 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.385+0000 7efe6153a640 1 -- 172.21.15.99:0/4016082651 learned_addr learned my addr 172.21.15.99:0/4016082651 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:01:47.393 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.385+0000 7efe527fc640 1 --2- 172.21.15.99:0/3301585420 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7efe480a28a0 0x7efe480a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:47.393 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.385+0000 7efe61d3b640 1 --2- 172.21.15.99:0/3301585420 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7efe480a28a0 0x7efe480a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:47.393 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.385+0000 7efe6153a640 1 -- 172.21.15.99:0/4016082651 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d49cdf7310 msgr2=0x55d49cdf7730 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:01:47.393 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.386+0000 7efe6153a640 1 --2- 172.21.15.99:0/4016082651 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d49cdf7310 0x55d49cdf7730 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:47.393 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.386+0000 7efe6153a640 1 -- 172.21.15.99:0/4016082651 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d49cdfce70 msgr2=0x55d49cdff300 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:01:47.393 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.386+0000 7efe6153a640 1 --2- 172.21.15.99:0/4016082651 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d49cdfce70 0x55d49cdff300 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:47.393 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.386+0000 7efe527fc640 1 -- 172.21.15.99:0/3301585420 <== mon.2 v2:172.21.15.112:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7efe580cee00 con 0x7efe5c100750 2024-03-19T02:01:47.393 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.386+0000 7efe6153a640 1 -- 172.21.15.99:0/4016082651 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55d49ccf9d20 con 0x55d49cdf7d00 2024-03-19T02:01:47.393 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.386+0000 7efe60d39640 1 --2- 172.21.15.99:0/4016082651 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d49cdfce70 0x55d49cdff300 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:01:47.393 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.386+0000 7efe6153a640 1 --2- 172.21.15.99:0/4016082651 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d49cdf7d00 0x55d49cdfc800 secure :-1 s=READY pgs=211 cs=0 l=1 rev1=1 crypto rx=0x7efe5809b6f0 tx=0x7efe58003f10 comp rx=0 tx=0).ready entity=mon.2 client_cookie=35ce9cdfc800b33e server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:01:47.394 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.386+0000 7efe61d3b640 1 --2- 172.21.15.99:0/3301585420 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7efe480a28a0 0x7efe480a4d60 secure :-1 s=READY pgs=365 cs=0 l=1 rev1=1 crypto rx=0x7efe54005e30 tx=0x7efe5402c0f0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:01:47.394 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.386+0000 7efe477fe640 1 -- 172.21.15.99:0/4016082651 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efe5809b0c0 con 0x55d49cdf7d00 2024-03-19T02:01:47.394 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.386+0000 7efe477fe640 1 -- 172.21.15.99:0/4016082651 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7efe5809be60 con 0x55d49cdf7d00 2024-03-19T02:01:47.394 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.386+0000 7efe477fe640 1 -- 172.21.15.99:0/4016082651 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efe58099af0 con 0x55d49cdf7d00 2024-03-19T02:01:47.394 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.387+0000 7efe66388740 1 -- 172.21.15.99:0/4016082651 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d49cdf7d00 msgr2=0x55d49cdfc800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:01:47.394 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.387+0000 7efe66388740 1 --2- 172.21.15.99:0/4016082651 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d49cdf7d00 0x55d49cdfc800 secure :-1 s=READY pgs=211 cs=0 l=1 rev1=1 crypto rx=0x7efe5809b6f0 tx=0x7efe58003f10 comp rx=0 tx=0).stop 2024-03-19T02:01:47.394 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.387+0000 7efe66388740 1 -- 172.21.15.99:0/4016082651 shutdown_connections 2024-03-19T02:01:47.394 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.387+0000 7efe66388740 1 --2- 172.21.15.99:0/4016082651 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d49cdf7d00 0x55d49cdfc800 unknown :-1 s=CLOSED pgs=211 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:47.394 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.387+0000 7efe66388740 1 --2- 172.21.15.99:0/4016082651 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d49cdfce70 0x55d49cdff300 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:47.394 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.387+0000 7efe66388740 1 --2- 172.21.15.99:0/4016082651 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d49cdf7310 0x55d49cdf7730 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:47.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.387+0000 7efe66388740 1 -- 172.21.15.99:0/4016082651 >> 172.21.15.99:0/4016082651 conn(0x55d49cdf0670 msgr2=0x55d49cdf0a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:01:47.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.387+0000 7efe66388740 1 -- 172.21.15.99:0/4016082651 shutdown_connections 2024-03-19T02:01:47.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.387+0000 7efe66388740 1 -- 172.21.15.99:0/4016082651 wait complete. 2024-03-19T02:01:47.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.387+0000 7efe66388740 1 -- start start 2024-03-19T02:01:47.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.387+0000 7efe66388740 1 Processor -- start 2024-03-19T02:01:47.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.387+0000 7efe66388740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d49cdf7310 0x55d49ce01220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:47.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.387+0000 7efe66388740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d49cdf7d00 0x55d49ce01760 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:47.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.387+0000 7efe66388740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d49cdfce70 0x55d49ce087a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:47.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.387+0000 7efe66388740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55d49cdffd60 con 0x55d49cdf7d00 2024-03-19T02:01:47.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.387+0000 7efe66388740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55d49cd15db0 con 0x55d49cdfce70 2024-03-19T02:01:47.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.387+0000 7efe66388740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55d49cdffee0 con 0x55d49cdf7310 2024-03-19T02:01:47.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.387+0000 7efe66388740 10 client.0 _ll_register_callbacks cb 0x55d49cd1ae70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T02:01:47.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.388+0000 7efe60d39640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d49cdf7d00 0x55d49ce01760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:47.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.388+0000 7efe60d39640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d49cdf7d00 0x55d49ce01760 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:33280/0 (socket says 172.21.15.99:33280) 2024-03-19T02:01:47.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.388+0000 7efe60d39640 1 -- 172.21.15.99:0/341620928 learned_addr learned my addr 172.21.15.99:0/341620928 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:01:47.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.388+0000 7efe61d3b640 1 --2- 172.21.15.99:0/341620928 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d49cdfce70 0x55d49ce087a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:47.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.388+0000 7efe6153a640 1 --2- 172.21.15.99:0/341620928 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d49cdf7310 0x55d49ce01220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:47.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.388+0000 7efe60d39640 1 -- 172.21.15.99:0/341620928 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d49cdf7310 msgr2=0x55d49ce01220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:01:47.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.388+0000 7efe60d39640 1 --2- 172.21.15.99:0/341620928 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d49cdf7310 0x55d49ce01220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:47.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.388+0000 7efe60d39640 1 -- 172.21.15.99:0/341620928 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d49cdfce70 msgr2=0x55d49ce087a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:01:47.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.388+0000 7efe60d39640 1 --2- 172.21.15.99:0/341620928 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d49cdfce70 0x55d49ce087a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:47.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.388+0000 7efe60d39640 1 -- 172.21.15.99:0/341620928 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55d49cd14c60 con 0x55d49cdf7d00 2024-03-19T02:01:47.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.388+0000 7efe60d39640 1 --2- 172.21.15.99:0/341620928 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d49cdf7d00 0x55d49ce01760 secure :-1 s=READY pgs=735 cs=0 l=1 rev1=1 crypto rx=0x7efe4c012700 tx=0x7efe4c012bc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8e47ad8ace88a8ab server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:01:47.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.388+0000 7efe44ff9640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T02:01:47.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.388+0000 7efe44ff9640 1 -- 172.21.15.99:0/341620928 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efe4c004050 con 0x55d49cdf7d00 2024-03-19T02:01:47.397 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.388+0000 7efe44ff9640 1 -- 172.21.15.99:0/341620928 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7efe4c0041f0 con 0x55d49cdf7d00 2024-03-19T02:01:47.397 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.388+0000 7efe66388740 1 -- 172.21.15.99:0/341620928 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7efe5809be60 con 0x55d49cdf7d00 2024-03-19T02:01:47.397 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.388+0000 7efe44ff9640 1 -- 172.21.15.99:0/341620928 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7efe4c004430 con 0x55d49cdf7d00 2024-03-19T02:01:47.397 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.389+0000 7efe66388740 10 client.16992 fetch_fsmap learned FSMap version 37 2024-03-19T02:01:47.397 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.389+0000 7efe66388740 1 -- 172.21.15.99:0/341620928 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x55d49ca6e210 con 0x55d49cdf7d00 2024-03-19T02:01:47.397 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.389+0000 7efe44ff9640 1 -- 172.21.15.99:0/341620928 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7efe5809be60 con 0x55d49cdf7d00 2024-03-19T02:01:47.397 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.389+0000 7efe44ff9640 1 -- 172.21.15.99:0/341620928 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7efe4c003ca0 con 0x55d49cdf7d00 2024-03-19T02:01:47.397 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.389+0000 7efe66388740 10 client.16992 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T02:01:47.397 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.389+0000 7efe66388740 10 client.16992 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T02:01:47.397 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.389+0000 7efe66388740 20 client.16992 populate_metadata read hostname 'smithi099' 2024-03-19T02:01:47.397 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.389+0000 7efe66388740 1 --2- 172.21.15.99:0/341620928 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55d49ce0d100 0x55d49ce0f560 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:47.397 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.389+0000 7efe66388740 4 client.16992 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T02:01:47.397 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.389+0000 7efe66388740 1 -- 172.21.15.99:0/341620928 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55d49cd67120 con 0x55d49ce0d100 2024-03-19T02:01:47.397 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.389+0000 7efe6153a640 1 --2- 172.21.15.99:0/341620928 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55d49ce0d100 0x55d49ce0f560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:47.398 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.390+0000 7efe44ff9640 10 client.16992 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T02:01:47.398 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.390+0000 7efe6153a640 1 --2- 172.21.15.99:0/341620928 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55d49ce0d100 0x55d49ce0f560 crc :-1 s=READY pgs=213 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=d0103db9f44b6bad server_cookie=5703dc2ea0cad0cb in_seq=0 out_seq=0 2024-03-19T02:01:47.399 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.394+0000 7efe44ff9640 1 -- 172.21.15.99:0/341620928 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x55d49cd67120 con 0x55d49ce0d100 2024-03-19T02:01:47.399 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.394+0000 7efe44ff9640 10 client.16992 handle_command_reply: tid=0 2024-03-19T02:01:47.399 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.394+0000 7efe66388740 1 client.16992 shutdown 2024-03-19T02:01:47.399 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.394+0000 7efe66388740 1 -- 172.21.15.99:0/341620928 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d49cdf7d00 msgr2=0x55d49ce01760 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:01:47.399 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.394+0000 7efe66388740 1 --2- 172.21.15.99:0/341620928 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d49cdf7d00 0x55d49ce01760 secure :-1 s=READY pgs=735 cs=0 l=1 rev1=1 crypto rx=0x7efe4c012700 tx=0x7efe4c012bc0 comp rx=0 tx=0).stop 2024-03-19T02:01:47.399 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.394+0000 7efe66388740 1 -- 172.21.15.99:0/341620928 shutdown_connections 2024-03-19T02:01:47.399 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.394+0000 7efe66388740 1 --2- 172.21.15.99:0/341620928 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55d49ce0d100 0x55d49ce0f560 crc :-1 s=READY pgs=213 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:47.399 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.394+0000 7efe66388740 1 --2- 172.21.15.99:0/341620928 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d49cdfce70 0x55d49ce087a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:47.399 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.394+0000 7efe44ff9640 0 client.16992 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T02:01:47.399 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.394+0000 7efe66388740 1 --2- 172.21.15.99:0/341620928 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d49cdf7d00 0x55d49ce01760 unknown :-1 s=CLOSED pgs=735 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:47.399 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.394+0000 7efe66388740 1 --2- 172.21.15.99:0/341620928 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d49cdf7310 0x55d49ce01220 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:47.400 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.394+0000 7efe66388740 1 -- 172.21.15.99:0/341620928 >> 172.21.15.99:0/341620928 conn(0x55d49cdf0670 msgr2=0x55d49cdf18c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:01:47.400 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.394+0000 7efe66388740 1 -- 172.21.15.99:0/341620928 shutdown_connections 2024-03-19T02:01:47.400 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.394+0000 7efe66388740 1 -- 172.21.15.99:0/341620928 wait complete. 2024-03-19T02:01:47.400 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.394+0000 7efe66388740 20 client.16992 trim_cache size 0 max 16384 2024-03-19T02:01:47.411 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.410+0000 7efe66388740 1 Processor -- start 2024-03-19T02:01:47.411 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.410+0000 7efe66388740 1 -- start start 2024-03-19T02:01:47.411 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.410+0000 7efe66388740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d49cdf7310 0x55d49cdef220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:47.411 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.410+0000 7efe66388740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d49cdf7d00 0x55d49ce08760 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:47.411 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.410+0000 7efe66388740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d49cdfce70 0x55d49cdef760 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:47.411 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.410+0000 7efe66388740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7efe5c1a9530 con 0x55d49cdfce70 2024-03-19T02:01:47.411 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.410+0000 7efe66388740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7efe5c1a6e00 con 0x55d49cdf7d00 2024-03-19T02:01:47.411 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.410+0000 7efe66388740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7efe5c1a4c50 con 0x55d49cdf7310 2024-03-19T02:01:47.412 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.410+0000 7efe61d3b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d49cdfce70 0x55d49cdef760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:47.412 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.410+0000 7efe61d3b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d49cdfce70 0x55d49cdef760 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:33290/0 (socket says 172.21.15.99:33290) 2024-03-19T02:01:47.412 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.410+0000 7efe61d3b640 1 -- 172.21.15.99:0/173450105 learned_addr learned my addr 172.21.15.99:0/173450105 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:01:47.412 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.410+0000 7efe6153a640 1 --2- 172.21.15.99:0/173450105 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d49cdf7310 0x55d49cdef220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:47.412 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.410+0000 7efe60d39640 1 --2- 172.21.15.99:0/173450105 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d49cdf7d00 0x55d49ce08760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:47.412 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.411+0000 7efe61d3b640 1 -- 172.21.15.99:0/173450105 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d49cdf7310 msgr2=0x55d49cdef220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:01:47.412 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.411+0000 7efe61d3b640 1 --2- 172.21.15.99:0/173450105 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d49cdf7310 0x55d49cdef220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:47.412 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.411+0000 7efe61d3b640 1 -- 172.21.15.99:0/173450105 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d49cdf7d00 msgr2=0x55d49ce08760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:01:47.412 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.411+0000 7efe61d3b640 1 --2- 172.21.15.99:0/173450105 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d49cdf7d00 0x55d49ce08760 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:47.412 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.411+0000 7efe61d3b640 1 -- 172.21.15.99:0/173450105 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55d49cb3a620 con 0x55d49cdfce70 2024-03-19T02:01:47.412 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.411+0000 7efe61d3b640 1 --2- 172.21.15.99:0/173450105 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d49cdfce70 0x55d49cdef760 secure :-1 s=READY pgs=736 cs=0 l=1 rev1=1 crypto rx=0x7efe54036000 tx=0x7efe54009360 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d9bf3c9ccd4f4c3a server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:01:47.412 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.411+0000 7efe46ffd640 1 -- 172.21.15.99:0/173450105 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efe54039040 con 0x55d49cdfce70 2024-03-19T02:01:47.412 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.411+0000 7efe46ffd640 1 -- 172.21.15.99:0/173450105 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7efe54054070 con 0x55d49cdfce70 2024-03-19T02:01:47.412 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.411+0000 7efe46ffd640 1 -- 172.21.15.99:0/173450105 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efe5404f690 con 0x55d49cdfce70 2024-03-19T02:01:47.415 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.412+0000 7efe66388740 1 -- 172.21.15.99:0/173450105 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d49cdfce70 msgr2=0x55d49cdef760 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:01:47.416 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.412+0000 7efe66388740 1 --2- 172.21.15.99:0/173450105 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d49cdfce70 0x55d49cdef760 secure :-1 s=READY pgs=736 cs=0 l=1 rev1=1 crypto rx=0x7efe54036000 tx=0x7efe54009360 comp rx=0 tx=0).stop 2024-03-19T02:01:47.416 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.412+0000 7efe66388740 1 -- 172.21.15.99:0/173450105 shutdown_connections 2024-03-19T02:01:47.416 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.412+0000 7efe66388740 1 --2- 172.21.15.99:0/173450105 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d49cdf7d00 0x55d49ce08760 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:47.416 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.412+0000 7efe66388740 1 --2- 172.21.15.99:0/173450105 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d49cdfce70 0x55d49cdef760 unknown :-1 s=CLOSED pgs=736 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:47.416 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.412+0000 7efe66388740 1 --2- 172.21.15.99:0/173450105 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d49cdf7310 0x55d49cdef220 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:47.416 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.412+0000 7efe66388740 1 -- 172.21.15.99:0/173450105 >> 172.21.15.99:0/173450105 conn(0x55d49cdf0670 msgr2=0x55d49ce01220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:01:47.416 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.412+0000 7efe66388740 1 -- 172.21.15.99:0/173450105 shutdown_connections 2024-03-19T02:01:47.416 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.412+0000 7efe66388740 1 -- 172.21.15.99:0/173450105 wait complete. 2024-03-19T02:01:47.416 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.412+0000 7efe66388740 1 -- start start 2024-03-19T02:01:47.416 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.412+0000 7efe66388740 1 Processor -- start 2024-03-19T02:01:47.416 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.413+0000 7efe66388740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d49cdf7310 0x55d49ce063f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:47.416 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.413+0000 7efe66388740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d49cdf7d00 0x55d49ce06930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:47.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.413+0000 7efe66388740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d49cdfce70 0x55d49ce07940 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:47.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.413+0000 7efe66388740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7efe5c1a52c0 con 0x55d49cdf7d00 2024-03-19T02:01:47.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.413+0000 7efe66388740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7efe5c1a6730 con 0x55d49cdf7310 2024-03-19T02:01:47.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.413+0000 7efe66388740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7efe5c1a5990 con 0x55d49cdfce70 2024-03-19T02:01:47.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.413+0000 7efe66388740 10 client.0 _ll_register_callbacks cb 0x7efe4c009930 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T02:01:47.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.413+0000 7efe60d39640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d49cdf7d00 0x55d49ce06930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:47.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.413+0000 7efe60d39640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d49cdf7d00 0x55d49ce06930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:33306/0 (socket says 172.21.15.99:33306) 2024-03-19T02:01:47.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.413+0000 7efe60d39640 1 -- 172.21.15.99:0/4069079581 learned_addr learned my addr 172.21.15.99:0/4069079581 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:01:47.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.413+0000 7efe6153a640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d49cdf7310 0x55d49ce063f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:47.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.413+0000 7efe61d3b640 1 --2- 172.21.15.99:0/4069079581 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d49cdfce70 0x55d49ce07940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:47.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.413+0000 7efe60d39640 1 -- 172.21.15.99:0/4069079581 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d49cdfce70 msgr2=0x55d49ce07940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:01:47.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.413+0000 7efe60d39640 1 --2- 172.21.15.99:0/4069079581 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d49cdfce70 0x55d49ce07940 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:47.417 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.413+0000 7efe60d39640 1 -- 172.21.15.99:0/4069079581 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d49cdf7310 msgr2=0x55d49ce063f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:01:47.418 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.413+0000 7efe60d39640 1 --2- 172.21.15.99:0/4069079581 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d49cdf7310 0x55d49ce063f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:47.418 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.413+0000 7efe60d39640 1 -- 172.21.15.99:0/4069079581 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55d49ce07f30 con 0x55d49cdf7d00 2024-03-19T02:01:47.418 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.413+0000 7efe60d39640 1 --2- 172.21.15.99:0/4069079581 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d49cdf7d00 0x55d49ce06930 secure :-1 s=READY pgs=737 cs=0 l=1 rev1=1 crypto rx=0x7efe4c012420 tx=0x7efe4c009ea0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b9212193f7c53bde server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:01:47.418 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.413+0000 7efe44ff9640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T02:01:47.418 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.413+0000 7efe44ff9640 1 -- 172.21.15.99:0/4069079581 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efe4c034060 con 0x55d49cdf7d00 2024-03-19T02:01:47.418 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.413+0000 7efe66388740 1 -- 172.21.15.99:0/4069079581 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7efe54054070 con 0x55d49cdf7d00 2024-03-19T02:01:47.418 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.413+0000 7efe44ff9640 1 -- 172.21.15.99:0/4069079581 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7efe4c018040 con 0x55d49cdf7d00 2024-03-19T02:01:47.418 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.413+0000 7efe44ff9640 1 -- 172.21.15.99:0/4069079581 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7efe4c00f790 con 0x55d49cdf7d00 2024-03-19T02:01:47.418 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.413+0000 7efe66388740 10 client.16998 fetch_fsmap learned FSMap version 37 2024-03-19T02:01:47.418 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.414+0000 7efe66388740 1 -- 172.21.15.99:0/4069079581 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x55d49ce739f0 con 0x55d49cdf7d00 2024-03-19T02:01:47.418 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.414+0000 7efe44ff9640 1 -- 172.21.15.99:0/4069079581 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7efe54054070 con 0x55d49cdf7d00 2024-03-19T02:01:47.418 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.414+0000 7efe44ff9640 1 -- 172.21.15.99:0/4069079581 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7efe4c005560 con 0x55d49cdf7d00 2024-03-19T02:01:47.418 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.414+0000 7efe66388740 10 client.16998 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T02:01:47.418 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.414+0000 7efe66388740 10 client.16998 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T02:01:47.419 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.414+0000 7efe66388740 20 client.16998 populate_metadata read hostname 'smithi099' 2024-03-19T02:01:47.419 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.414+0000 7efe66388740 1 --2- 172.21.15.99:0/4069079581 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55d49ce0d100 0x55d49ce76400 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:01:47.419 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.414+0000 7efe66388740 4 client.16998 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T02:01:47.419 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.414+0000 7efe66388740 1 -- 172.21.15.99:0/4069079581 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55d49cd61470 con 0x55d49ce0d100 2024-03-19T02:01:47.419 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.414+0000 7efe6153a640 1 --2- 172.21.15.99:0/4069079581 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55d49ce0d100 0x55d49ce76400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:01:47.419 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.414+0000 7efe44ff9640 10 client.16998 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T02:01:47.419 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.414+0000 7efe6153a640 1 --2- 172.21.15.99:0/4069079581 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55d49ce0d100 0x55d49ce76400 crc :-1 s=READY pgs=214 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=db8303d9fedb1cab server_cookie=f2a17d3f45c97ddf in_seq=0 out_seq=0 2024-03-19T02:01:47.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:01:47 smithi115 ceph-mon[67332]: pgmap v972: 97 pgs: 97 active+clean; 633 MiB data, 8.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 6.9 MiB/s wr, 584 op/s 2024-03-19T02:01:47.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:01:47 smithi099 ceph-mon[62951]: pgmap v972: 97 pgs: 97 active+clean; 633 MiB data, 8.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 6.9 MiB/s wr, 584 op/s 2024-03-19T02:01:47.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:01:47 smithi112 ceph-mon[66819]: pgmap v972: 97 pgs: 97 active+clean; 633 MiB data, 8.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 6.9 MiB/s wr, 584 op/s 2024-03-19T02:01:47.981 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.980+0000 7efe44ff9640 1 -- 172.21.15.99:0/4069079581 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x55d49cd61470 con 0x55d49ce0d100 2024-03-19T02:01:47.981 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.980+0000 7efe44ff9640 10 client.16998 handle_command_reply: tid=0 2024-03-19T02:01:47.981 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.980+0000 7efe66388740 1 client.16998 shutdown 2024-03-19T02:01:47.981 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.980+0000 7efe66388740 1 -- 172.21.15.99:0/4069079581 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d49cdf7d00 msgr2=0x55d49ce06930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:01:47.981 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.980+0000 7efe66388740 1 --2- 172.21.15.99:0/4069079581 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d49cdf7d00 0x55d49ce06930 secure :-1 s=READY pgs=737 cs=0 l=1 rev1=1 crypto rx=0x7efe4c012420 tx=0x7efe4c009ea0 comp rx=0 tx=0).stop 2024-03-19T02:01:47.981 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.980+0000 7efe66388740 1 -- 172.21.15.99:0/4069079581 shutdown_connections 2024-03-19T02:01:47.981 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.980+0000 7efe66388740 1 --2- 172.21.15.99:0/4069079581 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55d49ce0d100 0x55d49ce76400 crc :-1 s=READY pgs=214 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:47.981 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.980+0000 7efe66388740 1 --2- 172.21.15.99:0/4069079581 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d49cdfce70 0x55d49ce07940 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:47.981 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.980+0000 7efe66388740 1 --2- 172.21.15.99:0/4069079581 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d49cdf7d00 0x55d49ce06930 unknown :-1 s=CLOSED pgs=737 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:47.981 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.980+0000 7efe66388740 1 --2- 172.21.15.99:0/4069079581 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d49cdf7310 0x55d49ce063f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:47.981 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.980+0000 7efe44ff9640 0 client.16998 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T02:01:47.982 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.980+0000 7efe66388740 1 -- 172.21.15.99:0/4069079581 >> 172.21.15.99:0/4069079581 conn(0x55d49cdf0670 msgr2=0x55d49cdef220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:01:47.982 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.981+0000 7efe66388740 1 -- 172.21.15.99:0/4069079581 shutdown_connections 2024-03-19T02:01:47.982 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.981+0000 7efe66388740 1 -- 172.21.15.99:0/4069079581 wait complete. 2024-03-19T02:01:47.982 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.981+0000 7efe66388740 20 client.16998 trim_cache size 0 max 16384 2024-03-19T02:01:47.982 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T02:01:47.982 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T02:01:47.982 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T02:01:47.982 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T02:01:47.983 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.981+0000 7efe6379e640 1 -- 172.21.15.99:0/3301585420 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7efe480a28a0 msgr2=0x7efe480a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:01:47.983 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.981+0000 7efe6379e640 1 --2- 172.21.15.99:0/3301585420 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7efe480a28a0 0x7efe480a4d60 secure :-1 s=READY pgs=365 cs=0 l=1 rev1=1 crypto rx=0x7efe54005e30 tx=0x7efe5402c0f0 comp rx=0 tx=0).stop 2024-03-19T02:01:47.983 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.981+0000 7efe6379e640 1 -- 172.21.15.99:0/3301585420 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7efe5c100750 msgr2=0x7efe5c1969c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:01:47.983 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.981+0000 7efe6379e640 1 --2- 172.21.15.99:0/3301585420 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7efe5c100750 0x7efe5c1969c0 secure :-1 s=READY pgs=210 cs=0 l=1 rev1=1 crypto rx=0x7efe5800e9c0 tx=0x7efe5800ee80 comp rx=0 tx=0).stop 2024-03-19T02:01:47.983 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.981+0000 7efe6379e640 1 -- 172.21.15.99:0/3301585420 shutdown_connections 2024-03-19T02:01:47.983 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.981+0000 7efe6379e640 1 --2- 172.21.15.99:0/3301585420 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7efe5c102300 0x7efe5c19df70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:47.983 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.981+0000 7efe6379e640 1 --2- 172.21.15.99:0/3301585420 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7efe5c101940 0x7efe5c196f00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:47.984 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.981+0000 7efe6379e640 1 --2- 172.21.15.99:0/3301585420 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7efe480a28a0 0x7efe480a4d60 unknown :-1 s=CLOSED pgs=365 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:47.984 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.982+0000 7efe6379e640 1 --2- 172.21.15.99:0/3301585420 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7efe5c100750 0x7efe5c1969c0 unknown :-1 s=CLOSED pgs=210 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:01:47.984 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.982+0000 7efe6379e640 1 -- 172.21.15.99:0/3301585420 >> 172.21.15.99:0/3301585420 conn(0x7efe5c0fbce0 msgr2=0x7efe5c104b70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:01:47.984 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.982+0000 7efe6379e640 1 -- 172.21.15.99:0/3301585420 shutdown_connections 2024-03-19T02:01:47.984 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:01:47.982+0000 7efe6379e640 1 -- 172.21.15.99:0/3301585420 wait complete. 2024-03-19T02:01:48.115 INFO:tasks.workunit.client.0.smithi099.stdout: 10 212529 22.57 MB/sec execute 377 sec latency 348.877 ms 2024-03-19T02:01:49.115 INFO:tasks.workunit.client.0.smithi099.stdout: 10 213069 22.59 MB/sec execute 378 sec latency 81.968 ms 2024-03-19T02:01:49.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:01:49 smithi115 ceph-mon[67332]: pgmap v973: 97 pgs: 97 active+clean; 632 MiB data, 8.2 GiB used, 1.0 TiB / 1.0 TiB avail; 2.6 MiB/s rd, 5.2 MiB/s wr, 388 op/s 2024-03-19T02:01:49.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:01:49 smithi099 ceph-mon[62951]: pgmap v973: 97 pgs: 97 active+clean; 632 MiB data, 8.2 GiB used, 1.0 TiB / 1.0 TiB avail; 2.6 MiB/s rd, 5.2 MiB/s wr, 388 op/s 2024-03-19T02:01:49.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:01:49 smithi112 ceph-mon[66819]: pgmap v973: 97 pgs: 97 active+clean; 632 MiB data, 8.2 GiB used, 1.0 TiB / 1.0 TiB avail; 2.6 MiB/s rd, 5.2 MiB/s wr, 388 op/s 2024-03-19T02:01:50.115 INFO:tasks.workunit.client.0.smithi099.stdout: 10 213465 22.57 MB/sec execute 379 sec latency 272.444 ms 2024-03-19T02:01:51.115 INFO:tasks.workunit.client.0.smithi099.stdout: 10 213852 22.58 MB/sec execute 380 sec latency 191.335 ms 2024-03-19T02:01:51.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:01:51 smithi115 ceph-mon[67332]: pgmap v974: 97 pgs: 97 active+clean; 630 MiB data, 8.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.1 MiB/s rd, 5.8 MiB/s wr, 481 op/s 2024-03-19T02:01:51.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:01:51 smithi099 ceph-mon[62951]: pgmap v974: 97 pgs: 97 active+clean; 630 MiB data, 8.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.1 MiB/s rd, 5.8 MiB/s wr, 481 op/s 2024-03-19T02:01:51.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:01:51 smithi112 ceph-mon[66819]: pgmap v974: 97 pgs: 97 active+clean; 630 MiB data, 8.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.1 MiB/s rd, 5.8 MiB/s wr, 481 op/s 2024-03-19T02:01:52.115 INFO:tasks.workunit.client.0.smithi099.stdout: 10 214287 22.59 MB/sec execute 381 sec latency 103.363 ms 2024-03-19T02:01:53.115 INFO:tasks.workunit.client.0.smithi099.stdout: 10 214730 22.60 MB/sec execute 382 sec latency 88.045 ms 2024-03-19T02:01:53.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:01:53 smithi099 ceph-mon[62951]: pgmap v975: 97 pgs: 97 active+clean; 628 MiB data, 8.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.3 MiB/s rd, 6.5 MiB/s wr, 557 op/s 2024-03-19T02:01:53.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:01:53 smithi115 ceph-mon[67332]: pgmap v975: 97 pgs: 97 active+clean; 628 MiB data, 8.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.3 MiB/s rd, 6.5 MiB/s wr, 557 op/s 2024-03-19T02:01:53.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:01:53 smithi112 ceph-mon[66819]: pgmap v975: 97 pgs: 97 active+clean; 628 MiB data, 8.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.3 MiB/s rd, 6.5 MiB/s wr, 557 op/s 2024-03-19T02:01:54.115 INFO:tasks.workunit.client.0.smithi099.stdout: 10 215284 22.63 MB/sec execute 383 sec latency 64.116 ms 2024-03-19T02:01:55.115 INFO:tasks.workunit.client.0.smithi099.stdout: 10 215606 22.63 MB/sec execute 384 sec latency 251.274 ms 2024-03-19T02:01:55.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:01:55 smithi115 ceph-mon[67332]: pgmap v976: 97 pgs: 97 active+clean; 630 MiB data, 8.2 GiB used, 1.0 TiB / 1.0 TiB avail; 2.8 MiB/s rd, 5.5 MiB/s wr, 503 op/s 2024-03-19T02:01:55.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:01:55 smithi099 ceph-mon[62951]: pgmap v976: 97 pgs: 97 active+clean; 630 MiB data, 8.2 GiB used, 1.0 TiB / 1.0 TiB avail; 2.8 MiB/s rd, 5.5 MiB/s wr, 503 op/s 2024-03-19T02:01:55.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:01:55 smithi112 ceph-mon[66819]: pgmap v976: 97 pgs: 97 active+clean; 630 MiB data, 8.2 GiB used, 1.0 TiB / 1.0 TiB avail; 2.8 MiB/s rd, 5.5 MiB/s wr, 503 op/s 2024-03-19T02:01:56.115 INFO:tasks.workunit.client.0.smithi099.stdout: 10 215938 22.62 MB/sec execute 385 sec latency 156.050 ms 2024-03-19T02:01:57.115 INFO:tasks.workunit.client.0.smithi099.stdout: 10 216233 22.58 MB/sec execute 386 sec latency 102.463 ms 2024-03-19T02:01:57.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:01:57 smithi115 ceph-mon[67332]: pgmap v977: 97 pgs: 97 active+clean; 634 MiB data, 8.3 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 6.7 MiB/s wr, 599 op/s 2024-03-19T02:01:57.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:01:57 smithi099 ceph-mon[62951]: pgmap v977: 97 pgs: 97 active+clean; 634 MiB data, 8.3 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 6.7 MiB/s wr, 599 op/s 2024-03-19T02:01:57.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:01:57 smithi112 ceph-mon[66819]: pgmap v977: 97 pgs: 97 active+clean; 634 MiB data, 8.3 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 6.7 MiB/s wr, 599 op/s 2024-03-19T02:01:58.115 INFO:tasks.workunit.client.0.smithi099.stdout: 10 216555 22.55 MB/sec execute 387 sec latency 100.248 ms 2024-03-19T02:01:59.115 INFO:tasks.workunit.client.0.smithi099.stdout: 10 217057 22.57 MB/sec execute 388 sec latency 77.391 ms 2024-03-19T02:01:59.414 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:01:59 smithi112 ceph-mon[66819]: pgmap v978: 97 pgs: 97 active+clean; 634 MiB data, 8.3 GiB used, 1.0 TiB / 1.0 TiB avail; 3.3 MiB/s rd, 5.5 MiB/s wr, 512 op/s 2024-03-19T02:01:59.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:01:59 smithi115 ceph-mon[67332]: pgmap v978: 97 pgs: 97 active+clean; 634 MiB data, 8.3 GiB used, 1.0 TiB / 1.0 TiB avail; 3.3 MiB/s rd, 5.5 MiB/s wr, 512 op/s 2024-03-19T02:01:59.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:01:59 smithi099 ceph-mon[62951]: pgmap v978: 97 pgs: 97 active+clean; 634 MiB data, 8.3 GiB used, 1.0 TiB / 1.0 TiB avail; 3.3 MiB/s rd, 5.5 MiB/s wr, 512 op/s 2024-03-19T02:02:00.115 INFO:tasks.workunit.client.0.smithi099.stdout: 10 217463 22.56 MB/sec execute 389 sec latency 343.062 ms 2024-03-19T02:02:01.086 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.084+0000 7fd8a0085640 1 -- 172.21.15.99:0/900743172 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd898102a40 msgr2=0x7fd898102ec0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:02:01.087 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.084+0000 7fd8a0085640 1 --2- 172.21.15.99:0/900743172 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd898102a40 0x7fd898102ec0 secure :-1 s=READY pgs=738 cs=0 l=1 rev1=1 crypto rx=0x7fd888009900 tx=0x7fd88802bc30 comp rx=0 tx=0).stop 2024-03-19T02:02:01.087 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.084+0000 7fd8a0085640 1 -- 172.21.15.99:0/900743172 shutdown_connections 2024-03-19T02:02:01.087 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.084+0000 7fd8a0085640 1 --2- 172.21.15.99:0/900743172 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd898103400 0x7fd898109c90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:01.087 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.084+0000 7fd8a0085640 1 --2- 172.21.15.99:0/900743172 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd898102a40 0x7fd898102ec0 unknown :-1 s=CLOSED pgs=738 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:01.087 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.084+0000 7fd8a0085640 1 --2- 172.21.15.99:0/900743172 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd898101850 0x7fd898101c70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:01.087 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.084+0000 7fd8a0085640 1 -- 172.21.15.99:0/900743172 >> 172.21.15.99:0/900743172 conn(0x7fd8980fcde0 msgr2=0x7fd8980ff200 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:02:01.087 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.085+0000 7fd8a0085640 1 -- 172.21.15.99:0/900743172 shutdown_connections 2024-03-19T02:02:01.087 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.085+0000 7fd8a0085640 1 -- 172.21.15.99:0/900743172 wait complete. 2024-03-19T02:02:01.087 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.085+0000 7fd8a0085640 1 Processor -- start 2024-03-19T02:02:01.087 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.085+0000 7fd8a0085640 1 -- start start 2024-03-19T02:02:01.087 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.085+0000 7fd8a0085640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd898101850 0x7fd898197b20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:01.087 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.085+0000 7fd8a0085640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd898102a40 0x7fd898198060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:01.087 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.085+0000 7fd8a0085640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd898103400 0x7fd89819f0d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:01.088 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.085+0000 7fd8a0085640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fd89810d540 con 0x7fd898101850 2024-03-19T02:02:01.088 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.085+0000 7fd8a0085640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fd89810d3c0 con 0x7fd898103400 2024-03-19T02:02:01.088 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.085+0000 7fd8a0085640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fd89810d6c0 con 0x7fd898102a40 2024-03-19T02:02:01.088 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.085+0000 7fd89de21640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd898101850 0x7fd898197b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:01.088 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.085+0000 7fd89de21640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd898101850 0x7fd898197b20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:56978/0 (socket says 172.21.15.99:56978) 2024-03-19T02:02:01.088 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.086+0000 7fd89de21640 1 -- 172.21.15.99:0/344465618 learned_addr learned my addr 172.21.15.99:0/344465618 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:02:01.088 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.086+0000 7fd89e622640 1 --2- 172.21.15.99:0/344465618 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd898103400 0x7fd89819f0d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:01.088 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.086+0000 7fd89d620640 1 --2- 172.21.15.99:0/344465618 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd898102a40 0x7fd898198060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:01.088 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.086+0000 7fd89de21640 1 -- 172.21.15.99:0/344465618 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd898102a40 msgr2=0x7fd898198060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:02:01.088 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.086+0000 7fd89de21640 1 --2- 172.21.15.99:0/344465618 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd898102a40 0x7fd898198060 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:01.088 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.086+0000 7fd89de21640 1 -- 172.21.15.99:0/344465618 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd898103400 msgr2=0x7fd89819f0d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:02:01.088 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.086+0000 7fd89de21640 1 --2- 172.21.15.99:0/344465618 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd898103400 0x7fd89819f0d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:01.088 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.086+0000 7fd89de21640 1 -- 172.21.15.99:0/344465618 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd89819f740 con 0x7fd898101850 2024-03-19T02:02:01.088 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.086+0000 7fd89de21640 1 --2- 172.21.15.99:0/344465618 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd898101850 0x7fd898197b20 secure :-1 s=READY pgs=739 cs=0 l=1 rev1=1 crypto rx=0x7fd89400b450 tx=0x7fd89400b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:02:01.089 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.086+0000 7fd88effd640 1 -- 172.21.15.99:0/344465618 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd894015020 con 0x7fd898101850 2024-03-19T02:02:01.089 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.086+0000 7fd88effd640 1 -- 172.21.15.99:0/344465618 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd894011d70 con 0x7fd898101850 2024-03-19T02:02:01.089 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.086+0000 7fd8a0085640 1 -- 172.21.15.99:0/344465618 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd89819fa30 con 0x7fd898101850 2024-03-19T02:02:01.089 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.086+0000 7fd88effd640 1 -- 172.21.15.99:0/344465618 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd89400f340 con 0x7fd898101850 2024-03-19T02:02:01.089 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.086+0000 7fd8a0085640 1 -- 172.21.15.99:0/344465618 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fd89819ff50 con 0x7fd898101850 2024-03-19T02:02:01.094 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.088+0000 7fd88effd640 1 -- 172.21.15.99:0/344465618 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7fd894018020 con 0x7fd898101850 2024-03-19T02:02:01.094 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.089+0000 7fd88effd640 1 --2- 172.21.15.99:0/344465618 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd8840a2920 0x7fd8840a4de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:01.094 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.089+0000 7fd8a2c74740 1 Processor -- start 2024-03-19T02:02:01.094 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.089+0000 7fd89d620640 1 --2- 172.21.15.99:0/344465618 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd8840a2920 0x7fd8840a4de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:01.094 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.089+0000 7fd88effd640 1 -- 172.21.15.99:0/344465618 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fd8940ce450 con 0x7fd898101850 2024-03-19T02:02:01.094 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.089+0000 7fd8a2c74740 1 -- start start 2024-03-19T02:02:01.095 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.089+0000 7fd8a2c74740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56268e776100 0x56268e776520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:01.095 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.089+0000 7fd8a2c74740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56268e776af0 0x56268e77b5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:01.095 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.089+0000 7fd8a2c74740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56268e77bc60 0x56268e77e0f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:01.095 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.089+0000 7fd8a2c74740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x56268e6dabe0 con 0x56268e776100 2024-03-19T02:02:01.095 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.089+0000 7fd8a2c74740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x56268e683960 con 0x56268e77bc60 2024-03-19T02:02:01.095 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.089+0000 7fd8a2c74740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x56268e6d6c80 con 0x56268e776af0 2024-03-19T02:02:01.095 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.089+0000 7fd89de21640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56268e776100 0x56268e776520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:01.095 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.089+0000 7fd89de21640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56268e776100 0x56268e776520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:56986/0 (socket says 172.21.15.99:56986) 2024-03-19T02:02:01.095 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.089+0000 7fd89de21640 1 -- 172.21.15.99:0/58476452 learned_addr learned my addr 172.21.15.99:0/58476452 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:02:01.095 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.089+0000 7fd89e622640 1 --2- 172.21.15.99:0/58476452 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56268e77bc60 0x56268e77e0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:01.095 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.089+0000 7fd89d620640 1 --2- 172.21.15.99:0/344465618 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd8840a2920 0x7fd8840a4de0 secure :-1 s=READY pgs=366 cs=0 l=1 rev1=1 crypto rx=0x7fd88802c7b0 tx=0x7fd888005fd0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:02:01.095 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.090+0000 7fd89de21640 1 -- 172.21.15.99:0/58476452 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56268e776af0 msgr2=0x56268e77b5f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:02:01.095 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.090+0000 7fd89de21640 1 --2- 172.21.15.99:0/58476452 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56268e776af0 0x56268e77b5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:01.095 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.090+0000 7fd89de21640 1 -- 172.21.15.99:0/58476452 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56268e77bc60 msgr2=0x56268e77e0f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:02:01.096 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.090+0000 7fd89de21640 1 --2- 172.21.15.99:0/58476452 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56268e77bc60 0x56268e77e0f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:01.096 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.090+0000 7fd89de21640 1 -- 172.21.15.99:0/58476452 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x56268e67ad20 con 0x56268e776100 2024-03-19T02:02:01.096 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.090+0000 7fd89de21640 1 --2- 172.21.15.99:0/58476452 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56268e776100 0x56268e776520 secure :-1 s=READY pgs=740 cs=0 l=1 rev1=1 crypto rx=0x7fd89400be10 tx=0x7fd894009280 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f2c852cfb04be9a6 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:02:01.096 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.090+0000 7fd883fff640 1 -- 172.21.15.99:0/58476452 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd89409ef00 con 0x56268e776100 2024-03-19T02:02:01.096 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.090+0000 7fd883fff640 1 -- 172.21.15.99:0/58476452 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd8940a3550 con 0x56268e776100 2024-03-19T02:02:01.096 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.090+0000 7fd883fff640 1 -- 172.21.15.99:0/58476452 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd89409a200 con 0x56268e776100 2024-03-19T02:02:01.096 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.091+0000 7fd8a2c74740 1 -- 172.21.15.99:0/58476452 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56268e776100 msgr2=0x56268e776520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:02:01.096 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.091+0000 7fd8a2c74740 1 --2- 172.21.15.99:0/58476452 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56268e776100 0x56268e776520 secure :-1 s=READY pgs=740 cs=0 l=1 rev1=1 crypto rx=0x7fd89400be10 tx=0x7fd894009280 comp rx=0 tx=0).stop 2024-03-19T02:02:01.096 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.091+0000 7fd8a2c74740 1 -- 172.21.15.99:0/58476452 shutdown_connections 2024-03-19T02:02:01.096 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.091+0000 7fd8a2c74740 1 --2- 172.21.15.99:0/58476452 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56268e77bc60 0x56268e77e0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:01.096 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.091+0000 7fd8a2c74740 1 --2- 172.21.15.99:0/58476452 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56268e776af0 0x56268e77b5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:01.096 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.091+0000 7fd8a2c74740 1 --2- 172.21.15.99:0/58476452 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56268e776100 0x56268e776520 unknown :-1 s=CLOSED pgs=740 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:01.096 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.091+0000 7fd8a2c74740 1 -- 172.21.15.99:0/58476452 >> 172.21.15.99:0/58476452 conn(0x56268e771670 msgr2=0x56268e771a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:02:01.097 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.091+0000 7fd8a2c74740 1 -- 172.21.15.99:0/58476452 shutdown_connections 2024-03-19T02:02:01.097 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.091+0000 7fd8a2c74740 1 -- 172.21.15.99:0/58476452 wait complete. 2024-03-19T02:02:01.097 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.091+0000 7fd8a2c74740 1 -- start start 2024-03-19T02:02:01.097 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.091+0000 7fd8a2c74740 1 Processor -- start 2024-03-19T02:02:01.097 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.092+0000 7fd8a2c74740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56268e776100 0x56268e780190 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:01.097 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.092+0000 7fd8a2c74740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56268e776af0 0x56268e7806d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:01.097 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.092+0000 7fd8a2c74740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56268e77bc60 0x56268e787710 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:01.097 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.092+0000 7fd8a2c74740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x56268e77ec10 con 0x56268e776100 2024-03-19T02:02:01.097 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.092+0000 7fd8a2c74740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x56268e696db0 con 0x56268e776af0 2024-03-19T02:02:01.097 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.092+0000 7fd8a2c74740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x56268e77ed90 con 0x56268e77bc60 2024-03-19T02:02:01.097 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.092+0000 7fd8a2c74740 10 client.0 _ll_register_callbacks cb 0x56268e69be70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T02:02:01.097 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.092+0000 7fd89de21640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56268e776100 0x56268e780190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:01.097 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.092+0000 7fd89de21640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56268e776100 0x56268e780190 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:56996/0 (socket says 172.21.15.99:56996) 2024-03-19T02:02:01.097 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.092+0000 7fd89de21640 1 -- 172.21.15.99:0/3861667758 learned_addr learned my addr 172.21.15.99:0/3861667758 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:02:01.097 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.092+0000 7fd89e622640 1 --2- 172.21.15.99:0/3861667758 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56268e77bc60 0x56268e787710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:01.097 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.092+0000 7fd89d620640 1 --2- 172.21.15.99:0/3861667758 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56268e776af0 0x56268e7806d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:01.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.092+0000 7fd89de21640 1 -- 172.21.15.99:0/3861667758 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56268e77bc60 msgr2=0x56268e787710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:02:01.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.092+0000 7fd89de21640 1 --2- 172.21.15.99:0/3861667758 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56268e77bc60 0x56268e787710 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:01.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.092+0000 7fd89de21640 1 -- 172.21.15.99:0/3861667758 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56268e776af0 msgr2=0x56268e7806d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:02:01.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.092+0000 7fd89de21640 1 --2- 172.21.15.99:0/3861667758 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56268e776af0 0x56268e7806d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:01.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.092+0000 7fd89de21640 1 -- 172.21.15.99:0/3861667758 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x56268e695c60 con 0x56268e776100 2024-03-19T02:02:01.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.092+0000 7fd89de21640 1 --2- 172.21.15.99:0/3861667758 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56268e776100 0x56268e780190 secure :-1 s=READY pgs=741 cs=0 l=1 rev1=1 crypto rx=0x7fd8940cbf60 tx=0x7fd894011ad0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f530eadbe856453c server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:02:01.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.092+0000 7fd8817fa640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T02:02:01.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.092+0000 7fd8817fa640 1 -- 172.21.15.99:0/3861667758 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd89409a200 con 0x56268e776100 2024-03-19T02:02:01.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.093+0000 7fd8817fa640 1 -- 172.21.15.99:0/3861667758 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fd89409a3a0 con 0x56268e776100 2024-03-19T02:02:01.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.093+0000 7fd8a2c74740 1 -- 172.21.15.99:0/3861667758 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fd8940a3550 con 0x56268e776100 2024-03-19T02:02:01.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.093+0000 7fd8817fa640 1 -- 172.21.15.99:0/3861667758 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7fd894096b00 con 0x56268e776100 2024-03-19T02:02:01.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.093+0000 7fd8a2c74740 10 client.17010 fetch_fsmap learned FSMap version 37 2024-03-19T02:02:01.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.093+0000 7fd8a2c74740 1 -- 172.21.15.99:0/3861667758 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x56268e3ef210 con 0x56268e776100 2024-03-19T02:02:01.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.093+0000 7fd8817fa640 1 -- 172.21.15.99:0/3861667758 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd8940a3550 con 0x56268e776100 2024-03-19T02:02:01.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.093+0000 7fd8817fa640 1 -- 172.21.15.99:0/3861667758 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7fd8940853b0 con 0x56268e776100 2024-03-19T02:02:01.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.093+0000 7fd8a2c74740 10 client.17010 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T02:02:01.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.093+0000 7fd8a2c74740 10 client.17010 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T02:02:01.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.093+0000 7fd8a2c74740 20 client.17010 populate_metadata read hostname 'smithi099' 2024-03-19T02:02:01.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.093+0000 7fd8a2c74740 1 --2- 172.21.15.99:0/3861667758 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56268e78bc00 0x56268e78dff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:01.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.093+0000 7fd8a2c74740 4 client.17010 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T02:02:01.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.093+0000 7fd8a2c74740 1 -- 172.21.15.99:0/3861667758 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x56268e6e8120 con 0x56268e78bc00 2024-03-19T02:02:01.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.094+0000 7fd89d620640 1 --2- 172.21.15.99:0/3861667758 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56268e78bc00 0x56268e78dff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:01.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.095+0000 7fd8817fa640 10 client.17010 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T02:02:01.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.095+0000 7fd89d620640 1 --2- 172.21.15.99:0/3861667758 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56268e78bc00 0x56268e78dff0 crc :-1 s=READY pgs=215 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=212e86226616d129 server_cookie=ceb5d3f0cc06fb30 in_seq=0 out_seq=0 2024-03-19T02:02:01.108 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.108+0000 7fd8817fa640 1 -- 172.21.15.99:0/3861667758 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x56268e6e8120 con 0x56268e78bc00 2024-03-19T02:02:01.108 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.108+0000 7fd8817fa640 10 client.17010 handle_command_reply: tid=0 2024-03-19T02:02:01.108 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.108+0000 7fd8a2c74740 1 client.17010 shutdown 2024-03-19T02:02:01.108 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.108+0000 7fd8a2c74740 1 -- 172.21.15.99:0/3861667758 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56268e776100 msgr2=0x56268e780190 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:02:01.108 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.108+0000 7fd8a2c74740 1 --2- 172.21.15.99:0/3861667758 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56268e776100 0x56268e780190 secure :-1 s=READY pgs=741 cs=0 l=1 rev1=1 crypto rx=0x7fd8940cbf60 tx=0x7fd894011ad0 comp rx=0 tx=0).stop 2024-03-19T02:02:01.108 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.108+0000 7fd8a2c74740 1 -- 172.21.15.99:0/3861667758 shutdown_connections 2024-03-19T02:02:01.109 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.108+0000 7fd8a2c74740 1 --2- 172.21.15.99:0/3861667758 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56268e78bc00 0x56268e78dff0 crc :-1 s=READY pgs=215 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:01.109 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.108+0000 7fd8a2c74740 1 --2- 172.21.15.99:0/3861667758 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56268e776af0 0x56268e7806d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:01.109 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.108+0000 7fd8817fa640 0 client.17010 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T02:02:01.109 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.108+0000 7fd8a2c74740 1 --2- 172.21.15.99:0/3861667758 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56268e77bc60 0x56268e787710 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:01.109 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.108+0000 7fd8a2c74740 1 --2- 172.21.15.99:0/3861667758 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56268e776100 0x56268e780190 unknown :-1 s=CLOSED pgs=741 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:01.109 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.108+0000 7fd8a2c74740 1 -- 172.21.15.99:0/3861667758 >> 172.21.15.99:0/3861667758 conn(0x56268e771670 msgr2=0x56268e7728c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:02:01.109 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.108+0000 7fd8a2c74740 1 -- 172.21.15.99:0/3861667758 shutdown_connections 2024-03-19T02:02:01.109 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.108+0000 7fd8a2c74740 1 -- 172.21.15.99:0/3861667758 wait complete. 2024-03-19T02:02:01.109 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.108+0000 7fd8a2c74740 20 client.17010 trim_cache size 0 max 16384 2024-03-19T02:02:01.115 INFO:tasks.workunit.client.0.smithi099.stdout: 10 217826 22.56 MB/sec execute 390 sec latency 101.586 ms 2024-03-19T02:02:01.128 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.127+0000 7fd8a2c74740 1 Processor -- start 2024-03-19T02:02:01.128 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.127+0000 7fd8a2c74740 1 -- start start 2024-03-19T02:02:01.128 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.127+0000 7fd8a2c74740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56268e776100 0x56268e7804b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:01.128 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.127+0000 7fd8a2c74740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56268e776af0 0x56268e7876d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:01.128 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.127+0000 7fd8a2c74740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56268e77bc60 0x56268e777350 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:01.128 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.127+0000 7fd8a2c74740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fd8981ab740 con 0x56268e776100 2024-03-19T02:02:01.128 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.127+0000 7fd8a2c74740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fd8981a9010 con 0x56268e77bc60 2024-03-19T02:02:01.128 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.127+0000 7fd8a2c74740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fd8981a6e60 con 0x56268e776af0 2024-03-19T02:02:01.128 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.128+0000 7fd89de21640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56268e776100 0x56268e7804b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:01.128 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.128+0000 7fd89de21640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56268e776100 0x56268e7804b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:57010/0 (socket says 172.21.15.99:57010) 2024-03-19T02:02:01.128 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.128+0000 7fd89de21640 1 -- 172.21.15.99:0/342913988 learned_addr learned my addr 172.21.15.99:0/342913988 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:02:01.128 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.128+0000 7fd89e622640 1 --2- 172.21.15.99:0/342913988 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56268e77bc60 0x56268e777350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:01.129 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.128+0000 7fd89de21640 1 -- 172.21.15.99:0/342913988 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56268e776af0 msgr2=0x56268e7876d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:02:01.129 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.128+0000 7fd89de21640 1 --2- 172.21.15.99:0/342913988 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56268e776af0 0x56268e7876d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:01.129 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.128+0000 7fd89de21640 1 -- 172.21.15.99:0/342913988 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56268e77bc60 msgr2=0x56268e777350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:02:01.129 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.128+0000 7fd89de21640 1 --2- 172.21.15.99:0/342913988 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56268e77bc60 0x56268e777350 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:01.129 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.128+0000 7fd89de21640 1 -- 172.21.15.99:0/342913988 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x56268e4bb620 con 0x56268e776100 2024-03-19T02:02:01.129 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.128+0000 7fd89de21640 1 --2- 172.21.15.99:0/342913988 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56268e776100 0x56268e7804b0 secure :-1 s=READY pgs=742 cs=0 l=1 rev1=1 crypto rx=0x7fd894097730 tx=0x7fd8940992c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c1f613af7d37d6ad server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:02:01.129 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.128+0000 7fd8837fe640 1 -- 172.21.15.99:0/342913988 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd8940a01f0 con 0x56268e776100 2024-03-19T02:02:01.129 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.128+0000 7fd8837fe640 1 -- 172.21.15.99:0/342913988 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd89407e070 con 0x56268e776100 2024-03-19T02:02:01.129 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.128+0000 7fd8837fe640 1 -- 172.21.15.99:0/342913988 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd894097050 con 0x56268e776100 2024-03-19T02:02:01.131 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.129+0000 7fd8a2c74740 1 -- 172.21.15.99:0/342913988 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56268e776100 msgr2=0x56268e7804b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:02:01.131 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.129+0000 7fd8a2c74740 1 --2- 172.21.15.99:0/342913988 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56268e776100 0x56268e7804b0 secure :-1 s=READY pgs=742 cs=0 l=1 rev1=1 crypto rx=0x7fd894097730 tx=0x7fd8940992c0 comp rx=0 tx=0).stop 2024-03-19T02:02:01.131 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.129+0000 7fd8a2c74740 1 -- 172.21.15.99:0/342913988 shutdown_connections 2024-03-19T02:02:01.131 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.129+0000 7fd8a2c74740 1 --2- 172.21.15.99:0/342913988 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56268e77bc60 0x56268e777350 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:01.131 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.129+0000 7fd8a2c74740 1 --2- 172.21.15.99:0/342913988 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56268e776af0 0x56268e7876d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:01.131 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.129+0000 7fd8a2c74740 1 --2- 172.21.15.99:0/342913988 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56268e776100 0x56268e7804b0 unknown :-1 s=CLOSED pgs=742 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:01.132 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.129+0000 7fd8a2c74740 1 -- 172.21.15.99:0/342913988 >> 172.21.15.99:0/342913988 conn(0x56268e771670 msgr2=0x56268e77fda0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:02:01.132 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.129+0000 7fd8a2c74740 1 -- 172.21.15.99:0/342913988 shutdown_connections 2024-03-19T02:02:01.132 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.129+0000 7fd8a2c74740 1 -- 172.21.15.99:0/342913988 wait complete. 2024-03-19T02:02:01.132 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.129+0000 7fd8a2c74740 1 -- start start 2024-03-19T02:02:01.132 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.129+0000 7fd8a2c74740 1 Processor -- start 2024-03-19T02:02:01.132 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.130+0000 7fd8a2c74740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56268e776100 0x56268e7848c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:01.132 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.130+0000 7fd8a2c74740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56268e776af0 0x56268e784e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:01.132 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.130+0000 7fd8a2c74740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56268e77bc60 0x56268e785e10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:01.132 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.130+0000 7fd8a2c74740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fd8981a74d0 con 0x56268e77bc60 2024-03-19T02:02:01.132 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.130+0000 7fd8a2c74740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fd8981a8940 con 0x56268e776af0 2024-03-19T02:02:01.132 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.130+0000 7fd8a2c74740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fd8981a7ba0 con 0x56268e776100 2024-03-19T02:02:01.132 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.130+0000 7fd8a2c74740 10 client.0 _ll_register_callbacks cb 0x7fd890009b30 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T02:02:01.132 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.130+0000 7fd89de21640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56268e776100 0x56268e7848c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:01.132 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.130+0000 7fd89e622640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56268e77bc60 0x56268e785e10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:01.132 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.130+0000 7fd89de21640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56268e776100 0x56268e7848c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:56228/0 (socket says 172.21.15.99:56228) 2024-03-19T02:02:01.132 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.130+0000 7fd89e622640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56268e77bc60 0x56268e785e10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:57026/0 (socket says 172.21.15.99:57026) 2024-03-19T02:02:01.133 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.130+0000 7fd89e622640 1 -- 172.21.15.99:0/1882727084 learned_addr learned my addr 172.21.15.99:0/1882727084 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:02:01.133 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.130+0000 7fd89d620640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56268e776af0 0x56268e784e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:01.133 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.130+0000 7fd89de21640 1 -- 172.21.15.99:0/1882727084 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56268e776af0 msgr2=0x56268e784e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:02:01.133 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.130+0000 7fd89de21640 1 --2- 172.21.15.99:0/1882727084 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56268e776af0 0x56268e784e00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:01.133 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.130+0000 7fd89de21640 1 -- 172.21.15.99:0/1882727084 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56268e77bc60 msgr2=0x56268e785e10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:02:01.133 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.130+0000 7fd89de21640 1 --2- 172.21.15.99:0/1882727084 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56268e77bc60 0x56268e785e10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:01.133 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.130+0000 7fd89de21640 1 -- 172.21.15.99:0/1882727084 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({osdmap=0}) -- 0x56268e786400 con 0x56268e776100 2024-03-19T02:02:01.133 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.131+0000 7fd89e622640 1 --2- 172.21.15.99:0/1882727084 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56268e77bc60 0x56268e785e10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:02:01.133 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.131+0000 7fd89d620640 1 --2- 172.21.15.99:0/1882727084 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56268e776af0 0x56268e784e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:02:01.133 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.131+0000 7fd89de21640 1 --2- 172.21.15.99:0/1882727084 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56268e776100 0x56268e7848c0 secure :-1 s=READY pgs=128 cs=0 l=1 rev1=1 crypto rx=0x56268e7635b0 tx=0x7fd894010970 comp rx=0 tx=0).ready entity=mon.1 client_cookie=5ffbcc35984a12aa server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:02:01.133 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.131+0000 7fd8817fa640 10 client.0 ms_handle_connect on v2:172.21.15.115:3300/0 2024-03-19T02:02:01.135 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.133+0000 7fd8817fa640 1 -- 172.21.15.99:0/1882727084 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd894097050 con 0x56268e776100 2024-03-19T02:02:01.135 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.133+0000 7fd8a2c74740 1 -- 172.21.15.99:0/1882727084 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fd89407e070 con 0x56268e776100 2024-03-19T02:02:01.135 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.133+0000 7fd8817fa640 1 -- 172.21.15.99:0/1882727084 <== mon.1 v2:172.21.15.115:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fd89409a400 con 0x56268e776100 2024-03-19T02:02:01.135 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.134+0000 7fd8817fa640 1 -- 172.21.15.99:0/1882727084 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7fd894094050 con 0x56268e776100 2024-03-19T02:02:01.135 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.134+0000 7fd8a2c74740 10 client.26791 fetch_fsmap learned FSMap version 37 2024-03-19T02:02:01.135 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.134+0000 7fd8a2c74740 1 -- 172.21.15.99:0/1882727084 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x56268e786400 con 0x56268e776100 2024-03-19T02:02:01.135 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.134+0000 7fd8817fa640 1 -- 172.21.15.99:0/1882727084 <== mon.1 v2:172.21.15.115:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd89407e070 con 0x56268e776100 2024-03-19T02:02:01.135 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.134+0000 7fd8817fa640 1 -- 172.21.15.99:0/1882727084 <== mon.1 v2:172.21.15.115:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7fd89409fe20 con 0x56268e776100 2024-03-19T02:02:01.135 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.134+0000 7fd8a2c74740 10 client.26791 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T02:02:01.135 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.134+0000 7fd8a2c74740 10 client.26791 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T02:02:01.135 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.134+0000 7fd8a2c74740 20 client.26791 populate_metadata read hostname 'smithi099' 2024-03-19T02:02:01.136 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.134+0000 7fd8a2c74740 1 --2- 172.21.15.99:0/1882727084 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56268e7f1c00 0x56268e7f3ff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:01.136 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.134+0000 7fd8a2c74740 4 client.26791 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T02:02:01.136 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.134+0000 7fd8a2c74740 1 -- 172.21.15.99:0/1882727084 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x56268e6e2470 con 0x56268e7f1c00 2024-03-19T02:02:01.136 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.135+0000 7fd89d620640 1 --2- 172.21.15.99:0/1882727084 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56268e7f1c00 0x56268e7f3ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:01.136 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.135+0000 7fd89d620640 1 --2- 172.21.15.99:0/1882727084 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56268e7f1c00 0x56268e7f3ff0 crc :-1 s=READY pgs=216 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=8aa88d90d52c370a server_cookie=cb115f7616e25ae5 in_seq=0 out_seq=0 2024-03-19T02:02:01.136 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.135+0000 7fd8817fa640 10 client.26791 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T02:02:01.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:02:01 smithi115 ceph-mon[67332]: pgmap v979: 97 pgs: 97 active+clean; 640 MiB data, 8.3 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 6.0 MiB/s wr, 532 op/s 2024-03-19T02:02:01.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:02:01 smithi099 ceph-mon[62951]: pgmap v979: 97 pgs: 97 active+clean; 640 MiB data, 8.3 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 6.0 MiB/s wr, 532 op/s 2024-03-19T02:02:01.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:02:01 smithi112 ceph-mon[66819]: pgmap v979: 97 pgs: 97 active+clean; 640 MiB data, 8.3 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 6.0 MiB/s wr, 532 op/s 2024-03-19T02:02:01.703 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.702+0000 7fd8817fa640 1 -- 172.21.15.99:0/1882727084 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x56268e6e2470 con 0x56268e7f1c00 2024-03-19T02:02:01.703 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.702+0000 7fd8817fa640 10 client.26791 handle_command_reply: tid=0 2024-03-19T02:02:01.703 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.702+0000 7fd8a2c74740 1 client.26791 shutdown 2024-03-19T02:02:01.703 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.702+0000 7fd8a2c74740 1 -- 172.21.15.99:0/1882727084 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56268e776100 msgr2=0x56268e7848c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:02:01.703 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.702+0000 7fd8a2c74740 1 --2- 172.21.15.99:0/1882727084 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56268e776100 0x56268e7848c0 secure :-1 s=READY pgs=128 cs=0 l=1 rev1=1 crypto rx=0x56268e7635b0 tx=0x7fd894010970 comp rx=0 tx=0).stop 2024-03-19T02:02:01.703 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.702+0000 7fd8a2c74740 1 -- 172.21.15.99:0/1882727084 shutdown_connections 2024-03-19T02:02:01.703 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.702+0000 7fd8a2c74740 1 --2- 172.21.15.99:0/1882727084 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56268e7f1c00 0x56268e7f3ff0 crc :-1 s=READY pgs=216 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:01.703 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.702+0000 7fd8a2c74740 1 --2- 172.21.15.99:0/1882727084 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56268e776af0 0x56268e784e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:01.703 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.702+0000 7fd8817fa640 0 client.26791 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T02:02:01.704 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.702+0000 7fd8a2c74740 1 --2- 172.21.15.99:0/1882727084 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56268e77bc60 0x56268e785e10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:01.704 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.702+0000 7fd8a2c74740 1 --2- 172.21.15.99:0/1882727084 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56268e776100 0x56268e7848c0 unknown :-1 s=CLOSED pgs=128 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:01.704 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.702+0000 7fd8a2c74740 1 -- 172.21.15.99:0/1882727084 >> 172.21.15.99:0/1882727084 conn(0x56268e771670 msgr2=0x56268e7805a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:02:01.704 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.703+0000 7fd8a2c74740 1 -- 172.21.15.99:0/1882727084 shutdown_connections 2024-03-19T02:02:01.704 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.703+0000 7fd8a2c74740 1 -- 172.21.15.99:0/1882727084 wait complete. 2024-03-19T02:02:01.704 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.703+0000 7fd8a2c74740 20 client.26791 trim_cache size 0 max 16384 2024-03-19T02:02:01.704 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T02:02:01.704 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T02:02:01.704 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T02:02:01.704 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T02:02:01.705 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.703+0000 7fd8a0085640 1 -- 172.21.15.99:0/344465618 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd8840a2920 msgr2=0x7fd8840a4de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:02:01.705 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.703+0000 7fd8a0085640 1 --2- 172.21.15.99:0/344465618 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd8840a2920 0x7fd8840a4de0 secure :-1 s=READY pgs=366 cs=0 l=1 rev1=1 crypto rx=0x7fd88802c7b0 tx=0x7fd888005fd0 comp rx=0 tx=0).stop 2024-03-19T02:02:01.705 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.703+0000 7fd8a0085640 1 -- 172.21.15.99:0/344465618 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd898101850 msgr2=0x7fd898197b20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:02:01.706 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.703+0000 7fd8a0085640 1 --2- 172.21.15.99:0/344465618 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd898101850 0x7fd898197b20 secure :-1 s=READY pgs=739 cs=0 l=1 rev1=1 crypto rx=0x7fd89400b450 tx=0x7fd89400b910 comp rx=0 tx=0).stop 2024-03-19T02:02:01.706 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.703+0000 7fd8a0085640 1 -- 172.21.15.99:0/344465618 shutdown_connections 2024-03-19T02:02:01.706 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.703+0000 7fd8a0085640 1 --2- 172.21.15.99:0/344465618 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd8840a2920 0x7fd8840a4de0 unknown :-1 s=CLOSED pgs=366 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:01.706 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.703+0000 7fd8a0085640 1 --2- 172.21.15.99:0/344465618 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd898103400 0x7fd89819f0d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:01.706 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.703+0000 7fd8a0085640 1 --2- 172.21.15.99:0/344465618 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd898102a40 0x7fd898198060 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:01.706 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.703+0000 7fd8a0085640 1 --2- 172.21.15.99:0/344465618 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd898101850 0x7fd898197b20 unknown :-1 s=CLOSED pgs=739 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:01.706 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.703+0000 7fd8a0085640 1 -- 172.21.15.99:0/344465618 >> 172.21.15.99:0/344465618 conn(0x7fd8980fcde0 msgr2=0x7fd898105c70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:02:01.706 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.703+0000 7fd8a0085640 1 -- 172.21.15.99:0/344465618 shutdown_connections 2024-03-19T02:02:01.706 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:01.703+0000 7fd8a0085640 1 -- 172.21.15.99:0/344465618 wait complete. 2024-03-19T02:02:02.115 INFO:tasks.workunit.client.0.smithi099.stdout: 10 218065 22.53 MB/sec execute 391 sec latency 370.646 ms 2024-03-19T02:02:03.115 INFO:tasks.workunit.client.0.smithi099.stdout: 10 218610 22.54 MB/sec execute 392 sec latency 66.885 ms 2024-03-19T02:02:03.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:02:03 smithi115 ceph-mon[67332]: pgmap v980: 97 pgs: 97 active+clean; 638 MiB data, 8.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.6 MiB/s rd, 6.4 MiB/s wr, 553 op/s 2024-03-19T02:02:03.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:02:03 smithi099 ceph-mon[62951]: pgmap v980: 97 pgs: 97 active+clean; 638 MiB data, 8.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.6 MiB/s rd, 6.4 MiB/s wr, 553 op/s 2024-03-19T02:02:03.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:02:03 smithi112 ceph-mon[66819]: pgmap v980: 97 pgs: 97 active+clean; 638 MiB data, 8.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.6 MiB/s rd, 6.4 MiB/s wr, 553 op/s 2024-03-19T02:02:04.116 INFO:tasks.workunit.client.0.smithi099.stdout: 10 219057 22.55 MB/sec execute 393 sec latency 82.996 ms 2024-03-19T02:02:05.116 INFO:tasks.workunit.client.0.smithi099.stdout: 10 219493 22.56 MB/sec execute 394 sec latency 213.796 ms 2024-03-19T02:02:05.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:02:05 smithi115 ceph-mon[67332]: pgmap v981: 97 pgs: 97 active+clean; 635 MiB data, 8.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 5.2 MiB/s wr, 457 op/s 2024-03-19T02:02:05.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:02:05 smithi099 ceph-mon[62951]: pgmap v981: 97 pgs: 97 active+clean; 635 MiB data, 8.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 5.2 MiB/s wr, 457 op/s 2024-03-19T02:02:05.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:02:05 smithi112 ceph-mon[66819]: pgmap v981: 97 pgs: 97 active+clean; 635 MiB data, 8.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 5.2 MiB/s wr, 457 op/s 2024-03-19T02:02:06.116 INFO:tasks.workunit.client.0.smithi099.stdout: 10 219959 22.58 MB/sec execute 395 sec latency 102.287 ms 2024-03-19T02:02:07.116 INFO:tasks.workunit.client.0.smithi099.stdout: 10 220333 22.58 MB/sec execute 396 sec latency 76.610 ms 2024-03-19T02:02:08.116 INFO:tasks.workunit.client.0.smithi099.stdout: 10 220744 22.58 MB/sec execute 397 sec latency 101.610 ms 2024-03-19T02:02:08.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:02:08 smithi115 ceph-mon[67332]: pgmap v982: 97 pgs: 97 active+clean; 635 MiB data, 8.3 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 MiB/s rd, 6.2 MiB/s wr, 588 op/s 2024-03-19T02:02:08.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:02:08 smithi099 ceph-mon[62951]: pgmap v982: 97 pgs: 97 active+clean; 635 MiB data, 8.3 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 MiB/s rd, 6.2 MiB/s wr, 588 op/s 2024-03-19T02:02:08.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:02:08 smithi112 ceph-mon[66819]: pgmap v982: 97 pgs: 97 active+clean; 635 MiB data, 8.3 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 MiB/s rd, 6.2 MiB/s wr, 588 op/s 2024-03-19T02:02:09.116 INFO:tasks.workunit.client.0.smithi099.stdout: 10 221074 22.55 MB/sec execute 398 sec latency 84.784 ms 2024-03-19T02:02:10.116 INFO:tasks.workunit.client.0.smithi099.stdout: 10 221299 22.52 MB/sec execute 399 sec latency 261.527 ms 2024-03-19T02:02:10.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:02:10 smithi115 ceph-mon[67332]: pgmap v983: 97 pgs: 97 active+clean; 635 MiB data, 8.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 5.0 MiB/s wr, 476 op/s 2024-03-19T02:02:10.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:02:10 smithi099 ceph-mon[62951]: pgmap v983: 97 pgs: 97 active+clean; 635 MiB data, 8.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 5.0 MiB/s wr, 476 op/s 2024-03-19T02:02:10.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:02:10 smithi112 ceph-mon[66819]: pgmap v983: 97 pgs: 97 active+clean; 635 MiB data, 8.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 5.0 MiB/s wr, 476 op/s 2024-03-19T02:02:11.116 INFO:tasks.workunit.client.0.smithi099.stdout: 10 221553 22.50 MB/sec execute 400 sec latency 441.949 ms 2024-03-19T02:02:12.116 INFO:tasks.workunit.client.0.smithi099.stdout: 10 222112 22.51 MB/sec execute 401 sec latency 79.802 ms 2024-03-19T02:02:12.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:02:12 smithi115 ceph-mon[67332]: pgmap v984: 97 pgs: 97 active+clean; 642 MiB data, 8.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.5 MiB/s rd, 5.9 MiB/s wr, 557 op/s 2024-03-19T02:02:12.392 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:02:12 smithi099 ceph-mon[62951]: pgmap v984: 97 pgs: 97 active+clean; 642 MiB data, 8.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.5 MiB/s rd, 5.9 MiB/s wr, 557 op/s 2024-03-19T02:02:12.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:02:12 smithi112 ceph-mon[66819]: pgmap v984: 97 pgs: 97 active+clean; 642 MiB data, 8.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.5 MiB/s rd, 5.9 MiB/s wr, 557 op/s 2024-03-19T02:02:13.116 INFO:tasks.workunit.client.0.smithi099.stdout: 10 222627 22.53 MB/sec execute 402 sec latency 60.227 ms 2024-03-19T02:02:14.116 INFO:tasks.workunit.client.0.smithi099.stdout: 10 223226 22.54 MB/sec execute 403 sec latency 61.651 ms 2024-03-19T02:02:14.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:02:14 smithi115 ceph-mon[67332]: pgmap v985: 97 pgs: 97 active+clean; 642 MiB data, 8.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.5 MiB/s rd, 6.3 MiB/s wr, 569 op/s 2024-03-19T02:02:14.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:02:14 smithi099 ceph-mon[62951]: pgmap v985: 97 pgs: 97 active+clean; 642 MiB data, 8.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.5 MiB/s rd, 6.3 MiB/s wr, 569 op/s 2024-03-19T02:02:14.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:02:14 smithi112 ceph-mon[66819]: pgmap v985: 97 pgs: 97 active+clean; 642 MiB data, 8.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.5 MiB/s rd, 6.3 MiB/s wr, 569 op/s 2024-03-19T02:02:14.801 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.798+0000 7f1c52d5b640 1 -- 172.21.15.99:0/1705001961 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1c4c100750 msgr2=0x7f1c4c100b70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:02:14.801 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.798+0000 7f1c52d5b640 1 --2- 172.21.15.99:0/1705001961 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1c4c100750 0x7f1c4c100b70 secure :-1 s=READY pgs=743 cs=0 l=1 rev1=1 crypto rx=0x7f1c44009ca0 tx=0x7f1c4402c0f0 comp rx=0 tx=0).stop 2024-03-19T02:02:14.801 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.798+0000 7f1c52d5b640 1 -- 172.21.15.99:0/1705001961 shutdown_connections 2024-03-19T02:02:14.801 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.798+0000 7f1c52d5b640 1 --2- 172.21.15.99:0/1705001961 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1c4c102300 0x7f1c4c108b90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:14.801 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.798+0000 7f1c52d5b640 1 --2- 172.21.15.99:0/1705001961 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1c4c101940 0x7f1c4c101dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:14.801 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.798+0000 7f1c52d5b640 1 --2- 172.21.15.99:0/1705001961 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1c4c100750 0x7f1c4c100b70 unknown :-1 s=CLOSED pgs=743 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:14.801 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.798+0000 7f1c52d5b640 1 -- 172.21.15.99:0/1705001961 >> 172.21.15.99:0/1705001961 conn(0x7f1c4c0fbce0 msgr2=0x7f1c4c0fe100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:02:14.801 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.798+0000 7f1c52d5b640 1 -- 172.21.15.99:0/1705001961 shutdown_connections 2024-03-19T02:02:14.801 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.798+0000 7f1c52d5b640 1 -- 172.21.15.99:0/1705001961 wait complete. 2024-03-19T02:02:14.801 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.799+0000 7f1c52d5b640 1 Processor -- start 2024-03-19T02:02:14.801 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.799+0000 7f1c52d5b640 1 -- start start 2024-03-19T02:02:14.801 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.799+0000 7f1c52d5b640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1c4c100750 0x7f1c4c06ac30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:14.802 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.799+0000 7f1c52d5b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1c4c101940 0x7f1c4c19c800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:14.802 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.799+0000 7f1c52d5b640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1c4c102300 0x7f1c4c19ee40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:14.802 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.799+0000 7f1c52d5b640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f1c4c10c3c0 con 0x7f1c4c101940 2024-03-19T02:02:14.802 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.799+0000 7f1c52d5b640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f1c4c10c240 con 0x7f1c4c100750 2024-03-19T02:02:14.802 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.799+0000 7f1c52d5b640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f1c4c10c540 con 0x7f1c4c102300 2024-03-19T02:02:14.802 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.799+0000 7f1c4bfff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1c4c101940 0x7f1c4c19c800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:14.802 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.799+0000 7f1c4bfff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1c4c101940 0x7f1c4c19c800 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:36308/0 (socket says 172.21.15.99:36308) 2024-03-19T02:02:14.802 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.799+0000 7f1c4bfff640 1 -- 172.21.15.99:0/1086828522 learned_addr learned my addr 172.21.15.99:0/1086828522 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:02:14.802 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.799+0000 7f1c50af7640 1 --2- 172.21.15.99:0/1086828522 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1c4c100750 0x7f1c4c06ac30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:14.802 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.799+0000 7f1c512f8640 1 --2- 172.21.15.99:0/1086828522 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1c4c102300 0x7f1c4c19ee40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:14.802 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.800+0000 7f1c50af7640 1 -- 172.21.15.99:0/1086828522 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1c4c102300 msgr2=0x7f1c4c19ee40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:02:14.802 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.800+0000 7f1c50af7640 1 --2- 172.21.15.99:0/1086828522 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1c4c102300 0x7f1c4c19ee40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:14.802 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.800+0000 7f1c50af7640 1 -- 172.21.15.99:0/1086828522 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1c4c101940 msgr2=0x7f1c4c19c800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:02:14.802 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.800+0000 7f1c50af7640 1 --2- 172.21.15.99:0/1086828522 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1c4c101940 0x7f1c4c19c800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:14.803 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.800+0000 7f1c50af7640 1 -- 172.21.15.99:0/1086828522 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f1c4c19f700 con 0x7f1c4c100750 2024-03-19T02:02:14.803 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.800+0000 7f1c4bfff640 1 --2- 172.21.15.99:0/1086828522 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1c4c101940 0x7f1c4c19c800 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:02:14.803 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.800+0000 7f1c50af7640 1 --2- 172.21.15.99:0/1086828522 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1c4c100750 0x7f1c4c06ac30 secure :-1 s=READY pgs=212 cs=0 l=1 rev1=1 crypto rx=0x7f1c44037040 tx=0x7f1c44032ea0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:02:14.803 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.800+0000 7f1c49ffb640 1 -- 172.21.15.99:0/1086828522 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1c44041020 con 0x7f1c4c100750 2024-03-19T02:02:14.803 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.800+0000 7f1c52d5b640 1 -- 172.21.15.99:0/1086828522 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f1c4c19f990 con 0x7f1c4c100750 2024-03-19T02:02:14.803 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.800+0000 7f1c49ffb640 1 -- 172.21.15.99:0/1086828522 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f1c44034cd0 con 0x7f1c4c100750 2024-03-19T02:02:14.803 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.800+0000 7f1c49ffb640 1 -- 172.21.15.99:0/1086828522 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1c44039040 con 0x7f1c4c100750 2024-03-19T02:02:14.803 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.800+0000 7f1c52d5b640 1 -- 172.21.15.99:0/1086828522 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f1c4c19fea0 con 0x7f1c4c100750 2024-03-19T02:02:14.808 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.803+0000 7f1c49ffb640 1 -- 172.21.15.99:0/1086828522 <== mon.2 v2:172.21.15.112:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f1c44044020 con 0x7f1c4c100750 2024-03-19T02:02:14.808 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.803+0000 7f1c55a25740 1 Processor -- start 2024-03-19T02:02:14.808 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.803+0000 7f1c55a25740 1 -- start start 2024-03-19T02:02:14.808 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.803+0000 7f1c55a25740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564b4ac97100 0x564b4ac97520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:14.808 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.803+0000 7f1c55a25740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564b4ac97af0 0x564b4ac9c5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:14.808 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.803+0000 7f1c55a25740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564b4ac9cc60 0x564b4ac9f0f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:14.808 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.803+0000 7f1c49ffb640 1 --2- 172.21.15.99:0/1086828522 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f1c380a28a0 0x7f1c380a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:14.809 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.803+0000 7f1c55a25740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x564b4abfbbe0 con 0x564b4ac97100 2024-03-19T02:02:14.809 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.803+0000 7f1c55a25740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x564b4aba4960 con 0x564b4ac97af0 2024-03-19T02:02:14.809 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.803+0000 7f1c55a25740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x564b4abf7c80 con 0x564b4ac9cc60 2024-03-19T02:02:14.809 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.804+0000 7f1c4bfff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564b4ac9cc60 0x564b4ac9f0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:14.809 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.804+0000 7f1c512f8640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564b4ac97100 0x564b4ac97520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:14.809 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.804+0000 7f1c4bfff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564b4ac9cc60 0x564b4ac9f0f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:52072/0 (socket says 172.21.15.99:52072) 2024-03-19T02:02:14.809 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.804+0000 7f1c4bfff640 1 -- 172.21.15.99:0/2093420210 learned_addr learned my addr 172.21.15.99:0/2093420210 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:02:14.809 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.804+0000 7f1c50af7640 1 --2- 172.21.15.99:0/2093420210 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564b4ac97af0 0x564b4ac9c5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:14.809 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.804+0000 7f1c512f8640 1 --2- 172.21.15.99:0/1086828522 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f1c380a28a0 0x7f1c380a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:14.809 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.804+0000 7f1c49ffb640 1 -- 172.21.15.99:0/1086828522 <== mon.2 v2:172.21.15.112:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f1c440f0860 con 0x7f1c4c100750 2024-03-19T02:02:14.809 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.804+0000 7f1c4bfff640 1 -- 172.21.15.99:0/2093420210 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564b4ac97af0 msgr2=0x564b4ac9c5f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:02:14.809 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.804+0000 7f1c4bfff640 1 --2- 172.21.15.99:0/2093420210 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564b4ac97af0 0x564b4ac9c5f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:14.809 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.804+0000 7f1c4bfff640 1 -- 172.21.15.99:0/2093420210 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564b4ac97100 msgr2=0x564b4ac97520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:02:14.809 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.804+0000 7f1c4bfff640 1 --2- 172.21.15.99:0/2093420210 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564b4ac97100 0x564b4ac97520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:14.809 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.804+0000 7f1c4bfff640 1 -- 172.21.15.99:0/2093420210 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x564b4ab9bd20 con 0x564b4ac9cc60 2024-03-19T02:02:14.809 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.804+0000 7f1c4bfff640 1 --2- 172.21.15.99:0/2093420210 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564b4ac9cc60 0x564b4ac9f0f0 secure :-1 s=READY pgs=129 cs=0 l=1 rev1=1 crypto rx=0x7f1c340115f0 tx=0x7f1c34011ab0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=f906549639214894 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:02:14.810 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.804+0000 7f1c512f8640 1 --2- 172.21.15.99:0/1086828522 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f1c380a28a0 0x7f1c380a4d60 secure :-1 s=READY pgs=372 cs=0 l=1 rev1=1 crypto rx=0x7f1c3c009710 tx=0x7f1c3c005f90 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:02:14.810 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.804+0000 7f1c42ffd640 1 -- 172.21.15.99:0/2093420210 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1c3401e070 con 0x564b4ac9cc60 2024-03-19T02:02:14.810 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.804+0000 7f1c42ffd640 1 -- 172.21.15.99:0/2093420210 <== mon.1 v2:172.21.15.115:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f1c3400fba0 con 0x564b4ac9cc60 2024-03-19T02:02:14.810 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.804+0000 7f1c42ffd640 1 -- 172.21.15.99:0/2093420210 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1c34019c70 con 0x564b4ac9cc60 2024-03-19T02:02:14.810 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.805+0000 7f1c55a25740 1 -- 172.21.15.99:0/2093420210 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564b4ac9cc60 msgr2=0x564b4ac9f0f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:02:14.810 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.805+0000 7f1c55a25740 1 --2- 172.21.15.99:0/2093420210 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564b4ac9cc60 0x564b4ac9f0f0 secure :-1 s=READY pgs=129 cs=0 l=1 rev1=1 crypto rx=0x7f1c340115f0 tx=0x7f1c34011ab0 comp rx=0 tx=0).stop 2024-03-19T02:02:14.810 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.805+0000 7f1c55a25740 1 -- 172.21.15.99:0/2093420210 shutdown_connections 2024-03-19T02:02:14.810 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.805+0000 7f1c55a25740 1 --2- 172.21.15.99:0/2093420210 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564b4ac97af0 0x564b4ac9c5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:14.810 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.805+0000 7f1c55a25740 1 --2- 172.21.15.99:0/2093420210 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564b4ac9cc60 0x564b4ac9f0f0 unknown :-1 s=CLOSED pgs=129 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:14.810 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.805+0000 7f1c55a25740 1 --2- 172.21.15.99:0/2093420210 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564b4ac97100 0x564b4ac97520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:14.810 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.805+0000 7f1c55a25740 1 -- 172.21.15.99:0/2093420210 >> 172.21.15.99:0/2093420210 conn(0x564b4ac92670 msgr2=0x564b4ac92a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:02:14.810 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.805+0000 7f1c55a25740 1 -- 172.21.15.99:0/2093420210 shutdown_connections 2024-03-19T02:02:14.810 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.805+0000 7f1c55a25740 1 -- 172.21.15.99:0/2093420210 wait complete. 2024-03-19T02:02:14.810 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.805+0000 7f1c55a25740 1 -- start start 2024-03-19T02:02:14.810 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.806+0000 7f1c55a25740 1 Processor -- start 2024-03-19T02:02:14.811 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.806+0000 7f1c55a25740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564b4ac97100 0x564b4aca1190 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:14.811 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.806+0000 7f1c55a25740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564b4ac97af0 0x564b4aca16d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:14.811 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.806+0000 7f1c55a25740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564b4ac9cc60 0x564b4aca8710 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:14.811 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.806+0000 7f1c55a25740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x564b4ac9fc10 con 0x564b4ac97100 2024-03-19T02:02:14.811 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.806+0000 7f1c55a25740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x564b4abb7db0 con 0x564b4ac9cc60 2024-03-19T02:02:14.811 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.806+0000 7f1c55a25740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x564b4ac9fd90 con 0x564b4ac97af0 2024-03-19T02:02:14.811 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.806+0000 7f1c50af7640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564b4ac97100 0x564b4aca1190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:14.811 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.806+0000 7f1c55a25740 10 client.0 _ll_register_callbacks cb 0x564b4abbce70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T02:02:14.811 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.806+0000 7f1c50af7640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564b4ac97100 0x564b4aca1190 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:36318/0 (socket says 172.21.15.99:36318) 2024-03-19T02:02:14.811 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.806+0000 7f1c50af7640 1 -- 172.21.15.99:0/3542123338 learned_addr learned my addr 172.21.15.99:0/3542123338 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:02:14.811 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.806+0000 7f1c4bfff640 1 --2- 172.21.15.99:0/3542123338 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564b4ac97af0 0x564b4aca16d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:14.811 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.806+0000 7f1c512f8640 1 --2- 172.21.15.99:0/3542123338 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564b4ac9cc60 0x564b4aca8710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:14.811 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.806+0000 7f1c50af7640 1 -- 172.21.15.99:0/3542123338 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564b4ac97af0 msgr2=0x564b4aca16d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:02:14.811 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.806+0000 7f1c50af7640 1 --2- 172.21.15.99:0/3542123338 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564b4ac97af0 0x564b4aca16d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:14.811 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.806+0000 7f1c50af7640 1 -- 172.21.15.99:0/3542123338 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564b4ac9cc60 msgr2=0x564b4aca8710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:02:14.812 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.806+0000 7f1c50af7640 1 --2- 172.21.15.99:0/3542123338 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564b4ac9cc60 0x564b4aca8710 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:14.812 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.806+0000 7f1c50af7640 1 -- 172.21.15.99:0/3542123338 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x564b4abb6c60 con 0x564b4ac97100 2024-03-19T02:02:14.812 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.806+0000 7f1c512f8640 1 --2- 172.21.15.99:0/3542123338 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564b4ac9cc60 0x564b4aca8710 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:02:14.812 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.807+0000 7f1c50af7640 1 --2- 172.21.15.99:0/3542123338 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564b4ac97100 0x564b4aca1190 secure :-1 s=READY pgs=744 cs=0 l=1 rev1=1 crypto rx=0x7f1c44005e30 tx=0x7f1c44000f40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=db6fe0d62e683315 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:02:14.812 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.807+0000 7f1c2bfff640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T02:02:14.812 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.807+0000 7f1c2bfff640 1 -- 172.21.15.99:0/3542123338 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1c440bcb50 con 0x564b4ac97100 2024-03-19T02:02:14.812 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.807+0000 7f1c2bfff640 1 -- 172.21.15.99:0/3542123338 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f1c440ee3d0 con 0x564b4ac97100 2024-03-19T02:02:14.812 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.807+0000 7f1c55a25740 1 -- 172.21.15.99:0/3542123338 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f1c3400fba0 con 0x564b4ac97100 2024-03-19T02:02:14.812 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.807+0000 7f1c2bfff640 1 -- 172.21.15.99:0/3542123338 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7f1c44049120 con 0x564b4ac97100 2024-03-19T02:02:14.812 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.807+0000 7f1c55a25740 10 client.17028 fetch_fsmap learned FSMap version 37 2024-03-19T02:02:14.812 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.807+0000 7f1c55a25740 1 -- 172.21.15.99:0/3542123338 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x564b4a910210 con 0x564b4ac97100 2024-03-19T02:02:14.812 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.807+0000 7f1c2bfff640 1 -- 172.21.15.99:0/3542123338 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f1c44080020 con 0x564b4ac97100 2024-03-19T02:02:14.812 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.807+0000 7f1c2bfff640 1 -- 172.21.15.99:0/3542123338 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7f1c4403c030 con 0x564b4ac97100 2024-03-19T02:02:14.812 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.807+0000 7f1c55a25740 10 client.17028 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T02:02:14.812 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.807+0000 7f1c55a25740 10 client.17028 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T02:02:14.812 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.807+0000 7f1c55a25740 20 client.17028 populate_metadata read hostname 'smithi099' 2024-03-19T02:02:14.812 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.807+0000 7f1c55a25740 1 --2- 172.21.15.99:0/3542123338 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x564b4acad0b0 0x564b4acaf540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:14.813 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.807+0000 7f1c55a25740 4 client.17028 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T02:02:14.813 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.807+0000 7f1c55a25740 1 -- 172.21.15.99:0/3542123338 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x564b4ac09120 con 0x564b4acad0b0 2024-03-19T02:02:14.813 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.808+0000 7f1c4bfff640 1 --2- 172.21.15.99:0/3542123338 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x564b4acad0b0 0x564b4acaf540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:14.813 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.808+0000 7f1c2bfff640 10 client.17028 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T02:02:14.813 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.808+0000 7f1c4bfff640 1 --2- 172.21.15.99:0/3542123338 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x564b4acad0b0 0x564b4acaf540 crc :-1 s=READY pgs=217 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=9fd79eea8aa8a06f server_cookie=a8ce83f27939f5f1 in_seq=0 out_seq=0 2024-03-19T02:02:14.814 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.811+0000 7f1c2bfff640 1 -- 172.21.15.99:0/3542123338 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x564b4ac09120 con 0x564b4acad0b0 2024-03-19T02:02:14.814 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.811+0000 7f1c2bfff640 10 client.17028 handle_command_reply: tid=0 2024-03-19T02:02:14.814 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.811+0000 7f1c55a25740 1 client.17028 shutdown 2024-03-19T02:02:14.814 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.811+0000 7f1c55a25740 1 -- 172.21.15.99:0/3542123338 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564b4ac97100 msgr2=0x564b4aca1190 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:02:14.814 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.811+0000 7f1c55a25740 1 --2- 172.21.15.99:0/3542123338 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564b4ac97100 0x564b4aca1190 secure :-1 s=READY pgs=744 cs=0 l=1 rev1=1 crypto rx=0x7f1c44005e30 tx=0x7f1c44000f40 comp rx=0 tx=0).stop 2024-03-19T02:02:14.814 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.811+0000 7f1c55a25740 1 -- 172.21.15.99:0/3542123338 shutdown_connections 2024-03-19T02:02:14.814 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.811+0000 7f1c55a25740 1 --2- 172.21.15.99:0/3542123338 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x564b4acad0b0 0x564b4acaf540 crc :-1 s=READY pgs=217 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:14.814 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.811+0000 7f1c2bfff640 0 client.17028 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T02:02:14.815 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.811+0000 7f1c55a25740 1 --2- 172.21.15.99:0/3542123338 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564b4ac9cc60 0x564b4aca8710 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:14.815 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.811+0000 7f1c55a25740 1 --2- 172.21.15.99:0/3542123338 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564b4ac97af0 0x564b4aca16d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:14.815 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.811+0000 7f1c55a25740 1 --2- 172.21.15.99:0/3542123338 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564b4ac97100 0x564b4aca1190 unknown :-1 s=CLOSED pgs=744 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:14.815 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.811+0000 7f1c55a25740 1 -- 172.21.15.99:0/3542123338 >> 172.21.15.99:0/3542123338 conn(0x564b4ac92670 msgr2=0x564b4ac938c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:02:14.815 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.811+0000 7f1c55a25740 1 -- 172.21.15.99:0/3542123338 shutdown_connections 2024-03-19T02:02:14.815 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.811+0000 7f1c55a25740 1 -- 172.21.15.99:0/3542123338 wait complete. 2024-03-19T02:02:14.815 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.811+0000 7f1c55a25740 20 client.17028 trim_cache size 0 max 16384 2024-03-19T02:02:14.828 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.827+0000 7f1c55a25740 1 Processor -- start 2024-03-19T02:02:14.829 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.827+0000 7f1c55a25740 1 -- start start 2024-03-19T02:02:14.829 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.827+0000 7f1c55a25740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564b4ac97100 0x564b4aca0460 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:14.829 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.827+0000 7f1c55a25740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564b4ac97af0 0x564b4aca86a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:14.829 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.827+0000 7f1c55a25740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564b4ac9cc60 0x564b4aca09a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:14.829 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.827+0000 7f1c55a25740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f1c4c1ab570 con 0x564b4ac97af0 2024-03-19T02:02:14.829 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.827+0000 7f1c55a25740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f1c4c1a8e40 con 0x564b4ac9cc60 2024-03-19T02:02:14.829 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.827+0000 7f1c55a25740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f1c4c1a6c90 con 0x564b4ac97100 2024-03-19T02:02:14.829 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.827+0000 7f1c4bfff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564b4ac97af0 0x564b4aca86a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:14.829 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.827+0000 7f1c512f8640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564b4ac9cc60 0x564b4aca09a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:14.829 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.827+0000 7f1c4bfff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564b4ac97af0 0x564b4aca86a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:36326/0 (socket says 172.21.15.99:36326) 2024-03-19T02:02:14.829 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.827+0000 7f1c4bfff640 1 -- 172.21.15.99:0/3190220270 learned_addr learned my addr 172.21.15.99:0/3190220270 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:02:14.830 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.827+0000 7f1c50af7640 1 --2- 172.21.15.99:0/3190220270 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564b4ac97100 0x564b4aca0460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:14.830 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.827+0000 7f1c4bfff640 1 -- 172.21.15.99:0/3190220270 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564b4ac97100 msgr2=0x564b4aca0460 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:02:14.830 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.827+0000 7f1c4bfff640 1 --2- 172.21.15.99:0/3190220270 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564b4ac97100 0x564b4aca0460 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:14.830 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.827+0000 7f1c4bfff640 1 -- 172.21.15.99:0/3190220270 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564b4ac9cc60 msgr2=0x564b4aca09a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:02:14.830 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.827+0000 7f1c4bfff640 1 --2- 172.21.15.99:0/3190220270 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564b4ac9cc60 0x564b4aca09a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:14.830 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.827+0000 7f1c4bfff640 1 -- 172.21.15.99:0/3190220270 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x564b4a9dc620 con 0x564b4ac97af0 2024-03-19T02:02:14.830 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.827+0000 7f1c512f8640 1 --2- 172.21.15.99:0/3190220270 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564b4ac9cc60 0x564b4aca09a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:02:14.830 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.828+0000 7f1c4bfff640 1 --2- 172.21.15.99:0/3190220270 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564b4ac97af0 0x564b4aca86a0 secure :-1 s=READY pgs=745 cs=0 l=1 rev1=1 crypto rx=0x7f1c34011510 tx=0x7f1c34009dc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f061fd61a0ffeb64 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:02:14.830 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.828+0000 7f1c427fc640 1 -- 172.21.15.99:0/3190220270 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1c3402d020 con 0x564b4ac97af0 2024-03-19T02:02:14.830 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.828+0000 7f1c427fc640 1 -- 172.21.15.99:0/3190220270 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f1c34005040 con 0x564b4ac97af0 2024-03-19T02:02:14.830 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.828+0000 7f1c427fc640 1 -- 172.21.15.99:0/3190220270 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1c3401e040 con 0x564b4ac97af0 2024-03-19T02:02:14.830 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.828+0000 7f1c55a25740 1 -- 172.21.15.99:0/3190220270 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564b4ac97af0 msgr2=0x564b4aca86a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:02:14.830 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.829+0000 7f1c55a25740 1 --2- 172.21.15.99:0/3190220270 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564b4ac97af0 0x564b4aca86a0 secure :-1 s=READY pgs=745 cs=0 l=1 rev1=1 crypto rx=0x7f1c34011510 tx=0x7f1c34009dc0 comp rx=0 tx=0).stop 2024-03-19T02:02:14.831 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.829+0000 7f1c55a25740 1 -- 172.21.15.99:0/3190220270 shutdown_connections 2024-03-19T02:02:14.831 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.829+0000 7f1c55a25740 1 --2- 172.21.15.99:0/3190220270 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564b4ac9cc60 0x564b4aca09a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:14.831 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.829+0000 7f1c55a25740 1 --2- 172.21.15.99:0/3190220270 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564b4ac97af0 0x564b4aca86a0 unknown :-1 s=CLOSED pgs=745 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:14.831 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.829+0000 7f1c55a25740 1 --2- 172.21.15.99:0/3190220270 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564b4ac97100 0x564b4aca0460 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:14.831 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.829+0000 7f1c55a25740 1 -- 172.21.15.99:0/3190220270 >> 172.21.15.99:0/3190220270 conn(0x564b4ac92670 msgr2=0x564b4ac96160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:02:14.831 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.829+0000 7f1c55a25740 1 -- 172.21.15.99:0/3190220270 shutdown_connections 2024-03-19T02:02:14.831 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.829+0000 7f1c55a25740 1 -- 172.21.15.99:0/3190220270 wait complete. 2024-03-19T02:02:14.831 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.829+0000 7f1c55a25740 1 -- start start 2024-03-19T02:02:14.831 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.829+0000 7f1c55a25740 1 Processor -- start 2024-03-19T02:02:14.834 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.829+0000 7f1c55a25740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564b4ac97100 0x564b4aca6000 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:14.834 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.829+0000 7f1c55a25740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564b4ac97af0 0x564b4aca6540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:14.834 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.829+0000 7f1c55a25740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564b4ac9cc60 0x564b4aca7550 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:14.834 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.829+0000 7f1c55a25740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f1c4c1a7300 con 0x564b4ac97af0 2024-03-19T02:02:14.834 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.829+0000 7f1c55a25740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f1c4c1a8770 con 0x564b4ac97100 2024-03-19T02:02:14.834 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.829+0000 7f1c55a25740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f1c4c1a79d0 con 0x564b4ac9cc60 2024-03-19T02:02:14.834 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.829+0000 7f1c55a25740 10 client.0 _ll_register_callbacks cb 0x7f1c440b9490 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T02:02:14.834 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.829+0000 7f1c4bfff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564b4ac97af0 0x564b4aca6540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:14.834 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.830+0000 7f1c4bfff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564b4ac97af0 0x564b4aca6540 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:36342/0 (socket says 172.21.15.99:36342) 2024-03-19T02:02:14.834 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.830+0000 7f1c50af7640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564b4ac97100 0x564b4aca6000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:14.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.830+0000 7f1c4bfff640 1 -- 172.21.15.99:0/3135173218 learned_addr learned my addr 172.21.15.99:0/3135173218 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:02:14.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.830+0000 7f1c512f8640 1 --2- 172.21.15.99:0/3135173218 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564b4ac9cc60 0x564b4aca7550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:14.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.830+0000 7f1c4bfff640 1 -- 172.21.15.99:0/3135173218 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564b4ac9cc60 msgr2=0x564b4aca7550 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:02:14.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.830+0000 7f1c4bfff640 1 --2- 172.21.15.99:0/3135173218 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564b4ac9cc60 0x564b4aca7550 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:14.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.830+0000 7f1c4bfff640 1 -- 172.21.15.99:0/3135173218 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564b4ac97100 msgr2=0x564b4aca6000 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:02:14.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.830+0000 7f1c4bfff640 1 --2- 172.21.15.99:0/3135173218 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564b4ac97100 0x564b4aca6000 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:14.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.830+0000 7f1c4bfff640 1 -- 172.21.15.99:0/3135173218 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x564b4aca7b40 con 0x564b4ac97af0 2024-03-19T02:02:14.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.830+0000 7f1c50af7640 1 --2- 172.21.15.99:0/3135173218 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564b4ac97100 0x564b4aca6000 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:02:14.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.830+0000 7f1c4bfff640 1 --2- 172.21.15.99:0/3135173218 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564b4ac97af0 0x564b4aca6540 secure :-1 s=READY pgs=746 cs=0 l=1 rev1=1 crypto rx=0x7f1c34009250 tx=0x7f1c340095c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5d750384e4a179e8 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:02:14.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.830+0000 7f1c2bfff640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T02:02:14.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.830+0000 7f1c2bfff640 1 -- 172.21.15.99:0/3135173218 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1c3402d030 con 0x564b4ac97af0 2024-03-19T02:02:14.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.830+0000 7f1c2bfff640 1 -- 172.21.15.99:0/3135173218 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f1c3400f800 con 0x564b4ac97af0 2024-03-19T02:02:14.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.830+0000 7f1c55a25740 1 -- 172.21.15.99:0/3135173218 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f1c34005040 con 0x564b4ac97af0 2024-03-19T02:02:14.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.830+0000 7f1c2bfff640 1 -- 172.21.15.99:0/3135173218 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7f1c34043050 con 0x564b4ac97af0 2024-03-19T02:02:14.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.830+0000 7f1c55a25740 10 client.17034 fetch_fsmap learned FSMap version 37 2024-03-19T02:02:14.835 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.830+0000 7f1c55a25740 1 -- 172.21.15.99:0/3135173218 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x564b4ad136f0 con 0x564b4ac97af0 2024-03-19T02:02:14.836 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.831+0000 7f1c2bfff640 1 -- 172.21.15.99:0/3135173218 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f1c34005040 con 0x564b4ac97af0 2024-03-19T02:02:14.836 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.831+0000 7f1c2bfff640 1 -- 172.21.15.99:0/3135173218 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7f1c34004680 con 0x564b4ac97af0 2024-03-19T02:02:14.836 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.831+0000 7f1c55a25740 10 client.17034 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T02:02:14.836 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.831+0000 7f1c55a25740 10 client.17034 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T02:02:14.836 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.831+0000 7f1c55a25740 20 client.17034 populate_metadata read hostname 'smithi099' 2024-03-19T02:02:14.836 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.831+0000 7f1c55a25740 1 --2- 172.21.15.99:0/3135173218 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x564b4ad13ec0 0x564b4ad162e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:14.836 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.831+0000 7f1c55a25740 4 client.17034 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T02:02:14.836 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.831+0000 7f1c55a25740 1 -- 172.21.15.99:0/3135173218 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x564b4ac03470 con 0x564b4ad13ec0 2024-03-19T02:02:14.836 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.831+0000 7f1c50af7640 1 --2- 172.21.15.99:0/3135173218 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x564b4ad13ec0 0x564b4ad162e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:14.836 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.831+0000 7f1c2bfff640 10 client.17034 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T02:02:14.836 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:14.831+0000 7f1c50af7640 1 --2- 172.21.15.99:0/3135173218 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x564b4ad13ec0 0x564b4ad162e0 crc :-1 s=READY pgs=218 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=a27467378679f05f server_cookie=1c7a181fb5a3b81d in_seq=0 out_seq=0 2024-03-19T02:02:15.116 INFO:tasks.workunit.client.0.smithi099.stdout: 10 223571 22.54 MB/sec execute 404 sec latency 272.102 ms 2024-03-19T02:02:15.516 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:15.515+0000 7f1c2bfff640 1 -- 172.21.15.99:0/3135173218 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x564b4ac03470 con 0x564b4ad13ec0 2024-03-19T02:02:15.516 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:15.515+0000 7f1c2bfff640 10 client.17034 handle_command_reply: tid=0 2024-03-19T02:02:15.516 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:15.515+0000 7f1c55a25740 1 client.17034 shutdown 2024-03-19T02:02:15.516 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:15.515+0000 7f1c55a25740 1 -- 172.21.15.99:0/3135173218 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564b4ac97af0 msgr2=0x564b4aca6540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:02:15.516 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:15.515+0000 7f1c55a25740 1 --2- 172.21.15.99:0/3135173218 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564b4ac97af0 0x564b4aca6540 secure :-1 s=READY pgs=746 cs=0 l=1 rev1=1 crypto rx=0x7f1c34009250 tx=0x7f1c340095c0 comp rx=0 tx=0).stop 2024-03-19T02:02:15.516 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:15.515+0000 7f1c55a25740 1 -- 172.21.15.99:0/3135173218 shutdown_connections 2024-03-19T02:02:15.516 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:15.515+0000 7f1c55a25740 1 --2- 172.21.15.99:0/3135173218 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x564b4ad13ec0 0x564b4ad162e0 crc :-1 s=READY pgs=218 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:15.517 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:15.515+0000 7f1c2bfff640 0 client.17034 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T02:02:15.517 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:15.515+0000 7f1c55a25740 1 --2- 172.21.15.99:0/3135173218 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x564b4ac9cc60 0x564b4aca7550 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:15.517 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:15.515+0000 7f1c55a25740 1 --2- 172.21.15.99:0/3135173218 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x564b4ac97af0 0x564b4aca6540 unknown :-1 s=CLOSED pgs=746 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:15.517 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:15.515+0000 7f1c55a25740 1 --2- 172.21.15.99:0/3135173218 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x564b4ac97100 0x564b4aca6000 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:15.517 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:15.515+0000 7f1c55a25740 1 -- 172.21.15.99:0/3135173218 >> 172.21.15.99:0/3135173218 conn(0x564b4ac92670 msgr2=0x564b4ac98380 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:02:15.517 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:15.516+0000 7f1c55a25740 1 -- 172.21.15.99:0/3135173218 shutdown_connections 2024-03-19T02:02:15.517 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:15.516+0000 7f1c55a25740 1 -- 172.21.15.99:0/3135173218 wait complete. 2024-03-19T02:02:15.517 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:15.516+0000 7f1c55a25740 20 client.17034 trim_cache size 0 max 16384 2024-03-19T02:02:15.517 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T02:02:15.517 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T02:02:15.517 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T02:02:15.518 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T02:02:15.518 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:15.516+0000 7f1c52d5b640 1 -- 172.21.15.99:0/1086828522 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f1c380a28a0 msgr2=0x7f1c380a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:02:15.518 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:15.516+0000 7f1c52d5b640 1 --2- 172.21.15.99:0/1086828522 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f1c380a28a0 0x7f1c380a4d60 secure :-1 s=READY pgs=372 cs=0 l=1 rev1=1 crypto rx=0x7f1c3c009710 tx=0x7f1c3c005f90 comp rx=0 tx=0).stop 2024-03-19T02:02:15.519 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:15.516+0000 7f1c52d5b640 1 -- 172.21.15.99:0/1086828522 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1c4c100750 msgr2=0x7f1c4c06ac30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:02:15.519 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:15.516+0000 7f1c52d5b640 1 --2- 172.21.15.99:0/1086828522 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1c4c100750 0x7f1c4c06ac30 secure :-1 s=READY pgs=212 cs=0 l=1 rev1=1 crypto rx=0x7f1c44037040 tx=0x7f1c44032ea0 comp rx=0 tx=0).stop 2024-03-19T02:02:15.519 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:15.517+0000 7f1c52d5b640 1 -- 172.21.15.99:0/1086828522 shutdown_connections 2024-03-19T02:02:15.519 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:15.517+0000 7f1c52d5b640 1 --2- 172.21.15.99:0/1086828522 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1c4c102300 0x7f1c4c19ee40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:15.519 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:15.517+0000 7f1c52d5b640 1 --2- 172.21.15.99:0/1086828522 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1c4c101940 0x7f1c4c19c800 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:15.519 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:15.517+0000 7f1c52d5b640 1 --2- 172.21.15.99:0/1086828522 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f1c380a28a0 0x7f1c380a4d60 unknown :-1 s=CLOSED pgs=372 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:15.519 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:15.517+0000 7f1c52d5b640 1 --2- 172.21.15.99:0/1086828522 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1c4c100750 0x7f1c4c06ac30 unknown :-1 s=CLOSED pgs=212 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:15.519 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:15.517+0000 7f1c52d5b640 1 -- 172.21.15.99:0/1086828522 >> 172.21.15.99:0/1086828522 conn(0x7f1c4c0fbce0 msgr2=0x7f1c4c104c40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:02:15.519 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:15.517+0000 7f1c52d5b640 1 -- 172.21.15.99:0/1086828522 shutdown_connections 2024-03-19T02:02:15.519 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:15.517+0000 7f1c52d5b640 1 -- 172.21.15.99:0/1086828522 wait complete. 2024-03-19T02:02:16.116 INFO:tasks.workunit.client.0.smithi099.stdout: 10 223726 22.49 MB/sec execute 405 sec latency 723.064 ms 2024-03-19T02:02:16.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:02:16 smithi115 ceph-mon[67332]: pgmap v986: 97 pgs: 97 active+clean; 643 MiB data, 8.4 GiB used, 1.0 TiB / 1.0 TiB avail; 3.2 MiB/s rd, 4.7 MiB/s wr, 393 op/s 2024-03-19T02:02:16.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:02:16 smithi099 ceph-mon[62951]: pgmap v986: 97 pgs: 97 active+clean; 643 MiB data, 8.4 GiB used, 1.0 TiB / 1.0 TiB avail; 3.2 MiB/s rd, 4.7 MiB/s wr, 393 op/s 2024-03-19T02:02:16.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:02:16 smithi112 ceph-mon[66819]: pgmap v986: 97 pgs: 97 active+clean; 643 MiB data, 8.4 GiB used, 1.0 TiB / 1.0 TiB avail; 3.2 MiB/s rd, 4.7 MiB/s wr, 393 op/s 2024-03-19T02:02:17.116 INFO:tasks.workunit.client.0.smithi099.stdout: 10 224268 22.52 MB/sec execute 406 sec latency 63.943 ms 2024-03-19T02:02:18.116 INFO:tasks.workunit.client.0.smithi099.stdout: 10 224853 22.57 MB/sec execute 407 sec latency 76.289 ms 2024-03-19T02:02:18.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:02:18 smithi115 ceph-mon[67332]: pgmap v987: 97 pgs: 97 active+clean; 642 MiB data, 8.4 GiB used, 1.0 TiB / 1.0 TiB avail; 4.4 MiB/s rd, 6.1 MiB/s wr, 558 op/s 2024-03-19T02:02:18.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:02:18 smithi099 ceph-mon[62951]: pgmap v987: 97 pgs: 97 active+clean; 642 MiB data, 8.4 GiB used, 1.0 TiB / 1.0 TiB avail; 4.4 MiB/s rd, 6.1 MiB/s wr, 558 op/s 2024-03-19T02:02:18.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:02:18 smithi112 ceph-mon[66819]: pgmap v987: 97 pgs: 97 active+clean; 642 MiB data, 8.4 GiB used, 1.0 TiB / 1.0 TiB avail; 4.4 MiB/s rd, 6.1 MiB/s wr, 558 op/s 2024-03-19T02:02:19.116 INFO:tasks.workunit.client.0.smithi099.stdout: 10 225278 22.56 MB/sec execute 408 sec latency 54.056 ms 2024-03-19T02:02:20.116 INFO:tasks.workunit.client.0.smithi099.stdout: 10 225611 22.56 MB/sec execute 409 sec latency 83.974 ms 2024-03-19T02:02:20.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:02:20 smithi115 ceph-mon[67332]: pgmap v988: 97 pgs: 97 active+clean; 647 MiB data, 8.4 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 5.7 MiB/s wr, 505 op/s 2024-03-19T02:02:20.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:02:20 smithi099 ceph-mon[62951]: pgmap v988: 97 pgs: 97 active+clean; 647 MiB data, 8.4 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 5.7 MiB/s wr, 505 op/s 2024-03-19T02:02:20.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:02:20 smithi112 ceph-mon[66819]: pgmap v988: 97 pgs: 97 active+clean; 647 MiB data, 8.4 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 5.7 MiB/s wr, 505 op/s 2024-03-19T02:02:21.116 INFO:tasks.workunit.client.0.smithi099.stdout: 10 225817 22.52 MB/sec execute 410 sec latency 302.052 ms 2024-03-19T02:02:22.116 INFO:tasks.workunit.client.0.smithi099.stdout: 10 226138 22.50 MB/sec execute 411 sec latency 105.991 ms 2024-03-19T02:02:22.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:02:22 smithi115 ceph-mon[67332]: pgmap v989: 97 pgs: 97 active+clean; 647 MiB data, 8.4 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 6.5 MiB/s wr, 573 op/s 2024-03-19T02:02:22.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:02:22 smithi099 ceph-mon[62951]: pgmap v989: 97 pgs: 97 active+clean; 647 MiB data, 8.4 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 6.5 MiB/s wr, 573 op/s 2024-03-19T02:02:22.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:02:22 smithi112 ceph-mon[66819]: pgmap v989: 97 pgs: 97 active+clean; 647 MiB data, 8.4 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 6.5 MiB/s wr, 573 op/s 2024-03-19T02:02:23.116 INFO:tasks.workunit.client.0.smithi099.stdout: 10 226581 22.50 MB/sec execute 412 sec latency 119.030 ms 2024-03-19T02:02:24.116 INFO:tasks.workunit.client.0.smithi099.stdout: 10 227164 22.52 MB/sec execute 413 sec latency 58.126 ms 2024-03-19T02:02:24.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:02:24 smithi115 ceph-mon[67332]: pgmap v990: 97 pgs: 97 active+clean; 649 MiB data, 8.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.6 MiB/s rd, 6.6 MiB/s wr, 574 op/s 2024-03-19T02:02:24.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:02:24 smithi099 ceph-mon[62951]: pgmap v990: 97 pgs: 97 active+clean; 649 MiB data, 8.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.6 MiB/s rd, 6.6 MiB/s wr, 574 op/s 2024-03-19T02:02:24.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:02:24 smithi112 ceph-mon[66819]: pgmap v990: 97 pgs: 97 active+clean; 649 MiB data, 8.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.6 MiB/s rd, 6.6 MiB/s wr, 574 op/s 2024-03-19T02:02:25.116 INFO:tasks.workunit.client.0.smithi099.stdout: 10 227610 22.53 MB/sec execute 414 sec latency 56.272 ms 2024-03-19T02:02:26.116 INFO:tasks.workunit.client.0.smithi099.stdout: 10 227910 22.50 MB/sec execute 415 sec latency 344.487 ms 2024-03-19T02:02:26.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:02:26 smithi115 ceph-mon[67332]: pgmap v991: 97 pgs: 97 active+clean; 649 MiB data, 8.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 5.2 MiB/s wr, 513 op/s 2024-03-19T02:02:26.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:02:26 smithi099 ceph-mon[62951]: pgmap v991: 97 pgs: 97 active+clean; 649 MiB data, 8.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 5.2 MiB/s wr, 513 op/s 2024-03-19T02:02:26.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:02:26 smithi112 ceph-mon[66819]: pgmap v991: 97 pgs: 97 active+clean; 649 MiB data, 8.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 5.2 MiB/s wr, 513 op/s 2024-03-19T02:02:27.116 INFO:tasks.workunit.client.0.smithi099.stdout: 10 228428 22.51 MB/sec execute 416 sec latency 71.757 ms 2024-03-19T02:02:28.116 INFO:tasks.workunit.client.0.smithi099.stdout: 10 228900 22.52 MB/sec execute 417 sec latency 73.843 ms 2024-03-19T02:02:28.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:02:28 smithi115 ceph-mon[67332]: pgmap v992: 97 pgs: 97 active+clean; 659 MiB data, 8.5 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 MiB/s rd, 7.4 MiB/s wr, 640 op/s 2024-03-19T02:02:28.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:02:28 smithi099 ceph-mon[62951]: pgmap v992: 97 pgs: 97 active+clean; 659 MiB data, 8.5 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 MiB/s rd, 7.4 MiB/s wr, 640 op/s 2024-03-19T02:02:28.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:02:28 smithi112 ceph-mon[66819]: pgmap v992: 97 pgs: 97 active+clean; 659 MiB data, 8.5 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 MiB/s rd, 7.4 MiB/s wr, 640 op/s 2024-03-19T02:02:28.620 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.618+0000 7f9fb3352640 1 -- 172.21.15.99:0/3219919574 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9fac1022f0 msgr2=0x7f9fac108b80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:02:28.620 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.618+0000 7f9fb3352640 1 --2- 172.21.15.99:0/3219919574 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9fac1022f0 0x7f9fac108b80 secure :-1 s=READY pgs=747 cs=0 l=1 rev1=1 crypto rx=0x7f9fa4009ca0 tx=0x7f9fa402c0f0 comp rx=0 tx=0).stop 2024-03-19T02:02:28.620 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.618+0000 7f9fb3352640 1 -- 172.21.15.99:0/3219919574 shutdown_connections 2024-03-19T02:02:28.620 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.618+0000 7f9fb3352640 1 --2- 172.21.15.99:0/3219919574 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9fac1022f0 0x7f9fac108b80 unknown :-1 s=CLOSED pgs=747 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:28.620 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.618+0000 7f9fb3352640 1 --2- 172.21.15.99:0/3219919574 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9fac101930 0x7f9fac101db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:28.621 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.618+0000 7f9fb3352640 1 --2- 172.21.15.99:0/3219919574 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9fac100740 0x7f9fac100b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:28.621 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.618+0000 7f9fb3352640 1 -- 172.21.15.99:0/3219919574 >> 172.21.15.99:0/3219919574 conn(0x7f9fac0fbcb0 msgr2=0x7f9fac0fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:02:28.621 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.618+0000 7f9fb3352640 1 -- 172.21.15.99:0/3219919574 shutdown_connections 2024-03-19T02:02:28.621 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.618+0000 7f9fb3352640 1 -- 172.21.15.99:0/3219919574 wait complete. 2024-03-19T02:02:28.621 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.619+0000 7f9fb3352640 1 Processor -- start 2024-03-19T02:02:28.621 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.619+0000 7f9fb3352640 1 -- start start 2024-03-19T02:02:28.621 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.619+0000 7f9fb3352640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9fac100740 0x7f9fac069a30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:28.621 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.619+0000 7f9fb3352640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9fac101930 0x7f9fac19c750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:28.621 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.619+0000 7f9fb3352640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9fac1022f0 0x7f9fac19ed90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:28.621 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.619+0000 7f9fb3352640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f9fac10c3a0 con 0x7f9fac100740 2024-03-19T02:02:28.621 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.619+0000 7f9fb3352640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f9fac10c220 con 0x7f9fac1022f0 2024-03-19T02:02:28.621 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.619+0000 7f9fb3352640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f9fac10c520 con 0x7f9fac101930 2024-03-19T02:02:28.621 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.619+0000 7f9fb10ee640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9fac100740 0x7f9fac069a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:28.621 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.619+0000 7f9fb10ee640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9fac100740 0x7f9fac069a30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:38660/0 (socket says 172.21.15.99:38660) 2024-03-19T02:02:28.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.619+0000 7f9fb10ee640 1 -- 172.21.15.99:0/296009616 learned_addr learned my addr 172.21.15.99:0/296009616 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:02:28.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.619+0000 7f9fb18ef640 1 --2- 172.21.15.99:0/296009616 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9fac1022f0 0x7f9fac19ed90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:28.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.619+0000 7f9fb08ed640 1 --2- 172.21.15.99:0/296009616 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9fac101930 0x7f9fac19c750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:28.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.620+0000 7f9fb10ee640 1 -- 172.21.15.99:0/296009616 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9fac101930 msgr2=0x7f9fac19c750 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:02:28.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.620+0000 7f9fb10ee640 1 --2- 172.21.15.99:0/296009616 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9fac101930 0x7f9fac19c750 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:28.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.620+0000 7f9fb10ee640 1 -- 172.21.15.99:0/296009616 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9fac1022f0 msgr2=0x7f9fac19ed90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:02:28.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.620+0000 7f9fb10ee640 1 --2- 172.21.15.99:0/296009616 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9fac1022f0 0x7f9fac19ed90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:28.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.620+0000 7f9fb10ee640 1 -- 172.21.15.99:0/296009616 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f9fac19f650 con 0x7f9fac100740 2024-03-19T02:02:28.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.620+0000 7f9fb10ee640 1 --2- 172.21.15.99:0/296009616 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9fac100740 0x7f9fac069a30 secure :-1 s=READY pgs=748 cs=0 l=1 rev1=1 crypto rx=0x7f9fa800d950 tx=0x7f9fa800de10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:02:28.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.620+0000 7f9fa27fc640 1 -- 172.21.15.99:0/296009616 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9fa8017020 con 0x7f9fac100740 2024-03-19T02:02:28.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.620+0000 7f9fa27fc640 1 -- 172.21.15.99:0/296009616 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f9fa8009c30 con 0x7f9fac100740 2024-03-19T02:02:28.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.620+0000 7f9fb3352640 1 -- 172.21.15.99:0/296009616 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f9fac19f940 con 0x7f9fac100740 2024-03-19T02:02:28.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.620+0000 7f9fa27fc640 1 -- 172.21.15.99:0/296009616 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9fa800f740 con 0x7f9fac100740 2024-03-19T02:02:28.623 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.620+0000 7f9fb3352640 1 -- 172.21.15.99:0/296009616 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f9fac19fe90 con 0x7f9fac100740 2024-03-19T02:02:28.625 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.623+0000 7f9fa27fc640 1 -- 172.21.15.99:0/296009616 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f9fa801a020 con 0x7f9fac100740 2024-03-19T02:02:28.625 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.623+0000 7f9fb600e740 1 Processor -- start 2024-03-19T02:02:28.626 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.623+0000 7f9fb600e740 1 -- start start 2024-03-19T02:02:28.626 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.623+0000 7f9fb600e740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a29a145100 0x55a29a145520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:28.626 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.623+0000 7f9fb600e740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a29a145af0 0x55a29a14a5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:28.626 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.623+0000 7f9fb600e740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a29a14ac60 0x55a29a14d0f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:28.626 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.623+0000 7f9fb600e740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55a29a0a9be0 con 0x55a29a145100 2024-03-19T02:02:28.626 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.623+0000 7f9fb600e740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55a29a052960 con 0x55a29a14ac60 2024-03-19T02:02:28.626 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.623+0000 7f9fb600e740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55a29a0a5c80 con 0x55a29a145af0 2024-03-19T02:02:28.626 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.623+0000 7f9fb18ef640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a29a145100 0x55a29a145520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:28.626 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.623+0000 7f9fb18ef640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a29a145100 0x55a29a145520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:38668/0 (socket says 172.21.15.99:38668) 2024-03-19T02:02:28.626 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.623+0000 7f9fb18ef640 1 -- 172.21.15.99:0/276907634 learned_addr learned my addr 172.21.15.99:0/276907634 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:02:28.626 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.623+0000 7f9fb08ed640 1 --2- 172.21.15.99:0/276907634 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a29a14ac60 0x55a29a14d0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:28.626 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.623+0000 7f9fa27fc640 1 --2- 172.21.15.99:0/296009616 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f9f980a28a0 0x7f9f980a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:28.626 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.624+0000 7f9fb08ed640 1 -- 172.21.15.99:0/276907634 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a29a145af0 msgr2=0x55a29a14a5f0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-03-19T02:02:28.626 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.624+0000 7f9fb08ed640 1 --2- 172.21.15.99:0/276907634 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a29a145af0 0x55a29a14a5f0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:28.627 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.624+0000 7f9fb08ed640 1 -- 172.21.15.99:0/276907634 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a29a145100 msgr2=0x55a29a145520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:02:28.627 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.624+0000 7f9fb08ed640 1 --2- 172.21.15.99:0/276907634 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a29a145100 0x55a29a145520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:28.627 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.624+0000 7f9fb08ed640 1 -- 172.21.15.99:0/276907634 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55a29a049d20 con 0x55a29a14ac60 2024-03-19T02:02:28.627 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.624+0000 7f9fa27fc640 1 -- 172.21.15.99:0/296009616 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f9fa80ce440 con 0x7f9fac100740 2024-03-19T02:02:28.627 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.624+0000 7f9fb18ef640 1 --2- 172.21.15.99:0/296009616 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f9f980a28a0 0x7f9f980a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:28.627 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.624+0000 7f9fb08ed640 1 --2- 172.21.15.99:0/276907634 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a29a14ac60 0x55a29a14d0f0 secure :-1 s=READY pgs=213 cs=0 l=1 rev1=1 crypto rx=0x7f9f9c005850 tx=0x7f9f9c005880 comp rx=0 tx=0).ready entity=mon.2 client_cookie=85cdda8084533f70 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:02:28.627 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.624+0000 7f9f977fe640 1 -- 172.21.15.99:0/276907634 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9f9c004050 con 0x55a29a14ac60 2024-03-19T02:02:28.627 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.624+0000 7f9f977fe640 1 -- 172.21.15.99:0/276907634 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f9f9c01a070 con 0x55a29a14ac60 2024-03-19T02:02:28.627 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.624+0000 7f9f977fe640 1 -- 172.21.15.99:0/276907634 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9f9c004b40 con 0x55a29a14ac60 2024-03-19T02:02:28.627 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.625+0000 7f9fb600e740 1 -- 172.21.15.99:0/276907634 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a29a14ac60 msgr2=0x55a29a14d0f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:02:28.627 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.625+0000 7f9fb18ef640 1 --2- 172.21.15.99:0/296009616 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f9f980a28a0 0x7f9f980a4d60 secure :-1 s=READY pgs=373 cs=0 l=1 rev1=1 crypto rx=0x7f9fa4006010 tx=0x7f9fa402e040 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:02:28.627 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.625+0000 7f9fb600e740 1 --2- 172.21.15.99:0/276907634 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a29a14ac60 0x55a29a14d0f0 secure :-1 s=READY pgs=213 cs=0 l=1 rev1=1 crypto rx=0x7f9f9c005850 tx=0x7f9f9c005880 comp rx=0 tx=0).stop 2024-03-19T02:02:28.627 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.625+0000 7f9fb600e740 1 -- 172.21.15.99:0/276907634 shutdown_connections 2024-03-19T02:02:28.627 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.625+0000 7f9fb600e740 1 --2- 172.21.15.99:0/276907634 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a29a14ac60 0x55a29a14d0f0 unknown :-1 s=CLOSED pgs=213 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:28.628 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.625+0000 7f9fb600e740 1 --2- 172.21.15.99:0/276907634 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a29a145af0 0x55a29a14a5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:28.628 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.625+0000 7f9fb600e740 1 --2- 172.21.15.99:0/276907634 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a29a145100 0x55a29a145520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:28.628 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.625+0000 7f9fb600e740 1 -- 172.21.15.99:0/276907634 >> 172.21.15.99:0/276907634 conn(0x55a29a140670 msgr2=0x55a29a140a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:02:28.628 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.625+0000 7f9fb600e740 1 -- 172.21.15.99:0/276907634 shutdown_connections 2024-03-19T02:02:28.628 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.625+0000 7f9fb600e740 1 -- 172.21.15.99:0/276907634 wait complete. 2024-03-19T02:02:28.628 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.626+0000 7f9fb600e740 1 -- start start 2024-03-19T02:02:28.628 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.626+0000 7f9fb600e740 1 Processor -- start 2024-03-19T02:02:28.632 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.626+0000 7f9fb600e740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a29a145100 0x55a29a14f0e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:28.632 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.626+0000 7f9fb600e740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a29a145af0 0x55a29a14f620 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:28.632 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.626+0000 7f9fb600e740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a29a14ac60 0x55a29a156660 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:28.632 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.626+0000 7f9fb600e740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55a29a14dc10 con 0x55a29a14ac60 2024-03-19T02:02:28.632 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.626+0000 7f9fb600e740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55a29a065db0 con 0x55a29a145100 2024-03-19T02:02:28.632 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.626+0000 7f9fb600e740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55a29a14dd90 con 0x55a29a145af0 2024-03-19T02:02:28.632 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.626+0000 7f9fb600e740 10 client.0 _ll_register_callbacks cb 0x55a29a06ae70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T02:02:28.632 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.626+0000 7f9fb18ef640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a29a14ac60 0x55a29a156660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:28.632 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.626+0000 7f9fb10ee640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a29a145100 0x55a29a14f0e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:28.632 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.626+0000 7f9fb10ee640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a29a145100 0x55a29a14f0e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:57000/0 (socket says 172.21.15.99:57000) 2024-03-19T02:02:28.632 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.626+0000 7f9fb18ef640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a29a14ac60 0x55a29a156660 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:38684/0 (socket says 172.21.15.99:38684) 2024-03-19T02:02:28.633 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.626+0000 7f9fb10ee640 1 -- 172.21.15.99:0/1589177355 learned_addr learned my addr 172.21.15.99:0/1589177355 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:02:28.633 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.626+0000 7f9fb08ed640 1 --2- 172.21.15.99:0/1589177355 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a29a145af0 0x55a29a14f620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:28.633 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.627+0000 7f9fb10ee640 1 -- 172.21.15.99:0/1589177355 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a29a145af0 msgr2=0x55a29a14f620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:02:28.633 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.627+0000 7f9fb10ee640 1 --2- 172.21.15.99:0/1589177355 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a29a145af0 0x55a29a14f620 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:28.633 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.627+0000 7f9fb10ee640 1 -- 172.21.15.99:0/1589177355 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a29a14ac60 msgr2=0x55a29a156660 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:02:28.633 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.627+0000 7f9fb10ee640 1 --2- 172.21.15.99:0/1589177355 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a29a14ac60 0x55a29a156660 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:28.633 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.627+0000 7f9fb10ee640 1 -- 172.21.15.99:0/1589177355 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55a29a064c60 con 0x55a29a145100 2024-03-19T02:02:28.633 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.627+0000 7f9fb10ee640 1 --2- 172.21.15.99:0/1589177355 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a29a145100 0x55a29a14f0e0 secure :-1 s=READY pgs=214 cs=0 l=1 rev1=1 crypto rx=0x7f9fa8097200 tx=0x7f9fa8009f20 comp rx=0 tx=0).ready entity=mon.2 client_cookie=1d8ac2448b85d6e7 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:02:28.633 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.627+0000 7f9f94ff9640 10 client.0 ms_handle_connect on v2:172.21.15.112:3300/0 2024-03-19T02:02:28.633 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.627+0000 7f9f94ff9640 1 -- 172.21.15.99:0/1589177355 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9fa80b3e60 con 0x55a29a145100 2024-03-19T02:02:28.633 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.627+0000 7f9f94ff9640 1 -- 172.21.15.99:0/1589177355 <== mon.2 v2:172.21.15.112:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f9fa8097410 con 0x55a29a145100 2024-03-19T02:02:28.633 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.627+0000 7f9fb600e740 1 -- 172.21.15.99:0/1589177355 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f9f9c01a070 con 0x55a29a145100 2024-03-19T02:02:28.633 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.627+0000 7f9f94ff9640 1 -- 172.21.15.99:0/1589177355 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7f9fa809e070 con 0x55a29a145100 2024-03-19T02:02:28.633 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.627+0000 7f9fb600e740 10 client.26822 fetch_fsmap learned FSMap version 37 2024-03-19T02:02:28.633 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.627+0000 7f9fb600e740 1 -- 172.21.15.99:0/1589177355 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x55a299dbe210 con 0x55a29a145100 2024-03-19T02:02:28.634 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.628+0000 7f9f94ff9640 1 -- 172.21.15.99:0/1589177355 <== mon.2 v2:172.21.15.112:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f9f9c01a070 con 0x55a29a145100 2024-03-19T02:02:28.634 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.628+0000 7f9f94ff9640 1 -- 172.21.15.99:0/1589177355 <== mon.2 v2:172.21.15.112:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7f9fa8099c60 con 0x55a29a145100 2024-03-19T02:02:28.634 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.628+0000 7f9fb600e740 10 client.26822 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T02:02:28.634 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.628+0000 7f9fb600e740 10 client.26822 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T02:02:28.634 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.628+0000 7f9fb600e740 20 client.26822 populate_metadata read hostname 'smithi099' 2024-03-19T02:02:28.634 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.628+0000 7f9fb600e740 1 --2- 172.21.15.99:0/1589177355 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55a29a15af00 0x55a29a15d2f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:28.634 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.628+0000 7f9fb600e740 4 client.26822 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T02:02:28.634 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.628+0000 7f9fb600e740 1 -- 172.21.15.99:0/1589177355 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55a29a0b7120 con 0x55a29a15af00 2024-03-19T02:02:28.634 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.628+0000 7f9fb08ed640 1 --2- 172.21.15.99:0/1589177355 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55a29a15af00 0x55a29a15d2f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:28.634 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.629+0000 7f9f94ff9640 10 client.26822 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T02:02:28.634 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.629+0000 7f9fb08ed640 1 --2- 172.21.15.99:0/1589177355 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55a29a15af00 0x55a29a15d2f0 crc :-1 s=READY pgs=219 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=14b5146ea5cd4db server_cookie=f07438a07a9648ad in_seq=0 out_seq=0 2024-03-19T02:02:28.634 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.631+0000 7f9f94ff9640 1 -- 172.21.15.99:0/1589177355 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x55a29a0b7120 con 0x55a29a15af00 2024-03-19T02:02:28.634 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.632+0000 7f9f94ff9640 10 client.26822 handle_command_reply: tid=0 2024-03-19T02:02:28.635 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.632+0000 7f9fb600e740 1 client.26822 shutdown 2024-03-19T02:02:28.635 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.632+0000 7f9fb600e740 1 -- 172.21.15.99:0/1589177355 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a29a145100 msgr2=0x55a29a14f0e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:02:28.635 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.632+0000 7f9fb600e740 1 --2- 172.21.15.99:0/1589177355 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a29a145100 0x55a29a14f0e0 secure :-1 s=READY pgs=214 cs=0 l=1 rev1=1 crypto rx=0x7f9fa8097200 tx=0x7f9fa8009f20 comp rx=0 tx=0).stop 2024-03-19T02:02:28.635 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.632+0000 7f9fb600e740 1 -- 172.21.15.99:0/1589177355 shutdown_connections 2024-03-19T02:02:28.635 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.632+0000 7f9fb600e740 1 --2- 172.21.15.99:0/1589177355 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55a29a15af00 0x55a29a15d2f0 crc :-1 s=READY pgs=219 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:28.635 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.632+0000 7f9fb600e740 1 --2- 172.21.15.99:0/1589177355 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a29a14ac60 0x55a29a156660 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:28.635 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.632+0000 7f9fb600e740 1 --2- 172.21.15.99:0/1589177355 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a29a145af0 0x55a29a14f620 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:28.635 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.632+0000 7f9f94ff9640 0 client.26822 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T02:02:28.635 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.632+0000 7f9fb600e740 1 --2- 172.21.15.99:0/1589177355 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a29a145100 0x55a29a14f0e0 unknown :-1 s=CLOSED pgs=214 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:28.635 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.632+0000 7f9fb600e740 1 -- 172.21.15.99:0/1589177355 >> 172.21.15.99:0/1589177355 conn(0x55a29a140670 msgr2=0x55a29a1418c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:02:28.635 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.632+0000 7f9fb600e740 1 -- 172.21.15.99:0/1589177355 shutdown_connections 2024-03-19T02:02:28.635 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.632+0000 7f9fb600e740 1 -- 172.21.15.99:0/1589177355 wait complete. 2024-03-19T02:02:28.635 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.632+0000 7f9fb600e740 20 client.26822 trim_cache size 0 max 16384 2024-03-19T02:02:28.649 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.648+0000 7f9fb600e740 1 Processor -- start 2024-03-19T02:02:28.650 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.648+0000 7f9fb600e740 1 -- start start 2024-03-19T02:02:28.650 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.649+0000 7f9fb600e740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a29a145100 0x55a29a146560 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:28.650 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.649+0000 7f9fb600e740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a29a145af0 0x55a29a156620 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:28.650 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.649+0000 7f9fb600e740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a29a14ac60 0x55a29a146aa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:28.650 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.649+0000 7f9fb600e740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f9fac1a9460 con 0x55a29a14ac60 2024-03-19T02:02:28.650 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.649+0000 7f9fb600e740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f9fac1a6d30 con 0x55a29a145af0 2024-03-19T02:02:28.650 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.649+0000 7f9fb600e740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f9fac1a4b80 con 0x55a29a145100 2024-03-19T02:02:28.650 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.649+0000 7f9fb18ef640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a29a14ac60 0x55a29a146aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:28.650 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.649+0000 7f9fb18ef640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a29a14ac60 0x55a29a146aa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:38688/0 (socket says 172.21.15.99:38688) 2024-03-19T02:02:28.650 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.649+0000 7f9fb18ef640 1 -- 172.21.15.99:0/3375412842 learned_addr learned my addr 172.21.15.99:0/3375412842 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:02:28.650 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.649+0000 7f9fb08ed640 1 --2- 172.21.15.99:0/3375412842 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a29a145af0 0x55a29a156620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:28.650 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.649+0000 7f9fb10ee640 1 --2- 172.21.15.99:0/3375412842 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a29a145100 0x55a29a146560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:28.650 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.649+0000 7f9fb08ed640 1 -- 172.21.15.99:0/3375412842 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a29a145100 msgr2=0x55a29a146560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:02:28.651 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.649+0000 7f9fb08ed640 1 --2- 172.21.15.99:0/3375412842 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a29a145100 0x55a29a146560 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:28.651 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.649+0000 7f9fb08ed640 1 -- 172.21.15.99:0/3375412842 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a29a14ac60 msgr2=0x55a29a146aa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:02:28.651 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.649+0000 7f9fb08ed640 1 --2- 172.21.15.99:0/3375412842 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a29a14ac60 0x55a29a146aa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:28.651 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.649+0000 7f9fb08ed640 1 -- 172.21.15.99:0/3375412842 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55a29a15d660 con 0x55a29a145af0 2024-03-19T02:02:28.651 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.649+0000 7f9fb18ef640 1 --2- 172.21.15.99:0/3375412842 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a29a14ac60 0x55a29a146aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:02:28.651 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.649+0000 7f9fb08ed640 1 --2- 172.21.15.99:0/3375412842 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a29a145af0 0x55a29a156620 secure :-1 s=READY pgs=215 cs=0 l=1 rev1=1 crypto rx=0x7f9f9c012010 tx=0x7f9f9c009950 comp rx=0 tx=0).ready entity=mon.2 client_cookie=d20080f6f627144c server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:02:28.651 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.650+0000 7f9f96ffd640 1 -- 172.21.15.99:0/3375412842 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9f9c02e070 con 0x55a29a145af0 2024-03-19T02:02:28.651 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.650+0000 7f9f96ffd640 1 -- 172.21.15.99:0/3375412842 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f9f9c01a040 con 0x55a29a145af0 2024-03-19T02:02:28.651 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.650+0000 7f9f96ffd640 1 -- 172.21.15.99:0/3375412842 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9f9c023040 con 0x55a29a145af0 2024-03-19T02:02:28.655 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.650+0000 7f9fb600e740 1 -- 172.21.15.99:0/3375412842 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a29a145af0 msgr2=0x55a29a156620 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:02:28.655 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.650+0000 7f9fb600e740 1 --2- 172.21.15.99:0/3375412842 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a29a145af0 0x55a29a156620 secure :-1 s=READY pgs=215 cs=0 l=1 rev1=1 crypto rx=0x7f9f9c012010 tx=0x7f9f9c009950 comp rx=0 tx=0).stop 2024-03-19T02:02:28.655 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.650+0000 7f9fb600e740 1 -- 172.21.15.99:0/3375412842 shutdown_connections 2024-03-19T02:02:28.655 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.650+0000 7f9fb600e740 1 --2- 172.21.15.99:0/3375412842 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a29a145af0 0x55a29a156620 unknown :-1 s=CLOSED pgs=215 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:28.655 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.650+0000 7f9fb600e740 1 --2- 172.21.15.99:0/3375412842 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a29a14ac60 0x55a29a146aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:28.655 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.650+0000 7f9fb600e740 1 --2- 172.21.15.99:0/3375412842 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a29a145100 0x55a29a146560 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:28.655 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.650+0000 7f9fb600e740 1 -- 172.21.15.99:0/3375412842 >> 172.21.15.99:0/3375412842 conn(0x55a29a140670 msgr2=0x55a29a14ecf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:02:28.655 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.650+0000 7f9fb600e740 1 -- 172.21.15.99:0/3375412842 shutdown_connections 2024-03-19T02:02:28.655 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.650+0000 7f9fb600e740 1 -- 172.21.15.99:0/3375412842 wait complete. 2024-03-19T02:02:28.655 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.651+0000 7f9fb600e740 1 -- start start 2024-03-19T02:02:28.655 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.651+0000 7f9fb600e740 1 Processor -- start 2024-03-19T02:02:28.655 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.651+0000 7f9fb600e740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a29a145100 0x55a29a154760 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:28.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.651+0000 7f9fb600e740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a29a145af0 0x55a29a154ca0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:28.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.651+0000 7f9fb600e740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a29a14ac60 0x55a29a155b20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:28.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.651+0000 7f9fb600e740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f9fac1a51f0 con 0x55a29a145100 2024-03-19T02:02:28.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.651+0000 7f9fb600e740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f9fac1a6660 con 0x55a29a145af0 2024-03-19T02:02:28.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.651+0000 7f9fb600e740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f9fac1a58c0 con 0x55a29a14ac60 2024-03-19T02:02:28.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.651+0000 7f9fb600e740 10 client.0 _ll_register_callbacks cb 0x7f9fa8097040 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T02:02:28.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.651+0000 7f9fb10ee640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a29a145100 0x55a29a154760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:28.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.651+0000 7f9fb10ee640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a29a145100 0x55a29a154760 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:38692/0 (socket says 172.21.15.99:38692) 2024-03-19T02:02:28.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.651+0000 7f9fb08ed640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a29a145af0 0x55a29a154ca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:28.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.651+0000 7f9fb10ee640 1 -- 172.21.15.99:0/934306143 learned_addr learned my addr 172.21.15.99:0/934306143 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:02:28.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.651+0000 7f9fb18ef640 1 --2- 172.21.15.99:0/934306143 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a29a14ac60 0x55a29a155b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:28.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.652+0000 7f9fb08ed640 1 -- 172.21.15.99:0/934306143 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a29a14ac60 msgr2=0x55a29a155b20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:02:28.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.652+0000 7f9fb08ed640 1 --2- 172.21.15.99:0/934306143 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a29a14ac60 0x55a29a155b20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:28.657 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.652+0000 7f9fb08ed640 1 -- 172.21.15.99:0/934306143 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a29a145100 msgr2=0x55a29a154760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:02:28.657 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.652+0000 7f9fb08ed640 1 --2- 172.21.15.99:0/934306143 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a29a145100 0x55a29a154760 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:28.657 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.652+0000 7f9fb08ed640 1 -- 172.21.15.99:0/934306143 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55a299e8a620 con 0x55a29a145af0 2024-03-19T02:02:28.657 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.652+0000 7f9fb10ee640 1 --2- 172.21.15.99:0/934306143 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a29a145100 0x55a29a154760 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:02:28.657 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.652+0000 7f9fb08ed640 1 --2- 172.21.15.99:0/934306143 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a29a145af0 0x55a29a154ca0 secure :-1 s=READY pgs=216 cs=0 l=1 rev1=1 crypto rx=0x7f9f9c009b40 tx=0x7f9f9c005db0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=f904f1f68342e652 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:02:28.657 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.652+0000 7f9f94ff9640 10 client.0 ms_handle_connect on v2:172.21.15.112:3300/0 2024-03-19T02:02:28.657 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.652+0000 7f9f94ff9640 1 -- 172.21.15.99:0/934306143 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9f9c02e070 con 0x55a29a145af0 2024-03-19T02:02:28.657 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.652+0000 7f9f94ff9640 1 -- 172.21.15.99:0/934306143 <== mon.2 v2:172.21.15.112:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f9f9c0055f0 con 0x55a29a145af0 2024-03-19T02:02:28.657 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.652+0000 7f9fb600e740 1 -- 172.21.15.99:0/934306143 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f9f9c01a040 con 0x55a29a145af0 2024-03-19T02:02:28.657 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.652+0000 7f9f94ff9640 1 -- 172.21.15.99:0/934306143 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7f9f9c02f260 con 0x55a29a145af0 2024-03-19T02:02:28.657 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.652+0000 7f9fb600e740 10 client.26828 fetch_fsmap learned FSMap version 37 2024-03-19T02:02:28.657 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.652+0000 7f9fb600e740 1 -- 172.21.15.99:0/934306143 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x55a29a1c1990 con 0x55a29a145af0 2024-03-19T02:02:28.657 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.653+0000 7f9f94ff9640 1 -- 172.21.15.99:0/934306143 <== mon.2 v2:172.21.15.112:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f9f9c01a040 con 0x55a29a145af0 2024-03-19T02:02:28.657 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.653+0000 7f9f94ff9640 1 -- 172.21.15.99:0/934306143 <== mon.2 v2:172.21.15.112:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7f9f9c029040 con 0x55a29a145af0 2024-03-19T02:02:28.658 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.653+0000 7f9fb600e740 10 client.26828 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T02:02:28.658 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.653+0000 7f9fb600e740 10 client.26828 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T02:02:28.658 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.653+0000 7f9fb600e740 20 client.26828 populate_metadata read hostname 'smithi099' 2024-03-19T02:02:28.658 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.653+0000 7f9fb600e740 1 --2- 172.21.15.99:0/934306143 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55a29a1c20b0 0x55a29a1c44d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:28.658 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.653+0000 7f9fb600e740 4 client.26828 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T02:02:28.658 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.653+0000 7f9fb600e740 1 -- 172.21.15.99:0/934306143 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55a29a0b1470 con 0x55a29a1c20b0 2024-03-19T02:02:28.658 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.653+0000 7f9fb10ee640 1 --2- 172.21.15.99:0/934306143 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55a29a1c20b0 0x55a29a1c44d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:28.658 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.653+0000 7f9f94ff9640 10 client.26828 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T02:02:28.658 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:28.654+0000 7f9fb10ee640 1 --2- 172.21.15.99:0/934306143 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55a29a1c20b0 0x55a29a1c44d0 crc :-1 s=READY pgs=220 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=8a05758830baea6e server_cookie=1cedc2100708a7f3 in_seq=0 out_seq=0 2024-03-19T02:02:29.116 INFO:tasks.workunit.client.0.smithi099.stdout: 10 229252 22.53 MB/sec execute 418 sec latency 359.656 ms 2024-03-19T02:02:29.253 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:29.252+0000 7f9f94ff9640 1 -- 172.21.15.99:0/934306143 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x55a29a0b1470 con 0x55a29a1c20b0 2024-03-19T02:02:29.253 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:29.252+0000 7f9f94ff9640 10 client.26828 handle_command_reply: tid=0 2024-03-19T02:02:29.253 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:29.252+0000 7f9fb600e740 1 client.26828 shutdown 2024-03-19T02:02:29.253 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:29.252+0000 7f9fb600e740 1 -- 172.21.15.99:0/934306143 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a29a145af0 msgr2=0x55a29a154ca0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:02:29.253 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:29.252+0000 7f9fb600e740 1 --2- 172.21.15.99:0/934306143 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a29a145af0 0x55a29a154ca0 secure :-1 s=READY pgs=216 cs=0 l=1 rev1=1 crypto rx=0x7f9f9c009b40 tx=0x7f9f9c005db0 comp rx=0 tx=0).stop 2024-03-19T02:02:29.253 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:29.253+0000 7f9fb600e740 1 -- 172.21.15.99:0/934306143 shutdown_connections 2024-03-19T02:02:29.253 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:29.253+0000 7f9fb600e740 1 --2- 172.21.15.99:0/934306143 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55a29a1c20b0 0x55a29a1c44d0 crc :-1 s=READY pgs=220 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:29.254 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:29.253+0000 7f9fb600e740 1 --2- 172.21.15.99:0/934306143 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a29a145af0 0x55a29a154ca0 unknown :-1 s=CLOSED pgs=216 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:29.254 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:29.253+0000 7f9fb600e740 1 --2- 172.21.15.99:0/934306143 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a29a14ac60 0x55a29a155b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:29.254 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:29.253+0000 7f9f94ff9640 0 client.26828 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T02:02:29.254 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:29.253+0000 7f9fb600e740 1 --2- 172.21.15.99:0/934306143 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a29a145100 0x55a29a154760 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:29.254 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:29.253+0000 7f9fb600e740 1 -- 172.21.15.99:0/934306143 >> 172.21.15.99:0/934306143 conn(0x55a29a140670 msgr2=0x55a29a1438d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:02:29.254 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:29.253+0000 7f9fb600e740 1 -- 172.21.15.99:0/934306143 shutdown_connections 2024-03-19T02:02:29.254 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:29.253+0000 7f9fb600e740 1 -- 172.21.15.99:0/934306143 wait complete. 2024-03-19T02:02:29.254 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:29.253+0000 7f9fb600e740 20 client.26828 trim_cache size 0 max 16384 2024-03-19T02:02:29.254 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T02:02:29.254 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T02:02:29.255 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T02:02:29.255 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T02:02:29.255 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:29.254+0000 7f9fb3352640 1 -- 172.21.15.99:0/296009616 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f9f980a28a0 msgr2=0x7f9f980a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:02:29.255 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:29.254+0000 7f9fb3352640 1 --2- 172.21.15.99:0/296009616 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f9f980a28a0 0x7f9f980a4d60 secure :-1 s=READY pgs=373 cs=0 l=1 rev1=1 crypto rx=0x7f9fa4006010 tx=0x7f9fa402e040 comp rx=0 tx=0).stop 2024-03-19T02:02:29.256 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:29.254+0000 7f9fb3352640 1 -- 172.21.15.99:0/296009616 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9fac100740 msgr2=0x7f9fac069a30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:02:29.256 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:29.254+0000 7f9fb3352640 1 --2- 172.21.15.99:0/296009616 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9fac100740 0x7f9fac069a30 secure :-1 s=READY pgs=748 cs=0 l=1 rev1=1 crypto rx=0x7f9fa800d950 tx=0x7f9fa800de10 comp rx=0 tx=0).stop 2024-03-19T02:02:29.256 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:29.254+0000 7f9fb3352640 1 -- 172.21.15.99:0/296009616 shutdown_connections 2024-03-19T02:02:29.256 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:29.254+0000 7f9fb3352640 1 --2- 172.21.15.99:0/296009616 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f9f980a28a0 0x7f9f980a4d60 unknown :-1 s=CLOSED pgs=373 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:29.256 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:29.254+0000 7f9fb3352640 1 --2- 172.21.15.99:0/296009616 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9fac1022f0 0x7f9fac19ed90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:29.256 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:29.254+0000 7f9fb3352640 1 --2- 172.21.15.99:0/296009616 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9fac101930 0x7f9fac19c750 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:29.256 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:29.254+0000 7f9fb3352640 1 --2- 172.21.15.99:0/296009616 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9fac100740 0x7f9fac069a30 unknown :-1 s=CLOSED pgs=748 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:29.256 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:29.254+0000 7f9fb3352640 1 -- 172.21.15.99:0/296009616 >> 172.21.15.99:0/296009616 conn(0x7f9fac0fbcb0 msgr2=0x7f9fac104bd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:02:29.256 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:29.254+0000 7f9fb3352640 1 -- 172.21.15.99:0/296009616 shutdown_connections 2024-03-19T02:02:29.256 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:29.254+0000 7f9fb3352640 1 -- 172.21.15.99:0/296009616 wait complete. 2024-03-19T02:02:30.116 INFO:tasks.workunit.client.0.smithi099.stdout: 10 229699 22.54 MB/sec execute 419 sec latency 368.991 ms 2024-03-19T02:02:30.413 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:02:30 smithi112 ceph-mon[66819]: pgmap v993: 97 pgs: 97 active+clean; 660 MiB data, 8.5 GiB used, 1.0 TiB / 1.0 TiB avail; 3.9 MiB/s rd, 6.3 MiB/s wr, 461 op/s 2024-03-19T02:02:30.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:02:30 smithi115 ceph-mon[67332]: pgmap v993: 97 pgs: 97 active+clean; 660 MiB data, 8.5 GiB used, 1.0 TiB / 1.0 TiB avail; 3.9 MiB/s rd, 6.3 MiB/s wr, 461 op/s 2024-03-19T02:02:30.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:02:30 smithi099 ceph-mon[62951]: pgmap v993: 97 pgs: 97 active+clean; 660 MiB data, 8.5 GiB used, 1.0 TiB / 1.0 TiB avail; 3.9 MiB/s rd, 6.3 MiB/s wr, 461 op/s 2024-03-19T02:02:31.116 INFO:tasks.workunit.client.0.smithi099.stdout: 10 229906 22.51 MB/sec execute 420 sec latency 248.938 ms 2024-03-19T02:02:32.116 INFO:tasks.workunit.client.0.smithi099.stdout: 10 230262 22.51 MB/sec execute 421 sec latency 140.582 ms 2024-03-19T02:02:32.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:02:32 smithi115 ceph-mon[67332]: pgmap v994: 97 pgs: 97 active+clean; 651 MiB data, 8.5 GiB used, 1.0 TiB / 1.0 TiB avail; 3.6 MiB/s rd, 5.9 MiB/s wr, 464 op/s 2024-03-19T02:02:32.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:02:32 smithi099 ceph-mon[62951]: pgmap v994: 97 pgs: 97 active+clean; 651 MiB data, 8.5 GiB used, 1.0 TiB / 1.0 TiB avail; 3.6 MiB/s rd, 5.9 MiB/s wr, 464 op/s 2024-03-19T02:02:32.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:02:32 smithi112 ceph-mon[66819]: pgmap v994: 97 pgs: 97 active+clean; 651 MiB data, 8.5 GiB used, 1.0 TiB / 1.0 TiB avail; 3.6 MiB/s rd, 5.9 MiB/s wr, 464 op/s 2024-03-19T02:02:33.117 INFO:tasks.workunit.client.0.smithi099.stdout: 10 230641 22.50 MB/sec execute 422 sec latency 70.046 ms 2024-03-19T02:02:34.117 INFO:tasks.workunit.client.0.smithi099.stdout: 10 231031 22.48 MB/sec execute 423 sec latency 85.196 ms 2024-03-19T02:02:34.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:02:34 smithi115 ceph-mon[67332]: pgmap v995: 97 pgs: 97 active+clean; 636 MiB data, 8.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.5 MiB/s rd, 6.4 MiB/s wr, 610 op/s 2024-03-19T02:02:34.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:02:34 smithi099 ceph-mon[62951]: pgmap v995: 97 pgs: 97 active+clean; 636 MiB data, 8.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.5 MiB/s rd, 6.4 MiB/s wr, 610 op/s 2024-03-19T02:02:34.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:02:34 smithi112 ceph-mon[66819]: pgmap v995: 97 pgs: 97 active+clean; 636 MiB data, 8.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.5 MiB/s rd, 6.4 MiB/s wr, 610 op/s 2024-03-19T02:02:35.117 INFO:tasks.workunit.client.0.smithi099.stdout: 10 231528 22.49 MB/sec execute 424 sec latency 70.913 ms 2024-03-19T02:02:36.117 INFO:tasks.workunit.client.0.smithi099.stdout: 10 231768 22.47 MB/sec execute 425 sec latency 242.143 ms 2024-03-19T02:02:36.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:02:36 smithi115 ceph-mon[67332]: pgmap v996: 97 pgs: 97 active+clean; 636 MiB data, 8.5 GiB used, 1.0 TiB / 1.0 TiB avail; 3.3 MiB/s rd, 5.2 MiB/s wr, 515 op/s 2024-03-19T02:02:36.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:02:36 smithi099 ceph-mon[62951]: pgmap v996: 97 pgs: 97 active+clean; 636 MiB data, 8.5 GiB used, 1.0 TiB / 1.0 TiB avail; 3.3 MiB/s rd, 5.2 MiB/s wr, 515 op/s 2024-03-19T02:02:36.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:02:36 smithi112 ceph-mon[66819]: pgmap v996: 97 pgs: 97 active+clean; 636 MiB data, 8.5 GiB used, 1.0 TiB / 1.0 TiB avail; 3.3 MiB/s rd, 5.2 MiB/s wr, 515 op/s 2024-03-19T02:02:37.117 INFO:tasks.workunit.client.0.smithi099.stdout: 10 232235 22.47 MB/sec execute 426 sec latency 60.974 ms 2024-03-19T02:02:38.117 INFO:tasks.workunit.client.0.smithi099.stdout: 10 232813 22.50 MB/sec execute 427 sec latency 62.962 ms 2024-03-19T02:02:38.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:02:38 smithi115 ceph-mon[67332]: pgmap v997: 97 pgs: 97 active+clean; 633 MiB data, 8.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 6.9 MiB/s wr, 621 op/s 2024-03-19T02:02:38.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:02:38 smithi099 ceph-mon[62951]: pgmap v997: 97 pgs: 97 active+clean; 633 MiB data, 8.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 6.9 MiB/s wr, 621 op/s 2024-03-19T02:02:38.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:02:38 smithi112 ceph-mon[66819]: pgmap v997: 97 pgs: 97 active+clean; 633 MiB data, 8.5 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 6.9 MiB/s wr, 621 op/s 2024-03-19T02:02:39.117 INFO:tasks.workunit.client.0.smithi099.stdout: 10 233376 22.51 MB/sec execute 428 sec latency 55.210 ms 2024-03-19T02:02:40.117 INFO:tasks.workunit.client.0.smithi099.stdout: 10 233820 22.52 MB/sec execute 429 sec latency 95.243 ms 2024-03-19T02:02:40.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:02:40 smithi115 ceph-mon[67332]: pgmap v998: 97 pgs: 97 active+clean; 634 MiB data, 8.5 GiB used, 1.0 TiB / 1.0 TiB avail; 2.9 MiB/s rd, 4.9 MiB/s wr, 503 op/s 2024-03-19T02:02:40.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:02:40 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T02:02:40.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:02:40 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:02:40.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:02:40 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:02:40.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:02:40 smithi099 ceph-mon[62951]: pgmap v998: 97 pgs: 97 active+clean; 634 MiB data, 8.5 GiB used, 1.0 TiB / 1.0 TiB avail; 2.9 MiB/s rd, 4.9 MiB/s wr, 503 op/s 2024-03-19T02:02:40.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:02:40 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T02:02:40.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:02:40 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:02:40.592 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:02:40 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:02:40.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:02:40 smithi112 ceph-mon[66819]: pgmap v998: 97 pgs: 97 active+clean; 634 MiB data, 8.5 GiB used, 1.0 TiB / 1.0 TiB avail; 2.9 MiB/s rd, 4.9 MiB/s wr, 503 op/s 2024-03-19T02:02:40.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:02:40 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T02:02:40.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:02:40 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:02:40.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:02:40 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:02:41.117 INFO:tasks.workunit.client.0.smithi099.stdout: 10 234101 22.51 MB/sec execute 430 sec latency 293.438 ms 2024-03-19T02:02:42.117 INFO:tasks.workunit.client.0.smithi099.stdout: 10 234357 22.49 MB/sec execute 431 sec latency 236.037 ms 2024-03-19T02:02:42.355 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.353+0000 7ff8cf710640 1 -- 172.21.15.99:0/3859889529 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff8c81033c0 msgr2=0x7ff8c8109c50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:02:42.355 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.353+0000 7ff8cf710640 1 --2- 172.21.15.99:0/3859889529 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff8c81033c0 0x7ff8c8109c50 secure :-1 s=READY pgs=749 cs=0 l=1 rev1=1 crypto rx=0x7ff8c0009ca0 tx=0x7ff8c002c0f0 comp rx=0 tx=0).stop 2024-03-19T02:02:42.356 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.353+0000 7ff8cf710640 1 -- 172.21.15.99:0/3859889529 shutdown_connections 2024-03-19T02:02:42.356 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.353+0000 7ff8cf710640 1 --2- 172.21.15.99:0/3859889529 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff8c8102a00 0x7ff8c8102e80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:42.356 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.353+0000 7ff8cf710640 1 --2- 172.21.15.99:0/3859889529 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff8c81033c0 0x7ff8c8109c50 unknown :-1 s=CLOSED pgs=749 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:42.356 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.353+0000 7ff8cf710640 1 --2- 172.21.15.99:0/3859889529 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff8c8101810 0x7ff8c8101c30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:42.356 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.353+0000 7ff8cf710640 1 -- 172.21.15.99:0/3859889529 >> 172.21.15.99:0/3859889529 conn(0x7ff8c80fcda0 msgr2=0x7ff8c80ff1c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:02:42.356 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.353+0000 7ff8cf710640 1 -- 172.21.15.99:0/3859889529 shutdown_connections 2024-03-19T02:02:42.356 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.353+0000 7ff8cf710640 1 -- 172.21.15.99:0/3859889529 wait complete. 2024-03-19T02:02:42.356 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.354+0000 7ff8cf710640 1 Processor -- start 2024-03-19T02:02:42.356 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.354+0000 7ff8cf710640 1 -- start start 2024-03-19T02:02:42.356 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.354+0000 7ff8cf710640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff8c8101810 0x7ff8c81979f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:42.356 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.354+0000 7ff8cf710640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff8c8102a00 0x7ff8c8197f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:42.356 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.354+0000 7ff8cf710640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff8c81033c0 0x7ff8c819efa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:42.356 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.354+0000 7ff8cf710640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7ff8c810d470 con 0x7ff8c8102a00 2024-03-19T02:02:42.356 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.354+0000 7ff8cf710640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7ff8c810d2f0 con 0x7ff8c81033c0 2024-03-19T02:02:42.357 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.354+0000 7ff8cf710640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7ff8c810d5f0 con 0x7ff8c8101810 2024-03-19T02:02:42.357 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.354+0000 7ff8cccab640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff8c8102a00 0x7ff8c8197f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:42.357 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.354+0000 7ff8cccab640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff8c8102a00 0x7ff8c8197f30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58918/0 (socket says 172.21.15.99:58918) 2024-03-19T02:02:42.357 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.354+0000 7ff8cdcad640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff8c81033c0 0x7ff8c819efa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:42.357 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.354+0000 7ff8cccab640 1 -- 172.21.15.99:0/1950007060 learned_addr learned my addr 172.21.15.99:0/1950007060 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:02:42.357 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.354+0000 7ff8cd4ac640 1 --2- 172.21.15.99:0/1950007060 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff8c8101810 0x7ff8c81979f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:42.357 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.354+0000 7ff8cccab640 1 -- 172.21.15.99:0/1950007060 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff8c8101810 msgr2=0x7ff8c81979f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:02:42.357 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.354+0000 7ff8cccab640 1 --2- 172.21.15.99:0/1950007060 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff8c8101810 0x7ff8c81979f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:42.357 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.354+0000 7ff8cccab640 1 -- 172.21.15.99:0/1950007060 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff8c81033c0 msgr2=0x7ff8c819efa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:02:42.357 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.354+0000 7ff8cccab640 1 --2- 172.21.15.99:0/1950007060 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff8c81033c0 0x7ff8c819efa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:42.357 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.354+0000 7ff8cccab640 1 -- 172.21.15.99:0/1950007060 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ff8c819f610 con 0x7ff8c8102a00 2024-03-19T02:02:42.357 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.355+0000 7ff8cccab640 1 --2- 172.21.15.99:0/1950007060 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff8c8102a00 0x7ff8c8197f30 secure :-1 s=READY pgs=750 cs=0 l=1 rev1=1 crypto rx=0x7ff8b800e9f0 tx=0x7ff8b800eeb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:02:42.357 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.355+0000 7ff8be7fc640 1 -- 172.21.15.99:0/1950007060 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff8b800cd80 con 0x7ff8c8102a00 2024-03-19T02:02:42.357 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.355+0000 7ff8be7fc640 1 -- 172.21.15.99:0/1950007060 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7ff8b8010910 con 0x7ff8c8102a00 2024-03-19T02:02:42.357 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.355+0000 7ff8be7fc640 1 -- 172.21.15.99:0/1950007060 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff8b8018990 con 0x7ff8c8102a00 2024-03-19T02:02:42.358 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.355+0000 7ff8cf710640 1 -- 172.21.15.99:0/1950007060 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ff8c819f900 con 0x7ff8c8102a00 2024-03-19T02:02:42.358 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.355+0000 7ff8cf710640 1 -- 172.21.15.99:0/1950007060 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7ff8c819fe50 con 0x7ff8c8102a00 2024-03-19T02:02:42.364 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.358+0000 7ff8d23da740 1 Processor -- start 2024-03-19T02:02:42.364 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.359+0000 7ff8d23da740 1 -- start start 2024-03-19T02:02:42.364 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.359+0000 7ff8d23da740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56268604f510 0x56268604f930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:42.364 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.359+0000 7ff8d23da740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56268604ff00 0x562686054a00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:42.364 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.359+0000 7ff8d23da740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562686055070 0x562686057500 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:42.364 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.359+0000 7ff8d23da740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x562685fafbe0 con 0x56268604f510 2024-03-19T02:02:42.365 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.359+0000 7ff8d23da740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x562685f58960 con 0x562686055070 2024-03-19T02:02:42.365 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.359+0000 7ff8d23da740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x562685fabc80 con 0x56268604ff00 2024-03-19T02:02:42.365 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.358+0000 7ff8be7fc640 1 -- 172.21.15.99:0/1950007060 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7ff8b8010ac0 con 0x7ff8c8102a00 2024-03-19T02:02:42.365 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.359+0000 7ff8cdcad640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56268604f510 0x56268604f930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:42.365 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.359+0000 7ff8cdcad640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56268604f510 0x56268604f930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58922/0 (socket says 172.21.15.99:58922) 2024-03-19T02:02:42.365 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.359+0000 7ff8cdcad640 1 -- 172.21.15.99:0/4185700423 learned_addr learned my addr 172.21.15.99:0/4185700423 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:02:42.365 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.359+0000 7ff8cd4ac640 1 --2- 172.21.15.99:0/4185700423 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56268604ff00 0x562686054a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:42.365 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.359+0000 7ff8cccab640 1 --2- 172.21.15.99:0/4185700423 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562686055070 0x562686057500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:42.365 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.360+0000 7ff8cd4ac640 1 -- 172.21.15.99:0/4185700423 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562686055070 msgr2=0x562686057500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:02:42.365 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.360+0000 7ff8cd4ac640 1 --2- 172.21.15.99:0/4185700423 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562686055070 0x562686057500 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:42.365 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.360+0000 7ff8cd4ac640 1 -- 172.21.15.99:0/4185700423 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56268604f510 msgr2=0x56268604f930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:02:42.365 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.360+0000 7ff8cd4ac640 1 --2- 172.21.15.99:0/4185700423 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56268604f510 0x56268604f930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:42.365 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.360+0000 7ff8cd4ac640 1 -- 172.21.15.99:0/4185700423 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x562685f4fd20 con 0x56268604ff00 2024-03-19T02:02:42.366 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.360+0000 7ff8cccab640 1 --2- 172.21.15.99:0/4185700423 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562686055070 0x562686057500 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-19T02:02:42.366 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.360+0000 7ff8cdcad640 1 --2- 172.21.15.99:0/4185700423 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56268604f510 0x56268604f930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:02:42.366 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.360+0000 7ff8be7fc640 1 --2- 172.21.15.99:0/1950007060 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff8b40a2920 0x7ff8b40a4de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:42.366 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.360+0000 7ff8cd4ac640 1 --2- 172.21.15.99:0/4185700423 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56268604ff00 0x562686054a00 secure :-1 s=READY pgs=130 cs=0 l=1 rev1=1 crypto rx=0x7ff8c4009bf0 tx=0x7ff8c4011810 comp rx=0 tx=0).ready entity=mon.1 client_cookie=b2fb18823a22a201 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:02:42.366 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.360+0000 7ff8cdcad640 1 --2- 172.21.15.99:0/1950007060 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff8b40a2920 0x7ff8b40a4de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:42.366 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.360+0000 7ff8be7fc640 1 -- 172.21.15.99:0/1950007060 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7ff8b80cee50 con 0x7ff8c8102a00 2024-03-19T02:02:42.366 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.360+0000 7ff8b37fe640 1 -- 172.21.15.99:0/4185700423 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff8c4009da0 con 0x56268604ff00 2024-03-19T02:02:42.366 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.360+0000 7ff8b37fe640 1 -- 172.21.15.99:0/4185700423 <== mon.1 v2:172.21.15.115:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7ff8c4005020 con 0x56268604ff00 2024-03-19T02:02:42.366 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.360+0000 7ff8b37fe640 1 -- 172.21.15.99:0/4185700423 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff8c4004bc0 con 0x56268604ff00 2024-03-19T02:02:42.366 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.361+0000 7ff8d23da740 1 -- 172.21.15.99:0/4185700423 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56268604ff00 msgr2=0x562686054a00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:02:42.366 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.361+0000 7ff8cdcad640 1 --2- 172.21.15.99:0/1950007060 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff8b40a2920 0x7ff8b40a4de0 secure :-1 s=READY pgs=374 cs=0 l=1 rev1=1 crypto rx=0x7ff8c0006010 tx=0x7ff8c0032590 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:02:42.366 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.361+0000 7ff8d23da740 1 --2- 172.21.15.99:0/4185700423 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56268604ff00 0x562686054a00 secure :-1 s=READY pgs=130 cs=0 l=1 rev1=1 crypto rx=0x7ff8c4009bf0 tx=0x7ff8c4011810 comp rx=0 tx=0).stop 2024-03-19T02:02:42.367 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.361+0000 7ff8d23da740 1 -- 172.21.15.99:0/4185700423 shutdown_connections 2024-03-19T02:02:42.367 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.361+0000 7ff8d23da740 1 --2- 172.21.15.99:0/4185700423 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562686055070 0x562686057500 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:42.367 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.361+0000 7ff8d23da740 1 --2- 172.21.15.99:0/4185700423 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56268604ff00 0x562686054a00 unknown :-1 s=CLOSED pgs=130 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:42.367 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.361+0000 7ff8d23da740 1 --2- 172.21.15.99:0/4185700423 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56268604f510 0x56268604f930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:42.367 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.361+0000 7ff8d23da740 1 -- 172.21.15.99:0/4185700423 >> 172.21.15.99:0/4185700423 conn(0x562686046670 msgr2=0x562686046a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:02:42.367 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.361+0000 7ff8d23da740 1 -- 172.21.15.99:0/4185700423 shutdown_connections 2024-03-19T02:02:42.367 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.361+0000 7ff8d23da740 1 -- 172.21.15.99:0/4185700423 wait complete. 2024-03-19T02:02:42.367 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.361+0000 7ff8d23da740 1 -- start start 2024-03-19T02:02:42.367 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.362+0000 7ff8d23da740 1 Processor -- start 2024-03-19T02:02:42.367 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.362+0000 7ff8d23da740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56268604f510 0x5626860594b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:42.367 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.362+0000 7ff8d23da740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56268604ff00 0x5626860599f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:42.367 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.362+0000 7ff8d23da740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562686055070 0x562686060a30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:42.367 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.362+0000 7ff8d23da740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x562686058020 con 0x56268604ff00 2024-03-19T02:02:42.368 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.362+0000 7ff8d23da740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x562685f6bdb0 con 0x562686055070 2024-03-19T02:02:42.368 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.362+0000 7ff8d23da740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x5626860581a0 con 0x56268604f510 2024-03-19T02:02:42.368 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.362+0000 7ff8d23da740 10 client.0 _ll_register_callbacks cb 0x562685f70e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T02:02:42.368 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.362+0000 7ff8cccab640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56268604ff00 0x5626860599f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:42.368 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.362+0000 7ff8cccab640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56268604ff00 0x5626860599f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58936/0 (socket says 172.21.15.99:58936) 2024-03-19T02:02:42.368 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.362+0000 7ff8cd4ac640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56268604f510 0x5626860594b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:42.368 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.362+0000 7ff8cccab640 1 -- 172.21.15.99:0/3100031409 learned_addr learned my addr 172.21.15.99:0/3100031409 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:02:42.368 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.362+0000 7ff8cdcad640 1 --2- 172.21.15.99:0/3100031409 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562686055070 0x562686060a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:42.368 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.362+0000 7ff8cccab640 1 -- 172.21.15.99:0/3100031409 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56268604f510 msgr2=0x5626860594b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:02:42.368 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.362+0000 7ff8cccab640 1 --2- 172.21.15.99:0/3100031409 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56268604f510 0x5626860594b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:42.368 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.362+0000 7ff8cccab640 1 -- 172.21.15.99:0/3100031409 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562686055070 msgr2=0x562686060a30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:02:42.368 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.362+0000 7ff8cccab640 1 --2- 172.21.15.99:0/3100031409 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562686055070 0x562686060a30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:42.368 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.362+0000 7ff8cccab640 1 -- 172.21.15.99:0/3100031409 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x562685f6ac60 con 0x56268604ff00 2024-03-19T02:02:42.368 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.362+0000 7ff8cd4ac640 1 --2- 172.21.15.99:0/3100031409 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56268604f510 0x5626860594b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:02:42.369 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.362+0000 7ff8cdcad640 1 --2- 172.21.15.99:0/3100031409 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562686055070 0x562686060a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:02:42.369 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.363+0000 7ff8cccab640 1 --2- 172.21.15.99:0/3100031409 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56268604ff00 0x5626860599f0 secure :-1 s=READY pgs=751 cs=0 l=1 rev1=1 crypto rx=0x7ff8b8097c60 tx=0x7ff8b809b690 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d2a8d286488b16f4 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:02:42.369 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.363+0000 7ff8b0ff9640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T02:02:42.369 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.363+0000 7ff8b0ff9640 1 -- 172.21.15.99:0/3100031409 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff8b809f8a0 con 0x56268604ff00 2024-03-19T02:02:42.369 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.363+0000 7ff8b0ff9640 1 -- 172.21.15.99:0/3100031409 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7ff8b8018b30 con 0x56268604ff00 2024-03-19T02:02:42.369 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.363+0000 7ff8d23da740 1 -- 172.21.15.99:0/3100031409 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7ff8c4005020 con 0x56268604ff00 2024-03-19T02:02:42.369 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.363+0000 7ff8b0ff9640 1 -- 172.21.15.99:0/3100031409 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7ff8b80a3ef0 con 0x56268604ff00 2024-03-19T02:02:42.369 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.363+0000 7ff8d23da740 10 client.17064 fetch_fsmap learned FSMap version 37 2024-03-19T02:02:42.369 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.363+0000 7ff8d23da740 1 -- 172.21.15.99:0/3100031409 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x562685cc4210 con 0x56268604ff00 2024-03-19T02:02:42.369 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.363+0000 7ff8b0ff9640 1 -- 172.21.15.99:0/3100031409 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7ff8c4005020 con 0x56268604ff00 2024-03-19T02:02:42.369 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.363+0000 7ff8b0ff9640 1 -- 172.21.15.99:0/3100031409 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7ff8b809ac50 con 0x56268604ff00 2024-03-19T02:02:42.369 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.363+0000 7ff8d23da740 10 client.17064 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T02:02:42.369 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.363+0000 7ff8d23da740 10 client.17064 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T02:02:42.369 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.363+0000 7ff8d23da740 20 client.17064 populate_metadata read hostname 'smithi099' 2024-03-19T02:02:42.370 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.363+0000 7ff8d23da740 1 --2- 172.21.15.99:0/3100031409 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5626860653e0 0x562686067810 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:42.370 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.363+0000 7ff8d23da740 4 client.17064 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T02:02:42.370 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.363+0000 7ff8d23da740 1 -- 172.21.15.99:0/3100031409 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x562685fbd120 con 0x5626860653e0 2024-03-19T02:02:42.370 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.364+0000 7ff8cd4ac640 1 --2- 172.21.15.99:0/3100031409 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5626860653e0 0x562686067810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:42.370 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.364+0000 7ff8cd4ac640 1 --2- 172.21.15.99:0/3100031409 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5626860653e0 0x562686067810 crc :-1 s=READY pgs=221 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=d765351b8d71d9e1 server_cookie=8393ea80b9c3dd27 in_seq=0 out_seq=0 2024-03-19T02:02:42.370 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.364+0000 7ff8b0ff9640 10 client.17064 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T02:02:42.373 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.372+0000 7ff8b0ff9640 1 -- 172.21.15.99:0/3100031409 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x562685fbd120 con 0x5626860653e0 2024-03-19T02:02:42.373 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.372+0000 7ff8b0ff9640 10 client.17064 handle_command_reply: tid=0 2024-03-19T02:02:42.373 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.372+0000 7ff8d23da740 1 client.17064 shutdown 2024-03-19T02:02:42.373 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.372+0000 7ff8d23da740 1 -- 172.21.15.99:0/3100031409 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56268604ff00 msgr2=0x5626860599f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:02:42.373 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.372+0000 7ff8d23da740 1 --2- 172.21.15.99:0/3100031409 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56268604ff00 0x5626860599f0 secure :-1 s=READY pgs=751 cs=0 l=1 rev1=1 crypto rx=0x7ff8b8097c60 tx=0x7ff8b809b690 comp rx=0 tx=0).stop 2024-03-19T02:02:42.373 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.372+0000 7ff8d23da740 1 -- 172.21.15.99:0/3100031409 shutdown_connections 2024-03-19T02:02:42.373 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.372+0000 7ff8d23da740 1 --2- 172.21.15.99:0/3100031409 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5626860653e0 0x562686067810 crc :-1 s=READY pgs=221 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:42.373 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.373+0000 7ff8d23da740 1 --2- 172.21.15.99:0/3100031409 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562686055070 0x562686060a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:42.373 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.373+0000 7ff8b0ff9640 0 client.17064 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T02:02:42.373 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.373+0000 7ff8d23da740 1 --2- 172.21.15.99:0/3100031409 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56268604ff00 0x5626860599f0 unknown :-1 s=CLOSED pgs=751 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:42.373 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.373+0000 7ff8d23da740 1 --2- 172.21.15.99:0/3100031409 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56268604f510 0x5626860594b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:42.373 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.373+0000 7ff8d23da740 1 -- 172.21.15.99:0/3100031409 >> 172.21.15.99:0/3100031409 conn(0x562686046670 msgr2=0x5626860478c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:02:42.374 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.373+0000 7ff8d23da740 1 -- 172.21.15.99:0/3100031409 shutdown_connections 2024-03-19T02:02:42.374 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.373+0000 7ff8d23da740 1 -- 172.21.15.99:0/3100031409 wait complete. 2024-03-19T02:02:42.374 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.373+0000 7ff8d23da740 20 client.17064 trim_cache size 0 max 16384 2024-03-19T02:02:42.389 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.388+0000 7ff8d23da740 1 Processor -- start 2024-03-19T02:02:42.389 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.388+0000 7ff8d23da740 1 -- start start 2024-03-19T02:02:42.389 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.388+0000 7ff8d23da740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56268604f510 0x562686045ab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:42.390 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.388+0000 7ff8d23da740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56268604ff00 0x5626860609f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:42.390 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.388+0000 7ff8d23da740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562686055070 0x5626860546d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:42.390 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.388+0000 7ff8d23da740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7ff8c81a9410 con 0x56268604f510 2024-03-19T02:02:42.390 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.388+0000 7ff8d23da740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7ff8c81a6ce0 con 0x56268604ff00 2024-03-19T02:02:42.390 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.388+0000 7ff8d23da740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7ff8c81a4b30 con 0x562686055070 2024-03-19T02:02:42.390 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.389+0000 7ff8cd4ac640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56268604f510 0x562686045ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:42.390 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.389+0000 7ff8cdcad640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562686055070 0x5626860546d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:42.390 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.389+0000 7ff8cd4ac640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56268604f510 0x562686045ab0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58952/0 (socket says 172.21.15.99:58952) 2024-03-19T02:02:42.390 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.389+0000 7ff8cd4ac640 1 -- 172.21.15.99:0/645338787 learned_addr learned my addr 172.21.15.99:0/645338787 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:02:42.390 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.389+0000 7ff8cccab640 1 --2- 172.21.15.99:0/645338787 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56268604ff00 0x5626860609f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:42.390 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.389+0000 7ff8cdcad640 1 -- 172.21.15.99:0/645338787 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56268604ff00 msgr2=0x5626860609f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:02:42.390 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.389+0000 7ff8cdcad640 1 --2- 172.21.15.99:0/645338787 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56268604ff00 0x5626860609f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:42.391 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.389+0000 7ff8cdcad640 1 -- 172.21.15.99:0/645338787 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56268604f510 msgr2=0x562686045ab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:02:42.391 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.389+0000 7ff8cdcad640 1 --2- 172.21.15.99:0/645338787 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56268604f510 0x562686045ab0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:42.391 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.389+0000 7ff8cdcad640 1 -- 172.21.15.99:0/645338787 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x562685d90620 con 0x562686055070 2024-03-19T02:02:42.391 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.389+0000 7ff8cccab640 1 --2- 172.21.15.99:0/645338787 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56268604ff00 0x5626860609f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:02:42.391 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.389+0000 7ff8cd4ac640 1 --2- 172.21.15.99:0/645338787 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56268604f510 0x562686045ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:02:42.391 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.389+0000 7ff8cdcad640 1 --2- 172.21.15.99:0/645338787 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562686055070 0x5626860546d0 secure :-1 s=READY pgs=131 cs=0 l=1 rev1=1 crypto rx=0x7ff8c0032420 tx=0x7ff8c0041000 comp rx=0 tx=0).ready entity=mon.1 client_cookie=8b77693feb62871a server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:02:42.391 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.390+0000 7ff8b2ffd640 1 -- 172.21.15.99:0/645338787 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff8c0039040 con 0x562686055070 2024-03-19T02:02:42.391 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.390+0000 7ff8b2ffd640 1 -- 172.21.15.99:0/645338787 <== mon.1 v2:172.21.15.115:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7ff8c0004040 con 0x562686055070 2024-03-19T02:02:42.391 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.390+0000 7ff8b2ffd640 1 -- 172.21.15.99:0/645338787 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff8c004d6b0 con 0x562686055070 2024-03-19T02:02:42.394 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.391+0000 7ff8d23da740 1 -- 172.21.15.99:0/645338787 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562686055070 msgr2=0x5626860546d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:02:42.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.391+0000 7ff8d23da740 1 --2- 172.21.15.99:0/645338787 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562686055070 0x5626860546d0 secure :-1 s=READY pgs=131 cs=0 l=1 rev1=1 crypto rx=0x7ff8c0032420 tx=0x7ff8c0041000 comp rx=0 tx=0).stop 2024-03-19T02:02:42.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.391+0000 7ff8d23da740 1 -- 172.21.15.99:0/645338787 shutdown_connections 2024-03-19T02:02:42.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.391+0000 7ff8d23da740 1 --2- 172.21.15.99:0/645338787 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56268604ff00 0x5626860609f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:42.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.391+0000 7ff8d23da740 1 --2- 172.21.15.99:0/645338787 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562686055070 0x5626860546d0 unknown :-1 s=CLOSED pgs=131 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:42.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.391+0000 7ff8d23da740 1 --2- 172.21.15.99:0/645338787 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56268604f510 0x562686045ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:42.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.391+0000 7ff8d23da740 1 -- 172.21.15.99:0/645338787 >> 172.21.15.99:0/645338787 conn(0x562686046670 msgr2=0x5626860509a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:02:42.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.391+0000 7ff8d23da740 1 -- 172.21.15.99:0/645338787 shutdown_connections 2024-03-19T02:02:42.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.391+0000 7ff8d23da740 1 -- 172.21.15.99:0/645338787 wait complete. 2024-03-19T02:02:42.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.391+0000 7ff8d23da740 1 -- start start 2024-03-19T02:02:42.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.391+0000 7ff8d23da740 1 Processor -- start 2024-03-19T02:02:42.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.391+0000 7ff8d23da740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56268604f510 0x56268605e980 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:42.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.391+0000 7ff8d23da740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56268604ff00 0x56268605eec0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:42.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.392+0000 7ff8d23da740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562686055070 0x56268605fe60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:42.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.392+0000 7ff8d23da740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7ff8c81a51a0 con 0x56268604f510 2024-03-19T02:02:42.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.392+0000 7ff8d23da740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7ff8c81a6610 con 0x562686055070 2024-03-19T02:02:42.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.392+0000 7ff8d23da740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7ff8c81a5870 con 0x56268604ff00 2024-03-19T02:02:42.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.392+0000 7ff8d23da740 10 client.0 _ll_register_callbacks cb 0x7ff8b8097aa0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T02:02:42.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.392+0000 7ff8cdcad640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562686055070 0x56268605fe60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:42.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.392+0000 7ff8cd4ac640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56268604f510 0x56268605e980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:42.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.392+0000 7ff8cdcad640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562686055070 0x56268605fe60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:37570/0 (socket says 172.21.15.99:37570) 2024-03-19T02:02:42.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.392+0000 7ff8cdcad640 1 -- 172.21.15.99:0/1068614515 learned_addr learned my addr 172.21.15.99:0/1068614515 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:02:42.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.392+0000 7ff8cccab640 1 --2- 172.21.15.99:0/1068614515 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56268604ff00 0x56268605eec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:42.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.392+0000 7ff8cdcad640 1 -- 172.21.15.99:0/1068614515 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56268604ff00 msgr2=0x56268605eec0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:02:42.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.392+0000 7ff8cdcad640 1 --2- 172.21.15.99:0/1068614515 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56268604ff00 0x56268605eec0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:42.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.392+0000 7ff8cdcad640 1 -- 172.21.15.99:0/1068614515 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56268604f510 msgr2=0x56268605e980 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:02:42.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.392+0000 7ff8cdcad640 1 --2- 172.21.15.99:0/1068614515 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56268604f510 0x56268605e980 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:42.397 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.392+0000 7ff8cdcad640 1 -- 172.21.15.99:0/1068614515 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x562686060450 con 0x562686055070 2024-03-19T02:02:42.397 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.392+0000 7ff8cdcad640 1 --2- 172.21.15.99:0/1068614515 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562686055070 0x56268605fe60 secure :-1 s=READY pgs=217 cs=0 l=1 rev1=1 crypto rx=0x7ff8c002c630 tx=0x7ff8c0003da0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=5d0c4eba31d99774 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:02:42.397 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.392+0000 7ff8b0ff9640 10 client.0 ms_handle_connect on v2:172.21.15.112:3300/0 2024-03-19T02:02:42.397 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.392+0000 7ff8b0ff9640 1 -- 172.21.15.99:0/1068614515 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff8c0039040 con 0x562686055070 2024-03-19T02:02:42.397 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.393+0000 7ff8b0ff9640 1 -- 172.21.15.99:0/1068614515 <== mon.2 v2:172.21.15.112:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7ff8c00043f0 con 0x562686055070 2024-03-19T02:02:42.397 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.393+0000 7ff8d23da740 1 -- 172.21.15.99:0/1068614515 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7ff8c0004040 con 0x562686055070 2024-03-19T02:02:42.397 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.393+0000 7ff8b0ff9640 1 -- 172.21.15.99:0/1068614515 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7ff8c0004630 con 0x562686055070 2024-03-19T02:02:42.397 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.393+0000 7ff8d23da740 10 client.26846 fetch_fsmap learned FSMap version 37 2024-03-19T02:02:42.397 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.393+0000 7ff8d23da740 1 -- 172.21.15.99:0/1068614515 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x5626860cbf80 con 0x562686055070 2024-03-19T02:02:42.397 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.393+0000 7ff8b0ff9640 1 -- 172.21.15.99:0/1068614515 <== mon.2 v2:172.21.15.112:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7ff8c0004040 con 0x562686055070 2024-03-19T02:02:42.397 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.393+0000 7ff8b0ff9640 1 -- 172.21.15.99:0/1068614515 <== mon.2 v2:172.21.15.112:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7ff8c00684d0 con 0x562686055070 2024-03-19T02:02:42.398 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.394+0000 7ff8d23da740 10 client.26846 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T02:02:42.398 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.394+0000 7ff8d23da740 10 client.26846 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T02:02:42.398 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.394+0000 7ff8d23da740 20 client.26846 populate_metadata read hostname 'smithi099' 2024-03-19T02:02:42.398 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.394+0000 7ff8d23da740 1 --2- 172.21.15.99:0/1068614515 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5626860cc930 0x5626860ced50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:42.398 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.394+0000 7ff8d23da740 4 client.26846 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T02:02:42.398 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.394+0000 7ff8d23da740 1 -- 172.21.15.99:0/1068614515 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x562685fb7470 con 0x5626860cc930 2024-03-19T02:02:42.398 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.394+0000 7ff8cd4ac640 1 --2- 172.21.15.99:0/1068614515 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5626860cc930 0x5626860ced50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:42.398 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.394+0000 7ff8cd4ac640 1 --2- 172.21.15.99:0/1068614515 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5626860cc930 0x5626860ced50 crc :-1 s=READY pgs=222 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=1de04583e6670474 server_cookie=43a7bcf9643156f8 in_seq=0 out_seq=0 2024-03-19T02:02:42.398 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.394+0000 7ff8b0ff9640 10 client.26846 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T02:02:42.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:02:42 smithi115 ceph-mon[67332]: pgmap v999: 97 pgs: 97 active+clean; 635 MiB data, 8.5 GiB used, 1.0 TiB / 1.0 TiB avail; 3.5 MiB/s rd, 5.4 MiB/s wr, 575 op/s 2024-03-19T02:02:42.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:02:42 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:02:42.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:02:42 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:02:42.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:02:42 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:02:42.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:02:42 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:02:42.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:02:42 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:02:42.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:02:42 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:02:42.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:02:42 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T02:02:42.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:02:42 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T02:02:42.557 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:02:42 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:02:42.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:02:42 smithi099 ceph-mon[62951]: pgmap v999: 97 pgs: 97 active+clean; 635 MiB data, 8.5 GiB used, 1.0 TiB / 1.0 TiB avail; 3.5 MiB/s rd, 5.4 MiB/s wr, 575 op/s 2024-03-19T02:02:42.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:02:42 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:02:42.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:02:42 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:02:42.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:02:42 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:02:42.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:02:42 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:02:42.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:02:42 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:02:42.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:02:42 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:02:42.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:02:42 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T02:02:42.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:02:42 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T02:02:42.591 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:02:42 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:02:42.620 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:02:42 smithi112 ceph-mon[66819]: pgmap v999: 97 pgs: 97 active+clean; 635 MiB data, 8.5 GiB used, 1.0 TiB / 1.0 TiB avail; 3.5 MiB/s rd, 5.4 MiB/s wr, 575 op/s 2024-03-19T02:02:42.620 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:02:42 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:02:42.621 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:02:42 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:02:42.621 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:02:42 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:02:42.621 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:02:42 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:02:42.621 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:02:42 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:02:42.621 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:02:42 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:02:42.621 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:02:42 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T02:02:42.621 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:02:42 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T02:02:42.621 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:02:42 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:02:42.937 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.937+0000 7ff8b0ff9640 1 -- 172.21.15.99:0/1068614515 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x562685fb7470 con 0x5626860cc930 2024-03-19T02:02:42.937 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.937+0000 7ff8b0ff9640 10 client.26846 handle_command_reply: tid=0 2024-03-19T02:02:42.938 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.937+0000 7ff8d23da740 1 client.26846 shutdown 2024-03-19T02:02:42.938 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.937+0000 7ff8d23da740 1 -- 172.21.15.99:0/1068614515 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562686055070 msgr2=0x56268605fe60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:02:42.938 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.937+0000 7ff8d23da740 1 --2- 172.21.15.99:0/1068614515 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562686055070 0x56268605fe60 secure :-1 s=READY pgs=217 cs=0 l=1 rev1=1 crypto rx=0x7ff8c002c630 tx=0x7ff8c0003da0 comp rx=0 tx=0).stop 2024-03-19T02:02:42.938 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.937+0000 7ff8d23da740 1 -- 172.21.15.99:0/1068614515 shutdown_connections 2024-03-19T02:02:42.938 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.937+0000 7ff8d23da740 1 --2- 172.21.15.99:0/1068614515 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5626860cc930 0x5626860ced50 crc :-1 s=READY pgs=222 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:42.938 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.937+0000 7ff8b0ff9640 0 client.26846 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T02:02:42.938 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.937+0000 7ff8d23da740 1 --2- 172.21.15.99:0/1068614515 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562686055070 0x56268605fe60 unknown :-1 s=CLOSED pgs=217 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:42.938 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.937+0000 7ff8d23da740 1 --2- 172.21.15.99:0/1068614515 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56268604ff00 0x56268605eec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:42.938 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.937+0000 7ff8d23da740 1 --2- 172.21.15.99:0/1068614515 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56268604f510 0x56268605e980 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:42.938 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.937+0000 7ff8d23da740 1 -- 172.21.15.99:0/1068614515 >> 172.21.15.99:0/1068614515 conn(0x562686046670 msgr2=0x562686051230 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:02:42.938 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.937+0000 7ff8d23da740 1 -- 172.21.15.99:0/1068614515 shutdown_connections 2024-03-19T02:02:42.938 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.937+0000 7ff8d23da740 1 -- 172.21.15.99:0/1068614515 wait complete. 2024-03-19T02:02:42.938 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.938+0000 7ff8d23da740 20 client.26846 trim_cache size 0 max 16384 2024-03-19T02:02:42.939 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T02:02:42.939 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T02:02:42.939 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T02:02:42.939 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T02:02:42.940 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.938+0000 7ff8cf710640 1 -- 172.21.15.99:0/1950007060 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff8b40a2920 msgr2=0x7ff8b40a4de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:02:42.940 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.938+0000 7ff8cf710640 1 --2- 172.21.15.99:0/1950007060 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff8b40a2920 0x7ff8b40a4de0 secure :-1 s=READY pgs=374 cs=0 l=1 rev1=1 crypto rx=0x7ff8c0006010 tx=0x7ff8c0032590 comp rx=0 tx=0).stop 2024-03-19T02:02:42.940 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.938+0000 7ff8cf710640 1 -- 172.21.15.99:0/1950007060 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff8c8102a00 msgr2=0x7ff8c8197f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:02:42.940 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.938+0000 7ff8cf710640 1 --2- 172.21.15.99:0/1950007060 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff8c8102a00 0x7ff8c8197f30 secure :-1 s=READY pgs=750 cs=0 l=1 rev1=1 crypto rx=0x7ff8b800e9f0 tx=0x7ff8b800eeb0 comp rx=0 tx=0).stop 2024-03-19T02:02:42.940 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.938+0000 7ff8cf710640 1 -- 172.21.15.99:0/1950007060 shutdown_connections 2024-03-19T02:02:42.940 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.938+0000 7ff8cf710640 1 --2- 172.21.15.99:0/1950007060 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff8b40a2920 0x7ff8b40a4de0 unknown :-1 s=CLOSED pgs=374 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:42.940 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.938+0000 7ff8cf710640 1 --2- 172.21.15.99:0/1950007060 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff8c81033c0 0x7ff8c819efa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:42.940 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.938+0000 7ff8cf710640 1 --2- 172.21.15.99:0/1950007060 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff8c8102a00 0x7ff8c8197f30 unknown :-1 s=CLOSED pgs=750 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:42.940 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.938+0000 7ff8cf710640 1 --2- 172.21.15.99:0/1950007060 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff8c8101810 0x7ff8c81979f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:42.940 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.938+0000 7ff8cf710640 1 -- 172.21.15.99:0/1950007060 >> 172.21.15.99:0/1950007060 conn(0x7ff8c80fcda0 msgr2=0x7ff8c8105c30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:02:42.940 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.938+0000 7ff8cf710640 1 -- 172.21.15.99:0/1950007060 shutdown_connections 2024-03-19T02:02:42.940 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:42.938+0000 7ff8cf710640 1 -- 172.21.15.99:0/1950007060 wait complete. 2024-03-19T02:02:43.117 INFO:tasks.workunit.client.0.smithi099.stdout: 10 234595 22.47 MB/sec execute 432 sec latency 357.324 ms 2024-03-19T02:02:44.117 INFO:tasks.workunit.client.0.smithi099.stdout: 10 235056 22.48 MB/sec execute 433 sec latency 70.885 ms 2024-03-19T02:02:44.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:02:44 smithi115 ceph-mon[67332]: pgmap v1000: 97 pgs: 97 active+clean; 639 MiB data, 8.5 GiB used, 1.0 TiB / 1.0 TiB avail; 3.9 MiB/s rd, 5.8 MiB/s wr, 547 op/s 2024-03-19T02:02:44.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:02:44 smithi099 ceph-mon[62951]: pgmap v1000: 97 pgs: 97 active+clean; 639 MiB data, 8.5 GiB used, 1.0 TiB / 1.0 TiB avail; 3.9 MiB/s rd, 5.8 MiB/s wr, 547 op/s 2024-03-19T02:02:44.626 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:02:44 smithi112 ceph-mon[66819]: pgmap v1000: 97 pgs: 97 active+clean; 639 MiB data, 8.5 GiB used, 1.0 TiB / 1.0 TiB avail; 3.9 MiB/s rd, 5.8 MiB/s wr, 547 op/s 2024-03-19T02:02:45.117 INFO:tasks.workunit.client.0.smithi099.stdout: 10 235501 22.47 MB/sec execute 434 sec latency 65.414 ms 2024-03-19T02:02:46.117 INFO:tasks.workunit.client.0.smithi099.stdout: 10 235790 22.45 MB/sec execute 435 sec latency 267.216 ms 2024-03-19T02:02:46.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:02:46 smithi115 ceph-mon[67332]: pgmap v1001: 97 pgs: 97 active+clean; 639 MiB data, 8.5 GiB used, 1.0 TiB / 1.0 TiB avail; 2.6 MiB/s rd, 4.4 MiB/s wr, 336 op/s 2024-03-19T02:02:46.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:02:46 smithi099 ceph-mon[62951]: pgmap v1001: 97 pgs: 97 active+clean; 639 MiB data, 8.5 GiB used, 1.0 TiB / 1.0 TiB avail; 2.6 MiB/s rd, 4.4 MiB/s wr, 336 op/s 2024-03-19T02:02:46.689 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:02:46 smithi112 ceph-mon[66819]: pgmap v1001: 97 pgs: 97 active+clean; 639 MiB data, 8.5 GiB used, 1.0 TiB / 1.0 TiB avail; 2.6 MiB/s rd, 4.4 MiB/s wr, 336 op/s 2024-03-19T02:02:47.117 INFO:tasks.workunit.client.0.smithi099.stdout: 10 236162 22.44 MB/sec execute 436 sec latency 216.976 ms 2024-03-19T02:02:48.117 INFO:tasks.workunit.client.0.smithi099.stdout: 10 236506 22.43 MB/sec execute 437 sec latency 157.214 ms 2024-03-19T02:02:48.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:02:48 smithi115 ceph-mon[67332]: pgmap v1002: 97 pgs: 97 active+clean; 643 MiB data, 8.6 GiB used, 1.0 TiB / 1.0 TiB avail; 3.7 MiB/s rd, 6.3 MiB/s wr, 557 op/s 2024-03-19T02:02:48.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:02:48 smithi099 ceph-mon[62951]: pgmap v1002: 97 pgs: 97 active+clean; 643 MiB data, 8.6 GiB used, 1.0 TiB / 1.0 TiB avail; 3.7 MiB/s rd, 6.3 MiB/s wr, 557 op/s 2024-03-19T02:02:48.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:02:48 smithi112 ceph-mon[66819]: pgmap v1002: 97 pgs: 97 active+clean; 643 MiB data, 8.6 GiB used, 1.0 TiB / 1.0 TiB avail; 3.7 MiB/s rd, 6.3 MiB/s wr, 557 op/s 2024-03-19T02:02:49.117 INFO:tasks.workunit.client.0.smithi099.stdout: 10 237049 22.45 MB/sec execute 438 sec latency 73.518 ms 2024-03-19T02:02:50.117 INFO:tasks.workunit.client.0.smithi099.stdout: 10 237629 22.47 MB/sec execute 439 sec latency 65.530 ms 2024-03-19T02:02:50.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:02:50 smithi115 ceph-mon[67332]: pgmap v1003: 97 pgs: 97 active+clean; 641 MiB data, 8.5 GiB used, 1.0 TiB / 1.0 TiB avail; 3.1 MiB/s rd, 5.1 MiB/s wr, 490 op/s 2024-03-19T02:02:50.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:02:50 smithi099 ceph-mon[62951]: pgmap v1003: 97 pgs: 97 active+clean; 641 MiB data, 8.5 GiB used, 1.0 TiB / 1.0 TiB avail; 3.1 MiB/s rd, 5.1 MiB/s wr, 490 op/s 2024-03-19T02:02:50.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:02:50 smithi112 ceph-mon[66819]: pgmap v1003: 97 pgs: 97 active+clean; 641 MiB data, 8.5 GiB used, 1.0 TiB / 1.0 TiB avail; 3.1 MiB/s rd, 5.1 MiB/s wr, 490 op/s 2024-03-19T02:02:51.117 INFO:tasks.workunit.client.0.smithi099.stdout: 10 238099 22.48 MB/sec execute 440 sec latency 294.821 ms 2024-03-19T02:02:52.117 INFO:tasks.workunit.client.0.smithi099.stdout: 10 238463 22.48 MB/sec execute 441 sec latency 85.910 ms 2024-03-19T02:02:52.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:02:52 smithi115 ceph-mon[67332]: pgmap v1004: 97 pgs: 97 active+clean; 645 MiB data, 8.6 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 5.8 MiB/s wr, 558 op/s 2024-03-19T02:02:52.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:02:52 smithi099 ceph-mon[62951]: pgmap v1004: 97 pgs: 97 active+clean; 645 MiB data, 8.6 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 5.8 MiB/s wr, 558 op/s 2024-03-19T02:02:52.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:02:52 smithi112 ceph-mon[66819]: pgmap v1004: 97 pgs: 97 active+clean; 645 MiB data, 8.6 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 5.8 MiB/s wr, 558 op/s 2024-03-19T02:02:53.117 INFO:tasks.workunit.client.0.smithi099.stdout: 10 238887 22.48 MB/sec execute 442 sec latency 122.295 ms 2024-03-19T02:02:54.117 INFO:tasks.workunit.client.0.smithi099.stdout: 10 239305 22.48 MB/sec execute 443 sec latency 91.233 ms 2024-03-19T02:02:54.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:02:54 smithi115 ceph-mon[67332]: pgmap v1005: 97 pgs: 97 active+clean; 648 MiB data, 8.6 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 6.1 MiB/s wr, 547 op/s 2024-03-19T02:02:54.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:02:54 smithi099 ceph-mon[62951]: pgmap v1005: 97 pgs: 97 active+clean; 648 MiB data, 8.6 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 6.1 MiB/s wr, 547 op/s 2024-03-19T02:02:54.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:02:54 smithi112 ceph-mon[66819]: pgmap v1005: 97 pgs: 97 active+clean; 648 MiB data, 8.6 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 6.1 MiB/s wr, 547 op/s 2024-03-19T02:02:55.117 INFO:tasks.workunit.client.0.smithi099.stdout: 10 239780 22.50 MB/sec execute 444 sec latency 71.213 ms 2024-03-19T02:02:56.070 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.068+0000 7f120fae8640 1 -- 172.21.15.99:0/858318180 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1208101900 msgr2=0x7f1208101d80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:02:56.070 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.068+0000 7f120fae8640 1 --2- 172.21.15.99:0/858318180 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1208101900 0x7f1208101d80 secure :-1 s=READY pgs=752 cs=0 l=1 rev1=1 crypto rx=0x7f11f8009900 tx=0x7f11f802bc30 comp rx=0 tx=0).stop 2024-03-19T02:02:56.070 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.068+0000 7f120fae8640 1 -- 172.21.15.99:0/858318180 shutdown_connections 2024-03-19T02:02:56.070 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.068+0000 7f120fae8640 1 --2- 172.21.15.99:0/858318180 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f12081022c0 0x7f1208108b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:56.070 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.068+0000 7f120fae8640 1 --2- 172.21.15.99:0/858318180 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1208101900 0x7f1208101d80 unknown :-1 s=CLOSED pgs=752 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:56.070 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.068+0000 7f120fae8640 1 --2- 172.21.15.99:0/858318180 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1208100710 0x7f1208100b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:56.070 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.068+0000 7f120fae8640 1 -- 172.21.15.99:0/858318180 >> 172.21.15.99:0/858318180 conn(0x7f12080fbc80 msgr2=0x7f12080fe0c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:02:56.070 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.068+0000 7f120fae8640 1 -- 172.21.15.99:0/858318180 shutdown_connections 2024-03-19T02:02:56.071 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.068+0000 7f120fae8640 1 -- 172.21.15.99:0/858318180 wait complete. 2024-03-19T02:02:56.071 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.069+0000 7f120fae8640 1 Processor -- start 2024-03-19T02:02:56.071 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.069+0000 7f120fae8640 1 -- start start 2024-03-19T02:02:56.071 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.069+0000 7f120fae8640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1208100710 0x7f1208197880 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:56.071 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.069+0000 7f120fae8640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1208101900 0x7f1208197dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:56.071 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.069+0000 7f120fae8640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f12081022c0 0x7f120819ee30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:56.071 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.069+0000 7f120fae8640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f120810d510 con 0x7f1208100710 2024-03-19T02:02:56.071 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.069+0000 7f120fae8640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f120810d390 con 0x7f1208101900 2024-03-19T02:02:56.071 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.069+0000 7f120fae8640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f120810d690 con 0x7f12081022c0 2024-03-19T02:02:56.072 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.069+0000 7f120d884640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1208100710 0x7f1208197880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:56.072 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.069+0000 7f120d884640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1208100710 0x7f1208197880 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:57782/0 (socket says 172.21.15.99:57782) 2024-03-19T02:02:56.072 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.069+0000 7f120d884640 1 -- 172.21.15.99:0/258771076 learned_addr learned my addr 172.21.15.99:0/258771076 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:02:56.072 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.069+0000 7f120d083640 1 --2- 172.21.15.99:0/258771076 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1208101900 0x7f1208197dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:56.072 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.069+0000 7f120e085640 1 --2- 172.21.15.99:0/258771076 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f12081022c0 0x7f120819ee30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:56.072 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.069+0000 7f120d884640 1 -- 172.21.15.99:0/258771076 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f12081022c0 msgr2=0x7f120819ee30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:02:56.072 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.069+0000 7f120d884640 1 --2- 172.21.15.99:0/258771076 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f12081022c0 0x7f120819ee30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:56.072 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.069+0000 7f120d884640 1 -- 172.21.15.99:0/258771076 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1208101900 msgr2=0x7f1208197dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:02:56.072 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.069+0000 7f120d884640 1 --2- 172.21.15.99:0/258771076 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1208101900 0x7f1208197dc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:56.072 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.069+0000 7f120d884640 1 -- 172.21.15.99:0/258771076 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f120819f4a0 con 0x7f1208100710 2024-03-19T02:02:56.072 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.069+0000 7f120d083640 1 --2- 172.21.15.99:0/258771076 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1208101900 0x7f1208197dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:02:56.072 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.069+0000 7f120d884640 1 --2- 172.21.15.99:0/258771076 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1208100710 0x7f1208197880 secure :-1 s=READY pgs=753 cs=0 l=1 rev1=1 crypto rx=0x7f120400e9c0 tx=0x7f120400ee80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:02:56.072 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.070+0000 7f11feffd640 1 -- 172.21.15.99:0/258771076 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f120400cde0 con 0x7f1208100710 2024-03-19T02:02:56.073 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.070+0000 7f120fae8640 1 -- 172.21.15.99:0/258771076 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f120819f790 con 0x7f1208100710 2024-03-19T02:02:56.073 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.070+0000 7f11feffd640 1 -- 172.21.15.99:0/258771076 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f1204010910 con 0x7f1208100710 2024-03-19T02:02:56.073 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.070+0000 7f120fae8640 1 -- 172.21.15.99:0/258771076 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f120819fd00 con 0x7f1208100710 2024-03-19T02:02:56.073 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.070+0000 7f11feffd640 1 -- 172.21.15.99:0/258771076 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1204018a10 con 0x7f1208100710 2024-03-19T02:02:56.080 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.072+0000 7f11feffd640 1 -- 172.21.15.99:0/258771076 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f120401f020 con 0x7f1208100710 2024-03-19T02:02:56.080 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.073+0000 7f12127ad740 1 Processor -- start 2024-03-19T02:02:56.080 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.073+0000 7f12127ad740 1 -- start start 2024-03-19T02:02:56.081 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.073+0000 7f12127ad740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c376710080 0x55c3767104a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:56.081 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.073+0000 7f12127ad740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c376710a70 0x55c376713560 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:56.081 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.073+0000 7f12127ad740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c376713bd0 0x55c376720a40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:56.081 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.073+0000 7f12127ad740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55c376674be0 con 0x55c376713bd0 2024-03-19T02:02:56.081 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.073+0000 7f12127ad740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55c37661d960 con 0x55c376710080 2024-03-19T02:02:56.081 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.073+0000 7f12127ad740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55c376670c80 con 0x55c376710a70 2024-03-19T02:02:56.081 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.073+0000 7f120d083640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c376713bd0 0x55c376720a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:56.081 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.073+0000 7f11feffd640 1 --2- 172.21.15.99:0/258771076 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f11f40a2920 0x7f11f40a4de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:56.081 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.073+0000 7f120e085640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c376710080 0x55c3767104a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:56.081 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.073+0000 7f120d083640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c376713bd0 0x55c376720a40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:57784/0 (socket says 172.21.15.99:57784) 2024-03-19T02:02:56.081 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.073+0000 7f120d083640 1 -- 172.21.15.99:0/292301681 learned_addr learned my addr 172.21.15.99:0/292301681 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:02:56.081 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.073+0000 7f120d884640 1 --2- 172.21.15.99:0/292301681 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c376710a70 0x55c376713560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:56.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.073+0000 7f120e085640 1 --2- 172.21.15.99:0/258771076 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f11f40a2920 0x7f11f40a4de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:56.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.073+0000 7f120d083640 1 -- 172.21.15.99:0/292301681 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c376710a70 msgr2=0x55c376713560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:02:56.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.073+0000 7f120d083640 1 --2- 172.21.15.99:0/292301681 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c376710a70 0x55c376713560 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:56.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.073+0000 7f120d083640 1 -- 172.21.15.99:0/292301681 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c376710080 msgr2=0x55c3767104a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:02:56.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.073+0000 7f120d083640 1 --2- 172.21.15.99:0/292301681 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c376710080 0x55c3767104a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:56.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.073+0000 7f120d083640 1 -- 172.21.15.99:0/292301681 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55c376614d20 con 0x55c376713bd0 2024-03-19T02:02:56.082 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.073+0000 7f120e085640 1 --2- 172.21.15.99:0/292301681 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c376710080 0x55c3767104a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:02:56.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.074+0000 7f11feffd640 1 -- 172.21.15.99:0/258771076 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f12040ced20 con 0x7f1208100710 2024-03-19T02:02:56.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.074+0000 7f120d083640 1 --2- 172.21.15.99:0/292301681 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c376713bd0 0x55c376720a40 secure :-1 s=READY pgs=754 cs=0 l=1 rev1=1 crypto rx=0x7f11f8009900 tx=0x7f11f8000bc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=93d3f8da33a021d7 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:02:56.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.074+0000 7f120e085640 1 --2- 172.21.15.99:0/258771076 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f11f40a2920 0x7f11f40a4de0 secure :-1 s=READY pgs=375 cs=0 l=1 rev1=1 crypto rx=0x55c3767112d0 tx=0x7f1200004ab0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:02:56.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.074+0000 7f11f3fff640 1 -- 172.21.15.99:0/292301681 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f11f8045070 con 0x55c376713bd0 2024-03-19T02:02:56.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.074+0000 7f11f3fff640 1 -- 172.21.15.99:0/292301681 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f11f8004c80 con 0x55c376713bd0 2024-03-19T02:02:56.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.074+0000 7f11f3fff640 1 -- 172.21.15.99:0/292301681 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f11f80052d0 con 0x55c376713bd0 2024-03-19T02:02:56.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.075+0000 7f12127ad740 1 -- 172.21.15.99:0/292301681 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c376713bd0 msgr2=0x55c376720a40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:02:56.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.075+0000 7f12127ad740 1 --2- 172.21.15.99:0/292301681 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c376713bd0 0x55c376720a40 secure :-1 s=READY pgs=754 cs=0 l=1 rev1=1 crypto rx=0x7f11f8009900 tx=0x7f11f8000bc0 comp rx=0 tx=0).stop 2024-03-19T02:02:56.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.075+0000 7f12127ad740 1 -- 172.21.15.99:0/292301681 shutdown_connections 2024-03-19T02:02:56.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.075+0000 7f12127ad740 1 --2- 172.21.15.99:0/292301681 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c376713bd0 0x55c376720a40 unknown :-1 s=CLOSED pgs=754 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:56.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.075+0000 7f12127ad740 1 --2- 172.21.15.99:0/292301681 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c376710a70 0x55c376713560 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:56.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.075+0000 7f12127ad740 1 --2- 172.21.15.99:0/292301681 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c376710080 0x55c3767104a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:56.083 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.075+0000 7f12127ad740 1 -- 172.21.15.99:0/292301681 >> 172.21.15.99:0/292301681 conn(0x55c37670b670 msgr2=0x55c37670ba70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:02:56.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.075+0000 7f12127ad740 1 -- 172.21.15.99:0/292301681 shutdown_connections 2024-03-19T02:02:56.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.075+0000 7f12127ad740 1 -- 172.21.15.99:0/292301681 wait complete. 2024-03-19T02:02:56.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.075+0000 7f12127ad740 1 -- start start 2024-03-19T02:02:56.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.075+0000 7f12127ad740 1 Processor -- start 2024-03-19T02:02:56.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.076+0000 7f12127ad740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c376710080 0x55c376722840 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:56.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.076+0000 7f12127ad740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c376710a70 0x55c376722d80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:56.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.076+0000 7f12127ad740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c376713bd0 0x55c376729dc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:56.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.076+0000 7f12127ad740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55c3767213e0 con 0x55c376710a70 2024-03-19T02:02:56.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.076+0000 7f12127ad740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55c376630db0 con 0x55c376710080 2024-03-19T02:02:56.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.076+0000 7f12127ad740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55c376721560 con 0x55c376713bd0 2024-03-19T02:02:56.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.076+0000 7f120d083640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c376710a70 0x55c376722d80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:56.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.076+0000 7f120d083640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c376710a70 0x55c376722d80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:57786/0 (socket says 172.21.15.99:57786) 2024-03-19T02:02:56.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.076+0000 7f120d083640 1 -- 172.21.15.99:0/2302377803 learned_addr learned my addr 172.21.15.99:0/2302377803 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:02:56.084 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.076+0000 7f12127ad740 10 client.0 _ll_register_callbacks cb 0x55c376635e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T02:02:56.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.076+0000 7f120d884640 1 --2- 172.21.15.99:0/2302377803 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c376710080 0x55c376722840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:56.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.076+0000 7f120e085640 1 --2- 172.21.15.99:0/2302377803 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c376713bd0 0x55c376729dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:56.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.076+0000 7f120d083640 1 -- 172.21.15.99:0/2302377803 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c376713bd0 msgr2=0x55c376729dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:02:56.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.076+0000 7f120d083640 1 --2- 172.21.15.99:0/2302377803 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c376713bd0 0x55c376729dc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:56.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.076+0000 7f120d083640 1 -- 172.21.15.99:0/2302377803 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c376710080 msgr2=0x55c376722840 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:02:56.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.076+0000 7f120d083640 1 --2- 172.21.15.99:0/2302377803 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c376710080 0x55c376722840 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:56.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.076+0000 7f120d083640 1 -- 172.21.15.99:0/2302377803 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55c37662fc60 con 0x55c376710a70 2024-03-19T02:02:56.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.076+0000 7f120d083640 1 --2- 172.21.15.99:0/2302377803 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c376710a70 0x55c376722d80 secure :-1 s=READY pgs=755 cs=0 l=1 rev1=1 crypto rx=0x7f11f8009230 tx=0x7f11f802c670 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3d19c27ad7f7ea87 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:02:56.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.076+0000 7f11f17fa640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T02:02:56.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.076+0000 7f11f17fa640 1 -- 172.21.15.99:0/2302377803 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f11f8045040 con 0x55c376710a70 2024-03-19T02:02:56.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.076+0000 7f11f17fa640 1 -- 172.21.15.99:0/2302377803 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f11f8005560 con 0x55c376710a70 2024-03-19T02:02:56.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.076+0000 7f12127ad740 1 -- 172.21.15.99:0/2302377803 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f11f8004c80 con 0x55c376710a70 2024-03-19T02:02:56.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.076+0000 7f11f17fa640 1 -- 172.21.15.99:0/2302377803 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7f11f805b050 con 0x55c376710a70 2024-03-19T02:02:56.085 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.077+0000 7f12127ad740 10 client.17082 fetch_fsmap learned FSMap version 37 2024-03-19T02:02:56.086 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.077+0000 7f12127ad740 1 -- 172.21.15.99:0/2302377803 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x55c376389210 con 0x55c376710a70 2024-03-19T02:02:56.086 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.077+0000 7f11f17fa640 1 -- 172.21.15.99:0/2302377803 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f11f8004c80 con 0x55c376710a70 2024-03-19T02:02:56.086 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.077+0000 7f11f17fa640 1 -- 172.21.15.99:0/2302377803 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7f11f803f560 con 0x55c376710a70 2024-03-19T02:02:56.086 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.077+0000 7f12127ad740 10 client.17082 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T02:02:56.086 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.077+0000 7f12127ad740 10 client.17082 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T02:02:56.086 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.077+0000 7f12127ad740 20 client.17082 populate_metadata read hostname 'smithi099' 2024-03-19T02:02:56.086 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.077+0000 7f12127ad740 1 --2- 172.21.15.99:0/2302377803 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55c37672e290 0x55c376730700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:56.086 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.077+0000 7f12127ad740 4 client.17082 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T02:02:56.086 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.077+0000 7f12127ad740 1 -- 172.21.15.99:0/2302377803 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55c376682120 con 0x55c37672e290 2024-03-19T02:02:56.086 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.077+0000 7f120d884640 1 --2- 172.21.15.99:0/2302377803 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55c37672e290 0x55c376730700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:56.086 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.077+0000 7f120d884640 1 --2- 172.21.15.99:0/2302377803 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55c37672e290 0x55c376730700 crc :-1 s=READY pgs=223 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=743e3eb074143000 server_cookie=b6b3b7a646f0123a in_seq=0 out_seq=0 2024-03-19T02:02:56.086 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.077+0000 7f11f17fa640 10 client.17082 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T02:02:56.087 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.079+0000 7f11f17fa640 1 -- 172.21.15.99:0/2302377803 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x55c376682120 con 0x55c37672e290 2024-03-19T02:02:56.087 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.079+0000 7f11f17fa640 10 client.17082 handle_command_reply: tid=0 2024-03-19T02:02:56.087 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.079+0000 7f12127ad740 1 client.17082 shutdown 2024-03-19T02:02:56.087 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.080+0000 7f12127ad740 1 -- 172.21.15.99:0/2302377803 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c376710a70 msgr2=0x55c376722d80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:02:56.087 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.080+0000 7f12127ad740 1 --2- 172.21.15.99:0/2302377803 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c376710a70 0x55c376722d80 secure :-1 s=READY pgs=755 cs=0 l=1 rev1=1 crypto rx=0x7f11f8009230 tx=0x7f11f802c670 comp rx=0 tx=0).stop 2024-03-19T02:02:56.087 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.080+0000 7f12127ad740 1 -- 172.21.15.99:0/2302377803 shutdown_connections 2024-03-19T02:02:56.087 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.080+0000 7f12127ad740 1 --2- 172.21.15.99:0/2302377803 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55c37672e290 0x55c376730700 crc :-1 s=READY pgs=223 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:56.087 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.080+0000 7f12127ad740 1 --2- 172.21.15.99:0/2302377803 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c376713bd0 0x55c376729dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:56.087 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.080+0000 7f11f17fa640 0 client.17082 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T02:02:56.087 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.080+0000 7f12127ad740 1 --2- 172.21.15.99:0/2302377803 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c376710a70 0x55c376722d80 unknown :-1 s=CLOSED pgs=755 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:56.087 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.080+0000 7f12127ad740 1 --2- 172.21.15.99:0/2302377803 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c376710080 0x55c376722840 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:56.087 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.080+0000 7f12127ad740 1 -- 172.21.15.99:0/2302377803 >> 172.21.15.99:0/2302377803 conn(0x55c37670b670 msgr2=0x55c37670c8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:02:56.087 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.080+0000 7f12127ad740 1 -- 172.21.15.99:0/2302377803 shutdown_connections 2024-03-19T02:02:56.087 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.080+0000 7f12127ad740 1 -- 172.21.15.99:0/2302377803 wait complete. 2024-03-19T02:02:56.087 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.080+0000 7f12127ad740 20 client.17082 trim_cache size 0 max 16384 2024-03-19T02:02:56.097 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.095+0000 7f12127ad740 1 Processor -- start 2024-03-19T02:02:56.097 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.095+0000 7f12127ad740 1 -- start start 2024-03-19T02:02:56.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.096+0000 7f12127ad740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c376710080 0x55c37671ca20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:56.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.096+0000 7f12127ad740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c376710a70 0x55c376729d80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:56.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.096+0000 7f12127ad740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c376713bd0 0x55c37671cf60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:56.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.096+0000 7f12127ad740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f12081a9280 con 0x55c376710080 2024-03-19T02:02:56.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.096+0000 7f12127ad740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f12081a6b50 con 0x55c376710a70 2024-03-19T02:02:56.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.096+0000 7f12127ad740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f12081a49a0 con 0x55c376713bd0 2024-03-19T02:02:56.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.096+0000 7f120d884640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c376710080 0x55c37671ca20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:56.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.096+0000 7f120d884640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c376710080 0x55c37671ca20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:57798/0 (socket says 172.21.15.99:57798) 2024-03-19T02:02:56.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.096+0000 7f120d884640 1 -- 172.21.15.99:0/299138242 learned_addr learned my addr 172.21.15.99:0/299138242 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:02:56.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.096+0000 7f120d083640 1 --2- 172.21.15.99:0/299138242 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c376710a70 0x55c376729d80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:56.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.096+0000 7f120e085640 1 --2- 172.21.15.99:0/299138242 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c376713bd0 0x55c37671cf60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:56.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.096+0000 7f120d884640 1 -- 172.21.15.99:0/299138242 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c376713bd0 msgr2=0x55c37671cf60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:02:56.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.096+0000 7f120d884640 1 --2- 172.21.15.99:0/299138242 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c376713bd0 0x55c37671cf60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:56.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.096+0000 7f120d884640 1 -- 172.21.15.99:0/299138242 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c376710a70 msgr2=0x55c376729d80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:02:56.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.096+0000 7f120d884640 1 --2- 172.21.15.99:0/299138242 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c376710a70 0x55c376729d80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:56.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.096+0000 7f120d884640 1 -- 172.21.15.99:0/299138242 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55c376455620 con 0x55c376710080 2024-03-19T02:02:56.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.096+0000 7f120e085640 1 --2- 172.21.15.99:0/299138242 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c376713bd0 0x55c37671cf60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-19T02:02:56.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.096+0000 7f120d083640 1 --2- 172.21.15.99:0/299138242 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c376710a70 0x55c376729d80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:02:56.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.096+0000 7f120d884640 1 --2- 172.21.15.99:0/299138242 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c376710080 0x55c37671ca20 secure :-1 s=READY pgs=756 cs=0 l=1 rev1=1 crypto rx=0x7f1204010f70 tx=0x7f120409b4a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=77ab412639d94ba0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:02:56.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.096+0000 7f11f37fe640 1 -- 172.21.15.99:0/299138242 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f120409b620 con 0x55c376710080 2024-03-19T02:02:56.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.096+0000 7f11f37fe640 1 -- 172.21.15.99:0/299138242 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f120409f770 con 0x55c376710080 2024-03-19T02:02:56.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.096+0000 7f11f37fe640 1 -- 172.21.15.99:0/299138242 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1204097c90 con 0x55c376710080 2024-03-19T02:02:56.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.097+0000 7f12127ad740 1 -- 172.21.15.99:0/299138242 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c376710080 msgr2=0x55c37671ca20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:02:56.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.097+0000 7f12127ad740 1 --2- 172.21.15.99:0/299138242 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c376710080 0x55c37671ca20 secure :-1 s=READY pgs=756 cs=0 l=1 rev1=1 crypto rx=0x7f1204010f70 tx=0x7f120409b4a0 comp rx=0 tx=0).stop 2024-03-19T02:02:56.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.097+0000 7f12127ad740 1 -- 172.21.15.99:0/299138242 shutdown_connections 2024-03-19T02:02:56.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.097+0000 7f12127ad740 1 --2- 172.21.15.99:0/299138242 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c376710a70 0x55c376729d80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:56.100 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.097+0000 7f12127ad740 1 --2- 172.21.15.99:0/299138242 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c376713bd0 0x55c37671cf60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:56.100 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.097+0000 7f12127ad740 1 --2- 172.21.15.99:0/299138242 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c376710080 0x55c37671ca20 unknown :-1 s=CLOSED pgs=756 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:56.100 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.097+0000 7f12127ad740 1 -- 172.21.15.99:0/299138242 >> 172.21.15.99:0/299138242 conn(0x55c37670b670 msgr2=0x55c376722840 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:02:56.100 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.097+0000 7f12127ad740 1 -- 172.21.15.99:0/299138242 shutdown_connections 2024-03-19T02:02:56.100 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.097+0000 7f12127ad740 1 -- 172.21.15.99:0/299138242 wait complete. 2024-03-19T02:02:56.100 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.098+0000 7f12127ad740 1 -- start start 2024-03-19T02:02:56.100 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.098+0000 7f12127ad740 1 Processor -- start 2024-03-19T02:02:56.102 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.098+0000 7f12127ad740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c376710080 0x55c376727cd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:56.103 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.098+0000 7f12127ad740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c376710a70 0x55c376728210 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:56.103 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.098+0000 7f12127ad740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c376713bd0 0x55c376729130 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:56.103 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.098+0000 7f12127ad740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f12081a5010 con 0x55c376713bd0 2024-03-19T02:02:56.103 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.098+0000 7f12127ad740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f12081a6480 con 0x55c376710080 2024-03-19T02:02:56.103 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.098+0000 7f12127ad740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f12081a56e0 con 0x55c376710a70 2024-03-19T02:02:56.103 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.098+0000 7f12127ad740 10 client.0 _ll_register_callbacks cb 0x7f1200009a00 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T02:02:56.103 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.098+0000 7f120e085640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c376713bd0 0x55c376729130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:56.103 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.098+0000 7f120d083640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c376710a70 0x55c376728210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:56.103 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.098+0000 7f120e085640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c376713bd0 0x55c376729130 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:57802/0 (socket says 172.21.15.99:57802) 2024-03-19T02:02:56.103 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.098+0000 7f120d884640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c376710080 0x55c376727cd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:56.103 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.098+0000 7f120e085640 1 -- 172.21.15.99:0/2934674986 learned_addr learned my addr 172.21.15.99:0/2934674986 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:02:56.103 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.099+0000 7f120e085640 1 -- 172.21.15.99:0/2934674986 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c376710a70 msgr2=0x55c376728210 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:02:56.103 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.099+0000 7f120e085640 1 --2- 172.21.15.99:0/2934674986 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c376710a70 0x55c376728210 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:56.104 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.099+0000 7f120e085640 1 -- 172.21.15.99:0/2934674986 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c376710080 msgr2=0x55c376727cd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:02:56.104 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.099+0000 7f120e085640 1 --2- 172.21.15.99:0/2934674986 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c376710080 0x55c376727cd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:56.104 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.099+0000 7f120e085640 1 -- 172.21.15.99:0/2934674986 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55c376729720 con 0x55c376713bd0 2024-03-19T02:02:56.104 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.099+0000 7f120d884640 1 --2- 172.21.15.99:0/2934674986 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c376710080 0x55c376727cd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:02:56.104 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.099+0000 7f120e085640 1 --2- 172.21.15.99:0/2934674986 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c376713bd0 0x55c376729130 secure :-1 s=READY pgs=757 cs=0 l=1 rev1=1 crypto rx=0x7f1200012b10 tx=0x7f1200004a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9bbcba5b97ed5de3 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:02:56.104 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.099+0000 7f11f17fa640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T02:02:56.104 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.099+0000 7f11f17fa640 1 -- 172.21.15.99:0/2934674986 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f120001a040 con 0x55c376713bd0 2024-03-19T02:02:56.104 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.099+0000 7f11f17fa640 1 -- 172.21.15.99:0/2934674986 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f1200004500 con 0x55c376713bd0 2024-03-19T02:02:56.104 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.099+0000 7f12127ad740 1 -- 172.21.15.99:0/2934674986 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f120409f770 con 0x55c376713bd0 2024-03-19T02:02:56.104 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.099+0000 7f11f17fa640 1 -- 172.21.15.99:0/2934674986 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7f1200004700 con 0x55c376713bd0 2024-03-19T02:02:56.104 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.099+0000 7f12127ad740 10 client.17088 fetch_fsmap learned FSMap version 37 2024-03-19T02:02:56.105 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.099+0000 7f12127ad740 1 -- 172.21.15.99:0/2934674986 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x55c376794de0 con 0x55c376713bd0 2024-03-19T02:02:56.105 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.099+0000 7f11f17fa640 1 -- 172.21.15.99:0/2934674986 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f120409f770 con 0x55c376713bd0 2024-03-19T02:02:56.105 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.099+0000 7f11f17fa640 1 -- 172.21.15.99:0/2934674986 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7f1200020030 con 0x55c376713bd0 2024-03-19T02:02:56.105 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.100+0000 7f12127ad740 10 client.17088 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T02:02:56.105 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.100+0000 7f12127ad740 10 client.17088 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T02:02:56.105 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.100+0000 7f12127ad740 20 client.17088 populate_metadata read hostname 'smithi099' 2024-03-19T02:02:56.105 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.100+0000 7f12127ad740 1 --2- 172.21.15.99:0/2934674986 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55c376795840 0x55c376797c60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:02:56.105 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.100+0000 7f12127ad740 4 client.17088 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T02:02:56.105 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.100+0000 7f12127ad740 1 -- 172.21.15.99:0/2934674986 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55c37667c470 con 0x55c376795840 2024-03-19T02:02:56.105 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.100+0000 7f120d884640 1 --2- 172.21.15.99:0/2934674986 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55c376795840 0x55c376797c60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:02:56.105 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.100+0000 7f11f17fa640 10 client.17088 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T02:02:56.105 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.100+0000 7f120d884640 1 --2- 172.21.15.99:0/2934674986 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55c376795840 0x55c376797c60 crc :-1 s=READY pgs=224 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=7071b0d7ae5d2060 server_cookie=b570665b76e13040 in_seq=0 out_seq=0 2024-03-19T02:02:56.117 INFO:tasks.workunit.client.0.smithi099.stdout: 10 240150 22.49 MB/sec execute 445 sec latency 219.211 ms 2024-03-19T02:02:56.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:02:56 smithi115 ceph-mon[67332]: pgmap v1006: 97 pgs: 97 active+clean; 649 MiB data, 8.6 GiB used, 1.0 TiB / 1.0 TiB avail; 3.4 MiB/s rd, 5.4 MiB/s wr, 499 op/s 2024-03-19T02:02:56.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:02:56 smithi099 ceph-mon[62951]: pgmap v1006: 97 pgs: 97 active+clean; 649 MiB data, 8.6 GiB used, 1.0 TiB / 1.0 TiB avail; 3.4 MiB/s rd, 5.4 MiB/s wr, 499 op/s 2024-03-19T02:02:56.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:02:56 smithi112 ceph-mon[66819]: pgmap v1006: 97 pgs: 97 active+clean; 649 MiB data, 8.6 GiB used, 1.0 TiB / 1.0 TiB avail; 3.4 MiB/s rd, 5.4 MiB/s wr, 499 op/s 2024-03-19T02:02:56.863 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.863+0000 7f11f17fa640 1 -- 172.21.15.99:0/2934674986 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x55c37667c470 con 0x55c376795840 2024-03-19T02:02:56.864 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.863+0000 7f11f17fa640 10 client.17088 handle_command_reply: tid=0 2024-03-19T02:02:56.864 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.863+0000 7f12127ad740 1 client.17088 shutdown 2024-03-19T02:02:56.864 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.863+0000 7f12127ad740 1 -- 172.21.15.99:0/2934674986 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c376713bd0 msgr2=0x55c376729130 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:02:56.864 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.863+0000 7f12127ad740 1 --2- 172.21.15.99:0/2934674986 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c376713bd0 0x55c376729130 secure :-1 s=READY pgs=757 cs=0 l=1 rev1=1 crypto rx=0x7f1200012b10 tx=0x7f1200004a40 comp rx=0 tx=0).stop 2024-03-19T02:02:56.864 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.863+0000 7f12127ad740 1 -- 172.21.15.99:0/2934674986 shutdown_connections 2024-03-19T02:02:56.864 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.863+0000 7f12127ad740 1 --2- 172.21.15.99:0/2934674986 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55c376795840 0x55c376797c60 crc :-1 s=READY pgs=224 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:56.864 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.863+0000 7f12127ad740 1 --2- 172.21.15.99:0/2934674986 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55c376713bd0 0x55c376729130 unknown :-1 s=CLOSED pgs=757 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:56.864 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.863+0000 7f12127ad740 1 --2- 172.21.15.99:0/2934674986 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55c376710a70 0x55c376728210 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:56.864 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.863+0000 7f12127ad740 1 --2- 172.21.15.99:0/2934674986 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55c376710080 0x55c376727cd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:56.864 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.863+0000 7f12127ad740 1 -- 172.21.15.99:0/2934674986 >> 172.21.15.99:0/2934674986 conn(0x55c37670b670 msgr2=0x55c37671ca20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:02:56.864 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.863+0000 7f11f17fa640 0 client.17088 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T02:02:56.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.864+0000 7f12127ad740 1 -- 172.21.15.99:0/2934674986 shutdown_connections 2024-03-19T02:02:56.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.864+0000 7f12127ad740 1 -- 172.21.15.99:0/2934674986 wait complete. 2024-03-19T02:02:56.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.864+0000 7f12127ad740 20 client.17088 trim_cache size 0 max 16384 2024-03-19T02:02:56.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T02:02:56.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T02:02:56.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T02:02:56.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T02:02:56.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.865+0000 7f120fae8640 1 -- 172.21.15.99:0/258771076 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f11f40a2920 msgr2=0x7f11f40a4de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:02:56.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.865+0000 7f120fae8640 1 --2- 172.21.15.99:0/258771076 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f11f40a2920 0x7f11f40a4de0 secure :-1 s=READY pgs=375 cs=0 l=1 rev1=1 crypto rx=0x55c3767112d0 tx=0x7f1200004ab0 comp rx=0 tx=0).stop 2024-03-19T02:02:56.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.865+0000 7f120fae8640 1 -- 172.21.15.99:0/258771076 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1208100710 msgr2=0x7f1208197880 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:02:56.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.865+0000 7f120fae8640 1 --2- 172.21.15.99:0/258771076 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1208100710 0x7f1208197880 secure :-1 s=READY pgs=753 cs=0 l=1 rev1=1 crypto rx=0x7f120400e9c0 tx=0x7f120400ee80 comp rx=0 tx=0).stop 2024-03-19T02:02:56.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.865+0000 7f120fae8640 1 -- 172.21.15.99:0/258771076 shutdown_connections 2024-03-19T02:02:56.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.865+0000 7f120fae8640 1 --2- 172.21.15.99:0/258771076 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f11f40a2920 0x7f11f40a4de0 unknown :-1 s=CLOSED pgs=375 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:56.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.865+0000 7f120fae8640 1 --2- 172.21.15.99:0/258771076 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1208101900 0x7f1208197dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:56.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.865+0000 7f120fae8640 1 --2- 172.21.15.99:0/258771076 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f12081022c0 0x7f120819ee30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:56.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.865+0000 7f120fae8640 1 --2- 172.21.15.99:0/258771076 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1208100710 0x7f1208197880 unknown :-1 s=CLOSED pgs=753 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:02:56.867 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.865+0000 7f120fae8640 1 -- 172.21.15.99:0/258771076 >> 172.21.15.99:0/258771076 conn(0x7f12080fbc80 msgr2=0x7f1208104b30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:02:56.867 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.865+0000 7f120fae8640 1 -- 172.21.15.99:0/258771076 shutdown_connections 2024-03-19T02:02:56.867 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:02:56.865+0000 7f120fae8640 1 -- 172.21.15.99:0/258771076 wait complete. 2024-03-19T02:02:57.117 INFO:tasks.workunit.client.0.smithi099.stdout: 10 240285 22.45 MB/sec execute 446 sec latency 478.061 ms 2024-03-19T02:02:58.117 INFO:tasks.workunit.client.0.smithi099.stdout: 10 240685 22.45 MB/sec execute 447 sec latency 105.434 ms 2024-03-19T02:02:58.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:02:58 smithi115 ceph-mon[67332]: pgmap v1007: 97 pgs: 97 active+clean; 652 MiB data, 8.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.4 MiB/s rd, 7.2 MiB/s wr, 624 op/s 2024-03-19T02:02:58.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:02:58 smithi099 ceph-mon[62951]: pgmap v1007: 97 pgs: 97 active+clean; 652 MiB data, 8.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.4 MiB/s rd, 7.2 MiB/s wr, 624 op/s 2024-03-19T02:02:58.758 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:02:58 smithi112 ceph-mon[66819]: pgmap v1007: 97 pgs: 97 active+clean; 652 MiB data, 8.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.4 MiB/s rd, 7.2 MiB/s wr, 624 op/s 2024-03-19T02:02:59.117 INFO:tasks.workunit.client.0.smithi099.stdout: 10 241113 22.44 MB/sec execute 448 sec latency 102.990 ms 2024-03-19T02:03:00.117 INFO:tasks.workunit.client.0.smithi099.stdout: 10 241529 22.43 MB/sec execute 449 sec latency 76.803 ms 2024-03-19T02:03:00.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:03:00 smithi115 ceph-mon[67332]: pgmap v1008: 97 pgs: 97 active+clean; 655 MiB data, 8.7 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 6.0 MiB/s wr, 480 op/s 2024-03-19T02:03:00.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:03:00 smithi099 ceph-mon[62951]: pgmap v1008: 97 pgs: 97 active+clean; 655 MiB data, 8.7 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 6.0 MiB/s wr, 480 op/s 2024-03-19T02:03:00.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:03:00 smithi112 ceph-mon[66819]: pgmap v1008: 97 pgs: 97 active+clean; 655 MiB data, 8.7 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 6.0 MiB/s wr, 480 op/s 2024-03-19T02:03:01.117 INFO:tasks.workunit.client.0.smithi099.stdout: 10 241969 22.44 MB/sec execute 450 sec latency 133.439 ms 2024-03-19T02:03:02.118 INFO:tasks.workunit.client.0.smithi099.stdout: 10 242348 22.45 MB/sec execute 451 sec latency 285.582 ms 2024-03-19T02:03:02.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:03:02 smithi115 ceph-mon[67332]: pgmap v1009: 97 pgs: 97 active+clean; 650 MiB data, 8.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 6.2 MiB/s wr, 583 op/s 2024-03-19T02:03:02.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:03:02 smithi099 ceph-mon[62951]: pgmap v1009: 97 pgs: 97 active+clean; 650 MiB data, 8.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 6.2 MiB/s wr, 583 op/s 2024-03-19T02:03:02.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:03:02 smithi112 ceph-mon[66819]: pgmap v1009: 97 pgs: 97 active+clean; 650 MiB data, 8.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 6.2 MiB/s wr, 583 op/s 2024-03-19T02:03:03.118 INFO:tasks.workunit.client.0.smithi099.stdout: 10 242804 22.44 MB/sec execute 452 sec latency 190.490 ms 2024-03-19T02:03:04.118 INFO:tasks.workunit.client.0.smithi099.stdout: 10 243246 22.44 MB/sec execute 453 sec latency 71.472 ms 2024-03-19T02:03:04.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:03:04 smithi115 ceph-mon[67332]: pgmap v1010: 97 pgs: 97 active+clean; 648 MiB data, 8.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 6.5 MiB/s wr, 588 op/s 2024-03-19T02:03:04.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:03:04 smithi099 ceph-mon[62951]: pgmap v1010: 97 pgs: 97 active+clean; 648 MiB data, 8.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 6.5 MiB/s wr, 588 op/s 2024-03-19T02:03:04.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:03:04 smithi112 ceph-mon[66819]: pgmap v1010: 97 pgs: 97 active+clean; 648 MiB data, 8.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 6.5 MiB/s wr, 588 op/s 2024-03-19T02:03:05.118 INFO:tasks.workunit.client.0.smithi099.stdout: 10 243672 22.46 MB/sec execute 454 sec latency 76.650 ms 2024-03-19T02:03:06.118 INFO:tasks.workunit.client.0.smithi099.stdout: 10 244016 22.45 MB/sec execute 455 sec latency 256.877 ms 2024-03-19T02:03:06.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:03:06 smithi115 ceph-mon[67332]: pgmap v1011: 97 pgs: 97 active+clean; 650 MiB data, 8.7 GiB used, 1.0 TiB / 1.0 TiB avail; 3.2 MiB/s rd, 5.4 MiB/s wr, 524 op/s 2024-03-19T02:03:06.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:03:06 smithi099 ceph-mon[62951]: pgmap v1011: 97 pgs: 97 active+clean; 650 MiB data, 8.7 GiB used, 1.0 TiB / 1.0 TiB avail; 3.2 MiB/s rd, 5.4 MiB/s wr, 524 op/s 2024-03-19T02:03:06.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:03:06 smithi112 ceph-mon[66819]: pgmap v1011: 97 pgs: 97 active+clean; 650 MiB data, 8.7 GiB used, 1.0 TiB / 1.0 TiB avail; 3.2 MiB/s rd, 5.4 MiB/s wr, 524 op/s 2024-03-19T02:03:07.118 INFO:tasks.workunit.client.0.smithi099.stdout: 10 244301 22.45 MB/sec execute 456 sec latency 287.241 ms 2024-03-19T02:03:08.118 INFO:tasks.workunit.client.0.smithi099.stdout: 10 244593 22.42 MB/sec execute 457 sec latency 222.435 ms 2024-03-19T02:03:08.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:03:08 smithi115 ceph-mon[67332]: pgmap v1012: 97 pgs: 97 active+clean; 655 MiB data, 8.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.7 MiB/s rd, 6.8 MiB/s wr, 620 op/s 2024-03-19T02:03:08.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:03:08 smithi099 ceph-mon[62951]: pgmap v1012: 97 pgs: 97 active+clean; 655 MiB data, 8.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.7 MiB/s rd, 6.8 MiB/s wr, 620 op/s 2024-03-19T02:03:08.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:03:08 smithi112 ceph-mon[66819]: pgmap v1012: 97 pgs: 97 active+clean; 655 MiB data, 8.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.7 MiB/s rd, 6.8 MiB/s wr, 620 op/s 2024-03-19T02:03:09.118 INFO:tasks.workunit.client.0.smithi099.stdout: 10 245012 22.41 MB/sec execute 458 sec latency 83.803 ms 2024-03-19T02:03:09.961 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.959+0000 7f828033b640 1 -- 172.21.15.99:0/1080882172 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8278101940 msgr2=0x7f8278101dc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:03:09.961 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.959+0000 7f828033b640 1 --2- 172.21.15.99:0/1080882172 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8278101940 0x7f8278101dc0 secure :-1 s=READY pgs=758 cs=0 l=1 rev1=1 crypto rx=0x7f8268009900 tx=0x7f826802bc30 comp rx=0 tx=0).stop 2024-03-19T02:03:09.961 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.959+0000 7f828033b640 1 -- 172.21.15.99:0/1080882172 shutdown_connections 2024-03-19T02:03:09.961 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.959+0000 7f828033b640 1 --2- 172.21.15.99:0/1080882172 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8278102300 0x7f8278108b90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:09.961 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.959+0000 7f828033b640 1 --2- 172.21.15.99:0/1080882172 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8278101940 0x7f8278101dc0 unknown :-1 s=CLOSED pgs=758 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:09.961 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.959+0000 7f828033b640 1 --2- 172.21.15.99:0/1080882172 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8278100750 0x7f8278100b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:09.961 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.959+0000 7f828033b640 1 -- 172.21.15.99:0/1080882172 >> 172.21.15.99:0/1080882172 conn(0x7f82780fbcc0 msgr2=0x7f82780fe100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:03:09.961 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.959+0000 7f828033b640 1 -- 172.21.15.99:0/1080882172 shutdown_connections 2024-03-19T02:03:09.961 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.959+0000 7f828033b640 1 -- 172.21.15.99:0/1080882172 wait complete. 2024-03-19T02:03:09.962 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.959+0000 7f828033b640 1 Processor -- start 2024-03-19T02:03:09.962 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.959+0000 7f828033b640 1 -- start start 2024-03-19T02:03:09.962 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.959+0000 7f828033b640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8278100750 0x7f8278197ab0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:09.962 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.959+0000 7f828033b640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8278101940 0x7f8278197ff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:09.962 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.959+0000 7f828033b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8278102300 0x7f827819f060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:09.962 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.960+0000 7f828033b640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f827810c3d0 con 0x7f8278102300 2024-03-19T02:03:09.962 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.960+0000 7f828033b640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f827810c250 con 0x7f8278100750 2024-03-19T02:03:09.962 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.960+0000 7f828033b640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f827810c550 con 0x7f8278101940 2024-03-19T02:03:09.962 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.960+0000 7f827e8d8640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8278102300 0x7f827819f060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:09.962 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.960+0000 7f827e0d7640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8278100750 0x7f8278197ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:09.962 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.960+0000 7f827e8d8640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8278102300 0x7f827819f060 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:57194/0 (socket says 172.21.15.99:57194) 2024-03-19T02:03:09.963 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.960+0000 7f827e8d8640 1 -- 172.21.15.99:0/682291383 learned_addr learned my addr 172.21.15.99:0/682291383 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:03:09.963 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.960+0000 7f827d8d6640 1 --2- 172.21.15.99:0/682291383 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8278101940 0x7f8278197ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:09.963 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.960+0000 7f827e8d8640 1 -- 172.21.15.99:0/682291383 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8278101940 msgr2=0x7f8278197ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:03:09.963 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.960+0000 7f827e8d8640 1 --2- 172.21.15.99:0/682291383 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8278101940 0x7f8278197ff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:09.963 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.960+0000 7f827e8d8640 1 -- 172.21.15.99:0/682291383 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8278100750 msgr2=0x7f8278197ab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:03:09.963 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.960+0000 7f827e8d8640 1 --2- 172.21.15.99:0/682291383 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8278100750 0x7f8278197ab0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:09.963 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.960+0000 7f827e8d8640 1 -- 172.21.15.99:0/682291383 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f827819f6d0 con 0x7f8278102300 2024-03-19T02:03:09.963 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.960+0000 7f827e8d8640 1 --2- 172.21.15.99:0/682291383 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8278102300 0x7f827819f060 secure :-1 s=READY pgs=759 cs=0 l=1 rev1=1 crypto rx=0x7f827000efd0 tx=0x7f827000c6a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:03:09.963 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.960+0000 7f826f7fe640 1 -- 172.21.15.99:0/682291383 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8270003b70 con 0x7f8278102300 2024-03-19T02:03:09.963 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.960+0000 7f826f7fe640 1 -- 172.21.15.99:0/682291383 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f8270003d10 con 0x7f8278102300 2024-03-19T02:03:09.963 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.960+0000 7f828033b640 1 -- 172.21.15.99:0/682291383 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f827819f9c0 con 0x7f8278102300 2024-03-19T02:03:09.963 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.960+0000 7f826f7fe640 1 -- 172.21.15.99:0/682291383 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f82700109f0 con 0x7f8278102300 2024-03-19T02:03:09.964 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.960+0000 7f828033b640 1 -- 172.21.15.99:0/682291383 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f827819fe90 con 0x7f8278102300 2024-03-19T02:03:09.971 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.962+0000 7f826f7fe640 1 -- 172.21.15.99:0/682291383 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f8270004020 con 0x7f8278102300 2024-03-19T02:03:09.971 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.963+0000 7f8283000740 1 Processor -- start 2024-03-19T02:03:09.971 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.963+0000 7f8283000740 1 -- start start 2024-03-19T02:03:09.971 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.963+0000 7f8283000740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56400ff26100 0x56400ff26520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:09.971 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.963+0000 7f8283000740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56400ff26af0 0x56400ff2b5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:09.971 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.963+0000 7f8283000740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56400ff2bc60 0x56400ff2e0f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:09.971 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.963+0000 7f826f7fe640 1 --2- 172.21.15.99:0/682291383 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f82640a28a0 0x7f82640a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:09.972 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.963+0000 7f8283000740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x56400fe8abe0 con 0x56400ff26100 2024-03-19T02:03:09.972 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.963+0000 7f8283000740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x56400fe33960 con 0x56400ff26af0 2024-03-19T02:03:09.972 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.963+0000 7f8283000740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x56400fe86c80 con 0x56400ff2bc60 2024-03-19T02:03:09.972 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.963+0000 7f827d8d6640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56400ff26100 0x56400ff26520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:09.972 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.963+0000 7f827d8d6640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56400ff26100 0x56400ff26520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:57204/0 (socket says 172.21.15.99:57204) 2024-03-19T02:03:09.972 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.963+0000 7f827e8d8640 1 --2- 172.21.15.99:0/682291383 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f82640a28a0 0x7f82640a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:09.972 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.963+0000 7f827d8d6640 1 -- 172.21.15.99:0/2506062052 learned_addr learned my addr 172.21.15.99:0/2506062052 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:03:09.972 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.963+0000 7f827e0d7640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56400ff26af0 0x56400ff2b5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:09.972 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.964+0000 7f827d8d6640 1 --2- 172.21.15.99:0/2506062052 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56400ff2bc60 0x56400ff2e0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:09.972 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.964+0000 7f826f7fe640 1 -- 172.21.15.99:0/682291383 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f82700ceea0 con 0x7f8278102300 2024-03-19T02:03:09.973 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.964+0000 7f827e0d7640 1 -- 172.21.15.99:0/2506062052 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56400ff2bc60 msgr2=0x56400ff2e0f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:03:09.973 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.964+0000 7f827e0d7640 1 --2- 172.21.15.99:0/2506062052 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56400ff2bc60 0x56400ff2e0f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:09.973 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.964+0000 7f827e0d7640 1 -- 172.21.15.99:0/2506062052 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56400ff26100 msgr2=0x56400ff26520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:03:09.973 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.964+0000 7f827e0d7640 1 --2- 172.21.15.99:0/2506062052 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56400ff26100 0x56400ff26520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:09.973 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.964+0000 7f827e0d7640 1 -- 172.21.15.99:0/2506062052 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x56400fe2ad20 con 0x56400ff26af0 2024-03-19T02:03:09.973 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.964+0000 7f827e8d8640 1 --2- 172.21.15.99:0/682291383 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f82640a28a0 0x7f82640a4d60 secure :-1 s=READY pgs=378 cs=0 l=1 rev1=1 crypto rx=0x7f82700a3a70 tx=0x7f8270090ce0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:03:09.973 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.964+0000 7f827e0d7640 1 --2- 172.21.15.99:0/2506062052 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56400ff26af0 0x56400ff2b5f0 secure :-1 s=READY pgs=218 cs=0 l=1 rev1=1 crypto rx=0x7f8274009bd0 tx=0x7f8274011810 comp rx=0 tx=0).ready entity=mon.2 client_cookie=abd788767724d336 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:03:09.973 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.964+0000 7f826cbfe640 1 -- 172.21.15.99:0/2506062052 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f827401e070 con 0x56400ff26af0 2024-03-19T02:03:09.973 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.964+0000 7f826cbfe640 1 -- 172.21.15.99:0/2506062052 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f8274009d80 con 0x56400ff26af0 2024-03-19T02:03:09.973 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.964+0000 7f826cbfe640 1 -- 172.21.15.99:0/2506062052 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8274019450 con 0x56400ff26af0 2024-03-19T02:03:09.973 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.965+0000 7f8283000740 1 -- 172.21.15.99:0/2506062052 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56400ff26af0 msgr2=0x56400ff2b5f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:03:09.973 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.965+0000 7f8283000740 1 --2- 172.21.15.99:0/2506062052 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56400ff26af0 0x56400ff2b5f0 secure :-1 s=READY pgs=218 cs=0 l=1 rev1=1 crypto rx=0x7f8274009bd0 tx=0x7f8274011810 comp rx=0 tx=0).stop 2024-03-19T02:03:09.973 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.965+0000 7f8283000740 1 -- 172.21.15.99:0/2506062052 shutdown_connections 2024-03-19T02:03:09.974 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.965+0000 7f8283000740 1 --2- 172.21.15.99:0/2506062052 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56400ff26af0 0x56400ff2b5f0 unknown :-1 s=CLOSED pgs=218 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:09.974 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.965+0000 7f8283000740 1 --2- 172.21.15.99:0/2506062052 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56400ff2bc60 0x56400ff2e0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:09.974 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.965+0000 7f8283000740 1 --2- 172.21.15.99:0/2506062052 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56400ff26100 0x56400ff26520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:09.974 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.965+0000 7f8283000740 1 -- 172.21.15.99:0/2506062052 >> 172.21.15.99:0/2506062052 conn(0x56400ff21670 msgr2=0x56400ff21a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:03:09.974 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.965+0000 7f8283000740 1 -- 172.21.15.99:0/2506062052 shutdown_connections 2024-03-19T02:03:09.974 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.965+0000 7f8283000740 1 -- 172.21.15.99:0/2506062052 wait complete. 2024-03-19T02:03:09.974 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.966+0000 7f8283000740 1 -- start start 2024-03-19T02:03:09.974 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.966+0000 7f8283000740 1 Processor -- start 2024-03-19T02:03:09.974 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.966+0000 7f8283000740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56400ff26100 0x56400ff34540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:09.974 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.966+0000 7f8283000740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56400ff26af0 0x56400ff34a80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:09.974 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.966+0000 7f8283000740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56400ff2bc60 0x56400ff3bac0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:09.974 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.966+0000 7f8283000740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x56400ff2ec10 con 0x56400ff26af0 2024-03-19T02:03:09.974 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.966+0000 7f8283000740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x56400fe46db0 con 0x56400ff2bc60 2024-03-19T02:03:09.975 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.966+0000 7f8283000740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x56400ff2ed90 con 0x56400ff26100 2024-03-19T02:03:09.975 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.966+0000 7f8283000740 10 client.0 _ll_register_callbacks cb 0x56400fe4be70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T02:03:09.975 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.966+0000 7f827e0d7640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56400ff26af0 0x56400ff34a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:09.975 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.966+0000 7f827e0d7640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56400ff26af0 0x56400ff34a80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:57214/0 (socket says 172.21.15.99:57214) 2024-03-19T02:03:09.975 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.966+0000 7f827e0d7640 1 -- 172.21.15.99:0/1994017592 learned_addr learned my addr 172.21.15.99:0/1994017592 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:03:09.975 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.966+0000 7f827d8d6640 1 --2- 172.21.15.99:0/1994017592 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56400ff2bc60 0x56400ff3bac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:09.975 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.966+0000 7f827d8d6640 1 --2- 172.21.15.99:0/1994017592 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56400ff26100 0x56400ff34540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:09.975 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.966+0000 7f827e0d7640 1 -- 172.21.15.99:0/1994017592 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56400ff26100 msgr2=0x56400ff34540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:03:09.976 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.966+0000 7f827e0d7640 1 --2- 172.21.15.99:0/1994017592 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56400ff26100 0x56400ff34540 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:09.976 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.966+0000 7f827e0d7640 1 -- 172.21.15.99:0/1994017592 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56400ff2bc60 msgr2=0x56400ff3bac0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:03:09.976 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.966+0000 7f827e0d7640 1 --2- 172.21.15.99:0/1994017592 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56400ff2bc60 0x56400ff3bac0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:09.976 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.966+0000 7f827e0d7640 1 -- 172.21.15.99:0/1994017592 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x56400fe45c60 con 0x56400ff26af0 2024-03-19T02:03:09.976 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.967+0000 7f827e0d7640 1 --2- 172.21.15.99:0/1994017592 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56400ff26af0 0x56400ff34a80 secure :-1 s=READY pgs=760 cs=0 l=1 rev1=1 crypto rx=0x7f8274012fd0 tx=0x7f82740039a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=891a4a5c95a92c6a server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:03:09.976 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.967+0000 7f8261ffb640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T02:03:09.976 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.967+0000 7f8261ffb640 1 -- 172.21.15.99:0/1994017592 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f827401e040 con 0x56400ff26af0 2024-03-19T02:03:09.976 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.967+0000 7f8261ffb640 1 -- 172.21.15.99:0/1994017592 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f8274004000 con 0x56400ff26af0 2024-03-19T02:03:09.976 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.967+0000 7f8283000740 1 -- 172.21.15.99:0/1994017592 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f8274009d80 con 0x56400ff26af0 2024-03-19T02:03:09.976 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.967+0000 7f8261ffb640 1 -- 172.21.15.99:0/1994017592 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7f8274004220 con 0x56400ff26af0 2024-03-19T02:03:09.976 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.967+0000 7f8283000740 10 client.17100 fetch_fsmap learned FSMap version 37 2024-03-19T02:03:09.976 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.967+0000 7f8283000740 1 -- 172.21.15.99:0/1994017592 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x56400fb9f210 con 0x56400ff26af0 2024-03-19T02:03:09.976 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.967+0000 7f8261ffb640 1 -- 172.21.15.99:0/1994017592 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f8274009d80 con 0x56400ff26af0 2024-03-19T02:03:09.976 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.967+0000 7f8261ffb640 1 -- 172.21.15.99:0/1994017592 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7f8274021030 con 0x56400ff26af0 2024-03-19T02:03:09.977 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.967+0000 7f8283000740 10 client.17100 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T02:03:09.977 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.967+0000 7f8283000740 10 client.17100 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T02:03:09.977 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.967+0000 7f8283000740 20 client.17100 populate_metadata read hostname 'smithi099' 2024-03-19T02:03:09.977 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.967+0000 7f8283000740 1 --2- 172.21.15.99:0/1994017592 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56400ff401e0 0x56400ff425d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:09.977 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.967+0000 7f8283000740 4 client.17100 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T02:03:09.977 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.967+0000 7f8283000740 1 -- 172.21.15.99:0/1994017592 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x56400fe98120 con 0x56400ff401e0 2024-03-19T02:03:09.977 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.967+0000 7f827d8d6640 1 --2- 172.21.15.99:0/1994017592 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56400ff401e0 0x56400ff425d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:09.977 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.968+0000 7f827d8d6640 1 --2- 172.21.15.99:0/1994017592 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56400ff401e0 0x56400ff425d0 crc :-1 s=READY pgs=225 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=93be9145396e6e7 server_cookie=d86c196a604446c5 in_seq=0 out_seq=0 2024-03-19T02:03:09.977 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.968+0000 7f8261ffb640 10 client.17100 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T02:03:09.978 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.977+0000 7f8261ffb640 1 -- 172.21.15.99:0/1994017592 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x56400fe98120 con 0x56400ff401e0 2024-03-19T02:03:09.978 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.977+0000 7f8261ffb640 10 client.17100 handle_command_reply: tid=0 2024-03-19T02:03:09.978 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.978+0000 7f8283000740 1 client.17100 shutdown 2024-03-19T02:03:09.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.978+0000 7f8283000740 1 -- 172.21.15.99:0/1994017592 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56400ff26af0 msgr2=0x56400ff34a80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:03:09.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.978+0000 7f8283000740 1 --2- 172.21.15.99:0/1994017592 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56400ff26af0 0x56400ff34a80 secure :-1 s=READY pgs=760 cs=0 l=1 rev1=1 crypto rx=0x7f8274012fd0 tx=0x7f82740039a0 comp rx=0 tx=0).stop 2024-03-19T02:03:09.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.978+0000 7f8283000740 1 -- 172.21.15.99:0/1994017592 shutdown_connections 2024-03-19T02:03:09.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.978+0000 7f8283000740 1 --2- 172.21.15.99:0/1994017592 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56400ff401e0 0x56400ff425d0 crc :-1 s=READY pgs=225 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:09.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.978+0000 7f8283000740 1 --2- 172.21.15.99:0/1994017592 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56400ff2bc60 0x56400ff3bac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:09.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.978+0000 7f8283000740 1 --2- 172.21.15.99:0/1994017592 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56400ff26af0 0x56400ff34a80 unknown :-1 s=CLOSED pgs=760 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:09.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.978+0000 7f8283000740 1 --2- 172.21.15.99:0/1994017592 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56400ff26100 0x56400ff34540 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:09.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.978+0000 7f8261ffb640 0 client.17100 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T02:03:09.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.978+0000 7f8283000740 1 -- 172.21.15.99:0/1994017592 >> 172.21.15.99:0/1994017592 conn(0x56400ff21670 msgr2=0x56400ff228c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:03:09.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.978+0000 7f8283000740 1 -- 172.21.15.99:0/1994017592 shutdown_connections 2024-03-19T02:03:09.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.978+0000 7f8283000740 1 -- 172.21.15.99:0/1994017592 wait complete. 2024-03-19T02:03:09.979 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.978+0000 7f8283000740 20 client.17100 trim_cache size 0 max 16384 2024-03-19T02:03:09.997 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.993+0000 7f8283000740 1 Processor -- start 2024-03-19T02:03:09.997 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.993+0000 7f8283000740 1 -- start start 2024-03-19T02:03:09.997 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.993+0000 7f8283000740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56400ff26100 0x56400ff2f8a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:09.997 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.993+0000 7f8283000740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56400ff26af0 0x56400ff3ba80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:09.997 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.993+0000 7f8283000740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56400ff2bc60 0x56400ff20220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:09.997 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.993+0000 7f8283000740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f82781a93a0 con 0x56400ff26100 2024-03-19T02:03:09.997 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.993+0000 7f8283000740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f82781a6c70 con 0x56400ff2bc60 2024-03-19T02:03:09.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.993+0000 7f8283000740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f82781a4ac0 con 0x56400ff26af0 2024-03-19T02:03:09.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.993+0000 7f827d8d6640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56400ff26100 0x56400ff2f8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:09.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.993+0000 7f827d8d6640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56400ff26100 0x56400ff2f8a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:57226/0 (socket says 172.21.15.99:57226) 2024-03-19T02:03:09.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.993+0000 7f827d8d6640 1 -- 172.21.15.99:0/1094510342 learned_addr learned my addr 172.21.15.99:0/1094510342 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:03:09.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.993+0000 7f827d8d6640 1 --2- 172.21.15.99:0/1094510342 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56400ff2bc60 0x56400ff20220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:09.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.993+0000 7f827d8d6640 1 -- 172.21.15.99:0/1094510342 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56400ff26af0 msgr2=0x56400ff3ba80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:03:09.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.993+0000 7f827d8d6640 1 --2- 172.21.15.99:0/1094510342 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56400ff26af0 0x56400ff3ba80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:09.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.993+0000 7f827d8d6640 1 -- 172.21.15.99:0/1094510342 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56400ff2bc60 msgr2=0x56400ff20220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:03:09.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.993+0000 7f827d8d6640 1 --2- 172.21.15.99:0/1094510342 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56400ff2bc60 0x56400ff20220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:09.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.993+0000 7f827d8d6640 1 -- 172.21.15.99:0/1094510342 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x56400fc6b620 con 0x56400ff26100 2024-03-19T02:03:09.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.993+0000 7f827d8d6640 1 --2- 172.21.15.99:0/1094510342 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56400ff26100 0x56400ff2f8a0 secure :-1 s=READY pgs=761 cs=0 l=1 rev1=1 crypto rx=0x7f826802c7b0 tx=0x7f8268009620 comp rx=0 tx=0).ready entity=mon.0 client_cookie=71ca2694562ee164 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:03:09.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.993+0000 7f8263fff640 1 -- 172.21.15.99:0/1094510342 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f826804ee20 con 0x56400ff26100 2024-03-19T02:03:09.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.993+0000 7f8263fff640 1 -- 172.21.15.99:0/1094510342 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f8268005340 con 0x56400ff26100 2024-03-19T02:03:09.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.993+0000 7f8263fff640 1 -- 172.21.15.99:0/1094510342 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8268004920 con 0x56400ff26100 2024-03-19T02:03:09.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.994+0000 7f8283000740 1 -- 172.21.15.99:0/1094510342 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56400ff26100 msgr2=0x56400ff2f8a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:03:09.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.994+0000 7f8283000740 1 --2- 172.21.15.99:0/1094510342 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56400ff26100 0x56400ff2f8a0 secure :-1 s=READY pgs=761 cs=0 l=1 rev1=1 crypto rx=0x7f826802c7b0 tx=0x7f8268009620 comp rx=0 tx=0).stop 2024-03-19T02:03:09.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.994+0000 7f8283000740 1 -- 172.21.15.99:0/1094510342 shutdown_connections 2024-03-19T02:03:09.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.994+0000 7f8283000740 1 --2- 172.21.15.99:0/1094510342 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56400ff2bc60 0x56400ff20220 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:09.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.994+0000 7f8283000740 1 --2- 172.21.15.99:0/1094510342 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56400ff26af0 0x56400ff3ba80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:09.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.994+0000 7f8283000740 1 --2- 172.21.15.99:0/1094510342 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56400ff26100 0x56400ff2f8a0 unknown :-1 s=CLOSED pgs=761 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:09.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.994+0000 7f8283000740 1 -- 172.21.15.99:0/1094510342 >> 172.21.15.99:0/1094510342 conn(0x56400ff21670 msgr2=0x56400ff2f190 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:03:09.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.994+0000 7f8283000740 1 -- 172.21.15.99:0/1094510342 shutdown_connections 2024-03-19T02:03:09.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.994+0000 7f8283000740 1 -- 172.21.15.99:0/1094510342 wait complete. 2024-03-19T02:03:09.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.994+0000 7f8283000740 1 -- start start 2024-03-19T02:03:09.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.995+0000 7f8283000740 1 Processor -- start 2024-03-19T02:03:09.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.995+0000 7f8283000740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56400ff26100 0x56400ff38dc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:09.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.995+0000 7f8283000740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56400ff26af0 0x56400ff39300 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:09.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.995+0000 7f8283000740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56400ff2bc60 0x56400ff3a310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:09.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.995+0000 7f8283000740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f82781a5130 con 0x56400ff2bc60 2024-03-19T02:03:09.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.995+0000 7f8283000740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f82781a65a0 con 0x56400ff26100 2024-03-19T02:03:10.000 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.995+0000 7f8283000740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f82781a5800 con 0x56400ff26af0 2024-03-19T02:03:10.000 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.995+0000 7f8283000740 10 client.0 _ll_register_callbacks cb 0x7f827401c070 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T02:03:10.000 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.995+0000 7f827d8d6640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56400ff2bc60 0x56400ff3a310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:10.000 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.995+0000 7f827d8d6640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56400ff2bc60 0x56400ff3a310 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:57236/0 (socket says 172.21.15.99:57236) 2024-03-19T02:03:10.000 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.995+0000 7f827d8d6640 1 -- 172.21.15.99:0/598345157 learned_addr learned my addr 172.21.15.99:0/598345157 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:03:10.000 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.995+0000 7f827d8d6640 1 --2- 172.21.15.99:0/598345157 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56400ff26100 0x56400ff38dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:10.000 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.995+0000 7f827e0d7640 1 --2- 172.21.15.99:0/598345157 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56400ff26af0 0x56400ff39300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:10.000 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.995+0000 7f827d8d6640 1 -- 172.21.15.99:0/598345157 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56400ff26af0 msgr2=0x56400ff39300 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:03:10.000 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.995+0000 7f827d8d6640 1 --2- 172.21.15.99:0/598345157 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56400ff26af0 0x56400ff39300 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:10.000 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.995+0000 7f827d8d6640 1 -- 172.21.15.99:0/598345157 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56400ff26100 msgr2=0x56400ff38dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:03:10.000 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.995+0000 7f827d8d6640 1 --2- 172.21.15.99:0/598345157 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56400ff26100 0x56400ff38dc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:10.000 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.995+0000 7f827d8d6640 1 -- 172.21.15.99:0/598345157 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x56400ff3a900 con 0x56400ff2bc60 2024-03-19T02:03:10.000 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.995+0000 7f827e0d7640 1 --2- 172.21.15.99:0/598345157 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56400ff26af0 0x56400ff39300 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-19T02:03:10.000 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.995+0000 7f827d8d6640 1 --2- 172.21.15.99:0/598345157 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56400ff2bc60 0x56400ff3a310 secure :-1 s=READY pgs=762 cs=0 l=1 rev1=1 crypto rx=0x7f8268005ec0 tx=0x7f826802c330 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4d9ed439b6841b56 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:03:10.001 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.995+0000 7f8261ffb640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T02:03:10.001 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.995+0000 7f8261ffb640 1 -- 172.21.15.99:0/598345157 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f826804e5b0 con 0x56400ff2bc60 2024-03-19T02:03:10.001 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.995+0000 7f8261ffb640 1 -- 172.21.15.99:0/598345157 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f826804e750 con 0x56400ff2bc60 2024-03-19T02:03:10.001 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.995+0000 7f8283000740 1 -- 172.21.15.99:0/598345157 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f8268005340 con 0x56400ff2bc60 2024-03-19T02:03:10.001 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.996+0000 7f8261ffb640 1 -- 172.21.15.99:0/598345157 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) ==== 24+0+0 (secure 0 0 0) 0x7f826804edd0 con 0x56400ff2bc60 2024-03-19T02:03:10.001 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.996+0000 7f8283000740 10 client.17106 fetch_fsmap learned FSMap version 37 2024-03-19T02:03:10.001 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.996+0000 7f8283000740 1 -- 172.21.15.99:0/598345157 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) -- 0x56400ff3bc10 con 0x56400ff2bc60 2024-03-19T02:03:10.001 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.996+0000 7f8261ffb640 1 -- 172.21.15.99:0/598345157 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f8268005340 con 0x56400ff2bc60 2024-03-19T02:03:10.001 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.996+0000 7f8261ffb640 1 -- 172.21.15.99:0/598345157 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 37) ==== 8338+0+0 (secure 0 0 0) 0x7f8268065080 con 0x56400ff2bc60 2024-03-19T02:03:10.001 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.996+0000 7f8283000740 10 client.17106 fetch_fsmap finished waiting for FSMap version 37 2024-03-19T02:03:10.001 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.996+0000 7f8283000740 10 client.17106 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T02:03:10.001 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.996+0000 7f8283000740 20 client.17106 populate_metadata read hostname 'smithi099' 2024-03-19T02:03:10.001 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.996+0000 7f8283000740 1 --2- 172.21.15.99:0/598345157 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56400ffa6a70 0x56400ffa8e90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:10.001 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.996+0000 7f8283000740 4 client.17106 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T02:03:10.002 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.996+0000 7f8283000740 1 -- 172.21.15.99:0/598345157 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x56400fe92470 con 0x56400ffa6a70 2024-03-19T02:03:10.002 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.996+0000 7f827e0d7640 1 --2- 172.21.15.99:0/598345157 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56400ffa6a70 0x56400ffa8e90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:10.002 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.997+0000 7f8261ffb640 10 client.17106 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T02:03:10.002 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:09.997+0000 7f827e0d7640 1 --2- 172.21.15.99:0/598345157 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56400ffa6a70 0x56400ffa8e90 crc :-1 s=READY pgs=226 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=3379ffe14f3513c9 server_cookie=76f49b618e3990f2 in_seq=0 out_seq=0 2024-03-19T02:03:10.118 INFO:tasks.workunit.client.0.smithi099.stdout: 10 245389 22.41 MB/sec execute 459 sec latency 105.917 ms 2024-03-19T02:03:10.556 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:03:10 smithi115 ceph-mon[67332]: pgmap v1013: 97 pgs: 97 active+clean; 653 MiB data, 8.8 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 5.4 MiB/s wr, 497 op/s 2024-03-19T02:03:10.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:03:10 smithi099 ceph-mon[62951]: pgmap v1013: 97 pgs: 97 active+clean; 653 MiB data, 8.8 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 5.4 MiB/s wr, 497 op/s 2024-03-19T02:03:10.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:03:10 smithi112 ceph-mon[66819]: pgmap v1013: 97 pgs: 97 active+clean; 653 MiB data, 8.8 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 5.4 MiB/s wr, 497 op/s 2024-03-19T02:03:10.745 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:10.744+0000 7f8261ffb640 1 -- 172.21.15.99:0/598345157 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x56400fe92470 con 0x56400ffa6a70 2024-03-19T02:03:10.745 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:10.744+0000 7f8261ffb640 10 client.17106 handle_command_reply: tid=0 2024-03-19T02:03:10.745 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:10.744+0000 7f8283000740 1 client.17106 shutdown 2024-03-19T02:03:10.745 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:10.744+0000 7f8283000740 1 -- 172.21.15.99:0/598345157 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56400ff2bc60 msgr2=0x56400ff3a310 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:03:10.745 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:10.744+0000 7f8283000740 1 --2- 172.21.15.99:0/598345157 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56400ff2bc60 0x56400ff3a310 secure :-1 s=READY pgs=762 cs=0 l=1 rev1=1 crypto rx=0x7f8268005ec0 tx=0x7f826802c330 comp rx=0 tx=0).stop 2024-03-19T02:03:10.745 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:10.744+0000 7f8283000740 1 -- 172.21.15.99:0/598345157 shutdown_connections 2024-03-19T02:03:10.745 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:10.744+0000 7f8283000740 1 --2- 172.21.15.99:0/598345157 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56400ffa6a70 0x56400ffa8e90 crc :-1 s=READY pgs=226 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:10.746 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:10.744+0000 7f8283000740 1 --2- 172.21.15.99:0/598345157 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56400ff2bc60 0x56400ff3a310 unknown :-1 s=CLOSED pgs=762 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:10.746 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:10.744+0000 7f8283000740 1 --2- 172.21.15.99:0/598345157 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56400ff26af0 0x56400ff39300 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:10.746 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:10.744+0000 7f8283000740 1 --2- 172.21.15.99:0/598345157 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56400ff26100 0x56400ff38dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:10.746 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:10.744+0000 7f8283000740 1 -- 172.21.15.99:0/598345157 >> 172.21.15.99:0/598345157 conn(0x56400ff21670 msgr2=0x56400ff2f8a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:03:10.746 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:10.744+0000 7f8261ffb640 0 client.17106 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T02:03:10.746 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:10.745+0000 7f8283000740 1 -- 172.21.15.99:0/598345157 shutdown_connections 2024-03-19T02:03:10.746 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:10.745+0000 7f8283000740 1 -- 172.21.15.99:0/598345157 wait complete. 2024-03-19T02:03:10.746 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:10.745+0000 7f8283000740 20 client.17106 trim_cache size 0 max 16384 2024-03-19T02:03:10.746 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T02:03:10.746 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T02:03:10.747 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T02:03:10.747 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T02:03:10.747 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:10.745+0000 7f828033b640 1 -- 172.21.15.99:0/682291383 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f82640a28a0 msgr2=0x7f82640a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:03:10.748 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:10.745+0000 7f828033b640 1 --2- 172.21.15.99:0/682291383 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f82640a28a0 0x7f82640a4d60 secure :-1 s=READY pgs=378 cs=0 l=1 rev1=1 crypto rx=0x7f82700a3a70 tx=0x7f8270090ce0 comp rx=0 tx=0).stop 2024-03-19T02:03:10.748 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:10.745+0000 7f828033b640 1 -- 172.21.15.99:0/682291383 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8278102300 msgr2=0x7f827819f060 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:03:10.748 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:10.745+0000 7f828033b640 1 --2- 172.21.15.99:0/682291383 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8278102300 0x7f827819f060 secure :-1 s=READY pgs=759 cs=0 l=1 rev1=1 crypto rx=0x7f827000efd0 tx=0x7f827000c6a0 comp rx=0 tx=0).stop 2024-03-19T02:03:10.748 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:10.745+0000 7f828033b640 1 -- 172.21.15.99:0/682291383 shutdown_connections 2024-03-19T02:03:10.748 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:10.745+0000 7f828033b640 1 --2- 172.21.15.99:0/682291383 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f8278102300 0x7f827819f060 unknown :-1 s=CLOSED pgs=759 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:10.748 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:10.745+0000 7f828033b640 1 --2- 172.21.15.99:0/682291383 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f8278101940 0x7f8278197ff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:10.748 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:10.745+0000 7f828033b640 1 --2- 172.21.15.99:0/682291383 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f82640a28a0 0x7f82640a4d60 unknown :-1 s=CLOSED pgs=378 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:10.748 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:10.745+0000 7f828033b640 1 --2- 172.21.15.99:0/682291383 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f8278100750 0x7f8278197ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:10.748 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:10.745+0000 7f828033b640 1 -- 172.21.15.99:0/682291383 >> 172.21.15.99:0/682291383 conn(0x7f82780fbcc0 msgr2=0x7f82780fd6d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:03:10.748 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:10.745+0000 7f828033b640 1 -- 172.21.15.99:0/682291383 shutdown_connections 2024-03-19T02:03:10.748 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:10.745+0000 7f828033b640 1 -- 172.21.15.99:0/682291383 wait complete. 2024-03-19T02:03:11.118 INFO:tasks.workunit.client.0.smithi099.stdout: 10 245615 22.38 MB/sec execute 460 sec latency 488.901 ms 2024-03-19T02:03:12.118 INFO:tasks.workunit.client.0.smithi099.stdout: 10 245846 22.36 MB/sec execute 461 sec latency 376.081 ms 2024-03-19T02:03:12.840 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:03:12 smithi099 ceph-mon[62951]: pgmap v1014: 97 pgs: 97 active+clean; 650 MiB data, 8.8 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 5.3 MiB/s wr, 517 op/s 2024-03-19T02:03:12.861 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:03:12 smithi112 ceph-mon[66819]: pgmap v1014: 97 pgs: 97 active+clean; 650 MiB data, 8.8 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 5.3 MiB/s wr, 517 op/s 2024-03-19T02:03:13.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:03:12 smithi115 ceph-mon[67332]: pgmap v1014: 97 pgs: 97 active+clean; 650 MiB data, 8.8 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 5.3 MiB/s wr, 517 op/s 2024-03-19T02:03:13.118 INFO:tasks.workunit.client.0.smithi099.stdout: 10 246306 22.37 MB/sec execute 462 sec latency 84.872 ms 2024-03-19T02:03:14.118 INFO:tasks.workunit.client.0.smithi099.stdout: 10 246727 22.38 MB/sec execute 463 sec latency 72.311 ms 2024-03-19T02:03:14.840 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:03:14 smithi099 ceph-mon[62951]: pgmap v1015: 97 pgs: 97 active+clean; 649 MiB data, 8.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 6.1 MiB/s wr, 560 op/s 2024-03-19T02:03:14.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:03:14 smithi112 ceph-mon[66819]: pgmap v1015: 97 pgs: 97 active+clean; 649 MiB data, 8.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 6.1 MiB/s wr, 560 op/s 2024-03-19T02:03:15.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:03:14 smithi115 ceph-mon[67332]: pgmap v1015: 97 pgs: 97 active+clean; 649 MiB data, 8.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 6.1 MiB/s wr, 560 op/s 2024-03-19T02:03:15.118 INFO:tasks.workunit.client.0.smithi099.stdout: 10 247171 22.37 MB/sec execute 464 sec latency 96.111 ms 2024-03-19T02:03:16.118 INFO:tasks.workunit.client.0.smithi099.stdout: 10 247755 22.37 MB/sec execute 465 sec latency 66.706 ms 2024-03-19T02:03:16.840 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:03:16 smithi099 ceph-mon[62951]: pgmap v1016: 97 pgs: 97 active+clean; 654 MiB data, 8.7 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 5.2 MiB/s wr, 532 op/s 2024-03-19T02:03:16.861 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:03:16 smithi112 ceph-mon[66819]: pgmap v1016: 97 pgs: 97 active+clean; 654 MiB data, 8.7 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 5.2 MiB/s wr, 532 op/s 2024-03-19T02:03:17.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:03:16 smithi115 ceph-mon[67332]: pgmap v1016: 97 pgs: 97 active+clean; 654 MiB data, 8.7 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 5.2 MiB/s wr, 532 op/s 2024-03-19T02:03:17.118 INFO:tasks.workunit.client.0.smithi099.stdout: 10 248128 22.39 MB/sec execute 466 sec latency 187.787 ms 2024-03-19T02:03:18.118 INFO:tasks.workunit.client.0.smithi099.stdout: 10 248573 22.42 MB/sec execute 467 sec latency 70.571 ms 2024-03-19T02:03:18.840 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:03:18 smithi099 ceph-mon[62951]: pgmap v1017: 97 pgs: 97 active+clean; 651 MiB data, 8.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.4 MiB/s rd, 6.0 MiB/s wr, 572 op/s 2024-03-19T02:03:18.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:03:18 smithi112 ceph-mon[66819]: pgmap v1017: 97 pgs: 97 active+clean; 651 MiB data, 8.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.4 MiB/s rd, 6.0 MiB/s wr, 572 op/s 2024-03-19T02:03:19.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:03:18 smithi115 ceph-mon[67332]: pgmap v1017: 97 pgs: 97 active+clean; 651 MiB data, 8.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.4 MiB/s rd, 6.0 MiB/s wr, 572 op/s 2024-03-19T02:03:19.118 INFO:tasks.workunit.client.0.smithi099.stdout: 10 248949 22.41 MB/sec execute 468 sec latency 85.618 ms 2024-03-19T02:03:20.118 INFO:tasks.workunit.client.0.smithi099.stdout: 10 249184 22.38 MB/sec execute 469 sec latency 96.122 ms 2024-03-19T02:03:20.840 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:03:20 smithi099 ceph-mon[62951]: pgmap v1018: 97 pgs: 97 active+clean; 653 MiB data, 8.8 GiB used, 1.0 TiB / 1.0 TiB avail; 3.4 MiB/s rd, 5.1 MiB/s wr, 519 op/s 2024-03-19T02:03:20.861 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:03:20 smithi112 ceph-mon[66819]: pgmap v1018: 97 pgs: 97 active+clean; 653 MiB data, 8.8 GiB used, 1.0 TiB / 1.0 TiB avail; 3.4 MiB/s rd, 5.1 MiB/s wr, 519 op/s 2024-03-19T02:03:21.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:03:20 smithi115 ceph-mon[67332]: pgmap v1018: 97 pgs: 97 active+clean; 653 MiB data, 8.8 GiB used, 1.0 TiB / 1.0 TiB avail; 3.4 MiB/s rd, 5.1 MiB/s wr, 519 op/s 2024-03-19T02:03:21.118 INFO:tasks.workunit.client.0.smithi099.stdout: 10 249536 22.40 MB/sec execute 470 sec latency 67.708 ms 2024-03-19T02:03:22.118 INFO:tasks.workunit.client.0.smithi099.stdout: 10 249801 22.38 MB/sec execute 471 sec latency 434.523 ms 2024-03-19T02:03:22.840 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:03:22 smithi099 ceph-mon[62951]: pgmap v1019: 97 pgs: 97 active+clean; 654 MiB data, 8.8 GiB used, 1.0 TiB / 1.0 TiB avail; 3.4 MiB/s rd, 5.3 MiB/s wr, 550 op/s 2024-03-19T02:03:22.861 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:03:22 smithi112 ceph-mon[66819]: pgmap v1019: 97 pgs: 97 active+clean; 654 MiB data, 8.8 GiB used, 1.0 TiB / 1.0 TiB avail; 3.4 MiB/s rd, 5.3 MiB/s wr, 550 op/s 2024-03-19T02:03:23.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:03:22 smithi115 ceph-mon[67332]: pgmap v1019: 97 pgs: 97 active+clean; 654 MiB data, 8.8 GiB used, 1.0 TiB / 1.0 TiB avail; 3.4 MiB/s rd, 5.3 MiB/s wr, 550 op/s 2024-03-19T02:03:23.118 INFO:tasks.workunit.client.0.smithi099.stdout: 10 250263 22.38 MB/sec execute 472 sec latency 99.293 ms 2024-03-19T02:03:23.848 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.845+0000 7fd49931d640 1 -- 172.21.15.99:0/746901351 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd494102330 msgr2=0x7fd494108bc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:03:23.849 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.845+0000 7fd49931d640 1 --2- 172.21.15.99:0/746901351 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd494102330 0x7fd494108bc0 secure :-1 s=READY pgs=763 cs=0 l=1 rev1=1 crypto rx=0x7fd48c009ca0 tx=0x7fd48c02c0f0 comp rx=0 tx=0).stop 2024-03-19T02:03:23.849 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.845+0000 7fd49931d640 1 -- 172.21.15.99:0/746901351 shutdown_connections 2024-03-19T02:03:23.849 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.845+0000 7fd49931d640 1 --2- 172.21.15.99:0/746901351 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd494101970 0x7fd494101df0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:23.849 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.845+0000 7fd49931d640 1 --2- 172.21.15.99:0/746901351 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd494102330 0x7fd494108bc0 unknown :-1 s=CLOSED pgs=763 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:23.849 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.845+0000 7fd49931d640 1 --2- 172.21.15.99:0/746901351 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd494100c60 0x7fd494101080 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:23.849 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.845+0000 7fd49931d640 1 -- 172.21.15.99:0/746901351 >> 172.21.15.99:0/746901351 conn(0x7fd4940fb470 msgr2=0x7fd4940fd890 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:03:23.849 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.845+0000 7fd49931d640 1 -- 172.21.15.99:0/746901351 shutdown_connections 2024-03-19T02:03:23.849 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.846+0000 7fd49931d640 1 -- 172.21.15.99:0/746901351 wait complete. 2024-03-19T02:03:23.849 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.846+0000 7fd49931d640 1 Processor -- start 2024-03-19T02:03:23.849 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.846+0000 7fd49931d640 1 -- start start 2024-03-19T02:03:23.849 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.846+0000 7fd49931d640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd494100c60 0x7fd494196910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:23.850 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.846+0000 7fd49931d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd494101970 0x7fd494196e50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:23.850 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.846+0000 7fd49931d640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd494102330 0x7fd49419dec0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:23.850 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.846+0000 7fd49931d640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fd49410c3b0 con 0x7fd494101970 2024-03-19T02:03:23.850 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.846+0000 7fd49931d640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fd49410c230 con 0x7fd494100c60 2024-03-19T02:03:23.850 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.846+0000 7fd49931d640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fd49410c530 con 0x7fd494102330 2024-03-19T02:03:23.850 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.846+0000 7fd48bfff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd494101970 0x7fd494196e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:23.850 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.846+0000 7fd492ffd640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd494100c60 0x7fd494196910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:23.850 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.846+0000 7fd48bfff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd494101970 0x7fd494196e50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:48616/0 (socket says 172.21.15.99:48616) 2024-03-19T02:03:23.850 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.846+0000 7fd48bfff640 1 -- 172.21.15.99:0/841354227 learned_addr learned my addr 172.21.15.99:0/841354227 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:03:23.850 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.846+0000 7fd4937fe640 1 --2- 172.21.15.99:0/841354227 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd494102330 0x7fd49419dec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:23.850 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.847+0000 7fd492ffd640 1 -- 172.21.15.99:0/841354227 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd494102330 msgr2=0x7fd49419dec0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:03:23.850 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.847+0000 7fd492ffd640 1 --2- 172.21.15.99:0/841354227 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd494102330 0x7fd49419dec0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:23.850 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.847+0000 7fd492ffd640 1 -- 172.21.15.99:0/841354227 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd494101970 msgr2=0x7fd494196e50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:03:23.851 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.847+0000 7fd492ffd640 1 --2- 172.21.15.99:0/841354227 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd494101970 0x7fd494196e50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:23.851 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.847+0000 7fd492ffd640 1 -- 172.21.15.99:0/841354227 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd49419e530 con 0x7fd494100c60 2024-03-19T02:03:23.851 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.847+0000 7fd4937fe640 1 --2- 172.21.15.99:0/841354227 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd494102330 0x7fd49419dec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-19T02:03:23.851 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.847+0000 7fd48bfff640 1 --2- 172.21.15.99:0/841354227 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd494101970 0x7fd494196e50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:03:23.851 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.847+0000 7fd492ffd640 1 --2- 172.21.15.99:0/841354227 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd494100c60 0x7fd494196910 secure :-1 s=READY pgs=219 cs=0 l=1 rev1=1 crypto rx=0x7fd48400ead0 tx=0x7fd48400ef90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:03:23.851 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.847+0000 7fd490ff9640 1 -- 172.21.15.99:0/841354227 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd48400bb70 con 0x7fd494100c60 2024-03-19T02:03:23.851 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.847+0000 7fd490ff9640 1 -- 172.21.15.99:0/841354227 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd48400bd10 con 0x7fd494100c60 2024-03-19T02:03:23.851 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.847+0000 7fd49931d640 1 -- 172.21.15.99:0/841354227 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd4940ff810 con 0x7fd494100c60 2024-03-19T02:03:23.852 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.847+0000 7fd490ff9640 1 -- 172.21.15.99:0/841354227 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd484017410 con 0x7fd494100c60 2024-03-19T02:03:23.852 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.847+0000 7fd49931d640 1 -- 172.21.15.99:0/841354227 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fd4940ffdb0 con 0x7fd494100c60 2024-03-19T02:03:23.859 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.849+0000 7fd490ff9640 1 -- 172.21.15.99:0/841354227 <== mon.2 v2:172.21.15.112:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7fd48401e020 con 0x7fd494100c60 2024-03-19T02:03:23.859 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.850+0000 7fd49bfe0740 1 Processor -- start 2024-03-19T02:03:23.859 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.850+0000 7fd49bfe0740 1 -- start start 2024-03-19T02:03:23.859 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.850+0000 7fd490ff9640 1 --2- 172.21.15.99:0/841354227 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd4800a2920 0x7fd4800a4de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:23.859 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.850+0000 7fd4937fe640 1 --2- 172.21.15.99:0/841354227 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd4800a2920 0x7fd4800a4de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:23.859 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.850+0000 7fd490ff9640 1 -- 172.21.15.99:0/841354227 <== mon.2 v2:172.21.15.112:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fd4840d0260 con 0x7fd494100c60 2024-03-19T02:03:23.859 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.850+0000 7fd49bfe0740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562859016100 0x562859016520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:23.859 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.850+0000 7fd49bfe0740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562859016af0 0x56285901b5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:23.859 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.850+0000 7fd49bfe0740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56285901bc60 0x56285901e0f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:23.859 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.850+0000 7fd49bfe0740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x562858f7abe0 con 0x56285901bc60 2024-03-19T02:03:23.860 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.850+0000 7fd49bfe0740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x562858f23960 con 0x562859016100 2024-03-19T02:03:23.860 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.850+0000 7fd49bfe0740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x562858f76c80 con 0x562859016af0 2024-03-19T02:03:23.860 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.851+0000 7fd492ffd640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562859016100 0x562859016520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:23.860 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.851+0000 7fd492ffd640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562859016100 0x562859016520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:49860/0 (socket says 172.21.15.99:49860) 2024-03-19T02:03:23.860 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.851+0000 7fd492ffd640 1 -- 172.21.15.99:0/1619927050 learned_addr learned my addr 172.21.15.99:0/1619927050 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:03:23.860 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.851+0000 7fd48bfff640 1 --2- 172.21.15.99:0/1619927050 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562859016af0 0x56285901b5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:23.860 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.851+0000 7fd4937fe640 1 --2- 172.21.15.99:0/1619927050 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56285901bc60 0x56285901e0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:23.860 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.851+0000 7fd492ffd640 1 -- 172.21.15.99:0/1619927050 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562859016af0 msgr2=0x56285901b5f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:03:23.860 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.851+0000 7fd492ffd640 1 --2- 172.21.15.99:0/1619927050 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562859016af0 0x56285901b5f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:23.860 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.851+0000 7fd492ffd640 1 -- 172.21.15.99:0/1619927050 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56285901bc60 msgr2=0x56285901e0f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:03:23.860 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.851+0000 7fd492ffd640 1 --2- 172.21.15.99:0/1619927050 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56285901bc60 0x56285901e0f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:23.860 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.851+0000 7fd492ffd640 1 -- 172.21.15.99:0/1619927050 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x562858f1ad20 con 0x562859016100 2024-03-19T02:03:23.860 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.851+0000 7fd4937fe640 1 --2- 172.21.15.99:0/1619927050 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56285901bc60 0x56285901e0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-19T02:03:23.860 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.851+0000 7fd4937fe640 1 --2- 172.21.15.99:0/841354227 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd4800a2920 0x7fd4800a4de0 secure :-1 s=READY pgs=380 cs=0 l=1 rev1=1 crypto rx=0x7fd48c006010 tx=0x7fd48c009920 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:03:23.861 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.851+0000 7fd492ffd640 1 --2- 172.21.15.99:0/1619927050 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562859016100 0x562859016520 secure :-1 s=READY pgs=220 cs=0 l=1 rev1=1 crypto rx=0x7fd484099590 tx=0x7fd48409d380 comp rx=0 tx=0).ready entity=mon.2 client_cookie=ff982bf6407fb111 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:03:23.861 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.851+0000 7fd4897fa640 1 -- 172.21.15.99:0/1619927050 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd4840b5070 con 0x562859016100 2024-03-19T02:03:23.861 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.851+0000 7fd4897fa640 1 -- 172.21.15.99:0/1619927050 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd4840a5360 con 0x562859016100 2024-03-19T02:03:23.861 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.851+0000 7fd4897fa640 1 -- 172.21.15.99:0/1619927050 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd4840a5130 con 0x562859016100 2024-03-19T02:03:23.861 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.852+0000 7fd49bfe0740 1 -- 172.21.15.99:0/1619927050 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562859016100 msgr2=0x562859016520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:03:23.861 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.852+0000 7fd49bfe0740 1 --2- 172.21.15.99:0/1619927050 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562859016100 0x562859016520 secure :-1 s=READY pgs=220 cs=0 l=1 rev1=1 crypto rx=0x7fd484099590 tx=0x7fd48409d380 comp rx=0 tx=0).stop 2024-03-19T02:03:23.861 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.852+0000 7fd49bfe0740 1 -- 172.21.15.99:0/1619927050 shutdown_connections 2024-03-19T02:03:23.861 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.852+0000 7fd49bfe0740 1 --2- 172.21.15.99:0/1619927050 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56285901bc60 0x56285901e0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:23.861 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.852+0000 7fd49bfe0740 1 --2- 172.21.15.99:0/1619927050 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562859016af0 0x56285901b5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:23.861 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.852+0000 7fd49bfe0740 1 --2- 172.21.15.99:0/1619927050 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562859016100 0x562859016520 unknown :-1 s=CLOSED pgs=220 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:23.861 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.852+0000 7fd49bfe0740 1 -- 172.21.15.99:0/1619927050 >> 172.21.15.99:0/1619927050 conn(0x562859011670 msgr2=0x562859011a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:03:23.861 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.852+0000 7fd49bfe0740 1 -- 172.21.15.99:0/1619927050 shutdown_connections 2024-03-19T02:03:23.862 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.852+0000 7fd49bfe0740 1 -- 172.21.15.99:0/1619927050 wait complete. 2024-03-19T02:03:23.862 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.852+0000 7fd49bfe0740 1 -- start start 2024-03-19T02:03:23.862 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.852+0000 7fd49bfe0740 1 Processor -- start 2024-03-19T02:03:23.862 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.853+0000 7fd49bfe0740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562859016100 0x5628590201c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:23.862 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.853+0000 7fd49bfe0740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562859016af0 0x562859020700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:23.862 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.853+0000 7fd49bfe0740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56285901bc60 0x562859027740 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:23.862 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.853+0000 7fd49bfe0740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x56285901ec10 con 0x56285901bc60 2024-03-19T02:03:23.862 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.853+0000 7fd49bfe0740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x562858f36db0 con 0x562859016af0 2024-03-19T02:03:23.862 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.853+0000 7fd49bfe0740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x56285901ed90 con 0x562859016100 2024-03-19T02:03:23.863 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.853+0000 7fd49bfe0740 10 client.0 _ll_register_callbacks cb 0x562858f3be70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T02:03:23.863 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.853+0000 7fd4937fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56285901bc60 0x562859027740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:23.863 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.853+0000 7fd4937fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56285901bc60 0x562859027740 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:48634/0 (socket says 172.21.15.99:48634) 2024-03-19T02:03:23.863 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.853+0000 7fd48bfff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562859016af0 0x562859020700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:23.863 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.853+0000 7fd4937fe640 1 -- 172.21.15.99:0/2129544602 learned_addr learned my addr 172.21.15.99:0/2129544602 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:03:23.863 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.853+0000 7fd492ffd640 1 --2- 172.21.15.99:0/2129544602 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562859016100 0x5628590201c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:23.863 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.853+0000 7fd48bfff640 1 -- 172.21.15.99:0/2129544602 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562859016100 msgr2=0x5628590201c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:03:23.863 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.853+0000 7fd48bfff640 1 --2- 172.21.15.99:0/2129544602 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562859016100 0x5628590201c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:23.863 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.853+0000 7fd48bfff640 1 -- 172.21.15.99:0/2129544602 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56285901bc60 msgr2=0x562859027740 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:03:23.863 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.853+0000 7fd48bfff640 1 --2- 172.21.15.99:0/2129544602 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56285901bc60 0x562859027740 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:23.863 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.853+0000 7fd48bfff640 1 -- 172.21.15.99:0/2129544602 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x562858f35c60 con 0x562859016af0 2024-03-19T02:03:23.863 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.853+0000 7fd4937fe640 1 --2- 172.21.15.99:0/2129544602 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56285901bc60 0x562859027740 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:03:23.863 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.853+0000 7fd48bfff640 1 --2- 172.21.15.99:0/2129544602 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562859016af0 0x562859020700 secure :-1 s=READY pgs=221 cs=0 l=1 rev1=1 crypto rx=0x562859016ef0 tx=0x7fd47c00fad0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=e039e657f19f13ed server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:03:23.863 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.853+0000 7fd47a7fc640 10 client.0 ms_handle_connect on v2:172.21.15.112:3300/0 2024-03-19T02:03:23.864 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.853+0000 7fd47a7fc640 1 -- 172.21.15.99:0/2129544602 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd47c004320 con 0x562859016af0 2024-03-19T02:03:23.864 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.853+0000 7fd49bfe0740 1 -- 172.21.15.99:0/2129544602 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fd4840a5360 con 0x562859016af0 2024-03-19T02:03:23.864 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.853+0000 7fd47a7fc640 1 -- 172.21.15.99:0/2129544602 <== mon.2 v2:172.21.15.112:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fd47c016110 con 0x562859016af0 2024-03-19T02:03:23.864 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.853+0000 7fd47a7fc640 1 -- 172.21.15.99:0/2129544602 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_get_version_reply(handle=1 version=38) ==== 24+0+0 (secure 0 0 0) 0x7fd47c0044c0 con 0x562859016af0 2024-03-19T02:03:23.864 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.853+0000 7fd49bfe0740 10 client.26894 fetch_fsmap learned FSMap version 38 2024-03-19T02:03:23.864 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.853+0000 7fd49bfe0740 1 -- 172.21.15.99:0/2129544602 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,fsmap=38,monmap=4+}) -- 0x562858c8f210 con 0x562859016af0 2024-03-19T02:03:23.864 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.854+0000 7fd47a7fc640 1 -- 172.21.15.99:0/2129544602 <== mon.2 v2:172.21.15.112:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd4840a5360 con 0x562859016af0 2024-03-19T02:03:23.864 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.854+0000 7fd47a7fc640 1 -- 172.21.15.99:0/2129544602 <== mon.2 v2:172.21.15.112:3300/0 5 ==== fsmap(e 38) ==== 8342+0+0 (secure 0 0 0) 0x7fd47c003630 con 0x562859016af0 2024-03-19T02:03:23.864 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.854+0000 7fd49bfe0740 10 client.26894 fetch_fsmap finished waiting for FSMap version 38 2024-03-19T02:03:23.864 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.854+0000 7fd49bfe0740 10 client.26894 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T02:03:23.864 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.854+0000 7fd49bfe0740 20 client.26894 populate_metadata read hostname 'smithi099' 2024-03-19T02:03:23.864 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.854+0000 7fd49bfe0740 1 --2- 172.21.15.99:0/2129544602 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56285902c0e0 0x56285902e510 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:23.864 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.854+0000 7fd49bfe0740 4 client.26894 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T02:03:23.864 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.854+0000 7fd49bfe0740 1 -- 172.21.15.99:0/2129544602 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x562858f88120 con 0x56285902c0e0 2024-03-19T02:03:23.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.854+0000 7fd492ffd640 1 --2- 172.21.15.99:0/2129544602 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56285902c0e0 0x56285902e510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:23.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.854+0000 7fd47a7fc640 10 client.26894 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T02:03:23.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.854+0000 7fd492ffd640 1 --2- 172.21.15.99:0/2129544602 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56285902c0e0 0x56285902e510 crc :-1 s=READY pgs=227 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=c1cc273a7d199db0 server_cookie=9c1d0ea37ee65170 in_seq=0 out_seq=0 2024-03-19T02:03:23.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.857+0000 7fd47a7fc640 1 -- 172.21.15.99:0/2129544602 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x562858f88120 con 0x56285902c0e0 2024-03-19T02:03:23.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.857+0000 7fd47a7fc640 10 client.26894 handle_command_reply: tid=0 2024-03-19T02:03:23.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.857+0000 7fd49bfe0740 1 client.26894 shutdown 2024-03-19T02:03:23.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.858+0000 7fd49bfe0740 1 -- 172.21.15.99:0/2129544602 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562859016af0 msgr2=0x562859020700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:03:23.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.858+0000 7fd49bfe0740 1 --2- 172.21.15.99:0/2129544602 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562859016af0 0x562859020700 secure :-1 s=READY pgs=221 cs=0 l=1 rev1=1 crypto rx=0x562859016ef0 tx=0x7fd47c00fad0 comp rx=0 tx=0).stop 2024-03-19T02:03:23.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.858+0000 7fd49bfe0740 1 -- 172.21.15.99:0/2129544602 shutdown_connections 2024-03-19T02:03:23.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.858+0000 7fd49bfe0740 1 --2- 172.21.15.99:0/2129544602 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56285902c0e0 0x56285902e510 crc :-1 s=READY pgs=227 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:23.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.858+0000 7fd49bfe0740 1 --2- 172.21.15.99:0/2129544602 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562859016af0 0x562859020700 unknown :-1 s=CLOSED pgs=221 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:23.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.858+0000 7fd49bfe0740 1 --2- 172.21.15.99:0/2129544602 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56285901bc60 0x562859027740 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:23.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.858+0000 7fd47a7fc640 0 client.26894 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T02:03:23.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.858+0000 7fd49bfe0740 1 --2- 172.21.15.99:0/2129544602 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x562859016100 0x5628590201c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:23.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.858+0000 7fd49bfe0740 1 -- 172.21.15.99:0/2129544602 >> 172.21.15.99:0/2129544602 conn(0x562859011670 msgr2=0x5628590128c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:03:23.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.858+0000 7fd49bfe0740 1 -- 172.21.15.99:0/2129544602 shutdown_connections 2024-03-19T02:03:23.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.858+0000 7fd49bfe0740 1 -- 172.21.15.99:0/2129544602 wait complete. 2024-03-19T02:03:23.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.858+0000 7fd49bfe0740 20 client.26894 trim_cache size 0 max 16384 2024-03-19T02:03:23.866 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:03:23 smithi112 ceph-mon[66819]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T02:03:23.882 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.875+0000 7fd49bfe0740 1 Processor -- start 2024-03-19T02:03:23.883 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.875+0000 7fd49bfe0740 1 -- start start 2024-03-19T02:03:23.883 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.875+0000 7fd49bfe0740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562859016100 0x562859010ab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:23.883 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.875+0000 7fd49bfe0740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562859016af0 0x562859027700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:23.883 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.875+0000 7fd49bfe0740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56285901bc60 0x562859014fe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:23.883 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.875+0000 7fd49bfe0740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fd4941a93c0 con 0x562859016af0 2024-03-19T02:03:23.883 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.875+0000 7fd49bfe0740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fd4941a6c90 con 0x562859016100 2024-03-19T02:03:23.883 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.875+0000 7fd49bfe0740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fd4941a4ae0 con 0x56285901bc60 2024-03-19T02:03:23.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.875+0000 7fd48bfff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562859016af0 0x562859027700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:23.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.875+0000 7fd492ffd640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562859016100 0x562859010ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:23.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.875+0000 7fd492ffd640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562859016100 0x562859010ab0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:49882/0 (socket says 172.21.15.99:49882) 2024-03-19T02:03:23.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.875+0000 7fd492ffd640 1 -- 172.21.15.99:0/3412935357 learned_addr learned my addr 172.21.15.99:0/3412935357 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:03:23.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.875+0000 7fd4937fe640 1 --2- 172.21.15.99:0/3412935357 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56285901bc60 0x562859014fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:23.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.876+0000 7fd48bfff640 1 -- 172.21.15.99:0/3412935357 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56285901bc60 msgr2=0x562859014fe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:03:23.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.876+0000 7fd48bfff640 1 --2- 172.21.15.99:0/3412935357 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56285901bc60 0x562859014fe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:23.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.876+0000 7fd48bfff640 1 -- 172.21.15.99:0/3412935357 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562859016100 msgr2=0x562859010ab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:03:23.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.876+0000 7fd48bfff640 1 --2- 172.21.15.99:0/3412935357 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562859016100 0x562859010ab0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:23.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.876+0000 7fd48bfff640 1 -- 172.21.15.99:0/3412935357 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x562858d5b620 con 0x562859016af0 2024-03-19T02:03:23.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.876+0000 7fd492ffd640 1 --2- 172.21.15.99:0/3412935357 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562859016100 0x562859010ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:03:23.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.876+0000 7fd48bfff640 1 --2- 172.21.15.99:0/3412935357 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562859016af0 0x562859027700 secure :-1 s=READY pgs=764 cs=0 l=1 rev1=1 crypto rx=0x7fd47c003c90 tx=0x7fd47c016d20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=29f4f5237a40b519 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:03:23.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.876+0000 7fd488ff9640 1 -- 172.21.15.99:0/3412935357 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd47c02f070 con 0x562859016af0 2024-03-19T02:03:23.884 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.876+0000 7fd488ff9640 1 -- 172.21.15.99:0/3412935357 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd47c0165f0 con 0x562859016af0 2024-03-19T02:03:23.885 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.876+0000 7fd488ff9640 1 -- 172.21.15.99:0/3412935357 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd47c0052a0 con 0x562859016af0 2024-03-19T02:03:23.885 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.877+0000 7fd49bfe0740 1 -- 172.21.15.99:0/3412935357 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562859016af0 msgr2=0x562859027700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:03:23.885 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.877+0000 7fd49bfe0740 1 --2- 172.21.15.99:0/3412935357 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562859016af0 0x562859027700 secure :-1 s=READY pgs=764 cs=0 l=1 rev1=1 crypto rx=0x7fd47c003c90 tx=0x7fd47c016d20 comp rx=0 tx=0).stop 2024-03-19T02:03:23.885 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.877+0000 7fd49bfe0740 1 -- 172.21.15.99:0/3412935357 shutdown_connections 2024-03-19T02:03:23.885 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.877+0000 7fd49bfe0740 1 --2- 172.21.15.99:0/3412935357 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56285901bc60 0x562859014fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:23.885 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.877+0000 7fd49bfe0740 1 --2- 172.21.15.99:0/3412935357 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562859016af0 0x562859027700 unknown :-1 s=CLOSED pgs=764 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:23.885 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.877+0000 7fd49bfe0740 1 --2- 172.21.15.99:0/3412935357 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562859016100 0x562859010ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:23.885 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.877+0000 7fd49bfe0740 1 -- 172.21.15.99:0/3412935357 >> 172.21.15.99:0/3412935357 conn(0x562859011670 msgr2=0x5628590148d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:03:23.885 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.877+0000 7fd49bfe0740 1 -- 172.21.15.99:0/3412935357 shutdown_connections 2024-03-19T02:03:23.885 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.877+0000 7fd49bfe0740 1 -- 172.21.15.99:0/3412935357 wait complete. 2024-03-19T02:03:23.885 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.877+0000 7fd49bfe0740 1 -- start start 2024-03-19T02:03:23.885 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.877+0000 7fd49bfe0740 1 Processor -- start 2024-03-19T02:03:23.885 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.878+0000 7fd49bfe0740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562859016100 0x562859025360 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:23.885 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.878+0000 7fd49bfe0740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562859016af0 0x5628590258a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:23.886 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.878+0000 7fd49bfe0740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56285901bc60 0x5628590268b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:23.886 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.878+0000 7fd49bfe0740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fd4941a5150 con 0x562859016100 2024-03-19T02:03:23.886 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.878+0000 7fd49bfe0740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fd4941a65c0 con 0x562859016af0 2024-03-19T02:03:23.886 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.878+0000 7fd49bfe0740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fd4941a5820 con 0x56285901bc60 2024-03-19T02:03:23.886 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.878+0000 7fd49bfe0740 10 client.0 _ll_register_callbacks cb 0x7fd47c009cf0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T02:03:23.886 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.878+0000 7fd492ffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562859016100 0x562859025360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:23.886 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.878+0000 7fd492ffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562859016100 0x562859025360 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:48652/0 (socket says 172.21.15.99:48652) 2024-03-19T02:03:23.886 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.878+0000 7fd492ffd640 1 -- 172.21.15.99:0/2125025022 learned_addr learned my addr 172.21.15.99:0/2125025022 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:03:23.886 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.878+0000 7fd48bfff640 1 --2- 172.21.15.99:0/2125025022 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562859016af0 0x5628590258a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:23.886 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.878+0000 7fd4937fe640 1 --2- 172.21.15.99:0/2125025022 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56285901bc60 0x5628590268b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:23.886 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.878+0000 7fd48bfff640 1 -- 172.21.15.99:0/2125025022 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56285901bc60 msgr2=0x5628590268b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:03:23.886 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.878+0000 7fd48bfff640 1 --2- 172.21.15.99:0/2125025022 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56285901bc60 0x5628590268b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:23.886 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.878+0000 7fd48bfff640 1 -- 172.21.15.99:0/2125025022 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562859016100 msgr2=0x562859025360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:03:23.887 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.878+0000 7fd48bfff640 1 --2- 172.21.15.99:0/2125025022 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562859016100 0x562859025360 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:23.887 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.878+0000 7fd48bfff640 1 -- 172.21.15.99:0/2125025022 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x562859026ea0 con 0x562859016af0 2024-03-19T02:03:23.887 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.878+0000 7fd492ffd640 1 --2- 172.21.15.99:0/2125025022 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562859016100 0x562859025360 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:03:23.887 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.878+0000 7fd48bfff640 1 --2- 172.21.15.99:0/2125025022 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562859016af0 0x5628590258a0 secure :-1 s=READY pgs=222 cs=0 l=1 rev1=1 crypto rx=0x56285901e6f0 tx=0x7fd47c003820 comp rx=0 tx=0).ready entity=mon.2 client_cookie=e6cbaae96f58a2df server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:03:23.887 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.878+0000 7fd47a7fc640 10 client.0 ms_handle_connect on v2:172.21.15.112:3300/0 2024-03-19T02:03:23.887 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.878+0000 7fd47a7fc640 1 -- 172.21.15.99:0/2125025022 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd47c038020 con 0x562859016af0 2024-03-19T02:03:23.887 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.878+0000 7fd49bfe0740 1 -- 172.21.15.99:0/2125025022 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fd47c0165f0 con 0x562859016af0 2024-03-19T02:03:23.887 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.878+0000 7fd47a7fc640 1 -- 172.21.15.99:0/2125025022 <== mon.2 v2:172.21.15.112:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fd47c014840 con 0x562859016af0 2024-03-19T02:03:23.887 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.879+0000 7fd47a7fc640 1 -- 172.21.15.99:0/2125025022 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_get_version_reply(handle=1 version=38) ==== 24+0+0 (secure 0 0 0) 0x7fd47c043050 con 0x562859016af0 2024-03-19T02:03:23.887 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.879+0000 7fd49bfe0740 10 client.26900 fetch_fsmap learned FSMap version 38 2024-03-19T02:03:23.887 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.879+0000 7fd49bfe0740 1 -- 172.21.15.99:0/2125025022 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,fsmap=38,monmap=4+}) -- 0x5628590928d0 con 0x562859016af0 2024-03-19T02:03:23.888 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.879+0000 7fd47a7fc640 1 -- 172.21.15.99:0/2125025022 <== mon.2 v2:172.21.15.112:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fd47c0165f0 con 0x562859016af0 2024-03-19T02:03:23.888 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.879+0000 7fd47a7fc640 1 -- 172.21.15.99:0/2125025022 <== mon.2 v2:172.21.15.112:3300/0 5 ==== fsmap(e 38) ==== 8342+0+0 (secure 0 0 0) 0x7fd47c004660 con 0x562859016af0 2024-03-19T02:03:23.888 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.879+0000 7fd49bfe0740 10 client.26900 fetch_fsmap finished waiting for FSMap version 38 2024-03-19T02:03:23.888 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.879+0000 7fd49bfe0740 10 client.26900 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T02:03:23.888 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.879+0000 7fd49bfe0740 20 client.26900 populate_metadata read hostname 'smithi099' 2024-03-19T02:03:23.888 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.879+0000 7fd49bfe0740 1 --2- 172.21.15.99:0/2125025022 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5628590931e0 0x562859095600 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:23.888 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.879+0000 7fd49bfe0740 4 client.26900 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T02:03:23.888 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.879+0000 7fd49bfe0740 1 -- 172.21.15.99:0/2125025022 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x56285901ffb0 con 0x5628590931e0 2024-03-19T02:03:23.888 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.879+0000 7fd492ffd640 1 --2- 172.21.15.99:0/2125025022 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5628590931e0 0x562859095600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:23.888 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.879+0000 7fd492ffd640 1 --2- 172.21.15.99:0/2125025022 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5628590931e0 0x562859095600 crc :-1 s=READY pgs=228 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=cd8ec9987c72b2ec server_cookie=69d06ee4bf00b055 in_seq=0 out_seq=0 2024-03-19T02:03:23.888 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:23.879+0000 7fd47a7fc640 10 client.26900 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T02:03:24.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:03:23 smithi115 ceph-mon[67332]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T02:03:24.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:03:23 smithi099 ceph-mon[62951]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T02:03:24.118 INFO:tasks.workunit.client.0.smithi099.stdout: 10 250638 22.37 MB/sec execute 473 sec latency 157.244 ms 2024-03-19T02:03:24.508 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:24.507+0000 7fd47a7fc640 1 -- 172.21.15.99:0/2125025022 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x56285901ffb0 con 0x5628590931e0 2024-03-19T02:03:24.508 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:24.507+0000 7fd47a7fc640 10 client.26900 handle_command_reply: tid=0 2024-03-19T02:03:24.508 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:24.507+0000 7fd49bfe0740 1 client.26900 shutdown 2024-03-19T02:03:24.508 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:24.507+0000 7fd49bfe0740 1 -- 172.21.15.99:0/2125025022 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562859016af0 msgr2=0x5628590258a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:03:24.508 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:24.507+0000 7fd49bfe0740 1 --2- 172.21.15.99:0/2125025022 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562859016af0 0x5628590258a0 secure :-1 s=READY pgs=222 cs=0 l=1 rev1=1 crypto rx=0x56285901e6f0 tx=0x7fd47c003820 comp rx=0 tx=0).stop 2024-03-19T02:03:24.508 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:24.508+0000 7fd49bfe0740 1 -- 172.21.15.99:0/2125025022 shutdown_connections 2024-03-19T02:03:24.508 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:24.508+0000 7fd49bfe0740 1 --2- 172.21.15.99:0/2125025022 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5628590931e0 0x562859095600 crc :-1 s=READY pgs=228 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:24.509 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:24.508+0000 7fd49bfe0740 1 --2- 172.21.15.99:0/2125025022 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x562859016af0 0x5628590258a0 unknown :-1 s=CLOSED pgs=222 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:24.509 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:24.508+0000 7fd47a7fc640 0 client.26900 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T02:03:24.509 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:24.508+0000 7fd49bfe0740 1 --2- 172.21.15.99:0/2125025022 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56285901bc60 0x5628590268b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:24.509 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:24.508+0000 7fd49bfe0740 1 --2- 172.21.15.99:0/2125025022 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x562859016100 0x562859025360 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:24.509 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:24.508+0000 7fd49bfe0740 1 -- 172.21.15.99:0/2125025022 >> 172.21.15.99:0/2125025022 conn(0x562859011670 msgr2=0x562859015160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:03:24.509 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:24.508+0000 7fd49bfe0740 1 -- 172.21.15.99:0/2125025022 shutdown_connections 2024-03-19T02:03:24.509 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:24.508+0000 7fd49bfe0740 1 -- 172.21.15.99:0/2125025022 wait complete. 2024-03-19T02:03:24.509 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:24.508+0000 7fd49bfe0740 20 client.26900 trim_cache size 0 max 16384 2024-03-19T02:03:24.509 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T02:03:24.509 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T02:03:24.509 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T02:03:24.510 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T02:03:24.510 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:24.508+0000 7fd49931d640 1 -- 172.21.15.99:0/841354227 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd4800a2920 msgr2=0x7fd4800a4de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:03:24.510 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:24.508+0000 7fd49931d640 1 --2- 172.21.15.99:0/841354227 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd4800a2920 0x7fd4800a4de0 secure :-1 s=READY pgs=380 cs=0 l=1 rev1=1 crypto rx=0x7fd48c006010 tx=0x7fd48c009920 comp rx=0 tx=0).stop 2024-03-19T02:03:24.511 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:24.508+0000 7fd49931d640 1 -- 172.21.15.99:0/841354227 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd494100c60 msgr2=0x7fd494196910 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:03:24.511 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:24.508+0000 7fd49931d640 1 --2- 172.21.15.99:0/841354227 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd494100c60 0x7fd494196910 secure :-1 s=READY pgs=219 cs=0 l=1 rev1=1 crypto rx=0x7fd48400ead0 tx=0x7fd48400ef90 comp rx=0 tx=0).stop 2024-03-19T02:03:24.511 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:24.508+0000 7fd49931d640 1 -- 172.21.15.99:0/841354227 shutdown_connections 2024-03-19T02:03:24.511 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:24.508+0000 7fd49931d640 1 --2- 172.21.15.99:0/841354227 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fd494102330 0x7fd49419dec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:24.511 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:24.508+0000 7fd49931d640 1 --2- 172.21.15.99:0/841354227 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fd494101970 0x7fd494196e50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:24.511 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:24.508+0000 7fd49931d640 1 --2- 172.21.15.99:0/841354227 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fd4800a2920 0x7fd4800a4de0 unknown :-1 s=CLOSED pgs=380 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:24.511 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:24.508+0000 7fd49931d640 1 --2- 172.21.15.99:0/841354227 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fd494100c60 0x7fd494196910 unknown :-1 s=CLOSED pgs=219 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:24.511 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:24.508+0000 7fd49931d640 1 -- 172.21.15.99:0/841354227 >> 172.21.15.99:0/841354227 conn(0x7fd4940fb470 msgr2=0x7fd494104c10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:03:24.511 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:24.508+0000 7fd49931d640 1 -- 172.21.15.99:0/841354227 shutdown_connections 2024-03-19T02:03:24.511 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:24.508+0000 7fd49931d640 1 -- 172.21.15.99:0/841354227 wait complete. 2024-03-19T02:03:24.861 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:03:24 smithi112 ceph-mon[66819]: pgmap v1020: 97 pgs: 97 active+clean; 656 MiB data, 8.8 GiB used, 1.0 TiB / 1.0 TiB avail; 3.6 MiB/s rd, 6.2 MiB/s wr, 534 op/s 2024-03-19T02:03:25.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:03:24 smithi115 ceph-mon[67332]: pgmap v1020: 97 pgs: 97 active+clean; 656 MiB data, 8.8 GiB used, 1.0 TiB / 1.0 TiB avail; 3.6 MiB/s rd, 6.2 MiB/s wr, 534 op/s 2024-03-19T02:03:25.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:03:24 smithi099 ceph-mon[62951]: pgmap v1020: 97 pgs: 97 active+clean; 656 MiB data, 8.8 GiB used, 1.0 TiB / 1.0 TiB avail; 3.6 MiB/s rd, 6.2 MiB/s wr, 534 op/s 2024-03-19T02:03:25.118 INFO:tasks.workunit.client.0.smithi099.stdout: 10 250947 22.37 MB/sec execute 474 sec latency 471.957 ms 2024-03-19T02:03:26.118 INFO:tasks.workunit.client.0.smithi099.stdout: 10 251389 22.37 MB/sec execute 475 sec latency 68.599 ms 2024-03-19T02:03:27.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:03:26 smithi115 ceph-mon[67332]: pgmap v1021: 97 pgs: 97 active+clean; 658 MiB data, 8.8 GiB used, 1.0 TiB / 1.0 TiB avail; 3.3 MiB/s rd, 5.2 MiB/s wr, 405 op/s 2024-03-19T02:03:27.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:03:26 smithi115 ceph-mon[67332]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T02:03:27.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:03:26 smithi099 ceph-mon[62951]: pgmap v1021: 97 pgs: 97 active+clean; 658 MiB data, 8.8 GiB used, 1.0 TiB / 1.0 TiB avail; 3.3 MiB/s rd, 5.2 MiB/s wr, 405 op/s 2024-03-19T02:03:27.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:03:26 smithi099 ceph-mon[62951]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T02:03:27.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:03:26 smithi112 ceph-mon[66819]: pgmap v1021: 97 pgs: 97 active+clean; 658 MiB data, 8.8 GiB used, 1.0 TiB / 1.0 TiB avail; 3.3 MiB/s rd, 5.2 MiB/s wr, 405 op/s 2024-03-19T02:03:27.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:03:26 smithi112 ceph-mon[66819]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T02:03:27.118 INFO:tasks.workunit.client.0.smithi099.stdout: 10 251700 22.35 MB/sec execute 476 sec latency 268.829 ms 2024-03-19T02:03:28.119 INFO:tasks.workunit.client.0.smithi099.stdout: 10 252080 22.35 MB/sec execute 477 sec latency 160.719 ms 2024-03-19T02:03:29.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:03:28 smithi115 ceph-mon[67332]: pgmap v1022: 97 pgs: 97 active+clean; 654 MiB data, 8.8 GiB used, 1.0 TiB / 1.0 TiB avail; 3.2 MiB/s rd, 6.3 MiB/s wr, 518 op/s 2024-03-19T02:03:29.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:03:28 smithi115 ceph-mon[67332]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T02:03:29.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:03:28 smithi099 ceph-mon[62951]: pgmap v1022: 97 pgs: 97 active+clean; 654 MiB data, 8.8 GiB used, 1.0 TiB / 1.0 TiB avail; 3.2 MiB/s rd, 6.3 MiB/s wr, 518 op/s 2024-03-19T02:03:29.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:03:28 smithi099 ceph-mon[62951]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T02:03:29.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:03:28 smithi112 ceph-mon[66819]: pgmap v1022: 97 pgs: 97 active+clean; 654 MiB data, 8.8 GiB used, 1.0 TiB / 1.0 TiB avail; 3.2 MiB/s rd, 6.3 MiB/s wr, 518 op/s 2024-03-19T02:03:29.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:03:28 smithi112 ceph-mon[66819]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T02:03:29.118 INFO:tasks.workunit.client.0.smithi099.stdout: 10 252638 22.37 MB/sec execute 478 sec latency 75.756 ms 2024-03-19T02:03:30.119 INFO:tasks.workunit.client.0.smithi099.stdout: 10 253079 22.38 MB/sec execute 479 sec latency 72.366 ms 2024-03-19T02:03:30.913 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:03:30 smithi112 ceph-mon[66819]: pgmap v1023: 97 pgs: 97 active+clean; 650 MiB data, 8.8 GiB used, 1.0 TiB / 1.0 TiB avail; 2.7 MiB/s rd, 5.6 MiB/s wr, 513 op/s 2024-03-19T02:03:31.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:03:30 smithi115 ceph-mon[67332]: pgmap v1023: 97 pgs: 97 active+clean; 650 MiB data, 8.8 GiB used, 1.0 TiB / 1.0 TiB avail; 2.7 MiB/s rd, 5.6 MiB/s wr, 513 op/s 2024-03-19T02:03:31.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:03:30 smithi099 ceph-mon[62951]: pgmap v1023: 97 pgs: 97 active+clean; 650 MiB data, 8.8 GiB used, 1.0 TiB / 1.0 TiB avail; 2.7 MiB/s rd, 5.6 MiB/s wr, 513 op/s 2024-03-19T02:03:31.119 INFO:tasks.workunit.client.0.smithi099.stdout: 10 253525 22.39 MB/sec execute 480 sec latency 73.892 ms 2024-03-19T02:03:32.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:03:31 smithi115 ceph-mon[67332]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T02:03:32.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:03:31 smithi099 ceph-mon[62951]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T02:03:32.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:03:31 smithi112 ceph-mon[66819]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T02:03:32.119 INFO:tasks.workunit.client.0.smithi099.stdout: 10 253726 22.36 MB/sec execute 481 sec latency 299.765 ms 2024-03-19T02:03:33.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:03:32 smithi115 ceph-mon[67332]: pgmap v1024: 97 pgs: 97 active+clean; 652 MiB data, 8.8 GiB used, 1.0 TiB / 1.0 TiB avail; 2.9 MiB/s rd, 6.1 MiB/s wr, 542 op/s 2024-03-19T02:03:33.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:03:32 smithi099 ceph-mon[62951]: pgmap v1024: 97 pgs: 97 active+clean; 652 MiB data, 8.8 GiB used, 1.0 TiB / 1.0 TiB avail; 2.9 MiB/s rd, 6.1 MiB/s wr, 542 op/s 2024-03-19T02:03:33.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:03:32 smithi112 ceph-mon[66819]: pgmap v1024: 97 pgs: 97 active+clean; 652 MiB data, 8.8 GiB used, 1.0 TiB / 1.0 TiB avail; 2.9 MiB/s rd, 6.1 MiB/s wr, 542 op/s 2024-03-19T02:03:33.119 INFO:tasks.workunit.client.0.smithi099.stdout: 10 254056 22.36 MB/sec execute 482 sec latency 156.393 ms 2024-03-19T02:03:34.119 INFO:tasks.workunit.client.0.smithi099.stdout: 10 254406 22.33 MB/sec execute 483 sec latency 105.610 ms 2024-03-19T02:03:35.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:03:34 smithi115 ceph-mon[67332]: pgmap v1025: 97 pgs: 97 active+clean; 647 MiB data, 8.8 GiB used, 1.0 TiB / 1.0 TiB avail; 3.4 MiB/s rd, 6.4 MiB/s wr, 560 op/s 2024-03-19T02:03:35.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:03:34 smithi099 ceph-mon[62951]: pgmap v1025: 97 pgs: 97 active+clean; 647 MiB data, 8.8 GiB used, 1.0 TiB / 1.0 TiB avail; 3.4 MiB/s rd, 6.4 MiB/s wr, 560 op/s 2024-03-19T02:03:35.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:03:34 smithi112 ceph-mon[66819]: pgmap v1025: 97 pgs: 97 active+clean; 647 MiB data, 8.8 GiB used, 1.0 TiB / 1.0 TiB avail; 3.4 MiB/s rd, 6.4 MiB/s wr, 560 op/s 2024-03-19T02:03:35.119 INFO:tasks.workunit.client.0.smithi099.stdout: 10 254891 22.35 MB/sec execute 484 sec latency 98.897 ms 2024-03-19T02:03:36.119 INFO:tasks.workunit.client.0.smithi099.stdout: 10 255336 22.35 MB/sec execute 485 sec latency 70.929 ms 2024-03-19T02:03:37.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:03:36 smithi115 ceph-mon[67332]: pgmap v1026: 97 pgs: 97 active+clean; 650 MiB data, 8.8 GiB used, 1.0 TiB / 1.0 TiB avail; 2.9 MiB/s rd, 5.2 MiB/s wr, 508 op/s 2024-03-19T02:03:37.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:03:36 smithi099 ceph-mon[62951]: pgmap v1026: 97 pgs: 97 active+clean; 650 MiB data, 8.8 GiB used, 1.0 TiB / 1.0 TiB avail; 2.9 MiB/s rd, 5.2 MiB/s wr, 508 op/s 2024-03-19T02:03:37.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:03:36 smithi112 ceph-mon[66819]: pgmap v1026: 97 pgs: 97 active+clean; 650 MiB data, 8.8 GiB used, 1.0 TiB / 1.0 TiB avail; 2.9 MiB/s rd, 5.2 MiB/s wr, 508 op/s 2024-03-19T02:03:37.119 INFO:tasks.workunit.client.0.smithi099.stdout: 10 255653 22.33 MB/sec execute 486 sec latency 269.969 ms 2024-03-19T02:03:37.605 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.603+0000 7f8053f3b640 1 -- 172.21.15.99:0/1383296713 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f804c102a50 msgr2=0x7f804c102ed0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:03:37.605 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.603+0000 7f8053f3b640 1 --2- 172.21.15.99:0/1383296713 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f804c102a50 0x7f804c102ed0 secure :-1 s=READY pgs=765 cs=0 l=1 rev1=1 crypto rx=0x7f803c009900 tx=0x7f803c02bc30 comp rx=0 tx=0).stop 2024-03-19T02:03:37.605 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.603+0000 7f8053f3b640 1 -- 172.21.15.99:0/1383296713 shutdown_connections 2024-03-19T02:03:37.605 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.603+0000 7f8053f3b640 1 --2- 172.21.15.99:0/1383296713 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f804c103410 0x7f804c109ca0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:37.605 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.603+0000 7f8053f3b640 1 --2- 172.21.15.99:0/1383296713 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f804c102a50 0x7f804c102ed0 unknown :-1 s=CLOSED pgs=765 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:37.606 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.603+0000 7f8053f3b640 1 --2- 172.21.15.99:0/1383296713 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f804c101860 0x7f804c101c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:37.606 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.603+0000 7f8053f3b640 1 -- 172.21.15.99:0/1383296713 >> 172.21.15.99:0/1383296713 conn(0x7f804c0fcdb0 msgr2=0x7f804c0ff210 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:03:37.606 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.603+0000 7f8053f3b640 1 -- 172.21.15.99:0/1383296713 shutdown_connections 2024-03-19T02:03:37.606 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.603+0000 7f8053f3b640 1 -- 172.21.15.99:0/1383296713 wait complete. 2024-03-19T02:03:37.606 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.604+0000 7f8053f3b640 1 Processor -- start 2024-03-19T02:03:37.606 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.604+0000 7f8053f3b640 1 -- start start 2024-03-19T02:03:37.606 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.604+0000 7f8053f3b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f804c101860 0x7f804c197b20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:37.606 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.604+0000 7f8053f3b640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f804c102a50 0x7f804c198060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:37.606 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.604+0000 7f8053f3b640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f804c103410 0x7f804c19f0d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:37.606 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.604+0000 7f8053f3b640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f804c10d530 con 0x7f804c101860 2024-03-19T02:03:37.607 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.604+0000 7f8053f3b640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f804c10d3b0 con 0x7f804c102a50 2024-03-19T02:03:37.607 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.604+0000 7f8053f3b640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f804c10d6b0 con 0x7f804c103410 2024-03-19T02:03:37.607 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.604+0000 7f80514d6640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f804c102a50 0x7f804c198060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:37.607 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.604+0000 7f80524d8640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f804c103410 0x7f804c19f0d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:37.607 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.604+0000 7f8051cd7640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f804c101860 0x7f804c197b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:37.607 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.604+0000 7f80514d6640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f804c102a50 0x7f804c198060 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:58052/0 (socket says 172.21.15.99:58052) 2024-03-19T02:03:37.607 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.604+0000 7f80524d8640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f804c103410 0x7f804c19f0d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:55306/0 (socket says 172.21.15.99:55306) 2024-03-19T02:03:37.607 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.604+0000 7f80514d6640 1 -- 172.21.15.99:0/490610158 learned_addr learned my addr 172.21.15.99:0/490610158 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:03:37.607 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.604+0000 7f8051cd7640 1 -- 172.21.15.99:0/490610158 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f804c103410 msgr2=0x7f804c19f0d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:03:37.607 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.604+0000 7f8051cd7640 1 --2- 172.21.15.99:0/490610158 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f804c103410 0x7f804c19f0d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:37.607 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.604+0000 7f8051cd7640 1 -- 172.21.15.99:0/490610158 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f804c102a50 msgr2=0x7f804c198060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:03:37.607 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.604+0000 7f8051cd7640 1 --2- 172.21.15.99:0/490610158 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f804c102a50 0x7f804c198060 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:37.607 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.604+0000 7f8051cd7640 1 -- 172.21.15.99:0/490610158 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f804c19f740 con 0x7f804c101860 2024-03-19T02:03:37.608 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.604+0000 7f8051cd7640 1 --2- 172.21.15.99:0/490610158 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f804c101860 0x7f804c197b20 secure :-1 s=READY pgs=766 cs=0 l=1 rev1=1 crypto rx=0x7f804800e9c0 tx=0x7f804800edc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:03:37.608 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.605+0000 7f8042ffd640 1 -- 172.21.15.99:0/490610158 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f804800ccc0 con 0x7f804c101860 2024-03-19T02:03:37.608 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.605+0000 7f8053f3b640 1 -- 172.21.15.99:0/490610158 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f804c073ab0 con 0x7f804c101860 2024-03-19T02:03:37.608 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.605+0000 7f8042ffd640 1 -- 172.21.15.99:0/490610158 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f80480035f0 con 0x7f804c101860 2024-03-19T02:03:37.608 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.605+0000 7f8042ffd640 1 -- 172.21.15.99:0/490610158 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f804800ce60 con 0x7f804c101860 2024-03-19T02:03:37.608 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.605+0000 7f8053f3b640 1 -- 172.21.15.99:0/490610158 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f804c073fd0 con 0x7f804c101860 2024-03-19T02:03:37.614 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.607+0000 7f8042ffd640 1 -- 172.21.15.99:0/490610158 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f80480037a0 con 0x7f804c101860 2024-03-19T02:03:37.614 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.607+0000 7f8056c01740 1 Processor -- start 2024-03-19T02:03:37.614 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.607+0000 7f8056c01740 1 -- start start 2024-03-19T02:03:37.614 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.607+0000 7f8056c01740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563e57bef100 0x563e57bef520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:37.614 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.608+0000 7f8056c01740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563e57befaf0 0x563e57bf45f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:37.614 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.608+0000 7f8056c01740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563e57bf4c60 0x563e57bf70f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:37.614 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.608+0000 7f8056c01740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x563e57b53be0 con 0x563e57befaf0 2024-03-19T02:03:37.614 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.608+0000 7f8056c01740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x563e57afc960 con 0x563e57bef100 2024-03-19T02:03:37.614 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.608+0000 7f8056c01740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x563e57b4fc80 con 0x563e57bf4c60 2024-03-19T02:03:37.615 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.608+0000 7f80524d8640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563e57bef100 0x563e57bef520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:37.615 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.608+0000 7f8051cd7640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563e57befaf0 0x563e57bf45f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:37.615 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.608+0000 7f80524d8640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563e57bef100 0x563e57bef520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:58062/0 (socket says 172.21.15.99:58062) 2024-03-19T02:03:37.615 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.608+0000 7f8051cd7640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563e57befaf0 0x563e57bf45f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:52404/0 (socket says 172.21.15.99:52404) 2024-03-19T02:03:37.615 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.608+0000 7f80524d8640 1 -- 172.21.15.99:0/2403122882 learned_addr learned my addr 172.21.15.99:0/2403122882 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:03:37.615 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.608+0000 7f8042ffd640 1 --2- 172.21.15.99:0/490610158 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f80380a28a0 0x7f80380a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:37.615 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.608+0000 7f80514d6640 1 --2- 172.21.15.99:0/2403122882 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563e57bf4c60 0x563e57bf70f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:37.615 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.608+0000 7f80524d8640 1 --2- 172.21.15.99:0/490610158 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f80380a28a0 0x7f80380a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:37.615 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.608+0000 7f80524d8640 1 -- 172.21.15.99:0/2403122882 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563e57bf4c60 msgr2=0x563e57bf70f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:03:37.615 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.608+0000 7f80524d8640 1 --2- 172.21.15.99:0/2403122882 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563e57bf4c60 0x563e57bf70f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:37.615 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.608+0000 7f80524d8640 1 -- 172.21.15.99:0/2403122882 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563e57befaf0 msgr2=0x563e57bf45f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:03:37.615 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.608+0000 7f80524d8640 1 --2- 172.21.15.99:0/2403122882 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563e57befaf0 0x563e57bf45f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:37.615 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.608+0000 7f80524d8640 1 -- 172.21.15.99:0/2403122882 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x563e57af3d20 con 0x563e57bef100 2024-03-19T02:03:37.616 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.608+0000 7f8042ffd640 1 -- 172.21.15.99:0/490610158 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f80480ceda0 con 0x7f804c101860 2024-03-19T02:03:37.616 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.608+0000 7f80514d6640 1 --2- 172.21.15.99:0/2403122882 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563e57bf4c60 0x563e57bf70f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:03:37.616 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.608+0000 7f80524d8640 1 --2- 172.21.15.99:0/2403122882 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563e57bef100 0x563e57bef520 secure :-1 s=READY pgs=223 cs=0 l=1 rev1=1 crypto rx=0x7f8044009b80 tx=0x7f804400b690 comp rx=0 tx=0).ready entity=mon.2 client_cookie=5ed1b6b4391e9589 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:03:37.616 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.609+0000 7f8037fff640 1 -- 172.21.15.99:0/2403122882 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8044023070 con 0x563e57bef100 2024-03-19T02:03:37.616 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.609+0000 7f80524d8640 1 --2- 172.21.15.99:0/490610158 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f80380a28a0 0x7f80380a4d60 secure :-1 s=READY pgs=382 cs=0 l=1 rev1=1 crypto rx=0x7f8044026000 tx=0x7f8044023000 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:03:37.616 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.609+0000 7f8037fff640 1 -- 172.21.15.99:0/2403122882 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f80440106e0 con 0x563e57bef100 2024-03-19T02:03:37.616 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.609+0000 7f8037fff640 1 -- 172.21.15.99:0/2403122882 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f804401fc00 con 0x563e57bef100 2024-03-19T02:03:37.616 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.609+0000 7f8056c01740 1 -- 172.21.15.99:0/2403122882 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563e57bef100 msgr2=0x563e57bef520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:03:37.616 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.609+0000 7f8056c01740 1 --2- 172.21.15.99:0/2403122882 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563e57bef100 0x563e57bef520 secure :-1 s=READY pgs=223 cs=0 l=1 rev1=1 crypto rx=0x7f8044009b80 tx=0x7f804400b690 comp rx=0 tx=0).stop 2024-03-19T02:03:37.616 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.609+0000 7f8056c01740 1 -- 172.21.15.99:0/2403122882 shutdown_connections 2024-03-19T02:03:37.616 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.609+0000 7f8056c01740 1 --2- 172.21.15.99:0/2403122882 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563e57bf4c60 0x563e57bf70f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:37.617 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.609+0000 7f8056c01740 1 --2- 172.21.15.99:0/2403122882 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563e57befaf0 0x563e57bf45f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:37.617 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.609+0000 7f8056c01740 1 --2- 172.21.15.99:0/2403122882 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563e57bef100 0x563e57bef520 unknown :-1 s=CLOSED pgs=223 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:37.617 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.609+0000 7f8056c01740 1 -- 172.21.15.99:0/2403122882 >> 172.21.15.99:0/2403122882 conn(0x563e57bea670 msgr2=0x563e57beaa70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:03:37.617 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.610+0000 7f8056c01740 1 -- 172.21.15.99:0/2403122882 shutdown_connections 2024-03-19T02:03:37.617 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.610+0000 7f8056c01740 1 -- 172.21.15.99:0/2403122882 wait complete. 2024-03-19T02:03:37.617 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.610+0000 7f8056c01740 1 -- start start 2024-03-19T02:03:37.617 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.610+0000 7f8056c01740 1 Processor -- start 2024-03-19T02:03:37.617 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.610+0000 7f8056c01740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563e57bef100 0x563e57bf9110 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:37.617 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.610+0000 7f8056c01740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563e57befaf0 0x563e57bf9650 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:37.618 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.610+0000 7f8056c01740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563e57bf4c60 0x563e57c00690 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:37.618 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.610+0000 7f8056c01740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x563e57bf7c10 con 0x563e57bf4c60 2024-03-19T02:03:37.618 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.610+0000 7f8056c01740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x563e57b0fdb0 con 0x563e57bef100 2024-03-19T02:03:37.618 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.610+0000 7f8056c01740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x563e57bf7d90 con 0x563e57befaf0 2024-03-19T02:03:37.618 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.610+0000 7f8056c01740 10 client.0 _ll_register_callbacks cb 0x563e57b14e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T02:03:37.618 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.610+0000 7f80524d8640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563e57bf4c60 0x563e57c00690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:37.618 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.610+0000 7f8051cd7640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563e57bef100 0x563e57bf9110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:37.618 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.610+0000 7f8051cd7640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563e57bef100 0x563e57bf9110 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:58078/0 (socket says 172.21.15.99:58078) 2024-03-19T02:03:37.618 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.611+0000 7f8051cd7640 1 -- 172.21.15.99:0/35768863 learned_addr learned my addr 172.21.15.99:0/35768863 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:03:37.618 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.610+0000 7f80524d8640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563e57bf4c60 0x563e57c00690 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:52420/0 (socket says 172.21.15.99:52420) 2024-03-19T02:03:37.618 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.611+0000 7f80514d6640 1 --2- 172.21.15.99:0/35768863 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563e57befaf0 0x563e57bf9650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:37.618 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.611+0000 7f8051cd7640 1 -- 172.21.15.99:0/35768863 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563e57befaf0 msgr2=0x563e57bf9650 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:03:37.618 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.611+0000 7f8051cd7640 1 --2- 172.21.15.99:0/35768863 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563e57befaf0 0x563e57bf9650 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:37.619 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.611+0000 7f8051cd7640 1 -- 172.21.15.99:0/35768863 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563e57bf4c60 msgr2=0x563e57c00690 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:03:37.619 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.611+0000 7f8051cd7640 1 --2- 172.21.15.99:0/35768863 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563e57bf4c60 0x563e57c00690 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:37.619 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.611+0000 7f8051cd7640 1 -- 172.21.15.99:0/35768863 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x563e57b0ec60 con 0x563e57bef100 2024-03-19T02:03:37.619 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.611+0000 7f80524d8640 1 --2- 172.21.15.99:0/35768863 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563e57bf4c60 0x563e57c00690 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:03:37.619 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.611+0000 7f8051cd7640 1 --2- 172.21.15.99:0/35768863 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563e57bef100 0x563e57bf9110 secure :-1 s=READY pgs=224 cs=0 l=1 rev1=1 crypto rx=0x7f804800f010 tx=0x7f804801df80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=160bef977b0e208f server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:03:37.619 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.611+0000 7f80357fa640 10 client.0 ms_handle_connect on v2:172.21.15.112:3300/0 2024-03-19T02:03:37.619 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.611+0000 7f80357fa640 1 -- 172.21.15.99:0/35768863 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f804800a250 con 0x563e57bef100 2024-03-19T02:03:37.619 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.611+0000 7f8056c01740 1 -- 172.21.15.99:0/35768863 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f80440106e0 con 0x563e57bef100 2024-03-19T02:03:37.619 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.611+0000 7f80357fa640 1 -- 172.21.15.99:0/35768863 <== mon.2 v2:172.21.15.112:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f804809b960 con 0x563e57bef100 2024-03-19T02:03:37.619 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.611+0000 7f80357fa640 1 -- 172.21.15.99:0/35768863 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_get_version_reply(handle=1 version=41) ==== 24+0+0 (secure 0 0 0) 0x7f804809be00 con 0x563e57bef100 2024-03-19T02:03:37.619 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.611+0000 7f8056c01740 10 client.26912 fetch_fsmap learned FSMap version 41 2024-03-19T02:03:37.619 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.611+0000 7f8056c01740 1 -- 172.21.15.99:0/35768863 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,fsmap=41,monmap=4+}) -- 0x563e57868210 con 0x563e57bef100 2024-03-19T02:03:37.619 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.612+0000 7f80357fa640 1 -- 172.21.15.99:0/35768863 <== mon.2 v2:172.21.15.112:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f80440106e0 con 0x563e57bef100 2024-03-19T02:03:37.619 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.612+0000 7f80357fa640 1 -- 172.21.15.99:0/35768863 <== mon.2 v2:172.21.15.112:3300/0 5 ==== fsmap(e 41) ==== 8354+0+0 (secure 0 0 0) 0x7f80480a3aa0 con 0x563e57bef100 2024-03-19T02:03:37.620 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.612+0000 7f8056c01740 10 client.26912 fetch_fsmap finished waiting for FSMap version 41 2024-03-19T02:03:37.620 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.612+0000 7f8056c01740 10 client.26912 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T02:03:37.620 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.612+0000 7f8056c01740 20 client.26912 populate_metadata read hostname 'smithi099' 2024-03-19T02:03:37.620 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.612+0000 7f8056c01740 1 --2- 172.21.15.99:0/35768863 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x563e57c04f00 0x563e57c072f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:37.620 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.612+0000 7f8056c01740 4 client.26912 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T02:03:37.620 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.612+0000 7f8056c01740 1 -- 172.21.15.99:0/35768863 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x563e57b61120 con 0x563e57c04f00 2024-03-19T02:03:37.620 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.612+0000 7f80514d6640 1 --2- 172.21.15.99:0/35768863 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x563e57c04f00 0x563e57c072f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:37.620 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.613+0000 7f80357fa640 10 client.26912 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T02:03:37.620 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.613+0000 7f80514d6640 1 --2- 172.21.15.99:0/35768863 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x563e57c04f00 0x563e57c072f0 crc :-1 s=READY pgs=229 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=272f0ca66713871f server_cookie=33465fdd5566c66c in_seq=0 out_seq=0 2024-03-19T02:03:37.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.619+0000 7f80357fa640 1 -- 172.21.15.99:0/35768863 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x563e57b61120 con 0x563e57c04f00 2024-03-19T02:03:37.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.619+0000 7f80357fa640 10 client.26912 handle_command_reply: tid=0 2024-03-19T02:03:37.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.619+0000 7f8056c01740 1 client.26912 shutdown 2024-03-19T02:03:37.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.619+0000 7f8056c01740 1 -- 172.21.15.99:0/35768863 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563e57bef100 msgr2=0x563e57bf9110 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:03:37.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.619+0000 7f8056c01740 1 --2- 172.21.15.99:0/35768863 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563e57bef100 0x563e57bf9110 secure :-1 s=READY pgs=224 cs=0 l=1 rev1=1 crypto rx=0x7f804800f010 tx=0x7f804801df80 comp rx=0 tx=0).stop 2024-03-19T02:03:37.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.619+0000 7f8056c01740 1 -- 172.21.15.99:0/35768863 shutdown_connections 2024-03-19T02:03:37.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.619+0000 7f8056c01740 1 --2- 172.21.15.99:0/35768863 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x563e57c04f00 0x563e57c072f0 crc :-1 s=READY pgs=229 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:37.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.619+0000 7f8056c01740 1 --2- 172.21.15.99:0/35768863 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563e57bf4c60 0x563e57c00690 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:37.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.619+0000 7f8056c01740 1 --2- 172.21.15.99:0/35768863 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563e57befaf0 0x563e57bf9650 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:37.623 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.619+0000 7f8056c01740 1 --2- 172.21.15.99:0/35768863 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563e57bef100 0x563e57bf9110 unknown :-1 s=CLOSED pgs=224 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:37.623 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.619+0000 7f80357fa640 0 client.26912 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T02:03:37.623 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.619+0000 7f8056c01740 1 -- 172.21.15.99:0/35768863 >> 172.21.15.99:0/35768863 conn(0x563e57bea670 msgr2=0x563e57beb8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:03:37.623 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.619+0000 7f8056c01740 1 -- 172.21.15.99:0/35768863 shutdown_connections 2024-03-19T02:03:37.623 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.619+0000 7f8056c01740 1 -- 172.21.15.99:0/35768863 wait complete. 2024-03-19T02:03:37.623 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.619+0000 7f8056c01740 20 client.26912 trim_cache size 0 max 16384 2024-03-19T02:03:37.636 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.636+0000 7f8056c01740 1 Processor -- start 2024-03-19T02:03:37.637 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.636+0000 7f8056c01740 1 -- start start 2024-03-19T02:03:37.637 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.636+0000 7f8056c01740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563e57bef100 0x563e57be9ab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:37.637 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.636+0000 7f8056c01740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563e57befaf0 0x563e57c00650 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:37.637 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.636+0000 7f8056c01740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563e57bf4c60 0x563e57bf0380 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:37.637 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.636+0000 7f8056c01740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f804c1a9550 con 0x563e57bf4c60 2024-03-19T02:03:37.637 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.636+0000 7f8056c01740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f804c1a6dc0 con 0x563e57bef100 2024-03-19T02:03:37.637 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.636+0000 7f8056c01740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f804c1a4c10 con 0x563e57befaf0 2024-03-19T02:03:37.637 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.636+0000 7f80524d8640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563e57bf4c60 0x563e57bf0380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:37.637 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.636+0000 7f8051cd7640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563e57bef100 0x563e57be9ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:37.637 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.636+0000 7f80524d8640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563e57bf4c60 0x563e57bf0380 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:52422/0 (socket says 172.21.15.99:52422) 2024-03-19T02:03:37.638 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.636+0000 7f80524d8640 1 -- 172.21.15.99:0/3116465009 learned_addr learned my addr 172.21.15.99:0/3116465009 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:03:37.638 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.636+0000 7f80514d6640 1 --2- 172.21.15.99:0/3116465009 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563e57befaf0 0x563e57c00650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:37.638 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.636+0000 7f80524d8640 1 -- 172.21.15.99:0/3116465009 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563e57befaf0 msgr2=0x563e57c00650 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:03:37.638 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.636+0000 7f80524d8640 1 --2- 172.21.15.99:0/3116465009 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563e57befaf0 0x563e57c00650 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:37.638 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.636+0000 7f80524d8640 1 -- 172.21.15.99:0/3116465009 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563e57bef100 msgr2=0x563e57be9ab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:03:37.638 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.636+0000 7f80524d8640 1 --2- 172.21.15.99:0/3116465009 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563e57bef100 0x563e57be9ab0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:37.639 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.636+0000 7f80524d8640 1 -- 172.21.15.99:0/3116465009 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x563e57934620 con 0x563e57bf4c60 2024-03-19T02:03:37.639 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.636+0000 7f80524d8640 1 --2- 172.21.15.99:0/3116465009 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563e57bf4c60 0x563e57bf0380 secure :-1 s=READY pgs=767 cs=0 l=1 rev1=1 crypto rx=0x7f8044009fd0 tx=0x7f8044010870 comp rx=0 tx=0).ready entity=mon.0 client_cookie=178cdd626401bfc8 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:03:37.639 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.637+0000 7f80377fe640 1 -- 172.21.15.99:0/3116465009 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f804401f5f0 con 0x563e57bf4c60 2024-03-19T02:03:37.639 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.637+0000 7f80377fe640 1 -- 172.21.15.99:0/3116465009 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f804401fc70 con 0x563e57bf4c60 2024-03-19T02:03:37.639 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.637+0000 7f80377fe640 1 -- 172.21.15.99:0/3116465009 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8044005820 con 0x563e57bf4c60 2024-03-19T02:03:37.643 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.637+0000 7f8056c01740 1 -- 172.21.15.99:0/3116465009 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563e57bf4c60 msgr2=0x563e57bf0380 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:03:37.643 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.637+0000 7f8056c01740 1 --2- 172.21.15.99:0/3116465009 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563e57bf4c60 0x563e57bf0380 secure :-1 s=READY pgs=767 cs=0 l=1 rev1=1 crypto rx=0x7f8044009fd0 tx=0x7f8044010870 comp rx=0 tx=0).stop 2024-03-19T02:03:37.643 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.637+0000 7f8056c01740 1 -- 172.21.15.99:0/3116465009 shutdown_connections 2024-03-19T02:03:37.644 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.637+0000 7f8056c01740 1 --2- 172.21.15.99:0/3116465009 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563e57bf4c60 0x563e57bf0380 unknown :-1 s=CLOSED pgs=767 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:37.644 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.637+0000 7f8056c01740 1 --2- 172.21.15.99:0/3116465009 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563e57befaf0 0x563e57c00650 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:37.644 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.637+0000 7f8056c01740 1 --2- 172.21.15.99:0/3116465009 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563e57bef100 0x563e57be9ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:37.644 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.637+0000 7f8056c01740 1 -- 172.21.15.99:0/3116465009 >> 172.21.15.99:0/3116465009 conn(0x563e57bea670 msgr2=0x563e57bf9390 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:03:37.644 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.637+0000 7f8056c01740 1 -- 172.21.15.99:0/3116465009 shutdown_connections 2024-03-19T02:03:37.644 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.637+0000 7f8056c01740 1 -- 172.21.15.99:0/3116465009 wait complete. 2024-03-19T02:03:37.644 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.638+0000 7f8056c01740 1 -- start start 2024-03-19T02:03:37.644 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.638+0000 7f8056c01740 1 Processor -- start 2024-03-19T02:03:37.644 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.638+0000 7f8056c01740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563e57bef100 0x563e57bfd770 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:37.644 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.638+0000 7f8056c01740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563e57befaf0 0x563e57bfdcb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:37.644 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.638+0000 7f8056c01740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563e57bf4c60 0x563e57bfecc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:37.644 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.638+0000 7f8056c01740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f804c1a5280 con 0x563e57befaf0 2024-03-19T02:03:37.644 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.638+0000 7f8056c01740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f804c1a66f0 con 0x563e57bef100 2024-03-19T02:03:37.644 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.638+0000 7f8056c01740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f804c1a5950 con 0x563e57bf4c60 2024-03-19T02:03:37.645 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.639+0000 7f8056c01740 10 client.0 _ll_register_callbacks cb 0x7f80480979a0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T02:03:37.645 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.639+0000 7f80514d6640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563e57befaf0 0x563e57bfdcb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:37.645 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.639+0000 7f80514d6640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563e57befaf0 0x563e57bfdcb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:52428/0 (socket says 172.21.15.99:52428) 2024-03-19T02:03:37.645 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.639+0000 7f80514d6640 1 -- 172.21.15.99:0/2025966650 learned_addr learned my addr 172.21.15.99:0/2025966650 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:03:37.645 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.639+0000 7f8051cd7640 1 --2- 172.21.15.99:0/2025966650 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563e57bef100 0x563e57bfd770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:37.645 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.639+0000 7f80524d8640 1 --2- 172.21.15.99:0/2025966650 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563e57bf4c60 0x563e57bfecc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:37.645 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.639+0000 7f80514d6640 1 -- 172.21.15.99:0/2025966650 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563e57bf4c60 msgr2=0x563e57bfecc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:03:37.645 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.639+0000 7f80514d6640 1 --2- 172.21.15.99:0/2025966650 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563e57bf4c60 0x563e57bfecc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:37.645 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.639+0000 7f80514d6640 1 -- 172.21.15.99:0/2025966650 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563e57bef100 msgr2=0x563e57bfd770 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:03:37.645 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.639+0000 7f80514d6640 1 --2- 172.21.15.99:0/2025966650 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563e57bef100 0x563e57bfd770 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:37.645 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.639+0000 7f80514d6640 1 -- 172.21.15.99:0/2025966650 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x563e57bff2b0 con 0x563e57befaf0 2024-03-19T02:03:37.645 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.639+0000 7f80514d6640 1 --2- 172.21.15.99:0/2025966650 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563e57befaf0 0x563e57bfdcb0 secure :-1 s=READY pgs=768 cs=0 l=1 rev1=1 crypto rx=0x7f803c02c240 tx=0x7f803c02cb40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a85ee513baa9ec08 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:03:37.645 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.639+0000 7f80357fa640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T02:03:37.645 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.639+0000 7f80357fa640 1 -- 172.21.15.99:0/2025966650 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f803c051070 con 0x563e57befaf0 2024-03-19T02:03:37.645 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.639+0000 7f80357fa640 1 -- 172.21.15.99:0/2025966650 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f803c04ad20 con 0x563e57befaf0 2024-03-19T02:03:37.646 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.639+0000 7f8056c01740 1 -- 172.21.15.99:0/2025966650 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f804401fc70 con 0x563e57befaf0 2024-03-19T02:03:37.646 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.639+0000 7f80357fa640 1 -- 172.21.15.99:0/2025966650 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=41) ==== 24+0+0 (secure 0 0 0) 0x7f803c05c050 con 0x563e57befaf0 2024-03-19T02:03:37.646 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.639+0000 7f8056c01740 10 client.17142 fetch_fsmap learned FSMap version 41 2024-03-19T02:03:37.646 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.639+0000 7f8056c01740 1 -- 172.21.15.99:0/2025966650 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=41,monmap=4+}) -- 0x563e57c00660 con 0x563e57befaf0 2024-03-19T02:03:37.646 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.640+0000 7f80357fa640 1 -- 172.21.15.99:0/2025966650 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f804401fc70 con 0x563e57befaf0 2024-03-19T02:03:37.646 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.640+0000 7f80357fa640 1 -- 172.21.15.99:0/2025966650 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 41) ==== 8354+0+0 (secure 0 0 0) 0x7f803c004750 con 0x563e57befaf0 2024-03-19T02:03:37.646 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.640+0000 7f8056c01740 10 client.17142 fetch_fsmap finished waiting for FSMap version 41 2024-03-19T02:03:37.646 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.640+0000 7f8056c01740 10 client.17142 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T02:03:37.646 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.640+0000 7f8056c01740 20 client.17142 populate_metadata read hostname 'smithi099' 2024-03-19T02:03:37.646 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.640+0000 7f8056c01740 1 --2- 172.21.15.99:0/2025966650 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x563e57c6b5b0 0x563e57c6d9d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:37.646 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.640+0000 7f8056c01740 4 client.17142 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T02:03:37.646 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.640+0000 7f8056c01740 1 -- 172.21.15.99:0/2025966650 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x563e57b5b470 con 0x563e57c6b5b0 2024-03-19T02:03:37.647 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.640+0000 7f8051cd7640 1 --2- 172.21.15.99:0/2025966650 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x563e57c6b5b0 0x563e57c6d9d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:37.647 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.640+0000 7f8051cd7640 1 --2- 172.21.15.99:0/2025966650 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x563e57c6b5b0 0x563e57c6d9d0 crc :-1 s=READY pgs=230 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=24dcf84cbaef94b9 server_cookie=1ddc9d5b864baf3b in_seq=0 out_seq=0 2024-03-19T02:03:37.647 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:37.640+0000 7f80357fa640 10 client.17142 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T02:03:38.119 INFO:tasks.workunit.client.0.smithi099.stdout: 10 255966 22.33 MB/sec execute 487 sec latency 375.309 ms 2024-03-19T02:03:38.285 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:38.284+0000 7f80357fa640 1 -- 172.21.15.99:0/2025966650 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x563e57b5b470 con 0x563e57c6b5b0 2024-03-19T02:03:38.285 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:38.284+0000 7f80357fa640 10 client.17142 handle_command_reply: tid=0 2024-03-19T02:03:38.285 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:38.284+0000 7f8056c01740 1 client.17142 shutdown 2024-03-19T02:03:38.285 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:38.284+0000 7f8056c01740 1 -- 172.21.15.99:0/2025966650 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563e57befaf0 msgr2=0x563e57bfdcb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:03:38.285 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:38.284+0000 7f8056c01740 1 --2- 172.21.15.99:0/2025966650 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563e57befaf0 0x563e57bfdcb0 secure :-1 s=READY pgs=768 cs=0 l=1 rev1=1 crypto rx=0x7f803c02c240 tx=0x7f803c02cb40 comp rx=0 tx=0).stop 2024-03-19T02:03:38.285 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:38.284+0000 7f8056c01740 1 -- 172.21.15.99:0/2025966650 shutdown_connections 2024-03-19T02:03:38.285 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:38.284+0000 7f8056c01740 1 --2- 172.21.15.99:0/2025966650 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x563e57c6b5b0 0x563e57c6d9d0 crc :-1 s=READY pgs=230 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:38.285 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:38.284+0000 7f8056c01740 1 --2- 172.21.15.99:0/2025966650 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x563e57bf4c60 0x563e57bfecc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:38.285 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:38.284+0000 7f80357fa640 0 client.17142 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T02:03:38.285 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:38.284+0000 7f8056c01740 1 --2- 172.21.15.99:0/2025966650 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x563e57befaf0 0x563e57bfdcb0 unknown :-1 s=CLOSED pgs=768 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:38.286 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:38.284+0000 7f8056c01740 1 --2- 172.21.15.99:0/2025966650 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x563e57bef100 0x563e57bfd770 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:38.286 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:38.284+0000 7f8056c01740 1 -- 172.21.15.99:0/2025966650 >> 172.21.15.99:0/2025966650 conn(0x563e57bea670 msgr2=0x563e57bf3d10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:03:38.286 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:38.285+0000 7f8056c01740 1 -- 172.21.15.99:0/2025966650 shutdown_connections 2024-03-19T02:03:38.286 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:38.285+0000 7f8056c01740 1 -- 172.21.15.99:0/2025966650 wait complete. 2024-03-19T02:03:38.286 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:38.285+0000 7f8056c01740 20 client.17142 trim_cache size 0 max 16384 2024-03-19T02:03:38.286 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T02:03:38.286 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T02:03:38.286 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T02:03:38.286 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T02:03:38.287 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:38.285+0000 7f8053f3b640 1 -- 172.21.15.99:0/490610158 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f80380a28a0 msgr2=0x7f80380a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:03:38.287 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:38.285+0000 7f8053f3b640 1 --2- 172.21.15.99:0/490610158 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f80380a28a0 0x7f80380a4d60 secure :-1 s=READY pgs=382 cs=0 l=1 rev1=1 crypto rx=0x7f8044026000 tx=0x7f8044023000 comp rx=0 tx=0).stop 2024-03-19T02:03:38.287 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:38.285+0000 7f8053f3b640 1 -- 172.21.15.99:0/490610158 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f804c101860 msgr2=0x7f804c197b20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:03:38.287 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:38.285+0000 7f8053f3b640 1 --2- 172.21.15.99:0/490610158 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f804c101860 0x7f804c197b20 secure :-1 s=READY pgs=766 cs=0 l=1 rev1=1 crypto rx=0x7f804800e9c0 tx=0x7f804800edc0 comp rx=0 tx=0).stop 2024-03-19T02:03:38.288 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:38.285+0000 7f8053f3b640 1 -- 172.21.15.99:0/490610158 shutdown_connections 2024-03-19T02:03:38.288 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:38.285+0000 7f8053f3b640 1 --2- 172.21.15.99:0/490610158 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f80380a28a0 0x7f80380a4d60 unknown :-1 s=CLOSED pgs=382 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:38.288 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:38.285+0000 7f8053f3b640 1 --2- 172.21.15.99:0/490610158 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f804c102a50 0x7f804c198060 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:38.288 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:38.285+0000 7f8053f3b640 1 --2- 172.21.15.99:0/490610158 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f804c103410 0x7f804c19f0d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:38.288 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:38.285+0000 7f8053f3b640 1 --2- 172.21.15.99:0/490610158 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f804c101860 0x7f804c197b20 unknown :-1 s=CLOSED pgs=766 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:38.288 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:38.285+0000 7f8053f3b640 1 -- 172.21.15.99:0/490610158 >> 172.21.15.99:0/490610158 conn(0x7f804c0fcdb0 msgr2=0x7f804c105d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:03:38.288 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:38.285+0000 7f8053f3b640 1 -- 172.21.15.99:0/490610158 shutdown_connections 2024-03-19T02:03:38.288 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:38.285+0000 7f8053f3b640 1 -- 172.21.15.99:0/490610158 wait complete. 2024-03-19T02:03:39.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:03:38 smithi115 ceph-mon[67332]: pgmap v1027: 97 pgs: 97 active+clean; 652 MiB data, 8.9 GiB used, 1.0 TiB / 1.0 TiB avail; 3.2 MiB/s rd, 6.5 MiB/s wr, 552 op/s 2024-03-19T02:03:39.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:03:38 smithi099 ceph-mon[62951]: pgmap v1027: 97 pgs: 97 active+clean; 652 MiB data, 8.9 GiB used, 1.0 TiB / 1.0 TiB avail; 3.2 MiB/s rd, 6.5 MiB/s wr, 552 op/s 2024-03-19T02:03:39.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:03:38 smithi112 ceph-mon[66819]: pgmap v1027: 97 pgs: 97 active+clean; 652 MiB data, 8.9 GiB used, 1.0 TiB / 1.0 TiB avail; 3.2 MiB/s rd, 6.5 MiB/s wr, 552 op/s 2024-03-19T02:03:39.119 INFO:tasks.workunit.client.0.smithi099.stdout: 10 256362 22.32 MB/sec execute 488 sec latency 381.921 ms 2024-03-19T02:03:40.119 INFO:tasks.workunit.client.0.smithi099.stdout: 10 256828 22.32 MB/sec execute 489 sec latency 69.718 ms 2024-03-19T02:03:41.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:03:40 smithi115 ceph-mon[67332]: pgmap v1028: 97 pgs: 97 active+clean; 654 MiB data, 8.9 GiB used, 1.0 TiB / 1.0 TiB avail; 3.2 MiB/s rd, 5.3 MiB/s wr, 397 op/s 2024-03-19T02:03:41.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:03:40 smithi099 ceph-mon[62951]: pgmap v1028: 97 pgs: 97 active+clean; 654 MiB data, 8.9 GiB used, 1.0 TiB / 1.0 TiB avail; 3.2 MiB/s rd, 5.3 MiB/s wr, 397 op/s 2024-03-19T02:03:41.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:03:40 smithi112 ceph-mon[66819]: pgmap v1028: 97 pgs: 97 active+clean; 654 MiB data, 8.9 GiB used, 1.0 TiB / 1.0 TiB avail; 3.2 MiB/s rd, 5.3 MiB/s wr, 397 op/s 2024-03-19T02:03:41.119 INFO:tasks.workunit.client.0.smithi099.stdout: 10 257322 22.35 MB/sec execute 490 sec latency 64.695 ms 2024-03-19T02:03:42.119 INFO:tasks.workunit.client.0.smithi099.stdout: 10 257772 22.34 MB/sec execute 491 sec latency 156.920 ms 2024-03-19T02:03:43.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:03:42 smithi115 ceph-mon[67332]: pgmap v1029: 97 pgs: 97 active+clean; 644 MiB data, 8.9 GiB used, 1.0 TiB / 1.0 TiB avail; 3.7 MiB/s rd, 6.0 MiB/s wr, 460 op/s 2024-03-19T02:03:43.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:03:42 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T02:03:43.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:03:42 smithi099 ceph-mon[62951]: pgmap v1029: 97 pgs: 97 active+clean; 644 MiB data, 8.9 GiB used, 1.0 TiB / 1.0 TiB avail; 3.7 MiB/s rd, 6.0 MiB/s wr, 460 op/s 2024-03-19T02:03:43.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:03:42 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T02:03:43.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:03:42 smithi112 ceph-mon[66819]: pgmap v1029: 97 pgs: 97 active+clean; 644 MiB data, 8.9 GiB used, 1.0 TiB / 1.0 TiB avail; 3.7 MiB/s rd, 6.0 MiB/s wr, 460 op/s 2024-03-19T02:03:43.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:03:42 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T02:03:43.119 INFO:tasks.workunit.client.0.smithi099.stdout: 10 258077 22.33 MB/sec execute 492 sec latency 212.148 ms 2024-03-19T02:03:44.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:03:43 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:03:44.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:03:43 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:03:44.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:03:43 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:03:44.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:03:43 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:03:44.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:03:43 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:03:44.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:03:43 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:03:44.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:03:43 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:03:44.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:03:43 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:03:44.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:03:43 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:03:44.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:03:43 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:03:44.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:03:43 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:03:44.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:03:43 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:03:44.119 INFO:tasks.workunit.client.0.smithi099.stdout: 10 258513 22.34 MB/sec execute 493 sec latency 86.919 ms 2024-03-19T02:03:45.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:03:44 smithi115 ceph-mon[67332]: pgmap v1030: 97 pgs: 97 active+clean; 645 MiB data, 8.9 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 6.1 MiB/s wr, 541 op/s 2024-03-19T02:03:45.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:03:44 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T02:03:45.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:03:44 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T02:03:45.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:03:44 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:03:45.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:03:44 smithi099 ceph-mon[62951]: pgmap v1030: 97 pgs: 97 active+clean; 645 MiB data, 8.9 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 6.1 MiB/s wr, 541 op/s 2024-03-19T02:03:45.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:03:44 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T02:03:45.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:03:44 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T02:03:45.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:03:44 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:03:45.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:03:44 smithi112 ceph-mon[66819]: pgmap v1030: 97 pgs: 97 active+clean; 645 MiB data, 8.9 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 6.1 MiB/s wr, 541 op/s 2024-03-19T02:03:45.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:03:44 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T02:03:45.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:03:44 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T02:03:45.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:03:44 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:03:45.119 INFO:tasks.workunit.client.0.smithi099.stdout: 10 258888 22.33 MB/sec execute 494 sec latency 98.153 ms 2024-03-19T02:03:46.119 INFO:tasks.workunit.client.0.smithi099.stdout: 10 259227 22.32 MB/sec execute 495 sec latency 98.438 ms 2024-03-19T02:03:47.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:03:46 smithi115 ceph-mon[67332]: pgmap v1031: 97 pgs: 97 active+clean; 645 MiB data, 8.9 GiB used, 1.0 TiB / 1.0 TiB avail; 3.3 MiB/s rd, 5.3 MiB/s wr, 476 op/s 2024-03-19T02:03:47.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:03:46 smithi099 ceph-mon[62951]: pgmap v1031: 97 pgs: 97 active+clean; 645 MiB data, 8.9 GiB used, 1.0 TiB / 1.0 TiB avail; 3.3 MiB/s rd, 5.3 MiB/s wr, 476 op/s 2024-03-19T02:03:47.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:03:46 smithi112 ceph-mon[66819]: pgmap v1031: 97 pgs: 97 active+clean; 645 MiB data, 8.9 GiB used, 1.0 TiB / 1.0 TiB avail; 3.3 MiB/s rd, 5.3 MiB/s wr, 476 op/s 2024-03-19T02:03:47.119 INFO:tasks.workunit.client.0.smithi099.stdout: 10 259595 22.32 MB/sec execute 496 sec latency 96.797 ms 2024-03-19T02:03:48.119 INFO:tasks.workunit.client.0.smithi099.stdout: 10 259939 22.31 MB/sec execute 497 sec latency 387.011 ms 2024-03-19T02:03:49.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:03:48 smithi115 ceph-mon[67332]: pgmap v1032: 97 pgs: 97 active+clean; 647 MiB data, 9.0 GiB used, 1.0 TiB / 1.0 TiB avail; 3.7 MiB/s rd, 5.9 MiB/s wr, 528 op/s 2024-03-19T02:03:49.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:03:48 smithi099 ceph-mon[62951]: pgmap v1032: 97 pgs: 97 active+clean; 647 MiB data, 9.0 GiB used, 1.0 TiB / 1.0 TiB avail; 3.7 MiB/s rd, 5.9 MiB/s wr, 528 op/s 2024-03-19T02:03:49.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:03:48 smithi112 ceph-mon[66819]: pgmap v1032: 97 pgs: 97 active+clean; 647 MiB data, 9.0 GiB used, 1.0 TiB / 1.0 TiB avail; 3.7 MiB/s rd, 5.9 MiB/s wr, 528 op/s 2024-03-19T02:03:49.119 INFO:tasks.workunit.client.0.smithi099.stdout: 10 260382 22.31 MB/sec execute 498 sec latency 86.108 ms 2024-03-19T02:03:50.119 INFO:tasks.workunit.client.0.smithi099.stdout: 10 260893 22.31 MB/sec execute 499 sec latency 55.054 ms 2024-03-19T02:03:51.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:03:50 smithi115 ceph-mon[67332]: pgmap v1033: 97 pgs: 97 active+clean; 654 MiB data, 9.0 GiB used, 1.0 TiB / 1.0 TiB avail; 3.5 MiB/s rd, 4.9 MiB/s wr, 479 op/s 2024-03-19T02:03:51.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:03:50 smithi099 ceph-mon[62951]: pgmap v1033: 97 pgs: 97 active+clean; 654 MiB data, 9.0 GiB used, 1.0 TiB / 1.0 TiB avail; 3.5 MiB/s rd, 4.9 MiB/s wr, 479 op/s 2024-03-19T02:03:51.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:03:50 smithi112 ceph-mon[66819]: pgmap v1033: 97 pgs: 97 active+clean; 654 MiB data, 9.0 GiB used, 1.0 TiB / 1.0 TiB avail; 3.5 MiB/s rd, 4.9 MiB/s wr, 479 op/s 2024-03-19T02:03:51.119 INFO:tasks.workunit.client.0.smithi099.stdout: 10 261318 22.32 MB/sec execute 500 sec latency 70.292 ms 2024-03-19T02:03:51.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.391+0000 7f4767deb640 1 -- 172.21.15.99:0/2421142025 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f47601022f0 msgr2=0x7f4760108b80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:03:51.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.391+0000 7f4767deb640 1 --2- 172.21.15.99:0/2421142025 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f47601022f0 0x7f4760108b80 secure :-1 s=READY pgs=769 cs=0 l=1 rev1=1 crypto rx=0x7f4758009ca0 tx=0x7f475802c0f0 comp rx=0 tx=0).stop 2024-03-19T02:03:51.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.391+0000 7f4767deb640 1 -- 172.21.15.99:0/2421142025 shutdown_connections 2024-03-19T02:03:51.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.391+0000 7f4767deb640 1 --2- 172.21.15.99:0/2421142025 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f47601022f0 0x7f4760108b80 unknown :-1 s=CLOSED pgs=769 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:51.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.391+0000 7f4767deb640 1 --2- 172.21.15.99:0/2421142025 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4760101930 0x7f4760101db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:51.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.391+0000 7f4767deb640 1 --2- 172.21.15.99:0/2421142025 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4760100740 0x7f4760100b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:51.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.391+0000 7f4767deb640 1 -- 172.21.15.99:0/2421142025 >> 172.21.15.99:0/2421142025 conn(0x7f47600fbcb0 msgr2=0x7f47600fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:03:51.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.391+0000 7f4767deb640 1 -- 172.21.15.99:0/2421142025 shutdown_connections 2024-03-19T02:03:51.395 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.391+0000 7f4767deb640 1 -- 172.21.15.99:0/2421142025 wait complete. 2024-03-19T02:03:51.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.392+0000 7f4767deb640 1 Processor -- start 2024-03-19T02:03:51.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.392+0000 7f4767deb640 1 -- start start 2024-03-19T02:03:51.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.392+0000 7f4767deb640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4760100740 0x7f4760069a70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:51.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.392+0000 7f4767deb640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4760101930 0x7f476019c760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:51.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.392+0000 7f4767deb640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f47601022f0 0x7f476019edd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:51.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.392+0000 7f4767deb640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f476010c3c0 con 0x7f4760100740 2024-03-19T02:03:51.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.392+0000 7f4767deb640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f476010c240 con 0x7f47601022f0 2024-03-19T02:03:51.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.392+0000 7f4767deb640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f476010c540 con 0x7f4760101930 2024-03-19T02:03:51.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.392+0000 7f4765b87640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4760100740 0x7f4760069a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:51.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.392+0000 7f4765b87640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4760100740 0x7f4760069a70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:49756/0 (socket says 172.21.15.99:49756) 2024-03-19T02:03:51.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.392+0000 7f4765b87640 1 -- 172.21.15.99:0/647650333 learned_addr learned my addr 172.21.15.99:0/647650333 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:03:51.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.392+0000 7f4766388640 1 --2- 172.21.15.99:0/647650333 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f47601022f0 0x7f476019edd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:51.396 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.392+0000 7f4765386640 1 --2- 172.21.15.99:0/647650333 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4760101930 0x7f476019c760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:51.397 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.392+0000 7f4765b87640 1 -- 172.21.15.99:0/647650333 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4760101930 msgr2=0x7f476019c760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:03:51.397 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.392+0000 7f4765b87640 1 --2- 172.21.15.99:0/647650333 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4760101930 0x7f476019c760 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:51.397 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.392+0000 7f4765b87640 1 -- 172.21.15.99:0/647650333 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f47601022f0 msgr2=0x7f476019edd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:03:51.397 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.392+0000 7f4765b87640 1 --2- 172.21.15.99:0/647650333 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f47601022f0 0x7f476019edd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:51.397 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.392+0000 7f4765b87640 1 -- 172.21.15.99:0/647650333 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f476019f690 con 0x7f4760100740 2024-03-19T02:03:51.397 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.392+0000 7f4765b87640 1 --2- 172.21.15.99:0/647650333 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4760100740 0x7f4760069a70 secure :-1 s=READY pgs=770 cs=0 l=1 rev1=1 crypto rx=0x7f475c00e9c0 tx=0x7f475c00ee80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:03:51.397 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.393+0000 7f4756ffd640 1 -- 172.21.15.99:0/647650333 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f475c00cd80 con 0x7f4760100740 2024-03-19T02:03:51.397 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.393+0000 7f4756ffd640 1 -- 172.21.15.99:0/647650333 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f475c0035f0 con 0x7f4760100740 2024-03-19T02:03:51.397 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.393+0000 7f4767deb640 1 -- 172.21.15.99:0/647650333 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f476019f980 con 0x7f4760100740 2024-03-19T02:03:51.397 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.393+0000 7f4756ffd640 1 -- 172.21.15.99:0/647650333 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f475c010690 con 0x7f4760100740 2024-03-19T02:03:51.397 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.393+0000 7f4767deb640 1 -- 172.21.15.99:0/647650333 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f476019fed0 con 0x7f4760100740 2024-03-19T02:03:51.403 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.395+0000 7f4756ffd640 1 -- 172.21.15.99:0/647650333 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f475c0037a0 con 0x7f4760100740 2024-03-19T02:03:51.404 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.395+0000 7f476a9cd740 1 Processor -- start 2024-03-19T02:03:51.404 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.396+0000 7f476a9cd740 1 -- start start 2024-03-19T02:03:51.404 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.396+0000 7f476a9cd740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556898a99100 0x556898a99520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:51.404 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.396+0000 7f476a9cd740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556898a99af0 0x556898a9e5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:51.404 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.396+0000 7f476a9cd740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556898a9ec60 0x556898aa10f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:51.404 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.396+0000 7f476a9cd740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x5568989fdbe0 con 0x556898a99af0 2024-03-19T02:03:51.404 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.396+0000 7f476a9cd740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x5568989a6960 con 0x556898a9ec60 2024-03-19T02:03:51.404 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.396+0000 7f476a9cd740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x5568989f9c80 con 0x556898a99100 2024-03-19T02:03:51.404 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.396+0000 7f4765b87640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556898a99af0 0x556898a9e5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:51.404 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.396+0000 7f4765386640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556898a9ec60 0x556898aa10f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:51.404 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.396+0000 7f4765b87640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556898a99af0 0x556898a9e5f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:49762/0 (socket says 172.21.15.99:49762) 2024-03-19T02:03:51.404 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.396+0000 7f4766388640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556898a99100 0x556898a99520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:51.404 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.396+0000 7f4765b87640 1 -- 172.21.15.99:0/3559784760 learned_addr learned my addr 172.21.15.99:0/3559784760 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:03:51.404 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.396+0000 7f4756ffd640 1 --2- 172.21.15.99:0/647650333 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f474c0a28a0 0x7f474c0a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:51.405 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.396+0000 7f4766388640 1 --2- 172.21.15.99:0/647650333 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f474c0a28a0 0x7f474c0a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:51.405 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.396+0000 7f4765386640 1 -- 172.21.15.99:0/3559784760 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556898a99100 msgr2=0x556898a99520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:03:51.405 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.396+0000 7f4765386640 1 --2- 172.21.15.99:0/3559784760 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556898a99100 0x556898a99520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:51.405 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.396+0000 7f4756ffd640 1 -- 172.21.15.99:0/647650333 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f475c0cee50 con 0x7f4760100740 2024-03-19T02:03:51.405 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.396+0000 7f4765386640 1 -- 172.21.15.99:0/3559784760 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556898a99af0 msgr2=0x556898a9e5f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:03:51.405 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.396+0000 7f4765386640 1 --2- 172.21.15.99:0/3559784760 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556898a99af0 0x556898a9e5f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:51.405 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.396+0000 7f4765386640 1 -- 172.21.15.99:0/3559784760 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55689899dd20 con 0x556898a9ec60 2024-03-19T02:03:51.405 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.397+0000 7f4765386640 1 --2- 172.21.15.99:0/3559784760 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556898a9ec60 0x556898aa10f0 secure :-1 s=READY pgs=225 cs=0 l=1 rev1=1 crypto rx=0x7f4750009bd0 tx=0x7f4750011810 comp rx=0 tx=0).ready entity=mon.2 client_cookie=ab664959e31cff05 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:03:51.405 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.397+0000 7f4766388640 1 --2- 172.21.15.99:0/647650333 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f474c0a28a0 0x7f474c0a4d60 secure :-1 s=READY pgs=384 cs=0 l=1 rev1=1 crypto rx=0x7f4758005e30 tx=0x7f475802c0f0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:03:51.405 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.397+0000 7f474bfff640 1 -- 172.21.15.99:0/3559784760 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4750004230 con 0x556898a9ec60 2024-03-19T02:03:51.405 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.397+0000 7f474bfff640 1 -- 172.21.15.99:0/3559784760 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f4750009e40 con 0x556898a9ec60 2024-03-19T02:03:51.405 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.397+0000 7f474bfff640 1 -- 172.21.15.99:0/3559784760 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4750004db0 con 0x556898a9ec60 2024-03-19T02:03:51.405 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.397+0000 7f476a9cd740 1 -- 172.21.15.99:0/3559784760 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556898a9ec60 msgr2=0x556898aa10f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:03:51.405 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.398+0000 7f476a9cd740 1 --2- 172.21.15.99:0/3559784760 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556898a9ec60 0x556898aa10f0 secure :-1 s=READY pgs=225 cs=0 l=1 rev1=1 crypto rx=0x7f4750009bd0 tx=0x7f4750011810 comp rx=0 tx=0).stop 2024-03-19T02:03:51.405 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.398+0000 7f476a9cd740 1 -- 172.21.15.99:0/3559784760 shutdown_connections 2024-03-19T02:03:51.406 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.398+0000 7f476a9cd740 1 --2- 172.21.15.99:0/3559784760 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556898a9ec60 0x556898aa10f0 unknown :-1 s=CLOSED pgs=225 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:51.406 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.398+0000 7f476a9cd740 1 --2- 172.21.15.99:0/3559784760 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556898a99af0 0x556898a9e5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:51.406 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.398+0000 7f476a9cd740 1 --2- 172.21.15.99:0/3559784760 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556898a99100 0x556898a99520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:51.406 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.398+0000 7f476a9cd740 1 -- 172.21.15.99:0/3559784760 >> 172.21.15.99:0/3559784760 conn(0x556898a94670 msgr2=0x556898a94a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:03:51.406 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.398+0000 7f476a9cd740 1 -- 172.21.15.99:0/3559784760 shutdown_connections 2024-03-19T02:03:51.406 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.398+0000 7f476a9cd740 1 -- 172.21.15.99:0/3559784760 wait complete. 2024-03-19T02:03:51.406 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.398+0000 7f476a9cd740 1 -- start start 2024-03-19T02:03:51.406 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.398+0000 7f476a9cd740 1 Processor -- start 2024-03-19T02:03:51.406 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.398+0000 7f476a9cd740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556898a99100 0x556898aa30b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:51.406 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.398+0000 7f476a9cd740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556898a99af0 0x556898aa35f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:51.406 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.398+0000 7f476a9cd740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556898a9ec60 0x556898aaa630 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:51.406 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.398+0000 7f476a9cd740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x556898aa1c10 con 0x556898a9ec60 2024-03-19T02:03:51.406 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.398+0000 7f476a9cd740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x5568989b9db0 con 0x556898a99100 2024-03-19T02:03:51.406 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.398+0000 7f476a9cd740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x556898aa1d90 con 0x556898a99af0 2024-03-19T02:03:51.407 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.399+0000 7f476a9cd740 10 client.0 _ll_register_callbacks cb 0x5568989bee70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T02:03:51.407 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.399+0000 7f4766388640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556898a9ec60 0x556898aaa630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:51.407 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.399+0000 7f4765b87640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556898a99100 0x556898aa30b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:51.407 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.399+0000 7f4765b87640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556898a99100 0x556898aa30b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:49978/0 (socket says 172.21.15.99:49978) 2024-03-19T02:03:51.407 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.399+0000 7f4765b87640 1 -- 172.21.15.99:0/4179635664 learned_addr learned my addr 172.21.15.99:0/4179635664 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:03:51.407 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.399+0000 7f4765386640 1 --2- 172.21.15.99:0/4179635664 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556898a99af0 0x556898aa35f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:51.407 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.399+0000 7f4766388640 1 -- 172.21.15.99:0/4179635664 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556898a99af0 msgr2=0x556898aa35f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:03:51.407 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.399+0000 7f4766388640 1 --2- 172.21.15.99:0/4179635664 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556898a99af0 0x556898aa35f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:51.407 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.399+0000 7f4766388640 1 -- 172.21.15.99:0/4179635664 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556898a99100 msgr2=0x556898aa30b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:03:51.407 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.399+0000 7f4766388640 1 --2- 172.21.15.99:0/4179635664 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556898a99100 0x556898aa30b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:51.407 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.399+0000 7f4766388640 1 -- 172.21.15.99:0/4179635664 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5568989b8c60 con 0x556898a9ec60 2024-03-19T02:03:51.407 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.399+0000 7f4765b87640 1 --2- 172.21.15.99:0/4179635664 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556898a99100 0x556898aa30b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:03:51.407 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.399+0000 7f4766388640 1 --2- 172.21.15.99:0/4179635664 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556898a9ec60 0x556898aaa630 secure :-1 s=READY pgs=771 cs=0 l=1 rev1=1 crypto rx=0x7f475802c630 tx=0x7f47580095c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2892418c11d24362 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:03:51.407 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.399+0000 7f47497fa640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T02:03:51.408 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.399+0000 7f47497fa640 1 -- 172.21.15.99:0/4179635664 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4758053060 con 0x556898a9ec60 2024-03-19T02:03:51.408 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.399+0000 7f47497fa640 1 -- 172.21.15.99:0/4179635664 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f4758004040 con 0x556898a9ec60 2024-03-19T02:03:51.408 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.399+0000 7f476a9cd740 1 -- 172.21.15.99:0/4179635664 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x556898a0b120 con 0x556898a9ec60 2024-03-19T02:03:51.408 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.399+0000 7f47497fa640 1 -- 172.21.15.99:0/4179635664 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=41) ==== 24+0+0 (secure 0 0 0) 0x7f4758047070 con 0x556898a9ec60 2024-03-19T02:03:51.408 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.399+0000 7f476a9cd740 10 client.17154 fetch_fsmap learned FSMap version 41 2024-03-19T02:03:51.408 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.399+0000 7f476a9cd740 1 -- 172.21.15.99:0/4179635664 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=41,monmap=4+}) -- 0x556898712210 con 0x556898a9ec60 2024-03-19T02:03:51.408 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.399+0000 7f47497fa640 1 -- 172.21.15.99:0/4179635664 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x556898a0b120 con 0x556898a9ec60 2024-03-19T02:03:51.408 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.400+0000 7f47497fa640 1 -- 172.21.15.99:0/4179635664 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 41) ==== 8354+0+0 (secure 0 0 0) 0x7f475803f070 con 0x556898a9ec60 2024-03-19T02:03:51.408 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.400+0000 7f476a9cd740 10 client.17154 fetch_fsmap finished waiting for FSMap version 41 2024-03-19T02:03:51.408 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.400+0000 7f476a9cd740 10 client.17154 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T02:03:51.408 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.400+0000 7f476a9cd740 20 client.17154 populate_metadata read hostname 'smithi099' 2024-03-19T02:03:51.408 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.400+0000 7f476a9cd740 1 --2- 172.21.15.99:0/4179635664 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x556898aaef60 0x556898ab13f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:51.408 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.400+0000 7f476a9cd740 4 client.17154 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T02:03:51.408 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.400+0000 7f476a9cd740 1 -- 172.21.15.99:0/4179635664 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x556898a05470 con 0x556898aaef60 2024-03-19T02:03:51.408 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.400+0000 7f4765b87640 1 --2- 172.21.15.99:0/4179635664 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x556898aaef60 0x556898ab13f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:51.408 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.400+0000 7f47497fa640 10 client.17154 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T02:03:51.409 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.400+0000 7f4765b87640 1 --2- 172.21.15.99:0/4179635664 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x556898aaef60 0x556898ab13f0 crc :-1 s=READY pgs=231 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=775f8eafe7a51191 server_cookie=dd0f82b656629f7f in_seq=0 out_seq=0 2024-03-19T02:03:51.410 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.404+0000 7f47497fa640 1 -- 172.21.15.99:0/4179635664 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x556898a05470 con 0x556898aaef60 2024-03-19T02:03:51.410 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.404+0000 7f47497fa640 10 client.17154 handle_command_reply: tid=0 2024-03-19T02:03:51.410 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.404+0000 7f476a9cd740 1 client.17154 shutdown 2024-03-19T02:03:51.410 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.405+0000 7f476a9cd740 1 -- 172.21.15.99:0/4179635664 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556898a9ec60 msgr2=0x556898aaa630 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:03:51.410 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.405+0000 7f476a9cd740 1 --2- 172.21.15.99:0/4179635664 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556898a9ec60 0x556898aaa630 secure :-1 s=READY pgs=771 cs=0 l=1 rev1=1 crypto rx=0x7f475802c630 tx=0x7f47580095c0 comp rx=0 tx=0).stop 2024-03-19T02:03:51.410 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.405+0000 7f476a9cd740 1 -- 172.21.15.99:0/4179635664 shutdown_connections 2024-03-19T02:03:51.410 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.405+0000 7f476a9cd740 1 --2- 172.21.15.99:0/4179635664 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x556898aaef60 0x556898ab13f0 crc :-1 s=READY pgs=231 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:51.410 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.405+0000 7f47497fa640 0 client.17154 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T02:03:51.411 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.405+0000 7f476a9cd740 1 --2- 172.21.15.99:0/4179635664 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556898a9ec60 0x556898aaa630 unknown :-1 s=CLOSED pgs=771 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:51.411 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.405+0000 7f476a9cd740 1 --2- 172.21.15.99:0/4179635664 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556898a99af0 0x556898aa35f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:51.411 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.405+0000 7f476a9cd740 1 --2- 172.21.15.99:0/4179635664 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556898a99100 0x556898aa30b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:51.411 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.405+0000 7f476a9cd740 1 -- 172.21.15.99:0/4179635664 >> 172.21.15.99:0/4179635664 conn(0x556898a94670 msgr2=0x556898a958c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:03:51.411 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.405+0000 7f476a9cd740 1 -- 172.21.15.99:0/4179635664 shutdown_connections 2024-03-19T02:03:51.411 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.405+0000 7f476a9cd740 1 -- 172.21.15.99:0/4179635664 wait complete. 2024-03-19T02:03:51.411 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.405+0000 7f476a9cd740 20 client.17154 trim_cache size 0 max 16384 2024-03-19T02:03:51.430 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.424+0000 7f476a9cd740 1 Processor -- start 2024-03-19T02:03:51.430 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.424+0000 7f476a9cd740 1 -- start start 2024-03-19T02:03:51.430 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.424+0000 7f476a9cd740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556898a99100 0x556898a97fe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:51.430 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.424+0000 7f476a9cd740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556898a99af0 0x556898ab5cf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:51.430 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.424+0000 7f476a9cd740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556898a9ec60 0x556898ab6230 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:51.430 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.424+0000 7f476a9cd740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f47601ab6b0 con 0x556898a99af0 2024-03-19T02:03:51.430 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.424+0000 7f476a9cd740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f47601a8f20 con 0x556898a9ec60 2024-03-19T02:03:51.430 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.424+0000 7f476a9cd740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f47601a6d70 con 0x556898a99100 2024-03-19T02:03:51.430 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.424+0000 7f4765386640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556898a99af0 0x556898ab5cf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:51.431 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.424+0000 7f4765386640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556898a99af0 0x556898ab5cf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:49792/0 (socket says 172.21.15.99:49792) 2024-03-19T02:03:51.431 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.424+0000 7f4765386640 1 -- 172.21.15.99:0/2501224653 learned_addr learned my addr 172.21.15.99:0/2501224653 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:03:51.431 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.424+0000 7f4765b87640 1 --2- 172.21.15.99:0/2501224653 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556898a99100 0x556898a97fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:51.431 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.424+0000 7f4766388640 1 --2- 172.21.15.99:0/2501224653 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556898a9ec60 0x556898ab6230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:51.431 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.424+0000 7f4765386640 1 -- 172.21.15.99:0/2501224653 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556898a99100 msgr2=0x556898a97fe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:03:51.431 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.424+0000 7f4765386640 1 --2- 172.21.15.99:0/2501224653 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556898a99100 0x556898a97fe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:51.431 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.424+0000 7f4765386640 1 -- 172.21.15.99:0/2501224653 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556898a9ec60 msgr2=0x556898ab6230 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:03:51.431 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.424+0000 7f4765386640 1 --2- 172.21.15.99:0/2501224653 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556898a9ec60 0x556898ab6230 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:51.431 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.424+0000 7f4765386640 1 -- 172.21.15.99:0/2501224653 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x556898a8fee0 con 0x556898a99af0 2024-03-19T02:03:51.431 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.424+0000 7f4765386640 1 --2- 172.21.15.99:0/2501224653 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556898a99af0 0x556898ab5cf0 secure :-1 s=READY pgs=772 cs=0 l=1 rev1=1 crypto rx=0x7f4750009bd0 tx=0x7f4750005d50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4fe5d25c33153b25 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:03:51.431 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.425+0000 7f474b7fe640 1 -- 172.21.15.99:0/2501224653 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f475001e040 con 0x556898a99af0 2024-03-19T02:03:51.431 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.425+0000 7f474b7fe640 1 -- 172.21.15.99:0/2501224653 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f47500035f0 con 0x556898a99af0 2024-03-19T02:03:51.432 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.425+0000 7f474b7fe640 1 -- 172.21.15.99:0/2501224653 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4750023650 con 0x556898a99af0 2024-03-19T02:03:51.432 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.425+0000 7f476a9cd740 1 -- 172.21.15.99:0/2501224653 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556898a99af0 msgr2=0x556898ab5cf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:03:51.432 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.425+0000 7f476a9cd740 1 --2- 172.21.15.99:0/2501224653 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556898a99af0 0x556898ab5cf0 secure :-1 s=READY pgs=772 cs=0 l=1 rev1=1 crypto rx=0x7f4750009bd0 tx=0x7f4750005d50 comp rx=0 tx=0).stop 2024-03-19T02:03:51.432 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.425+0000 7f476a9cd740 1 -- 172.21.15.99:0/2501224653 shutdown_connections 2024-03-19T02:03:51.432 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.425+0000 7f476a9cd740 1 --2- 172.21.15.99:0/2501224653 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556898a9ec60 0x556898ab6230 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:51.432 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.425+0000 7f476a9cd740 1 --2- 172.21.15.99:0/2501224653 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556898a99af0 0x556898ab5cf0 unknown :-1 s=CLOSED pgs=772 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:51.432 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.425+0000 7f476a9cd740 1 --2- 172.21.15.99:0/2501224653 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556898a99100 0x556898a97fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:51.432 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.425+0000 7f476a9cd740 1 -- 172.21.15.99:0/2501224653 >> 172.21.15.99:0/2501224653 conn(0x556898a94670 msgr2=0x556898a978d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:03:51.432 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.426+0000 7f476a9cd740 1 -- 172.21.15.99:0/2501224653 shutdown_connections 2024-03-19T02:03:51.432 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.426+0000 7f476a9cd740 1 -- 172.21.15.99:0/2501224653 wait complete. 2024-03-19T02:03:51.432 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.426+0000 7f476a9cd740 1 -- start start 2024-03-19T02:03:51.432 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.426+0000 7f476a9cd740 1 Processor -- start 2024-03-19T02:03:51.433 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.426+0000 7f476a9cd740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556898a99100 0x556898b10b30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:51.433 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.426+0000 7f476a9cd740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556898a99af0 0x556898b11070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:51.433 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.426+0000 7f476a9cd740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556898a9ec60 0x556898b180b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:51.433 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.426+0000 7f476a9cd740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f47601a73e0 con 0x556898a99af0 2024-03-19T02:03:51.433 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.426+0000 7f476a9cd740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f47601a8850 con 0x556898a99100 2024-03-19T02:03:51.433 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.426+0000 7f476a9cd740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f47601a7ab0 con 0x556898a9ec60 2024-03-19T02:03:51.433 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.426+0000 7f476a9cd740 10 client.0 _ll_register_callbacks cb 0x7f4750005e30 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T02:03:51.433 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.426+0000 7f4765386640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556898a99af0 0x556898b11070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:51.433 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.426+0000 7f4765386640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556898a99af0 0x556898b11070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:49800/0 (socket says 172.21.15.99:49800) 2024-03-19T02:03:51.433 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.426+0000 7f4765b87640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556898a99100 0x556898b10b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:51.433 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.426+0000 7f4765386640 1 -- 172.21.15.99:0/1171257373 learned_addr learned my addr 172.21.15.99:0/1171257373 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:03:51.433 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.426+0000 7f4766388640 1 --2- 172.21.15.99:0/1171257373 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556898a9ec60 0x556898b180b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:51.434 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.427+0000 7f4765386640 1 -- 172.21.15.99:0/1171257373 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556898a9ec60 msgr2=0x556898b180b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:03:51.434 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.427+0000 7f4765386640 1 --2- 172.21.15.99:0/1171257373 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556898a9ec60 0x556898b180b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:51.434 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.427+0000 7f4765386640 1 -- 172.21.15.99:0/1171257373 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556898a99100 msgr2=0x556898b10b30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:03:51.434 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.427+0000 7f4765386640 1 --2- 172.21.15.99:0/1171257373 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556898a99100 0x556898b10b30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:51.434 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.427+0000 7f4765386640 1 -- 172.21.15.99:0/1171257373 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5568987de620 con 0x556898a99af0 2024-03-19T02:03:51.434 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.427+0000 7f4765b87640 1 --2- 172.21.15.99:0/1171257373 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556898a99100 0x556898b10b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:03:51.434 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.427+0000 7f4765386640 1 --2- 172.21.15.99:0/1171257373 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556898a99af0 0x556898b11070 secure :-1 s=READY pgs=773 cs=0 l=1 rev1=1 crypto rx=0x7f4750009340 tx=0x7f47500050d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2862f2c9ced7f778 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:03:51.434 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.427+0000 7f47497fa640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T02:03:51.434 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.427+0000 7f47497fa640 1 -- 172.21.15.99:0/1171257373 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4750003bb0 con 0x556898a99af0 2024-03-19T02:03:51.434 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.427+0000 7f47497fa640 1 -- 172.21.15.99:0/1171257373 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f475002c430 con 0x556898a99af0 2024-03-19T02:03:51.434 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.427+0000 7f476a9cd740 1 -- 172.21.15.99:0/1171257373 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f47500035f0 con 0x556898a99af0 2024-03-19T02:03:51.434 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.427+0000 7f47497fa640 1 -- 172.21.15.99:0/1171257373 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=41) ==== 24+0+0 (secure 0 0 0) 0x7f475003c050 con 0x556898a99af0 2024-03-19T02:03:51.434 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.427+0000 7f476a9cd740 10 client.17160 fetch_fsmap learned FSMap version 41 2024-03-19T02:03:51.434 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.427+0000 7f476a9cd740 1 -- 172.21.15.99:0/1171257373 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=41,monmap=4+}) -- 0x556898b197e0 con 0x556898a99af0 2024-03-19T02:03:51.435 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.427+0000 7f47497fa640 1 -- 172.21.15.99:0/1171257373 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f47500035f0 con 0x556898a99af0 2024-03-19T02:03:51.435 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.428+0000 7f47497fa640 1 -- 172.21.15.99:0/1171257373 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 41) ==== 8354+0+0 (secure 0 0 0) 0x7f4750018070 con 0x556898a99af0 2024-03-19T02:03:51.435 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.428+0000 7f476a9cd740 10 client.17160 fetch_fsmap finished waiting for FSMap version 41 2024-03-19T02:03:51.435 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.428+0000 7f476a9cd740 10 client.17160 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T02:03:51.435 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.428+0000 7f476a9cd740 20 client.17160 populate_metadata read hostname 'smithi099' 2024-03-19T02:03:51.435 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.428+0000 7f476a9cd740 1 --2- 172.21.15.99:0/1171257373 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x556898aaef60 0x556898b1bfb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:03:51.435 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.428+0000 7f476a9cd740 4 client.17160 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T02:03:51.435 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.428+0000 7f476a9cd740 1 -- 172.21.15.99:0/1171257373 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x556898983770 con 0x556898aaef60 2024-03-19T02:03:51.435 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.428+0000 7f4765b87640 1 --2- 172.21.15.99:0/1171257373 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x556898aaef60 0x556898b1bfb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:03:51.435 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.428+0000 7f47497fa640 10 client.17160 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T02:03:51.435 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.428+0000 7f4765b87640 1 --2- 172.21.15.99:0/1171257373 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x556898aaef60 0x556898b1bfb0 crc :-1 s=READY pgs=232 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=e355057d38664874 server_cookie=61fe06055083f934 in_seq=0 out_seq=0 2024-03-19T02:03:51.941 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.939+0000 7f47497fa640 1 -- 172.21.15.99:0/1171257373 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x556898983770 con 0x556898aaef60 2024-03-19T02:03:51.941 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.939+0000 7f47497fa640 10 client.17160 handle_command_reply: tid=0 2024-03-19T02:03:51.941 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.939+0000 7f476a9cd740 1 client.17160 shutdown 2024-03-19T02:03:51.941 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.940+0000 7f476a9cd740 1 -- 172.21.15.99:0/1171257373 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556898a99af0 msgr2=0x556898b11070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:03:51.941 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.940+0000 7f476a9cd740 1 --2- 172.21.15.99:0/1171257373 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556898a99af0 0x556898b11070 secure :-1 s=READY pgs=773 cs=0 l=1 rev1=1 crypto rx=0x7f4750009340 tx=0x7f47500050d0 comp rx=0 tx=0).stop 2024-03-19T02:03:51.941 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.940+0000 7f476a9cd740 1 -- 172.21.15.99:0/1171257373 shutdown_connections 2024-03-19T02:03:51.941 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.940+0000 7f476a9cd740 1 --2- 172.21.15.99:0/1171257373 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x556898aaef60 0x556898b1bfb0 crc :-1 s=READY pgs=232 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:51.941 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.940+0000 7f47497fa640 0 client.17160 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T02:03:51.942 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.940+0000 7f476a9cd740 1 --2- 172.21.15.99:0/1171257373 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x556898a9ec60 0x556898b180b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:51.942 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.940+0000 7f476a9cd740 1 --2- 172.21.15.99:0/1171257373 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x556898a99af0 0x556898b11070 unknown :-1 s=CLOSED pgs=773 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:51.942 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.940+0000 7f476a9cd740 1 --2- 172.21.15.99:0/1171257373 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x556898a99100 0x556898b10b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:51.942 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.940+0000 7f476a9cd740 1 -- 172.21.15.99:0/1171257373 >> 172.21.15.99:0/1171257373 conn(0x556898a94670 msgr2=0x556898a98160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:03:51.942 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.940+0000 7f476a9cd740 1 -- 172.21.15.99:0/1171257373 shutdown_connections 2024-03-19T02:03:51.942 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.940+0000 7f476a9cd740 1 -- 172.21.15.99:0/1171257373 wait complete. 2024-03-19T02:03:51.942 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.940+0000 7f476a9cd740 20 client.17160 trim_cache size 0 max 16384 2024-03-19T02:03:51.942 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T02:03:51.942 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T02:03:51.942 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T02:03:51.942 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T02:03:51.943 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.940+0000 7f4767deb640 1 -- 172.21.15.99:0/647650333 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f474c0a28a0 msgr2=0x7f474c0a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:03:51.943 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.940+0000 7f4767deb640 1 --2- 172.21.15.99:0/647650333 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f474c0a28a0 0x7f474c0a4d60 secure :-1 s=READY pgs=384 cs=0 l=1 rev1=1 crypto rx=0x7f4758005e30 tx=0x7f475802c0f0 comp rx=0 tx=0).stop 2024-03-19T02:03:51.943 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.940+0000 7f4767deb640 1 -- 172.21.15.99:0/647650333 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4760100740 msgr2=0x7f4760069a70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:03:51.943 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.940+0000 7f4767deb640 1 --2- 172.21.15.99:0/647650333 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4760100740 0x7f4760069a70 secure :-1 s=READY pgs=770 cs=0 l=1 rev1=1 crypto rx=0x7f475c00e9c0 tx=0x7f475c00ee80 comp rx=0 tx=0).stop 2024-03-19T02:03:51.944 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.941+0000 7f4767deb640 1 -- 172.21.15.99:0/647650333 shutdown_connections 2024-03-19T02:03:51.944 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.941+0000 7f4767deb640 1 --2- 172.21.15.99:0/647650333 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f474c0a28a0 0x7f474c0a4d60 unknown :-1 s=CLOSED pgs=384 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:51.944 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.941+0000 7f4767deb640 1 --2- 172.21.15.99:0/647650333 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f47601022f0 0x7f476019edd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:51.944 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.941+0000 7f4767deb640 1 --2- 172.21.15.99:0/647650333 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4760101930 0x7f476019c760 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:51.944 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.941+0000 7f4767deb640 1 --2- 172.21.15.99:0/647650333 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4760100740 0x7f4760069a70 unknown :-1 s=CLOSED pgs=770 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:03:51.944 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.941+0000 7f4767deb640 1 -- 172.21.15.99:0/647650333 >> 172.21.15.99:0/647650333 conn(0x7f47600fbcb0 msgr2=0x7f4760104bd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:03:51.944 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.941+0000 7f4767deb640 1 -- 172.21.15.99:0/647650333 shutdown_connections 2024-03-19T02:03:51.944 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:03:51.941+0000 7f4767deb640 1 -- 172.21.15.99:0/647650333 wait complete. 2024-03-19T02:03:52.120 INFO:tasks.workunit.client.0.smithi099.stdout: 10 261531 22.30 MB/sec execute 501 sec latency 333.352 ms 2024-03-19T02:03:53.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:03:52 smithi115 ceph-mon[67332]: pgmap v1034: 97 pgs: 97 active+clean; 655 MiB data, 8.9 GiB used, 1.0 TiB / 1.0 TiB avail; 3.7 MiB/s rd, 5.5 MiB/s wr, 527 op/s 2024-03-19T02:03:53.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:03:52 smithi099 ceph-mon[62951]: pgmap v1034: 97 pgs: 97 active+clean; 655 MiB data, 8.9 GiB used, 1.0 TiB / 1.0 TiB avail; 3.7 MiB/s rd, 5.5 MiB/s wr, 527 op/s 2024-03-19T02:03:53.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:03:52 smithi112 ceph-mon[66819]: pgmap v1034: 97 pgs: 97 active+clean; 655 MiB data, 8.9 GiB used, 1.0 TiB / 1.0 TiB avail; 3.7 MiB/s rd, 5.5 MiB/s wr, 527 op/s 2024-03-19T02:03:53.119 INFO:tasks.workunit.client.0.smithi099.stdout: 10 261832 22.28 MB/sec execute 502 sec latency 225.571 ms 2024-03-19T02:03:54.120 INFO:tasks.workunit.client.0.smithi099.stdout: 10 262350 22.30 MB/sec execute 503 sec latency 114.650 ms 2024-03-19T02:03:55.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:03:54 smithi115 ceph-mon[67332]: pgmap v1035: 97 pgs: 97 active+clean; 652 MiB data, 9.0 GiB used, 1.0 TiB / 1.0 TiB avail; 3.9 MiB/s rd, 6.2 MiB/s wr, 562 op/s 2024-03-19T02:03:55.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:03:54 smithi099 ceph-mon[62951]: pgmap v1035: 97 pgs: 97 active+clean; 652 MiB data, 9.0 GiB used, 1.0 TiB / 1.0 TiB avail; 3.9 MiB/s rd, 6.2 MiB/s wr, 562 op/s 2024-03-19T02:03:55.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:03:54 smithi112 ceph-mon[66819]: pgmap v1035: 97 pgs: 97 active+clean; 652 MiB data, 9.0 GiB used, 1.0 TiB / 1.0 TiB avail; 3.9 MiB/s rd, 6.2 MiB/s wr, 562 op/s 2024-03-19T02:03:55.119 INFO:tasks.workunit.client.0.smithi099.stdout: 10 262725 22.29 MB/sec execute 504 sec latency 133.861 ms 2024-03-19T02:03:56.120 INFO:tasks.workunit.client.0.smithi099.stdout: 10 263153 22.31 MB/sec execute 505 sec latency 132.158 ms 2024-03-19T02:03:57.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:03:56 smithi115 ceph-mon[67332]: pgmap v1036: 97 pgs: 97 active+clean; 654 MiB data, 9.0 GiB used, 1.0 TiB / 1.0 TiB avail; 2.7 MiB/s rd, 5.2 MiB/s wr, 416 op/s 2024-03-19T02:03:57.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:03:56 smithi099 ceph-mon[62951]: pgmap v1036: 97 pgs: 97 active+clean; 654 MiB data, 9.0 GiB used, 1.0 TiB / 1.0 TiB avail; 2.7 MiB/s rd, 5.2 MiB/s wr, 416 op/s 2024-03-19T02:03:57.113 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:03:56 smithi112 ceph-mon[66819]: pgmap v1036: 97 pgs: 97 active+clean; 654 MiB data, 9.0 GiB used, 1.0 TiB / 1.0 TiB avail; 2.7 MiB/s rd, 5.2 MiB/s wr, 416 op/s 2024-03-19T02:03:57.119 INFO:tasks.workunit.client.0.smithi099.stdout: 10 263519 22.31 MB/sec execute 506 sec latency 83.705 ms 2024-03-19T02:03:58.120 INFO:tasks.workunit.client.0.smithi099.stdout: 10 263765 22.27 MB/sec execute 507 sec latency 282.101 ms 2024-03-19T02:03:59.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:03:58 smithi115 ceph-mon[67332]: pgmap v1037: 97 pgs: 97 active+clean; 669 MiB data, 9.0 GiB used, 1.0 TiB / 1.0 TiB avail; 3.2 MiB/s rd, 7.1 MiB/s wr, 536 op/s 2024-03-19T02:03:59.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:03:58 smithi099 ceph-mon[62951]: pgmap v1037: 97 pgs: 97 active+clean; 669 MiB data, 9.0 GiB used, 1.0 TiB / 1.0 TiB avail; 3.2 MiB/s rd, 7.1 MiB/s wr, 536 op/s 2024-03-19T02:03:59.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:03:58 smithi112 ceph-mon[66819]: pgmap v1037: 97 pgs: 97 active+clean; 669 MiB data, 9.0 GiB used, 1.0 TiB / 1.0 TiB avail; 3.2 MiB/s rd, 7.1 MiB/s wr, 536 op/s 2024-03-19T02:03:59.120 INFO:tasks.workunit.client.0.smithi099.stdout: 10 264069 22.26 MB/sec execute 508 sec latency 143.036 ms 2024-03-19T02:04:00.120 INFO:tasks.workunit.client.0.smithi099.stdout: 10 264499 22.26 MB/sec execute 509 sec latency 171.216 ms 2024-03-19T02:04:01.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:04:00 smithi115 ceph-mon[67332]: pgmap v1038: 97 pgs: 97 active+clean; 674 MiB data, 9.1 GiB used, 1.0 TiB / 1.0 TiB avail; 3.1 MiB/s rd, 6.9 MiB/s wr, 501 op/s 2024-03-19T02:04:01.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:04:00 smithi099 ceph-mon[62951]: pgmap v1038: 97 pgs: 97 active+clean; 674 MiB data, 9.1 GiB used, 1.0 TiB / 1.0 TiB avail; 3.1 MiB/s rd, 6.9 MiB/s wr, 501 op/s 2024-03-19T02:04:01.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:04:00 smithi112 ceph-mon[66819]: pgmap v1038: 97 pgs: 97 active+clean; 674 MiB data, 9.1 GiB used, 1.0 TiB / 1.0 TiB avail; 3.1 MiB/s rd, 6.9 MiB/s wr, 501 op/s 2024-03-19T02:04:01.120 INFO:tasks.workunit.client.0.smithi099.stdout: 10 265008 22.28 MB/sec execute 510 sec latency 61.260 ms 2024-03-19T02:04:02.121 INFO:tasks.workunit.client.0.smithi099.stdout: 10 265450 22.27 MB/sec execute 511 sec latency 73.397 ms 2024-03-19T02:04:03.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:04:02 smithi115 ceph-mon[67332]: pgmap v1039: 97 pgs: 97 active+clean; 678 MiB data, 9.1 GiB used, 1.0 TiB / 1.0 TiB avail; 3.2 MiB/s rd, 7.1 MiB/s wr, 533 op/s 2024-03-19T02:04:03.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:04:02 smithi099 ceph-mon[62951]: pgmap v1039: 97 pgs: 97 active+clean; 678 MiB data, 9.1 GiB used, 1.0 TiB / 1.0 TiB avail; 3.2 MiB/s rd, 7.1 MiB/s wr, 533 op/s 2024-03-19T02:04:03.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:04:02 smithi112 ceph-mon[66819]: pgmap v1039: 97 pgs: 97 active+clean; 678 MiB data, 9.1 GiB used, 1.0 TiB / 1.0 TiB avail; 3.2 MiB/s rd, 7.1 MiB/s wr, 533 op/s 2024-03-19T02:04:03.120 INFO:tasks.workunit.client.0.smithi099.stdout: 10 265769 22.27 MB/sec execute 512 sec latency 234.179 ms 2024-03-19T02:04:04.120 INFO:tasks.workunit.client.0.smithi099.stdout: 10 266187 22.27 MB/sec execute 513 sec latency 125.850 ms 2024-03-19T02:04:05.034 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.034+0000 7f4f41fa9640 1 -- 172.21.15.99:0/195337902 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4f3c101930 msgr2=0x7f4f3c101db0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:04:05.035 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.034+0000 7f4f41fa9640 1 --2- 172.21.15.99:0/195337902 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4f3c101930 0x7f4f3c101db0 secure :-1 s=READY pgs=774 cs=0 l=1 rev1=1 crypto rx=0x7f4f24009900 tx=0x7f4f2402bc30 comp rx=0 tx=0).stop 2024-03-19T02:04:05.035 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.034+0000 7f4f41fa9640 1 -- 172.21.15.99:0/195337902 shutdown_connections 2024-03-19T02:04:05.035 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.034+0000 7f4f41fa9640 1 --2- 172.21.15.99:0/195337902 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4f3c1022f0 0x7f4f3c108b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:05.035 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.034+0000 7f4f41fa9640 1 --2- 172.21.15.99:0/195337902 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4f3c101930 0x7f4f3c101db0 unknown :-1 s=CLOSED pgs=774 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:05.035 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.034+0000 7f4f41fa9640 1 --2- 172.21.15.99:0/195337902 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4f3c100740 0x7f4f3c100b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:05.035 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.034+0000 7f4f41fa9640 1 -- 172.21.15.99:0/195337902 >> 172.21.15.99:0/195337902 conn(0x7f4f3c0fbcb0 msgr2=0x7f4f3c0fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:04:05.035 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.034+0000 7f4f41fa9640 1 -- 172.21.15.99:0/195337902 shutdown_connections 2024-03-19T02:04:05.035 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.034+0000 7f4f41fa9640 1 -- 172.21.15.99:0/195337902 wait complete. 2024-03-19T02:04:05.035 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.035+0000 7f4f41fa9640 1 Processor -- start 2024-03-19T02:04:05.037 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.035+0000 7f4f41fa9640 1 -- start start 2024-03-19T02:04:05.038 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.035+0000 7f4f41fa9640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4f3c100740 0x7f4f3c1046d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:05.038 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.035+0000 7f4f41fa9640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4f3c101930 0x7f4f3c102d10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:05.038 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.036+0000 7f4f41fa9640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4f3c1022f0 0x7f4f3c103250 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:05.038 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.036+0000 7f4f41fa9640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f4f3c10c3c0 con 0x7f4f3c101930 2024-03-19T02:04:05.038 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.036+0000 7f4f41fa9640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f4f3c10c240 con 0x7f4f3c1022f0 2024-03-19T02:04:05.038 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.036+0000 7f4f41fa9640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f4f3c10c540 con 0x7f4f3c100740 2024-03-19T02:04:05.038 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.036+0000 7f4f3affd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4f3c101930 0x7f4f3c102d10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:05.038 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.036+0000 7f4f3affd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4f3c101930 0x7f4f3c102d10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:59852/0 (socket says 172.21.15.99:59852) 2024-03-19T02:04:05.038 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.036+0000 7f4f3affd640 1 -- 172.21.15.99:0/3338784395 learned_addr learned my addr 172.21.15.99:0/3338784395 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:04:05.038 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.036+0000 7f4f3b7fe640 1 --2- 172.21.15.99:0/3338784395 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4f3c100740 0x7f4f3c1046d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:05.038 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.036+0000 7f4f3bfff640 1 --2- 172.21.15.99:0/3338784395 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4f3c1022f0 0x7f4f3c103250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:05.039 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.036+0000 7f4f3affd640 1 -- 172.21.15.99:0/3338784395 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4f3c100740 msgr2=0x7f4f3c1046d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:04:05.039 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.036+0000 7f4f3affd640 1 --2- 172.21.15.99:0/3338784395 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4f3c100740 0x7f4f3c1046d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:05.039 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.036+0000 7f4f3affd640 1 -- 172.21.15.99:0/3338784395 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4f3c1022f0 msgr2=0x7f4f3c103250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:04:05.039 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.036+0000 7f4f3affd640 1 --2- 172.21.15.99:0/3338784395 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4f3c1022f0 0x7f4f3c103250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:05.039 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.036+0000 7f4f3affd640 1 -- 172.21.15.99:0/3338784395 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f4f3c103ae0 con 0x7f4f3c101930 2024-03-19T02:04:05.039 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.036+0000 7f4f3bfff640 1 --2- 172.21.15.99:0/3338784395 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4f3c1022f0 0x7f4f3c103250 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:04:05.039 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.036+0000 7f4f3affd640 1 --2- 172.21.15.99:0/3338784395 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4f3c101930 0x7f4f3c102d10 secure :-1 s=READY pgs=775 cs=0 l=1 rev1=1 crypto rx=0x7f4f24009230 tx=0x7f4f2402c650 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:04:05.039 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.036+0000 7f4f38ff9640 1 -- 172.21.15.99:0/3338784395 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4f24041070 con 0x7f4f3c101930 2024-03-19T02:04:05.039 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.036+0000 7f4f41fa9640 1 -- 172.21.15.99:0/3338784395 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f4f3c1a1b60 con 0x7f4f3c101930 2024-03-19T02:04:05.039 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.036+0000 7f4f38ff9640 1 -- 172.21.15.99:0/3338784395 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f4f24004450 con 0x7f4f3c101930 2024-03-19T02:04:05.039 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.036+0000 7f4f38ff9640 1 -- 172.21.15.99:0/3338784395 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4f24003e20 con 0x7f4f3c101930 2024-03-19T02:04:05.039 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.037+0000 7f4f41fa9640 1 -- 172.21.15.99:0/3338784395 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f4f3c1a2100 con 0x7f4f3c101930 2024-03-19T02:04:05.045 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.039+0000 7f4f38ff9640 1 -- 172.21.15.99:0/3338784395 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f4f24044070 con 0x7f4f3c101930 2024-03-19T02:04:05.045 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.039+0000 7f4f44b91740 1 Processor -- start 2024-03-19T02:04:05.045 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.039+0000 7f4f44b91740 1 -- start start 2024-03-19T02:04:05.045 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.039+0000 7f4f38ff9640 1 --2- 172.21.15.99:0/3338784395 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f4f280a28a0 0x7f4f280a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:05.045 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.039+0000 7f4f44b91740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5604ce9af100 0x5604ce9af520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:05.045 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.039+0000 7f4f44b91740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5604ce9afaf0 0x5604ce9b45f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:05.045 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.039+0000 7f4f44b91740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5604ce9b4c60 0x5604ce9b70f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:05.045 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.039+0000 7f4f44b91740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x5604ce913be0 con 0x5604ce9af100 2024-03-19T02:04:05.045 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.039+0000 7f4f44b91740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x5604ce8bc960 con 0x5604ce9b4c60 2024-03-19T02:04:05.046 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.039+0000 7f4f44b91740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x5604ce90fc80 con 0x5604ce9afaf0 2024-03-19T02:04:05.046 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.039+0000 7f4f3b7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5604ce9af100 0x5604ce9af520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:05.046 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.040+0000 7f4f38ff9640 1 -- 172.21.15.99:0/3338784395 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f4f240f0620 con 0x7f4f3c101930 2024-03-19T02:04:05.046 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.040+0000 7f4f3b7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5604ce9af100 0x5604ce9af520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:59858/0 (socket says 172.21.15.99:59858) 2024-03-19T02:04:05.046 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.040+0000 7f4f3b7fe640 1 -- 172.21.15.99:0/1432870141 learned_addr learned my addr 172.21.15.99:0/1432870141 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:04:05.046 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.040+0000 7f4f3affd640 1 --2- 172.21.15.99:0/1432870141 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5604ce9afaf0 0x5604ce9b45f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:05.046 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.040+0000 7f4f3bfff640 1 --2- 172.21.15.99:0/3338784395 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f4f280a28a0 0x7f4f280a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:05.046 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.040+0000 7f4f3bfff640 1 --2- 172.21.15.99:0/1432870141 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5604ce9b4c60 0x5604ce9b70f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:05.046 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.040+0000 7f4f3b7fe640 1 -- 172.21.15.99:0/1432870141 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5604ce9afaf0 msgr2=0x5604ce9b45f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:04:05.046 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.040+0000 7f4f3b7fe640 1 --2- 172.21.15.99:0/1432870141 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5604ce9afaf0 0x5604ce9b45f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:05.046 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.040+0000 7f4f3b7fe640 1 -- 172.21.15.99:0/1432870141 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5604ce9b4c60 msgr2=0x5604ce9b70f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:04:05.046 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.040+0000 7f4f3b7fe640 1 --2- 172.21.15.99:0/1432870141 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5604ce9b4c60 0x5604ce9b70f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:05.046 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.040+0000 7f4f3b7fe640 1 -- 172.21.15.99:0/1432870141 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5604ce8b3d20 con 0x5604ce9af100 2024-03-19T02:04:05.047 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.040+0000 7f4f3b7fe640 1 --2- 172.21.15.99:0/1432870141 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5604ce9af100 0x5604ce9af520 secure :-1 s=READY pgs=776 cs=0 l=1 rev1=1 crypto rx=0x7f4f2c011470 tx=0x7f4f2c011930 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3470270a8c66619f server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:04:05.047 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.040+0000 7f4f3bfff640 1 --2- 172.21.15.99:0/3338784395 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f4f280a28a0 0x7f4f280a4d60 secure :-1 s=READY pgs=385 cs=0 l=1 rev1=1 crypto rx=0x7f4f34009710 tx=0x7f4f34009f50 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:04:05.047 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.040+0000 7f4f31ffb640 1 -- 172.21.15.99:0/1432870141 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4f2c01e070 con 0x5604ce9af100 2024-03-19T02:04:05.047 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.040+0000 7f4f31ffb640 1 -- 172.21.15.99:0/1432870141 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f4f2c003c30 con 0x5604ce9af100 2024-03-19T02:04:05.047 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.040+0000 7f4f31ffb640 1 -- 172.21.15.99:0/1432870141 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4f2c00f9d0 con 0x5604ce9af100 2024-03-19T02:04:05.047 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.041+0000 7f4f44b91740 1 -- 172.21.15.99:0/1432870141 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5604ce9af100 msgr2=0x5604ce9af520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:04:05.047 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.041+0000 7f4f44b91740 1 --2- 172.21.15.99:0/1432870141 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5604ce9af100 0x5604ce9af520 secure :-1 s=READY pgs=776 cs=0 l=1 rev1=1 crypto rx=0x7f4f2c011470 tx=0x7f4f2c011930 comp rx=0 tx=0).stop 2024-03-19T02:04:05.047 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.041+0000 7f4f44b91740 1 -- 172.21.15.99:0/1432870141 shutdown_connections 2024-03-19T02:04:05.047 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.041+0000 7f4f44b91740 1 --2- 172.21.15.99:0/1432870141 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5604ce9b4c60 0x5604ce9b70f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:05.047 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.041+0000 7f4f44b91740 1 --2- 172.21.15.99:0/1432870141 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5604ce9afaf0 0x5604ce9b45f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:05.047 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.041+0000 7f4f44b91740 1 --2- 172.21.15.99:0/1432870141 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5604ce9af100 0x5604ce9af520 unknown :-1 s=CLOSED pgs=776 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:05.047 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.041+0000 7f4f44b91740 1 -- 172.21.15.99:0/1432870141 >> 172.21.15.99:0/1432870141 conn(0x5604ce9aa670 msgr2=0x5604ce9aaa70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:04:05.047 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.041+0000 7f4f44b91740 1 -- 172.21.15.99:0/1432870141 shutdown_connections 2024-03-19T02:04:05.047 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.041+0000 7f4f44b91740 1 -- 172.21.15.99:0/1432870141 wait complete. 2024-03-19T02:04:05.048 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.041+0000 7f4f44b91740 1 -- start start 2024-03-19T02:04:05.048 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.042+0000 7f4f44b91740 1 Processor -- start 2024-03-19T02:04:05.048 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.042+0000 7f4f44b91740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5604ce9af100 0x5604ce9b9070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:05.048 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.042+0000 7f4f44b91740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5604ce9afaf0 0x5604ce9b95b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:05.048 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.042+0000 7f4f44b91740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5604ce9b4c60 0x5604ce9c05f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:05.048 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.042+0000 7f4f44b91740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x5604ce9b7c10 con 0x5604ce9afaf0 2024-03-19T02:04:05.048 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.042+0000 7f4f44b91740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x5604ce8cfdb0 con 0x5604ce9af100 2024-03-19T02:04:05.048 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.042+0000 7f4f44b91740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x5604ce9b7d90 con 0x5604ce9b4c60 2024-03-19T02:04:05.048 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.042+0000 7f4f44b91740 10 client.0 _ll_register_callbacks cb 0x5604ce8d4e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T02:04:05.048 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.042+0000 7f4f3affd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5604ce9afaf0 0x5604ce9b95b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:05.048 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.042+0000 7f4f3affd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5604ce9afaf0 0x5604ce9b95b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:59874/0 (socket says 172.21.15.99:59874) 2024-03-19T02:04:05.048 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.042+0000 7f4f3affd640 1 -- 172.21.15.99:0/4290445705 learned_addr learned my addr 172.21.15.99:0/4290445705 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:04:05.048 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.042+0000 7f4f3b7fe640 1 --2- 172.21.15.99:0/4290445705 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5604ce9af100 0x5604ce9b9070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:05.048 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.042+0000 7f4f3bfff640 1 --2- 172.21.15.99:0/4290445705 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5604ce9b4c60 0x5604ce9c05f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:05.049 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.042+0000 7f4f3affd640 1 -- 172.21.15.99:0/4290445705 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5604ce9b4c60 msgr2=0x5604ce9c05f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:04:05.049 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.042+0000 7f4f3affd640 1 --2- 172.21.15.99:0/4290445705 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5604ce9b4c60 0x5604ce9c05f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:05.049 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.042+0000 7f4f3affd640 1 -- 172.21.15.99:0/4290445705 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5604ce9af100 msgr2=0x5604ce9b9070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:04:05.049 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.042+0000 7f4f3affd640 1 --2- 172.21.15.99:0/4290445705 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5604ce9af100 0x5604ce9b9070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:05.049 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.042+0000 7f4f3affd640 1 -- 172.21.15.99:0/4290445705 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5604ce8cec60 con 0x5604ce9afaf0 2024-03-19T02:04:05.049 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.042+0000 7f4f3b7fe640 1 --2- 172.21.15.99:0/4290445705 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5604ce9af100 0x5604ce9b9070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-19T02:04:05.049 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.043+0000 7f4f3affd640 1 --2- 172.21.15.99:0/4290445705 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5604ce9afaf0 0x5604ce9b95b0 secure :-1 s=READY pgs=777 cs=0 l=1 rev1=1 crypto rx=0x7f4f240ee0e0 tx=0x7f4f240bcc30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1bdad42907d2ad13 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:04:05.049 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.043+0000 7f4f1affd640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T02:04:05.049 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.043+0000 7f4f1affd640 1 -- 172.21.15.99:0/4290445705 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4f2404b040 con 0x5604ce9afaf0 2024-03-19T02:04:05.049 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.043+0000 7f4f1affd640 1 -- 172.21.15.99:0/4290445705 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f4f240b96a0 con 0x5604ce9afaf0 2024-03-19T02:04:05.049 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.043+0000 7f4f44b91740 1 -- 172.21.15.99:0/4290445705 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f4f2c003c30 con 0x5604ce9afaf0 2024-03-19T02:04:05.049 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.043+0000 7f4f1affd640 1 -- 172.21.15.99:0/4290445705 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=41) ==== 24+0+0 (secure 0 0 0) 0x7f4f2409a4c0 con 0x5604ce9afaf0 2024-03-19T02:04:05.049 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.043+0000 7f4f44b91740 10 client.17172 fetch_fsmap learned FSMap version 41 2024-03-19T02:04:05.049 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.043+0000 7f4f44b91740 1 -- 172.21.15.99:0/4290445705 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=41,monmap=4+}) -- 0x5604ce628210 con 0x5604ce9afaf0 2024-03-19T02:04:05.049 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.043+0000 7f4f1affd640 1 -- 172.21.15.99:0/4290445705 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f4f2c003c30 con 0x5604ce9afaf0 2024-03-19T02:04:05.050 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.043+0000 7f4f1affd640 1 -- 172.21.15.99:0/4290445705 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 41) ==== 8354+0+0 (secure 0 0 0) 0x7f4f240c5290 con 0x5604ce9afaf0 2024-03-19T02:04:05.050 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.044+0000 7f4f44b91740 10 client.17172 fetch_fsmap finished waiting for FSMap version 41 2024-03-19T02:04:05.050 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.044+0000 7f4f44b91740 10 client.17172 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T02:04:05.050 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.044+0000 7f4f44b91740 20 client.17172 populate_metadata read hostname 'smithi099' 2024-03-19T02:04:05.050 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.044+0000 7f4f44b91740 1 --2- 172.21.15.99:0/4290445705 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5604ce9c4f90 0x5604ce9c7420 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:05.050 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.044+0000 7f4f44b91740 4 client.17172 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T02:04:05.050 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.044+0000 7f4f44b91740 1 -- 172.21.15.99:0/4290445705 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5604ce921120 con 0x5604ce9c4f90 2024-03-19T02:04:05.050 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.044+0000 7f4f3b7fe640 1 --2- 172.21.15.99:0/4290445705 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5604ce9c4f90 0x5604ce9c7420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:05.050 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.044+0000 7f4f1affd640 10 client.17172 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T02:04:05.050 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.044+0000 7f4f3b7fe640 1 --2- 172.21.15.99:0/4290445705 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5604ce9c4f90 0x5604ce9c7420 crc :-1 s=READY pgs=233 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=ee24af289c40219e server_cookie=a5c7c85ca749f77a in_seq=0 out_seq=0 2024-03-19T02:04:05.051 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.046+0000 7f4f1affd640 1 -- 172.21.15.99:0/4290445705 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x5604ce921120 con 0x5604ce9c4f90 2024-03-19T02:04:05.052 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.046+0000 7f4f1affd640 10 client.17172 handle_command_reply: tid=0 2024-03-19T02:04:05.052 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.046+0000 7f4f44b91740 1 client.17172 shutdown 2024-03-19T02:04:05.052 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.046+0000 7f4f44b91740 1 -- 172.21.15.99:0/4290445705 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5604ce9afaf0 msgr2=0x5604ce9b95b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:04:05.052 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.046+0000 7f4f44b91740 1 --2- 172.21.15.99:0/4290445705 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5604ce9afaf0 0x5604ce9b95b0 secure :-1 s=READY pgs=777 cs=0 l=1 rev1=1 crypto rx=0x7f4f240ee0e0 tx=0x7f4f240bcc30 comp rx=0 tx=0).stop 2024-03-19T02:04:05.052 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.046+0000 7f4f44b91740 1 -- 172.21.15.99:0/4290445705 shutdown_connections 2024-03-19T02:04:05.052 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.046+0000 7f4f44b91740 1 --2- 172.21.15.99:0/4290445705 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5604ce9c4f90 0x5604ce9c7420 crc :-1 s=READY pgs=233 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:05.052 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.046+0000 7f4f44b91740 1 --2- 172.21.15.99:0/4290445705 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5604ce9b4c60 0x5604ce9c05f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:05.052 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.046+0000 7f4f1affd640 0 client.17172 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T02:04:05.052 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.046+0000 7f4f44b91740 1 --2- 172.21.15.99:0/4290445705 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5604ce9afaf0 0x5604ce9b95b0 unknown :-1 s=CLOSED pgs=777 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:05.052 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.046+0000 7f4f44b91740 1 --2- 172.21.15.99:0/4290445705 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5604ce9af100 0x5604ce9b9070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:05.052 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.046+0000 7f4f44b91740 1 -- 172.21.15.99:0/4290445705 >> 172.21.15.99:0/4290445705 conn(0x5604ce9aa670 msgr2=0x5604ce9ab8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:04:05.052 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.046+0000 7f4f44b91740 1 -- 172.21.15.99:0/4290445705 shutdown_connections 2024-03-19T02:04:05.052 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.046+0000 7f4f44b91740 1 -- 172.21.15.99:0/4290445705 wait complete. 2024-03-19T02:04:05.052 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.047+0000 7f4f44b91740 20 client.17172 trim_cache size 0 max 16384 2024-03-19T02:04:05.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:04:04 smithi115 ceph-mon[67332]: pgmap v1040: 97 pgs: 97 active+clean; 682 MiB data, 9.1 GiB used, 1.0 TiB / 1.0 TiB avail; 3.5 MiB/s rd, 7.4 MiB/s wr, 526 op/s 2024-03-19T02:04:05.068 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.063+0000 7f4f44b91740 1 Processor -- start 2024-03-19T02:04:05.068 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.063+0000 7f4f44b91740 1 -- start start 2024-03-19T02:04:05.068 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.063+0000 7f4f44b91740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5604ce9af100 0x5604ce9a9220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:05.068 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.063+0000 7f4f44b91740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5604ce9afaf0 0x5604ce9a9760 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:05.068 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.063+0000 7f4f44b91740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5604ce9b4c60 0x5604ce9c05b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:05.068 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.063+0000 7f4f44b91740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f4f3c1ad810 con 0x5604ce9af100 2024-03-19T02:04:05.068 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.063+0000 7f4f44b91740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f4f3c1ab080 con 0x5604ce9afaf0 2024-03-19T02:04:05.068 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.063+0000 7f4f44b91740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f4f3c1a8ed0 con 0x5604ce9b4c60 2024-03-19T02:04:05.068 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.063+0000 7f4f3b7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5604ce9af100 0x5604ce9a9220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:05.069 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.063+0000 7f4f3b7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5604ce9af100 0x5604ce9a9220 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:59882/0 (socket says 172.21.15.99:59882) 2024-03-19T02:04:05.069 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.063+0000 7f4f3b7fe640 1 -- 172.21.15.99:0/2142295820 learned_addr learned my addr 172.21.15.99:0/2142295820 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:04:05.069 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.063+0000 7f4f3affd640 1 --2- 172.21.15.99:0/2142295820 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5604ce9afaf0 0x5604ce9a9760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:05.069 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.063+0000 7f4f3bfff640 1 --2- 172.21.15.99:0/2142295820 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5604ce9b4c60 0x5604ce9c05b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:05.069 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.064+0000 7f4f3b7fe640 1 -- 172.21.15.99:0/2142295820 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5604ce9b4c60 msgr2=0x5604ce9c05b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:04:05.069 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.064+0000 7f4f3b7fe640 1 --2- 172.21.15.99:0/2142295820 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5604ce9b4c60 0x5604ce9c05b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:05.069 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.064+0000 7f4f3b7fe640 1 -- 172.21.15.99:0/2142295820 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5604ce9afaf0 msgr2=0x5604ce9a9760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:04:05.069 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.064+0000 7f4f3b7fe640 1 --2- 172.21.15.99:0/2142295820 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5604ce9afaf0 0x5604ce9a9760 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:05.069 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.064+0000 7f4f3b7fe640 1 -- 172.21.15.99:0/2142295820 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5604ce6f4620 con 0x5604ce9af100 2024-03-19T02:04:05.069 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.064+0000 7f4f3b7fe640 1 --2- 172.21.15.99:0/2142295820 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5604ce9af100 0x5604ce9a9220 secure :-1 s=READY pgs=778 cs=0 l=1 rev1=1 crypto rx=0x7f4f2c000f80 tx=0x7f4f2c005af0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e10ecefe22378869 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:04:05.069 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.064+0000 7f4f317fa640 1 -- 172.21.15.99:0/2142295820 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4f2c01e040 con 0x5604ce9af100 2024-03-19T02:04:05.069 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.064+0000 7f4f317fa640 1 -- 172.21.15.99:0/2142295820 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f4f2c02a7e0 con 0x5604ce9af100 2024-03-19T02:04:05.069 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.064+0000 7f4f317fa640 1 -- 172.21.15.99:0/2142295820 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4f2c00fda0 con 0x5604ce9af100 2024-03-19T02:04:05.069 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.065+0000 7f4f44b91740 1 -- 172.21.15.99:0/2142295820 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5604ce9af100 msgr2=0x5604ce9a9220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:04:05.070 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.065+0000 7f4f44b91740 1 --2- 172.21.15.99:0/2142295820 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5604ce9af100 0x5604ce9a9220 secure :-1 s=READY pgs=778 cs=0 l=1 rev1=1 crypto rx=0x7f4f2c000f80 tx=0x7f4f2c005af0 comp rx=0 tx=0).stop 2024-03-19T02:04:05.070 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.065+0000 7f4f44b91740 1 -- 172.21.15.99:0/2142295820 shutdown_connections 2024-03-19T02:04:05.070 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.065+0000 7f4f44b91740 1 --2- 172.21.15.99:0/2142295820 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5604ce9afaf0 0x5604ce9a9760 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:05.070 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.065+0000 7f4f44b91740 1 --2- 172.21.15.99:0/2142295820 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5604ce9b4c60 0x5604ce9c05b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:05.070 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.065+0000 7f4f44b91740 1 --2- 172.21.15.99:0/2142295820 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5604ce9af100 0x5604ce9a9220 unknown :-1 s=CLOSED pgs=778 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:05.070 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.065+0000 7f4f44b91740 1 -- 172.21.15.99:0/2142295820 >> 172.21.15.99:0/2142295820 conn(0x5604ce9aa670 msgr2=0x5604ce9aaa50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:04:05.070 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.065+0000 7f4f44b91740 1 -- 172.21.15.99:0/2142295820 shutdown_connections 2024-03-19T02:04:05.070 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.065+0000 7f4f44b91740 1 -- 172.21.15.99:0/2142295820 wait complete. 2024-03-19T02:04:05.070 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.065+0000 7f4f44b91740 1 -- start start 2024-03-19T02:04:05.070 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.065+0000 7f4f44b91740 1 Processor -- start 2024-03-19T02:04:05.070 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.065+0000 7f4f44b91740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5604ce9af100 0x5604ce9bde60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:05.070 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.065+0000 7f4f44b91740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5604ce9afaf0 0x5604ce9be3a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:05.070 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.065+0000 7f4f44b91740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5604ce9b4c60 0x5604ce9bf360 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:05.070 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.065+0000 7f4f44b91740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f4f3c1a9540 con 0x5604ce9af100 2024-03-19T02:04:05.070 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.065+0000 7f4f44b91740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f4f3c1aa9b0 con 0x5604ce9afaf0 2024-03-19T02:04:05.071 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.065+0000 7f4f44b91740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f4f3c1a9c10 con 0x5604ce9b4c60 2024-03-19T02:04:05.071 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.066+0000 7f4f44b91740 10 client.0 _ll_register_callbacks cb 0x7f4f240b92f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T02:04:05.071 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.066+0000 7f4f3b7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5604ce9af100 0x5604ce9bde60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:05.071 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.066+0000 7f4f3affd640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5604ce9afaf0 0x5604ce9be3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:05.071 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.066+0000 7f4f3affd640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5604ce9afaf0 0x5604ce9be3a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:59734/0 (socket says 172.21.15.99:59734) 2024-03-19T02:04:05.071 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.066+0000 7f4f3b7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5604ce9af100 0x5604ce9bde60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:59890/0 (socket says 172.21.15.99:59890) 2024-03-19T02:04:05.071 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.066+0000 7f4f3affd640 1 -- 172.21.15.99:0/2172538513 learned_addr learned my addr 172.21.15.99:0/2172538513 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:04:05.071 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.066+0000 7f4f3bfff640 1 --2- 172.21.15.99:0/2172538513 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5604ce9b4c60 0x5604ce9bf360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:05.071 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.066+0000 7f4f3affd640 1 -- 172.21.15.99:0/2172538513 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5604ce9b4c60 msgr2=0x5604ce9bf360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:04:05.071 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.066+0000 7f4f3affd640 1 --2- 172.21.15.99:0/2172538513 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5604ce9b4c60 0x5604ce9bf360 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:05.071 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.066+0000 7f4f3affd640 1 -- 172.21.15.99:0/2172538513 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5604ce9af100 msgr2=0x5604ce9bde60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:04:05.071 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.066+0000 7f4f3affd640 1 --2- 172.21.15.99:0/2172538513 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5604ce9af100 0x5604ce9bde60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:05.071 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.066+0000 7f4f3affd640 1 -- 172.21.15.99:0/2172538513 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5604ce9bf950 con 0x5604ce9afaf0 2024-03-19T02:04:05.072 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.066+0000 7f4f3affd640 1 --2- 172.21.15.99:0/2172538513 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5604ce9afaf0 0x5604ce9be3a0 secure :-1 s=READY pgs=226 cs=0 l=1 rev1=1 crypto rx=0x7f4f2402cfd0 tx=0x7f4f240dd050 comp rx=0 tx=0).ready entity=mon.2 client_cookie=9e89979b3c5cd39d server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:04:05.072 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.066+0000 7f4f1affd640 10 client.0 ms_handle_connect on v2:172.21.15.112:3300/0 2024-03-19T02:04:05.072 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.066+0000 7f4f1affd640 1 -- 172.21.15.99:0/2172538513 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4f2404b070 con 0x5604ce9afaf0 2024-03-19T02:04:05.072 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.066+0000 7f4f44b91740 1 -- 172.21.15.99:0/2172538513 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f4f2c02a7e0 con 0x5604ce9afaf0 2024-03-19T02:04:05.072 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.066+0000 7f4f1affd640 1 -- 172.21.15.99:0/2172538513 <== mon.2 v2:172.21.15.112:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f4f240b96a0 con 0x5604ce9afaf0 2024-03-19T02:04:05.072 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.066+0000 7f4f1affd640 1 -- 172.21.15.99:0/2172538513 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_get_version_reply(handle=1 version=41) ==== 24+0+0 (secure 0 0 0) 0x7f4f240bce00 con 0x5604ce9afaf0 2024-03-19T02:04:05.072 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.066+0000 7f4f44b91740 10 client.26954 fetch_fsmap learned FSMap version 41 2024-03-19T02:04:05.072 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.066+0000 7f4f44b91740 1 -- 172.21.15.99:0/2172538513 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,fsmap=41,monmap=4+}) -- 0x5604cea2b550 con 0x5604ce9afaf0 2024-03-19T02:04:05.072 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.067+0000 7f4f1affd640 1 -- 172.21.15.99:0/2172538513 <== mon.2 v2:172.21.15.112:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f4f2c02a7e0 con 0x5604ce9afaf0 2024-03-19T02:04:05.072 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.067+0000 7f4f1affd640 1 -- 172.21.15.99:0/2172538513 <== mon.2 v2:172.21.15.112:3300/0 5 ==== fsmap(e 41) ==== 8354+0+0 (secure 0 0 0) 0x7f4f240bc480 con 0x5604ce9afaf0 2024-03-19T02:04:05.073 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.067+0000 7f4f44b91740 10 client.26954 fetch_fsmap finished waiting for FSMap version 41 2024-03-19T02:04:05.073 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.067+0000 7f4f44b91740 10 client.26954 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T02:04:05.073 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.067+0000 7f4f44b91740 20 client.26954 populate_metadata read hostname 'smithi099' 2024-03-19T02:04:05.073 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.067+0000 7f4f44b91740 1 --2- 172.21.15.99:0/2172538513 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5604ce9c4f90 0x5604cea2dcf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:05.073 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.067+0000 7f4f44b91740 4 client.26954 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T02:04:05.073 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.067+0000 7f4f44b91740 1 -- 172.21.15.99:0/2172538513 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x5604ce91b470 con 0x5604ce9c4f90 2024-03-19T02:04:05.073 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.067+0000 7f4f3b7fe640 1 --2- 172.21.15.99:0/2172538513 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5604ce9c4f90 0x5604cea2dcf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:05.073 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.067+0000 7f4f3b7fe640 1 --2- 172.21.15.99:0/2172538513 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5604ce9c4f90 0x5604cea2dcf0 crc :-1 s=READY pgs=234 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=9d37005e97192545 server_cookie=eb38e98c9dfa2b87 in_seq=0 out_seq=0 2024-03-19T02:04:05.073 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.067+0000 7f4f1affd640 10 client.26954 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T02:04:05.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:04:04 smithi099 ceph-mon[62951]: pgmap v1040: 97 pgs: 97 active+clean; 682 MiB data, 9.1 GiB used, 1.0 TiB / 1.0 TiB avail; 3.5 MiB/s rd, 7.4 MiB/s wr, 526 op/s 2024-03-19T02:04:05.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:04:04 smithi112 ceph-mon[66819]: pgmap v1040: 97 pgs: 97 active+clean; 682 MiB data, 9.1 GiB used, 1.0 TiB / 1.0 TiB avail; 3.5 MiB/s rd, 7.4 MiB/s wr, 526 op/s 2024-03-19T02:04:05.120 INFO:tasks.workunit.client.0.smithi099.stdout: 10 266611 22.26 MB/sec execute 514 sec latency 70.939 ms 2024-03-19T02:04:05.633 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.632+0000 7f4f1affd640 1 -- 172.21.15.99:0/2172538513 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x5604ce91b470 con 0x5604ce9c4f90 2024-03-19T02:04:05.633 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.632+0000 7f4f1affd640 10 client.26954 handle_command_reply: tid=0 2024-03-19T02:04:05.633 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.632+0000 7f4f44b91740 1 client.26954 shutdown 2024-03-19T02:04:05.633 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.632+0000 7f4f44b91740 1 -- 172.21.15.99:0/2172538513 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5604ce9afaf0 msgr2=0x5604ce9be3a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:04:05.633 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.632+0000 7f4f44b91740 1 --2- 172.21.15.99:0/2172538513 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5604ce9afaf0 0x5604ce9be3a0 secure :-1 s=READY pgs=226 cs=0 l=1 rev1=1 crypto rx=0x7f4f2402cfd0 tx=0x7f4f240dd050 comp rx=0 tx=0).stop 2024-03-19T02:04:05.633 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.632+0000 7f4f44b91740 1 -- 172.21.15.99:0/2172538513 shutdown_connections 2024-03-19T02:04:05.633 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.632+0000 7f4f44b91740 1 --2- 172.21.15.99:0/2172538513 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x5604ce9c4f90 0x5604cea2dcf0 crc :-1 s=READY pgs=234 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:05.633 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.632+0000 7f4f44b91740 1 --2- 172.21.15.99:0/2172538513 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x5604ce9afaf0 0x5604ce9be3a0 unknown :-1 s=CLOSED pgs=226 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:05.634 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.632+0000 7f4f44b91740 1 --2- 172.21.15.99:0/2172538513 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x5604ce9b4c60 0x5604ce9bf360 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:05.634 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.632+0000 7f4f44b91740 1 --2- 172.21.15.99:0/2172538513 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x5604ce9af100 0x5604ce9bde60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:05.634 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.632+0000 7f4f44b91740 1 -- 172.21.15.99:0/2172538513 >> 172.21.15.99:0/2172538513 conn(0x5604ce9aa670 msgr2=0x5604ce9b3fa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:04:05.634 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.632+0000 7f4f1affd640 0 client.26954 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T02:04:05.634 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.632+0000 7f4f44b91740 1 -- 172.21.15.99:0/2172538513 shutdown_connections 2024-03-19T02:04:05.634 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.632+0000 7f4f44b91740 1 -- 172.21.15.99:0/2172538513 wait complete. 2024-03-19T02:04:05.634 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.632+0000 7f4f44b91740 20 client.26954 trim_cache size 0 max 16384 2024-03-19T02:04:05.637 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T02:04:05.637 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T02:04:05.637 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T02:04:05.637 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T02:04:05.638 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.633+0000 7f4f41fa9640 1 -- 172.21.15.99:0/3338784395 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f4f280a28a0 msgr2=0x7f4f280a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:04:05.638 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.633+0000 7f4f41fa9640 1 --2- 172.21.15.99:0/3338784395 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f4f280a28a0 0x7f4f280a4d60 secure :-1 s=READY pgs=385 cs=0 l=1 rev1=1 crypto rx=0x7f4f34009710 tx=0x7f4f34009f50 comp rx=0 tx=0).stop 2024-03-19T02:04:05.638 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.633+0000 7f4f41fa9640 1 -- 172.21.15.99:0/3338784395 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4f3c101930 msgr2=0x7f4f3c102d10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:04:05.639 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.633+0000 7f4f41fa9640 1 --2- 172.21.15.99:0/3338784395 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4f3c101930 0x7f4f3c102d10 secure :-1 s=READY pgs=775 cs=0 l=1 rev1=1 crypto rx=0x7f4f24009230 tx=0x7f4f2402c650 comp rx=0 tx=0).stop 2024-03-19T02:04:05.639 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.633+0000 7f4f41fa9640 1 -- 172.21.15.99:0/3338784395 shutdown_connections 2024-03-19T02:04:05.639 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.633+0000 7f4f41fa9640 1 --2- 172.21.15.99:0/3338784395 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f4f280a28a0 0x7f4f280a4d60 unknown :-1 s=CLOSED pgs=385 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:05.639 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.633+0000 7f4f41fa9640 1 --2- 172.21.15.99:0/3338784395 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f4f3c1022f0 0x7f4f3c103250 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:05.639 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.633+0000 7f4f41fa9640 1 --2- 172.21.15.99:0/3338784395 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f4f3c101930 0x7f4f3c102d10 unknown :-1 s=CLOSED pgs=775 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:05.639 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.633+0000 7f4f41fa9640 1 --2- 172.21.15.99:0/3338784395 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f4f3c100740 0x7f4f3c1046d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:05.639 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.633+0000 7f4f41fa9640 1 -- 172.21.15.99:0/3338784395 >> 172.21.15.99:0/3338784395 conn(0x7f4f3c0fbcb0 msgr2=0x7f4f3c0fd6d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:04:05.639 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.633+0000 7f4f41fa9640 1 -- 172.21.15.99:0/3338784395 shutdown_connections 2024-03-19T02:04:05.639 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:05.633+0000 7f4f41fa9640 1 -- 172.21.15.99:0/3338784395 wait complete. 2024-03-19T02:04:06.120 INFO:tasks.workunit.client.0.smithi099.stdout: 10 266842 22.25 MB/sec execute 515 sec latency 357.871 ms 2024-03-19T02:04:07.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:04:06 smithi115 ceph-mon[67332]: pgmap v1041: 97 pgs: 97 active+clean; 683 MiB data, 9.1 GiB used, 1.0 TiB / 1.0 TiB avail; 2.7 MiB/s rd, 5.8 MiB/s wr, 395 op/s 2024-03-19T02:04:07.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:04:06 smithi099 ceph-mon[62951]: pgmap v1041: 97 pgs: 97 active+clean; 683 MiB data, 9.1 GiB used, 1.0 TiB / 1.0 TiB avail; 2.7 MiB/s rd, 5.8 MiB/s wr, 395 op/s 2024-03-19T02:04:07.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:04:06 smithi112 ceph-mon[66819]: pgmap v1041: 97 pgs: 97 active+clean; 683 MiB data, 9.1 GiB used, 1.0 TiB / 1.0 TiB avail; 2.7 MiB/s rd, 5.8 MiB/s wr, 395 op/s 2024-03-19T02:04:07.120 INFO:tasks.workunit.client.0.smithi099.stdout: 10 267196 22.25 MB/sec execute 516 sec latency 107.606 ms 2024-03-19T02:04:08.121 INFO:tasks.workunit.client.0.smithi099.stdout: 10 267491 22.24 MB/sec execute 517 sec latency 218.110 ms 2024-03-19T02:04:09.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:04:08 smithi115 ceph-mon[67332]: pgmap v1042: 97 pgs: 97 active+clean; 681 MiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.4 MiB/s rd, 6.9 MiB/s wr, 529 op/s 2024-03-19T02:04:09.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:04:08 smithi099 ceph-mon[62951]: pgmap v1042: 97 pgs: 97 active+clean; 681 MiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.4 MiB/s rd, 6.9 MiB/s wr, 529 op/s 2024-03-19T02:04:09.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:04:08 smithi112 ceph-mon[66819]: pgmap v1042: 97 pgs: 97 active+clean; 681 MiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.4 MiB/s rd, 6.9 MiB/s wr, 529 op/s 2024-03-19T02:04:09.122 INFO:tasks.workunit.client.0.smithi099.stdout: 10 267899 22.24 MB/sec execute 518 sec latency 107.812 ms 2024-03-19T02:04:10.120 INFO:tasks.workunit.client.0.smithi099.stdout: 10 268321 22.23 MB/sec execute 519 sec latency 79.288 ms 2024-03-19T02:04:11.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:04:10 smithi115 ceph-mon[67332]: pgmap v1043: 97 pgs: 97 active+clean; 680 MiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.2 MiB/s rd, 5.9 MiB/s wr, 495 op/s 2024-03-19T02:04:11.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:04:10 smithi099 ceph-mon[62951]: pgmap v1043: 97 pgs: 97 active+clean; 680 MiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.2 MiB/s rd, 5.9 MiB/s wr, 495 op/s 2024-03-19T02:04:11.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:04:10 smithi112 ceph-mon[66819]: pgmap v1043: 97 pgs: 97 active+clean; 680 MiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.2 MiB/s rd, 5.9 MiB/s wr, 495 op/s 2024-03-19T02:04:11.120 INFO:tasks.workunit.client.0.smithi099.stdout: 10 268722 22.23 MB/sec execute 520 sec latency 167.447 ms 2024-03-19T02:04:12.120 INFO:tasks.workunit.client.0.smithi099.stdout: 10 269137 22.24 MB/sec execute 521 sec latency 142.555 ms 2024-03-19T02:04:13.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:04:12 smithi115 ceph-mon[67332]: pgmap v1044: 97 pgs: 97 active+clean; 675 MiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.5 MiB/s rd, 6.0 MiB/s wr, 535 op/s 2024-03-19T02:04:13.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:04:12 smithi099 ceph-mon[62951]: pgmap v1044: 97 pgs: 97 active+clean; 675 MiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.5 MiB/s rd, 6.0 MiB/s wr, 535 op/s 2024-03-19T02:04:13.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:04:12 smithi112 ceph-mon[66819]: pgmap v1044: 97 pgs: 97 active+clean; 675 MiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.5 MiB/s rd, 6.0 MiB/s wr, 535 op/s 2024-03-19T02:04:13.120 INFO:tasks.workunit.client.0.smithi099.stdout: 10 269351 22.21 MB/sec execute 522 sec latency 421.228 ms 2024-03-19T02:04:14.120 INFO:tasks.workunit.client.0.smithi099.stdout: 10 269716 22.21 MB/sec execute 523 sec latency 106.846 ms 2024-03-19T02:04:15.058 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:04:14 smithi115 ceph-mon[67332]: pgmap v1045: 97 pgs: 97 active+clean; 670 MiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 6.4 MiB/s wr, 548 op/s 2024-03-19T02:04:15.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:04:14 smithi099 ceph-mon[62951]: pgmap v1045: 97 pgs: 97 active+clean; 670 MiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 6.4 MiB/s wr, 548 op/s 2024-03-19T02:04:15.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:04:14 smithi112 ceph-mon[66819]: pgmap v1045: 97 pgs: 97 active+clean; 670 MiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 6.4 MiB/s wr, 548 op/s 2024-03-19T02:04:15.120 INFO:tasks.workunit.client.0.smithi099.stdout: 10 270183 22.21 MB/sec execute 524 sec latency 61.155 ms 2024-03-19T02:04:16.120 INFO:tasks.workunit.client.0.smithi099.stdout: 10 270639 22.21 MB/sec execute 525 sec latency 101.139 ms 2024-03-19T02:04:17.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:04:16 smithi115 ceph-mon[67332]: pgmap v1046: 97 pgs: 97 active+clean; 671 MiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.3 MiB/s rd, 5.2 MiB/s wr, 501 op/s 2024-03-19T02:04:17.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:04:16 smithi099 ceph-mon[62951]: pgmap v1046: 97 pgs: 97 active+clean; 671 MiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.3 MiB/s rd, 5.2 MiB/s wr, 501 op/s 2024-03-19T02:04:17.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:04:16 smithi112 ceph-mon[66819]: pgmap v1046: 97 pgs: 97 active+clean; 671 MiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.3 MiB/s rd, 5.2 MiB/s wr, 501 op/s 2024-03-19T02:04:17.120 INFO:tasks.workunit.client.0.smithi099.stdout: 10 271174 22.23 MB/sec execute 526 sec latency 70.810 ms 2024-03-19T02:04:18.122 INFO:tasks.workunit.client.0.smithi099.stdout: 10 271467 22.22 MB/sec execute 527 sec latency 202.178 ms 2024-03-19T02:04:18.740 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.727+0000 7fb3dbec0640 1 -- 172.21.15.99:0/1203534475 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb3d4102360 msgr2=0x7fb3d4104750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:04:18.741 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.727+0000 7fb3dbec0640 1 --2- 172.21.15.99:0/1203534475 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb3d4102360 0x7fb3d4104750 secure :-1 s=READY pgs=779 cs=0 l=1 rev1=1 crypto rx=0x7fb3d0009900 tx=0x7fb3d002bc30 comp rx=0 tx=0).stop 2024-03-19T02:04:18.741 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.727+0000 7fb3dbec0640 1 -- 172.21.15.99:0/1203534475 shutdown_connections 2024-03-19T02:04:18.741 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.727+0000 7fb3dbec0640 1 --2- 172.21.15.99:0/1203534475 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb3d4104c90 0x7fb3d4107080 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:18.741 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.727+0000 7fb3dbec0640 1 --2- 172.21.15.99:0/1203534475 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb3d41075c0 0x7fb3d41099b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:18.741 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.727+0000 7fb3dbec0640 1 --2- 172.21.15.99:0/1203534475 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb3d4102360 0x7fb3d4104750 unknown :-1 s=CLOSED pgs=779 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:18.741 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.727+0000 7fb3dbec0640 1 -- 172.21.15.99:0/1203534475 >> 172.21.15.99:0/1203534475 conn(0x7fb3d40fbcb0 msgr2=0x7fb3d40fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:04:18.742 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.727+0000 7fb3dbec0640 1 -- 172.21.15.99:0/1203534475 shutdown_connections 2024-03-19T02:04:18.742 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.727+0000 7fb3dbec0640 1 -- 172.21.15.99:0/1203534475 wait complete. 2024-03-19T02:04:18.742 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.728+0000 7fb3dbec0640 1 Processor -- start 2024-03-19T02:04:18.742 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.728+0000 7fb3dbec0640 1 -- start start 2024-03-19T02:04:18.742 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.728+0000 7fb3dbec0640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb3d4102360 0x7fb3d4197ae0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:18.742 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.728+0000 7fb3dbec0640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb3d4104c90 0x7fb3d4198020 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:18.742 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.728+0000 7fb3dbec0640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb3d41075c0 0x7fb3d419f090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:18.743 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.728+0000 7fb3dbec0640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fb3d410d4d0 con 0x7fb3d41075c0 2024-03-19T02:04:18.743 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.728+0000 7fb3dbec0640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fb3d410d350 con 0x7fb3d4104c90 2024-03-19T02:04:18.743 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.728+0000 7fb3dbec0640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fb3d410d650 con 0x7fb3d4102360 2024-03-19T02:04:18.743 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.728+0000 7fb3da45d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb3d41075c0 0x7fb3d419f090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:18.743 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.728+0000 7fb3d945b640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb3d4104c90 0x7fb3d4198020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:18.743 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.728+0000 7fb3da45d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb3d41075c0 0x7fb3d419f090 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:40056/0 (socket says 172.21.15.99:40056) 2024-03-19T02:04:18.743 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.728+0000 7fb3da45d640 1 -- 172.21.15.99:0/475824888 learned_addr learned my addr 172.21.15.99:0/475824888 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:04:18.743 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.728+0000 7fb3d9c5c640 1 --2- 172.21.15.99:0/475824888 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb3d4102360 0x7fb3d4197ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:18.743 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.728+0000 7fb3da45d640 1 -- 172.21.15.99:0/475824888 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb3d4102360 msgr2=0x7fb3d4197ae0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:04:18.743 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.728+0000 7fb3da45d640 1 --2- 172.21.15.99:0/475824888 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb3d4102360 0x7fb3d4197ae0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:18.743 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.728+0000 7fb3da45d640 1 -- 172.21.15.99:0/475824888 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb3d4104c90 msgr2=0x7fb3d4198020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:04:18.743 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.728+0000 7fb3da45d640 1 --2- 172.21.15.99:0/475824888 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb3d4104c90 0x7fb3d4198020 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:18.744 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.728+0000 7fb3da45d640 1 -- 172.21.15.99:0/475824888 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb3d419f700 con 0x7fb3d41075c0 2024-03-19T02:04:18.744 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.728+0000 7fb3da45d640 1 --2- 172.21.15.99:0/475824888 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb3d41075c0 0x7fb3d419f090 secure :-1 s=READY pgs=780 cs=0 l=1 rev1=1 crypto rx=0x7fb3cc00eb10 tx=0x7fb3cc00c2d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:04:18.744 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.728+0000 7fb3caffd640 1 -- 172.21.15.99:0/475824888 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb3cc004b20 con 0x7fb3d41075c0 2024-03-19T02:04:18.744 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.729+0000 7fb3caffd640 1 -- 172.21.15.99:0/475824888 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fb3cc004cc0 con 0x7fb3d41075c0 2024-03-19T02:04:18.744 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.729+0000 7fb3dbec0640 1 -- 172.21.15.99:0/475824888 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb3d419f9f0 con 0x7fb3d41075c0 2024-03-19T02:04:18.744 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.729+0000 7fb3caffd640 1 -- 172.21.15.99:0/475824888 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb3cc010760 con 0x7fb3d41075c0 2024-03-19T02:04:18.744 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.729+0000 7fb3dbec0640 1 -- 172.21.15.99:0/475824888 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fb3d419ff40 con 0x7fb3d41075c0 2024-03-19T02:04:18.744 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.731+0000 7fb3caffd640 1 -- 172.21.15.99:0/475824888 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7fb3cc005020 con 0x7fb3d41075c0 2024-03-19T02:04:18.744 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.731+0000 7fb3deb87740 1 Processor -- start 2024-03-19T02:04:18.744 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.732+0000 7fb3deb87740 1 -- start start 2024-03-19T02:04:18.744 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.732+0000 7fb3caffd640 1 --2- 172.21.15.99:0/475824888 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb3c00a28a0 0x7fb3c00a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:18.744 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.732+0000 7fb3deb87740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56456a3b3100 0x56456a3b3520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:18.744 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.732+0000 7fb3deb87740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56456a3b3af0 0x56456a3b85f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:18.745 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.732+0000 7fb3deb87740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56456a3b8c60 0x56456a3bb0f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:18.745 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.732+0000 7fb3deb87740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x56456a317be0 con 0x56456a3b3100 2024-03-19T02:04:18.745 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.732+0000 7fb3deb87740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x56456a2c0960 con 0x56456a3b3af0 2024-03-19T02:04:18.745 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.732+0000 7fb3deb87740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x56456a313c80 con 0x56456a3b8c60 2024-03-19T02:04:18.745 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.732+0000 7fb3d945b640 1 --2- 172.21.15.99:0/475824888 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb3c00a28a0 0x7fb3c00a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:18.745 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.732+0000 7fb3caffd640 1 -- 172.21.15.99:0/475824888 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fb3cc0cf180 con 0x7fb3d41075c0 2024-03-19T02:04:18.745 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.732+0000 7fb3d9c5c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56456a3b3100 0x56456a3b3520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:18.745 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.732+0000 7fb3d945b640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56456a3b3af0 0x56456a3b85f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:18.745 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.732+0000 7fb3d945b640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56456a3b3af0 0x56456a3b85f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:55204/0 (socket says 172.21.15.99:55204) 2024-03-19T02:04:18.745 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.732+0000 7fb3da45d640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56456a3b8c60 0x56456a3bb0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:18.746 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.732+0000 7fb3d945b640 1 -- 172.21.15.99:0/3907522868 learned_addr learned my addr 172.21.15.99:0/3907522868 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:04:18.746 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.732+0000 7fb3d9c5c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56456a3b3100 0x56456a3b3520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:40064/0 (socket says 172.21.15.99:40064) 2024-03-19T02:04:18.746 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.732+0000 7fb3d945b640 1 -- 172.21.15.99:0/3907522868 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56456a3b8c60 msgr2=0x56456a3bb0f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:04:18.746 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.732+0000 7fb3d945b640 1 --2- 172.21.15.99:0/3907522868 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56456a3b8c60 0x56456a3bb0f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:18.746 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.732+0000 7fb3d945b640 1 -- 172.21.15.99:0/3907522868 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56456a3b3100 msgr2=0x56456a3b3520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:04:18.746 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.732+0000 7fb3d945b640 1 --2- 172.21.15.99:0/3907522868 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56456a3b3100 0x56456a3b3520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:18.746 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.732+0000 7fb3d945b640 1 -- 172.21.15.99:0/3907522868 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x56456a2b7d20 con 0x56456a3b3af0 2024-03-19T02:04:18.746 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.732+0000 7fb3d9c5c640 1 --2- 172.21.15.99:0/3907522868 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56456a3b3100 0x56456a3b3520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:04:18.746 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.732+0000 7fb3d945b640 1 --2- 172.21.15.99:0/475824888 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb3c00a28a0 0x7fb3c00a4d60 secure :-1 s=READY pgs=387 cs=0 l=1 rev1=1 crypto rx=0x7fb3c40119b0 tx=0x7fb3c4009be0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:04:18.746 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.732+0000 7fb3d945b640 1 --2- 172.21.15.99:0/3907522868 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56456a3b3af0 0x56456a3b85f0 secure :-1 s=READY pgs=227 cs=0 l=1 rev1=1 crypto rx=0x7fb3c400f790 tx=0x7fb3c401d000 comp rx=0 tx=0).ready entity=mon.2 client_cookie=1ef1aa5a9c0830b3 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:04:18.746 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.733+0000 7fb3bffff640 1 -- 172.21.15.99:0/3907522868 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb3c4024070 con 0x56456a3b3af0 2024-03-19T02:04:18.746 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.733+0000 7fb3bffff640 1 -- 172.21.15.99:0/3907522868 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fb3c400fe50 con 0x56456a3b3af0 2024-03-19T02:04:18.746 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.733+0000 7fb3bffff640 1 -- 172.21.15.99:0/3907522868 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb3c4020e30 con 0x56456a3b3af0 2024-03-19T02:04:18.747 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.733+0000 7fb3deb87740 1 -- 172.21.15.99:0/3907522868 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56456a3b3af0 msgr2=0x56456a3b85f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:04:18.747 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.733+0000 7fb3deb87740 1 --2- 172.21.15.99:0/3907522868 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56456a3b3af0 0x56456a3b85f0 secure :-1 s=READY pgs=227 cs=0 l=1 rev1=1 crypto rx=0x7fb3c400f790 tx=0x7fb3c401d000 comp rx=0 tx=0).stop 2024-03-19T02:04:18.747 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.733+0000 7fb3deb87740 1 -- 172.21.15.99:0/3907522868 shutdown_connections 2024-03-19T02:04:18.747 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.733+0000 7fb3deb87740 1 --2- 172.21.15.99:0/3907522868 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56456a3b3af0 0x56456a3b85f0 unknown :-1 s=CLOSED pgs=227 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:18.747 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.733+0000 7fb3deb87740 1 --2- 172.21.15.99:0/3907522868 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56456a3b8c60 0x56456a3bb0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:18.747 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.733+0000 7fb3deb87740 1 --2- 172.21.15.99:0/3907522868 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56456a3b3100 0x56456a3b3520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:18.747 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.733+0000 7fb3deb87740 1 -- 172.21.15.99:0/3907522868 >> 172.21.15.99:0/3907522868 conn(0x56456a3ae670 msgr2=0x56456a3aea70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:04:18.747 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.734+0000 7fb3deb87740 1 -- 172.21.15.99:0/3907522868 shutdown_connections 2024-03-19T02:04:18.747 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.734+0000 7fb3deb87740 1 -- 172.21.15.99:0/3907522868 wait complete. 2024-03-19T02:04:18.747 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.734+0000 7fb3deb87740 1 -- start start 2024-03-19T02:04:18.747 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.734+0000 7fb3deb87740 1 Processor -- start 2024-03-19T02:04:18.747 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.734+0000 7fb3deb87740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56456a3b3100 0x56456a3bd0c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:18.747 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.734+0000 7fb3deb87740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56456a3b3af0 0x56456a3bd600 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:18.747 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.734+0000 7fb3deb87740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56456a3b8c60 0x56456a3c4640 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:18.748 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.734+0000 7fb3deb87740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x56456a3bbbb0 con 0x56456a3b8c60 2024-03-19T02:04:18.748 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.734+0000 7fb3deb87740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x56456a2d3db0 con 0x56456a3b3af0 2024-03-19T02:04:18.748 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.734+0000 7fb3deb87740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x56456a3bbd30 con 0x56456a3b3100 2024-03-19T02:04:18.748 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.734+0000 7fb3deb87740 10 client.0 _ll_register_callbacks cb 0x56456a2d8e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T02:04:18.748 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.734+0000 7fb3da45d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56456a3b8c60 0x56456a3c4640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:18.748 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.734+0000 7fb3d9c5c640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56456a3b3100 0x56456a3bd0c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:18.748 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.734+0000 7fb3da45d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56456a3b8c60 0x56456a3c4640 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:40072/0 (socket says 172.21.15.99:40072) 2024-03-19T02:04:18.748 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.734+0000 7fb3da45d640 1 -- 172.21.15.99:0/1272032792 learned_addr learned my addr 172.21.15.99:0/1272032792 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:04:18.748 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.734+0000 7fb3d945b640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56456a3b3af0 0x56456a3bd600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:18.748 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.735+0000 7fb3da45d640 1 -- 172.21.15.99:0/1272032792 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56456a3b3100 msgr2=0x56456a3bd0c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:04:18.748 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.735+0000 7fb3da45d640 1 --2- 172.21.15.99:0/1272032792 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56456a3b3100 0x56456a3bd0c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:18.748 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.735+0000 7fb3da45d640 1 -- 172.21.15.99:0/1272032792 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56456a3b3af0 msgr2=0x56456a3bd600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:04:18.748 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.735+0000 7fb3da45d640 1 --2- 172.21.15.99:0/1272032792 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56456a3b3af0 0x56456a3bd600 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:18.749 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.735+0000 7fb3da45d640 1 -- 172.21.15.99:0/1272032792 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x56456a2d2c60 con 0x56456a3b8c60 2024-03-19T02:04:18.749 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.735+0000 7fb3da45d640 1 --2- 172.21.15.99:0/1272032792 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56456a3b8c60 0x56456a3c4640 secure :-1 s=READY pgs=781 cs=0 l=1 rev1=1 crypto rx=0x7fb3cc0ccc40 tx=0x7fb3cc09b440 comp rx=0 tx=0).ready entity=mon.0 client_cookie=656e8a9d62e418ab server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:04:18.749 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.735+0000 7fb3bd7fa640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T02:04:18.749 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.735+0000 7fb3bd7fa640 1 -- 172.21.15.99:0/1272032792 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb3cc0b3070 con 0x56456a3b8c60 2024-03-19T02:04:18.749 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.735+0000 7fb3bd7fa640 1 -- 172.21.15.99:0/1272032792 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fb3cc0ba230 con 0x56456a3b8c60 2024-03-19T02:04:18.749 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.735+0000 7fb3deb87740 1 -- 172.21.15.99:0/1272032792 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fb3c400fe50 con 0x56456a3b8c60 2024-03-19T02:04:18.749 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.735+0000 7fb3bd7fa640 1 -- 172.21.15.99:0/1272032792 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=41) ==== 24+0+0 (secure 0 0 0) 0x7fb3cc01aeb0 con 0x56456a3b8c60 2024-03-19T02:04:18.749 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.735+0000 7fb3deb87740 10 client.17190 fetch_fsmap learned FSMap version 41 2024-03-19T02:04:18.749 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.735+0000 7fb3deb87740 1 -- 172.21.15.99:0/1272032792 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=41,monmap=4+}) -- 0x56456a02c210 con 0x56456a3b8c60 2024-03-19T02:04:18.749 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.735+0000 7fb3bd7fa640 1 -- 172.21.15.99:0/1272032792 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fb3c400fe50 con 0x56456a3b8c60 2024-03-19T02:04:18.749 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.735+0000 7fb3bd7fa640 1 -- 172.21.15.99:0/1272032792 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 41) ==== 8354+0+0 (secure 0 0 0) 0x7fb3cc005310 con 0x56456a3b8c60 2024-03-19T02:04:18.749 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.735+0000 7fb3deb87740 10 client.17190 fetch_fsmap finished waiting for FSMap version 41 2024-03-19T02:04:18.749 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.735+0000 7fb3deb87740 10 client.17190 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T02:04:18.749 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.735+0000 7fb3deb87740 20 client.17190 populate_metadata read hostname 'smithi099' 2024-03-19T02:04:18.749 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.735+0000 7fb3deb87740 1 --2- 172.21.15.99:0/1272032792 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56456a3c8f60 0x56456a3cb350 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:18.750 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.735+0000 7fb3deb87740 4 client.17190 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T02:04:18.750 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.736+0000 7fb3deb87740 1 -- 172.21.15.99:0/1272032792 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x56456a325120 con 0x56456a3c8f60 2024-03-19T02:04:18.750 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.736+0000 7fb3d9c5c640 1 --2- 172.21.15.99:0/1272032792 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56456a3c8f60 0x56456a3cb350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:18.750 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.736+0000 7fb3d9c5c640 1 --2- 172.21.15.99:0/1272032792 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56456a3c8f60 0x56456a3cb350 crc :-1 s=READY pgs=235 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=1be6d309aeb77d2a server_cookie=49464cefa7b21d7d in_seq=0 out_seq=0 2024-03-19T02:04:18.750 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.736+0000 7fb3bd7fa640 10 client.17190 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T02:04:18.751 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.741+0000 7fb3bd7fa640 1 -- 172.21.15.99:0/1272032792 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x56456a325120 con 0x56456a3c8f60 2024-03-19T02:04:18.751 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.741+0000 7fb3bd7fa640 10 client.17190 handle_command_reply: tid=0 2024-03-19T02:04:18.751 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.741+0000 7fb3deb87740 1 client.17190 shutdown 2024-03-19T02:04:18.751 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.741+0000 7fb3deb87740 1 -- 172.21.15.99:0/1272032792 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56456a3b8c60 msgr2=0x56456a3c4640 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:04:18.751 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.741+0000 7fb3deb87740 1 --2- 172.21.15.99:0/1272032792 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56456a3b8c60 0x56456a3c4640 secure :-1 s=READY pgs=781 cs=0 l=1 rev1=1 crypto rx=0x7fb3cc0ccc40 tx=0x7fb3cc09b440 comp rx=0 tx=0).stop 2024-03-19T02:04:18.751 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.742+0000 7fb3deb87740 1 -- 172.21.15.99:0/1272032792 shutdown_connections 2024-03-19T02:04:18.751 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.742+0000 7fb3deb87740 1 --2- 172.21.15.99:0/1272032792 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56456a3c8f60 0x56456a3cb350 crc :-1 s=READY pgs=235 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:18.752 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.742+0000 7fb3deb87740 1 --2- 172.21.15.99:0/1272032792 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56456a3b3af0 0x56456a3bd600 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:18.752 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.742+0000 7fb3deb87740 1 --2- 172.21.15.99:0/1272032792 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56456a3b8c60 0x56456a3c4640 unknown :-1 s=CLOSED pgs=781 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:18.752 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.742+0000 7fb3deb87740 1 --2- 172.21.15.99:0/1272032792 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56456a3b3100 0x56456a3bd0c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:18.752 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.742+0000 7fb3bd7fa640 0 client.17190 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T02:04:18.752 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.742+0000 7fb3deb87740 1 -- 172.21.15.99:0/1272032792 >> 172.21.15.99:0/1272032792 conn(0x56456a3ae670 msgr2=0x56456a3af8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:04:18.752 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.742+0000 7fb3deb87740 1 -- 172.21.15.99:0/1272032792 shutdown_connections 2024-03-19T02:04:18.752 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.742+0000 7fb3deb87740 1 -- 172.21.15.99:0/1272032792 wait complete. 2024-03-19T02:04:18.752 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.742+0000 7fb3deb87740 20 client.17190 trim_cache size 0 max 16384 2024-03-19T02:04:18.758 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.758+0000 7fb3deb87740 1 Processor -- start 2024-03-19T02:04:18.758 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.758+0000 7fb3deb87740 1 -- start start 2024-03-19T02:04:18.759 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.758+0000 7fb3deb87740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56456a3b3100 0x56456a3b1fe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:18.759 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.758+0000 7fb3deb87740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56456a3b3af0 0x56456a3c4600 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:18.759 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.758+0000 7fb3deb87740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56456a3b8c60 0x56456a3cfbe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:18.759 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.758+0000 7fb3deb87740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fb3d41ab640 con 0x56456a3b8c60 2024-03-19T02:04:18.759 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.758+0000 7fb3deb87740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fb3d41a8eb0 con 0x56456a3b3100 2024-03-19T02:04:18.759 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.758+0000 7fb3deb87740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fb3d41a6d00 con 0x56456a3b3af0 2024-03-19T02:04:18.759 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.758+0000 7fb3da45d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56456a3b8c60 0x56456a3cfbe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:18.759 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.758+0000 7fb3da45d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56456a3b8c60 0x56456a3cfbe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:40086/0 (socket says 172.21.15.99:40086) 2024-03-19T02:04:18.759 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.758+0000 7fb3da45d640 1 -- 172.21.15.99:0/2379789314 learned_addr learned my addr 172.21.15.99:0/2379789314 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:04:18.759 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.758+0000 7fb3d9c5c640 1 --2- 172.21.15.99:0/2379789314 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56456a3b3100 0x56456a3b1fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:18.759 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.758+0000 7fb3d945b640 1 --2- 172.21.15.99:0/2379789314 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56456a3b3af0 0x56456a3c4600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:18.759 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.758+0000 7fb3da45d640 1 -- 172.21.15.99:0/2379789314 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56456a3b3af0 msgr2=0x56456a3c4600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:04:18.759 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.758+0000 7fb3da45d640 1 --2- 172.21.15.99:0/2379789314 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56456a3b3af0 0x56456a3c4600 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:18.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.758+0000 7fb3da45d640 1 -- 172.21.15.99:0/2379789314 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56456a3b3100 msgr2=0x56456a3b1fe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:04:18.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.758+0000 7fb3da45d640 1 --2- 172.21.15.99:0/2379789314 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56456a3b3100 0x56456a3b1fe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:18.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.758+0000 7fb3da45d640 1 -- 172.21.15.99:0/2379789314 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x56456a0f8620 con 0x56456a3b8c60 2024-03-19T02:04:18.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.758+0000 7fb3da45d640 1 --2- 172.21.15.99:0/2379789314 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56456a3b8c60 0x56456a3cfbe0 secure :-1 s=READY pgs=782 cs=0 l=1 rev1=1 crypto rx=0x7fb3cc000f80 tx=0x7fb3cc099ff0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a85abc7718ae1947 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:04:18.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.759+0000 7fb3bf7fe640 1 -- 172.21.15.99:0/2379789314 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb3cc0b3050 con 0x56456a3b8c60 2024-03-19T02:04:18.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.759+0000 7fb3bf7fe640 1 -- 172.21.15.99:0/2379789314 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fb3cc09c1e0 con 0x56456a3b8c60 2024-03-19T02:04:18.760 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.759+0000 7fb3bf7fe640 1 -- 172.21.15.99:0/2379789314 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb3cc09af80 con 0x56456a3b8c60 2024-03-19T02:04:18.764 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.759+0000 7fb3deb87740 1 -- 172.21.15.99:0/2379789314 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56456a3b8c60 msgr2=0x56456a3cfbe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:04:18.764 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.759+0000 7fb3deb87740 1 --2- 172.21.15.99:0/2379789314 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56456a3b8c60 0x56456a3cfbe0 secure :-1 s=READY pgs=782 cs=0 l=1 rev1=1 crypto rx=0x7fb3cc000f80 tx=0x7fb3cc099ff0 comp rx=0 tx=0).stop 2024-03-19T02:04:18.764 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.759+0000 7fb3deb87740 1 -- 172.21.15.99:0/2379789314 shutdown_connections 2024-03-19T02:04:18.764 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.759+0000 7fb3deb87740 1 --2- 172.21.15.99:0/2379789314 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56456a3b8c60 0x56456a3cfbe0 unknown :-1 s=CLOSED pgs=782 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:18.764 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.759+0000 7fb3deb87740 1 --2- 172.21.15.99:0/2379789314 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56456a3b3af0 0x56456a3c4600 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:18.764 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.759+0000 7fb3deb87740 1 --2- 172.21.15.99:0/2379789314 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56456a3b3100 0x56456a3b1fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:18.764 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.759+0000 7fb3deb87740 1 -- 172.21.15.99:0/2379789314 >> 172.21.15.99:0/2379789314 conn(0x56456a3ae670 msgr2=0x56456a3b18d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:04:18.764 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.759+0000 7fb3deb87740 1 -- 172.21.15.99:0/2379789314 shutdown_connections 2024-03-19T02:04:18.764 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.759+0000 7fb3deb87740 1 -- 172.21.15.99:0/2379789314 wait complete. 2024-03-19T02:04:18.764 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.760+0000 7fb3deb87740 1 -- start start 2024-03-19T02:04:18.764 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.760+0000 7fb3deb87740 1 Processor -- start 2024-03-19T02:04:18.764 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.760+0000 7fb3deb87740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56456a3b3100 0x56456a3c2550 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:18.765 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.760+0000 7fb3deb87740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56456a3b3af0 0x56456a3c2a90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:18.765 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.760+0000 7fb3deb87740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56456a3b8c60 0x56456a3c3aa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:18.765 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.760+0000 7fb3deb87740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fb3d41a7370 con 0x56456a3b3100 2024-03-19T02:04:18.765 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.760+0000 7fb3deb87740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fb3d41a87e0 con 0x56456a3b8c60 2024-03-19T02:04:18.765 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.760+0000 7fb3deb87740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fb3d41a7a40 con 0x56456a3b3af0 2024-03-19T02:04:18.765 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.760+0000 7fb3deb87740 10 client.0 _ll_register_callbacks cb 0x7fb3cc004e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T02:04:18.765 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.760+0000 7fb3d9c5c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56456a3b3100 0x56456a3c2550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:18.765 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.760+0000 7fb3d9c5c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56456a3b3100 0x56456a3c2550 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:40100/0 (socket says 172.21.15.99:40100) 2024-03-19T02:04:18.765 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.760+0000 7fb3d9c5c640 1 -- 172.21.15.99:0/1729183641 learned_addr learned my addr 172.21.15.99:0/1729183641 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:04:18.765 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.760+0000 7fb3da45d640 1 --2- 172.21.15.99:0/1729183641 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56456a3b8c60 0x56456a3c3aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:18.765 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.760+0000 7fb3d9c5c640 1 -- 172.21.15.99:0/1729183641 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56456a3b3af0 msgr2=0x56456a3c2a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:04:18.765 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.760+0000 7fb3d9c5c640 1 --2- 172.21.15.99:0/1729183641 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56456a3b3af0 0x56456a3c2a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:18.765 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.760+0000 7fb3d9c5c640 1 -- 172.21.15.99:0/1729183641 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56456a3b8c60 msgr2=0x56456a3c3aa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:04:18.765 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.760+0000 7fb3d9c5c640 1 --2- 172.21.15.99:0/1729183641 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56456a3b8c60 0x56456a3c3aa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:18.766 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.760+0000 7fb3d9c5c640 1 -- 172.21.15.99:0/1729183641 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x56456a3c4090 con 0x56456a3b3100 2024-03-19T02:04:18.766 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.761+0000 7fb3d9c5c640 1 --2- 172.21.15.99:0/1729183641 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56456a3b3100 0x56456a3c2550 secure :-1 s=READY pgs=783 cs=0 l=1 rev1=1 crypto rx=0x7fb3d002c710 tx=0x7fb3d002cca0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=db13c185efc0f3f1 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:04:18.766 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.761+0000 7fb3bd7fa640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T02:04:18.766 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.761+0000 7fb3bd7fa640 1 -- 172.21.15.99:0/1729183641 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb3d004a070 con 0x56456a3b3100 2024-03-19T02:04:18.766 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.761+0000 7fb3bd7fa640 1 -- 172.21.15.99:0/1729183641 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fb3d0004040 con 0x56456a3b3100 2024-03-19T02:04:18.766 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.761+0000 7fb3deb87740 1 -- 172.21.15.99:0/1729183641 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fb3cc09c1e0 con 0x56456a3b3100 2024-03-19T02:04:18.766 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.761+0000 7fb3bd7fa640 1 -- 172.21.15.99:0/1729183641 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=41) ==== 24+0+0 (secure 0 0 0) 0x7fb3d0056050 con 0x56456a3b3100 2024-03-19T02:04:18.766 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.761+0000 7fb3deb87740 10 client.17196 fetch_fsmap learned FSMap version 41 2024-03-19T02:04:18.766 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.761+0000 7fb3deb87740 1 -- 172.21.15.99:0/1729183641 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=41,monmap=4+}) -- 0x56456a42faa0 con 0x56456a3b3100 2024-03-19T02:04:18.766 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.761+0000 7fb3bd7fa640 1 -- 172.21.15.99:0/1729183641 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fb3cc09c1e0 con 0x56456a3b3100 2024-03-19T02:04:18.766 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.761+0000 7fb3bd7fa640 1 -- 172.21.15.99:0/1729183641 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 41) ==== 8354+0+0 (secure 0 0 0) 0x7fb3d0005300 con 0x56456a3b3100 2024-03-19T02:04:18.766 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.761+0000 7fb3deb87740 10 client.17196 fetch_fsmap finished waiting for FSMap version 41 2024-03-19T02:04:18.766 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.761+0000 7fb3deb87740 10 client.17196 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T02:04:18.767 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.761+0000 7fb3deb87740 20 client.17196 populate_metadata read hostname 'smithi099' 2024-03-19T02:04:18.767 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.761+0000 7fb3deb87740 1 --2- 172.21.15.99:0/1729183641 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56456a4304a0 0x56456a4328c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:18.767 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.761+0000 7fb3deb87740 4 client.17196 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T02:04:18.767 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.761+0000 7fb3deb87740 1 -- 172.21.15.99:0/1729183641 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x56456a31f470 con 0x56456a4304a0 2024-03-19T02:04:18.767 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.762+0000 7fb3d945b640 1 --2- 172.21.15.99:0/1729183641 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56456a4304a0 0x56456a4328c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:18.767 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.763+0000 7fb3bd7fa640 10 client.17196 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T02:04:18.767 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:18.763+0000 7fb3d945b640 1 --2- 172.21.15.99:0/1729183641 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56456a4304a0 0x56456a4328c0 crc :-1 s=READY pgs=236 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=feca87f04a1f3482 server_cookie=a049d9da9fd67a44 in_seq=0 out_seq=0 2024-03-19T02:04:19.058 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:04:18 smithi115 ceph-mon[67332]: pgmap v1047: 97 pgs: 97 active+clean; 660 MiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 6.4 MiB/s wr, 546 op/s 2024-03-19T02:04:19.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:04:18 smithi099 ceph-mon[62951]: pgmap v1047: 97 pgs: 97 active+clean; 660 MiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 6.4 MiB/s wr, 546 op/s 2024-03-19T02:04:19.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:04:18 smithi112 ceph-mon[66819]: pgmap v1047: 97 pgs: 97 active+clean; 660 MiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 6.4 MiB/s wr, 546 op/s 2024-03-19T02:04:19.120 INFO:tasks.workunit.client.0.smithi099.stdout: 10 271699 22.21 MB/sec execute 528 sec latency 297.487 ms 2024-03-19T02:04:19.507 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:19.507+0000 7fb3bd7fa640 1 -- 172.21.15.99:0/1729183641 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x56456a31f470 con 0x56456a4304a0 2024-03-19T02:04:19.507 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:19.507+0000 7fb3bd7fa640 10 client.17196 handle_command_reply: tid=0 2024-03-19T02:04:19.508 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:19.507+0000 7fb3deb87740 1 client.17196 shutdown 2024-03-19T02:04:19.508 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:19.507+0000 7fb3deb87740 1 -- 172.21.15.99:0/1729183641 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56456a3b3100 msgr2=0x56456a3c2550 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:04:19.508 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:19.507+0000 7fb3deb87740 1 --2- 172.21.15.99:0/1729183641 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56456a3b3100 0x56456a3c2550 secure :-1 s=READY pgs=783 cs=0 l=1 rev1=1 crypto rx=0x7fb3d002c710 tx=0x7fb3d002cca0 comp rx=0 tx=0).stop 2024-03-19T02:04:19.508 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:19.507+0000 7fb3deb87740 1 -- 172.21.15.99:0/1729183641 shutdown_connections 2024-03-19T02:04:19.508 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:19.507+0000 7fb3deb87740 1 --2- 172.21.15.99:0/1729183641 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56456a4304a0 0x56456a4328c0 crc :-1 s=READY pgs=236 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:19.508 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:19.507+0000 7fb3deb87740 1 --2- 172.21.15.99:0/1729183641 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56456a3b8c60 0x56456a3c3aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:19.508 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:19.507+0000 7fb3deb87740 1 --2- 172.21.15.99:0/1729183641 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56456a3b3af0 0x56456a3c2a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:19.508 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:19.507+0000 7fb3deb87740 1 --2- 172.21.15.99:0/1729183641 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56456a3b3100 0x56456a3c2550 unknown :-1 s=CLOSED pgs=783 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:19.508 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:19.507+0000 7fb3deb87740 1 -- 172.21.15.99:0/1729183641 >> 172.21.15.99:0/1729183641 conn(0x56456a3ae670 msgr2=0x56456a3b2160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:04:19.508 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:19.507+0000 7fb3bd7fa640 0 client.17196 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T02:04:19.508 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:19.507+0000 7fb3deb87740 1 -- 172.21.15.99:0/1729183641 shutdown_connections 2024-03-19T02:04:19.508 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:19.507+0000 7fb3deb87740 1 -- 172.21.15.99:0/1729183641 wait complete. 2024-03-19T02:04:19.508 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:19.507+0000 7fb3deb87740 20 client.17196 trim_cache size 0 max 16384 2024-03-19T02:04:19.509 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T02:04:19.509 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T02:04:19.509 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T02:04:19.509 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T02:04:19.510 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:19.508+0000 7fb3dbec0640 1 -- 172.21.15.99:0/475824888 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb3c00a28a0 msgr2=0x7fb3c00a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:04:19.510 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:19.508+0000 7fb3dbec0640 1 --2- 172.21.15.99:0/475824888 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb3c00a28a0 0x7fb3c00a4d60 secure :-1 s=READY pgs=387 cs=0 l=1 rev1=1 crypto rx=0x7fb3c40119b0 tx=0x7fb3c4009be0 comp rx=0 tx=0).stop 2024-03-19T02:04:19.510 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:19.508+0000 7fb3dbec0640 1 -- 172.21.15.99:0/475824888 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb3d41075c0 msgr2=0x7fb3d419f090 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:04:19.510 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:19.508+0000 7fb3dbec0640 1 --2- 172.21.15.99:0/475824888 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb3d41075c0 0x7fb3d419f090 secure :-1 s=READY pgs=780 cs=0 l=1 rev1=1 crypto rx=0x7fb3cc00eb10 tx=0x7fb3cc00c2d0 comp rx=0 tx=0).stop 2024-03-19T02:04:19.510 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:19.508+0000 7fb3dbec0640 1 -- 172.21.15.99:0/475824888 shutdown_connections 2024-03-19T02:04:19.510 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:19.508+0000 7fb3dbec0640 1 --2- 172.21.15.99:0/475824888 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb3c00a28a0 0x7fb3c00a4d60 unknown :-1 s=CLOSED pgs=387 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:19.510 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:19.508+0000 7fb3dbec0640 1 --2- 172.21.15.99:0/475824888 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb3d4104c90 0x7fb3d4198020 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:19.510 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:19.508+0000 7fb3dbec0640 1 --2- 172.21.15.99:0/475824888 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb3d41075c0 0x7fb3d419f090 unknown :-1 s=CLOSED pgs=780 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:19.510 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:19.508+0000 7fb3dbec0640 1 --2- 172.21.15.99:0/475824888 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb3d4102360 0x7fb3d4197ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:19.510 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:19.508+0000 7fb3dbec0640 1 -- 172.21.15.99:0/475824888 >> 172.21.15.99:0/475824888 conn(0x7fb3d40fbcb0 msgr2=0x7fb3d4102ca0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:04:19.511 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:19.508+0000 7fb3dbec0640 1 -- 172.21.15.99:0/475824888 shutdown_connections 2024-03-19T02:04:19.511 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:19.508+0000 7fb3dbec0640 1 -- 172.21.15.99:0/475824888 wait complete. 2024-03-19T02:04:20.120 INFO:tasks.workunit.client.0.smithi099.stdout: 10 271952 22.19 MB/sec execute 529 sec latency 466.263 ms 2024-03-19T02:04:21.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:04:20 smithi115 ceph-mon[67332]: pgmap v1048: 97 pgs: 97 active+clean; 661 MiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 5.5 MiB/s wr, 436 op/s 2024-03-19T02:04:21.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:04:20 smithi099 ceph-mon[62951]: pgmap v1048: 97 pgs: 97 active+clean; 661 MiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 5.5 MiB/s wr, 436 op/s 2024-03-19T02:04:21.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:04:20 smithi112 ceph-mon[66819]: pgmap v1048: 97 pgs: 97 active+clean; 661 MiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 5.5 MiB/s wr, 436 op/s 2024-03-19T02:04:21.120 INFO:tasks.workunit.client.0.smithi099.stdout: 10 272310 22.18 MB/sec execute 530 sec latency 74.909 ms 2024-03-19T02:04:22.121 INFO:tasks.workunit.client.0.smithi099.stdout: 10 272718 22.18 MB/sec execute 531 sec latency 79.386 ms 2024-03-19T02:04:23.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:04:22 smithi115 ceph-mon[67332]: pgmap v1049: 97 pgs: 97 active+clean; 663 MiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 5.7 MiB/s wr, 494 op/s 2024-03-19T02:04:23.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:04:22 smithi099 ceph-mon[62951]: pgmap v1049: 97 pgs: 97 active+clean; 663 MiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 5.7 MiB/s wr, 494 op/s 2024-03-19T02:04:23.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:04:22 smithi112 ceph-mon[66819]: pgmap v1049: 97 pgs: 97 active+clean; 663 MiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 5.7 MiB/s wr, 494 op/s 2024-03-19T02:04:23.121 INFO:tasks.workunit.client.0.smithi099.stdout: 10 273117 22.17 MB/sec execute 532 sec latency 203.222 ms 2024-03-19T02:04:24.121 INFO:tasks.workunit.client.0.smithi099.stdout: 10 273460 22.16 MB/sec execute 533 sec latency 261.803 ms 2024-03-19T02:04:25.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:04:24 smithi115 ceph-mon[67332]: pgmap v1050: 97 pgs: 97 active+clean; 635 MiB data, 9.1 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 5.7 MiB/s wr, 541 op/s 2024-03-19T02:04:25.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:04:24 smithi099 ceph-mon[62951]: pgmap v1050: 97 pgs: 97 active+clean; 635 MiB data, 9.1 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 5.7 MiB/s wr, 541 op/s 2024-03-19T02:04:25.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:04:24 smithi112 ceph-mon[66819]: pgmap v1050: 97 pgs: 97 active+clean; 635 MiB data, 9.1 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 5.7 MiB/s wr, 541 op/s 2024-03-19T02:04:25.120 INFO:tasks.workunit.client.0.smithi099.stdout: 10 273911 22.17 MB/sec execute 534 sec latency 66.887 ms 2024-03-19T02:04:26.121 INFO:tasks.workunit.client.0.smithi099.stdout: 10 274385 22.18 MB/sec execute 535 sec latency 56.033 ms 2024-03-19T02:04:27.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:04:26 smithi115 ceph-mon[67332]: pgmap v1051: 97 pgs: 97 active+clean; 635 MiB data, 9.1 GiB used, 1.0 TiB / 1.0 TiB avail; 3.1 MiB/s rd, 4.5 MiB/s wr, 451 op/s 2024-03-19T02:04:27.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:04:26 smithi099 ceph-mon[62951]: pgmap v1051: 97 pgs: 97 active+clean; 635 MiB data, 9.1 GiB used, 1.0 TiB / 1.0 TiB avail; 3.1 MiB/s rd, 4.5 MiB/s wr, 451 op/s 2024-03-19T02:04:27.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:04:26 smithi112 ceph-mon[66819]: pgmap v1051: 97 pgs: 97 active+clean; 635 MiB data, 9.1 GiB used, 1.0 TiB / 1.0 TiB avail; 3.1 MiB/s rd, 4.5 MiB/s wr, 451 op/s 2024-03-19T02:04:27.121 INFO:tasks.workunit.client.0.smithi099.stdout: 10 274783 22.16 MB/sec execute 536 sec latency 71.729 ms 2024-03-19T02:04:28.121 INFO:tasks.workunit.client.0.smithi099.stdout: 10 275206 22.18 MB/sec execute 537 sec latency 91.366 ms 2024-03-19T02:04:29.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:04:28 smithi115 ceph-mon[67332]: pgmap v1052: 97 pgs: 97 active+clean; 641 MiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.9 MiB/s rd, 5.8 MiB/s wr, 538 op/s 2024-03-19T02:04:29.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:04:28 smithi099 ceph-mon[62951]: pgmap v1052: 97 pgs: 97 active+clean; 641 MiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.9 MiB/s rd, 5.8 MiB/s wr, 538 op/s 2024-03-19T02:04:29.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:04:28 smithi112 ceph-mon[66819]: pgmap v1052: 97 pgs: 97 active+clean; 641 MiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.9 MiB/s rd, 5.8 MiB/s wr, 538 op/s 2024-03-19T02:04:29.121 INFO:tasks.workunit.client.0.smithi099.stdout: 10 275617 22.18 MB/sec execute 538 sec latency 296.951 ms 2024-03-19T02:04:30.121 INFO:tasks.workunit.client.0.smithi099.stdout: 10 276010 22.17 MB/sec execute 539 sec latency 77.043 ms 2024-03-19T02:04:31.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:04:30 smithi115 ceph-mon[67332]: pgmap v1053: 97 pgs: 97 active+clean; 642 MiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.4 MiB/s rd, 5.2 MiB/s wr, 507 op/s 2024-03-19T02:04:31.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:04:30 smithi099 ceph-mon[62951]: pgmap v1053: 97 pgs: 97 active+clean; 642 MiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.4 MiB/s rd, 5.2 MiB/s wr, 507 op/s 2024-03-19T02:04:31.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:04:30 smithi112 ceph-mon[66819]: pgmap v1053: 97 pgs: 97 active+clean; 642 MiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.4 MiB/s rd, 5.2 MiB/s wr, 507 op/s 2024-03-19T02:04:31.121 INFO:tasks.workunit.client.0.smithi099.stdout: 10 276441 22.19 MB/sec execute 540 sec latency 75.730 ms 2024-03-19T02:04:32.121 INFO:tasks.workunit.client.0.smithi099.stdout: 10 276840 22.18 MB/sec execute 541 sec latency 71.181 ms 2024-03-19T02:04:32.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.620+0000 7fb01c634640 1 -- 172.21.15.99:0/141131770 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb014101850 msgr2=0x7fb014101c70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:04:32.622 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.620+0000 7fb01c634640 1 --2- 172.21.15.99:0/141131770 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb014101850 0x7fb014101c70 secure :-1 s=READY pgs=784 cs=0 l=1 rev1=1 crypto rx=0x7fb010009900 tx=0x7fb01002bc30 comp rx=0 tx=0).stop 2024-03-19T02:04:32.623 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.621+0000 7fb01c634640 1 -- 172.21.15.99:0/141131770 shutdown_connections 2024-03-19T02:04:32.623 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.621+0000 7fb01c634640 1 --2- 172.21.15.99:0/141131770 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb014102a40 0x7fb014102ec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:32.623 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.621+0000 7fb01c634640 1 --2- 172.21.15.99:0/141131770 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb014103400 0x7fb014109c90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:32.623 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.621+0000 7fb01c634640 1 --2- 172.21.15.99:0/141131770 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb014101850 0x7fb014101c70 unknown :-1 s=CLOSED pgs=784 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:32.623 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.621+0000 7fb01c634640 1 -- 172.21.15.99:0/141131770 >> 172.21.15.99:0/141131770 conn(0x7fb0140fcde0 msgr2=0x7fb0140ff200 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:04:32.623 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.621+0000 7fb01c634640 1 -- 172.21.15.99:0/141131770 shutdown_connections 2024-03-19T02:04:32.623 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.621+0000 7fb01c634640 1 -- 172.21.15.99:0/141131770 wait complete. 2024-03-19T02:04:32.623 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.621+0000 7fb01c634640 1 Processor -- start 2024-03-19T02:04:32.623 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.621+0000 7fb01c634640 1 -- start start 2024-03-19T02:04:32.623 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.621+0000 7fb01c634640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb014101850 0x7fb014197ab0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:32.623 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.621+0000 7fb01c634640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb014102a40 0x7fb014197ff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:32.623 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.621+0000 7fb01c634640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb014103400 0x7fb01419f060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:32.623 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.621+0000 7fb01c634640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fb01410d4d0 con 0x7fb014103400 2024-03-19T02:04:32.624 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.621+0000 7fb01c634640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fb01410d350 con 0x7fb014101850 2024-03-19T02:04:32.624 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.621+0000 7fb01c634640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fb01410d650 con 0x7fb014102a40 2024-03-19T02:04:32.624 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.622+0000 7fb01abd1640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb014103400 0x7fb01419f060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:32.624 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.622+0000 7fb019bcf640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb014102a40 0x7fb014197ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:32.624 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.622+0000 7fb01abd1640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb014103400 0x7fb01419f060 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:57636/0 (socket says 172.21.15.99:57636) 2024-03-19T02:04:32.624 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.622+0000 7fb01abd1640 1 -- 172.21.15.99:0/1642349795 learned_addr learned my addr 172.21.15.99:0/1642349795 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:04:32.624 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.622+0000 7fb01a3d0640 1 --2- 172.21.15.99:0/1642349795 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb014101850 0x7fb014197ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:32.624 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.622+0000 7fb019bcf640 1 -- 172.21.15.99:0/1642349795 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb014101850 msgr2=0x7fb014197ab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:04:32.624 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.622+0000 7fb019bcf640 1 --2- 172.21.15.99:0/1642349795 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb014101850 0x7fb014197ab0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:32.624 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.622+0000 7fb019bcf640 1 -- 172.21.15.99:0/1642349795 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb014103400 msgr2=0x7fb01419f060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:04:32.624 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.622+0000 7fb019bcf640 1 --2- 172.21.15.99:0/1642349795 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb014103400 0x7fb01419f060 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:32.624 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.622+0000 7fb019bcf640 1 -- 172.21.15.99:0/1642349795 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb01419f6d0 con 0x7fb014102a40 2024-03-19T02:04:32.624 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.622+0000 7fb019bcf640 1 --2- 172.21.15.99:0/1642349795 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb014102a40 0x7fb014197ff0 secure :-1 s=READY pgs=132 cs=0 l=1 rev1=1 crypto rx=0x7fb00400e9f0 tx=0x7fb00400eeb0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:04:32.624 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.622+0000 7fb00b7fe640 1 -- 172.21.15.99:0/1642349795 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb00400ccc0 con 0x7fb014102a40 2024-03-19T02:04:32.625 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.622+0000 7fb01c634640 1 -- 172.21.15.99:0/1642349795 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb01419f9c0 con 0x7fb014102a40 2024-03-19T02:04:32.625 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.622+0000 7fb00b7fe640 1 -- 172.21.15.99:0/1642349795 <== mon.1 v2:172.21.15.115:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fb004010910 con 0x7fb014102a40 2024-03-19T02:04:32.625 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.622+0000 7fb00b7fe640 1 -- 172.21.15.99:0/1642349795 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb0040189c0 con 0x7fb014102a40 2024-03-19T02:04:32.625 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.622+0000 7fb01c634640 1 -- 172.21.15.99:0/1642349795 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fb01419ff60 con 0x7fb014102a40 2024-03-19T02:04:32.631 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.625+0000 7fb00b7fe640 1 -- 172.21.15.99:0/1642349795 <== mon.1 v2:172.21.15.115:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7fb004010ac0 con 0x7fb014102a40 2024-03-19T02:04:32.631 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.625+0000 7fb01f21c740 1 Processor -- start 2024-03-19T02:04:32.631 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.625+0000 7fb01f21c740 1 -- start start 2024-03-19T02:04:32.631 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.626+0000 7fb01f21c740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560cae084ff0 0x560cae0914f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:32.631 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.626+0000 7fb01f21c740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560cae091a30 0x560cae093e40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:32.631 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.626+0000 7fb00b7fe640 1 --2- 172.21.15.99:0/1642349795 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb0000a28a0 0x7fb0000a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:32.631 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.626+0000 7fb01f21c740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560cae0944d0 0x560cae096960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:32.631 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.626+0000 7fb01f21c740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x560cadfeabe0 con 0x560cae0944d0 2024-03-19T02:04:32.631 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.626+0000 7fb01f21c740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x560cadf93960 con 0x560cae091a30 2024-03-19T02:04:32.631 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.626+0000 7fb01f21c740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x560cadfe6c80 con 0x560cae084ff0 2024-03-19T02:04:32.631 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.626+0000 7fb01a3d0640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560cae091a30 0x560cae093e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:32.631 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.626+0000 7fb01a3d0640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560cae091a30 0x560cae093e40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:49872/0 (socket says 172.21.15.99:49872) 2024-03-19T02:04:32.632 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.626+0000 7fb01abd1640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560cae084ff0 0x560cae0914f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:32.632 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.626+0000 7fb01a3d0640 1 -- 172.21.15.99:0/3536575886 learned_addr learned my addr 172.21.15.99:0/3536575886 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:04:32.632 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.626+0000 7fb00b7fe640 1 -- 172.21.15.99:0/1642349795 <== mon.1 v2:172.21.15.115:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fb0040cee50 con 0x7fb014102a40 2024-03-19T02:04:32.632 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.626+0000 7fb019bcf640 1 --2- 172.21.15.99:0/1642349795 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb0000a28a0 0x7fb0000a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:32.632 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.626+0000 7fb01abd1640 1 --2- 172.21.15.99:0/3536575886 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560cae0944d0 0x560cae096960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:32.632 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.626+0000 7fb01a3d0640 1 -- 172.21.15.99:0/3536575886 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560cae084ff0 msgr2=0x560cae0914f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:04:32.632 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.626+0000 7fb01a3d0640 1 --2- 172.21.15.99:0/3536575886 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560cae084ff0 0x560cae0914f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:32.632 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.626+0000 7fb01a3d0640 1 -- 172.21.15.99:0/3536575886 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560cae0944d0 msgr2=0x560cae096960 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:04:32.632 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.626+0000 7fb01a3d0640 1 --2- 172.21.15.99:0/3536575886 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560cae0944d0 0x560cae096960 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:32.632 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.626+0000 7fb01a3d0640 1 -- 172.21.15.99:0/3536575886 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x560cadf8ad20 con 0x560cae091a30 2024-03-19T02:04:32.632 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.626+0000 7fb01a3d0640 1 --2- 172.21.15.99:0/3536575886 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560cae091a30 0x560cae093e40 secure :-1 s=READY pgs=228 cs=0 l=1 rev1=1 crypto rx=0x7fb010009900 tx=0x7fb010009560 comp rx=0 tx=0).ready entity=mon.2 client_cookie=62a56ec22b80f782 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:04:32.632 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.627+0000 7fb019bcf640 1 --2- 172.21.15.99:0/1642349795 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb0000a28a0 0x7fb0000a4d60 secure :-1 s=READY pgs=390 cs=0 l=1 rev1=1 crypto rx=0x7fb004000c00 tx=0x7fb004018420 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:04:32.632 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.627+0000 7fb008bfe640 1 -- 172.21.15.99:0/3536575886 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb01002cb80 con 0x560cae091a30 2024-03-19T02:04:32.632 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.627+0000 7fb008bfe640 1 -- 172.21.15.99:0/3536575886 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fb010005040 con 0x560cae091a30 2024-03-19T02:04:32.633 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.627+0000 7fb008bfe640 1 -- 172.21.15.99:0/3536575886 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb010004920 con 0x560cae091a30 2024-03-19T02:04:32.633 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.627+0000 7fb01f21c740 1 -- 172.21.15.99:0/3536575886 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560cae091a30 msgr2=0x560cae093e40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:04:32.633 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.627+0000 7fb01f21c740 1 --2- 172.21.15.99:0/3536575886 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560cae091a30 0x560cae093e40 secure :-1 s=READY pgs=228 cs=0 l=1 rev1=1 crypto rx=0x7fb010009900 tx=0x7fb010009560 comp rx=0 tx=0).stop 2024-03-19T02:04:32.633 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.627+0000 7fb01f21c740 1 -- 172.21.15.99:0/3536575886 shutdown_connections 2024-03-19T02:04:32.633 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.627+0000 7fb01f21c740 1 --2- 172.21.15.99:0/3536575886 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560cae091a30 0x560cae093e40 unknown :-1 s=CLOSED pgs=228 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:32.633 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.628+0000 7fb01f21c740 1 --2- 172.21.15.99:0/3536575886 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560cae0944d0 0x560cae096960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:32.633 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.628+0000 7fb01f21c740 1 --2- 172.21.15.99:0/3536575886 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560cae084ff0 0x560cae0914f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:32.633 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.628+0000 7fb01f21c740 1 -- 172.21.15.99:0/3536575886 >> 172.21.15.99:0/3536575886 conn(0x560cae085a80 msgr2=0x560cae085e80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:04:32.633 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.628+0000 7fb01f21c740 1 -- 172.21.15.99:0/3536575886 shutdown_connections 2024-03-19T02:04:32.633 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.628+0000 7fb01f21c740 1 -- 172.21.15.99:0/3536575886 wait complete. 2024-03-19T02:04:32.633 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.628+0000 7fb01f21c740 1 -- start start 2024-03-19T02:04:32.633 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.628+0000 7fb01f21c740 1 Processor -- start 2024-03-19T02:04:32.633 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.628+0000 7fb01f21c740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560cae084ff0 0x560cae098a00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:32.633 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.628+0000 7fb01f21c740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560cae091a30 0x560cae098f40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:32.634 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.628+0000 7fb01f21c740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560cae0944d0 0x560cae09fea0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:32.634 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.628+0000 7fb01f21c740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x560cae097420 con 0x560cae091a30 2024-03-19T02:04:32.634 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.628+0000 7fb01f21c740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x560cadfa6db0 con 0x560cae084ff0 2024-03-19T02:04:32.634 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.628+0000 7fb01f21c740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x560cae0975a0 con 0x560cae0944d0 2024-03-19T02:04:32.634 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.628+0000 7fb01f21c740 10 client.0 _ll_register_callbacks cb 0x560cadfabe70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T02:04:32.634 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.628+0000 7fb01a3d0640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560cae091a30 0x560cae098f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:32.634 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.628+0000 7fb01abd1640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560cae084ff0 0x560cae098a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:32.634 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.628+0000 7fb01a3d0640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560cae091a30 0x560cae098f40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:57648/0 (socket says 172.21.15.99:57648) 2024-03-19T02:04:32.634 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.628+0000 7fb01a3d0640 1 -- 172.21.15.99:0/3261980094 learned_addr learned my addr 172.21.15.99:0/3261980094 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:04:32.634 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.628+0000 7fb01abd1640 1 --2- 172.21.15.99:0/3261980094 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560cae0944d0 0x560cae09fea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:32.634 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.629+0000 7fb01abd1640 1 -- 172.21.15.99:0/3261980094 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560cae0944d0 msgr2=0x560cae09fea0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:04:32.634 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.629+0000 7fb01abd1640 1 --2- 172.21.15.99:0/3261980094 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560cae0944d0 0x560cae09fea0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:32.634 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.629+0000 7fb01abd1640 1 -- 172.21.15.99:0/3261980094 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560cae091a30 msgr2=0x560cae098f40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:04:32.634 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.629+0000 7fb01abd1640 1 --2- 172.21.15.99:0/3261980094 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560cae091a30 0x560cae098f40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:32.635 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.629+0000 7fb01abd1640 1 -- 172.21.15.99:0/3261980094 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x560cadfa5c60 con 0x560cae084ff0 2024-03-19T02:04:32.635 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.629+0000 7fb01a3d0640 1 --2- 172.21.15.99:0/3261980094 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560cae091a30 0x560cae098f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:04:32.635 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.629+0000 7fb01abd1640 1 --2- 172.21.15.99:0/3261980094 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560cae084ff0 0x560cae098a00 secure :-1 s=READY pgs=229 cs=0 l=1 rev1=1 crypto rx=0x7fb00c009aa0 tx=0x7fb00c004ab0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=27c06e89f6313df7 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:04:32.635 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.629+0000 7faffdffb640 10 client.0 ms_handle_connect on v2:172.21.15.112:3300/0 2024-03-19T02:04:32.635 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.629+0000 7faffdffb640 1 -- 172.21.15.99:0/3261980094 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb00c011e90 con 0x560cae084ff0 2024-03-19T02:04:32.635 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.629+0000 7faffdffb640 1 -- 172.21.15.99:0/3261980094 <== mon.2 v2:172.21.15.112:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fb00c009040 con 0x560cae084ff0 2024-03-19T02:04:32.635 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.629+0000 7fb01f21c740 1 -- 172.21.15.99:0/3261980094 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fb010005040 con 0x560cae084ff0 2024-03-19T02:04:32.635 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.629+0000 7faffdffb640 1 -- 172.21.15.99:0/3261980094 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_get_version_reply(handle=1 version=41) ==== 24+0+0 (secure 0 0 0) 0x7fb00c004b20 con 0x560cae084ff0 2024-03-19T02:04:32.635 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.629+0000 7fb01f21c740 10 client.26984 fetch_fsmap learned FSMap version 41 2024-03-19T02:04:32.637 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.629+0000 7fb01f21c740 1 -- 172.21.15.99:0/3261980094 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,fsmap=41,monmap=4+}) -- 0x560cadcff210 con 0x560cae084ff0 2024-03-19T02:04:32.637 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.629+0000 7faffdffb640 1 -- 172.21.15.99:0/3261980094 <== mon.2 v2:172.21.15.112:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fb010005040 con 0x560cae084ff0 2024-03-19T02:04:32.637 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.630+0000 7faffdffb640 1 -- 172.21.15.99:0/3261980094 <== mon.2 v2:172.21.15.112:3300/0 5 ==== fsmap(e 41) ==== 8354+0+0 (secure 0 0 0) 0x7fb00c022920 con 0x560cae084ff0 2024-03-19T02:04:32.637 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.630+0000 7fb01f21c740 10 client.26984 fetch_fsmap finished waiting for FSMap version 41 2024-03-19T02:04:32.637 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.630+0000 7fb01f21c740 10 client.26984 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T02:04:32.637 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.630+0000 7fb01f21c740 20 client.26984 populate_metadata read hostname 'smithi099' 2024-03-19T02:04:32.637 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.630+0000 7fb01f21c740 1 --2- 172.21.15.99:0/3261980094 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x560cae0a4580 0x560cae0a69d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:32.637 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.630+0000 7fb01f21c740 4 client.26984 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T02:04:32.637 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.630+0000 7fb01f21c740 1 -- 172.21.15.99:0/3261980094 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x560cadff8120 con 0x560cae0a4580 2024-03-19T02:04:32.638 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.630+0000 7fb01a3d0640 1 --2- 172.21.15.99:0/3261980094 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x560cae0a4580 0x560cae0a69d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:32.638 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.630+0000 7faffdffb640 10 client.26984 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T02:04:32.638 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.630+0000 7fb01a3d0640 1 --2- 172.21.15.99:0/3261980094 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x560cae0a4580 0x560cae0a69d0 crc :-1 s=READY pgs=237 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=cc6f98e83385d73 server_cookie=f50c2aae0f5f2dca in_seq=0 out_seq=0 2024-03-19T02:04:32.639 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.632+0000 7faffdffb640 1 -- 172.21.15.99:0/3261980094 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x560cadff8120 con 0x560cae0a4580 2024-03-19T02:04:32.639 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.632+0000 7faffdffb640 10 client.26984 handle_command_reply: tid=0 2024-03-19T02:04:32.639 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.632+0000 7fb01f21c740 1 client.26984 shutdown 2024-03-19T02:04:32.639 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.632+0000 7fb01f21c740 1 -- 172.21.15.99:0/3261980094 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560cae084ff0 msgr2=0x560cae098a00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:04:32.639 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.632+0000 7fb01f21c740 1 --2- 172.21.15.99:0/3261980094 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560cae084ff0 0x560cae098a00 secure :-1 s=READY pgs=229 cs=0 l=1 rev1=1 crypto rx=0x7fb00c009aa0 tx=0x7fb00c004ab0 comp rx=0 tx=0).stop 2024-03-19T02:04:32.639 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.632+0000 7fb01f21c740 1 -- 172.21.15.99:0/3261980094 shutdown_connections 2024-03-19T02:04:32.639 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.632+0000 7fb01f21c740 1 --2- 172.21.15.99:0/3261980094 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x560cae0a4580 0x560cae0a69d0 crc :-1 s=READY pgs=237 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:32.640 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.632+0000 7faffdffb640 0 client.26984 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T02:04:32.640 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.632+0000 7fb01f21c740 1 --2- 172.21.15.99:0/3261980094 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560cae0944d0 0x560cae09fea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:32.640 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.632+0000 7fb01f21c740 1 --2- 172.21.15.99:0/3261980094 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560cae091a30 0x560cae098f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:32.640 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.632+0000 7fb01f21c740 1 --2- 172.21.15.99:0/3261980094 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560cae084ff0 0x560cae098a00 unknown :-1 s=CLOSED pgs=229 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:32.640 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.632+0000 7fb01f21c740 1 -- 172.21.15.99:0/3261980094 >> 172.21.15.99:0/3261980094 conn(0x560cae085a80 msgr2=0x560cae08fd70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:04:32.640 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.632+0000 7fb01f21c740 1 -- 172.21.15.99:0/3261980094 shutdown_connections 2024-03-19T02:04:32.640 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.632+0000 7fb01f21c740 1 -- 172.21.15.99:0/3261980094 wait complete. 2024-03-19T02:04:32.640 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.632+0000 7fb01f21c740 20 client.26984 trim_cache size 0 max 16384 2024-03-19T02:04:32.653 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.649+0000 7fb01f21c740 1 Processor -- start 2024-03-19T02:04:32.653 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.649+0000 7fb01f21c740 1 -- start start 2024-03-19T02:04:32.653 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.649+0000 7fb01f21c740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560cae084ff0 0x560cae093a70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:32.653 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.649+0000 7fb01f21c740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560cae091a30 0x560cae09fe60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:32.653 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.649+0000 7fb01f21c740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560cae0944d0 0x560cae0ac2e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:32.654 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.649+0000 7fb01f21c740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fb0141a9460 con 0x560cae091a30 2024-03-19T02:04:32.654 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.649+0000 7fb01f21c740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fb0141a6cd0 con 0x560cae0944d0 2024-03-19T02:04:32.654 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.649+0000 7fb01f21c740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fb0141a4b20 con 0x560cae084ff0 2024-03-19T02:04:32.654 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.649+0000 7fb01a3d0640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560cae091a30 0x560cae09fe60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:32.654 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.649+0000 7fb01a3d0640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560cae091a30 0x560cae09fe60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:57656/0 (socket says 172.21.15.99:57656) 2024-03-19T02:04:32.654 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.649+0000 7fb01a3d0640 1 -- 172.21.15.99:0/3823940858 learned_addr learned my addr 172.21.15.99:0/3823940858 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:04:32.654 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.649+0000 7fb01abd1640 1 --2- 172.21.15.99:0/3823940858 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560cae084ff0 0x560cae093a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:32.654 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.649+0000 7fb01abd1640 1 --2- 172.21.15.99:0/3823940858 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560cae0944d0 0x560cae0ac2e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:32.654 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.649+0000 7fb01a3d0640 1 -- 172.21.15.99:0/3823940858 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560cae084ff0 msgr2=0x560cae093a70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:04:32.654 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.649+0000 7fb01a3d0640 1 --2- 172.21.15.99:0/3823940858 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560cae084ff0 0x560cae093a70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:32.654 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.650+0000 7fb01a3d0640 1 -- 172.21.15.99:0/3823940858 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560cae0944d0 msgr2=0x560cae0ac2e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:04:32.654 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.650+0000 7fb01a3d0640 1 --2- 172.21.15.99:0/3823940858 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560cae0944d0 0x560cae0ac2e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:32.654 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.650+0000 7fb01a3d0640 1 -- 172.21.15.99:0/3823940858 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x560caddcb620 con 0x560cae091a30 2024-03-19T02:04:32.655 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.650+0000 7fb01a3d0640 1 --2- 172.21.15.99:0/3823940858 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560cae091a30 0x560cae09fe60 secure :-1 s=READY pgs=785 cs=0 l=1 rev1=1 crypto rx=0x7fb010005bc0 tx=0x7fb010005b50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2c1cf37890f5bae1 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:04:32.655 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.650+0000 7faffffff640 1 -- 172.21.15.99:0/3823940858 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb01004a040 con 0x560cae091a30 2024-03-19T02:04:32.655 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.650+0000 7faffffff640 1 -- 172.21.15.99:0/3823940858 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fb01002cc90 con 0x560cae091a30 2024-03-19T02:04:32.655 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.650+0000 7faffffff640 1 -- 172.21.15.99:0/3823940858 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb01004f9c0 con 0x560cae091a30 2024-03-19T02:04:32.655 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.651+0000 7fb01f21c740 1 -- 172.21.15.99:0/3823940858 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560cae091a30 msgr2=0x560cae09fe60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:04:32.655 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.651+0000 7fb01f21c740 1 --2- 172.21.15.99:0/3823940858 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560cae091a30 0x560cae09fe60 secure :-1 s=READY pgs=785 cs=0 l=1 rev1=1 crypto rx=0x7fb010005bc0 tx=0x7fb010005b50 comp rx=0 tx=0).stop 2024-03-19T02:04:32.655 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.651+0000 7fb01f21c740 1 -- 172.21.15.99:0/3823940858 shutdown_connections 2024-03-19T02:04:32.655 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.651+0000 7fb01f21c740 1 --2- 172.21.15.99:0/3823940858 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560cae0944d0 0x560cae0ac2e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:32.655 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.651+0000 7fb01f21c740 1 --2- 172.21.15.99:0/3823940858 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560cae091a30 0x560cae09fe60 unknown :-1 s=CLOSED pgs=785 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:32.655 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.651+0000 7fb01f21c740 1 --2- 172.21.15.99:0/3823940858 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560cae084ff0 0x560cae093a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:32.655 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.651+0000 7fb01f21c740 1 -- 172.21.15.99:0/3823940858 >> 172.21.15.99:0/3823940858 conn(0x560cae085a80 msgr2=0x560cae085e60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:04:32.655 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.651+0000 7fb01f21c740 1 -- 172.21.15.99:0/3823940858 shutdown_connections 2024-03-19T02:04:32.655 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.651+0000 7fb01f21c740 1 -- 172.21.15.99:0/3823940858 wait complete. 2024-03-19T02:04:32.655 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.651+0000 7fb01f21c740 1 -- start start 2024-03-19T02:04:32.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.651+0000 7fb01f21c740 1 Processor -- start 2024-03-19T02:04:32.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.651+0000 7fb01f21c740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560cae084ff0 0x560cae09cf40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:32.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.651+0000 7fb01f21c740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560cae091a30 0x560cae09d480 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:32.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.651+0000 7fb01f21c740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560cae0944d0 0x560cae09e490 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:32.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.651+0000 7fb01f21c740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fb0141a5190 con 0x560cae091a30 2024-03-19T02:04:32.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.651+0000 7fb01f21c740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fb0141a6600 con 0x560cae084ff0 2024-03-19T02:04:32.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.651+0000 7fb01f21c740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fb0141a5860 con 0x560cae0944d0 2024-03-19T02:04:32.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.651+0000 7fb01f21c740 10 client.0 _ll_register_callbacks cb 0x7fb00c0146b0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T02:04:32.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.651+0000 7fb01a3d0640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560cae091a30 0x560cae09d480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:32.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.651+0000 7fb01abd1640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560cae084ff0 0x560cae09cf40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:32.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.651+0000 7fb01abd1640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560cae0944d0 0x560cae09e490 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:32.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.651+0000 7fb01a3d0640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560cae091a30 0x560cae09d480 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:57670/0 (socket says 172.21.15.99:57670) 2024-03-19T02:04:32.656 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.651+0000 7fb01a3d0640 1 -- 172.21.15.99:0/3859083598 learned_addr learned my addr 172.21.15.99:0/3859083598 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:04:32.657 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.652+0000 7fb01a3d0640 1 -- 172.21.15.99:0/3859083598 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560cae0944d0 msgr2=0x560cae09e490 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:04:32.657 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.652+0000 7fb01a3d0640 1 --2- 172.21.15.99:0/3859083598 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560cae0944d0 0x560cae09e490 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:32.657 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.652+0000 7fb01a3d0640 1 -- 172.21.15.99:0/3859083598 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560cae084ff0 msgr2=0x560cae09cf40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:04:32.657 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.652+0000 7fb01a3d0640 1 --2- 172.21.15.99:0/3859083598 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560cae084ff0 0x560cae09cf40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:32.657 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.652+0000 7fb01a3d0640 1 -- 172.21.15.99:0/3859083598 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x560cae09ea80 con 0x560cae091a30 2024-03-19T02:04:32.657 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.652+0000 7fb01a3d0640 1 --2- 172.21.15.99:0/3859083598 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560cae091a30 0x560cae09d480 secure :-1 s=READY pgs=786 cs=0 l=1 rev1=1 crypto rx=0x7fb010009850 tx=0x7fb010005410 comp rx=0 tx=0).ready entity=mon.0 client_cookie=83eabadbbf7aa8ce server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:04:32.657 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.652+0000 7faffdffb640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T02:04:32.657 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.652+0000 7faffdffb640 1 -- 172.21.15.99:0/3859083598 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb01005c060 con 0x560cae091a30 2024-03-19T02:04:32.657 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.652+0000 7faffdffb640 1 -- 172.21.15.99:0/3859083598 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fb010004240 con 0x560cae091a30 2024-03-19T02:04:32.657 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.652+0000 7fb01f21c740 1 -- 172.21.15.99:0/3859083598 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fb01002cc90 con 0x560cae091a30 2024-03-19T02:04:32.657 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.652+0000 7faffdffb640 1 -- 172.21.15.99:0/3859083598 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=41) ==== 24+0+0 (secure 0 0 0) 0x7fb01004a030 con 0x560cae091a30 2024-03-19T02:04:32.657 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.652+0000 7fb01f21c740 10 client.17214 fetch_fsmap learned FSMap version 41 2024-03-19T02:04:32.657 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.652+0000 7fb01f21c740 1 -- 172.21.15.99:0/3859083598 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=41,monmap=4+}) -- 0x560cae09fdc0 con 0x560cae091a30 2024-03-19T02:04:32.657 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.652+0000 7faffdffb640 1 -- 172.21.15.99:0/3859083598 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fb01002cc90 con 0x560cae091a30 2024-03-19T02:04:32.657 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.652+0000 7faffdffb640 1 -- 172.21.15.99:0/3859083598 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 41) ==== 8354+0+0 (secure 0 0 0) 0x7fb010034600 con 0x560cae091a30 2024-03-19T02:04:32.658 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.652+0000 7fb01f21c740 10 client.17214 fetch_fsmap finished waiting for FSMap version 41 2024-03-19T02:04:32.658 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.652+0000 7fb01f21c740 10 client.17214 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T02:04:32.658 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.653+0000 7fb01f21c740 20 client.17214 populate_metadata read hostname 'smithi099' 2024-03-19T02:04:32.658 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.653+0000 7fb01f21c740 1 --2- 172.21.15.99:0/3859083598 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x560cae0a4580 0x560cae10c9a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:32.658 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.653+0000 7fb01f21c740 4 client.17214 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T02:04:32.658 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.653+0000 7fb01f21c740 1 -- 172.21.15.99:0/3859083598 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x560cadff2470 con 0x560cae0a4580 2024-03-19T02:04:32.658 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.653+0000 7fb01abd1640 1 --2- 172.21.15.99:0/3859083598 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x560cae0a4580 0x560cae10c9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:32.658 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.653+0000 7faffdffb640 10 client.17214 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T02:04:32.658 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:32.653+0000 7fb01abd1640 1 --2- 172.21.15.99:0/3859083598 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x560cae0a4580 0x560cae10c9a0 crc :-1 s=READY pgs=238 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=1f573a72b2226ef server_cookie=7f9d244c4d016ec2 in_seq=0 out_seq=0 2024-03-19T02:04:33.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:04:32 smithi115 ceph-mon[67332]: pgmap v1054: 97 pgs: 97 active+clean; 645 MiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.5 MiB/s rd, 5.6 MiB/s wr, 524 op/s 2024-03-19T02:04:33.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:04:32 smithi099 ceph-mon[62951]: pgmap v1054: 97 pgs: 97 active+clean; 645 MiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.5 MiB/s rd, 5.6 MiB/s wr, 524 op/s 2024-03-19T02:04:33.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:04:32 smithi112 ceph-mon[66819]: pgmap v1054: 97 pgs: 97 active+clean; 645 MiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.5 MiB/s rd, 5.6 MiB/s wr, 524 op/s 2024-03-19T02:04:33.121 INFO:tasks.workunit.client.0.smithi099.stdout: 10 277045 22.16 MB/sec execute 542 sec latency 459.846 ms 2024-03-19T02:04:33.390 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:33.389+0000 7faffdffb640 1 -- 172.21.15.99:0/3859083598 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x560cadff2470 con 0x560cae0a4580 2024-03-19T02:04:33.390 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:33.389+0000 7faffdffb640 10 client.17214 handle_command_reply: tid=0 2024-03-19T02:04:33.390 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:33.389+0000 7fb01f21c740 1 client.17214 shutdown 2024-03-19T02:04:33.391 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:33.390+0000 7fb01f21c740 1 -- 172.21.15.99:0/3859083598 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560cae091a30 msgr2=0x560cae09d480 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:04:33.391 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:33.390+0000 7fb01f21c740 1 --2- 172.21.15.99:0/3859083598 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560cae091a30 0x560cae09d480 secure :-1 s=READY pgs=786 cs=0 l=1 rev1=1 crypto rx=0x7fb010009850 tx=0x7fb010005410 comp rx=0 tx=0).stop 2024-03-19T02:04:33.391 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:33.390+0000 7fb01f21c740 1 -- 172.21.15.99:0/3859083598 shutdown_connections 2024-03-19T02:04:33.391 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:33.390+0000 7fb01f21c740 1 --2- 172.21.15.99:0/3859083598 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x560cae0a4580 0x560cae10c9a0 crc :-1 s=READY pgs=238 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:33.391 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:33.390+0000 7fb01f21c740 1 --2- 172.21.15.99:0/3859083598 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x560cae0944d0 0x560cae09e490 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:33.391 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:33.390+0000 7fb01f21c740 1 --2- 172.21.15.99:0/3859083598 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x560cae091a30 0x560cae09d480 unknown :-1 s=CLOSED pgs=786 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:33.391 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:33.390+0000 7faffdffb640 0 client.17214 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T02:04:33.391 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:33.390+0000 7fb01f21c740 1 --2- 172.21.15.99:0/3859083598 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x560cae084ff0 0x560cae09cf40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:33.391 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:33.390+0000 7fb01f21c740 1 -- 172.21.15.99:0/3859083598 >> 172.21.15.99:0/3859083598 conn(0x560cae085a80 msgr2=0x560cae098480 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:04:33.391 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:33.390+0000 7fb01f21c740 1 -- 172.21.15.99:0/3859083598 shutdown_connections 2024-03-19T02:04:33.391 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:33.390+0000 7fb01f21c740 1 -- 172.21.15.99:0/3859083598 wait complete. 2024-03-19T02:04:33.391 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:33.390+0000 7fb01f21c740 20 client.17214 trim_cache size 0 max 16384 2024-03-19T02:04:33.392 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T02:04:33.392 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T02:04:33.392 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T02:04:33.392 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T02:04:33.393 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:33.390+0000 7fb01c634640 1 -- 172.21.15.99:0/1642349795 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb0000a28a0 msgr2=0x7fb0000a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:04:33.393 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:33.390+0000 7fb01c634640 1 --2- 172.21.15.99:0/1642349795 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb0000a28a0 0x7fb0000a4d60 secure :-1 s=READY pgs=390 cs=0 l=1 rev1=1 crypto rx=0x7fb004000c00 tx=0x7fb004018420 comp rx=0 tx=0).stop 2024-03-19T02:04:33.393 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:33.390+0000 7fb01c634640 1 -- 172.21.15.99:0/1642349795 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb014102a40 msgr2=0x7fb014197ff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:04:33.393 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:33.390+0000 7fb01c634640 1 --2- 172.21.15.99:0/1642349795 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb014102a40 0x7fb014197ff0 secure :-1 s=READY pgs=132 cs=0 l=1 rev1=1 crypto rx=0x7fb00400e9f0 tx=0x7fb00400eeb0 comp rx=0 tx=0).stop 2024-03-19T02:04:33.393 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:33.390+0000 7fb01c634640 1 -- 172.21.15.99:0/1642349795 shutdown_connections 2024-03-19T02:04:33.393 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:33.390+0000 7fb01c634640 1 --2- 172.21.15.99:0/1642349795 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb014103400 0x7fb01419f060 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:33.393 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:33.390+0000 7fb01c634640 1 --2- 172.21.15.99:0/1642349795 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb014102a40 0x7fb014197ff0 unknown :-1 s=CLOSED pgs=132 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:33.393 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:33.390+0000 7fb01c634640 1 --2- 172.21.15.99:0/1642349795 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb0000a28a0 0x7fb0000a4d60 unknown :-1 s=CLOSED pgs=390 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:33.393 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:33.390+0000 7fb01c634640 1 --2- 172.21.15.99:0/1642349795 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb014101850 0x7fb014197ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:33.393 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:33.390+0000 7fb01c634640 1 -- 172.21.15.99:0/1642349795 >> 172.21.15.99:0/1642349795 conn(0x7fb0140fcde0 msgr2=0x7fb014105ce0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:04:33.393 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:33.391+0000 7fb01c634640 1 -- 172.21.15.99:0/1642349795 shutdown_connections 2024-03-19T02:04:33.393 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:33.391+0000 7fb01c634640 1 -- 172.21.15.99:0/1642349795 wait complete. 2024-03-19T02:04:34.121 INFO:tasks.workunit.client.0.smithi099.stdout: 10 277245 22.13 MB/sec execute 543 sec latency 669.031 ms 2024-03-19T02:04:35.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:04:34 smithi115 ceph-mon[67332]: pgmap v1055: 97 pgs: 97 active+clean; 649 MiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 5.8 MiB/s wr, 468 op/s 2024-03-19T02:04:35.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:04:34 smithi099 ceph-mon[62951]: pgmap v1055: 97 pgs: 97 active+clean; 649 MiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 5.8 MiB/s wr, 468 op/s 2024-03-19T02:04:35.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:04:34 smithi112 ceph-mon[66819]: pgmap v1055: 97 pgs: 97 active+clean; 649 MiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 5.8 MiB/s wr, 468 op/s 2024-03-19T02:04:35.121 INFO:tasks.workunit.client.0.smithi099.stdout: 10 277671 22.14 MB/sec execute 544 sec latency 115.707 ms 2024-03-19T02:04:36.121 INFO:tasks.workunit.client.0.smithi099.stdout: 10 278092 22.13 MB/sec execute 545 sec latency 88.858 ms 2024-03-19T02:04:37.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:04:36 smithi115 ceph-mon[67332]: pgmap v1056: 97 pgs: 97 active+clean; 648 MiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.5 MiB/s rd, 5.2 MiB/s wr, 394 op/s 2024-03-19T02:04:37.098 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:04:36 smithi099 ceph-mon[62951]: pgmap v1056: 97 pgs: 97 active+clean; 648 MiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.5 MiB/s rd, 5.2 MiB/s wr, 394 op/s 2024-03-19T02:04:37.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:04:36 smithi112 ceph-mon[66819]: pgmap v1056: 97 pgs: 97 active+clean; 648 MiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 3.5 MiB/s rd, 5.2 MiB/s wr, 394 op/s 2024-03-19T02:04:37.121 INFO:tasks.workunit.client.0.smithi099.stdout: 10 278572 22.14 MB/sec execute 546 sec latency 252.564 ms 2024-03-19T02:04:38.121 INFO:tasks.workunit.client.0.smithi099.stdout: 10 279112 22.16 MB/sec execute 547 sec latency 282.876 ms 2024-03-19T02:04:39.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:04:38 smithi115 ceph-mon[67332]: pgmap v1057: 97 pgs: 97 active+clean; 640 MiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 4.3 MiB/s rd, 6.2 MiB/s wr, 528 op/s 2024-03-19T02:04:39.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:04:38 smithi099 ceph-mon[62951]: pgmap v1057: 97 pgs: 97 active+clean; 640 MiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 4.3 MiB/s rd, 6.2 MiB/s wr, 528 op/s 2024-03-19T02:04:39.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:04:38 smithi112 ceph-mon[66819]: pgmap v1057: 97 pgs: 97 active+clean; 640 MiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 4.3 MiB/s rd, 6.2 MiB/s wr, 528 op/s 2024-03-19T02:04:39.121 INFO:tasks.workunit.client.0.smithi099.stdout: 10 279441 22.16 MB/sec execute 548 sec latency 259.708 ms 2024-03-19T02:04:40.121 INFO:tasks.workunit.client.0.smithi099.stdout: 10 279803 22.14 MB/sec execute 549 sec latency 114.393 ms 2024-03-19T02:04:41.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:04:40 smithi115 ceph-mon[67332]: pgmap v1058: 97 pgs: 97 active+clean; 643 MiB data, 9.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 5.7 MiB/s wr, 504 op/s 2024-03-19T02:04:41.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:04:40 smithi099 ceph-mon[62951]: pgmap v1058: 97 pgs: 97 active+clean; 643 MiB data, 9.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 5.7 MiB/s wr, 504 op/s 2024-03-19T02:04:41.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:04:40 smithi112 ceph-mon[66819]: pgmap v1058: 97 pgs: 97 active+clean; 643 MiB data, 9.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 5.7 MiB/s wr, 504 op/s 2024-03-19T02:04:41.121 INFO:tasks.workunit.client.0.smithi099.stdout: 10 280247 22.14 MB/sec execute 550 sec latency 64.222 ms 2024-03-19T02:04:42.121 INFO:tasks.workunit.client.0.smithi099.stdout: 10 280681 22.15 MB/sec execute 551 sec latency 86.417 ms 2024-03-19T02:04:43.057 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:04:42 smithi115 ceph-mon[67332]: pgmap v1059: 97 pgs: 97 active+clean; 646 MiB data, 9.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.6 MiB/s rd, 5.7 MiB/s wr, 536 op/s 2024-03-19T02:04:43.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:04:42 smithi099 ceph-mon[62951]: pgmap v1059: 97 pgs: 97 active+clean; 646 MiB data, 9.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.6 MiB/s rd, 5.7 MiB/s wr, 536 op/s 2024-03-19T02:04:43.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:04:42 smithi112 ceph-mon[66819]: pgmap v1059: 97 pgs: 97 active+clean; 646 MiB data, 9.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.6 MiB/s rd, 5.7 MiB/s wr, 536 op/s 2024-03-19T02:04:43.121 INFO:tasks.workunit.client.0.smithi099.stdout: 10 281091 22.15 MB/sec execute 552 sec latency 121.109 ms 2024-03-19T02:04:44.121 INFO:tasks.workunit.client.0.smithi099.stdout: 10 281442 22.15 MB/sec execute 553 sec latency 286.661 ms 2024-03-19T02:04:45.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:04:44 smithi115 ceph-mon[67332]: pgmap v1060: 97 pgs: 97 active+clean; 651 MiB data, 9.3 GiB used, 1.0 TiB / 1.0 TiB avail; 5.3 MiB/s rd, 6.4 MiB/s wr, 569 op/s 2024-03-19T02:04:45.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:04:44 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T02:04:45.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:04:44 smithi099 ceph-mon[62951]: pgmap v1060: 97 pgs: 97 active+clean; 651 MiB data, 9.3 GiB used, 1.0 TiB / 1.0 TiB avail; 5.3 MiB/s rd, 6.4 MiB/s wr, 569 op/s 2024-03-19T02:04:45.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:04:44 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T02:04:45.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:04:44 smithi112 ceph-mon[66819]: pgmap v1060: 97 pgs: 97 active+clean; 651 MiB data, 9.3 GiB used, 1.0 TiB / 1.0 TiB avail; 5.3 MiB/s rd, 6.4 MiB/s wr, 569 op/s 2024-03-19T02:04:45.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:04:44 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T02:04:45.121 INFO:tasks.workunit.client.0.smithi099.stdout: 10 281818 22.15 MB/sec execute 554 sec latency 121.465 ms 2024-03-19T02:04:46.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:04:45 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T02:04:46.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:04:45 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T02:04:46.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:04:45 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:04:46.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:04:45 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T02:04:46.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:04:45 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T02:04:46.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:04:45 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:04:46.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:04:45 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T02:04:46.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:04:45 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T02:04:46.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:04:45 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:04:46.121 INFO:tasks.workunit.client.0.smithi099.stdout: 10 282153 22.13 MB/sec execute 555 sec latency 103.038 ms 2024-03-19T02:04:46.483 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.481+0000 7f1918c4b640 1 -- 172.21.15.99:0/316279590 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1914102340 msgr2=0x7f1914104730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:04:46.483 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.481+0000 7f1918c4b640 1 --2- 172.21.15.99:0/316279590 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1914102340 0x7f1914104730 secure :-1 s=READY pgs=787 cs=0 l=1 rev1=1 crypto rx=0x7f1904009900 tx=0x7f190402bc30 comp rx=0 tx=0).stop 2024-03-19T02:04:46.484 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.481+0000 7f1918c4b640 1 -- 172.21.15.99:0/316279590 shutdown_connections 2024-03-19T02:04:46.484 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.481+0000 7f1918c4b640 1 --2- 172.21.15.99:0/316279590 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1914104c70 0x7f1914107060 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:46.484 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.481+0000 7f1918c4b640 1 --2- 172.21.15.99:0/316279590 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f19141075a0 0x7f1914109990 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:46.484 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.481+0000 7f1918c4b640 1 --2- 172.21.15.99:0/316279590 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1914102340 0x7f1914104730 unknown :-1 s=CLOSED pgs=787 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:46.484 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.481+0000 7f1918c4b640 1 -- 172.21.15.99:0/316279590 >> 172.21.15.99:0/316279590 conn(0x7f19140fbcb0 msgr2=0x7f19140fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:04:46.484 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.481+0000 7f1918c4b640 1 -- 172.21.15.99:0/316279590 shutdown_connections 2024-03-19T02:04:46.484 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.481+0000 7f1918c4b640 1 -- 172.21.15.99:0/316279590 wait complete. 2024-03-19T02:04:46.484 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.482+0000 7f1918c4b640 1 Processor -- start 2024-03-19T02:04:46.484 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.482+0000 7f1918c4b640 1 -- start start 2024-03-19T02:04:46.484 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.482+0000 7f1918c4b640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1914102340 0x7f1914197af0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:46.485 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.482+0000 7f1918c4b640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1914104c70 0x7f1914198030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:46.485 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.482+0000 7f1918c4b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f19141075a0 0x7f191419f0a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:46.485 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.482+0000 7f1918c4b640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f191410d4b0 con 0x7f19141075a0 2024-03-19T02:04:46.485 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.482+0000 7f1918c4b640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f191410d330 con 0x7f1914104c70 2024-03-19T02:04:46.485 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.482+0000 7f1918c4b640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f191410d630 con 0x7f1914102340 2024-03-19T02:04:46.485 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.482+0000 7f1912d9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f19141075a0 0x7f191419f0a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:46.485 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.482+0000 7f1912d9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f19141075a0 0x7f191419f0a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:32810/0 (socket says 172.21.15.99:32810) 2024-03-19T02:04:46.485 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.482+0000 7f191259c640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1914102340 0x7f1914197af0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:46.485 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.482+0000 7f1912d9d640 1 -- 172.21.15.99:0/2135570197 learned_addr learned my addr 172.21.15.99:0/2135570197 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:04:46.486 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.482+0000 7f190bfff640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1914104c70 0x7f1914198030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:46.486 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.482+0000 7f1912d9d640 1 -- 172.21.15.99:0/2135570197 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1914102340 msgr2=0x7f1914197af0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:04:46.486 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.482+0000 7f1912d9d640 1 --2- 172.21.15.99:0/2135570197 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1914102340 0x7f1914197af0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:46.486 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.482+0000 7f1912d9d640 1 -- 172.21.15.99:0/2135570197 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1914104c70 msgr2=0x7f1914198030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:04:46.486 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.482+0000 7f1912d9d640 1 --2- 172.21.15.99:0/2135570197 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1914104c70 0x7f1914198030 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:46.486 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.482+0000 7f1912d9d640 1 -- 172.21.15.99:0/2135570197 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f191419f710 con 0x7f19141075a0 2024-03-19T02:04:46.486 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.483+0000 7f190bfff640 1 --2- 172.21.15.99:0/2135570197 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1914104c70 0x7f1914198030 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:04:46.486 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.483+0000 7f1912d9d640 1 --2- 172.21.15.99:0/2135570197 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f19141075a0 0x7f191419f0a0 secure :-1 s=READY pgs=788 cs=0 l=1 rev1=1 crypto rx=0x7f190c00eb10 tx=0x7f190c00c2d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:04:46.486 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.483+0000 7f190b7fe640 1 -- 172.21.15.99:0/2135570197 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f190c004b20 con 0x7f19141075a0 2024-03-19T02:04:46.486 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.483+0000 7f1918c4b640 1 -- 172.21.15.99:0/2135570197 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f191419fa00 con 0x7f19141075a0 2024-03-19T02:04:46.487 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.483+0000 7f190b7fe640 1 -- 172.21.15.99:0/2135570197 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f190c004cc0 con 0x7f19141075a0 2024-03-19T02:04:46.487 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.483+0000 7f190b7fe640 1 -- 172.21.15.99:0/2135570197 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f190c010760 con 0x7f19141075a0 2024-03-19T02:04:46.487 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.483+0000 7f1918c4b640 1 -- 172.21.15.99:0/2135570197 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f191419ff50 con 0x7f19141075a0 2024-03-19T02:04:46.492 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.485+0000 7f190b7fe640 1 -- 172.21.15.99:0/2135570197 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f190c005020 con 0x7f19141075a0 2024-03-19T02:04:46.492 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.485+0000 7f191b835740 1 Processor -- start 2024-03-19T02:04:46.492 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.486+0000 7f191b835740 1 -- start start 2024-03-19T02:04:46.492 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.486+0000 7f191b835740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56073a638100 0x56073a638520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:46.492 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.486+0000 7f190b7fe640 1 --2- 172.21.15.99:0/2135570197 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f19000a28a0 0x7f19000a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:46.493 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.486+0000 7f191b835740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56073a638af0 0x56073a63d5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:46.493 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.486+0000 7f191b835740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56073a63dc60 0x56073a6400f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:46.493 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.486+0000 7f191b835740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x56073a59cbe0 con 0x56073a638100 2024-03-19T02:04:46.493 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.486+0000 7f191b835740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x56073a545960 con 0x56073a63dc60 2024-03-19T02:04:46.493 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.486+0000 7f191b835740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x56073a598c80 con 0x56073a638af0 2024-03-19T02:04:46.493 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.486+0000 7f190bfff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56073a638100 0x56073a638520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:46.493 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.486+0000 7f191259c640 1 --2- 172.21.15.99:0/2135570197 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f19000a28a0 0x7f19000a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:46.493 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.486+0000 7f1912d9d640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56073a63dc60 0x56073a6400f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:46.493 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.486+0000 7f190bfff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56073a638100 0x56073a638520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:32824/0 (socket says 172.21.15.99:32824) 2024-03-19T02:04:46.493 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.486+0000 7f190bfff640 1 -- 172.21.15.99:0/4006012739 learned_addr learned my addr 172.21.15.99:0/4006012739 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:04:46.493 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.486+0000 7f190b7fe640 1 -- 172.21.15.99:0/2135570197 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f190c0cf010 con 0x7f19141075a0 2024-03-19T02:04:46.493 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.486+0000 7f190bfff640 1 --2- 172.21.15.99:0/4006012739 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56073a638af0 0x56073a63d5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:46.493 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.486+0000 7f1912d9d640 1 -- 172.21.15.99:0/4006012739 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56073a638af0 msgr2=0x56073a63d5f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:04:46.494 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.486+0000 7f1912d9d640 1 --2- 172.21.15.99:0/4006012739 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56073a638af0 0x56073a63d5f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:46.494 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.486+0000 7f1912d9d640 1 -- 172.21.15.99:0/4006012739 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56073a638100 msgr2=0x56073a638520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:04:46.494 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.486+0000 7f1912d9d640 1 --2- 172.21.15.99:0/4006012739 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56073a638100 0x56073a638520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:46.494 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.486+0000 7f1912d9d640 1 -- 172.21.15.99:0/4006012739 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x56073a53cd20 con 0x56073a63dc60 2024-03-19T02:04:46.494 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.486+0000 7f190bfff640 1 --2- 172.21.15.99:0/4006012739 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56073a638100 0x56073a638520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:04:46.494 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.487+0000 7f191259c640 1 --2- 172.21.15.99:0/2135570197 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f19000a28a0 0x7f19000a4d60 secure :-1 s=READY pgs=391 cs=0 l=1 rev1=1 crypto rx=0x7f190402c7b0 tx=0x7f1904005fd0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:04:46.494 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.487+0000 7f1912d9d640 1 --2- 172.21.15.99:0/4006012739 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56073a63dc60 0x56073a6400f0 secure :-1 s=READY pgs=230 cs=0 l=1 rev1=1 crypto rx=0x7f190c0ccb20 tx=0x7f190c0975b0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=7d7fa0cc962f2b3c server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:04:46.494 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.487+0000 7f1908ff9640 1 -- 172.21.15.99:0/4006012739 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f190c09b2d0 con 0x56073a63dc60 2024-03-19T02:04:46.494 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.487+0000 7f1908ff9640 1 -- 172.21.15.99:0/4006012739 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f190c09c070 con 0x56073a63dc60 2024-03-19T02:04:46.494 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.487+0000 7f1908ff9640 1 -- 172.21.15.99:0/4006012739 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f190c099d40 con 0x56073a63dc60 2024-03-19T02:04:46.494 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.488+0000 7f191b835740 1 -- 172.21.15.99:0/4006012739 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56073a63dc60 msgr2=0x56073a6400f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:04:46.494 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.488+0000 7f191b835740 1 --2- 172.21.15.99:0/4006012739 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56073a63dc60 0x56073a6400f0 secure :-1 s=READY pgs=230 cs=0 l=1 rev1=1 crypto rx=0x7f190c0ccb20 tx=0x7f190c0975b0 comp rx=0 tx=0).stop 2024-03-19T02:04:46.494 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.488+0000 7f191b835740 1 -- 172.21.15.99:0/4006012739 shutdown_connections 2024-03-19T02:04:46.494 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.488+0000 7f191b835740 1 --2- 172.21.15.99:0/4006012739 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56073a63dc60 0x56073a6400f0 unknown :-1 s=CLOSED pgs=230 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:46.495 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.488+0000 7f191b835740 1 --2- 172.21.15.99:0/4006012739 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56073a638af0 0x56073a63d5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:46.495 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.488+0000 7f191b835740 1 --2- 172.21.15.99:0/4006012739 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56073a638100 0x56073a638520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:46.495 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.488+0000 7f191b835740 1 -- 172.21.15.99:0/4006012739 >> 172.21.15.99:0/4006012739 conn(0x56073a633670 msgr2=0x56073a633a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:04:46.495 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.488+0000 7f191b835740 1 -- 172.21.15.99:0/4006012739 shutdown_connections 2024-03-19T02:04:46.495 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.488+0000 7f191b835740 1 -- 172.21.15.99:0/4006012739 wait complete. 2024-03-19T02:04:46.495 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.488+0000 7f191b835740 1 -- start start 2024-03-19T02:04:46.495 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.488+0000 7f191b835740 1 Processor -- start 2024-03-19T02:04:46.495 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.488+0000 7f191b835740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56073a638100 0x56073a642080 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:46.495 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.488+0000 7f191b835740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56073a638af0 0x56073a6425c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:46.495 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.488+0000 7f191b835740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56073a63dc60 0x56073a649600 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:46.495 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.488+0000 7f191b835740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x56073a640bb0 con 0x56073a63dc60 2024-03-19T02:04:46.495 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.488+0000 7f191b835740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x56073a558db0 con 0x56073a638100 2024-03-19T02:04:46.495 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.488+0000 7f191b835740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x56073a640d30 con 0x56073a638af0 2024-03-19T02:04:46.495 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.488+0000 7f191b835740 10 client.0 _ll_register_callbacks cb 0x56073a55de70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T02:04:46.496 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.488+0000 7f1912d9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56073a63dc60 0x56073a649600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:46.496 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.488+0000 7f190bfff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56073a638af0 0x56073a6425c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:46.496 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.488+0000 7f191259c640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56073a638100 0x56073a642080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:46.496 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.488+0000 7f1912d9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56073a63dc60 0x56073a649600 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:32832/0 (socket says 172.21.15.99:32832) 2024-03-19T02:04:46.496 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.488+0000 7f190bfff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56073a638af0 0x56073a6425c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:53866/0 (socket says 172.21.15.99:53866) 2024-03-19T02:04:46.496 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.488+0000 7f1912d9d640 1 -- 172.21.15.99:0/649052647 learned_addr learned my addr 172.21.15.99:0/649052647 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:04:46.496 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.489+0000 7f1912d9d640 1 -- 172.21.15.99:0/649052647 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56073a638af0 msgr2=0x56073a6425c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:04:46.496 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.489+0000 7f1912d9d640 1 --2- 172.21.15.99:0/649052647 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56073a638af0 0x56073a6425c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:46.496 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.489+0000 7f1912d9d640 1 -- 172.21.15.99:0/649052647 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56073a638100 msgr2=0x56073a642080 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:04:46.496 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.489+0000 7f1912d9d640 1 --2- 172.21.15.99:0/649052647 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56073a638100 0x56073a642080 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:46.496 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.489+0000 7f1912d9d640 1 -- 172.21.15.99:0/649052647 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x56073a557c60 con 0x56073a63dc60 2024-03-19T02:04:46.496 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.489+0000 7f1912d9d640 1 --2- 172.21.15.99:0/649052647 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56073a63dc60 0x56073a649600 secure :-1 s=READY pgs=789 cs=0 l=1 rev1=1 crypto rx=0x7f190c0a3be0 tx=0x7f190c09b350 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8cfdf8c237b78a3e server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:04:46.497 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.489+0000 7f18f9ffb640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T02:04:46.497 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.489+0000 7f18f9ffb640 1 -- 172.21.15.99:0/649052647 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f190c0b4060 con 0x56073a63dc60 2024-03-19T02:04:46.497 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.489+0000 7f18f9ffb640 1 -- 172.21.15.99:0/649052647 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f190c086110 con 0x56073a63dc60 2024-03-19T02:04:46.497 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.489+0000 7f191b835740 1 -- 172.21.15.99:0/649052647 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f190c09c070 con 0x56073a63dc60 2024-03-19T02:04:46.497 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.489+0000 7f18f9ffb640 1 -- 172.21.15.99:0/649052647 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=41) ==== 24+0+0 (secure 0 0 0) 0x7f190c09fac0 con 0x56073a63dc60 2024-03-19T02:04:46.497 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.489+0000 7f191b835740 10 client.17226 fetch_fsmap learned FSMap version 41 2024-03-19T02:04:46.497 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.489+0000 7f191b835740 1 -- 172.21.15.99:0/649052647 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=41,monmap=4+}) -- 0x56073a2b1210 con 0x56073a63dc60 2024-03-19T02:04:46.497 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.489+0000 7f18f9ffb640 1 -- 172.21.15.99:0/649052647 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f190c09c070 con 0x56073a63dc60 2024-03-19T02:04:46.497 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.490+0000 7f18f9ffb640 1 -- 172.21.15.99:0/649052647 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 41) ==== 8354+0+0 (secure 0 0 0) 0x7f190c005430 con 0x56073a63dc60 2024-03-19T02:04:46.497 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.490+0000 7f191b835740 10 client.17226 fetch_fsmap finished waiting for FSMap version 41 2024-03-19T02:04:46.497 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.490+0000 7f191b835740 10 client.17226 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T02:04:46.498 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.490+0000 7f191b835740 20 client.17226 populate_metadata read hostname 'smithi099' 2024-03-19T02:04:46.498 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.490+0000 7f191b835740 1 --2- 172.21.15.99:0/649052647 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56073a64daf0 0x56073a64fee0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:46.498 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.490+0000 7f191b835740 4 client.17226 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T02:04:46.498 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.490+0000 7f191b835740 1 -- 172.21.15.99:0/649052647 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x56073a5aa120 con 0x56073a64daf0 2024-03-19T02:04:46.498 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.490+0000 7f191259c640 1 --2- 172.21.15.99:0/649052647 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56073a64daf0 0x56073a64fee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:46.498 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.490+0000 7f18f9ffb640 10 client.17226 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T02:04:46.498 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.490+0000 7f191259c640 1 --2- 172.21.15.99:0/649052647 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56073a64daf0 0x56073a64fee0 crc :-1 s=READY pgs=239 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=573444b7f447653c server_cookie=757ed52cddc166c6 in_seq=0 out_seq=0 2024-03-19T02:04:46.499 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.498+0000 7f18f9ffb640 1 -- 172.21.15.99:0/649052647 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x56073a5aa120 con 0x56073a64daf0 2024-03-19T02:04:46.499 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.498+0000 7f18f9ffb640 10 client.17226 handle_command_reply: tid=0 2024-03-19T02:04:46.499 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.498+0000 7f191b835740 1 client.17226 shutdown 2024-03-19T02:04:46.500 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.498+0000 7f191b835740 1 -- 172.21.15.99:0/649052647 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56073a63dc60 msgr2=0x56073a649600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:04:46.500 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.498+0000 7f191b835740 1 --2- 172.21.15.99:0/649052647 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56073a63dc60 0x56073a649600 secure :-1 s=READY pgs=789 cs=0 l=1 rev1=1 crypto rx=0x7f190c0a3be0 tx=0x7f190c09b350 comp rx=0 tx=0).stop 2024-03-19T02:04:46.500 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.498+0000 7f191b835740 1 -- 172.21.15.99:0/649052647 shutdown_connections 2024-03-19T02:04:46.500 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.498+0000 7f191b835740 1 --2- 172.21.15.99:0/649052647 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56073a64daf0 0x56073a64fee0 crc :-1 s=READY pgs=239 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:46.500 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.498+0000 7f191b835740 1 --2- 172.21.15.99:0/649052647 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56073a63dc60 0x56073a649600 unknown :-1 s=CLOSED pgs=789 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:46.500 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.498+0000 7f191b835740 1 --2- 172.21.15.99:0/649052647 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56073a638af0 0x56073a6425c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:46.500 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.498+0000 7f191b835740 1 --2- 172.21.15.99:0/649052647 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56073a638100 0x56073a642080 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:46.500 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.498+0000 7f18f9ffb640 0 client.17226 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T02:04:46.500 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.498+0000 7f191b835740 1 -- 172.21.15.99:0/649052647 >> 172.21.15.99:0/649052647 conn(0x56073a633670 msgr2=0x56073a6348c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:04:46.500 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.498+0000 7f191b835740 1 -- 172.21.15.99:0/649052647 shutdown_connections 2024-03-19T02:04:46.500 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.498+0000 7f191b835740 1 -- 172.21.15.99:0/649052647 wait complete. 2024-03-19T02:04:46.500 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.498+0000 7f191b835740 20 client.17226 trim_cache size 0 max 16384 2024-03-19T02:04:46.519 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.514+0000 7f191b835740 1 Processor -- start 2024-03-19T02:04:46.519 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.514+0000 7f191b835740 1 -- start start 2024-03-19T02:04:46.519 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.514+0000 7f191b835740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56073a638100 0x56073a639590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:46.520 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.514+0000 7f191b835740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56073a638af0 0x56073a6495c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:46.520 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.514+0000 7f191b835740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56073a63dc60 0x56073a639ad0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:46.520 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.514+0000 7f191b835740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f19141ab710 con 0x56073a63dc60 2024-03-19T02:04:46.520 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.514+0000 7f191b835740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f19141a8f80 con 0x56073a638100 2024-03-19T02:04:46.520 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.514+0000 7f191b835740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f19141a6dd0 con 0x56073a638af0 2024-03-19T02:04:46.520 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.515+0000 7f1912d9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56073a63dc60 0x56073a639ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:46.520 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.515+0000 7f190bfff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56073a638af0 0x56073a6495c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:46.520 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.515+0000 7f190bfff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56073a638af0 0x56073a6495c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:53876/0 (socket says 172.21.15.99:53876) 2024-03-19T02:04:46.520 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.515+0000 7f191259c640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56073a638100 0x56073a639590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:46.520 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.515+0000 7f1912d9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56073a63dc60 0x56073a639ad0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:32846/0 (socket says 172.21.15.99:32846) 2024-03-19T02:04:46.520 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.515+0000 7f190bfff640 1 -- 172.21.15.99:0/4242229609 learned_addr learned my addr 172.21.15.99:0/4242229609 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:04:46.521 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.515+0000 7f1912d9d640 1 -- 172.21.15.99:0/4242229609 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56073a638af0 msgr2=0x56073a6495c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:04:46.521 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.515+0000 7f1912d9d640 1 --2- 172.21.15.99:0/4242229609 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56073a638af0 0x56073a6495c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:46.521 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.515+0000 7f1912d9d640 1 -- 172.21.15.99:0/4242229609 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56073a638100 msgr2=0x56073a639590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:04:46.521 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.515+0000 7f1912d9d640 1 --2- 172.21.15.99:0/4242229609 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56073a638100 0x56073a639590 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:46.521 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.515+0000 7f1912d9d640 1 -- 172.21.15.99:0/4242229609 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x56073a37d620 con 0x56073a63dc60 2024-03-19T02:04:46.521 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.515+0000 7f1912d9d640 1 --2- 172.21.15.99:0/4242229609 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56073a63dc60 0x56073a639ad0 secure :-1 s=READY pgs=790 cs=0 l=1 rev1=1 crypto rx=0x7f190c09b670 tx=0x7f190c0975b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=39c2a49a4c7f62b5 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:04:46.521 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.515+0000 7f18fbfff640 1 -- 172.21.15.99:0/4242229609 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f190c0b4040 con 0x56073a63dc60 2024-03-19T02:04:46.521 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.515+0000 7f18fbfff640 1 -- 172.21.15.99:0/4242229609 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f190c09c070 con 0x56073a63dc60 2024-03-19T02:04:46.521 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.515+0000 7f18fbfff640 1 -- 172.21.15.99:0/4242229609 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f190c09af60 con 0x56073a63dc60 2024-03-19T02:04:46.521 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.516+0000 7f191b835740 1 -- 172.21.15.99:0/4242229609 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56073a63dc60 msgr2=0x56073a639ad0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:04:46.521 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.516+0000 7f191b835740 1 --2- 172.21.15.99:0/4242229609 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56073a63dc60 0x56073a639ad0 secure :-1 s=READY pgs=790 cs=0 l=1 rev1=1 crypto rx=0x7f190c09b670 tx=0x7f190c0975b0 comp rx=0 tx=0).stop 2024-03-19T02:04:46.521 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.516+0000 7f191b835740 1 -- 172.21.15.99:0/4242229609 shutdown_connections 2024-03-19T02:04:46.522 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.516+0000 7f191b835740 1 --2- 172.21.15.99:0/4242229609 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56073a63dc60 0x56073a639ad0 unknown :-1 s=CLOSED pgs=790 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:46.522 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.516+0000 7f191b835740 1 --2- 172.21.15.99:0/4242229609 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56073a638af0 0x56073a6495c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:46.522 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.516+0000 7f191b835740 1 --2- 172.21.15.99:0/4242229609 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56073a638100 0x56073a639590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:46.522 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.516+0000 7f191b835740 1 -- 172.21.15.99:0/4242229609 >> 172.21.15.99:0/4242229609 conn(0x56073a633670 msgr2=0x56073a642080 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:04:46.522 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.516+0000 7f191b835740 1 -- 172.21.15.99:0/4242229609 shutdown_connections 2024-03-19T02:04:46.522 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.516+0000 7f191b835740 1 -- 172.21.15.99:0/4242229609 wait complete. 2024-03-19T02:04:46.522 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.516+0000 7f191b835740 1 -- start start 2024-03-19T02:04:46.522 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.516+0000 7f191b835740 1 Processor -- start 2024-03-19T02:04:46.522 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.517+0000 7f191b835740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56073a638100 0x56073a647030 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:46.522 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.517+0000 7f191b835740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56073a638af0 0x56073a647570 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:46.523 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.517+0000 7f191b835740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56073a63dc60 0x56073a648580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:46.523 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.517+0000 7f191b835740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f19141a7440 con 0x56073a638100 2024-03-19T02:04:46.523 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.517+0000 7f191b835740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f19141a88b0 con 0x56073a638af0 2024-03-19T02:04:46.523 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.517+0000 7f191b835740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f19141a7b10 con 0x56073a63dc60 2024-03-19T02:04:46.524 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.517+0000 7f191b835740 10 client.0 _ll_register_callbacks cb 0x7f18fc009c00 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T02:04:46.524 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.517+0000 7f191259c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56073a638100 0x56073a647030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:46.525 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.517+0000 7f191259c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56073a638100 0x56073a647030 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:32852/0 (socket says 172.21.15.99:32852) 2024-03-19T02:04:46.525 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.517+0000 7f191259c640 1 -- 172.21.15.99:0/2859683208 learned_addr learned my addr 172.21.15.99:0/2859683208 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:04:46.525 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.517+0000 7f190bfff640 1 --2- 172.21.15.99:0/2859683208 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56073a638af0 0x56073a647570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:46.525 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.517+0000 7f1912d9d640 1 --2- 172.21.15.99:0/2859683208 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56073a63dc60 0x56073a648580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:46.525 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.517+0000 7f191259c640 1 -- 172.21.15.99:0/2859683208 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56073a63dc60 msgr2=0x56073a648580 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:04:46.525 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.517+0000 7f191259c640 1 --2- 172.21.15.99:0/2859683208 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56073a63dc60 0x56073a648580 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:46.525 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.517+0000 7f191259c640 1 -- 172.21.15.99:0/2859683208 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56073a638af0 msgr2=0x56073a647570 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:04:46.525 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.517+0000 7f191259c640 1 --2- 172.21.15.99:0/2859683208 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56073a638af0 0x56073a647570 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:46.525 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.517+0000 7f191259c640 1 -- 172.21.15.99:0/2859683208 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x56073a648b70 con 0x56073a638100 2024-03-19T02:04:46.525 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.517+0000 7f191259c640 1 --2- 172.21.15.99:0/2859683208 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56073a638100 0x56073a647030 secure :-1 s=READY pgs=791 cs=0 l=1 rev1=1 crypto rx=0x7f1904005ec0 tx=0x7f190403e000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5f16ee45b8901a5f server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:04:46.525 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.517+0000 7f18f9ffb640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T02:04:46.525 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.517+0000 7f18f9ffb640 1 -- 172.21.15.99:0/2859683208 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1904004030 con 0x56073a638100 2024-03-19T02:04:46.526 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.518+0000 7f191b835740 1 -- 172.21.15.99:0/2859683208 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f190c09c070 con 0x56073a638100 2024-03-19T02:04:46.526 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.518+0000 7f18f9ffb640 1 -- 172.21.15.99:0/2859683208 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f19040041d0 con 0x56073a638100 2024-03-19T02:04:46.526 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.518+0000 7f18f9ffb640 1 -- 172.21.15.99:0/2859683208 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=41) ==== 24+0+0 (secure 0 0 0) 0x7f1904062050 con 0x56073a638100 2024-03-19T02:04:46.526 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.518+0000 7f191b835740 10 client.17232 fetch_fsmap learned FSMap version 41 2024-03-19T02:04:46.526 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.518+0000 7f191b835740 1 -- 172.21.15.99:0/2859683208 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=41,monmap=4+}) -- 0x56073a6b41c0 con 0x56073a638100 2024-03-19T02:04:46.526 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.518+0000 7f18f9ffb640 1 -- 172.21.15.99:0/2859683208 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f190c09c070 con 0x56073a638100 2024-03-19T02:04:46.526 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.518+0000 7f18f9ffb640 1 -- 172.21.15.99:0/2859683208 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 41) ==== 8354+0+0 (secure 0 0 0) 0x7f19040046c0 con 0x56073a638100 2024-03-19T02:04:46.526 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.518+0000 7f191b835740 10 client.17232 fetch_fsmap finished waiting for FSMap version 41 2024-03-19T02:04:46.526 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.518+0000 7f191b835740 10 client.17232 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T02:04:46.526 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.518+0000 7f191b835740 20 client.17232 populate_metadata read hostname 'smithi099' 2024-03-19T02:04:46.526 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.518+0000 7f191b835740 1 --2- 172.21.15.99:0/2859683208 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56073a6b4ba0 0x56073a6b6fc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:04:46.526 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.518+0000 7f191b835740 4 client.17232 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T02:04:46.527 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.518+0000 7f191b835740 1 -- 172.21.15.99:0/2859683208 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x56073a5a4470 con 0x56073a6b4ba0 2024-03-19T02:04:46.527 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.518+0000 7f190bfff640 1 --2- 172.21.15.99:0/2859683208 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56073a6b4ba0 0x56073a6b6fc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:04:46.527 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.519+0000 7f18f9ffb640 10 client.17232 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T02:04:46.527 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:46.519+0000 7f190bfff640 1 --2- 172.21.15.99:0/2859683208 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56073a6b4ba0 0x56073a6b6fc0 crc :-1 s=READY pgs=240 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=85e306aa600b4004 server_cookie=5d7ad92362a1553f in_seq=0 out_seq=0 2024-03-19T02:04:47.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:04:46 smithi115 ceph-mon[67332]: pgmap v1061: 97 pgs: 97 active+clean; 651 MiB data, 9.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.3 MiB/s rd, 5.3 MiB/s wr, 488 op/s 2024-03-19T02:04:47.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:04:46 smithi099 ceph-mon[62951]: pgmap v1061: 97 pgs: 97 active+clean; 651 MiB data, 9.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.3 MiB/s rd, 5.3 MiB/s wr, 488 op/s 2024-03-19T02:04:47.097 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:47.097+0000 7f18f9ffb640 1 -- 172.21.15.99:0/2859683208 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x56073a5a4470 con 0x56073a6b4ba0 2024-03-19T02:04:47.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:47.097+0000 7f18f9ffb640 10 client.17232 handle_command_reply: tid=0 2024-03-19T02:04:47.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:47.097+0000 7f191b835740 1 client.17232 shutdown 2024-03-19T02:04:47.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:47.097+0000 7f191b835740 1 -- 172.21.15.99:0/2859683208 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56073a638100 msgr2=0x56073a647030 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:04:47.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:47.097+0000 7f191b835740 1 --2- 172.21.15.99:0/2859683208 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56073a638100 0x56073a647030 secure :-1 s=READY pgs=791 cs=0 l=1 rev1=1 crypto rx=0x7f1904005ec0 tx=0x7f190403e000 comp rx=0 tx=0).stop 2024-03-19T02:04:47.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:47.097+0000 7f191b835740 1 -- 172.21.15.99:0/2859683208 shutdown_connections 2024-03-19T02:04:47.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:47.097+0000 7f191b835740 1 --2- 172.21.15.99:0/2859683208 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x56073a6b4ba0 0x56073a6b6fc0 crc :-1 s=READY pgs=240 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:47.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:47.097+0000 7f191b835740 1 --2- 172.21.15.99:0/2859683208 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x56073a638af0 0x56073a647570 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:47.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:47.097+0000 7f18f9ffb640 0 client.17232 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T02:04:47.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:47.097+0000 7f191b835740 1 --2- 172.21.15.99:0/2859683208 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x56073a63dc60 0x56073a648580 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:47.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:47.097+0000 7f191b835740 1 --2- 172.21.15.99:0/2859683208 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x56073a638100 0x56073a647030 unknown :-1 s=CLOSED pgs=791 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:47.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:47.097+0000 7f191b835740 1 -- 172.21.15.99:0/2859683208 >> 172.21.15.99:0/2859683208 conn(0x56073a633670 msgr2=0x56073a6368d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:04:47.098 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:47.097+0000 7f191b835740 1 -- 172.21.15.99:0/2859683208 shutdown_connections 2024-03-19T02:04:47.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:47.097+0000 7f191b835740 1 -- 172.21.15.99:0/2859683208 wait complete. 2024-03-19T02:04:47.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:47.097+0000 7f191b835740 20 client.17232 trim_cache size 0 max 16384 2024-03-19T02:04:47.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T02:04:47.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T02:04:47.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T02:04:47.099 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T02:04:47.100 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:47.098+0000 7f1918c4b640 1 -- 172.21.15.99:0/2135570197 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f19000a28a0 msgr2=0x7f19000a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:04:47.100 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:47.098+0000 7f1918c4b640 1 --2- 172.21.15.99:0/2135570197 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f19000a28a0 0x7f19000a4d60 secure :-1 s=READY pgs=391 cs=0 l=1 rev1=1 crypto rx=0x7f190402c7b0 tx=0x7f1904005fd0 comp rx=0 tx=0).stop 2024-03-19T02:04:47.100 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:47.098+0000 7f1918c4b640 1 -- 172.21.15.99:0/2135570197 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f19141075a0 msgr2=0x7f191419f0a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:04:47.100 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:47.098+0000 7f1918c4b640 1 --2- 172.21.15.99:0/2135570197 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f19141075a0 0x7f191419f0a0 secure :-1 s=READY pgs=788 cs=0 l=1 rev1=1 crypto rx=0x7f190c00eb10 tx=0x7f190c00c2d0 comp rx=0 tx=0).stop 2024-03-19T02:04:47.100 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:47.098+0000 7f1918c4b640 1 -- 172.21.15.99:0/2135570197 shutdown_connections 2024-03-19T02:04:47.100 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:47.098+0000 7f1918c4b640 1 --2- 172.21.15.99:0/2135570197 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f19000a28a0 0x7f19000a4d60 unknown :-1 s=CLOSED pgs=391 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:47.100 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:47.098+0000 7f1918c4b640 1 --2- 172.21.15.99:0/2135570197 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1914104c70 0x7f1914198030 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:47.101 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:47.098+0000 7f1918c4b640 1 --2- 172.21.15.99:0/2135570197 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f19141075a0 0x7f191419f0a0 unknown :-1 s=CLOSED pgs=788 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:47.101 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:47.098+0000 7f1918c4b640 1 --2- 172.21.15.99:0/2135570197 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1914102340 0x7f1914197af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:04:47.101 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:47.098+0000 7f1918c4b640 1 -- 172.21.15.99:0/2135570197 >> 172.21.15.99:0/2135570197 conn(0x7f19140fbcb0 msgr2=0x7f1914102c80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:04:47.101 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:47.098+0000 7f1918c4b640 1 -- 172.21.15.99:0/2135570197 shutdown_connections 2024-03-19T02:04:47.101 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:04:47.098+0000 7f1918c4b640 1 -- 172.21.15.99:0/2135570197 wait complete. 2024-03-19T02:04:47.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:04:46 smithi112 ceph-mon[66819]: pgmap v1061: 97 pgs: 97 active+clean; 651 MiB data, 9.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.3 MiB/s rd, 5.3 MiB/s wr, 488 op/s 2024-03-19T02:04:47.121 INFO:tasks.workunit.client.0.smithi099.stdout: 10 282398 22.12 MB/sec execute 556 sec latency 473.138 ms 2024-03-19T02:04:48.121 INFO:tasks.workunit.client.0.smithi099.stdout: 10 282789 22.10 MB/sec execute 557 sec latency 301.737 ms 2024-03-19T02:04:49.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:04:48 smithi115 ceph-mon[67332]: pgmap v1062: 97 pgs: 97 active+clean; 657 MiB data, 9.3 GiB used, 1.0 TiB / 1.0 TiB avail; 5.2 MiB/s rd, 6.5 MiB/s wr, 558 op/s 2024-03-19T02:04:49.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:04:48 smithi099 ceph-mon[62951]: pgmap v1062: 97 pgs: 97 active+clean; 657 MiB data, 9.3 GiB used, 1.0 TiB / 1.0 TiB avail; 5.2 MiB/s rd, 6.5 MiB/s wr, 558 op/s 2024-03-19T02:04:49.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:04:48 smithi112 ceph-mon[66819]: pgmap v1062: 97 pgs: 97 active+clean; 657 MiB data, 9.3 GiB used, 1.0 TiB / 1.0 TiB avail; 5.2 MiB/s rd, 6.5 MiB/s wr, 558 op/s 2024-03-19T02:04:49.121 INFO:tasks.workunit.client.0.smithi099.stdout: 10 283147 22.10 MB/sec execute 558 sec latency 313.042 ms 2024-03-19T02:04:50.122 INFO:tasks.workunit.client.0.smithi099.stdout: 10 283620 22.12 MB/sec execute 559 sec latency 152.424 ms 2024-03-19T02:04:51.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:04:50 smithi115 ceph-mon[67332]: pgmap v1063: 97 pgs: 97 active+clean; 653 MiB data, 9.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.6 MiB/s rd, 5.8 MiB/s wr, 480 op/s 2024-03-19T02:04:51.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:04:50 smithi099 ceph-mon[62951]: pgmap v1063: 97 pgs: 97 active+clean; 653 MiB data, 9.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.6 MiB/s rd, 5.8 MiB/s wr, 480 op/s 2024-03-19T02:04:51.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:04:50 smithi112 ceph-mon[66819]: pgmap v1063: 97 pgs: 97 active+clean; 653 MiB data, 9.3 GiB used, 1.0 TiB / 1.0 TiB avail; 4.6 MiB/s rd, 5.8 MiB/s wr, 480 op/s 2024-03-19T02:04:51.121 INFO:tasks.workunit.client.0.smithi099.stdout: 10 284102 22.13 MB/sec execute 560 sec latency 71.720 ms 2024-03-19T02:04:52.122 INFO:tasks.workunit.client.0.smithi099.stdout: 10 284501 22.12 MB/sec execute 561 sec latency 77.371 ms 2024-03-19T02:04:53.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:04:52 smithi115 ceph-mon[67332]: pgmap v1064: 97 pgs: 97 active+clean; 654 MiB data, 9.3 GiB used, 1.0 TiB / 1.0 TiB avail; 3.9 MiB/s rd, 5.7 MiB/s wr, 500 op/s 2024-03-19T02:04:53.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:04:52 smithi099 ceph-mon[62951]: pgmap v1064: 97 pgs: 97 active+clean; 654 MiB data, 9.3 GiB used, 1.0 TiB / 1.0 TiB avail; 3.9 MiB/s rd, 5.7 MiB/s wr, 500 op/s 2024-03-19T02:04:53.114 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:04:52 smithi112 ceph-mon[66819]: pgmap v1064: 97 pgs: 97 active+clean; 654 MiB data, 9.3 GiB used, 1.0 TiB / 1.0 TiB avail; 3.9 MiB/s rd, 5.7 MiB/s wr, 500 op/s 2024-03-19T02:04:53.122 INFO:tasks.workunit.client.0.smithi099.stdout: 10 284928 22.13 MB/sec execute 562 sec latency 182.840 ms 2024-03-19T02:04:54.122 INFO:tasks.workunit.client.0.smithi099.stdout: 10 285291 22.12 MB/sec execute 563 sec latency 197.046 ms 2024-03-19T02:04:55.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:04:54 smithi115 ceph-mon[67332]: pgmap v1065: 97 pgs: 97 active+clean; 652 MiB data, 9.4 GiB used, 1.0 TiB / 1.0 TiB avail; 4.3 MiB/s rd, 6.6 MiB/s wr, 579 op/s 2024-03-19T02:04:55.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:04:54 smithi099 ceph-mon[62951]: pgmap v1065: 97 pgs: 97 active+clean; 652 MiB data, 9.4 GiB used, 1.0 TiB / 1.0 TiB avail; 4.3 MiB/s rd, 6.6 MiB/s wr, 579 op/s 2024-03-19T02:04:55.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:04:54 smithi112 ceph-mon[66819]: pgmap v1065: 97 pgs: 97 active+clean; 652 MiB data, 9.4 GiB used, 1.0 TiB / 1.0 TiB avail; 4.3 MiB/s rd, 6.6 MiB/s wr, 579 op/s 2024-03-19T02:04:55.122 INFO:tasks.workunit.client.0.smithi099.stdout: 10 285760 22.14 MB/sec execute 564 sec latency 86.618 ms 2024-03-19T02:04:56.122 INFO:tasks.workunit.client.0.smithi099.stdout: 10 286132 22.13 MB/sec execute 565 sec latency 73.204 ms 2024-03-19T02:04:57.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:04:56 smithi115 ceph-mon[67332]: pgmap v1066: 97 pgs: 97 active+clean; 652 MiB data, 9.4 GiB used, 1.0 TiB / 1.0 TiB avail; 3.3 MiB/s rd, 5.2 MiB/s wr, 489 op/s 2024-03-19T02:04:57.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:04:56 smithi099 ceph-mon[62951]: pgmap v1066: 97 pgs: 97 active+clean; 652 MiB data, 9.4 GiB used, 1.0 TiB / 1.0 TiB avail; 3.3 MiB/s rd, 5.2 MiB/s wr, 489 op/s 2024-03-19T02:04:57.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:04:56 smithi112 ceph-mon[66819]: pgmap v1066: 97 pgs: 97 active+clean; 652 MiB data, 9.4 GiB used, 1.0 TiB / 1.0 TiB avail; 3.3 MiB/s rd, 5.2 MiB/s wr, 489 op/s 2024-03-19T02:04:57.122 INFO:tasks.workunit.client.0.smithi099.stdout: 10 286544 22.12 MB/sec execute 566 sec latency 122.917 ms 2024-03-19T02:04:58.122 INFO:tasks.workunit.client.0.smithi099.stdout: 10 287001 22.12 MB/sec execute 567 sec latency 81.857 ms 2024-03-19T02:04:59.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:04:58 smithi099 ceph-mon[62951]: pgmap v1067: 97 pgs: 97 active+clean; 655 MiB data, 9.4 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 5.9 MiB/s wr, 556 op/s 2024-03-19T02:04:59.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:04:58 smithi112 ceph-mon[66819]: pgmap v1067: 97 pgs: 97 active+clean; 655 MiB data, 9.4 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 5.9 MiB/s wr, 556 op/s 2024-03-19T02:04:59.122 INFO:tasks.workunit.client.0.smithi099.stdout: 10 287352 22.12 MB/sec execute 568 sec latency 317.252 ms 2024-03-19T02:04:59.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:04:58 smithi115 ceph-mon[67332]: pgmap v1067: 97 pgs: 97 active+clean; 655 MiB data, 9.4 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 MiB/s rd, 5.9 MiB/s wr, 556 op/s 2024-03-19T02:05:00.122 INFO:tasks.workunit.client.0.smithi099.stdout: 10 287770 22.11 MB/sec execute 569 sec latency 169.255 ms 2024-03-19T02:05:00.200 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.198+0000 7fb36c8ea640 1 -- 172.21.15.99:0/2874572349 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb368101870 msgr2=0x7fb368101c90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:05:00.200 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.198+0000 7fb36c8ea640 1 --2- 172.21.15.99:0/2874572349 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb368101870 0x7fb368101c90 secure :-1 s=READY pgs=792 cs=0 l=1 rev1=1 crypto rx=0x7fb358009900 tx=0x7fb35802bc30 comp rx=0 tx=0).stop 2024-03-19T02:05:00.200 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.198+0000 7fb36c8ea640 1 -- 172.21.15.99:0/2874572349 shutdown_connections 2024-03-19T02:05:00.200 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.198+0000 7fb36c8ea640 1 --2- 172.21.15.99:0/2874572349 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb368103420 0x7fb368109cb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:00.200 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.198+0000 7fb36c8ea640 1 --2- 172.21.15.99:0/2874572349 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb368102a60 0x7fb368102ee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:00.200 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.198+0000 7fb36c8ea640 1 --2- 172.21.15.99:0/2874572349 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb368101870 0x7fb368101c90 unknown :-1 s=CLOSED pgs=792 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:00.200 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.198+0000 7fb36c8ea640 1 -- 172.21.15.99:0/2874572349 >> 172.21.15.99:0/2874572349 conn(0x7fb3680fcde0 msgr2=0x7fb3680ff220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:05:00.200 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.198+0000 7fb36c8ea640 1 -- 172.21.15.99:0/2874572349 shutdown_connections 2024-03-19T02:05:00.200 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.198+0000 7fb36c8ea640 1 -- 172.21.15.99:0/2874572349 wait complete. 2024-03-19T02:05:00.200 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.198+0000 7fb36c8ea640 1 Processor -- start 2024-03-19T02:05:00.201 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.198+0000 7fb36c8ea640 1 -- start start 2024-03-19T02:05:00.201 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.199+0000 7fb36c8ea640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb368101870 0x7fb368197b00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:00.201 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.199+0000 7fb36c8ea640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb368102a60 0x7fb368198040 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:00.201 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.199+0000 7fb36c8ea640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb368103420 0x7fb36819f0b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:00.201 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.199+0000 7fb36c8ea640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fb36810d4e0 con 0x7fb368103420 2024-03-19T02:05:00.201 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.199+0000 7fb36c8ea640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fb36810d360 con 0x7fb368102a60 2024-03-19T02:05:00.201 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.199+0000 7fb36c8ea640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fb36810d660 con 0x7fb368101870 2024-03-19T02:05:00.201 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.199+0000 7fb366d9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb368103420 0x7fb36819f0b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:00.201 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.199+0000 7fb365d9b640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb368102a60 0x7fb368198040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:00.201 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.199+0000 7fb36659c640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb368101870 0x7fb368197b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:00.201 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.199+0000 7fb366d9d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb368103420 0x7fb36819f0b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:50140/0 (socket says 172.21.15.99:50140) 2024-03-19T02:05:00.201 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.199+0000 7fb366d9d640 1 -- 172.21.15.99:0/1347374105 learned_addr learned my addr 172.21.15.99:0/1347374105 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:05:00.202 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.199+0000 7fb366d9d640 1 -- 172.21.15.99:0/1347374105 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb368101870 msgr2=0x7fb368197b00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:05:00.202 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.199+0000 7fb366d9d640 1 --2- 172.21.15.99:0/1347374105 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb368101870 0x7fb368197b00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:00.202 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.199+0000 7fb366d9d640 1 -- 172.21.15.99:0/1347374105 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb368102a60 msgr2=0x7fb368198040 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:05:00.202 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.199+0000 7fb366d9d640 1 --2- 172.21.15.99:0/1347374105 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb368102a60 0x7fb368198040 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:00.202 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.199+0000 7fb366d9d640 1 -- 172.21.15.99:0/1347374105 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb36819f720 con 0x7fb368103420 2024-03-19T02:05:00.202 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.199+0000 7fb365d9b640 1 --2- 172.21.15.99:0/1347374105 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb368102a60 0x7fb368198040 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:05:00.202 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.199+0000 7fb366d9d640 1 --2- 172.21.15.99:0/1347374105 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb368103420 0x7fb36819f0b0 secure :-1 s=READY pgs=793 cs=0 l=1 rev1=1 crypto rx=0x7fb36000eb10 tx=0x7fb36000c2d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:05:00.202 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.199+0000 7fb3577fe640 1 -- 172.21.15.99:0/1347374105 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb360004b20 con 0x7fb368103420 2024-03-19T02:05:00.202 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.199+0000 7fb36c8ea640 1 -- 172.21.15.99:0/1347374105 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb36819fa10 con 0x7fb368103420 2024-03-19T02:05:00.202 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.199+0000 7fb3577fe640 1 -- 172.21.15.99:0/1347374105 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fb360004cc0 con 0x7fb368103420 2024-03-19T02:05:00.202 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.199+0000 7fb3577fe640 1 -- 172.21.15.99:0/1347374105 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb360010760 con 0x7fb368103420 2024-03-19T02:05:00.202 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.199+0000 7fb36c8ea640 1 -- 172.21.15.99:0/1347374105 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fb36819ff60 con 0x7fb368103420 2024-03-19T02:05:00.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.202+0000 7fb3577fe640 1 -- 172.21.15.99:0/1347374105 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7fb360010900 con 0x7fb368103420 2024-03-19T02:05:00.209 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.202+0000 7fb36f5b3740 1 Processor -- start 2024-03-19T02:05:00.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.202+0000 7fb36f5b3740 1 -- start start 2024-03-19T02:05:00.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.202+0000 7fb36f5b3740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a1a4528100 0x55a1a4528520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:00.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.202+0000 7fb36f5b3740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a1a4528af0 0x55a1a452d5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:00.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.202+0000 7fb36f5b3740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a1a452dc60 0x55a1a45300f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:00.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.202+0000 7fb36f5b3740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55a1a448cbe0 con 0x55a1a4528af0 2024-03-19T02:05:00.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.202+0000 7fb36f5b3740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55a1a4435960 con 0x55a1a4528100 2024-03-19T02:05:00.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.202+0000 7fb36f5b3740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55a1a4488c80 con 0x55a1a452dc60 2024-03-19T02:05:00.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.202+0000 7fb36659c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a1a4528af0 0x55a1a452d5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:00.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.203+0000 7fb365d9b640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a1a4528100 0x55a1a4528520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:00.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.203+0000 7fb36659c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a1a4528af0 0x55a1a452d5f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:50146/0 (socket says 172.21.15.99:50146) 2024-03-19T02:05:00.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.203+0000 7fb36659c640 1 -- 172.21.15.99:0/1840319061 learned_addr learned my addr 172.21.15.99:0/1840319061 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:05:00.210 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.203+0000 7fb365d9b640 1 --2- 172.21.15.99:0/1840319061 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a1a452dc60 0x55a1a45300f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:00.211 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.203+0000 7fb3577fe640 1 --2- 172.21.15.99:0/1347374105 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb3500a28a0 0x7fb3500a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:00.211 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.203+0000 7fb365d9b640 1 -- 172.21.15.99:0/1840319061 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a1a452dc60 msgr2=0x55a1a45300f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:05:00.211 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.203+0000 7fb365d9b640 1 --2- 172.21.15.99:0/1840319061 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a1a452dc60 0x55a1a45300f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:00.211 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.203+0000 7fb365d9b640 1 -- 172.21.15.99:0/1840319061 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a1a4528af0 msgr2=0x55a1a452d5f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:05:00.211 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.203+0000 7fb365d9b640 1 --2- 172.21.15.99:0/1840319061 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a1a4528af0 0x55a1a452d5f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:00.211 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.203+0000 7fb365d9b640 1 -- 172.21.15.99:0/1840319061 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55a1a442cd20 con 0x55a1a4528100 2024-03-19T02:05:00.211 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.203+0000 7fb36659c640 1 --2- 172.21.15.99:0/1840319061 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a1a4528af0 0x55a1a452d5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:05:00.211 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.203+0000 7fb366d9d640 1 --2- 172.21.15.99:0/1347374105 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb3500a28a0 0x7fb3500a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:00.211 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.203+0000 7fb3577fe640 1 -- 172.21.15.99:0/1347374105 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fb3600cf060 con 0x7fb368103420 2024-03-19T02:05:00.211 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.203+0000 7fb365d9b640 1 --2- 172.21.15.99:0/1840319061 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a1a4528100 0x55a1a4528520 secure :-1 s=READY pgs=231 cs=0 l=1 rev1=1 crypto rx=0x7fb368199010 tx=0x7fb35c005850 comp rx=0 tx=0).ready entity=mon.2 client_cookie=323aed0e2c5766de server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:05:00.211 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.203+0000 7fb354ff9640 1 -- 172.21.15.99:0/1840319061 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb35c011050 con 0x55a1a4528100 2024-03-19T02:05:00.211 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.203+0000 7fb366d9d640 1 --2- 172.21.15.99:0/1347374105 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb3500a28a0 0x7fb3500a4d60 secure :-1 s=READY pgs=393 cs=0 l=1 rev1=1 crypto rx=0x7fb360098390 tx=0x7fb360066c20 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:05:00.212 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.203+0000 7fb354ff9640 1 -- 172.21.15.99:0/1840319061 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fb35c00fd60 con 0x55a1a4528100 2024-03-19T02:05:00.212 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.203+0000 7fb354ff9640 1 -- 172.21.15.99:0/1840319061 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb35c0047a0 con 0x55a1a4528100 2024-03-19T02:05:00.212 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.204+0000 7fb36f5b3740 1 -- 172.21.15.99:0/1840319061 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a1a4528100 msgr2=0x55a1a4528520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:05:00.212 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.204+0000 7fb36f5b3740 1 --2- 172.21.15.99:0/1840319061 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a1a4528100 0x55a1a4528520 secure :-1 s=READY pgs=231 cs=0 l=1 rev1=1 crypto rx=0x7fb368199010 tx=0x7fb35c005850 comp rx=0 tx=0).stop 2024-03-19T02:05:00.212 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.204+0000 7fb36f5b3740 1 -- 172.21.15.99:0/1840319061 shutdown_connections 2024-03-19T02:05:00.212 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.204+0000 7fb36f5b3740 1 --2- 172.21.15.99:0/1840319061 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a1a452dc60 0x55a1a45300f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:00.212 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.204+0000 7fb36f5b3740 1 --2- 172.21.15.99:0/1840319061 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a1a4528af0 0x55a1a452d5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:00.212 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.204+0000 7fb36f5b3740 1 --2- 172.21.15.99:0/1840319061 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a1a4528100 0x55a1a4528520 secure :-1 s=CLOSED pgs=231 cs=0 l=1 rev1=1 crypto rx=0x7fb368199010 tx=0x7fb35c005850 comp rx=0 tx=0).stop 2024-03-19T02:05:00.212 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.204+0000 7fb36f5b3740 1 -- 172.21.15.99:0/1840319061 >> 172.21.15.99:0/1840319061 conn(0x55a1a4523670 msgr2=0x55a1a4523a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:05:00.212 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.204+0000 7fb36f5b3740 1 -- 172.21.15.99:0/1840319061 shutdown_connections 2024-03-19T02:05:00.212 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.204+0000 7fb36f5b3740 1 -- 172.21.15.99:0/1840319061 wait complete. 2024-03-19T02:05:00.212 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.205+0000 7fb36f5b3740 1 -- start start 2024-03-19T02:05:00.212 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.205+0000 7fb36f5b3740 1 Processor -- start 2024-03-19T02:05:00.213 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.205+0000 7fb36f5b3740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a1a4528af0 0x55a1a45322e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:00.213 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.205+0000 7fb36f5b3740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a1a452dc60 0x55a1a4532820 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:00.213 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.205+0000 7fb36f5b3740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a1a4537850 0x55a1a4539c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:00.213 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.205+0000 7fb36f5b3740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55a1a4530c10 con 0x55a1a4537850 2024-03-19T02:05:00.213 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.205+0000 7fb36f5b3740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55a1a4448db0 con 0x55a1a4528af0 2024-03-19T02:05:00.213 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.205+0000 7fb36f5b3740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55a1a4530d90 con 0x55a1a452dc60 2024-03-19T02:05:00.213 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.205+0000 7fb36f5b3740 10 client.0 _ll_register_callbacks cb 0x55a1a444de70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T02:05:00.213 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.205+0000 7fb365d9b640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a1a4528af0 0x55a1a45322e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:00.213 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.205+0000 7fb36659c640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a1a452dc60 0x55a1a4532820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:00.213 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.205+0000 7fb365d9b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a1a4537850 0x55a1a4539c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:00.213 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.205+0000 7fb36659c640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a1a452dc60 0x55a1a4532820 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:57132/0 (socket says 172.21.15.99:57132) 2024-03-19T02:05:00.213 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.205+0000 7fb36659c640 1 -- 172.21.15.99:0/3783329407 learned_addr learned my addr 172.21.15.99:0/3783329407 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:05:00.214 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.205+0000 7fb36659c640 1 -- 172.21.15.99:0/3783329407 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a1a4528af0 msgr2=0x55a1a45322e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:05:00.214 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.205+0000 7fb36659c640 1 --2- 172.21.15.99:0/3783329407 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a1a4528af0 0x55a1a45322e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:00.214 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.205+0000 7fb36659c640 1 -- 172.21.15.99:0/3783329407 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a1a4537850 msgr2=0x55a1a4539c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:05:00.214 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.205+0000 7fb36659c640 1 --2- 172.21.15.99:0/3783329407 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a1a4537850 0x55a1a4539c40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:00.214 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.205+0000 7fb36659c640 1 -- 172.21.15.99:0/3783329407 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55a1a4447c60 con 0x55a1a452dc60 2024-03-19T02:05:00.214 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.206+0000 7fb36659c640 1 --2- 172.21.15.99:0/3783329407 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a1a452dc60 0x55a1a4532820 secure :-1 s=READY pgs=133 cs=0 l=1 rev1=1 crypto rx=0x7fb35802cfd0 tx=0x7fb358004270 comp rx=0 tx=0).ready entity=mon.1 client_cookie=690dc31668d09c02 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:05:00.214 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.206+0000 7fb34dffb640 10 client.0 ms_handle_connect on v2:172.21.15.115:3300/0 2024-03-19T02:05:00.214 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.206+0000 7fb34dffb640 1 -- 172.21.15.99:0/3783329407 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb358005d50 con 0x55a1a452dc60 2024-03-19T02:05:00.214 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.206+0000 7fb36f5b3740 1 -- 172.21.15.99:0/3783329407 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fb35c00fd60 con 0x55a1a452dc60 2024-03-19T02:05:00.214 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.206+0000 7fb34dffb640 1 -- 172.21.15.99:0/3783329407 <== mon.1 v2:172.21.15.115:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fb358005220 con 0x55a1a452dc60 2024-03-19T02:05:00.214 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.206+0000 7fb34dffb640 1 -- 172.21.15.99:0/3783329407 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_get_version_reply(handle=1 version=41) ==== 24+0+0 (secure 0 0 0) 0x7fb358005420 con 0x55a1a452dc60 2024-03-19T02:05:00.214 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.206+0000 7fb36f5b3740 10 client.27010 fetch_fsmap learned FSMap version 41 2024-03-19T02:05:00.215 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.206+0000 7fb36f5b3740 1 -- 172.21.15.99:0/3783329407 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,fsmap=41,monmap=4+}) -- 0x55a1a41a1210 con 0x55a1a452dc60 2024-03-19T02:05:00.215 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.206+0000 7fb34dffb640 1 -- 172.21.15.99:0/3783329407 <== mon.1 v2:172.21.15.115:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fb35c00fd60 con 0x55a1a452dc60 2024-03-19T02:05:00.215 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.206+0000 7fb34dffb640 1 -- 172.21.15.99:0/3783329407 <== mon.1 v2:172.21.15.115:3300/0 5 ==== fsmap(e 41) ==== 8354+0+0 (secure 0 0 0) 0x7fb358003710 con 0x55a1a452dc60 2024-03-19T02:05:00.215 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.207+0000 7fb36f5b3740 10 client.27010 fetch_fsmap finished waiting for FSMap version 41 2024-03-19T02:05:00.215 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.207+0000 7fb36f5b3740 10 client.27010 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T02:05:00.215 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.207+0000 7fb36f5b3740 20 client.27010 populate_metadata read hostname 'smithi099' 2024-03-19T02:05:00.215 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.207+0000 7fb36f5b3740 1 --2- 172.21.15.99:0/3783329407 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55a1a453e570 0x55a1a4540960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:00.215 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.207+0000 7fb36f5b3740 4 client.27010 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T02:05:00.215 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.207+0000 7fb36f5b3740 1 -- 172.21.15.99:0/3783329407 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55a1a449a120 con 0x55a1a453e570 2024-03-19T02:05:00.215 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.207+0000 7fb365d9b640 1 --2- 172.21.15.99:0/3783329407 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55a1a453e570 0x55a1a4540960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:00.215 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.207+0000 7fb34dffb640 10 client.27010 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T02:05:00.215 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.207+0000 7fb365d9b640 1 --2- 172.21.15.99:0/3783329407 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55a1a453e570 0x55a1a4540960 crc :-1 s=READY pgs=241 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=a84dd197f8c217c4 server_cookie=19725f2ce1c52e1 in_seq=0 out_seq=0 2024-03-19T02:05:00.215 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.209+0000 7fb34dffb640 1 -- 172.21.15.99:0/3783329407 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x55a1a449a120 con 0x55a1a453e570 2024-03-19T02:05:00.216 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.209+0000 7fb34dffb640 10 client.27010 handle_command_reply: tid=0 2024-03-19T02:05:00.216 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.209+0000 7fb36f5b3740 1 client.27010 shutdown 2024-03-19T02:05:00.216 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.209+0000 7fb36f5b3740 1 -- 172.21.15.99:0/3783329407 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a1a452dc60 msgr2=0x55a1a4532820 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:05:00.216 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.209+0000 7fb36f5b3740 1 --2- 172.21.15.99:0/3783329407 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a1a452dc60 0x55a1a4532820 secure :-1 s=READY pgs=133 cs=0 l=1 rev1=1 crypto rx=0x7fb35802cfd0 tx=0x7fb358004270 comp rx=0 tx=0).stop 2024-03-19T02:05:00.216 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.209+0000 7fb36f5b3740 1 -- 172.21.15.99:0/3783329407 shutdown_connections 2024-03-19T02:05:00.216 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.209+0000 7fb36f5b3740 1 --2- 172.21.15.99:0/3783329407 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55a1a453e570 0x55a1a4540960 crc :-1 s=READY pgs=241 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:00.216 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.209+0000 7fb36f5b3740 1 --2- 172.21.15.99:0/3783329407 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a1a4537850 0x55a1a4539c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:00.216 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.209+0000 7fb36f5b3740 1 --2- 172.21.15.99:0/3783329407 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a1a452dc60 0x55a1a4532820 unknown :-1 s=CLOSED pgs=133 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:00.216 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.209+0000 7fb34dffb640 0 client.27010 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T02:05:00.216 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.209+0000 7fb36f5b3740 1 --2- 172.21.15.99:0/3783329407 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a1a4528af0 0x55a1a45322e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:00.216 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.209+0000 7fb36f5b3740 1 -- 172.21.15.99:0/3783329407 >> 172.21.15.99:0/3783329407 conn(0x55a1a4523670 msgr2=0x55a1a45248c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:05:00.216 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.209+0000 7fb36f5b3740 1 -- 172.21.15.99:0/3783329407 shutdown_connections 2024-03-19T02:05:00.217 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.209+0000 7fb36f5b3740 1 -- 172.21.15.99:0/3783329407 wait complete. 2024-03-19T02:05:00.217 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.209+0000 7fb36f5b3740 20 client.27010 trim_cache size 0 max 16384 2024-03-19T02:05:00.233 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.227+0000 7fb36f5b3740 1 Processor -- start 2024-03-19T02:05:00.233 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.227+0000 7fb36f5b3740 1 -- start start 2024-03-19T02:05:00.233 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.228+0000 7fb36f5b3740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a1a4528af0 0x55a1a4522ab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:00.234 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.228+0000 7fb36f5b3740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a1a452dc60 0x55a1a4526fe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:00.234 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.228+0000 7fb36f5b3740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a1a4537850 0x55a1a4545220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:00.234 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.228+0000 7fb36f5b3740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fb3681ab700 con 0x55a1a452dc60 2024-03-19T02:05:00.234 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.228+0000 7fb36f5b3740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fb3681a8f70 con 0x55a1a4537850 2024-03-19T02:05:00.234 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.228+0000 7fb36f5b3740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fb3681a6dc0 con 0x55a1a4528af0 2024-03-19T02:05:00.234 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.228+0000 7fb36659c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a1a452dc60 0x55a1a4526fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:00.234 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.228+0000 7fb36659c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a1a452dc60 0x55a1a4526fe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:50156/0 (socket says 172.21.15.99:50156) 2024-03-19T02:05:00.235 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.228+0000 7fb36659c640 1 -- 172.21.15.99:0/3272244766 learned_addr learned my addr 172.21.15.99:0/3272244766 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:05:00.235 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.228+0000 7fb365d9b640 1 --2- 172.21.15.99:0/3272244766 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a1a4528af0 0x55a1a4522ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:00.235 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.228+0000 7fb365d9b640 1 --2- 172.21.15.99:0/3272244766 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a1a4537850 0x55a1a4545220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:00.235 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.228+0000 7fb36659c640 1 -- 172.21.15.99:0/3272244766 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a1a4528af0 msgr2=0x55a1a4522ab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:05:00.235 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.228+0000 7fb36659c640 1 --2- 172.21.15.99:0/3272244766 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a1a4528af0 0x55a1a4522ab0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:00.235 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.228+0000 7fb36659c640 1 -- 172.21.15.99:0/3272244766 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a1a4537850 msgr2=0x55a1a4545220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:05:00.235 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.228+0000 7fb36659c640 1 --2- 172.21.15.99:0/3272244766 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a1a4537850 0x55a1a4545220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:00.236 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.228+0000 7fb36659c640 1 -- 172.21.15.99:0/3272244766 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55a1a426d620 con 0x55a1a452dc60 2024-03-19T02:05:00.236 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.228+0000 7fb36659c640 1 --2- 172.21.15.99:0/3272244766 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a1a452dc60 0x55a1a4526fe0 secure :-1 s=READY pgs=794 cs=0 l=1 rev1=1 crypto rx=0x7fb358009850 tx=0x7fb3580045f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ea31adae9e40424d server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:05:00.236 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.228+0000 7fb34ffff640 1 -- 172.21.15.99:0/3272244766 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb358005d50 con 0x55a1a452dc60 2024-03-19T02:05:00.236 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.228+0000 7fb34ffff640 1 -- 172.21.15.99:0/3272244766 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fb358041070 con 0x55a1a452dc60 2024-03-19T02:05:00.236 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.228+0000 7fb34ffff640 1 -- 172.21.15.99:0/3272244766 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb358034840 con 0x55a1a452dc60 2024-03-19T02:05:00.236 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.229+0000 7fb36f5b3740 1 -- 172.21.15.99:0/3272244766 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a1a452dc60 msgr2=0x55a1a4526fe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:05:00.236 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.229+0000 7fb36f5b3740 1 --2- 172.21.15.99:0/3272244766 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a1a452dc60 0x55a1a4526fe0 secure :-1 s=READY pgs=794 cs=0 l=1 rev1=1 crypto rx=0x7fb358009850 tx=0x7fb3580045f0 comp rx=0 tx=0).stop 2024-03-19T02:05:00.236 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.229+0000 7fb36f5b3740 1 -- 172.21.15.99:0/3272244766 shutdown_connections 2024-03-19T02:05:00.236 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.229+0000 7fb36f5b3740 1 --2- 172.21.15.99:0/3272244766 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a1a4537850 0x55a1a4545220 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:00.236 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.229+0000 7fb36f5b3740 1 --2- 172.21.15.99:0/3272244766 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a1a452dc60 0x55a1a4526fe0 unknown :-1 s=CLOSED pgs=794 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:00.237 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.229+0000 7fb36f5b3740 1 --2- 172.21.15.99:0/3272244766 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a1a4528af0 0x55a1a4522ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:00.237 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.229+0000 7fb36f5b3740 1 -- 172.21.15.99:0/3272244766 >> 172.21.15.99:0/3272244766 conn(0x55a1a4523670 msgr2=0x55a1a45268d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:05:00.237 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.229+0000 7fb36f5b3740 1 -- 172.21.15.99:0/3272244766 shutdown_connections 2024-03-19T02:05:00.237 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.229+0000 7fb36f5b3740 1 -- 172.21.15.99:0/3272244766 wait complete. 2024-03-19T02:05:00.237 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.229+0000 7fb36f5b3740 1 -- start start 2024-03-19T02:05:00.237 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.230+0000 7fb36f5b3740 1 Processor -- start 2024-03-19T02:05:00.237 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.230+0000 7fb36f5b3740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a1a4528af0 0x55a1a454f9f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:00.237 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.230+0000 7fb36f5b3740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a1a452dc60 0x55a1a454ff30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:00.237 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.230+0000 7fb36f5b3740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a1a4537850 0x55a1a4550ef0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:00.237 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.230+0000 7fb36f5b3740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fb3681a7430 con 0x55a1a4528af0 2024-03-19T02:05:00.238 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.230+0000 7fb36f5b3740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fb3681a88a0 con 0x55a1a4537850 2024-03-19T02:05:00.238 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.230+0000 7fb36f5b3740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fb3681a7b00 con 0x55a1a452dc60 2024-03-19T02:05:00.238 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.230+0000 7fb36f5b3740 10 client.0 _ll_register_callbacks cb 0x7fb358009780 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T02:05:00.238 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.230+0000 7fb365d9b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a1a4528af0 0x55a1a454f9f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:00.238 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.230+0000 7fb365d9b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a1a4528af0 0x55a1a454f9f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:50158/0 (socket says 172.21.15.99:50158) 2024-03-19T02:05:00.238 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.230+0000 7fb365d9b640 1 -- 172.21.15.99:0/2122041333 learned_addr learned my addr 172.21.15.99:0/2122041333 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:05:00.238 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.230+0000 7fb36659c640 1 --2- 172.21.15.99:0/2122041333 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a1a452dc60 0x55a1a454ff30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:00.238 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.230+0000 7fb365d9b640 1 --2- 172.21.15.99:0/2122041333 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a1a4537850 0x55a1a4550ef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:00.238 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.230+0000 7fb365d9b640 1 -- 172.21.15.99:0/2122041333 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a1a452dc60 msgr2=0x55a1a454ff30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:05:00.239 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.230+0000 7fb365d9b640 1 --2- 172.21.15.99:0/2122041333 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a1a452dc60 0x55a1a454ff30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:00.239 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.230+0000 7fb365d9b640 1 -- 172.21.15.99:0/2122041333 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a1a4537850 msgr2=0x55a1a4550ef0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:05:00.239 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.230+0000 7fb365d9b640 1 --2- 172.21.15.99:0/2122041333 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a1a4537850 0x55a1a4550ef0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:00.239 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.230+0000 7fb365d9b640 1 -- 172.21.15.99:0/2122041333 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55a1a45514e0 con 0x55a1a4528af0 2024-03-19T02:05:00.239 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.230+0000 7fb365d9b640 1 --2- 172.21.15.99:0/2122041333 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a1a4528af0 0x55a1a454f9f0 secure :-1 s=READY pgs=795 cs=0 l=1 rev1=1 crypto rx=0x7fb35c011af0 tx=0x7fb35802cdd0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1b023048ed57ad75 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:05:00.239 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.230+0000 7fb34dffb640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T02:05:00.239 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.231+0000 7fb34dffb640 1 -- 172.21.15.99:0/2122041333 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb35c028040 con 0x55a1a4528af0 2024-03-19T02:05:00.239 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.231+0000 7fb34dffb640 1 -- 172.21.15.99:0/2122041333 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fb35c030740 con 0x55a1a4528af0 2024-03-19T02:05:00.239 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.231+0000 7fb36f5b3740 1 -- 172.21.15.99:0/2122041333 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fb358041070 con 0x55a1a4528af0 2024-03-19T02:05:00.239 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.231+0000 7fb34dffb640 1 -- 172.21.15.99:0/2122041333 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=41) ==== 24+0+0 (secure 0 0 0) 0x7fb35c00f850 con 0x55a1a4528af0 2024-03-19T02:05:00.239 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.231+0000 7fb36f5b3740 10 client.17250 fetch_fsmap learned FSMap version 41 2024-03-19T02:05:00.239 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.231+0000 7fb36f5b3740 1 -- 172.21.15.99:0/2122041333 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=41,monmap=4+}) -- 0x55a1a45a4a90 con 0x55a1a4528af0 2024-03-19T02:05:00.240 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.231+0000 7fb34dffb640 1 -- 172.21.15.99:0/2122041333 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fb358041070 con 0x55a1a4528af0 2024-03-19T02:05:00.240 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.231+0000 7fb34dffb640 1 -- 172.21.15.99:0/2122041333 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 41) ==== 8354+0+0 (secure 0 0 0) 0x7fb35c003cc0 con 0x55a1a4528af0 2024-03-19T02:05:00.240 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.231+0000 7fb36f5b3740 10 client.17250 fetch_fsmap finished waiting for FSMap version 41 2024-03-19T02:05:00.240 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.231+0000 7fb36f5b3740 10 client.17250 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T02:05:00.240 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.231+0000 7fb36f5b3740 20 client.17250 populate_metadata read hostname 'smithi099' 2024-03-19T02:05:00.240 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.231+0000 7fb36f5b3740 1 --2- 172.21.15.99:0/2122041333 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55a1a45a5130 0x55a1a45a7520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:00.240 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.231+0000 7fb36f5b3740 4 client.17250 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T02:05:00.240 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.231+0000 7fb36f5b3740 1 -- 172.21.15.99:0/2122041333 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55a1a4494470 con 0x55a1a45a5130 2024-03-19T02:05:00.240 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.231+0000 7fb36659c640 1 --2- 172.21.15.99:0/2122041333 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55a1a45a5130 0x55a1a45a7520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:00.240 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.232+0000 7fb36659c640 1 --2- 172.21.15.99:0/2122041333 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55a1a45a5130 0x55a1a45a7520 crc :-1 s=READY pgs=242 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=55661c747015355f server_cookie=e88713a8561e6736 in_seq=0 out_seq=0 2024-03-19T02:05:00.240 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.232+0000 7fb34dffb640 10 client.17250 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T02:05:00.863 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.863+0000 7fb34dffb640 1 -- 172.21.15.99:0/2122041333 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x55a1a4494470 con 0x55a1a45a5130 2024-03-19T02:05:00.863 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.863+0000 7fb34dffb640 10 client.17250 handle_command_reply: tid=0 2024-03-19T02:05:00.863 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.863+0000 7fb36f5b3740 1 client.17250 shutdown 2024-03-19T02:05:00.863 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.863+0000 7fb36f5b3740 1 -- 172.21.15.99:0/2122041333 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a1a4528af0 msgr2=0x55a1a454f9f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:05:00.863 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.863+0000 7fb36f5b3740 1 --2- 172.21.15.99:0/2122041333 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a1a4528af0 0x55a1a454f9f0 secure :-1 s=READY pgs=795 cs=0 l=1 rev1=1 crypto rx=0x7fb35c011af0 tx=0x7fb35802cdd0 comp rx=0 tx=0).stop 2024-03-19T02:05:00.864 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.863+0000 7fb36f5b3740 1 -- 172.21.15.99:0/2122041333 shutdown_connections 2024-03-19T02:05:00.864 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.863+0000 7fb36f5b3740 1 --2- 172.21.15.99:0/2122041333 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55a1a45a5130 0x55a1a45a7520 crc :-1 s=READY pgs=242 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:00.864 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.863+0000 7fb36f5b3740 1 --2- 172.21.15.99:0/2122041333 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55a1a4537850 0x55a1a4550ef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:00.864 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.863+0000 7fb36f5b3740 1 --2- 172.21.15.99:0/2122041333 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55a1a452dc60 0x55a1a454ff30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:00.864 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.863+0000 7fb36f5b3740 1 --2- 172.21.15.99:0/2122041333 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55a1a4528af0 0x55a1a454f9f0 unknown :-1 s=CLOSED pgs=795 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:00.864 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.863+0000 7fb36f5b3740 1 -- 172.21.15.99:0/2122041333 >> 172.21.15.99:0/2122041333 conn(0x55a1a4523670 msgr2=0x55a1a4527160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:05:00.864 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.863+0000 7fb34dffb640 0 client.17250 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T02:05:00.864 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.863+0000 7fb36f5b3740 1 -- 172.21.15.99:0/2122041333 shutdown_connections 2024-03-19T02:05:00.864 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.863+0000 7fb36f5b3740 1 -- 172.21.15.99:0/2122041333 wait complete. 2024-03-19T02:05:00.864 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.863+0000 7fb36f5b3740 20 client.17250 trim_cache size 0 max 16384 2024-03-19T02:05:00.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T02:05:00.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T02:05:00.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T02:05:00.865 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T02:05:00.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.864+0000 7fb36c8ea640 1 -- 172.21.15.99:0/1347374105 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb3500a28a0 msgr2=0x7fb3500a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:05:00.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.864+0000 7fb36c8ea640 1 --2- 172.21.15.99:0/1347374105 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb3500a28a0 0x7fb3500a4d60 secure :-1 s=READY pgs=393 cs=0 l=1 rev1=1 crypto rx=0x7fb360098390 tx=0x7fb360066c20 comp rx=0 tx=0).stop 2024-03-19T02:05:00.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.864+0000 7fb36c8ea640 1 -- 172.21.15.99:0/1347374105 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb368103420 msgr2=0x7fb36819f0b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:05:00.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.864+0000 7fb36c8ea640 1 --2- 172.21.15.99:0/1347374105 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb368103420 0x7fb36819f0b0 secure :-1 s=READY pgs=793 cs=0 l=1 rev1=1 crypto rx=0x7fb36000eb10 tx=0x7fb36000c2d0 comp rx=0 tx=0).stop 2024-03-19T02:05:00.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.864+0000 7fb36c8ea640 1 -- 172.21.15.99:0/1347374105 shutdown_connections 2024-03-19T02:05:00.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.864+0000 7fb36c8ea640 1 --2- 172.21.15.99:0/1347374105 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb3500a28a0 0x7fb3500a4d60 unknown :-1 s=CLOSED pgs=393 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:00.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.864+0000 7fb36c8ea640 1 --2- 172.21.15.99:0/1347374105 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb368102a60 0x7fb368198040 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:00.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.864+0000 7fb36c8ea640 1 --2- 172.21.15.99:0/1347374105 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb368103420 0x7fb36819f0b0 unknown :-1 s=CLOSED pgs=793 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:00.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.864+0000 7fb36c8ea640 1 --2- 172.21.15.99:0/1347374105 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb368101870 0x7fb368197b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:00.866 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.864+0000 7fb36c8ea640 1 -- 172.21.15.99:0/1347374105 >> 172.21.15.99:0/1347374105 conn(0x7fb3680fcde0 msgr2=0x7fb368105d60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:05:00.867 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.864+0000 7fb36c8ea640 1 -- 172.21.15.99:0/1347374105 shutdown_connections 2024-03-19T02:05:00.867 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:00.864+0000 7fb36c8ea640 1 -- 172.21.15.99:0/1347374105 wait complete. 2024-03-19T02:05:01.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:05:00 smithi099 ceph-mon[62951]: pgmap v1068: 97 pgs: 97 active+clean; 658 MiB data, 9.4 GiB used, 1.0 TiB / 1.0 TiB avail; 3.4 MiB/s rd, 5.1 MiB/s wr, 476 op/s 2024-03-19T02:05:01.107 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:05:00 smithi115 ceph-mon[67332]: pgmap v1068: 97 pgs: 97 active+clean; 658 MiB data, 9.4 GiB used, 1.0 TiB / 1.0 TiB avail; 3.4 MiB/s rd, 5.1 MiB/s wr, 476 op/s 2024-03-19T02:05:01.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:05:00 smithi112 ceph-mon[66819]: pgmap v1068: 97 pgs: 97 active+clean; 658 MiB data, 9.4 GiB used, 1.0 TiB / 1.0 TiB avail; 3.4 MiB/s rd, 5.1 MiB/s wr, 476 op/s 2024-03-19T02:05:01.122 INFO:tasks.workunit.client.0.smithi099.stdout: 10 288001 22.11 MB/sec execute 570 sec latency 447.064 ms 2024-03-19T02:05:02.122 INFO:tasks.workunit.client.0.smithi099.stdout: 10 288488 22.11 MB/sec execute 571 sec latency 244.615 ms 2024-03-19T02:05:03.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:05:02 smithi099 ceph-mon[62951]: pgmap v1069: 97 pgs: 97 active+clean; 663 MiB data, 9.4 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 5.6 MiB/s wr, 494 op/s 2024-03-19T02:05:03.198 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:05:02 smithi112 ceph-mon[66819]: pgmap v1069: 97 pgs: 97 active+clean; 663 MiB data, 9.4 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 5.6 MiB/s wr, 494 op/s 2024-03-19T02:05:03.198 INFO:tasks.workunit.client.0.smithi099.stdout: 10 288967 22.11 MB/sec execute 572 sec latency 77.815 ms 2024-03-19T02:05:03.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:05:02 smithi115 ceph-mon[67332]: pgmap v1069: 97 pgs: 97 active+clean; 663 MiB data, 9.4 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 5.6 MiB/s wr, 494 op/s 2024-03-19T02:05:04.122 INFO:tasks.workunit.client.0.smithi099.stdout: 10 289305 22.12 MB/sec execute 573 sec latency 186.964 ms 2024-03-19T02:05:05.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:05:04 smithi099 ceph-mon[62951]: pgmap v1070: 97 pgs: 97 active+clean; 657 MiB data, 9.4 GiB used, 1.0 TiB / 1.0 TiB avail; 4.4 MiB/s rd, 6.2 MiB/s wr, 563 op/s 2024-03-19T02:05:05.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:05:04 smithi112 ceph-mon[66819]: pgmap v1070: 97 pgs: 97 active+clean; 657 MiB data, 9.4 GiB used, 1.0 TiB / 1.0 TiB avail; 4.4 MiB/s rd, 6.2 MiB/s wr, 563 op/s 2024-03-19T02:05:05.122 INFO:tasks.workunit.client.0.smithi099.stdout: 10 289663 22.11 MB/sec execute 574 sec latency 250.259 ms 2024-03-19T02:05:05.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:05:04 smithi115 ceph-mon[67332]: pgmap v1070: 97 pgs: 97 active+clean; 657 MiB data, 9.4 GiB used, 1.0 TiB / 1.0 TiB avail; 4.4 MiB/s rd, 6.2 MiB/s wr, 563 op/s 2024-03-19T02:05:06.122 INFO:tasks.workunit.client.0.smithi099.stdout: 10 290019 22.10 MB/sec execute 575 sec latency 74.768 ms 2024-03-19T02:05:07.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:05:06 smithi099 ceph-mon[62951]: pgmap v1071: 97 pgs: 97 active+clean; 662 MiB data, 9.5 GiB used, 1.0 TiB / 1.0 TiB avail; 3.5 MiB/s rd, 5.2 MiB/s wr, 503 op/s 2024-03-19T02:05:07.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:05:06 smithi112 ceph-mon[66819]: pgmap v1071: 97 pgs: 97 active+clean; 662 MiB data, 9.5 GiB used, 1.0 TiB / 1.0 TiB avail; 3.5 MiB/s rd, 5.2 MiB/s wr, 503 op/s 2024-03-19T02:05:07.123 INFO:tasks.workunit.client.0.smithi099.stdout: 10 290471 22.11 MB/sec execute 576 sec latency 77.298 ms 2024-03-19T02:05:07.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:05:06 smithi115 ceph-mon[67332]: pgmap v1071: 97 pgs: 97 active+clean; 662 MiB data, 9.5 GiB used, 1.0 TiB / 1.0 TiB avail; 3.5 MiB/s rd, 5.2 MiB/s wr, 503 op/s 2024-03-19T02:05:08.122 INFO:tasks.workunit.client.0.smithi099.stdout: 10 290967 22.11 MB/sec execute 577 sec latency 67.148 ms 2024-03-19T02:05:09.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:05:08 smithi099 ceph-mon[62951]: pgmap v1072: 97 pgs: 97 active+clean; 662 MiB data, 9.5 GiB used, 1.0 TiB / 1.0 TiB avail; 3.5 MiB/s rd, 5.5 MiB/s wr, 523 op/s 2024-03-19T02:05:09.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:05:08 smithi112 ceph-mon[66819]: pgmap v1072: 97 pgs: 97 active+clean; 662 MiB data, 9.5 GiB used, 1.0 TiB / 1.0 TiB avail; 3.5 MiB/s rd, 5.5 MiB/s wr, 523 op/s 2024-03-19T02:05:09.122 INFO:tasks.workunit.client.0.smithi099.stdout: 10 291376 22.11 MB/sec execute 578 sec latency 110.018 ms 2024-03-19T02:05:09.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:05:08 smithi115 ceph-mon[67332]: pgmap v1072: 97 pgs: 97 active+clean; 662 MiB data, 9.5 GiB used, 1.0 TiB / 1.0 TiB avail; 3.5 MiB/s rd, 5.5 MiB/s wr, 523 op/s 2024-03-19T02:05:10.122 INFO:tasks.workunit.client.0.smithi099.stdout: 10 291723 22.11 MB/sec execute 579 sec latency 283.004 ms 2024-03-19T02:05:11.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:05:10 smithi099 ceph-mon[62951]: pgmap v1073: 97 pgs: 97 active+clean; 665 MiB data, 9.5 GiB used, 1.0 TiB / 1.0 TiB avail; 3.0 MiB/s rd, 5.4 MiB/s wr, 511 op/s 2024-03-19T02:05:11.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:05:10 smithi112 ceph-mon[66819]: pgmap v1073: 97 pgs: 97 active+clean; 665 MiB data, 9.5 GiB used, 1.0 TiB / 1.0 TiB avail; 3.0 MiB/s rd, 5.4 MiB/s wr, 511 op/s 2024-03-19T02:05:11.124 INFO:tasks.workunit.client.0.smithi099.stdout: 10 292063 22.09 MB/sec execute 580 sec latency 77.564 ms 2024-03-19T02:05:11.308 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:05:10 smithi115 ceph-mon[67332]: pgmap v1073: 97 pgs: 97 active+clean; 665 MiB data, 9.5 GiB used, 1.0 TiB / 1.0 TiB avail; 3.0 MiB/s rd, 5.4 MiB/s wr, 511 op/s 2024-03-19T02:05:12.122 INFO:tasks.workunit.client.0.smithi099.stdout: 10 292651 22.11 MB/sec execute 581 sec latency 64.443 ms 2024-03-19T02:05:13.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:05:12 smithi099 ceph-mon[62951]: pgmap v1074: 97 pgs: 97 active+clean; 667 MiB data, 9.5 GiB used, 1.0 TiB / 1.0 TiB avail; 3.3 MiB/s rd, 5.6 MiB/s wr, 510 op/s 2024-03-19T02:05:13.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:05:12 smithi112 ceph-mon[66819]: pgmap v1074: 97 pgs: 97 active+clean; 667 MiB data, 9.5 GiB used, 1.0 TiB / 1.0 TiB avail; 3.3 MiB/s rd, 5.6 MiB/s wr, 510 op/s 2024-03-19T02:05:13.123 INFO:tasks.workunit.client.0.smithi099.stdout: 10 293105 22.12 MB/sec execute 582 sec latency 102.872 ms 2024-03-19T02:05:13.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:05:12 smithi115 ceph-mon[67332]: pgmap v1074: 97 pgs: 97 active+clean; 667 MiB data, 9.5 GiB used, 1.0 TiB / 1.0 TiB avail; 3.3 MiB/s rd, 5.6 MiB/s wr, 510 op/s 2024-03-19T02:05:13.965 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.962+0000 7f3a3a99d640 1 -- 172.21.15.99:0/3107989844 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3a34100740 msgr2=0x7f3a34100b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:05:13.965 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.962+0000 7f3a3a99d640 1 --2- 172.21.15.99:0/3107989844 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3a34100740 0x7f3a34100b60 secure :-1 s=READY pgs=796 cs=0 l=1 rev1=1 crypto rx=0x7f3a24009900 tx=0x7f3a2402bc30 comp rx=0 tx=0).stop 2024-03-19T02:05:13.965 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.962+0000 7f3a3a99d640 1 -- 172.21.15.99:0/3107989844 shutdown_connections 2024-03-19T02:05:13.966 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.962+0000 7f3a3a99d640 1 --2- 172.21.15.99:0/3107989844 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3a341022f0 0x7f3a34108b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:13.966 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.962+0000 7f3a3a99d640 1 --2- 172.21.15.99:0/3107989844 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f3a34101930 0x7f3a34101db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:13.966 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.962+0000 7f3a3a99d640 1 --2- 172.21.15.99:0/3107989844 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3a34100740 0x7f3a34100b60 unknown :-1 s=CLOSED pgs=796 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:13.966 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.962+0000 7f3a3a99d640 1 -- 172.21.15.99:0/3107989844 >> 172.21.15.99:0/3107989844 conn(0x7f3a340fbcb0 msgr2=0x7f3a340fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:05:13.966 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.963+0000 7f3a3a99d640 1 -- 172.21.15.99:0/3107989844 shutdown_connections 2024-03-19T02:05:13.966 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.963+0000 7f3a3a99d640 1 -- 172.21.15.99:0/3107989844 wait complete. 2024-03-19T02:05:13.966 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.963+0000 7f3a3a99d640 1 Processor -- start 2024-03-19T02:05:13.967 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.963+0000 7f3a3a99d640 1 -- start start 2024-03-19T02:05:13.967 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.963+0000 7f3a3a99d640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3a34100740 0x7f3a34069a30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:13.967 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.964+0000 7f3a3a99d640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f3a34101930 0x7f3a3419c820 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:13.967 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.964+0000 7f3a3a99d640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3a341022f0 0x7f3a3419ee60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:13.967 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.964+0000 7f3a3a99d640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f3a3410c3b0 con 0x7f3a34100740 2024-03-19T02:05:13.967 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.964+0000 7f3a3a99d640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f3a3410c230 con 0x7f3a341022f0 2024-03-19T02:05:13.968 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.964+0000 7f3a3a99d640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f3a3410c530 con 0x7f3a34101930 2024-03-19T02:05:13.968 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.964+0000 7f3a33fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3a34100740 0x7f3a34069a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:13.968 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.964+0000 7f3a33fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3a34100740 0x7f3a34069a30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:35454/0 (socket says 172.21.15.99:35454) 2024-03-19T02:05:13.968 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.964+0000 7f3a33fff640 1 -- 172.21.15.99:0/2550555787 learned_addr learned my addr 172.21.15.99:0/2550555787 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:05:13.968 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.964+0000 7f3a38f3a640 1 --2- 172.21.15.99:0/2550555787 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3a341022f0 0x7f3a3419ee60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:13.968 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.964+0000 7f3a337fe640 1 --2- 172.21.15.99:0/2550555787 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f3a34101930 0x7f3a3419c820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:13.969 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.964+0000 7f3a38f3a640 1 -- 172.21.15.99:0/2550555787 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f3a34101930 msgr2=0x7f3a3419c820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:05:13.969 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.964+0000 7f3a38f3a640 1 --2- 172.21.15.99:0/2550555787 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f3a34101930 0x7f3a3419c820 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:13.969 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.964+0000 7f3a38f3a640 1 -- 172.21.15.99:0/2550555787 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3a34100740 msgr2=0x7f3a34069a30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:05:13.969 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.964+0000 7f3a38f3a640 1 --2- 172.21.15.99:0/2550555787 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3a34100740 0x7f3a34069a30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:13.969 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.964+0000 7f3a38f3a640 1 -- 172.21.15.99:0/2550555787 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f3a3419f720 con 0x7f3a341022f0 2024-03-19T02:05:13.969 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.964+0000 7f3a33fff640 1 --2- 172.21.15.99:0/2550555787 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3a34100740 0x7f3a34069a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:05:13.969 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.964+0000 7f3a38f3a640 1 --2- 172.21.15.99:0/2550555787 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3a341022f0 0x7f3a3419ee60 secure :-1 s=READY pgs=232 cs=0 l=1 rev1=1 crypto rx=0x7f3a2c009fd0 tx=0x7f3a2c00ef10 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:05:13.969 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.964+0000 7f3a317fa640 1 -- 172.21.15.99:0/2550555787 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3a2c00ce20 con 0x7f3a341022f0 2024-03-19T02:05:13.969 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.964+0000 7f3a3a99d640 1 -- 172.21.15.99:0/2550555787 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f3a3419fa10 con 0x7f3a341022f0 2024-03-19T02:05:13.969 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.964+0000 7f3a317fa640 1 -- 172.21.15.99:0/2550555787 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f3a2c005020 con 0x7f3a341022f0 2024-03-19T02:05:13.969 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.964+0000 7f3a317fa640 1 -- 172.21.15.99:0/2550555787 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3a2c010720 con 0x7f3a341022f0 2024-03-19T02:05:13.969 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.964+0000 7f3a3a99d640 1 -- 172.21.15.99:0/2550555787 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f3a3419ff60 con 0x7f3a341022f0 2024-03-19T02:05:13.980 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.966+0000 7f3a317fa640 1 -- 172.21.15.99:0/2550555787 <== mon.2 v2:172.21.15.112:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f3a2c005540 con 0x7f3a341022f0 2024-03-19T02:05:13.980 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.967+0000 7f3a317fa640 1 --2- 172.21.15.99:0/2550555787 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f3a1c0a28a0 0x7f3a1c0a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:13.981 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.967+0000 7f3a33fff640 1 --2- 172.21.15.99:0/2550555787 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f3a1c0a28a0 0x7f3a1c0a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:13.981 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.967+0000 7f3a317fa640 1 -- 172.21.15.99:0/2550555787 <== mon.2 v2:172.21.15.112:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f3a2c0cf180 con 0x7f3a341022f0 2024-03-19T02:05:13.981 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.968+0000 7f3a33fff640 1 --2- 172.21.15.99:0/2550555787 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f3a1c0a28a0 0x7f3a1c0a4d60 secure :-1 s=READY pgs=394 cs=0 l=1 rev1=1 crypto rx=0x7f3a2402c8e0 tx=0x7f3a24005b50 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:05:13.981 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.968+0000 7f3a3d581740 1 Processor -- start 2024-03-19T02:05:13.981 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.968+0000 7f3a3d581740 1 -- start start 2024-03-19T02:05:13.981 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.968+0000 7f3a3d581740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ad8a04b510 0x55ad8a04b930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:13.981 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.968+0000 7f3a3d581740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ad8a04bf00 0x55ad8a050a00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:13.981 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.968+0000 7f3a3d581740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ad8a051070 0x55ad8a053500 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:13.981 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.968+0000 7f3a3d581740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55ad89fabbe0 con 0x55ad8a04bf00 2024-03-19T02:05:13.981 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.968+0000 7f3a3d581740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55ad89f54960 con 0x55ad8a051070 2024-03-19T02:05:13.981 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.968+0000 7f3a3d581740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55ad89fa7c80 con 0x55ad8a04b510 2024-03-19T02:05:13.982 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.968+0000 7f3a337fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ad8a04bf00 0x55ad8a050a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:13.982 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.968+0000 7f3a337fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ad8a04bf00 0x55ad8a050a00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:35466/0 (socket says 172.21.15.99:35466) 2024-03-19T02:05:13.982 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.968+0000 7f3a337fe640 1 -- 172.21.15.99:0/3498265617 learned_addr learned my addr 172.21.15.99:0/3498265617 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:05:13.982 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.968+0000 7f3a33fff640 1 --2- 172.21.15.99:0/3498265617 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ad8a04b510 0x55ad8a04b930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:13.982 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.968+0000 7f3a38f3a640 1 --2- 172.21.15.99:0/3498265617 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ad8a051070 0x55ad8a053500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:13.982 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.968+0000 7f3a337fe640 1 -- 172.21.15.99:0/3498265617 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ad8a04b510 msgr2=0x55ad8a04b930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:05:13.982 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.968+0000 7f3a337fe640 1 --2- 172.21.15.99:0/3498265617 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ad8a04b510 0x55ad8a04b930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:13.982 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.968+0000 7f3a337fe640 1 -- 172.21.15.99:0/3498265617 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ad8a051070 msgr2=0x55ad8a053500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:05:13.982 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.968+0000 7f3a337fe640 1 --2- 172.21.15.99:0/3498265617 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ad8a051070 0x55ad8a053500 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:13.982 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.968+0000 7f3a337fe640 1 -- 172.21.15.99:0/3498265617 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55ad89f4bd20 con 0x55ad8a04bf00 2024-03-19T02:05:13.983 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.969+0000 7f3a337fe640 1 --2- 172.21.15.99:0/3498265617 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ad8a04bf00 0x55ad8a050a00 secure :-1 s=READY pgs=797 cs=0 l=1 rev1=1 crypto rx=0x7f3a28009ca0 tx=0x7f3a28011810 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e945a75192c4d379 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:05:13.983 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.969+0000 7f3a1a7fc640 1 -- 172.21.15.99:0/3498265617 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3a2801e070 con 0x55ad8a04bf00 2024-03-19T02:05:13.983 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.969+0000 7f3a1a7fc640 1 -- 172.21.15.99:0/3498265617 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f3a28019b10 con 0x55ad8a04bf00 2024-03-19T02:05:13.983 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.969+0000 7f3a1a7fc640 1 -- 172.21.15.99:0/3498265617 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3a2800ea30 con 0x55ad8a04bf00 2024-03-19T02:05:13.983 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.970+0000 7f3a3d581740 1 -- 172.21.15.99:0/3498265617 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ad8a04bf00 msgr2=0x55ad8a050a00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:05:13.983 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.970+0000 7f3a3d581740 1 --2- 172.21.15.99:0/3498265617 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ad8a04bf00 0x55ad8a050a00 secure :-1 s=READY pgs=797 cs=0 l=1 rev1=1 crypto rx=0x7f3a28009ca0 tx=0x7f3a28011810 comp rx=0 tx=0).stop 2024-03-19T02:05:13.983 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.970+0000 7f3a3d581740 1 -- 172.21.15.99:0/3498265617 shutdown_connections 2024-03-19T02:05:13.983 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.970+0000 7f3a3d581740 1 --2- 172.21.15.99:0/3498265617 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ad8a051070 0x55ad8a053500 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:13.983 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.970+0000 7f3a3d581740 1 --2- 172.21.15.99:0/3498265617 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ad8a04bf00 0x55ad8a050a00 unknown :-1 s=CLOSED pgs=797 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:13.983 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.970+0000 7f3a3d581740 1 --2- 172.21.15.99:0/3498265617 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ad8a04b510 0x55ad8a04b930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:13.983 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.970+0000 7f3a3d581740 1 -- 172.21.15.99:0/3498265617 >> 172.21.15.99:0/3498265617 conn(0x55ad8a042670 msgr2=0x55ad8a042a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:05:13.984 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.970+0000 7f3a3d581740 1 -- 172.21.15.99:0/3498265617 shutdown_connections 2024-03-19T02:05:13.984 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.970+0000 7f3a3d581740 1 -- 172.21.15.99:0/3498265617 wait complete. 2024-03-19T02:05:13.984 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.970+0000 7f3a3d581740 1 -- start start 2024-03-19T02:05:13.984 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.970+0000 7f3a3d581740 1 Processor -- start 2024-03-19T02:05:13.984 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.970+0000 7f3a3d581740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ad8a04b510 0x55ad8a0554f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:13.984 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.970+0000 7f3a3d581740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ad8a04bf00 0x55ad8a055a30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:13.984 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.970+0000 7f3a3d581740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ad8a051070 0x55ad8a05ca70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:13.984 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.970+0000 7f3a3d581740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55ad8a054020 con 0x55ad8a04bf00 2024-03-19T02:05:13.984 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.970+0000 7f3a3d581740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55ad89f67db0 con 0x55ad8a051070 2024-03-19T02:05:13.984 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.970+0000 7f3a3d581740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55ad8a0541a0 con 0x55ad8a04b510 2024-03-19T02:05:13.984 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.970+0000 7f3a3d581740 10 client.0 _ll_register_callbacks cb 0x55ad89f6ce70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T02:05:13.985 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.971+0000 7f3a337fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ad8a04bf00 0x55ad8a055a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:13.985 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.971+0000 7f3a33fff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ad8a04b510 0x55ad8a0554f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:13.985 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.971+0000 7f3a337fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ad8a04bf00 0x55ad8a055a30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:35478/0 (socket says 172.21.15.99:35478) 2024-03-19T02:05:13.985 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.971+0000 7f3a33fff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ad8a04b510 0x55ad8a0554f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:52506/0 (socket says 172.21.15.99:52506) 2024-03-19T02:05:13.985 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.971+0000 7f3a33fff640 1 -- 172.21.15.99:0/3462278656 learned_addr learned my addr 172.21.15.99:0/3462278656 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:05:13.985 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.971+0000 7f3a38f3a640 1 --2- 172.21.15.99:0/3462278656 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ad8a051070 0x55ad8a05ca70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:13.985 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.971+0000 7f3a33fff640 1 -- 172.21.15.99:0/3462278656 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ad8a051070 msgr2=0x55ad8a05ca70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:05:13.985 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.971+0000 7f3a33fff640 1 --2- 172.21.15.99:0/3462278656 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ad8a051070 0x55ad8a05ca70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:13.985 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.971+0000 7f3a33fff640 1 -- 172.21.15.99:0/3462278656 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ad8a04bf00 msgr2=0x55ad8a055a30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:05:13.985 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.971+0000 7f3a33fff640 1 --2- 172.21.15.99:0/3462278656 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ad8a04bf00 0x55ad8a055a30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:13.985 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.971+0000 7f3a33fff640 1 -- 172.21.15.99:0/3462278656 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55ad89f66c60 con 0x55ad8a04b510 2024-03-19T02:05:13.986 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.971+0000 7f3a33fff640 1 --2- 172.21.15.99:0/3462278656 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ad8a04b510 0x55ad8a0554f0 secure :-1 s=READY pgs=134 cs=0 l=1 rev1=1 crypto rx=0x7f3a24005c40 tx=0x7f3a2402bc30 comp rx=0 tx=0).ready entity=mon.1 client_cookie=d34549bc06c5ed8b server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:05:13.986 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.971+0000 7f3a137fe640 10 client.0 ms_handle_connect on v2:172.21.15.115:3300/0 2024-03-19T02:05:13.986 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.971+0000 7f3a137fe640 1 -- 172.21.15.99:0/3462278656 <== mon.1 v2:172.21.15.115:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3a2404f070 con 0x55ad8a04b510 2024-03-19T02:05:13.986 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.971+0000 7f3a137fe640 1 -- 172.21.15.99:0/3462278656 <== mon.1 v2:172.21.15.115:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f3a24005260 con 0x55ad8a04b510 2024-03-19T02:05:13.986 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.971+0000 7f3a3d581740 1 -- 172.21.15.99:0/3462278656 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f3a28019b10 con 0x55ad8a04b510 2024-03-19T02:05:13.986 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.972+0000 7f3a137fe640 1 -- 172.21.15.99:0/3462278656 <== mon.1 v2:172.21.15.115:3300/0 3 ==== mon_get_version_reply(handle=1 version=41) ==== 24+0+0 (secure 0 0 0) 0x7f3a24005460 con 0x55ad8a04b510 2024-03-19T02:05:13.986 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.972+0000 7f3a3d581740 10 client.27028 fetch_fsmap learned FSMap version 41 2024-03-19T02:05:13.986 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.972+0000 7f3a3d581740 1 -- 172.21.15.99:0/3462278656 --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_subscribe({config=0+,fsmap=41,monmap=4+}) -- 0x55ad89cc0210 con 0x55ad8a04b510 2024-03-19T02:05:13.986 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.972+0000 7f3a137fe640 1 -- 172.21.15.99:0/3462278656 <== mon.1 v2:172.21.15.115:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f3a28019b10 con 0x55ad8a04b510 2024-03-19T02:05:13.986 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.972+0000 7f3a137fe640 1 -- 172.21.15.99:0/3462278656 <== mon.1 v2:172.21.15.115:3300/0 5 ==== fsmap(e 41) ==== 8354+0+0 (secure 0 0 0) 0x7f3a240545e0 con 0x55ad8a04b510 2024-03-19T02:05:13.986 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.972+0000 7f3a3d581740 10 client.27028 fetch_fsmap finished waiting for FSMap version 41 2024-03-19T02:05:13.987 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.972+0000 7f3a3d581740 10 client.27028 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T02:05:13.987 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.972+0000 7f3a3d581740 20 client.27028 populate_metadata read hostname 'smithi099' 2024-03-19T02:05:13.987 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.972+0000 7f3a3d581740 1 --2- 172.21.15.99:0/3462278656 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55ad8a0613a0 0x55ad8a063830 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:13.987 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.972+0000 7f3a3d581740 4 client.27028 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T02:05:13.987 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.972+0000 7f3a3d581740 1 -- 172.21.15.99:0/3462278656 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55ad89fb9120 con 0x55ad8a0613a0 2024-03-19T02:05:13.987 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.972+0000 7f3a337fe640 1 --2- 172.21.15.99:0/3462278656 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55ad8a0613a0 0x55ad8a063830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:13.987 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.973+0000 7f3a137fe640 10 client.27028 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T02:05:13.987 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.973+0000 7f3a337fe640 1 --2- 172.21.15.99:0/3462278656 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55ad8a0613a0 0x55ad8a063830 crc :-1 s=READY pgs=243 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=b2088c88d6657e69 server_cookie=ec71b9eb8b07053e in_seq=0 out_seq=0 2024-03-19T02:05:13.987 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.974+0000 7f3a137fe640 1 -- 172.21.15.99:0/3462278656 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x55ad89fb9120 con 0x55ad8a0613a0 2024-03-19T02:05:13.987 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.974+0000 7f3a137fe640 10 client.27028 handle_command_reply: tid=0 2024-03-19T02:05:13.987 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.974+0000 7f3a3d581740 1 client.27028 shutdown 2024-03-19T02:05:13.987 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.974+0000 7f3a3d581740 1 -- 172.21.15.99:0/3462278656 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ad8a04b510 msgr2=0x55ad8a0554f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:05:13.987 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.974+0000 7f3a3d581740 1 --2- 172.21.15.99:0/3462278656 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ad8a04b510 0x55ad8a0554f0 secure :-1 s=READY pgs=134 cs=0 l=1 rev1=1 crypto rx=0x7f3a24005c40 tx=0x7f3a2402bc30 comp rx=0 tx=0).stop 2024-03-19T02:05:13.988 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.974+0000 7f3a3d581740 1 -- 172.21.15.99:0/3462278656 shutdown_connections 2024-03-19T02:05:13.988 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.974+0000 7f3a3d581740 1 --2- 172.21.15.99:0/3462278656 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55ad8a0613a0 0x55ad8a063830 crc :-1 s=READY pgs=243 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:13.988 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.974+0000 7f3a3d581740 1 --2- 172.21.15.99:0/3462278656 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ad8a051070 0x55ad8a05ca70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:13.988 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.974+0000 7f3a3d581740 1 --2- 172.21.15.99:0/3462278656 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ad8a04bf00 0x55ad8a055a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:13.988 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.974+0000 7f3a3d581740 1 --2- 172.21.15.99:0/3462278656 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ad8a04b510 0x55ad8a0554f0 unknown :-1 s=CLOSED pgs=134 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:13.988 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.974+0000 7f3a137fe640 0 client.27028 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T02:05:13.988 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.974+0000 7f3a3d581740 1 -- 172.21.15.99:0/3462278656 >> 172.21.15.99:0/3462278656 conn(0x55ad8a042670 msgr2=0x55ad8a0438c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:05:13.988 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.974+0000 7f3a3d581740 1 -- 172.21.15.99:0/3462278656 shutdown_connections 2024-03-19T02:05:13.988 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.974+0000 7f3a3d581740 1 -- 172.21.15.99:0/3462278656 wait complete. 2024-03-19T02:05:13.988 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.974+0000 7f3a3d581740 20 client.27028 trim_cache size 0 max 16384 2024-03-19T02:05:13.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.990+0000 7f3a3d581740 1 Processor -- start 2024-03-19T02:05:13.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.990+0000 7f3a3d581740 1 -- start start 2024-03-19T02:05:13.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.990+0000 7f3a3d581740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ad8a04b510 0x55ad8a041ab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:13.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.990+0000 7f3a3d581740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ad8a04bf00 0x55ad8a045fe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:13.995 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.990+0000 7f3a3d581740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ad8a051070 0x55ad8a068130 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:13.996 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.990+0000 7f3a3d581740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f3a341a9490 con 0x55ad8a051070 2024-03-19T02:05:13.996 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.990+0000 7f3a3d581740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f3a341a6d00 con 0x55ad8a04bf00 2024-03-19T02:05:13.996 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.990+0000 7f3a3d581740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f3a341a4b50 con 0x55ad8a04b510 2024-03-19T02:05:13.996 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.990+0000 7f3a38f3a640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ad8a051070 0x55ad8a068130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:13.996 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.991+0000 7f3a38f3a640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ad8a051070 0x55ad8a068130 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:35486/0 (socket says 172.21.15.99:35486) 2024-03-19T02:05:13.996 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.991+0000 7f3a38f3a640 1 -- 172.21.15.99:0/395056179 learned_addr learned my addr 172.21.15.99:0/395056179 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:05:13.996 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.991+0000 7f3a337fe640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ad8a04bf00 0x55ad8a045fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:13.996 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.991+0000 7f3a33fff640 1 --2- 172.21.15.99:0/395056179 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ad8a04b510 0x55ad8a041ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:13.996 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.991+0000 7f3a38f3a640 1 -- 172.21.15.99:0/395056179 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ad8a04b510 msgr2=0x55ad8a041ab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:05:13.996 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.991+0000 7f3a38f3a640 1 --2- 172.21.15.99:0/395056179 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ad8a04b510 0x55ad8a041ab0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:13.996 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.991+0000 7f3a38f3a640 1 -- 172.21.15.99:0/395056179 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ad8a04bf00 msgr2=0x55ad8a045fe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:05:13.997 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.991+0000 7f3a38f3a640 1 --2- 172.21.15.99:0/395056179 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ad8a04bf00 0x55ad8a045fe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:13.997 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.991+0000 7f3a38f3a640 1 -- 172.21.15.99:0/395056179 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55ad89d8c620 con 0x55ad8a051070 2024-03-19T02:05:13.997 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.991+0000 7f3a337fe640 1 --2- 172.21.15.99:0/395056179 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ad8a04bf00 0x55ad8a045fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:05:13.997 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.991+0000 7f3a38f3a640 1 --2- 172.21.15.99:0/395056179 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ad8a051070 0x55ad8a068130 secure :-1 s=READY pgs=798 cs=0 l=1 rev1=1 crypto rx=0x7f3a2c09ba20 tx=0x7f3a2c0ccb10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=fcd90daca8c15081 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:05:13.997 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.991+0000 7f3a19ffb640 1 -- 172.21.15.99:0/395056179 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3a2c0ba2b0 con 0x55ad8a051070 2024-03-19T02:05:13.997 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.991+0000 7f3a19ffb640 1 -- 172.21.15.99:0/395056179 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f3a2c0051d0 con 0x55ad8a051070 2024-03-19T02:05:13.997 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.991+0000 7f3a19ffb640 1 -- 172.21.15.99:0/395056179 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3a2c0b4ba0 con 0x55ad8a051070 2024-03-19T02:05:13.997 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.992+0000 7f3a3d581740 1 -- 172.21.15.99:0/395056179 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ad8a051070 msgr2=0x55ad8a068130 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:05:13.997 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.992+0000 7f3a3d581740 1 --2- 172.21.15.99:0/395056179 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ad8a051070 0x55ad8a068130 secure :-1 s=READY pgs=798 cs=0 l=1 rev1=1 crypto rx=0x7f3a2c09ba20 tx=0x7f3a2c0ccb10 comp rx=0 tx=0).stop 2024-03-19T02:05:13.997 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.992+0000 7f3a3d581740 1 -- 172.21.15.99:0/395056179 shutdown_connections 2024-03-19T02:05:13.997 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.992+0000 7f3a3d581740 1 --2- 172.21.15.99:0/395056179 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ad8a04bf00 0x55ad8a045fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:13.997 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.992+0000 7f3a3d581740 1 --2- 172.21.15.99:0/395056179 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ad8a051070 0x55ad8a068130 unknown :-1 s=CLOSED pgs=798 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:13.997 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.992+0000 7f3a3d581740 1 --2- 172.21.15.99:0/395056179 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ad8a04b510 0x55ad8a041ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:13.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.992+0000 7f3a3d581740 1 -- 172.21.15.99:0/395056179 >> 172.21.15.99:0/395056179 conn(0x55ad8a042670 msgr2=0x55ad8a0458d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:05:13.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.992+0000 7f3a3d581740 1 -- 172.21.15.99:0/395056179 shutdown_connections 2024-03-19T02:05:13.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.992+0000 7f3a3d581740 1 -- 172.21.15.99:0/395056179 wait complete. 2024-03-19T02:05:13.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.992+0000 7f3a3d581740 1 -- start start 2024-03-19T02:05:13.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.993+0000 7f3a3d581740 1 Processor -- start 2024-03-19T02:05:13.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.993+0000 7f3a3d581740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ad8a04b510 0x55ad8a05a1d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:13.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.993+0000 7f3a3d581740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ad8a04bf00 0x55ad8a05a710 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:13.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.993+0000 7f3a3d581740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ad8a051070 0x55ad8a05b6b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:13.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.993+0000 7f3a3d581740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f3a341a51c0 con 0x55ad8a04bf00 2024-03-19T02:05:13.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.993+0000 7f3a3d581740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f3a341a6630 con 0x55ad8a051070 2024-03-19T02:05:13.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.993+0000 7f3a3d581740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f3a341a5890 con 0x55ad8a04b510 2024-03-19T02:05:13.998 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.993+0000 7f3a3d581740 10 client.0 _ll_register_callbacks cb 0x7f3a28005e30 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T02:05:13.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.993+0000 7f3a33fff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ad8a04b510 0x55ad8a05a1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:13.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.993+0000 7f3a337fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ad8a04bf00 0x55ad8a05a710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:13.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.993+0000 7f3a38f3a640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ad8a051070 0x55ad8a05b6b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:13.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.993+0000 7f3a38f3a640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ad8a051070 0x55ad8a05b6b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:60358/0 (socket says 172.21.15.99:60358) 2024-03-19T02:05:13.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.993+0000 7f3a33fff640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ad8a04b510 0x55ad8a05a1d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:52520/0 (socket says 172.21.15.99:52520) 2024-03-19T02:05:13.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.993+0000 7f3a38f3a640 1 -- 172.21.15.99:0/3570896806 learned_addr learned my addr 172.21.15.99:0/3570896806 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:05:13.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.993+0000 7f3a38f3a640 1 -- 172.21.15.99:0/3570896806 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ad8a04b510 msgr2=0x55ad8a05a1d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:05:13.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.993+0000 7f3a38f3a640 1 --2- 172.21.15.99:0/3570896806 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ad8a04b510 0x55ad8a05a1d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:13.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.993+0000 7f3a38f3a640 1 -- 172.21.15.99:0/3570896806 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ad8a04bf00 msgr2=0x55ad8a05a710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:05:13.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.993+0000 7f3a38f3a640 1 --2- 172.21.15.99:0/3570896806 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ad8a04bf00 0x55ad8a05a710 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:13.999 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.993+0000 7f3a38f3a640 1 -- 172.21.15.99:0/3570896806 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55ad8a05bca0 con 0x55ad8a051070 2024-03-19T02:05:14.000 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.994+0000 7f3a38f3a640 1 --2- 172.21.15.99:0/3570896806 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ad8a051070 0x55ad8a05b6b0 secure :-1 s=READY pgs=233 cs=0 l=1 rev1=1 crypto rx=0x7f3a2c005fd0 tx=0x7f3a2c0054b0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=aec8b9c7bd198aa4 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:05:14.000 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.994+0000 7f3a137fe640 10 client.0 ms_handle_connect on v2:172.21.15.112:3300/0 2024-03-19T02:05:14.000 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.994+0000 7f3a137fe640 1 -- 172.21.15.99:0/3570896806 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3a2c098150 con 0x55ad8a051070 2024-03-19T02:05:14.000 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.994+0000 7f3a137fe640 1 -- 172.21.15.99:0/3570896806 <== mon.2 v2:172.21.15.112:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f3a2c0b4ba0 con 0x55ad8a051070 2024-03-19T02:05:14.000 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.994+0000 7f3a3d581740 1 -- 172.21.15.99:0/3570896806 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f3a2c0051d0 con 0x55ad8a051070 2024-03-19T02:05:14.000 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.994+0000 7f3a137fe640 1 -- 172.21.15.99:0/3570896806 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_get_version_reply(handle=1 version=41) ==== 24+0+0 (secure 0 0 0) 0x7f3a2c09af30 con 0x55ad8a051070 2024-03-19T02:05:14.000 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.994+0000 7f3a3d581740 10 client.27044 fetch_fsmap learned FSMap version 41 2024-03-19T02:05:14.000 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.994+0000 7f3a3d581740 1 -- 172.21.15.99:0/3570896806 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,fsmap=41,monmap=4+}) -- 0x55ad8a0c7ac0 con 0x55ad8a051070 2024-03-19T02:05:14.000 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.994+0000 7f3a137fe640 1 -- 172.21.15.99:0/3570896806 <== mon.2 v2:172.21.15.112:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f3a2c0051d0 con 0x55ad8a051070 2024-03-19T02:05:14.000 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.995+0000 7f3a137fe640 1 -- 172.21.15.99:0/3570896806 <== mon.2 v2:172.21.15.112:3300/0 5 ==== fsmap(e 41) ==== 8354+0+0 (secure 0 0 0) 0x7f3a2c09abb0 con 0x55ad8a051070 2024-03-19T02:05:14.000 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.995+0000 7f3a3d581740 10 client.27044 fetch_fsmap finished waiting for FSMap version 41 2024-03-19T02:05:14.000 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.995+0000 7f3a3d581740 10 client.27044 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T02:05:14.001 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.995+0000 7f3a3d581740 20 client.27044 populate_metadata read hostname 'smithi099' 2024-03-19T02:05:14.001 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.995+0000 7f3a3d581740 1 --2- 172.21.15.99:0/3570896806 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55ad8a0c8330 0x55ad8a0ca750 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:14.001 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.995+0000 7f3a3d581740 4 client.27044 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T02:05:14.001 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.995+0000 7f3a3d581740 1 -- 172.21.15.99:0/3570896806 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55ad89fb3470 con 0x55ad8a0c8330 2024-03-19T02:05:14.001 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.995+0000 7f3a33fff640 1 --2- 172.21.15.99:0/3570896806 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55ad8a0c8330 0x55ad8a0ca750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:14.001 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.995+0000 7f3a33fff640 1 --2- 172.21.15.99:0/3570896806 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55ad8a0c8330 0x55ad8a0ca750 crc :-1 s=READY pgs=244 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=eb63ad968ca7f497 server_cookie=b6c4f4a99e243e29 in_seq=0 out_seq=0 2024-03-19T02:05:14.001 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:13.995+0000 7f3a137fe640 10 client.27044 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T02:05:14.127 INFO:tasks.workunit.client.0.smithi099.stdout: 10 293579 22.13 MB/sec execute 583 sec latency 128.513 ms 2024-03-19T02:05:14.808 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:14.807+0000 7f3a137fe640 1 -- 172.21.15.99:0/3570896806 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x55ad89fb3470 con 0x55ad8a0c8330 2024-03-19T02:05:14.808 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:14.807+0000 7f3a137fe640 10 client.27044 handle_command_reply: tid=0 2024-03-19T02:05:14.808 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:14.807+0000 7f3a3d581740 1 client.27044 shutdown 2024-03-19T02:05:14.808 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:14.808+0000 7f3a3d581740 1 -- 172.21.15.99:0/3570896806 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ad8a051070 msgr2=0x55ad8a05b6b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:05:14.808 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:14.808+0000 7f3a3d581740 1 --2- 172.21.15.99:0/3570896806 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ad8a051070 0x55ad8a05b6b0 secure :-1 s=READY pgs=233 cs=0 l=1 rev1=1 crypto rx=0x7f3a2c005fd0 tx=0x7f3a2c0054b0 comp rx=0 tx=0).stop 2024-03-19T02:05:14.809 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:14.808+0000 7f3a3d581740 1 -- 172.21.15.99:0/3570896806 shutdown_connections 2024-03-19T02:05:14.809 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:14.808+0000 7f3a3d581740 1 --2- 172.21.15.99:0/3570896806 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55ad8a0c8330 0x55ad8a0ca750 crc :-1 s=READY pgs=244 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:14.809 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:14.808+0000 7f3a3d581740 1 --2- 172.21.15.99:0/3570896806 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55ad8a051070 0x55ad8a05b6b0 unknown :-1 s=CLOSED pgs=233 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:14.809 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:14.808+0000 7f3a3d581740 1 --2- 172.21.15.99:0/3570896806 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55ad8a04bf00 0x55ad8a05a710 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:14.809 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:14.808+0000 7f3a137fe640 0 client.27044 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T02:05:14.809 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:14.808+0000 7f3a3d581740 1 --2- 172.21.15.99:0/3570896806 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55ad8a04b510 0x55ad8a05a1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:14.809 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:14.808+0000 7f3a3d581740 1 -- 172.21.15.99:0/3570896806 >> 172.21.15.99:0/3570896806 conn(0x55ad8a042670 msgr2=0x55ad8a046160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:05:14.809 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:14.808+0000 7f3a3d581740 1 -- 172.21.15.99:0/3570896806 shutdown_connections 2024-03-19T02:05:14.809 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:14.808+0000 7f3a3d581740 1 -- 172.21.15.99:0/3570896806 wait complete. 2024-03-19T02:05:14.809 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:14.808+0000 7f3a3d581740 20 client.27044 trim_cache size 0 max 16384 2024-03-19T02:05:14.809 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T02:05:14.810 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T02:05:14.810 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T02:05:14.810 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T02:05:14.811 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:14.809+0000 7f3a3a99d640 1 -- 172.21.15.99:0/2550555787 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f3a1c0a28a0 msgr2=0x7f3a1c0a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:05:14.811 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:14.809+0000 7f3a3a99d640 1 --2- 172.21.15.99:0/2550555787 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f3a1c0a28a0 0x7f3a1c0a4d60 secure :-1 s=READY pgs=394 cs=0 l=1 rev1=1 crypto rx=0x7f3a2402c8e0 tx=0x7f3a24005b50 comp rx=0 tx=0).stop 2024-03-19T02:05:14.811 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:14.809+0000 7f3a3a99d640 1 -- 172.21.15.99:0/2550555787 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3a341022f0 msgr2=0x7f3a3419ee60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:05:14.811 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:14.809+0000 7f3a3a99d640 1 --2- 172.21.15.99:0/2550555787 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3a341022f0 0x7f3a3419ee60 secure :-1 s=READY pgs=232 cs=0 l=1 rev1=1 crypto rx=0x7f3a2c009fd0 tx=0x7f3a2c00ef10 comp rx=0 tx=0).stop 2024-03-19T02:05:14.811 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:14.809+0000 7f3a3a99d640 1 -- 172.21.15.99:0/2550555787 shutdown_connections 2024-03-19T02:05:14.811 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:14.809+0000 7f3a3a99d640 1 --2- 172.21.15.99:0/2550555787 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f3a1c0a28a0 0x7f3a1c0a4d60 unknown :-1 s=CLOSED pgs=394 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:14.811 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:14.809+0000 7f3a3a99d640 1 --2- 172.21.15.99:0/2550555787 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3a341022f0 0x7f3a3419ee60 unknown :-1 s=CLOSED pgs=232 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:14.811 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:14.809+0000 7f3a3a99d640 1 --2- 172.21.15.99:0/2550555787 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f3a34101930 0x7f3a3419c820 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:14.811 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:14.809+0000 7f3a3a99d640 1 --2- 172.21.15.99:0/2550555787 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3a34100740 0x7f3a34069a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:14.811 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:14.809+0000 7f3a3a99d640 1 -- 172.21.15.99:0/2550555787 >> 172.21.15.99:0/2550555787 conn(0x7f3a340fbcb0 msgr2=0x7f3a34104c30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:05:14.811 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:14.809+0000 7f3a3a99d640 1 -- 172.21.15.99:0/2550555787 shutdown_connections 2024-03-19T02:05:14.812 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:14.809+0000 7f3a3a99d640 1 -- 172.21.15.99:0/2550555787 wait complete. 2024-03-19T02:05:15.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:05:14 smithi099 ceph-mon[62951]: pgmap v1075: 97 pgs: 97 active+clean; 673 MiB data, 9.6 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 6.4 MiB/s wr, 558 op/s 2024-03-19T02:05:15.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:05:14 smithi112 ceph-mon[66819]: pgmap v1075: 97 pgs: 97 active+clean; 673 MiB data, 9.6 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 6.4 MiB/s wr, 558 op/s 2024-03-19T02:05:15.122 INFO:tasks.workunit.client.0.smithi099.stdout: 10 293645 22.10 MB/sec execute 584 sec latency 633.548 ms 2024-03-19T02:05:15.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:05:14 smithi115 ceph-mon[67332]: pgmap v1075: 97 pgs: 97 active+clean; 673 MiB data, 9.6 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 MiB/s rd, 6.4 MiB/s wr, 558 op/s 2024-03-19T02:05:16.123 INFO:tasks.workunit.client.0.smithi099.stdout: 10 293970 22.09 MB/sec execute 585 sec latency 134.398 ms 2024-03-19T02:05:17.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:05:16 smithi099 ceph-mon[62951]: pgmap v1076: 97 pgs: 97 active+clean; 665 MiB data, 9.6 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 5.4 MiB/s wr, 440 op/s 2024-03-19T02:05:17.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:05:16 smithi112 ceph-mon[66819]: pgmap v1076: 97 pgs: 97 active+clean; 665 MiB data, 9.6 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 5.4 MiB/s wr, 440 op/s 2024-03-19T02:05:17.122 INFO:tasks.workunit.client.0.smithi099.stdout: 10 294356 22.09 MB/sec execute 586 sec latency 93.595 ms 2024-03-19T02:05:17.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:05:16 smithi115 ceph-mon[67332]: pgmap v1076: 97 pgs: 97 active+clean; 665 MiB data, 9.6 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 MiB/s rd, 5.4 MiB/s wr, 440 op/s 2024-03-19T02:05:18.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:05:17 smithi099 ceph-mon[62951]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T02:05:18.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:05:17 smithi112 ceph-mon[66819]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T02:05:18.122 INFO:tasks.workunit.client.0.smithi099.stdout: 10 294775 22.09 MB/sec execute 587 sec latency 119.660 ms 2024-03-19T02:05:18.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:05:17 smithi115 ceph-mon[67332]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T02:05:19.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:05:18 smithi099 ceph-mon[62951]: pgmap v1077: 97 pgs: 97 active+clean; 668 MiB data, 9.6 GiB used, 1.0 TiB / 1.0 TiB avail; 4.6 MiB/s rd, 6.1 MiB/s wr, 507 op/s 2024-03-19T02:05:19.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:05:18 smithi112 ceph-mon[66819]: pgmap v1077: 97 pgs: 97 active+clean; 668 MiB data, 9.6 GiB used, 1.0 TiB / 1.0 TiB avail; 4.6 MiB/s rd, 6.1 MiB/s wr, 507 op/s 2024-03-19T02:05:19.123 INFO:tasks.workunit.client.0.smithi099.stdout: 10 295247 22.09 MB/sec execute 588 sec latency 76.862 ms 2024-03-19T02:05:19.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:05:18 smithi115 ceph-mon[67332]: pgmap v1077: 97 pgs: 97 active+clean; 668 MiB data, 9.6 GiB used, 1.0 TiB / 1.0 TiB avail; 4.6 MiB/s rd, 6.1 MiB/s wr, 507 op/s 2024-03-19T02:05:20.123 INFO:tasks.workunit.client.0.smithi099.stdout: 10 295563 22.08 MB/sec execute 589 sec latency 226.506 ms 2024-03-19T02:05:21.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:05:20 smithi112 ceph-mon[66819]: pgmap v1078: 97 pgs: 97 active+clean; 668 MiB data, 9.6 GiB used, 1.0 TiB / 1.0 TiB avail; 4.6 MiB/s rd, 6.3 MiB/s wr, 548 op/s 2024-03-19T02:05:21.123 INFO:tasks.workunit.client.0.smithi099.stdout: 10 295862 22.07 MB/sec execute 590 sec latency 227.425 ms 2024-03-19T02:05:21.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:05:20 smithi115 ceph-mon[67332]: pgmap v1078: 97 pgs: 97 active+clean; 668 MiB data, 9.6 GiB used, 1.0 TiB / 1.0 TiB avail; 4.6 MiB/s rd, 6.3 MiB/s wr, 548 op/s 2024-03-19T02:05:21.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:05:20 smithi099 ceph-mon[62951]: pgmap v1078: 97 pgs: 97 active+clean; 668 MiB data, 9.6 GiB used, 1.0 TiB / 1.0 TiB avail; 4.6 MiB/s rd, 6.3 MiB/s wr, 548 op/s 2024-03-19T02:05:22.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:05:21 smithi112 ceph-mon[66819]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T02:05:22.123 INFO:tasks.workunit.client.0.smithi099.stdout: 10 296340 22.08 MB/sec execute 591 sec latency 85.217 ms 2024-03-19T02:05:22.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:05:21 smithi115 ceph-mon[67332]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T02:05:22.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:05:21 smithi099 ceph-mon[62951]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T02:05:23.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:05:22 smithi112 ceph-mon[66819]: pgmap v1079: 97 pgs: 97 active+clean; 666 MiB data, 9.6 GiB used, 1.0 TiB / 1.0 TiB avail; 5.0 MiB/s rd, 6.0 MiB/s wr, 542 op/s 2024-03-19T02:05:23.123 INFO:tasks.workunit.client.0.smithi099.stdout: 10 296833 22.08 MB/sec execute 592 sec latency 103.368 ms 2024-03-19T02:05:23.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:05:22 smithi115 ceph-mon[67332]: pgmap v1079: 97 pgs: 97 active+clean; 666 MiB data, 9.6 GiB used, 1.0 TiB / 1.0 TiB avail; 5.0 MiB/s rd, 6.0 MiB/s wr, 542 op/s 2024-03-19T02:05:23.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:05:22 smithi099 ceph-mon[62951]: pgmap v1079: 97 pgs: 97 active+clean; 666 MiB data, 9.6 GiB used, 1.0 TiB / 1.0 TiB avail; 5.0 MiB/s rd, 6.0 MiB/s wr, 542 op/s 2024-03-19T02:05:24.123 INFO:tasks.workunit.client.0.smithi099.stdout: 10 297232 22.08 MB/sec execute 593 sec latency 69.746 ms 2024-03-19T02:05:25.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:05:24 smithi112 ceph-mon[66819]: pgmap v1080: 97 pgs: 97 active+clean; 671 MiB data, 9.6 GiB used, 1.0 TiB / 1.0 TiB avail; 4.9 MiB/s rd, 6.6 MiB/s wr, 590 op/s 2024-03-19T02:05:25.123 INFO:tasks.workunit.client.0.smithi099.stdout: 10 297569 22.09 MB/sec execute 594 sec latency 353.026 ms 2024-03-19T02:05:25.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:05:24 smithi115 ceph-mon[67332]: pgmap v1080: 97 pgs: 97 active+clean; 671 MiB data, 9.6 GiB used, 1.0 TiB / 1.0 TiB avail; 4.9 MiB/s rd, 6.6 MiB/s wr, 590 op/s 2024-03-19T02:05:25.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:05:24 smithi099 ceph-mon[62951]: pgmap v1080: 97 pgs: 97 active+clean; 671 MiB data, 9.6 GiB used, 1.0 TiB / 1.0 TiB avail; 4.9 MiB/s rd, 6.6 MiB/s wr, 590 op/s 2024-03-19T02:05:26.123 INFO:tasks.workunit.client.0.smithi099.stdout: 10 297939 22.07 MB/sec execute 595 sec latency 146.827 ms 2024-03-19T02:05:27.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:05:26 smithi112 ceph-mon[66819]: pgmap v1081: 97 pgs: 97 active+clean; 673 MiB data, 9.7 GiB used, 1.0 TiB / 1.0 TiB avail; 3.6 MiB/s rd, 5.3 MiB/s wr, 491 op/s 2024-03-19T02:05:27.123 INFO:tasks.workunit.client.0.smithi099.stdout: 10 298424 22.09 MB/sec execute 596 sec latency 61.405 ms 2024-03-19T02:05:27.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:05:26 smithi115 ceph-mon[67332]: pgmap v1081: 97 pgs: 97 active+clean; 673 MiB data, 9.7 GiB used, 1.0 TiB / 1.0 TiB avail; 3.6 MiB/s rd, 5.3 MiB/s wr, 491 op/s 2024-03-19T02:05:27.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:05:26 smithi099 ceph-mon[62951]: pgmap v1081: 97 pgs: 97 active+clean; 673 MiB data, 9.7 GiB used, 1.0 TiB / 1.0 TiB avail; 3.6 MiB/s rd, 5.3 MiB/s wr, 491 op/s 2024-03-19T02:05:27.906 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.903+0000 7fb72fac6640 1 -- 172.21.15.99:0/120083140 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb728100760 msgr2=0x7fb728100b80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:05:27.906 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.903+0000 7fb72fac6640 1 --2- 172.21.15.99:0/120083140 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb728100760 0x7fb728100b80 secure :-1 s=READY pgs=799 cs=0 l=1 rev1=1 crypto rx=0x7fb724009900 tx=0x7fb72402bc30 comp rx=0 tx=0).stop 2024-03-19T02:05:27.906 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.903+0000 7fb72fac6640 1 -- 172.21.15.99:0/120083140 shutdown_connections 2024-03-19T02:05:27.906 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.903+0000 7fb72fac6640 1 --2- 172.21.15.99:0/120083140 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb728102310 0x7fb728108ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:27.906 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.903+0000 7fb72fac6640 1 --2- 172.21.15.99:0/120083140 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb728101950 0x7fb728101dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:27.907 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.903+0000 7fb72fac6640 1 --2- 172.21.15.99:0/120083140 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb728100760 0x7fb728100b80 unknown :-1 s=CLOSED pgs=799 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:27.907 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.903+0000 7fb72fac6640 1 -- 172.21.15.99:0/120083140 >> 172.21.15.99:0/120083140 conn(0x7fb7280fbcb0 msgr2=0x7fb7280fe110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:05:27.907 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.903+0000 7fb72fac6640 1 -- 172.21.15.99:0/120083140 shutdown_connections 2024-03-19T02:05:27.907 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.903+0000 7fb72fac6640 1 -- 172.21.15.99:0/120083140 wait complete. 2024-03-19T02:05:27.907 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.904+0000 7fb72fac6640 1 Processor -- start 2024-03-19T02:05:27.907 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.904+0000 7fb72fac6640 1 -- start start 2024-03-19T02:05:27.907 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.904+0000 7fb72fac6640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb728100760 0x7fb728197b30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:27.907 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.904+0000 7fb72fac6640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb728101950 0x7fb728198070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:27.907 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.904+0000 7fb72fac6640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb728102310 0x7fb72819f0e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:27.907 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.904+0000 7fb72fac6640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fb72810d540 con 0x7fb728100760 2024-03-19T02:05:27.907 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.904+0000 7fb72fac6640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fb72810d3c0 con 0x7fb728102310 2024-03-19T02:05:27.907 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.904+0000 7fb72fac6640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fb72810d6c0 con 0x7fb728101950 2024-03-19T02:05:27.907 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.904+0000 7fb72d862640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb728100760 0x7fb728197b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:27.907 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.904+0000 7fb72d862640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb728100760 0x7fb728197b30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:51614/0 (socket says 172.21.15.99:51614) 2024-03-19T02:05:27.908 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.904+0000 7fb72d862640 1 -- 172.21.15.99:0/239265696 learned_addr learned my addr 172.21.15.99:0/239265696 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:05:27.908 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.904+0000 7fb72e063640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb728102310 0x7fb72819f0e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:27.908 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.904+0000 7fb72d061640 1 --2- 172.21.15.99:0/239265696 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb728101950 0x7fb728198070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:27.908 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.905+0000 7fb72e063640 1 -- 172.21.15.99:0/239265696 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb728101950 msgr2=0x7fb728198070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:05:27.908 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.905+0000 7fb72e063640 1 --2- 172.21.15.99:0/239265696 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb728101950 0x7fb728198070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:27.908 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.905+0000 7fb72e063640 1 -- 172.21.15.99:0/239265696 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb728100760 msgr2=0x7fb728197b30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:05:27.908 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.905+0000 7fb72e063640 1 --2- 172.21.15.99:0/239265696 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb728100760 0x7fb728197b30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:27.908 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.905+0000 7fb72e063640 1 -- 172.21.15.99:0/239265696 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb72819f750 con 0x7fb728102310 2024-03-19T02:05:27.908 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.905+0000 7fb72d862640 1 --2- 172.21.15.99:0/239265696 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb728100760 0x7fb728197b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:05:27.908 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.905+0000 7fb72e063640 1 --2- 172.21.15.99:0/239265696 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb728102310 0x7fb72819f0e0 secure :-1 s=READY pgs=234 cs=0 l=1 rev1=1 crypto rx=0x7fb720009fd0 tx=0x7fb72000ee50 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:05:27.908 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.905+0000 7fb71effd640 1 -- 172.21.15.99:0/239265696 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb72000cd60 con 0x7fb728102310 2024-03-19T02:05:27.908 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.905+0000 7fb71effd640 1 -- 172.21.15.99:0/239265696 <== mon.2 v2:172.21.15.112:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fb720005020 con 0x7fb728102310 2024-03-19T02:05:27.908 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.905+0000 7fb72fac6640 1 -- 172.21.15.99:0/239265696 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb72819fa40 con 0x7fb728102310 2024-03-19T02:05:27.909 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.905+0000 7fb71effd640 1 -- 172.21.15.99:0/239265696 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb720010690 con 0x7fb728102310 2024-03-19T02:05:27.909 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.905+0000 7fb72fac6640 1 -- 172.21.15.99:0/239265696 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fb72819ff40 con 0x7fb728102310 2024-03-19T02:05:27.915 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.907+0000 7fb71effd640 1 -- 172.21.15.99:0/239265696 <== mon.2 v2:172.21.15.112:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7fb720005540 con 0x7fb728102310 2024-03-19T02:05:27.915 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.908+0000 7fb71effd640 1 --2- 172.21.15.99:0/239265696 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb7140a28a0 0x7fb7140a4d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:27.915 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.908+0000 7fb732793740 1 Processor -- start 2024-03-19T02:05:27.915 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.908+0000 7fb72d862640 1 --2- 172.21.15.99:0/239265696 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb7140a28a0 0x7fb7140a4d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:27.915 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.908+0000 7fb71effd640 1 -- 172.21.15.99:0/239265696 <== mon.2 v2:172.21.15.112:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fb7200cf120 con 0x7fb728102310 2024-03-19T02:05:27.915 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.908+0000 7fb732793740 1 -- start start 2024-03-19T02:05:27.915 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.908+0000 7fb732793740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d37d971100 0x55d37d971520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:27.915 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.908+0000 7fb732793740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d37d971af0 0x55d37d9765f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:27.915 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.908+0000 7fb732793740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d37d976c60 0x55d37d9790f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:27.915 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.908+0000 7fb732793740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55d37d8d5be0 con 0x55d37d971af0 2024-03-19T02:05:27.915 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.908+0000 7fb732793740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55d37d87e960 con 0x55d37d976c60 2024-03-19T02:05:27.915 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.908+0000 7fb732793740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55d37d8d1c80 con 0x55d37d971100 2024-03-19T02:05:27.916 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.909+0000 7fb72d862640 1 --2- 172.21.15.99:0/239265696 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb7140a28a0 0x7fb7140a4d60 secure :-1 s=READY pgs=396 cs=0 l=1 rev1=1 crypto rx=0x7fb72402c8e0 tx=0x7fb724004ed0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:05:27.916 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.909+0000 7fb72d061640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d37d971af0 0x55d37d9765f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:27.916 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.909+0000 7fb72d061640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d37d971af0 0x55d37d9765f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:51620/0 (socket says 172.21.15.99:51620) 2024-03-19T02:05:27.916 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.909+0000 7fb72d862640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d37d971100 0x55d37d971520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:27.916 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.909+0000 7fb72d061640 1 -- 172.21.15.99:0/3565883981 learned_addr learned my addr 172.21.15.99:0/3565883981 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:05:27.916 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.909+0000 7fb72e063640 1 --2- 172.21.15.99:0/3565883981 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d37d976c60 0x55d37d9790f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:27.916 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.909+0000 7fb72d061640 1 -- 172.21.15.99:0/3565883981 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d37d971100 msgr2=0x55d37d971520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:05:27.916 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.909+0000 7fb72d061640 1 --2- 172.21.15.99:0/3565883981 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d37d971100 0x55d37d971520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:27.916 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.909+0000 7fb72d061640 1 -- 172.21.15.99:0/3565883981 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d37d976c60 msgr2=0x55d37d9790f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:05:27.916 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.909+0000 7fb72d061640 1 --2- 172.21.15.99:0/3565883981 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d37d976c60 0x55d37d9790f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:27.916 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.909+0000 7fb72d061640 1 -- 172.21.15.99:0/3565883981 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55d37d875d20 con 0x55d37d971af0 2024-03-19T02:05:27.916 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.909+0000 7fb72e063640 1 --2- 172.21.15.99:0/3565883981 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d37d976c60 0x55d37d9790f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:05:27.916 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.909+0000 7fb72d061640 1 --2- 172.21.15.99:0/3565883981 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d37d971af0 0x55d37d9765f0 secure :-1 s=READY pgs=800 cs=0 l=1 rev1=1 crypto rx=0x7fb718009a40 tx=0x7fb718009f00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4dc9aaf1aec654b2 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:05:27.916 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.909+0000 7fb713fff640 1 -- 172.21.15.99:0/3565883981 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb718017070 con 0x55d37d971af0 2024-03-19T02:05:27.917 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.909+0000 7fb713fff640 1 -- 172.21.15.99:0/3565883981 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fb718013470 con 0x55d37d971af0 2024-03-19T02:05:27.917 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.909+0000 7fb713fff640 1 -- 172.21.15.99:0/3565883981 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb718012610 con 0x55d37d971af0 2024-03-19T02:05:27.917 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.910+0000 7fb732793740 1 -- 172.21.15.99:0/3565883981 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d37d971af0 msgr2=0x55d37d9765f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:05:27.917 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.910+0000 7fb732793740 1 --2- 172.21.15.99:0/3565883981 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d37d971af0 0x55d37d9765f0 secure :-1 s=READY pgs=800 cs=0 l=1 rev1=1 crypto rx=0x7fb718009a40 tx=0x7fb718009f00 comp rx=0 tx=0).stop 2024-03-19T02:05:27.917 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.910+0000 7fb732793740 1 -- 172.21.15.99:0/3565883981 shutdown_connections 2024-03-19T02:05:27.917 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.910+0000 7fb732793740 1 --2- 172.21.15.99:0/3565883981 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d37d976c60 0x55d37d9790f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:27.917 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.910+0000 7fb732793740 1 --2- 172.21.15.99:0/3565883981 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d37d971af0 0x55d37d9765f0 unknown :-1 s=CLOSED pgs=800 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:27.917 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.910+0000 7fb732793740 1 --2- 172.21.15.99:0/3565883981 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d37d971100 0x55d37d971520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:27.917 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.910+0000 7fb732793740 1 -- 172.21.15.99:0/3565883981 >> 172.21.15.99:0/3565883981 conn(0x55d37d96c670 msgr2=0x55d37d96ca70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:05:27.917 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.910+0000 7fb732793740 1 -- 172.21.15.99:0/3565883981 shutdown_connections 2024-03-19T02:05:27.917 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.910+0000 7fb732793740 1 -- 172.21.15.99:0/3565883981 wait complete. 2024-03-19T02:05:27.917 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.910+0000 7fb732793740 1 -- start start 2024-03-19T02:05:27.917 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.911+0000 7fb732793740 1 Processor -- start 2024-03-19T02:05:27.917 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.911+0000 7fb732793740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d37d971100 0x55d37d97b160 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:27.918 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.911+0000 7fb732793740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d37d971af0 0x55d37d97b6a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:27.918 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.911+0000 7fb732793740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d37d976c60 0x55d37d9826e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:27.918 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.911+0000 7fb732793740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x55d37d979c10 con 0x55d37d971af0 2024-03-19T02:05:27.918 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.911+0000 7fb732793740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x55d37d891db0 con 0x55d37d976c60 2024-03-19T02:05:27.918 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.911+0000 7fb732793740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x55d37d979d90 con 0x55d37d971100 2024-03-19T02:05:27.918 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.911+0000 7fb72d862640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d37d971100 0x55d37d97b160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:27.918 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.911+0000 7fb72d061640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d37d971af0 0x55d37d97b6a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:27.918 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.911+0000 7fb72e063640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d37d976c60 0x55d37d9826e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:27.918 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.911+0000 7fb72e063640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d37d976c60 0x55d37d9826e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:59548/0 (socket says 172.21.15.99:59548) 2024-03-19T02:05:27.918 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.911+0000 7fb72e063640 1 -- 172.21.15.99:0/229344344 learned_addr learned my addr 172.21.15.99:0/229344344 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:05:27.918 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.911+0000 7fb72d061640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d37d971af0 0x55d37d97b6a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:51622/0 (socket says 172.21.15.99:51622) 2024-03-19T02:05:27.918 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.911+0000 7fb732793740 10 client.0 _ll_register_callbacks cb 0x55d37d896e70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T02:05:27.918 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.911+0000 7fb72e063640 1 -- 172.21.15.99:0/229344344 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d37d971100 msgr2=0x55d37d97b160 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:05:27.918 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.911+0000 7fb72e063640 1 --2- 172.21.15.99:0/229344344 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d37d971100 0x55d37d97b160 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:27.919 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.911+0000 7fb72e063640 1 -- 172.21.15.99:0/229344344 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d37d971af0 msgr2=0x55d37d97b6a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:05:27.919 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.911+0000 7fb72e063640 1 --2- 172.21.15.99:0/229344344 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d37d971af0 0x55d37d97b6a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:27.919 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.911+0000 7fb72e063640 1 -- 172.21.15.99:0/229344344 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55d37d890c60 con 0x55d37d976c60 2024-03-19T02:05:27.919 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.911+0000 7fb72d061640 1 --2- 172.21.15.99:0/229344344 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d37d971af0 0x55d37d97b6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:05:27.919 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.912+0000 7fb72e063640 1 --2- 172.21.15.99:0/229344344 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d37d976c60 0x55d37d9826e0 secure :-1 s=READY pgs=235 cs=0 l=1 rev1=1 crypto rx=0x7fb7200ccbe0 tx=0x7fb72009fd10 comp rx=0 tx=0).ready entity=mon.2 client_cookie=ab64e6bd2d4026a1 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:05:27.919 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.912+0000 7fb7117fa640 10 client.0 ms_handle_connect on v2:172.21.15.112:3300/0 2024-03-19T02:05:27.919 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.912+0000 7fb7117fa640 1 -- 172.21.15.99:0/229344344 <== mon.2 v2:172.21.15.112:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb7200afe60 con 0x55d37d976c60 2024-03-19T02:05:27.919 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.912+0000 7fb7117fa640 1 -- 172.21.15.99:0/229344344 <== mon.2 v2:172.21.15.112:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fb720013190 con 0x55d37d976c60 2024-03-19T02:05:27.919 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.912+0000 7fb732793740 1 -- 172.21.15.99:0/229344344 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fb718013470 con 0x55d37d976c60 2024-03-19T02:05:27.919 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.912+0000 7fb7117fa640 1 -- 172.21.15.99:0/229344344 <== mon.2 v2:172.21.15.112:3300/0 3 ==== mon_get_version_reply(handle=1 version=43) ==== 24+0+0 (secure 0 0 0) 0x7fb7200ba170 con 0x55d37d976c60 2024-03-19T02:05:27.919 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.912+0000 7fb732793740 10 client.27056 fetch_fsmap learned FSMap version 43 2024-03-19T02:05:27.919 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.912+0000 7fb732793740 1 -- 172.21.15.99:0/229344344 --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_subscribe({config=0+,fsmap=43,monmap=4+}) -- 0x55d37d5ea210 con 0x55d37d976c60 2024-03-19T02:05:27.919 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.912+0000 7fb7117fa640 1 -- 172.21.15.99:0/229344344 <== mon.2 v2:172.21.15.112:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fb718013470 con 0x55d37d976c60 2024-03-19T02:05:27.919 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.912+0000 7fb7117fa640 1 -- 172.21.15.99:0/229344344 <== mon.2 v2:172.21.15.112:3300/0 5 ==== fsmap(e 43) ==== 8346+0+0 (secure 0 0 0) 0x7fb720019be0 con 0x55d37d976c60 2024-03-19T02:05:27.919 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.912+0000 7fb732793740 10 client.27056 fetch_fsmap finished waiting for FSMap version 43 2024-03-19T02:05:27.920 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.912+0000 7fb732793740 10 client.27056 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T02:05:27.920 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.912+0000 7fb732793740 20 client.27056 populate_metadata read hostname 'smithi099' 2024-03-19T02:05:27.920 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.912+0000 7fb732793740 1 --2- 172.21.15.99:0/229344344 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55d37d986fa0 0x55d37d9893b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:27.920 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.912+0000 7fb732793740 4 client.27056 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-03-19T02:05:27.920 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.912+0000 7fb732793740 1 -- 172.21.15.99:0/229344344 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55d37d8e3120 con 0x55d37d986fa0 2024-03-19T02:05:27.920 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.913+0000 7fb72d862640 1 --2- 172.21.15.99:0/229344344 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55d37d986fa0 0x55d37d9893b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:27.920 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.913+0000 7fb7117fa640 10 client.27056 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T02:05:27.920 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.913+0000 7fb72d862640 1 --2- 172.21.15.99:0/229344344 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55d37d986fa0 0x55d37d9893b0 crc :-1 s=READY pgs=245 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=eb9a325b825b24c1 server_cookie=c37874f2a4ad3e97 in_seq=0 out_seq=0 2024-03-19T02:05:27.921 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.916+0000 7fb7117fa640 1 -- 172.21.15.99:0/229344344 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+22966 (crc 0 0 0) 0x55d37d8e3120 con 0x55d37d986fa0 2024-03-19T02:05:27.921 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.916+0000 7fb7117fa640 10 client.27056 handle_command_reply: tid=0 2024-03-19T02:05:27.921 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.916+0000 7fb732793740 1 client.27056 shutdown 2024-03-19T02:05:27.921 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.916+0000 7fb732793740 1 -- 172.21.15.99:0/229344344 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d37d976c60 msgr2=0x55d37d9826e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:05:27.922 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.916+0000 7fb732793740 1 --2- 172.21.15.99:0/229344344 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d37d976c60 0x55d37d9826e0 secure :-1 s=READY pgs=235 cs=0 l=1 rev1=1 crypto rx=0x7fb7200ccbe0 tx=0x7fb72009fd10 comp rx=0 tx=0).stop 2024-03-19T02:05:27.922 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.916+0000 7fb732793740 1 -- 172.21.15.99:0/229344344 shutdown_connections 2024-03-19T02:05:27.922 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.916+0000 7fb732793740 1 --2- 172.21.15.99:0/229344344 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55d37d986fa0 0x55d37d9893b0 crc :-1 s=READY pgs=245 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:27.922 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.916+0000 7fb732793740 1 --2- 172.21.15.99:0/229344344 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d37d976c60 0x55d37d9826e0 unknown :-1 s=CLOSED pgs=235 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:27.922 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.916+0000 7fb732793740 1 --2- 172.21.15.99:0/229344344 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d37d971af0 0x55d37d97b6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:27.922 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.916+0000 7fb7117fa640 0 client.27056 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T02:05:27.922 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.916+0000 7fb732793740 1 --2- 172.21.15.99:0/229344344 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d37d971100 0x55d37d97b160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:27.922 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.916+0000 7fb732793740 1 -- 172.21.15.99:0/229344344 >> 172.21.15.99:0/229344344 conn(0x55d37d96c670 msgr2=0x55d37d96d8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:05:27.922 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.916+0000 7fb732793740 1 -- 172.21.15.99:0/229344344 shutdown_connections 2024-03-19T02:05:27.922 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.916+0000 7fb732793740 1 -- 172.21.15.99:0/229344344 wait complete. 2024-03-19T02:05:27.922 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.916+0000 7fb732793740 20 client.27056 trim_cache size 0 max 16384 2024-03-19T02:05:27.934 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.932+0000 7fb732793740 1 Processor -- start 2024-03-19T02:05:27.934 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.932+0000 7fb732793740 1 -- start start 2024-03-19T02:05:27.934 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.932+0000 7fb732793740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d37d971100 0x55d37d96b930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:27.934 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.932+0000 7fb732793740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d37d971af0 0x55d37d9826a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:27.934 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.932+0000 7fb732793740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d37d976c60 0x55d37d972590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:27.934 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.932+0000 7fb732793740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fb7281ab690 con 0x55d37d976c60 2024-03-19T02:05:27.934 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.932+0000 7fb732793740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fb7281a8f30 con 0x55d37d971100 2024-03-19T02:05:27.935 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.932+0000 7fb732793740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fb7281a6d80 con 0x55d37d971af0 2024-03-19T02:05:27.935 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.933+0000 7fb72d862640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d37d971100 0x55d37d96b930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:27.935 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.932+0000 7fb72e063640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d37d976c60 0x55d37d972590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:27.935 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.933+0000 7fb72e063640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d37d976c60 0x55d37d972590 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:51624/0 (socket says 172.21.15.99:51624) 2024-03-19T02:05:27.935 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.933+0000 7fb72d862640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d37d971100 0x55d37d96b930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:59556/0 (socket says 172.21.15.99:59556) 2024-03-19T02:05:27.935 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.933+0000 7fb72e063640 1 -- 172.21.15.99:0/236749086 learned_addr learned my addr 172.21.15.99:0/236749086 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:05:27.935 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.933+0000 7fb72d061640 1 --2- 172.21.15.99:0/236749086 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d37d971af0 0x55d37d9826a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:27.935 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.933+0000 7fb72e063640 1 -- 172.21.15.99:0/236749086 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d37d971af0 msgr2=0x55d37d9826a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:05:27.935 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.933+0000 7fb72e063640 1 --2- 172.21.15.99:0/236749086 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d37d971af0 0x55d37d9826a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:27.935 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.933+0000 7fb72e063640 1 -- 172.21.15.99:0/236749086 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d37d971100 msgr2=0x55d37d96b930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:05:27.935 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.933+0000 7fb72e063640 1 --2- 172.21.15.99:0/236749086 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d37d971100 0x55d37d96b930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:27.935 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.933+0000 7fb72e063640 1 -- 172.21.15.99:0/236749086 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55d37d6b6620 con 0x55d37d976c60 2024-03-19T02:05:27.935 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.933+0000 7fb72e063640 1 --2- 172.21.15.99:0/236749086 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d37d976c60 0x55d37d972590 secure :-1 s=READY pgs=801 cs=0 l=1 rev1=1 crypto rx=0x7fb72000cfd0 tx=0x7fb720097e50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8f39376a2de7a118 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:05:27.935 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.933+0000 7fb7137fe640 1 -- 172.21.15.99:0/236749086 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb7200afe60 con 0x55d37d976c60 2024-03-19T02:05:27.936 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.933+0000 7fb7137fe640 1 -- 172.21.15.99:0/236749086 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fb7200ba310 con 0x55d37d976c60 2024-03-19T02:05:27.936 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.933+0000 7fb7137fe640 1 -- 172.21.15.99:0/236749086 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb72009b130 con 0x55d37d976c60 2024-03-19T02:05:27.939 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.934+0000 7fb732793740 1 -- 172.21.15.99:0/236749086 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d37d976c60 msgr2=0x55d37d972590 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:05:27.939 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.934+0000 7fb732793740 1 --2- 172.21.15.99:0/236749086 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d37d976c60 0x55d37d972590 secure :-1 s=READY pgs=801 cs=0 l=1 rev1=1 crypto rx=0x7fb72000cfd0 tx=0x7fb720097e50 comp rx=0 tx=0).stop 2024-03-19T02:05:27.940 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.934+0000 7fb732793740 1 -- 172.21.15.99:0/236749086 shutdown_connections 2024-03-19T02:05:27.940 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.934+0000 7fb732793740 1 --2- 172.21.15.99:0/236749086 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d37d976c60 0x55d37d972590 unknown :-1 s=CLOSED pgs=801 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:27.940 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.934+0000 7fb732793740 1 --2- 172.21.15.99:0/236749086 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d37d971af0 0x55d37d9826a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:27.940 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.934+0000 7fb732793740 1 --2- 172.21.15.99:0/236749086 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d37d971100 0x55d37d96b930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:27.940 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.934+0000 7fb732793740 1 -- 172.21.15.99:0/236749086 >> 172.21.15.99:0/236749086 conn(0x55d37d96c670 msgr2=0x55d37d96b220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:05:27.940 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.934+0000 7fb732793740 1 -- 172.21.15.99:0/236749086 shutdown_connections 2024-03-19T02:05:27.940 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.934+0000 7fb732793740 1 -- 172.21.15.99:0/236749086 wait complete. 2024-03-19T02:05:27.940 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.934+0000 7fb732793740 1 -- start start 2024-03-19T02:05:27.940 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.935+0000 7fb732793740 1 Processor -- start 2024-03-19T02:05:27.940 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.935+0000 7fb732793740 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d37d971100 0x55d37d980120 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:27.940 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.935+0000 7fb732793740 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d37d971af0 0x55d37d980660 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:27.940 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.935+0000 7fb732793740 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d37d976c60 0x55d37d981620 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:27.940 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.935+0000 7fb732793740 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fb7281a73f0 con 0x55d37d971af0 2024-03-19T02:05:27.940 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.935+0000 7fb732793740 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fb7281a8860 con 0x55d37d971100 2024-03-19T02:05:27.941 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.935+0000 7fb732793740 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fb7281a7ac0 con 0x55d37d976c60 2024-03-19T02:05:27.941 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.935+0000 7fb72d061640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d37d971af0 0x55d37d980660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:27.941 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.935+0000 7fb732793740 10 client.0 _ll_register_callbacks cb 0x7fb72009b3e0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-03-19T02:05:27.941 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.935+0000 7fb72d862640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d37d971100 0x55d37d980120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:27.941 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.935+0000 7fb72d061640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d37d971af0 0x55d37d980660 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:51634/0 (socket says 172.21.15.99:51634) 2024-03-19T02:05:27.941 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.935+0000 7fb72d061640 1 -- 172.21.15.99:0/4241943415 learned_addr learned my addr 172.21.15.99:0/4241943415 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:05:27.941 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.935+0000 7fb72e063640 1 --2- 172.21.15.99:0/4241943415 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d37d976c60 0x55d37d981620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:27.941 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.936+0000 7fb72d061640 1 -- 172.21.15.99:0/4241943415 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d37d976c60 msgr2=0x55d37d981620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:05:27.941 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.936+0000 7fb72d061640 1 --2- 172.21.15.99:0/4241943415 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d37d976c60 0x55d37d981620 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:27.941 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.936+0000 7fb72d061640 1 -- 172.21.15.99:0/4241943415 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d37d971100 msgr2=0x55d37d980120 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-03-19T02:05:27.941 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.936+0000 7fb72d061640 1 --2- 172.21.15.99:0/4241943415 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d37d971100 0x55d37d980120 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:27.941 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.936+0000 7fb72d061640 1 -- 172.21.15.99:0/4241943415 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55d37d981c10 con 0x55d37d971af0 2024-03-19T02:05:27.941 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.936+0000 7fb72d061640 1 --2- 172.21.15.99:0/4241943415 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d37d971af0 0x55d37d980660 secure :-1 s=READY pgs=802 cs=0 l=1 rev1=1 crypto rx=0x7fb718004700 tx=0x7fb7180136a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e0b1ae905fb3de3d server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:05:27.941 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.936+0000 7fb7117fa640 10 client.0 ms_handle_connect on v2:172.21.15.99:3300/0 2024-03-19T02:05:27.942 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.936+0000 7fb7117fa640 1 -- 172.21.15.99:0/4241943415 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb718017040 con 0x55d37d971af0 2024-03-19T02:05:27.942 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.936+0000 7fb7117fa640 1 -- 172.21.15.99:0/4241943415 <== mon.0 v2:172.21.15.99:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fb718013920 con 0x55d37d971af0 2024-03-19T02:05:27.942 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.936+0000 7fb732793740 1 -- 172.21.15.99:0/4241943415 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fb7200ba310 con 0x55d37d971af0 2024-03-19T02:05:27.942 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.936+0000 7fb7117fa640 1 -- 172.21.15.99:0/4241943415 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_get_version_reply(handle=1 version=43) ==== 24+0+0 (secure 0 0 0) 0x7fb718013b20 con 0x55d37d971af0 2024-03-19T02:05:27.942 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.936+0000 7fb732793740 10 client.17286 fetch_fsmap learned FSMap version 43 2024-03-19T02:05:27.942 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.936+0000 7fb732793740 1 -- 172.21.15.99:0/4241943415 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,fsmap=43,monmap=4+}) -- 0x55d37d9ed630 con 0x55d37d971af0 2024-03-19T02:05:27.942 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.936+0000 7fb7117fa640 1 -- 172.21.15.99:0/4241943415 <== mon.0 v2:172.21.15.99:3300/0 4 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fb7200ba310 con 0x55d37d971af0 2024-03-19T02:05:27.942 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.936+0000 7fb7117fa640 1 -- 172.21.15.99:0/4241943415 <== mon.0 v2:172.21.15.99:3300/0 5 ==== fsmap(e 43) ==== 8346+0+0 (secure 0 0 0) 0x7fb7180065b0 con 0x55d37d971af0 2024-03-19T02:05:27.942 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.936+0000 7fb732793740 10 client.17286 fetch_fsmap finished waiting for FSMap version 43 2024-03-19T02:05:27.942 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.937+0000 7fb732793740 10 client.17286 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.i 2024-03-19T02:05:27.942 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.937+0000 7fb732793740 20 client.17286 populate_metadata read hostname 'smithi099' 2024-03-19T02:05:27.942 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.937+0000 7fb732793740 1 --2- 172.21.15.99:0/4241943415 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55d37d9edd00 0x55d37d9f0120 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:27.942 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.937+0000 7fb732793740 4 client.17286 mds_command: new command op to 24487 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-03-19T02:05:27.942 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.937+0000 7fb732793740 1 -- 172.21.15.99:0/4241943415 --> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55d37d8dd470 con 0x55d37d9edd00 2024-03-19T02:05:27.943 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.937+0000 7fb72d862640 1 --2- 172.21.15.99:0/4241943415 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55d37d9edd00 0x55d37d9f0120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:27.943 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.937+0000 7fb7117fa640 10 client.17286 ms_handle_connect on v2:172.21.15.115:6838/2743761968 2024-03-19T02:05:27.943 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:27.937+0000 7fb72d862640 1 --2- 172.21.15.99:0/4241943415 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55d37d9edd00 0x55d37d9f0120 crc :-1 s=READY pgs=246 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=2ae260f0bd0bc64e server_cookie=e050c5abac839e4d in_seq=0 out_seq=0 2024-03-19T02:05:28.123 INFO:tasks.workunit.client.0.smithi099.stdout: 10 298854 22.08 MB/sec execute 597 sec latency 142.393 ms 2024-03-19T02:05:28.634 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:28.633+0000 7fb7117fa640 1 -- 172.21.15.99:0/4241943415 <== mds.0 v2:172.21.15.115:6838/2743761968 1 ==== command_reply(tid 0: 0 ) ==== 8+0+44 (crc 0 0 0) 0x55d37d8dd470 con 0x55d37d9edd00 2024-03-19T02:05:28.635 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:28.633+0000 7fb7117fa640 10 client.17286 handle_command_reply: tid=0 2024-03-19T02:05:28.635 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:28.633+0000 7fb732793740 1 client.17286 shutdown 2024-03-19T02:05:28.635 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:28.633+0000 7fb732793740 1 -- 172.21.15.99:0/4241943415 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d37d971af0 msgr2=0x55d37d980660 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:05:28.635 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:28.633+0000 7fb732793740 1 --2- 172.21.15.99:0/4241943415 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d37d971af0 0x55d37d980660 secure :-1 s=READY pgs=802 cs=0 l=1 rev1=1 crypto rx=0x7fb718004700 tx=0x7fb7180136a0 comp rx=0 tx=0).stop 2024-03-19T02:05:28.635 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:28.633+0000 7fb732793740 1 -- 172.21.15.99:0/4241943415 shutdown_connections 2024-03-19T02:05:28.635 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:28.633+0000 7fb732793740 1 --2- 172.21.15.99:0/4241943415 >> [v2:172.21.15.115:6838/2743761968,v1:172.21.15.115:6839/2743761968] conn(0x55d37d9edd00 0x55d37d9f0120 crc :-1 s=READY pgs=246 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:28.635 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:28.633+0000 7fb732793740 1 --2- 172.21.15.99:0/4241943415 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x55d37d976c60 0x55d37d981620 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:28.635 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:28.633+0000 7fb732793740 1 --2- 172.21.15.99:0/4241943415 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x55d37d971af0 0x55d37d980660 unknown :-1 s=CLOSED pgs=802 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:28.635 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:28.633+0000 7fb732793740 1 --2- 172.21.15.99:0/4241943415 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x55d37d971100 0x55d37d980120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:28.635 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:28.633+0000 7fb7117fa640 0 client.17286 ms_handle_reset on v2:172.21.15.115:6838/2743761968 2024-03-19T02:05:28.635 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:28.633+0000 7fb732793740 1 -- 172.21.15.99:0/4241943415 >> 172.21.15.99:0/4241943415 conn(0x55d37d96c670 msgr2=0x55d37d96f8d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:05:28.636 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:28.633+0000 7fb732793740 1 -- 172.21.15.99:0/4241943415 shutdown_connections 2024-03-19T02:05:28.636 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:28.633+0000 7fb732793740 1 -- 172.21.15.99:0/4241943415 wait complete. 2024-03-19T02:05:28.636 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:28.634+0000 7fb732793740 20 client.17286 trim_cache size 0 max 16384 2024-03-19T02:05:28.636 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:{ 2024-03-19T02:05:28.636 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "message": "", 2024-03-19T02:05:28.636 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout: "return_code": 0 2024-03-19T02:05:28.636 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stdout:} 2024-03-19T02:05:28.637 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:28.634+0000 7fb72fac6640 1 -- 172.21.15.99:0/239265696 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb7140a28a0 msgr2=0x7fb7140a4d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:05:28.637 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:28.634+0000 7fb72fac6640 1 --2- 172.21.15.99:0/239265696 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb7140a28a0 0x7fb7140a4d60 secure :-1 s=READY pgs=396 cs=0 l=1 rev1=1 crypto rx=0x7fb72402c8e0 tx=0x7fb724004ed0 comp rx=0 tx=0).stop 2024-03-19T02:05:28.637 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:28.634+0000 7fb72fac6640 1 -- 172.21.15.99:0/239265696 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb728102310 msgr2=0x7fb72819f0e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:05:28.637 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:28.634+0000 7fb72fac6640 1 --2- 172.21.15.99:0/239265696 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb728102310 0x7fb72819f0e0 secure :-1 s=READY pgs=234 cs=0 l=1 rev1=1 crypto rx=0x7fb720009fd0 tx=0x7fb72000ee50 comp rx=0 tx=0).stop 2024-03-19T02:05:28.637 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:28.634+0000 7fb72fac6640 1 -- 172.21.15.99:0/239265696 shutdown_connections 2024-03-19T02:05:28.637 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:28.634+0000 7fb72fac6640 1 --2- 172.21.15.99:0/239265696 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb7140a28a0 0x7fb7140a4d60 unknown :-1 s=CLOSED pgs=396 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:28.637 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:28.634+0000 7fb72fac6640 1 --2- 172.21.15.99:0/239265696 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb728102310 0x7fb72819f0e0 unknown :-1 s=CLOSED pgs=234 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:28.637 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:28.634+0000 7fb72fac6640 1 --2- 172.21.15.99:0/239265696 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb728101950 0x7fb728198070 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:28.638 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:28.634+0000 7fb72fac6640 1 --2- 172.21.15.99:0/239265696 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb728100760 0x7fb728197b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:28.638 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:28.634+0000 7fb72fac6640 1 -- 172.21.15.99:0/239265696 >> 172.21.15.99:0/239265696 conn(0x7fb7280fbcb0 msgr2=0x7fb728104c50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:05:28.638 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:28.635+0000 7fb72fac6640 1 -- 172.21.15.99:0/239265696 shutdown_connections 2024-03-19T02:05:28.638 INFO:teuthology.task.background_exec.ubuntu@smithi099.front.sepia.ceph.com.smithi099.stderr:2024-03-19T02:05:28.635+0000 7fb72fac6640 1 -- 172.21.15.99:0/239265696 wait complete. 2024-03-19T02:05:29.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:05:28 smithi112 ceph-mon[66819]: pgmap v1082: 97 pgs: 97 active+clean; 674 MiB data, 9.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 5.8 MiB/s wr, 490 op/s 2024-03-19T02:05:29.123 INFO:tasks.workunit.client.0.smithi099.stdout: 10 299073 22.07 MB/sec execute 598 sec latency 393.943 ms 2024-03-19T02:05:29.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:05:28 smithi115 ceph-mon[67332]: pgmap v1082: 97 pgs: 97 active+clean; 674 MiB data, 9.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 5.8 MiB/s wr, 490 op/s 2024-03-19T02:05:29.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:05:28 smithi099 ceph-mon[62951]: pgmap v1082: 97 pgs: 97 active+clean; 674 MiB data, 9.7 GiB used, 1.0 TiB / 1.0 TiB avail; 4.0 MiB/s rd, 5.8 MiB/s wr, 490 op/s 2024-03-19T02:05:30.123 INFO:tasks.workunit.client.0.smithi099.stdout: 10 299340 22.07 MB/sec execute 599 sec latency 344.137 ms 2024-03-19T02:05:31.126 INFO:tasks.workunit.client.0.smithi099.stdout: 10 cleanup 600 sec 2024-03-19T02:05:31.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:05:30 smithi115 ceph-mon[67332]: pgmap v1083: 97 pgs: 97 active+clean; 680 MiB data, 9.7 GiB used, 1.0 TiB / 1.0 TiB avail; 3.5 MiB/s rd, 5.6 MiB/s wr, 425 op/s 2024-03-19T02:05:31.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:05:30 smithi099 ceph-mon[62951]: pgmap v1083: 97 pgs: 97 active+clean; 680 MiB data, 9.7 GiB used, 1.0 TiB / 1.0 TiB avail; 3.5 MiB/s rd, 5.6 MiB/s wr, 425 op/s 2024-03-19T02:05:31.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:05:30 smithi112 ceph-mon[66819]: pgmap v1083: 97 pgs: 97 active+clean; 680 MiB data, 9.7 GiB used, 1.0 TiB / 1.0 TiB avail; 3.5 MiB/s rd, 5.6 MiB/s wr, 425 op/s 2024-03-19T02:05:32.123 INFO:tasks.workunit.client.0.smithi099.stdout: 7 cleanup 601 sec 2024-03-19T02:05:32.950 INFO:tasks.workunit.client.0.smithi099.stdout: 0 cleanup 602 sec 2024-03-19T02:05:32.950 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-03-19T02:05:32.950 INFO:tasks.workunit.client.0.smithi099.stdout: Operation Count AvgLat MaxLat 2024-03-19T02:05:32.951 INFO:tasks.workunit.client.0.smithi099.stdout: ---------------------------------------- 2024-03-19T02:05:32.951 INFO:tasks.workunit.client.0.smithi099.stdout: NTCreateX 423842 6.243 723.060 2024-03-19T02:05:32.951 INFO:tasks.workunit.client.0.smithi099.stdout: Close 311214 0.002 0.178 2024-03-19T02:05:32.951 INFO:tasks.workunit.client.0.smithi099.stdout: Rename 17963 27.165 441.937 2024-03-19T02:05:32.951 INFO:tasks.workunit.client.0.smithi099.stdout: Unlink 85695 8.945 397.945 2024-03-19T02:05:32.951 INFO:tasks.workunit.client.0.smithi099.stdout: Deltree 2 167.710 257.251 2024-03-19T02:05:32.951 INFO:tasks.workunit.client.0.smithi099.stdout: Mkdir 1 0.006 0.006 2024-03-19T02:05:32.951 INFO:tasks.workunit.client.0.smithi099.stdout: Qpathinfo 384362 4.219 669.025 2024-03-19T02:05:32.951 INFO:tasks.workunit.client.0.smithi099.stdout: Qfileinfo 67058 0.002 0.074 2024-03-19T02:05:32.951 INFO:tasks.workunit.client.0.smithi099.stdout: Qfsinfo 70486 0.218 77.959 2024-03-19T02:05:32.951 INFO:tasks.workunit.client.0.smithi099.stdout: Sfileinfo 34530 0.426 267.952 2024-03-19T02:05:32.951 INFO:tasks.workunit.client.0.smithi099.stdout: Find 148535 0.524 473.132 2024-03-19T02:05:32.951 INFO:tasks.workunit.client.0.smithi099.stdout: WriteX 209756 0.062 488.290 2024-03-19T02:05:32.951 INFO:tasks.workunit.client.0.smithi099.stdout: ReadX 664848 0.019 301.889 2024-03-19T02:05:32.952 INFO:tasks.workunit.client.0.smithi099.stdout: LockX 1382 7.042 139.616 2024-03-19T02:05:32.952 INFO:tasks.workunit.client.0.smithi099.stdout: UnlockX 1382 5.854 104.462 2024-03-19T02:05:32.952 INFO:tasks.workunit.client.0.smithi099.stdout: Flush 29704 10.790 429.654 2024-03-19T02:05:32.952 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-03-19T02:05:32.952 INFO:tasks.workunit.client.0.smithi099.stdout:Throughput 22.0653 MB/sec 10 clients 10 procs max_latency=723.064 ms 2024-03-19T02:05:32.952 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-03-19T02:05:32.952 DEBUG:teuthology.orchestra.run.smithi099:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-03-19T02:05:33.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:05:32 smithi115 ceph-mon[67332]: pgmap v1084: 97 pgs: 97 active+clean; 677 MiB data, 9.7 GiB used, 1.0 TiB / 1.0 TiB avail; 3.5 MiB/s rd, 5.5 MiB/s wr, 461 op/s 2024-03-19T02:05:33.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:05:32 smithi099 ceph-mon[62951]: pgmap v1084: 97 pgs: 97 active+clean; 677 MiB data, 9.7 GiB used, 1.0 TiB / 1.0 TiB avail; 3.5 MiB/s rd, 5.5 MiB/s wr, 461 op/s 2024-03-19T02:05:33.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:05:32 smithi112 ceph-mon[66819]: pgmap v1084: 97 pgs: 97 active+clean; 677 MiB data, 9.7 GiB used, 1.0 TiB / 1.0 TiB avail; 3.5 MiB/s rd, 5.5 MiB/s wr, 461 op/s 2024-03-19T02:05:34.305 INFO:tasks.workunit:Stopping ['suites/dbench.sh'] on client.0... 2024-03-19T02:05:34.305 DEBUG:teuthology.orchestra.run.smithi099:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2024-03-19T02:05:34.579 DEBUG:teuthology.parallel:result is None 2024-03-19T02:05:34.579 DEBUG:teuthology.orchestra.run.smithi099:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2024-03-19T02:05:34.888 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2024-03-19T02:05:34.888 DEBUG:teuthology.run_tasks:Unwinding manager background_exec 2024-03-19T02:05:34.900 INFO:teuthology.task.background_exec:Stopping background command on ubuntu@smithi099.front.sepia.ceph.com 2024-03-19T02:05:34.908 DEBUG:teuthology.run_tasks:Unwinding manager check-counter 2024-03-19T02:05:34.919 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-19T02:05:35.056 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.053+0000 7f94ab4cc640 1 -- 172.21.15.99:0/2342286924 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f94a4104c50 msgr2=0x7f94a4107040 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:05:35.056 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.053+0000 7f94ab4cc640 1 --2- 172.21.15.99:0/2342286924 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f94a4104c50 0x7f94a4107040 secure :-1 s=READY pgs=803 cs=0 l=1 rev1=1 crypto rx=0x7f9494009900 tx=0x7f949402bc30 comp rx=0 tx=0).stop 2024-03-19T02:05:35.057 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.053+0000 7f94ab4cc640 1 -- 172.21.15.99:0/2342286924 shutdown_connections 2024-03-19T02:05:35.057 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.053+0000 7f94ab4cc640 1 --2- 172.21.15.99:0/2342286924 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f94a4107580 0x7f94a4109970 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:35.057 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.053+0000 7f94ab4cc640 1 --2- 172.21.15.99:0/2342286924 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f94a4104c50 0x7f94a4107040 unknown :-1 s=CLOSED pgs=803 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:35.057 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.053+0000 7f94ab4cc640 1 --2- 172.21.15.99:0/2342286924 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f94a4102320 0x7f94a4104710 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:35.057 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.053+0000 7f94ab4cc640 1 -- 172.21.15.99:0/2342286924 >> 172.21.15.99:0/2342286924 conn(0x7f94a40fbcb0 msgr2=0x7f94a40fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:05:35.057 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.053+0000 7f94ab4cc640 1 -- 172.21.15.99:0/2342286924 shutdown_connections 2024-03-19T02:05:35.057 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.054+0000 7f94ab4cc640 1 -- 172.21.15.99:0/2342286924 wait complete. 2024-03-19T02:05:35.057 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.054+0000 7f94ab4cc640 1 Processor -- start 2024-03-19T02:05:35.057 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.054+0000 7f94ab4cc640 1 -- start start 2024-03-19T02:05:35.057 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.054+0000 7f94ab4cc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f94a4102320 0x7f94a4197b90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:35.057 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.054+0000 7f94ab4cc640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f94a4104c50 0x7f94a41980d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:35.057 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.054+0000 7f94ab4cc640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f94a4107580 0x7f94a419f140 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:35.057 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.054+0000 7f94ab4cc640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f94a410d490 con 0x7f94a4102320 2024-03-19T02:05:35.058 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.054+0000 7f94ab4cc640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f94a410d310 con 0x7f94a4104c50 2024-03-19T02:05:35.058 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.054+0000 7f94ab4cc640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f94a410d610 con 0x7f94a4107580 2024-03-19T02:05:35.058 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.054+0000 7f94a9268640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f94a4102320 0x7f94a4197b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:35.058 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.054+0000 7f94a9268640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f94a4102320 0x7f94a4197b90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:44988/0 (socket says 172.21.15.99:44988) 2024-03-19T02:05:35.058 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.054+0000 7f94a9268640 1 -- 172.21.15.99:0/2944248752 learned_addr learned my addr 172.21.15.99:0/2944248752 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:05:35.058 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.054+0000 7f94a9a69640 1 --2- 172.21.15.99:0/2944248752 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f94a4107580 0x7f94a419f140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:35.058 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.054+0000 7f94a8a67640 1 --2- 172.21.15.99:0/2944248752 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f94a4104c50 0x7f94a41980d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:35.058 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.055+0000 7f94a9268640 1 -- 172.21.15.99:0/2944248752 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f94a4107580 msgr2=0x7f94a419f140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:05:35.058 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.055+0000 7f94a9268640 1 --2- 172.21.15.99:0/2944248752 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f94a4107580 0x7f94a419f140 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:35.058 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.055+0000 7f94a9268640 1 -- 172.21.15.99:0/2944248752 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f94a4104c50 msgr2=0x7f94a41980d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:05:35.058 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.055+0000 7f94a9268640 1 --2- 172.21.15.99:0/2944248752 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f94a4104c50 0x7f94a41980d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:35.058 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.055+0000 7f94a9268640 1 -- 172.21.15.99:0/2944248752 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f94a419f7b0 con 0x7f94a4102320 2024-03-19T02:05:35.058 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.055+0000 7f94a8a67640 1 --2- 172.21.15.99:0/2944248752 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f94a4104c50 0x7f94a41980d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:05:35.058 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.055+0000 7f94a9268640 1 --2- 172.21.15.99:0/2944248752 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f94a4102320 0x7f94a4197b90 secure :-1 s=READY pgs=804 cs=0 l=1 rev1=1 crypto rx=0x7f94a000ebc0 tx=0x7f94a000c6a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:05:35.059 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.055+0000 7f949a7fc640 1 -- 172.21.15.99:0/2944248752 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f94a0010630 con 0x7f94a4102320 2024-03-19T02:05:35.059 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.055+0000 7f949a7fc640 1 -- 172.21.15.99:0/2944248752 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f94a0010cb0 con 0x7f94a4102320 2024-03-19T02:05:35.059 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.055+0000 7f94ab4cc640 1 -- 172.21.15.99:0/2944248752 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f94a419faa0 con 0x7f94a4102320 2024-03-19T02:05:35.059 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.055+0000 7f949a7fc640 1 -- 172.21.15.99:0/2944248752 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f94a0018bd0 con 0x7f94a4102320 2024-03-19T02:05:35.059 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.055+0000 7f94ab4cc640 1 -- 172.21.15.99:0/2944248752 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f94a41a0010 con 0x7f94a4102320 2024-03-19T02:05:35.059 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.056+0000 7f94ab4cc640 1 -- 172.21.15.99:0/2944248752 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f94a40fd7a0 con 0x7f94a4102320 2024-03-19T02:05:35.060 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.057+0000 7f949a7fc640 1 -- 172.21.15.99:0/2944248752 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f94a00107d0 con 0x7f94a4102320 2024-03-19T02:05:35.060 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.057+0000 7f949a7fc640 1 --2- 172.21.15.99:0/2944248752 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f94900a2880 0x7f94900a4d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:35.060 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.058+0000 7f94a8a67640 1 --2- 172.21.15.99:0/2944248752 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f94900a2880 0x7f94900a4d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:35.060 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.058+0000 7f949a7fc640 1 -- 172.21.15.99:0/2944248752 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f94a00cf260 con 0x7f94a4102320 2024-03-19T02:05:35.060 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.058+0000 7f94a8a67640 1 --2- 172.21.15.99:0/2944248752 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f94900a2880 0x7f94900a4d40 secure :-1 s=READY pgs=397 cs=0 l=1 rev1=1 crypto rx=0x7f9494000c00 tx=0x7f9494005fd0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:05:35.061 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.061+0000 7f949a7fc640 1 -- 172.21.15.99:0/2944248752 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f94a009bc70 con 0x7f94a4102320 2024-03-19T02:05:35.262 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.263+0000 7f94ab4cc640 1 -- 172.21.15.99:0/2944248752 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f94a402d480 con 0x7f94a4102320 2024-03-19T02:05:35.264 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.264+0000 7f949a7fc640 1 -- 172.21.15.99:0/2944248752 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v92) ==== 74+0+22804 (secure 0 0 0) 0x7f94a009fd10 con 0x7f94a4102320 2024-03-19T02:05:35.264 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T02:05:35.264 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":92,"fsid":"438fb69c-e590-11ee-95c9-87774f69a715","created":"2024-03-19T01:31:10.165073+0000","modified":"2024-03-19T01:52:30.348737+0000","last_up_change":"2024-03-19T01:38:09.082688+0000","last_in_change":"2024-03-19T01:37:49.115793+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-19T01:34:31.001309+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-19T01:38:54.018623+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"86","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-19T01:38:55.319036+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"87","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"0c407a0e-a40e-43d7-ad9a-bb8f3cd4a9ba","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6803","nonce":2935260112}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6805","nonce":2935260112}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6808","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6809","nonce":2935260112}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6807","nonce":2935260112}]},"public_addr":"172.21.15.99:6803/2935260112","cluster_addr":"172.21.15.99:6805/2935260112","heartbeat_back_addr":"172.21.15.99:6809/2935260112","heartbeat_front_addr":"172.21.15.99:6807/2935260112","state":["exists","up"]},{"osd":1,"uuid":"a48f9173-4a06-49f5-bca2-2c9047b8f666","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6800","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6801","nonce":1468854289}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6802","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6803","nonce":1468854289}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6806","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6807","nonce":1468854289}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6804","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6805","nonce":1468854289}]},"public_addr":"172.21.15.112:6801/1468854289","cluster_addr":"172.21.15.112:6803/1468854289","heartbeat_back_addr":"172.21.15.112:6807/1468854289","heartbeat_front_addr":"172.21.15.112:6805/1468854289","state":["exists","up"]},{"osd":2,"uuid":"b7715fdd-b6c9-4348-bd16-5eb61b427eef","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6800","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6801","nonce":3766690472}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6802","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6803","nonce":3766690472}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6806","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6807","nonce":3766690472}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6804","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6805","nonce":3766690472}]},"public_addr":"172.21.15.115:6801/3766690472","cluster_addr":"172.21.15.115:6803/3766690472","heartbeat_back_addr":"172.21.15.115:6807/3766690472","heartbeat_front_addr":"172.21.15.115:6805/3766690472","state":["exists","up"]},{"osd":3,"uuid":"6697ed1b-96bf-42de-8440-d17e9785b359","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6810","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6811","nonce":3756732895}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6812","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6813","nonce":3756732895}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6816","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6817","nonce":3756732895}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6814","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6815","nonce":3756732895}]},"public_addr":"172.21.15.99:6811/3756732895","cluster_addr":"172.21.15.99:6813/3756732895","heartbeat_back_addr":"172.21.15.99:6817/3756732895","heartbeat_front_addr":"172.21.15.99:6815/3756732895","state":["exists","up"]},{"osd":4,"uuid":"d958d6b9-67f6-43d0-b704-aa55dc9df83a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":34,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6808","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6809","nonce":3717986391}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6810","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6811","nonce":3717986391}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6814","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6815","nonce":3717986391}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6812","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6813","nonce":3717986391}]},"public_addr":"172.21.15.112:6809/3717986391","cluster_addr":"172.21.15.112:6811/3717986391","heartbeat_back_addr":"172.21.15.112:6815/3717986391","heartbeat_front_addr":"172.21.15.112:6813/3717986391","state":["exists","up"]},{"osd":5,"uuid":"a73eb49a-7925-49ba-bc4f-515e17f8b8ae","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6808","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6809","nonce":1971806408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6810","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6811","nonce":1971806408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6814","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6815","nonce":1971806408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6812","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6813","nonce":1971806408}]},"public_addr":"172.21.15.115:6809/1971806408","cluster_addr":"172.21.15.115:6811/1971806408","heartbeat_back_addr":"172.21.15.115:6815/1971806408","heartbeat_front_addr":"172.21.15.115:6813/1971806408","state":["exists","up"]},{"osd":6,"uuid":"caa718cd-e08b-48de-832a-29546d71e508","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6818","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6819","nonce":1130832772}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6820","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6821","nonce":1130832772}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6824","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6825","nonce":1130832772}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6822","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6823","nonce":1130832772}]},"public_addr":"172.21.15.99:6819/1130832772","cluster_addr":"172.21.15.99:6821/1130832772","heartbeat_back_addr":"172.21.15.99:6825/1130832772","heartbeat_front_addr":"172.21.15.99:6823/1130832772","state":["exists","up"]},{"osd":7,"uuid":"dbee19a1-2315-40d3-88e7-876fcde28903","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":53,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6816","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6817","nonce":3610356576}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6818","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6819","nonce":3610356576}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6822","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6823","nonce":3610356576}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6820","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6821","nonce":3610356576}]},"public_addr":"172.21.15.112:6817/3610356576","cluster_addr":"172.21.15.112:6819/3610356576","heartbeat_back_addr":"172.21.15.112:6823/3610356576","heartbeat_front_addr":"172.21.15.112:6821/3610356576","state":["exists","up"]},{"osd":8,"uuid":"14a866dc-5ce3-42fc-8bcf-9088b68726cb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":60,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6816","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6817","nonce":4255803111}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6818","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6819","nonce":4255803111}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6822","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6823","nonce":4255803111}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6820","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6821","nonce":4255803111}]},"public_addr":"172.21.15.115:6817/4255803111","cluster_addr":"172.21.15.115:6819/4255803111","heartbeat_back_addr":"172.21.15.115:6823/4255803111","heartbeat_front_addr":"172.21.15.115:6821/4255803111","state":["exists","up"]},{"osd":9,"uuid":"1db93652-6035-4906-91ae-d64d5a909502","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":68,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6826","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6827","nonce":1410921523}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6828","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6829","nonce":1410921523}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6832","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6833","nonce":1410921523}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6830","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6831","nonce":1410921523}]},"public_addr":"172.21.15.99:6827/1410921523","cluster_addr":"172.21.15.99:6829/1410921523","heartbeat_back_addr":"172.21.15.99:6833/1410921523","heartbeat_front_addr":"172.21.15.99:6831/1410921523","state":["exists","up"]},{"osd":10,"uuid":"6c50a8a7-3345-4f4a-af0e-5606c7aca453","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":74,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6824","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6825","nonce":891805922}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6826","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6827","nonce":891805922}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6830","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6831","nonce":891805922}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6828","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6829","nonce":891805922}]},"public_addr":"172.21.15.112:6825/891805922","cluster_addr":"172.21.15.112:6827/891805922","heartbeat_back_addr":"172.21.15.112:6831/891805922","heartbeat_front_addr":"172.21.15.112:6829/891805922","state":["exists","up"]},{"osd":11,"uuid":"fd63b7fd-0db5-46d5-82fa-9707bbd0b495","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":82,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6824","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6825","nonce":938436244}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6826","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6827","nonce":938436244}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6830","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6831","nonce":938436244}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6828","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6829","nonce":938436244}]},"public_addr":"172.21.15.115:6825/938436244","cluster_addr":"172.21.15.115:6827/938436244","heartbeat_back_addr":"172.21.15.115:6831/938436244","heartbeat_front_addr":"172.21.15.115:6829/938436244","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:33:39.036822+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:34:03.339566+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:34:27.149273+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:34:51.315561+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:35:15.396031+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:35:38.957168+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:36:03.213914+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:36:27.763699+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:36:51.688670+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:37:17.280165+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:37:42.554633+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:38:07.438039+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.0","mappings":[{"from":8,"to":1}]},{"pgid":"2.8","mappings":[{"from":2,"to":11}]},{"pgid":"2.b","mappings":[{"from":8,"to":11}]},{"pgid":"2.e","mappings":[{"from":2,"to":4}]},{"pgid":"3.0","mappings":[{"from":5,"to":9}]},{"pgid":"3.2","mappings":[{"from":3,"to":9}]},{"pgid":"3.14","mappings":[{"from":1,"to":9}]},{"pgid":"3.17","mappings":[{"from":4,"to":8}]},{"pgid":"3.31","mappings":[{"from":4,"to":8}]},{"pgid":"3.37","mappings":[{"from":5,"to":7}]},{"pgid":"3.3a","mappings":[{"from":4,"to":6}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.99:6801/1328245934":"2024-03-20T01:32:06.661002+0000","172.21.15.99:0/1618965837":"2024-03-20T01:32:06.661002+0000","172.21.15.99:0/2491429851":"2024-03-20T01:32:06.661002+0000","172.21.15.99:6800/1328245934":"2024-03-20T01:32:06.661002+0000","172.21.15.99:0/2112231766":"2024-03-20T01:31:40.161751+0000","172.21.15.99:0/3341836545":"2024-03-20T01:31:40.161751+0000","172.21.15.99:6800/4223048293":"2024-03-20T01:31:40.161751+0000","172.21.15.99:6801/4223048293":"2024-03-20T01:31:40.161751+0000","172.21.15.99:0/2377621863":"2024-03-20T01:31:40.161751+0000","172.21.15.99:0/1904400935":"2024-03-20T01:32:06.661002+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-19T02:05:35.266 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.266+0000 7f94ab4cc640 1 -- 172.21.15.99:0/2944248752 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f94900a2880 msgr2=0x7f94900a4d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:05:35.266 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.266+0000 7f94ab4cc640 1 --2- 172.21.15.99:0/2944248752 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f94900a2880 0x7f94900a4d40 secure :-1 s=READY pgs=397 cs=0 l=1 rev1=1 crypto rx=0x7f9494000c00 tx=0x7f9494005fd0 comp rx=0 tx=0).stop 2024-03-19T02:05:35.266 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.266+0000 7f94ab4cc640 1 -- 172.21.15.99:0/2944248752 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f94a4102320 msgr2=0x7f94a4197b90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:05:35.267 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.266+0000 7f94ab4cc640 1 --2- 172.21.15.99:0/2944248752 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f94a4102320 0x7f94a4197b90 secure :-1 s=READY pgs=804 cs=0 l=1 rev1=1 crypto rx=0x7f94a000ebc0 tx=0x7f94a000c6a0 comp rx=0 tx=0).stop 2024-03-19T02:05:35.267 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.266+0000 7f94ab4cc640 1 -- 172.21.15.99:0/2944248752 shutdown_connections 2024-03-19T02:05:35.267 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.266+0000 7f94ab4cc640 1 --2- 172.21.15.99:0/2944248752 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f94900a2880 0x7f94900a4d40 unknown :-1 s=CLOSED pgs=397 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:35.267 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.266+0000 7f94ab4cc640 1 --2- 172.21.15.99:0/2944248752 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f94a4104c50 0x7f94a41980d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:35.267 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.266+0000 7f94ab4cc640 1 --2- 172.21.15.99:0/2944248752 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f94a4107580 0x7f94a419f140 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:35.267 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.266+0000 7f94ab4cc640 1 --2- 172.21.15.99:0/2944248752 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f94a4102320 0x7f94a4197b90 unknown :-1 s=CLOSED pgs=804 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:35.267 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.266+0000 7f94ab4cc640 1 -- 172.21.15.99:0/2944248752 >> 172.21.15.99:0/2944248752 conn(0x7f94a40fbcb0 msgr2=0x7f94a4102d60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:05:35.267 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.266+0000 7f94ab4cc640 1 -- 172.21.15.99:0/2944248752 shutdown_connections 2024-03-19T02:05:35.267 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.266+0000 7f94ab4cc640 1 -- 172.21.15.99:0/2944248752 wait complete. 2024-03-19T02:05:35.276 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:05:34 smithi099 ceph-mon[62951]: pgmap v1085: 97 pgs: 97 active+clean; 668 MiB data, 9.7 GiB used, 1.0 TiB / 1.0 TiB avail; 3.5 MiB/s rd, 6.5 MiB/s wr, 573 op/s 2024-03-19T02:05:35.278 INFO:tasks.check_counter.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-19T01:34:31.001309+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 12, 'score_stable': 12, 'optimal_score': 0.25, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-19T01:38:54.018623+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '86', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.8799999952316284, 'score_stable': 1.8799999952316284, 'optimal_score': 1, 'raw_score_acting': 1.8799999952316284, 'raw_score_stable': 1.8799999952316284, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-03-19T01:38:55.319036+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '87', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-19T02:05:35.278 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-19T02:05:35.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:05:34 smithi115 ceph-mon[67332]: pgmap v1085: 97 pgs: 97 active+clean; 668 MiB data, 9.7 GiB used, 1.0 TiB / 1.0 TiB avail; 3.5 MiB/s rd, 6.5 MiB/s wr, 573 op/s 2024-03-19T02:05:35.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:05:34 smithi112 ceph-mon[66819]: pgmap v1085: 97 pgs: 97 active+clean; 668 MiB data, 9.7 GiB used, 1.0 TiB / 1.0 TiB avail; 3.5 MiB/s rd, 6.5 MiB/s wr, 573 op/s 2024-03-19T02:05:35.388 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.383+0000 7f361ac4b640 1 -- 172.21.15.99:0/3804269404 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f36141022e0 msgr2=0x7f3614108b70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:05:35.388 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.383+0000 7f361ac4b640 1 --2- 172.21.15.99:0/3804269404 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f36141022e0 0x7f3614108b70 secure :-1 s=READY pgs=805 cs=0 l=1 rev1=1 crypto rx=0x7f360c009ca0 tx=0x7f360c02c0f0 comp rx=0 tx=0).stop 2024-03-19T02:05:35.388 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.383+0000 7f361ac4b640 1 -- 172.21.15.99:0/3804269404 shutdown_connections 2024-03-19T02:05:35.389 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.383+0000 7f361ac4b640 1 --2- 172.21.15.99:0/3804269404 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3614101920 0x7f3614101da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:35.389 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.383+0000 7f361ac4b640 1 --2- 172.21.15.99:0/3804269404 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f36141022e0 0x7f3614108b70 unknown :-1 s=CLOSED pgs=805 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:35.389 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.383+0000 7f361ac4b640 1 --2- 172.21.15.99:0/3804269404 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f3614100730 0x7f3614100b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:35.389 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.383+0000 7f361ac4b640 1 -- 172.21.15.99:0/3804269404 >> 172.21.15.99:0/3804269404 conn(0x7f36140fbcc0 msgr2=0x7f36140fe0e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:05:35.389 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.383+0000 7f361ac4b640 1 -- 172.21.15.99:0/3804269404 shutdown_connections 2024-03-19T02:05:35.390 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.383+0000 7f361ac4b640 1 -- 172.21.15.99:0/3804269404 wait complete. 2024-03-19T02:05:35.390 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.383+0000 7f361ac4b640 1 Processor -- start 2024-03-19T02:05:35.390 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.383+0000 7f361ac4b640 1 -- start start 2024-03-19T02:05:35.390 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.383+0000 7f361ac4b640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3614100730 0x7f36141969f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:35.390 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.383+0000 7f361ac4b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3614101920 0x7f3614196f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:35.390 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.383+0000 7f361ac4b640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f36141022e0 0x7f361419dfa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:35.391 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.383+0000 7f361ac4b640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f361410c3b0 con 0x7f3614101920 2024-03-19T02:05:35.391 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.384+0000 7f361ac4b640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f361410c230 con 0x7f3614100730 2024-03-19T02:05:35.391 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.384+0000 7f361ac4b640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f361410c530 con 0x7f36141022e0 2024-03-19T02:05:35.391 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.384+0000 7f3613fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3614101920 0x7f3614196f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:35.391 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.384+0000 7f3613fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3614101920 0x7f3614196f30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:45004/0 (socket says 172.21.15.99:45004) 2024-03-19T02:05:35.391 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.384+0000 7f3613fff640 1 -- 172.21.15.99:0/890064291 learned_addr learned my addr 172.21.15.99:0/890064291 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:05:35.392 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.384+0000 7f36189e7640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3614100730 0x7f36141969f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:35.392 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.384+0000 7f36191e8640 1 --2- 172.21.15.99:0/890064291 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f36141022e0 0x7f361419dfa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:35.392 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.384+0000 7f3613fff640 1 -- 172.21.15.99:0/890064291 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f36141022e0 msgr2=0x7f361419dfa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:05:35.392 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.384+0000 7f3613fff640 1 --2- 172.21.15.99:0/890064291 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f36141022e0 0x7f361419dfa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:35.392 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.384+0000 7f3613fff640 1 -- 172.21.15.99:0/890064291 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3614100730 msgr2=0x7f36141969f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:05:35.392 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.384+0000 7f3613fff640 1 --2- 172.21.15.99:0/890064291 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3614100730 0x7f36141969f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:35.393 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.384+0000 7f3613fff640 1 -- 172.21.15.99:0/890064291 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f361419e610 con 0x7f3614101920 2024-03-19T02:05:35.393 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.384+0000 7f36189e7640 1 --2- 172.21.15.99:0/890064291 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3614100730 0x7f36141969f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:05:35.393 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.384+0000 7f3613fff640 1 --2- 172.21.15.99:0/890064291 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3614101920 0x7f3614196f30 secure :-1 s=READY pgs=806 cs=0 l=1 rev1=1 crypto rx=0x7f360000e9f0 tx=0x7f360000eeb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:05:35.393 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.384+0000 7f3611ffb640 1 -- 172.21.15.99:0/890064291 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f360000cd80 con 0x7f3614101920 2024-03-19T02:05:35.393 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.384+0000 7f3611ffb640 1 -- 172.21.15.99:0/890064291 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f3600010910 con 0x7f3614101920 2024-03-19T02:05:35.393 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.384+0000 7f361ac4b640 1 -- 172.21.15.99:0/890064291 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f361419e900 con 0x7f3614101920 2024-03-19T02:05:35.394 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.384+0000 7f3611ffb640 1 -- 172.21.15.99:0/890064291 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3600018990 con 0x7f3614101920 2024-03-19T02:05:35.394 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.384+0000 7f361ac4b640 1 -- 172.21.15.99:0/890064291 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f361419ee50 con 0x7f3614101920 2024-03-19T02:05:35.394 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.385+0000 7f361ac4b640 1 -- 172.21.15.99:0/890064291 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f3614107a50 con 0x7f3614101920 2024-03-19T02:05:35.394 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.386+0000 7f3611ffb640 1 -- 172.21.15.99:0/890064291 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f3600010ac0 con 0x7f3614101920 2024-03-19T02:05:35.394 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.387+0000 7f3611ffb640 1 --2- 172.21.15.99:0/890064291 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f36040a2800 0x7f36040a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:35.394 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.387+0000 7f36189e7640 1 --2- 172.21.15.99:0/890064291 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f36040a2800 0x7f36040a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:35.395 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.387+0000 7f3611ffb640 1 -- 172.21.15.99:0/890064291 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f36000cf1c0 con 0x7f3614101920 2024-03-19T02:05:35.395 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.387+0000 7f36189e7640 1 --2- 172.21.15.99:0/890064291 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f36040a2800 0x7f36040a4cc0 secure :-1 s=READY pgs=398 cs=0 l=1 rev1=1 crypto rx=0x7f3614101780 tx=0x7f3608005f90 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:05:35.395 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.390+0000 7f3611ffb640 1 -- 172.21.15.99:0/890064291 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f360009bbd0 con 0x7f3614101920 2024-03-19T02:05:35.572 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.572+0000 7f361ac4b640 1 -- 172.21.15.99:0/890064291 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) -- 0x7f361402d380 con 0x7f3614101920 2024-03-19T02:05:35.572 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.572+0000 7f3611ffb640 1 -- 172.21.15.99:0/890064291 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v92) ==== 93+0+10 (secure 0 0 0) 0x7f360009fc70 con 0x7f3614101920 2024-03-19T02:05:35.572 INFO:teuthology.orchestra.run.smithi099.stdout:pg_num: 1 2024-03-19T02:05:35.574 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.574+0000 7f361ac4b640 1 -- 172.21.15.99:0/890064291 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f36040a2800 msgr2=0x7f36040a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:05:35.575 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.574+0000 7f361ac4b640 1 --2- 172.21.15.99:0/890064291 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f36040a2800 0x7f36040a4cc0 secure :-1 s=READY pgs=398 cs=0 l=1 rev1=1 crypto rx=0x7f3614101780 tx=0x7f3608005f90 comp rx=0 tx=0).stop 2024-03-19T02:05:35.575 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.574+0000 7f361ac4b640 1 -- 172.21.15.99:0/890064291 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3614101920 msgr2=0x7f3614196f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:05:35.575 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.574+0000 7f361ac4b640 1 --2- 172.21.15.99:0/890064291 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3614101920 0x7f3614196f30 secure :-1 s=READY pgs=806 cs=0 l=1 rev1=1 crypto rx=0x7f360000e9f0 tx=0x7f360000eeb0 comp rx=0 tx=0).stop 2024-03-19T02:05:35.575 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.574+0000 7f361ac4b640 1 -- 172.21.15.99:0/890064291 shutdown_connections 2024-03-19T02:05:35.575 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.574+0000 7f361ac4b640 1 --2- 172.21.15.99:0/890064291 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f36141022e0 0x7f361419dfa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:35.575 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.574+0000 7f361ac4b640 1 --2- 172.21.15.99:0/890064291 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f3614101920 0x7f3614196f30 unknown :-1 s=CLOSED pgs=806 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:35.575 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.574+0000 7f361ac4b640 1 --2- 172.21.15.99:0/890064291 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f36040a2800 0x7f36040a4cc0 unknown :-1 s=CLOSED pgs=398 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:35.575 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.574+0000 7f361ac4b640 1 --2- 172.21.15.99:0/890064291 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f3614100730 0x7f36141969f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:35.575 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.574+0000 7f361ac4b640 1 -- 172.21.15.99:0/890064291 >> 172.21.15.99:0/890064291 conn(0x7f36140fbcc0 msgr2=0x7f3614104c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:05:35.576 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.574+0000 7f361ac4b640 1 -- 172.21.15.99:0/890064291 shutdown_connections 2024-03-19T02:05:35.576 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.574+0000 7f361ac4b640 1 -- 172.21.15.99:0/890064291 wait complete. 2024-03-19T02:05:35.585 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-19T02:05:35.696 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.694+0000 7f6a97553640 1 -- 172.21.15.99:0/1769574757 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6a901022d0 msgr2=0x7f6a90108b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:05:35.696 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.694+0000 7f6a97553640 1 --2- 172.21.15.99:0/1769574757 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6a901022d0 0x7f6a90108b60 secure :-1 s=READY pgs=807 cs=0 l=1 rev1=1 crypto rx=0x7f6a8c009900 tx=0x7f6a8c02bc30 comp rx=0 tx=0).stop 2024-03-19T02:05:35.696 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.694+0000 7f6a97553640 1 -- 172.21.15.99:0/1769574757 shutdown_connections 2024-03-19T02:05:35.696 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.694+0000 7f6a97553640 1 --2- 172.21.15.99:0/1769574757 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6a90101910 0x7f6a90101d90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:35.697 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.694+0000 7f6a97553640 1 --2- 172.21.15.99:0/1769574757 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6a901022d0 0x7f6a90108b60 unknown :-1 s=CLOSED pgs=807 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:35.697 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.694+0000 7f6a97553640 1 --2- 172.21.15.99:0/1769574757 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f6a90100720 0x7f6a90100b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:35.697 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.694+0000 7f6a97553640 1 -- 172.21.15.99:0/1769574757 >> 172.21.15.99:0/1769574757 conn(0x7f6a900fbcb0 msgr2=0x7f6a900fe0d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:05:35.697 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.694+0000 7f6a97553640 1 -- 172.21.15.99:0/1769574757 shutdown_connections 2024-03-19T02:05:35.697 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.694+0000 7f6a97553640 1 -- 172.21.15.99:0/1769574757 wait complete. 2024-03-19T02:05:35.697 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.695+0000 7f6a97553640 1 Processor -- start 2024-03-19T02:05:35.697 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.695+0000 7f6a97553640 1 -- start start 2024-03-19T02:05:35.697 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.695+0000 7f6a97553640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6a90100720 0x7f6a90197ac0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:35.697 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.695+0000 7f6a97553640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6a90101910 0x7f6a90198000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:35.697 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.695+0000 7f6a97553640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f6a901022d0 0x7f6a9019f070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:35.697 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.695+0000 7f6a97553640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f6a9010d4c0 con 0x7f6a90100720 2024-03-19T02:05:35.697 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.695+0000 7f6a97553640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f6a9010d340 con 0x7f6a90101910 2024-03-19T02:05:35.697 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.695+0000 7f6a97553640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f6a9010d640 con 0x7f6a901022d0 2024-03-19T02:05:35.697 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.695+0000 7f6a952ef640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6a90100720 0x7f6a90197ac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:35.698 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.695+0000 7f6a952ef640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6a90100720 0x7f6a90197ac0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:45024/0 (socket says 172.21.15.99:45024) 2024-03-19T02:05:35.698 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.695+0000 7f6a952ef640 1 -- 172.21.15.99:0/3983860069 learned_addr learned my addr 172.21.15.99:0/3983860069 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:05:35.698 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.695+0000 7f6a94aee640 1 --2- 172.21.15.99:0/3983860069 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6a90101910 0x7f6a90198000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:35.698 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.695+0000 7f6a95af0640 1 --2- 172.21.15.99:0/3983860069 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f6a901022d0 0x7f6a9019f070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:35.698 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.695+0000 7f6a952ef640 1 -- 172.21.15.99:0/3983860069 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f6a901022d0 msgr2=0x7f6a9019f070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:05:35.698 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.695+0000 7f6a952ef640 1 --2- 172.21.15.99:0/3983860069 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f6a901022d0 0x7f6a9019f070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:35.698 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.695+0000 7f6a952ef640 1 -- 172.21.15.99:0/3983860069 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6a90101910 msgr2=0x7f6a90198000 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:05:35.698 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.695+0000 7f6a952ef640 1 --2- 172.21.15.99:0/3983860069 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6a90101910 0x7f6a90198000 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:35.698 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.695+0000 7f6a952ef640 1 -- 172.21.15.99:0/3983860069 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f6a9019f6e0 con 0x7f6a90100720 2024-03-19T02:05:35.698 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.696+0000 7f6a952ef640 1 --2- 172.21.15.99:0/3983860069 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6a90100720 0x7f6a90197ac0 secure :-1 s=READY pgs=808 cs=0 l=1 rev1=1 crypto rx=0x7f6a8800ebc0 tx=0x7f6a8800c6a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:05:35.698 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.696+0000 7f6a867fc640 1 -- 172.21.15.99:0/3983860069 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6a88003b70 con 0x7f6a90100720 2024-03-19T02:05:35.698 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.696+0000 7f6a867fc640 1 -- 172.21.15.99:0/3983860069 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f6a88003d10 con 0x7f6a90100720 2024-03-19T02:05:35.698 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.696+0000 7f6a97553640 1 -- 172.21.15.99:0/3983860069 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f6a9019f9d0 con 0x7f6a90100720 2024-03-19T02:05:35.699 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.696+0000 7f6a867fc640 1 -- 172.21.15.99:0/3983860069 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6a88019980 con 0x7f6a90100720 2024-03-19T02:05:35.699 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.696+0000 7f6a97553640 1 -- 172.21.15.99:0/3983860069 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f6a9019ff20 con 0x7f6a90100720 2024-03-19T02:05:35.699 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.697+0000 7f6a97553640 1 -- 172.21.15.99:0/3983860069 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f6a90068610 con 0x7f6a90100720 2024-03-19T02:05:35.699 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.698+0000 7f6a867fc640 1 -- 172.21.15.99:0/3983860069 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f6a88019b20 con 0x7f6a90100720 2024-03-19T02:05:35.700 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.698+0000 7f6a867fc640 1 --2- 172.21.15.99:0/3983860069 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f6a7c0a2800 0x7f6a7c0a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:35.700 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.698+0000 7f6a94aee640 1 --2- 172.21.15.99:0/3983860069 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f6a7c0a2800 0x7f6a7c0a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:35.700 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.698+0000 7f6a867fc640 1 -- 172.21.15.99:0/3983860069 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f6a880cf3c0 con 0x7f6a90100720 2024-03-19T02:05:35.700 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.699+0000 7f6a94aee640 1 --2- 172.21.15.99:0/3983860069 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f6a7c0a2800 0x7f6a7c0a4cc0 secure :-1 s=READY pgs=399 cs=0 l=1 rev1=1 crypto rx=0x7f6a800095f0 tx=0x7f6a80005f90 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:05:35.701 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.702+0000 7f6a867fc640 1 -- 172.21.15.99:0/3983860069 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f6a8809c4b0 con 0x7f6a90100720 2024-03-19T02:05:35.883 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.882+0000 7f6a97553640 1 -- 172.21.15.99:0/3983860069 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) -- 0x7f6a9002d270 con 0x7f6a90100720 2024-03-19T02:05:35.883 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.882+0000 7f6a867fc640 1 -- 172.21.15.99:0/3983860069 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v92) ==== 104+0+11 (secure 0 0 0) 0x7f6a880a0e20 con 0x7f6a90100720 2024-03-19T02:05:35.883 INFO:teuthology.orchestra.run.smithi099.stdout:pg_num: 32 2024-03-19T02:05:35.884 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.884+0000 7f6a97553640 1 -- 172.21.15.99:0/3983860069 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f6a7c0a2800 msgr2=0x7f6a7c0a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:05:35.884 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.884+0000 7f6a97553640 1 --2- 172.21.15.99:0/3983860069 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f6a7c0a2800 0x7f6a7c0a4cc0 secure :-1 s=READY pgs=399 cs=0 l=1 rev1=1 crypto rx=0x7f6a800095f0 tx=0x7f6a80005f90 comp rx=0 tx=0).stop 2024-03-19T02:05:35.884 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.884+0000 7f6a97553640 1 -- 172.21.15.99:0/3983860069 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6a90100720 msgr2=0x7f6a90197ac0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:05:35.885 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.884+0000 7f6a97553640 1 --2- 172.21.15.99:0/3983860069 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6a90100720 0x7f6a90197ac0 secure :-1 s=READY pgs=808 cs=0 l=1 rev1=1 crypto rx=0x7f6a8800ebc0 tx=0x7f6a8800c6a0 comp rx=0 tx=0).stop 2024-03-19T02:05:35.885 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.884+0000 7f6a97553640 1 -- 172.21.15.99:0/3983860069 shutdown_connections 2024-03-19T02:05:35.885 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.884+0000 7f6a97553640 1 --2- 172.21.15.99:0/3983860069 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f6a7c0a2800 0x7f6a7c0a4cc0 unknown :-1 s=CLOSED pgs=399 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:35.885 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.884+0000 7f6a97553640 1 --2- 172.21.15.99:0/3983860069 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f6a90101910 0x7f6a90198000 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:35.885 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.884+0000 7f6a97553640 1 --2- 172.21.15.99:0/3983860069 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f6a901022d0 0x7f6a9019f070 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:35.885 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.884+0000 7f6a97553640 1 --2- 172.21.15.99:0/3983860069 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f6a90100720 0x7f6a90197ac0 unknown :-1 s=CLOSED pgs=808 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:35.885 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.884+0000 7f6a97553640 1 -- 172.21.15.99:0/3983860069 >> 172.21.15.99:0/3983860069 conn(0x7f6a900fbcb0 msgr2=0x7f6a90104bb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:05:35.885 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.884+0000 7f6a97553640 1 -- 172.21.15.99:0/3983860069 shutdown_connections 2024-03-19T02:05:35.885 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:35.885+0000 7f6a97553640 1 -- 172.21.15.99:0/3983860069 wait complete. 2024-03-19T02:05:35.896 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-19T02:05:36.016 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.014+0000 7ff1af097640 1 -- 172.21.15.99:0/469160240 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff1a81033e0 msgr2=0x7ff1a8109c70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:05:36.016 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.014+0000 7ff1af097640 1 --2- 172.21.15.99:0/469160240 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff1a81033e0 0x7ff1a8109c70 secure :-1 s=READY pgs=809 cs=0 l=1 rev1=1 crypto rx=0x7ff198009900 tx=0x7ff19802bc30 comp rx=0 tx=0).stop 2024-03-19T02:05:36.016 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.014+0000 7ff1af097640 1 -- 172.21.15.99:0/469160240 shutdown_connections 2024-03-19T02:05:36.016 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.014+0000 7ff1af097640 1 --2- 172.21.15.99:0/469160240 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff1a8102a20 0x7ff1a8102ea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:36.016 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.014+0000 7ff1af097640 1 --2- 172.21.15.99:0/469160240 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff1a81033e0 0x7ff1a8109c70 unknown :-1 s=CLOSED pgs=809 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:36.016 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.014+0000 7ff1af097640 1 --2- 172.21.15.99:0/469160240 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff1a8101830 0x7ff1a8101c50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:36.017 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.014+0000 7ff1af097640 1 -- 172.21.15.99:0/469160240 >> 172.21.15.99:0/469160240 conn(0x7ff1a80fcdc0 msgr2=0x7ff1a80ff1e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:05:36.017 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.014+0000 7ff1af097640 1 -- 172.21.15.99:0/469160240 shutdown_connections 2024-03-19T02:05:36.017 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.014+0000 7ff1af097640 1 -- 172.21.15.99:0/469160240 wait complete. 2024-03-19T02:05:36.017 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.014+0000 7ff1af097640 1 Processor -- start 2024-03-19T02:05:36.017 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.014+0000 7ff1af097640 1 -- start start 2024-03-19T02:05:36.017 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.015+0000 7ff1af097640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff1a8101830 0x7ff1a8197af0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:36.017 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.015+0000 7ff1af097640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff1a8102a20 0x7ff1a8198030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:36.017 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.015+0000 7ff1af097640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff1a81033e0 0x7ff1a819f0a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:36.017 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.015+0000 7ff1af097640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7ff1a810d4b0 con 0x7ff1a8102a20 2024-03-19T02:05:36.017 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.015+0000 7ff1af097640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7ff1a810d330 con 0x7ff1a8101830 2024-03-19T02:05:36.017 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.015+0000 7ff1af097640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7ff1a810d630 con 0x7ff1a81033e0 2024-03-19T02:05:36.017 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.015+0000 7ff1a7fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff1a8102a20 0x7ff1a8198030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:36.017 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.015+0000 7ff1ace33640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff1a8101830 0x7ff1a8197af0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:36.017 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.015+0000 7ff1a7fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff1a8102a20 0x7ff1a8198030 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:45048/0 (socket says 172.21.15.99:45048) 2024-03-19T02:05:36.018 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.015+0000 7ff1ace33640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff1a8101830 0x7ff1a8197af0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:44778/0 (socket says 172.21.15.99:44778) 2024-03-19T02:05:36.018 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.015+0000 7ff1a7fff640 1 -- 172.21.15.99:0/1086307220 learned_addr learned my addr 172.21.15.99:0/1086307220 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:05:36.018 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.015+0000 7ff1ad634640 1 --2- 172.21.15.99:0/1086307220 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff1a81033e0 0x7ff1a819f0a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:36.018 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.015+0000 7ff1a7fff640 1 -- 172.21.15.99:0/1086307220 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff1a81033e0 msgr2=0x7ff1a819f0a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:05:36.018 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.015+0000 7ff1a7fff640 1 --2- 172.21.15.99:0/1086307220 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff1a81033e0 0x7ff1a819f0a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:36.018 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.015+0000 7ff1a7fff640 1 -- 172.21.15.99:0/1086307220 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff1a8101830 msgr2=0x7ff1a8197af0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:05:36.018 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.015+0000 7ff1a7fff640 1 --2- 172.21.15.99:0/1086307220 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff1a8101830 0x7ff1a8197af0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:36.018 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.015+0000 7ff1a7fff640 1 -- 172.21.15.99:0/1086307220 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ff1a819f710 con 0x7ff1a8102a20 2024-03-19T02:05:36.018 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.015+0000 7ff1ace33640 1 --2- 172.21.15.99:0/1086307220 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff1a8101830 0x7ff1a8197af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:05:36.018 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.015+0000 7ff1a7fff640 1 --2- 172.21.15.99:0/1086307220 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff1a8102a20 0x7ff1a8198030 secure :-1 s=READY pgs=810 cs=0 l=1 rev1=1 crypto rx=0x7ff19c00ea60 tx=0x7ff19c00ef20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:05:36.018 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.015+0000 7ff1a5ffb640 1 -- 172.21.15.99:0/1086307220 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff19c00cde0 con 0x7ff1a8102a20 2024-03-19T02:05:36.018 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.015+0000 7ff1a5ffb640 1 -- 172.21.15.99:0/1086307220 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7ff19c0035f0 con 0x7ff1a8102a20 2024-03-19T02:05:36.018 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.015+0000 7ff1af097640 1 -- 172.21.15.99:0/1086307220 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ff1a819fa00 con 0x7ff1a8102a20 2024-03-19T02:05:36.018 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.015+0000 7ff1a5ffb640 1 -- 172.21.15.99:0/1086307220 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff19c010670 con 0x7ff1a8102a20 2024-03-19T02:05:36.019 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.016+0000 7ff1af097640 1 -- 172.21.15.99:0/1086307220 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7ff1a819ff50 con 0x7ff1a8102a20 2024-03-19T02:05:36.019 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.017+0000 7ff1af097640 1 -- 172.21.15.99:0/1086307220 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7ff1a8108b50 con 0x7ff1a8102a20 2024-03-19T02:05:36.020 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.020+0000 7ff1a5ffb640 1 -- 172.21.15.99:0/1086307220 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7ff19c0037a0 con 0x7ff1a8102a20 2024-03-19T02:05:36.021 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.021+0000 7ff1a5ffb640 1 --2- 172.21.15.99:0/1086307220 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff1900a2880 0x7ff1900a4d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:36.021 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.021+0000 7ff1ace33640 1 --2- 172.21.15.99:0/1086307220 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff1900a2880 0x7ff1900a4d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:36.021 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.021+0000 7ff1a5ffb640 1 -- 172.21.15.99:0/1086307220 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7ff19c0ceeb0 con 0x7ff1a8102a20 2024-03-19T02:05:36.022 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.022+0000 7ff1ace33640 1 --2- 172.21.15.99:0/1086307220 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff1900a2880 0x7ff1900a4d40 secure :-1 s=READY pgs=400 cs=0 l=1 rev1=1 crypto rx=0x7ff1a8102880 tx=0x7ff1a0009500 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:05:36.025 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.025+0000 7ff1a5ffb640 1 -- 172.21.15.99:0/1086307220 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7ff19c09bf40 con 0x7ff1a8102a20 2024-03-19T02:05:36.248 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.248+0000 7ff1af097640 1 -- 172.21.15.99:0/1086307220 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) -- 0x7ff1a802d380 con 0x7ff1a8102a20 2024-03-19T02:05:36.248 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.248+0000 7ff1a5ffb640 1 -- 172.21.15.99:0/1086307220 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v92) ==== 100+0+11 (secure 0 0 0) 0x7ff19c09b840 con 0x7ff1a8102a20 2024-03-19T02:05:36.248 INFO:teuthology.orchestra.run.smithi099.stdout:pg_num: 64 2024-03-19T02:05:36.250 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.250+0000 7ff1af097640 1 -- 172.21.15.99:0/1086307220 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff1900a2880 msgr2=0x7ff1900a4d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:05:36.250 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.250+0000 7ff1af097640 1 --2- 172.21.15.99:0/1086307220 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff1900a2880 0x7ff1900a4d40 secure :-1 s=READY pgs=400 cs=0 l=1 rev1=1 crypto rx=0x7ff1a8102880 tx=0x7ff1a0009500 comp rx=0 tx=0).stop 2024-03-19T02:05:36.250 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.250+0000 7ff1af097640 1 -- 172.21.15.99:0/1086307220 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff1a8102a20 msgr2=0x7ff1a8198030 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:05:36.251 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.250+0000 7ff1af097640 1 --2- 172.21.15.99:0/1086307220 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff1a8102a20 0x7ff1a8198030 secure :-1 s=READY pgs=810 cs=0 l=1 rev1=1 crypto rx=0x7ff19c00ea60 tx=0x7ff19c00ef20 comp rx=0 tx=0).stop 2024-03-19T02:05:36.251 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.250+0000 7ff1af097640 1 -- 172.21.15.99:0/1086307220 shutdown_connections 2024-03-19T02:05:36.251 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.250+0000 7ff1af097640 1 --2- 172.21.15.99:0/1086307220 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ff1a81033e0 0x7ff1a819f0a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:36.251 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.250+0000 7ff1af097640 1 --2- 172.21.15.99:0/1086307220 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ff1a8102a20 0x7ff1a8198030 unknown :-1 s=CLOSED pgs=810 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:36.251 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.250+0000 7ff1af097640 1 --2- 172.21.15.99:0/1086307220 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ff1900a2880 0x7ff1900a4d40 unknown :-1 s=CLOSED pgs=400 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:36.251 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.250+0000 7ff1af097640 1 --2- 172.21.15.99:0/1086307220 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ff1a8101830 0x7ff1a8197af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:36.251 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.250+0000 7ff1af097640 1 -- 172.21.15.99:0/1086307220 >> 172.21.15.99:0/1086307220 conn(0x7ff1a80fcdc0 msgr2=0x7ff1a8105d20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:05:36.251 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.250+0000 7ff1af097640 1 -- 172.21.15.99:0/1086307220 shutdown_connections 2024-03-19T02:05:36.251 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.250+0000 7ff1af097640 1 -- 172.21.15.99:0/1086307220 wait complete. 2024-03-19T02:05:36.260 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:05:35 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/2944248752' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-19T02:05:36.260 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:05:35 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/890064291' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-03-19T02:05:36.261 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph mgr dump --format=json-pretty 2024-03-19T02:05:36.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:05:35 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/2944248752' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-19T02:05:36.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:05:35 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/890064291' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-03-19T02:05:36.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:05:35 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/2944248752' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-19T02:05:36.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:05:35 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/890064291' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-03-19T02:05:36.363 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.363+0000 7fde82459640 1 -- 172.21.15.99:0/1323979863 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fde7c102a30 msgr2=0x7fde7c102eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:05:36.363 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.363+0000 7fde82459640 1 --2- 172.21.15.99:0/1323979863 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fde7c102a30 0x7fde7c102eb0 secure :-1 s=READY pgs=811 cs=0 l=1 rev1=1 crypto rx=0x7fde70009900 tx=0x7fde7002bc30 comp rx=0 tx=0).stop 2024-03-19T02:05:36.363 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.363+0000 7fde82459640 1 -- 172.21.15.99:0/1323979863 shutdown_connections 2024-03-19T02:05:36.363 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.363+0000 7fde82459640 1 --2- 172.21.15.99:0/1323979863 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fde7c1033f0 0x7fde7c109c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:36.363 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.363+0000 7fde82459640 1 --2- 172.21.15.99:0/1323979863 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fde7c102a30 0x7fde7c102eb0 unknown :-1 s=CLOSED pgs=811 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:36.364 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.363+0000 7fde82459640 1 --2- 172.21.15.99:0/1323979863 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fde7c101840 0x7fde7c101c60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:36.364 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.363+0000 7fde82459640 1 -- 172.21.15.99:0/1323979863 >> 172.21.15.99:0/1323979863 conn(0x7fde7c0fcdd0 msgr2=0x7fde7c0ff1f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:05:36.364 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.363+0000 7fde82459640 1 -- 172.21.15.99:0/1323979863 shutdown_connections 2024-03-19T02:05:36.364 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.363+0000 7fde82459640 1 -- 172.21.15.99:0/1323979863 wait complete. 2024-03-19T02:05:36.364 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.363+0000 7fde82459640 1 Processor -- start 2024-03-19T02:05:36.364 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.363+0000 7fde82459640 1 -- start start 2024-03-19T02:05:36.366 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.364+0000 7fde82459640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fde7c101840 0x7fde7c197b40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:36.366 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.364+0000 7fde82459640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fde7c102a30 0x7fde7c198080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:36.366 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.364+0000 7fde82459640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fde7c1033f0 0x7fde7c19f0f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:36.366 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.364+0000 7fde82459640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fde7c10d4c0 con 0x7fde7c102a30 2024-03-19T02:05:36.366 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.364+0000 7fde82459640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fde7c10d340 con 0x7fde7c1033f0 2024-03-19T02:05:36.366 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.364+0000 7fde82459640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fde7c10d640 con 0x7fde7c101840 2024-03-19T02:05:36.366 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.364+0000 7fde7b7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fde7c102a30 0x7fde7c198080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:36.366 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.364+0000 7fde7b7fe640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fde7c102a30 0x7fde7c198080 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:45068/0 (socket says 172.21.15.99:45068) 2024-03-19T02:05:36.367 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.364+0000 7fde7b7fe640 1 -- 172.21.15.99:0/3495611872 learned_addr learned my addr 172.21.15.99:0/3495611872 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:05:36.367 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.364+0000 7fde7bfff640 1 --2- 172.21.15.99:0/3495611872 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fde7c101840 0x7fde7c197b40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:36.367 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.364+0000 7fde809f6640 1 --2- 172.21.15.99:0/3495611872 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fde7c1033f0 0x7fde7c19f0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:36.367 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.364+0000 7fde7b7fe640 1 -- 172.21.15.99:0/3495611872 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fde7c101840 msgr2=0x7fde7c197b40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:05:36.367 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.364+0000 7fde7b7fe640 1 --2- 172.21.15.99:0/3495611872 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fde7c101840 0x7fde7c197b40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:36.367 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.364+0000 7fde7b7fe640 1 -- 172.21.15.99:0/3495611872 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fde7c1033f0 msgr2=0x7fde7c19f0f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:05:36.367 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.364+0000 7fde7b7fe640 1 --2- 172.21.15.99:0/3495611872 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fde7c1033f0 0x7fde7c19f0f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:36.367 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.364+0000 7fde7b7fe640 1 -- 172.21.15.99:0/3495611872 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fde7c19f760 con 0x7fde7c102a30 2024-03-19T02:05:36.367 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.364+0000 7fde7b7fe640 1 --2- 172.21.15.99:0/3495611872 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fde7c102a30 0x7fde7c198080 secure :-1 s=READY pgs=812 cs=0 l=1 rev1=1 crypto rx=0x7fde70009230 tx=0x7fde7002c650 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:05:36.367 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.364+0000 7fde797fa640 1 -- 172.21.15.99:0/3495611872 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fde70041070 con 0x7fde7c102a30 2024-03-19T02:05:36.367 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.364+0000 7fde797fa640 1 -- 172.21.15.99:0/3495611872 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fde70004450 con 0x7fde7c102a30 2024-03-19T02:05:36.367 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.364+0000 7fde82459640 1 -- 172.21.15.99:0/3495611872 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fde7c19f9f0 con 0x7fde7c102a30 2024-03-19T02:05:36.367 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.364+0000 7fde797fa640 1 -- 172.21.15.99:0/3495611872 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fde70003e20 con 0x7fde7c102a30 2024-03-19T02:05:36.367 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.365+0000 7fde82459640 1 -- 172.21.15.99:0/3495611872 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fde7c19fee0 con 0x7fde7c102a30 2024-03-19T02:05:36.368 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.366+0000 7fde82459640 1 -- 172.21.15.99:0/3495611872 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fde7c108b60 con 0x7fde7c102a30 2024-03-19T02:05:36.368 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.367+0000 7fde797fa640 1 -- 172.21.15.99:0/3495611872 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7fde70044070 con 0x7fde7c102a30 2024-03-19T02:05:36.368 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.367+0000 7fde797fa640 1 --2- 172.21.15.99:0/3495611872 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fde640a2800 0x7fde640a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:05:36.368 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.368+0000 7fde7bfff640 1 --2- 172.21.15.99:0/3495611872 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fde640a2800 0x7fde640a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:05:36.369 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.368+0000 7fde797fa640 1 -- 172.21.15.99:0/3495611872 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fde700f0630 con 0x7fde7c102a30 2024-03-19T02:05:36.369 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.368+0000 7fde7bfff640 1 --2- 172.21.15.99:0/3495611872 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fde640a2800 0x7fde640a4cc0 secure :-1 s=READY pgs=401 cs=0 l=1 rev1=1 crypto rx=0x7fde7c102890 tx=0x7fde6c005f90 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:05:36.371 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.372+0000 7fde797fa640 1 -- 172.21.15.99:0/3495611872 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7fde700bd040 con 0x7fde7c102a30 2024-03-19T02:05:36.596 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.596+0000 7fde82459640 1 -- 172.21.15.99:0/3495611872 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "mgr dump", "format": "json-pretty"} v 0) -- 0x7fde7c062450 con 0x7fde7c102a30 2024-03-19T02:05:36.604 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.604+0000 7fde797fa640 1 -- 172.21.15.99:0/3495611872 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "mgr dump", "format": "json-pretty"}]=0 v20) ==== 81+0+739828 (secure 0 0 0) 0x7fde700c10e0 con 0x7fde7c102a30 2024-03-19T02:05:36.614 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T02:05:36.614 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2024-03-19T02:05:36.614 INFO:teuthology.orchestra.run.smithi099.stdout: "epoch": 20, 2024-03-19T02:05:36.614 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.614 INFO:teuthology.orchestra.run.smithi099.stdout: "active_gid": 14150, 2024-03-19T02:05:36.614 INFO:teuthology.orchestra.run.smithi099.stdout: "active_name": "x", 2024-03-19T02:05:36.614 INFO:teuthology.orchestra.run.smithi099.stdout: "active_addrs": { 2024-03-19T02:05:36.615 INFO:teuthology.orchestra.run.smithi099.stdout: "addrvec": [ 2024-03-19T02:05:36.615 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:36.615 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "v2", 2024-03-19T02:05:36.615 INFO:teuthology.orchestra.run.smithi099.stdout: "addr": "172.21.15.99:6800", 2024-03-19T02:05:36.615 INFO:teuthology.orchestra.run.smithi099.stdout: "nonce": 4272410295 2024-03-19T02:05:36.615 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.615 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:36.615 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "v1", 2024-03-19T02:05:36.615 INFO:teuthology.orchestra.run.smithi099.stdout: "addr": "172.21.15.99:6801", 2024-03-19T02:05:36.615 INFO:teuthology.orchestra.run.smithi099.stdout: "nonce": 4272410295 2024-03-19T02:05:36.615 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.615 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-03-19T02:05:36.615 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.615 INFO:teuthology.orchestra.run.smithi099.stdout: "active_addr": "172.21.15.99:6801/4272410295", 2024-03-19T02:05:36.616 INFO:teuthology.orchestra.run.smithi099.stdout: "active_change": "2024-03-19T01:32:06.661109+0000", 2024-03-19T02:05:36.616 INFO:teuthology.orchestra.run.smithi099.stdout: "active_mgr_features": 4540701547738038271, 2024-03-19T02:05:36.616 INFO:teuthology.orchestra.run.smithi099.stdout: "available": true, 2024-03-19T02:05:36.616 INFO:teuthology.orchestra.run.smithi099.stdout: "standbys": [ 2024-03-19T02:05:36.616 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:36.616 INFO:teuthology.orchestra.run.smithi099.stdout: "gid": 24106, 2024-03-19T02:05:36.616 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "z", 2024-03-19T02:05:36.616 INFO:teuthology.orchestra.run.smithi099.stdout: "mgr_features": 4540701547738038271, 2024-03-19T02:05:36.616 INFO:teuthology.orchestra.run.smithi099.stdout: "available_modules": [ 2024-03-19T02:05:36.616 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:36.616 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "alerts", 2024-03-19T02:05:36.616 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:36.616 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:36.616 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:36.617 INFO:teuthology.orchestra.run.smithi099.stdout: "interval": { 2024-03-19T02:05:36.617 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "interval", 2024-03-19T02:05:36.617 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:36.617 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.617 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.617 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "60", 2024-03-19T02:05:36.617 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.617 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.617 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.617 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "How frequently to reexamine health status", 2024-03-19T02:05:36.618 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.618 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.618 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.618 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.618 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:36.618 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:36.618 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.618 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.618 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.618 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.618 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.618 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.618 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.619 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.619 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.619 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.619 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.619 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.619 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.619 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.619 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.619 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.619 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.619 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.619 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.619 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:36.619 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:36.620 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.620 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.620 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.620 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.620 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.620 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.620 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.620 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.620 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.620 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.620 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.620 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.620 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:36.620 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:36.621 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.621 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.621 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.621 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:36.621 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.621 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.621 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.621 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.621 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.621 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.621 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.621 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.621 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.621 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.621 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.622 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.622 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.622 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.622 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.622 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:36.622 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:36.622 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.622 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.622 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.622 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.622 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.622 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.622 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.622 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.623 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.623 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.623 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.623 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.623 INFO:teuthology.orchestra.run.smithi099.stdout: "smtp_destination": { 2024-03-19T02:05:36.623 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "smtp_destination", 2024-03-19T02:05:36.623 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.623 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.623 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.623 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.624 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.624 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.624 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.624 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Email address to send alerts to", 2024-03-19T02:05:36.624 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.624 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.624 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.624 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.624 INFO:teuthology.orchestra.run.smithi099.stdout: "smtp_from_name": { 2024-03-19T02:05:36.624 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "smtp_from_name", 2024-03-19T02:05:36.624 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.624 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.624 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.624 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "Ceph", 2024-03-19T02:05:36.625 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.625 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.625 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.625 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Email From: name", 2024-03-19T02:05:36.625 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.625 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.625 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.625 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.625 INFO:teuthology.orchestra.run.smithi099.stdout: "smtp_host": { 2024-03-19T02:05:36.625 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "smtp_host", 2024-03-19T02:05:36.625 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.625 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.625 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.625 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.625 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.626 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.626 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.626 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "SMTP server", 2024-03-19T02:05:36.626 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.626 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.626 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.626 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.626 INFO:teuthology.orchestra.run.smithi099.stdout: "smtp_password": { 2024-03-19T02:05:36.626 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "smtp_password", 2024-03-19T02:05:36.626 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.626 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.626 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.626 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.626 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.627 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.627 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.627 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Password to authenticate with", 2024-03-19T02:05:36.627 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.627 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.627 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.627 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.627 INFO:teuthology.orchestra.run.smithi099.stdout: "smtp_port": { 2024-03-19T02:05:36.627 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "smtp_port", 2024-03-19T02:05:36.627 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:36.627 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.627 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.628 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "465", 2024-03-19T02:05:36.628 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.628 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.628 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.628 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "SMTP port", 2024-03-19T02:05:36.628 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.628 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.628 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.628 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.628 INFO:teuthology.orchestra.run.smithi099.stdout: "smtp_sender": { 2024-03-19T02:05:36.628 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "smtp_sender", 2024-03-19T02:05:36.628 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.628 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.628 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.628 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.629 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.629 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.629 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.629 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "SMTP envelope sender", 2024-03-19T02:05:36.629 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.629 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.629 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.629 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.629 INFO:teuthology.orchestra.run.smithi099.stdout: "smtp_ssl": { 2024-03-19T02:05:36.629 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "smtp_ssl", 2024-03-19T02:05:36.629 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.629 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.629 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.630 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:36.630 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.630 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.630 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.630 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Use SSL to connect to SMTP server", 2024-03-19T02:05:36.630 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.630 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.630 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.630 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.630 INFO:teuthology.orchestra.run.smithi099.stdout: "smtp_user": { 2024-03-19T02:05:36.630 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "smtp_user", 2024-03-19T02:05:36.631 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.631 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.631 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.631 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.631 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.631 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.631 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.631 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "User to authenticate as", 2024-03-19T02:05:36.631 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.631 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.631 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.631 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.631 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.631 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.632 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:36.632 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "balancer", 2024-03-19T02:05:36.632 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:36.632 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:36.632 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:36.632 INFO:teuthology.orchestra.run.smithi099.stdout: "active": { 2024-03-19T02:05:36.632 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "active", 2024-03-19T02:05:36.632 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.632 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.632 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.632 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:36.632 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.632 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.632 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.633 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "automatically balance PGs across cluster", 2024-03-19T02:05:36.633 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.633 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.633 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.633 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.633 INFO:teuthology.orchestra.run.smithi099.stdout: "begin_time": { 2024-03-19T02:05:36.633 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "begin_time", 2024-03-19T02:05:36.633 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.633 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.633 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.633 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "0000", 2024-03-19T02:05:36.633 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.633 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.633 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.633 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "beginning time of day to automatically balance", 2024-03-19T02:05:36.634 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "This is a time of day in the format HHMM.", 2024-03-19T02:05:36.634 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.634 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.634 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.634 INFO:teuthology.orchestra.run.smithi099.stdout: "begin_weekday": { 2024-03-19T02:05:36.634 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "begin_weekday", 2024-03-19T02:05:36.634 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "uint", 2024-03-19T02:05:36.634 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.634 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.634 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "0", 2024-03-19T02:05:36.634 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "0", 2024-03-19T02:05:36.634 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "6", 2024-03-19T02:05:36.634 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.635 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Restrict automatic balancing to this day of the week or later", 2024-03-19T02:05:36.635 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "0 = Sunday, 1 = Monday, etc.", 2024-03-19T02:05:36.635 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.635 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.635 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.635 INFO:teuthology.orchestra.run.smithi099.stdout: "crush_compat_max_iterations": { 2024-03-19T02:05:36.635 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "crush_compat_max_iterations", 2024-03-19T02:05:36.635 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "uint", 2024-03-19T02:05:36.635 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.635 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.635 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "25", 2024-03-19T02:05:36.635 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "1", 2024-03-19T02:05:36.635 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "250", 2024-03-19T02:05:36.635 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.635 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "maximum number of iterations to attempt optimization", 2024-03-19T02:05:36.636 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.637 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.637 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.637 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.637 INFO:teuthology.orchestra.run.smithi099.stdout: "crush_compat_metrics": { 2024-03-19T02:05:36.637 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "crush_compat_metrics", 2024-03-19T02:05:36.637 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.637 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.637 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.637 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "pgs,objects,bytes", 2024-03-19T02:05:36.638 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.638 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.638 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.638 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "metrics with which to calculate OSD utilization", 2024-03-19T02:05:36.638 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.", 2024-03-19T02:05:36.638 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.638 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.638 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.638 INFO:teuthology.orchestra.run.smithi099.stdout: "crush_compat_step": { 2024-03-19T02:05:36.638 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "crush_compat_step", 2024-03-19T02:05:36.638 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "float", 2024-03-19T02:05:36.638 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.638 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.638 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "0.5", 2024-03-19T02:05:36.639 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "0.001", 2024-03-19T02:05:36.639 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "0.999", 2024-03-19T02:05:36.639 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.639 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "aggressiveness of optimization", 2024-03-19T02:05:36.639 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": ".99 is very aggressive, .01 is less aggressive", 2024-03-19T02:05:36.639 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.639 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.639 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.639 INFO:teuthology.orchestra.run.smithi099.stdout: "end_time": { 2024-03-19T02:05:36.639 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "end_time", 2024-03-19T02:05:36.639 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.639 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.639 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.639 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "2359", 2024-03-19T02:05:36.639 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.640 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.640 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.640 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "ending time of day to automatically balance", 2024-03-19T02:05:36.640 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "This is a time of day in the format HHMM.", 2024-03-19T02:05:36.640 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.640 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.640 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.640 INFO:teuthology.orchestra.run.smithi099.stdout: "end_weekday": { 2024-03-19T02:05:36.640 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "end_weekday", 2024-03-19T02:05:36.640 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "uint", 2024-03-19T02:05:36.640 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.640 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.640 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "0", 2024-03-19T02:05:36.640 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "0", 2024-03-19T02:05:36.641 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "6", 2024-03-19T02:05:36.641 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.641 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Restrict automatic balancing to days of the week earlier than this", 2024-03-19T02:05:36.641 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "0 = Sunday, 1 = Monday, etc.", 2024-03-19T02:05:36.641 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.641 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.641 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.641 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:36.641 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:36.641 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.641 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.641 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.641 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.641 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.642 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.642 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.642 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.642 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.642 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.642 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.642 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.642 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.642 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.642 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.642 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.642 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.642 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.643 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.643 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:36.643 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:36.643 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.643 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.643 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.643 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.643 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.643 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.643 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.643 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.643 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.644 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.644 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.644 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.644 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:36.644 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:36.644 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.644 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.644 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.644 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:36.644 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.644 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.644 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.644 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.644 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.645 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.645 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.645 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.645 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.645 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.645 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.645 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.645 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.645 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.645 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.645 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:36.645 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:36.645 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.645 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.646 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.646 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.646 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.646 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.646 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.646 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.646 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.646 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.646 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.646 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.646 INFO:teuthology.orchestra.run.smithi099.stdout: "min_score": { 2024-03-19T02:05:36.646 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "min_score", 2024-03-19T02:05:36.646 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "float", 2024-03-19T02:05:36.646 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.646 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.647 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "0", 2024-03-19T02:05:36.647 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.647 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.647 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.647 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "minimum score, below which no optimization is attempted", 2024-03-19T02:05:36.647 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.647 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.647 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.647 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.647 INFO:teuthology.orchestra.run.smithi099.stdout: "mode": { 2024-03-19T02:05:36.647 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "mode", 2024-03-19T02:05:36.647 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.647 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.647 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.648 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "upmap", 2024-03-19T02:05:36.648 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.648 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.648 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.648 INFO:teuthology.orchestra.run.smithi099.stdout: "crush-compat", 2024-03-19T02:05:36.648 INFO:teuthology.orchestra.run.smithi099.stdout: "none", 2024-03-19T02:05:36.648 INFO:teuthology.orchestra.run.smithi099.stdout: "read", 2024-03-19T02:05:36.648 INFO:teuthology.orchestra.run.smithi099.stdout: "upmap", 2024-03-19T02:05:36.648 INFO:teuthology.orchestra.run.smithi099.stdout: "upmap-read" 2024-03-19T02:05:36.648 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.648 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Balancer mode", 2024-03-19T02:05:36.648 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.648 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.649 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.649 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.649 INFO:teuthology.orchestra.run.smithi099.stdout: "pool_ids": { 2024-03-19T02:05:36.649 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "pool_ids", 2024-03-19T02:05:36.649 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.649 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.649 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.649 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.649 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.649 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.649 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.649 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "pools which the automatic balancing will be limited to", 2024-03-19T02:05:36.649 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.649 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.650 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.650 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.650 INFO:teuthology.orchestra.run.smithi099.stdout: "sleep_interval": { 2024-03-19T02:05:36.650 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "sleep_interval", 2024-03-19T02:05:36.650 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:36.650 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.650 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.650 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "60", 2024-03-19T02:05:36.650 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.650 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.650 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.650 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "how frequently to wake up and attempt optimization", 2024-03-19T02:05:36.650 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.650 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.651 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.651 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.651 INFO:teuthology.orchestra.run.smithi099.stdout: "upmap_max_deviation": { 2024-03-19T02:05:36.651 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "upmap_max_deviation", 2024-03-19T02:05:36.651 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:36.651 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.651 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.651 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "5", 2024-03-19T02:05:36.651 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "1", 2024-03-19T02:05:36.651 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.651 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.651 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "deviation below which no optimization is attempted", 2024-03-19T02:05:36.651 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "If the number of PGs are within this count then no optimization is attempted", 2024-03-19T02:05:36.651 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.651 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.652 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.652 INFO:teuthology.orchestra.run.smithi099.stdout: "upmap_max_optimizations": { 2024-03-19T02:05:36.652 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "upmap_max_optimizations", 2024-03-19T02:05:36.652 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "uint", 2024-03-19T02:05:36.652 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.652 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.652 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "10", 2024-03-19T02:05:36.652 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.652 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.652 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.652 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "maximum upmap optimizations to make per attempt", 2024-03-19T02:05:36.652 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.652 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.652 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.652 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.653 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.653 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.653 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:36.653 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "cephadm", 2024-03-19T02:05:36.653 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:36.653 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:36.653 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:36.653 INFO:teuthology.orchestra.run.smithi099.stdout: "agent_down_multiplier": { 2024-03-19T02:05:36.653 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "agent_down_multiplier", 2024-03-19T02:05:36.653 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "float", 2024-03-19T02:05:36.653 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.653 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.653 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "3.0", 2024-03-19T02:05:36.653 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.653 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.654 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.654 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Multiplied by agent refresh rate to calculate how long agent must not report before being marked down", 2024-03-19T02:05:36.654 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.654 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.654 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.654 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.654 INFO:teuthology.orchestra.run.smithi099.stdout: "agent_refresh_rate": { 2024-03-19T02:05:36.654 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "agent_refresh_rate", 2024-03-19T02:05:36.654 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:36.654 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.654 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.654 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "20", 2024-03-19T02:05:36.654 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.654 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.655 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.655 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "How often agent on each host will try to gather and send metadata", 2024-03-19T02:05:36.655 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.655 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.655 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.655 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.655 INFO:teuthology.orchestra.run.smithi099.stdout: "agent_starting_port": { 2024-03-19T02:05:36.655 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "agent_starting_port", 2024-03-19T02:05:36.655 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:36.655 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.655 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.655 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "4721", 2024-03-19T02:05:36.655 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.655 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.656 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.656 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)", 2024-03-19T02:05:36.656 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.656 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.656 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.656 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.656 INFO:teuthology.orchestra.run.smithi099.stdout: "allow_ptrace": { 2024-03-19T02:05:36.656 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "allow_ptrace", 2024-03-19T02:05:36.656 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.656 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.656 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.656 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.656 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.656 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.657 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.657 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "allow SYS_PTRACE capability on ceph containers", 2024-03-19T02:05:36.657 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.", 2024-03-19T02:05:36.657 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.657 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.657 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.657 INFO:teuthology.orchestra.run.smithi099.stdout: "autotune_interval": { 2024-03-19T02:05:36.657 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "autotune_interval", 2024-03-19T02:05:36.657 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:36.657 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.657 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.657 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "600", 2024-03-19T02:05:36.657 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.657 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.657 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.658 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "how frequently to autotune daemon memory", 2024-03-19T02:05:36.658 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.658 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.658 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.658 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.658 INFO:teuthology.orchestra.run.smithi099.stdout: "autotune_memory_target_ratio": { 2024-03-19T02:05:36.658 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "autotune_memory_target_ratio", 2024-03-19T02:05:36.658 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "float", 2024-03-19T02:05:36.658 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.658 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.658 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "0.7", 2024-03-19T02:05:36.658 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.658 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.659 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.659 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "ratio of total system memory to divide amongst autotuned daemons", 2024-03-19T02:05:36.659 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.659 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.659 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.659 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.659 INFO:teuthology.orchestra.run.smithi099.stdout: "cephadm_log_destination": { 2024-03-19T02:05:36.659 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "cephadm_log_destination", 2024-03-19T02:05:36.659 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.659 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.659 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.659 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.659 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.659 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.659 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.660 INFO:teuthology.orchestra.run.smithi099.stdout: "file", 2024-03-19T02:05:36.660 INFO:teuthology.orchestra.run.smithi099.stdout: "file,syslog", 2024-03-19T02:05:36.660 INFO:teuthology.orchestra.run.smithi099.stdout: "syslog" 2024-03-19T02:05:36.660 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.660 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Destination for cephadm command's persistent logging", 2024-03-19T02:05:36.660 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.660 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.660 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.660 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.660 INFO:teuthology.orchestra.run.smithi099.stdout: "cgroups_split": { 2024-03-19T02:05:36.660 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "cgroups_split", 2024-03-19T02:05:36.660 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.660 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.660 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.661 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:36.661 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.661 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.661 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.661 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Pass --cgroups=split when cephadm creates containers (currently podman only)", 2024-03-19T02:05:36.661 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.661 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.661 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.661 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.661 INFO:teuthology.orchestra.run.smithi099.stdout: "config_checks_enabled": { 2024-03-19T02:05:36.661 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "config_checks_enabled", 2024-03-19T02:05:36.661 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.661 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.661 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.661 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.662 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.662 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.662 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.662 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Enable or disable the cephadm configuration analysis", 2024-03-19T02:05:36.662 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.662 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.662 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.662 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.662 INFO:teuthology.orchestra.run.smithi099.stdout: "config_dashboard": { 2024-03-19T02:05:36.662 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "config_dashboard", 2024-03-19T02:05:36.662 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.662 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.662 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.662 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:36.663 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.663 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.663 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.663 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "manage configs like API endpoints in Dashboard.", 2024-03-19T02:05:36.663 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.663 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.663 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.663 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.663 INFO:teuthology.orchestra.run.smithi099.stdout: "container_image_alertmanager": { 2024-03-19T02:05:36.663 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "container_image_alertmanager", 2024-03-19T02:05:36.663 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.663 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.663 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.663 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "quay.io/prometheus/alertmanager:v0.25.0", 2024-03-19T02:05:36.663 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.664 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.664 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.664 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Prometheus container image", 2024-03-19T02:05:36.664 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.664 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.664 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.664 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.664 INFO:teuthology.orchestra.run.smithi099.stdout: "container_image_base": { 2024-03-19T02:05:36.664 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "container_image_base", 2024-03-19T02:05:36.664 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.664 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.664 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.665 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "quay.io/ceph/ceph", 2024-03-19T02:05:36.665 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.665 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.665 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.665 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Container image name, without the tag", 2024-03-19T02:05:36.665 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.665 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.665 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.665 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.665 INFO:teuthology.orchestra.run.smithi099.stdout: "container_image_elasticsearch": { 2024-03-19T02:05:36.665 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "container_image_elasticsearch", 2024-03-19T02:05:36.665 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.665 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.665 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.665 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "quay.io/omrizeneva/elasticsearch:6.8.23", 2024-03-19T02:05:36.666 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.666 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.666 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.666 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "elasticsearch container image", 2024-03-19T02:05:36.666 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.666 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.666 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.666 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.666 INFO:teuthology.orchestra.run.smithi099.stdout: "container_image_grafana": { 2024-03-19T02:05:36.666 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "container_image_grafana", 2024-03-19T02:05:36.666 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.666 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.666 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.666 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "quay.io/ceph/ceph-grafana:9.4.12", 2024-03-19T02:05:36.667 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.667 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.667 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.667 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Prometheus container image", 2024-03-19T02:05:36.667 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.667 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.667 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.667 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.667 INFO:teuthology.orchestra.run.smithi099.stdout: "container_image_haproxy": { 2024-03-19T02:05:36.667 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "container_image_haproxy", 2024-03-19T02:05:36.667 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.667 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.667 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.667 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "quay.io/ceph/haproxy:2.3", 2024-03-19T02:05:36.668 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.668 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.668 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.668 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "HAproxy container image", 2024-03-19T02:05:36.668 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.668 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.668 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.668 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.668 INFO:teuthology.orchestra.run.smithi099.stdout: "container_image_jaeger_agent": { 2024-03-19T02:05:36.668 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "container_image_jaeger_agent", 2024-03-19T02:05:36.668 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.668 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.668 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.668 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "quay.io/jaegertracing/jaeger-agent:1.29", 2024-03-19T02:05:36.669 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.669 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.669 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.669 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Jaeger agent container image", 2024-03-19T02:05:36.669 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.669 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.669 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.669 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.669 INFO:teuthology.orchestra.run.smithi099.stdout: "container_image_jaeger_collector": { 2024-03-19T02:05:36.669 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "container_image_jaeger_collector", 2024-03-19T02:05:36.669 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.669 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.669 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.669 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "quay.io/jaegertracing/jaeger-collector:1.29", 2024-03-19T02:05:36.670 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.670 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.670 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.670 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Jaeger collector container image", 2024-03-19T02:05:36.670 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.670 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.670 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.670 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.670 INFO:teuthology.orchestra.run.smithi099.stdout: "container_image_jaeger_query": { 2024-03-19T02:05:36.670 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "container_image_jaeger_query", 2024-03-19T02:05:36.670 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.670 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.670 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.670 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "quay.io/jaegertracing/jaeger-query:1.29", 2024-03-19T02:05:36.670 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.671 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.671 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.671 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Jaeger query container image", 2024-03-19T02:05:36.671 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.671 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.671 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.671 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.671 INFO:teuthology.orchestra.run.smithi099.stdout: "container_image_keepalived": { 2024-03-19T02:05:36.671 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "container_image_keepalived", 2024-03-19T02:05:36.671 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.671 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.671 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.671 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "quay.io/ceph/keepalived:2.2.4", 2024-03-19T02:05:36.671 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.672 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.672 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.672 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Keepalived container image", 2024-03-19T02:05:36.672 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.672 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.672 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.672 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.672 INFO:teuthology.orchestra.run.smithi099.stdout: "container_image_loki": { 2024-03-19T02:05:36.672 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "container_image_loki", 2024-03-19T02:05:36.672 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.672 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.672 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.672 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "docker.io/grafana/loki:2.4.0", 2024-03-19T02:05:36.672 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.673 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.673 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.673 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Loki container image", 2024-03-19T02:05:36.673 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.673 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.673 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.673 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.673 INFO:teuthology.orchestra.run.smithi099.stdout: "container_image_node_exporter": { 2024-03-19T02:05:36.673 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "container_image_node_exporter", 2024-03-19T02:05:36.673 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.673 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.673 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.673 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "quay.io/prometheus/node-exporter:v1.5.0", 2024-03-19T02:05:36.673 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.673 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.674 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.674 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Prometheus container image", 2024-03-19T02:05:36.674 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.674 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.674 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.674 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.674 INFO:teuthology.orchestra.run.smithi099.stdout: "container_image_nvmeof": { 2024-03-19T02:05:36.674 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "container_image_nvmeof", 2024-03-19T02:05:36.674 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.674 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.674 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.674 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "quay.io/ceph/nvmeof:1.0.0", 2024-03-19T02:05:36.674 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.674 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.675 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.675 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Nvme-of container image", 2024-03-19T02:05:36.675 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.675 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.675 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.675 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.675 INFO:teuthology.orchestra.run.smithi099.stdout: "container_image_prometheus": { 2024-03-19T02:05:36.675 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "container_image_prometheus", 2024-03-19T02:05:36.675 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.675 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.675 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.675 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "quay.io/prometheus/prometheus:v2.43.0", 2024-03-19T02:05:36.675 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.675 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.676 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.676 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Prometheus container image", 2024-03-19T02:05:36.676 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.676 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.676 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.676 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.676 INFO:teuthology.orchestra.run.smithi099.stdout: "container_image_promtail": { 2024-03-19T02:05:36.676 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "container_image_promtail", 2024-03-19T02:05:36.676 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.676 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.676 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.676 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "docker.io/grafana/promtail:2.4.0", 2024-03-19T02:05:36.676 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.676 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.677 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.677 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Promtail container image", 2024-03-19T02:05:36.677 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.677 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.677 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.677 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.677 INFO:teuthology.orchestra.run.smithi099.stdout: "container_image_snmp_gateway": { 2024-03-19T02:05:36.677 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "container_image_snmp_gateway", 2024-03-19T02:05:36.677 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.677 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.677 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.677 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "docker.io/maxwo/snmp-notifier:v1.2.1", 2024-03-19T02:05:36.677 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.677 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.678 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.678 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "SNMP Gateway container image", 2024-03-19T02:05:36.678 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.678 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.678 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.678 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.678 INFO:teuthology.orchestra.run.smithi099.stdout: "container_init": { 2024-03-19T02:05:36.678 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "container_init", 2024-03-19T02:05:36.678 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.678 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.678 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.678 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:36.678 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.678 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.679 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.679 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Run podman/docker with `--init`", 2024-03-19T02:05:36.679 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.679 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.679 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.679 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.679 INFO:teuthology.orchestra.run.smithi099.stdout: "daemon_cache_timeout": { 2024-03-19T02:05:36.679 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "daemon_cache_timeout", 2024-03-19T02:05:36.679 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:36.679 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.679 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.679 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "600", 2024-03-19T02:05:36.679 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.679 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.680 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.680 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "seconds to cache service (daemon) inventory", 2024-03-19T02:05:36.680 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.680 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.680 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.680 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.680 INFO:teuthology.orchestra.run.smithi099.stdout: "default_cephadm_command_timeout": { 2024-03-19T02:05:36.680 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "default_cephadm_command_timeout", 2024-03-19T02:05:36.680 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:36.680 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.680 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.680 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "900", 2024-03-19T02:05:36.680 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.680 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.680 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.681 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Default timeout applied to cephadm commands run directly on the host (in seconds)", 2024-03-19T02:05:36.681 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.681 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.681 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.681 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.681 INFO:teuthology.orchestra.run.smithi099.stdout: "default_registry": { 2024-03-19T02:05:36.681 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "default_registry", 2024-03-19T02:05:36.681 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.681 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.681 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.681 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "docker.io", 2024-03-19T02:05:36.681 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.681 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.681 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.682 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Search-registry to which we should normalize unqualified image names. This is not the default registry", 2024-03-19T02:05:36.682 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.682 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.682 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.682 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.682 INFO:teuthology.orchestra.run.smithi099.stdout: "device_cache_timeout": { 2024-03-19T02:05:36.682 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "device_cache_timeout", 2024-03-19T02:05:36.682 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:36.682 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.682 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.682 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "1800", 2024-03-19T02:05:36.682 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.682 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.682 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.682 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "seconds to cache device inventory", 2024-03-19T02:05:36.683 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.683 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.683 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.683 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.683 INFO:teuthology.orchestra.run.smithi099.stdout: "device_enhanced_scan": { 2024-03-19T02:05:36.683 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "device_enhanced_scan", 2024-03-19T02:05:36.683 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.683 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.683 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.683 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.683 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.683 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.683 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.683 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Use libstoragemgmt during device scans", 2024-03-19T02:05:36.684 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.684 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.684 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.684 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.684 INFO:teuthology.orchestra.run.smithi099.stdout: "facts_cache_timeout": { 2024-03-19T02:05:36.684 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "facts_cache_timeout", 2024-03-19T02:05:36.684 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:36.684 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.684 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.684 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "60", 2024-03-19T02:05:36.684 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.684 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.684 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.684 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "seconds to cache host facts data", 2024-03-19T02:05:36.685 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.685 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.685 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.685 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.685 INFO:teuthology.orchestra.run.smithi099.stdout: "host_check_interval": { 2024-03-19T02:05:36.685 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "host_check_interval", 2024-03-19T02:05:36.685 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:36.685 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.685 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.685 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "600", 2024-03-19T02:05:36.685 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.685 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.685 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.685 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "how frequently to perform a host check", 2024-03-19T02:05:36.685 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.686 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.686 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.686 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.686 INFO:teuthology.orchestra.run.smithi099.stdout: "hw_monitoring": { 2024-03-19T02:05:36.686 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "hw_monitoring", 2024-03-19T02:05:36.686 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.686 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.686 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.686 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.686 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.686 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.686 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.686 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Deploy hw monitoring daemon on every host.", 2024-03-19T02:05:36.686 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.687 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.687 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.687 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.687 INFO:teuthology.orchestra.run.smithi099.stdout: "inventory_list_all": { 2024-03-19T02:05:36.687 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "inventory_list_all", 2024-03-19T02:05:36.687 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.687 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.687 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.687 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.687 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.687 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.687 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.687 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)", 2024-03-19T02:05:36.687 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.687 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.688 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.688 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.688 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:36.688 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:36.688 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.688 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.688 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.688 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.688 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.688 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.688 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.688 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.688 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.688 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.688 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.689 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.689 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.689 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.689 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.689 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.689 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.689 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.689 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.689 INFO:teuthology.orchestra.run.smithi099.stdout: "log_refresh_metadata": { 2024-03-19T02:05:36.689 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_refresh_metadata", 2024-03-19T02:05:36.689 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.689 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.689 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.689 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.689 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.690 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.690 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.690 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level", 2024-03-19T02:05:36.690 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.690 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.690 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.690 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.690 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:36.690 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:36.690 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.690 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.690 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.690 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:36.690 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.691 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.691 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.691 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "log to the \"cephadm\" cluster log channel\"", 2024-03-19T02:05:36.691 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.691 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.691 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.691 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.691 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:36.691 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:36.691 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.691 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.691 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.691 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:36.691 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.692 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.692 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.692 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.692 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.692 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.692 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.692 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.692 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.692 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.692 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.692 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.692 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.692 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.692 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.692 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:36.693 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:36.693 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.693 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.693 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.693 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.693 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.693 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.693 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.693 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.693 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.693 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.693 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.693 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.693 INFO:teuthology.orchestra.run.smithi099.stdout: "manage_etc_ceph_ceph_conf": { 2024-03-19T02:05:36.693 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "manage_etc_ceph_ceph_conf", 2024-03-19T02:05:36.694 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.694 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.694 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.694 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.694 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.694 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.694 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.694 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Manage and own /etc/ceph/ceph.conf on the hosts.", 2024-03-19T02:05:36.694 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.694 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.694 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.694 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.694 INFO:teuthology.orchestra.run.smithi099.stdout: "manage_etc_ceph_ceph_conf_hosts": { 2024-03-19T02:05:36.694 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "manage_etc_ceph_ceph_conf_hosts", 2024-03-19T02:05:36.695 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.695 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.695 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.695 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "*", 2024-03-19T02:05:36.695 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.695 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.695 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.695 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf", 2024-03-19T02:05:36.695 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.695 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.695 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.695 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.695 INFO:teuthology.orchestra.run.smithi099.stdout: "max_count_per_host": { 2024-03-19T02:05:36.695 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "max_count_per_host", 2024-03-19T02:05:36.696 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:36.696 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.696 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.696 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "10", 2024-03-19T02:05:36.696 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.696 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.696 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.696 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "max number of daemons per service per host", 2024-03-19T02:05:36.696 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.696 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.696 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.696 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.696 INFO:teuthology.orchestra.run.smithi099.stdout: "max_osd_draining_count": { 2024-03-19T02:05:36.696 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "max_osd_draining_count", 2024-03-19T02:05:36.696 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:36.697 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.697 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.697 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "10", 2024-03-19T02:05:36.697 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.697 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.697 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.697 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "max number of osds that will be drained simultaneously when osds are removed", 2024-03-19T02:05:36.697 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.697 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.697 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.697 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.697 INFO:teuthology.orchestra.run.smithi099.stdout: "migration_current": { 2024-03-19T02:05:36.697 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "migration_current", 2024-03-19T02:05:36.697 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:36.697 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.698 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.698 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.698 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.698 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.698 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.698 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "internal - do not modify", 2024-03-19T02:05:36.698 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.698 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.698 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.698 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.698 INFO:teuthology.orchestra.run.smithi099.stdout: "mode": { 2024-03-19T02:05:36.698 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "mode", 2024-03-19T02:05:36.698 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.698 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.699 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.699 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "root", 2024-03-19T02:05:36.699 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.699 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.699 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.699 INFO:teuthology.orchestra.run.smithi099.stdout: "cephadm-package", 2024-03-19T02:05:36.699 INFO:teuthology.orchestra.run.smithi099.stdout: "root" 2024-03-19T02:05:36.699 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.699 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "mode for remote execution of cephadm", 2024-03-19T02:05:36.699 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.699 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.699 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.699 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.699 INFO:teuthology.orchestra.run.smithi099.stdout: "oob_default_addr": { 2024-03-19T02:05:36.700 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "oob_default_addr", 2024-03-19T02:05:36.700 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.700 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.700 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.700 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "169.254.1.1", 2024-03-19T02:05:36.700 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.700 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.700 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.700 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Default address for RedFish API (oob management).", 2024-03-19T02:05:36.700 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.700 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.700 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.700 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.701 INFO:teuthology.orchestra.run.smithi099.stdout: "prometheus_alerts_path": { 2024-03-19T02:05:36.701 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "prometheus_alerts_path", 2024-03-19T02:05:36.701 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.701 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.701 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.701 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "/etc/prometheus/ceph/ceph_default_alerts.yml", 2024-03-19T02:05:36.701 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.701 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.701 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.701 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "location of alerts to include in prometheus deployments", 2024-03-19T02:05:36.701 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.701 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.701 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.701 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.702 INFO:teuthology.orchestra.run.smithi099.stdout: "registry_insecure": { 2024-03-19T02:05:36.702 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "registry_insecure", 2024-03-19T02:05:36.702 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.702 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.702 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.702 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.702 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.702 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.702 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.702 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Registry is to be considered insecure (no TLS available). Only for development purposes.", 2024-03-19T02:05:36.702 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.702 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.702 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.702 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.702 INFO:teuthology.orchestra.run.smithi099.stdout: "registry_password": { 2024-03-19T02:05:36.703 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "registry_password", 2024-03-19T02:05:36.703 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.703 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.703 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.703 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.703 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.703 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.703 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.703 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Custom repository password. Only used for logging into a registry.", 2024-03-19T02:05:36.703 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.703 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.703 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.703 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.703 INFO:teuthology.orchestra.run.smithi099.stdout: "registry_url": { 2024-03-19T02:05:36.703 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "registry_url", 2024-03-19T02:05:36.704 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.704 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.704 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.704 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.704 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.704 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.704 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.704 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Registry url for login purposes. This is not the default registry", 2024-03-19T02:05:36.704 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.704 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.704 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.704 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.704 INFO:teuthology.orchestra.run.smithi099.stdout: "registry_username": { 2024-03-19T02:05:36.705 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "registry_username", 2024-03-19T02:05:36.705 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.705 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.705 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.705 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.705 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.705 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.705 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.705 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Custom repository username. Only used for logging into a registry.", 2024-03-19T02:05:36.705 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.705 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.705 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.705 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.705 INFO:teuthology.orchestra.run.smithi099.stdout: "secure_monitoring_stack": { 2024-03-19T02:05:36.705 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "secure_monitoring_stack", 2024-03-19T02:05:36.706 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.706 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.706 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.706 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.706 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.706 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.706 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.706 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Enable TLS security for all the monitoring stack daemons", 2024-03-19T02:05:36.706 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.706 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.706 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.706 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.706 INFO:teuthology.orchestra.run.smithi099.stdout: "service_discovery_port": { 2024-03-19T02:05:36.706 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "service_discovery_port", 2024-03-19T02:05:36.707 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:36.707 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.707 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.707 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "8765", 2024-03-19T02:05:36.707 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.707 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.707 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.707 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "cephadm service discovery port", 2024-03-19T02:05:36.707 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.707 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.707 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.707 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.707 INFO:teuthology.orchestra.run.smithi099.stdout: "ssh_config_file": { 2024-03-19T02:05:36.707 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "ssh_config_file", 2024-03-19T02:05:36.707 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.708 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.708 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.708 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.708 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.708 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.708 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.708 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "customized SSH config file to connect to managed hosts", 2024-03-19T02:05:36.708 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.708 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.708 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.708 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.708 INFO:teuthology.orchestra.run.smithi099.stdout: "use_agent": { 2024-03-19T02:05:36.708 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "use_agent", 2024-03-19T02:05:36.708 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.708 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.709 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.709 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.709 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.709 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.709 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.709 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Use cephadm agent on each host to gather and send metadata", 2024-03-19T02:05:36.709 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.709 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.709 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.709 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.709 INFO:teuthology.orchestra.run.smithi099.stdout: "use_repo_digest": { 2024-03-19T02:05:36.709 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "use_repo_digest", 2024-03-19T02:05:36.709 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.709 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.709 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.710 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:36.710 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.710 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.710 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.710 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Automatically convert image tags to image digest. Make sure all daemons use the same image", 2024-03-19T02:05:36.710 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.710 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.710 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.710 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.710 INFO:teuthology.orchestra.run.smithi099.stdout: "warn_on_failed_host_check": { 2024-03-19T02:05:36.710 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "warn_on_failed_host_check", 2024-03-19T02:05:36.710 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.710 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.710 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.711 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:36.711 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.711 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.711 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.711 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "raise a health warning if the host check fails", 2024-03-19T02:05:36.711 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.711 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.711 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.711 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.711 INFO:teuthology.orchestra.run.smithi099.stdout: "warn_on_stray_daemons": { 2024-03-19T02:05:36.711 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "warn_on_stray_daemons", 2024-03-19T02:05:36.711 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.711 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.711 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.712 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:36.712 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.712 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.712 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.712 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "raise a health warning if daemons are detected that are not managed by cephadm", 2024-03-19T02:05:36.712 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.712 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.712 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.712 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.712 INFO:teuthology.orchestra.run.smithi099.stdout: "warn_on_stray_hosts": { 2024-03-19T02:05:36.712 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "warn_on_stray_hosts", 2024-03-19T02:05:36.712 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.712 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.712 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.712 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:36.713 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.713 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.713 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.713 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "raise a health warning if daemons are detected on a host that is not managed by cephadm", 2024-03-19T02:05:36.713 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.713 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.713 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.713 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.713 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.713 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.713 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:36.713 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "crash", 2024-03-19T02:05:36.713 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:36.713 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:36.713 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:36.714 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:36.714 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:36.714 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.714 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.714 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.714 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.714 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.714 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.714 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.714 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.714 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.714 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.714 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.714 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.714 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.715 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.715 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.715 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.715 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.715 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.715 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.715 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:36.715 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:36.715 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.715 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.715 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.715 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.715 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.715 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.715 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.716 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.716 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.716 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.716 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.716 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.716 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:36.716 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:36.716 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.716 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.716 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.716 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:36.716 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.716 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.717 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.717 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.717 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.717 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.717 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.717 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.717 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.717 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.717 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.717 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.717 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.717 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.717 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.717 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:36.717 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:36.718 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.718 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.718 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.718 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.718 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.718 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.718 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.718 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.718 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.718 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.718 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.718 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.718 INFO:teuthology.orchestra.run.smithi099.stdout: "retain_interval": { 2024-03-19T02:05:36.718 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "retain_interval", 2024-03-19T02:05:36.719 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:36.719 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.719 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.719 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "31536000", 2024-03-19T02:05:36.719 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.719 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.719 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.719 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "how long to retain crashes before pruning them", 2024-03-19T02:05:36.719 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.719 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.719 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.719 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.719 INFO:teuthology.orchestra.run.smithi099.stdout: "warn_recent_interval": { 2024-03-19T02:05:36.719 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "warn_recent_interval", 2024-03-19T02:05:36.719 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:36.720 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.720 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.720 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "1209600", 2024-03-19T02:05:36.720 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.720 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.720 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.720 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "time interval in which to warn about recent crashes", 2024-03-19T02:05:36.720 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.720 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.720 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.720 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.720 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.720 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.720 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:36.721 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "dashboard", 2024-03-19T02:05:36.721 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:36.721 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:36.721 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:36.721 INFO:teuthology.orchestra.run.smithi099.stdout: "ACCOUNT_LOCKOUT_ATTEMPTS": { 2024-03-19T02:05:36.721 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "ACCOUNT_LOCKOUT_ATTEMPTS", 2024-03-19T02:05:36.721 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:36.721 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.721 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.721 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "10", 2024-03-19T02:05:36.721 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.721 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.721 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.721 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.721 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.722 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.722 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.722 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.722 INFO:teuthology.orchestra.run.smithi099.stdout: "ALERTMANAGER_API_HOST": { 2024-03-19T02:05:36.722 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "ALERTMANAGER_API_HOST", 2024-03-19T02:05:36.722 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.722 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.722 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.722 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.722 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.722 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.722 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.722 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.722 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.723 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.723 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.723 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.723 INFO:teuthology.orchestra.run.smithi099.stdout: "ALERTMANAGER_API_SSL_VERIFY": { 2024-03-19T02:05:36.723 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "ALERTMANAGER_API_SSL_VERIFY", 2024-03-19T02:05:36.723 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.723 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.723 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.723 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:36.723 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.723 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.723 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.723 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.723 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.723 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.724 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.724 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.724 INFO:teuthology.orchestra.run.smithi099.stdout: "AUDIT_API_ENABLED": { 2024-03-19T02:05:36.724 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "AUDIT_API_ENABLED", 2024-03-19T02:05:36.724 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.724 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.724 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.724 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.724 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.724 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.724 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.724 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.724 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.724 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.724 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.725 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.725 INFO:teuthology.orchestra.run.smithi099.stdout: "AUDIT_API_LOG_PAYLOAD": { 2024-03-19T02:05:36.725 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "AUDIT_API_LOG_PAYLOAD", 2024-03-19T02:05:36.725 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.725 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.725 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.725 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:36.725 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.725 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.725 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.725 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.725 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.725 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.725 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.725 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.726 INFO:teuthology.orchestra.run.smithi099.stdout: "ENABLE_BROWSABLE_API": { 2024-03-19T02:05:36.726 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "ENABLE_BROWSABLE_API", 2024-03-19T02:05:36.726 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.726 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.726 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.726 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:36.726 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.726 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.726 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.726 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.726 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.726 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.726 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.726 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.727 INFO:teuthology.orchestra.run.smithi099.stdout: "FEATURE_TOGGLE_CEPHFS": { 2024-03-19T02:05:36.727 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "FEATURE_TOGGLE_CEPHFS", 2024-03-19T02:05:36.727 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.727 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.727 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.727 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:36.727 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.727 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.727 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.727 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.727 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.727 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.727 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.727 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.727 INFO:teuthology.orchestra.run.smithi099.stdout: "FEATURE_TOGGLE_DASHBOARD": { 2024-03-19T02:05:36.728 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "FEATURE_TOGGLE_DASHBOARD", 2024-03-19T02:05:36.728 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.728 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.728 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.728 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:36.728 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.728 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.728 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.728 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.728 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.728 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.728 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.728 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.728 INFO:teuthology.orchestra.run.smithi099.stdout: "FEATURE_TOGGLE_ISCSI": { 2024-03-19T02:05:36.729 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "FEATURE_TOGGLE_ISCSI", 2024-03-19T02:05:36.729 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.729 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.729 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.729 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:36.729 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.729 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.729 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.729 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.729 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.729 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.729 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.729 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.729 INFO:teuthology.orchestra.run.smithi099.stdout: "FEATURE_TOGGLE_MIRRORING": { 2024-03-19T02:05:36.730 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "FEATURE_TOGGLE_MIRRORING", 2024-03-19T02:05:36.730 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.730 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.730 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.730 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:36.730 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.730 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.730 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.730 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.730 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.730 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.730 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.730 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.730 INFO:teuthology.orchestra.run.smithi099.stdout: "FEATURE_TOGGLE_NFS": { 2024-03-19T02:05:36.731 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "FEATURE_TOGGLE_NFS", 2024-03-19T02:05:36.731 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.731 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.731 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.731 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:36.731 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.731 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.731 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.731 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.731 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.731 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.731 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.731 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.731 INFO:teuthology.orchestra.run.smithi099.stdout: "FEATURE_TOGGLE_RBD": { 2024-03-19T02:05:36.731 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "FEATURE_TOGGLE_RBD", 2024-03-19T02:05:36.732 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.732 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.732 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.732 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:36.732 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.732 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.732 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.732 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.732 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.732 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.732 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.732 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.732 INFO:teuthology.orchestra.run.smithi099.stdout: "FEATURE_TOGGLE_RGW": { 2024-03-19T02:05:36.732 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "FEATURE_TOGGLE_RGW", 2024-03-19T02:05:36.732 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.733 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.733 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.733 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:36.733 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.733 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.733 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.733 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.733 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.733 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.733 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.733 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.733 INFO:teuthology.orchestra.run.smithi099.stdout: "GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE": { 2024-03-19T02:05:36.733 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE", 2024-03-19T02:05:36.733 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.733 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.734 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.734 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.734 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.734 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.734 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.734 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.734 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.734 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.734 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.734 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.734 INFO:teuthology.orchestra.run.smithi099.stdout: "GRAFANA_API_PASSWORD": { 2024-03-19T02:05:36.734 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "GRAFANA_API_PASSWORD", 2024-03-19T02:05:36.734 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.734 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.735 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.735 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "admin", 2024-03-19T02:05:36.735 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.735 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.735 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.735 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.735 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.735 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.735 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.735 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.735 INFO:teuthology.orchestra.run.smithi099.stdout: "GRAFANA_API_SSL_VERIFY": { 2024-03-19T02:05:36.735 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "GRAFANA_API_SSL_VERIFY", 2024-03-19T02:05:36.735 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.735 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.736 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.736 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:36.736 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.736 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.736 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.736 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.736 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.736 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.736 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.736 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.736 INFO:teuthology.orchestra.run.smithi099.stdout: "GRAFANA_API_URL": { 2024-03-19T02:05:36.736 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "GRAFANA_API_URL", 2024-03-19T02:05:36.736 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.736 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.736 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.737 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.737 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.737 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.737 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.737 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.737 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.737 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.737 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.737 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.737 INFO:teuthology.orchestra.run.smithi099.stdout: "GRAFANA_API_USERNAME": { 2024-03-19T02:05:36.737 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "GRAFANA_API_USERNAME", 2024-03-19T02:05:36.737 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.737 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.737 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.737 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "admin", 2024-03-19T02:05:36.738 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.738 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.738 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.738 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.738 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.738 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.738 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.738 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.738 INFO:teuthology.orchestra.run.smithi099.stdout: "GRAFANA_FRONTEND_API_URL": { 2024-03-19T02:05:36.738 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "GRAFANA_FRONTEND_API_URL", 2024-03-19T02:05:36.738 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.738 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.738 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.738 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.738 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.739 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.739 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.739 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.739 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.739 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.739 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.739 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.739 INFO:teuthology.orchestra.run.smithi099.stdout: "GRAFANA_UPDATE_DASHBOARDS": { 2024-03-19T02:05:36.739 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "GRAFANA_UPDATE_DASHBOARDS", 2024-03-19T02:05:36.739 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.739 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.739 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.739 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.739 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.739 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.740 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.740 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.740 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.740 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.740 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.740 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.740 INFO:teuthology.orchestra.run.smithi099.stdout: "ISCSI_API_SSL_VERIFICATION": { 2024-03-19T02:05:36.740 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "ISCSI_API_SSL_VERIFICATION", 2024-03-19T02:05:36.740 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.740 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.740 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.740 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:36.740 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.740 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.740 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.741 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.741 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.741 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.741 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.741 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.741 INFO:teuthology.orchestra.run.smithi099.stdout: "ISSUE_TRACKER_API_KEY": { 2024-03-19T02:05:36.741 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "ISSUE_TRACKER_API_KEY", 2024-03-19T02:05:36.741 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.741 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.741 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.741 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.741 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.741 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.742 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.742 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.742 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.742 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.742 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.742 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.742 INFO:teuthology.orchestra.run.smithi099.stdout: "MULTICLUSTER_CONFIG": { 2024-03-19T02:05:36.742 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "MULTICLUSTER_CONFIG", 2024-03-19T02:05:36.742 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.742 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.742 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.742 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "{}", 2024-03-19T02:05:36.742 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.742 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.743 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.743 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.743 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.743 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.743 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.743 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.743 INFO:teuthology.orchestra.run.smithi099.stdout: "PROMETHEUS_API_HOST": { 2024-03-19T02:05:36.743 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "PROMETHEUS_API_HOST", 2024-03-19T02:05:36.743 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.743 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.743 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.743 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.743 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.743 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.743 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.744 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.744 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.744 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.744 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.744 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.744 INFO:teuthology.orchestra.run.smithi099.stdout: "PROMETHEUS_API_SSL_VERIFY": { 2024-03-19T02:05:36.744 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "PROMETHEUS_API_SSL_VERIFY", 2024-03-19T02:05:36.744 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.744 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.744 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.744 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:36.744 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.744 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.744 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.745 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.745 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.745 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.745 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.745 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.745 INFO:teuthology.orchestra.run.smithi099.stdout: "PWD_POLICY_CHECK_COMPLEXITY_ENABLED": { 2024-03-19T02:05:36.745 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "PWD_POLICY_CHECK_COMPLEXITY_ENABLED", 2024-03-19T02:05:36.745 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.745 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.745 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.745 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.745 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.745 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.745 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.745 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.746 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.746 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.746 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.746 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.746 INFO:teuthology.orchestra.run.smithi099.stdout: "PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED": { 2024-03-19T02:05:36.746 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED", 2024-03-19T02:05:36.746 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.746 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.746 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.746 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.746 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.746 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.746 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.746 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.747 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.747 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.747 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.747 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.747 INFO:teuthology.orchestra.run.smithi099.stdout: "PWD_POLICY_CHECK_LENGTH_ENABLED": { 2024-03-19T02:05:36.747 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "PWD_POLICY_CHECK_LENGTH_ENABLED", 2024-03-19T02:05:36.747 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.747 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.747 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.747 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:36.747 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.747 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.747 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.747 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.747 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.748 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.748 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.748 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.748 INFO:teuthology.orchestra.run.smithi099.stdout: "PWD_POLICY_CHECK_OLDPWD_ENABLED": { 2024-03-19T02:05:36.748 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "PWD_POLICY_CHECK_OLDPWD_ENABLED", 2024-03-19T02:05:36.748 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.748 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.748 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.748 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:36.748 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.748 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.748 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.748 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.748 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.748 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.749 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.749 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.749 INFO:teuthology.orchestra.run.smithi099.stdout: "PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED": { 2024-03-19T02:05:36.749 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED", 2024-03-19T02:05:36.749 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.749 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.749 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.749 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.749 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.749 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.749 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.749 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.749 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.749 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.749 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.750 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.750 INFO:teuthology.orchestra.run.smithi099.stdout: "PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED": { 2024-03-19T02:05:36.750 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED", 2024-03-19T02:05:36.750 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.750 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.750 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.750 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.750 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.750 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.750 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.750 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.750 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.750 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.750 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.751 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.751 INFO:teuthology.orchestra.run.smithi099.stdout: "PWD_POLICY_CHECK_USERNAME_ENABLED": { 2024-03-19T02:05:36.751 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "PWD_POLICY_CHECK_USERNAME_ENABLED", 2024-03-19T02:05:36.751 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.751 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.751 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.751 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.751 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.751 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.751 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.751 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.751 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.751 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.751 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.751 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.752 INFO:teuthology.orchestra.run.smithi099.stdout: "PWD_POLICY_ENABLED": { 2024-03-19T02:05:36.752 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "PWD_POLICY_ENABLED", 2024-03-19T02:05:36.752 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.752 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.752 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.752 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:36.752 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.752 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.752 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.752 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.752 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.752 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.752 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.752 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.752 INFO:teuthology.orchestra.run.smithi099.stdout: "PWD_POLICY_EXCLUSION_LIST": { 2024-03-19T02:05:36.753 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "PWD_POLICY_EXCLUSION_LIST", 2024-03-19T02:05:36.753 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.753 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.753 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.753 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps", 2024-03-19T02:05:36.753 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.753 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.753 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.753 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.753 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.753 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.753 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.753 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.753 INFO:teuthology.orchestra.run.smithi099.stdout: "PWD_POLICY_MIN_COMPLEXITY": { 2024-03-19T02:05:36.754 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "PWD_POLICY_MIN_COMPLEXITY", 2024-03-19T02:05:36.754 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:36.754 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.754 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.754 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "10", 2024-03-19T02:05:36.754 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.754 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.754 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.754 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.754 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.754 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.754 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.754 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.754 INFO:teuthology.orchestra.run.smithi099.stdout: "PWD_POLICY_MIN_LENGTH": { 2024-03-19T02:05:36.755 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "PWD_POLICY_MIN_LENGTH", 2024-03-19T02:05:36.755 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:36.755 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.755 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.755 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "8", 2024-03-19T02:05:36.755 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.755 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.755 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.755 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.755 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.755 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.755 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.755 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.755 INFO:teuthology.orchestra.run.smithi099.stdout: "REST_REQUESTS_TIMEOUT": { 2024-03-19T02:05:36.755 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "REST_REQUESTS_TIMEOUT", 2024-03-19T02:05:36.756 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:36.756 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.756 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.756 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "45", 2024-03-19T02:05:36.756 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.756 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.756 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.756 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.756 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.756 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.756 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.756 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.756 INFO:teuthology.orchestra.run.smithi099.stdout: "RGW_API_ACCESS_KEY": { 2024-03-19T02:05:36.756 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "RGW_API_ACCESS_KEY", 2024-03-19T02:05:36.757 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.757 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.757 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.757 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.757 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.757 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.757 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.757 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.757 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.757 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.757 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.757 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.757 INFO:teuthology.orchestra.run.smithi099.stdout: "RGW_API_ADMIN_RESOURCE": { 2024-03-19T02:05:36.757 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "RGW_API_ADMIN_RESOURCE", 2024-03-19T02:05:36.757 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.758 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.758 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.758 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "admin", 2024-03-19T02:05:36.758 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.758 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.758 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.758 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.758 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.758 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.758 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.758 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.758 INFO:teuthology.orchestra.run.smithi099.stdout: "RGW_API_SECRET_KEY": { 2024-03-19T02:05:36.758 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "RGW_API_SECRET_KEY", 2024-03-19T02:05:36.758 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.759 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.759 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.759 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.759 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.759 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.759 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.759 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.759 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.759 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.759 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.759 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.759 INFO:teuthology.orchestra.run.smithi099.stdout: "RGW_API_SSL_VERIFY": { 2024-03-19T02:05:36.759 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "RGW_API_SSL_VERIFY", 2024-03-19T02:05:36.759 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.760 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.760 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.760 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:36.760 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.760 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.760 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.760 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.760 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.760 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.760 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.760 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.760 INFO:teuthology.orchestra.run.smithi099.stdout: "UNSAFE_TLS_v1_2": { 2024-03-19T02:05:36.760 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "UNSAFE_TLS_v1_2", 2024-03-19T02:05:36.760 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.760 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.761 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.761 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.761 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.761 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.761 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.761 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.761 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.761 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.761 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.761 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.761 INFO:teuthology.orchestra.run.smithi099.stdout: "USER_PWD_EXPIRATION_SPAN": { 2024-03-19T02:05:36.761 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "USER_PWD_EXPIRATION_SPAN", 2024-03-19T02:05:36.761 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:36.761 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.761 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.762 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "0", 2024-03-19T02:05:36.762 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.762 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.762 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.762 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.762 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.762 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.762 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.762 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.762 INFO:teuthology.orchestra.run.smithi099.stdout: "USER_PWD_EXPIRATION_WARNING_1": { 2024-03-19T02:05:36.762 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "USER_PWD_EXPIRATION_WARNING_1", 2024-03-19T02:05:36.762 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:36.762 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.762 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.763 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "10", 2024-03-19T02:05:36.763 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.763 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.763 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.763 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.763 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.763 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.763 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.763 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.763 INFO:teuthology.orchestra.run.smithi099.stdout: "USER_PWD_EXPIRATION_WARNING_2": { 2024-03-19T02:05:36.763 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "USER_PWD_EXPIRATION_WARNING_2", 2024-03-19T02:05:36.763 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:36.763 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.763 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.763 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "5", 2024-03-19T02:05:36.764 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.764 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.764 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.764 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.764 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.764 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.764 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.764 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.764 INFO:teuthology.orchestra.run.smithi099.stdout: "cross_origin_url": { 2024-03-19T02:05:36.764 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "cross_origin_url", 2024-03-19T02:05:36.764 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.764 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.764 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.764 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.764 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.765 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.765 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.765 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.765 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.765 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.765 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.765 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.765 INFO:teuthology.orchestra.run.smithi099.stdout: "crt_file": { 2024-03-19T02:05:36.765 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "crt_file", 2024-03-19T02:05:36.765 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.765 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.765 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.765 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.765 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.765 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.766 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.766 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.766 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.766 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.766 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.766 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.766 INFO:teuthology.orchestra.run.smithi099.stdout: "debug": { 2024-03-19T02:05:36.766 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "debug", 2024-03-19T02:05:36.766 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.766 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.766 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.766 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.766 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.767 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.767 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.767 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Enable/disable debug options", 2024-03-19T02:05:36.767 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.767 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.767 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.767 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.767 INFO:teuthology.orchestra.run.smithi099.stdout: "jwt_token_ttl": { 2024-03-19T02:05:36.767 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "jwt_token_ttl", 2024-03-19T02:05:36.767 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:36.767 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.767 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.767 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "28800", 2024-03-19T02:05:36.767 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.768 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.768 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.768 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.768 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.768 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.768 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.768 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.768 INFO:teuthology.orchestra.run.smithi099.stdout: "key_file": { 2024-03-19T02:05:36.768 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "key_file", 2024-03-19T02:05:36.768 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.768 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.768 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.768 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.768 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.768 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.769 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.769 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.769 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.769 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.769 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.769 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.769 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:36.769 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:36.769 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.769 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.769 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.769 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.769 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.769 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.770 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.770 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.770 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.770 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.770 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.770 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.770 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.770 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.770 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.770 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.770 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.770 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.770 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.770 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:36.770 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:36.771 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.771 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.771 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.771 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.771 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.771 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.771 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.771 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.771 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.771 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.771 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.771 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.771 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:36.771 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:36.772 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.772 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.772 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.772 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:36.772 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.772 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.772 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.772 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.772 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.772 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.772 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.772 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.772 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.772 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.772 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.773 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.773 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.773 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.773 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.773 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:36.773 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:36.773 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.773 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.773 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.773 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.773 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.773 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.773 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.773 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.774 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.774 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.774 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.774 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.774 INFO:teuthology.orchestra.run.smithi099.stdout: "motd": { 2024-03-19T02:05:36.774 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "motd", 2024-03-19T02:05:36.774 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.774 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.774 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.774 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.774 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.774 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.774 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.774 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "The message of the day", 2024-03-19T02:05:36.774 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.775 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.775 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.775 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.775 INFO:teuthology.orchestra.run.smithi099.stdout: "redirect_resolve_ip_addr": { 2024-03-19T02:05:36.775 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "redirect_resolve_ip_addr", 2024-03-19T02:05:36.775 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.775 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.775 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.775 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.775 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.775 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.775 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.775 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.775 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.775 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.776 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.776 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.776 INFO:teuthology.orchestra.run.smithi099.stdout: "server_addr": { 2024-03-19T02:05:36.776 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "server_addr", 2024-03-19T02:05:36.776 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.776 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.776 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.776 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "::", 2024-03-19T02:05:36.776 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.776 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.776 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.776 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.776 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.776 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.777 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.777 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.777 INFO:teuthology.orchestra.run.smithi099.stdout: "server_port": { 2024-03-19T02:05:36.777 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "server_port", 2024-03-19T02:05:36.777 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:36.777 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.777 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.777 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "8080", 2024-03-19T02:05:36.777 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.777 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.777 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.777 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.777 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.777 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.777 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.778 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.778 INFO:teuthology.orchestra.run.smithi099.stdout: "ssl": { 2024-03-19T02:05:36.778 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "ssl", 2024-03-19T02:05:36.778 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.778 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.778 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.778 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:36.778 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.778 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.778 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.778 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.778 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.778 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.778 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.778 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.779 INFO:teuthology.orchestra.run.smithi099.stdout: "ssl_server_port": { 2024-03-19T02:05:36.779 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "ssl_server_port", 2024-03-19T02:05:36.779 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:36.779 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.779 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.779 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "8443", 2024-03-19T02:05:36.779 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.779 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.779 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.779 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.779 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.779 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.779 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.779 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.779 INFO:teuthology.orchestra.run.smithi099.stdout: "standby_behaviour": { 2024-03-19T02:05:36.780 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "standby_behaviour", 2024-03-19T02:05:36.780 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.780 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.780 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.780 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "redirect", 2024-03-19T02:05:36.780 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.780 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.780 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.780 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.780 INFO:teuthology.orchestra.run.smithi099.stdout: "redirect" 2024-03-19T02:05:36.780 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.780 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.780 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.781 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.781 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.781 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.781 INFO:teuthology.orchestra.run.smithi099.stdout: "standby_error_status_code": { 2024-03-19T02:05:36.781 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "standby_error_status_code", 2024-03-19T02:05:36.781 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:36.781 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.781 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.781 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "500", 2024-03-19T02:05:36.781 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "400", 2024-03-19T02:05:36.781 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "599", 2024-03-19T02:05:36.781 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.781 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.781 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.782 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.782 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.782 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.782 INFO:teuthology.orchestra.run.smithi099.stdout: "url_prefix": { 2024-03-19T02:05:36.782 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "url_prefix", 2024-03-19T02:05:36.782 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.782 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.782 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.782 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.782 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.782 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.782 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.782 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.782 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.782 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.783 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.783 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.783 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.783 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.783 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:36.783 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "devicehealth", 2024-03-19T02:05:36.783 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:36.783 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:36.783 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:36.783 INFO:teuthology.orchestra.run.smithi099.stdout: "enable_monitoring": { 2024-03-19T02:05:36.783 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "enable_monitoring", 2024-03-19T02:05:36.783 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.783 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.783 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.784 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:36.784 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.784 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.784 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.784 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "monitor device health metrics", 2024-03-19T02:05:36.784 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.784 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.784 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.784 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.784 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:36.784 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:36.784 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.784 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.784 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.784 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.785 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.785 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.785 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.785 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.785 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.785 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.785 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.785 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.785 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.785 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.785 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.785 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.785 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.785 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.785 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.786 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:36.786 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:36.786 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.786 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.786 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.786 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.786 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.786 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.786 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.786 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.786 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.786 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.786 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.786 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.787 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:36.787 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:36.787 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.787 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.787 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.787 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:36.787 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.787 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.787 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.787 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.787 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.787 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.787 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.787 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.787 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.788 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.788 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.788 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.788 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.788 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.788 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.788 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:36.788 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:36.788 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.788 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.788 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.788 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.788 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.788 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.788 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.789 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.789 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.789 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.789 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.789 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.789 INFO:teuthology.orchestra.run.smithi099.stdout: "mark_out_threshold": { 2024-03-19T02:05:36.789 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "mark_out_threshold", 2024-03-19T02:05:36.789 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:36.789 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.789 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.789 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "2419200", 2024-03-19T02:05:36.789 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.789 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.789 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.789 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "automatically mark OSD if it may fail before this long", 2024-03-19T02:05:36.790 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.790 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.790 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.790 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.790 INFO:teuthology.orchestra.run.smithi099.stdout: "pool_name": { 2024-03-19T02:05:36.790 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "pool_name", 2024-03-19T02:05:36.790 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.790 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.790 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.790 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "device_health_metrics", 2024-03-19T02:05:36.790 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.790 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.790 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.790 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "name of pool in which to store device health metrics", 2024-03-19T02:05:36.790 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.791 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.791 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.791 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.791 INFO:teuthology.orchestra.run.smithi099.stdout: "retention_period": { 2024-03-19T02:05:36.791 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "retention_period", 2024-03-19T02:05:36.791 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:36.791 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.791 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.791 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "15552000", 2024-03-19T02:05:36.791 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.791 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.791 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.791 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "how long to retain device health metrics", 2024-03-19T02:05:36.791 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.792 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.792 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.792 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.792 INFO:teuthology.orchestra.run.smithi099.stdout: "scrape_frequency": { 2024-03-19T02:05:36.792 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "scrape_frequency", 2024-03-19T02:05:36.792 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:36.792 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.792 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.792 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "86400", 2024-03-19T02:05:36.792 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.792 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.792 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.792 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "how frequently to scrape device health metrics", 2024-03-19T02:05:36.792 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.793 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.793 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.793 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.793 INFO:teuthology.orchestra.run.smithi099.stdout: "self_heal": { 2024-03-19T02:05:36.793 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "self_heal", 2024-03-19T02:05:36.793 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.793 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.793 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.793 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:36.793 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.793 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.793 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.793 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "preemptively heal cluster around devices that may fail", 2024-03-19T02:05:36.793 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.794 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.794 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.794 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.794 INFO:teuthology.orchestra.run.smithi099.stdout: "sleep_interval": { 2024-03-19T02:05:36.794 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "sleep_interval", 2024-03-19T02:05:36.794 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:36.794 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.794 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.794 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "600", 2024-03-19T02:05:36.794 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.794 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.794 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.794 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "how frequently to wake up and check device health", 2024-03-19T02:05:36.794 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.794 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.795 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.795 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.795 INFO:teuthology.orchestra.run.smithi099.stdout: "warn_threshold": { 2024-03-19T02:05:36.795 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "warn_threshold", 2024-03-19T02:05:36.795 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:36.795 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.795 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.795 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "7257600", 2024-03-19T02:05:36.795 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.795 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.795 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.795 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "raise health warning if OSD may fail before this long", 2024-03-19T02:05:36.795 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.795 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.796 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.796 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.796 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.796 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.796 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:36.796 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "diskprediction_local", 2024-03-19T02:05:36.796 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:36.796 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:36.796 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:36.796 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:36.796 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:36.796 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.796 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.796 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.796 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.797 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.797 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.797 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.797 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.797 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.797 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.797 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.797 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.797 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.797 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.797 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.797 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.797 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.797 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.797 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.798 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:36.798 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:36.798 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.798 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.798 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.798 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.798 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.798 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.798 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.798 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.798 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.798 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.798 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.798 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.799 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:36.799 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:36.799 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.799 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.799 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.799 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:36.799 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.799 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.799 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.799 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.799 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.799 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.799 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.799 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.800 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.800 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.800 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.800 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.800 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.800 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.800 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.800 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:36.800 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:36.800 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.800 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.800 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.800 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.800 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.800 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.801 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.801 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.801 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.801 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.801 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.801 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.801 INFO:teuthology.orchestra.run.smithi099.stdout: "predict_interval": { 2024-03-19T02:05:36.801 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "predict_interval", 2024-03-19T02:05:36.801 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.801 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.801 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.801 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "86400", 2024-03-19T02:05:36.801 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.801 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.801 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.802 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.802 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.802 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.802 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.802 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.802 INFO:teuthology.orchestra.run.smithi099.stdout: "predictor_model": { 2024-03-19T02:05:36.802 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "predictor_model", 2024-03-19T02:05:36.802 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.802 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.802 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.802 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "prophetstor", 2024-03-19T02:05:36.802 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.802 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.802 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.803 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.803 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.803 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.803 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.803 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.803 INFO:teuthology.orchestra.run.smithi099.stdout: "sleep_interval": { 2024-03-19T02:05:36.803 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "sleep_interval", 2024-03-19T02:05:36.803 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.803 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.803 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.803 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "600", 2024-03-19T02:05:36.803 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.803 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.803 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.803 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.804 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.804 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.804 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.804 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.804 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.804 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.804 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:36.804 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "influx", 2024-03-19T02:05:36.804 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": false, 2024-03-19T02:05:36.804 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "influxdb python module not found", 2024-03-19T02:05:36.804 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:36.804 INFO:teuthology.orchestra.run.smithi099.stdout: "batch_size": { 2024-03-19T02:05:36.804 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "batch_size", 2024-03-19T02:05:36.804 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:36.804 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.805 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.805 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "5000", 2024-03-19T02:05:36.805 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.805 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.805 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.805 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "How big batches of data points should be when sending to InfluxDB.", 2024-03-19T02:05:36.805 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.805 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.805 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.805 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.805 INFO:teuthology.orchestra.run.smithi099.stdout: "database": { 2024-03-19T02:05:36.805 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "database", 2024-03-19T02:05:36.805 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.805 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.806 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.806 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "ceph", 2024-03-19T02:05:36.806 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.806 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.806 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.806 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.", 2024-03-19T02:05:36.806 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.806 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.806 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.806 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.806 INFO:teuthology.orchestra.run.smithi099.stdout: "hostname": { 2024-03-19T02:05:36.806 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "hostname", 2024-03-19T02:05:36.806 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.806 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.807 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.807 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.807 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.807 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.807 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.807 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "InfluxDB server hostname", 2024-03-19T02:05:36.807 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.807 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.807 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.807 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.807 INFO:teuthology.orchestra.run.smithi099.stdout: "interval": { 2024-03-19T02:05:36.807 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "interval", 2024-03-19T02:05:36.807 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:36.807 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.807 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.808 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "30", 2024-03-19T02:05:36.808 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "5", 2024-03-19T02:05:36.808 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.808 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.808 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Time between reports to InfluxDB. Default 30 seconds.", 2024-03-19T02:05:36.808 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.808 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.808 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.808 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.808 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:36.808 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:36.808 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.808 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.808 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.809 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.809 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.809 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.809 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.809 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.809 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.809 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.809 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.809 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.809 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.809 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.809 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.809 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.809 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.809 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.810 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.810 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:36.810 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:36.810 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.810 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.810 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.810 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.810 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.810 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.810 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.810 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.810 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.810 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.810 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.810 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.811 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:36.811 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:36.811 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.811 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.811 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.811 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:36.811 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.811 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.811 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.811 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.811 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.811 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.811 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.811 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.812 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.812 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.812 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.812 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.812 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.812 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.812 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.812 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:36.812 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:36.812 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.812 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.812 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.812 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.812 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.812 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.813 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.813 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.813 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.813 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.813 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.813 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.813 INFO:teuthology.orchestra.run.smithi099.stdout: "password": { 2024-03-19T02:05:36.813 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "password", 2024-03-19T02:05:36.813 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.813 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.813 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.813 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.813 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.813 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.813 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.814 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "password of InfluxDB server user", 2024-03-19T02:05:36.814 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.814 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.814 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.814 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.814 INFO:teuthology.orchestra.run.smithi099.stdout: "port": { 2024-03-19T02:05:36.814 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "port", 2024-03-19T02:05:36.814 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:36.814 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.814 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.814 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "8086", 2024-03-19T02:05:36.814 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.814 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.814 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.814 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "InfluxDB server port", 2024-03-19T02:05:36.815 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.815 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.815 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.815 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.815 INFO:teuthology.orchestra.run.smithi099.stdout: "ssl": { 2024-03-19T02:05:36.815 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "ssl", 2024-03-19T02:05:36.815 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.815 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.815 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.815 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "false", 2024-03-19T02:05:36.815 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.815 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.815 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.815 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Use https connection for InfluxDB server. Use \"true\" or \"false\".", 2024-03-19T02:05:36.815 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.816 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.816 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.816 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.816 INFO:teuthology.orchestra.run.smithi099.stdout: "threads": { 2024-03-19T02:05:36.816 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "threads", 2024-03-19T02:05:36.816 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:36.816 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.816 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.816 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "5", 2024-03-19T02:05:36.816 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "1", 2024-03-19T02:05:36.816 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "32", 2024-03-19T02:05:36.816 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.816 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "How many worker threads should be spawned for sending data to InfluxDB.", 2024-03-19T02:05:36.816 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.816 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.817 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.817 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.817 INFO:teuthology.orchestra.run.smithi099.stdout: "username": { 2024-03-19T02:05:36.817 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "username", 2024-03-19T02:05:36.817 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.817 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.817 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.817 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.817 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.817 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.817 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.817 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "username of InfluxDB server user", 2024-03-19T02:05:36.817 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.817 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.817 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.818 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.818 INFO:teuthology.orchestra.run.smithi099.stdout: "verify_ssl": { 2024-03-19T02:05:36.818 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "verify_ssl", 2024-03-19T02:05:36.818 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.818 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.818 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.818 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "true", 2024-03-19T02:05:36.818 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.818 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.818 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.818 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Verify https cert for InfluxDB server. Use \"true\" or \"false\".", 2024-03-19T02:05:36.818 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.818 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.819 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.819 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.819 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.819 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.819 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:36.819 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "insights", 2024-03-19T02:05:36.819 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:36.819 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:36.819 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:36.819 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:36.819 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:36.819 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.819 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.819 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.819 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.820 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.820 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.820 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.820 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.820 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.820 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.820 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.820 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.820 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.820 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.820 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.820 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.820 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.820 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.821 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.821 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:36.821 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:36.821 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.821 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.821 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.821 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.821 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.821 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.821 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.821 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.821 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.821 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.821 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.821 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.822 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:36.822 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:36.822 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.822 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.822 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.822 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:36.822 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.822 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.822 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.822 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.822 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.822 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.822 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.822 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.823 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.823 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.823 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.823 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.823 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.823 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.823 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.823 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:36.823 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:36.823 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.823 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.823 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.823 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.823 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.823 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.824 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.824 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.824 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.824 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.824 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.824 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.824 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.824 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.824 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:36.824 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "iostat", 2024-03-19T02:05:36.824 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:36.824 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:36.824 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:36.824 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:36.824 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:36.825 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.825 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.825 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.825 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.825 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.825 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.825 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.825 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.825 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.825 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.825 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.825 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.825 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.825 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.826 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.826 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.826 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.826 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.826 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.826 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:36.826 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:36.826 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.826 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.826 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.826 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.826 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.826 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.826 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.826 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.827 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.827 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.827 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.827 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.827 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:36.827 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:36.827 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.827 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.827 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.827 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:36.827 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.827 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.827 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.827 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.827 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.828 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.828 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.828 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.828 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.828 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.828 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.828 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.828 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.828 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.828 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.828 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:36.828 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:36.828 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.828 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.828 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.829 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.829 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.829 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.829 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.829 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.829 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.829 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.829 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.829 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.829 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.829 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.829 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:36.829 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "k8sevents", 2024-03-19T02:05:36.829 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:36.829 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:36.830 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:36.830 INFO:teuthology.orchestra.run.smithi099.stdout: "ceph_event_retention_days": { 2024-03-19T02:05:36.830 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "ceph_event_retention_days", 2024-03-19T02:05:36.830 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:36.830 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.830 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.830 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "7", 2024-03-19T02:05:36.830 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.830 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.830 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.830 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Days to hold ceph event information within local cache", 2024-03-19T02:05:36.830 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.830 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.830 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.831 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.831 INFO:teuthology.orchestra.run.smithi099.stdout: "config_check_secs": { 2024-03-19T02:05:36.831 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "config_check_secs", 2024-03-19T02:05:36.831 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:36.831 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.831 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.831 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "10", 2024-03-19T02:05:36.831 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "10", 2024-03-19T02:05:36.831 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.831 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.831 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "interval (secs) to check for cluster configuration changes", 2024-03-19T02:05:36.831 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.831 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.831 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.832 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.832 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:36.832 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:36.832 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.832 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.832 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.832 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.832 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.832 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.832 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.832 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.832 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.832 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.832 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.832 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.833 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.833 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.833 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.833 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.833 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.833 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.833 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.833 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:36.833 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:36.833 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.833 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.833 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.833 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.833 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.834 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.834 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.834 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.834 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.834 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.834 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.834 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.834 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:36.834 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:36.834 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.834 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.834 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.834 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:36.834 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.834 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.835 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.835 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.835 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.835 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.835 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.835 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.835 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.835 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.835 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.835 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.835 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.835 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.835 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.835 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:36.836 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:36.836 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.836 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.836 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.836 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.836 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.836 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.836 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.836 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.836 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.836 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.836 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.836 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.836 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.836 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.837 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:36.837 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "localpool", 2024-03-19T02:05:36.837 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:36.837 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:36.837 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:36.837 INFO:teuthology.orchestra.run.smithi099.stdout: "failure_domain": { 2024-03-19T02:05:36.837 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "failure_domain", 2024-03-19T02:05:36.837 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.837 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.837 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.837 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "host", 2024-03-19T02:05:36.837 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.837 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.837 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.837 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "failure domain for any created local pool", 2024-03-19T02:05:36.838 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "what failure domain we should separate data replicas across.", 2024-03-19T02:05:36.838 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.838 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.838 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.838 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:36.838 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:36.838 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.838 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.838 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.838 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.838 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.838 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.838 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.838 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.839 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.839 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.839 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.839 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.839 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.839 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.839 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.839 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.839 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.839 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.839 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.839 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:36.839 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:36.839 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.839 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.840 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.841 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.841 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.841 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.841 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.841 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.841 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.841 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.841 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.841 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.841 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:36.842 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:36.842 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.842 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.842 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.842 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:36.842 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.842 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.842 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.842 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.842 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.842 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.842 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.842 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.842 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.842 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.843 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.843 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.843 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.843 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.843 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.843 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:36.843 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:36.843 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.843 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.843 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.843 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.843 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.843 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.843 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.844 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.844 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.844 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.844 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.844 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.844 INFO:teuthology.orchestra.run.smithi099.stdout: "min_size": { 2024-03-19T02:05:36.844 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "min_size", 2024-03-19T02:05:36.844 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:36.844 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.844 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.844 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.844 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.844 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.844 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.844 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "default min_size for any created local pool", 2024-03-19T02:05:36.845 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "value to set min_size to (unchanged from Ceph's default if this option is not set)", 2024-03-19T02:05:36.845 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.845 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.845 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.845 INFO:teuthology.orchestra.run.smithi099.stdout: "num_rep": { 2024-03-19T02:05:36.845 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "num_rep", 2024-03-19T02:05:36.845 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:36.845 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.845 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.845 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "3", 2024-03-19T02:05:36.845 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.845 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.845 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.845 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "default replica count for any created local pool", 2024-03-19T02:05:36.846 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.846 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.846 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.846 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.846 INFO:teuthology.orchestra.run.smithi099.stdout: "pg_num": { 2024-03-19T02:05:36.846 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "pg_num", 2024-03-19T02:05:36.846 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:36.846 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.846 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.846 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "128", 2024-03-19T02:05:36.846 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.846 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.846 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.846 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "default pg_num for any created local pool", 2024-03-19T02:05:36.847 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.847 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.847 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.847 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.847 INFO:teuthology.orchestra.run.smithi099.stdout: "prefix": { 2024-03-19T02:05:36.847 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "prefix", 2024-03-19T02:05:36.847 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.847 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.847 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.847 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.847 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.847 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.847 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.847 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "name prefix for any created local pool", 2024-03-19T02:05:36.847 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.848 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.848 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.848 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.848 INFO:teuthology.orchestra.run.smithi099.stdout: "subtree": { 2024-03-19T02:05:36.848 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "subtree", 2024-03-19T02:05:36.848 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.848 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.848 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.848 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "rack", 2024-03-19T02:05:36.848 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.848 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.848 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.848 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "CRUSH level for which to create a local pool", 2024-03-19T02:05:36.848 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "which CRUSH subtree type the module should create a pool for.", 2024-03-19T02:05:36.848 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.849 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.849 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.849 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.849 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.849 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:36.849 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "mds_autoscaler", 2024-03-19T02:05:36.849 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:36.849 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:36.849 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:36.849 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:36.849 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:36.849 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.849 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.849 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.850 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.850 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.850 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.850 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.850 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.850 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.850 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.850 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.850 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.850 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.850 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.850 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.850 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.850 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.850 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.851 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.851 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:36.851 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:36.851 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.851 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.851 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.851 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.851 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.851 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.851 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.851 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.851 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.851 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.851 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.851 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.852 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:36.852 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:36.852 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.852 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.852 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.852 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:36.852 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.852 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.852 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.852 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.852 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.852 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.852 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.852 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.853 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.853 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.853 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.853 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.853 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.853 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.853 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.853 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:36.853 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:36.853 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.853 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.853 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.853 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.853 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.853 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.854 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.854 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.854 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.854 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.854 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.854 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.854 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.854 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.854 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:36.854 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "mirroring", 2024-03-19T02:05:36.854 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:36.854 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:36.854 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:36.854 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:36.854 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:36.855 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.855 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.855 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.855 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.855 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.855 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.855 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.855 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.855 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.855 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.855 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.855 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.855 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.855 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.855 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.856 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.856 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.856 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.856 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.856 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:36.856 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:36.856 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.856 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.856 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.856 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.856 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.856 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.856 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.856 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.856 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.857 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.857 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.857 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.857 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:36.857 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:36.857 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.857 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.857 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.857 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:36.857 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.857 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.857 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.857 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.857 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.857 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.858 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.858 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.858 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.858 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.858 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.858 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.858 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.858 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.858 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.858 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:36.858 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:36.858 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.858 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.858 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.858 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.859 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.859 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.859 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.859 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.859 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.859 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.859 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.859 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.859 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.859 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.859 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:36.859 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "nfs", 2024-03-19T02:05:36.859 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:36.859 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:36.860 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:36.860 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:36.860 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:36.860 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.860 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.860 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.860 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.860 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.860 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.860 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.860 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.860 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.860 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.860 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.860 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.861 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.861 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.861 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.861 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.861 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.861 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.861 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.861 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:36.861 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:36.861 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.861 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.861 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.861 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.861 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.862 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.862 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.862 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.862 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.862 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.862 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.862 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.862 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:36.862 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:36.862 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.862 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.862 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.862 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:36.862 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.862 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.863 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.863 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.863 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.863 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.863 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.863 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.863 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.863 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.863 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.863 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.863 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.863 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.863 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.863 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:36.863 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:36.864 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.864 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.864 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.864 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.864 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.864 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.864 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.864 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.864 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.864 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.864 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.864 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.864 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.864 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.865 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:36.865 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "orchestrator", 2024-03-19T02:05:36.865 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:36.865 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:36.865 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:36.865 INFO:teuthology.orchestra.run.smithi099.stdout: "fail_fs": { 2024-03-19T02:05:36.865 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "fail_fs", 2024-03-19T02:05:36.865 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.865 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.865 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.865 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.865 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.865 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.865 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.865 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Fail filesystem for rapid multi-rank mds upgrade", 2024-03-19T02:05:36.866 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.866 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.866 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.866 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.866 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:36.866 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:36.866 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.866 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.866 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.866 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.866 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.866 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.866 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.866 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.867 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.867 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.867 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.867 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.867 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.867 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.867 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.867 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.867 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.867 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.867 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.867 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:36.867 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:36.867 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.867 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.868 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.868 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.868 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.868 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.868 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.868 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.868 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.868 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.868 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.868 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.868 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:36.868 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:36.868 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.868 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.868 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.869 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:36.869 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.869 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.869 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.869 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.869 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.869 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.869 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.869 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.869 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.869 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.869 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.869 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.869 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.869 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.870 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.870 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:36.870 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:36.870 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.870 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.870 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.870 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.870 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.870 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.870 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.870 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.870 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.870 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.870 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.871 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.871 INFO:teuthology.orchestra.run.smithi099.stdout: "orchestrator": { 2024-03-19T02:05:36.871 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "orchestrator", 2024-03-19T02:05:36.871 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.871 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.871 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.871 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.871 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.871 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.871 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.871 INFO:teuthology.orchestra.run.smithi099.stdout: "cephadm", 2024-03-19T02:05:36.871 INFO:teuthology.orchestra.run.smithi099.stdout: "rook", 2024-03-19T02:05:36.871 INFO:teuthology.orchestra.run.smithi099.stdout: "test_orchestrator" 2024-03-19T02:05:36.871 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.871 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Orchestrator backend", 2024-03-19T02:05:36.872 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.872 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.872 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.872 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.872 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.872 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.872 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:36.872 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "osd_perf_query", 2024-03-19T02:05:36.872 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:36.872 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:36.872 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:36.872 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:36.872 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:36.872 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.873 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.873 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.873 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.873 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.873 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.873 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.873 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.873 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.873 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.873 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.873 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.873 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.873 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.873 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.873 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.874 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.874 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.874 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.874 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:36.874 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:36.874 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.874 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.874 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.874 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.874 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.874 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.874 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.874 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.874 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.874 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.875 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.875 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.875 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:36.875 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:36.875 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.875 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.875 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.875 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:36.875 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.875 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.875 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.875 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.875 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.875 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.875 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.876 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.876 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.876 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.876 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.876 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.876 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.876 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.876 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.876 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:36.876 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:36.876 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.876 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.876 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.876 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.877 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.877 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.877 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.877 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.877 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.877 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.877 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.877 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.877 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.877 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.877 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:36.877 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "osd_support", 2024-03-19T02:05:36.877 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:36.877 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:36.877 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:36.878 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:36.878 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:36.878 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.878 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.878 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.878 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.878 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.878 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.878 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.878 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.878 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.878 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.878 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.878 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.879 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.879 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.879 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.879 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.879 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.879 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.879 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.879 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:36.879 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:36.879 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.879 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.879 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.879 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.879 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.879 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.880 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.880 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.880 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.880 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.880 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.880 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.880 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:36.880 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:36.880 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.880 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.880 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.880 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:36.880 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.880 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.880 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.881 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.881 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.881 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.881 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.881 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.881 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.881 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.881 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.881 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.881 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.881 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.881 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.881 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:36.881 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:36.881 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.882 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.882 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.882 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.882 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.882 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.882 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.882 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.882 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.882 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.882 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.882 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.882 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.882 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.882 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:36.882 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "pg_autoscaler", 2024-03-19T02:05:36.883 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:36.883 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:36.883 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:36.883 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:36.883 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:36.883 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.883 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.883 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.883 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.883 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.883 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.883 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.883 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.883 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.884 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.884 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.884 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.884 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.884 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.884 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.884 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.884 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.884 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.884 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.884 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:36.884 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:36.884 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.884 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.884 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.884 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.885 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.885 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.885 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.885 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.885 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.885 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.885 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.885 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.885 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:36.885 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:36.885 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.885 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.885 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.885 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:36.885 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.886 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.886 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.886 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.886 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.886 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.886 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.886 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.886 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.886 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.886 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.886 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.886 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.887 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.887 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.887 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:36.887 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:36.887 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.887 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.887 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.887 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.887 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.887 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.887 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.887 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.887 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.887 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.887 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.888 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.888 INFO:teuthology.orchestra.run.smithi099.stdout: "sleep_interval": { 2024-03-19T02:05:36.888 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "sleep_interval", 2024-03-19T02:05:36.888 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:36.888 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.888 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.888 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "60", 2024-03-19T02:05:36.888 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.888 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.888 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.888 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.888 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.888 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.888 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.888 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.889 INFO:teuthology.orchestra.run.smithi099.stdout: "threshold": { 2024-03-19T02:05:36.889 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "threshold", 2024-03-19T02:05:36.889 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "float", 2024-03-19T02:05:36.889 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.889 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.889 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "3.0", 2024-03-19T02:05:36.889 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "1.0", 2024-03-19T02:05:36.889 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.889 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.889 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "scaling threshold", 2024-03-19T02:05:36.889 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0", 2024-03-19T02:05:36.889 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.889 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.889 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.890 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.890 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.890 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:36.890 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "progress", 2024-03-19T02:05:36.890 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:36.890 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:36.890 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:36.890 INFO:teuthology.orchestra.run.smithi099.stdout: "allow_pg_recovery_event": { 2024-03-19T02:05:36.890 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "allow_pg_recovery_event", 2024-03-19T02:05:36.890 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.890 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.890 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.890 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.890 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.891 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.891 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.891 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "allow the module to show pg recovery progress", 2024-03-19T02:05:36.891 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.891 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.891 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.891 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.891 INFO:teuthology.orchestra.run.smithi099.stdout: "enabled": { 2024-03-19T02:05:36.891 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "enabled", 2024-03-19T02:05:36.891 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.891 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.891 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.891 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:36.891 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.891 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.892 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.892 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.892 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.892 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.892 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.892 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.892 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:36.892 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:36.892 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.892 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.892 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.892 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.892 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.892 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.892 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.893 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.893 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.893 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.893 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.893 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.893 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.893 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.893 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.893 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.893 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.893 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.893 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.893 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:36.893 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:36.893 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.894 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.894 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.894 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.894 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.894 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.894 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.894 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.894 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.894 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.894 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.894 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.894 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:36.894 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:36.894 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.895 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.895 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.895 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:36.895 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.895 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.895 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.895 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.895 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.895 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.895 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.895 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.895 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.895 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.895 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.895 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.896 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.896 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.896 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.896 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:36.896 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:36.896 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.896 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.896 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.896 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.896 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.896 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.896 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.896 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.896 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.896 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.897 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.897 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.897 INFO:teuthology.orchestra.run.smithi099.stdout: "max_completed_events": { 2024-03-19T02:05:36.897 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "max_completed_events", 2024-03-19T02:05:36.897 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:36.897 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.897 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.897 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "50", 2024-03-19T02:05:36.897 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.897 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.897 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.897 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "number of past completed events to remember", 2024-03-19T02:05:36.897 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.897 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.897 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.898 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.898 INFO:teuthology.orchestra.run.smithi099.stdout: "sleep_interval": { 2024-03-19T02:05:36.898 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "sleep_interval", 2024-03-19T02:05:36.898 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:36.898 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.898 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.898 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "5", 2024-03-19T02:05:36.898 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.898 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.898 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.898 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "how long the module is going to sleep", 2024-03-19T02:05:36.898 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.898 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.898 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.899 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.899 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.899 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.899 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:36.899 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "prometheus", 2024-03-19T02:05:36.899 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:36.899 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:36.899 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:36.899 INFO:teuthology.orchestra.run.smithi099.stdout: "cache": { 2024-03-19T02:05:36.899 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "cache", 2024-03-19T02:05:36.899 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.899 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.899 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.900 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:36.900 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.900 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.900 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.900 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.900 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.900 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.900 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.900 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.900 INFO:teuthology.orchestra.run.smithi099.stdout: "exclude_perf_counters": { 2024-03-19T02:05:36.900 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "exclude_perf_counters", 2024-03-19T02:05:36.900 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.900 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.900 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.900 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:36.901 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.901 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.901 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.901 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Do not include perf-counters in the metrics output", 2024-03-19T02:05:36.901 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.", 2024-03-19T02:05:36.901 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.901 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.901 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.901 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:36.901 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:36.901 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.901 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.901 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.901 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.901 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.902 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.902 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.902 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.902 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.902 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.902 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.902 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.902 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.902 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.902 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.902 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.902 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.902 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.902 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.902 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:36.903 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:36.903 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.903 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.903 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.903 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.903 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.903 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.903 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.903 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.903 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.903 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.903 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.903 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.903 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:36.904 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:36.904 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.904 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.904 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.904 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:36.904 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.904 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.904 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.904 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.904 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.904 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.904 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.904 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.904 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.904 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.905 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.905 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.905 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.905 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.905 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.905 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:36.905 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:36.905 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.905 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.905 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.905 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.905 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.905 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.905 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.905 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.906 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.906 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.906 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.906 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.906 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_stats_pools": { 2024-03-19T02:05:36.906 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "rbd_stats_pools", 2024-03-19T02:05:36.906 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.906 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.906 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.906 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.906 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.906 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.906 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.906 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.907 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.907 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.907 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.907 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.907 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_stats_pools_refresh_interval": { 2024-03-19T02:05:36.907 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "rbd_stats_pools_refresh_interval", 2024-03-19T02:05:36.907 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:36.907 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.907 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.907 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "300", 2024-03-19T02:05:36.907 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.907 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.907 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.907 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.907 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.908 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.908 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.908 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.908 INFO:teuthology.orchestra.run.smithi099.stdout: "scrape_interval": { 2024-03-19T02:05:36.908 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "scrape_interval", 2024-03-19T02:05:36.908 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "float", 2024-03-19T02:05:36.908 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.908 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.908 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "15.0", 2024-03-19T02:05:36.908 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.908 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.908 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.908 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.908 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.908 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.909 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.909 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.909 INFO:teuthology.orchestra.run.smithi099.stdout: "server_addr": { 2024-03-19T02:05:36.909 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "server_addr", 2024-03-19T02:05:36.909 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.909 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.909 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.909 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "::", 2024-03-19T02:05:36.909 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.909 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.909 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.909 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "the IPv4 or IPv6 address on which the module listens for HTTP requests", 2024-03-19T02:05:36.909 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.909 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.909 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.910 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.910 INFO:teuthology.orchestra.run.smithi099.stdout: "server_port": { 2024-03-19T02:05:36.910 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "server_port", 2024-03-19T02:05:36.910 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:36.910 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.910 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.910 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "9283", 2024-03-19T02:05:36.910 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.910 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.910 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.910 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "the port on which the module listens for HTTP requests", 2024-03-19T02:05:36.910 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.910 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.910 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.911 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.911 INFO:teuthology.orchestra.run.smithi099.stdout: "stale_cache_strategy": { 2024-03-19T02:05:36.911 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "stale_cache_strategy", 2024-03-19T02:05:36.911 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.911 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.911 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.911 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "log", 2024-03-19T02:05:36.911 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.911 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.911 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.911 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.911 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.911 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.911 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.912 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.912 INFO:teuthology.orchestra.run.smithi099.stdout: "standby_behaviour": { 2024-03-19T02:05:36.912 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "standby_behaviour", 2024-03-19T02:05:36.912 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.912 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.912 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.912 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "default", 2024-03-19T02:05:36.912 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.912 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.912 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.912 INFO:teuthology.orchestra.run.smithi099.stdout: "default", 2024-03-19T02:05:36.912 INFO:teuthology.orchestra.run.smithi099.stdout: "error" 2024-03-19T02:05:36.912 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.912 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.912 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.913 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.913 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.913 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.913 INFO:teuthology.orchestra.run.smithi099.stdout: "standby_error_status_code": { 2024-03-19T02:05:36.913 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "standby_error_status_code", 2024-03-19T02:05:36.913 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:36.913 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.913 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.913 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "500", 2024-03-19T02:05:36.913 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "400", 2024-03-19T02:05:36.913 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "599", 2024-03-19T02:05:36.913 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.913 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.913 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.913 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.914 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.914 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.914 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.914 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.914 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:36.914 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "rbd_support", 2024-03-19T02:05:36.914 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:36.914 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:36.914 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:36.914 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:36.914 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:36.914 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.914 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.914 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.915 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.915 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.915 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.915 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.915 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.915 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.915 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.915 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.915 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.915 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.915 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.915 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.915 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.915 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.915 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.916 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.916 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:36.916 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:36.916 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.916 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.916 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.916 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.916 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.916 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.916 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.916 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.916 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.916 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.916 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.916 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.917 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:36.917 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:36.917 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.917 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.917 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.917 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:36.917 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.917 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.917 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.917 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.917 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.917 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.917 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.917 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.917 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.918 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.918 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.918 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.918 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.918 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.918 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.918 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:36.918 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:36.918 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.918 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.918 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.918 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.918 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.918 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.919 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.919 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.919 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.919 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.919 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.919 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.919 INFO:teuthology.orchestra.run.smithi099.stdout: "max_concurrent_snap_create": { 2024-03-19T02:05:36.919 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "max_concurrent_snap_create", 2024-03-19T02:05:36.919 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:36.919 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.919 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.919 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "10", 2024-03-19T02:05:36.919 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.919 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.919 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.920 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.920 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.920 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.920 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.920 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.920 INFO:teuthology.orchestra.run.smithi099.stdout: "mirror_snapshot_schedule": { 2024-03-19T02:05:36.920 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "mirror_snapshot_schedule", 2024-03-19T02:05:36.920 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.920 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.920 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.920 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.920 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.920 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.920 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.920 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.921 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.921 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.921 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.921 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.921 INFO:teuthology.orchestra.run.smithi099.stdout: "trash_purge_schedule": { 2024-03-19T02:05:36.921 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "trash_purge_schedule", 2024-03-19T02:05:36.921 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.921 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.921 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.921 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.921 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.921 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.921 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.921 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.921 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.922 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.922 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.922 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.922 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.922 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.922 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:36.922 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "restful", 2024-03-19T02:05:36.922 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:36.922 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:36.922 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:36.922 INFO:teuthology.orchestra.run.smithi099.stdout: "enable_auth": { 2024-03-19T02:05:36.922 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "enable_auth", 2024-03-19T02:05:36.922 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.922 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.923 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.923 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:36.923 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.923 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.923 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.923 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.923 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.923 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.923 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.923 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.923 INFO:teuthology.orchestra.run.smithi099.stdout: "key_file": { 2024-03-19T02:05:36.923 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "key_file", 2024-03-19T02:05:36.923 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.923 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.923 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.924 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.924 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.924 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.924 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.924 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.924 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.924 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.924 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.924 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.924 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:36.924 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:36.924 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.925 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.925 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.925 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.925 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.925 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.925 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.925 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.925 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.925 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.925 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.925 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.925 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.925 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.925 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.926 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.926 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.926 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.926 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.926 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:36.926 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:36.926 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.926 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.926 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.926 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.926 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.926 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.926 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.926 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.927 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.927 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.927 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.927 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.927 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:36.927 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:36.927 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.927 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.927 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.927 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:36.927 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.927 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.927 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.927 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.927 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.928 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.928 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.928 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.928 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.928 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.928 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.928 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.928 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.928 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.928 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.928 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:36.928 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:36.928 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.928 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.928 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.929 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.929 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.929 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.929 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.929 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.929 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.929 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.929 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.929 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.929 INFO:teuthology.orchestra.run.smithi099.stdout: "server_addr": { 2024-03-19T02:05:36.929 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "server_addr", 2024-03-19T02:05:36.929 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.929 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.929 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.929 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.930 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.930 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.930 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.930 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.930 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.930 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.930 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.930 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.930 INFO:teuthology.orchestra.run.smithi099.stdout: "server_port": { 2024-03-19T02:05:36.930 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "server_port", 2024-03-19T02:05:36.930 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.930 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.930 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.930 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.931 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.931 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.931 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.931 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.931 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.931 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.931 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.931 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.931 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.931 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.931 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:36.931 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "rgw", 2024-03-19T02:05:36.931 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:36.931 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:36.931 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:36.932 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:36.932 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:36.932 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.932 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.932 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.932 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.932 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.932 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.932 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.932 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.932 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.932 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.932 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.932 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.932 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.933 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.933 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.933 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.933 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.933 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.933 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.933 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:36.933 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:36.933 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.933 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.933 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.933 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.933 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.933 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.933 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.934 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.934 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.934 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.934 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.934 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.934 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:36.934 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:36.934 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.934 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.934 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.934 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:36.934 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.934 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.934 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.935 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.935 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.935 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.935 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.935 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.935 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.935 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.935 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.935 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.935 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.935 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.935 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.935 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:36.935 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:36.935 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.936 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.936 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.936 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.936 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.936 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.936 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.936 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.936 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.936 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.936 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.936 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.936 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.936 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.936 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:36.936 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "rook", 2024-03-19T02:05:36.937 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:36.937 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:36.937 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:36.937 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:36.937 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:36.937 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.937 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.937 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.937 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.937 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.937 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.937 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.937 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.937 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.937 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.938 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.938 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.938 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.938 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.938 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.938 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.938 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.938 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.938 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.938 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:36.938 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:36.938 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.938 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.938 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.939 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.939 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.939 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.939 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.939 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.939 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.939 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.939 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.939 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.939 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:36.939 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:36.939 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.939 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.939 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.939 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:36.940 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.940 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.940 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.940 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.940 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.940 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.940 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.940 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.940 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.940 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.940 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.940 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.940 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.940 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.940 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.941 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:36.941 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:36.941 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.941 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.941 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.941 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.941 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.941 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.941 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.941 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.941 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.941 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.941 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.941 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.941 INFO:teuthology.orchestra.run.smithi099.stdout: "storage_class": { 2024-03-19T02:05:36.942 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "storage_class", 2024-03-19T02:05:36.942 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.942 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.942 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.942 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "local", 2024-03-19T02:05:36.942 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.942 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.942 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.942 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "storage class name for LSO-discovered PVs", 2024-03-19T02:05:36.942 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.942 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.942 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.942 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.942 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.943 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.943 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:36.943 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "selftest", 2024-03-19T02:05:36.943 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:36.943 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:36.943 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:36.943 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:36.943 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:36.943 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.943 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.943 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.943 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.943 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.943 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.944 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.944 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.944 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.944 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.944 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.944 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.944 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.944 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.944 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.944 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.944 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.944 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.944 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.944 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:36.944 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:36.945 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.945 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.945 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.945 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.945 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.945 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.945 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.945 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.945 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.945 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.945 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.945 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.945 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:36.945 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:36.945 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.946 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.946 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.946 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:36.946 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.946 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.946 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.946 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.946 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.946 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.946 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.946 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.946 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.946 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.946 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.947 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.947 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.947 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.947 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.947 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:36.947 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:36.947 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.947 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.947 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.947 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.947 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.947 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.947 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.947 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.947 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.948 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.948 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.948 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.948 INFO:teuthology.orchestra.run.smithi099.stdout: "roption1": { 2024-03-19T02:05:36.948 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "roption1", 2024-03-19T02:05:36.948 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.948 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.948 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.948 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.948 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.948 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.948 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.948 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.948 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.949 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.949 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.949 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.949 INFO:teuthology.orchestra.run.smithi099.stdout: "roption2": { 2024-03-19T02:05:36.949 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "roption2", 2024-03-19T02:05:36.949 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.949 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.949 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.949 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "xyz", 2024-03-19T02:05:36.949 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.949 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.949 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.949 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.949 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.949 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.950 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.950 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.950 INFO:teuthology.orchestra.run.smithi099.stdout: "rwoption1": { 2024-03-19T02:05:36.950 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "rwoption1", 2024-03-19T02:05:36.950 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.950 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.950 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.950 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.950 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.950 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.950 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.950 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.950 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.950 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.951 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.951 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.951 INFO:teuthology.orchestra.run.smithi099.stdout: "rwoption2": { 2024-03-19T02:05:36.951 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "rwoption2", 2024-03-19T02:05:36.951 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:36.951 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.951 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.951 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.951 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.951 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.951 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.951 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.951 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.952 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.952 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.952 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.952 INFO:teuthology.orchestra.run.smithi099.stdout: "rwoption3": { 2024-03-19T02:05:36.952 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "rwoption3", 2024-03-19T02:05:36.952 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "float", 2024-03-19T02:05:36.952 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.952 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.952 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.952 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.952 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.952 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.952 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.952 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.952 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.953 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.953 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.953 INFO:teuthology.orchestra.run.smithi099.stdout: "rwoption4": { 2024-03-19T02:05:36.953 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "rwoption4", 2024-03-19T02:05:36.953 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.953 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.953 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.953 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.953 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.953 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.953 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.953 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.953 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.953 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.953 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.954 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.954 INFO:teuthology.orchestra.run.smithi099.stdout: "rwoption5": { 2024-03-19T02:05:36.954 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "rwoption5", 2024-03-19T02:05:36.954 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.954 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.954 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.954 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.954 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.954 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.954 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.954 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.954 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.954 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.954 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.955 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.955 INFO:teuthology.orchestra.run.smithi099.stdout: "rwoption6": { 2024-03-19T02:05:36.955 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "rwoption6", 2024-03-19T02:05:36.955 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.955 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.955 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.955 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:36.955 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.955 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.955 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.955 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.955 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.955 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.955 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.955 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.956 INFO:teuthology.orchestra.run.smithi099.stdout: "rwoption7": { 2024-03-19T02:05:36.956 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "rwoption7", 2024-03-19T02:05:36.956 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:36.956 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.956 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.956 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.956 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "1", 2024-03-19T02:05:36.956 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "42", 2024-03-19T02:05:36.956 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.956 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.956 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.956 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.956 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.956 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.957 INFO:teuthology.orchestra.run.smithi099.stdout: "testkey": { 2024-03-19T02:05:36.957 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "testkey", 2024-03-19T02:05:36.957 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.957 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.957 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.957 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.957 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.957 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.957 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.957 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.957 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.957 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.957 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.957 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.957 INFO:teuthology.orchestra.run.smithi099.stdout: "testlkey": { 2024-03-19T02:05:36.958 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "testlkey", 2024-03-19T02:05:36.958 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.958 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.958 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.958 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.958 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.958 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.958 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.958 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.958 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.958 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.958 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.958 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.958 INFO:teuthology.orchestra.run.smithi099.stdout: "testnewline": { 2024-03-19T02:05:36.959 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "testnewline", 2024-03-19T02:05:36.959 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.959 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.959 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.959 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.959 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.959 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.959 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.959 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.959 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.959 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.959 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.959 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.959 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.959 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.960 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:36.960 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "snap_schedule", 2024-03-19T02:05:36.960 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:36.960 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:36.960 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:36.960 INFO:teuthology.orchestra.run.smithi099.stdout: "allow_m_granularity": { 2024-03-19T02:05:36.960 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "allow_m_granularity", 2024-03-19T02:05:36.960 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.960 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.960 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.960 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.960 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.960 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.960 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.960 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "allow minute scheduled snapshots", 2024-03-19T02:05:36.961 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.961 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.961 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.961 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.961 INFO:teuthology.orchestra.run.smithi099.stdout: "dump_on_update": { 2024-03-19T02:05:36.961 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "dump_on_update", 2024-03-19T02:05:36.961 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.961 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.961 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.961 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.961 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.961 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.961 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.961 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "dump database to debug log on update", 2024-03-19T02:05:36.961 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.962 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.962 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.962 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.962 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:36.962 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:36.962 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.962 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.962 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.962 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.962 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.962 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.962 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.962 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.962 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.963 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.963 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.963 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.963 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.963 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.963 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.963 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.963 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.963 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.963 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.963 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:36.963 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:36.963 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.963 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.963 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.964 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.964 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.964 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.964 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.964 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.964 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.964 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.964 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.964 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.964 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:36.964 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:36.964 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.964 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.964 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.964 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:36.965 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.965 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.965 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.965 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.965 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.965 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.965 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.965 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.965 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.965 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.965 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.965 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.965 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.965 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.966 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.966 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:36.966 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:36.966 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.966 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.966 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.966 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.966 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.966 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.966 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.966 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.966 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.966 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.966 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.967 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.967 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.967 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.967 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:36.967 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "stats", 2024-03-19T02:05:36.967 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:36.967 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:36.967 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:36.967 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:36.967 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:36.967 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.967 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.967 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.967 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.967 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.968 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.968 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.968 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.968 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.968 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.968 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.968 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.968 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.968 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.968 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.968 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.968 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.968 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.968 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.969 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:36.969 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:36.969 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.969 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.969 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.969 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.969 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.969 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.969 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.969 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.969 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.969 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.969 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.969 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.969 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:36.970 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:36.970 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.970 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.970 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.970 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:36.970 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.970 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.970 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.970 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.970 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.970 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.970 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.970 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.970 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.971 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.971 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.971 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.971 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.971 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.971 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.971 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:36.971 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:36.971 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.971 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.971 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.971 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.971 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.971 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.971 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.972 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.972 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.972 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.972 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.972 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.972 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.972 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.972 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:36.972 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "status", 2024-03-19T02:05:36.972 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:36.972 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:36.972 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:36.972 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:36.972 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:36.972 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.973 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.973 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.973 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.973 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.973 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.973 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.973 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.973 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.973 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.973 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.973 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.973 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.973 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.973 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.974 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.974 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.974 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.974 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.974 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:36.974 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:36.974 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.974 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.974 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.974 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.974 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.974 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.974 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.974 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.974 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.975 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.975 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.975 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.975 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:36.975 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:36.975 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.975 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.975 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.975 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:36.975 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.975 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.975 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.975 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.975 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.976 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.976 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.976 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.976 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.976 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.976 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.976 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.976 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.976 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.976 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.976 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:36.976 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:36.976 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.976 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.976 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.977 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.977 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.977 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.977 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.977 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.977 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.977 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.977 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.977 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.977 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.977 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.977 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:36.977 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "telegraf", 2024-03-19T02:05:36.977 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:36.977 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:36.978 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:36.978 INFO:teuthology.orchestra.run.smithi099.stdout: "address": { 2024-03-19T02:05:36.978 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "address", 2024-03-19T02:05:36.978 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.978 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.978 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.978 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "unixgram:///tmp/telegraf.sock", 2024-03-19T02:05:36.978 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.978 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.978 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.978 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.978 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.978 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.979 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.979 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.979 INFO:teuthology.orchestra.run.smithi099.stdout: "interval": { 2024-03-19T02:05:36.979 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "interval", 2024-03-19T02:05:36.979 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:36.979 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.979 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.979 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "15", 2024-03-19T02:05:36.979 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.979 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.979 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.979 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.979 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.979 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.979 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.980 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.980 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:36.980 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:36.980 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.980 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.980 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.980 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.980 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.980 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.980 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.980 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.980 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.980 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.980 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.981 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.981 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.981 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.981 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.981 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.981 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.981 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.981 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.981 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:36.981 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:36.981 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.981 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.981 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.981 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.981 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.982 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.982 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.982 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.982 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.982 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.982 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.982 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.982 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:36.982 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:36.982 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.982 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.982 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.982 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:36.982 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.982 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.983 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.983 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.983 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.983 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.983 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.983 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.983 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.983 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.983 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.983 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.983 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.983 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.983 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.983 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:36.984 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:36.984 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.984 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.984 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.984 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.984 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.984 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.984 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.984 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.984 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.984 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.984 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.984 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.984 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:36.984 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.985 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:36.985 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "telemetry", 2024-03-19T02:05:36.985 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:36.985 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:36.985 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:36.985 INFO:teuthology.orchestra.run.smithi099.stdout: "channel_basic": { 2024-03-19T02:05:36.985 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "channel_basic", 2024-03-19T02:05:36.985 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.985 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.985 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.985 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:36.985 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.985 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.985 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.985 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Share basic cluster information (size, version)", 2024-03-19T02:05:36.986 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.986 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.986 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.986 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.986 INFO:teuthology.orchestra.run.smithi099.stdout: "channel_crash": { 2024-03-19T02:05:36.986 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "channel_crash", 2024-03-19T02:05:36.986 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.986 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.986 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.986 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:36.986 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.986 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.986 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.986 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Share metadata about Ceph daemon crashes (version, stack straces, etc)", 2024-03-19T02:05:36.987 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.987 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.987 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.987 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.987 INFO:teuthology.orchestra.run.smithi099.stdout: "channel_device": { 2024-03-19T02:05:36.987 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "channel_device", 2024-03-19T02:05:36.987 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.987 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.987 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.987 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:36.987 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.987 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.987 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.987 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)", 2024-03-19T02:05:36.987 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.988 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.988 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.988 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.988 INFO:teuthology.orchestra.run.smithi099.stdout: "channel_ident": { 2024-03-19T02:05:36.988 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "channel_ident", 2024-03-19T02:05:36.988 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.988 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.988 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.988 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.988 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.988 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.988 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.988 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Share a user-provided description and/or contact email for the cluster", 2024-03-19T02:05:36.988 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.988 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.989 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.989 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.989 INFO:teuthology.orchestra.run.smithi099.stdout: "channel_perf": { 2024-03-19T02:05:36.989 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "channel_perf", 2024-03-19T02:05:36.989 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.989 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.989 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.989 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.989 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.989 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.989 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.989 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Share various performance metrics of a cluster", 2024-03-19T02:05:36.989 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.989 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.990 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.990 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.990 INFO:teuthology.orchestra.run.smithi099.stdout: "contact": { 2024-03-19T02:05:36.990 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "contact", 2024-03-19T02:05:36.990 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.990 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.990 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.990 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.990 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.990 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.990 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.990 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.990 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.990 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.990 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.991 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.991 INFO:teuthology.orchestra.run.smithi099.stdout: "description": { 2024-03-19T02:05:36.991 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "description", 2024-03-19T02:05:36.991 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.991 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.991 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.991 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.991 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.991 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.991 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.991 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.991 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.991 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.991 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.992 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.992 INFO:teuthology.orchestra.run.smithi099.stdout: "device_url": { 2024-03-19T02:05:36.992 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "device_url", 2024-03-19T02:05:36.992 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.992 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.992 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.992 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "https://telemetry.ceph.com/device", 2024-03-19T02:05:36.992 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.992 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.992 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.992 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.992 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.992 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.992 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.993 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.993 INFO:teuthology.orchestra.run.smithi099.stdout: "enabled": { 2024-03-19T02:05:36.993 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "enabled", 2024-03-19T02:05:36.993 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.993 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.993 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.993 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.993 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.993 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.993 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.993 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.993 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.993 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.993 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.993 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.994 INFO:teuthology.orchestra.run.smithi099.stdout: "interval": { 2024-03-19T02:05:36.994 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "interval", 2024-03-19T02:05:36.994 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:36.994 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.994 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.994 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "24", 2024-03-19T02:05:36.994 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "8", 2024-03-19T02:05:36.994 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.994 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.994 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.994 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.994 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.994 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.994 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.994 INFO:teuthology.orchestra.run.smithi099.stdout: "last_opt_revision": { 2024-03-19T02:05:36.995 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "last_opt_revision", 2024-03-19T02:05:36.995 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:36.995 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.995 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.995 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "1", 2024-03-19T02:05:36.995 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.995 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.995 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.995 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.995 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.995 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.995 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.995 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.995 INFO:teuthology.orchestra.run.smithi099.stdout: "leaderboard": { 2024-03-19T02:05:36.996 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "leaderboard", 2024-03-19T02:05:36.996 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.996 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.996 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.996 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.996 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.996 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.996 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.996 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.996 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.996 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.996 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.996 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.996 INFO:teuthology.orchestra.run.smithi099.stdout: "leaderboard_description": { 2024-03-19T02:05:36.996 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "leaderboard_description", 2024-03-19T02:05:36.997 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.997 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.997 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:36.997 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.997 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.997 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.997 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.997 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.997 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.997 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.997 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.997 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.997 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:36.997 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:36.997 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:36.998 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.998 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.998 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:36.998 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.998 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.998 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:36.998 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:36.998 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:36.998 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:36.998 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:36.998 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:36.998 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:36.998 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:36.998 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.998 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.999 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:36.999 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:36.999 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:36.999 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:36.999 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:36.999 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:36.999 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:36.999 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:36.999 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:36.999 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:36.999 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:36.999 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:36.999 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:36.999 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:36.999 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.000 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.000 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.000 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.000 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.000 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.000 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.000 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.000 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.000 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.000 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.000 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.000 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.000 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.000 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.000 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.001 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.001 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.001 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.001 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.001 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.001 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.001 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.001 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.001 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.001 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.001 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.001 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.001 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.001 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.001 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.002 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.002 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.002 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.002 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.002 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.002 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.002 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.002 INFO:teuthology.orchestra.run.smithi099.stdout: "organization": { 2024-03-19T02:05:37.002 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "organization", 2024-03-19T02:05:37.002 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.002 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.002 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.002 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.002 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.002 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.003 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.003 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.003 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.003 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.003 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.003 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.003 INFO:teuthology.orchestra.run.smithi099.stdout: "proxy": { 2024-03-19T02:05:37.003 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "proxy", 2024-03-19T02:05:37.003 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.003 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.003 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.003 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.003 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.003 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.003 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.004 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.004 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.004 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.004 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.004 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.004 INFO:teuthology.orchestra.run.smithi099.stdout: "url": { 2024-03-19T02:05:37.004 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "url", 2024-03-19T02:05:37.004 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.004 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.004 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.004 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "https://telemetry.ceph.com/report", 2024-03-19T02:05:37.004 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.004 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.005 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.005 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.005 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.005 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.005 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.005 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.005 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.005 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.005 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.005 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "test_orchestrator", 2024-03-19T02:05:37.005 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.005 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.005 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.005 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.005 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.006 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.006 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.006 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.006 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.006 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.006 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.006 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.006 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.006 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.006 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.006 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.006 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.006 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.006 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.007 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.007 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.007 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.007 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.007 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.007 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.007 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.007 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.007 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.007 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.007 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.007 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.007 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.007 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.008 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.008 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.008 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.008 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.008 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.008 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.008 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.008 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.008 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.008 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.008 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.008 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.008 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.008 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.008 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.009 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.009 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.009 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.009 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.009 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.009 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.009 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.009 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.009 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.009 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.009 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.009 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.009 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.009 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.009 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.010 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.010 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.010 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.010 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.010 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.010 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.010 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.010 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.010 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.010 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.010 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.010 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.010 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.010 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "volumes", 2024-03-19T02:05:37.011 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.011 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.011 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.011 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.011 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.011 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.011 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.011 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.011 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.011 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.011 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.011 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.011 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.011 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.012 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.012 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.012 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.012 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.012 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.012 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.012 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.012 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.012 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.012 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.012 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.012 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.012 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.012 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.012 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.013 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.013 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.013 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.013 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.013 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.013 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.013 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.013 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.013 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.013 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.013 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.013 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.013 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.013 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.013 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.014 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.014 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.014 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.014 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.014 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.014 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.014 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.014 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.014 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.014 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.014 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.014 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.014 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.014 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.014 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.015 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.015 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.015 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.015 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.015 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.015 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.015 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.015 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.015 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.015 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.015 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.015 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.015 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.015 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.015 INFO:teuthology.orchestra.run.smithi099.stdout: "max_concurrent_clones": { 2024-03-19T02:05:37.016 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "max_concurrent_clones", 2024-03-19T02:05:37.016 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.016 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.016 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.016 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "4", 2024-03-19T02:05:37.016 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.016 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.016 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.016 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Number of asynchronous cloner threads", 2024-03-19T02:05:37.016 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.016 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.016 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.016 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.016 INFO:teuthology.orchestra.run.smithi099.stdout: "periodic_async_work": { 2024-03-19T02:05:37.016 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "periodic_async_work", 2024-03-19T02:05:37.017 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.017 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.017 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.017 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.017 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.017 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.017 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.017 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Periodically check for async work", 2024-03-19T02:05:37.017 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.017 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.017 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.017 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.017 INFO:teuthology.orchestra.run.smithi099.stdout: "snapshot_clone_delay": { 2024-03-19T02:05:37.017 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "snapshot_clone_delay", 2024-03-19T02:05:37.018 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.018 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.018 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.018 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "0", 2024-03-19T02:05:37.018 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.018 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.018 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.018 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Delay clone begin operation by snapshot_clone_delay seconds", 2024-03-19T02:05:37.018 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.018 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.018 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.018 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.018 INFO:teuthology.orchestra.run.smithi099.stdout: "snapshot_clone_no_wait": { 2024-03-19T02:05:37.018 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "snapshot_clone_no_wait", 2024-03-19T02:05:37.019 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.019 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.019 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.019 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.019 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.019 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.019 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.019 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Reject subvolume clone request when cloner threads are busy", 2024-03-19T02:05:37.019 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.019 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.019 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.019 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.019 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.019 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.019 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.020 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "zabbix", 2024-03-19T02:05:37.020 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.020 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.020 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.020 INFO:teuthology.orchestra.run.smithi099.stdout: "discovery_interval": { 2024-03-19T02:05:37.020 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "discovery_interval", 2024-03-19T02:05:37.020 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "uint", 2024-03-19T02:05:37.020 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.020 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.020 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "100", 2024-03-19T02:05:37.020 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.020 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.020 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.020 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.021 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.021 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.021 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.021 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.021 INFO:teuthology.orchestra.run.smithi099.stdout: "identifier": { 2024-03-19T02:05:37.021 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "identifier", 2024-03-19T02:05:37.021 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.021 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.021 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.021 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.021 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.021 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.021 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.021 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.021 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.022 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.022 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.022 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.022 INFO:teuthology.orchestra.run.smithi099.stdout: "interval": { 2024-03-19T02:05:37.022 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "interval", 2024-03-19T02:05:37.022 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:37.022 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.022 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.022 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "60", 2024-03-19T02:05:37.022 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.022 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.022 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.022 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.022 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.023 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.023 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.023 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.023 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.023 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.023 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.023 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.023 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.023 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.023 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.023 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.023 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.023 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.023 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.023 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.024 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.024 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.024 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.024 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.024 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.024 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.024 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.024 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.024 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.024 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.024 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.024 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.024 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.024 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.025 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.025 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.025 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.025 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.025 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.025 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.025 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.025 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.025 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.025 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.025 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.025 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.025 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.025 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.025 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.026 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.026 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.026 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.026 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.026 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.026 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.026 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.026 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.026 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.026 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.026 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.026 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.026 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.026 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.026 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.027 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.027 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.027 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.027 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.027 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.027 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.027 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.027 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.027 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.027 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.027 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.027 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.027 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.027 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.028 INFO:teuthology.orchestra.run.smithi099.stdout: "zabbix_host": { 2024-03-19T02:05:37.028 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "zabbix_host", 2024-03-19T02:05:37.028 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.028 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.028 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.028 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.028 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.028 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.028 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.028 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.028 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.028 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.028 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.028 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.028 INFO:teuthology.orchestra.run.smithi099.stdout: "zabbix_port": { 2024-03-19T02:05:37.029 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "zabbix_port", 2024-03-19T02:05:37.029 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.029 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.029 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.029 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "10051", 2024-03-19T02:05:37.029 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.029 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.029 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.029 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.029 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.029 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.029 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.029 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.029 INFO:teuthology.orchestra.run.smithi099.stdout: "zabbix_sender": { 2024-03-19T02:05:37.029 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "zabbix_sender", 2024-03-19T02:05:37.030 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.030 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.030 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.030 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "/usr/bin/zabbix_sender", 2024-03-19T02:05:37.030 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.030 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.030 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.030 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.030 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.030 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.030 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.030 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.030 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.030 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.030 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-03-19T02:05:37.031 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.031 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.031 INFO:teuthology.orchestra.run.smithi099.stdout: "gid": 24110, 2024-03-19T02:05:37.031 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "y", 2024-03-19T02:05:37.031 INFO:teuthology.orchestra.run.smithi099.stdout: "mgr_features": 4540701547738038271, 2024-03-19T02:05:37.031 INFO:teuthology.orchestra.run.smithi099.stdout: "available_modules": [ 2024-03-19T02:05:37.031 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.031 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "alerts", 2024-03-19T02:05:37.031 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.031 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.031 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.031 INFO:teuthology.orchestra.run.smithi099.stdout: "interval": { 2024-03-19T02:05:37.031 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "interval", 2024-03-19T02:05:37.031 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:37.032 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.032 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.032 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "60", 2024-03-19T02:05:37.032 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.032 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.032 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.032 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "How frequently to reexamine health status", 2024-03-19T02:05:37.032 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.032 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.032 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.032 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.032 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.032 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.032 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.032 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.033 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.033 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.033 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.033 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.033 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.033 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.033 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.033 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.033 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.033 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.033 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.033 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.033 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.033 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.033 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.034 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.034 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.034 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.034 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.034 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.034 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.034 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.034 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.034 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.034 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.034 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.034 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.034 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.034 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.035 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.035 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.035 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.035 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.035 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.035 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.035 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.035 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.035 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.035 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.035 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.035 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.035 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.035 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.036 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.036 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.036 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.036 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.036 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.036 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.036 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.036 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.036 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.036 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.036 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.036 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.036 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.036 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.036 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.037 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.037 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.037 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.037 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.037 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.037 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.037 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.037 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.037 INFO:teuthology.orchestra.run.smithi099.stdout: "smtp_destination": { 2024-03-19T02:05:37.037 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "smtp_destination", 2024-03-19T02:05:37.037 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.037 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.037 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.037 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.038 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.038 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.038 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.038 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Email address to send alerts to", 2024-03-19T02:05:37.038 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.038 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.038 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.038 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.038 INFO:teuthology.orchestra.run.smithi099.stdout: "smtp_from_name": { 2024-03-19T02:05:37.038 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "smtp_from_name", 2024-03-19T02:05:37.038 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.038 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.038 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.038 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "Ceph", 2024-03-19T02:05:37.039 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.039 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.039 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.039 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Email From: name", 2024-03-19T02:05:37.039 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.039 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.039 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.039 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.039 INFO:teuthology.orchestra.run.smithi099.stdout: "smtp_host": { 2024-03-19T02:05:37.039 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "smtp_host", 2024-03-19T02:05:37.039 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.039 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.039 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.039 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.039 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.040 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.040 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.040 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "SMTP server", 2024-03-19T02:05:37.040 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.040 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.040 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.040 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.040 INFO:teuthology.orchestra.run.smithi099.stdout: "smtp_password": { 2024-03-19T02:05:37.040 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "smtp_password", 2024-03-19T02:05:37.040 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.040 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.040 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.040 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.040 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.040 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.041 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.041 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Password to authenticate with", 2024-03-19T02:05:37.041 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.041 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.041 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.041 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.041 INFO:teuthology.orchestra.run.smithi099.stdout: "smtp_port": { 2024-03-19T02:05:37.041 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "smtp_port", 2024-03-19T02:05:37.041 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.041 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.041 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.041 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "465", 2024-03-19T02:05:37.041 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.041 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.041 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.042 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "SMTP port", 2024-03-19T02:05:37.042 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.042 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.042 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.042 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.042 INFO:teuthology.orchestra.run.smithi099.stdout: "smtp_sender": { 2024-03-19T02:05:37.042 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "smtp_sender", 2024-03-19T02:05:37.042 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.042 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.042 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.042 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.042 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.042 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.042 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.043 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "SMTP envelope sender", 2024-03-19T02:05:37.043 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.043 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.043 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.043 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.043 INFO:teuthology.orchestra.run.smithi099.stdout: "smtp_ssl": { 2024-03-19T02:05:37.043 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "smtp_ssl", 2024-03-19T02:05:37.043 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.043 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.043 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.043 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.043 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.043 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.043 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.043 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Use SSL to connect to SMTP server", 2024-03-19T02:05:37.043 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.044 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.044 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.045 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.045 INFO:teuthology.orchestra.run.smithi099.stdout: "smtp_user": { 2024-03-19T02:05:37.045 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "smtp_user", 2024-03-19T02:05:37.045 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.045 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.045 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.045 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.045 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.046 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.046 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.046 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "User to authenticate as", 2024-03-19T02:05:37.046 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.046 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.046 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.046 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.046 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.046 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.046 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.046 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "balancer", 2024-03-19T02:05:37.046 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.046 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.046 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.047 INFO:teuthology.orchestra.run.smithi099.stdout: "active": { 2024-03-19T02:05:37.047 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "active", 2024-03-19T02:05:37.047 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.047 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.047 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.047 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.047 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.047 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.047 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.047 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "automatically balance PGs across cluster", 2024-03-19T02:05:37.047 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.047 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.047 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.047 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.047 INFO:teuthology.orchestra.run.smithi099.stdout: "begin_time": { 2024-03-19T02:05:37.048 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "begin_time", 2024-03-19T02:05:37.048 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.048 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.048 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.048 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "0000", 2024-03-19T02:05:37.048 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.048 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.048 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.048 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "beginning time of day to automatically balance", 2024-03-19T02:05:37.048 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "This is a time of day in the format HHMM.", 2024-03-19T02:05:37.048 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.048 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.048 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.048 INFO:teuthology.orchestra.run.smithi099.stdout: "begin_weekday": { 2024-03-19T02:05:37.049 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "begin_weekday", 2024-03-19T02:05:37.049 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "uint", 2024-03-19T02:05:37.049 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.049 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.049 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "0", 2024-03-19T02:05:37.049 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "0", 2024-03-19T02:05:37.049 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "6", 2024-03-19T02:05:37.049 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.049 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Restrict automatic balancing to this day of the week or later", 2024-03-19T02:05:37.049 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "0 = Sunday, 1 = Monday, etc.", 2024-03-19T02:05:37.049 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.049 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.049 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.049 INFO:teuthology.orchestra.run.smithi099.stdout: "crush_compat_max_iterations": { 2024-03-19T02:05:37.049 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "crush_compat_max_iterations", 2024-03-19T02:05:37.050 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "uint", 2024-03-19T02:05:37.050 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.050 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.050 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "25", 2024-03-19T02:05:37.050 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "1", 2024-03-19T02:05:37.050 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "250", 2024-03-19T02:05:37.050 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.050 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "maximum number of iterations to attempt optimization", 2024-03-19T02:05:37.050 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.050 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.050 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.050 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.050 INFO:teuthology.orchestra.run.smithi099.stdout: "crush_compat_metrics": { 2024-03-19T02:05:37.050 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "crush_compat_metrics", 2024-03-19T02:05:37.051 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.051 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.051 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.051 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "pgs,objects,bytes", 2024-03-19T02:05:37.051 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.051 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.051 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.051 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "metrics with which to calculate OSD utilization", 2024-03-19T02:05:37.051 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.", 2024-03-19T02:05:37.051 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.051 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.051 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.051 INFO:teuthology.orchestra.run.smithi099.stdout: "crush_compat_step": { 2024-03-19T02:05:37.051 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "crush_compat_step", 2024-03-19T02:05:37.051 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "float", 2024-03-19T02:05:37.052 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.052 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.052 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "0.5", 2024-03-19T02:05:37.052 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "0.001", 2024-03-19T02:05:37.052 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "0.999", 2024-03-19T02:05:37.052 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.052 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "aggressiveness of optimization", 2024-03-19T02:05:37.052 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": ".99 is very aggressive, .01 is less aggressive", 2024-03-19T02:05:37.052 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.052 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.052 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.052 INFO:teuthology.orchestra.run.smithi099.stdout: "end_time": { 2024-03-19T02:05:37.052 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "end_time", 2024-03-19T02:05:37.052 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.052 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.053 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.053 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "2359", 2024-03-19T02:05:37.053 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.053 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.053 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.053 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "ending time of day to automatically balance", 2024-03-19T02:05:37.053 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "This is a time of day in the format HHMM.", 2024-03-19T02:05:37.053 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.053 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.053 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.053 INFO:teuthology.orchestra.run.smithi099.stdout: "end_weekday": { 2024-03-19T02:05:37.053 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "end_weekday", 2024-03-19T02:05:37.053 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "uint", 2024-03-19T02:05:37.053 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.054 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.054 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "0", 2024-03-19T02:05:37.054 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "0", 2024-03-19T02:05:37.054 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "6", 2024-03-19T02:05:37.054 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.054 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Restrict automatic balancing to days of the week earlier than this", 2024-03-19T02:05:37.054 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "0 = Sunday, 1 = Monday, etc.", 2024-03-19T02:05:37.054 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.054 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.054 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.054 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.054 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.054 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.054 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.054 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.055 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.055 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.055 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.055 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.055 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.055 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.055 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.055 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.055 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.055 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.055 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.055 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.055 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.055 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.055 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.056 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.056 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.056 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.056 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.056 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.056 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.056 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.056 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.056 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.056 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.056 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.056 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.056 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.056 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.056 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.057 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.057 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.057 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.057 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.057 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.057 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.057 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.057 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.057 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.057 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.057 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.057 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.057 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.057 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.058 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.058 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.058 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.058 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.058 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.058 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.058 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.058 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.058 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.058 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.058 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.058 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.058 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.058 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.059 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.059 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.059 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.059 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.059 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.059 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.059 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.059 INFO:teuthology.orchestra.run.smithi099.stdout: "min_score": { 2024-03-19T02:05:37.059 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "min_score", 2024-03-19T02:05:37.059 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "float", 2024-03-19T02:05:37.059 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.059 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.059 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "0", 2024-03-19T02:05:37.059 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.059 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.060 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.060 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "minimum score, below which no optimization is attempted", 2024-03-19T02:05:37.060 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.060 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.060 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.060 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.060 INFO:teuthology.orchestra.run.smithi099.stdout: "mode": { 2024-03-19T02:05:37.060 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "mode", 2024-03-19T02:05:37.060 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.060 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.060 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.060 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "upmap", 2024-03-19T02:05:37.060 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.060 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.060 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.061 INFO:teuthology.orchestra.run.smithi099.stdout: "crush-compat", 2024-03-19T02:05:37.061 INFO:teuthology.orchestra.run.smithi099.stdout: "none", 2024-03-19T02:05:37.061 INFO:teuthology.orchestra.run.smithi099.stdout: "read", 2024-03-19T02:05:37.061 INFO:teuthology.orchestra.run.smithi099.stdout: "upmap", 2024-03-19T02:05:37.061 INFO:teuthology.orchestra.run.smithi099.stdout: "upmap-read" 2024-03-19T02:05:37.061 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.061 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Balancer mode", 2024-03-19T02:05:37.061 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.061 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.061 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.061 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.061 INFO:teuthology.orchestra.run.smithi099.stdout: "pool_ids": { 2024-03-19T02:05:37.061 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "pool_ids", 2024-03-19T02:05:37.061 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.061 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.062 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.062 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.062 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.062 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.062 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.062 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "pools which the automatic balancing will be limited to", 2024-03-19T02:05:37.062 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.062 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.062 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.062 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.062 INFO:teuthology.orchestra.run.smithi099.stdout: "sleep_interval": { 2024-03-19T02:05:37.062 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "sleep_interval", 2024-03-19T02:05:37.062 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:37.062 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.063 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.063 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "60", 2024-03-19T02:05:37.063 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.063 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.063 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.063 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "how frequently to wake up and attempt optimization", 2024-03-19T02:05:37.063 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.063 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.063 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.063 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.063 INFO:teuthology.orchestra.run.smithi099.stdout: "upmap_max_deviation": { 2024-03-19T02:05:37.063 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "upmap_max_deviation", 2024-03-19T02:05:37.063 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.063 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.063 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.064 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "5", 2024-03-19T02:05:37.064 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "1", 2024-03-19T02:05:37.064 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.064 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.064 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "deviation below which no optimization is attempted", 2024-03-19T02:05:37.064 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "If the number of PGs are within this count then no optimization is attempted", 2024-03-19T02:05:37.064 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.064 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.064 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.064 INFO:teuthology.orchestra.run.smithi099.stdout: "upmap_max_optimizations": { 2024-03-19T02:05:37.064 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "upmap_max_optimizations", 2024-03-19T02:05:37.064 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "uint", 2024-03-19T02:05:37.064 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.064 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.064 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "10", 2024-03-19T02:05:37.065 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.065 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.065 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.065 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "maximum upmap optimizations to make per attempt", 2024-03-19T02:05:37.065 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.065 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.065 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.065 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.065 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.065 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.065 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.065 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "cephadm", 2024-03-19T02:05:37.065 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.065 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.065 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.066 INFO:teuthology.orchestra.run.smithi099.stdout: "agent_down_multiplier": { 2024-03-19T02:05:37.066 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "agent_down_multiplier", 2024-03-19T02:05:37.066 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "float", 2024-03-19T02:05:37.066 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.066 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.066 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "3.0", 2024-03-19T02:05:37.066 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.066 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.066 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.066 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Multiplied by agent refresh rate to calculate how long agent must not report before being marked down", 2024-03-19T02:05:37.066 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.066 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.066 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.066 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.067 INFO:teuthology.orchestra.run.smithi099.stdout: "agent_refresh_rate": { 2024-03-19T02:05:37.067 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "agent_refresh_rate", 2024-03-19T02:05:37.067 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:37.067 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.067 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.067 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "20", 2024-03-19T02:05:37.067 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.067 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.067 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.067 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "How often agent on each host will try to gather and send metadata", 2024-03-19T02:05:37.067 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.067 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.067 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.067 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.067 INFO:teuthology.orchestra.run.smithi099.stdout: "agent_starting_port": { 2024-03-19T02:05:37.068 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "agent_starting_port", 2024-03-19T02:05:37.068 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.068 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.068 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.068 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "4721", 2024-03-19T02:05:37.068 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.068 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.068 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.068 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)", 2024-03-19T02:05:37.068 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.068 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.068 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.068 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.068 INFO:teuthology.orchestra.run.smithi099.stdout: "allow_ptrace": { 2024-03-19T02:05:37.068 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "allow_ptrace", 2024-03-19T02:05:37.069 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.069 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.069 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.069 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.069 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.069 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.069 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.069 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "allow SYS_PTRACE capability on ceph containers", 2024-03-19T02:05:37.069 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.", 2024-03-19T02:05:37.069 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.069 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.069 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.069 INFO:teuthology.orchestra.run.smithi099.stdout: "autotune_interval": { 2024-03-19T02:05:37.069 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "autotune_interval", 2024-03-19T02:05:37.070 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:37.070 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.070 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.070 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "600", 2024-03-19T02:05:37.070 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.070 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.070 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.070 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "how frequently to autotune daemon memory", 2024-03-19T02:05:37.070 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.070 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.070 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.070 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.070 INFO:teuthology.orchestra.run.smithi099.stdout: "autotune_memory_target_ratio": { 2024-03-19T02:05:37.070 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "autotune_memory_target_ratio", 2024-03-19T02:05:37.070 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "float", 2024-03-19T02:05:37.071 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.071 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.071 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "0.7", 2024-03-19T02:05:37.071 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.071 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.071 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.071 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "ratio of total system memory to divide amongst autotuned daemons", 2024-03-19T02:05:37.071 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.071 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.071 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.071 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.071 INFO:teuthology.orchestra.run.smithi099.stdout: "cephadm_log_destination": { 2024-03-19T02:05:37.071 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "cephadm_log_destination", 2024-03-19T02:05:37.071 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.071 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.072 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.072 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.072 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.072 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.072 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.072 INFO:teuthology.orchestra.run.smithi099.stdout: "file", 2024-03-19T02:05:37.072 INFO:teuthology.orchestra.run.smithi099.stdout: "file,syslog", 2024-03-19T02:05:37.072 INFO:teuthology.orchestra.run.smithi099.stdout: "syslog" 2024-03-19T02:05:37.072 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.072 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Destination for cephadm command's persistent logging", 2024-03-19T02:05:37.072 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.072 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.072 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.072 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.073 INFO:teuthology.orchestra.run.smithi099.stdout: "cgroups_split": { 2024-03-19T02:05:37.073 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "cgroups_split", 2024-03-19T02:05:37.073 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.073 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.073 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.073 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.073 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.073 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.073 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.073 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Pass --cgroups=split when cephadm creates containers (currently podman only)", 2024-03-19T02:05:37.073 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.073 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.073 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.073 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.073 INFO:teuthology.orchestra.run.smithi099.stdout: "config_checks_enabled": { 2024-03-19T02:05:37.074 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "config_checks_enabled", 2024-03-19T02:05:37.074 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.074 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.074 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.074 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.074 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.074 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.074 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.074 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Enable or disable the cephadm configuration analysis", 2024-03-19T02:05:37.074 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.074 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.074 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.074 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.074 INFO:teuthology.orchestra.run.smithi099.stdout: "config_dashboard": { 2024-03-19T02:05:37.075 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "config_dashboard", 2024-03-19T02:05:37.075 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.075 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.075 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.075 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.075 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.075 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.075 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.075 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "manage configs like API endpoints in Dashboard.", 2024-03-19T02:05:37.075 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.075 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.075 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.075 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.075 INFO:teuthology.orchestra.run.smithi099.stdout: "container_image_alertmanager": { 2024-03-19T02:05:37.075 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "container_image_alertmanager", 2024-03-19T02:05:37.076 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.076 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.076 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.076 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "quay.io/prometheus/alertmanager:v0.25.0", 2024-03-19T02:05:37.076 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.076 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.076 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.076 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Prometheus container image", 2024-03-19T02:05:37.076 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.076 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.076 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.076 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.076 INFO:teuthology.orchestra.run.smithi099.stdout: "container_image_base": { 2024-03-19T02:05:37.076 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "container_image_base", 2024-03-19T02:05:37.077 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.077 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.077 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.077 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "quay.io/ceph/ceph", 2024-03-19T02:05:37.077 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.077 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.077 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.077 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Container image name, without the tag", 2024-03-19T02:05:37.077 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.077 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.077 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.077 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.077 INFO:teuthology.orchestra.run.smithi099.stdout: "container_image_elasticsearch": { 2024-03-19T02:05:37.077 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "container_image_elasticsearch", 2024-03-19T02:05:37.077 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.078 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.078 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.078 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "quay.io/omrizeneva/elasticsearch:6.8.23", 2024-03-19T02:05:37.078 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.078 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.078 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.078 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "elasticsearch container image", 2024-03-19T02:05:37.078 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.078 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.078 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.078 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.078 INFO:teuthology.orchestra.run.smithi099.stdout: "container_image_grafana": { 2024-03-19T02:05:37.078 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "container_image_grafana", 2024-03-19T02:05:37.078 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.078 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.079 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.079 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "quay.io/ceph/ceph-grafana:9.4.12", 2024-03-19T02:05:37.079 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.079 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.079 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.079 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Prometheus container image", 2024-03-19T02:05:37.079 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.079 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.079 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.079 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.079 INFO:teuthology.orchestra.run.smithi099.stdout: "container_image_haproxy": { 2024-03-19T02:05:37.079 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "container_image_haproxy", 2024-03-19T02:05:37.079 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.079 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.079 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.080 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "quay.io/ceph/haproxy:2.3", 2024-03-19T02:05:37.080 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.080 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.080 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.080 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "HAproxy container image", 2024-03-19T02:05:37.080 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.080 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.080 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.080 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.080 INFO:teuthology.orchestra.run.smithi099.stdout: "container_image_jaeger_agent": { 2024-03-19T02:05:37.080 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "container_image_jaeger_agent", 2024-03-19T02:05:37.080 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.080 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.080 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.080 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "quay.io/jaegertracing/jaeger-agent:1.29", 2024-03-19T02:05:37.081 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.081 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.081 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.081 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Jaeger agent container image", 2024-03-19T02:05:37.081 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.081 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.081 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.081 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.081 INFO:teuthology.orchestra.run.smithi099.stdout: "container_image_jaeger_collector": { 2024-03-19T02:05:37.081 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "container_image_jaeger_collector", 2024-03-19T02:05:37.081 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.081 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.081 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.082 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "quay.io/jaegertracing/jaeger-collector:1.29", 2024-03-19T02:05:37.082 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.082 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.082 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.082 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Jaeger collector container image", 2024-03-19T02:05:37.082 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.082 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.082 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.082 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.082 INFO:teuthology.orchestra.run.smithi099.stdout: "container_image_jaeger_query": { 2024-03-19T02:05:37.082 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "container_image_jaeger_query", 2024-03-19T02:05:37.082 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.082 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.082 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.082 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "quay.io/jaegertracing/jaeger-query:1.29", 2024-03-19T02:05:37.083 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.083 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.083 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.083 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Jaeger query container image", 2024-03-19T02:05:37.083 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.083 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.083 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.083 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.083 INFO:teuthology.orchestra.run.smithi099.stdout: "container_image_keepalived": { 2024-03-19T02:05:37.083 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "container_image_keepalived", 2024-03-19T02:05:37.083 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.083 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.083 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.083 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "quay.io/ceph/keepalived:2.2.4", 2024-03-19T02:05:37.084 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.084 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.084 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.084 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Keepalived container image", 2024-03-19T02:05:37.084 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.084 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.084 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.084 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.084 INFO:teuthology.orchestra.run.smithi099.stdout: "container_image_loki": { 2024-03-19T02:05:37.084 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "container_image_loki", 2024-03-19T02:05:37.084 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.084 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.084 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.084 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "docker.io/grafana/loki:2.4.0", 2024-03-19T02:05:37.084 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.085 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.085 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.085 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Loki container image", 2024-03-19T02:05:37.085 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.085 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.085 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.085 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.085 INFO:teuthology.orchestra.run.smithi099.stdout: "container_image_node_exporter": { 2024-03-19T02:05:37.085 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "container_image_node_exporter", 2024-03-19T02:05:37.085 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.085 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.085 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.085 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "quay.io/prometheus/node-exporter:v1.5.0", 2024-03-19T02:05:37.085 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.086 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.086 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.086 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Prometheus container image", 2024-03-19T02:05:37.086 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.086 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.086 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.086 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.086 INFO:teuthology.orchestra.run.smithi099.stdout: "container_image_nvmeof": { 2024-03-19T02:05:37.086 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "container_image_nvmeof", 2024-03-19T02:05:37.086 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.086 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.086 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.086 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "quay.io/ceph/nvmeof:1.0.0", 2024-03-19T02:05:37.086 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.087 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.087 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.087 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Nvme-of container image", 2024-03-19T02:05:37.087 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.087 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.087 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.087 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.087 INFO:teuthology.orchestra.run.smithi099.stdout: "container_image_prometheus": { 2024-03-19T02:05:37.087 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "container_image_prometheus", 2024-03-19T02:05:37.087 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.087 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.087 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.087 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "quay.io/prometheus/prometheus:v2.43.0", 2024-03-19T02:05:37.087 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.087 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.088 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.088 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Prometheus container image", 2024-03-19T02:05:37.088 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.088 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.088 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.088 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.088 INFO:teuthology.orchestra.run.smithi099.stdout: "container_image_promtail": { 2024-03-19T02:05:37.088 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "container_image_promtail", 2024-03-19T02:05:37.088 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.088 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.088 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.088 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "docker.io/grafana/promtail:2.4.0", 2024-03-19T02:05:37.088 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.088 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.088 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.089 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Promtail container image", 2024-03-19T02:05:37.089 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.089 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.089 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.089 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.089 INFO:teuthology.orchestra.run.smithi099.stdout: "container_image_snmp_gateway": { 2024-03-19T02:05:37.089 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "container_image_snmp_gateway", 2024-03-19T02:05:37.089 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.089 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.089 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.089 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "docker.io/maxwo/snmp-notifier:v1.2.1", 2024-03-19T02:05:37.089 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.089 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.089 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.089 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "SNMP Gateway container image", 2024-03-19T02:05:37.090 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.090 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.090 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.090 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.090 INFO:teuthology.orchestra.run.smithi099.stdout: "container_init": { 2024-03-19T02:05:37.090 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "container_init", 2024-03-19T02:05:37.090 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.090 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.090 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.090 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.090 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.090 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.090 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.090 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Run podman/docker with `--init`", 2024-03-19T02:05:37.091 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.091 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.091 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.091 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.091 INFO:teuthology.orchestra.run.smithi099.stdout: "daemon_cache_timeout": { 2024-03-19T02:05:37.091 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "daemon_cache_timeout", 2024-03-19T02:05:37.091 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:37.091 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.091 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.091 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "600", 2024-03-19T02:05:37.091 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.091 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.091 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.091 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "seconds to cache service (daemon) inventory", 2024-03-19T02:05:37.091 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.092 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.092 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.092 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.092 INFO:teuthology.orchestra.run.smithi099.stdout: "default_cephadm_command_timeout": { 2024-03-19T02:05:37.092 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "default_cephadm_command_timeout", 2024-03-19T02:05:37.092 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.092 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.092 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.092 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "900", 2024-03-19T02:05:37.092 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.092 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.092 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.092 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Default timeout applied to cephadm commands run directly on the host (in seconds)", 2024-03-19T02:05:37.092 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.093 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.093 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.093 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.093 INFO:teuthology.orchestra.run.smithi099.stdout: "default_registry": { 2024-03-19T02:05:37.093 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "default_registry", 2024-03-19T02:05:37.093 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.093 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.093 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.093 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "docker.io", 2024-03-19T02:05:37.093 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.093 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.093 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.093 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Search-registry to which we should normalize unqualified image names. This is not the default registry", 2024-03-19T02:05:37.093 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.094 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.094 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.094 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.094 INFO:teuthology.orchestra.run.smithi099.stdout: "device_cache_timeout": { 2024-03-19T02:05:37.094 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "device_cache_timeout", 2024-03-19T02:05:37.094 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:37.094 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.094 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.094 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "1800", 2024-03-19T02:05:37.094 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.094 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.094 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.094 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "seconds to cache device inventory", 2024-03-19T02:05:37.094 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.095 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.095 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.095 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.095 INFO:teuthology.orchestra.run.smithi099.stdout: "device_enhanced_scan": { 2024-03-19T02:05:37.095 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "device_enhanced_scan", 2024-03-19T02:05:37.095 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.095 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.095 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.095 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.095 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.095 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.095 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.095 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Use libstoragemgmt during device scans", 2024-03-19T02:05:37.095 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.095 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.096 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.096 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.096 INFO:teuthology.orchestra.run.smithi099.stdout: "facts_cache_timeout": { 2024-03-19T02:05:37.096 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "facts_cache_timeout", 2024-03-19T02:05:37.096 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:37.096 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.096 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.096 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "60", 2024-03-19T02:05:37.096 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.096 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.096 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.096 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "seconds to cache host facts data", 2024-03-19T02:05:37.096 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.096 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.097 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.097 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.097 INFO:teuthology.orchestra.run.smithi099.stdout: "host_check_interval": { 2024-03-19T02:05:37.097 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "host_check_interval", 2024-03-19T02:05:37.097 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:37.097 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.097 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.097 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "600", 2024-03-19T02:05:37.097 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.097 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.097 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.097 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "how frequently to perform a host check", 2024-03-19T02:05:37.097 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.097 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.097 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.098 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.098 INFO:teuthology.orchestra.run.smithi099.stdout: "hw_monitoring": { 2024-03-19T02:05:37.098 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "hw_monitoring", 2024-03-19T02:05:37.098 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.098 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.098 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.098 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.098 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.098 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.098 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.098 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Deploy hw monitoring daemon on every host.", 2024-03-19T02:05:37.098 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.098 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.098 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.099 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.099 INFO:teuthology.orchestra.run.smithi099.stdout: "inventory_list_all": { 2024-03-19T02:05:37.099 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "inventory_list_all", 2024-03-19T02:05:37.099 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.099 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.099 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.099 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.099 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.099 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.099 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.099 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)", 2024-03-19T02:05:37.099 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.099 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.099 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.099 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.100 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.100 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.100 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.100 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.100 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.100 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.100 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.100 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.100 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.100 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.100 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.100 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.100 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.100 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.100 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.101 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.101 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.101 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.101 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.101 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.101 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.101 INFO:teuthology.orchestra.run.smithi099.stdout: "log_refresh_metadata": { 2024-03-19T02:05:37.101 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_refresh_metadata", 2024-03-19T02:05:37.101 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.101 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.101 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.101 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.101 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.101 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.102 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.102 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level", 2024-03-19T02:05:37.102 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.102 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.102 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.102 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.102 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.102 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.102 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.102 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.102 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.102 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.102 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.102 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.102 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.103 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "log to the \"cephadm\" cluster log channel\"", 2024-03-19T02:05:37.103 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.103 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.103 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.103 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.103 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.103 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.103 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.103 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.103 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.103 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.103 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.103 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.103 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.103 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.104 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.104 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.104 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.104 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.104 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.104 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.104 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.104 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.104 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.104 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.104 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.104 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.104 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.104 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.105 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.105 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.105 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.105 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.105 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.105 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.105 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.105 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.105 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.105 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.105 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.105 INFO:teuthology.orchestra.run.smithi099.stdout: "manage_etc_ceph_ceph_conf": { 2024-03-19T02:05:37.105 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "manage_etc_ceph_ceph_conf", 2024-03-19T02:05:37.105 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.106 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.106 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.106 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.106 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.106 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.106 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.106 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Manage and own /etc/ceph/ceph.conf on the hosts.", 2024-03-19T02:05:37.106 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.106 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.106 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.106 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.106 INFO:teuthology.orchestra.run.smithi099.stdout: "manage_etc_ceph_ceph_conf_hosts": { 2024-03-19T02:05:37.106 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "manage_etc_ceph_ceph_conf_hosts", 2024-03-19T02:05:37.106 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.106 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.107 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.107 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "*", 2024-03-19T02:05:37.107 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.107 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.107 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.107 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf", 2024-03-19T02:05:37.107 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.107 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.107 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.107 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.107 INFO:teuthology.orchestra.run.smithi099.stdout: "max_count_per_host": { 2024-03-19T02:05:37.107 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "max_count_per_host", 2024-03-19T02:05:37.107 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.107 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.108 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.108 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "10", 2024-03-19T02:05:37.108 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.108 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.108 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.108 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "max number of daemons per service per host", 2024-03-19T02:05:37.108 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.108 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.108 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.108 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.108 INFO:teuthology.orchestra.run.smithi099.stdout: "max_osd_draining_count": { 2024-03-19T02:05:37.108 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "max_osd_draining_count", 2024-03-19T02:05:37.108 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.108 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.108 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.109 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "10", 2024-03-19T02:05:37.109 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.109 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.109 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.109 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "max number of osds that will be drained simultaneously when osds are removed", 2024-03-19T02:05:37.109 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.109 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.109 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.109 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.109 INFO:teuthology.orchestra.run.smithi099.stdout: "migration_current": { 2024-03-19T02:05:37.109 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "migration_current", 2024-03-19T02:05:37.109 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.109 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.109 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.109 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.110 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.110 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.110 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.110 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "internal - do not modify", 2024-03-19T02:05:37.110 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.110 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.110 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.110 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.110 INFO:teuthology.orchestra.run.smithi099.stdout: "mode": { 2024-03-19T02:05:37.110 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "mode", 2024-03-19T02:05:37.110 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.110 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.110 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.110 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "root", 2024-03-19T02:05:37.111 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.111 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.111 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.111 INFO:teuthology.orchestra.run.smithi099.stdout: "cephadm-package", 2024-03-19T02:05:37.111 INFO:teuthology.orchestra.run.smithi099.stdout: "root" 2024-03-19T02:05:37.111 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.111 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "mode for remote execution of cephadm", 2024-03-19T02:05:37.111 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.111 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.111 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.111 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.111 INFO:teuthology.orchestra.run.smithi099.stdout: "oob_default_addr": { 2024-03-19T02:05:37.111 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "oob_default_addr", 2024-03-19T02:05:37.111 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.111 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.112 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.112 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "169.254.1.1", 2024-03-19T02:05:37.112 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.112 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.112 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.112 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Default address for RedFish API (oob management).", 2024-03-19T02:05:37.112 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.112 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.112 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.112 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.112 INFO:teuthology.orchestra.run.smithi099.stdout: "prometheus_alerts_path": { 2024-03-19T02:05:37.112 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "prometheus_alerts_path", 2024-03-19T02:05:37.112 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.112 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.112 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.113 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "/etc/prometheus/ceph/ceph_default_alerts.yml", 2024-03-19T02:05:37.113 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.113 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.113 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.113 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "location of alerts to include in prometheus deployments", 2024-03-19T02:05:37.113 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.113 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.113 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.113 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.113 INFO:teuthology.orchestra.run.smithi099.stdout: "registry_insecure": { 2024-03-19T02:05:37.113 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "registry_insecure", 2024-03-19T02:05:37.113 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.113 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.113 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.113 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.114 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.114 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.114 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.114 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Registry is to be considered insecure (no TLS available). Only for development purposes.", 2024-03-19T02:05:37.114 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.114 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.114 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.114 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.114 INFO:teuthology.orchestra.run.smithi099.stdout: "registry_password": { 2024-03-19T02:05:37.114 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "registry_password", 2024-03-19T02:05:37.114 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.114 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.114 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.114 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.115 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.115 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.115 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.115 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Custom repository password. Only used for logging into a registry.", 2024-03-19T02:05:37.115 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.115 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.115 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.115 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.115 INFO:teuthology.orchestra.run.smithi099.stdout: "registry_url": { 2024-03-19T02:05:37.115 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "registry_url", 2024-03-19T02:05:37.115 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.115 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.115 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.115 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.115 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.116 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.116 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.116 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Registry url for login purposes. This is not the default registry", 2024-03-19T02:05:37.116 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.116 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.116 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.116 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.116 INFO:teuthology.orchestra.run.smithi099.stdout: "registry_username": { 2024-03-19T02:05:37.116 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "registry_username", 2024-03-19T02:05:37.116 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.116 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.116 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.116 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.116 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.117 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.117 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.117 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Custom repository username. Only used for logging into a registry.", 2024-03-19T02:05:37.117 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.117 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.117 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.117 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.117 INFO:teuthology.orchestra.run.smithi099.stdout: "secure_monitoring_stack": { 2024-03-19T02:05:37.117 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "secure_monitoring_stack", 2024-03-19T02:05:37.117 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.117 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.117 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.117 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.117 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.117 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.118 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.118 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Enable TLS security for all the monitoring stack daemons", 2024-03-19T02:05:37.118 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.118 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.118 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.118 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.118 INFO:teuthology.orchestra.run.smithi099.stdout: "service_discovery_port": { 2024-03-19T02:05:37.118 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "service_discovery_port", 2024-03-19T02:05:37.118 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.118 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.118 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.118 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "8765", 2024-03-19T02:05:37.118 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.118 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.118 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.119 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "cephadm service discovery port", 2024-03-19T02:05:37.119 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.119 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.119 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.119 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.119 INFO:teuthology.orchestra.run.smithi099.stdout: "ssh_config_file": { 2024-03-19T02:05:37.119 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "ssh_config_file", 2024-03-19T02:05:37.119 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.119 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.119 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.119 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.119 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.119 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.119 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.120 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "customized SSH config file to connect to managed hosts", 2024-03-19T02:05:37.120 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.120 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.120 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.120 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.120 INFO:teuthology.orchestra.run.smithi099.stdout: "use_agent": { 2024-03-19T02:05:37.120 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "use_agent", 2024-03-19T02:05:37.120 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.120 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.120 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.120 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.120 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.120 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.120 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.120 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Use cephadm agent on each host to gather and send metadata", 2024-03-19T02:05:37.121 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.121 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.121 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.121 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.121 INFO:teuthology.orchestra.run.smithi099.stdout: "use_repo_digest": { 2024-03-19T02:05:37.121 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "use_repo_digest", 2024-03-19T02:05:37.121 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.121 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.121 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.121 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.121 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.121 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.121 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.121 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Automatically convert image tags to image digest. Make sure all daemons use the same image", 2024-03-19T02:05:37.121 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.122 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.122 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.122 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.122 INFO:teuthology.orchestra.run.smithi099.stdout: "warn_on_failed_host_check": { 2024-03-19T02:05:37.122 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "warn_on_failed_host_check", 2024-03-19T02:05:37.122 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.122 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.122 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.122 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.122 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.122 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.122 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.122 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "raise a health warning if the host check fails", 2024-03-19T02:05:37.122 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.123 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.123 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.123 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.123 INFO:teuthology.orchestra.run.smithi099.stdout: "warn_on_stray_daemons": { 2024-03-19T02:05:37.123 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "warn_on_stray_daemons", 2024-03-19T02:05:37.123 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.123 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.123 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.123 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.123 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.123 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.123 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.123 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "raise a health warning if daemons are detected that are not managed by cephadm", 2024-03-19T02:05:37.123 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.123 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.124 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.124 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.124 INFO:teuthology.orchestra.run.smithi099.stdout: "warn_on_stray_hosts": { 2024-03-19T02:05:37.124 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "warn_on_stray_hosts", 2024-03-19T02:05:37.124 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.124 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.124 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.124 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.124 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.124 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.124 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.124 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "raise a health warning if daemons are detected on a host that is not managed by cephadm", 2024-03-19T02:05:37.124 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.124 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.124 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.125 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.125 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.125 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.125 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.125 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "crash", 2024-03-19T02:05:37.125 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.125 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.125 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.125 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.125 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.125 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.125 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.125 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.125 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.125 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.126 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.126 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.126 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.126 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.126 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.126 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.126 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.126 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.126 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.126 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.126 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.126 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.126 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.126 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.126 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.127 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.127 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.127 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.127 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.127 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.127 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.127 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.127 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.127 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.127 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.127 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.127 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.127 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.127 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.127 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.128 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.128 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.128 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.128 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.128 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.128 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.128 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.128 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.128 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.128 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.128 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.128 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.128 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.129 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.129 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.129 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.129 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.129 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.129 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.129 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.129 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.129 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.129 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.129 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.129 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.129 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.129 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.129 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.130 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.130 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.130 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.130 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.130 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.130 INFO:teuthology.orchestra.run.smithi099.stdout: "retain_interval": { 2024-03-19T02:05:37.130 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "retain_interval", 2024-03-19T02:05:37.130 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:37.130 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.130 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.130 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "31536000", 2024-03-19T02:05:37.130 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.130 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.130 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.130 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "how long to retain crashes before pruning them", 2024-03-19T02:05:37.131 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.131 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.131 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.131 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.131 INFO:teuthology.orchestra.run.smithi099.stdout: "warn_recent_interval": { 2024-03-19T02:05:37.131 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "warn_recent_interval", 2024-03-19T02:05:37.131 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:37.131 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.131 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.131 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "1209600", 2024-03-19T02:05:37.131 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.131 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.131 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.131 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "time interval in which to warn about recent crashes", 2024-03-19T02:05:37.131 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.132 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.132 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.132 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.132 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.132 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.132 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.132 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "dashboard", 2024-03-19T02:05:37.132 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.132 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.132 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.132 INFO:teuthology.orchestra.run.smithi099.stdout: "ACCOUNT_LOCKOUT_ATTEMPTS": { 2024-03-19T02:05:37.132 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "ACCOUNT_LOCKOUT_ATTEMPTS", 2024-03-19T02:05:37.132 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.132 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.133 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.133 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "10", 2024-03-19T02:05:37.133 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.133 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.133 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.133 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.133 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.133 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.133 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.133 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.133 INFO:teuthology.orchestra.run.smithi099.stdout: "ALERTMANAGER_API_HOST": { 2024-03-19T02:05:37.133 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "ALERTMANAGER_API_HOST", 2024-03-19T02:05:37.133 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.133 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.133 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.134 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.134 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.134 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.134 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.134 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.134 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.134 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.134 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.134 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.134 INFO:teuthology.orchestra.run.smithi099.stdout: "ALERTMANAGER_API_SSL_VERIFY": { 2024-03-19T02:05:37.134 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "ALERTMANAGER_API_SSL_VERIFY", 2024-03-19T02:05:37.134 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.134 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.134 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.135 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.135 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.135 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.135 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.135 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.135 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.135 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.135 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.135 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.135 INFO:teuthology.orchestra.run.smithi099.stdout: "AUDIT_API_ENABLED": { 2024-03-19T02:05:37.135 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "AUDIT_API_ENABLED", 2024-03-19T02:05:37.135 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.135 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.135 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.135 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.136 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.136 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.136 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.136 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.136 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.136 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.136 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.136 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.136 INFO:teuthology.orchestra.run.smithi099.stdout: "AUDIT_API_LOG_PAYLOAD": { 2024-03-19T02:05:37.136 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "AUDIT_API_LOG_PAYLOAD", 2024-03-19T02:05:37.136 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.136 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.136 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.136 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.136 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.137 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.137 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.137 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.137 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.137 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.137 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.137 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.137 INFO:teuthology.orchestra.run.smithi099.stdout: "ENABLE_BROWSABLE_API": { 2024-03-19T02:05:37.137 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "ENABLE_BROWSABLE_API", 2024-03-19T02:05:37.137 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.137 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.137 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.137 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.137 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.137 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.138 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.138 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.138 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.138 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.138 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.138 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.138 INFO:teuthology.orchestra.run.smithi099.stdout: "FEATURE_TOGGLE_CEPHFS": { 2024-03-19T02:05:37.138 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "FEATURE_TOGGLE_CEPHFS", 2024-03-19T02:05:37.138 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.138 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.138 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.138 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.138 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.138 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.138 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.139 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.139 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.139 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.139 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.139 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.139 INFO:teuthology.orchestra.run.smithi099.stdout: "FEATURE_TOGGLE_DASHBOARD": { 2024-03-19T02:05:37.139 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "FEATURE_TOGGLE_DASHBOARD", 2024-03-19T02:05:37.139 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.139 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.139 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.139 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.139 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.139 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.139 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.139 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.140 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.140 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.140 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.140 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.140 INFO:teuthology.orchestra.run.smithi099.stdout: "FEATURE_TOGGLE_ISCSI": { 2024-03-19T02:05:37.140 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "FEATURE_TOGGLE_ISCSI", 2024-03-19T02:05:37.140 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.140 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.140 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.140 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.140 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.140 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.140 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.140 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.140 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.141 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.141 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.141 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.141 INFO:teuthology.orchestra.run.smithi099.stdout: "FEATURE_TOGGLE_MIRRORING": { 2024-03-19T02:05:37.141 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "FEATURE_TOGGLE_MIRRORING", 2024-03-19T02:05:37.141 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.141 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.141 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.141 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.141 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.141 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.141 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.141 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.141 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.142 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.142 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.142 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.142 INFO:teuthology.orchestra.run.smithi099.stdout: "FEATURE_TOGGLE_NFS": { 2024-03-19T02:05:37.142 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "FEATURE_TOGGLE_NFS", 2024-03-19T02:05:37.142 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.142 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.142 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.142 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.142 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.142 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.142 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.142 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.142 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.142 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.143 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.143 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.143 INFO:teuthology.orchestra.run.smithi099.stdout: "FEATURE_TOGGLE_RBD": { 2024-03-19T02:05:37.143 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "FEATURE_TOGGLE_RBD", 2024-03-19T02:05:37.143 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.143 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.143 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.143 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.143 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.143 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.143 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.143 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.143 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.143 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.143 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.144 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.144 INFO:teuthology.orchestra.run.smithi099.stdout: "FEATURE_TOGGLE_RGW": { 2024-03-19T02:05:37.144 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "FEATURE_TOGGLE_RGW", 2024-03-19T02:05:37.144 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.144 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.144 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.144 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.144 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.144 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.144 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.144 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.144 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.144 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.144 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.144 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.145 INFO:teuthology.orchestra.run.smithi099.stdout: "GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE": { 2024-03-19T02:05:37.145 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE", 2024-03-19T02:05:37.145 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.145 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.145 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.145 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.145 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.145 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.145 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.145 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.145 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.145 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.145 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.145 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.146 INFO:teuthology.orchestra.run.smithi099.stdout: "GRAFANA_API_PASSWORD": { 2024-03-19T02:05:37.146 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "GRAFANA_API_PASSWORD", 2024-03-19T02:05:37.146 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.146 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.146 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.146 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "admin", 2024-03-19T02:05:37.146 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.146 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.146 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.146 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.146 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.146 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.146 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.146 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.147 INFO:teuthology.orchestra.run.smithi099.stdout: "GRAFANA_API_SSL_VERIFY": { 2024-03-19T02:05:37.147 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "GRAFANA_API_SSL_VERIFY", 2024-03-19T02:05:37.147 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.147 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.147 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.147 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.147 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.147 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.147 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.147 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.147 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.147 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.147 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.147 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.147 INFO:teuthology.orchestra.run.smithi099.stdout: "GRAFANA_API_URL": { 2024-03-19T02:05:37.148 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "GRAFANA_API_URL", 2024-03-19T02:05:37.148 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.148 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.148 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.148 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.148 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.148 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.148 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.148 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.148 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.148 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.148 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.148 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.148 INFO:teuthology.orchestra.run.smithi099.stdout: "GRAFANA_API_USERNAME": { 2024-03-19T02:05:37.148 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "GRAFANA_API_USERNAME", 2024-03-19T02:05:37.149 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.149 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.149 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.149 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "admin", 2024-03-19T02:05:37.149 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.149 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.149 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.149 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.149 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.149 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.149 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.149 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.149 INFO:teuthology.orchestra.run.smithi099.stdout: "GRAFANA_FRONTEND_API_URL": { 2024-03-19T02:05:37.149 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "GRAFANA_FRONTEND_API_URL", 2024-03-19T02:05:37.149 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.149 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.150 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.150 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.150 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.150 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.150 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.150 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.150 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.150 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.150 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.150 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.150 INFO:teuthology.orchestra.run.smithi099.stdout: "GRAFANA_UPDATE_DASHBOARDS": { 2024-03-19T02:05:37.150 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "GRAFANA_UPDATE_DASHBOARDS", 2024-03-19T02:05:37.150 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.150 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.151 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.151 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.151 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.151 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.151 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.151 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.151 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.151 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.151 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.151 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.151 INFO:teuthology.orchestra.run.smithi099.stdout: "ISCSI_API_SSL_VERIFICATION": { 2024-03-19T02:05:37.151 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "ISCSI_API_SSL_VERIFICATION", 2024-03-19T02:05:37.151 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.151 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.151 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.152 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.152 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.152 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.152 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.152 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.152 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.152 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.152 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.152 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.152 INFO:teuthology.orchestra.run.smithi099.stdout: "ISSUE_TRACKER_API_KEY": { 2024-03-19T02:05:37.152 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "ISSUE_TRACKER_API_KEY", 2024-03-19T02:05:37.152 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.152 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.152 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.152 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.153 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.153 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.153 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.153 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.153 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.153 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.153 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.153 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.153 INFO:teuthology.orchestra.run.smithi099.stdout: "MULTICLUSTER_CONFIG": { 2024-03-19T02:05:37.153 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "MULTICLUSTER_CONFIG", 2024-03-19T02:05:37.153 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.153 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.153 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.153 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "{}", 2024-03-19T02:05:37.154 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.154 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.154 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.154 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.154 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.154 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.154 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.154 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.154 INFO:teuthology.orchestra.run.smithi099.stdout: "PROMETHEUS_API_HOST": { 2024-03-19T02:05:37.154 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "PROMETHEUS_API_HOST", 2024-03-19T02:05:37.154 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.154 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.154 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.154 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.155 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.155 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.155 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.155 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.155 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.155 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.155 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.155 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.155 INFO:teuthology.orchestra.run.smithi099.stdout: "PROMETHEUS_API_SSL_VERIFY": { 2024-03-19T02:05:37.155 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "PROMETHEUS_API_SSL_VERIFY", 2024-03-19T02:05:37.155 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.155 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.155 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.155 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.155 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.156 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.156 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.156 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.156 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.156 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.156 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.156 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.156 INFO:teuthology.orchestra.run.smithi099.stdout: "PWD_POLICY_CHECK_COMPLEXITY_ENABLED": { 2024-03-19T02:05:37.156 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "PWD_POLICY_CHECK_COMPLEXITY_ENABLED", 2024-03-19T02:05:37.156 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.156 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.156 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.156 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.156 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.156 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.157 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.157 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.157 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.157 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.157 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.157 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.157 INFO:teuthology.orchestra.run.smithi099.stdout: "PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED": { 2024-03-19T02:05:37.157 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED", 2024-03-19T02:05:37.157 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.157 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.157 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.157 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.157 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.157 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.158 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.158 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.158 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.158 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.158 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.158 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.158 INFO:teuthology.orchestra.run.smithi099.stdout: "PWD_POLICY_CHECK_LENGTH_ENABLED": { 2024-03-19T02:05:37.158 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "PWD_POLICY_CHECK_LENGTH_ENABLED", 2024-03-19T02:05:37.158 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.158 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.158 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.158 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.158 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.158 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.158 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.159 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.159 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.159 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.159 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.159 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.159 INFO:teuthology.orchestra.run.smithi099.stdout: "PWD_POLICY_CHECK_OLDPWD_ENABLED": { 2024-03-19T02:05:37.159 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "PWD_POLICY_CHECK_OLDPWD_ENABLED", 2024-03-19T02:05:37.159 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.159 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.159 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.159 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.159 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.159 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.159 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.159 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.160 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.160 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.160 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.160 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.160 INFO:teuthology.orchestra.run.smithi099.stdout: "PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED": { 2024-03-19T02:05:37.160 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED", 2024-03-19T02:05:37.160 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.160 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.160 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.160 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.160 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.160 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.160 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.160 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.160 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.161 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.161 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.161 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.161 INFO:teuthology.orchestra.run.smithi099.stdout: "PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED": { 2024-03-19T02:05:37.161 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED", 2024-03-19T02:05:37.161 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.161 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.161 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.161 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.161 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.161 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.161 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.161 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.161 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.161 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.162 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.162 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.162 INFO:teuthology.orchestra.run.smithi099.stdout: "PWD_POLICY_CHECK_USERNAME_ENABLED": { 2024-03-19T02:05:37.162 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "PWD_POLICY_CHECK_USERNAME_ENABLED", 2024-03-19T02:05:37.162 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.162 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.162 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.162 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.162 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.162 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.162 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.162 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.162 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.162 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.162 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.163 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.163 INFO:teuthology.orchestra.run.smithi099.stdout: "PWD_POLICY_ENABLED": { 2024-03-19T02:05:37.163 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "PWD_POLICY_ENABLED", 2024-03-19T02:05:37.163 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.163 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.163 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.163 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.163 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.163 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.163 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.163 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.163 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.163 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.163 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.163 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.164 INFO:teuthology.orchestra.run.smithi099.stdout: "PWD_POLICY_EXCLUSION_LIST": { 2024-03-19T02:05:37.164 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "PWD_POLICY_EXCLUSION_LIST", 2024-03-19T02:05:37.164 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.164 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.164 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.164 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps", 2024-03-19T02:05:37.164 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.164 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.164 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.164 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.164 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.164 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.164 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.164 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.164 INFO:teuthology.orchestra.run.smithi099.stdout: "PWD_POLICY_MIN_COMPLEXITY": { 2024-03-19T02:05:37.165 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "PWD_POLICY_MIN_COMPLEXITY", 2024-03-19T02:05:37.165 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.165 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.165 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.165 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "10", 2024-03-19T02:05:37.165 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.165 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.165 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.165 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.165 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.165 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.165 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.165 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.165 INFO:teuthology.orchestra.run.smithi099.stdout: "PWD_POLICY_MIN_LENGTH": { 2024-03-19T02:05:37.165 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "PWD_POLICY_MIN_LENGTH", 2024-03-19T02:05:37.166 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.166 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.166 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.166 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "8", 2024-03-19T02:05:37.166 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.166 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.166 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.166 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.166 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.166 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.166 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.166 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.166 INFO:teuthology.orchestra.run.smithi099.stdout: "REST_REQUESTS_TIMEOUT": { 2024-03-19T02:05:37.167 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "REST_REQUESTS_TIMEOUT", 2024-03-19T02:05:37.167 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.167 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.167 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.167 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "45", 2024-03-19T02:05:37.167 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.167 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.167 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.167 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.167 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.167 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.167 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.167 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.167 INFO:teuthology.orchestra.run.smithi099.stdout: "RGW_API_ACCESS_KEY": { 2024-03-19T02:05:37.167 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "RGW_API_ACCESS_KEY", 2024-03-19T02:05:37.168 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.168 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.168 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.168 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.168 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.168 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.168 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.168 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.168 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.168 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.168 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.168 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.168 INFO:teuthology.orchestra.run.smithi099.stdout: "RGW_API_ADMIN_RESOURCE": { 2024-03-19T02:05:37.168 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "RGW_API_ADMIN_RESOURCE", 2024-03-19T02:05:37.168 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.169 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.169 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.169 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "admin", 2024-03-19T02:05:37.169 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.169 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.169 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.169 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.169 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.169 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.169 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.169 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.169 INFO:teuthology.orchestra.run.smithi099.stdout: "RGW_API_SECRET_KEY": { 2024-03-19T02:05:37.169 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "RGW_API_SECRET_KEY", 2024-03-19T02:05:37.169 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.170 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.170 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.170 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.170 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.170 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.170 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.170 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.170 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.170 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.170 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.170 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.170 INFO:teuthology.orchestra.run.smithi099.stdout: "RGW_API_SSL_VERIFY": { 2024-03-19T02:05:37.170 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "RGW_API_SSL_VERIFY", 2024-03-19T02:05:37.170 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.171 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.171 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.171 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.171 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.171 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.171 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.171 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.171 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.171 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.171 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.171 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.171 INFO:teuthology.orchestra.run.smithi099.stdout: "UNSAFE_TLS_v1_2": { 2024-03-19T02:05:37.171 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "UNSAFE_TLS_v1_2", 2024-03-19T02:05:37.171 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.171 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.172 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.172 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.172 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.172 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.172 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.172 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.172 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.172 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.172 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.172 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.172 INFO:teuthology.orchestra.run.smithi099.stdout: "USER_PWD_EXPIRATION_SPAN": { 2024-03-19T02:05:37.172 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "USER_PWD_EXPIRATION_SPAN", 2024-03-19T02:05:37.172 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.172 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.173 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.173 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "0", 2024-03-19T02:05:37.173 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.173 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.173 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.173 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.173 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.173 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.173 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.173 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.173 INFO:teuthology.orchestra.run.smithi099.stdout: "USER_PWD_EXPIRATION_WARNING_1": { 2024-03-19T02:05:37.173 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "USER_PWD_EXPIRATION_WARNING_1", 2024-03-19T02:05:37.173 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.173 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.173 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.174 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "10", 2024-03-19T02:05:37.174 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.174 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.174 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.174 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.174 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.174 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.174 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.174 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.174 INFO:teuthology.orchestra.run.smithi099.stdout: "USER_PWD_EXPIRATION_WARNING_2": { 2024-03-19T02:05:37.174 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "USER_PWD_EXPIRATION_WARNING_2", 2024-03-19T02:05:37.174 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.174 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.174 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.175 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "5", 2024-03-19T02:05:37.175 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.175 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.175 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.175 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.175 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.175 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.175 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.175 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.175 INFO:teuthology.orchestra.run.smithi099.stdout: "cross_origin_url": { 2024-03-19T02:05:37.175 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "cross_origin_url", 2024-03-19T02:05:37.175 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.175 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.175 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.175 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.176 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.176 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.176 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.176 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.176 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.176 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.176 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.176 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.176 INFO:teuthology.orchestra.run.smithi099.stdout: "crt_file": { 2024-03-19T02:05:37.176 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "crt_file", 2024-03-19T02:05:37.176 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.176 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.176 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.176 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.176 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.177 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.177 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.177 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.177 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.177 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.177 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.177 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.177 INFO:teuthology.orchestra.run.smithi099.stdout: "debug": { 2024-03-19T02:05:37.177 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "debug", 2024-03-19T02:05:37.177 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.177 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.177 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.177 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.177 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.177 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.178 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.178 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Enable/disable debug options", 2024-03-19T02:05:37.178 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.178 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.178 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.178 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.178 INFO:teuthology.orchestra.run.smithi099.stdout: "jwt_token_ttl": { 2024-03-19T02:05:37.178 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "jwt_token_ttl", 2024-03-19T02:05:37.178 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.178 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.178 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.178 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "28800", 2024-03-19T02:05:37.178 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.178 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.179 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.179 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.179 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.179 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.179 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.179 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.179 INFO:teuthology.orchestra.run.smithi099.stdout: "key_file": { 2024-03-19T02:05:37.179 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "key_file", 2024-03-19T02:05:37.179 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.179 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.179 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.179 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.179 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.179 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.180 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.180 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.180 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.180 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.180 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.180 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.180 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.180 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.180 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.180 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.180 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.180 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.180 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.180 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.180 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.181 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.181 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.181 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.181 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.181 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.181 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.181 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.181 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.181 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.181 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.181 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.181 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.181 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.181 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.181 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.182 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.182 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.182 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.182 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.182 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.182 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.182 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.182 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.182 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.182 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.182 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.182 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.182 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.182 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.183 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.183 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.183 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.183 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.183 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.183 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.183 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.183 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.183 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.183 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.183 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.183 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.183 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.183 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.184 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.184 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.184 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.184 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.184 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.184 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.184 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.184 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.184 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.184 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.184 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.184 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.184 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.184 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.184 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.185 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.185 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.185 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.185 INFO:teuthology.orchestra.run.smithi099.stdout: "motd": { 2024-03-19T02:05:37.185 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "motd", 2024-03-19T02:05:37.185 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.185 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.185 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.185 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.185 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.185 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.185 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.185 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "The message of the day", 2024-03-19T02:05:37.185 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.185 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.186 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.186 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.186 INFO:teuthology.orchestra.run.smithi099.stdout: "redirect_resolve_ip_addr": { 2024-03-19T02:05:37.186 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "redirect_resolve_ip_addr", 2024-03-19T02:05:37.186 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.186 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.186 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.186 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.186 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.186 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.186 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.186 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.186 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.186 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.187 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.187 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.187 INFO:teuthology.orchestra.run.smithi099.stdout: "server_addr": { 2024-03-19T02:05:37.187 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "server_addr", 2024-03-19T02:05:37.187 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.187 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.187 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.187 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "::", 2024-03-19T02:05:37.187 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.187 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.187 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.187 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.187 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.187 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.187 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.188 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.188 INFO:teuthology.orchestra.run.smithi099.stdout: "server_port": { 2024-03-19T02:05:37.188 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "server_port", 2024-03-19T02:05:37.188 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.188 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.188 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.188 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "8080", 2024-03-19T02:05:37.188 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.188 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.188 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.188 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.188 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.188 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.188 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.188 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.189 INFO:teuthology.orchestra.run.smithi099.stdout: "ssl": { 2024-03-19T02:05:37.189 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "ssl", 2024-03-19T02:05:37.189 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.189 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.189 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.189 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.189 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.189 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.189 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.189 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.189 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.189 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.189 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.189 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.189 INFO:teuthology.orchestra.run.smithi099.stdout: "ssl_server_port": { 2024-03-19T02:05:37.189 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "ssl_server_port", 2024-03-19T02:05:37.190 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.190 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.190 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.190 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "8443", 2024-03-19T02:05:37.190 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.190 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.190 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.190 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.190 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.190 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.190 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.190 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.190 INFO:teuthology.orchestra.run.smithi099.stdout: "standby_behaviour": { 2024-03-19T02:05:37.190 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "standby_behaviour", 2024-03-19T02:05:37.191 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.191 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.191 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.191 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "redirect", 2024-03-19T02:05:37.191 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.191 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.191 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.191 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.191 INFO:teuthology.orchestra.run.smithi099.stdout: "redirect" 2024-03-19T02:05:37.191 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.191 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.191 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.191 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.191 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.191 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.192 INFO:teuthology.orchestra.run.smithi099.stdout: "standby_error_status_code": { 2024-03-19T02:05:37.192 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "standby_error_status_code", 2024-03-19T02:05:37.192 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.192 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.192 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.192 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "500", 2024-03-19T02:05:37.192 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "400", 2024-03-19T02:05:37.192 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "599", 2024-03-19T02:05:37.192 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.192 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.192 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.192 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.192 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.192 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.193 INFO:teuthology.orchestra.run.smithi099.stdout: "url_prefix": { 2024-03-19T02:05:37.193 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "url_prefix", 2024-03-19T02:05:37.193 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.193 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.193 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.193 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.193 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.193 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.193 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.193 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.193 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.193 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.193 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.193 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.193 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.194 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.194 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.194 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "devicehealth", 2024-03-19T02:05:37.194 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.194 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.194 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.194 INFO:teuthology.orchestra.run.smithi099.stdout: "enable_monitoring": { 2024-03-19T02:05:37.194 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "enable_monitoring", 2024-03-19T02:05:37.194 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.194 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.194 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.194 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.194 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.194 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.195 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.195 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "monitor device health metrics", 2024-03-19T02:05:37.195 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.195 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.195 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.195 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.195 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.195 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.195 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.195 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.195 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.195 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.195 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.195 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.195 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.196 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.196 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.196 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.196 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.196 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.196 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.196 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.196 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.196 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.196 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.196 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.196 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.196 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.196 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.197 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.197 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.197 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.197 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.197 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.197 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.197 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.197 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.197 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.197 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.197 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.197 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.197 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.197 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.197 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.198 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.198 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.198 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.198 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.198 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.198 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.198 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.198 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.198 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.198 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.198 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.198 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.198 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.198 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.199 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.199 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.199 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.199 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.199 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.199 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.199 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.199 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.199 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.199 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.199 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.199 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.199 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.199 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.199 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.200 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.200 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.200 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.200 INFO:teuthology.orchestra.run.smithi099.stdout: "mark_out_threshold": { 2024-03-19T02:05:37.200 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "mark_out_threshold", 2024-03-19T02:05:37.200 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:37.200 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.200 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.200 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "2419200", 2024-03-19T02:05:37.200 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.200 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.200 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.200 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "automatically mark OSD if it may fail before this long", 2024-03-19T02:05:37.200 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.200 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.201 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.201 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.201 INFO:teuthology.orchestra.run.smithi099.stdout: "pool_name": { 2024-03-19T02:05:37.201 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "pool_name", 2024-03-19T02:05:37.201 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.201 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.201 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.201 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "device_health_metrics", 2024-03-19T02:05:37.201 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.201 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.201 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.201 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "name of pool in which to store device health metrics", 2024-03-19T02:05:37.201 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.201 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.202 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.202 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.202 INFO:teuthology.orchestra.run.smithi099.stdout: "retention_period": { 2024-03-19T02:05:37.202 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "retention_period", 2024-03-19T02:05:37.202 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:37.202 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.202 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.202 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "15552000", 2024-03-19T02:05:37.202 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.202 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.202 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.202 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "how long to retain device health metrics", 2024-03-19T02:05:37.202 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.202 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.202 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.203 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.203 INFO:teuthology.orchestra.run.smithi099.stdout: "scrape_frequency": { 2024-03-19T02:05:37.203 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "scrape_frequency", 2024-03-19T02:05:37.203 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:37.203 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.203 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.203 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "86400", 2024-03-19T02:05:37.203 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.203 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.203 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.203 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "how frequently to scrape device health metrics", 2024-03-19T02:05:37.203 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.203 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.203 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.203 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.204 INFO:teuthology.orchestra.run.smithi099.stdout: "self_heal": { 2024-03-19T02:05:37.204 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "self_heal", 2024-03-19T02:05:37.204 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.204 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.204 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.204 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.204 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.204 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.204 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.204 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "preemptively heal cluster around devices that may fail", 2024-03-19T02:05:37.204 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.204 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.204 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.204 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.205 INFO:teuthology.orchestra.run.smithi099.stdout: "sleep_interval": { 2024-03-19T02:05:37.205 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "sleep_interval", 2024-03-19T02:05:37.205 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:37.205 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.205 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.205 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "600", 2024-03-19T02:05:37.205 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.205 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.205 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.205 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "how frequently to wake up and check device health", 2024-03-19T02:05:37.205 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.205 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.205 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.205 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.206 INFO:teuthology.orchestra.run.smithi099.stdout: "warn_threshold": { 2024-03-19T02:05:37.206 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "warn_threshold", 2024-03-19T02:05:37.206 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:37.206 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.206 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.206 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "7257600", 2024-03-19T02:05:37.206 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.206 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.206 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.206 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "raise health warning if OSD may fail before this long", 2024-03-19T02:05:37.206 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.206 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.206 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.206 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.206 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.207 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.207 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.207 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "diskprediction_local", 2024-03-19T02:05:37.207 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.207 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.207 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.207 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.207 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.207 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.207 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.207 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.207 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.207 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.207 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.208 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.208 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.208 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.208 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.208 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.208 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.208 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.208 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.208 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.208 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.208 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.208 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.208 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.208 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.208 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.209 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.209 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.209 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.209 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.209 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.209 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.209 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.209 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.209 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.209 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.209 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.209 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.209 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.209 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.210 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.210 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.210 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.210 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.210 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.210 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.210 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.210 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.210 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.210 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.210 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.210 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.210 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.210 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.210 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.211 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.211 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.211 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.211 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.211 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.211 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.211 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.211 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.211 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.211 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.211 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.211 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.211 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.211 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.211 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.212 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.212 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.212 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.212 INFO:teuthology.orchestra.run.smithi099.stdout: "predict_interval": { 2024-03-19T02:05:37.212 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "predict_interval", 2024-03-19T02:05:37.212 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.212 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.212 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.212 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "86400", 2024-03-19T02:05:37.212 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.212 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.212 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.212 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.212 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.212 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.213 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.213 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.213 INFO:teuthology.orchestra.run.smithi099.stdout: "predictor_model": { 2024-03-19T02:05:37.213 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "predictor_model", 2024-03-19T02:05:37.213 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.213 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.213 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.213 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "prophetstor", 2024-03-19T02:05:37.213 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.213 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.213 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.213 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.213 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.213 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.214 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.214 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.214 INFO:teuthology.orchestra.run.smithi099.stdout: "sleep_interval": { 2024-03-19T02:05:37.214 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "sleep_interval", 2024-03-19T02:05:37.214 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.214 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.214 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.214 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "600", 2024-03-19T02:05:37.214 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.214 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.214 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.214 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.214 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.214 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.215 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.215 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.215 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.215 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.215 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.215 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "influx", 2024-03-19T02:05:37.215 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": false, 2024-03-19T02:05:37.215 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "influxdb python module not found", 2024-03-19T02:05:37.215 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.215 INFO:teuthology.orchestra.run.smithi099.stdout: "batch_size": { 2024-03-19T02:05:37.215 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "batch_size", 2024-03-19T02:05:37.215 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.215 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.215 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.215 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "5000", 2024-03-19T02:05:37.216 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.216 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.216 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.216 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "How big batches of data points should be when sending to InfluxDB.", 2024-03-19T02:05:37.216 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.216 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.216 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.216 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.216 INFO:teuthology.orchestra.run.smithi099.stdout: "database": { 2024-03-19T02:05:37.216 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "database", 2024-03-19T02:05:37.216 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.216 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.216 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.216 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "ceph", 2024-03-19T02:05:37.216 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.217 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.217 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.217 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.", 2024-03-19T02:05:37.217 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.217 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.217 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.217 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.217 INFO:teuthology.orchestra.run.smithi099.stdout: "hostname": { 2024-03-19T02:05:37.217 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "hostname", 2024-03-19T02:05:37.217 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.217 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.217 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.217 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.217 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.218 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.218 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.218 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "InfluxDB server hostname", 2024-03-19T02:05:37.218 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.218 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.218 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.218 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.218 INFO:teuthology.orchestra.run.smithi099.stdout: "interval": { 2024-03-19T02:05:37.218 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "interval", 2024-03-19T02:05:37.218 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:37.218 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.218 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.218 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "30", 2024-03-19T02:05:37.218 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "5", 2024-03-19T02:05:37.219 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.219 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.219 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Time between reports to InfluxDB. Default 30 seconds.", 2024-03-19T02:05:37.219 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.219 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.219 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.219 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.219 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.219 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.219 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.219 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.219 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.219 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.219 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.219 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.220 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.220 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.220 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.220 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.220 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.220 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.220 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.220 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.220 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.220 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.220 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.220 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.220 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.220 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.220 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.221 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.221 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.221 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.221 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.221 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.221 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.221 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.221 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.221 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.221 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.221 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.221 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.221 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.221 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.222 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.222 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.222 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.222 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.222 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.222 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.222 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.222 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.222 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.222 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.222 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.222 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.222 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.222 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.222 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.223 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.223 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.223 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.223 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.223 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.223 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.223 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.223 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.223 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.223 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.223 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.223 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.223 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.223 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.223 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.224 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.224 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.224 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.224 INFO:teuthology.orchestra.run.smithi099.stdout: "password": { 2024-03-19T02:05:37.224 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "password", 2024-03-19T02:05:37.224 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.224 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.224 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.224 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.224 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.224 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.224 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.224 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "password of InfluxDB server user", 2024-03-19T02:05:37.224 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.224 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.225 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.225 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.225 INFO:teuthology.orchestra.run.smithi099.stdout: "port": { 2024-03-19T02:05:37.225 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "port", 2024-03-19T02:05:37.225 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.225 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.225 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.225 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "8086", 2024-03-19T02:05:37.225 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.225 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.225 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.225 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "InfluxDB server port", 2024-03-19T02:05:37.225 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.225 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.225 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.226 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.226 INFO:teuthology.orchestra.run.smithi099.stdout: "ssl": { 2024-03-19T02:05:37.226 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "ssl", 2024-03-19T02:05:37.226 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.226 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.226 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.226 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "false", 2024-03-19T02:05:37.226 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.226 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.226 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.226 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Use https connection for InfluxDB server. Use \"true\" or \"false\".", 2024-03-19T02:05:37.226 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.226 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.226 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.226 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.227 INFO:teuthology.orchestra.run.smithi099.stdout: "threads": { 2024-03-19T02:05:37.227 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "threads", 2024-03-19T02:05:37.227 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.227 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.227 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.227 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "5", 2024-03-19T02:05:37.227 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "1", 2024-03-19T02:05:37.227 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "32", 2024-03-19T02:05:37.227 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.227 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "How many worker threads should be spawned for sending data to InfluxDB.", 2024-03-19T02:05:37.227 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.227 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.227 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.227 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.227 INFO:teuthology.orchestra.run.smithi099.stdout: "username": { 2024-03-19T02:05:37.228 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "username", 2024-03-19T02:05:37.228 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.228 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.228 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.228 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.228 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.228 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.228 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.228 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "username of InfluxDB server user", 2024-03-19T02:05:37.228 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.228 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.228 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.228 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.228 INFO:teuthology.orchestra.run.smithi099.stdout: "verify_ssl": { 2024-03-19T02:05:37.228 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "verify_ssl", 2024-03-19T02:05:37.229 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.229 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.229 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.229 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "true", 2024-03-19T02:05:37.229 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.229 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.229 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.229 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Verify https cert for InfluxDB server. Use \"true\" or \"false\".", 2024-03-19T02:05:37.229 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.229 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.229 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.229 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.229 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.229 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.229 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.230 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "insights", 2024-03-19T02:05:37.230 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.230 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.230 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.230 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.230 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.230 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.230 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.230 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.230 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.230 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.230 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.230 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.230 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.231 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.231 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.231 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.231 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.231 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.231 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.231 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.231 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.231 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.231 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.231 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.231 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.231 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.231 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.232 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.232 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.232 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.232 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.232 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.232 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.232 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.232 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.232 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.232 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.232 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.232 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.232 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.232 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.232 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.233 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.233 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.233 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.233 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.233 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.233 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.233 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.233 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.233 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.233 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.233 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.233 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.233 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.233 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.233 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.234 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.234 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.234 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.234 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.234 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.234 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.234 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.234 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.234 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.234 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.234 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.234 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.234 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.234 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.235 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.235 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.235 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.235 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.235 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.235 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "iostat", 2024-03-19T02:05:37.235 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.235 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.235 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.235 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.235 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.235 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.235 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.235 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.235 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.236 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.236 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.236 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.236 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.236 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.236 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.236 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.236 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.236 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.236 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.236 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.236 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.236 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.236 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.236 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.237 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.237 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.237 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.237 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.237 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.237 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.237 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.237 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.237 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.237 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.237 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.237 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.237 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.237 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.237 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.238 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.238 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.238 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.238 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.238 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.238 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.238 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.238 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.238 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.238 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.238 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.238 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.238 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.238 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.239 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.239 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.239 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.239 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.239 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.239 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.239 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.239 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.239 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.239 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.239 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.239 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.239 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.239 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.239 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.240 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.240 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.240 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.240 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.240 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.240 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.240 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.240 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.240 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "k8sevents", 2024-03-19T02:05:37.240 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.240 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.240 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.240 INFO:teuthology.orchestra.run.smithi099.stdout: "ceph_event_retention_days": { 2024-03-19T02:05:37.240 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "ceph_event_retention_days", 2024-03-19T02:05:37.240 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.241 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.241 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.241 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "7", 2024-03-19T02:05:37.241 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.241 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.241 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.241 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Days to hold ceph event information within local cache", 2024-03-19T02:05:37.241 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.241 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.241 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.241 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.241 INFO:teuthology.orchestra.run.smithi099.stdout: "config_check_secs": { 2024-03-19T02:05:37.241 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "config_check_secs", 2024-03-19T02:05:37.241 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.241 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.242 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.242 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "10", 2024-03-19T02:05:37.242 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "10", 2024-03-19T02:05:37.242 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.242 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.242 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "interval (secs) to check for cluster configuration changes", 2024-03-19T02:05:37.242 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.242 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.242 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.242 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.242 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.242 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.242 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.242 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.242 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.243 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.243 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.243 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.243 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.243 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.243 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.243 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.243 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.243 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.243 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.243 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.243 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.243 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.243 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.244 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.244 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.244 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.244 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.244 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.244 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.244 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.244 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.244 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.244 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.244 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.244 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.244 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.244 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.245 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.245 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.245 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.245 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.245 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.245 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.245 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.245 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.245 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.245 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.245 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.245 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.245 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.245 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.245 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.246 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.246 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.246 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.246 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.246 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.246 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.246 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.246 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.246 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.246 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.246 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.246 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.246 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.246 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.247 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.247 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.247 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.247 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.247 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.247 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.247 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.247 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.247 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.247 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.247 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.247 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "localpool", 2024-03-19T02:05:37.247 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.247 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.249 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.249 INFO:teuthology.orchestra.run.smithi099.stdout: "failure_domain": { 2024-03-19T02:05:37.249 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "failure_domain", 2024-03-19T02:05:37.249 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.249 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.249 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.249 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "host", 2024-03-19T02:05:37.249 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.249 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.249 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.250 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "failure domain for any created local pool", 2024-03-19T02:05:37.250 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "what failure domain we should separate data replicas across.", 2024-03-19T02:05:37.250 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.250 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.250 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.250 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.250 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.250 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.250 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.250 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.250 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.250 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.250 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.250 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.251 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.251 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.251 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.251 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.251 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.251 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.251 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.251 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.251 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.251 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.251 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.251 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.251 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.251 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.251 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.252 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.252 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.252 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.252 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.252 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.252 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.252 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.252 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.252 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.252 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.252 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.252 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.252 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.252 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.252 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.253 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.253 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.253 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.253 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.253 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.253 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.253 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.253 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.253 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.253 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.253 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.253 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.253 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.253 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.253 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.254 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.254 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.254 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.254 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.254 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.254 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.254 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.254 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.254 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.254 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.254 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.254 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.254 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.254 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.255 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.255 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.255 INFO:teuthology.orchestra.run.smithi099.stdout: "min_size": { 2024-03-19T02:05:37.255 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "min_size", 2024-03-19T02:05:37.255 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.255 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.255 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.255 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.255 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.255 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.255 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.255 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "default min_size for any created local pool", 2024-03-19T02:05:37.255 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "value to set min_size to (unchanged from Ceph's default if this option is not set)", 2024-03-19T02:05:37.255 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.255 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.255 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.256 INFO:teuthology.orchestra.run.smithi099.stdout: "num_rep": { 2024-03-19T02:05:37.256 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "num_rep", 2024-03-19T02:05:37.256 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.256 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.256 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.256 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "3", 2024-03-19T02:05:37.256 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.256 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.256 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.256 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "default replica count for any created local pool", 2024-03-19T02:05:37.256 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.256 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.256 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.256 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.256 INFO:teuthology.orchestra.run.smithi099.stdout: "pg_num": { 2024-03-19T02:05:37.257 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "pg_num", 2024-03-19T02:05:37.257 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.257 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.257 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.257 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "128", 2024-03-19T02:05:37.257 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.257 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.257 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.257 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "default pg_num for any created local pool", 2024-03-19T02:05:37.257 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.257 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.258 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.258 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.258 INFO:teuthology.orchestra.run.smithi099.stdout: "prefix": { 2024-03-19T02:05:37.258 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "prefix", 2024-03-19T02:05:37.258 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.258 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.258 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.258 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.258 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.258 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.258 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.258 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "name prefix for any created local pool", 2024-03-19T02:05:37.258 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.258 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.259 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.259 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.259 INFO:teuthology.orchestra.run.smithi099.stdout: "subtree": { 2024-03-19T02:05:37.259 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "subtree", 2024-03-19T02:05:37.259 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.259 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.259 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.259 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "rack", 2024-03-19T02:05:37.259 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.259 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.259 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.259 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "CRUSH level for which to create a local pool", 2024-03-19T02:05:37.259 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "which CRUSH subtree type the module should create a pool for.", 2024-03-19T02:05:37.259 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.259 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.260 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.260 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.260 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.260 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.260 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "mds_autoscaler", 2024-03-19T02:05:37.260 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.260 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.260 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.260 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.260 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.260 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.260 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.260 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.260 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.261 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.261 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.261 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.261 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.261 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.261 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.261 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.261 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.261 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.261 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.261 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.261 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.261 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.261 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.261 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.262 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.262 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.262 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.262 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.262 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.262 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.262 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.262 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.262 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.262 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.262 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.262 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.262 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.262 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.263 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.263 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.263 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.263 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.263 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.263 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.263 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.263 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.263 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.263 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.263 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.263 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.263 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.263 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.263 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.264 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.264 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.264 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.264 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.264 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.264 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.264 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.264 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.264 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.264 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.264 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.264 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.264 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.264 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.264 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.265 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.265 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.265 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.265 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.265 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.265 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.265 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.265 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.265 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "mirroring", 2024-03-19T02:05:37.265 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.265 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.265 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.265 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.265 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.265 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.266 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.266 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.266 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.266 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.266 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.266 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.266 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.266 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.266 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.266 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.266 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.266 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.266 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.266 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.266 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.267 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.267 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.267 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.267 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.267 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.267 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.267 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.267 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.267 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.267 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.267 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.267 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.267 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.267 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.267 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.268 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.268 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.268 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.268 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.268 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.268 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.268 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.268 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.268 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.268 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.268 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.268 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.268 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.268 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.268 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.269 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.269 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.269 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.269 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.269 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.269 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.269 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.269 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.269 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.269 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.269 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.269 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.269 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.269 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.269 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.270 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.270 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.270 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.270 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.270 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.270 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.270 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.270 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.270 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.270 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.270 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "nfs", 2024-03-19T02:05:37.270 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.270 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.270 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.271 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.271 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.271 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.271 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.271 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.271 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.271 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.271 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.271 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.271 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.271 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.271 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.271 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.271 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.272 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.272 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.272 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.272 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.272 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.272 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.272 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.272 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.272 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.272 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.272 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.272 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.272 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.272 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.272 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.273 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.273 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.273 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.273 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.273 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.273 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.273 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.273 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.273 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.273 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.273 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.273 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.273 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.273 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.273 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.274 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.274 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.274 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.274 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.274 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.274 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.274 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.274 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.274 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.274 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.274 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.274 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.274 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.274 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.275 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.275 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.275 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.275 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.275 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.275 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.275 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.275 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.275 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.275 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.275 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.275 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.275 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.275 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.275 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.276 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "orchestrator", 2024-03-19T02:05:37.276 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.276 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.276 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.276 INFO:teuthology.orchestra.run.smithi099.stdout: "fail_fs": { 2024-03-19T02:05:37.276 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "fail_fs", 2024-03-19T02:05:37.276 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.276 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.276 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.276 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.276 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.276 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.276 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.276 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Fail filesystem for rapid multi-rank mds upgrade", 2024-03-19T02:05:37.277 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.277 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.277 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.277 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.277 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.277 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.277 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.277 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.277 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.277 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.277 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.277 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.277 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.277 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.277 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.278 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.278 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.278 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.278 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.278 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.278 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.278 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.278 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.278 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.278 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.278 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.278 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.278 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.278 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.279 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.279 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.279 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.279 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.279 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.279 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.279 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.279 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.279 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.279 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.279 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.279 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.279 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.279 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.279 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.280 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.280 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.280 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.280 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.280 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.280 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.280 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.280 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.280 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.280 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.280 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.280 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.280 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.280 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.280 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.281 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.281 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.281 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.281 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.281 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.281 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.281 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.281 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.281 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.281 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.281 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.281 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.281 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.281 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.281 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.282 INFO:teuthology.orchestra.run.smithi099.stdout: "orchestrator": { 2024-03-19T02:05:37.282 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "orchestrator", 2024-03-19T02:05:37.282 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.282 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.282 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.282 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.282 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.282 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.282 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.282 INFO:teuthology.orchestra.run.smithi099.stdout: "cephadm", 2024-03-19T02:05:37.282 INFO:teuthology.orchestra.run.smithi099.stdout: "rook", 2024-03-19T02:05:37.282 INFO:teuthology.orchestra.run.smithi099.stdout: "test_orchestrator" 2024-03-19T02:05:37.282 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.282 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Orchestrator backend", 2024-03-19T02:05:37.282 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.283 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.283 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.283 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.283 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.283 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.283 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.283 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "osd_perf_query", 2024-03-19T02:05:37.283 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.283 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.283 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.283 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.283 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.283 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.283 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.284 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.284 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.284 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.284 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.284 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.284 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.284 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.284 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.284 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.284 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.284 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.284 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.284 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.284 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.285 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.285 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.285 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.285 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.285 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.285 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.285 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.285 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.285 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.285 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.285 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.285 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.285 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.285 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.285 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.286 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.286 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.286 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.286 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.286 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.286 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.286 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.286 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.286 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.286 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.286 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.286 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.286 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.286 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.287 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.287 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.287 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.287 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.287 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.287 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.287 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.287 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.287 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.287 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.287 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.287 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.287 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.287 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.287 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.288 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.288 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.288 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.288 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.288 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.288 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.288 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.288 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.288 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.288 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.288 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.288 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "osd_support", 2024-03-19T02:05:37.288 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.288 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.289 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.289 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.289 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.289 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.289 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.289 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.289 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.289 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.289 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.289 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.289 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.289 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.289 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.289 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.289 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.290 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.290 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.290 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.290 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.290 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.290 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.290 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.290 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.290 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.290 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.290 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.290 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.290 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.290 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.291 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.291 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.291 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.291 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.291 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.291 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.291 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.291 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.291 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.291 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.291 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.291 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.291 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.291 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.291 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.292 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.292 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.292 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.292 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.292 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.292 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.292 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.292 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.292 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.292 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.292 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.292 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.292 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.292 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.292 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.293 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.293 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.293 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.293 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.293 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.293 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.293 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.293 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.293 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.293 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.293 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.293 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.293 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.293 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.293 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.294 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "pg_autoscaler", 2024-03-19T02:05:37.294 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.294 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.294 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.294 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.294 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.294 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.294 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.294 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.294 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.294 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.294 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.294 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.294 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.295 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.295 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.295 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.295 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.295 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.295 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.295 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.295 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.295 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.295 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.295 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.295 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.295 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.295 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.296 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.296 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.296 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.296 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.296 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.296 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.296 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.296 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.296 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.296 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.296 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.296 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.296 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.296 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.296 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.297 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.297 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.297 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.297 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.297 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.297 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.297 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.297 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.297 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.297 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.297 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.297 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.297 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.297 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.297 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.298 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.298 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.298 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.298 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.298 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.298 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.298 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.298 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.298 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.298 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.298 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.298 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.298 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.298 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.299 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.299 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.299 INFO:teuthology.orchestra.run.smithi099.stdout: "sleep_interval": { 2024-03-19T02:05:37.299 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "sleep_interval", 2024-03-19T02:05:37.299 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:37.299 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.299 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.299 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "60", 2024-03-19T02:05:37.299 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.299 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.299 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.299 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.299 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.299 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.299 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.300 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.300 INFO:teuthology.orchestra.run.smithi099.stdout: "threshold": { 2024-03-19T02:05:37.300 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "threshold", 2024-03-19T02:05:37.300 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "float", 2024-03-19T02:05:37.300 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.300 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.300 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "3.0", 2024-03-19T02:05:37.300 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "1.0", 2024-03-19T02:05:37.300 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.300 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.300 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "scaling threshold", 2024-03-19T02:05:37.300 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0", 2024-03-19T02:05:37.300 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.300 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.301 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.301 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.301 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.301 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.301 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "progress", 2024-03-19T02:05:37.301 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.301 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.301 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.301 INFO:teuthology.orchestra.run.smithi099.stdout: "allow_pg_recovery_event": { 2024-03-19T02:05:37.301 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "allow_pg_recovery_event", 2024-03-19T02:05:37.301 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.301 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.301 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.301 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.301 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.302 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.302 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.302 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "allow the module to show pg recovery progress", 2024-03-19T02:05:37.302 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.302 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.302 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.302 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.302 INFO:teuthology.orchestra.run.smithi099.stdout: "enabled": { 2024-03-19T02:05:37.302 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "enabled", 2024-03-19T02:05:37.302 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.302 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.302 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.302 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.302 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.303 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.303 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.303 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.303 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.303 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.303 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.303 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.303 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.303 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.303 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.303 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.303 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.303 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.303 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.303 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.304 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.304 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.304 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.304 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.304 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.304 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.304 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.304 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.304 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.304 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.304 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.304 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.304 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.304 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.304 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.305 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.305 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.305 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.305 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.305 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.305 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.305 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.305 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.305 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.305 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.305 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.305 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.305 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.305 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.305 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.306 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.306 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.306 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.306 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.306 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.306 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.306 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.306 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.306 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.306 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.306 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.306 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.306 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.306 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.307 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.307 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.307 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.307 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.307 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.307 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.307 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.307 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.307 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.307 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.307 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.307 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.307 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.307 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.307 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.308 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.308 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.308 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.308 INFO:teuthology.orchestra.run.smithi099.stdout: "max_completed_events": { 2024-03-19T02:05:37.308 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "max_completed_events", 2024-03-19T02:05:37.308 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.308 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.308 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.308 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "50", 2024-03-19T02:05:37.308 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.308 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.308 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.308 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "number of past completed events to remember", 2024-03-19T02:05:37.308 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.308 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.309 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.309 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.309 INFO:teuthology.orchestra.run.smithi099.stdout: "sleep_interval": { 2024-03-19T02:05:37.309 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "sleep_interval", 2024-03-19T02:05:37.309 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:37.309 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.309 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.309 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "5", 2024-03-19T02:05:37.309 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.309 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.309 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.309 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "how long the module is going to sleep", 2024-03-19T02:05:37.309 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.309 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.309 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.310 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.310 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.310 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.310 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.310 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "prometheus", 2024-03-19T02:05:37.310 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.310 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.310 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.310 INFO:teuthology.orchestra.run.smithi099.stdout: "cache": { 2024-03-19T02:05:37.310 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "cache", 2024-03-19T02:05:37.310 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.310 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.310 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.310 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.311 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.311 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.311 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.311 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.311 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.311 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.311 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.311 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.311 INFO:teuthology.orchestra.run.smithi099.stdout: "exclude_perf_counters": { 2024-03-19T02:05:37.311 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "exclude_perf_counters", 2024-03-19T02:05:37.311 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.311 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.311 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.311 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.312 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.312 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.312 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.312 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Do not include perf-counters in the metrics output", 2024-03-19T02:05:37.312 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.", 2024-03-19T02:05:37.312 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.312 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.312 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.312 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.312 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.312 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.312 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.312 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.312 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.312 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.313 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.313 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.313 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.313 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.313 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.313 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.313 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.313 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.313 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.313 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.313 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.313 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.313 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.313 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.313 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.314 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.314 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.314 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.314 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.314 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.314 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.314 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.314 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.314 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.314 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.314 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.314 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.314 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.314 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.315 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.315 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.315 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.315 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.315 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.315 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.315 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.315 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.315 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.315 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.315 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.315 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.315 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.315 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.315 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.316 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.316 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.316 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.316 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.316 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.316 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.316 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.316 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.316 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.316 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.316 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.316 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.316 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.316 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.317 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.317 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.317 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.317 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.317 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.317 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_stats_pools": { 2024-03-19T02:05:37.317 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "rbd_stats_pools", 2024-03-19T02:05:37.317 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.317 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.317 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.317 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.317 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.317 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.317 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.317 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.317 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.318 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.318 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.318 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.318 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_stats_pools_refresh_interval": { 2024-03-19T02:05:37.318 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "rbd_stats_pools_refresh_interval", 2024-03-19T02:05:37.318 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.318 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.318 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.318 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "300", 2024-03-19T02:05:37.318 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.318 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.318 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.318 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.318 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.319 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.319 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.319 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.319 INFO:teuthology.orchestra.run.smithi099.stdout: "scrape_interval": { 2024-03-19T02:05:37.319 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "scrape_interval", 2024-03-19T02:05:37.319 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "float", 2024-03-19T02:05:37.319 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.319 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.319 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "15.0", 2024-03-19T02:05:37.319 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.319 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.319 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.319 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.319 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.319 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.320 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.320 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.320 INFO:teuthology.orchestra.run.smithi099.stdout: "server_addr": { 2024-03-19T02:05:37.320 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "server_addr", 2024-03-19T02:05:37.320 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.320 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.320 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.320 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "::", 2024-03-19T02:05:37.320 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.320 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.320 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.320 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "the IPv4 or IPv6 address on which the module listens for HTTP requests", 2024-03-19T02:05:37.320 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.320 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.320 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.321 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.321 INFO:teuthology.orchestra.run.smithi099.stdout: "server_port": { 2024-03-19T02:05:37.321 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "server_port", 2024-03-19T02:05:37.321 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.321 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.321 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.321 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "9283", 2024-03-19T02:05:37.321 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.321 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.321 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.321 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "the port on which the module listens for HTTP requests", 2024-03-19T02:05:37.321 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.321 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.321 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.321 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.322 INFO:teuthology.orchestra.run.smithi099.stdout: "stale_cache_strategy": { 2024-03-19T02:05:37.322 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "stale_cache_strategy", 2024-03-19T02:05:37.322 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.322 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.322 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.322 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "log", 2024-03-19T02:05:37.322 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.322 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.322 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.322 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.322 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.322 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.322 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.322 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.322 INFO:teuthology.orchestra.run.smithi099.stdout: "standby_behaviour": { 2024-03-19T02:05:37.323 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "standby_behaviour", 2024-03-19T02:05:37.323 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.323 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.323 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.323 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "default", 2024-03-19T02:05:37.323 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.323 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.323 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.323 INFO:teuthology.orchestra.run.smithi099.stdout: "default", 2024-03-19T02:05:37.323 INFO:teuthology.orchestra.run.smithi099.stdout: "error" 2024-03-19T02:05:37.323 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.323 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.323 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.324 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.324 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.324 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.324 INFO:teuthology.orchestra.run.smithi099.stdout: "standby_error_status_code": { 2024-03-19T02:05:37.324 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "standby_error_status_code", 2024-03-19T02:05:37.324 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.324 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.324 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.324 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "500", 2024-03-19T02:05:37.324 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "400", 2024-03-19T02:05:37.324 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "599", 2024-03-19T02:05:37.324 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.324 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.324 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.325 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.325 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.325 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.325 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.325 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.325 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.325 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "rbd_support", 2024-03-19T02:05:37.325 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.325 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.325 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.325 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.325 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.325 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.325 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.326 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.326 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.326 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.326 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.326 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.326 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.326 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.326 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.326 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.326 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.326 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.326 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.326 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.326 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.326 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.327 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.327 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.327 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.327 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.327 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.327 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.327 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.327 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.327 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.327 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.327 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.327 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.327 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.327 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.328 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.328 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.328 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.328 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.328 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.328 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.328 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.328 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.328 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.328 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.328 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.328 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.328 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.328 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.328 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.329 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.329 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.329 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.329 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.329 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.329 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.329 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.329 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.329 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.329 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.329 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.329 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.329 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.329 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.329 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.330 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.330 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.330 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.330 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.330 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.330 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.330 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.330 INFO:teuthology.orchestra.run.smithi099.stdout: "max_concurrent_snap_create": { 2024-03-19T02:05:37.330 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "max_concurrent_snap_create", 2024-03-19T02:05:37.330 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.330 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.330 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.330 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "10", 2024-03-19T02:05:37.330 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.331 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.331 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.331 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.331 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.331 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.331 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.331 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.331 INFO:teuthology.orchestra.run.smithi099.stdout: "mirror_snapshot_schedule": { 2024-03-19T02:05:37.331 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "mirror_snapshot_schedule", 2024-03-19T02:05:37.331 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.331 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.331 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.331 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.331 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.331 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.332 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.332 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.332 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.332 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.332 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.332 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.332 INFO:teuthology.orchestra.run.smithi099.stdout: "trash_purge_schedule": { 2024-03-19T02:05:37.332 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "trash_purge_schedule", 2024-03-19T02:05:37.332 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.332 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.332 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.332 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.332 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.332 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.332 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.332 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.333 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.333 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.333 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.333 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.333 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.333 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.333 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.333 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "restful", 2024-03-19T02:05:37.333 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.333 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.333 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.333 INFO:teuthology.orchestra.run.smithi099.stdout: "enable_auth": { 2024-03-19T02:05:37.333 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "enable_auth", 2024-03-19T02:05:37.333 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.333 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.334 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.334 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.334 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.334 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.334 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.334 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.334 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.334 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.334 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.334 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.334 INFO:teuthology.orchestra.run.smithi099.stdout: "key_file": { 2024-03-19T02:05:37.334 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "key_file", 2024-03-19T02:05:37.334 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.334 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.335 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.335 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.335 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.335 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.335 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.335 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.335 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.335 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.335 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.335 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.335 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.335 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.335 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.335 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.335 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.336 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.336 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.336 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.336 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.336 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.336 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.336 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.336 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.336 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.336 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.336 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.336 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.336 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.336 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.336 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.337 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.337 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.337 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.337 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.337 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.337 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.337 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.337 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.337 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.337 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.337 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.337 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.337 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.337 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.337 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.338 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.338 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.338 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.338 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.338 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.338 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.338 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.338 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.338 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.338 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.338 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.338 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.338 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.338 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.339 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.339 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.339 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.339 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.339 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.339 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.339 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.339 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.339 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.339 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.339 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.339 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.339 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.339 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.339 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.340 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.340 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.340 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.340 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.340 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.340 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.340 INFO:teuthology.orchestra.run.smithi099.stdout: "server_addr": { 2024-03-19T02:05:37.340 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "server_addr", 2024-03-19T02:05:37.340 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.340 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.340 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.340 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.340 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.340 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.340 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.341 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.341 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.341 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.341 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.341 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.341 INFO:teuthology.orchestra.run.smithi099.stdout: "server_port": { 2024-03-19T02:05:37.341 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "server_port", 2024-03-19T02:05:37.341 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.341 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.341 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.341 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.341 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.341 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.341 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.341 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.342 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.342 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.342 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.342 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.342 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.342 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.342 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.342 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "rgw", 2024-03-19T02:05:37.342 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.342 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.342 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.342 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.342 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.342 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.343 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.343 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.343 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.343 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.343 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.343 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.343 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.343 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.343 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.343 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.343 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.343 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.343 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.343 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.343 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.344 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.344 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.344 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.344 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.344 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.344 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.344 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.344 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.344 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.344 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.344 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.344 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.344 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.344 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.344 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.345 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.345 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.345 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.345 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.345 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.345 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.345 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.345 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.345 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.345 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.345 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.345 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.345 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.345 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.345 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.346 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.346 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.346 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.346 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.346 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.346 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.346 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.346 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.346 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.346 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.346 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.346 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.346 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.346 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.346 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.347 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.347 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.347 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.347 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.347 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.347 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.347 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.347 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.347 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.347 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.347 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "rook", 2024-03-19T02:05:37.347 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.347 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.347 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.347 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.348 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.348 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.348 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.348 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.348 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.348 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.348 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.348 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.348 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.348 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.348 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.348 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.348 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.348 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.348 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.348 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.349 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.349 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.349 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.349 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.349 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.349 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.349 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.349 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.349 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.349 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.349 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.349 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.349 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.349 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.349 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.350 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.350 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.350 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.350 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.350 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.350 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.350 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.350 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.350 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.350 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.350 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.350 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.350 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.351 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.351 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.351 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.351 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.351 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.351 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.351 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.351 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.351 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.351 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.351 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.351 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.351 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.351 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.351 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.352 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.352 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.352 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.352 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.352 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.352 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.352 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.352 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.352 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.352 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.352 INFO:teuthology.orchestra.run.smithi099.stdout: "storage_class": { 2024-03-19T02:05:37.352 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "storage_class", 2024-03-19T02:05:37.352 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.352 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.352 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.353 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "local", 2024-03-19T02:05:37.353 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.353 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.353 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.353 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "storage class name for LSO-discovered PVs", 2024-03-19T02:05:37.353 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.353 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.353 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.353 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.353 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.353 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.353 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.353 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "selftest", 2024-03-19T02:05:37.353 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.353 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.354 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.354 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.354 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.354 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.354 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.354 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.354 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.354 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.354 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.354 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.354 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.354 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.354 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.354 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.355 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.355 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.355 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.355 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.355 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.355 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.355 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.355 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.355 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.355 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.355 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.355 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.355 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.355 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.355 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.356 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.356 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.356 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.356 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.356 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.356 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.356 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.356 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.356 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.356 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.356 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.356 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.356 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.356 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.356 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.357 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.357 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.357 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.357 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.357 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.357 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.357 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.357 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.357 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.357 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.357 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.357 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.357 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.357 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.357 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.358 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.358 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.358 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.358 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.358 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.358 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.358 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.358 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.358 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.358 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.358 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.358 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.358 INFO:teuthology.orchestra.run.smithi099.stdout: "roption1": { 2024-03-19T02:05:37.358 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "roption1", 2024-03-19T02:05:37.359 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.359 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.359 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.359 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.359 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.359 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.359 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.359 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.359 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.359 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.359 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.359 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.359 INFO:teuthology.orchestra.run.smithi099.stdout: "roption2": { 2024-03-19T02:05:37.359 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "roption2", 2024-03-19T02:05:37.359 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.359 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.360 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.360 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "xyz", 2024-03-19T02:05:37.360 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.360 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.360 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.360 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.360 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.360 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.360 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.360 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.360 INFO:teuthology.orchestra.run.smithi099.stdout: "rwoption1": { 2024-03-19T02:05:37.360 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "rwoption1", 2024-03-19T02:05:37.360 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.360 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.361 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.361 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.361 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.361 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.361 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.361 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.361 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.361 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.361 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.361 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.361 INFO:teuthology.orchestra.run.smithi099.stdout: "rwoption2": { 2024-03-19T02:05:37.361 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "rwoption2", 2024-03-19T02:05:37.361 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.361 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.361 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.361 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.362 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.362 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.362 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.362 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.362 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.362 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.362 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.362 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.362 INFO:teuthology.orchestra.run.smithi099.stdout: "rwoption3": { 2024-03-19T02:05:37.362 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "rwoption3", 2024-03-19T02:05:37.362 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "float", 2024-03-19T02:05:37.362 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.362 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.362 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.363 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.363 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.363 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.363 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.363 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.363 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.363 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.363 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.363 INFO:teuthology.orchestra.run.smithi099.stdout: "rwoption4": { 2024-03-19T02:05:37.363 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "rwoption4", 2024-03-19T02:05:37.363 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.363 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.363 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.363 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.363 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.364 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.364 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.364 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.364 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.364 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.364 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.364 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.364 INFO:teuthology.orchestra.run.smithi099.stdout: "rwoption5": { 2024-03-19T02:05:37.364 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "rwoption5", 2024-03-19T02:05:37.364 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.364 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.364 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.364 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.364 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.364 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.365 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.365 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.365 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.365 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.365 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.365 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.365 INFO:teuthology.orchestra.run.smithi099.stdout: "rwoption6": { 2024-03-19T02:05:37.365 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "rwoption6", 2024-03-19T02:05:37.365 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.365 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.365 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.365 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.365 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.365 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.366 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.366 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.366 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.366 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.366 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.366 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.366 INFO:teuthology.orchestra.run.smithi099.stdout: "rwoption7": { 2024-03-19T02:05:37.366 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "rwoption7", 2024-03-19T02:05:37.366 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.366 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.366 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.366 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.366 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "1", 2024-03-19T02:05:37.366 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "42", 2024-03-19T02:05:37.366 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.367 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.367 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.367 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.367 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.367 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.367 INFO:teuthology.orchestra.run.smithi099.stdout: "testkey": { 2024-03-19T02:05:37.367 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "testkey", 2024-03-19T02:05:37.367 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.367 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.367 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.367 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.367 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.367 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.367 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.367 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.368 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.368 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.368 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.368 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.368 INFO:teuthology.orchestra.run.smithi099.stdout: "testlkey": { 2024-03-19T02:05:37.368 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "testlkey", 2024-03-19T02:05:37.368 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.368 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.368 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.368 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.368 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.368 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.368 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.368 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.368 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.369 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.369 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.369 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.369 INFO:teuthology.orchestra.run.smithi099.stdout: "testnewline": { 2024-03-19T02:05:37.369 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "testnewline", 2024-03-19T02:05:37.369 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.369 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.369 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.369 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.369 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.369 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.369 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.369 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.369 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.369 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.370 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.370 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.370 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.370 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.370 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.370 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "snap_schedule", 2024-03-19T02:05:37.370 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.370 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.370 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.371 INFO:teuthology.orchestra.run.smithi099.stdout: "allow_m_granularity": { 2024-03-19T02:05:37.371 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "allow_m_granularity", 2024-03-19T02:05:37.371 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.371 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.371 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.371 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.371 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.371 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.371 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.371 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "allow minute scheduled snapshots", 2024-03-19T02:05:37.371 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.371 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.371 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.371 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.371 INFO:teuthology.orchestra.run.smithi099.stdout: "dump_on_update": { 2024-03-19T02:05:37.372 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "dump_on_update", 2024-03-19T02:05:37.372 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.372 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.372 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.372 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.372 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.372 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.372 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.372 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "dump database to debug log on update", 2024-03-19T02:05:37.372 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.372 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.372 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.372 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.372 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.372 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.373 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.373 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.373 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.373 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.373 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.373 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.373 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.373 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.373 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.373 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.373 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.373 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.373 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.373 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.373 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.374 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.374 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.374 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.374 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.374 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.374 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.374 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.374 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.374 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.374 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.374 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.374 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.374 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.374 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.374 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.375 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.375 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.375 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.375 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.375 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.375 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.375 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.375 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.375 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.375 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.375 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.375 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.375 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.375 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.375 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.376 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.376 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.376 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.376 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.376 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.376 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.376 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.376 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.376 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.376 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.376 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.376 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.376 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.376 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.377 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.377 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.377 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.377 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.377 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.377 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.377 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.377 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.377 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.377 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.377 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.377 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.377 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "stats", 2024-03-19T02:05:37.377 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.377 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.378 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.378 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.378 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.378 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.378 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.378 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.378 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.378 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.378 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.378 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.378 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.378 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.378 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.378 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.379 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.379 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.379 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.379 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.379 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.379 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.379 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.379 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.379 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.379 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.379 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.379 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.379 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.379 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.379 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.380 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.380 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.380 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.380 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.380 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.380 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.380 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.380 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.380 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.380 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.380 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.380 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.380 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.380 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.381 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.381 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.381 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.381 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.381 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.381 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.381 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.381 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.381 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.381 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.381 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.381 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.381 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.381 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.381 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.382 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.382 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.382 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.382 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.382 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.382 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.382 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.382 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.382 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.382 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.382 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.382 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.382 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.383 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.383 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.383 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.383 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "status", 2024-03-19T02:05:37.383 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.383 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.383 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.383 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.383 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.383 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.383 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.383 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.383 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.383 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.383 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.384 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.384 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.384 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.384 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.384 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.384 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.384 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.384 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.384 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.384 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.384 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.384 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.384 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.384 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.384 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.385 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.385 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.385 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.385 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.385 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.385 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.385 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.385 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.385 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.385 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.385 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.385 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.385 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.385 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.386 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.386 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.386 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.386 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.386 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.386 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.386 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.386 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.386 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.386 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.386 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.386 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.386 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.386 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.386 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.387 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.387 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.387 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.387 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.387 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.387 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.387 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.387 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.387 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.387 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.387 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.387 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.387 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.387 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.387 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.388 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.388 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.388 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.388 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.388 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.388 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.388 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "telegraf", 2024-03-19T02:05:37.388 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.388 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.388 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.388 INFO:teuthology.orchestra.run.smithi099.stdout: "address": { 2024-03-19T02:05:37.388 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "address", 2024-03-19T02:05:37.388 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.388 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.388 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.389 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "unixgram:///tmp/telegraf.sock", 2024-03-19T02:05:37.389 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.389 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.389 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.389 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.389 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.389 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.389 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.389 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.389 INFO:teuthology.orchestra.run.smithi099.stdout: "interval": { 2024-03-19T02:05:37.389 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "interval", 2024-03-19T02:05:37.389 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:37.389 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.389 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.389 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "15", 2024-03-19T02:05:37.389 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.390 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.390 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.390 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.390 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.390 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.390 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.390 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.390 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.390 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.390 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.390 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.390 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.390 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.391 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.391 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.391 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.391 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.391 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.391 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.391 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.391 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.391 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.391 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.391 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.391 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.391 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.391 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.391 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.392 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.392 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.392 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.392 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.392 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.392 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.392 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.392 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.392 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.392 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.392 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.392 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.392 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.392 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.392 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.393 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.393 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.393 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.393 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.393 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.393 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.393 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.393 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.393 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.393 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.393 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.393 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.393 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.393 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.393 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.394 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.394 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.394 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.394 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.394 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.394 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.394 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.394 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.394 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.394 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.394 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.394 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.394 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.394 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.394 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.395 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.395 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.395 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.395 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.395 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.395 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.395 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.395 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "telemetry", 2024-03-19T02:05:37.395 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.395 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.395 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.395 INFO:teuthology.orchestra.run.smithi099.stdout: "channel_basic": { 2024-03-19T02:05:37.395 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "channel_basic", 2024-03-19T02:05:37.395 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.396 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.396 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.396 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.396 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.396 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.396 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.396 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Share basic cluster information (size, version)", 2024-03-19T02:05:37.396 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.396 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.396 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.396 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.396 INFO:teuthology.orchestra.run.smithi099.stdout: "channel_crash": { 2024-03-19T02:05:37.396 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "channel_crash", 2024-03-19T02:05:37.396 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.396 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.397 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.397 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.397 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.397 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.397 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.397 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Share metadata about Ceph daemon crashes (version, stack straces, etc)", 2024-03-19T02:05:37.397 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.397 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.397 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.397 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.397 INFO:teuthology.orchestra.run.smithi099.stdout: "channel_device": { 2024-03-19T02:05:37.397 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "channel_device", 2024-03-19T02:05:37.397 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.397 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.397 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.398 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.398 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.398 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.398 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.398 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)", 2024-03-19T02:05:37.398 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.398 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.398 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.398 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.398 INFO:teuthology.orchestra.run.smithi099.stdout: "channel_ident": { 2024-03-19T02:05:37.398 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "channel_ident", 2024-03-19T02:05:37.398 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.398 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.398 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.398 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.399 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.399 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.399 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.399 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Share a user-provided description and/or contact email for the cluster", 2024-03-19T02:05:37.399 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.399 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.399 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.399 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.399 INFO:teuthology.orchestra.run.smithi099.stdout: "channel_perf": { 2024-03-19T02:05:37.399 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "channel_perf", 2024-03-19T02:05:37.399 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.399 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.399 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.399 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.399 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.400 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.400 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.400 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Share various performance metrics of a cluster", 2024-03-19T02:05:37.400 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.400 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.400 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.400 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.400 INFO:teuthology.orchestra.run.smithi099.stdout: "contact": { 2024-03-19T02:05:37.400 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "contact", 2024-03-19T02:05:37.400 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.400 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.400 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.400 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.400 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.400 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.401 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.401 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.401 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.401 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.401 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.401 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.401 INFO:teuthology.orchestra.run.smithi099.stdout: "description": { 2024-03-19T02:05:37.401 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "description", 2024-03-19T02:05:37.401 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.401 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.401 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.401 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.401 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.401 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.401 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.402 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.402 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.402 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.402 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.402 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.402 INFO:teuthology.orchestra.run.smithi099.stdout: "device_url": { 2024-03-19T02:05:37.402 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "device_url", 2024-03-19T02:05:37.402 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.402 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.402 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.402 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "https://telemetry.ceph.com/device", 2024-03-19T02:05:37.402 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.402 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.402 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.403 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.403 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.403 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.403 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.403 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.403 INFO:teuthology.orchestra.run.smithi099.stdout: "enabled": { 2024-03-19T02:05:37.403 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "enabled", 2024-03-19T02:05:37.403 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.403 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.403 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.403 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.403 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.403 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.403 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.403 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.404 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.404 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.404 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.404 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.404 INFO:teuthology.orchestra.run.smithi099.stdout: "interval": { 2024-03-19T02:05:37.404 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "interval", 2024-03-19T02:05:37.404 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.404 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.404 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.404 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "24", 2024-03-19T02:05:37.404 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "8", 2024-03-19T02:05:37.404 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.404 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.404 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.404 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.405 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.405 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.405 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.405 INFO:teuthology.orchestra.run.smithi099.stdout: "last_opt_revision": { 2024-03-19T02:05:37.405 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "last_opt_revision", 2024-03-19T02:05:37.405 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.405 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.405 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.405 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "1", 2024-03-19T02:05:37.405 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.405 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.405 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.405 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.405 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.405 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.406 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.406 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.406 INFO:teuthology.orchestra.run.smithi099.stdout: "leaderboard": { 2024-03-19T02:05:37.406 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "leaderboard", 2024-03-19T02:05:37.406 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.406 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.406 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.406 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.406 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.406 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.406 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.406 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.406 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.406 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.407 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.407 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.407 INFO:teuthology.orchestra.run.smithi099.stdout: "leaderboard_description": { 2024-03-19T02:05:37.407 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "leaderboard_description", 2024-03-19T02:05:37.407 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.407 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.407 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.407 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.407 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.407 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.407 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.407 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.407 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.407 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.407 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.408 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.408 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.408 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.408 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.408 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.408 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.408 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.408 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.408 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.408 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.408 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.408 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.408 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.408 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.408 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.409 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.409 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.409 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.409 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.409 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.409 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.409 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.409 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.409 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.409 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.409 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.409 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.409 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.409 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.410 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.410 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.410 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.410 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.410 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.410 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.410 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.410 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.410 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.410 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.410 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.410 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.410 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.410 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.410 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.411 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.411 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.411 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.411 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.411 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.411 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.411 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.411 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.411 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.411 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.411 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.411 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.411 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.411 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.411 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.412 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.412 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.412 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.412 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.412 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.412 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.412 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.412 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.412 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.412 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.412 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.412 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.412 INFO:teuthology.orchestra.run.smithi099.stdout: "organization": { 2024-03-19T02:05:37.412 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "organization", 2024-03-19T02:05:37.412 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.413 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.413 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.413 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.413 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.413 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.413 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.413 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.413 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.413 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.413 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.413 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.413 INFO:teuthology.orchestra.run.smithi099.stdout: "proxy": { 2024-03-19T02:05:37.413 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "proxy", 2024-03-19T02:05:37.413 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.413 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.414 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.414 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.414 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.414 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.414 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.414 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.414 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.414 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.414 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.414 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.414 INFO:teuthology.orchestra.run.smithi099.stdout: "url": { 2024-03-19T02:05:37.414 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "url", 2024-03-19T02:05:37.414 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.414 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.414 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.415 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "https://telemetry.ceph.com/report", 2024-03-19T02:05:37.415 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.415 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.415 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.415 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.415 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.415 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.415 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.415 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.415 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.415 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.415 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.415 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "test_orchestrator", 2024-03-19T02:05:37.415 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.415 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.416 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.416 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.416 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.416 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.416 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.416 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.416 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.416 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.416 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.416 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.416 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.416 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.416 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.416 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.417 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.417 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.417 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.417 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.417 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.417 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.417 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.417 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.417 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.417 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.417 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.417 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.417 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.417 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.417 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.418 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.418 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.418 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.418 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.418 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.418 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.418 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.418 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.418 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.418 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.418 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.418 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.418 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.418 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.419 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.419 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.419 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.419 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.419 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.419 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.419 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.419 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.419 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.419 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.419 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.419 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.419 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.419 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.419 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.420 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.420 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.420 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.420 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.420 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.420 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.420 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.420 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.420 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.420 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.420 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.420 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.420 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.420 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.420 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.421 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.421 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "volumes", 2024-03-19T02:05:37.421 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.421 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.421 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.421 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.421 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.421 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.421 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.421 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.421 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.421 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.421 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.421 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.421 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.422 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.422 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.422 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.422 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.422 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.422 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.422 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.422 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.422 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.422 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.422 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.422 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.422 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.422 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.422 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.423 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.423 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.423 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.423 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.423 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.423 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.423 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.423 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.423 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.423 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.423 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.423 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.423 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.423 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.423 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.424 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.424 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.424 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.424 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.424 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.424 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.424 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.424 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.424 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.424 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.424 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.424 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.424 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.424 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.424 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.425 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.425 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.425 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.425 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.425 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.425 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.425 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.425 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.425 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.425 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.425 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.425 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.425 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.425 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.426 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.426 INFO:teuthology.orchestra.run.smithi099.stdout: "max_concurrent_clones": { 2024-03-19T02:05:37.426 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "max_concurrent_clones", 2024-03-19T02:05:37.426 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.426 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.426 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.426 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "4", 2024-03-19T02:05:37.426 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.426 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.426 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.426 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Number of asynchronous cloner threads", 2024-03-19T02:05:37.426 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.426 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.426 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.426 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.427 INFO:teuthology.orchestra.run.smithi099.stdout: "periodic_async_work": { 2024-03-19T02:05:37.427 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "periodic_async_work", 2024-03-19T02:05:37.427 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.427 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.427 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.427 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.427 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.427 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.427 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.427 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Periodically check for async work", 2024-03-19T02:05:37.427 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.427 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.427 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.427 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.427 INFO:teuthology.orchestra.run.smithi099.stdout: "snapshot_clone_delay": { 2024-03-19T02:05:37.428 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "snapshot_clone_delay", 2024-03-19T02:05:37.428 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.428 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.428 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.428 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "0", 2024-03-19T02:05:37.428 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.428 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.428 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.428 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Delay clone begin operation by snapshot_clone_delay seconds", 2024-03-19T02:05:37.428 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.428 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.428 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.428 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.428 INFO:teuthology.orchestra.run.smithi099.stdout: "snapshot_clone_no_wait": { 2024-03-19T02:05:37.428 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "snapshot_clone_no_wait", 2024-03-19T02:05:37.429 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.429 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.429 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.429 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.429 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.429 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.429 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.429 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Reject subvolume clone request when cloner threads are busy", 2024-03-19T02:05:37.429 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.429 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.429 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.429 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.429 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.429 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.430 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.430 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "zabbix", 2024-03-19T02:05:37.430 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.430 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.430 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.430 INFO:teuthology.orchestra.run.smithi099.stdout: "discovery_interval": { 2024-03-19T02:05:37.430 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "discovery_interval", 2024-03-19T02:05:37.430 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "uint", 2024-03-19T02:05:37.430 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.430 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.430 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "100", 2024-03-19T02:05:37.430 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.430 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.430 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.430 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.431 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.431 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.431 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.431 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.431 INFO:teuthology.orchestra.run.smithi099.stdout: "identifier": { 2024-03-19T02:05:37.431 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "identifier", 2024-03-19T02:05:37.431 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.431 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.431 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.431 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.431 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.431 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.431 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.431 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.432 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.432 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.432 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.432 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.432 INFO:teuthology.orchestra.run.smithi099.stdout: "interval": { 2024-03-19T02:05:37.432 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "interval", 2024-03-19T02:05:37.432 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:37.432 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.432 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.432 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "60", 2024-03-19T02:05:37.432 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.432 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.432 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.432 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.432 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.433 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.433 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.433 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.433 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.433 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.433 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.433 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.433 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.433 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.433 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.433 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.433 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.433 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.433 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.433 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.434 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.434 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.434 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.434 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.434 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.434 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.434 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.434 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.434 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.434 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.434 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.434 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.434 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.434 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.435 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.435 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.435 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.435 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.435 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.435 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.435 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.435 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.435 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.435 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.435 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.435 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.435 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.435 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.435 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.436 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.436 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.436 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.436 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.436 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.436 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.436 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.436 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.436 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.436 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.436 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.436 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.436 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.436 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.437 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.437 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.437 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.437 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.437 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.437 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.437 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.437 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.437 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.437 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.437 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.437 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.437 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.437 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.437 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.437 INFO:teuthology.orchestra.run.smithi099.stdout: "zabbix_host": { 2024-03-19T02:05:37.438 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "zabbix_host", 2024-03-19T02:05:37.438 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.438 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.438 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.438 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.438 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.438 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.438 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.438 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.438 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.438 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.438 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.438 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.438 INFO:teuthology.orchestra.run.smithi099.stdout: "zabbix_port": { 2024-03-19T02:05:37.439 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "zabbix_port", 2024-03-19T02:05:37.439 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.439 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.439 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.439 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "10051", 2024-03-19T02:05:37.439 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.439 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.439 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.439 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.439 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.439 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.439 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.439 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.439 INFO:teuthology.orchestra.run.smithi099.stdout: "zabbix_sender": { 2024-03-19T02:05:37.439 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "zabbix_sender", 2024-03-19T02:05:37.440 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.440 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.440 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.440 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "/usr/bin/zabbix_sender", 2024-03-19T02:05:37.440 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.440 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.440 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.440 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.440 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.440 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.440 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.440 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.440 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.440 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.440 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-03-19T02:05:37.441 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.441 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.441 INFO:teuthology.orchestra.run.smithi099.stdout: "modules": [ 2024-03-19T02:05:37.441 INFO:teuthology.orchestra.run.smithi099.stdout: "cephadm", 2024-03-19T02:05:37.441 INFO:teuthology.orchestra.run.smithi099.stdout: "dashboard", 2024-03-19T02:05:37.441 INFO:teuthology.orchestra.run.smithi099.stdout: "iostat", 2024-03-19T02:05:37.441 INFO:teuthology.orchestra.run.smithi099.stdout: "nfs", 2024-03-19T02:05:37.441 INFO:teuthology.orchestra.run.smithi099.stdout: "restful" 2024-03-19T02:05:37.441 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.441 INFO:teuthology.orchestra.run.smithi099.stdout: "available_modules": [ 2024-03-19T02:05:37.441 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.441 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "alerts", 2024-03-19T02:05:37.441 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.441 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.441 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.442 INFO:teuthology.orchestra.run.smithi099.stdout: "interval": { 2024-03-19T02:05:37.442 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "interval", 2024-03-19T02:05:37.442 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:37.442 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.442 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.442 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "60", 2024-03-19T02:05:37.442 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.442 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.442 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.442 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "How frequently to reexamine health status", 2024-03-19T02:05:37.442 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.442 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.442 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.442 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.443 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.443 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.443 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.443 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.443 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.443 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.443 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.443 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.443 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.443 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.443 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.443 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.443 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.443 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.444 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.444 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.444 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.444 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.444 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.444 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.444 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.444 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.444 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.444 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.444 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.444 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.444 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.444 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.444 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.445 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.445 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.445 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.445 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.445 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.445 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.445 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.445 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.445 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.445 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.445 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.445 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.445 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.445 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.445 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.446 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.446 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.446 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.446 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.446 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.446 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.446 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.446 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.446 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.446 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.446 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.446 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.446 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.446 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.447 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.447 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.447 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.447 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.447 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.447 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.447 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.447 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.447 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.447 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.447 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.447 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.447 INFO:teuthology.orchestra.run.smithi099.stdout: "smtp_destination": { 2024-03-19T02:05:37.447 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "smtp_destination", 2024-03-19T02:05:37.447 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.448 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.448 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.448 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.448 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.448 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.448 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.448 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Email address to send alerts to", 2024-03-19T02:05:37.448 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.448 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.448 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.448 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.448 INFO:teuthology.orchestra.run.smithi099.stdout: "smtp_from_name": { 2024-03-19T02:05:37.448 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "smtp_from_name", 2024-03-19T02:05:37.448 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.448 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.449 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.449 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "Ceph", 2024-03-19T02:05:37.449 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.449 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.449 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.449 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Email From: name", 2024-03-19T02:05:37.449 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.449 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.449 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.449 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.449 INFO:teuthology.orchestra.run.smithi099.stdout: "smtp_host": { 2024-03-19T02:05:37.449 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "smtp_host", 2024-03-19T02:05:37.449 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.449 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.450 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.450 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.450 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.450 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.450 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.450 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "SMTP server", 2024-03-19T02:05:37.450 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.450 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.450 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.450 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.450 INFO:teuthology.orchestra.run.smithi099.stdout: "smtp_password": { 2024-03-19T02:05:37.450 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "smtp_password", 2024-03-19T02:05:37.450 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.450 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.451 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.451 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.451 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.451 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.451 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.451 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Password to authenticate with", 2024-03-19T02:05:37.451 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.451 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.451 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.451 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.451 INFO:teuthology.orchestra.run.smithi099.stdout: "smtp_port": { 2024-03-19T02:05:37.451 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "smtp_port", 2024-03-19T02:05:37.451 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.451 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.451 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.452 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "465", 2024-03-19T02:05:37.452 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.452 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.452 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.452 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "SMTP port", 2024-03-19T02:05:37.452 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.452 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.452 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.452 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.452 INFO:teuthology.orchestra.run.smithi099.stdout: "smtp_sender": { 2024-03-19T02:05:37.452 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "smtp_sender", 2024-03-19T02:05:37.452 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.452 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.452 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.452 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.453 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.453 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.453 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.453 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "SMTP envelope sender", 2024-03-19T02:05:37.453 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.453 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.453 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.453 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.453 INFO:teuthology.orchestra.run.smithi099.stdout: "smtp_ssl": { 2024-03-19T02:05:37.453 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "smtp_ssl", 2024-03-19T02:05:37.453 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.453 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.453 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.453 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.453 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.454 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.454 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.454 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Use SSL to connect to SMTP server", 2024-03-19T02:05:37.454 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.454 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.454 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.454 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.454 INFO:teuthology.orchestra.run.smithi099.stdout: "smtp_user": { 2024-03-19T02:05:37.454 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "smtp_user", 2024-03-19T02:05:37.454 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.454 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.454 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.454 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.454 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.454 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.455 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.455 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "User to authenticate as", 2024-03-19T02:05:37.455 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.455 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.455 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.455 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.455 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.455 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.455 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.455 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "balancer", 2024-03-19T02:05:37.455 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.455 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.455 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.455 INFO:teuthology.orchestra.run.smithi099.stdout: "active": { 2024-03-19T02:05:37.455 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "active", 2024-03-19T02:05:37.456 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.456 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.456 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.456 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.456 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.456 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.456 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.456 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "automatically balance PGs across cluster", 2024-03-19T02:05:37.456 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.456 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.456 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.456 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.456 INFO:teuthology.orchestra.run.smithi099.stdout: "begin_time": { 2024-03-19T02:05:37.456 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "begin_time", 2024-03-19T02:05:37.456 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.457 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.457 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.457 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "0000", 2024-03-19T02:05:37.457 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.457 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.457 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.457 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "beginning time of day to automatically balance", 2024-03-19T02:05:37.457 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "This is a time of day in the format HHMM.", 2024-03-19T02:05:37.457 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.457 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.457 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.457 INFO:teuthology.orchestra.run.smithi099.stdout: "begin_weekday": { 2024-03-19T02:05:37.457 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "begin_weekday", 2024-03-19T02:05:37.457 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "uint", 2024-03-19T02:05:37.458 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.458 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.458 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "0", 2024-03-19T02:05:37.458 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "0", 2024-03-19T02:05:37.458 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "6", 2024-03-19T02:05:37.458 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.458 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Restrict automatic balancing to this day of the week or later", 2024-03-19T02:05:37.458 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "0 = Sunday, 1 = Monday, etc.", 2024-03-19T02:05:37.458 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.458 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.458 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.458 INFO:teuthology.orchestra.run.smithi099.stdout: "crush_compat_max_iterations": { 2024-03-19T02:05:37.458 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "crush_compat_max_iterations", 2024-03-19T02:05:37.458 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "uint", 2024-03-19T02:05:37.459 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.459 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.459 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "25", 2024-03-19T02:05:37.459 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "1", 2024-03-19T02:05:37.459 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "250", 2024-03-19T02:05:37.459 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.459 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "maximum number of iterations to attempt optimization", 2024-03-19T02:05:37.459 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.459 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.459 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.459 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.459 INFO:teuthology.orchestra.run.smithi099.stdout: "crush_compat_metrics": { 2024-03-19T02:05:37.459 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "crush_compat_metrics", 2024-03-19T02:05:37.459 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.459 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.460 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.460 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "pgs,objects,bytes", 2024-03-19T02:05:37.460 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.460 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.460 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.460 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "metrics with which to calculate OSD utilization", 2024-03-19T02:05:37.460 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.", 2024-03-19T02:05:37.460 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.460 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.460 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.460 INFO:teuthology.orchestra.run.smithi099.stdout: "crush_compat_step": { 2024-03-19T02:05:37.460 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "crush_compat_step", 2024-03-19T02:05:37.460 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "float", 2024-03-19T02:05:37.460 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.461 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.461 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "0.5", 2024-03-19T02:05:37.461 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "0.001", 2024-03-19T02:05:37.461 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "0.999", 2024-03-19T02:05:37.461 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.461 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "aggressiveness of optimization", 2024-03-19T02:05:37.461 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": ".99 is very aggressive, .01 is less aggressive", 2024-03-19T02:05:37.461 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.461 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.461 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.461 INFO:teuthology.orchestra.run.smithi099.stdout: "end_time": { 2024-03-19T02:05:37.461 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "end_time", 2024-03-19T02:05:37.461 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.461 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.461 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.462 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "2359", 2024-03-19T02:05:37.462 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.462 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.462 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.462 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "ending time of day to automatically balance", 2024-03-19T02:05:37.462 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "This is a time of day in the format HHMM.", 2024-03-19T02:05:37.462 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.462 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.462 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.462 INFO:teuthology.orchestra.run.smithi099.stdout: "end_weekday": { 2024-03-19T02:05:37.462 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "end_weekday", 2024-03-19T02:05:37.462 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "uint", 2024-03-19T02:05:37.462 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.463 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.463 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "0", 2024-03-19T02:05:37.463 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "0", 2024-03-19T02:05:37.463 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "6", 2024-03-19T02:05:37.463 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.463 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Restrict automatic balancing to days of the week earlier than this", 2024-03-19T02:05:37.463 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "0 = Sunday, 1 = Monday, etc.", 2024-03-19T02:05:37.463 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.463 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.463 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.463 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.463 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.463 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.463 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.463 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.464 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.464 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.464 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.464 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.464 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.464 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.464 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.464 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.464 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.464 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.464 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.464 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.464 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.464 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.465 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.465 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.465 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.465 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.465 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.465 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.465 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.465 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.465 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.465 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.465 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.465 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.465 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.465 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.465 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.466 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.466 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.466 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.466 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.466 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.466 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.466 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.466 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.466 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.466 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.466 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.466 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.466 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.466 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.466 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.467 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.467 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.467 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.467 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.467 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.467 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.467 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.467 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.467 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.467 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.467 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.467 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.467 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.467 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.467 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.468 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.468 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.468 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.468 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.468 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.468 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.468 INFO:teuthology.orchestra.run.smithi099.stdout: "min_score": { 2024-03-19T02:05:37.468 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "min_score", 2024-03-19T02:05:37.468 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "float", 2024-03-19T02:05:37.468 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.468 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.468 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "0", 2024-03-19T02:05:37.468 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.468 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.468 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.469 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "minimum score, below which no optimization is attempted", 2024-03-19T02:05:37.469 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.469 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.469 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.469 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.469 INFO:teuthology.orchestra.run.smithi099.stdout: "mode": { 2024-03-19T02:05:37.469 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "mode", 2024-03-19T02:05:37.469 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.469 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.469 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.469 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "upmap", 2024-03-19T02:05:37.469 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.469 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.469 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.469 INFO:teuthology.orchestra.run.smithi099.stdout: "crush-compat", 2024-03-19T02:05:37.470 INFO:teuthology.orchestra.run.smithi099.stdout: "none", 2024-03-19T02:05:37.470 INFO:teuthology.orchestra.run.smithi099.stdout: "read", 2024-03-19T02:05:37.470 INFO:teuthology.orchestra.run.smithi099.stdout: "upmap", 2024-03-19T02:05:37.470 INFO:teuthology.orchestra.run.smithi099.stdout: "upmap-read" 2024-03-19T02:05:37.470 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.470 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Balancer mode", 2024-03-19T02:05:37.470 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.470 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.470 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.470 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.470 INFO:teuthology.orchestra.run.smithi099.stdout: "pool_ids": { 2024-03-19T02:05:37.470 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "pool_ids", 2024-03-19T02:05:37.470 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.470 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.470 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.471 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.471 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.471 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.471 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.471 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "pools which the automatic balancing will be limited to", 2024-03-19T02:05:37.471 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.471 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.471 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.471 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.471 INFO:teuthology.orchestra.run.smithi099.stdout: "sleep_interval": { 2024-03-19T02:05:37.471 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "sleep_interval", 2024-03-19T02:05:37.471 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:37.471 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.472 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.472 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "60", 2024-03-19T02:05:37.472 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.472 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.472 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.472 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "how frequently to wake up and attempt optimization", 2024-03-19T02:05:37.472 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.472 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.472 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.472 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.472 INFO:teuthology.orchestra.run.smithi099.stdout: "upmap_max_deviation": { 2024-03-19T02:05:37.472 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "upmap_max_deviation", 2024-03-19T02:05:37.472 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.472 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.472 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.473 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "5", 2024-03-19T02:05:37.473 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "1", 2024-03-19T02:05:37.473 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.473 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.473 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "deviation below which no optimization is attempted", 2024-03-19T02:05:37.473 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "If the number of PGs are within this count then no optimization is attempted", 2024-03-19T02:05:37.473 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.473 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.473 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.473 INFO:teuthology.orchestra.run.smithi099.stdout: "upmap_max_optimizations": { 2024-03-19T02:05:37.473 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "upmap_max_optimizations", 2024-03-19T02:05:37.473 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "uint", 2024-03-19T02:05:37.473 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.473 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.474 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "10", 2024-03-19T02:05:37.474 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.474 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.474 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.474 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "maximum upmap optimizations to make per attempt", 2024-03-19T02:05:37.474 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.474 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.474 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.474 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.474 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.474 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.474 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.474 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "cephadm", 2024-03-19T02:05:37.474 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.475 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.475 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.475 INFO:teuthology.orchestra.run.smithi099.stdout: "agent_down_multiplier": { 2024-03-19T02:05:37.475 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "agent_down_multiplier", 2024-03-19T02:05:37.475 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "float", 2024-03-19T02:05:37.475 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.475 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.475 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "3.0", 2024-03-19T02:05:37.475 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.475 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.475 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.475 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Multiplied by agent refresh rate to calculate how long agent must not report before being marked down", 2024-03-19T02:05:37.475 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.475 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.476 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.476 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.476 INFO:teuthology.orchestra.run.smithi099.stdout: "agent_refresh_rate": { 2024-03-19T02:05:37.476 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "agent_refresh_rate", 2024-03-19T02:05:37.476 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:37.476 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.476 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.476 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "20", 2024-03-19T02:05:37.476 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.476 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.476 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.476 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "How often agent on each host will try to gather and send metadata", 2024-03-19T02:05:37.476 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.476 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.476 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.477 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.477 INFO:teuthology.orchestra.run.smithi099.stdout: "agent_starting_port": { 2024-03-19T02:05:37.477 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "agent_starting_port", 2024-03-19T02:05:37.477 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.477 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.477 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.477 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "4721", 2024-03-19T02:05:37.477 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.477 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.477 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.477 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)", 2024-03-19T02:05:37.477 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.477 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.477 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.478 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.478 INFO:teuthology.orchestra.run.smithi099.stdout: "allow_ptrace": { 2024-03-19T02:05:37.478 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "allow_ptrace", 2024-03-19T02:05:37.478 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.478 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.478 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.478 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.478 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.478 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.478 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.478 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "allow SYS_PTRACE capability on ceph containers", 2024-03-19T02:05:37.478 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.", 2024-03-19T02:05:37.478 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.478 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.479 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.479 INFO:teuthology.orchestra.run.smithi099.stdout: "autotune_interval": { 2024-03-19T02:05:37.479 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "autotune_interval", 2024-03-19T02:05:37.479 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:37.479 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.479 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.479 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "600", 2024-03-19T02:05:37.479 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.479 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.479 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.479 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "how frequently to autotune daemon memory", 2024-03-19T02:05:37.479 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.479 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.479 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.479 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.480 INFO:teuthology.orchestra.run.smithi099.stdout: "autotune_memory_target_ratio": { 2024-03-19T02:05:37.480 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "autotune_memory_target_ratio", 2024-03-19T02:05:37.480 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "float", 2024-03-19T02:05:37.480 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.480 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.480 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "0.7", 2024-03-19T02:05:37.480 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.480 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.480 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.480 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "ratio of total system memory to divide amongst autotuned daemons", 2024-03-19T02:05:37.480 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.480 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.480 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.480 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.481 INFO:teuthology.orchestra.run.smithi099.stdout: "cephadm_log_destination": { 2024-03-19T02:05:37.481 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "cephadm_log_destination", 2024-03-19T02:05:37.481 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.481 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.481 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.481 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.481 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.481 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.481 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.481 INFO:teuthology.orchestra.run.smithi099.stdout: "file", 2024-03-19T02:05:37.481 INFO:teuthology.orchestra.run.smithi099.stdout: "file,syslog", 2024-03-19T02:05:37.481 INFO:teuthology.orchestra.run.smithi099.stdout: "syslog" 2024-03-19T02:05:37.481 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.481 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Destination for cephadm command's persistent logging", 2024-03-19T02:05:37.481 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.482 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.482 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.482 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.482 INFO:teuthology.orchestra.run.smithi099.stdout: "cgroups_split": { 2024-03-19T02:05:37.482 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "cgroups_split", 2024-03-19T02:05:37.482 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.482 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.482 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.482 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.482 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.482 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.482 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.482 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Pass --cgroups=split when cephadm creates containers (currently podman only)", 2024-03-19T02:05:37.482 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.483 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.483 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.483 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.483 INFO:teuthology.orchestra.run.smithi099.stdout: "config_checks_enabled": { 2024-03-19T02:05:37.483 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "config_checks_enabled", 2024-03-19T02:05:37.483 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.483 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.483 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.483 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.483 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.483 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.483 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.483 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Enable or disable the cephadm configuration analysis", 2024-03-19T02:05:37.483 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.483 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.484 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.484 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.484 INFO:teuthology.orchestra.run.smithi099.stdout: "config_dashboard": { 2024-03-19T02:05:37.484 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "config_dashboard", 2024-03-19T02:05:37.484 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.484 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.484 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.484 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.484 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.484 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.484 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.484 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "manage configs like API endpoints in Dashboard.", 2024-03-19T02:05:37.484 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.484 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.484 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.485 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.485 INFO:teuthology.orchestra.run.smithi099.stdout: "container_image_alertmanager": { 2024-03-19T02:05:37.485 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "container_image_alertmanager", 2024-03-19T02:05:37.485 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.485 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.485 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.485 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "quay.io/prometheus/alertmanager:v0.25.0", 2024-03-19T02:05:37.485 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.485 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.485 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.485 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Prometheus container image", 2024-03-19T02:05:37.485 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.485 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.485 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.485 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.486 INFO:teuthology.orchestra.run.smithi099.stdout: "container_image_base": { 2024-03-19T02:05:37.486 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "container_image_base", 2024-03-19T02:05:37.486 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.486 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.486 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.486 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "quay.io/ceph/ceph", 2024-03-19T02:05:37.486 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.486 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.486 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.486 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Container image name, without the tag", 2024-03-19T02:05:37.486 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.486 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.486 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.486 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.486 INFO:teuthology.orchestra.run.smithi099.stdout: "container_image_elasticsearch": { 2024-03-19T02:05:37.487 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "container_image_elasticsearch", 2024-03-19T02:05:37.487 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.487 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.487 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.487 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "quay.io/omrizeneva/elasticsearch:6.8.23", 2024-03-19T02:05:37.487 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.487 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.487 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.487 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "elasticsearch container image", 2024-03-19T02:05:37.487 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.487 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.487 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.487 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.488 INFO:teuthology.orchestra.run.smithi099.stdout: "container_image_grafana": { 2024-03-19T02:05:37.488 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "container_image_grafana", 2024-03-19T02:05:37.488 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.488 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.488 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.488 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "quay.io/ceph/ceph-grafana:9.4.12", 2024-03-19T02:05:37.488 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.488 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.488 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.488 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Prometheus container image", 2024-03-19T02:05:37.488 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.488 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.488 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.488 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.488 INFO:teuthology.orchestra.run.smithi099.stdout: "container_image_haproxy": { 2024-03-19T02:05:37.489 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "container_image_haproxy", 2024-03-19T02:05:37.489 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.489 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.489 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.489 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "quay.io/ceph/haproxy:2.3", 2024-03-19T02:05:37.489 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.489 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.489 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.489 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "HAproxy container image", 2024-03-19T02:05:37.489 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.489 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.489 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.489 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.489 INFO:teuthology.orchestra.run.smithi099.stdout: "container_image_jaeger_agent": { 2024-03-19T02:05:37.489 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "container_image_jaeger_agent", 2024-03-19T02:05:37.490 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.490 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.490 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.490 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "quay.io/jaegertracing/jaeger-agent:1.29", 2024-03-19T02:05:37.490 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.490 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.490 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.490 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Jaeger agent container image", 2024-03-19T02:05:37.490 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.490 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.490 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.490 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.490 INFO:teuthology.orchestra.run.smithi099.stdout: "container_image_jaeger_collector": { 2024-03-19T02:05:37.490 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "container_image_jaeger_collector", 2024-03-19T02:05:37.491 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.491 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.491 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.491 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "quay.io/jaegertracing/jaeger-collector:1.29", 2024-03-19T02:05:37.491 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.491 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.491 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.491 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Jaeger collector container image", 2024-03-19T02:05:37.491 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.491 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.491 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.491 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.491 INFO:teuthology.orchestra.run.smithi099.stdout: "container_image_jaeger_query": { 2024-03-19T02:05:37.491 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "container_image_jaeger_query", 2024-03-19T02:05:37.492 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.492 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.492 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.492 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "quay.io/jaegertracing/jaeger-query:1.29", 2024-03-19T02:05:37.492 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.492 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.492 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.492 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Jaeger query container image", 2024-03-19T02:05:37.492 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.492 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.492 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.492 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.492 INFO:teuthology.orchestra.run.smithi099.stdout: "container_image_keepalived": { 2024-03-19T02:05:37.492 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "container_image_keepalived", 2024-03-19T02:05:37.492 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.493 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.493 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.493 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "quay.io/ceph/keepalived:2.2.4", 2024-03-19T02:05:37.493 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.493 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.493 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.493 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Keepalived container image", 2024-03-19T02:05:37.493 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.493 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.493 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.493 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.493 INFO:teuthology.orchestra.run.smithi099.stdout: "container_image_loki": { 2024-03-19T02:05:37.493 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "container_image_loki", 2024-03-19T02:05:37.493 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.493 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.494 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.494 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "docker.io/grafana/loki:2.4.0", 2024-03-19T02:05:37.494 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.494 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.494 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.494 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Loki container image", 2024-03-19T02:05:37.494 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.494 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.494 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.494 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.494 INFO:teuthology.orchestra.run.smithi099.stdout: "container_image_node_exporter": { 2024-03-19T02:05:37.494 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "container_image_node_exporter", 2024-03-19T02:05:37.494 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.494 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.495 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.495 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "quay.io/prometheus/node-exporter:v1.5.0", 2024-03-19T02:05:37.495 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.495 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.495 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.495 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Prometheus container image", 2024-03-19T02:05:37.495 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.495 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.495 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.495 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.495 INFO:teuthology.orchestra.run.smithi099.stdout: "container_image_nvmeof": { 2024-03-19T02:05:37.495 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "container_image_nvmeof", 2024-03-19T02:05:37.495 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.495 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.495 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.496 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "quay.io/ceph/nvmeof:1.0.0", 2024-03-19T02:05:37.496 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.496 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.496 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.496 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Nvme-of container image", 2024-03-19T02:05:37.496 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.496 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.496 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.496 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.496 INFO:teuthology.orchestra.run.smithi099.stdout: "container_image_prometheus": { 2024-03-19T02:05:37.496 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "container_image_prometheus", 2024-03-19T02:05:37.496 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.496 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.496 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.496 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "quay.io/prometheus/prometheus:v2.43.0", 2024-03-19T02:05:37.497 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.497 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.497 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.497 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Prometheus container image", 2024-03-19T02:05:37.497 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.497 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.497 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.497 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.497 INFO:teuthology.orchestra.run.smithi099.stdout: "container_image_promtail": { 2024-03-19T02:05:37.497 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "container_image_promtail", 2024-03-19T02:05:37.497 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.497 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.497 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.497 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "docker.io/grafana/promtail:2.4.0", 2024-03-19T02:05:37.497 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.498 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.498 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.498 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Promtail container image", 2024-03-19T02:05:37.498 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.498 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.498 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.498 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.498 INFO:teuthology.orchestra.run.smithi099.stdout: "container_image_snmp_gateway": { 2024-03-19T02:05:37.498 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "container_image_snmp_gateway", 2024-03-19T02:05:37.498 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.498 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.498 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.498 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "docker.io/maxwo/snmp-notifier:v1.2.1", 2024-03-19T02:05:37.498 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.499 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.499 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.499 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "SNMP Gateway container image", 2024-03-19T02:05:37.499 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.499 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.499 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.499 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.499 INFO:teuthology.orchestra.run.smithi099.stdout: "container_init": { 2024-03-19T02:05:37.499 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "container_init", 2024-03-19T02:05:37.499 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.499 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.499 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.499 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.499 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.499 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.500 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.500 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Run podman/docker with `--init`", 2024-03-19T02:05:37.500 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.500 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.500 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.500 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.500 INFO:teuthology.orchestra.run.smithi099.stdout: "daemon_cache_timeout": { 2024-03-19T02:05:37.500 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "daemon_cache_timeout", 2024-03-19T02:05:37.500 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:37.500 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.500 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.500 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "600", 2024-03-19T02:05:37.500 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.500 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.500 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.501 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "seconds to cache service (daemon) inventory", 2024-03-19T02:05:37.501 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.501 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.501 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.501 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.501 INFO:teuthology.orchestra.run.smithi099.stdout: "default_cephadm_command_timeout": { 2024-03-19T02:05:37.501 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "default_cephadm_command_timeout", 2024-03-19T02:05:37.501 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.501 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.501 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.501 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "900", 2024-03-19T02:05:37.501 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.501 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.501 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.502 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Default timeout applied to cephadm commands run directly on the host (in seconds)", 2024-03-19T02:05:37.502 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.502 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.502 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.502 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.502 INFO:teuthology.orchestra.run.smithi099.stdout: "default_registry": { 2024-03-19T02:05:37.502 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "default_registry", 2024-03-19T02:05:37.502 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.502 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.502 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.502 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "docker.io", 2024-03-19T02:05:37.502 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.502 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.502 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.502 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Search-registry to which we should normalize unqualified image names. This is not the default registry", 2024-03-19T02:05:37.503 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.503 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.503 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.503 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.503 INFO:teuthology.orchestra.run.smithi099.stdout: "device_cache_timeout": { 2024-03-19T02:05:37.503 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "device_cache_timeout", 2024-03-19T02:05:37.503 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:37.503 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.503 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.503 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "1800", 2024-03-19T02:05:37.503 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.503 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.503 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.503 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "seconds to cache device inventory", 2024-03-19T02:05:37.504 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.504 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.504 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.504 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.504 INFO:teuthology.orchestra.run.smithi099.stdout: "device_enhanced_scan": { 2024-03-19T02:05:37.504 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "device_enhanced_scan", 2024-03-19T02:05:37.504 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.504 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.504 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.504 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.504 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.504 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.504 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.504 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Use libstoragemgmt during device scans", 2024-03-19T02:05:37.504 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.505 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.505 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.505 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.505 INFO:teuthology.orchestra.run.smithi099.stdout: "facts_cache_timeout": { 2024-03-19T02:05:37.505 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "facts_cache_timeout", 2024-03-19T02:05:37.505 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:37.505 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.505 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.505 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "60", 2024-03-19T02:05:37.505 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.505 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.505 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.505 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "seconds to cache host facts data", 2024-03-19T02:05:37.505 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.506 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.506 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.506 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.506 INFO:teuthology.orchestra.run.smithi099.stdout: "host_check_interval": { 2024-03-19T02:05:37.506 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "host_check_interval", 2024-03-19T02:05:37.506 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:37.506 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.506 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.506 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "600", 2024-03-19T02:05:37.506 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.506 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.506 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.506 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "how frequently to perform a host check", 2024-03-19T02:05:37.506 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.507 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.507 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.507 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.507 INFO:teuthology.orchestra.run.smithi099.stdout: "hw_monitoring": { 2024-03-19T02:05:37.507 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "hw_monitoring", 2024-03-19T02:05:37.507 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.507 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.507 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.507 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.507 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.507 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.507 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.507 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Deploy hw monitoring daemon on every host.", 2024-03-19T02:05:37.507 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.507 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.508 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.508 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.508 INFO:teuthology.orchestra.run.smithi099.stdout: "inventory_list_all": { 2024-03-19T02:05:37.508 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "inventory_list_all", 2024-03-19T02:05:37.508 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.508 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.508 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.508 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.508 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.508 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.508 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.508 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)", 2024-03-19T02:05:37.508 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.508 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.508 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.509 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.509 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.509 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.509 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.509 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.509 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.509 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.509 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.509 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.509 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.509 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.509 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.509 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.509 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.509 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.510 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.510 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.510 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.510 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.510 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.510 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.510 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.510 INFO:teuthology.orchestra.run.smithi099.stdout: "log_refresh_metadata": { 2024-03-19T02:05:37.510 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_refresh_metadata", 2024-03-19T02:05:37.510 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.510 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.510 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.510 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.510 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.511 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.511 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.511 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level", 2024-03-19T02:05:37.511 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.511 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.511 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.511 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.511 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.511 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.511 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.511 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.511 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.511 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.511 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.511 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.512 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.512 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "log to the \"cephadm\" cluster log channel\"", 2024-03-19T02:05:37.512 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.512 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.512 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.512 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.512 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.512 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.512 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.512 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.512 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.512 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.512 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.512 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.512 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.513 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.513 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.513 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.513 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.513 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.513 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.513 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.513 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.513 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.513 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.513 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.513 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.513 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.513 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.513 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.514 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.514 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.514 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.514 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.514 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.514 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.514 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.514 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.514 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.514 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.514 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.514 INFO:teuthology.orchestra.run.smithi099.stdout: "manage_etc_ceph_ceph_conf": { 2024-03-19T02:05:37.514 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "manage_etc_ceph_ceph_conf", 2024-03-19T02:05:37.514 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.515 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.515 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.515 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.515 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.515 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.515 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.515 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Manage and own /etc/ceph/ceph.conf on the hosts.", 2024-03-19T02:05:37.515 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.515 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.515 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.515 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.515 INFO:teuthology.orchestra.run.smithi099.stdout: "manage_etc_ceph_ceph_conf_hosts": { 2024-03-19T02:05:37.515 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "manage_etc_ceph_ceph_conf_hosts", 2024-03-19T02:05:37.515 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.516 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.516 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.516 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "*", 2024-03-19T02:05:37.516 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.516 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.516 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.516 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf", 2024-03-19T02:05:37.516 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.516 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.516 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.516 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.516 INFO:teuthology.orchestra.run.smithi099.stdout: "max_count_per_host": { 2024-03-19T02:05:37.516 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "max_count_per_host", 2024-03-19T02:05:37.516 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.517 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.517 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.517 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "10", 2024-03-19T02:05:37.517 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.517 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.517 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.517 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "max number of daemons per service per host", 2024-03-19T02:05:37.517 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.517 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.517 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.517 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.517 INFO:teuthology.orchestra.run.smithi099.stdout: "max_osd_draining_count": { 2024-03-19T02:05:37.517 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "max_osd_draining_count", 2024-03-19T02:05:37.517 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.518 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.518 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.518 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "10", 2024-03-19T02:05:37.518 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.518 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.518 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.518 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "max number of osds that will be drained simultaneously when osds are removed", 2024-03-19T02:05:37.518 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.518 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.518 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.518 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.518 INFO:teuthology.orchestra.run.smithi099.stdout: "migration_current": { 2024-03-19T02:05:37.518 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "migration_current", 2024-03-19T02:05:37.519 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.519 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.519 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.519 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.519 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.519 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.519 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.519 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "internal - do not modify", 2024-03-19T02:05:37.519 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.519 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.519 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.519 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.519 INFO:teuthology.orchestra.run.smithi099.stdout: "mode": { 2024-03-19T02:05:37.519 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "mode", 2024-03-19T02:05:37.520 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.520 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.520 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.520 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "root", 2024-03-19T02:05:37.520 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.520 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.520 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.520 INFO:teuthology.orchestra.run.smithi099.stdout: "cephadm-package", 2024-03-19T02:05:37.520 INFO:teuthology.orchestra.run.smithi099.stdout: "root" 2024-03-19T02:05:37.520 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.520 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "mode for remote execution of cephadm", 2024-03-19T02:05:37.520 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.520 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.520 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.520 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.521 INFO:teuthology.orchestra.run.smithi099.stdout: "oob_default_addr": { 2024-03-19T02:05:37.521 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "oob_default_addr", 2024-03-19T02:05:37.521 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.521 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.521 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.521 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "169.254.1.1", 2024-03-19T02:05:37.521 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.521 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.521 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.521 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Default address for RedFish API (oob management).", 2024-03-19T02:05:37.521 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.521 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.521 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.522 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.522 INFO:teuthology.orchestra.run.smithi099.stdout: "prometheus_alerts_path": { 2024-03-19T02:05:37.522 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "prometheus_alerts_path", 2024-03-19T02:05:37.522 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.522 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.522 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.522 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "/etc/prometheus/ceph/ceph_default_alerts.yml", 2024-03-19T02:05:37.522 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.522 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.522 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.522 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "location of alerts to include in prometheus deployments", 2024-03-19T02:05:37.522 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.522 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.522 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.523 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.523 INFO:teuthology.orchestra.run.smithi099.stdout: "registry_insecure": { 2024-03-19T02:05:37.523 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "registry_insecure", 2024-03-19T02:05:37.523 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.523 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.523 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.523 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.523 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.523 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.523 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.523 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Registry is to be considered insecure (no TLS available). Only for development purposes.", 2024-03-19T02:05:37.523 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.523 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.523 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.524 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.524 INFO:teuthology.orchestra.run.smithi099.stdout: "registry_password": { 2024-03-19T02:05:37.524 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "registry_password", 2024-03-19T02:05:37.524 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.524 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.524 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.524 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.524 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.524 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.524 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.524 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Custom repository password. Only used for logging into a registry.", 2024-03-19T02:05:37.524 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.524 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.524 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.524 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.525 INFO:teuthology.orchestra.run.smithi099.stdout: "registry_url": { 2024-03-19T02:05:37.525 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "registry_url", 2024-03-19T02:05:37.525 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.525 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.525 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.525 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.525 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.525 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.525 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.525 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Registry url for login purposes. This is not the default registry", 2024-03-19T02:05:37.525 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.525 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.525 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.525 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.525 INFO:teuthology.orchestra.run.smithi099.stdout: "registry_username": { 2024-03-19T02:05:37.526 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "registry_username", 2024-03-19T02:05:37.526 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.526 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.526 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.526 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.526 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.526 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.526 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.526 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Custom repository username. Only used for logging into a registry.", 2024-03-19T02:05:37.526 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.526 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.526 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.526 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.526 INFO:teuthology.orchestra.run.smithi099.stdout: "secure_monitoring_stack": { 2024-03-19T02:05:37.527 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "secure_monitoring_stack", 2024-03-19T02:05:37.527 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.527 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.527 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.527 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.527 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.527 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.527 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.527 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Enable TLS security for all the monitoring stack daemons", 2024-03-19T02:05:37.527 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.527 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.527 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.527 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.527 INFO:teuthology.orchestra.run.smithi099.stdout: "service_discovery_port": { 2024-03-19T02:05:37.527 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "service_discovery_port", 2024-03-19T02:05:37.528 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.528 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.528 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.528 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "8765", 2024-03-19T02:05:37.528 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.528 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.528 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.528 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "cephadm service discovery port", 2024-03-19T02:05:37.528 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.528 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.528 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.528 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.528 INFO:teuthology.orchestra.run.smithi099.stdout: "ssh_config_file": { 2024-03-19T02:05:37.528 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "ssh_config_file", 2024-03-19T02:05:37.528 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.529 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.529 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.529 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.529 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.529 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.529 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.529 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "customized SSH config file to connect to managed hosts", 2024-03-19T02:05:37.529 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.529 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.529 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.529 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.529 INFO:teuthology.orchestra.run.smithi099.stdout: "use_agent": { 2024-03-19T02:05:37.529 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "use_agent", 2024-03-19T02:05:37.529 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.530 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.530 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.530 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.530 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.530 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.530 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.530 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Use cephadm agent on each host to gather and send metadata", 2024-03-19T02:05:37.530 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.530 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.530 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.530 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.530 INFO:teuthology.orchestra.run.smithi099.stdout: "use_repo_digest": { 2024-03-19T02:05:37.530 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "use_repo_digest", 2024-03-19T02:05:37.530 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.531 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.531 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.531 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.531 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.531 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.531 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.531 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Automatically convert image tags to image digest. Make sure all daemons use the same image", 2024-03-19T02:05:37.531 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.531 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.531 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.531 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.531 INFO:teuthology.orchestra.run.smithi099.stdout: "warn_on_failed_host_check": { 2024-03-19T02:05:37.531 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "warn_on_failed_host_check", 2024-03-19T02:05:37.531 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.532 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.532 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.532 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.532 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.532 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.532 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.532 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "raise a health warning if the host check fails", 2024-03-19T02:05:37.532 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.532 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.532 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.532 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.532 INFO:teuthology.orchestra.run.smithi099.stdout: "warn_on_stray_daemons": { 2024-03-19T02:05:37.532 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "warn_on_stray_daemons", 2024-03-19T02:05:37.532 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.532 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.533 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.533 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.533 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.533 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.533 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.533 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "raise a health warning if daemons are detected that are not managed by cephadm", 2024-03-19T02:05:37.533 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.533 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.533 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.533 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.533 INFO:teuthology.orchestra.run.smithi099.stdout: "warn_on_stray_hosts": { 2024-03-19T02:05:37.533 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "warn_on_stray_hosts", 2024-03-19T02:05:37.533 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.533 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.534 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.534 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.534 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.534 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.534 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.534 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "raise a health warning if daemons are detected on a host that is not managed by cephadm", 2024-03-19T02:05:37.534 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.534 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.534 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.534 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.534 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.534 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.534 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.535 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "crash", 2024-03-19T02:05:37.535 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.535 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.535 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.535 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.535 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.535 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.535 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.535 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.535 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.535 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.535 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.535 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.535 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.535 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.536 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.536 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.536 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.536 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.536 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.536 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.536 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.536 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.536 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.536 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.536 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.536 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.536 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.536 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.537 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.537 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.537 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.537 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.537 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.537 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.537 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.537 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.537 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.537 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.537 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.537 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.537 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.537 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.537 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.538 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.538 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.538 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.538 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.538 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.538 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.538 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.538 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.538 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.538 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.538 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.538 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.538 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.538 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.539 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.539 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.539 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.539 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.539 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.539 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.539 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.539 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.539 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.539 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.539 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.539 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.539 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.539 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.539 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.540 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.540 INFO:teuthology.orchestra.run.smithi099.stdout: "retain_interval": { 2024-03-19T02:05:37.540 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "retain_interval", 2024-03-19T02:05:37.540 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:37.540 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.540 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.540 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "31536000", 2024-03-19T02:05:37.540 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.540 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.540 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.540 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "how long to retain crashes before pruning them", 2024-03-19T02:05:37.540 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.540 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.540 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.540 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.541 INFO:teuthology.orchestra.run.smithi099.stdout: "warn_recent_interval": { 2024-03-19T02:05:37.541 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "warn_recent_interval", 2024-03-19T02:05:37.541 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:37.541 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.541 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.541 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "1209600", 2024-03-19T02:05:37.541 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.541 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.541 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.541 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "time interval in which to warn about recent crashes", 2024-03-19T02:05:37.541 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.541 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.541 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.541 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.541 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.542 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.542 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.542 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "dashboard", 2024-03-19T02:05:37.542 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.542 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.542 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.542 INFO:teuthology.orchestra.run.smithi099.stdout: "ACCOUNT_LOCKOUT_ATTEMPTS": { 2024-03-19T02:05:37.542 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "ACCOUNT_LOCKOUT_ATTEMPTS", 2024-03-19T02:05:37.542 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.542 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.542 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.542 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "10", 2024-03-19T02:05:37.542 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.542 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.542 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.543 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.543 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.543 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.543 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.543 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.543 INFO:teuthology.orchestra.run.smithi099.stdout: "ALERTMANAGER_API_HOST": { 2024-03-19T02:05:37.543 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "ALERTMANAGER_API_HOST", 2024-03-19T02:05:37.543 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.543 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.543 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.543 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.543 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.543 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.543 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.543 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.544 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.544 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.544 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.544 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.544 INFO:teuthology.orchestra.run.smithi099.stdout: "ALERTMANAGER_API_SSL_VERIFY": { 2024-03-19T02:05:37.544 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "ALERTMANAGER_API_SSL_VERIFY", 2024-03-19T02:05:37.544 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.544 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.544 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.544 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.544 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.544 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.544 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.544 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.544 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.545 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.545 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.545 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.545 INFO:teuthology.orchestra.run.smithi099.stdout: "AUDIT_API_ENABLED": { 2024-03-19T02:05:37.545 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "AUDIT_API_ENABLED", 2024-03-19T02:05:37.545 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.545 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.545 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.545 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.545 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.545 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.545 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.545 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.545 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.546 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.546 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.546 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.546 INFO:teuthology.orchestra.run.smithi099.stdout: "AUDIT_API_LOG_PAYLOAD": { 2024-03-19T02:05:37.546 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "AUDIT_API_LOG_PAYLOAD", 2024-03-19T02:05:37.546 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.546 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.546 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.546 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.546 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.546 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.546 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.546 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.546 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.546 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.547 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.547 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.547 INFO:teuthology.orchestra.run.smithi099.stdout: "ENABLE_BROWSABLE_API": { 2024-03-19T02:05:37.547 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "ENABLE_BROWSABLE_API", 2024-03-19T02:05:37.547 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.547 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.547 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.547 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.547 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.547 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.547 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.547 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.547 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.547 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.548 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.548 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.548 INFO:teuthology.orchestra.run.smithi099.stdout: "FEATURE_TOGGLE_CEPHFS": { 2024-03-19T02:05:37.548 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "FEATURE_TOGGLE_CEPHFS", 2024-03-19T02:05:37.548 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.548 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.548 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.548 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.548 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.548 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.548 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.548 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.548 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.548 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.548 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.549 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.549 INFO:teuthology.orchestra.run.smithi099.stdout: "FEATURE_TOGGLE_DASHBOARD": { 2024-03-19T02:05:37.549 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "FEATURE_TOGGLE_DASHBOARD", 2024-03-19T02:05:37.549 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.549 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.549 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.549 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.549 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.549 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.549 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.549 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.549 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.549 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.549 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.549 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.550 INFO:teuthology.orchestra.run.smithi099.stdout: "FEATURE_TOGGLE_ISCSI": { 2024-03-19T02:05:37.550 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "FEATURE_TOGGLE_ISCSI", 2024-03-19T02:05:37.550 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.550 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.550 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.550 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.550 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.550 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.550 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.550 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.550 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.550 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.550 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.550 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.551 INFO:teuthology.orchestra.run.smithi099.stdout: "FEATURE_TOGGLE_MIRRORING": { 2024-03-19T02:05:37.551 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "FEATURE_TOGGLE_MIRRORING", 2024-03-19T02:05:37.551 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.551 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.551 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.551 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.551 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.551 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.551 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.551 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.551 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.551 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.551 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.551 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.552 INFO:teuthology.orchestra.run.smithi099.stdout: "FEATURE_TOGGLE_NFS": { 2024-03-19T02:05:37.552 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "FEATURE_TOGGLE_NFS", 2024-03-19T02:05:37.552 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.552 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.552 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.552 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.552 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.552 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.552 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.552 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.552 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.552 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.552 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.552 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.553 INFO:teuthology.orchestra.run.smithi099.stdout: "FEATURE_TOGGLE_RBD": { 2024-03-19T02:05:37.553 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "FEATURE_TOGGLE_RBD", 2024-03-19T02:05:37.553 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.553 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.553 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.553 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.553 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.553 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.553 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.553 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.553 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.553 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.553 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.553 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.554 INFO:teuthology.orchestra.run.smithi099.stdout: "FEATURE_TOGGLE_RGW": { 2024-03-19T02:05:37.554 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "FEATURE_TOGGLE_RGW", 2024-03-19T02:05:37.554 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.554 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.554 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.554 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.554 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.554 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.554 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.554 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.554 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.554 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.554 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.554 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.555 INFO:teuthology.orchestra.run.smithi099.stdout: "GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE": { 2024-03-19T02:05:37.555 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE", 2024-03-19T02:05:37.555 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.555 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.555 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.555 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.555 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.555 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.555 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.555 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.555 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.555 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.555 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.556 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.556 INFO:teuthology.orchestra.run.smithi099.stdout: "GRAFANA_API_PASSWORD": { 2024-03-19T02:05:37.556 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "GRAFANA_API_PASSWORD", 2024-03-19T02:05:37.556 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.556 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.556 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.556 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "admin", 2024-03-19T02:05:37.556 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.556 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.556 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.556 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.556 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.556 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.556 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.557 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.557 INFO:teuthology.orchestra.run.smithi099.stdout: "GRAFANA_API_SSL_VERIFY": { 2024-03-19T02:05:37.557 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "GRAFANA_API_SSL_VERIFY", 2024-03-19T02:05:37.557 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.557 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.557 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.557 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.557 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.557 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.557 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.557 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.557 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.557 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.557 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.558 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.558 INFO:teuthology.orchestra.run.smithi099.stdout: "GRAFANA_API_URL": { 2024-03-19T02:05:37.558 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "GRAFANA_API_URL", 2024-03-19T02:05:37.558 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.558 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.558 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.558 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.558 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.558 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.558 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.558 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.558 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.558 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.558 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.558 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.559 INFO:teuthology.orchestra.run.smithi099.stdout: "GRAFANA_API_USERNAME": { 2024-03-19T02:05:37.559 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "GRAFANA_API_USERNAME", 2024-03-19T02:05:37.559 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.559 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.559 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.559 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "admin", 2024-03-19T02:05:37.559 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.559 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.559 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.559 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.559 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.559 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.559 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.559 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.559 INFO:teuthology.orchestra.run.smithi099.stdout: "GRAFANA_FRONTEND_API_URL": { 2024-03-19T02:05:37.560 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "GRAFANA_FRONTEND_API_URL", 2024-03-19T02:05:37.560 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.560 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.560 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.560 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.560 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.560 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.560 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.560 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.560 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.560 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.560 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.560 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.560 INFO:teuthology.orchestra.run.smithi099.stdout: "GRAFANA_UPDATE_DASHBOARDS": { 2024-03-19T02:05:37.561 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "GRAFANA_UPDATE_DASHBOARDS", 2024-03-19T02:05:37.561 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.561 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.561 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.561 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.561 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.561 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.561 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.561 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.561 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.561 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.561 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.561 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.562 INFO:teuthology.orchestra.run.smithi099.stdout: "ISCSI_API_SSL_VERIFICATION": { 2024-03-19T02:05:37.562 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "ISCSI_API_SSL_VERIFICATION", 2024-03-19T02:05:37.562 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.562 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.562 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.562 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.562 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.562 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.562 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.562 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.562 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.562 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.562 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.562 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.562 INFO:teuthology.orchestra.run.smithi099.stdout: "ISSUE_TRACKER_API_KEY": { 2024-03-19T02:05:37.563 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "ISSUE_TRACKER_API_KEY", 2024-03-19T02:05:37.563 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.563 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.563 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.563 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.563 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.563 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.563 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.563 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.563 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.563 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.563 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.563 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.563 INFO:teuthology.orchestra.run.smithi099.stdout: "MULTICLUSTER_CONFIG": { 2024-03-19T02:05:37.563 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "MULTICLUSTER_CONFIG", 2024-03-19T02:05:37.564 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.564 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.564 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.564 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "{}", 2024-03-19T02:05:37.564 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.564 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.564 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.564 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.564 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.564 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.564 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.564 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.564 INFO:teuthology.orchestra.run.smithi099.stdout: "PROMETHEUS_API_HOST": { 2024-03-19T02:05:37.565 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "PROMETHEUS_API_HOST", 2024-03-19T02:05:37.565 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.565 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.565 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.565 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.565 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.565 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.565 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.565 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.565 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.565 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.565 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.565 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.565 INFO:teuthology.orchestra.run.smithi099.stdout: "PROMETHEUS_API_SSL_VERIFY": { 2024-03-19T02:05:37.565 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "PROMETHEUS_API_SSL_VERIFY", 2024-03-19T02:05:37.566 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.566 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.566 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.566 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.566 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.566 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.566 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.566 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.566 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.566 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.566 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.566 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.566 INFO:teuthology.orchestra.run.smithi099.stdout: "PWD_POLICY_CHECK_COMPLEXITY_ENABLED": { 2024-03-19T02:05:37.566 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "PWD_POLICY_CHECK_COMPLEXITY_ENABLED", 2024-03-19T02:05:37.567 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.567 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.567 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.567 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.567 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.567 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.567 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.567 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.567 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.567 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.567 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.567 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.567 INFO:teuthology.orchestra.run.smithi099.stdout: "PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED": { 2024-03-19T02:05:37.567 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED", 2024-03-19T02:05:37.568 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.568 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.568 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.568 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.568 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.568 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.568 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.568 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.568 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.568 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.568 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.568 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.568 INFO:teuthology.orchestra.run.smithi099.stdout: "PWD_POLICY_CHECK_LENGTH_ENABLED": { 2024-03-19T02:05:37.569 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "PWD_POLICY_CHECK_LENGTH_ENABLED", 2024-03-19T02:05:37.569 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.569 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.569 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.569 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.569 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.569 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.569 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.569 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.569 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.569 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.569 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.569 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.569 INFO:teuthology.orchestra.run.smithi099.stdout: "PWD_POLICY_CHECK_OLDPWD_ENABLED": { 2024-03-19T02:05:37.569 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "PWD_POLICY_CHECK_OLDPWD_ENABLED", 2024-03-19T02:05:37.570 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.570 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.570 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.570 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.570 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.570 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.570 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.570 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.570 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.570 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.570 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.570 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.570 INFO:teuthology.orchestra.run.smithi099.stdout: "PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED": { 2024-03-19T02:05:37.570 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED", 2024-03-19T02:05:37.571 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.571 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.571 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.571 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.571 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.571 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.571 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.571 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.571 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.571 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.571 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.571 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.571 INFO:teuthology.orchestra.run.smithi099.stdout: "PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED": { 2024-03-19T02:05:37.571 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED", 2024-03-19T02:05:37.571 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.572 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.572 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.572 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.572 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.572 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.572 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.572 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.572 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.572 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.572 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.572 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.572 INFO:teuthology.orchestra.run.smithi099.stdout: "PWD_POLICY_CHECK_USERNAME_ENABLED": { 2024-03-19T02:05:37.572 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "PWD_POLICY_CHECK_USERNAME_ENABLED", 2024-03-19T02:05:37.572 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.572 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.573 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.573 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.573 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.573 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.573 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.573 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.573 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.573 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.573 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.573 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.573 INFO:teuthology.orchestra.run.smithi099.stdout: "PWD_POLICY_ENABLED": { 2024-03-19T02:05:37.573 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "PWD_POLICY_ENABLED", 2024-03-19T02:05:37.573 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.573 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.574 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.574 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.574 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.574 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.574 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.574 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.574 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.574 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.574 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.574 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.574 INFO:teuthology.orchestra.run.smithi099.stdout: "PWD_POLICY_EXCLUSION_LIST": { 2024-03-19T02:05:37.574 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "PWD_POLICY_EXCLUSION_LIST", 2024-03-19T02:05:37.574 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.574 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.575 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.575 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps", 2024-03-19T02:05:37.575 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.575 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.575 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.575 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.575 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.575 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.575 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.575 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.575 INFO:teuthology.orchestra.run.smithi099.stdout: "PWD_POLICY_MIN_COMPLEXITY": { 2024-03-19T02:05:37.575 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "PWD_POLICY_MIN_COMPLEXITY", 2024-03-19T02:05:37.575 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.575 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.575 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.576 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "10", 2024-03-19T02:05:37.576 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.576 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.576 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.576 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.576 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.576 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.576 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.576 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.576 INFO:teuthology.orchestra.run.smithi099.stdout: "PWD_POLICY_MIN_LENGTH": { 2024-03-19T02:05:37.576 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "PWD_POLICY_MIN_LENGTH", 2024-03-19T02:05:37.576 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.576 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.576 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.577 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "8", 2024-03-19T02:05:37.577 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.577 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.577 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.577 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.577 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.577 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.577 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.577 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.577 INFO:teuthology.orchestra.run.smithi099.stdout: "REST_REQUESTS_TIMEOUT": { 2024-03-19T02:05:37.577 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "REST_REQUESTS_TIMEOUT", 2024-03-19T02:05:37.577 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.577 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.577 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.577 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "45", 2024-03-19T02:05:37.578 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.578 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.578 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.578 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.578 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.578 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.578 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.578 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.578 INFO:teuthology.orchestra.run.smithi099.stdout: "RGW_API_ACCESS_KEY": { 2024-03-19T02:05:37.578 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "RGW_API_ACCESS_KEY", 2024-03-19T02:05:37.578 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.578 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.578 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.578 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.579 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.579 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.579 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.579 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.579 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.579 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.579 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.579 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.579 INFO:teuthology.orchestra.run.smithi099.stdout: "RGW_API_ADMIN_RESOURCE": { 2024-03-19T02:05:37.579 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "RGW_API_ADMIN_RESOURCE", 2024-03-19T02:05:37.579 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.579 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.579 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.579 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "admin", 2024-03-19T02:05:37.579 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.580 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.580 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.580 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.580 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.580 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.580 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.580 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.580 INFO:teuthology.orchestra.run.smithi099.stdout: "RGW_API_SECRET_KEY": { 2024-03-19T02:05:37.580 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "RGW_API_SECRET_KEY", 2024-03-19T02:05:37.580 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.580 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.580 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.580 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.580 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.580 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.581 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.581 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.581 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.581 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.581 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.581 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.581 INFO:teuthology.orchestra.run.smithi099.stdout: "RGW_API_SSL_VERIFY": { 2024-03-19T02:05:37.581 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "RGW_API_SSL_VERIFY", 2024-03-19T02:05:37.581 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.581 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.581 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.581 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.581 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.581 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.581 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.582 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.582 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.582 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.582 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.582 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.582 INFO:teuthology.orchestra.run.smithi099.stdout: "UNSAFE_TLS_v1_2": { 2024-03-19T02:05:37.582 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "UNSAFE_TLS_v1_2", 2024-03-19T02:05:37.582 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.582 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.582 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.582 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.582 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.582 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.582 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.583 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.583 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.583 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.583 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.583 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.583 INFO:teuthology.orchestra.run.smithi099.stdout: "USER_PWD_EXPIRATION_SPAN": { 2024-03-19T02:05:37.583 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "USER_PWD_EXPIRATION_SPAN", 2024-03-19T02:05:37.583 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.583 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.583 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.583 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "0", 2024-03-19T02:05:37.583 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.583 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.583 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.583 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.584 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.584 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.584 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.584 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.584 INFO:teuthology.orchestra.run.smithi099.stdout: "USER_PWD_EXPIRATION_WARNING_1": { 2024-03-19T02:05:37.584 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "USER_PWD_EXPIRATION_WARNING_1", 2024-03-19T02:05:37.584 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.584 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.584 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.584 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "10", 2024-03-19T02:05:37.584 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.584 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.584 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.584 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.585 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.585 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.585 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.585 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.585 INFO:teuthology.orchestra.run.smithi099.stdout: "USER_PWD_EXPIRATION_WARNING_2": { 2024-03-19T02:05:37.585 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "USER_PWD_EXPIRATION_WARNING_2", 2024-03-19T02:05:37.585 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.585 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.585 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.585 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "5", 2024-03-19T02:05:37.585 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.585 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.585 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.585 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.585 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.586 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.586 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.586 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.586 INFO:teuthology.orchestra.run.smithi099.stdout: "cross_origin_url": { 2024-03-19T02:05:37.586 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "cross_origin_url", 2024-03-19T02:05:37.586 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.586 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.586 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.586 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.586 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.586 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.586 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.586 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.586 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.586 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.587 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.587 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.587 INFO:teuthology.orchestra.run.smithi099.stdout: "crt_file": { 2024-03-19T02:05:37.587 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "crt_file", 2024-03-19T02:05:37.587 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.587 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.587 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.587 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.587 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.587 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.587 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.587 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.587 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.587 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.588 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.588 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.588 INFO:teuthology.orchestra.run.smithi099.stdout: "debug": { 2024-03-19T02:05:37.588 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "debug", 2024-03-19T02:05:37.588 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.588 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.588 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.588 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.588 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.588 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.588 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.588 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Enable/disable debug options", 2024-03-19T02:05:37.588 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.588 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.588 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.589 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.589 INFO:teuthology.orchestra.run.smithi099.stdout: "jwt_token_ttl": { 2024-03-19T02:05:37.589 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "jwt_token_ttl", 2024-03-19T02:05:37.589 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.589 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.589 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.589 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "28800", 2024-03-19T02:05:37.589 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.589 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.589 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.589 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.589 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.589 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.589 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.589 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.590 INFO:teuthology.orchestra.run.smithi099.stdout: "key_file": { 2024-03-19T02:05:37.590 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "key_file", 2024-03-19T02:05:37.590 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.590 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.590 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.590 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.590 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.590 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.590 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.590 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.590 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.590 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.590 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.590 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.591 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.591 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.591 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.591 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.591 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.591 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.591 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.591 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.591 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.591 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.591 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.591 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.591 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.591 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.591 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.592 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.592 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.592 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.592 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.592 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.592 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.592 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.592 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.592 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.592 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.592 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.592 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.592 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.592 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.593 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.593 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.593 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.593 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.593 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.593 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.593 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.593 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.593 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.593 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.593 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.593 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.593 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.593 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.594 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.594 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.594 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.594 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.594 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.594 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.594 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.594 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.594 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.594 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.594 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.594 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.594 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.594 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.595 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.595 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.595 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.595 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.595 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.595 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.595 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.595 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.595 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.595 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.595 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.595 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.595 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.595 INFO:teuthology.orchestra.run.smithi099.stdout: "motd": { 2024-03-19T02:05:37.596 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "motd", 2024-03-19T02:05:37.596 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.596 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.596 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.596 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.596 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.596 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.596 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.596 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "The message of the day", 2024-03-19T02:05:37.596 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.596 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.596 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.596 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.596 INFO:teuthology.orchestra.run.smithi099.stdout: "redirect_resolve_ip_addr": { 2024-03-19T02:05:37.596 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "redirect_resolve_ip_addr", 2024-03-19T02:05:37.597 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.597 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.597 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.597 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.597 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.597 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.597 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.597 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.597 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.597 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.597 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.597 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.597 INFO:teuthology.orchestra.run.smithi099.stdout: "server_addr": { 2024-03-19T02:05:37.597 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "server_addr", 2024-03-19T02:05:37.598 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.598 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.598 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.598 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "::", 2024-03-19T02:05:37.598 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.598 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.598 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.598 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.598 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.598 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.598 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.598 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.598 INFO:teuthology.orchestra.run.smithi099.stdout: "server_port": { 2024-03-19T02:05:37.598 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "server_port", 2024-03-19T02:05:37.599 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.599 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.599 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.599 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "8080", 2024-03-19T02:05:37.599 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.599 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.599 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.599 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.599 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.599 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.599 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.599 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.599 INFO:teuthology.orchestra.run.smithi099.stdout: "ssl": { 2024-03-19T02:05:37.599 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "ssl", 2024-03-19T02:05:37.600 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.600 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.600 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.600 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.600 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.600 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.600 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.600 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.600 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.600 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.600 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.600 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.600 INFO:teuthology.orchestra.run.smithi099.stdout: "ssl_server_port": { 2024-03-19T02:05:37.600 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "ssl_server_port", 2024-03-19T02:05:37.600 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.601 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.601 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.601 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "8443", 2024-03-19T02:05:37.601 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.601 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.601 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.601 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.601 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.601 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.601 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.601 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.601 INFO:teuthology.orchestra.run.smithi099.stdout: "standby_behaviour": { 2024-03-19T02:05:37.601 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "standby_behaviour", 2024-03-19T02:05:37.601 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.601 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.602 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.602 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "redirect", 2024-03-19T02:05:37.602 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.602 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.602 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.602 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.602 INFO:teuthology.orchestra.run.smithi099.stdout: "redirect" 2024-03-19T02:05:37.602 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.602 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.602 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.602 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.602 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.602 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.602 INFO:teuthology.orchestra.run.smithi099.stdout: "standby_error_status_code": { 2024-03-19T02:05:37.602 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "standby_error_status_code", 2024-03-19T02:05:37.603 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.603 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.603 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.603 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "500", 2024-03-19T02:05:37.603 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "400", 2024-03-19T02:05:37.603 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "599", 2024-03-19T02:05:37.603 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.603 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.603 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.603 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.603 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.603 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.603 INFO:teuthology.orchestra.run.smithi099.stdout: "url_prefix": { 2024-03-19T02:05:37.603 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "url_prefix", 2024-03-19T02:05:37.603 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.604 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.604 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.604 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.604 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.604 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.604 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.604 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.604 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.604 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.604 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.604 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.604 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.604 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.604 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.604 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "devicehealth", 2024-03-19T02:05:37.605 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.605 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.605 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.605 INFO:teuthology.orchestra.run.smithi099.stdout: "enable_monitoring": { 2024-03-19T02:05:37.605 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "enable_monitoring", 2024-03-19T02:05:37.605 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.605 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.605 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.605 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.605 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.605 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.605 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.605 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "monitor device health metrics", 2024-03-19T02:05:37.605 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.606 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.606 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.606 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.606 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.606 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.606 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.606 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.606 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.606 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.606 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.606 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.606 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.606 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.606 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.606 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.607 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.607 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.607 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.607 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.607 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.607 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.607 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.607 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.607 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.607 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.607 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.607 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.607 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.607 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.608 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.608 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.608 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.608 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.608 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.608 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.608 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.608 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.608 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.608 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.608 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.608 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.608 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.608 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.609 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.609 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.609 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.609 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.609 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.609 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.609 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.609 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.609 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.609 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.609 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.609 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.609 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.609 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.609 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.610 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.610 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.610 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.610 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.610 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.610 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.610 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.610 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.610 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.610 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.610 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.610 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.610 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.610 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.611 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.611 INFO:teuthology.orchestra.run.smithi099.stdout: "mark_out_threshold": { 2024-03-19T02:05:37.611 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "mark_out_threshold", 2024-03-19T02:05:37.611 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:37.611 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.611 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.611 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "2419200", 2024-03-19T02:05:37.611 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.611 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.611 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.611 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "automatically mark OSD if it may fail before this long", 2024-03-19T02:05:37.611 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.611 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.611 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.611 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.612 INFO:teuthology.orchestra.run.smithi099.stdout: "pool_name": { 2024-03-19T02:05:37.612 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "pool_name", 2024-03-19T02:05:37.612 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.612 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.612 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.612 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "device_health_metrics", 2024-03-19T02:05:37.612 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.612 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.612 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.612 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "name of pool in which to store device health metrics", 2024-03-19T02:05:37.612 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.612 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.612 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.612 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.613 INFO:teuthology.orchestra.run.smithi099.stdout: "retention_period": { 2024-03-19T02:05:37.613 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "retention_period", 2024-03-19T02:05:37.613 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:37.613 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.613 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.613 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "15552000", 2024-03-19T02:05:37.613 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.613 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.613 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.613 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "how long to retain device health metrics", 2024-03-19T02:05:37.613 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.613 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.613 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.613 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.613 INFO:teuthology.orchestra.run.smithi099.stdout: "scrape_frequency": { 2024-03-19T02:05:37.614 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "scrape_frequency", 2024-03-19T02:05:37.614 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:37.614 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.614 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.614 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "86400", 2024-03-19T02:05:37.614 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.614 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.614 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.614 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "how frequently to scrape device health metrics", 2024-03-19T02:05:37.614 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.614 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.614 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.614 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.615 INFO:teuthology.orchestra.run.smithi099.stdout: "self_heal": { 2024-03-19T02:05:37.615 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "self_heal", 2024-03-19T02:05:37.615 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.615 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.615 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.615 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.615 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.615 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.615 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.615 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "preemptively heal cluster around devices that may fail", 2024-03-19T02:05:37.615 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.615 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.615 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.615 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.615 INFO:teuthology.orchestra.run.smithi099.stdout: "sleep_interval": { 2024-03-19T02:05:37.616 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "sleep_interval", 2024-03-19T02:05:37.616 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:37.616 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.616 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.616 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "600", 2024-03-19T02:05:37.616 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.616 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.616 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.616 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "how frequently to wake up and check device health", 2024-03-19T02:05:37.616 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.616 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.616 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.616 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.616 INFO:teuthology.orchestra.run.smithi099.stdout: "warn_threshold": { 2024-03-19T02:05:37.616 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "warn_threshold", 2024-03-19T02:05:37.617 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:37.617 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.617 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.617 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "7257600", 2024-03-19T02:05:37.617 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.617 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.617 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.617 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "raise health warning if OSD may fail before this long", 2024-03-19T02:05:37.617 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.617 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.617 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.617 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.617 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.617 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.617 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.618 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "diskprediction_local", 2024-03-19T02:05:37.618 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.618 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.618 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.618 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.618 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.618 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.618 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.618 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.618 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.618 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.618 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.618 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.618 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.619 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.619 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.619 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.619 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.619 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.619 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.619 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.619 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.619 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.619 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.619 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.619 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.619 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.619 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.619 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.620 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.620 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.620 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.620 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.620 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.620 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.620 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.620 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.620 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.620 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.620 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.620 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.620 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.620 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.620 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.621 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.621 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.621 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.621 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.621 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.621 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.621 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.621 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.621 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.621 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.621 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.621 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.621 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.621 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.621 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.622 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.622 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.622 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.622 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.622 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.622 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.622 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.622 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.622 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.622 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.622 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.622 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.622 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.622 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.622 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.623 INFO:teuthology.orchestra.run.smithi099.stdout: "predict_interval": { 2024-03-19T02:05:37.623 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "predict_interval", 2024-03-19T02:05:37.623 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.623 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.623 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.623 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "86400", 2024-03-19T02:05:37.623 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.623 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.623 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.623 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.623 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.623 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.623 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.623 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.623 INFO:teuthology.orchestra.run.smithi099.stdout: "predictor_model": { 2024-03-19T02:05:37.624 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "predictor_model", 2024-03-19T02:05:37.624 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.624 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.624 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.624 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "prophetstor", 2024-03-19T02:05:37.624 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.624 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.624 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.624 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.624 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.624 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.625 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.625 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.625 INFO:teuthology.orchestra.run.smithi099.stdout: "sleep_interval": { 2024-03-19T02:05:37.625 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "sleep_interval", 2024-03-19T02:05:37.625 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.625 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.625 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.625 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "600", 2024-03-19T02:05:37.625 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.625 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.625 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.625 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.625 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.625 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.625 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.626 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.626 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.626 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.626 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.626 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "influx", 2024-03-19T02:05:37.626 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": false, 2024-03-19T02:05:37.626 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "influxdb python module not found", 2024-03-19T02:05:37.626 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.626 INFO:teuthology.orchestra.run.smithi099.stdout: "batch_size": { 2024-03-19T02:05:37.626 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "batch_size", 2024-03-19T02:05:37.626 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.626 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.626 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.626 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "5000", 2024-03-19T02:05:37.627 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.627 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.627 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.627 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "How big batches of data points should be when sending to InfluxDB.", 2024-03-19T02:05:37.627 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.627 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.627 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.627 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.627 INFO:teuthology.orchestra.run.smithi099.stdout: "database": { 2024-03-19T02:05:37.627 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "database", 2024-03-19T02:05:37.627 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.627 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.627 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.627 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "ceph", 2024-03-19T02:05:37.627 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.628 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.628 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.628 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.", 2024-03-19T02:05:37.628 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.628 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.628 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.628 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.628 INFO:teuthology.orchestra.run.smithi099.stdout: "hostname": { 2024-03-19T02:05:37.628 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "hostname", 2024-03-19T02:05:37.628 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.628 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.628 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.628 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.628 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.629 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.629 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.629 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "InfluxDB server hostname", 2024-03-19T02:05:37.629 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.629 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.629 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.629 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.629 INFO:teuthology.orchestra.run.smithi099.stdout: "interval": { 2024-03-19T02:05:37.629 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "interval", 2024-03-19T02:05:37.629 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:37.629 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.629 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.629 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "30", 2024-03-19T02:05:37.629 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "5", 2024-03-19T02:05:37.630 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.630 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.630 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Time between reports to InfluxDB. Default 30 seconds.", 2024-03-19T02:05:37.630 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.630 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.630 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.630 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.630 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.630 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.630 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.630 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.630 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.630 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.631 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.631 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.631 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.631 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.631 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.631 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.631 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.631 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.631 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.631 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.631 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.631 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.631 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.631 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.631 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.632 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.632 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.632 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.632 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.632 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.632 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.632 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.632 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.632 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.632 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.632 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.632 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.632 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.632 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.633 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.633 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.633 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.633 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.633 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.633 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.633 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.633 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.633 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.633 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.633 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.633 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.633 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.633 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.633 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.634 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.634 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.634 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.634 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.634 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.634 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.634 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.634 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.634 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.634 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.634 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.634 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.634 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.634 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.634 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.635 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.635 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.635 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.635 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.635 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.635 INFO:teuthology.orchestra.run.smithi099.stdout: "password": { 2024-03-19T02:05:37.635 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "password", 2024-03-19T02:05:37.635 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.635 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.635 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.635 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.635 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.635 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.635 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.636 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "password of InfluxDB server user", 2024-03-19T02:05:37.636 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.636 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.636 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.636 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.636 INFO:teuthology.orchestra.run.smithi099.stdout: "port": { 2024-03-19T02:05:37.636 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "port", 2024-03-19T02:05:37.636 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.636 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.636 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.636 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "8086", 2024-03-19T02:05:37.636 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.636 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.636 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.636 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "InfluxDB server port", 2024-03-19T02:05:37.637 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.637 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.637 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.637 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.637 INFO:teuthology.orchestra.run.smithi099.stdout: "ssl": { 2024-03-19T02:05:37.637 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "ssl", 2024-03-19T02:05:37.637 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.637 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.637 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.637 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "false", 2024-03-19T02:05:37.637 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.637 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.637 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.637 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Use https connection for InfluxDB server. Use \"true\" or \"false\".", 2024-03-19T02:05:37.638 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.638 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.638 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.638 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.638 INFO:teuthology.orchestra.run.smithi099.stdout: "threads": { 2024-03-19T02:05:37.638 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "threads", 2024-03-19T02:05:37.638 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.638 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.638 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.638 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "5", 2024-03-19T02:05:37.638 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "1", 2024-03-19T02:05:37.638 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "32", 2024-03-19T02:05:37.638 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.638 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "How many worker threads should be spawned for sending data to InfluxDB.", 2024-03-19T02:05:37.638 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.639 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.639 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.639 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.639 INFO:teuthology.orchestra.run.smithi099.stdout: "username": { 2024-03-19T02:05:37.639 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "username", 2024-03-19T02:05:37.639 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.639 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.639 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.639 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.639 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.639 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.639 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.639 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "username of InfluxDB server user", 2024-03-19T02:05:37.639 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.639 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.640 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.640 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.640 INFO:teuthology.orchestra.run.smithi099.stdout: "verify_ssl": { 2024-03-19T02:05:37.640 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "verify_ssl", 2024-03-19T02:05:37.640 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.640 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.640 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.640 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "true", 2024-03-19T02:05:37.640 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.640 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.640 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.640 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Verify https cert for InfluxDB server. Use \"true\" or \"false\".", 2024-03-19T02:05:37.640 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.640 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.641 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.641 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.641 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.641 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.641 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.641 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "insights", 2024-03-19T02:05:37.641 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.641 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.641 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.641 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.641 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.641 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.641 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.641 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.641 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.642 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.642 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.642 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.642 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.642 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.642 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.642 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.642 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.642 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.642 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.642 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.642 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.642 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.642 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.642 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.643 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.643 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.643 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.643 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.643 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.643 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.643 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.643 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.643 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.643 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.643 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.643 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.643 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.643 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.643 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.644 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.644 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.644 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.644 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.644 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.644 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.644 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.644 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.644 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.644 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.644 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.644 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.644 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.644 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.644 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.645 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.645 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.645 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.645 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.645 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.645 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.645 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.645 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.645 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.645 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.645 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.645 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.645 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.645 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.646 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.646 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.646 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.646 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.646 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.646 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.646 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.646 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.646 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "iostat", 2024-03-19T02:05:37.646 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.646 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.646 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.646 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.646 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.647 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.647 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.647 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.647 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.647 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.647 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.647 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.647 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.647 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.647 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.647 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.647 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.647 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.647 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.647 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.648 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.648 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.648 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.648 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.648 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.648 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.648 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.648 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.648 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.648 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.648 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.648 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.648 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.648 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.648 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.649 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.649 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.649 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.649 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.649 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.649 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.649 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.649 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.649 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.649 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.649 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.649 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.649 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.649 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.649 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.650 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.650 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.650 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.650 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.650 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.650 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.650 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.650 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.650 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.650 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.650 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.650 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.650 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.650 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.650 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.651 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.651 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.651 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.651 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.651 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.651 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.651 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.651 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.651 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.651 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.651 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.651 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "k8sevents", 2024-03-19T02:05:37.651 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.651 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.651 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.652 INFO:teuthology.orchestra.run.smithi099.stdout: "ceph_event_retention_days": { 2024-03-19T02:05:37.652 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "ceph_event_retention_days", 2024-03-19T02:05:37.652 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.652 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.652 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.652 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "7", 2024-03-19T02:05:37.652 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.652 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.652 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.652 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Days to hold ceph event information within local cache", 2024-03-19T02:05:37.652 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.652 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.652 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.652 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.652 INFO:teuthology.orchestra.run.smithi099.stdout: "config_check_secs": { 2024-03-19T02:05:37.653 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "config_check_secs", 2024-03-19T02:05:37.653 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.653 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.653 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.653 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "10", 2024-03-19T02:05:37.653 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "10", 2024-03-19T02:05:37.653 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.653 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.653 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "interval (secs) to check for cluster configuration changes", 2024-03-19T02:05:37.653 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.653 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.653 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.653 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.653 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.653 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.654 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.654 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.654 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.654 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.654 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.654 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.654 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.654 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.654 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.654 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.654 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.654 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.654 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.654 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.654 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.655 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.655 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.655 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.655 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.655 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.655 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.655 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.655 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.655 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.655 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.655 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.655 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.655 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.655 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.655 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.656 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.656 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.656 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.656 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.656 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.656 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.656 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.656 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.656 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.656 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.656 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.656 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.656 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.656 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.657 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.657 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.657 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.657 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.657 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.657 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.657 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.657 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.657 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.657 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.657 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.657 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.657 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.657 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.657 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.658 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.658 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.658 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.658 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.658 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.658 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.658 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.658 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.658 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.658 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.658 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.658 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.658 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "localpool", 2024-03-19T02:05:37.658 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.659 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.659 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.659 INFO:teuthology.orchestra.run.smithi099.stdout: "failure_domain": { 2024-03-19T02:05:37.659 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "failure_domain", 2024-03-19T02:05:37.659 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.659 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.659 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.659 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "host", 2024-03-19T02:05:37.659 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.659 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.659 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.659 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "failure domain for any created local pool", 2024-03-19T02:05:37.659 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "what failure domain we should separate data replicas across.", 2024-03-19T02:05:37.659 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.659 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.660 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.660 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.660 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.660 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.660 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.660 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.660 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.660 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.660 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.660 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.660 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.660 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.660 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.660 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.661 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.661 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.661 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.661 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.661 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.661 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.661 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.661 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.661 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.661 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.661 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.661 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.661 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.661 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.662 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.662 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.662 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.662 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.662 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.662 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.662 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.662 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.662 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.662 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.662 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.662 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.662 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.662 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.663 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.663 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.663 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.663 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.663 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.663 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.663 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.663 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.663 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.663 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.663 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.663 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.663 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.663 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.663 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.664 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.664 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.664 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.664 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.664 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.664 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.664 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.664 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.664 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.664 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.664 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.664 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.664 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.664 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.665 INFO:teuthology.orchestra.run.smithi099.stdout: "min_size": { 2024-03-19T02:05:37.665 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "min_size", 2024-03-19T02:05:37.665 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.665 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.665 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.665 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.665 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.665 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.665 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.665 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "default min_size for any created local pool", 2024-03-19T02:05:37.665 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "value to set min_size to (unchanged from Ceph's default if this option is not set)", 2024-03-19T02:05:37.665 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.665 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.665 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.665 INFO:teuthology.orchestra.run.smithi099.stdout: "num_rep": { 2024-03-19T02:05:37.666 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "num_rep", 2024-03-19T02:05:37.666 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.666 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.666 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.666 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "3", 2024-03-19T02:05:37.666 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.666 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.666 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.666 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "default replica count for any created local pool", 2024-03-19T02:05:37.666 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.666 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.666 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.666 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.666 INFO:teuthology.orchestra.run.smithi099.stdout: "pg_num": { 2024-03-19T02:05:37.666 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "pg_num", 2024-03-19T02:05:37.667 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.667 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.667 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.667 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "128", 2024-03-19T02:05:37.667 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.667 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.667 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.667 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "default pg_num for any created local pool", 2024-03-19T02:05:37.667 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.667 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.667 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.667 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.667 INFO:teuthology.orchestra.run.smithi099.stdout: "prefix": { 2024-03-19T02:05:37.667 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "prefix", 2024-03-19T02:05:37.667 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.668 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.668 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.668 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.668 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.668 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.668 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.668 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "name prefix for any created local pool", 2024-03-19T02:05:37.668 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.668 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.668 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.668 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.668 INFO:teuthology.orchestra.run.smithi099.stdout: "subtree": { 2024-03-19T02:05:37.668 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "subtree", 2024-03-19T02:05:37.668 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.668 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.669 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.669 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "rack", 2024-03-19T02:05:37.669 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.669 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.669 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.669 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "CRUSH level for which to create a local pool", 2024-03-19T02:05:37.669 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "which CRUSH subtree type the module should create a pool for.", 2024-03-19T02:05:37.669 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.669 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.669 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.669 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.669 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.669 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.669 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "mds_autoscaler", 2024-03-19T02:05:37.669 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.670 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.670 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.670 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.670 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.670 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.670 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.670 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.670 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.670 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.670 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.670 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.670 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.670 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.670 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.670 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.671 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.671 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.671 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.671 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.671 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.671 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.671 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.671 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.671 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.671 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.671 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.671 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.671 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.671 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.671 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.672 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.672 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.672 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.672 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.672 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.672 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.672 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.672 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.672 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.672 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.672 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.672 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.672 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.673 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.673 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.673 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.673 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.673 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.673 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.673 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.673 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.673 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.673 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.673 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.673 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.673 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.673 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.673 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.674 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.674 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.674 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.674 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.674 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.674 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.674 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.674 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.674 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.674 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.674 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.674 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.674 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.674 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.674 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.675 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.675 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.675 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "mirroring", 2024-03-19T02:05:37.675 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.675 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.675 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.675 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.675 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.675 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.675 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.675 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.675 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.675 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.675 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.675 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.676 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.676 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.676 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.676 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.676 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.676 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.676 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.676 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.676 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.676 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.676 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.676 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.676 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.676 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.676 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.677 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.677 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.677 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.677 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.677 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.677 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.677 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.677 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.677 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.677 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.677 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.677 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.677 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.677 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.677 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.678 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.678 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.678 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.678 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.678 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.678 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.678 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.678 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.678 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.678 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.678 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.678 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.678 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.678 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.679 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.679 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.679 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.679 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.679 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.679 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.679 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.679 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.679 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.679 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.679 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.679 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.679 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.679 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.679 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.680 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.680 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.680 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.680 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.680 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.680 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "nfs", 2024-03-19T02:05:37.680 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.680 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.680 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.680 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.680 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.680 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.680 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.680 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.680 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.681 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.681 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.681 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.681 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.681 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.681 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.681 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.681 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.681 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.681 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.681 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.681 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.681 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.681 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.681 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.682 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.682 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.682 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.682 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.682 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.682 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.682 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.682 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.682 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.682 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.682 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.682 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.682 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.682 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.683 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.683 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.683 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.683 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.683 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.683 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.683 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.683 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.683 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.683 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.683 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.683 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.683 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.683 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.683 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.684 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.684 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.684 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.684 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.684 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.684 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.684 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.684 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.684 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.684 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.684 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.684 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.684 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.684 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.684 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.685 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.685 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.685 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.685 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.685 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.685 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.685 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.685 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.685 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "orchestrator", 2024-03-19T02:05:37.685 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.685 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.685 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.685 INFO:teuthology.orchestra.run.smithi099.stdout: "fail_fs": { 2024-03-19T02:05:37.685 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "fail_fs", 2024-03-19T02:05:37.685 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.686 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.686 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.686 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.686 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.686 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.686 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.686 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Fail filesystem for rapid multi-rank mds upgrade", 2024-03-19T02:05:37.686 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.686 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.686 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.686 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.686 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.686 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.686 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.686 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.687 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.687 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.687 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.687 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.687 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.687 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.687 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.687 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.687 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.687 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.687 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.687 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.687 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.687 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.687 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.688 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.688 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.688 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.688 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.688 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.688 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.688 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.688 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.688 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.688 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.688 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.688 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.688 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.688 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.689 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.689 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.689 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.689 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.689 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.689 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.689 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.689 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.689 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.689 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.689 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.689 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.689 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.689 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.689 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.690 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.690 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.690 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.690 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.690 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.690 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.690 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.690 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.690 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.690 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.690 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.690 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.690 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.690 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.690 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.691 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.691 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.691 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.691 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.691 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.691 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.691 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.691 INFO:teuthology.orchestra.run.smithi099.stdout: "orchestrator": { 2024-03-19T02:05:37.691 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "orchestrator", 2024-03-19T02:05:37.691 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.691 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.691 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.691 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.691 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.691 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.692 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.692 INFO:teuthology.orchestra.run.smithi099.stdout: "cephadm", 2024-03-19T02:05:37.692 INFO:teuthology.orchestra.run.smithi099.stdout: "rook", 2024-03-19T02:05:37.692 INFO:teuthology.orchestra.run.smithi099.stdout: "test_orchestrator" 2024-03-19T02:05:37.692 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.692 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Orchestrator backend", 2024-03-19T02:05:37.692 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.692 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.692 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.692 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.692 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.692 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.692 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.692 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "osd_perf_query", 2024-03-19T02:05:37.693 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.693 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.693 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.693 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.693 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.693 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.693 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.693 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.693 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.693 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.693 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.693 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.693 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.693 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.694 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.694 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.694 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.694 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.694 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.694 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.694 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.694 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.694 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.694 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.694 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.694 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.694 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.694 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.694 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.695 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.695 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.695 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.695 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.695 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.695 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.695 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.695 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.695 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.695 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.695 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.695 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.695 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.695 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.696 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.696 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.696 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.696 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.696 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.696 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.696 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.696 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.696 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.696 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.696 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.696 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.696 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.696 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.696 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.697 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.697 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.697 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.697 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.697 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.697 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.697 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.697 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.697 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.697 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.697 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.697 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.697 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.697 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.697 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.698 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.698 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.698 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.698 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "osd_support", 2024-03-19T02:05:37.698 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.698 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.698 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.698 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.698 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.698 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.698 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.698 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.698 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.698 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.698 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.699 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.699 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.699 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.699 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.699 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.699 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.699 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.699 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.699 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.699 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.699 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.699 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.699 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.699 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.699 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.700 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.700 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.700 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.700 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.700 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.700 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.700 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.700 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.700 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.700 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.700 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.700 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.700 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.700 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.700 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.701 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.701 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.701 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.701 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.701 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.701 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.701 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.701 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.701 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.701 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.701 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.701 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.701 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.701 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.701 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.702 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.702 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.702 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.702 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.702 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.702 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.702 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.702 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.702 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.702 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.702 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.702 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.702 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.702 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.702 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.703 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.703 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.703 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.703 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.703 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.703 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "pg_autoscaler", 2024-03-19T02:05:37.703 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.703 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.703 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.703 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.703 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.703 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.703 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.703 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.703 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.704 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.704 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.704 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.704 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.704 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.704 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.704 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.704 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.704 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.704 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.704 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.704 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.704 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.704 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.704 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.705 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.705 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.705 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.705 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.705 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.705 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.705 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.705 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.705 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.705 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.705 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.705 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.705 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.706 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.706 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.706 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.706 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.706 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.706 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.706 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.706 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.706 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.706 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.706 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.706 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.706 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.706 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.707 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.707 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.707 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.707 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.707 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.707 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.707 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.707 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.707 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.707 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.707 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.707 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.707 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.707 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.707 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.708 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.708 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.708 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.708 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.708 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.708 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.708 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.708 INFO:teuthology.orchestra.run.smithi099.stdout: "sleep_interval": { 2024-03-19T02:05:37.708 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "sleep_interval", 2024-03-19T02:05:37.708 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:37.708 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.708 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.708 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "60", 2024-03-19T02:05:37.708 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.708 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.709 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.709 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.709 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.709 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.709 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.709 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.709 INFO:teuthology.orchestra.run.smithi099.stdout: "threshold": { 2024-03-19T02:05:37.709 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "threshold", 2024-03-19T02:05:37.709 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "float", 2024-03-19T02:05:37.709 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.709 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.709 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "3.0", 2024-03-19T02:05:37.709 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "1.0", 2024-03-19T02:05:37.709 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.709 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.710 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "scaling threshold", 2024-03-19T02:05:37.710 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0", 2024-03-19T02:05:37.710 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.710 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.710 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.710 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.710 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.710 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.710 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "progress", 2024-03-19T02:05:37.710 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.710 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.710 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.710 INFO:teuthology.orchestra.run.smithi099.stdout: "allow_pg_recovery_event": { 2024-03-19T02:05:37.710 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "allow_pg_recovery_event", 2024-03-19T02:05:37.710 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.711 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.711 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.711 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.711 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.711 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.711 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.711 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "allow the module to show pg recovery progress", 2024-03-19T02:05:37.711 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.711 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.711 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.711 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.711 INFO:teuthology.orchestra.run.smithi099.stdout: "enabled": { 2024-03-19T02:05:37.711 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "enabled", 2024-03-19T02:05:37.711 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.712 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.712 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.712 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.712 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.712 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.712 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.712 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.712 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.712 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.712 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.712 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.712 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.712 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.712 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.712 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.713 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.713 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.713 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.713 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.713 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.713 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.713 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.713 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.713 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.713 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.713 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.713 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.713 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.713 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.713 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.714 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.714 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.714 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.714 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.714 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.714 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.714 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.714 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.714 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.714 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.714 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.714 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.714 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.714 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.715 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.715 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.715 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.715 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.715 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.715 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.715 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.715 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.715 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.715 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.715 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.715 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.715 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.715 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.715 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.716 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.716 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.716 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.716 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.716 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.716 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.716 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.716 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.716 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.716 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.716 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.716 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.716 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.716 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.716 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.717 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.717 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.717 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.717 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.717 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.717 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.717 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.717 INFO:teuthology.orchestra.run.smithi099.stdout: "max_completed_events": { 2024-03-19T02:05:37.717 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "max_completed_events", 2024-03-19T02:05:37.717 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.717 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.717 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.717 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "50", 2024-03-19T02:05:37.717 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.717 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.718 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.718 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "number of past completed events to remember", 2024-03-19T02:05:37.718 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.718 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.718 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.718 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.718 INFO:teuthology.orchestra.run.smithi099.stdout: "sleep_interval": { 2024-03-19T02:05:37.718 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "sleep_interval", 2024-03-19T02:05:37.718 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:37.718 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.718 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.718 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "5", 2024-03-19T02:05:37.718 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.718 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.719 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.719 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "how long the module is going to sleep", 2024-03-19T02:05:37.719 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.719 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.719 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.719 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.719 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.719 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.719 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.719 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "prometheus", 2024-03-19T02:05:37.719 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.719 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.719 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.719 INFO:teuthology.orchestra.run.smithi099.stdout: "cache": { 2024-03-19T02:05:37.719 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "cache", 2024-03-19T02:05:37.719 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.720 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.720 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.720 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.720 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.720 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.720 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.720 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.720 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.720 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.720 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.720 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.720 INFO:teuthology.orchestra.run.smithi099.stdout: "exclude_perf_counters": { 2024-03-19T02:05:37.720 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "exclude_perf_counters", 2024-03-19T02:05:37.720 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.720 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.721 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.721 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.721 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.721 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.721 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.721 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Do not include perf-counters in the metrics output", 2024-03-19T02:05:37.721 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.", 2024-03-19T02:05:37.721 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.721 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.721 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.721 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.721 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.721 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.721 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.722 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.722 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.722 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.722 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.722 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.722 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.722 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.722 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.722 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.722 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.722 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.722 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.722 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.722 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.723 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.723 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.723 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.723 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.723 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.723 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.723 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.723 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.723 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.723 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.723 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.723 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.723 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.723 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.723 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.724 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.724 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.724 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.724 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.724 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.724 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.724 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.724 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.724 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.724 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.724 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.724 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.724 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.724 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.724 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.725 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.725 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.725 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.725 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.725 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.725 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.725 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.725 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.725 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.725 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.725 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.725 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.725 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.725 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.725 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.726 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.726 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.726 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.726 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.726 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.726 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.726 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.726 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_stats_pools": { 2024-03-19T02:05:37.726 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "rbd_stats_pools", 2024-03-19T02:05:37.726 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.726 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.726 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.726 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.726 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.727 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.727 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.727 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.727 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.727 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.727 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.727 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.727 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_stats_pools_refresh_interval": { 2024-03-19T02:05:37.727 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "rbd_stats_pools_refresh_interval", 2024-03-19T02:05:37.727 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.727 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.727 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.727 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "300", 2024-03-19T02:05:37.727 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.728 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.728 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.728 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.728 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.728 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.728 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.728 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.728 INFO:teuthology.orchestra.run.smithi099.stdout: "scrape_interval": { 2024-03-19T02:05:37.728 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "scrape_interval", 2024-03-19T02:05:37.728 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "float", 2024-03-19T02:05:37.728 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.728 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.728 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "15.0", 2024-03-19T02:05:37.728 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.728 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.729 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.729 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.729 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.729 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.729 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.729 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.729 INFO:teuthology.orchestra.run.smithi099.stdout: "server_addr": { 2024-03-19T02:05:37.729 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "server_addr", 2024-03-19T02:05:37.729 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.729 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.729 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.729 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "::", 2024-03-19T02:05:37.729 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.729 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.729 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.730 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "the IPv4 or IPv6 address on which the module listens for HTTP requests", 2024-03-19T02:05:37.730 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.730 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.730 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.730 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.730 INFO:teuthology.orchestra.run.smithi099.stdout: "server_port": { 2024-03-19T02:05:37.730 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "server_port", 2024-03-19T02:05:37.730 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.730 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.730 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.730 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "9283", 2024-03-19T02:05:37.730 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.730 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.730 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.730 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "the port on which the module listens for HTTP requests", 2024-03-19T02:05:37.731 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.731 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.731 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.731 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.731 INFO:teuthology.orchestra.run.smithi099.stdout: "stale_cache_strategy": { 2024-03-19T02:05:37.731 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "stale_cache_strategy", 2024-03-19T02:05:37.731 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.731 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.731 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.731 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "log", 2024-03-19T02:05:37.731 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.731 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.731 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.731 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.731 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.732 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.732 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.732 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.732 INFO:teuthology.orchestra.run.smithi099.stdout: "standby_behaviour": { 2024-03-19T02:05:37.732 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "standby_behaviour", 2024-03-19T02:05:37.732 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.732 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.732 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.732 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "default", 2024-03-19T02:05:37.732 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.732 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.732 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.732 INFO:teuthology.orchestra.run.smithi099.stdout: "default", 2024-03-19T02:05:37.732 INFO:teuthology.orchestra.run.smithi099.stdout: "error" 2024-03-19T02:05:37.732 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.733 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.733 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.733 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.733 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.733 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.733 INFO:teuthology.orchestra.run.smithi099.stdout: "standby_error_status_code": { 2024-03-19T02:05:37.733 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "standby_error_status_code", 2024-03-19T02:05:37.733 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.733 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.733 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.733 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "500", 2024-03-19T02:05:37.733 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "400", 2024-03-19T02:05:37.733 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "599", 2024-03-19T02:05:37.733 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.733 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.734 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.734 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.734 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.734 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.734 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.734 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.734 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.734 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "rbd_support", 2024-03-19T02:05:37.734 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.734 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.734 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.734 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.734 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.734 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.734 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.735 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.735 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.735 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.735 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.735 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.735 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.735 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.735 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.735 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.735 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.735 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.735 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.735 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.735 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.735 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.736 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.736 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.736 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.736 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.736 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.736 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.736 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.736 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.736 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.736 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.736 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.736 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.736 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.736 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.737 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.737 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.737 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.737 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.737 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.737 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.737 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.737 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.737 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.737 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.737 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.737 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.737 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.737 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.737 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.738 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.738 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.738 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.738 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.738 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.738 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.738 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.738 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.738 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.738 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.738 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.738 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.738 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.738 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.738 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.739 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.739 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.739 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.739 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.739 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.739 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.739 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.739 INFO:teuthology.orchestra.run.smithi099.stdout: "max_concurrent_snap_create": { 2024-03-19T02:05:37.739 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "max_concurrent_snap_create", 2024-03-19T02:05:37.739 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.739 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.739 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.739 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "10", 2024-03-19T02:05:37.739 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.740 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.740 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.740 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.740 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.740 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.740 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.740 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.740 INFO:teuthology.orchestra.run.smithi099.stdout: "mirror_snapshot_schedule": { 2024-03-19T02:05:37.740 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "mirror_snapshot_schedule", 2024-03-19T02:05:37.740 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.740 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.740 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.740 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.740 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.740 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.741 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.741 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.741 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.741 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.741 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.741 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.741 INFO:teuthology.orchestra.run.smithi099.stdout: "trash_purge_schedule": { 2024-03-19T02:05:37.741 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "trash_purge_schedule", 2024-03-19T02:05:37.741 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.741 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.741 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.741 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.741 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.741 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.741 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.742 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.742 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.742 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.742 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.742 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.742 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.742 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.742 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.742 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "restful", 2024-03-19T02:05:37.742 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.742 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.742 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.742 INFO:teuthology.orchestra.run.smithi099.stdout: "enable_auth": { 2024-03-19T02:05:37.742 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "enable_auth", 2024-03-19T02:05:37.743 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.743 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.743 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.743 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.743 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.743 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.743 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.743 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.743 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.743 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.743 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.743 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.743 INFO:teuthology.orchestra.run.smithi099.stdout: "key_file": { 2024-03-19T02:05:37.743 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "key_file", 2024-03-19T02:05:37.743 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.744 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.744 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.744 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.744 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.744 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.744 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.744 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.744 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.744 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.744 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.744 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.744 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.744 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.744 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.744 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.745 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.745 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.745 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.745 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.745 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.745 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.745 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.745 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.745 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.745 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.745 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.745 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.745 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.745 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.746 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.746 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.746 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.746 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.746 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.746 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.746 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.746 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.746 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.746 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.746 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.746 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.746 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.746 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.746 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.747 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.747 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.747 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.747 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.747 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.747 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.747 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.747 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.747 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.747 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.747 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.747 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.747 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.747 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.747 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.748 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.748 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.748 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.748 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.748 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.748 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.748 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.748 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.748 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.748 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.748 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.748 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.748 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.748 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.748 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.749 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.749 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.749 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.749 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.749 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.749 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.749 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.749 INFO:teuthology.orchestra.run.smithi099.stdout: "server_addr": { 2024-03-19T02:05:37.749 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "server_addr", 2024-03-19T02:05:37.749 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.749 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.749 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.749 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.749 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.749 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.750 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.750 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.750 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.750 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.750 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.750 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.750 INFO:teuthology.orchestra.run.smithi099.stdout: "server_port": { 2024-03-19T02:05:37.750 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "server_port", 2024-03-19T02:05:37.750 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.750 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.750 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.750 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.750 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.750 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.750 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.751 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.751 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.751 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.751 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.751 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.751 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.751 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.751 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.751 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "rgw", 2024-03-19T02:05:37.751 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.751 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.751 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.751 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.751 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.751 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.752 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.752 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.752 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.752 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.752 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.752 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.752 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.752 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.752 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.752 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.752 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.752 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.752 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.752 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.753 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.753 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.753 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.753 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.753 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.753 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.753 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.753 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.753 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.753 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.753 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.753 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.753 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.753 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.754 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.754 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.754 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.754 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.754 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.754 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.754 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.754 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.754 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.754 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.754 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.754 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.754 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.754 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.754 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.755 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.755 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.755 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.755 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.755 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.755 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.755 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.755 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.755 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.755 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.755 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.755 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.755 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.755 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.755 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.756 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.756 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.756 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.756 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.756 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.756 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.756 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.756 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.756 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.756 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.756 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.756 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.756 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "rook", 2024-03-19T02:05:37.756 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.756 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.757 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.757 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.757 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.757 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.757 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.757 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.757 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.757 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.757 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.757 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.757 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.757 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.757 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.757 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.758 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.758 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.758 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.758 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.758 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.758 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.758 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.758 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.758 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.758 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.758 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.758 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.758 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.758 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.758 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.759 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.759 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.759 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.759 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.759 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.759 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.759 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.759 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.759 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.759 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.759 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.759 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.759 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.759 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.760 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.760 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.760 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.760 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.760 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.760 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.760 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.760 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.760 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.760 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.760 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.760 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.760 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.760 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.760 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.761 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.761 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.761 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.761 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.761 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.761 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.761 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.761 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.761 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.761 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.761 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.761 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.761 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.761 INFO:teuthology.orchestra.run.smithi099.stdout: "storage_class": { 2024-03-19T02:05:37.761 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "storage_class", 2024-03-19T02:05:37.762 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.762 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.762 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.762 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "local", 2024-03-19T02:05:37.762 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.762 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.762 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.762 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "storage class name for LSO-discovered PVs", 2024-03-19T02:05:37.762 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.762 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.762 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.762 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.762 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.762 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.763 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.763 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "selftest", 2024-03-19T02:05:37.763 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.763 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.763 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.763 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.763 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.763 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.763 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.763 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.763 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.763 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.763 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.763 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.763 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.764 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.764 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.764 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.764 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.764 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.764 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.764 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.764 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.764 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.764 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.764 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.764 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.764 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.764 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.764 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.765 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.765 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.765 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.765 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.765 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.765 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.765 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.765 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.765 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.765 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.765 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.765 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.765 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.765 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.765 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.766 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.766 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.766 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.766 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.766 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.766 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.766 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.766 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.766 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.766 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.766 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.766 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.766 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.766 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.767 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.767 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.767 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.767 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.767 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.767 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.767 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.767 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.767 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.767 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.767 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.767 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.767 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.767 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.768 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.768 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.768 INFO:teuthology.orchestra.run.smithi099.stdout: "roption1": { 2024-03-19T02:05:37.768 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "roption1", 2024-03-19T02:05:37.768 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.768 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.768 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.768 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.768 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.768 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.768 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.768 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.768 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.768 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.768 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.769 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.769 INFO:teuthology.orchestra.run.smithi099.stdout: "roption2": { 2024-03-19T02:05:37.769 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "roption2", 2024-03-19T02:05:37.769 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.769 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.769 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.769 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "xyz", 2024-03-19T02:05:37.769 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.769 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.769 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.769 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.769 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.769 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.769 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.769 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.770 INFO:teuthology.orchestra.run.smithi099.stdout: "rwoption1": { 2024-03-19T02:05:37.770 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "rwoption1", 2024-03-19T02:05:37.770 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.770 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.770 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.770 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.770 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.770 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.770 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.770 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.770 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.770 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.770 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.770 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.771 INFO:teuthology.orchestra.run.smithi099.stdout: "rwoption2": { 2024-03-19T02:05:37.771 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "rwoption2", 2024-03-19T02:05:37.771 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.771 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.771 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.771 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.771 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.771 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.771 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.771 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.771 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.771 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.771 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.771 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.771 INFO:teuthology.orchestra.run.smithi099.stdout: "rwoption3": { 2024-03-19T02:05:37.772 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "rwoption3", 2024-03-19T02:05:37.772 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "float", 2024-03-19T02:05:37.772 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.772 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.772 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.772 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.772 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.772 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.772 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.772 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.772 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.772 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.772 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.772 INFO:teuthology.orchestra.run.smithi099.stdout: "rwoption4": { 2024-03-19T02:05:37.772 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "rwoption4", 2024-03-19T02:05:37.773 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.773 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.773 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.773 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.773 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.773 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.773 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.773 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.773 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.773 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.773 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.773 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.773 INFO:teuthology.orchestra.run.smithi099.stdout: "rwoption5": { 2024-03-19T02:05:37.773 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "rwoption5", 2024-03-19T02:05:37.773 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.774 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.774 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.774 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.774 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.774 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.774 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.774 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.774 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.774 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.774 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.774 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.774 INFO:teuthology.orchestra.run.smithi099.stdout: "rwoption6": { 2024-03-19T02:05:37.774 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "rwoption6", 2024-03-19T02:05:37.774 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.775 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.775 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.775 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.775 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.775 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.775 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.775 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.775 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.775 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.775 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.775 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.775 INFO:teuthology.orchestra.run.smithi099.stdout: "rwoption7": { 2024-03-19T02:05:37.775 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "rwoption7", 2024-03-19T02:05:37.775 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.775 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.776 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.776 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.776 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "1", 2024-03-19T02:05:37.776 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "42", 2024-03-19T02:05:37.776 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.776 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.776 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.776 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.776 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.776 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.776 INFO:teuthology.orchestra.run.smithi099.stdout: "testkey": { 2024-03-19T02:05:37.776 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "testkey", 2024-03-19T02:05:37.776 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.776 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.777 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.777 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.777 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.777 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.777 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.777 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.777 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.777 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.777 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.777 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.777 INFO:teuthology.orchestra.run.smithi099.stdout: "testlkey": { 2024-03-19T02:05:37.777 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "testlkey", 2024-03-19T02:05:37.777 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.777 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.777 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.778 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.778 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.778 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.778 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.778 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.778 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.778 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.778 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.778 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.778 INFO:teuthology.orchestra.run.smithi099.stdout: "testnewline": { 2024-03-19T02:05:37.778 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "testnewline", 2024-03-19T02:05:37.778 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.778 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.778 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.778 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.779 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.779 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.779 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.779 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.779 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.779 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.779 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.779 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.779 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.779 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.779 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.779 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "snap_schedule", 2024-03-19T02:05:37.779 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.779 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.779 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.780 INFO:teuthology.orchestra.run.smithi099.stdout: "allow_m_granularity": { 2024-03-19T02:05:37.780 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "allow_m_granularity", 2024-03-19T02:05:37.780 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.780 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.780 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.780 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.780 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.780 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.780 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.780 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "allow minute scheduled snapshots", 2024-03-19T02:05:37.780 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.780 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.780 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.780 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.780 INFO:teuthology.orchestra.run.smithi099.stdout: "dump_on_update": { 2024-03-19T02:05:37.781 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "dump_on_update", 2024-03-19T02:05:37.781 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.781 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.781 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.781 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.781 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.781 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.781 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.781 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "dump database to debug log on update", 2024-03-19T02:05:37.781 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.781 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.781 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.781 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.781 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.781 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.782 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.782 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.782 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.782 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.782 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.782 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.782 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.782 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.782 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.782 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.782 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.782 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.782 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.782 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.782 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.783 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.783 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.783 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.783 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.783 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.783 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.783 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.783 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.783 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.783 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.783 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.783 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.783 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.783 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.783 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.784 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.784 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.784 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.784 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.784 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.784 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.784 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.784 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.784 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.784 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.784 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.784 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.784 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.784 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.784 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.785 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.785 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.785 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.785 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.785 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.785 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.785 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.785 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.785 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.785 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.785 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.785 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.785 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.785 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.786 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.786 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.786 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.786 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.786 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.786 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.786 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.786 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.786 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.786 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.786 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.786 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.786 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "stats", 2024-03-19T02:05:37.786 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.787 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.787 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.787 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.787 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.787 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.787 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.787 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.787 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.787 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.787 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.787 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.787 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.787 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.787 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.787 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.788 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.788 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.788 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.788 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.788 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.788 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.788 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.788 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.788 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.788 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.788 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.788 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.788 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.788 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.788 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.789 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.789 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.789 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.789 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.789 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.789 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.789 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.789 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.789 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.789 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.789 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.789 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.789 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.789 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.789 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.790 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.790 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.790 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.790 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.790 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.790 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.790 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.790 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.790 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.790 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.790 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.790 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.790 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.790 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.791 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.791 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.791 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.791 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.791 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.791 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.791 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.791 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.791 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.791 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.791 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.791 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.791 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.791 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.791 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.792 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.792 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "status", 2024-03-19T02:05:37.792 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.792 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.792 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.792 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.792 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.792 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.792 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.792 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.792 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.792 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.792 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.792 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.793 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.793 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.793 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.793 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.793 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.793 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.793 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.793 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.793 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.793 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.793 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.793 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.793 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.793 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.793 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.794 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.794 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.794 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.794 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.794 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.794 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.794 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.794 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.794 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.794 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.794 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.794 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.794 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.794 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.795 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.795 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.795 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.795 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.795 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.795 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.795 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.795 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.795 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.795 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.795 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.795 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.795 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.795 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.795 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.795 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.796 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.796 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.796 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.796 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.796 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.796 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.796 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.796 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.796 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.796 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.796 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.796 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.796 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.796 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.797 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.797 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.797 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.797 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.797 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.797 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "telegraf", 2024-03-19T02:05:37.797 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.797 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.797 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.797 INFO:teuthology.orchestra.run.smithi099.stdout: "address": { 2024-03-19T02:05:37.797 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "address", 2024-03-19T02:05:37.797 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.797 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.797 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.797 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "unixgram:///tmp/telegraf.sock", 2024-03-19T02:05:37.798 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.798 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.798 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.798 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.798 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.798 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.798 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.798 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.798 INFO:teuthology.orchestra.run.smithi099.stdout: "interval": { 2024-03-19T02:05:37.798 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "interval", 2024-03-19T02:05:37.798 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:37.798 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.798 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.798 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "15", 2024-03-19T02:05:37.798 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.799 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.799 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.799 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.799 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.799 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.799 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.799 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.799 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.799 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.799 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.799 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.799 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.799 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.799 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.799 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.800 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.800 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.800 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.800 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.800 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.800 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.800 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.800 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.800 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.800 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.800 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.800 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.800 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.800 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.800 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.801 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.801 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.801 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.801 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.801 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.801 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.801 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.801 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.801 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.801 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.801 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.801 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.801 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.801 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.801 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.802 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.802 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.802 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.802 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.802 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.802 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.802 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.802 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.802 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.802 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.802 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.802 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.802 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.802 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.803 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.803 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.803 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.803 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.803 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.803 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.803 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.803 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.803 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.803 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.803 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.803 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.803 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.803 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.803 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.804 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.804 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.804 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.804 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.804 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.804 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.804 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "telemetry", 2024-03-19T02:05:37.804 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.804 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.804 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.804 INFO:teuthology.orchestra.run.smithi099.stdout: "channel_basic": { 2024-03-19T02:05:37.804 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "channel_basic", 2024-03-19T02:05:37.804 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.804 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.804 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.805 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.805 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.805 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.805 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.805 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Share basic cluster information (size, version)", 2024-03-19T02:05:37.805 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.805 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.805 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.805 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.805 INFO:teuthology.orchestra.run.smithi099.stdout: "channel_crash": { 2024-03-19T02:05:37.805 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "channel_crash", 2024-03-19T02:05:37.805 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.805 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.805 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.805 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.806 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.806 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.806 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.806 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Share metadata about Ceph daemon crashes (version, stack straces, etc)", 2024-03-19T02:05:37.806 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.806 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.806 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.806 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.806 INFO:teuthology.orchestra.run.smithi099.stdout: "channel_device": { 2024-03-19T02:05:37.806 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "channel_device", 2024-03-19T02:05:37.806 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.806 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.806 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.806 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.807 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.807 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.807 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.807 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)", 2024-03-19T02:05:37.807 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.807 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.807 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.807 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.807 INFO:teuthology.orchestra.run.smithi099.stdout: "channel_ident": { 2024-03-19T02:05:37.807 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "channel_ident", 2024-03-19T02:05:37.807 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.807 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.807 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.807 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.808 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.808 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.808 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.808 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Share a user-provided description and/or contact email for the cluster", 2024-03-19T02:05:37.808 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.808 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.808 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.808 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.808 INFO:teuthology.orchestra.run.smithi099.stdout: "channel_perf": { 2024-03-19T02:05:37.808 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "channel_perf", 2024-03-19T02:05:37.808 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.808 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.808 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.808 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.808 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.809 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.809 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.809 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Share various performance metrics of a cluster", 2024-03-19T02:05:37.809 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.809 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.809 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.809 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.809 INFO:teuthology.orchestra.run.smithi099.stdout: "contact": { 2024-03-19T02:05:37.809 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "contact", 2024-03-19T02:05:37.809 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.809 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.809 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.809 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.809 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.809 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.810 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.810 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.810 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.810 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.810 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.810 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.810 INFO:teuthology.orchestra.run.smithi099.stdout: "description": { 2024-03-19T02:05:37.810 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "description", 2024-03-19T02:05:37.810 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.810 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.810 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.810 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.810 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.810 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.810 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.811 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.811 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.811 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.811 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.811 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.811 INFO:teuthology.orchestra.run.smithi099.stdout: "device_url": { 2024-03-19T02:05:37.811 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "device_url", 2024-03-19T02:05:37.811 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.811 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.811 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.811 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "https://telemetry.ceph.com/device", 2024-03-19T02:05:37.811 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.811 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.811 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.811 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.812 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.812 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.812 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.812 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.812 INFO:teuthology.orchestra.run.smithi099.stdout: "enabled": { 2024-03-19T02:05:37.812 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "enabled", 2024-03-19T02:05:37.812 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.812 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.812 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.812 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.812 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.812 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.812 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.812 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.812 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.813 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.813 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.813 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.813 INFO:teuthology.orchestra.run.smithi099.stdout: "interval": { 2024-03-19T02:05:37.813 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "interval", 2024-03-19T02:05:37.813 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.813 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.813 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.813 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "24", 2024-03-19T02:05:37.813 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "8", 2024-03-19T02:05:37.813 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.813 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.813 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.813 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.813 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.814 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.814 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.814 INFO:teuthology.orchestra.run.smithi099.stdout: "last_opt_revision": { 2024-03-19T02:05:37.814 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "last_opt_revision", 2024-03-19T02:05:37.814 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.814 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.814 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.814 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "1", 2024-03-19T02:05:37.814 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.814 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.814 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.814 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.814 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.814 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.815 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.815 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.815 INFO:teuthology.orchestra.run.smithi099.stdout: "leaderboard": { 2024-03-19T02:05:37.815 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "leaderboard", 2024-03-19T02:05:37.815 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.815 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.815 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.815 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.815 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.815 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.815 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.815 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.815 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.815 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.815 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.816 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.816 INFO:teuthology.orchestra.run.smithi099.stdout: "leaderboard_description": { 2024-03-19T02:05:37.816 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "leaderboard_description", 2024-03-19T02:05:37.816 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.816 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.816 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.816 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.816 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.816 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.816 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.816 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.816 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.816 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.816 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.816 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.816 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.817 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.817 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.817 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.817 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.817 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.817 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.817 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.817 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.817 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.817 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.817 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.817 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.817 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.817 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.818 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.818 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.818 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.818 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.818 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.818 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.818 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.818 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.818 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.818 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.818 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.818 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.818 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.818 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.819 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.819 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.819 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.819 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.819 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.819 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.819 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.819 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.819 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.819 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.819 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.819 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.819 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.819 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.819 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.820 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.820 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.820 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.820 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.820 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.820 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.820 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.820 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.820 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.820 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.820 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.820 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.820 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.820 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.820 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.821 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.821 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.821 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.821 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.821 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.821 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.821 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.821 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.821 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.821 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.821 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.821 INFO:teuthology.orchestra.run.smithi099.stdout: "organization": { 2024-03-19T02:05:37.821 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "organization", 2024-03-19T02:05:37.821 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.821 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.822 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.822 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.822 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.822 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.822 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.822 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.822 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.822 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.822 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.822 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.822 INFO:teuthology.orchestra.run.smithi099.stdout: "proxy": { 2024-03-19T02:05:37.822 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "proxy", 2024-03-19T02:05:37.822 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.822 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.823 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.823 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.823 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.823 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.823 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.823 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.823 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.823 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.823 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.823 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.823 INFO:teuthology.orchestra.run.smithi099.stdout: "url": { 2024-03-19T02:05:37.823 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "url", 2024-03-19T02:05:37.823 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.823 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.823 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.824 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "https://telemetry.ceph.com/report", 2024-03-19T02:05:37.824 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.824 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.824 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.824 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.824 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.824 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.824 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.824 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.824 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.824 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.824 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.824 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "test_orchestrator", 2024-03-19T02:05:37.824 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.824 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.825 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.825 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.825 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.825 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.825 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.825 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.825 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.825 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.825 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.825 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.825 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.825 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.825 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.825 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.826 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.826 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.826 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.826 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.826 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.826 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.826 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.826 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.826 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.826 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.826 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.826 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.826 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.826 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.826 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.827 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.827 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.827 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.827 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.827 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.827 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.827 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.827 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.827 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.827 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.827 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.827 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.827 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.827 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.828 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.828 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.828 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.828 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.828 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.828 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.828 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.828 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.828 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.828 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.828 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.828 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.828 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.828 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.829 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.829 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.829 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.829 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.829 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.829 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.829 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.829 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.829 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.829 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.829 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.829 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.829 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.829 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.829 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.830 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.830 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.830 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "volumes", 2024-03-19T02:05:37.830 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.830 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.830 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.830 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.830 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.830 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.830 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.830 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.830 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.830 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.830 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.830 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.831 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.831 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.831 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.831 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.831 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.831 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.831 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.831 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.831 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.831 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.831 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.831 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.831 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.831 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.832 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.832 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.832 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.832 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.832 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.832 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.832 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.832 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.832 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.832 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.832 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.832 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.832 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.832 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.832 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.833 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.833 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.833 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.833 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.833 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.833 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.833 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.833 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.833 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.833 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.833 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.833 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.833 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.833 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.834 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.834 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.834 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.834 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.834 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.834 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.834 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.834 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.834 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.834 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.834 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.834 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.834 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.834 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.835 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.835 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.835 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.835 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.835 INFO:teuthology.orchestra.run.smithi099.stdout: "max_concurrent_clones": { 2024-03-19T02:05:37.835 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "max_concurrent_clones", 2024-03-19T02:05:37.835 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.835 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.835 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.835 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "4", 2024-03-19T02:05:37.835 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.835 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.835 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.835 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Number of asynchronous cloner threads", 2024-03-19T02:05:37.835 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.836 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.836 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.836 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.836 INFO:teuthology.orchestra.run.smithi099.stdout: "periodic_async_work": { 2024-03-19T02:05:37.836 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "periodic_async_work", 2024-03-19T02:05:37.836 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.836 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.836 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.836 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.836 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.836 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.836 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.836 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Periodically check for async work", 2024-03-19T02:05:37.836 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.836 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.837 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.837 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.837 INFO:teuthology.orchestra.run.smithi099.stdout: "snapshot_clone_delay": { 2024-03-19T02:05:37.837 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "snapshot_clone_delay", 2024-03-19T02:05:37.837 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.837 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.837 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.837 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "0", 2024-03-19T02:05:37.837 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.837 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.837 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.837 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Delay clone begin operation by snapshot_clone_delay seconds", 2024-03-19T02:05:37.837 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.837 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.838 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.838 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.838 INFO:teuthology.orchestra.run.smithi099.stdout: "snapshot_clone_no_wait": { 2024-03-19T02:05:37.838 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "snapshot_clone_no_wait", 2024-03-19T02:05:37.838 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.838 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.838 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.838 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "True", 2024-03-19T02:05:37.838 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.838 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.838 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.838 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "Reject subvolume clone request when cloner threads are busy", 2024-03-19T02:05:37.838 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.838 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.839 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.839 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.839 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.839 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.839 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.839 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "zabbix", 2024-03-19T02:05:37.839 INFO:teuthology.orchestra.run.smithi099.stdout: "can_run": true, 2024-03-19T02:05:37.839 INFO:teuthology.orchestra.run.smithi099.stdout: "error_string": "", 2024-03-19T02:05:37.839 INFO:teuthology.orchestra.run.smithi099.stdout: "module_options": { 2024-03-19T02:05:37.839 INFO:teuthology.orchestra.run.smithi099.stdout: "discovery_interval": { 2024-03-19T02:05:37.839 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "discovery_interval", 2024-03-19T02:05:37.839 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "uint", 2024-03-19T02:05:37.839 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.839 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.840 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "100", 2024-03-19T02:05:37.840 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.840 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.840 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.840 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.840 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.840 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.840 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.840 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.840 INFO:teuthology.orchestra.run.smithi099.stdout: "identifier": { 2024-03-19T02:05:37.840 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "identifier", 2024-03-19T02:05:37.840 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.840 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.840 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.840 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.841 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.841 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.841 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.841 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.841 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.841 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.841 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.841 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.841 INFO:teuthology.orchestra.run.smithi099.stdout: "interval": { 2024-03-19T02:05:37.841 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "interval", 2024-03-19T02:05:37.841 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "secs", 2024-03-19T02:05:37.841 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.841 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.841 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "60", 2024-03-19T02:05:37.842 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.842 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.842 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.842 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.842 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.842 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.842 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.842 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.842 INFO:teuthology.orchestra.run.smithi099.stdout: "log_level": { 2024-03-19T02:05:37.842 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_level", 2024-03-19T02:05:37.842 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.842 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.842 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.843 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.843 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.843 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.843 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.843 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.843 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.843 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.843 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.843 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.843 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.843 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.843 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.843 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.843 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.844 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.844 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.844 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster": { 2024-03-19T02:05:37.844 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster", 2024-03-19T02:05:37.844 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.844 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.844 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.844 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.844 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.844 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.844 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.844 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.844 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.844 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.845 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.845 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.845 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_cluster_level": { 2024-03-19T02:05:37.845 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_cluster_level", 2024-03-19T02:05:37.845 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.845 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.845 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.845 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "info", 2024-03-19T02:05:37.845 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.845 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.845 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [ 2024-03-19T02:05:37.845 INFO:teuthology.orchestra.run.smithi099.stdout: "", 2024-03-19T02:05:37.845 INFO:teuthology.orchestra.run.smithi099.stdout: "critical", 2024-03-19T02:05:37.845 INFO:teuthology.orchestra.run.smithi099.stdout: "debug", 2024-03-19T02:05:37.845 INFO:teuthology.orchestra.run.smithi099.stdout: "error", 2024-03-19T02:05:37.846 INFO:teuthology.orchestra.run.smithi099.stdout: "info", 2024-03-19T02:05:37.846 INFO:teuthology.orchestra.run.smithi099.stdout: "warning" 2024-03-19T02:05:37.846 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.846 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.846 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.846 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.846 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.846 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.846 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": { 2024-03-19T02:05:37.846 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "log_to_file", 2024-03-19T02:05:37.846 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "bool", 2024-03-19T02:05:37.846 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.846 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 1, 2024-03-19T02:05:37.846 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "False", 2024-03-19T02:05:37.846 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.847 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.847 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.847 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.847 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.847 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.847 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.847 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.847 INFO:teuthology.orchestra.run.smithi099.stdout: "zabbix_host": { 2024-03-19T02:05:37.847 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "zabbix_host", 2024-03-19T02:05:37.847 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.847 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.847 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.847 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "", 2024-03-19T02:05:37.847 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.848 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.848 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.848 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.848 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.848 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.848 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.848 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.848 INFO:teuthology.orchestra.run.smithi099.stdout: "zabbix_port": { 2024-03-19T02:05:37.848 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "zabbix_port", 2024-03-19T02:05:37.848 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "int", 2024-03-19T02:05:37.848 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.848 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.848 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "10051", 2024-03-19T02:05:37.848 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.848 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.849 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.849 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.849 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.849 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.849 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.849 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.849 INFO:teuthology.orchestra.run.smithi099.stdout: "zabbix_sender": { 2024-03-19T02:05:37.849 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "zabbix_sender", 2024-03-19T02:05:37.849 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "str", 2024-03-19T02:05:37.849 INFO:teuthology.orchestra.run.smithi099.stdout: "level": "advanced", 2024-03-19T02:05:37.849 INFO:teuthology.orchestra.run.smithi099.stdout: "flags": 0, 2024-03-19T02:05:37.849 INFO:teuthology.orchestra.run.smithi099.stdout: "default_value": "/usr/bin/zabbix_sender", 2024-03-19T02:05:37.849 INFO:teuthology.orchestra.run.smithi099.stdout: "min": "", 2024-03-19T02:05:37.849 INFO:teuthology.orchestra.run.smithi099.stdout: "max": "", 2024-03-19T02:05:37.849 INFO:teuthology.orchestra.run.smithi099.stdout: "enum_allowed": [], 2024-03-19T02:05:37.850 INFO:teuthology.orchestra.run.smithi099.stdout: "desc": "", 2024-03-19T02:05:37.850 INFO:teuthology.orchestra.run.smithi099.stdout: "long_desc": "", 2024-03-19T02:05:37.850 INFO:teuthology.orchestra.run.smithi099.stdout: "tags": [], 2024-03-19T02:05:37.850 INFO:teuthology.orchestra.run.smithi099.stdout: "see_also": [] 2024-03-19T02:05:37.850 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.850 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.850 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.850 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.850 INFO:teuthology.orchestra.run.smithi099.stdout: "services": { 2024-03-19T02:05:37.850 INFO:teuthology.orchestra.run.smithi099.stdout: "dashboard": "https://172.21.15.99:8443/" 2024-03-19T02:05:37.850 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.850 INFO:teuthology.orchestra.run.smithi099.stdout: "always_on_modules": { 2024-03-19T02:05:37.850 INFO:teuthology.orchestra.run.smithi099.stdout: "octopus": [ 2024-03-19T02:05:37.851 INFO:teuthology.orchestra.run.smithi099.stdout: "balancer", 2024-03-19T02:05:37.851 INFO:teuthology.orchestra.run.smithi099.stdout: "crash", 2024-03-19T02:05:37.851 INFO:teuthology.orchestra.run.smithi099.stdout: "devicehealth", 2024-03-19T02:05:37.851 INFO:teuthology.orchestra.run.smithi099.stdout: "orchestrator", 2024-03-19T02:05:37.851 INFO:teuthology.orchestra.run.smithi099.stdout: "pg_autoscaler", 2024-03-19T02:05:37.851 INFO:teuthology.orchestra.run.smithi099.stdout: "progress", 2024-03-19T02:05:37.851 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_support", 2024-03-19T02:05:37.851 INFO:teuthology.orchestra.run.smithi099.stdout: "status", 2024-03-19T02:05:37.851 INFO:teuthology.orchestra.run.smithi099.stdout: "telemetry", 2024-03-19T02:05:37.851 INFO:teuthology.orchestra.run.smithi099.stdout: "volumes" 2024-03-19T02:05:37.851 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.851 INFO:teuthology.orchestra.run.smithi099.stdout: "pacific": [ 2024-03-19T02:05:37.851 INFO:teuthology.orchestra.run.smithi099.stdout: "balancer", 2024-03-19T02:05:37.851 INFO:teuthology.orchestra.run.smithi099.stdout: "crash", 2024-03-19T02:05:37.852 INFO:teuthology.orchestra.run.smithi099.stdout: "devicehealth", 2024-03-19T02:05:37.852 INFO:teuthology.orchestra.run.smithi099.stdout: "orchestrator", 2024-03-19T02:05:37.852 INFO:teuthology.orchestra.run.smithi099.stdout: "pg_autoscaler", 2024-03-19T02:05:37.852 INFO:teuthology.orchestra.run.smithi099.stdout: "progress", 2024-03-19T02:05:37.852 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_support", 2024-03-19T02:05:37.852 INFO:teuthology.orchestra.run.smithi099.stdout: "status", 2024-03-19T02:05:37.852 INFO:teuthology.orchestra.run.smithi099.stdout: "telemetry", 2024-03-19T02:05:37.852 INFO:teuthology.orchestra.run.smithi099.stdout: "volumes" 2024-03-19T02:05:37.852 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.852 INFO:teuthology.orchestra.run.smithi099.stdout: "quincy": [ 2024-03-19T02:05:37.852 INFO:teuthology.orchestra.run.smithi099.stdout: "balancer", 2024-03-19T02:05:37.852 INFO:teuthology.orchestra.run.smithi099.stdout: "crash", 2024-03-19T02:05:37.852 INFO:teuthology.orchestra.run.smithi099.stdout: "devicehealth", 2024-03-19T02:05:37.852 INFO:teuthology.orchestra.run.smithi099.stdout: "orchestrator", 2024-03-19T02:05:37.852 INFO:teuthology.orchestra.run.smithi099.stdout: "pg_autoscaler", 2024-03-19T02:05:37.853 INFO:teuthology.orchestra.run.smithi099.stdout: "progress", 2024-03-19T02:05:37.853 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_support", 2024-03-19T02:05:37.853 INFO:teuthology.orchestra.run.smithi099.stdout: "status", 2024-03-19T02:05:37.853 INFO:teuthology.orchestra.run.smithi099.stdout: "telemetry", 2024-03-19T02:05:37.853 INFO:teuthology.orchestra.run.smithi099.stdout: "volumes" 2024-03-19T02:05:37.853 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.853 INFO:teuthology.orchestra.run.smithi099.stdout: "reef": [ 2024-03-19T02:05:37.853 INFO:teuthology.orchestra.run.smithi099.stdout: "balancer", 2024-03-19T02:05:37.853 INFO:teuthology.orchestra.run.smithi099.stdout: "crash", 2024-03-19T02:05:37.853 INFO:teuthology.orchestra.run.smithi099.stdout: "devicehealth", 2024-03-19T02:05:37.853 INFO:teuthology.orchestra.run.smithi099.stdout: "orchestrator", 2024-03-19T02:05:37.853 INFO:teuthology.orchestra.run.smithi099.stdout: "pg_autoscaler", 2024-03-19T02:05:37.853 INFO:teuthology.orchestra.run.smithi099.stdout: "progress", 2024-03-19T02:05:37.853 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_support", 2024-03-19T02:05:37.853 INFO:teuthology.orchestra.run.smithi099.stdout: "status", 2024-03-19T02:05:37.854 INFO:teuthology.orchestra.run.smithi099.stdout: "telemetry", 2024-03-19T02:05:37.854 INFO:teuthology.orchestra.run.smithi099.stdout: "volumes" 2024-03-19T02:05:37.854 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-03-19T02:05:37.854 INFO:teuthology.orchestra.run.smithi099.stdout: "squid": [ 2024-03-19T02:05:37.854 INFO:teuthology.orchestra.run.smithi099.stdout: "balancer", 2024-03-19T02:05:37.854 INFO:teuthology.orchestra.run.smithi099.stdout: "crash", 2024-03-19T02:05:37.854 INFO:teuthology.orchestra.run.smithi099.stdout: "devicehealth", 2024-03-19T02:05:37.854 INFO:teuthology.orchestra.run.smithi099.stdout: "orchestrator", 2024-03-19T02:05:37.854 INFO:teuthology.orchestra.run.smithi099.stdout: "pg_autoscaler", 2024-03-19T02:05:37.854 INFO:teuthology.orchestra.run.smithi099.stdout: "progress", 2024-03-19T02:05:37.854 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_support", 2024-03-19T02:05:37.854 INFO:teuthology.orchestra.run.smithi099.stdout: "status", 2024-03-19T02:05:37.854 INFO:teuthology.orchestra.run.smithi099.stdout: "telemetry", 2024-03-19T02:05:37.854 INFO:teuthology.orchestra.run.smithi099.stdout: "volumes" 2024-03-19T02:05:37.855 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-03-19T02:05:37.855 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.855 INFO:teuthology.orchestra.run.smithi099.stdout: "last_failure_osd_epoch": 3, 2024-03-19T02:05:37.855 INFO:teuthology.orchestra.run.smithi099.stdout: "active_clients": [ 2024-03-19T02:05:37.855 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.855 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "devicehealth", 2024-03-19T02:05:37.855 INFO:teuthology.orchestra.run.smithi099.stdout: "addrvec": [ 2024-03-19T02:05:37.855 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.855 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "v2", 2024-03-19T02:05:37.855 INFO:teuthology.orchestra.run.smithi099.stdout: "addr": "172.21.15.99:0", 2024-03-19T02:05:37.855 INFO:teuthology.orchestra.run.smithi099.stdout: "nonce": 121881045 2024-03-19T02:05:37.855 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.855 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-03-19T02:05:37.855 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.855 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.856 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "libcephsqlite", 2024-03-19T02:05:37.856 INFO:teuthology.orchestra.run.smithi099.stdout: "addrvec": [ 2024-03-19T02:05:37.856 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.856 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "v2", 2024-03-19T02:05:37.856 INFO:teuthology.orchestra.run.smithi099.stdout: "addr": "172.21.15.99:0", 2024-03-19T02:05:37.856 INFO:teuthology.orchestra.run.smithi099.stdout: "nonce": 750780629 2024-03-19T02:05:37.856 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.856 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-03-19T02:05:37.856 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.856 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.856 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "rbd_support", 2024-03-19T02:05:37.856 INFO:teuthology.orchestra.run.smithi099.stdout: "addrvec": [ 2024-03-19T02:05:37.856 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.856 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "v2", 2024-03-19T02:05:37.857 INFO:teuthology.orchestra.run.smithi099.stdout: "addr": "172.21.15.99:0", 2024-03-19T02:05:37.857 INFO:teuthology.orchestra.run.smithi099.stdout: "nonce": 223376036 2024-03-19T02:05:37.857 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.857 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-03-19T02:05:37.857 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:37.857 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.857 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "volumes", 2024-03-19T02:05:37.857 INFO:teuthology.orchestra.run.smithi099.stdout: "addrvec": [ 2024-03-19T02:05:37.857 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-03-19T02:05:37.857 INFO:teuthology.orchestra.run.smithi099.stdout: "type": "v2", 2024-03-19T02:05:37.857 INFO:teuthology.orchestra.run.smithi099.stdout: "addr": "172.21.15.99:0", 2024-03-19T02:05:37.857 INFO:teuthology.orchestra.run.smithi099.stdout: "nonce": 1928788970 2024-03-19T02:05:37.857 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.857 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-03-19T02:05:37.857 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:37.857 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-03-19T02:05:37.858 INFO:teuthology.orchestra.run.smithi099.stdout:} 2024-03-19T02:05:37.861 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:05:36 smithi112 ceph-mon[66819]: pgmap v1086: 97 pgs: 97 active+clean; 644 MiB data, 9.6 GiB used, 1.0 TiB / 1.0 TiB avail; 2.7 MiB/s rd, 5.3 MiB/s wr, 501 op/s 2024-03-19T02:05:37.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:05:36 smithi112 ceph-mon[66819]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T02:05:37.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:05:36 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/3983860069' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-03-19T02:05:37.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:05:36 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/1086307220' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-03-19T02:05:37.862 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:05:36 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/3495611872' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json-pretty"}]: dispatch 2024-03-19T02:05:37.862 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:05:36 smithi115 ceph-mon[67332]: pgmap v1086: 97 pgs: 97 active+clean; 644 MiB data, 9.6 GiB used, 1.0 TiB / 1.0 TiB avail; 2.7 MiB/s rd, 5.3 MiB/s wr, 501 op/s 2024-03-19T02:05:37.862 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:05:36 smithi115 ceph-mon[67332]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T02:05:37.862 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:05:36 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/3983860069' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-03-19T02:05:37.862 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:05:36 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/1086307220' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-03-19T02:05:37.862 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:05:36 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/3495611872' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json-pretty"}]: dispatch 2024-03-19T02:05:37.863 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.609+0000 7fde82459640 1 -- 172.21.15.99:0/3495611872 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fde640a2800 msgr2=0x7fde640a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:05:37.863 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.609+0000 7fde82459640 1 --2- 172.21.15.99:0/3495611872 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fde640a2800 0x7fde640a4cc0 secure :-1 s=READY pgs=401 cs=0 l=1 rev1=1 crypto rx=0x7fde7c102890 tx=0x7fde6c005f90 comp rx=0 tx=0).stop 2024-03-19T02:05:37.863 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.609+0000 7fde82459640 1 -- 172.21.15.99:0/3495611872 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fde7c102a30 msgr2=0x7fde7c198080 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:05:37.863 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.609+0000 7fde82459640 1 --2- 172.21.15.99:0/3495611872 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fde7c102a30 0x7fde7c198080 secure :-1 s=READY pgs=812 cs=0 l=1 rev1=1 crypto rx=0x7fde70009230 tx=0x7fde7002c650 comp rx=0 tx=0).stop 2024-03-19T02:05:37.863 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.609+0000 7fde82459640 1 -- 172.21.15.99:0/3495611872 shutdown_connections 2024-03-19T02:05:37.863 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.609+0000 7fde82459640 1 --2- 172.21.15.99:0/3495611872 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fde640a2800 0x7fde640a4cc0 unknown :-1 s=CLOSED pgs=401 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:37.863 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.609+0000 7fde82459640 1 --2- 172.21.15.99:0/3495611872 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fde7c1033f0 0x7fde7c19f0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:37.863 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.609+0000 7fde82459640 1 --2- 172.21.15.99:0/3495611872 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fde7c102a30 0x7fde7c198080 unknown :-1 s=CLOSED pgs=812 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:37.864 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.609+0000 7fde82459640 1 --2- 172.21.15.99:0/3495611872 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fde7c101840 0x7fde7c197b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:05:37.864 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.609+0000 7fde82459640 1 -- 172.21.15.99:0/3495611872 >> 172.21.15.99:0/3495611872 conn(0x7fde7c0fcdd0 msgr2=0x7fde7c105d30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:05:37.864 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.610+0000 7fde82459640 1 -- 172.21.15.99:0/3495611872 shutdown_connections 2024-03-19T02:05:37.864 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:05:36.610+0000 7fde82459640 1 -- 172.21.15.99:0/3495611872 wait complete. 2024-03-19T02:05:37.869 DEBUG:tasks.check_counter:Getting stats from a 2024-03-19T02:05:37.869 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph daemon mds.a perf dump 2024-03-19T02:05:37.870 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:05:36 smithi099 ceph-mon[62951]: pgmap v1086: 97 pgs: 97 active+clean; 644 MiB data, 9.6 GiB used, 1.0 TiB / 1.0 TiB avail; 2.7 MiB/s rd, 5.3 MiB/s wr, 501 op/s 2024-03-19T02:05:37.870 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:05:36 smithi099 ceph-mon[62951]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T02:05:37.870 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:05:36 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/3983860069' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-03-19T02:05:37.870 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:05:36 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/1086307220' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-03-19T02:05:37.870 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:05:36 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/3495611872' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json-pretty"}]: dispatch 2024-03-19T02:05:38.125 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T02:05:38.246 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:05:37 smithi099 ceph-mon[62951]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T02:05:38.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:05:37 smithi115 ceph-mon[67332]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T02:05:38.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:05:37 smithi112 ceph-mon[66819]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T02:05:39.303 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:05:38 smithi099 ceph-mon[62951]: pgmap v1087: 97 pgs: 97 active+clean; 556 MiB data, 9.4 GiB used, 1.0 TiB / 1.0 TiB avail; 2.6 MiB/s rd, 5.7 MiB/s wr, 519 op/s 2024-03-19T02:05:39.304 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:05:38 smithi099 ceph-mon[62951]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T02:05:39.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:05:38 smithi115 ceph-mon[67332]: pgmap v1087: 97 pgs: 97 active+clean; 556 MiB data, 9.4 GiB used, 1.0 TiB / 1.0 TiB avail; 2.6 MiB/s rd, 5.7 MiB/s wr, 519 op/s 2024-03-19T02:05:39.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:05:38 smithi115 ceph-mon[67332]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T02:05:39.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:05:38 smithi112 ceph-mon[66819]: pgmap v1087: 97 pgs: 97 active+clean; 556 MiB data, 9.4 GiB used, 1.0 TiB / 1.0 TiB avail; 2.6 MiB/s rd, 5.7 MiB/s wr, 519 op/s 2024-03-19T02:05:39.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:05:38 smithi112 ceph-mon[66819]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T02:05:40.008 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2024-03-19T02:05:40.008 INFO:teuthology.orchestra.run.smithi099.stdout: "AsyncMessenger::Worker-0": { 2024-03-19T02:05:40.008 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_recv_messages": 4, 2024-03-19T02:05:40.008 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_send_messages": 4, 2024-03-19T02:05:40.008 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_recv_bytes": 3072, 2024-03-19T02:05:40.008 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_send_bytes": 448, 2024-03-19T02:05:40.008 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_created_connections": 4, 2024-03-19T02:05:40.009 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_active_connections": 0, 2024-03-19T02:05:40.009 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_running_total_time": 0.001509803, 2024-03-19T02:05:40.009 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_running_send_time": 0.000039067, 2024-03-19T02:05:40.009 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_running_recv_time": 0.000573357, 2024-03-19T02:05:40.009 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_running_fast_dispatch_time": 0.000000000, 2024-03-19T02:05:40.009 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_send_messages_queue_lat": { 2024-03-19T02:05:40.009 INFO:teuthology.orchestra.run.smithi099.stdout: "avgcount": 3, 2024-03-19T02:05:40.009 INFO:teuthology.orchestra.run.smithi099.stdout: "sum": 0.000735075, 2024-03-19T02:05:40.009 INFO:teuthology.orchestra.run.smithi099.stdout: "avgtime": 0.000245025 2024-03-19T02:05:40.009 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:40.009 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_handle_ack_lat": { 2024-03-19T02:05:40.009 INFO:teuthology.orchestra.run.smithi099.stdout: "avgcount": 0, 2024-03-19T02:05:40.009 INFO:teuthology.orchestra.run.smithi099.stdout: "sum": 0.000000000, 2024-03-19T02:05:40.010 INFO:teuthology.orchestra.run.smithi099.stdout: "avgtime": 0.000000000 2024-03-19T02:05:40.010 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:40.010 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_recv_encrypted_bytes": 3072, 2024-03-19T02:05:40.010 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_send_encrypted_bytes": 448 2024-03-19T02:05:40.010 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:40.010 INFO:teuthology.orchestra.run.smithi099.stdout: "AsyncMessenger::Worker-1": { 2024-03-19T02:05:40.010 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_recv_messages": 2, 2024-03-19T02:05:40.010 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_send_messages": 327, 2024-03-19T02:05:40.010 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_recv_bytes": 320, 2024-03-19T02:05:40.010 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_send_bytes": 213488, 2024-03-19T02:05:40.010 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_created_connections": 4, 2024-03-19T02:05:40.010 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_active_connections": 2, 2024-03-19T02:05:40.010 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_running_total_time": 0.020090819, 2024-03-19T02:05:40.011 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_running_send_time": 0.015879554, 2024-03-19T02:05:40.011 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_running_recv_time": 0.001642576, 2024-03-19T02:05:40.011 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_running_fast_dispatch_time": 0.000000000, 2024-03-19T02:05:40.011 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_send_messages_queue_lat": { 2024-03-19T02:05:40.011 INFO:teuthology.orchestra.run.smithi099.stdout: "avgcount": 325, 2024-03-19T02:05:40.011 INFO:teuthology.orchestra.run.smithi099.stdout: "sum": 0.011440458, 2024-03-19T02:05:40.011 INFO:teuthology.orchestra.run.smithi099.stdout: "avgtime": 0.000035201 2024-03-19T02:05:40.011 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:40.011 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_handle_ack_lat": { 2024-03-19T02:05:40.011 INFO:teuthology.orchestra.run.smithi099.stdout: "avgcount": 0, 2024-03-19T02:05:40.011 INFO:teuthology.orchestra.run.smithi099.stdout: "sum": 0.000000000, 2024-03-19T02:05:40.011 INFO:teuthology.orchestra.run.smithi099.stdout: "avgtime": 0.000000000 2024-03-19T02:05:40.011 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:40.011 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_recv_encrypted_bytes": 320, 2024-03-19T02:05:40.012 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_send_encrypted_bytes": 213488 2024-03-19T02:05:40.012 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:40.012 INFO:teuthology.orchestra.run.smithi099.stdout: "AsyncMessenger::Worker-2": { 2024-03-19T02:05:40.012 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_recv_messages": 433, 2024-03-19T02:05:40.012 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_send_messages": 428, 2024-03-19T02:05:40.012 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_recv_bytes": 812128, 2024-03-19T02:05:40.012 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_send_bytes": 264896, 2024-03-19T02:05:40.012 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_created_connections": 2, 2024-03-19T02:05:40.012 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_active_connections": 1, 2024-03-19T02:05:40.012 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_running_total_time": 0.051037183, 2024-03-19T02:05:40.012 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_running_send_time": 0.022073391, 2024-03-19T02:05:40.012 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_running_recv_time": 0.020246989, 2024-03-19T02:05:40.012 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_running_fast_dispatch_time": 0.007143937, 2024-03-19T02:05:40.012 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_send_messages_queue_lat": { 2024-03-19T02:05:40.012 INFO:teuthology.orchestra.run.smithi099.stdout: "avgcount": 427, 2024-03-19T02:05:40.013 INFO:teuthology.orchestra.run.smithi099.stdout: "sum": 0.015768311, 2024-03-19T02:05:40.013 INFO:teuthology.orchestra.run.smithi099.stdout: "avgtime": 0.000036928 2024-03-19T02:05:40.013 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:40.013 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_handle_ack_lat": { 2024-03-19T02:05:40.013 INFO:teuthology.orchestra.run.smithi099.stdout: "avgcount": 0, 2024-03-19T02:05:40.013 INFO:teuthology.orchestra.run.smithi099.stdout: "sum": 0.000000000, 2024-03-19T02:05:40.013 INFO:teuthology.orchestra.run.smithi099.stdout: "avgtime": 0.000000000 2024-03-19T02:05:40.013 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:40.013 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_recv_encrypted_bytes": 812128, 2024-03-19T02:05:40.013 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_send_encrypted_bytes": 264896 2024-03-19T02:05:40.013 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:40.013 INFO:teuthology.orchestra.run.smithi099.stdout: "cct": { 2024-03-19T02:05:40.013 INFO:teuthology.orchestra.run.smithi099.stdout: "total_workers": 0, 2024-03-19T02:05:40.013 INFO:teuthology.orchestra.run.smithi099.stdout: "unhealthy_workers": 0 2024-03-19T02:05:40.013 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:40.014 INFO:teuthology.orchestra.run.smithi099.stdout: "mempool": { 2024-03-19T02:05:40.014 INFO:teuthology.orchestra.run.smithi099.stdout: "bloom_filter_bytes": 0, 2024-03-19T02:05:40.014 INFO:teuthology.orchestra.run.smithi099.stdout: "bloom_filter_items": 0, 2024-03-19T02:05:40.014 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_alloc_bytes": 0, 2024-03-19T02:05:40.014 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_alloc_items": 0, 2024-03-19T02:05:40.014 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cache_data_bytes": 0, 2024-03-19T02:05:40.014 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cache_data_items": 0, 2024-03-19T02:05:40.014 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cache_onode_bytes": 0, 2024-03-19T02:05:40.014 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cache_onode_items": 0, 2024-03-19T02:05:40.014 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cache_meta_bytes": 0, 2024-03-19T02:05:40.015 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cache_meta_items": 0, 2024-03-19T02:05:40.015 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cache_other_bytes": 0, 2024-03-19T02:05:40.015 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cache_other_items": 0, 2024-03-19T02:05:40.015 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cache_buffer_bytes": 0, 2024-03-19T02:05:40.015 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cache_buffer_items": 0, 2024-03-19T02:05:40.015 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_extent_bytes": 0, 2024-03-19T02:05:40.015 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_extent_items": 0, 2024-03-19T02:05:40.015 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_blob_bytes": 0, 2024-03-19T02:05:40.015 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_blob_items": 0, 2024-03-19T02:05:40.015 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_shared_blob_bytes": 0, 2024-03-19T02:05:40.015 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_shared_blob_items": 0, 2024-03-19T02:05:40.015 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_inline_bl_bytes": 0, 2024-03-19T02:05:40.015 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_inline_bl_items": 0, 2024-03-19T02:05:40.015 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_fsck_bytes": 0, 2024-03-19T02:05:40.016 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_fsck_items": 0, 2024-03-19T02:05:40.016 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_txc_bytes": 0, 2024-03-19T02:05:40.016 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_txc_items": 0, 2024-03-19T02:05:40.016 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_writing_deferred_bytes": 0, 2024-03-19T02:05:40.016 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_writing_deferred_items": 0, 2024-03-19T02:05:40.016 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_writing_bytes": 0, 2024-03-19T02:05:40.016 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_writing_items": 0, 2024-03-19T02:05:40.016 INFO:teuthology.orchestra.run.smithi099.stdout: "bluefs_bytes": 0, 2024-03-19T02:05:40.016 INFO:teuthology.orchestra.run.smithi099.stdout: "bluefs_items": 0, 2024-03-19T02:05:40.016 INFO:teuthology.orchestra.run.smithi099.stdout: "bluefs_file_reader_bytes": 0, 2024-03-19T02:05:40.016 INFO:teuthology.orchestra.run.smithi099.stdout: "bluefs_file_reader_items": 0, 2024-03-19T02:05:40.016 INFO:teuthology.orchestra.run.smithi099.stdout: "bluefs_file_writer_bytes": 0, 2024-03-19T02:05:40.016 INFO:teuthology.orchestra.run.smithi099.stdout: "bluefs_file_writer_items": 0, 2024-03-19T02:05:40.016 INFO:teuthology.orchestra.run.smithi099.stdout: "buffer_anon_bytes": 148761, 2024-03-19T02:05:40.017 INFO:teuthology.orchestra.run.smithi099.stdout: "buffer_anon_items": 31, 2024-03-19T02:05:40.017 INFO:teuthology.orchestra.run.smithi099.stdout: "buffer_meta_bytes": 0, 2024-03-19T02:05:40.017 INFO:teuthology.orchestra.run.smithi099.stdout: "buffer_meta_items": 0, 2024-03-19T02:05:40.017 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_bytes": 0, 2024-03-19T02:05:40.017 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_items": 0, 2024-03-19T02:05:40.017 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_mapbl_bytes": 0, 2024-03-19T02:05:40.017 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_mapbl_items": 0, 2024-03-19T02:05:40.017 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_pglog_bytes": 0, 2024-03-19T02:05:40.017 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_pglog_items": 0, 2024-03-19T02:05:40.017 INFO:teuthology.orchestra.run.smithi099.stdout: "osdmap_bytes": 0, 2024-03-19T02:05:40.017 INFO:teuthology.orchestra.run.smithi099.stdout: "osdmap_items": 0, 2024-03-19T02:05:40.018 INFO:teuthology.orchestra.run.smithi099.stdout: "osdmap_mapping_bytes": 0, 2024-03-19T02:05:40.018 INFO:teuthology.orchestra.run.smithi099.stdout: "osdmap_mapping_items": 0, 2024-03-19T02:05:40.018 INFO:teuthology.orchestra.run.smithi099.stdout: "pgmap_bytes": 0, 2024-03-19T02:05:40.018 INFO:teuthology.orchestra.run.smithi099.stdout: "pgmap_items": 0, 2024-03-19T02:05:40.018 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_co_bytes": 928, 2024-03-19T02:05:40.018 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_co_items": 2, 2024-03-19T02:05:40.018 INFO:teuthology.orchestra.run.smithi099.stdout: "unittest_1_bytes": 0, 2024-03-19T02:05:40.018 INFO:teuthology.orchestra.run.smithi099.stdout: "unittest_1_items": 0, 2024-03-19T02:05:40.018 INFO:teuthology.orchestra.run.smithi099.stdout: "unittest_2_bytes": 0, 2024-03-19T02:05:40.018 INFO:teuthology.orchestra.run.smithi099.stdout: "unittest_2_items": 0 2024-03-19T02:05:40.018 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:40.018 INFO:teuthology.orchestra.run.smithi099.stdout: "throttle-msgr_dispatch_throttler-mds": { 2024-03-19T02:05:40.018 INFO:teuthology.orchestra.run.smithi099.stdout: "val": 0, 2024-03-19T02:05:40.018 INFO:teuthology.orchestra.run.smithi099.stdout: "max": 104857600, 2024-03-19T02:05:40.019 INFO:teuthology.orchestra.run.smithi099.stdout: "get_started": 0, 2024-03-19T02:05:40.019 INFO:teuthology.orchestra.run.smithi099.stdout: "get": 435, 2024-03-19T02:05:40.019 INFO:teuthology.orchestra.run.smithi099.stdout: "get_sum": 750913, 2024-03-19T02:05:40.019 INFO:teuthology.orchestra.run.smithi099.stdout: "get_or_fail_fail": 0, 2024-03-19T02:05:40.019 INFO:teuthology.orchestra.run.smithi099.stdout: "get_or_fail_success": 435, 2024-03-19T02:05:40.019 INFO:teuthology.orchestra.run.smithi099.stdout: "take": 0, 2024-03-19T02:05:40.019 INFO:teuthology.orchestra.run.smithi099.stdout: "take_sum": 0, 2024-03-19T02:05:40.019 INFO:teuthology.orchestra.run.smithi099.stdout: "put": 435, 2024-03-19T02:05:40.019 INFO:teuthology.orchestra.run.smithi099.stdout: "put_sum": 750913, 2024-03-19T02:05:40.019 INFO:teuthology.orchestra.run.smithi099.stdout: "wait": { 2024-03-19T02:05:40.019 INFO:teuthology.orchestra.run.smithi099.stdout: "avgcount": 0, 2024-03-19T02:05:40.019 INFO:teuthology.orchestra.run.smithi099.stdout: "sum": 0.000000000, 2024-03-19T02:05:40.019 INFO:teuthology.orchestra.run.smithi099.stdout: "avgtime": 0.000000000 2024-03-19T02:05:40.019 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:40.020 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:40.020 INFO:teuthology.orchestra.run.smithi099.stdout:} 2024-03-19T02:05:40.904 WARNING:tasks.check_counter:Counter 'mds.exported' not found on daemon mds.a 2024-03-19T02:05:40.904 WARNING:tasks.check_counter:Counter 'mds.imported' not found on daemon mds.a 2024-03-19T02:05:40.904 WARNING:tasks.check_counter:Counter 'mds_cache.dir_update' not found on daemon mds.a 2024-03-19T02:05:40.904 WARNING:tasks.check_counter:Counter 'mds_cache.dir_update_receipt' not found on daemon mds.a 2024-03-19T02:05:40.904 DEBUG:tasks.check_counter:Getting stats from d 2024-03-19T02:05:40.904 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph daemon mds.d perf dump 2024-03-19T02:05:41.147 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T02:05:41.270 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:05:40 smithi099 ceph-mon[62951]: pgmap v1088: 97 pgs: 97 active+clean; 510 MiB data, 9.3 GiB used, 1.0 TiB / 1.0 TiB avail; 1.9 MiB/s rd, 5.4 MiB/s wr, 496 op/s 2024-03-19T02:05:41.312 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:05:40 smithi115 ceph-mon[67332]: pgmap v1088: 97 pgs: 97 active+clean; 510 MiB data, 9.3 GiB used, 1.0 TiB / 1.0 TiB avail; 1.9 MiB/s rd, 5.4 MiB/s wr, 496 op/s 2024-03-19T02:05:41.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:05:40 smithi112 ceph-mon[66819]: pgmap v1088: 97 pgs: 97 active+clean; 510 MiB data, 9.3 GiB used, 1.0 TiB / 1.0 TiB avail; 1.9 MiB/s rd, 5.4 MiB/s wr, 496 op/s 2024-03-19T02:05:42.241 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:05:41 smithi099 ceph-mon[62951]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T02:05:42.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:05:41 smithi115 ceph-mon[67332]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T02:05:42.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:05:41 smithi112 ceph-mon[66819]: fsmap cephfs:3 {0=i=up:active,1=b=up:active,2=c=up:active} 3 up:standby-replay 6 up:standby 2024-03-19T02:05:42.794 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2024-03-19T02:05:42.794 INFO:teuthology.orchestra.run.smithi099.stdout: "AsyncMessenger::Worker-0": { 2024-03-19T02:05:42.794 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_recv_messages": 0, 2024-03-19T02:05:42.795 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_send_messages": 2, 2024-03-19T02:05:42.795 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_recv_bytes": 0, 2024-03-19T02:05:42.795 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_send_bytes": 0, 2024-03-19T02:05:42.795 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_created_connections": 4, 2024-03-19T02:05:42.795 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_active_connections": 0, 2024-03-19T02:05:42.795 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_running_total_time": 0.001205664, 2024-03-19T02:05:42.795 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_running_send_time": 0.000000000, 2024-03-19T02:05:42.795 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_running_recv_time": 0.000248672, 2024-03-19T02:05:42.795 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_running_fast_dispatch_time": 0.000000000, 2024-03-19T02:05:42.795 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_send_messages_queue_lat": { 2024-03-19T02:05:42.795 INFO:teuthology.orchestra.run.smithi099.stdout: "avgcount": 0, 2024-03-19T02:05:42.795 INFO:teuthology.orchestra.run.smithi099.stdout: "sum": 0.000000000, 2024-03-19T02:05:42.795 INFO:teuthology.orchestra.run.smithi099.stdout: "avgtime": 0.000000000 2024-03-19T02:05:42.796 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:42.796 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_handle_ack_lat": { 2024-03-19T02:05:42.796 INFO:teuthology.orchestra.run.smithi099.stdout: "avgcount": 0, 2024-03-19T02:05:42.796 INFO:teuthology.orchestra.run.smithi099.stdout: "sum": 0.000000000, 2024-03-19T02:05:42.796 INFO:teuthology.orchestra.run.smithi099.stdout: "avgtime": 0.000000000 2024-03-19T02:05:42.796 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:42.796 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_recv_encrypted_bytes": 0, 2024-03-19T02:05:42.796 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_send_encrypted_bytes": 0 2024-03-19T02:05:42.796 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:42.796 INFO:teuthology.orchestra.run.smithi099.stdout: "AsyncMessenger::Worker-1": { 2024-03-19T02:05:42.796 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_recv_messages": 443, 2024-03-19T02:05:42.796 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_send_messages": 436, 2024-03-19T02:05:42.796 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_recv_bytes": 817456, 2024-03-19T02:05:42.796 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_send_bytes": 268880, 2024-03-19T02:05:42.796 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_created_connections": 2, 2024-03-19T02:05:42.797 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_active_connections": 1, 2024-03-19T02:05:42.797 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_running_total_time": 0.050365621, 2024-03-19T02:05:42.797 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_running_send_time": 0.017242086, 2024-03-19T02:05:42.797 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_running_recv_time": 0.025115446, 2024-03-19T02:05:42.797 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_running_fast_dispatch_time": 0.007891960, 2024-03-19T02:05:42.797 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_send_messages_queue_lat": { 2024-03-19T02:05:42.797 INFO:teuthology.orchestra.run.smithi099.stdout: "avgcount": 436, 2024-03-19T02:05:42.797 INFO:teuthology.orchestra.run.smithi099.stdout: "sum": 0.012695832, 2024-03-19T02:05:42.797 INFO:teuthology.orchestra.run.smithi099.stdout: "avgtime": 0.000029118 2024-03-19T02:05:42.797 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:42.797 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_handle_ack_lat": { 2024-03-19T02:05:42.797 INFO:teuthology.orchestra.run.smithi099.stdout: "avgcount": 0, 2024-03-19T02:05:42.797 INFO:teuthology.orchestra.run.smithi099.stdout: "sum": 0.000000000, 2024-03-19T02:05:42.797 INFO:teuthology.orchestra.run.smithi099.stdout: "avgtime": 0.000000000 2024-03-19T02:05:42.798 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:42.798 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_recv_encrypted_bytes": 817456, 2024-03-19T02:05:42.798 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_send_encrypted_bytes": 268880 2024-03-19T02:05:42.798 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:42.798 INFO:teuthology.orchestra.run.smithi099.stdout: "AsyncMessenger::Worker-2": { 2024-03-19T02:05:42.798 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_recv_messages": 2, 2024-03-19T02:05:42.798 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_send_messages": 331, 2024-03-19T02:05:42.798 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_recv_bytes": 320, 2024-03-19T02:05:42.798 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_send_bytes": 214192, 2024-03-19T02:05:42.798 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_created_connections": 4, 2024-03-19T02:05:42.798 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_active_connections": 2, 2024-03-19T02:05:42.798 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_running_total_time": 0.020634263, 2024-03-19T02:05:42.799 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_running_send_time": 0.016088776, 2024-03-19T02:05:42.799 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_running_recv_time": 0.001988389, 2024-03-19T02:05:42.799 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_running_fast_dispatch_time": 0.000000000, 2024-03-19T02:05:42.799 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_send_messages_queue_lat": { 2024-03-19T02:05:42.799 INFO:teuthology.orchestra.run.smithi099.stdout: "avgcount": 329, 2024-03-19T02:05:42.799 INFO:teuthology.orchestra.run.smithi099.stdout: "sum": 0.011818554, 2024-03-19T02:05:42.799 INFO:teuthology.orchestra.run.smithi099.stdout: "avgtime": 0.000035922 2024-03-19T02:05:42.799 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:42.799 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_handle_ack_lat": { 2024-03-19T02:05:42.799 INFO:teuthology.orchestra.run.smithi099.stdout: "avgcount": 0, 2024-03-19T02:05:42.799 INFO:teuthology.orchestra.run.smithi099.stdout: "sum": 0.000000000, 2024-03-19T02:05:42.799 INFO:teuthology.orchestra.run.smithi099.stdout: "avgtime": 0.000000000 2024-03-19T02:05:42.799 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:42.800 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_recv_encrypted_bytes": 320, 2024-03-19T02:05:42.800 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_send_encrypted_bytes": 214192 2024-03-19T02:05:42.800 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:42.800 INFO:teuthology.orchestra.run.smithi099.stdout: "cct": { 2024-03-19T02:05:42.800 INFO:teuthology.orchestra.run.smithi099.stdout: "total_workers": 0, 2024-03-19T02:05:42.800 INFO:teuthology.orchestra.run.smithi099.stdout: "unhealthy_workers": 0 2024-03-19T02:05:42.800 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:42.800 INFO:teuthology.orchestra.run.smithi099.stdout: "mempool": { 2024-03-19T02:05:42.800 INFO:teuthology.orchestra.run.smithi099.stdout: "bloom_filter_bytes": 0, 2024-03-19T02:05:42.800 INFO:teuthology.orchestra.run.smithi099.stdout: "bloom_filter_items": 0, 2024-03-19T02:05:42.800 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_alloc_bytes": 0, 2024-03-19T02:05:42.800 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_alloc_items": 0, 2024-03-19T02:05:42.800 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cache_data_bytes": 0, 2024-03-19T02:05:42.800 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cache_data_items": 0, 2024-03-19T02:05:42.801 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cache_onode_bytes": 0, 2024-03-19T02:05:42.801 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cache_onode_items": 0, 2024-03-19T02:05:42.801 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cache_meta_bytes": 0, 2024-03-19T02:05:42.801 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cache_meta_items": 0, 2024-03-19T02:05:42.801 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cache_other_bytes": 0, 2024-03-19T02:05:42.801 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cache_other_items": 0, 2024-03-19T02:05:42.801 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cache_buffer_bytes": 0, 2024-03-19T02:05:42.801 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cache_buffer_items": 0, 2024-03-19T02:05:42.801 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_extent_bytes": 0, 2024-03-19T02:05:42.801 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_extent_items": 0, 2024-03-19T02:05:42.801 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_blob_bytes": 0, 2024-03-19T02:05:42.801 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_blob_items": 0, 2024-03-19T02:05:42.801 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_shared_blob_bytes": 0, 2024-03-19T02:05:42.801 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_shared_blob_items": 0, 2024-03-19T02:05:42.801 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_inline_bl_bytes": 0, 2024-03-19T02:05:42.802 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_inline_bl_items": 0, 2024-03-19T02:05:42.802 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_fsck_bytes": 0, 2024-03-19T02:05:42.802 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_fsck_items": 0, 2024-03-19T02:05:42.802 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_txc_bytes": 0, 2024-03-19T02:05:42.802 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_txc_items": 0, 2024-03-19T02:05:42.802 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_writing_deferred_bytes": 0, 2024-03-19T02:05:42.802 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_writing_deferred_items": 0, 2024-03-19T02:05:42.802 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_writing_bytes": 0, 2024-03-19T02:05:42.802 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_writing_items": 0, 2024-03-19T02:05:42.802 INFO:teuthology.orchestra.run.smithi099.stdout: "bluefs_bytes": 0, 2024-03-19T02:05:42.802 INFO:teuthology.orchestra.run.smithi099.stdout: "bluefs_items": 0, 2024-03-19T02:05:42.802 INFO:teuthology.orchestra.run.smithi099.stdout: "bluefs_file_reader_bytes": 0, 2024-03-19T02:05:42.802 INFO:teuthology.orchestra.run.smithi099.stdout: "bluefs_file_reader_items": 0, 2024-03-19T02:05:42.802 INFO:teuthology.orchestra.run.smithi099.stdout: "bluefs_file_writer_bytes": 0, 2024-03-19T02:05:42.802 INFO:teuthology.orchestra.run.smithi099.stdout: "bluefs_file_writer_items": 0, 2024-03-19T02:05:42.803 INFO:teuthology.orchestra.run.smithi099.stdout: "buffer_anon_bytes": 148704, 2024-03-19T02:05:42.803 INFO:teuthology.orchestra.run.smithi099.stdout: "buffer_anon_items": 29, 2024-03-19T02:05:42.803 INFO:teuthology.orchestra.run.smithi099.stdout: "buffer_meta_bytes": 0, 2024-03-19T02:05:42.803 INFO:teuthology.orchestra.run.smithi099.stdout: "buffer_meta_items": 0, 2024-03-19T02:05:42.803 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_bytes": 0, 2024-03-19T02:05:42.803 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_items": 0, 2024-03-19T02:05:42.803 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_mapbl_bytes": 0, 2024-03-19T02:05:42.803 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_mapbl_items": 0, 2024-03-19T02:05:42.803 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_pglog_bytes": 0, 2024-03-19T02:05:42.803 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_pglog_items": 0, 2024-03-19T02:05:42.803 INFO:teuthology.orchestra.run.smithi099.stdout: "osdmap_bytes": 0, 2024-03-19T02:05:42.803 INFO:teuthology.orchestra.run.smithi099.stdout: "osdmap_items": 0, 2024-03-19T02:05:42.803 INFO:teuthology.orchestra.run.smithi099.stdout: "osdmap_mapping_bytes": 0, 2024-03-19T02:05:42.803 INFO:teuthology.orchestra.run.smithi099.stdout: "osdmap_mapping_items": 0, 2024-03-19T02:05:42.803 INFO:teuthology.orchestra.run.smithi099.stdout: "pgmap_bytes": 0, 2024-03-19T02:05:42.804 INFO:teuthology.orchestra.run.smithi099.stdout: "pgmap_items": 0, 2024-03-19T02:05:42.804 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_co_bytes": 928, 2024-03-19T02:05:42.804 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_co_items": 2, 2024-03-19T02:05:42.804 INFO:teuthology.orchestra.run.smithi099.stdout: "unittest_1_bytes": 0, 2024-03-19T02:05:42.804 INFO:teuthology.orchestra.run.smithi099.stdout: "unittest_1_items": 0, 2024-03-19T02:05:42.804 INFO:teuthology.orchestra.run.smithi099.stdout: "unittest_2_bytes": 0, 2024-03-19T02:05:42.804 INFO:teuthology.orchestra.run.smithi099.stdout: "unittest_2_items": 0 2024-03-19T02:05:42.804 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:42.804 INFO:teuthology.orchestra.run.smithi099.stdout: "throttle-msgr_dispatch_throttler-mds": { 2024-03-19T02:05:42.804 INFO:teuthology.orchestra.run.smithi099.stdout: "val": 0, 2024-03-19T02:05:42.804 INFO:teuthology.orchestra.run.smithi099.stdout: "max": 104857600, 2024-03-19T02:05:42.804 INFO:teuthology.orchestra.run.smithi099.stdout: "get_started": 0, 2024-03-19T02:05:42.804 INFO:teuthology.orchestra.run.smithi099.stdout: "get": 441, 2024-03-19T02:05:42.804 INFO:teuthology.orchestra.run.smithi099.stdout: "get_sum": 752330, 2024-03-19T02:05:42.804 INFO:teuthology.orchestra.run.smithi099.stdout: "get_or_fail_fail": 0, 2024-03-19T02:05:42.805 INFO:teuthology.orchestra.run.smithi099.stdout: "get_or_fail_success": 441, 2024-03-19T02:05:42.805 INFO:teuthology.orchestra.run.smithi099.stdout: "take": 0, 2024-03-19T02:05:42.805 INFO:teuthology.orchestra.run.smithi099.stdout: "take_sum": 0, 2024-03-19T02:05:42.805 INFO:teuthology.orchestra.run.smithi099.stdout: "put": 441, 2024-03-19T02:05:42.805 INFO:teuthology.orchestra.run.smithi099.stdout: "put_sum": 752330, 2024-03-19T02:05:42.805 INFO:teuthology.orchestra.run.smithi099.stdout: "wait": { 2024-03-19T02:05:42.805 INFO:teuthology.orchestra.run.smithi099.stdout: "avgcount": 0, 2024-03-19T02:05:42.805 INFO:teuthology.orchestra.run.smithi099.stdout: "sum": 0.000000000, 2024-03-19T02:05:42.805 INFO:teuthology.orchestra.run.smithi099.stdout: "avgtime": 0.000000000 2024-03-19T02:05:42.805 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:42.805 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:42.805 INFO:teuthology.orchestra.run.smithi099.stdout:} 2024-03-19T02:05:43.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:05:42 smithi099 ceph-mon[62951]: pgmap v1089: 97 pgs: 97 active+clean; 502 MiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 1.4 MiB/s rd, 4.6 MiB/s wr, 432 op/s 2024-03-19T02:05:43.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:05:42 smithi115 ceph-mon[67332]: pgmap v1089: 97 pgs: 97 active+clean; 502 MiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 1.4 MiB/s rd, 4.6 MiB/s wr, 432 op/s 2024-03-19T02:05:43.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:05:42 smithi112 ceph-mon[66819]: pgmap v1089: 97 pgs: 97 active+clean; 502 MiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 1.4 MiB/s rd, 4.6 MiB/s wr, 432 op/s 2024-03-19T02:05:43.453 WARNING:tasks.check_counter:Counter 'mds.exported' not found on daemon mds.d 2024-03-19T02:05:43.453 WARNING:tasks.check_counter:Counter 'mds.imported' not found on daemon mds.d 2024-03-19T02:05:43.453 WARNING:tasks.check_counter:Counter 'mds_cache.dir_update' not found on daemon mds.d 2024-03-19T02:05:43.453 WARNING:tasks.check_counter:Counter 'mds_cache.dir_update_receipt' not found on daemon mds.d 2024-03-19T02:05:43.453 DEBUG:tasks.check_counter:Getting stats from g 2024-03-19T02:05:43.454 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph daemon mds.g perf dump 2024-03-19T02:05:43.697 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T02:05:45.034 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:05:44 smithi099 ceph-mon[62951]: pgmap v1090: 97 pgs: 97 active+clean; 500 MiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 4.7 MiB/s wr, 348 op/s 2024-03-19T02:05:45.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:05:44 smithi115 ceph-mon[67332]: pgmap v1090: 97 pgs: 97 active+clean; 500 MiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 4.7 MiB/s wr, 348 op/s 2024-03-19T02:05:45.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:05:44 smithi112 ceph-mon[66819]: pgmap v1090: 97 pgs: 97 active+clean; 500 MiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 4.7 MiB/s wr, 348 op/s 2024-03-19T02:05:45.438 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2024-03-19T02:05:45.439 INFO:teuthology.orchestra.run.smithi099.stdout: "AsyncMessenger::Worker-0": { 2024-03-19T02:05:45.439 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_recv_messages": 4, 2024-03-19T02:05:45.439 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_send_messages": 4, 2024-03-19T02:05:45.439 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_recv_bytes": 3072, 2024-03-19T02:05:45.439 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_send_bytes": 448, 2024-03-19T02:05:45.439 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_created_connections": 4, 2024-03-19T02:05:45.439 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_active_connections": 0, 2024-03-19T02:05:45.439 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_running_total_time": 0.001412673, 2024-03-19T02:05:45.439 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_running_send_time": 0.000030777, 2024-03-19T02:05:45.439 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_running_recv_time": 0.000533011, 2024-03-19T02:05:45.439 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_running_fast_dispatch_time": 0.000000000, 2024-03-19T02:05:45.439 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_send_messages_queue_lat": { 2024-03-19T02:05:45.439 INFO:teuthology.orchestra.run.smithi099.stdout: "avgcount": 3, 2024-03-19T02:05:45.440 INFO:teuthology.orchestra.run.smithi099.stdout: "sum": 0.000666111, 2024-03-19T02:05:45.440 INFO:teuthology.orchestra.run.smithi099.stdout: "avgtime": 0.000222037 2024-03-19T02:05:45.440 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:45.440 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_handle_ack_lat": { 2024-03-19T02:05:45.440 INFO:teuthology.orchestra.run.smithi099.stdout: "avgcount": 0, 2024-03-19T02:05:45.440 INFO:teuthology.orchestra.run.smithi099.stdout: "sum": 0.000000000, 2024-03-19T02:05:45.440 INFO:teuthology.orchestra.run.smithi099.stdout: "avgtime": 0.000000000 2024-03-19T02:05:45.440 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:45.440 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_recv_encrypted_bytes": 3072, 2024-03-19T02:05:45.440 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_send_encrypted_bytes": 448 2024-03-19T02:05:45.440 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:45.440 INFO:teuthology.orchestra.run.smithi099.stdout: "AsyncMessenger::Worker-1": { 2024-03-19T02:05:45.440 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_recv_messages": 436, 2024-03-19T02:05:45.440 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_send_messages": 431, 2024-03-19T02:05:45.440 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_recv_bytes": 813568, 2024-03-19T02:05:45.441 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_send_bytes": 266512, 2024-03-19T02:05:45.441 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_created_connections": 2, 2024-03-19T02:05:45.441 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_active_connections": 1, 2024-03-19T02:05:45.441 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_running_total_time": 0.052902273, 2024-03-19T02:05:45.441 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_running_send_time": 0.022949739, 2024-03-19T02:05:45.441 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_running_recv_time": 0.021274226, 2024-03-19T02:05:45.441 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_running_fast_dispatch_time": 0.007377198, 2024-03-19T02:05:45.441 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_send_messages_queue_lat": { 2024-03-19T02:05:45.441 INFO:teuthology.orchestra.run.smithi099.stdout: "avgcount": 430, 2024-03-19T02:05:45.441 INFO:teuthology.orchestra.run.smithi099.stdout: "sum": 0.016531463, 2024-03-19T02:05:45.441 INFO:teuthology.orchestra.run.smithi099.stdout: "avgtime": 0.000038445 2024-03-19T02:05:45.441 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:45.441 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_handle_ack_lat": { 2024-03-19T02:05:45.441 INFO:teuthology.orchestra.run.smithi099.stdout: "avgcount": 0, 2024-03-19T02:05:45.441 INFO:teuthology.orchestra.run.smithi099.stdout: "sum": 0.000000000, 2024-03-19T02:05:45.441 INFO:teuthology.orchestra.run.smithi099.stdout: "avgtime": 0.000000000 2024-03-19T02:05:45.442 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:45.442 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_recv_encrypted_bytes": 813568, 2024-03-19T02:05:45.442 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_send_encrypted_bytes": 266512 2024-03-19T02:05:45.442 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:45.442 INFO:teuthology.orchestra.run.smithi099.stdout: "AsyncMessenger::Worker-2": { 2024-03-19T02:05:45.442 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_recv_messages": 2, 2024-03-19T02:05:45.442 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_send_messages": 329, 2024-03-19T02:05:45.442 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_recv_bytes": 320, 2024-03-19T02:05:45.442 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_send_bytes": 213840, 2024-03-19T02:05:45.442 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_created_connections": 4, 2024-03-19T02:05:45.442 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_active_connections": 2, 2024-03-19T02:05:45.442 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_running_total_time": 0.018991306, 2024-03-19T02:05:45.442 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_running_send_time": 0.014524700, 2024-03-19T02:05:45.442 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_running_recv_time": 0.001911852, 2024-03-19T02:05:45.443 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_running_fast_dispatch_time": 0.000000000, 2024-03-19T02:05:45.443 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_send_messages_queue_lat": { 2024-03-19T02:05:45.443 INFO:teuthology.orchestra.run.smithi099.stdout: "avgcount": 327, 2024-03-19T02:05:45.443 INFO:teuthology.orchestra.run.smithi099.stdout: "sum": 0.011571923, 2024-03-19T02:05:45.443 INFO:teuthology.orchestra.run.smithi099.stdout: "avgtime": 0.000035388 2024-03-19T02:05:45.443 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:45.443 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_handle_ack_lat": { 2024-03-19T02:05:45.443 INFO:teuthology.orchestra.run.smithi099.stdout: "avgcount": 0, 2024-03-19T02:05:45.443 INFO:teuthology.orchestra.run.smithi099.stdout: "sum": 0.000000000, 2024-03-19T02:05:45.443 INFO:teuthology.orchestra.run.smithi099.stdout: "avgtime": 0.000000000 2024-03-19T02:05:45.443 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:45.443 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_recv_encrypted_bytes": 320, 2024-03-19T02:05:45.443 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_send_encrypted_bytes": 213840 2024-03-19T02:05:45.443 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:45.443 INFO:teuthology.orchestra.run.smithi099.stdout: "cct": { 2024-03-19T02:05:45.444 INFO:teuthology.orchestra.run.smithi099.stdout: "total_workers": 0, 2024-03-19T02:05:45.444 INFO:teuthology.orchestra.run.smithi099.stdout: "unhealthy_workers": 0 2024-03-19T02:05:45.444 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:45.444 INFO:teuthology.orchestra.run.smithi099.stdout: "mempool": { 2024-03-19T02:05:45.444 INFO:teuthology.orchestra.run.smithi099.stdout: "bloom_filter_bytes": 0, 2024-03-19T02:05:45.444 INFO:teuthology.orchestra.run.smithi099.stdout: "bloom_filter_items": 0, 2024-03-19T02:05:45.444 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_alloc_bytes": 0, 2024-03-19T02:05:45.444 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_alloc_items": 0, 2024-03-19T02:05:45.444 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cache_data_bytes": 0, 2024-03-19T02:05:45.444 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cache_data_items": 0, 2024-03-19T02:05:45.444 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cache_onode_bytes": 0, 2024-03-19T02:05:45.444 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cache_onode_items": 0, 2024-03-19T02:05:45.444 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cache_meta_bytes": 0, 2024-03-19T02:05:45.444 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cache_meta_items": 0, 2024-03-19T02:05:45.444 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cache_other_bytes": 0, 2024-03-19T02:05:45.445 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cache_other_items": 0, 2024-03-19T02:05:45.445 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cache_buffer_bytes": 0, 2024-03-19T02:05:45.445 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cache_buffer_items": 0, 2024-03-19T02:05:45.445 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_extent_bytes": 0, 2024-03-19T02:05:45.445 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_extent_items": 0, 2024-03-19T02:05:45.445 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_blob_bytes": 0, 2024-03-19T02:05:45.445 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_blob_items": 0, 2024-03-19T02:05:45.445 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_shared_blob_bytes": 0, 2024-03-19T02:05:45.445 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_shared_blob_items": 0, 2024-03-19T02:05:45.445 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_inline_bl_bytes": 0, 2024-03-19T02:05:45.445 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_inline_bl_items": 0, 2024-03-19T02:05:45.445 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_fsck_bytes": 0, 2024-03-19T02:05:45.445 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_fsck_items": 0, 2024-03-19T02:05:45.445 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_txc_bytes": 0, 2024-03-19T02:05:45.445 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_txc_items": 0, 2024-03-19T02:05:45.446 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_writing_deferred_bytes": 0, 2024-03-19T02:05:45.446 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_writing_deferred_items": 0, 2024-03-19T02:05:45.446 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_writing_bytes": 0, 2024-03-19T02:05:45.446 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_writing_items": 0, 2024-03-19T02:05:45.446 INFO:teuthology.orchestra.run.smithi099.stdout: "bluefs_bytes": 0, 2024-03-19T02:05:45.446 INFO:teuthology.orchestra.run.smithi099.stdout: "bluefs_items": 0, 2024-03-19T02:05:45.446 INFO:teuthology.orchestra.run.smithi099.stdout: "bluefs_file_reader_bytes": 0, 2024-03-19T02:05:45.446 INFO:teuthology.orchestra.run.smithi099.stdout: "bluefs_file_reader_items": 0, 2024-03-19T02:05:45.446 INFO:teuthology.orchestra.run.smithi099.stdout: "bluefs_file_writer_bytes": 0, 2024-03-19T02:05:45.446 INFO:teuthology.orchestra.run.smithi099.stdout: "bluefs_file_writer_items": 0, 2024-03-19T02:05:45.446 INFO:teuthology.orchestra.run.smithi099.stdout: "buffer_anon_bytes": 148721, 2024-03-19T02:05:45.446 INFO:teuthology.orchestra.run.smithi099.stdout: "buffer_anon_items": 30, 2024-03-19T02:05:45.446 INFO:teuthology.orchestra.run.smithi099.stdout: "buffer_meta_bytes": 0, 2024-03-19T02:05:45.446 INFO:teuthology.orchestra.run.smithi099.stdout: "buffer_meta_items": 0, 2024-03-19T02:05:45.447 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_bytes": 0, 2024-03-19T02:05:45.447 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_items": 0, 2024-03-19T02:05:45.447 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_mapbl_bytes": 0, 2024-03-19T02:05:45.447 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_mapbl_items": 0, 2024-03-19T02:05:45.447 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_pglog_bytes": 0, 2024-03-19T02:05:45.447 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_pglog_items": 0, 2024-03-19T02:05:45.447 INFO:teuthology.orchestra.run.smithi099.stdout: "osdmap_bytes": 0, 2024-03-19T02:05:45.447 INFO:teuthology.orchestra.run.smithi099.stdout: "osdmap_items": 0, 2024-03-19T02:05:45.447 INFO:teuthology.orchestra.run.smithi099.stdout: "osdmap_mapping_bytes": 0, 2024-03-19T02:05:45.447 INFO:teuthology.orchestra.run.smithi099.stdout: "osdmap_mapping_items": 0, 2024-03-19T02:05:45.447 INFO:teuthology.orchestra.run.smithi099.stdout: "pgmap_bytes": 0, 2024-03-19T02:05:45.447 INFO:teuthology.orchestra.run.smithi099.stdout: "pgmap_items": 0, 2024-03-19T02:05:45.447 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_co_bytes": 928, 2024-03-19T02:05:45.447 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_co_items": 2, 2024-03-19T02:05:45.447 INFO:teuthology.orchestra.run.smithi099.stdout: "unittest_1_bytes": 0, 2024-03-19T02:05:45.447 INFO:teuthology.orchestra.run.smithi099.stdout: "unittest_1_items": 0, 2024-03-19T02:05:45.447 INFO:teuthology.orchestra.run.smithi099.stdout: "unittest_2_bytes": 0, 2024-03-19T02:05:45.448 INFO:teuthology.orchestra.run.smithi099.stdout: "unittest_2_items": 0 2024-03-19T02:05:45.448 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:45.448 INFO:teuthology.orchestra.run.smithi099.stdout: "throttle-msgr_dispatch_throttler-mds": { 2024-03-19T02:05:45.448 INFO:teuthology.orchestra.run.smithi099.stdout: "val": 0, 2024-03-19T02:05:45.448 INFO:teuthology.orchestra.run.smithi099.stdout: "max": 104857600, 2024-03-19T02:05:45.448 INFO:teuthology.orchestra.run.smithi099.stdout: "get_started": 0, 2024-03-19T02:05:45.448 INFO:teuthology.orchestra.run.smithi099.stdout: "get": 438, 2024-03-19T02:05:45.448 INFO:teuthology.orchestra.run.smithi099.stdout: "get_sum": 751940, 2024-03-19T02:05:45.448 INFO:teuthology.orchestra.run.smithi099.stdout: "get_or_fail_fail": 0, 2024-03-19T02:05:45.448 INFO:teuthology.orchestra.run.smithi099.stdout: "get_or_fail_success": 438, 2024-03-19T02:05:45.448 INFO:teuthology.orchestra.run.smithi099.stdout: "take": 0, 2024-03-19T02:05:45.448 INFO:teuthology.orchestra.run.smithi099.stdout: "take_sum": 0, 2024-03-19T02:05:45.449 INFO:teuthology.orchestra.run.smithi099.stdout: "put": 438, 2024-03-19T02:05:45.449 INFO:teuthology.orchestra.run.smithi099.stdout: "put_sum": 751940, 2024-03-19T02:05:45.449 INFO:teuthology.orchestra.run.smithi099.stdout: "wait": { 2024-03-19T02:05:45.449 INFO:teuthology.orchestra.run.smithi099.stdout: "avgcount": 0, 2024-03-19T02:05:45.449 INFO:teuthology.orchestra.run.smithi099.stdout: "sum": 0.000000000, 2024-03-19T02:05:45.449 INFO:teuthology.orchestra.run.smithi099.stdout: "avgtime": 0.000000000 2024-03-19T02:05:45.449 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:45.449 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:45.449 INFO:teuthology.orchestra.run.smithi099.stdout:} 2024-03-19T02:05:46.019 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:05:45 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T02:05:46.047 WARNING:tasks.check_counter:Counter 'mds.exported' not found on daemon mds.g 2024-03-19T02:05:46.047 WARNING:tasks.check_counter:Counter 'mds.imported' not found on daemon mds.g 2024-03-19T02:05:46.047 WARNING:tasks.check_counter:Counter 'mds_cache.dir_update' not found on daemon mds.g 2024-03-19T02:05:46.047 WARNING:tasks.check_counter:Counter 'mds_cache.dir_update_receipt' not found on daemon mds.g 2024-03-19T02:05:46.047 DEBUG:tasks.check_counter:Getting stats from j 2024-03-19T02:05:46.047 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph daemon mds.j perf dump 2024-03-19T02:05:46.300 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.a/config 2024-03-19T02:05:46.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:05:45 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T02:05:46.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:05:45 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-19T02:05:47.090 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:05:46 smithi099 ceph-mon[62951]: pgmap v1091: 97 pgs: 97 active+clean; 496 MiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 321 KiB/s rd, 3.2 MiB/s wr, 187 op/s 2024-03-19T02:05:47.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:05:46 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T02:05:47.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:05:46 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T02:05:47.091 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:05:46 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:05:47.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:05:46 smithi115 ceph-mon[67332]: pgmap v1091: 97 pgs: 97 active+clean; 496 MiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 321 KiB/s rd, 3.2 MiB/s wr, 187 op/s 2024-03-19T02:05:47.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:05:46 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T02:05:47.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:05:46 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T02:05:47.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:05:46 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:05:47.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:05:46 smithi112 ceph-mon[66819]: pgmap v1091: 97 pgs: 97 active+clean; 496 MiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 321 KiB/s rd, 3.2 MiB/s wr, 187 op/s 2024-03-19T02:05:47.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:05:46 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-19T02:05:47.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:05:46 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-19T02:05:47.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:05:46 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' 2024-03-19T02:05:47.856 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2024-03-19T02:05:47.856 INFO:teuthology.orchestra.run.smithi099.stdout: "AsyncMessenger::Worker-0": { 2024-03-19T02:05:47.857 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_recv_messages": 4, 2024-03-19T02:05:47.857 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_send_messages": 4, 2024-03-19T02:05:47.857 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_recv_bytes": 3072, 2024-03-19T02:05:47.857 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_send_bytes": 448, 2024-03-19T02:05:47.857 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_created_connections": 4, 2024-03-19T02:05:47.857 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_active_connections": 0, 2024-03-19T02:05:47.857 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_running_total_time": 0.001403918, 2024-03-19T02:05:47.857 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_running_send_time": 0.000029396, 2024-03-19T02:05:47.857 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_running_recv_time": 0.000587959, 2024-03-19T02:05:47.857 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_running_fast_dispatch_time": 0.000000000, 2024-03-19T02:05:47.857 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_send_messages_queue_lat": { 2024-03-19T02:05:47.857 INFO:teuthology.orchestra.run.smithi099.stdout: "avgcount": 3, 2024-03-19T02:05:47.857 INFO:teuthology.orchestra.run.smithi099.stdout: "sum": 0.000739262, 2024-03-19T02:05:47.858 INFO:teuthology.orchestra.run.smithi099.stdout: "avgtime": 0.000246420 2024-03-19T02:05:47.858 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:47.858 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_handle_ack_lat": { 2024-03-19T02:05:47.858 INFO:teuthology.orchestra.run.smithi099.stdout: "avgcount": 0, 2024-03-19T02:05:47.858 INFO:teuthology.orchestra.run.smithi099.stdout: "sum": 0.000000000, 2024-03-19T02:05:47.858 INFO:teuthology.orchestra.run.smithi099.stdout: "avgtime": 0.000000000 2024-03-19T02:05:47.858 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:47.858 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_recv_encrypted_bytes": 3072, 2024-03-19T02:05:47.858 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_send_encrypted_bytes": 448 2024-03-19T02:05:47.858 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:47.858 INFO:teuthology.orchestra.run.smithi099.stdout: "AsyncMessenger::Worker-1": { 2024-03-19T02:05:47.858 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_recv_messages": 441, 2024-03-19T02:05:47.858 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_send_messages": 436, 2024-03-19T02:05:47.858 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_recv_bytes": 814928, 2024-03-19T02:05:47.859 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_send_bytes": 269712, 2024-03-19T02:05:47.859 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_created_connections": 2, 2024-03-19T02:05:47.859 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_active_connections": 1, 2024-03-19T02:05:47.859 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_running_total_time": 0.052096115, 2024-03-19T02:05:47.859 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_running_send_time": 0.021304027, 2024-03-19T02:05:47.859 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_running_recv_time": 0.022398241, 2024-03-19T02:05:47.859 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_running_fast_dispatch_time": 0.007394861, 2024-03-19T02:05:47.859 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_send_messages_queue_lat": { 2024-03-19T02:05:47.859 INFO:teuthology.orchestra.run.smithi099.stdout: "avgcount": 435, 2024-03-19T02:05:47.859 INFO:teuthology.orchestra.run.smithi099.stdout: "sum": 0.015012434, 2024-03-19T02:05:47.859 INFO:teuthology.orchestra.run.smithi099.stdout: "avgtime": 0.000034511 2024-03-19T02:05:47.859 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:47.859 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_handle_ack_lat": { 2024-03-19T02:05:47.860 INFO:teuthology.orchestra.run.smithi099.stdout: "avgcount": 0, 2024-03-19T02:05:47.860 INFO:teuthology.orchestra.run.smithi099.stdout: "sum": 0.000000000, 2024-03-19T02:05:47.860 INFO:teuthology.orchestra.run.smithi099.stdout: "avgtime": 0.000000000 2024-03-19T02:05:47.860 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:47.860 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_recv_encrypted_bytes": 814928, 2024-03-19T02:05:47.860 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_send_encrypted_bytes": 269712 2024-03-19T02:05:47.860 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:47.860 INFO:teuthology.orchestra.run.smithi099.stdout: "AsyncMessenger::Worker-2": { 2024-03-19T02:05:47.860 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_recv_messages": 2, 2024-03-19T02:05:47.860 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_send_messages": 333, 2024-03-19T02:05:47.860 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_recv_bytes": 320, 2024-03-19T02:05:47.860 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_send_bytes": 214544, 2024-03-19T02:05:47.861 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_created_connections": 4, 2024-03-19T02:05:47.861 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_active_connections": 2, 2024-03-19T02:05:47.861 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_running_total_time": 0.020367060, 2024-03-19T02:05:47.861 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_running_send_time": 0.016362394, 2024-03-19T02:05:47.861 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_running_recv_time": 0.001436787, 2024-03-19T02:05:47.861 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_running_fast_dispatch_time": 0.000000000, 2024-03-19T02:05:47.861 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_send_messages_queue_lat": { 2024-03-19T02:05:47.861 INFO:teuthology.orchestra.run.smithi099.stdout: "avgcount": 331, 2024-03-19T02:05:47.861 INFO:teuthology.orchestra.run.smithi099.stdout: "sum": 0.011829531, 2024-03-19T02:05:47.861 INFO:teuthology.orchestra.run.smithi099.stdout: "avgtime": 0.000035738 2024-03-19T02:05:47.861 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:47.861 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_handle_ack_lat": { 2024-03-19T02:05:47.861 INFO:teuthology.orchestra.run.smithi099.stdout: "avgcount": 0, 2024-03-19T02:05:47.862 INFO:teuthology.orchestra.run.smithi099.stdout: "sum": 0.000000000, 2024-03-19T02:05:47.862 INFO:teuthology.orchestra.run.smithi099.stdout: "avgtime": 0.000000000 2024-03-19T02:05:47.862 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:47.862 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_recv_encrypted_bytes": 320, 2024-03-19T02:05:47.862 INFO:teuthology.orchestra.run.smithi099.stdout: "msgr_send_encrypted_bytes": 214544 2024-03-19T02:05:47.862 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:47.862 INFO:teuthology.orchestra.run.smithi099.stdout: "cct": { 2024-03-19T02:05:47.862 INFO:teuthology.orchestra.run.smithi099.stdout: "total_workers": 0, 2024-03-19T02:05:47.862 INFO:teuthology.orchestra.run.smithi099.stdout: "unhealthy_workers": 0 2024-03-19T02:05:47.862 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:47.862 INFO:teuthology.orchestra.run.smithi099.stdout: "mempool": { 2024-03-19T02:05:47.862 INFO:teuthology.orchestra.run.smithi099.stdout: "bloom_filter_bytes": 0, 2024-03-19T02:05:47.862 INFO:teuthology.orchestra.run.smithi099.stdout: "bloom_filter_items": 0, 2024-03-19T02:05:47.862 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_alloc_bytes": 0, 2024-03-19T02:05:47.863 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_alloc_items": 0, 2024-03-19T02:05:47.863 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cache_data_bytes": 0, 2024-03-19T02:05:47.863 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cache_data_items": 0, 2024-03-19T02:05:47.863 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cache_onode_bytes": 0, 2024-03-19T02:05:47.863 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cache_onode_items": 0, 2024-03-19T02:05:47.863 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cache_meta_bytes": 0, 2024-03-19T02:05:47.863 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cache_meta_items": 0, 2024-03-19T02:05:47.863 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cache_other_bytes": 0, 2024-03-19T02:05:47.863 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cache_other_items": 0, 2024-03-19T02:05:47.863 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cache_buffer_bytes": 0, 2024-03-19T02:05:47.863 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cache_buffer_items": 0, 2024-03-19T02:05:47.863 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_extent_bytes": 0, 2024-03-19T02:05:47.863 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_extent_items": 0, 2024-03-19T02:05:47.863 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_blob_bytes": 0, 2024-03-19T02:05:47.863 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_blob_items": 0, 2024-03-19T02:05:47.863 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_shared_blob_bytes": 0, 2024-03-19T02:05:47.864 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_shared_blob_items": 0, 2024-03-19T02:05:47.864 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_inline_bl_bytes": 0, 2024-03-19T02:05:47.864 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_inline_bl_items": 0, 2024-03-19T02:05:47.864 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_fsck_bytes": 0, 2024-03-19T02:05:47.864 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_fsck_items": 0, 2024-03-19T02:05:47.864 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_txc_bytes": 0, 2024-03-19T02:05:47.864 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_txc_items": 0, 2024-03-19T02:05:47.864 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_writing_deferred_bytes": 0, 2024-03-19T02:05:47.864 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_writing_deferred_items": 0, 2024-03-19T02:05:47.864 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_writing_bytes": 0, 2024-03-19T02:05:47.864 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_writing_items": 0, 2024-03-19T02:05:47.864 INFO:teuthology.orchestra.run.smithi099.stdout: "bluefs_bytes": 0, 2024-03-19T02:05:47.864 INFO:teuthology.orchestra.run.smithi099.stdout: "bluefs_items": 0, 2024-03-19T02:05:47.864 INFO:teuthology.orchestra.run.smithi099.stdout: "bluefs_file_reader_bytes": 0, 2024-03-19T02:05:47.865 INFO:teuthology.orchestra.run.smithi099.stdout: "bluefs_file_reader_items": 0, 2024-03-19T02:05:47.865 INFO:teuthology.orchestra.run.smithi099.stdout: "bluefs_file_writer_bytes": 0, 2024-03-19T02:05:47.865 INFO:teuthology.orchestra.run.smithi099.stdout: "bluefs_file_writer_items": 0, 2024-03-19T02:05:47.865 INFO:teuthology.orchestra.run.smithi099.stdout: "buffer_anon_bytes": 148744, 2024-03-19T02:05:47.865 INFO:teuthology.orchestra.run.smithi099.stdout: "buffer_anon_items": 30, 2024-03-19T02:05:47.865 INFO:teuthology.orchestra.run.smithi099.stdout: "buffer_meta_bytes": 0, 2024-03-19T02:05:47.865 INFO:teuthology.orchestra.run.smithi099.stdout: "buffer_meta_items": 0, 2024-03-19T02:05:47.865 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_bytes": 0, 2024-03-19T02:05:47.865 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_items": 0, 2024-03-19T02:05:47.865 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_mapbl_bytes": 0, 2024-03-19T02:05:47.865 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_mapbl_items": 0, 2024-03-19T02:05:47.865 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_pglog_bytes": 0, 2024-03-19T02:05:47.865 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_pglog_items": 0, 2024-03-19T02:05:47.865 INFO:teuthology.orchestra.run.smithi099.stdout: "osdmap_bytes": 0, 2024-03-19T02:05:47.865 INFO:teuthology.orchestra.run.smithi099.stdout: "osdmap_items": 0, 2024-03-19T02:05:47.865 INFO:teuthology.orchestra.run.smithi099.stdout: "osdmap_mapping_bytes": 0, 2024-03-19T02:05:47.866 INFO:teuthology.orchestra.run.smithi099.stdout: "osdmap_mapping_items": 0, 2024-03-19T02:05:47.866 INFO:teuthology.orchestra.run.smithi099.stdout: "pgmap_bytes": 0, 2024-03-19T02:05:47.866 INFO:teuthology.orchestra.run.smithi099.stdout: "pgmap_items": 0, 2024-03-19T02:05:47.866 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_co_bytes": 928, 2024-03-19T02:05:47.866 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_co_items": 2, 2024-03-19T02:05:47.866 INFO:teuthology.orchestra.run.smithi099.stdout: "unittest_1_bytes": 0, 2024-03-19T02:05:47.866 INFO:teuthology.orchestra.run.smithi099.stdout: "unittest_1_items": 0, 2024-03-19T02:05:47.866 INFO:teuthology.orchestra.run.smithi099.stdout: "unittest_2_bytes": 0, 2024-03-19T02:05:47.866 INFO:teuthology.orchestra.run.smithi099.stdout: "unittest_2_items": 0 2024-03-19T02:05:47.866 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-03-19T02:05:47.866 INFO:teuthology.orchestra.run.smithi099.stdout: "throttle-msgr_dispatch_throttler-mds": { 2024-03-19T02:05:47.866 INFO:teuthology.orchestra.run.smithi099.stdout: "val": 0, 2024-03-19T02:05:47.866 INFO:teuthology.orchestra.run.smithi099.stdout: "max": 104857600, 2024-03-19T02:05:47.866 INFO:teuthology.orchestra.run.smithi099.stdout: "get_started": 0, 2024-03-19T02:05:47.867 INFO:teuthology.orchestra.run.smithi099.stdout: "get": 443, 2024-03-19T02:05:47.867 INFO:teuthology.orchestra.run.smithi099.stdout: "get_sum": 752590, 2024-03-19T02:05:47.867 INFO:teuthology.orchestra.run.smithi099.stdout: "get_or_fail_fail": 0, 2024-03-19T02:05:47.867 INFO:teuthology.orchestra.run.smithi099.stdout: "get_or_fail_success": 443, 2024-03-19T02:05:47.867 INFO:teuthology.orchestra.run.smithi099.stdout: "take": 0, 2024-03-19T02:05:47.867 INFO:teuthology.orchestra.run.smithi099.stdout: "take_sum": 0, 2024-03-19T02:05:47.867 INFO:teuthology.orchestra.run.smithi099.stdout: "put": 443, 2024-03-19T02:05:47.867 INFO:teuthology.orchestra.run.smithi099.stdout: "put_sum": 752590, 2024-03-19T02:05:47.867 INFO:teuthology.orchestra.run.smithi099.stdout: "wait": { 2024-03-19T02:05:47.867 INFO:teuthology.orchestra.run.smithi099.stdout: "avgcount": 0, 2024-03-19T02:05:47.867 INFO:teuthology.orchestra.run.smithi099.stdout: "sum": 0.000000000, 2024-03-19T02:05:47.867 INFO:teuthology.orchestra.run.smithi099.stdout: "avgtime": 0.000000000 2024-03-19T02:05:47.867 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:47.867 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-03-19T02:05:47.867 INFO:teuthology.orchestra.run.smithi099.stdout:} 2024-03-19T02:05:48.615 WARNING:tasks.check_counter:Counter 'mds.exported' not found on daemon mds.j 2024-03-19T02:05:48.615 WARNING:tasks.check_counter:Counter 'mds.imported' not found on daemon mds.j 2024-03-19T02:05:48.615 WARNING:tasks.check_counter:Counter 'mds_cache.dir_update' not found on daemon mds.j 2024-03-19T02:05:48.615 WARNING:tasks.check_counter:Counter 'mds_cache.dir_update_receipt' not found on daemon mds.j 2024-03-19T02:05:48.615 DEBUG:tasks.check_counter:Getting stats from b 2024-03-19T02:05:48.615 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph daemon mds.b perf dump 2024-03-19T02:05:48.979 INFO:teuthology.orchestra.run.smithi112.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.b/config 2024-03-19T02:05:49.103 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:05:48 smithi112 ceph-mon[66819]: pgmap v1092: 97 pgs: 97 active+clean; 491 MiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 261 KiB/s rd, 3.7 MiB/s wr, 181 op/s 2024-03-19T02:05:49.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:05:48 smithi115 ceph-mon[67332]: pgmap v1092: 97 pgs: 97 active+clean; 491 MiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 261 KiB/s rd, 3.7 MiB/s wr, 181 op/s 2024-03-19T02:05:49.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:05:48 smithi099 ceph-mon[62951]: pgmap v1092: 97 pgs: 97 active+clean; 491 MiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 261 KiB/s rd, 3.7 MiB/s wr, 181 op/s 2024-03-19T02:05:50.809 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-03-19T02:05:50.809 INFO:teuthology.orchestra.run.smithi112.stdout: "AsyncMessenger::Worker-0": { 2024-03-19T02:05:50.809 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_recv_messages": 639965, 2024-03-19T02:05:50.809 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_send_messages": 515821, 2024-03-19T02:05:50.809 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_recv_bytes": 160244978, 2024-03-19T02:05:50.810 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_send_bytes": 202392726, 2024-03-19T02:05:50.810 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_created_connections": 9, 2024-03-19T02:05:50.810 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_active_connections": 4, 2024-03-19T02:05:50.810 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_running_total_time": 25.321320829, 2024-03-19T02:05:50.810 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_running_send_time": 10.409395986, 2024-03-19T02:05:50.810 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_running_recv_time": 18.600115344, 2024-03-19T02:05:50.810 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_running_fast_dispatch_time": 0.583920444, 2024-03-19T02:05:50.810 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_send_messages_queue_lat": { 2024-03-19T02:05:50.810 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 515820, 2024-03-19T02:05:50.810 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 11.216380308, 2024-03-19T02:05:50.810 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000021744 2024-03-19T02:05:50.810 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.810 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_handle_ack_lat": { 2024-03-19T02:05:50.810 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 1103010, 2024-03-19T02:05:50.811 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.123815488, 2024-03-19T02:05:50.811 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000112 2024-03-19T02:05:50.811 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.811 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_recv_encrypted_bytes": 3072, 2024-03-19T02:05:50.811 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_send_encrypted_bytes": 448 2024-03-19T02:05:50.811 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.811 INFO:teuthology.orchestra.run.smithi112.stdout: "AsyncMessenger::Worker-1": { 2024-03-19T02:05:50.811 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_recv_messages": 32418, 2024-03-19T02:05:50.811 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_send_messages": 32392, 2024-03-19T02:05:50.811 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_recv_bytes": 8920484, 2024-03-19T02:05:50.811 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_send_bytes": 127963564, 2024-03-19T02:05:50.811 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_created_connections": 13, 2024-03-19T02:05:50.811 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_active_connections": 9, 2024-03-19T02:05:50.811 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_running_total_time": 2.797840894, 2024-03-19T02:05:50.811 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_running_send_time": 1.102462180, 2024-03-19T02:05:50.812 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_running_recv_time": 0.547593002, 2024-03-19T02:05:50.812 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_running_fast_dispatch_time": 0.980395259, 2024-03-19T02:05:50.812 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_send_messages_queue_lat": { 2024-03-19T02:05:50.812 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 32391, 2024-03-19T02:05:50.812 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 1.395347088, 2024-03-19T02:05:50.812 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000043078 2024-03-19T02:05:50.812 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.812 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_handle_ack_lat": { 2024-03-19T02:05:50.812 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 30, 2024-03-19T02:05:50.812 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000002524, 2024-03-19T02:05:50.812 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000084 2024-03-19T02:05:50.812 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.812 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_recv_encrypted_bytes": 960976, 2024-03-19T02:05:50.812 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_send_encrypted_bytes": 278560 2024-03-19T02:05:50.812 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.813 INFO:teuthology.orchestra.run.smithi112.stdout: "AsyncMessenger::Worker-2": { 2024-03-19T02:05:50.813 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_recv_messages": 355404, 2024-03-19T02:05:50.813 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_send_messages": 482983, 2024-03-19T02:05:50.813 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_recv_bytes": 106582925, 2024-03-19T02:05:50.813 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_send_bytes": 401693544, 2024-03-19T02:05:50.813 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_created_connections": 8, 2024-03-19T02:05:50.813 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_active_connections": 6, 2024-03-19T02:05:50.813 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_running_total_time": 18.769545683, 2024-03-19T02:05:50.813 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_running_send_time": 8.173042474, 2024-03-19T02:05:50.813 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_running_recv_time": 11.821977684, 2024-03-19T02:05:50.813 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_running_fast_dispatch_time": 0.765590015, 2024-03-19T02:05:50.813 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_send_messages_queue_lat": { 2024-03-19T02:05:50.813 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 482981, 2024-03-19T02:05:50.813 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 14.249003156, 2024-03-19T02:05:50.814 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000029502 2024-03-19T02:05:50.814 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.814 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_handle_ack_lat": { 2024-03-19T02:05:50.814 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 742105, 2024-03-19T02:05:50.814 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.105250633, 2024-03-19T02:05:50.814 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000141 2024-03-19T02:05:50.814 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.814 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_recv_encrypted_bytes": 160, 2024-03-19T02:05:50.814 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_send_encrypted_bytes": 664976 2024-03-19T02:05:50.814 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.814 INFO:teuthology.orchestra.run.smithi112.stdout: "cct": { 2024-03-19T02:05:50.814 INFO:teuthology.orchestra.run.smithi112.stdout: "total_workers": 1, 2024-03-19T02:05:50.814 INFO:teuthology.orchestra.run.smithi112.stdout: "unhealthy_workers": 0 2024-03-19T02:05:50.814 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.815 INFO:teuthology.orchestra.run.smithi112.stdout: "finisher-MDSRank": { 2024-03-19T02:05:50.815 INFO:teuthology.orchestra.run.smithi112.stdout: "queue_len": 0, 2024-03-19T02:05:50.815 INFO:teuthology.orchestra.run.smithi112.stdout: "complete_latency": { 2024-03-19T02:05:50.815 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 84088, 2024-03-19T02:05:50.815 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 99.656199775, 2024-03-19T02:05:50.815 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.001185141 2024-03-19T02:05:50.815 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-03-19T02:05:50.815 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.815 INFO:teuthology.orchestra.run.smithi112.stdout: "finisher-PurgeQueue": { 2024-03-19T02:05:50.815 INFO:teuthology.orchestra.run.smithi112.stdout: "queue_len": 0, 2024-03-19T02:05:50.815 INFO:teuthology.orchestra.run.smithi112.stdout: "complete_latency": { 2024-03-19T02:05:50.815 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 51974, 2024-03-19T02:05:50.815 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 5.040164451, 2024-03-19T02:05:50.815 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000096974 2024-03-19T02:05:50.816 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-03-19T02:05:50.816 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.816 INFO:teuthology.orchestra.run.smithi112.stdout: "mds": { 2024-03-19T02:05:50.816 INFO:teuthology.orchestra.run.smithi112.stdout: "request": 258823, 2024-03-19T02:05:50.816 INFO:teuthology.orchestra.run.smithi112.stdout: "reply": 189658, 2024-03-19T02:05:50.816 INFO:teuthology.orchestra.run.smithi112.stdout: "reply_latency": { 2024-03-19T02:05:50.816 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 189658, 2024-03-19T02:05:50.816 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 878.784619406, 2024-03-19T02:05:50.816 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.004633522 2024-03-19T02:05:50.816 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.816 INFO:teuthology.orchestra.run.smithi112.stdout: "slow_reply": 0, 2024-03-19T02:05:50.816 INFO:teuthology.orchestra.run.smithi112.stdout: "forward": 69165, 2024-03-19T02:05:50.816 INFO:teuthology.orchestra.run.smithi112.stdout: "dir_fetch_complete": 0, 2024-03-19T02:05:50.816 INFO:teuthology.orchestra.run.smithi112.stdout: "dir_fetch_keys": 0, 2024-03-19T02:05:50.817 INFO:teuthology.orchestra.run.smithi112.stdout: "dir_commit": 29, 2024-03-19T02:05:50.817 INFO:teuthology.orchestra.run.smithi112.stdout: "dir_split": 0, 2024-03-19T02:05:50.817 INFO:teuthology.orchestra.run.smithi112.stdout: "dir_merge": 0, 2024-03-19T02:05:50.817 INFO:teuthology.orchestra.run.smithi112.stdout: "inodes": 328, 2024-03-19T02:05:50.817 INFO:teuthology.orchestra.run.smithi112.stdout: "inodes_top": 5, 2024-03-19T02:05:50.817 INFO:teuthology.orchestra.run.smithi112.stdout: "inodes_bottom": 3, 2024-03-19T02:05:50.817 INFO:teuthology.orchestra.run.smithi112.stdout: "inodes_pin_tail": 320, 2024-03-19T02:05:50.817 INFO:teuthology.orchestra.run.smithi112.stdout: "inodes_pinned": 320, 2024-03-19T02:05:50.817 INFO:teuthology.orchestra.run.smithi112.stdout: "inodes_expired": 403, 2024-03-19T02:05:50.817 INFO:teuthology.orchestra.run.smithi112.stdout: "inodes_with_caps": 184, 2024-03-19T02:05:50.817 INFO:teuthology.orchestra.run.smithi112.stdout: "caps": 184, 2024-03-19T02:05:50.817 INFO:teuthology.orchestra.run.smithi112.stdout: "subtrees": 3, 2024-03-19T02:05:50.817 INFO:teuthology.orchestra.run.smithi112.stdout: "traverse": 398014, 2024-03-19T02:05:50.817 INFO:teuthology.orchestra.run.smithi112.stdout: "traverse_hit": 187775, 2024-03-19T02:05:50.817 INFO:teuthology.orchestra.run.smithi112.stdout: "traverse_forward": 69162, 2024-03-19T02:05:50.818 INFO:teuthology.orchestra.run.smithi112.stdout: "traverse_discover": 167, 2024-03-19T02:05:50.818 INFO:teuthology.orchestra.run.smithi112.stdout: "traverse_dir_fetch": 0, 2024-03-19T02:05:50.818 INFO:teuthology.orchestra.run.smithi112.stdout: "traverse_remote_ino": 0, 2024-03-19T02:05:50.818 INFO:teuthology.orchestra.run.smithi112.stdout: "traverse_lock": 4529, 2024-03-19T02:05:50.818 INFO:teuthology.orchestra.run.smithi112.stdout: "load_cent": 0, 2024-03-19T02:05:50.818 INFO:teuthology.orchestra.run.smithi112.stdout: "q": 0, 2024-03-19T02:05:50.818 INFO:teuthology.orchestra.run.smithi112.stdout: "exported": 5, 2024-03-19T02:05:50.818 INFO:teuthology.orchestra.run.smithi112.stdout: "exported_inodes": 99, 2024-03-19T02:05:50.818 INFO:teuthology.orchestra.run.smithi112.stdout: "imported": 6, 2024-03-19T02:05:50.818 INFO:teuthology.orchestra.run.smithi112.stdout: "imported_inodes": 105, 2024-03-19T02:05:50.818 INFO:teuthology.orchestra.run.smithi112.stdout: "openino_dir_fetch": 0, 2024-03-19T02:05:50.818 INFO:teuthology.orchestra.run.smithi112.stdout: "openino_backtrace_fetch": 0, 2024-03-19T02:05:50.818 INFO:teuthology.orchestra.run.smithi112.stdout: "openino_peer_discover": 0, 2024-03-19T02:05:50.818 INFO:teuthology.orchestra.run.smithi112.stdout: "root_rfiles": 4, 2024-03-19T02:05:50.819 INFO:teuthology.orchestra.run.smithi112.stdout: "root_rbytes": 234, 2024-03-19T02:05:50.819 INFO:teuthology.orchestra.run.smithi112.stdout: "root_rsnaps": 0, 2024-03-19T02:05:50.819 INFO:teuthology.orchestra.run.smithi112.stdout: "scrub_backtrace_fetch": 0, 2024-03-19T02:05:50.819 INFO:teuthology.orchestra.run.smithi112.stdout: "scrub_set_tag": 0, 2024-03-19T02:05:50.819 INFO:teuthology.orchestra.run.smithi112.stdout: "scrub_backtrace_repaired": 0, 2024-03-19T02:05:50.819 INFO:teuthology.orchestra.run.smithi112.stdout: "scrub_inotable_repaired": 0, 2024-03-19T02:05:50.819 INFO:teuthology.orchestra.run.smithi112.stdout: "scrub_dir_inodes": 0, 2024-03-19T02:05:50.819 INFO:teuthology.orchestra.run.smithi112.stdout: "scrub_dir_base_inodes": 0, 2024-03-19T02:05:50.819 INFO:teuthology.orchestra.run.smithi112.stdout: "scrub_dirfrag_rstats": 0, 2024-03-19T02:05:50.819 INFO:teuthology.orchestra.run.smithi112.stdout: "scrub_file_inodes": 0, 2024-03-19T02:05:50.820 INFO:teuthology.orchestra.run.smithi112.stdout: "handle_inode_file_caps": 30, 2024-03-19T02:05:50.820 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph_cap_op_revoke": 17600, 2024-03-19T02:05:50.820 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph_cap_op_grant": 91790, 2024-03-19T02:05:50.820 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph_cap_op_trunc": 0, 2024-03-19T02:05:50.820 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph_cap_op_flushsnap_ack": 0, 2024-03-19T02:05:50.820 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph_cap_op_flush_ack": 0, 2024-03-19T02:05:50.820 INFO:teuthology.orchestra.run.smithi112.stdout: "handle_client_caps": 52826, 2024-03-19T02:05:50.820 INFO:teuthology.orchestra.run.smithi112.stdout: "handle_client_caps_dirty": 18668, 2024-03-19T02:05:50.820 INFO:teuthology.orchestra.run.smithi112.stdout: "handle_client_cap_release": 9819, 2024-03-19T02:05:50.820 INFO:teuthology.orchestra.run.smithi112.stdout: "process_request_cap_release": 128697 2024-03-19T02:05:50.820 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.820 INFO:teuthology.orchestra.run.smithi112.stdout: "mds_cache": { 2024-03-19T02:05:50.820 INFO:teuthology.orchestra.run.smithi112.stdout: "dir_update": 31, 2024-03-19T02:05:50.820 INFO:teuthology.orchestra.run.smithi112.stdout: "dir_update_receipt": 930, 2024-03-19T02:05:50.820 INFO:teuthology.orchestra.run.smithi112.stdout: "dir_try_discover": 524, 2024-03-19T02:05:50.821 INFO:teuthology.orchestra.run.smithi112.stdout: "dir_send_discover": 868, 2024-03-19T02:05:50.821 INFO:teuthology.orchestra.run.smithi112.stdout: "dir_handle_discover": 534, 2024-03-19T02:05:50.821 INFO:teuthology.orchestra.run.smithi112.stdout: "num_strays": 215, 2024-03-19T02:05:50.821 INFO:teuthology.orchestra.run.smithi112.stdout: "num_strays_delayed": 46, 2024-03-19T02:05:50.821 INFO:teuthology.orchestra.run.smithi112.stdout: "num_strays_enqueuing": 0, 2024-03-19T02:05:50.821 INFO:teuthology.orchestra.run.smithi112.stdout: "strays_created": 29449, 2024-03-19T02:05:50.821 INFO:teuthology.orchestra.run.smithi112.stdout: "strays_enqueued": 29234, 2024-03-19T02:05:50.821 INFO:teuthology.orchestra.run.smithi112.stdout: "strays_reintegrated": 0, 2024-03-19T02:05:50.821 INFO:teuthology.orchestra.run.smithi112.stdout: "strays_migrated": 0, 2024-03-19T02:05:50.821 INFO:teuthology.orchestra.run.smithi112.stdout: "num_recovering_processing": 0, 2024-03-19T02:05:50.821 INFO:teuthology.orchestra.run.smithi112.stdout: "num_recovering_enqueued": 0, 2024-03-19T02:05:50.821 INFO:teuthology.orchestra.run.smithi112.stdout: "num_recovering_prioritized": 0, 2024-03-19T02:05:50.821 INFO:teuthology.orchestra.run.smithi112.stdout: "recovery_started": 0, 2024-03-19T02:05:50.821 INFO:teuthology.orchestra.run.smithi112.stdout: "recovery_completed": 0, 2024-03-19T02:05:50.821 INFO:teuthology.orchestra.run.smithi112.stdout: "ireq_quiesce_path": 0, 2024-03-19T02:05:50.822 INFO:teuthology.orchestra.run.smithi112.stdout: "ireq_quiesce_inode": 0, 2024-03-19T02:05:50.822 INFO:teuthology.orchestra.run.smithi112.stdout: "ireq_enqueue_scrub": 0, 2024-03-19T02:05:50.822 INFO:teuthology.orchestra.run.smithi112.stdout: "ireq_exportdir": 5, 2024-03-19T02:05:50.822 INFO:teuthology.orchestra.run.smithi112.stdout: "ireq_flush": 0, 2024-03-19T02:05:50.822 INFO:teuthology.orchestra.run.smithi112.stdout: "ireq_fragmentdir": 0, 2024-03-19T02:05:50.822 INFO:teuthology.orchestra.run.smithi112.stdout: "ireq_fragstats": 0, 2024-03-19T02:05:50.822 INFO:teuthology.orchestra.run.smithi112.stdout: "ireq_inodestats": 0 2024-03-19T02:05:50.822 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.822 INFO:teuthology.orchestra.run.smithi112.stdout: "mds_log": { 2024-03-19T02:05:50.822 INFO:teuthology.orchestra.run.smithi112.stdout: "evlrg": 0, 2024-03-19T02:05:50.822 INFO:teuthology.orchestra.run.smithi112.stdout: "evadd": 135675, 2024-03-19T02:05:50.822 INFO:teuthology.orchestra.run.smithi112.stdout: "evex": 25605, 2024-03-19T02:05:50.822 INFO:teuthology.orchestra.run.smithi112.stdout: "evtrm": 24581, 2024-03-19T02:05:50.822 INFO:teuthology.orchestra.run.smithi112.stdout: "ev": 111094, 2024-03-19T02:05:50.823 INFO:teuthology.orchestra.run.smithi112.stdout: "evexg": 0, 2024-03-19T02:05:50.823 INFO:teuthology.orchestra.run.smithi112.stdout: "evexd": 1024, 2024-03-19T02:05:50.823 INFO:teuthology.orchestra.run.smithi112.stdout: "segadd": 160, 2024-03-19T02:05:50.823 INFO:teuthology.orchestra.run.smithi112.stdout: "segex": 32, 2024-03-19T02:05:50.823 INFO:teuthology.orchestra.run.smithi112.stdout: "segtrm": 31, 2024-03-19T02:05:50.823 INFO:teuthology.orchestra.run.smithi112.stdout: "seg": 129, 2024-03-19T02:05:50.823 INFO:teuthology.orchestra.run.smithi112.stdout: "segmjr": 10, 2024-03-19T02:05:50.823 INFO:teuthology.orchestra.run.smithi112.stdout: "segexg": 0, 2024-03-19T02:05:50.823 INFO:teuthology.orchestra.run.smithi112.stdout: "segexd": 1, 2024-03-19T02:05:50.823 INFO:teuthology.orchestra.run.smithi112.stdout: "expos": 56004966, 2024-03-19T02:05:50.823 INFO:teuthology.orchestra.run.smithi112.stdout: "wrpos": 285269063, 2024-03-19T02:05:50.823 INFO:teuthology.orchestra.run.smithi112.stdout: "rdpos": 0, 2024-03-19T02:05:50.823 INFO:teuthology.orchestra.run.smithi112.stdout: "jlat": { 2024-03-19T02:05:50.823 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 44438, 2024-03-19T02:05:50.824 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 287.738528132, 2024-03-19T02:05:50.824 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.006475055 2024-03-19T02:05:50.824 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.824 INFO:teuthology.orchestra.run.smithi112.stdout: "replayed": 0 2024-03-19T02:05:50.824 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.824 INFO:teuthology.orchestra.run.smithi112.stdout: "mds_mem": { 2024-03-19T02:05:50.824 INFO:teuthology.orchestra.run.smithi112.stdout: "ino": 239, 2024-03-19T02:05:50.824 INFO:teuthology.orchestra.run.smithi112.stdout: "ino+": 29706, 2024-03-19T02:05:50.824 INFO:teuthology.orchestra.run.smithi112.stdout: "ino-": 29467, 2024-03-19T02:05:50.824 INFO:teuthology.orchestra.run.smithi112.stdout: "dir": 37, 2024-03-19T02:05:50.824 INFO:teuthology.orchestra.run.smithi112.stdout: "dir+": 492, 2024-03-19T02:05:50.824 INFO:teuthology.orchestra.run.smithi112.stdout: "dir-": 455, 2024-03-19T02:05:50.824 INFO:teuthology.orchestra.run.smithi112.stdout: "dn": 261, 2024-03-19T02:05:50.824 INFO:teuthology.orchestra.run.smithi112.stdout: "dn+": 57127, 2024-03-19T02:05:50.824 INFO:teuthology.orchestra.run.smithi112.stdout: "dn-": 56866, 2024-03-19T02:05:50.824 INFO:teuthology.orchestra.run.smithi112.stdout: "cap": 134, 2024-03-19T02:05:50.825 INFO:teuthology.orchestra.run.smithi112.stdout: "cap+": 31187, 2024-03-19T02:05:50.825 INFO:teuthology.orchestra.run.smithi112.stdout: "cap-": 31053, 2024-03-19T02:05:50.825 INFO:teuthology.orchestra.run.smithi112.stdout: "rss": 80628, 2024-03-19T02:05:50.825 INFO:teuthology.orchestra.run.smithi112.stdout: "heap": 223612 2024-03-19T02:05:50.825 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.825 INFO:teuthology.orchestra.run.smithi112.stdout: "mds_server": { 2024-03-19T02:05:50.825 INFO:teuthology.orchestra.run.smithi112.stdout: "dispatch_client_request": 346361, 2024-03-19T02:05:50.825 INFO:teuthology.orchestra.run.smithi112.stdout: "dispatch_server_request": 10889, 2024-03-19T02:05:50.825 INFO:teuthology.orchestra.run.smithi112.stdout: "handle_client_request": 258823, 2024-03-19T02:05:50.825 INFO:teuthology.orchestra.run.smithi112.stdout: "handle_client_session": 3741, 2024-03-19T02:05:50.825 INFO:teuthology.orchestra.run.smithi112.stdout: "handle_peer_request": 21816, 2024-03-19T02:05:50.825 INFO:teuthology.orchestra.run.smithi112.stdout: "req_create_latency": { 2024-03-19T02:05:50.825 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 29419, 2024-03-19T02:05:50.825 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 237.255966629, 2024-03-19T02:05:50.825 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.008064718 2024-03-19T02:05:50.826 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.826 INFO:teuthology.orchestra.run.smithi112.stdout: "req_getattr_latency": { 2024-03-19T02:05:50.826 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 17555, 2024-03-19T02:05:50.826 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 209.953039473, 2024-03-19T02:05:50.826 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.011959728 2024-03-19T02:05:50.826 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.826 INFO:teuthology.orchestra.run.smithi112.stdout: "req_getfilelock_latency": { 2024-03-19T02:05:50.826 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:50.826 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:50.826 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:50.826 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.826 INFO:teuthology.orchestra.run.smithi112.stdout: "req_link_latency": { 2024-03-19T02:05:50.826 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:50.826 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:50.827 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:50.827 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.827 INFO:teuthology.orchestra.run.smithi112.stdout: "req_lookup_latency": { 2024-03-19T02:05:50.827 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 62236, 2024-03-19T02:05:50.827 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 71.014959162, 2024-03-19T02:05:50.827 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.001141059 2024-03-19T02:05:50.827 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.827 INFO:teuthology.orchestra.run.smithi112.stdout: "req_lookuphash_latency": { 2024-03-19T02:05:50.827 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:50.827 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:50.827 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:50.827 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.827 INFO:teuthology.orchestra.run.smithi112.stdout: "req_lookupino_latency": { 2024-03-19T02:05:50.827 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 1, 2024-03-19T02:05:50.827 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000120388, 2024-03-19T02:05:50.828 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000120388 2024-03-19T02:05:50.828 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.828 INFO:teuthology.orchestra.run.smithi112.stdout: "req_lookupname_latency": { 2024-03-19T02:05:50.828 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:50.828 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:50.828 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:50.828 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.828 INFO:teuthology.orchestra.run.smithi112.stdout: "req_lookupparent_latency": { 2024-03-19T02:05:50.828 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:50.828 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:50.828 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:50.828 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.828 INFO:teuthology.orchestra.run.smithi112.stdout: "req_lookupsnap_latency": { 2024-03-19T02:05:50.828 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:50.829 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:50.829 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:50.829 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.829 INFO:teuthology.orchestra.run.smithi112.stdout: "req_lssnap_latency": { 2024-03-19T02:05:50.829 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:50.830 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:50.830 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:50.830 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.830 INFO:teuthology.orchestra.run.smithi112.stdout: "req_mkdir_latency": { 2024-03-19T02:05:50.830 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 10, 2024-03-19T02:05:50.830 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.110252953, 2024-03-19T02:05:50.830 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.011025295 2024-03-19T02:05:50.830 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.830 INFO:teuthology.orchestra.run.smithi112.stdout: "req_mknod_latency": { 2024-03-19T02:05:50.830 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:50.830 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:50.830 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:50.830 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.830 INFO:teuthology.orchestra.run.smithi112.stdout: "req_mksnap_latency": { 2024-03-19T02:05:50.831 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:50.831 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:50.831 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:50.831 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.831 INFO:teuthology.orchestra.run.smithi112.stdout: "req_open_latency": { 2024-03-19T02:05:50.831 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:50.831 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:50.831 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:50.831 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.831 INFO:teuthology.orchestra.run.smithi112.stdout: "req_readdir_latency": { 2024-03-19T02:05:50.831 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 43340, 2024-03-19T02:05:50.831 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 188.970946574, 2024-03-19T02:05:50.831 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.004360197 2024-03-19T02:05:50.832 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.832 INFO:teuthology.orchestra.run.smithi112.stdout: "req_rename_latency": { 2024-03-19T02:05:50.832 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 6540, 2024-03-19T02:05:50.832 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 72.443417945, 2024-03-19T02:05:50.832 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.011076975 2024-03-19T02:05:50.832 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.832 INFO:teuthology.orchestra.run.smithi112.stdout: "req_renamesnap_latency": { 2024-03-19T02:05:50.832 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:50.832 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:50.832 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:50.832 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.832 INFO:teuthology.orchestra.run.smithi112.stdout: "req_snapdiff_latency": { 2024-03-19T02:05:50.832 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:50.832 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:50.832 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:50.833 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.833 INFO:teuthology.orchestra.run.smithi112.stdout: "req_rmdir_latency": { 2024-03-19T02:05:50.833 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 19, 2024-03-19T02:05:50.833 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.176199054, 2024-03-19T02:05:50.833 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.009273634 2024-03-19T02:05:50.833 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.833 INFO:teuthology.orchestra.run.smithi112.stdout: "req_rmsnap_latency": { 2024-03-19T02:05:50.833 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:50.833 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:50.833 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:50.833 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.833 INFO:teuthology.orchestra.run.smithi112.stdout: "req_rmxattr_latency": { 2024-03-19T02:05:50.833 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:50.833 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:50.834 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:50.834 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.834 INFO:teuthology.orchestra.run.smithi112.stdout: "req_setattr_latency": { 2024-03-19T02:05:50.834 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 4, 2024-03-19T02:05:50.834 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.048409070, 2024-03-19T02:05:50.834 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.012102267 2024-03-19T02:05:50.834 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.834 INFO:teuthology.orchestra.run.smithi112.stdout: "req_setdirlayout_latency": { 2024-03-19T02:05:50.834 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:50.834 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:50.834 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:50.834 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.834 INFO:teuthology.orchestra.run.smithi112.stdout: "req_setfilelock_latency": { 2024-03-19T02:05:50.834 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 1104, 2024-03-19T02:05:50.835 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 1.084590427, 2024-03-19T02:05:50.835 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000982418 2024-03-19T02:05:50.835 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.835 INFO:teuthology.orchestra.run.smithi112.stdout: "req_setlayout_latency": { 2024-03-19T02:05:50.835 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:50.835 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:50.835 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:50.835 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.835 INFO:teuthology.orchestra.run.smithi112.stdout: "req_setxattr_latency": { 2024-03-19T02:05:50.835 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:50.835 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:50.835 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:50.835 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.835 INFO:teuthology.orchestra.run.smithi112.stdout: "req_symlink_latency": { 2024-03-19T02:05:50.835 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:50.836 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:50.836 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:50.836 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.836 INFO:teuthology.orchestra.run.smithi112.stdout: "req_unlink_latency": { 2024-03-19T02:05:50.836 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 29430, 2024-03-19T02:05:50.836 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 97.726717731, 2024-03-19T02:05:50.836 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.003320649 2024-03-19T02:05:50.836 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.836 INFO:teuthology.orchestra.run.smithi112.stdout: "cap_revoke_eviction": 0, 2024-03-19T02:05:50.836 INFO:teuthology.orchestra.run.smithi112.stdout: "cap_acquisition_throttle": 0, 2024-03-19T02:05:50.836 INFO:teuthology.orchestra.run.smithi112.stdout: "req_getvxattr_latency": { 2024-03-19T02:05:50.836 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:50.836 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:50.836 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:50.837 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-03-19T02:05:50.837 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.837 INFO:teuthology.orchestra.run.smithi112.stdout: "mds_sessions": { 2024-03-19T02:05:50.837 INFO:teuthology.orchestra.run.smithi112.stdout: "session_count": 1, 2024-03-19T02:05:50.837 INFO:teuthology.orchestra.run.smithi112.stdout: "session_add": 3, 2024-03-19T02:05:50.837 INFO:teuthology.orchestra.run.smithi112.stdout: "session_remove": 2, 2024-03-19T02:05:50.837 INFO:teuthology.orchestra.run.smithi112.stdout: "sessions_open": 1, 2024-03-19T02:05:50.837 INFO:teuthology.orchestra.run.smithi112.stdout: "sessions_stale": 0, 2024-03-19T02:05:50.837 INFO:teuthology.orchestra.run.smithi112.stdout: "total_load": 20268, 2024-03-19T02:05:50.837 INFO:teuthology.orchestra.run.smithi112.stdout: "average_load": 20268, 2024-03-19T02:05:50.837 INFO:teuthology.orchestra.run.smithi112.stdout: "avg_session_uptime": 1500, 2024-03-19T02:05:50.837 INFO:teuthology.orchestra.run.smithi112.stdout: "mdthresh_evicted": 0 2024-03-19T02:05:50.837 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.837 INFO:teuthology.orchestra.run.smithi112.stdout: "mempool": { 2024-03-19T02:05:50.838 INFO:teuthology.orchestra.run.smithi112.stdout: "bloom_filter_bytes": 0, 2024-03-19T02:05:50.838 INFO:teuthology.orchestra.run.smithi112.stdout: "bloom_filter_items": 0, 2024-03-19T02:05:50.838 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_alloc_bytes": 0, 2024-03-19T02:05:50.838 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_alloc_items": 0, 2024-03-19T02:05:50.838 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_cache_data_bytes": 0, 2024-03-19T02:05:50.838 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_cache_data_items": 0, 2024-03-19T02:05:50.838 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_cache_onode_bytes": 0, 2024-03-19T02:05:50.838 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_cache_onode_items": 0, 2024-03-19T02:05:50.838 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_cache_meta_bytes": 0, 2024-03-19T02:05:50.838 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_cache_meta_items": 0, 2024-03-19T02:05:50.838 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_cache_other_bytes": 0, 2024-03-19T02:05:50.838 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_cache_other_items": 0, 2024-03-19T02:05:50.838 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_cache_buffer_bytes": 0, 2024-03-19T02:05:50.838 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_cache_buffer_items": 0, 2024-03-19T02:05:50.838 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_extent_bytes": 0, 2024-03-19T02:05:50.839 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_extent_items": 0, 2024-03-19T02:05:50.839 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_blob_bytes": 0, 2024-03-19T02:05:50.839 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_blob_items": 0, 2024-03-19T02:05:50.839 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_shared_blob_bytes": 0, 2024-03-19T02:05:50.839 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_shared_blob_items": 0, 2024-03-19T02:05:50.839 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_inline_bl_bytes": 0, 2024-03-19T02:05:50.839 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_inline_bl_items": 0, 2024-03-19T02:05:50.839 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_fsck_bytes": 0, 2024-03-19T02:05:50.839 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_fsck_items": 0, 2024-03-19T02:05:50.839 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_txc_bytes": 0, 2024-03-19T02:05:50.839 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_txc_items": 0, 2024-03-19T02:05:50.839 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_writing_deferred_bytes": 0, 2024-03-19T02:05:50.839 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_writing_deferred_items": 0, 2024-03-19T02:05:50.839 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_writing_bytes": 0, 2024-03-19T02:05:50.839 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_writing_items": 0, 2024-03-19T02:05:50.840 INFO:teuthology.orchestra.run.smithi112.stdout: "bluefs_bytes": 0, 2024-03-19T02:05:50.840 INFO:teuthology.orchestra.run.smithi112.stdout: "bluefs_items": 0, 2024-03-19T02:05:50.840 INFO:teuthology.orchestra.run.smithi112.stdout: "bluefs_file_reader_bytes": 0, 2024-03-19T02:05:50.840 INFO:teuthology.orchestra.run.smithi112.stdout: "bluefs_file_reader_items": 0, 2024-03-19T02:05:50.840 INFO:teuthology.orchestra.run.smithi112.stdout: "bluefs_file_writer_bytes": 0, 2024-03-19T02:05:50.840 INFO:teuthology.orchestra.run.smithi112.stdout: "bluefs_file_writer_items": 0, 2024-03-19T02:05:50.840 INFO:teuthology.orchestra.run.smithi112.stdout: "buffer_anon_bytes": 921853, 2024-03-19T02:05:50.840 INFO:teuthology.orchestra.run.smithi112.stdout: "buffer_anon_items": 567, 2024-03-19T02:05:50.840 INFO:teuthology.orchestra.run.smithi112.stdout: "buffer_meta_bytes": 352, 2024-03-19T02:05:50.840 INFO:teuthology.orchestra.run.smithi112.stdout: "buffer_meta_items": 4, 2024-03-19T02:05:50.840 INFO:teuthology.orchestra.run.smithi112.stdout: "osd_bytes": 0, 2024-03-19T02:05:50.840 INFO:teuthology.orchestra.run.smithi112.stdout: "osd_items": 0, 2024-03-19T02:05:50.840 INFO:teuthology.orchestra.run.smithi112.stdout: "osd_mapbl_bytes": 0, 2024-03-19T02:05:50.840 INFO:teuthology.orchestra.run.smithi112.stdout: "osd_mapbl_items": 0, 2024-03-19T02:05:50.841 INFO:teuthology.orchestra.run.smithi112.stdout: "osd_pglog_bytes": 0, 2024-03-19T02:05:50.841 INFO:teuthology.orchestra.run.smithi112.stdout: "osd_pglog_items": 0, 2024-03-19T02:05:50.841 INFO:teuthology.orchestra.run.smithi112.stdout: "osdmap_bytes": 7816, 2024-03-19T02:05:50.841 INFO:teuthology.orchestra.run.smithi112.stdout: "osdmap_items": 164, 2024-03-19T02:05:50.841 INFO:teuthology.orchestra.run.smithi112.stdout: "osdmap_mapping_bytes": 0, 2024-03-19T02:05:50.841 INFO:teuthology.orchestra.run.smithi112.stdout: "osdmap_mapping_items": 0, 2024-03-19T02:05:50.841 INFO:teuthology.orchestra.run.smithi112.stdout: "pgmap_bytes": 0, 2024-03-19T02:05:50.841 INFO:teuthology.orchestra.run.smithi112.stdout: "pgmap_items": 0, 2024-03-19T02:05:50.841 INFO:teuthology.orchestra.run.smithi112.stdout: "mds_co_bytes": 1179145, 2024-03-19T02:05:50.841 INFO:teuthology.orchestra.run.smithi112.stdout: "mds_co_items": 44537, 2024-03-19T02:05:50.841 INFO:teuthology.orchestra.run.smithi112.stdout: "unittest_1_bytes": 0, 2024-03-19T02:05:50.841 INFO:teuthology.orchestra.run.smithi112.stdout: "unittest_1_items": 0, 2024-03-19T02:05:50.841 INFO:teuthology.orchestra.run.smithi112.stdout: "unittest_2_bytes": 0, 2024-03-19T02:05:50.841 INFO:teuthology.orchestra.run.smithi112.stdout: "unittest_2_items": 0 2024-03-19T02:05:50.841 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.841 INFO:teuthology.orchestra.run.smithi112.stdout: "objecter": { 2024-03-19T02:05:50.842 INFO:teuthology.orchestra.run.smithi112.stdout: "op_active": 0, 2024-03-19T02:05:50.842 INFO:teuthology.orchestra.run.smithi112.stdout: "op_laggy": 0, 2024-03-19T02:05:50.842 INFO:teuthology.orchestra.run.smithi112.stdout: "op_send": 76708, 2024-03-19T02:05:50.842 INFO:teuthology.orchestra.run.smithi112.stdout: "op_send_bytes": 287061065, 2024-03-19T02:05:50.842 INFO:teuthology.orchestra.run.smithi112.stdout: "op_resend": 28, 2024-03-19T02:05:50.842 INFO:teuthology.orchestra.run.smithi112.stdout: "op_reply": 76708, 2024-03-19T02:05:50.842 INFO:teuthology.orchestra.run.smithi112.stdout: "op_latency": { 2024-03-19T02:05:50.842 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 76708, 2024-03-19T02:05:50.842 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 579.935417014, 2024-03-19T02:05:50.842 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.007560299 2024-03-19T02:05:50.842 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.842 INFO:teuthology.orchestra.run.smithi112.stdout: "op_inflight": 0, 2024-03-19T02:05:50.842 INFO:teuthology.orchestra.run.smithi112.stdout: "oplen_avg": { 2024-03-19T02:05:50.842 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 76708, 2024-03-19T02:05:50.843 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 78164 2024-03-19T02:05:50.843 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.843 INFO:teuthology.orchestra.run.smithi112.stdout: "op": 76708, 2024-03-19T02:05:50.843 INFO:teuthology.orchestra.run.smithi112.stdout: "op_r": 826, 2024-03-19T02:05:50.843 INFO:teuthology.orchestra.run.smithi112.stdout: "op_w": 75882, 2024-03-19T02:05:50.843 INFO:teuthology.orchestra.run.smithi112.stdout: "op_rmw": 0, 2024-03-19T02:05:50.843 INFO:teuthology.orchestra.run.smithi112.stdout: "op_pg": 0, 2024-03-19T02:05:50.843 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_stat": 0, 2024-03-19T02:05:50.843 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_create": 121, 2024-03-19T02:05:50.843 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_read": 826, 2024-03-19T02:05:50.843 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_write": 45264, 2024-03-19T02:05:50.843 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_writefull": 412, 2024-03-19T02:05:50.843 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_writesame": 0, 2024-03-19T02:05:50.843 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_append": 0, 2024-03-19T02:05:50.843 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_zero": 0, 2024-03-19T02:05:50.844 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_truncate": 0, 2024-03-19T02:05:50.844 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_delete": 29325, 2024-03-19T02:05:50.844 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_mapext": 0, 2024-03-19T02:05:50.844 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_sparse_read": 0, 2024-03-19T02:05:50.844 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_clonerange": 0, 2024-03-19T02:05:50.844 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_getxattr": 0, 2024-03-19T02:05:50.844 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_setxattr": 242, 2024-03-19T02:05:50.844 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_cmpxattr": 0, 2024-03-19T02:05:50.844 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_rmxattr": 0, 2024-03-19T02:05:50.844 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_resetxattrs": 0, 2024-03-19T02:05:50.844 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_call": 0, 2024-03-19T02:05:50.844 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_watch": 0, 2024-03-19T02:05:50.844 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_notify": 0, 2024-03-19T02:05:50.844 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_src_cmpxattr": 0, 2024-03-19T02:05:50.844 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_pgls": 0, 2024-03-19T02:05:50.845 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_pgls_filter": 0, 2024-03-19T02:05:50.845 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_other": 0, 2024-03-19T02:05:50.845 INFO:teuthology.orchestra.run.smithi112.stdout: "linger_active": 0, 2024-03-19T02:05:50.845 INFO:teuthology.orchestra.run.smithi112.stdout: "linger_send": 0, 2024-03-19T02:05:50.845 INFO:teuthology.orchestra.run.smithi112.stdout: "linger_resend": 0, 2024-03-19T02:05:50.845 INFO:teuthology.orchestra.run.smithi112.stdout: "linger_ping": 0, 2024-03-19T02:05:50.845 INFO:teuthology.orchestra.run.smithi112.stdout: "poolop_active": 0, 2024-03-19T02:05:50.845 INFO:teuthology.orchestra.run.smithi112.stdout: "poolop_send": 0, 2024-03-19T02:05:50.845 INFO:teuthology.orchestra.run.smithi112.stdout: "poolop_resend": 0, 2024-03-19T02:05:50.845 INFO:teuthology.orchestra.run.smithi112.stdout: "poolstat_active": 0, 2024-03-19T02:05:50.845 INFO:teuthology.orchestra.run.smithi112.stdout: "poolstat_send": 0, 2024-03-19T02:05:50.845 INFO:teuthology.orchestra.run.smithi112.stdout: "poolstat_resend": 0, 2024-03-19T02:05:50.845 INFO:teuthology.orchestra.run.smithi112.stdout: "statfs_active": 0, 2024-03-19T02:05:50.845 INFO:teuthology.orchestra.run.smithi112.stdout: "statfs_send": 0, 2024-03-19T02:05:50.845 INFO:teuthology.orchestra.run.smithi112.stdout: "statfs_resend": 0, 2024-03-19T02:05:50.846 INFO:teuthology.orchestra.run.smithi112.stdout: "command_active": 0, 2024-03-19T02:05:50.846 INFO:teuthology.orchestra.run.smithi112.stdout: "command_send": 0, 2024-03-19T02:05:50.846 INFO:teuthology.orchestra.run.smithi112.stdout: "command_resend": 0, 2024-03-19T02:05:50.846 INFO:teuthology.orchestra.run.smithi112.stdout: "map_epoch": 92, 2024-03-19T02:05:50.846 INFO:teuthology.orchestra.run.smithi112.stdout: "map_full": 0, 2024-03-19T02:05:50.846 INFO:teuthology.orchestra.run.smithi112.stdout: "map_inc": 5, 2024-03-19T02:05:50.846 INFO:teuthology.orchestra.run.smithi112.stdout: "osd_sessions": 12, 2024-03-19T02:05:50.846 INFO:teuthology.orchestra.run.smithi112.stdout: "osd_session_open": 12, 2024-03-19T02:05:50.846 INFO:teuthology.orchestra.run.smithi112.stdout: "osd_session_close": 0, 2024-03-19T02:05:50.846 INFO:teuthology.orchestra.run.smithi112.stdout: "osd_laggy": 0, 2024-03-19T02:05:50.846 INFO:teuthology.orchestra.run.smithi112.stdout: "omap_wr": 1504, 2024-03-19T02:05:50.846 INFO:teuthology.orchestra.run.smithi112.stdout: "omap_rd": 0, 2024-03-19T02:05:50.846 INFO:teuthology.orchestra.run.smithi112.stdout: "omap_del": 470 2024-03-19T02:05:50.846 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.846 INFO:teuthology.orchestra.run.smithi112.stdout: "oft": { 2024-03-19T02:05:50.847 INFO:teuthology.orchestra.run.smithi112.stdout: "omap_total_objs": 1, 2024-03-19T02:05:50.847 INFO:teuthology.orchestra.run.smithi112.stdout: "omap_total_kv_pairs": 0, 2024-03-19T02:05:50.847 INFO:teuthology.orchestra.run.smithi112.stdout: "omap_total_updates": 0, 2024-03-19T02:05:50.847 INFO:teuthology.orchestra.run.smithi112.stdout: "omap_total_removes": 0 2024-03-19T02:05:50.847 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.847 INFO:teuthology.orchestra.run.smithi112.stdout: "osd-slow-ops": { 2024-03-19T02:05:50.847 INFO:teuthology.orchestra.run.smithi112.stdout: "slow_ops_count": 0 2024-03-19T02:05:50.847 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.847 INFO:teuthology.orchestra.run.smithi112.stdout: "purge_queue": { 2024-03-19T02:05:50.847 INFO:teuthology.orchestra.run.smithi112.stdout: "pq_executing_ops": 0, 2024-03-19T02:05:50.847 INFO:teuthology.orchestra.run.smithi112.stdout: "pq_executing_ops_high_water": 10, 2024-03-19T02:05:50.847 INFO:teuthology.orchestra.run.smithi112.stdout: "pq_executing": 0, 2024-03-19T02:05:50.847 INFO:teuthology.orchestra.run.smithi112.stdout: "pq_executing_high_water": 10, 2024-03-19T02:05:50.847 INFO:teuthology.orchestra.run.smithi112.stdout: "pq_executed_ops": 29234, 2024-03-19T02:05:50.847 INFO:teuthology.orchestra.run.smithi112.stdout: "pq_executed": 29234, 2024-03-19T02:05:50.848 INFO:teuthology.orchestra.run.smithi112.stdout: "pq_item_in_journal": 0 2024-03-19T02:05:50.848 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.848 INFO:teuthology.orchestra.run.smithi112.stdout: "throttle-msgr_dispatch_throttler-mds": { 2024-03-19T02:05:50.848 INFO:teuthology.orchestra.run.smithi112.stdout: "val": 0, 2024-03-19T02:05:50.848 INFO:teuthology.orchestra.run.smithi112.stdout: "max": 104857600, 2024-03-19T02:05:50.848 INFO:teuthology.orchestra.run.smithi112.stdout: "get_started": 0, 2024-03-19T02:05:50.848 INFO:teuthology.orchestra.run.smithi112.stdout: "get": 1027783, 2024-03-19T02:05:50.848 INFO:teuthology.orchestra.run.smithi112.stdout: "get_sum": 183218698, 2024-03-19T02:05:50.848 INFO:teuthology.orchestra.run.smithi112.stdout: "get_or_fail_fail": 0, 2024-03-19T02:05:50.848 INFO:teuthology.orchestra.run.smithi112.stdout: "get_or_fail_success": 1027783, 2024-03-19T02:05:50.848 INFO:teuthology.orchestra.run.smithi112.stdout: "take": 0, 2024-03-19T02:05:50.848 INFO:teuthology.orchestra.run.smithi112.stdout: "take_sum": 0, 2024-03-19T02:05:50.848 INFO:teuthology.orchestra.run.smithi112.stdout: "put": 1027783, 2024-03-19T02:05:50.848 INFO:teuthology.orchestra.run.smithi112.stdout: "put_sum": 183218698, 2024-03-19T02:05:50.848 INFO:teuthology.orchestra.run.smithi112.stdout: "wait": { 2024-03-19T02:05:50.849 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:50.849 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:50.849 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:50.849 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-03-19T02:05:50.849 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.849 INFO:teuthology.orchestra.run.smithi112.stdout: "throttle-objecter_bytes": { 2024-03-19T02:05:50.849 INFO:teuthology.orchestra.run.smithi112.stdout: "val": 0, 2024-03-19T02:05:50.849 INFO:teuthology.orchestra.run.smithi112.stdout: "max": 104857600, 2024-03-19T02:05:50.849 INFO:teuthology.orchestra.run.smithi112.stdout: "get_started": 0, 2024-03-19T02:05:50.849 INFO:teuthology.orchestra.run.smithi112.stdout: "get": 0, 2024-03-19T02:05:50.849 INFO:teuthology.orchestra.run.smithi112.stdout: "get_sum": 0, 2024-03-19T02:05:50.849 INFO:teuthology.orchestra.run.smithi112.stdout: "get_or_fail_fail": 0, 2024-03-19T02:05:50.849 INFO:teuthology.orchestra.run.smithi112.stdout: "get_or_fail_success": 0, 2024-03-19T02:05:50.849 INFO:teuthology.orchestra.run.smithi112.stdout: "take": 76708, 2024-03-19T02:05:50.850 INFO:teuthology.orchestra.run.smithi112.stdout: "take_sum": 290013699, 2024-03-19T02:05:50.850 INFO:teuthology.orchestra.run.smithi112.stdout: "put": 76708, 2024-03-19T02:05:50.850 INFO:teuthology.orchestra.run.smithi112.stdout: "put_sum": 290013699, 2024-03-19T02:05:50.850 INFO:teuthology.orchestra.run.smithi112.stdout: "wait": { 2024-03-19T02:05:50.850 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:50.850 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:50.850 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:50.850 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-03-19T02:05:50.850 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.850 INFO:teuthology.orchestra.run.smithi112.stdout: "throttle-objecter_ops": { 2024-03-19T02:05:50.850 INFO:teuthology.orchestra.run.smithi112.stdout: "val": 0, 2024-03-19T02:05:50.850 INFO:teuthology.orchestra.run.smithi112.stdout: "max": 1024, 2024-03-19T02:05:50.850 INFO:teuthology.orchestra.run.smithi112.stdout: "get_started": 0, 2024-03-19T02:05:50.850 INFO:teuthology.orchestra.run.smithi112.stdout: "get": 0, 2024-03-19T02:05:50.850 INFO:teuthology.orchestra.run.smithi112.stdout: "get_sum": 0, 2024-03-19T02:05:50.851 INFO:teuthology.orchestra.run.smithi112.stdout: "get_or_fail_fail": 0, 2024-03-19T02:05:50.851 INFO:teuthology.orchestra.run.smithi112.stdout: "get_or_fail_success": 0, 2024-03-19T02:05:50.851 INFO:teuthology.orchestra.run.smithi112.stdout: "take": 76708, 2024-03-19T02:05:50.851 INFO:teuthology.orchestra.run.smithi112.stdout: "take_sum": 76708, 2024-03-19T02:05:50.851 INFO:teuthology.orchestra.run.smithi112.stdout: "put": 76708, 2024-03-19T02:05:50.851 INFO:teuthology.orchestra.run.smithi112.stdout: "put_sum": 76708, 2024-03-19T02:05:50.851 INFO:teuthology.orchestra.run.smithi112.stdout: "wait": { 2024-03-19T02:05:50.851 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:50.851 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:50.851 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:50.851 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-03-19T02:05:50.851 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.851 INFO:teuthology.orchestra.run.smithi112.stdout: "throttle-write_buf_throttle": { 2024-03-19T02:05:50.851 INFO:teuthology.orchestra.run.smithi112.stdout: "val": 0, 2024-03-19T02:05:50.851 INFO:teuthology.orchestra.run.smithi112.stdout: "max": 3758096384, 2024-03-19T02:05:50.852 INFO:teuthology.orchestra.run.smithi112.stdout: "get_started": 0, 2024-03-19T02:05:50.852 INFO:teuthology.orchestra.run.smithi112.stdout: "get": 29234, 2024-03-19T02:05:50.852 INFO:teuthology.orchestra.run.smithi112.stdout: "get_sum": 2952634, 2024-03-19T02:05:50.852 INFO:teuthology.orchestra.run.smithi112.stdout: "get_or_fail_fail": 0, 2024-03-19T02:05:50.852 INFO:teuthology.orchestra.run.smithi112.stdout: "get_or_fail_success": 29234, 2024-03-19T02:05:50.852 INFO:teuthology.orchestra.run.smithi112.stdout: "take": 0, 2024-03-19T02:05:50.852 INFO:teuthology.orchestra.run.smithi112.stdout: "take_sum": 0, 2024-03-19T02:05:50.852 INFO:teuthology.orchestra.run.smithi112.stdout: "put": 826, 2024-03-19T02:05:50.852 INFO:teuthology.orchestra.run.smithi112.stdout: "put_sum": 2952634, 2024-03-19T02:05:50.852 INFO:teuthology.orchestra.run.smithi112.stdout: "wait": { 2024-03-19T02:05:50.852 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:50.852 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:50.852 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:50.852 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-03-19T02:05:50.852 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:50.853 INFO:teuthology.orchestra.run.smithi112.stdout: "throttle-write_buf_throttle-0x5653aa0a0500": { 2024-03-19T02:05:50.853 INFO:teuthology.orchestra.run.smithi112.stdout: "val": 56391, 2024-03-19T02:05:50.853 INFO:teuthology.orchestra.run.smithi112.stdout: "max": 3758096384, 2024-03-19T02:05:50.853 INFO:teuthology.orchestra.run.smithi112.stdout: "get_started": 0, 2024-03-19T02:05:50.853 INFO:teuthology.orchestra.run.smithi112.stdout: "get": 135675, 2024-03-19T02:05:50.853 INFO:teuthology.orchestra.run.smithi112.stdout: "get_sum": 281074759, 2024-03-19T02:05:50.853 INFO:teuthology.orchestra.run.smithi112.stdout: "get_or_fail_fail": 0, 2024-03-19T02:05:50.853 INFO:teuthology.orchestra.run.smithi112.stdout: "get_or_fail_success": 135675, 2024-03-19T02:05:50.853 INFO:teuthology.orchestra.run.smithi112.stdout: "take": 0, 2024-03-19T02:05:50.853 INFO:teuthology.orchestra.run.smithi112.stdout: "take_sum": 0, 2024-03-19T02:05:50.853 INFO:teuthology.orchestra.run.smithi112.stdout: "put": 44438, 2024-03-19T02:05:50.853 INFO:teuthology.orchestra.run.smithi112.stdout: "put_sum": 281018368, 2024-03-19T02:05:50.853 INFO:teuthology.orchestra.run.smithi112.stdout: "wait": { 2024-03-19T02:05:50.853 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:50.853 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:50.853 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:50.854 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-03-19T02:05:50.854 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-03-19T02:05:50.854 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-03-19T02:05:51.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:05:50 smithi112 ceph-mon[66819]: pgmap v1093: 97 pgs: 97 active+clean; 483 MiB data, 9.1 GiB used, 1.0 TiB / 1.0 TiB avail; 174 KiB/s rd, 3.4 MiB/s wr, 148 op/s 2024-03-19T02:05:51.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:05:50 smithi115 ceph-mon[67332]: pgmap v1093: 97 pgs: 97 active+clean; 483 MiB data, 9.1 GiB used, 1.0 TiB / 1.0 TiB avail; 174 KiB/s rd, 3.4 MiB/s wr, 148 op/s 2024-03-19T02:05:51.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:05:50 smithi099 ceph-mon[62951]: pgmap v1093: 97 pgs: 97 active+clean; 483 MiB data, 9.1 GiB used, 1.0 TiB / 1.0 TiB avail; 174 KiB/s rd, 3.4 MiB/s wr, 148 op/s 2024-03-19T02:05:51.733 INFO:tasks.check_counter:Daemon mds.b mds.exported=5 2024-03-19T02:05:51.733 INFO:tasks.check_counter:Daemon mds.b mds.imported=6 2024-03-19T02:05:51.733 INFO:tasks.check_counter:Daemon mds.b mds_cache.dir_update=31 2024-03-19T02:05:51.733 INFO:tasks.check_counter:Daemon mds.b mds_cache.dir_update_receipt=930 2024-03-19T02:05:51.733 DEBUG:tasks.check_counter:Getting stats from e 2024-03-19T02:05:51.734 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph daemon mds.e perf dump 2024-03-19T02:05:51.999 INFO:teuthology.orchestra.run.smithi112.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.b/config 2024-03-19T02:05:53.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:05:52 smithi112 ceph-mon[66819]: pgmap v1094: 97 pgs: 97 active+clean; 476 MiB data, 9.1 GiB used, 1.0 TiB / 1.0 TiB avail; 153 KiB/s rd, 3.3 MiB/s wr, 138 op/s 2024-03-19T02:05:53.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:05:52 smithi115 ceph-mon[67332]: pgmap v1094: 97 pgs: 97 active+clean; 476 MiB data, 9.1 GiB used, 1.0 TiB / 1.0 TiB avail; 153 KiB/s rd, 3.3 MiB/s wr, 138 op/s 2024-03-19T02:05:53.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:05:52 smithi099 ceph-mon[62951]: pgmap v1094: 97 pgs: 97 active+clean; 476 MiB data, 9.1 GiB used, 1.0 TiB / 1.0 TiB avail; 153 KiB/s rd, 3.3 MiB/s wr, 138 op/s 2024-03-19T02:05:55.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:05:54 smithi115 ceph-mon[67332]: pgmap v1095: 97 pgs: 97 active+clean; 468 MiB data, 9.1 GiB used, 1.0 TiB / 1.0 TiB avail; 194 KiB/s rd, 3.7 MiB/s wr, 142 op/s 2024-03-19T02:05:55.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:05:54 smithi099 ceph-mon[62951]: pgmap v1095: 97 pgs: 97 active+clean; 468 MiB data, 9.1 GiB used, 1.0 TiB / 1.0 TiB avail; 194 KiB/s rd, 3.7 MiB/s wr, 142 op/s 2024-03-19T02:05:55.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:05:54 smithi112 ceph-mon[66819]: pgmap v1095: 97 pgs: 97 active+clean; 468 MiB data, 9.1 GiB used, 1.0 TiB / 1.0 TiB avail; 194 KiB/s rd, 3.7 MiB/s wr, 142 op/s 2024-03-19T02:05:56.444 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-03-19T02:05:56.445 INFO:teuthology.orchestra.run.smithi112.stdout: "AsyncMessenger::Worker-0": { 2024-03-19T02:05:56.445 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_recv_messages": 1895, 2024-03-19T02:05:56.445 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_send_messages": 1895, 2024-03-19T02:05:56.445 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_recv_bytes": 235369520, 2024-03-19T02:05:56.445 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_send_bytes": 601850, 2024-03-19T02:05:56.445 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_created_connections": 8, 2024-03-19T02:05:56.445 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_active_connections": 4, 2024-03-19T02:05:56.445 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_running_total_time": 0.224268404, 2024-03-19T02:05:56.445 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_running_send_time": 0.032577785, 2024-03-19T02:05:56.445 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_running_recv_time": 0.144177599, 2024-03-19T02:05:56.445 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_running_fast_dispatch_time": 0.036981244, 2024-03-19T02:05:56.445 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_send_messages_queue_lat": { 2024-03-19T02:05:56.445 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 1894, 2024-03-19T02:05:56.446 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.041584409, 2024-03-19T02:05:56.446 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000021955 2024-03-19T02:05:56.446 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.446 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_handle_ack_lat": { 2024-03-19T02:05:56.446 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:56.446 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:56.446 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:56.446 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.446 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_recv_encrypted_bytes": 3072, 2024-03-19T02:05:56.446 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_send_encrypted_bytes": 448 2024-03-19T02:05:56.446 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.446 INFO:teuthology.orchestra.run.smithi112.stdout: "AsyncMessenger::Worker-1": { 2024-03-19T02:05:56.446 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_recv_messages": 2421, 2024-03-19T02:05:56.446 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_send_messages": 2389, 2024-03-19T02:05:56.447 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_recv_bytes": 236321981, 2024-03-19T02:05:56.447 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_send_bytes": 889625, 2024-03-19T02:05:56.447 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_created_connections": 6, 2024-03-19T02:05:56.447 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_active_connections": 5, 2024-03-19T02:05:56.447 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_running_total_time": 0.296430230, 2024-03-19T02:05:56.447 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_running_send_time": 0.046683007, 2024-03-19T02:05:56.447 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_running_recv_time": 0.192089318, 2024-03-19T02:05:56.447 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_running_fast_dispatch_time": 0.048489392, 2024-03-19T02:05:56.447 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_send_messages_queue_lat": { 2024-03-19T02:05:56.447 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 2388, 2024-03-19T02:05:56.447 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.060831018, 2024-03-19T02:05:56.447 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000025473 2024-03-19T02:05:56.447 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.447 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_handle_ack_lat": { 2024-03-19T02:05:56.448 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:56.448 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:56.448 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:56.448 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.448 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_recv_encrypted_bytes": 948288, 2024-03-19T02:05:56.448 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_send_encrypted_bytes": 272064 2024-03-19T02:05:56.448 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.448 INFO:teuthology.orchestra.run.smithi112.stdout: "AsyncMessenger::Worker-2": { 2024-03-19T02:05:56.448 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_recv_messages": 2541, 2024-03-19T02:05:56.448 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_send_messages": 2872, 2024-03-19T02:05:56.448 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_recv_bytes": 118215576, 2024-03-19T02:05:56.448 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_send_bytes": 1475192, 2024-03-19T02:05:56.448 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_created_connections": 6, 2024-03-19T02:05:56.448 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_active_connections": 5, 2024-03-19T02:05:56.449 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_running_total_time": 0.237131810, 2024-03-19T02:05:56.449 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_running_send_time": 0.058187369, 2024-03-19T02:05:56.449 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_running_recv_time": 0.103303391, 2024-03-19T02:05:56.449 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_running_fast_dispatch_time": 0.058988517, 2024-03-19T02:05:56.449 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_send_messages_queue_lat": { 2024-03-19T02:05:56.449 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 2870, 2024-03-19T02:05:56.449 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.076213653, 2024-03-19T02:05:56.449 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000026555 2024-03-19T02:05:56.449 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.449 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_handle_ack_lat": { 2024-03-19T02:05:56.449 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:56.449 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:56.449 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:56.449 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.450 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_recv_encrypted_bytes": 160, 2024-03-19T02:05:56.450 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_send_encrypted_bytes": 667472 2024-03-19T02:05:56.450 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.450 INFO:teuthology.orchestra.run.smithi112.stdout: "cct": { 2024-03-19T02:05:56.450 INFO:teuthology.orchestra.run.smithi112.stdout: "total_workers": 1, 2024-03-19T02:05:56.450 INFO:teuthology.orchestra.run.smithi112.stdout: "unhealthy_workers": 0 2024-03-19T02:05:56.450 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.450 INFO:teuthology.orchestra.run.smithi112.stdout: "finisher-MDSRank": { 2024-03-19T02:05:56.450 INFO:teuthology.orchestra.run.smithi112.stdout: "queue_len": 0, 2024-03-19T02:05:56.450 INFO:teuthology.orchestra.run.smithi112.stdout: "complete_latency": { 2024-03-19T02:05:56.450 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 10866, 2024-03-19T02:05:56.450 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.255445602, 2024-03-19T02:05:56.450 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000023508 2024-03-19T02:05:56.450 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-03-19T02:05:56.451 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.451 INFO:teuthology.orchestra.run.smithi112.stdout: "finisher-PurgeQueue": { 2024-03-19T02:05:56.451 INFO:teuthology.orchestra.run.smithi112.stdout: "queue_len": 0, 2024-03-19T02:05:56.451 INFO:teuthology.orchestra.run.smithi112.stdout: "complete_latency": { 2024-03-19T02:05:56.451 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:56.451 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:56.451 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:56.451 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-03-19T02:05:56.451 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.451 INFO:teuthology.orchestra.run.smithi112.stdout: "mds": { 2024-03-19T02:05:56.451 INFO:teuthology.orchestra.run.smithi112.stdout: "request": 0, 2024-03-19T02:05:56.451 INFO:teuthology.orchestra.run.smithi112.stdout: "reply": 0, 2024-03-19T02:05:56.451 INFO:teuthology.orchestra.run.smithi112.stdout: "reply_latency": { 2024-03-19T02:05:56.451 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:56.452 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:56.452 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:56.452 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.452 INFO:teuthology.orchestra.run.smithi112.stdout: "slow_reply": 0, 2024-03-19T02:05:56.452 INFO:teuthology.orchestra.run.smithi112.stdout: "forward": 0, 2024-03-19T02:05:56.452 INFO:teuthology.orchestra.run.smithi112.stdout: "dir_fetch_complete": 0, 2024-03-19T02:05:56.452 INFO:teuthology.orchestra.run.smithi112.stdout: "dir_fetch_keys": 0, 2024-03-19T02:05:56.452 INFO:teuthology.orchestra.run.smithi112.stdout: "dir_commit": 0, 2024-03-19T02:05:56.452 INFO:teuthology.orchestra.run.smithi112.stdout: "dir_split": 0, 2024-03-19T02:05:56.452 INFO:teuthology.orchestra.run.smithi112.stdout: "dir_merge": 0, 2024-03-19T02:05:56.452 INFO:teuthology.orchestra.run.smithi112.stdout: "inodes": 62511, 2024-03-19T02:05:56.452 INFO:teuthology.orchestra.run.smithi112.stdout: "inodes_top": 28435, 2024-03-19T02:05:56.452 INFO:teuthology.orchestra.run.smithi112.stdout: "inodes_bottom": 34076, 2024-03-19T02:05:56.452 INFO:teuthology.orchestra.run.smithi112.stdout: "inodes_pin_tail": 0, 2024-03-19T02:05:56.453 INFO:teuthology.orchestra.run.smithi112.stdout: "inodes_pinned": 21889, 2024-03-19T02:05:56.453 INFO:teuthology.orchestra.run.smithi112.stdout: "inodes_expired": 0, 2024-03-19T02:05:56.453 INFO:teuthology.orchestra.run.smithi112.stdout: "inodes_with_caps": 0, 2024-03-19T02:05:56.453 INFO:teuthology.orchestra.run.smithi112.stdout: "caps": 0, 2024-03-19T02:05:56.453 INFO:teuthology.orchestra.run.smithi112.stdout: "subtrees": 1, 2024-03-19T02:05:56.453 INFO:teuthology.orchestra.run.smithi112.stdout: "traverse": 0, 2024-03-19T02:05:56.453 INFO:teuthology.orchestra.run.smithi112.stdout: "traverse_hit": 0, 2024-03-19T02:05:56.453 INFO:teuthology.orchestra.run.smithi112.stdout: "traverse_forward": 0, 2024-03-19T02:05:56.453 INFO:teuthology.orchestra.run.smithi112.stdout: "traverse_discover": 0, 2024-03-19T02:05:56.453 INFO:teuthology.orchestra.run.smithi112.stdout: "traverse_dir_fetch": 0, 2024-03-19T02:05:56.453 INFO:teuthology.orchestra.run.smithi112.stdout: "traverse_remote_ino": 0, 2024-03-19T02:05:56.453 INFO:teuthology.orchestra.run.smithi112.stdout: "traverse_lock": 0, 2024-03-19T02:05:56.453 INFO:teuthology.orchestra.run.smithi112.stdout: "load_cent": 0, 2024-03-19T02:05:56.453 INFO:teuthology.orchestra.run.smithi112.stdout: "q": 0, 2024-03-19T02:05:56.454 INFO:teuthology.orchestra.run.smithi112.stdout: "exported": 0, 2024-03-19T02:05:56.454 INFO:teuthology.orchestra.run.smithi112.stdout: "exported_inodes": 0, 2024-03-19T02:05:56.454 INFO:teuthology.orchestra.run.smithi112.stdout: "imported": 0, 2024-03-19T02:05:56.454 INFO:teuthology.orchestra.run.smithi112.stdout: "imported_inodes": 0, 2024-03-19T02:05:56.454 INFO:teuthology.orchestra.run.smithi112.stdout: "openino_dir_fetch": 0, 2024-03-19T02:05:56.454 INFO:teuthology.orchestra.run.smithi112.stdout: "openino_backtrace_fetch": 0, 2024-03-19T02:05:56.454 INFO:teuthology.orchestra.run.smithi112.stdout: "openino_peer_discover": 0, 2024-03-19T02:05:56.454 INFO:teuthology.orchestra.run.smithi112.stdout: "root_rfiles": 0, 2024-03-19T02:05:56.454 INFO:teuthology.orchestra.run.smithi112.stdout: "root_rbytes": 0, 2024-03-19T02:05:56.454 INFO:teuthology.orchestra.run.smithi112.stdout: "root_rsnaps": 0, 2024-03-19T02:05:56.454 INFO:teuthology.orchestra.run.smithi112.stdout: "scrub_backtrace_fetch": 0, 2024-03-19T02:05:56.454 INFO:teuthology.orchestra.run.smithi112.stdout: "scrub_set_tag": 0, 2024-03-19T02:05:56.454 INFO:teuthology.orchestra.run.smithi112.stdout: "scrub_backtrace_repaired": 0, 2024-03-19T02:05:56.454 INFO:teuthology.orchestra.run.smithi112.stdout: "scrub_inotable_repaired": 0, 2024-03-19T02:05:56.455 INFO:teuthology.orchestra.run.smithi112.stdout: "scrub_dir_inodes": 0, 2024-03-19T02:05:56.455 INFO:teuthology.orchestra.run.smithi112.stdout: "scrub_dir_base_inodes": 0, 2024-03-19T02:05:56.455 INFO:teuthology.orchestra.run.smithi112.stdout: "scrub_dirfrag_rstats": 0, 2024-03-19T02:05:56.455 INFO:teuthology.orchestra.run.smithi112.stdout: "scrub_file_inodes": 0, 2024-03-19T02:05:56.455 INFO:teuthology.orchestra.run.smithi112.stdout: "handle_inode_file_caps": 0, 2024-03-19T02:05:56.455 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph_cap_op_revoke": 0, 2024-03-19T02:05:56.455 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph_cap_op_grant": 0, 2024-03-19T02:05:56.455 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph_cap_op_trunc": 0, 2024-03-19T02:05:56.455 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph_cap_op_flushsnap_ack": 0, 2024-03-19T02:05:56.455 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph_cap_op_flush_ack": 0, 2024-03-19T02:05:56.455 INFO:teuthology.orchestra.run.smithi112.stdout: "handle_client_caps": 0, 2024-03-19T02:05:56.455 INFO:teuthology.orchestra.run.smithi112.stdout: "handle_client_caps_dirty": 0, 2024-03-19T02:05:56.455 INFO:teuthology.orchestra.run.smithi112.stdout: "handle_client_cap_release": 0, 2024-03-19T02:05:56.455 INFO:teuthology.orchestra.run.smithi112.stdout: "process_request_cap_release": 0 2024-03-19T02:05:56.456 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.456 INFO:teuthology.orchestra.run.smithi112.stdout: "mds_cache": { 2024-03-19T02:05:56.456 INFO:teuthology.orchestra.run.smithi112.stdout: "dir_update": 0, 2024-03-19T02:05:56.456 INFO:teuthology.orchestra.run.smithi112.stdout: "dir_update_receipt": 0, 2024-03-19T02:05:56.456 INFO:teuthology.orchestra.run.smithi112.stdout: "dir_try_discover": 0, 2024-03-19T02:05:56.456 INFO:teuthology.orchestra.run.smithi112.stdout: "dir_send_discover": 0, 2024-03-19T02:05:56.456 INFO:teuthology.orchestra.run.smithi112.stdout: "dir_handle_discover": 0, 2024-03-19T02:05:56.456 INFO:teuthology.orchestra.run.smithi112.stdout: "num_strays": 0, 2024-03-19T02:05:56.456 INFO:teuthology.orchestra.run.smithi112.stdout: "num_strays_delayed": 0, 2024-03-19T02:05:56.456 INFO:teuthology.orchestra.run.smithi112.stdout: "num_strays_enqueuing": 0, 2024-03-19T02:05:56.456 INFO:teuthology.orchestra.run.smithi112.stdout: "strays_created": 0, 2024-03-19T02:05:56.456 INFO:teuthology.orchestra.run.smithi112.stdout: "strays_enqueued": 0, 2024-03-19T02:05:56.456 INFO:teuthology.orchestra.run.smithi112.stdout: "strays_reintegrated": 0, 2024-03-19T02:05:56.456 INFO:teuthology.orchestra.run.smithi112.stdout: "strays_migrated": 0, 2024-03-19T02:05:56.456 INFO:teuthology.orchestra.run.smithi112.stdout: "num_recovering_processing": 0, 2024-03-19T02:05:56.457 INFO:teuthology.orchestra.run.smithi112.stdout: "num_recovering_enqueued": 0, 2024-03-19T02:05:56.457 INFO:teuthology.orchestra.run.smithi112.stdout: "num_recovering_prioritized": 0, 2024-03-19T02:05:56.457 INFO:teuthology.orchestra.run.smithi112.stdout: "recovery_started": 0, 2024-03-19T02:05:56.457 INFO:teuthology.orchestra.run.smithi112.stdout: "recovery_completed": 0, 2024-03-19T02:05:56.457 INFO:teuthology.orchestra.run.smithi112.stdout: "ireq_quiesce_path": 0, 2024-03-19T02:05:56.457 INFO:teuthology.orchestra.run.smithi112.stdout: "ireq_quiesce_inode": 0, 2024-03-19T02:05:56.457 INFO:teuthology.orchestra.run.smithi112.stdout: "ireq_enqueue_scrub": 0, 2024-03-19T02:05:56.457 INFO:teuthology.orchestra.run.smithi112.stdout: "ireq_exportdir": 0, 2024-03-19T02:05:56.457 INFO:teuthology.orchestra.run.smithi112.stdout: "ireq_flush": 0, 2024-03-19T02:05:56.457 INFO:teuthology.orchestra.run.smithi112.stdout: "ireq_fragmentdir": 0, 2024-03-19T02:05:56.457 INFO:teuthology.orchestra.run.smithi112.stdout: "ireq_fragstats": 0, 2024-03-19T02:05:56.457 INFO:teuthology.orchestra.run.smithi112.stdout: "ireq_inodestats": 0 2024-03-19T02:05:56.457 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.457 INFO:teuthology.orchestra.run.smithi112.stdout: "mds_log": { 2024-03-19T02:05:56.458 INFO:teuthology.orchestra.run.smithi112.stdout: "evlrg": 0, 2024-03-19T02:05:56.458 INFO:teuthology.orchestra.run.smithi112.stdout: "evadd": 0, 2024-03-19T02:05:56.458 INFO:teuthology.orchestra.run.smithi112.stdout: "evex": 0, 2024-03-19T02:05:56.458 INFO:teuthology.orchestra.run.smithi112.stdout: "evtrm": 0, 2024-03-19T02:05:56.458 INFO:teuthology.orchestra.run.smithi112.stdout: "ev": 290524, 2024-03-19T02:05:56.458 INFO:teuthology.orchestra.run.smithi112.stdout: "evexg": 0, 2024-03-19T02:05:56.458 INFO:teuthology.orchestra.run.smithi112.stdout: "evexd": 0, 2024-03-19T02:05:56.458 INFO:teuthology.orchestra.run.smithi112.stdout: "segadd": 0, 2024-03-19T02:05:56.458 INFO:teuthology.orchestra.run.smithi112.stdout: "segex": 0, 2024-03-19T02:05:56.458 INFO:teuthology.orchestra.run.smithi112.stdout: "segtrm": 0, 2024-03-19T02:05:56.458 INFO:teuthology.orchestra.run.smithi112.stdout: "seg": 140, 2024-03-19T02:05:56.458 INFO:teuthology.orchestra.run.smithi112.stdout: "segmjr": 25, 2024-03-19T02:05:56.458 INFO:teuthology.orchestra.run.smithi112.stdout: "segexg": 0, 2024-03-19T02:05:56.458 INFO:teuthology.orchestra.run.smithi112.stdout: "segexd": 0, 2024-03-19T02:05:56.458 INFO:teuthology.orchestra.run.smithi112.stdout: "expos": 390069714, 2024-03-19T02:05:56.459 INFO:teuthology.orchestra.run.smithi112.stdout: "wrpos": 591396529, 2024-03-19T02:05:56.459 INFO:teuthology.orchestra.run.smithi112.stdout: "rdpos": 591394453, 2024-03-19T02:05:56.459 INFO:teuthology.orchestra.run.smithi112.stdout: "jlat": { 2024-03-19T02:05:56.459 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:56.459 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:56.459 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:56.459 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.459 INFO:teuthology.orchestra.run.smithi112.stdout: "replayed": 290524 2024-03-19T02:05:56.459 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.459 INFO:teuthology.orchestra.run.smithi112.stdout: "mds_mem": { 2024-03-19T02:05:56.459 INFO:teuthology.orchestra.run.smithi112.stdout: "ino": 129, 2024-03-19T02:05:56.459 INFO:teuthology.orchestra.run.smithi112.stdout: "ino+": 62572, 2024-03-19T02:05:56.459 INFO:teuthology.orchestra.run.smithi112.stdout: "ino-": 62443, 2024-03-19T02:05:56.459 INFO:teuthology.orchestra.run.smithi112.stdout: "dir": 12, 2024-03-19T02:05:56.459 INFO:teuthology.orchestra.run.smithi112.stdout: "dir+": 43, 2024-03-19T02:05:56.460 INFO:teuthology.orchestra.run.smithi112.stdout: "dir-": 31, 2024-03-19T02:05:56.460 INFO:teuthology.orchestra.run.smithi112.stdout: "dn": 62511, 2024-03-19T02:05:56.460 INFO:teuthology.orchestra.run.smithi112.stdout: "dn+": 62777, 2024-03-19T02:05:56.460 INFO:teuthology.orchestra.run.smithi112.stdout: "dn-": 266, 2024-03-19T02:05:56.460 INFO:teuthology.orchestra.run.smithi112.stdout: "cap": 0, 2024-03-19T02:05:56.460 INFO:teuthology.orchestra.run.smithi112.stdout: "cap+": 0, 2024-03-19T02:05:56.460 INFO:teuthology.orchestra.run.smithi112.stdout: "cap-": 0, 2024-03-19T02:05:56.460 INFO:teuthology.orchestra.run.smithi112.stdout: "rss": 104968, 2024-03-19T02:05:56.460 INFO:teuthology.orchestra.run.smithi112.stdout: "heap": 215420 2024-03-19T02:05:56.460 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.460 INFO:teuthology.orchestra.run.smithi112.stdout: "mds_server": { 2024-03-19T02:05:56.460 INFO:teuthology.orchestra.run.smithi112.stdout: "dispatch_client_request": 0, 2024-03-19T02:05:56.460 INFO:teuthology.orchestra.run.smithi112.stdout: "dispatch_server_request": 0, 2024-03-19T02:05:56.460 INFO:teuthology.orchestra.run.smithi112.stdout: "handle_client_request": 0, 2024-03-19T02:05:56.461 INFO:teuthology.orchestra.run.smithi112.stdout: "handle_client_session": 0, 2024-03-19T02:05:56.461 INFO:teuthology.orchestra.run.smithi112.stdout: "handle_peer_request": 0, 2024-03-19T02:05:56.461 INFO:teuthology.orchestra.run.smithi112.stdout: "req_create_latency": { 2024-03-19T02:05:56.461 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:56.461 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:56.461 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:56.461 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.461 INFO:teuthology.orchestra.run.smithi112.stdout: "req_getattr_latency": { 2024-03-19T02:05:56.461 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:56.461 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:56.461 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:56.461 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.461 INFO:teuthology.orchestra.run.smithi112.stdout: "req_getfilelock_latency": { 2024-03-19T02:05:56.461 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:56.461 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:56.462 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:56.462 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.462 INFO:teuthology.orchestra.run.smithi112.stdout: "req_link_latency": { 2024-03-19T02:05:56.462 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:56.462 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:56.462 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:56.462 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.462 INFO:teuthology.orchestra.run.smithi112.stdout: "req_lookup_latency": { 2024-03-19T02:05:56.462 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:56.462 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:56.462 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:56.462 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.462 INFO:teuthology.orchestra.run.smithi112.stdout: "req_lookuphash_latency": { 2024-03-19T02:05:56.462 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:56.462 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:56.463 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:56.463 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.463 INFO:teuthology.orchestra.run.smithi112.stdout: "req_lookupino_latency": { 2024-03-19T02:05:56.463 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:56.463 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:56.463 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:56.463 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.463 INFO:teuthology.orchestra.run.smithi112.stdout: "req_lookupname_latency": { 2024-03-19T02:05:56.463 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:56.463 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:56.463 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:56.463 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.463 INFO:teuthology.orchestra.run.smithi112.stdout: "req_lookupparent_latency": { 2024-03-19T02:05:56.463 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:56.464 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:56.464 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:56.464 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.464 INFO:teuthology.orchestra.run.smithi112.stdout: "req_lookupsnap_latency": { 2024-03-19T02:05:56.464 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:56.464 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:56.464 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:56.464 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.464 INFO:teuthology.orchestra.run.smithi112.stdout: "req_lssnap_latency": { 2024-03-19T02:05:56.464 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:56.464 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:56.464 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:56.464 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.464 INFO:teuthology.orchestra.run.smithi112.stdout: "req_mkdir_latency": { 2024-03-19T02:05:56.465 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:56.465 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:56.465 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:56.465 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.465 INFO:teuthology.orchestra.run.smithi112.stdout: "req_mknod_latency": { 2024-03-19T02:05:56.465 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:56.465 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:56.465 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:56.465 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.465 INFO:teuthology.orchestra.run.smithi112.stdout: "req_mksnap_latency": { 2024-03-19T02:05:56.465 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:56.465 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:56.465 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:56.465 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.465 INFO:teuthology.orchestra.run.smithi112.stdout: "req_open_latency": { 2024-03-19T02:05:56.466 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:56.466 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:56.466 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:56.466 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.466 INFO:teuthology.orchestra.run.smithi112.stdout: "req_readdir_latency": { 2024-03-19T02:05:56.466 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:56.466 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:56.466 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:56.466 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.466 INFO:teuthology.orchestra.run.smithi112.stdout: "req_rename_latency": { 2024-03-19T02:05:56.466 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:56.466 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:56.466 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:56.466 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.467 INFO:teuthology.orchestra.run.smithi112.stdout: "req_renamesnap_latency": { 2024-03-19T02:05:56.467 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:56.467 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:56.467 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:56.467 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.467 INFO:teuthology.orchestra.run.smithi112.stdout: "req_snapdiff_latency": { 2024-03-19T02:05:56.467 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:56.467 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:56.467 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:56.467 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.467 INFO:teuthology.orchestra.run.smithi112.stdout: "req_rmdir_latency": { 2024-03-19T02:05:56.467 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:56.467 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:56.467 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:56.468 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.468 INFO:teuthology.orchestra.run.smithi112.stdout: "req_rmsnap_latency": { 2024-03-19T02:05:56.468 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:56.468 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:56.468 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:56.468 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.468 INFO:teuthology.orchestra.run.smithi112.stdout: "req_rmxattr_latency": { 2024-03-19T02:05:56.468 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:56.468 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:56.468 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:56.468 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.468 INFO:teuthology.orchestra.run.smithi112.stdout: "req_setattr_latency": { 2024-03-19T02:05:56.468 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:56.468 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:56.468 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:56.469 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.469 INFO:teuthology.orchestra.run.smithi112.stdout: "req_setdirlayout_latency": { 2024-03-19T02:05:56.469 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:56.469 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:56.469 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:56.469 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.469 INFO:teuthology.orchestra.run.smithi112.stdout: "req_setfilelock_latency": { 2024-03-19T02:05:56.469 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:56.469 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:56.469 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:56.469 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.469 INFO:teuthology.orchestra.run.smithi112.stdout: "req_setlayout_latency": { 2024-03-19T02:05:56.469 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:56.469 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:56.470 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:56.470 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.470 INFO:teuthology.orchestra.run.smithi112.stdout: "req_setxattr_latency": { 2024-03-19T02:05:56.470 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:56.470 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:56.470 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:56.470 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.470 INFO:teuthology.orchestra.run.smithi112.stdout: "req_symlink_latency": { 2024-03-19T02:05:56.470 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:56.470 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:56.470 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:56.470 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.470 INFO:teuthology.orchestra.run.smithi112.stdout: "req_unlink_latency": { 2024-03-19T02:05:56.470 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:56.471 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:56.471 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:56.471 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.471 INFO:teuthology.orchestra.run.smithi112.stdout: "cap_revoke_eviction": 0, 2024-03-19T02:05:56.471 INFO:teuthology.orchestra.run.smithi112.stdout: "cap_acquisition_throttle": 0, 2024-03-19T02:05:56.471 INFO:teuthology.orchestra.run.smithi112.stdout: "req_getvxattr_latency": { 2024-03-19T02:05:56.471 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:56.471 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:56.471 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:56.471 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-03-19T02:05:56.471 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.471 INFO:teuthology.orchestra.run.smithi112.stdout: "mds_sessions": { 2024-03-19T02:05:56.471 INFO:teuthology.orchestra.run.smithi112.stdout: "session_count": 1, 2024-03-19T02:05:56.471 INFO:teuthology.orchestra.run.smithi112.stdout: "session_add": 2, 2024-03-19T02:05:56.472 INFO:teuthology.orchestra.run.smithi112.stdout: "session_remove": 1, 2024-03-19T02:05:56.472 INFO:teuthology.orchestra.run.smithi112.stdout: "sessions_open": 1, 2024-03-19T02:05:56.472 INFO:teuthology.orchestra.run.smithi112.stdout: "sessions_stale": 0, 2024-03-19T02:05:56.472 INFO:teuthology.orchestra.run.smithi112.stdout: "total_load": 0, 2024-03-19T02:05:56.472 INFO:teuthology.orchestra.run.smithi112.stdout: "average_load": 0, 2024-03-19T02:05:56.472 INFO:teuthology.orchestra.run.smithi112.stdout: "avg_session_uptime": 6290, 2024-03-19T02:05:56.472 INFO:teuthology.orchestra.run.smithi112.stdout: "mdthresh_evicted": 0 2024-03-19T02:05:56.472 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.472 INFO:teuthology.orchestra.run.smithi112.stdout: "mempool": { 2024-03-19T02:05:56.472 INFO:teuthology.orchestra.run.smithi112.stdout: "bloom_filter_bytes": 0, 2024-03-19T02:05:56.472 INFO:teuthology.orchestra.run.smithi112.stdout: "bloom_filter_items": 0, 2024-03-19T02:05:56.472 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_alloc_bytes": 0, 2024-03-19T02:05:56.472 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_alloc_items": 0, 2024-03-19T02:05:56.472 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_cache_data_bytes": 0, 2024-03-19T02:05:56.472 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_cache_data_items": 0, 2024-03-19T02:05:56.473 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_cache_onode_bytes": 0, 2024-03-19T02:05:56.473 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_cache_onode_items": 0, 2024-03-19T02:05:56.473 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_cache_meta_bytes": 0, 2024-03-19T02:05:56.473 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_cache_meta_items": 0, 2024-03-19T02:05:56.473 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_cache_other_bytes": 0, 2024-03-19T02:05:56.473 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_cache_other_items": 0, 2024-03-19T02:05:56.473 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_cache_buffer_bytes": 0, 2024-03-19T02:05:56.473 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_cache_buffer_items": 0, 2024-03-19T02:05:56.473 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_extent_bytes": 0, 2024-03-19T02:05:56.473 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_extent_items": 0, 2024-03-19T02:05:56.473 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_blob_bytes": 0, 2024-03-19T02:05:56.473 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_blob_items": 0, 2024-03-19T02:05:56.473 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_shared_blob_bytes": 0, 2024-03-19T02:05:56.473 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_shared_blob_items": 0, 2024-03-19T02:05:56.474 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_inline_bl_bytes": 0, 2024-03-19T02:05:56.474 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_inline_bl_items": 0, 2024-03-19T02:05:56.474 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_fsck_bytes": 0, 2024-03-19T02:05:56.474 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_fsck_items": 0, 2024-03-19T02:05:56.474 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_txc_bytes": 0, 2024-03-19T02:05:56.474 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_txc_items": 0, 2024-03-19T02:05:56.474 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_writing_deferred_bytes": 0, 2024-03-19T02:05:56.474 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_writing_deferred_items": 0, 2024-03-19T02:05:56.474 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_writing_bytes": 0, 2024-03-19T02:05:56.474 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_writing_items": 0, 2024-03-19T02:05:56.474 INFO:teuthology.orchestra.run.smithi112.stdout: "bluefs_bytes": 0, 2024-03-19T02:05:56.474 INFO:teuthology.orchestra.run.smithi112.stdout: "bluefs_items": 0, 2024-03-19T02:05:56.474 INFO:teuthology.orchestra.run.smithi112.stdout: "bluefs_file_reader_bytes": 0, 2024-03-19T02:05:56.474 INFO:teuthology.orchestra.run.smithi112.stdout: "bluefs_file_reader_items": 0, 2024-03-19T02:05:56.475 INFO:teuthology.orchestra.run.smithi112.stdout: "bluefs_file_writer_bytes": 0, 2024-03-19T02:05:56.475 INFO:teuthology.orchestra.run.smithi112.stdout: "bluefs_file_writer_items": 0, 2024-03-19T02:05:56.475 INFO:teuthology.orchestra.run.smithi112.stdout: "buffer_anon_bytes": 244524, 2024-03-19T02:05:56.475 INFO:teuthology.orchestra.run.smithi112.stdout: "buffer_anon_items": 197, 2024-03-19T02:05:56.475 INFO:teuthology.orchestra.run.smithi112.stdout: "buffer_meta_bytes": 0, 2024-03-19T02:05:56.475 INFO:teuthology.orchestra.run.smithi112.stdout: "buffer_meta_items": 0, 2024-03-19T02:05:56.475 INFO:teuthology.orchestra.run.smithi112.stdout: "osd_bytes": 0, 2024-03-19T02:05:56.475 INFO:teuthology.orchestra.run.smithi112.stdout: "osd_items": 0, 2024-03-19T02:05:56.475 INFO:teuthology.orchestra.run.smithi112.stdout: "osd_mapbl_bytes": 0, 2024-03-19T02:05:56.475 INFO:teuthology.orchestra.run.smithi112.stdout: "osd_mapbl_items": 0, 2024-03-19T02:05:56.475 INFO:teuthology.orchestra.run.smithi112.stdout: "osd_pglog_bytes": 0, 2024-03-19T02:05:56.475 INFO:teuthology.orchestra.run.smithi112.stdout: "osd_pglog_items": 0, 2024-03-19T02:05:56.475 INFO:teuthology.orchestra.run.smithi112.stdout: "osdmap_bytes": 7816, 2024-03-19T02:05:56.475 INFO:teuthology.orchestra.run.smithi112.stdout: "osdmap_items": 164, 2024-03-19T02:05:56.475 INFO:teuthology.orchestra.run.smithi112.stdout: "osdmap_mapping_bytes": 0, 2024-03-19T02:05:56.476 INFO:teuthology.orchestra.run.smithi112.stdout: "osdmap_mapping_items": 0, 2024-03-19T02:05:56.476 INFO:teuthology.orchestra.run.smithi112.stdout: "pgmap_bytes": 0, 2024-03-19T02:05:56.476 INFO:teuthology.orchestra.run.smithi112.stdout: "pgmap_items": 0, 2024-03-19T02:05:56.476 INFO:teuthology.orchestra.run.smithi112.stdout: "mds_co_bytes": 44333006, 2024-03-19T02:05:56.476 INFO:teuthology.orchestra.run.smithi112.stdout: "mds_co_items": 265450, 2024-03-19T02:05:56.476 INFO:teuthology.orchestra.run.smithi112.stdout: "unittest_1_bytes": 0, 2024-03-19T02:05:56.476 INFO:teuthology.orchestra.run.smithi112.stdout: "unittest_1_items": 0, 2024-03-19T02:05:56.476 INFO:teuthology.orchestra.run.smithi112.stdout: "unittest_2_bytes": 0, 2024-03-19T02:05:56.476 INFO:teuthology.orchestra.run.smithi112.stdout: "unittest_2_items": 0 2024-03-19T02:05:56.476 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.476 INFO:teuthology.orchestra.run.smithi112.stdout: "objecter": { 2024-03-19T02:05:56.476 INFO:teuthology.orchestra.run.smithi112.stdout: "op_active": 0, 2024-03-19T02:05:56.476 INFO:teuthology.orchestra.run.smithi112.stdout: "op_laggy": 0, 2024-03-19T02:05:56.476 INFO:teuthology.orchestra.run.smithi112.stdout: "op_send": 6373, 2024-03-19T02:05:56.476 INFO:teuthology.orchestra.run.smithi112.stdout: "op_send_bytes": 21, 2024-03-19T02:05:56.477 INFO:teuthology.orchestra.run.smithi112.stdout: "op_resend": 3, 2024-03-19T02:05:56.477 INFO:teuthology.orchestra.run.smithi112.stdout: "op_reply": 6373, 2024-03-19T02:05:56.477 INFO:teuthology.orchestra.run.smithi112.stdout: "op_latency": { 2024-03-19T02:05:56.477 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 6373, 2024-03-19T02:05:56.477 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 11.231716159, 2024-03-19T02:05:56.477 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.001762390 2024-03-19T02:05:56.477 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.477 INFO:teuthology.orchestra.run.smithi112.stdout: "op_inflight": 0, 2024-03-19T02:05:56.477 INFO:teuthology.orchestra.run.smithi112.stdout: "oplen_avg": { 2024-03-19T02:05:56.477 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 6373, 2024-03-19T02:05:56.477 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 6374 2024-03-19T02:05:56.477 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.477 INFO:teuthology.orchestra.run.smithi112.stdout: "op": 6373, 2024-03-19T02:05:56.477 INFO:teuthology.orchestra.run.smithi112.stdout: "op_r": 6373, 2024-03-19T02:05:56.478 INFO:teuthology.orchestra.run.smithi112.stdout: "op_w": 0, 2024-03-19T02:05:56.478 INFO:teuthology.orchestra.run.smithi112.stdout: "op_rmw": 0, 2024-03-19T02:05:56.478 INFO:teuthology.orchestra.run.smithi112.stdout: "op_pg": 0, 2024-03-19T02:05:56.478 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_stat": 3244, 2024-03-19T02:05:56.478 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_create": 0, 2024-03-19T02:05:56.478 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_read": 3127, 2024-03-19T02:05:56.478 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_write": 0, 2024-03-19T02:05:56.478 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_writefull": 0, 2024-03-19T02:05:56.478 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_writesame": 0, 2024-03-19T02:05:56.478 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_append": 0, 2024-03-19T02:05:56.478 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_zero": 0, 2024-03-19T02:05:56.478 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_truncate": 0, 2024-03-19T02:05:56.478 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_delete": 0, 2024-03-19T02:05:56.478 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_mapext": 0, 2024-03-19T02:05:56.478 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_sparse_read": 0, 2024-03-19T02:05:56.479 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_clonerange": 0, 2024-03-19T02:05:56.479 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_getxattr": 1, 2024-03-19T02:05:56.479 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_setxattr": 0, 2024-03-19T02:05:56.479 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_cmpxattr": 0, 2024-03-19T02:05:56.479 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_rmxattr": 0, 2024-03-19T02:05:56.479 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_resetxattrs": 0, 2024-03-19T02:05:56.479 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_call": 0, 2024-03-19T02:05:56.479 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_watch": 0, 2024-03-19T02:05:56.479 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_notify": 0, 2024-03-19T02:05:56.479 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_src_cmpxattr": 0, 2024-03-19T02:05:56.479 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_pgls": 0, 2024-03-19T02:05:56.479 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_pgls_filter": 0, 2024-03-19T02:05:56.479 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_other": 0, 2024-03-19T02:05:56.479 INFO:teuthology.orchestra.run.smithi112.stdout: "linger_active": 0, 2024-03-19T02:05:56.480 INFO:teuthology.orchestra.run.smithi112.stdout: "linger_send": 0, 2024-03-19T02:05:56.480 INFO:teuthology.orchestra.run.smithi112.stdout: "linger_resend": 0, 2024-03-19T02:05:56.480 INFO:teuthology.orchestra.run.smithi112.stdout: "linger_ping": 0, 2024-03-19T02:05:56.480 INFO:teuthology.orchestra.run.smithi112.stdout: "poolop_active": 0, 2024-03-19T02:05:56.480 INFO:teuthology.orchestra.run.smithi112.stdout: "poolop_send": 0, 2024-03-19T02:05:56.480 INFO:teuthology.orchestra.run.smithi112.stdout: "poolop_resend": 0, 2024-03-19T02:05:56.480 INFO:teuthology.orchestra.run.smithi112.stdout: "poolstat_active": 0, 2024-03-19T02:05:56.480 INFO:teuthology.orchestra.run.smithi112.stdout: "poolstat_send": 0, 2024-03-19T02:05:56.480 INFO:teuthology.orchestra.run.smithi112.stdout: "poolstat_resend": 0, 2024-03-19T02:05:56.480 INFO:teuthology.orchestra.run.smithi112.stdout: "statfs_active": 0, 2024-03-19T02:05:56.480 INFO:teuthology.orchestra.run.smithi112.stdout: "statfs_send": 0, 2024-03-19T02:05:56.480 INFO:teuthology.orchestra.run.smithi112.stdout: "statfs_resend": 0, 2024-03-19T02:05:56.480 INFO:teuthology.orchestra.run.smithi112.stdout: "command_active": 0, 2024-03-19T02:05:56.480 INFO:teuthology.orchestra.run.smithi112.stdout: "command_send": 0, 2024-03-19T02:05:56.480 INFO:teuthology.orchestra.run.smithi112.stdout: "command_resend": 0, 2024-03-19T02:05:56.481 INFO:teuthology.orchestra.run.smithi112.stdout: "map_epoch": 92, 2024-03-19T02:05:56.481 INFO:teuthology.orchestra.run.smithi112.stdout: "map_full": 0, 2024-03-19T02:05:56.481 INFO:teuthology.orchestra.run.smithi112.stdout: "map_inc": 5, 2024-03-19T02:05:56.481 INFO:teuthology.orchestra.run.smithi112.stdout: "osd_sessions": 11, 2024-03-19T02:05:56.481 INFO:teuthology.orchestra.run.smithi112.stdout: "osd_session_open": 11, 2024-03-19T02:05:56.481 INFO:teuthology.orchestra.run.smithi112.stdout: "osd_session_close": 0, 2024-03-19T02:05:56.481 INFO:teuthology.orchestra.run.smithi112.stdout: "osd_laggy": 0, 2024-03-19T02:05:56.481 INFO:teuthology.orchestra.run.smithi112.stdout: "omap_wr": 0, 2024-03-19T02:05:56.481 INFO:teuthology.orchestra.run.smithi112.stdout: "omap_rd": 2, 2024-03-19T02:05:56.481 INFO:teuthology.orchestra.run.smithi112.stdout: "omap_del": 0 2024-03-19T02:05:56.481 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.481 INFO:teuthology.orchestra.run.smithi112.stdout: "oft": { 2024-03-19T02:05:56.481 INFO:teuthology.orchestra.run.smithi112.stdout: "omap_total_objs": 0, 2024-03-19T02:05:56.481 INFO:teuthology.orchestra.run.smithi112.stdout: "omap_total_kv_pairs": 0, 2024-03-19T02:05:56.482 INFO:teuthology.orchestra.run.smithi112.stdout: "omap_total_updates": 0, 2024-03-19T02:05:56.482 INFO:teuthology.orchestra.run.smithi112.stdout: "omap_total_removes": 0 2024-03-19T02:05:56.482 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.482 INFO:teuthology.orchestra.run.smithi112.stdout: "osd-slow-ops": { 2024-03-19T02:05:56.482 INFO:teuthology.orchestra.run.smithi112.stdout: "slow_ops_count": 0 2024-03-19T02:05:56.482 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.482 INFO:teuthology.orchestra.run.smithi112.stdout: "purge_queue": { 2024-03-19T02:05:56.482 INFO:teuthology.orchestra.run.smithi112.stdout: "pq_executing_ops": 0, 2024-03-19T02:05:56.482 INFO:teuthology.orchestra.run.smithi112.stdout: "pq_executing_ops_high_water": 0, 2024-03-19T02:05:56.482 INFO:teuthology.orchestra.run.smithi112.stdout: "pq_executing": 0, 2024-03-19T02:05:56.482 INFO:teuthology.orchestra.run.smithi112.stdout: "pq_executing_high_water": 0, 2024-03-19T02:05:56.482 INFO:teuthology.orchestra.run.smithi112.stdout: "pq_executed_ops": 0, 2024-03-19T02:05:56.482 INFO:teuthology.orchestra.run.smithi112.stdout: "pq_executed": 0, 2024-03-19T02:05:56.482 INFO:teuthology.orchestra.run.smithi112.stdout: "pq_item_in_journal": 0 2024-03-19T02:05:56.482 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.483 INFO:teuthology.orchestra.run.smithi112.stdout: "throttle-msgr_dispatch_throttler-mds": { 2024-03-19T02:05:56.483 INFO:teuthology.orchestra.run.smithi112.stdout: "val": 0, 2024-03-19T02:05:56.483 INFO:teuthology.orchestra.run.smithi112.stdout: "max": 104857600, 2024-03-19T02:05:56.483 INFO:teuthology.orchestra.run.smithi112.stdout: "get_started": 0, 2024-03-19T02:05:56.483 INFO:teuthology.orchestra.run.smithi112.stdout: "get": 6853, 2024-03-19T02:05:56.483 INFO:teuthology.orchestra.run.smithi112.stdout: "get_sum": 589262678, 2024-03-19T02:05:56.483 INFO:teuthology.orchestra.run.smithi112.stdout: "get_or_fail_fail": 0, 2024-03-19T02:05:56.483 INFO:teuthology.orchestra.run.smithi112.stdout: "get_or_fail_success": 6853, 2024-03-19T02:05:56.483 INFO:teuthology.orchestra.run.smithi112.stdout: "take": 0, 2024-03-19T02:05:56.483 INFO:teuthology.orchestra.run.smithi112.stdout: "take_sum": 0, 2024-03-19T02:05:56.483 INFO:teuthology.orchestra.run.smithi112.stdout: "put": 6853, 2024-03-19T02:05:56.483 INFO:teuthology.orchestra.run.smithi112.stdout: "put_sum": 589262678, 2024-03-19T02:05:56.483 INFO:teuthology.orchestra.run.smithi112.stdout: "wait": { 2024-03-19T02:05:56.483 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:56.484 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:56.484 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:56.484 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-03-19T02:05:56.484 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.484 INFO:teuthology.orchestra.run.smithi112.stdout: "throttle-objecter_bytes": { 2024-03-19T02:05:56.484 INFO:teuthology.orchestra.run.smithi112.stdout: "val": 0, 2024-03-19T02:05:56.484 INFO:teuthology.orchestra.run.smithi112.stdout: "max": 104857600, 2024-03-19T02:05:56.484 INFO:teuthology.orchestra.run.smithi112.stdout: "get_started": 0, 2024-03-19T02:05:56.484 INFO:teuthology.orchestra.run.smithi112.stdout: "get": 0, 2024-03-19T02:05:56.484 INFO:teuthology.orchestra.run.smithi112.stdout: "get_sum": 0, 2024-03-19T02:05:56.484 INFO:teuthology.orchestra.run.smithi112.stdout: "get_or_fail_fail": 0, 2024-03-19T02:05:56.484 INFO:teuthology.orchestra.run.smithi112.stdout: "get_or_fail_success": 0, 2024-03-19T02:05:56.484 INFO:teuthology.orchestra.run.smithi112.stdout: "take": 6373, 2024-03-19T02:05:56.484 INFO:teuthology.orchestra.run.smithi112.stdout: "take_sum": 587205559, 2024-03-19T02:05:56.484 INFO:teuthology.orchestra.run.smithi112.stdout: "put": 6373, 2024-03-19T02:05:56.485 INFO:teuthology.orchestra.run.smithi112.stdout: "put_sum": 587205559, 2024-03-19T02:05:56.485 INFO:teuthology.orchestra.run.smithi112.stdout: "wait": { 2024-03-19T02:05:56.485 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:56.485 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:56.485 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:56.485 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-03-19T02:05:56.485 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.485 INFO:teuthology.orchestra.run.smithi112.stdout: "throttle-objecter_ops": { 2024-03-19T02:05:56.485 INFO:teuthology.orchestra.run.smithi112.stdout: "val": 0, 2024-03-19T02:05:56.485 INFO:teuthology.orchestra.run.smithi112.stdout: "max": 1024, 2024-03-19T02:05:56.485 INFO:teuthology.orchestra.run.smithi112.stdout: "get_started": 0, 2024-03-19T02:05:56.485 INFO:teuthology.orchestra.run.smithi112.stdout: "get": 0, 2024-03-19T02:05:56.485 INFO:teuthology.orchestra.run.smithi112.stdout: "get_sum": 0, 2024-03-19T02:05:56.485 INFO:teuthology.orchestra.run.smithi112.stdout: "get_or_fail_fail": 0, 2024-03-19T02:05:56.485 INFO:teuthology.orchestra.run.smithi112.stdout: "get_or_fail_success": 0, 2024-03-19T02:05:56.486 INFO:teuthology.orchestra.run.smithi112.stdout: "take": 6373, 2024-03-19T02:05:56.486 INFO:teuthology.orchestra.run.smithi112.stdout: "take_sum": 6373, 2024-03-19T02:05:56.486 INFO:teuthology.orchestra.run.smithi112.stdout: "put": 6373, 2024-03-19T02:05:56.486 INFO:teuthology.orchestra.run.smithi112.stdout: "put_sum": 6373, 2024-03-19T02:05:56.486 INFO:teuthology.orchestra.run.smithi112.stdout: "wait": { 2024-03-19T02:05:56.486 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:56.486 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:56.486 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:56.486 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-03-19T02:05:56.486 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.486 INFO:teuthology.orchestra.run.smithi112.stdout: "throttle-write_buf_throttle": { 2024-03-19T02:05:56.486 INFO:teuthology.orchestra.run.smithi112.stdout: "val": 0, 2024-03-19T02:05:56.486 INFO:teuthology.orchestra.run.smithi112.stdout: "max": 3758096384, 2024-03-19T02:05:56.487 INFO:teuthology.orchestra.run.smithi112.stdout: "get_started": 0, 2024-03-19T02:05:56.487 INFO:teuthology.orchestra.run.smithi112.stdout: "get": 0, 2024-03-19T02:05:56.487 INFO:teuthology.orchestra.run.smithi112.stdout: "get_sum": 0, 2024-03-19T02:05:56.487 INFO:teuthology.orchestra.run.smithi112.stdout: "get_or_fail_fail": 0, 2024-03-19T02:05:56.487 INFO:teuthology.orchestra.run.smithi112.stdout: "get_or_fail_success": 0, 2024-03-19T02:05:56.487 INFO:teuthology.orchestra.run.smithi112.stdout: "take": 0, 2024-03-19T02:05:56.487 INFO:teuthology.orchestra.run.smithi112.stdout: "take_sum": 0, 2024-03-19T02:05:56.487 INFO:teuthology.orchestra.run.smithi112.stdout: "put": 0, 2024-03-19T02:05:56.487 INFO:teuthology.orchestra.run.smithi112.stdout: "put_sum": 0, 2024-03-19T02:05:56.487 INFO:teuthology.orchestra.run.smithi112.stdout: "wait": { 2024-03-19T02:05:56.487 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:56.487 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:56.487 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:56.487 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-03-19T02:05:56.487 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:56.488 INFO:teuthology.orchestra.run.smithi112.stdout: "throttle-write_buf_throttle-0x5618beea46e0": { 2024-03-19T02:05:56.488 INFO:teuthology.orchestra.run.smithi112.stdout: "val": 0, 2024-03-19T02:05:56.488 INFO:teuthology.orchestra.run.smithi112.stdout: "max": 3758096384, 2024-03-19T02:05:56.488 INFO:teuthology.orchestra.run.smithi112.stdout: "get_started": 0, 2024-03-19T02:05:56.488 INFO:teuthology.orchestra.run.smithi112.stdout: "get": 0, 2024-03-19T02:05:56.488 INFO:teuthology.orchestra.run.smithi112.stdout: "get_sum": 0, 2024-03-19T02:05:56.488 INFO:teuthology.orchestra.run.smithi112.stdout: "get_or_fail_fail": 0, 2024-03-19T02:05:56.488 INFO:teuthology.orchestra.run.smithi112.stdout: "get_or_fail_success": 0, 2024-03-19T02:05:56.488 INFO:teuthology.orchestra.run.smithi112.stdout: "take": 0, 2024-03-19T02:05:56.488 INFO:teuthology.orchestra.run.smithi112.stdout: "take_sum": 0, 2024-03-19T02:05:56.488 INFO:teuthology.orchestra.run.smithi112.stdout: "put": 0, 2024-03-19T02:05:56.488 INFO:teuthology.orchestra.run.smithi112.stdout: "put_sum": 0, 2024-03-19T02:05:56.488 INFO:teuthology.orchestra.run.smithi112.stdout: "wait": { 2024-03-19T02:05:56.488 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:56.488 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:56.489 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:56.489 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-03-19T02:05:56.489 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-03-19T02:05:56.489 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-03-19T02:05:57.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:05:56 smithi112 ceph-mon[66819]: pgmap v1096: 97 pgs: 97 active+clean; 467 MiB data, 9.1 GiB used, 1.0 TiB / 1.0 TiB avail; 143 KiB/s rd, 3.1 MiB/s wr, 118 op/s 2024-03-19T02:05:57.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:05:56 smithi115 ceph-mon[67332]: pgmap v1096: 97 pgs: 97 active+clean; 467 MiB data, 9.1 GiB used, 1.0 TiB / 1.0 TiB avail; 143 KiB/s rd, 3.1 MiB/s wr, 118 op/s 2024-03-19T02:05:57.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:05:56 smithi099 ceph-mon[62951]: pgmap v1096: 97 pgs: 97 active+clean; 467 MiB data, 9.1 GiB used, 1.0 TiB / 1.0 TiB avail; 143 KiB/s rd, 3.1 MiB/s wr, 118 op/s 2024-03-19T02:05:57.342 INFO:tasks.check_counter:Daemon mds.e mds.exported=0 2024-03-19T02:05:57.343 INFO:tasks.check_counter:Daemon mds.e mds.imported=0 2024-03-19T02:05:57.343 INFO:tasks.check_counter:Daemon mds.e mds_cache.dir_update=0 2024-03-19T02:05:57.343 INFO:tasks.check_counter:Daemon mds.e mds_cache.dir_update_receipt=0 2024-03-19T02:05:57.343 DEBUG:tasks.check_counter:Getting stats from h 2024-03-19T02:05:57.343 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph daemon mds.h perf dump 2024-03-19T02:05:57.615 INFO:teuthology.orchestra.run.smithi112.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.b/config 2024-03-19T02:05:59.055 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:05:58 smithi112 ceph-mon[66819]: pgmap v1097: 97 pgs: 97 active+clean; 458 MiB data, 9.1 GiB used, 1.0 TiB / 1.0 TiB avail; 193 KiB/s rd, 3.5 MiB/s wr, 131 op/s 2024-03-19T02:05:59.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:05:58 smithi115 ceph-mon[67332]: pgmap v1097: 97 pgs: 97 active+clean; 458 MiB data, 9.1 GiB used, 1.0 TiB / 1.0 TiB avail; 193 KiB/s rd, 3.5 MiB/s wr, 131 op/s 2024-03-19T02:05:59.328 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-03-19T02:05:59.328 INFO:teuthology.orchestra.run.smithi112.stdout: "AsyncMessenger::Worker-0": { 2024-03-19T02:05:59.328 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_recv_messages": 4109, 2024-03-19T02:05:59.328 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_send_messages": 4111, 2024-03-19T02:05:59.328 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_recv_bytes": 101931743, 2024-03-19T02:05:59.329 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_send_bytes": 1306669, 2024-03-19T02:05:59.329 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_created_connections": 8, 2024-03-19T02:05:59.329 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_active_connections": 4, 2024-03-19T02:05:59.329 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_running_total_time": 0.286391428, 2024-03-19T02:05:59.329 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_running_send_time": 0.070328879, 2024-03-19T02:05:59.329 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_running_recv_time": 0.113265078, 2024-03-19T02:05:59.329 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_running_fast_dispatch_time": 0.084855836, 2024-03-19T02:05:59.329 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_send_messages_queue_lat": { 2024-03-19T02:05:59.329 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 4109, 2024-03-19T02:05:59.329 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.083477079, 2024-03-19T02:05:59.329 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000020315 2024-03-19T02:05:59.329 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.330 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_handle_ack_lat": { 2024-03-19T02:05:59.330 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:59.330 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:59.330 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:59.330 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.330 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_recv_encrypted_bytes": 0, 2024-03-19T02:05:59.330 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_send_encrypted_bytes": 0 2024-03-19T02:05:59.330 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.330 INFO:teuthology.orchestra.run.smithi112.stdout: "AsyncMessenger::Worker-1": { 2024-03-19T02:05:59.330 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_recv_messages": 1099, 2024-03-19T02:05:59.330 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_send_messages": 1067, 2024-03-19T02:05:59.330 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_recv_bytes": 80798860, 2024-03-19T02:05:59.330 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_send_bytes": 468964, 2024-03-19T02:05:59.330 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_created_connections": 6, 2024-03-19T02:05:59.331 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_active_connections": 5, 2024-03-19T02:05:59.331 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_running_total_time": 0.127429048, 2024-03-19T02:05:59.331 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_running_send_time": 0.029291972, 2024-03-19T02:05:59.331 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_running_recv_time": 0.075378488, 2024-03-19T02:05:59.331 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_running_fast_dispatch_time": 0.021337139, 2024-03-19T02:05:59.331 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_send_messages_queue_lat": { 2024-03-19T02:05:59.331 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 1066, 2024-03-19T02:05:59.331 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.025145662, 2024-03-19T02:05:59.331 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000023588 2024-03-19T02:05:59.331 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.331 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_handle_ack_lat": { 2024-03-19T02:05:59.331 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:59.331 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:59.331 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:59.332 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.332 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_recv_encrypted_bytes": 948016, 2024-03-19T02:05:59.332 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_send_encrypted_bytes": 271424 2024-03-19T02:05:59.332 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.332 INFO:teuthology.orchestra.run.smithi112.stdout: "AsyncMessenger::Worker-2": { 2024-03-19T02:05:59.332 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_recv_messages": 814, 2024-03-19T02:05:59.332 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_send_messages": 1142, 2024-03-19T02:05:59.332 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_recv_bytes": 100872390, 2024-03-19T02:05:59.332 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_send_bytes": 1012288, 2024-03-19T02:05:59.332 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_created_connections": 8, 2024-03-19T02:05:59.332 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_active_connections": 5, 2024-03-19T02:05:59.332 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_running_total_time": 0.118929742, 2024-03-19T02:05:59.332 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_running_send_time": 0.023634657, 2024-03-19T02:05:59.332 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_running_recv_time": 0.072143621, 2024-03-19T02:05:59.332 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_running_fast_dispatch_time": 0.017125978, 2024-03-19T02:05:59.333 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_send_messages_queue_lat": { 2024-03-19T02:05:59.333 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 1141, 2024-03-19T02:05:59.333 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.031065654, 2024-03-19T02:05:59.333 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000027226 2024-03-19T02:05:59.333 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.333 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_handle_ack_lat": { 2024-03-19T02:05:59.333 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:59.333 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:59.333 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:59.333 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.333 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_recv_encrypted_bytes": 3392, 2024-03-19T02:05:59.333 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_send_encrypted_bytes": 755344 2024-03-19T02:05:59.333 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.333 INFO:teuthology.orchestra.run.smithi112.stdout: "cct": { 2024-03-19T02:05:59.333 INFO:teuthology.orchestra.run.smithi112.stdout: "total_workers": 1, 2024-03-19T02:05:59.334 INFO:teuthology.orchestra.run.smithi112.stdout: "unhealthy_workers": 0 2024-03-19T02:05:59.334 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.334 INFO:teuthology.orchestra.run.smithi112.stdout: "finisher-MDSRank": { 2024-03-19T02:05:59.334 INFO:teuthology.orchestra.run.smithi112.stdout: "queue_len": 0, 2024-03-19T02:05:59.334 INFO:teuthology.orchestra.run.smithi112.stdout: "complete_latency": { 2024-03-19T02:05:59.334 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 9408, 2024-03-19T02:05:59.334 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.183057564, 2024-03-19T02:05:59.334 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000019457 2024-03-19T02:05:59.334 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-03-19T02:05:59.334 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.334 INFO:teuthology.orchestra.run.smithi112.stdout: "finisher-PurgeQueue": { 2024-03-19T02:05:59.334 INFO:teuthology.orchestra.run.smithi112.stdout: "queue_len": 0, 2024-03-19T02:05:59.334 INFO:teuthology.orchestra.run.smithi112.stdout: "complete_latency": { 2024-03-19T02:05:59.334 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:59.335 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:59.335 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:59.335 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-03-19T02:05:59.335 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.335 INFO:teuthology.orchestra.run.smithi112.stdout: "mds": { 2024-03-19T02:05:59.335 INFO:teuthology.orchestra.run.smithi112.stdout: "request": 0, 2024-03-19T02:05:59.335 INFO:teuthology.orchestra.run.smithi112.stdout: "reply": 0, 2024-03-19T02:05:59.335 INFO:teuthology.orchestra.run.smithi112.stdout: "reply_latency": { 2024-03-19T02:05:59.335 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:59.335 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:59.335 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:59.335 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.335 INFO:teuthology.orchestra.run.smithi112.stdout: "slow_reply": 0, 2024-03-19T02:05:59.335 INFO:teuthology.orchestra.run.smithi112.stdout: "forward": 0, 2024-03-19T02:05:59.336 INFO:teuthology.orchestra.run.smithi112.stdout: "dir_fetch_complete": 0, 2024-03-19T02:05:59.336 INFO:teuthology.orchestra.run.smithi112.stdout: "dir_fetch_keys": 0, 2024-03-19T02:05:59.336 INFO:teuthology.orchestra.run.smithi112.stdout: "dir_commit": 0, 2024-03-19T02:05:59.336 INFO:teuthology.orchestra.run.smithi112.stdout: "dir_split": 0, 2024-03-19T02:05:59.336 INFO:teuthology.orchestra.run.smithi112.stdout: "dir_merge": 0, 2024-03-19T02:05:59.336 INFO:teuthology.orchestra.run.smithi112.stdout: "inodes": 29547, 2024-03-19T02:05:59.336 INFO:teuthology.orchestra.run.smithi112.stdout: "inodes_top": 3406, 2024-03-19T02:05:59.336 INFO:teuthology.orchestra.run.smithi112.stdout: "inodes_bottom": 26141, 2024-03-19T02:05:59.336 INFO:teuthology.orchestra.run.smithi112.stdout: "inodes_pin_tail": 0, 2024-03-19T02:05:59.336 INFO:teuthology.orchestra.run.smithi112.stdout: "inodes_pinned": 24681, 2024-03-19T02:05:59.336 INFO:teuthology.orchestra.run.smithi112.stdout: "inodes_expired": 0, 2024-03-19T02:05:59.336 INFO:teuthology.orchestra.run.smithi112.stdout: "inodes_with_caps": 0, 2024-03-19T02:05:59.336 INFO:teuthology.orchestra.run.smithi112.stdout: "caps": 0, 2024-03-19T02:05:59.336 INFO:teuthology.orchestra.run.smithi112.stdout: "subtrees": 1, 2024-03-19T02:05:59.337 INFO:teuthology.orchestra.run.smithi112.stdout: "traverse": 0, 2024-03-19T02:05:59.337 INFO:teuthology.orchestra.run.smithi112.stdout: "traverse_hit": 0, 2024-03-19T02:05:59.337 INFO:teuthology.orchestra.run.smithi112.stdout: "traverse_forward": 0, 2024-03-19T02:05:59.337 INFO:teuthology.orchestra.run.smithi112.stdout: "traverse_discover": 0, 2024-03-19T02:05:59.337 INFO:teuthology.orchestra.run.smithi112.stdout: "traverse_dir_fetch": 0, 2024-03-19T02:05:59.337 INFO:teuthology.orchestra.run.smithi112.stdout: "traverse_remote_ino": 0, 2024-03-19T02:05:59.337 INFO:teuthology.orchestra.run.smithi112.stdout: "traverse_lock": 0, 2024-03-19T02:05:59.337 INFO:teuthology.orchestra.run.smithi112.stdout: "load_cent": 0, 2024-03-19T02:05:59.337 INFO:teuthology.orchestra.run.smithi112.stdout: "q": 0, 2024-03-19T02:05:59.337 INFO:teuthology.orchestra.run.smithi112.stdout: "exported": 0, 2024-03-19T02:05:59.337 INFO:teuthology.orchestra.run.smithi112.stdout: "exported_inodes": 0, 2024-03-19T02:05:59.337 INFO:teuthology.orchestra.run.smithi112.stdout: "imported": 0, 2024-03-19T02:05:59.337 INFO:teuthology.orchestra.run.smithi112.stdout: "imported_inodes": 0, 2024-03-19T02:05:59.337 INFO:teuthology.orchestra.run.smithi112.stdout: "openino_dir_fetch": 0, 2024-03-19T02:05:59.337 INFO:teuthology.orchestra.run.smithi112.stdout: "openino_backtrace_fetch": 0, 2024-03-19T02:05:59.338 INFO:teuthology.orchestra.run.smithi112.stdout: "openino_peer_discover": 0, 2024-03-19T02:05:59.338 INFO:teuthology.orchestra.run.smithi112.stdout: "root_rfiles": 0, 2024-03-19T02:05:59.338 INFO:teuthology.orchestra.run.smithi112.stdout: "root_rbytes": 0, 2024-03-19T02:05:59.338 INFO:teuthology.orchestra.run.smithi112.stdout: "root_rsnaps": 0, 2024-03-19T02:05:59.338 INFO:teuthology.orchestra.run.smithi112.stdout: "scrub_backtrace_fetch": 0, 2024-03-19T02:05:59.338 INFO:teuthology.orchestra.run.smithi112.stdout: "scrub_set_tag": 0, 2024-03-19T02:05:59.338 INFO:teuthology.orchestra.run.smithi112.stdout: "scrub_backtrace_repaired": 0, 2024-03-19T02:05:59.338 INFO:teuthology.orchestra.run.smithi112.stdout: "scrub_inotable_repaired": 0, 2024-03-19T02:05:59.338 INFO:teuthology.orchestra.run.smithi112.stdout: "scrub_dir_inodes": 0, 2024-03-19T02:05:59.338 INFO:teuthology.orchestra.run.smithi112.stdout: "scrub_dir_base_inodes": 0, 2024-03-19T02:05:59.339 INFO:teuthology.orchestra.run.smithi112.stdout: "scrub_dirfrag_rstats": 0, 2024-03-19T02:05:59.339 INFO:teuthology.orchestra.run.smithi112.stdout: "scrub_file_inodes": 0, 2024-03-19T02:05:59.339 INFO:teuthology.orchestra.run.smithi112.stdout: "handle_inode_file_caps": 0, 2024-03-19T02:05:59.339 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph_cap_op_revoke": 0, 2024-03-19T02:05:59.339 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph_cap_op_grant": 0, 2024-03-19T02:05:59.339 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph_cap_op_trunc": 0, 2024-03-19T02:05:59.339 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph_cap_op_flushsnap_ack": 0, 2024-03-19T02:05:59.339 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph_cap_op_flush_ack": 0, 2024-03-19T02:05:59.339 INFO:teuthology.orchestra.run.smithi112.stdout: "handle_client_caps": 0, 2024-03-19T02:05:59.339 INFO:teuthology.orchestra.run.smithi112.stdout: "handle_client_caps_dirty": 0, 2024-03-19T02:05:59.339 INFO:teuthology.orchestra.run.smithi112.stdout: "handle_client_cap_release": 0, 2024-03-19T02:05:59.339 INFO:teuthology.orchestra.run.smithi112.stdout: "process_request_cap_release": 0 2024-03-19T02:05:59.339 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.339 INFO:teuthology.orchestra.run.smithi112.stdout: "mds_cache": { 2024-03-19T02:05:59.339 INFO:teuthology.orchestra.run.smithi112.stdout: "dir_update": 0, 2024-03-19T02:05:59.340 INFO:teuthology.orchestra.run.smithi112.stdout: "dir_update_receipt": 0, 2024-03-19T02:05:59.340 INFO:teuthology.orchestra.run.smithi112.stdout: "dir_try_discover": 0, 2024-03-19T02:05:59.340 INFO:teuthology.orchestra.run.smithi112.stdout: "dir_send_discover": 0, 2024-03-19T02:05:59.340 INFO:teuthology.orchestra.run.smithi112.stdout: "dir_handle_discover": 0, 2024-03-19T02:05:59.340 INFO:teuthology.orchestra.run.smithi112.stdout: "num_strays": 0, 2024-03-19T02:05:59.340 INFO:teuthology.orchestra.run.smithi112.stdout: "num_strays_delayed": 0, 2024-03-19T02:05:59.340 INFO:teuthology.orchestra.run.smithi112.stdout: "num_strays_enqueuing": 0, 2024-03-19T02:05:59.340 INFO:teuthology.orchestra.run.smithi112.stdout: "strays_created": 0, 2024-03-19T02:05:59.340 INFO:teuthology.orchestra.run.smithi112.stdout: "strays_enqueued": 0, 2024-03-19T02:05:59.340 INFO:teuthology.orchestra.run.smithi112.stdout: "strays_reintegrated": 0, 2024-03-19T02:05:59.340 INFO:teuthology.orchestra.run.smithi112.stdout: "strays_migrated": 0, 2024-03-19T02:05:59.340 INFO:teuthology.orchestra.run.smithi112.stdout: "num_recovering_processing": 0, 2024-03-19T02:05:59.340 INFO:teuthology.orchestra.run.smithi112.stdout: "num_recovering_enqueued": 0, 2024-03-19T02:05:59.340 INFO:teuthology.orchestra.run.smithi112.stdout: "num_recovering_prioritized": 0, 2024-03-19T02:05:59.341 INFO:teuthology.orchestra.run.smithi112.stdout: "recovery_started": 0, 2024-03-19T02:05:59.341 INFO:teuthology.orchestra.run.smithi112.stdout: "recovery_completed": 0, 2024-03-19T02:05:59.341 INFO:teuthology.orchestra.run.smithi112.stdout: "ireq_quiesce_path": 0, 2024-03-19T02:05:59.341 INFO:teuthology.orchestra.run.smithi112.stdout: "ireq_quiesce_inode": 0, 2024-03-19T02:05:59.341 INFO:teuthology.orchestra.run.smithi112.stdout: "ireq_enqueue_scrub": 0, 2024-03-19T02:05:59.341 INFO:teuthology.orchestra.run.smithi112.stdout: "ireq_exportdir": 0, 2024-03-19T02:05:59.341 INFO:teuthology.orchestra.run.smithi112.stdout: "ireq_flush": 0, 2024-03-19T02:05:59.341 INFO:teuthology.orchestra.run.smithi112.stdout: "ireq_fragmentdir": 0, 2024-03-19T02:05:59.341 INFO:teuthology.orchestra.run.smithi112.stdout: "ireq_fragstats": 0, 2024-03-19T02:05:59.341 INFO:teuthology.orchestra.run.smithi112.stdout: "ireq_inodestats": 0 2024-03-19T02:05:59.341 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.341 INFO:teuthology.orchestra.run.smithi112.stdout: "mds_log": { 2024-03-19T02:05:59.341 INFO:teuthology.orchestra.run.smithi112.stdout: "evlrg": 0, 2024-03-19T02:05:59.341 INFO:teuthology.orchestra.run.smithi112.stdout: "evadd": 0, 2024-03-19T02:05:59.342 INFO:teuthology.orchestra.run.smithi112.stdout: "evex": 0, 2024-03-19T02:05:59.342 INFO:teuthology.orchestra.run.smithi112.stdout: "evtrm": 0, 2024-03-19T02:05:59.342 INFO:teuthology.orchestra.run.smithi112.stdout: "ev": 135721, 2024-03-19T02:05:59.342 INFO:teuthology.orchestra.run.smithi112.stdout: "evexg": 0, 2024-03-19T02:05:59.342 INFO:teuthology.orchestra.run.smithi112.stdout: "evexd": 0, 2024-03-19T02:05:59.342 INFO:teuthology.orchestra.run.smithi112.stdout: "segadd": 0, 2024-03-19T02:05:59.342 INFO:teuthology.orchestra.run.smithi112.stdout: "segex": 0, 2024-03-19T02:05:59.342 INFO:teuthology.orchestra.run.smithi112.stdout: "segtrm": 0, 2024-03-19T02:05:59.342 INFO:teuthology.orchestra.run.smithi112.stdout: "seg": 129, 2024-03-19T02:05:59.342 INFO:teuthology.orchestra.run.smithi112.stdout: "segmjr": 13, 2024-03-19T02:05:59.342 INFO:teuthology.orchestra.run.smithi112.stdout: "segexg": 0, 2024-03-19T02:05:59.342 INFO:teuthology.orchestra.run.smithi112.stdout: "segexd": 0, 2024-03-19T02:05:59.342 INFO:teuthology.orchestra.run.smithi112.stdout: "expos": 56004966, 2024-03-19T02:05:59.343 INFO:teuthology.orchestra.run.smithi112.stdout: "wrpos": 285296617, 2024-03-19T02:05:59.343 INFO:teuthology.orchestra.run.smithi112.stdout: "rdpos": 285296018, 2024-03-19T02:05:59.343 INFO:teuthology.orchestra.run.smithi112.stdout: "jlat": { 2024-03-19T02:05:59.343 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:59.343 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:59.343 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:59.343 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.343 INFO:teuthology.orchestra.run.smithi112.stdout: "replayed": 135721 2024-03-19T02:05:59.343 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.343 INFO:teuthology.orchestra.run.smithi112.stdout: "mds_mem": { 2024-03-19T02:05:59.343 INFO:teuthology.orchestra.run.smithi112.stdout: "ino": 182, 2024-03-19T02:05:59.343 INFO:teuthology.orchestra.run.smithi112.stdout: "ino+": 29523, 2024-03-19T02:05:59.343 INFO:teuthology.orchestra.run.smithi112.stdout: "ino-": 29341, 2024-03-19T02:05:59.343 INFO:teuthology.orchestra.run.smithi112.stdout: "dir": 19, 2024-03-19T02:05:59.343 INFO:teuthology.orchestra.run.smithi112.stdout: "dir+": 44, 2024-03-19T02:05:59.344 INFO:teuthology.orchestra.run.smithi112.stdout: "dir-": 25, 2024-03-19T02:05:59.344 INFO:teuthology.orchestra.run.smithi112.stdout: "dn": 29547, 2024-03-19T02:05:59.344 INFO:teuthology.orchestra.run.smithi112.stdout: "dn+": 29801, 2024-03-19T02:05:59.344 INFO:teuthology.orchestra.run.smithi112.stdout: "dn-": 254, 2024-03-19T02:05:59.344 INFO:teuthology.orchestra.run.smithi112.stdout: "cap": 0, 2024-03-19T02:05:59.344 INFO:teuthology.orchestra.run.smithi112.stdout: "cap+": 0, 2024-03-19T02:05:59.344 INFO:teuthology.orchestra.run.smithi112.stdout: "cap-": 0, 2024-03-19T02:05:59.344 INFO:teuthology.orchestra.run.smithi112.stdout: "rss": 77656, 2024-03-19T02:05:59.344 INFO:teuthology.orchestra.run.smithi112.stdout: "heap": 215420 2024-03-19T02:05:59.344 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.344 INFO:teuthology.orchestra.run.smithi112.stdout: "mds_server": { 2024-03-19T02:05:59.344 INFO:teuthology.orchestra.run.smithi112.stdout: "dispatch_client_request": 0, 2024-03-19T02:05:59.344 INFO:teuthology.orchestra.run.smithi112.stdout: "dispatch_server_request": 0, 2024-03-19T02:05:59.344 INFO:teuthology.orchestra.run.smithi112.stdout: "handle_client_request": 0, 2024-03-19T02:05:59.345 INFO:teuthology.orchestra.run.smithi112.stdout: "handle_client_session": 0, 2024-03-19T02:05:59.345 INFO:teuthology.orchestra.run.smithi112.stdout: "handle_peer_request": 0, 2024-03-19T02:05:59.345 INFO:teuthology.orchestra.run.smithi112.stdout: "req_create_latency": { 2024-03-19T02:05:59.345 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:59.345 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:59.345 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:59.345 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.345 INFO:teuthology.orchestra.run.smithi112.stdout: "req_getattr_latency": { 2024-03-19T02:05:59.345 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:59.345 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:59.345 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:59.345 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.345 INFO:teuthology.orchestra.run.smithi112.stdout: "req_getfilelock_latency": { 2024-03-19T02:05:59.345 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:59.346 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:59.346 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:59.346 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.346 INFO:teuthology.orchestra.run.smithi112.stdout: "req_link_latency": { 2024-03-19T02:05:59.346 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:59.346 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:59.346 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:59.346 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.346 INFO:teuthology.orchestra.run.smithi112.stdout: "req_lookup_latency": { 2024-03-19T02:05:59.346 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:59.346 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:59.346 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:59.346 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.346 INFO:teuthology.orchestra.run.smithi112.stdout: "req_lookuphash_latency": { 2024-03-19T02:05:59.347 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:59.347 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:59.347 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:59.347 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.347 INFO:teuthology.orchestra.run.smithi112.stdout: "req_lookupino_latency": { 2024-03-19T02:05:59.347 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:59.347 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:59.347 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:59.347 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.347 INFO:teuthology.orchestra.run.smithi112.stdout: "req_lookupname_latency": { 2024-03-19T02:05:59.347 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:59.347 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:59.347 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:59.347 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.348 INFO:teuthology.orchestra.run.smithi112.stdout: "req_lookupparent_latency": { 2024-03-19T02:05:59.348 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:59.348 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:59.348 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:59.348 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.348 INFO:teuthology.orchestra.run.smithi112.stdout: "req_lookupsnap_latency": { 2024-03-19T02:05:59.348 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:59.348 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:59.348 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:59.348 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.348 INFO:teuthology.orchestra.run.smithi112.stdout: "req_lssnap_latency": { 2024-03-19T02:05:59.348 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:59.348 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:59.348 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:59.349 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.349 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:05:58 smithi099 ceph-mon[62951]: pgmap v1097: 97 pgs: 97 active+clean; 458 MiB data, 9.1 GiB used, 1.0 TiB / 1.0 TiB avail; 193 KiB/s rd, 3.5 MiB/s wr, 131 op/s 2024-03-19T02:05:59.349 INFO:teuthology.orchestra.run.smithi112.stdout: "req_mkdir_latency": { 2024-03-19T02:05:59.350 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:59.350 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:59.350 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:59.350 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.350 INFO:teuthology.orchestra.run.smithi112.stdout: "req_mknod_latency": { 2024-03-19T02:05:59.350 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:59.350 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:59.350 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:59.350 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.350 INFO:teuthology.orchestra.run.smithi112.stdout: "req_mksnap_latency": { 2024-03-19T02:05:59.350 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:59.350 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:59.350 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:59.351 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.351 INFO:teuthology.orchestra.run.smithi112.stdout: "req_open_latency": { 2024-03-19T02:05:59.351 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:59.351 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:59.351 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:59.351 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.351 INFO:teuthology.orchestra.run.smithi112.stdout: "req_readdir_latency": { 2024-03-19T02:05:59.351 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:59.351 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:59.351 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:59.351 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.351 INFO:teuthology.orchestra.run.smithi112.stdout: "req_rename_latency": { 2024-03-19T02:05:59.351 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:59.351 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:59.352 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:59.352 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.352 INFO:teuthology.orchestra.run.smithi112.stdout: "req_renamesnap_latency": { 2024-03-19T02:05:59.352 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:59.352 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:59.352 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:59.352 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.352 INFO:teuthology.orchestra.run.smithi112.stdout: "req_snapdiff_latency": { 2024-03-19T02:05:59.352 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:59.352 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:59.352 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:59.352 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.352 INFO:teuthology.orchestra.run.smithi112.stdout: "req_rmdir_latency": { 2024-03-19T02:05:59.352 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:59.352 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:59.353 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:59.353 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.353 INFO:teuthology.orchestra.run.smithi112.stdout: "req_rmsnap_latency": { 2024-03-19T02:05:59.353 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:59.353 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:59.353 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:59.353 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.353 INFO:teuthology.orchestra.run.smithi112.stdout: "req_rmxattr_latency": { 2024-03-19T02:05:59.353 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:59.353 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:59.353 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:59.353 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.353 INFO:teuthology.orchestra.run.smithi112.stdout: "req_setattr_latency": { 2024-03-19T02:05:59.353 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:59.354 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:59.354 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:59.354 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.354 INFO:teuthology.orchestra.run.smithi112.stdout: "req_setdirlayout_latency": { 2024-03-19T02:05:59.354 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:59.354 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:59.354 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:59.354 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.354 INFO:teuthology.orchestra.run.smithi112.stdout: "req_setfilelock_latency": { 2024-03-19T02:05:59.354 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:59.354 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:59.354 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:59.354 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.354 INFO:teuthology.orchestra.run.smithi112.stdout: "req_setlayout_latency": { 2024-03-19T02:05:59.355 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:59.355 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:59.355 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:59.355 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.355 INFO:teuthology.orchestra.run.smithi112.stdout: "req_setxattr_latency": { 2024-03-19T02:05:59.355 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:59.355 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:59.355 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:59.355 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.355 INFO:teuthology.orchestra.run.smithi112.stdout: "req_symlink_latency": { 2024-03-19T02:05:59.355 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:59.355 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:59.355 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:59.355 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.355 INFO:teuthology.orchestra.run.smithi112.stdout: "req_unlink_latency": { 2024-03-19T02:05:59.356 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:59.356 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:59.356 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:59.356 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.356 INFO:teuthology.orchestra.run.smithi112.stdout: "cap_revoke_eviction": 0, 2024-03-19T02:05:59.356 INFO:teuthology.orchestra.run.smithi112.stdout: "cap_acquisition_throttle": 0, 2024-03-19T02:05:59.356 INFO:teuthology.orchestra.run.smithi112.stdout: "req_getvxattr_latency": { 2024-03-19T02:05:59.356 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:59.356 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:59.356 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:59.356 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-03-19T02:05:59.356 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.356 INFO:teuthology.orchestra.run.smithi112.stdout: "mds_sessions": { 2024-03-19T02:05:59.356 INFO:teuthology.orchestra.run.smithi112.stdout: "session_count": 1, 2024-03-19T02:05:59.357 INFO:teuthology.orchestra.run.smithi112.stdout: "session_add": 3, 2024-03-19T02:05:59.357 INFO:teuthology.orchestra.run.smithi112.stdout: "session_remove": 2, 2024-03-19T02:05:59.357 INFO:teuthology.orchestra.run.smithi112.stdout: "sessions_open": 2, 2024-03-19T02:05:59.357 INFO:teuthology.orchestra.run.smithi112.stdout: "sessions_stale": 0, 2024-03-19T02:05:59.357 INFO:teuthology.orchestra.run.smithi112.stdout: "total_load": 0, 2024-03-19T02:05:59.357 INFO:teuthology.orchestra.run.smithi112.stdout: "average_load": 0, 2024-03-19T02:05:59.357 INFO:teuthology.orchestra.run.smithi112.stdout: "avg_session_uptime": 11077, 2024-03-19T02:05:59.357 INFO:teuthology.orchestra.run.smithi112.stdout: "mdthresh_evicted": 0 2024-03-19T02:05:59.357 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.357 INFO:teuthology.orchestra.run.smithi112.stdout: "mempool": { 2024-03-19T02:05:59.357 INFO:teuthology.orchestra.run.smithi112.stdout: "bloom_filter_bytes": 0, 2024-03-19T02:05:59.357 INFO:teuthology.orchestra.run.smithi112.stdout: "bloom_filter_items": 0, 2024-03-19T02:05:59.357 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_alloc_bytes": 0, 2024-03-19T02:05:59.357 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_alloc_items": 0, 2024-03-19T02:05:59.357 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_cache_data_bytes": 0, 2024-03-19T02:05:59.358 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_cache_data_items": 0, 2024-03-19T02:05:59.358 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_cache_onode_bytes": 0, 2024-03-19T02:05:59.358 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_cache_onode_items": 0, 2024-03-19T02:05:59.358 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_cache_meta_bytes": 0, 2024-03-19T02:05:59.358 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_cache_meta_items": 0, 2024-03-19T02:05:59.358 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_cache_other_bytes": 0, 2024-03-19T02:05:59.358 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_cache_other_items": 0, 2024-03-19T02:05:59.358 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_cache_buffer_bytes": 0, 2024-03-19T02:05:59.358 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_cache_buffer_items": 0, 2024-03-19T02:05:59.358 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_extent_bytes": 0, 2024-03-19T02:05:59.358 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_extent_items": 0, 2024-03-19T02:05:59.358 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_blob_bytes": 0, 2024-03-19T02:05:59.358 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_blob_items": 0, 2024-03-19T02:05:59.359 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_shared_blob_bytes": 0, 2024-03-19T02:05:59.359 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_shared_blob_items": 0, 2024-03-19T02:05:59.359 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_inline_bl_bytes": 0, 2024-03-19T02:05:59.359 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_inline_bl_items": 0, 2024-03-19T02:05:59.359 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_fsck_bytes": 0, 2024-03-19T02:05:59.359 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_fsck_items": 0, 2024-03-19T02:05:59.359 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_txc_bytes": 0, 2024-03-19T02:05:59.359 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_txc_items": 0, 2024-03-19T02:05:59.359 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_writing_deferred_bytes": 0, 2024-03-19T02:05:59.359 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_writing_deferred_items": 0, 2024-03-19T02:05:59.359 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_writing_bytes": 0, 2024-03-19T02:05:59.359 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_writing_items": 0, 2024-03-19T02:05:59.359 INFO:teuthology.orchestra.run.smithi112.stdout: "bluefs_bytes": 0, 2024-03-19T02:05:59.359 INFO:teuthology.orchestra.run.smithi112.stdout: "bluefs_items": 0, 2024-03-19T02:05:59.359 INFO:teuthology.orchestra.run.smithi112.stdout: "bluefs_file_reader_bytes": 0, 2024-03-19T02:05:59.360 INFO:teuthology.orchestra.run.smithi112.stdout: "bluefs_file_reader_items": 0, 2024-03-19T02:05:59.360 INFO:teuthology.orchestra.run.smithi112.stdout: "bluefs_file_writer_bytes": 0, 2024-03-19T02:05:59.360 INFO:teuthology.orchestra.run.smithi112.stdout: "bluefs_file_writer_items": 0, 2024-03-19T02:05:59.360 INFO:teuthology.orchestra.run.smithi112.stdout: "buffer_anon_bytes": 245724, 2024-03-19T02:05:59.360 INFO:teuthology.orchestra.run.smithi112.stdout: "buffer_anon_items": 248, 2024-03-19T02:05:59.360 INFO:teuthology.orchestra.run.smithi112.stdout: "buffer_meta_bytes": 0, 2024-03-19T02:05:59.360 INFO:teuthology.orchestra.run.smithi112.stdout: "buffer_meta_items": 0, 2024-03-19T02:05:59.360 INFO:teuthology.orchestra.run.smithi112.stdout: "osd_bytes": 0, 2024-03-19T02:05:59.360 INFO:teuthology.orchestra.run.smithi112.stdout: "osd_items": 0, 2024-03-19T02:05:59.360 INFO:teuthology.orchestra.run.smithi112.stdout: "osd_mapbl_bytes": 0, 2024-03-19T02:05:59.360 INFO:teuthology.orchestra.run.smithi112.stdout: "osd_mapbl_items": 0, 2024-03-19T02:05:59.360 INFO:teuthology.orchestra.run.smithi112.stdout: "osd_pglog_bytes": 0, 2024-03-19T02:05:59.360 INFO:teuthology.orchestra.run.smithi112.stdout: "osd_pglog_items": 0, 2024-03-19T02:05:59.361 INFO:teuthology.orchestra.run.smithi112.stdout: "osdmap_bytes": 7816, 2024-03-19T02:05:59.361 INFO:teuthology.orchestra.run.smithi112.stdout: "osdmap_items": 164, 2024-03-19T02:05:59.361 INFO:teuthology.orchestra.run.smithi112.stdout: "osdmap_mapping_bytes": 0, 2024-03-19T02:05:59.361 INFO:teuthology.orchestra.run.smithi112.stdout: "osdmap_mapping_items": 0, 2024-03-19T02:05:59.361 INFO:teuthology.orchestra.run.smithi112.stdout: "pgmap_bytes": 0, 2024-03-19T02:05:59.361 INFO:teuthology.orchestra.run.smithi112.stdout: "pgmap_items": 0, 2024-03-19T02:05:59.361 INFO:teuthology.orchestra.run.smithi112.stdout: "mds_co_bytes": 21265255, 2024-03-19T02:05:59.361 INFO:teuthology.orchestra.run.smithi112.stdout: "mds_co_items": 140520, 2024-03-19T02:05:59.361 INFO:teuthology.orchestra.run.smithi112.stdout: "unittest_1_bytes": 0, 2024-03-19T02:05:59.361 INFO:teuthology.orchestra.run.smithi112.stdout: "unittest_1_items": 0, 2024-03-19T02:05:59.361 INFO:teuthology.orchestra.run.smithi112.stdout: "unittest_2_bytes": 0, 2024-03-19T02:05:59.361 INFO:teuthology.orchestra.run.smithi112.stdout: "unittest_2_items": 0 2024-03-19T02:05:59.361 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.362 INFO:teuthology.orchestra.run.smithi112.stdout: "objecter": { 2024-03-19T02:05:59.362 INFO:teuthology.orchestra.run.smithi112.stdout: "op_active": 0, 2024-03-19T02:05:59.362 INFO:teuthology.orchestra.run.smithi112.stdout: "op_laggy": 0, 2024-03-19T02:05:59.362 INFO:teuthology.orchestra.run.smithi112.stdout: "op_send": 5538, 2024-03-19T02:05:59.362 INFO:teuthology.orchestra.run.smithi112.stdout: "op_send_bytes": 21, 2024-03-19T02:05:59.362 INFO:teuthology.orchestra.run.smithi112.stdout: "op_resend": 3, 2024-03-19T02:05:59.362 INFO:teuthology.orchestra.run.smithi112.stdout: "op_reply": 5538, 2024-03-19T02:05:59.362 INFO:teuthology.orchestra.run.smithi112.stdout: "op_latency": { 2024-03-19T02:05:59.362 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 5538, 2024-03-19T02:05:59.362 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 8.823777584, 2024-03-19T02:05:59.362 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.001593314 2024-03-19T02:05:59.362 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.362 INFO:teuthology.orchestra.run.smithi112.stdout: "op_inflight": 0, 2024-03-19T02:05:59.362 INFO:teuthology.orchestra.run.smithi112.stdout: "oplen_avg": { 2024-03-19T02:05:59.363 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 5538, 2024-03-19T02:05:59.363 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 5539 2024-03-19T02:05:59.363 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.363 INFO:teuthology.orchestra.run.smithi112.stdout: "op": 5538, 2024-03-19T02:05:59.363 INFO:teuthology.orchestra.run.smithi112.stdout: "op_r": 5538, 2024-03-19T02:05:59.363 INFO:teuthology.orchestra.run.smithi112.stdout: "op_w": 0, 2024-03-19T02:05:59.363 INFO:teuthology.orchestra.run.smithi112.stdout: "op_rmw": 0, 2024-03-19T02:05:59.363 INFO:teuthology.orchestra.run.smithi112.stdout: "op_pg": 0, 2024-03-19T02:05:59.363 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_stat": 3155, 2024-03-19T02:05:59.363 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_create": 0, 2024-03-19T02:05:59.363 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_read": 2381, 2024-03-19T02:05:59.363 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_write": 0, 2024-03-19T02:05:59.363 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_writefull": 0, 2024-03-19T02:05:59.363 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_writesame": 0, 2024-03-19T02:05:59.364 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_append": 0, 2024-03-19T02:05:59.364 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_zero": 0, 2024-03-19T02:05:59.364 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_truncate": 0, 2024-03-19T02:05:59.364 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_delete": 0, 2024-03-19T02:05:59.364 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_mapext": 0, 2024-03-19T02:05:59.364 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_sparse_read": 0, 2024-03-19T02:05:59.364 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_clonerange": 0, 2024-03-19T02:05:59.364 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_getxattr": 1, 2024-03-19T02:05:59.364 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_setxattr": 0, 2024-03-19T02:05:59.364 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_cmpxattr": 0, 2024-03-19T02:05:59.364 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_rmxattr": 0, 2024-03-19T02:05:59.364 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_resetxattrs": 0, 2024-03-19T02:05:59.364 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_call": 0, 2024-03-19T02:05:59.364 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_watch": 0, 2024-03-19T02:05:59.365 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_notify": 0, 2024-03-19T02:05:59.365 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_src_cmpxattr": 0, 2024-03-19T02:05:59.365 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_pgls": 0, 2024-03-19T02:05:59.365 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_pgls_filter": 0, 2024-03-19T02:05:59.365 INFO:teuthology.orchestra.run.smithi112.stdout: "osdop_other": 0, 2024-03-19T02:05:59.365 INFO:teuthology.orchestra.run.smithi112.stdout: "linger_active": 0, 2024-03-19T02:05:59.365 INFO:teuthology.orchestra.run.smithi112.stdout: "linger_send": 0, 2024-03-19T02:05:59.365 INFO:teuthology.orchestra.run.smithi112.stdout: "linger_resend": 0, 2024-03-19T02:05:59.365 INFO:teuthology.orchestra.run.smithi112.stdout: "linger_ping": 0, 2024-03-19T02:05:59.365 INFO:teuthology.orchestra.run.smithi112.stdout: "poolop_active": 0, 2024-03-19T02:05:59.365 INFO:teuthology.orchestra.run.smithi112.stdout: "poolop_send": 0, 2024-03-19T02:05:59.365 INFO:teuthology.orchestra.run.smithi112.stdout: "poolop_resend": 0, 2024-03-19T02:05:59.365 INFO:teuthology.orchestra.run.smithi112.stdout: "poolstat_active": 0, 2024-03-19T02:05:59.365 INFO:teuthology.orchestra.run.smithi112.stdout: "poolstat_send": 0, 2024-03-19T02:05:59.366 INFO:teuthology.orchestra.run.smithi112.stdout: "poolstat_resend": 0, 2024-03-19T02:05:59.366 INFO:teuthology.orchestra.run.smithi112.stdout: "statfs_active": 0, 2024-03-19T02:05:59.366 INFO:teuthology.orchestra.run.smithi112.stdout: "statfs_send": 0, 2024-03-19T02:05:59.366 INFO:teuthology.orchestra.run.smithi112.stdout: "statfs_resend": 0, 2024-03-19T02:05:59.366 INFO:teuthology.orchestra.run.smithi112.stdout: "command_active": 0, 2024-03-19T02:05:59.366 INFO:teuthology.orchestra.run.smithi112.stdout: "command_send": 0, 2024-03-19T02:05:59.366 INFO:teuthology.orchestra.run.smithi112.stdout: "command_resend": 0, 2024-03-19T02:05:59.366 INFO:teuthology.orchestra.run.smithi112.stdout: "map_epoch": 92, 2024-03-19T02:05:59.366 INFO:teuthology.orchestra.run.smithi112.stdout: "map_full": 0, 2024-03-19T02:05:59.366 INFO:teuthology.orchestra.run.smithi112.stdout: "map_inc": 5, 2024-03-19T02:05:59.366 INFO:teuthology.orchestra.run.smithi112.stdout: "osd_sessions": 11, 2024-03-19T02:05:59.366 INFO:teuthology.orchestra.run.smithi112.stdout: "osd_session_open": 12, 2024-03-19T02:05:59.366 INFO:teuthology.orchestra.run.smithi112.stdout: "osd_session_close": 1, 2024-03-19T02:05:59.366 INFO:teuthology.orchestra.run.smithi112.stdout: "osd_laggy": 0, 2024-03-19T02:05:59.367 INFO:teuthology.orchestra.run.smithi112.stdout: "omap_wr": 0, 2024-03-19T02:05:59.367 INFO:teuthology.orchestra.run.smithi112.stdout: "omap_rd": 2, 2024-03-19T02:05:59.367 INFO:teuthology.orchestra.run.smithi112.stdout: "omap_del": 0 2024-03-19T02:05:59.367 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.367 INFO:teuthology.orchestra.run.smithi112.stdout: "oft": { 2024-03-19T02:05:59.367 INFO:teuthology.orchestra.run.smithi112.stdout: "omap_total_objs": 0, 2024-03-19T02:05:59.367 INFO:teuthology.orchestra.run.smithi112.stdout: "omap_total_kv_pairs": 0, 2024-03-19T02:05:59.367 INFO:teuthology.orchestra.run.smithi112.stdout: "omap_total_updates": 0, 2024-03-19T02:05:59.367 INFO:teuthology.orchestra.run.smithi112.stdout: "omap_total_removes": 0 2024-03-19T02:05:59.367 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.367 INFO:teuthology.orchestra.run.smithi112.stdout: "osd-slow-ops": { 2024-03-19T02:05:59.367 INFO:teuthology.orchestra.run.smithi112.stdout: "slow_ops_count": 0 2024-03-19T02:05:59.367 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.367 INFO:teuthology.orchestra.run.smithi112.stdout: "purge_queue": { 2024-03-19T02:05:59.368 INFO:teuthology.orchestra.run.smithi112.stdout: "pq_executing_ops": 0, 2024-03-19T02:05:59.368 INFO:teuthology.orchestra.run.smithi112.stdout: "pq_executing_ops_high_water": 0, 2024-03-19T02:05:59.368 INFO:teuthology.orchestra.run.smithi112.stdout: "pq_executing": 0, 2024-03-19T02:05:59.368 INFO:teuthology.orchestra.run.smithi112.stdout: "pq_executing_high_water": 0, 2024-03-19T02:05:59.368 INFO:teuthology.orchestra.run.smithi112.stdout: "pq_executed_ops": 0, 2024-03-19T02:05:59.368 INFO:teuthology.orchestra.run.smithi112.stdout: "pq_executed": 0, 2024-03-19T02:05:59.368 INFO:teuthology.orchestra.run.smithi112.stdout: "pq_item_in_journal": 0 2024-03-19T02:05:59.368 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.368 INFO:teuthology.orchestra.run.smithi112.stdout: "throttle-msgr_dispatch_throttler-mds": { 2024-03-19T02:05:59.368 INFO:teuthology.orchestra.run.smithi112.stdout: "val": 0, 2024-03-19T02:05:59.368 INFO:teuthology.orchestra.run.smithi112.stdout: "max": 104857600, 2024-03-19T02:05:59.368 INFO:teuthology.orchestra.run.smithi112.stdout: "get_started": 0, 2024-03-19T02:05:59.368 INFO:teuthology.orchestra.run.smithi112.stdout: "get": 6018, 2024-03-19T02:05:59.368 INFO:teuthology.orchestra.run.smithi112.stdout: "get_sum": 283033753, 2024-03-19T02:05:59.369 INFO:teuthology.orchestra.run.smithi112.stdout: "get_or_fail_fail": 0, 2024-03-19T02:05:59.369 INFO:teuthology.orchestra.run.smithi112.stdout: "get_or_fail_success": 6018, 2024-03-19T02:05:59.369 INFO:teuthology.orchestra.run.smithi112.stdout: "take": 0, 2024-03-19T02:05:59.369 INFO:teuthology.orchestra.run.smithi112.stdout: "take_sum": 0, 2024-03-19T02:05:59.369 INFO:teuthology.orchestra.run.smithi112.stdout: "put": 6018, 2024-03-19T02:05:59.369 INFO:teuthology.orchestra.run.smithi112.stdout: "put_sum": 283033753, 2024-03-19T02:05:59.369 INFO:teuthology.orchestra.run.smithi112.stdout: "wait": { 2024-03-19T02:05:59.369 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:59.369 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:59.369 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:59.369 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-03-19T02:05:59.369 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.369 INFO:teuthology.orchestra.run.smithi112.stdout: "throttle-objecter_bytes": { 2024-03-19T02:05:59.369 INFO:teuthology.orchestra.run.smithi112.stdout: "val": 0, 2024-03-19T02:05:59.369 INFO:teuthology.orchestra.run.smithi112.stdout: "max": 104857600, 2024-03-19T02:05:59.370 INFO:teuthology.orchestra.run.smithi112.stdout: "get_started": 0, 2024-03-19T02:05:59.370 INFO:teuthology.orchestra.run.smithi112.stdout: "get": 0, 2024-03-19T02:05:59.370 INFO:teuthology.orchestra.run.smithi112.stdout: "get_sum": 0, 2024-03-19T02:05:59.370 INFO:teuthology.orchestra.run.smithi112.stdout: "get_or_fail_fail": 0, 2024-03-19T02:05:59.370 INFO:teuthology.orchestra.run.smithi112.stdout: "get_or_fail_success": 0, 2024-03-19T02:05:59.370 INFO:teuthology.orchestra.run.smithi112.stdout: "take": 5538, 2024-03-19T02:05:59.370 INFO:teuthology.orchestra.run.smithi112.stdout: "take_sum": 281112231, 2024-03-19T02:05:59.370 INFO:teuthology.orchestra.run.smithi112.stdout: "put": 5538, 2024-03-19T02:05:59.370 INFO:teuthology.orchestra.run.smithi112.stdout: "put_sum": 281112231, 2024-03-19T02:05:59.370 INFO:teuthology.orchestra.run.smithi112.stdout: "wait": { 2024-03-19T02:05:59.370 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:59.370 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:59.370 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:59.371 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-03-19T02:05:59.371 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.371 INFO:teuthology.orchestra.run.smithi112.stdout: "throttle-objecter_ops": { 2024-03-19T02:05:59.371 INFO:teuthology.orchestra.run.smithi112.stdout: "val": 0, 2024-03-19T02:05:59.371 INFO:teuthology.orchestra.run.smithi112.stdout: "max": 1024, 2024-03-19T02:05:59.371 INFO:teuthology.orchestra.run.smithi112.stdout: "get_started": 0, 2024-03-19T02:05:59.371 INFO:teuthology.orchestra.run.smithi112.stdout: "get": 0, 2024-03-19T02:05:59.371 INFO:teuthology.orchestra.run.smithi112.stdout: "get_sum": 0, 2024-03-19T02:05:59.371 INFO:teuthology.orchestra.run.smithi112.stdout: "get_or_fail_fail": 0, 2024-03-19T02:05:59.371 INFO:teuthology.orchestra.run.smithi112.stdout: "get_or_fail_success": 0, 2024-03-19T02:05:59.371 INFO:teuthology.orchestra.run.smithi112.stdout: "take": 5538, 2024-03-19T02:05:59.371 INFO:teuthology.orchestra.run.smithi112.stdout: "take_sum": 5538, 2024-03-19T02:05:59.371 INFO:teuthology.orchestra.run.smithi112.stdout: "put": 5538, 2024-03-19T02:05:59.371 INFO:teuthology.orchestra.run.smithi112.stdout: "put_sum": 5538, 2024-03-19T02:05:59.372 INFO:teuthology.orchestra.run.smithi112.stdout: "wait": { 2024-03-19T02:05:59.372 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:59.372 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:59.372 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:59.372 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-03-19T02:05:59.372 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.372 INFO:teuthology.orchestra.run.smithi112.stdout: "throttle-write_buf_throttle": { 2024-03-19T02:05:59.372 INFO:teuthology.orchestra.run.smithi112.stdout: "val": 0, 2024-03-19T02:05:59.372 INFO:teuthology.orchestra.run.smithi112.stdout: "max": 3758096384, 2024-03-19T02:05:59.372 INFO:teuthology.orchestra.run.smithi112.stdout: "get_started": 0, 2024-03-19T02:05:59.372 INFO:teuthology.orchestra.run.smithi112.stdout: "get": 0, 2024-03-19T02:05:59.372 INFO:teuthology.orchestra.run.smithi112.stdout: "get_sum": 0, 2024-03-19T02:05:59.372 INFO:teuthology.orchestra.run.smithi112.stdout: "get_or_fail_fail": 0, 2024-03-19T02:05:59.372 INFO:teuthology.orchestra.run.smithi112.stdout: "get_or_fail_success": 0, 2024-03-19T02:05:59.372 INFO:teuthology.orchestra.run.smithi112.stdout: "take": 0, 2024-03-19T02:05:59.373 INFO:teuthology.orchestra.run.smithi112.stdout: "take_sum": 0, 2024-03-19T02:05:59.373 INFO:teuthology.orchestra.run.smithi112.stdout: "put": 0, 2024-03-19T02:05:59.373 INFO:teuthology.orchestra.run.smithi112.stdout: "put_sum": 0, 2024-03-19T02:05:59.373 INFO:teuthology.orchestra.run.smithi112.stdout: "wait": { 2024-03-19T02:05:59.373 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:59.373 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:59.373 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:59.373 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-03-19T02:05:59.373 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:05:59.373 INFO:teuthology.orchestra.run.smithi112.stdout: "throttle-write_buf_throttle-0x55f7470c26e0": { 2024-03-19T02:05:59.373 INFO:teuthology.orchestra.run.smithi112.stdout: "val": 0, 2024-03-19T02:05:59.373 INFO:teuthology.orchestra.run.smithi112.stdout: "max": 3758096384, 2024-03-19T02:05:59.373 INFO:teuthology.orchestra.run.smithi112.stdout: "get_started": 0, 2024-03-19T02:05:59.373 INFO:teuthology.orchestra.run.smithi112.stdout: "get": 0, 2024-03-19T02:05:59.373 INFO:teuthology.orchestra.run.smithi112.stdout: "get_sum": 0, 2024-03-19T02:05:59.374 INFO:teuthology.orchestra.run.smithi112.stdout: "get_or_fail_fail": 0, 2024-03-19T02:05:59.374 INFO:teuthology.orchestra.run.smithi112.stdout: "get_or_fail_success": 0, 2024-03-19T02:05:59.374 INFO:teuthology.orchestra.run.smithi112.stdout: "take": 0, 2024-03-19T02:05:59.374 INFO:teuthology.orchestra.run.smithi112.stdout: "take_sum": 0, 2024-03-19T02:05:59.374 INFO:teuthology.orchestra.run.smithi112.stdout: "put": 0, 2024-03-19T02:05:59.374 INFO:teuthology.orchestra.run.smithi112.stdout: "put_sum": 0, 2024-03-19T02:05:59.374 INFO:teuthology.orchestra.run.smithi112.stdout: "wait": { 2024-03-19T02:05:59.374 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:05:59.374 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:05:59.374 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:05:59.374 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-03-19T02:05:59.374 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-03-19T02:05:59.374 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-03-19T02:06:00.092 INFO:tasks.check_counter:Daemon mds.h mds.exported=0 2024-03-19T02:06:00.093 INFO:tasks.check_counter:Daemon mds.h mds.imported=0 2024-03-19T02:06:00.093 INFO:tasks.check_counter:Daemon mds.h mds_cache.dir_update=0 2024-03-19T02:06:00.093 INFO:tasks.check_counter:Daemon mds.h mds_cache.dir_update_receipt=0 2024-03-19T02:06:00.093 DEBUG:tasks.check_counter:Getting stats from k 2024-03-19T02:06:00.093 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph daemon mds.k perf dump 2024-03-19T02:06:00.435 INFO:teuthology.orchestra.run.smithi112.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.b/config 2024-03-19T02:06:01.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:06:00 smithi112 ceph-mon[66819]: pgmap v1098: 97 pgs: 97 active+clean; 456 MiB data, 9.0 GiB used, 1.0 TiB / 1.0 TiB avail; 163 KiB/s rd, 3.3 MiB/s wr, 124 op/s 2024-03-19T02:06:01.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:06:00 smithi115 ceph-mon[67332]: pgmap v1098: 97 pgs: 97 active+clean; 456 MiB data, 9.0 GiB used, 1.0 TiB / 1.0 TiB avail; 163 KiB/s rd, 3.3 MiB/s wr, 124 op/s 2024-03-19T02:06:01.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:06:00 smithi099 ceph-mon[62951]: pgmap v1098: 97 pgs: 97 active+clean; 456 MiB data, 9.0 GiB used, 1.0 TiB / 1.0 TiB avail; 163 KiB/s rd, 3.3 MiB/s wr, 124 op/s 2024-03-19T02:06:02.367 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-03-19T02:06:02.367 INFO:teuthology.orchestra.run.smithi112.stdout: "AsyncMessenger::Worker-0": { 2024-03-19T02:06:02.367 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_recv_messages": 445, 2024-03-19T02:06:02.367 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_send_messages": 440, 2024-03-19T02:06:02.367 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_recv_bytes": 816016, 2024-03-19T02:06:02.367 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_send_bytes": 272272, 2024-03-19T02:06:02.367 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_created_connections": 4, 2024-03-19T02:06:02.367 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_active_connections": 0, 2024-03-19T02:06:02.367 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_running_total_time": 0.048293961, 2024-03-19T02:06:02.367 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_running_send_time": 0.016887201, 2024-03-19T02:06:02.367 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_running_recv_time": 0.023717426, 2024-03-19T02:06:02.368 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_running_fast_dispatch_time": 0.007982969, 2024-03-19T02:06:02.368 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_send_messages_queue_lat": { 2024-03-19T02:06:02.368 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 439, 2024-03-19T02:06:02.368 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.011745573, 2024-03-19T02:06:02.368 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000026755 2024-03-19T02:06:02.368 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:06:02.368 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_handle_ack_lat": { 2024-03-19T02:06:02.368 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:06:02.368 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:06:02.368 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:06:02.368 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:06:02.368 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_recv_encrypted_bytes": 816016, 2024-03-19T02:06:02.368 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_send_encrypted_bytes": 272272 2024-03-19T02:06:02.368 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:06:02.369 INFO:teuthology.orchestra.run.smithi112.stdout: "AsyncMessenger::Worker-1": { 2024-03-19T02:06:02.369 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_recv_messages": 6, 2024-03-19T02:06:02.369 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_send_messages": 338, 2024-03-19T02:06:02.369 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_recv_bytes": 3392, 2024-03-19T02:06:02.369 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_send_bytes": 215520, 2024-03-19T02:06:02.369 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_created_connections": 4, 2024-03-19T02:06:02.369 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_active_connections": 2, 2024-03-19T02:06:02.369 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_running_total_time": 0.013363151, 2024-03-19T02:06:02.369 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_running_send_time": 0.009766531, 2024-03-19T02:06:02.369 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_running_recv_time": 0.001489359, 2024-03-19T02:06:02.369 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_running_fast_dispatch_time": 0.000000000, 2024-03-19T02:06:02.369 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_send_messages_queue_lat": { 2024-03-19T02:06:02.369 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 337, 2024-03-19T02:06:02.369 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.011937913, 2024-03-19T02:06:02.370 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000035424 2024-03-19T02:06:02.370 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:06:02.370 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_handle_ack_lat": { 2024-03-19T02:06:02.370 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:06:02.370 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:06:02.370 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:06:02.370 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:06:02.370 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_recv_encrypted_bytes": 3392, 2024-03-19T02:06:02.370 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_send_encrypted_bytes": 215520 2024-03-19T02:06:02.370 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:06:02.370 INFO:teuthology.orchestra.run.smithi112.stdout: "AsyncMessenger::Worker-2": { 2024-03-19T02:06:02.370 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_recv_messages": 0, 2024-03-19T02:06:02.370 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_send_messages": 2, 2024-03-19T02:06:02.371 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_recv_bytes": 0, 2024-03-19T02:06:02.371 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_send_bytes": 0, 2024-03-19T02:06:02.371 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_created_connections": 2, 2024-03-19T02:06:02.371 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_active_connections": 1, 2024-03-19T02:06:02.371 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_running_total_time": 0.000325785, 2024-03-19T02:06:02.371 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_running_send_time": 0.000000000, 2024-03-19T02:06:02.371 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_running_recv_time": 0.000235140, 2024-03-19T02:06:02.371 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_running_fast_dispatch_time": 0.000000000, 2024-03-19T02:06:02.371 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_send_messages_queue_lat": { 2024-03-19T02:06:02.371 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:06:02.371 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:06:02.371 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:06:02.371 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:06:02.372 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_handle_ack_lat": { 2024-03-19T02:06:02.372 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:06:02.372 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:06:02.372 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:06:02.372 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:06:02.372 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_recv_encrypted_bytes": 0, 2024-03-19T02:06:02.372 INFO:teuthology.orchestra.run.smithi112.stdout: "msgr_send_encrypted_bytes": 0 2024-03-19T02:06:02.372 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:06:02.372 INFO:teuthology.orchestra.run.smithi112.stdout: "cct": { 2024-03-19T02:06:02.372 INFO:teuthology.orchestra.run.smithi112.stdout: "total_workers": 0, 2024-03-19T02:06:02.372 INFO:teuthology.orchestra.run.smithi112.stdout: "unhealthy_workers": 0 2024-03-19T02:06:02.372 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:06:02.372 INFO:teuthology.orchestra.run.smithi112.stdout: "mempool": { 2024-03-19T02:06:02.372 INFO:teuthology.orchestra.run.smithi112.stdout: "bloom_filter_bytes": 0, 2024-03-19T02:06:02.372 INFO:teuthology.orchestra.run.smithi112.stdout: "bloom_filter_items": 0, 2024-03-19T02:06:02.373 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_alloc_bytes": 0, 2024-03-19T02:06:02.373 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_alloc_items": 0, 2024-03-19T02:06:02.373 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_cache_data_bytes": 0, 2024-03-19T02:06:02.373 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_cache_data_items": 0, 2024-03-19T02:06:02.373 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_cache_onode_bytes": 0, 2024-03-19T02:06:02.373 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_cache_onode_items": 0, 2024-03-19T02:06:02.373 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_cache_meta_bytes": 0, 2024-03-19T02:06:02.373 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_cache_meta_items": 0, 2024-03-19T02:06:02.373 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_cache_other_bytes": 0, 2024-03-19T02:06:02.373 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_cache_other_items": 0, 2024-03-19T02:06:02.373 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_cache_buffer_bytes": 0, 2024-03-19T02:06:02.373 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_cache_buffer_items": 0, 2024-03-19T02:06:02.373 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_extent_bytes": 0, 2024-03-19T02:06:02.373 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_extent_items": 0, 2024-03-19T02:06:02.374 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_blob_bytes": 0, 2024-03-19T02:06:02.374 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_blob_items": 0, 2024-03-19T02:06:02.374 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_shared_blob_bytes": 0, 2024-03-19T02:06:02.374 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_shared_blob_items": 0, 2024-03-19T02:06:02.374 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_inline_bl_bytes": 0, 2024-03-19T02:06:02.374 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_inline_bl_items": 0, 2024-03-19T02:06:02.374 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_fsck_bytes": 0, 2024-03-19T02:06:02.374 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_fsck_items": 0, 2024-03-19T02:06:02.374 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_txc_bytes": 0, 2024-03-19T02:06:02.374 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_txc_items": 0, 2024-03-19T02:06:02.374 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_writing_deferred_bytes": 0, 2024-03-19T02:06:02.374 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_writing_deferred_items": 0, 2024-03-19T02:06:02.374 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_writing_bytes": 0, 2024-03-19T02:06:02.374 INFO:teuthology.orchestra.run.smithi112.stdout: "bluestore_writing_items": 0, 2024-03-19T02:06:02.375 INFO:teuthology.orchestra.run.smithi112.stdout: "bluefs_bytes": 0, 2024-03-19T02:06:02.375 INFO:teuthology.orchestra.run.smithi112.stdout: "bluefs_items": 0, 2024-03-19T02:06:02.375 INFO:teuthology.orchestra.run.smithi112.stdout: "bluefs_file_reader_bytes": 0, 2024-03-19T02:06:02.375 INFO:teuthology.orchestra.run.smithi112.stdout: "bluefs_file_reader_items": 0, 2024-03-19T02:06:02.375 INFO:teuthology.orchestra.run.smithi112.stdout: "bluefs_file_writer_bytes": 0, 2024-03-19T02:06:02.375 INFO:teuthology.orchestra.run.smithi112.stdout: "bluefs_file_writer_items": 0, 2024-03-19T02:06:02.375 INFO:teuthology.orchestra.run.smithi112.stdout: "buffer_anon_bytes": 148721, 2024-03-19T02:06:02.375 INFO:teuthology.orchestra.run.smithi112.stdout: "buffer_anon_items": 30, 2024-03-19T02:06:02.375 INFO:teuthology.orchestra.run.smithi112.stdout: "buffer_meta_bytes": 0, 2024-03-19T02:06:02.375 INFO:teuthology.orchestra.run.smithi112.stdout: "buffer_meta_items": 0, 2024-03-19T02:06:02.375 INFO:teuthology.orchestra.run.smithi112.stdout: "osd_bytes": 0, 2024-03-19T02:06:02.375 INFO:teuthology.orchestra.run.smithi112.stdout: "osd_items": 0, 2024-03-19T02:06:02.375 INFO:teuthology.orchestra.run.smithi112.stdout: "osd_mapbl_bytes": 0, 2024-03-19T02:06:02.375 INFO:teuthology.orchestra.run.smithi112.stdout: "osd_mapbl_items": 0, 2024-03-19T02:06:02.376 INFO:teuthology.orchestra.run.smithi112.stdout: "osd_pglog_bytes": 0, 2024-03-19T02:06:02.376 INFO:teuthology.orchestra.run.smithi112.stdout: "osd_pglog_items": 0, 2024-03-19T02:06:02.376 INFO:teuthology.orchestra.run.smithi112.stdout: "osdmap_bytes": 0, 2024-03-19T02:06:02.376 INFO:teuthology.orchestra.run.smithi112.stdout: "osdmap_items": 0, 2024-03-19T02:06:02.376 INFO:teuthology.orchestra.run.smithi112.stdout: "osdmap_mapping_bytes": 0, 2024-03-19T02:06:02.376 INFO:teuthology.orchestra.run.smithi112.stdout: "osdmap_mapping_items": 0, 2024-03-19T02:06:02.376 INFO:teuthology.orchestra.run.smithi112.stdout: "pgmap_bytes": 0, 2024-03-19T02:06:02.376 INFO:teuthology.orchestra.run.smithi112.stdout: "pgmap_items": 0, 2024-03-19T02:06:02.376 INFO:teuthology.orchestra.run.smithi112.stdout: "mds_co_bytes": 928, 2024-03-19T02:06:02.376 INFO:teuthology.orchestra.run.smithi112.stdout: "mds_co_items": 2, 2024-03-19T02:06:02.376 INFO:teuthology.orchestra.run.smithi112.stdout: "unittest_1_bytes": 0, 2024-03-19T02:06:02.376 INFO:teuthology.orchestra.run.smithi112.stdout: "unittest_1_items": 0, 2024-03-19T02:06:02.376 INFO:teuthology.orchestra.run.smithi112.stdout: "unittest_2_bytes": 0, 2024-03-19T02:06:02.376 INFO:teuthology.orchestra.run.smithi112.stdout: "unittest_2_items": 0 2024-03-19T02:06:02.377 INFO:teuthology.orchestra.run.smithi112.stdout: }, 2024-03-19T02:06:02.377 INFO:teuthology.orchestra.run.smithi112.stdout: "throttle-msgr_dispatch_throttler-mds": { 2024-03-19T02:06:02.377 INFO:teuthology.orchestra.run.smithi112.stdout: "val": 0, 2024-03-19T02:06:02.377 INFO:teuthology.orchestra.run.smithi112.stdout: "max": 104857600, 2024-03-19T02:06:02.377 INFO:teuthology.orchestra.run.smithi112.stdout: "get_started": 0, 2024-03-19T02:06:02.377 INFO:teuthology.orchestra.run.smithi112.stdout: "get": 447, 2024-03-19T02:06:02.377 INFO:teuthology.orchestra.run.smithi112.stdout: "get_sum": 753110, 2024-03-19T02:06:02.377 INFO:teuthology.orchestra.run.smithi112.stdout: "get_or_fail_fail": 0, 2024-03-19T02:06:02.377 INFO:teuthology.orchestra.run.smithi112.stdout: "get_or_fail_success": 447, 2024-03-19T02:06:02.377 INFO:teuthology.orchestra.run.smithi112.stdout: "take": 0, 2024-03-19T02:06:02.377 INFO:teuthology.orchestra.run.smithi112.stdout: "take_sum": 0, 2024-03-19T02:06:02.377 INFO:teuthology.orchestra.run.smithi112.stdout: "put": 447, 2024-03-19T02:06:02.377 INFO:teuthology.orchestra.run.smithi112.stdout: "put_sum": 753110, 2024-03-19T02:06:02.377 INFO:teuthology.orchestra.run.smithi112.stdout: "wait": { 2024-03-19T02:06:02.377 INFO:teuthology.orchestra.run.smithi112.stdout: "avgcount": 0, 2024-03-19T02:06:02.378 INFO:teuthology.orchestra.run.smithi112.stdout: "sum": 0.000000000, 2024-03-19T02:06:02.378 INFO:teuthology.orchestra.run.smithi112.stdout: "avgtime": 0.000000000 2024-03-19T02:06:02.378 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-03-19T02:06:02.378 INFO:teuthology.orchestra.run.smithi112.stdout: } 2024-03-19T02:06:02.378 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-03-19T02:06:03.041 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:06:02 smithi112 ceph-mon[66819]: pgmap v1099: 97 pgs: 97 active+clean; 453 MiB data, 9.0 GiB used, 1.0 TiB / 1.0 TiB avail; 162 KiB/s rd, 3.2 MiB/s wr, 130 op/s 2024-03-19T02:06:03.071 WARNING:tasks.check_counter:Counter 'mds.exported' not found on daemon mds.k 2024-03-19T02:06:03.071 WARNING:tasks.check_counter:Counter 'mds.imported' not found on daemon mds.k 2024-03-19T02:06:03.071 WARNING:tasks.check_counter:Counter 'mds_cache.dir_update' not found on daemon mds.k 2024-03-19T02:06:03.071 WARNING:tasks.check_counter:Counter 'mds_cache.dir_update_receipt' not found on daemon mds.k 2024-03-19T02:06:03.071 DEBUG:tasks.check_counter:Getting stats from c 2024-03-19T02:06:03.071 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph daemon mds.c perf dump 2024-03-19T02:06:03.212 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:06:02 smithi115 ceph-mon[67332]: pgmap v1099: 97 pgs: 97 active+clean; 453 MiB data, 9.0 GiB used, 1.0 TiB / 1.0 TiB avail; 162 KiB/s rd, 3.2 MiB/s wr, 130 op/s 2024-03-19T02:06:03.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:06:02 smithi099 ceph-mon[62951]: pgmap v1099: 97 pgs: 97 active+clean; 453 MiB data, 9.0 GiB used, 1.0 TiB / 1.0 TiB avail; 162 KiB/s rd, 3.2 MiB/s wr, 130 op/s 2024-03-19T02:06:03.475 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.c/config 2024-03-19T02:06:05.176 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:06:04 smithi115 ceph-mon[67332]: pgmap v1100: 97 pgs: 97 active+clean; 453 MiB data, 9.0 GiB used, 1.0 TiB / 1.0 TiB avail; 180 KiB/s rd, 3.7 MiB/s wr, 131 op/s 2024-03-19T02:06:05.316 INFO:teuthology.orchestra.run.smithi115.stdout:{ 2024-03-19T02:06:05.317 INFO:teuthology.orchestra.run.smithi115.stdout: "AsyncMessenger::Worker-0": { 2024-03-19T02:06:05.317 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_recv_messages": 675529, 2024-03-19T02:06:05.317 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_send_messages": 961096, 2024-03-19T02:06:05.317 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_recv_bytes": 200751435, 2024-03-19T02:06:05.317 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_send_bytes": 907642675, 2024-03-19T02:06:05.317 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_created_connections": 15, 2024-03-19T02:06:05.317 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_active_connections": 6, 2024-03-19T02:06:05.317 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_running_total_time": 38.832636552, 2024-03-19T02:06:05.317 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_running_send_time": 16.844525488, 2024-03-19T02:06:05.317 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_running_recv_time": 25.601176953, 2024-03-19T02:06:05.317 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_running_fast_dispatch_time": 1.930460300, 2024-03-19T02:06:05.317 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_send_messages_queue_lat": { 2024-03-19T02:06:05.318 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 961095, 2024-03-19T02:06:05.318 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 30.737944083, 2024-03-19T02:06:05.318 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000031982 2024-03-19T02:06:05.318 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.318 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_handle_ack_lat": { 2024-03-19T02:06:05.318 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 1388517, 2024-03-19T02:06:05.318 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.213727836, 2024-03-19T02:06:05.318 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000153 2024-03-19T02:06:05.318 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.318 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_recv_encrypted_bytes": 4000, 2024-03-19T02:06:05.318 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_send_encrypted_bytes": 1184 2024-03-19T02:06:05.318 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.318 INFO:teuthology.orchestra.run.smithi115.stdout: "AsyncMessenger::Worker-1": { 2024-03-19T02:06:05.318 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_recv_messages": 328610, 2024-03-19T02:06:05.318 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_send_messages": 411706, 2024-03-19T02:06:05.319 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_recv_bytes": 75788230, 2024-03-19T02:06:05.319 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_send_bytes": 319855457, 2024-03-19T02:06:05.319 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_created_connections": 8, 2024-03-19T02:06:05.319 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_active_connections": 6, 2024-03-19T02:06:05.319 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_running_total_time": 21.672315935, 2024-03-19T02:06:05.319 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_running_send_time": 9.093758964, 2024-03-19T02:06:05.319 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_running_recv_time": 11.767987767, 2024-03-19T02:06:05.319 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_running_fast_dispatch_time": 1.813719166, 2024-03-19T02:06:05.319 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_send_messages_queue_lat": { 2024-03-19T02:06:05.319 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 411704, 2024-03-19T02:06:05.319 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 13.175768490, 2024-03-19T02:06:05.319 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000032003 2024-03-19T02:06:05.319 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.319 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_handle_ack_lat": { 2024-03-19T02:06:05.319 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 614140, 2024-03-19T02:06:05.319 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.107035148, 2024-03-19T02:06:05.320 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000174 2024-03-19T02:06:05.320 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.320 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_recv_encrypted_bytes": 160, 2024-03-19T02:06:05.320 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_send_encrypted_bytes": 670880 2024-03-19T02:06:05.320 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.320 INFO:teuthology.orchestra.run.smithi115.stdout: "AsyncMessenger::Worker-2": { 2024-03-19T02:06:05.320 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_recv_messages": 436203, 2024-03-19T02:06:05.320 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_send_messages": 434637, 2024-03-19T02:06:05.320 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_recv_bytes": 110741202, 2024-03-19T02:06:05.320 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_send_bytes": 218433384, 2024-03-19T02:06:05.320 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_created_connections": 7, 2024-03-19T02:06:05.320 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_active_connections": 6, 2024-03-19T02:06:05.320 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_running_total_time": 29.215567072, 2024-03-19T02:06:05.320 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_running_send_time": 13.311510679, 2024-03-19T02:06:05.321 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_running_recv_time": 17.689354580, 2024-03-19T02:06:05.321 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_running_fast_dispatch_time": 1.258303166, 2024-03-19T02:06:05.321 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_send_messages_queue_lat": { 2024-03-19T02:06:05.321 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 434636, 2024-03-19T02:06:05.321 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 10.894068636, 2024-03-19T02:06:05.321 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000025064 2024-03-19T02:06:05.321 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.321 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_handle_ack_lat": { 2024-03-19T02:06:05.321 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 775068, 2024-03-19T02:06:05.321 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.131720729, 2024-03-19T02:06:05.321 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000169 2024-03-19T02:06:05.321 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.321 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_recv_encrypted_bytes": 955952, 2024-03-19T02:06:05.321 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_send_encrypted_bytes": 278896 2024-03-19T02:06:05.321 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.322 INFO:teuthology.orchestra.run.smithi115.stdout: "cct": { 2024-03-19T02:06:05.322 INFO:teuthology.orchestra.run.smithi115.stdout: "total_workers": 1, 2024-03-19T02:06:05.322 INFO:teuthology.orchestra.run.smithi115.stdout: "unhealthy_workers": 0 2024-03-19T02:06:05.322 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.322 INFO:teuthology.orchestra.run.smithi115.stdout: "finisher-MDSRank": { 2024-03-19T02:06:05.322 INFO:teuthology.orchestra.run.smithi115.stdout: "queue_len": 0, 2024-03-19T02:06:05.322 INFO:teuthology.orchestra.run.smithi115.stdout: "complete_latency": { 2024-03-19T02:06:05.322 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 211924, 2024-03-19T02:06:05.322 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 216.864643117, 2024-03-19T02:06:05.322 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.001023313 2024-03-19T02:06:05.322 INFO:teuthology.orchestra.run.smithi115.stdout: } 2024-03-19T02:06:05.322 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.322 INFO:teuthology.orchestra.run.smithi115.stdout: "finisher-PurgeQueue": { 2024-03-19T02:06:05.323 INFO:teuthology.orchestra.run.smithi115.stdout: "queue_len": 0, 2024-03-19T02:06:05.323 INFO:teuthology.orchestra.run.smithi115.stdout: "complete_latency": { 2024-03-19T02:06:05.323 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 98971, 2024-03-19T02:06:05.323 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 13.312809803, 2024-03-19T02:06:05.323 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000134512 2024-03-19T02:06:05.323 INFO:teuthology.orchestra.run.smithi115.stdout: } 2024-03-19T02:06:05.323 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.323 INFO:teuthology.orchestra.run.smithi115.stdout: "mds": { 2024-03-19T02:06:05.323 INFO:teuthology.orchestra.run.smithi115.stdout: "request": 446668, 2024-03-19T02:06:05.323 INFO:teuthology.orchestra.run.smithi115.stdout: "reply": 390268, 2024-03-19T02:06:05.323 INFO:teuthology.orchestra.run.smithi115.stdout: "reply_latency": { 2024-03-19T02:06:05.323 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 390268, 2024-03-19T02:06:05.323 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 969.154040248, 2024-03-19T02:06:05.323 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.002483303 2024-03-19T02:06:05.324 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.324 INFO:teuthology.orchestra.run.smithi115.stdout: "slow_reply": 0, 2024-03-19T02:06:05.324 INFO:teuthology.orchestra.run.smithi115.stdout: "forward": 56400, 2024-03-19T02:06:05.324 INFO:teuthology.orchestra.run.smithi115.stdout: "dir_fetch_complete": 0, 2024-03-19T02:06:05.324 INFO:teuthology.orchestra.run.smithi115.stdout: "dir_fetch_keys": 0, 2024-03-19T02:06:05.324 INFO:teuthology.orchestra.run.smithi115.stdout: "dir_commit": 22, 2024-03-19T02:06:05.324 INFO:teuthology.orchestra.run.smithi115.stdout: "dir_split": 0, 2024-03-19T02:06:05.324 INFO:teuthology.orchestra.run.smithi115.stdout: "dir_merge": 0, 2024-03-19T02:06:05.324 INFO:teuthology.orchestra.run.smithi115.stdout: "inodes": 103, 2024-03-19T02:06:05.324 INFO:teuthology.orchestra.run.smithi115.stdout: "inodes_top": 4, 2024-03-19T02:06:05.324 INFO:teuthology.orchestra.run.smithi115.stdout: "inodes_bottom": 7, 2024-03-19T02:06:05.324 INFO:teuthology.orchestra.run.smithi115.stdout: "inodes_pin_tail": 92, 2024-03-19T02:06:05.324 INFO:teuthology.orchestra.run.smithi115.stdout: "inodes_pinned": 95, 2024-03-19T02:06:05.324 INFO:teuthology.orchestra.run.smithi115.stdout: "inodes_expired": 479, 2024-03-19T02:06:05.324 INFO:teuthology.orchestra.run.smithi115.stdout: "inodes_with_caps": 9, 2024-03-19T02:06:05.325 INFO:teuthology.orchestra.run.smithi115.stdout: "caps": 9, 2024-03-19T02:06:05.325 INFO:teuthology.orchestra.run.smithi115.stdout: "subtrees": 4, 2024-03-19T02:06:05.325 INFO:teuthology.orchestra.run.smithi115.stdout: "traverse": 732648, 2024-03-19T02:06:05.325 INFO:teuthology.orchestra.run.smithi115.stdout: "traverse_hit": 388420, 2024-03-19T02:06:05.325 INFO:teuthology.orchestra.run.smithi115.stdout: "traverse_forward": 56397, 2024-03-19T02:06:05.325 INFO:teuthology.orchestra.run.smithi115.stdout: "traverse_discover": 228, 2024-03-19T02:06:05.325 INFO:teuthology.orchestra.run.smithi115.stdout: "traverse_dir_fetch": 0, 2024-03-19T02:06:05.325 INFO:teuthology.orchestra.run.smithi115.stdout: "traverse_remote_ino": 0, 2024-03-19T02:06:05.325 INFO:teuthology.orchestra.run.smithi115.stdout: "traverse_lock": 8908, 2024-03-19T02:06:05.325 INFO:teuthology.orchestra.run.smithi115.stdout: "load_cent": 0, 2024-03-19T02:06:05.325 INFO:teuthology.orchestra.run.smithi115.stdout: "q": 0, 2024-03-19T02:06:05.325 INFO:teuthology.orchestra.run.smithi115.stdout: "exported": 5, 2024-03-19T02:06:05.325 INFO:teuthology.orchestra.run.smithi115.stdout: "exported_inodes": 109, 2024-03-19T02:06:05.325 INFO:teuthology.orchestra.run.smithi115.stdout: "imported": 14, 2024-03-19T02:06:05.326 INFO:teuthology.orchestra.run.smithi115.stdout: "imported_inodes": 227, 2024-03-19T02:06:05.326 INFO:teuthology.orchestra.run.smithi115.stdout: "openino_dir_fetch": 0, 2024-03-19T02:06:05.326 INFO:teuthology.orchestra.run.smithi115.stdout: "openino_backtrace_fetch": 0, 2024-03-19T02:06:05.326 INFO:teuthology.orchestra.run.smithi115.stdout: "openino_peer_discover": 0, 2024-03-19T02:06:05.326 INFO:teuthology.orchestra.run.smithi115.stdout: "root_rfiles": 4, 2024-03-19T02:06:05.326 INFO:teuthology.orchestra.run.smithi115.stdout: "root_rbytes": 234, 2024-03-19T02:06:05.326 INFO:teuthology.orchestra.run.smithi115.stdout: "root_rsnaps": 0, 2024-03-19T02:06:05.326 INFO:teuthology.orchestra.run.smithi115.stdout: "scrub_backtrace_fetch": 0, 2024-03-19T02:06:05.326 INFO:teuthology.orchestra.run.smithi115.stdout: "scrub_set_tag": 0, 2024-03-19T02:06:05.326 INFO:teuthology.orchestra.run.smithi115.stdout: "scrub_backtrace_repaired": 0, 2024-03-19T02:06:05.326 INFO:teuthology.orchestra.run.smithi115.stdout: "scrub_inotable_repaired": 0, 2024-03-19T02:06:05.326 INFO:teuthology.orchestra.run.smithi115.stdout: "scrub_dir_inodes": 0, 2024-03-19T02:06:05.326 INFO:teuthology.orchestra.run.smithi115.stdout: "scrub_dir_base_inodes": 0, 2024-03-19T02:06:05.326 INFO:teuthology.orchestra.run.smithi115.stdout: "scrub_dirfrag_rstats": 0, 2024-03-19T02:06:05.327 INFO:teuthology.orchestra.run.smithi115.stdout: "scrub_file_inodes": 0, 2024-03-19T02:06:05.327 INFO:teuthology.orchestra.run.smithi115.stdout: "handle_inode_file_caps": 6, 2024-03-19T02:06:05.327 INFO:teuthology.orchestra.run.smithi115.stdout: "ceph_cap_op_revoke": 47052, 2024-03-19T02:06:05.327 INFO:teuthology.orchestra.run.smithi115.stdout: "ceph_cap_op_grant": 202428, 2024-03-19T02:06:05.327 INFO:teuthology.orchestra.run.smithi115.stdout: "ceph_cap_op_trunc": 0, 2024-03-19T02:06:05.327 INFO:teuthology.orchestra.run.smithi115.stdout: "ceph_cap_op_flushsnap_ack": 0, 2024-03-19T02:06:05.327 INFO:teuthology.orchestra.run.smithi115.stdout: "ceph_cap_op_flush_ack": 0, 2024-03-19T02:06:05.327 INFO:teuthology.orchestra.run.smithi115.stdout: "handle_client_caps": 118297, 2024-03-19T02:06:05.327 INFO:teuthology.orchestra.run.smithi115.stdout: "handle_client_caps_dirty": 44863, 2024-03-19T02:06:05.327 INFO:teuthology.orchestra.run.smithi115.stdout: "handle_client_cap_release": 20792, 2024-03-19T02:06:05.327 INFO:teuthology.orchestra.run.smithi115.stdout: "process_request_cap_release": 268350 2024-03-19T02:06:05.327 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.327 INFO:teuthology.orchestra.run.smithi115.stdout: "mds_cache": { 2024-03-19T02:06:05.327 INFO:teuthology.orchestra.run.smithi115.stdout: "dir_update": 296, 2024-03-19T02:06:05.327 INFO:teuthology.orchestra.run.smithi115.stdout: "dir_update_receipt": 795, 2024-03-19T02:06:05.328 INFO:teuthology.orchestra.run.smithi115.stdout: "dir_try_discover": 485, 2024-03-19T02:06:05.328 INFO:teuthology.orchestra.run.smithi115.stdout: "dir_send_discover": 683, 2024-03-19T02:06:05.328 INFO:teuthology.orchestra.run.smithi115.stdout: "dir_handle_discover": 615, 2024-03-19T02:06:05.328 INFO:teuthology.orchestra.run.smithi115.stdout: "num_strays": 9, 2024-03-19T02:06:05.328 INFO:teuthology.orchestra.run.smithi115.stdout: "num_strays_delayed": 0, 2024-03-19T02:06:05.328 INFO:teuthology.orchestra.run.smithi115.stdout: "num_strays_enqueuing": 0, 2024-03-19T02:06:05.328 INFO:teuthology.orchestra.run.smithi115.stdout: "strays_created": 62492, 2024-03-19T02:06:05.328 INFO:teuthology.orchestra.run.smithi115.stdout: "strays_enqueued": 62483, 2024-03-19T02:06:05.328 INFO:teuthology.orchestra.run.smithi115.stdout: "strays_reintegrated": 0, 2024-03-19T02:06:05.328 INFO:teuthology.orchestra.run.smithi115.stdout: "strays_migrated": 0, 2024-03-19T02:06:05.328 INFO:teuthology.orchestra.run.smithi115.stdout: "num_recovering_processing": 0, 2024-03-19T02:06:05.328 INFO:teuthology.orchestra.run.smithi115.stdout: "num_recovering_enqueued": 0, 2024-03-19T02:06:05.328 INFO:teuthology.orchestra.run.smithi115.stdout: "num_recovering_prioritized": 0, 2024-03-19T02:06:05.328 INFO:teuthology.orchestra.run.smithi115.stdout: "recovery_started": 0, 2024-03-19T02:06:05.328 INFO:teuthology.orchestra.run.smithi115.stdout: "recovery_completed": 0, 2024-03-19T02:06:05.329 INFO:teuthology.orchestra.run.smithi115.stdout: "ireq_quiesce_path": 0, 2024-03-19T02:06:05.329 INFO:teuthology.orchestra.run.smithi115.stdout: "ireq_quiesce_inode": 0, 2024-03-19T02:06:05.329 INFO:teuthology.orchestra.run.smithi115.stdout: "ireq_enqueue_scrub": 0, 2024-03-19T02:06:05.329 INFO:teuthology.orchestra.run.smithi115.stdout: "ireq_exportdir": 6, 2024-03-19T02:06:05.329 INFO:teuthology.orchestra.run.smithi115.stdout: "ireq_flush": 0, 2024-03-19T02:06:05.329 INFO:teuthology.orchestra.run.smithi115.stdout: "ireq_fragmentdir": 0, 2024-03-19T02:06:05.329 INFO:teuthology.orchestra.run.smithi115.stdout: "ireq_fragstats": 0, 2024-03-19T02:06:05.329 INFO:teuthology.orchestra.run.smithi115.stdout: "ireq_inodestats": 0 2024-03-19T02:06:05.329 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.329 INFO:teuthology.orchestra.run.smithi115.stdout: "mds_log": { 2024-03-19T02:06:05.329 INFO:teuthology.orchestra.run.smithi115.stdout: "evlrg": 0, 2024-03-19T02:06:05.329 INFO:teuthology.orchestra.run.smithi115.stdout: "evadd": 290706, 2024-03-19T02:06:05.329 INFO:teuthology.orchestra.run.smithi115.stdout: "evex": 195500, 2024-03-19T02:06:05.329 INFO:teuthology.orchestra.run.smithi115.stdout: "evtrm": 184351, 2024-03-19T02:06:05.329 INFO:teuthology.orchestra.run.smithi115.stdout: "ev": 106355, 2024-03-19T02:06:05.330 INFO:teuthology.orchestra.run.smithi115.stdout: "evexg": 0, 2024-03-19T02:06:05.330 INFO:teuthology.orchestra.run.smithi115.stdout: "evexd": 11149, 2024-03-19T02:06:05.330 INFO:teuthology.orchestra.run.smithi115.stdout: "segadd": 365, 2024-03-19T02:06:05.330 INFO:teuthology.orchestra.run.smithi115.stdout: "segex": 237, 2024-03-19T02:06:05.330 INFO:teuthology.orchestra.run.smithi115.stdout: "segtrm": 224, 2024-03-19T02:06:05.330 INFO:teuthology.orchestra.run.smithi115.stdout: "seg": 141, 2024-03-19T02:06:05.330 INFO:teuthology.orchestra.run.smithi115.stdout: "segmjr": 9, 2024-03-19T02:06:05.330 INFO:teuthology.orchestra.run.smithi115.stdout: "segexg": 0, 2024-03-19T02:06:05.330 INFO:teuthology.orchestra.run.smithi115.stdout: "segexd": 13, 2024-03-19T02:06:05.330 INFO:teuthology.orchestra.run.smithi115.stdout: "expos": 390069714, 2024-03-19T02:06:05.330 INFO:teuthology.orchestra.run.smithi115.stdout: "wrpos": 591613875, 2024-03-19T02:06:05.330 INFO:teuthology.orchestra.run.smithi115.stdout: "rdpos": 0, 2024-03-19T02:06:05.330 INFO:teuthology.orchestra.run.smithi115.stdout: "jlat": { 2024-03-19T02:06:05.330 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 107626, 2024-03-19T02:06:05.331 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 353.306163118, 2024-03-19T02:06:05.331 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.003282721 2024-03-19T02:06:05.331 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.331 INFO:teuthology.orchestra.run.smithi115.stdout: "replayed": 0 2024-03-19T02:06:05.331 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.331 INFO:teuthology.orchestra.run.smithi115.stdout: "mds_mem": { 2024-03-19T02:06:05.331 INFO:teuthology.orchestra.run.smithi115.stdout: "ino": 78, 2024-03-19T02:06:05.331 INFO:teuthology.orchestra.run.smithi115.stdout: "ino+": 62797, 2024-03-19T02:06:05.331 INFO:teuthology.orchestra.run.smithi115.stdout: "ino-": 62719, 2024-03-19T02:06:05.331 INFO:teuthology.orchestra.run.smithi115.stdout: "dir": 34, 2024-03-19T02:06:05.331 INFO:teuthology.orchestra.run.smithi115.stdout: "dir+": 471, 2024-03-19T02:06:05.331 INFO:teuthology.orchestra.run.smithi115.stdout: "dir-": 437, 2024-03-19T02:06:05.331 INFO:teuthology.orchestra.run.smithi115.stdout: "dn": 76, 2024-03-19T02:06:05.331 INFO:teuthology.orchestra.run.smithi115.stdout: "dn+": 112301, 2024-03-19T02:06:05.331 INFO:teuthology.orchestra.run.smithi115.stdout: "dn-": 112225, 2024-03-19T02:06:05.332 INFO:teuthology.orchestra.run.smithi115.stdout: "cap": 9, 2024-03-19T02:06:05.332 INFO:teuthology.orchestra.run.smithi115.stdout: "cap+": 63792, 2024-03-19T02:06:05.332 INFO:teuthology.orchestra.run.smithi115.stdout: "cap-": 63783, 2024-03-19T02:06:05.332 INFO:teuthology.orchestra.run.smithi115.stdout: "rss": 89828, 2024-03-19T02:06:05.332 INFO:teuthology.orchestra.run.smithi115.stdout: "heap": 223612 2024-03-19T02:06:05.332 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.332 INFO:teuthology.orchestra.run.smithi115.stdout: "mds_server": { 2024-03-19T02:06:05.332 INFO:teuthology.orchestra.run.smithi115.stdout: "dispatch_client_request": 629417, 2024-03-19T02:06:05.332 INFO:teuthology.orchestra.run.smithi115.stdout: "dispatch_server_request": 19416, 2024-03-19T02:06:05.332 INFO:teuthology.orchestra.run.smithi115.stdout: "handle_client_request": 446668, 2024-03-19T02:06:05.332 INFO:teuthology.orchestra.run.smithi115.stdout: "handle_client_session": 8685, 2024-03-19T02:06:05.332 INFO:teuthology.orchestra.run.smithi115.stdout: "handle_peer_request": 46226, 2024-03-19T02:06:05.332 INFO:teuthology.orchestra.run.smithi115.stdout: "req_create_latency": { 2024-03-19T02:06:05.332 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 62421, 2024-03-19T02:06:05.333 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 197.975786895, 2024-03-19T02:06:05.333 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.003171621 2024-03-19T02:06:05.333 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.333 INFO:teuthology.orchestra.run.smithi115.stdout: "req_getattr_latency": { 2024-03-19T02:06:05.333 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 41842, 2024-03-19T02:06:05.333 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 264.095787930, 2024-03-19T02:06:05.333 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.006311739 2024-03-19T02:06:05.333 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.333 INFO:teuthology.orchestra.run.smithi115.stdout: "req_getfilelock_latency": { 2024-03-19T02:06:05.333 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:05.333 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:05.333 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:05.333 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.333 INFO:teuthology.orchestra.run.smithi115.stdout: "req_link_latency": { 2024-03-19T02:06:05.333 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:05.334 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:05.334 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:05.334 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.334 INFO:teuthology.orchestra.run.smithi115.stdout: "req_lookup_latency": { 2024-03-19T02:06:05.334 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 119839, 2024-03-19T02:06:05.334 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 83.700967496, 2024-03-19T02:06:05.334 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000698445 2024-03-19T02:06:05.334 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.334 INFO:teuthology.orchestra.run.smithi115.stdout: "req_lookuphash_latency": { 2024-03-19T02:06:05.334 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:05.334 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:05.334 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:05.334 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.334 INFO:teuthology.orchestra.run.smithi115.stdout: "req_lookupino_latency": { 2024-03-19T02:06:05.335 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 1, 2024-03-19T02:06:05.335 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000093367, 2024-03-19T02:06:05.335 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000093367 2024-03-19T02:06:05.335 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.335 INFO:teuthology.orchestra.run.smithi115.stdout: "req_lookupname_latency": { 2024-03-19T02:06:05.335 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:05.335 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:05.335 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:05.335 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.335 INFO:teuthology.orchestra.run.smithi115.stdout: "req_lookupparent_latency": { 2024-03-19T02:06:05.335 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:05.335 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:05.335 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:05.335 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.335 INFO:teuthology.orchestra.run.smithi115.stdout: "req_lookupsnap_latency": { 2024-03-19T02:06:05.335 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:05.336 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:05.336 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:05.336 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.336 INFO:teuthology.orchestra.run.smithi115.stdout: "req_lssnap_latency": { 2024-03-19T02:06:05.336 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:05.336 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:05.336 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:05.337 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.337 INFO:teuthology.orchestra.run.smithi115.stdout: "req_mkdir_latency": { 2024-03-19T02:06:05.337 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:05.337 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:05.337 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:05.337 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.337 INFO:teuthology.orchestra.run.smithi115.stdout: "req_mknod_latency": { 2024-03-19T02:06:05.337 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:05.337 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:05.337 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:05.337 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.337 INFO:teuthology.orchestra.run.smithi115.stdout: "req_mksnap_latency": { 2024-03-19T02:06:05.337 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:05.337 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:05.338 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:05.338 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.338 INFO:teuthology.orchestra.run.smithi115.stdout: "req_open_latency": { 2024-03-19T02:06:05.338 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:05.338 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:05.338 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:05.338 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.338 INFO:teuthology.orchestra.run.smithi115.stdout: "req_readdir_latency": { 2024-03-19T02:06:05.338 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 87580, 2024-03-19T02:06:05.338 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 239.151534835, 2024-03-19T02:06:05.338 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.002730663 2024-03-19T02:06:05.338 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.338 INFO:teuthology.orchestra.run.smithi115.stdout: "req_rename_latency": { 2024-03-19T02:06:05.338 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 13889, 2024-03-19T02:06:05.339 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 48.216705994, 2024-03-19T02:06:05.339 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.003471575 2024-03-19T02:06:05.339 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.339 INFO:teuthology.orchestra.run.smithi115.stdout: "req_renamesnap_latency": { 2024-03-19T02:06:05.339 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:05.339 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:05.339 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:05.339 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.339 INFO:teuthology.orchestra.run.smithi115.stdout: "req_snapdiff_latency": { 2024-03-19T02:06:05.339 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:05.339 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:05.339 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:05.339 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.339 INFO:teuthology.orchestra.run.smithi115.stdout: "req_rmdir_latency": { 2024-03-19T02:06:05.339 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 10, 2024-03-19T02:06:05.340 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.116200687, 2024-03-19T02:06:05.340 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.011620068 2024-03-19T02:06:05.340 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.340 INFO:teuthology.orchestra.run.smithi115.stdout: "req_rmsnap_latency": { 2024-03-19T02:06:05.340 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:05.340 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:05.340 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:05.340 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.340 INFO:teuthology.orchestra.run.smithi115.stdout: "req_rmxattr_latency": { 2024-03-19T02:06:05.340 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:05.340 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:05.340 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:05.340 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.340 INFO:teuthology.orchestra.run.smithi115.stdout: "req_setattr_latency": { 2024-03-19T02:06:05.341 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 8, 2024-03-19T02:06:05.341 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.115038674, 2024-03-19T02:06:05.341 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.014379834 2024-03-19T02:06:05.341 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.341 INFO:teuthology.orchestra.run.smithi115.stdout: "req_setdirlayout_latency": { 2024-03-19T02:06:05.341 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:05.341 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:05.341 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:05.341 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.341 INFO:teuthology.orchestra.run.smithi115.stdout: "req_setfilelock_latency": { 2024-03-19T02:06:05.341 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 2196, 2024-03-19T02:06:05.341 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 1.724382547, 2024-03-19T02:06:05.341 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000785237 2024-03-19T02:06:05.341 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.341 INFO:teuthology.orchestra.run.smithi115.stdout: "req_setlayout_latency": { 2024-03-19T02:06:05.342 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:05.342 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:05.342 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:05.342 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.342 INFO:teuthology.orchestra.run.smithi115.stdout: "req_setxattr_latency": { 2024-03-19T02:06:05.342 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:05.342 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:05.342 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:05.342 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.342 INFO:teuthology.orchestra.run.smithi115.stdout: "req_symlink_latency": { 2024-03-19T02:06:05.342 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:05.342 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:05.342 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:05.342 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.343 INFO:teuthology.orchestra.run.smithi115.stdout: "req_unlink_latency": { 2024-03-19T02:06:05.343 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 62482, 2024-03-19T02:06:05.343 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 134.057541823, 2024-03-19T02:06:05.343 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.002145538 2024-03-19T02:06:05.343 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.343 INFO:teuthology.orchestra.run.smithi115.stdout: "cap_revoke_eviction": 0, 2024-03-19T02:06:05.343 INFO:teuthology.orchestra.run.smithi115.stdout: "cap_acquisition_throttle": 0, 2024-03-19T02:06:05.343 INFO:teuthology.orchestra.run.smithi115.stdout: "req_getvxattr_latency": { 2024-03-19T02:06:05.343 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:05.343 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:05.343 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:05.343 INFO:teuthology.orchestra.run.smithi115.stdout: } 2024-03-19T02:06:05.343 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.343 INFO:teuthology.orchestra.run.smithi115.stdout: "mds_sessions": { 2024-03-19T02:06:05.344 INFO:teuthology.orchestra.run.smithi115.stdout: "session_count": 1, 2024-03-19T02:06:05.344 INFO:teuthology.orchestra.run.smithi115.stdout: "session_add": 2, 2024-03-19T02:06:05.344 INFO:teuthology.orchestra.run.smithi115.stdout: "session_remove": 1, 2024-03-19T02:06:05.344 INFO:teuthology.orchestra.run.smithi115.stdout: "sessions_open": 1, 2024-03-19T02:06:05.344 INFO:teuthology.orchestra.run.smithi115.stdout: "sessions_stale": 0, 2024-03-19T02:06:05.344 INFO:teuthology.orchestra.run.smithi115.stdout: "total_load": 11262, 2024-03-19T02:06:05.344 INFO:teuthology.orchestra.run.smithi115.stdout: "average_load": 11262, 2024-03-19T02:06:05.344 INFO:teuthology.orchestra.run.smithi115.stdout: "avg_session_uptime": 1514, 2024-03-19T02:06:05.344 INFO:teuthology.orchestra.run.smithi115.stdout: "mdthresh_evicted": 0 2024-03-19T02:06:05.344 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.344 INFO:teuthology.orchestra.run.smithi115.stdout: "mempool": { 2024-03-19T02:06:05.344 INFO:teuthology.orchestra.run.smithi115.stdout: "bloom_filter_bytes": 0, 2024-03-19T02:06:05.344 INFO:teuthology.orchestra.run.smithi115.stdout: "bloom_filter_items": 0, 2024-03-19T02:06:05.344 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_alloc_bytes": 0, 2024-03-19T02:06:05.344 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_alloc_items": 0, 2024-03-19T02:06:05.345 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_cache_data_bytes": 0, 2024-03-19T02:06:05.345 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_cache_data_items": 0, 2024-03-19T02:06:05.345 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_cache_onode_bytes": 0, 2024-03-19T02:06:05.345 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_cache_onode_items": 0, 2024-03-19T02:06:05.345 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_cache_meta_bytes": 0, 2024-03-19T02:06:05.345 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_cache_meta_items": 0, 2024-03-19T02:06:05.345 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_cache_other_bytes": 0, 2024-03-19T02:06:05.345 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_cache_other_items": 0, 2024-03-19T02:06:05.345 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_cache_buffer_bytes": 0, 2024-03-19T02:06:05.345 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_cache_buffer_items": 0, 2024-03-19T02:06:05.345 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_extent_bytes": 0, 2024-03-19T02:06:05.345 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_extent_items": 0, 2024-03-19T02:06:05.345 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_blob_bytes": 0, 2024-03-19T02:06:05.345 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_blob_items": 0, 2024-03-19T02:06:05.346 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_shared_blob_bytes": 0, 2024-03-19T02:06:05.346 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_shared_blob_items": 0, 2024-03-19T02:06:05.346 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_inline_bl_bytes": 0, 2024-03-19T02:06:05.346 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_inline_bl_items": 0, 2024-03-19T02:06:05.346 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_fsck_bytes": 0, 2024-03-19T02:06:05.346 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_fsck_items": 0, 2024-03-19T02:06:05.346 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_txc_bytes": 0, 2024-03-19T02:06:05.346 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_txc_items": 0, 2024-03-19T02:06:05.346 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_writing_deferred_bytes": 0, 2024-03-19T02:06:05.346 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_writing_deferred_items": 0, 2024-03-19T02:06:05.346 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_writing_bytes": 0, 2024-03-19T02:06:05.346 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_writing_items": 0, 2024-03-19T02:06:05.346 INFO:teuthology.orchestra.run.smithi115.stdout: "bluefs_bytes": 0, 2024-03-19T02:06:05.346 INFO:teuthology.orchestra.run.smithi115.stdout: "bluefs_items": 0, 2024-03-19T02:06:05.347 INFO:teuthology.orchestra.run.smithi115.stdout: "bluefs_file_reader_bytes": 0, 2024-03-19T02:06:05.347 INFO:teuthology.orchestra.run.smithi115.stdout: "bluefs_file_reader_items": 0, 2024-03-19T02:06:05.347 INFO:teuthology.orchestra.run.smithi115.stdout: "bluefs_file_writer_bytes": 0, 2024-03-19T02:06:05.347 INFO:teuthology.orchestra.run.smithi115.stdout: "bluefs_file_writer_items": 0, 2024-03-19T02:06:05.347 INFO:teuthology.orchestra.run.smithi115.stdout: "buffer_anon_bytes": 1076677, 2024-03-19T02:06:05.347 INFO:teuthology.orchestra.run.smithi115.stdout: "buffer_anon_items": 383, 2024-03-19T02:06:05.347 INFO:teuthology.orchestra.run.smithi115.stdout: "buffer_meta_bytes": 0, 2024-03-19T02:06:05.347 INFO:teuthology.orchestra.run.smithi115.stdout: "buffer_meta_items": 0, 2024-03-19T02:06:05.347 INFO:teuthology.orchestra.run.smithi115.stdout: "osd_bytes": 0, 2024-03-19T02:06:05.347 INFO:teuthology.orchestra.run.smithi115.stdout: "osd_items": 0, 2024-03-19T02:06:05.347 INFO:teuthology.orchestra.run.smithi115.stdout: "osd_mapbl_bytes": 0, 2024-03-19T02:06:05.347 INFO:teuthology.orchestra.run.smithi115.stdout: "osd_mapbl_items": 0, 2024-03-19T02:06:05.347 INFO:teuthology.orchestra.run.smithi115.stdout: "osd_pglog_bytes": 0, 2024-03-19T02:06:05.347 INFO:teuthology.orchestra.run.smithi115.stdout: "osd_pglog_items": 0, 2024-03-19T02:06:05.347 INFO:teuthology.orchestra.run.smithi115.stdout: "osdmap_bytes": 7816, 2024-03-19T02:06:05.348 INFO:teuthology.orchestra.run.smithi115.stdout: "osdmap_items": 164, 2024-03-19T02:06:05.348 INFO:teuthology.orchestra.run.smithi115.stdout: "osdmap_mapping_bytes": 0, 2024-03-19T02:06:05.348 INFO:teuthology.orchestra.run.smithi115.stdout: "osdmap_mapping_items": 0, 2024-03-19T02:06:05.348 INFO:teuthology.orchestra.run.smithi115.stdout: "pgmap_bytes": 0, 2024-03-19T02:06:05.348 INFO:teuthology.orchestra.run.smithi115.stdout: "pgmap_items": 0, 2024-03-19T02:06:05.348 INFO:teuthology.orchestra.run.smithi115.stdout: "mds_co_bytes": 487150, 2024-03-19T02:06:05.348 INFO:teuthology.orchestra.run.smithi115.stdout: "mds_co_items": 17991, 2024-03-19T02:06:05.348 INFO:teuthology.orchestra.run.smithi115.stdout: "unittest_1_bytes": 0, 2024-03-19T02:06:05.348 INFO:teuthology.orchestra.run.smithi115.stdout: "unittest_1_items": 0, 2024-03-19T02:06:05.348 INFO:teuthology.orchestra.run.smithi115.stdout: "unittest_2_bytes": 0, 2024-03-19T02:06:05.348 INFO:teuthology.orchestra.run.smithi115.stdout: "unittest_2_items": 0 2024-03-19T02:06:05.348 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.348 INFO:teuthology.orchestra.run.smithi115.stdout: "objecter": { 2024-03-19T02:06:05.348 INFO:teuthology.orchestra.run.smithi115.stdout: "op_active": 0, 2024-03-19T02:06:05.348 INFO:teuthology.orchestra.run.smithi115.stdout: "op_laggy": 0, 2024-03-19T02:06:05.349 INFO:teuthology.orchestra.run.smithi115.stdout: "op_send": 176036, 2024-03-19T02:06:05.349 INFO:teuthology.orchestra.run.smithi115.stdout: "op_send_bytes": 599584795, 2024-03-19T02:06:05.349 INFO:teuthology.orchestra.run.smithi115.stdout: "op_resend": 33, 2024-03-19T02:06:05.349 INFO:teuthology.orchestra.run.smithi115.stdout: "op_reply": 176036, 2024-03-19T02:06:05.349 INFO:teuthology.orchestra.run.smithi115.stdout: "op_latency": { 2024-03-19T02:06:05.349 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 176036, 2024-03-19T02:06:05.349 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 706.839127449, 2024-03-19T02:06:05.349 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.004015310 2024-03-19T02:06:05.349 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.349 INFO:teuthology.orchestra.run.smithi115.stdout: "op_inflight": 0, 2024-03-19T02:06:05.349 INFO:teuthology.orchestra.run.smithi115.stdout: "oplen_avg": { 2024-03-19T02:06:05.349 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 176036, 2024-03-19T02:06:05.349 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 178399 2024-03-19T02:06:05.349 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.349 INFO:teuthology.orchestra.run.smithi115.stdout: "op": 176036, 2024-03-19T02:06:05.349 INFO:teuthology.orchestra.run.smithi115.stdout: "op_r": 1703, 2024-03-19T02:06:05.350 INFO:teuthology.orchestra.run.smithi115.stdout: "op_w": 174333, 2024-03-19T02:06:05.350 INFO:teuthology.orchestra.run.smithi115.stdout: "op_rmw": 0, 2024-03-19T02:06:05.350 INFO:teuthology.orchestra.run.smithi115.stdout: "op_pg": 0, 2024-03-19T02:06:05.350 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_stat": 1, 2024-03-19T02:06:05.350 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_create": 31, 2024-03-19T02:06:05.350 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_read": 1703, 2024-03-19T02:06:05.350 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_write": 109329, 2024-03-19T02:06:05.350 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_writefull": 791, 2024-03-19T02:06:05.350 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_writesame": 0, 2024-03-19T02:06:05.350 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_append": 0, 2024-03-19T02:06:05.350 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_zero": 0, 2024-03-19T02:06:05.350 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_truncate": 0, 2024-03-19T02:06:05.350 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_delete": 62728, 2024-03-19T02:06:05.350 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_mapext": 0, 2024-03-19T02:06:05.350 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_sparse_read": 0, 2024-03-19T02:06:05.351 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_clonerange": 0, 2024-03-19T02:06:05.351 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_getxattr": 0, 2024-03-19T02:06:05.351 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_setxattr": 62, 2024-03-19T02:06:05.351 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_cmpxattr": 0, 2024-03-19T02:06:05.351 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_rmxattr": 0, 2024-03-19T02:06:05.351 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_resetxattrs": 0, 2024-03-19T02:06:05.351 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_call": 0, 2024-03-19T02:06:05.351 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_watch": 0, 2024-03-19T02:06:05.351 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_notify": 0, 2024-03-19T02:06:05.351 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_src_cmpxattr": 0, 2024-03-19T02:06:05.351 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_pgls": 0, 2024-03-19T02:06:05.351 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_pgls_filter": 0, 2024-03-19T02:06:05.351 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_other": 0, 2024-03-19T02:06:05.351 INFO:teuthology.orchestra.run.smithi115.stdout: "linger_active": 0, 2024-03-19T02:06:05.352 INFO:teuthology.orchestra.run.smithi115.stdout: "linger_send": 0, 2024-03-19T02:06:05.352 INFO:teuthology.orchestra.run.smithi115.stdout: "linger_resend": 0, 2024-03-19T02:06:05.352 INFO:teuthology.orchestra.run.smithi115.stdout: "linger_ping": 0, 2024-03-19T02:06:05.352 INFO:teuthology.orchestra.run.smithi115.stdout: "poolop_active": 0, 2024-03-19T02:06:05.352 INFO:teuthology.orchestra.run.smithi115.stdout: "poolop_send": 0, 2024-03-19T02:06:05.352 INFO:teuthology.orchestra.run.smithi115.stdout: "poolop_resend": 0, 2024-03-19T02:06:05.352 INFO:teuthology.orchestra.run.smithi115.stdout: "poolstat_active": 0, 2024-03-19T02:06:05.352 INFO:teuthology.orchestra.run.smithi115.stdout: "poolstat_send": 0, 2024-03-19T02:06:05.352 INFO:teuthology.orchestra.run.smithi115.stdout: "poolstat_resend": 0, 2024-03-19T02:06:05.352 INFO:teuthology.orchestra.run.smithi115.stdout: "statfs_active": 0, 2024-03-19T02:06:05.352 INFO:teuthology.orchestra.run.smithi115.stdout: "statfs_send": 0, 2024-03-19T02:06:05.352 INFO:teuthology.orchestra.run.smithi115.stdout: "statfs_resend": 0, 2024-03-19T02:06:05.352 INFO:teuthology.orchestra.run.smithi115.stdout: "command_active": 0, 2024-03-19T02:06:05.352 INFO:teuthology.orchestra.run.smithi115.stdout: "command_send": 0, 2024-03-19T02:06:05.352 INFO:teuthology.orchestra.run.smithi115.stdout: "command_resend": 0, 2024-03-19T02:06:05.353 INFO:teuthology.orchestra.run.smithi115.stdout: "map_epoch": 92, 2024-03-19T02:06:05.353 INFO:teuthology.orchestra.run.smithi115.stdout: "map_full": 0, 2024-03-19T02:06:05.353 INFO:teuthology.orchestra.run.smithi115.stdout: "map_inc": 5, 2024-03-19T02:06:05.353 INFO:teuthology.orchestra.run.smithi115.stdout: "osd_sessions": 12, 2024-03-19T02:06:05.353 INFO:teuthology.orchestra.run.smithi115.stdout: "osd_session_open": 12, 2024-03-19T02:06:05.353 INFO:teuthology.orchestra.run.smithi115.stdout: "osd_session_close": 0, 2024-03-19T02:06:05.353 INFO:teuthology.orchestra.run.smithi115.stdout: "osd_laggy": 0, 2024-03-19T02:06:05.353 INFO:teuthology.orchestra.run.smithi115.stdout: "omap_wr": 2869, 2024-03-19T02:06:05.353 INFO:teuthology.orchestra.run.smithi115.stdout: "omap_rd": 0, 2024-03-19T02:06:05.353 INFO:teuthology.orchestra.run.smithi115.stdout: "omap_del": 885 2024-03-19T02:06:05.353 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.353 INFO:teuthology.orchestra.run.smithi115.stdout: "oft": { 2024-03-19T02:06:05.353 INFO:teuthology.orchestra.run.smithi115.stdout: "omap_total_objs": 1, 2024-03-19T02:06:05.353 INFO:teuthology.orchestra.run.smithi115.stdout: "omap_total_kv_pairs": 0, 2024-03-19T02:06:05.353 INFO:teuthology.orchestra.run.smithi115.stdout: "omap_total_updates": 0, 2024-03-19T02:06:05.354 INFO:teuthology.orchestra.run.smithi115.stdout: "omap_total_removes": 0 2024-03-19T02:06:05.354 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.354 INFO:teuthology.orchestra.run.smithi115.stdout: "osd-slow-ops": { 2024-03-19T02:06:05.354 INFO:teuthology.orchestra.run.smithi115.stdout: "slow_ops_count": 0 2024-03-19T02:06:05.354 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.354 INFO:teuthology.orchestra.run.smithi115.stdout: "purge_queue": { 2024-03-19T02:06:05.354 INFO:teuthology.orchestra.run.smithi115.stdout: "pq_executing_ops": 0, 2024-03-19T02:06:05.354 INFO:teuthology.orchestra.run.smithi115.stdout: "pq_executing_ops_high_water": 10, 2024-03-19T02:06:05.354 INFO:teuthology.orchestra.run.smithi115.stdout: "pq_executing": 0, 2024-03-19T02:06:05.354 INFO:teuthology.orchestra.run.smithi115.stdout: "pq_executing_high_water": 10, 2024-03-19T02:06:05.354 INFO:teuthology.orchestra.run.smithi115.stdout: "pq_executed_ops": 62483, 2024-03-19T02:06:05.354 INFO:teuthology.orchestra.run.smithi115.stdout: "pq_executed": 62483, 2024-03-19T02:06:05.354 INFO:teuthology.orchestra.run.smithi115.stdout: "pq_item_in_journal": 0 2024-03-19T02:06:05.354 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.355 INFO:teuthology.orchestra.run.smithi115.stdout: "throttle-msgr_dispatch_throttler-mds": { 2024-03-19T02:06:05.355 INFO:teuthology.orchestra.run.smithi115.stdout: "val": 0, 2024-03-19T02:06:05.355 INFO:teuthology.orchestra.run.smithi115.stdout: "max": 104857600, 2024-03-19T02:06:05.355 INFO:teuthology.orchestra.run.smithi115.stdout: "get_started": 0, 2024-03-19T02:06:05.355 INFO:teuthology.orchestra.run.smithi115.stdout: "get": 1440338, 2024-03-19T02:06:05.355 INFO:teuthology.orchestra.run.smithi115.stdout: "get_sum": 257621170, 2024-03-19T02:06:05.355 INFO:teuthology.orchestra.run.smithi115.stdout: "get_or_fail_fail": 0, 2024-03-19T02:06:05.355 INFO:teuthology.orchestra.run.smithi115.stdout: "get_or_fail_success": 1440338, 2024-03-19T02:06:05.355 INFO:teuthology.orchestra.run.smithi115.stdout: "take": 0, 2024-03-19T02:06:05.355 INFO:teuthology.orchestra.run.smithi115.stdout: "take_sum": 0, 2024-03-19T02:06:05.355 INFO:teuthology.orchestra.run.smithi115.stdout: "put": 1440338, 2024-03-19T02:06:05.355 INFO:teuthology.orchestra.run.smithi115.stdout: "put_sum": 257621170, 2024-03-19T02:06:05.355 INFO:teuthology.orchestra.run.smithi115.stdout: "wait": { 2024-03-19T02:06:05.355 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:05.355 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:05.356 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:05.356 INFO:teuthology.orchestra.run.smithi115.stdout: } 2024-03-19T02:06:05.356 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.356 INFO:teuthology.orchestra.run.smithi115.stdout: "throttle-objecter_bytes": { 2024-03-19T02:06:05.356 INFO:teuthology.orchestra.run.smithi115.stdout: "val": 0, 2024-03-19T02:06:05.356 INFO:teuthology.orchestra.run.smithi115.stdout: "max": 104857600, 2024-03-19T02:06:05.356 INFO:teuthology.orchestra.run.smithi115.stdout: "get_started": 0, 2024-03-19T02:06:05.356 INFO:teuthology.orchestra.run.smithi115.stdout: "get": 0, 2024-03-19T02:06:05.356 INFO:teuthology.orchestra.run.smithi115.stdout: "get_sum": 0, 2024-03-19T02:06:05.356 INFO:teuthology.orchestra.run.smithi115.stdout: "get_or_fail_fail": 0, 2024-03-19T02:06:05.356 INFO:teuthology.orchestra.run.smithi115.stdout: "get_or_fail_success": 0, 2024-03-19T02:06:05.356 INFO:teuthology.orchestra.run.smithi115.stdout: "take": 176036, 2024-03-19T02:06:05.356 INFO:teuthology.orchestra.run.smithi115.stdout: "take_sum": 605895578, 2024-03-19T02:06:05.356 INFO:teuthology.orchestra.run.smithi115.stdout: "put": 176036, 2024-03-19T02:06:05.356 INFO:teuthology.orchestra.run.smithi115.stdout: "put_sum": 605895578, 2024-03-19T02:06:05.356 INFO:teuthology.orchestra.run.smithi115.stdout: "wait": { 2024-03-19T02:06:05.357 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:05.357 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:05.357 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:05.357 INFO:teuthology.orchestra.run.smithi115.stdout: } 2024-03-19T02:06:05.357 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.357 INFO:teuthology.orchestra.run.smithi115.stdout: "throttle-objecter_ops": { 2024-03-19T02:06:05.357 INFO:teuthology.orchestra.run.smithi115.stdout: "val": 0, 2024-03-19T02:06:05.357 INFO:teuthology.orchestra.run.smithi115.stdout: "max": 1024, 2024-03-19T02:06:05.357 INFO:teuthology.orchestra.run.smithi115.stdout: "get_started": 0, 2024-03-19T02:06:05.357 INFO:teuthology.orchestra.run.smithi115.stdout: "get": 0, 2024-03-19T02:06:05.357 INFO:teuthology.orchestra.run.smithi115.stdout: "get_sum": 0, 2024-03-19T02:06:05.357 INFO:teuthology.orchestra.run.smithi115.stdout: "get_or_fail_fail": 0, 2024-03-19T02:06:05.357 INFO:teuthology.orchestra.run.smithi115.stdout: "get_or_fail_success": 0, 2024-03-19T02:06:05.357 INFO:teuthology.orchestra.run.smithi115.stdout: "take": 176036, 2024-03-19T02:06:05.357 INFO:teuthology.orchestra.run.smithi115.stdout: "take_sum": 176036, 2024-03-19T02:06:05.357 INFO:teuthology.orchestra.run.smithi115.stdout: "put": 176036, 2024-03-19T02:06:05.358 INFO:teuthology.orchestra.run.smithi115.stdout: "put_sum": 176036, 2024-03-19T02:06:05.358 INFO:teuthology.orchestra.run.smithi115.stdout: "wait": { 2024-03-19T02:06:05.358 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:05.358 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:05.358 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:05.358 INFO:teuthology.orchestra.run.smithi115.stdout: } 2024-03-19T02:06:05.358 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.358 INFO:teuthology.orchestra.run.smithi115.stdout: "throttle-write_buf_throttle": { 2024-03-19T02:06:05.358 INFO:teuthology.orchestra.run.smithi115.stdout: "val": 0, 2024-03-19T02:06:05.358 INFO:teuthology.orchestra.run.smithi115.stdout: "max": 3758096384, 2024-03-19T02:06:05.358 INFO:teuthology.orchestra.run.smithi115.stdout: "get_started": 0, 2024-03-19T02:06:05.358 INFO:teuthology.orchestra.run.smithi115.stdout: "get": 62483, 2024-03-19T02:06:05.358 INFO:teuthology.orchestra.run.smithi115.stdout: "get_sum": 6310783, 2024-03-19T02:06:05.358 INFO:teuthology.orchestra.run.smithi115.stdout: "get_or_fail_fail": 0, 2024-03-19T02:06:05.358 INFO:teuthology.orchestra.run.smithi115.stdout: "get_or_fail_success": 62483, 2024-03-19T02:06:05.359 INFO:teuthology.orchestra.run.smithi115.stdout: "take": 0, 2024-03-19T02:06:05.359 INFO:teuthology.orchestra.run.smithi115.stdout: "take_sum": 0, 2024-03-19T02:06:05.359 INFO:teuthology.orchestra.run.smithi115.stdout: "put": 1703, 2024-03-19T02:06:05.359 INFO:teuthology.orchestra.run.smithi115.stdout: "put_sum": 6310783, 2024-03-19T02:06:05.359 INFO:teuthology.orchestra.run.smithi115.stdout: "wait": { 2024-03-19T02:06:05.359 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:05.359 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:05.359 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:05.359 INFO:teuthology.orchestra.run.smithi115.stdout: } 2024-03-19T02:06:05.359 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:05.359 INFO:teuthology.orchestra.run.smithi115.stdout: "throttle-write_buf_throttle-0x556d9605a500": { 2024-03-19T02:06:05.359 INFO:teuthology.orchestra.run.smithi115.stdout: "val": 27554, 2024-03-19T02:06:05.359 INFO:teuthology.orchestra.run.smithi115.stdout: "max": 3758096384, 2024-03-19T02:06:05.359 INFO:teuthology.orchestra.run.smithi115.stdout: "get_started": 0, 2024-03-19T02:06:05.359 INFO:teuthology.orchestra.run.smithi115.stdout: "get": 290706, 2024-03-19T02:06:05.359 INFO:teuthology.orchestra.run.smithi115.stdout: "get_sum": 587419571, 2024-03-19T02:06:05.359 INFO:teuthology.orchestra.run.smithi115.stdout: "get_or_fail_fail": 0, 2024-03-19T02:06:05.360 INFO:teuthology.orchestra.run.smithi115.stdout: "get_or_fail_success": 290706, 2024-03-19T02:06:05.360 INFO:teuthology.orchestra.run.smithi115.stdout: "take": 0, 2024-03-19T02:06:05.360 INFO:teuthology.orchestra.run.smithi115.stdout: "take_sum": 0, 2024-03-19T02:06:05.360 INFO:teuthology.orchestra.run.smithi115.stdout: "put": 107626, 2024-03-19T02:06:05.360 INFO:teuthology.orchestra.run.smithi115.stdout: "put_sum": 587392017, 2024-03-19T02:06:05.360 INFO:teuthology.orchestra.run.smithi115.stdout: "wait": { 2024-03-19T02:06:05.360 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:05.360 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:05.360 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:05.360 INFO:teuthology.orchestra.run.smithi115.stdout: } 2024-03-19T02:06:05.360 INFO:teuthology.orchestra.run.smithi115.stdout: } 2024-03-19T02:06:05.360 INFO:teuthology.orchestra.run.smithi115.stdout:} 2024-03-19T02:06:05.361 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:06:04 smithi099 ceph-mon[62951]: pgmap v1100: 97 pgs: 97 active+clean; 453 MiB data, 9.0 GiB used, 1.0 TiB / 1.0 TiB avail; 180 KiB/s rd, 3.7 MiB/s wr, 131 op/s 2024-03-19T02:06:05.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:06:04 smithi112 ceph-mon[66819]: pgmap v1100: 97 pgs: 97 active+clean; 453 MiB data, 9.0 GiB used, 1.0 TiB / 1.0 TiB avail; 180 KiB/s rd, 3.7 MiB/s wr, 131 op/s 2024-03-19T02:06:06.374 INFO:tasks.check_counter:Daemon mds.c mds.exported=5 2024-03-19T02:06:06.375 INFO:tasks.check_counter:Daemon mds.c mds.imported=14 2024-03-19T02:06:06.375 INFO:tasks.check_counter:Daemon mds.c mds_cache.dir_update=296 2024-03-19T02:06:06.375 INFO:tasks.check_counter:Daemon mds.c mds_cache.dir_update_receipt=795 2024-03-19T02:06:06.375 DEBUG:tasks.check_counter:Getting stats from f 2024-03-19T02:06:06.375 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph daemon mds.f perf dump 2024-03-19T02:06:06.628 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.c/config 2024-03-19T02:06:07.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:06:06 smithi115 ceph-mon[67332]: pgmap v1101: 97 pgs: 97 active+clean; 451 MiB data, 9.0 GiB used, 1.0 TiB / 1.0 TiB avail; 133 KiB/s rd, 3.1 MiB/s wr, 110 op/s 2024-03-19T02:06:07.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:06:06 smithi099 ceph-mon[62951]: pgmap v1101: 97 pgs: 97 active+clean; 451 MiB data, 9.0 GiB used, 1.0 TiB / 1.0 TiB avail; 133 KiB/s rd, 3.1 MiB/s wr, 110 op/s 2024-03-19T02:06:07.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:06:06 smithi112 ceph-mon[66819]: pgmap v1101: 97 pgs: 97 active+clean; 451 MiB data, 9.0 GiB used, 1.0 TiB / 1.0 TiB avail; 133 KiB/s rd, 3.1 MiB/s wr, 110 op/s 2024-03-19T02:06:08.869 INFO:teuthology.orchestra.run.smithi115.stdout:{ 2024-03-19T02:06:08.870 INFO:teuthology.orchestra.run.smithi115.stdout: "AsyncMessenger::Worker-0": { 2024-03-19T02:06:08.870 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_recv_messages": 955, 2024-03-19T02:06:08.870 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_send_messages": 957, 2024-03-19T02:06:08.870 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_recv_bytes": 218349029, 2024-03-19T02:06:08.870 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_send_bytes": 303705, 2024-03-19T02:06:08.870 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_created_connections": 8, 2024-03-19T02:06:08.870 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_active_connections": 4, 2024-03-19T02:06:08.870 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_running_total_time": 0.188602710, 2024-03-19T02:06:08.870 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_running_send_time": 0.020504717, 2024-03-19T02:06:08.870 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_running_recv_time": 0.137239141, 2024-03-19T02:06:08.870 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_running_fast_dispatch_time": 0.024069304, 2024-03-19T02:06:08.870 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_send_messages_queue_lat": { 2024-03-19T02:06:08.871 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 955, 2024-03-19T02:06:08.871 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.037831797, 2024-03-19T02:06:08.871 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000039614 2024-03-19T02:06:08.871 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.871 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_handle_ack_lat": { 2024-03-19T02:06:08.871 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:08.871 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:08.871 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:08.871 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.871 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_recv_encrypted_bytes": 0, 2024-03-19T02:06:08.871 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_send_encrypted_bytes": 0 2024-03-19T02:06:08.871 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.871 INFO:teuthology.orchestra.run.smithi115.stdout: "AsyncMessenger::Worker-1": { 2024-03-19T02:06:08.871 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_recv_messages": 1738, 2024-03-19T02:06:08.871 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_send_messages": 1717, 2024-03-19T02:06:08.872 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_recv_bytes": 160129441, 2024-03-19T02:06:08.872 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_send_bytes": 610265, 2024-03-19T02:06:08.872 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_created_connections": 6, 2024-03-19T02:06:08.872 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_active_connections": 5, 2024-03-19T02:06:08.872 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_running_total_time": 0.230079773, 2024-03-19T02:06:08.872 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_running_send_time": 0.037175207, 2024-03-19T02:06:08.872 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_running_recv_time": 0.143192038, 2024-03-19T02:06:08.872 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_running_fast_dispatch_time": 0.043482908, 2024-03-19T02:06:08.872 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_send_messages_queue_lat": { 2024-03-19T02:06:08.872 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 1717, 2024-03-19T02:06:08.872 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.068194613, 2024-03-19T02:06:08.872 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000039717 2024-03-19T02:06:08.872 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.872 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_handle_ack_lat": { 2024-03-19T02:06:08.873 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:08.873 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:08.873 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:08.873 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.873 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_recv_encrypted_bytes": 300912, 2024-03-19T02:06:08.873 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_send_encrypted_bytes": 136384 2024-03-19T02:06:08.873 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.873 INFO:teuthology.orchestra.run.smithi115.stdout: "AsyncMessenger::Worker-2": { 2024-03-19T02:06:08.873 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_recv_messages": 1133, 2024-03-19T02:06:08.873 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_send_messages": 1299, 2024-03-19T02:06:08.873 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_recv_bytes": 222619362, 2024-03-19T02:06:08.873 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_send_bytes": 741465, 2024-03-19T02:06:08.873 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_created_connections": 6, 2024-03-19T02:06:08.873 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_active_connections": 5, 2024-03-19T02:06:08.874 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_running_total_time": 0.211064617, 2024-03-19T02:06:08.874 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_running_send_time": 0.028092691, 2024-03-19T02:06:08.874 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_running_recv_time": 0.147896657, 2024-03-19T02:06:08.874 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_running_fast_dispatch_time": 0.027471247, 2024-03-19T02:06:08.874 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_send_messages_queue_lat": { 2024-03-19T02:06:08.874 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 1297, 2024-03-19T02:06:08.874 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.043355862, 2024-03-19T02:06:08.874 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000033427 2024-03-19T02:06:08.874 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.874 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_handle_ack_lat": { 2024-03-19T02:06:08.874 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:08.874 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:08.874 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:08.874 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.875 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_recv_encrypted_bytes": 160, 2024-03-19T02:06:08.875 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_send_encrypted_bytes": 381488 2024-03-19T02:06:08.875 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.875 INFO:teuthology.orchestra.run.smithi115.stdout: "cct": { 2024-03-19T02:06:08.875 INFO:teuthology.orchestra.run.smithi115.stdout: "total_workers": 1, 2024-03-19T02:06:08.875 INFO:teuthology.orchestra.run.smithi115.stdout: "unhealthy_workers": 0 2024-03-19T02:06:08.875 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.875 INFO:teuthology.orchestra.run.smithi115.stdout: "finisher-MDSRank": { 2024-03-19T02:06:08.875 INFO:teuthology.orchestra.run.smithi115.stdout: "queue_len": 0, 2024-03-19T02:06:08.875 INFO:teuthology.orchestra.run.smithi115.stdout: "complete_latency": { 2024-03-19T02:06:08.875 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 5796, 2024-03-19T02:06:08.875 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.873616066, 2024-03-19T02:06:08.875 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000150727 2024-03-19T02:06:08.875 INFO:teuthology.orchestra.run.smithi115.stdout: } 2024-03-19T02:06:08.875 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.876 INFO:teuthology.orchestra.run.smithi115.stdout: "finisher-PurgeQueue": { 2024-03-19T02:06:08.876 INFO:teuthology.orchestra.run.smithi115.stdout: "queue_len": 0, 2024-03-19T02:06:08.876 INFO:teuthology.orchestra.run.smithi115.stdout: "complete_latency": { 2024-03-19T02:06:08.876 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:08.876 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:08.876 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:08.876 INFO:teuthology.orchestra.run.smithi115.stdout: } 2024-03-19T02:06:08.876 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.876 INFO:teuthology.orchestra.run.smithi115.stdout: "mds": { 2024-03-19T02:06:08.876 INFO:teuthology.orchestra.run.smithi115.stdout: "request": 0, 2024-03-19T02:06:08.876 INFO:teuthology.orchestra.run.smithi115.stdout: "reply": 0, 2024-03-19T02:06:08.876 INFO:teuthology.orchestra.run.smithi115.stdout: "reply_latency": { 2024-03-19T02:06:08.876 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:08.876 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:08.876 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:08.876 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.877 INFO:teuthology.orchestra.run.smithi115.stdout: "slow_reply": 0, 2024-03-19T02:06:08.877 INFO:teuthology.orchestra.run.smithi115.stdout: "forward": 0, 2024-03-19T02:06:08.877 INFO:teuthology.orchestra.run.smithi115.stdout: "dir_fetch_complete": 0, 2024-03-19T02:06:08.877 INFO:teuthology.orchestra.run.smithi115.stdout: "dir_fetch_keys": 0, 2024-03-19T02:06:08.877 INFO:teuthology.orchestra.run.smithi115.stdout: "dir_commit": 0, 2024-03-19T02:06:08.877 INFO:teuthology.orchestra.run.smithi115.stdout: "dir_split": 0, 2024-03-19T02:06:08.877 INFO:teuthology.orchestra.run.smithi115.stdout: "dir_merge": 0, 2024-03-19T02:06:08.877 INFO:teuthology.orchestra.run.smithi115.stdout: "inodes": 59974, 2024-03-19T02:06:08.877 INFO:teuthology.orchestra.run.smithi115.stdout: "inodes_top": 40917, 2024-03-19T02:06:08.877 INFO:teuthology.orchestra.run.smithi115.stdout: "inodes_bottom": 19057, 2024-03-19T02:06:08.877 INFO:teuthology.orchestra.run.smithi115.stdout: "inodes_pin_tail": 0, 2024-03-19T02:06:08.877 INFO:teuthology.orchestra.run.smithi115.stdout: "inodes_pinned": 2186, 2024-03-19T02:06:08.877 INFO:teuthology.orchestra.run.smithi115.stdout: "inodes_expired": 0, 2024-03-19T02:06:08.877 INFO:teuthology.orchestra.run.smithi115.stdout: "inodes_with_caps": 0, 2024-03-19T02:06:08.877 INFO:teuthology.orchestra.run.smithi115.stdout: "caps": 0, 2024-03-19T02:06:08.877 INFO:teuthology.orchestra.run.smithi115.stdout: "subtrees": 2, 2024-03-19T02:06:08.878 INFO:teuthology.orchestra.run.smithi115.stdout: "traverse": 0, 2024-03-19T02:06:08.878 INFO:teuthology.orchestra.run.smithi115.stdout: "traverse_hit": 0, 2024-03-19T02:06:08.878 INFO:teuthology.orchestra.run.smithi115.stdout: "traverse_forward": 0, 2024-03-19T02:06:08.878 INFO:teuthology.orchestra.run.smithi115.stdout: "traverse_discover": 0, 2024-03-19T02:06:08.878 INFO:teuthology.orchestra.run.smithi115.stdout: "traverse_dir_fetch": 0, 2024-03-19T02:06:08.878 INFO:teuthology.orchestra.run.smithi115.stdout: "traverse_remote_ino": 0, 2024-03-19T02:06:08.878 INFO:teuthology.orchestra.run.smithi115.stdout: "traverse_lock": 0, 2024-03-19T02:06:08.878 INFO:teuthology.orchestra.run.smithi115.stdout: "load_cent": 0, 2024-03-19T02:06:08.878 INFO:teuthology.orchestra.run.smithi115.stdout: "q": 0, 2024-03-19T02:06:08.878 INFO:teuthology.orchestra.run.smithi115.stdout: "exported": 0, 2024-03-19T02:06:08.878 INFO:teuthology.orchestra.run.smithi115.stdout: "exported_inodes": 0, 2024-03-19T02:06:08.878 INFO:teuthology.orchestra.run.smithi115.stdout: "imported": 0, 2024-03-19T02:06:08.878 INFO:teuthology.orchestra.run.smithi115.stdout: "imported_inodes": 0, 2024-03-19T02:06:08.878 INFO:teuthology.orchestra.run.smithi115.stdout: "openino_dir_fetch": 0, 2024-03-19T02:06:08.879 INFO:teuthology.orchestra.run.smithi115.stdout: "openino_backtrace_fetch": 0, 2024-03-19T02:06:08.879 INFO:teuthology.orchestra.run.smithi115.stdout: "openino_peer_discover": 0, 2024-03-19T02:06:08.879 INFO:teuthology.orchestra.run.smithi115.stdout: "root_rfiles": 4, 2024-03-19T02:06:08.879 INFO:teuthology.orchestra.run.smithi115.stdout: "root_rbytes": 234, 2024-03-19T02:06:08.879 INFO:teuthology.orchestra.run.smithi115.stdout: "root_rsnaps": 0, 2024-03-19T02:06:08.879 INFO:teuthology.orchestra.run.smithi115.stdout: "scrub_backtrace_fetch": 0, 2024-03-19T02:06:08.879 INFO:teuthology.orchestra.run.smithi115.stdout: "scrub_set_tag": 0, 2024-03-19T02:06:08.879 INFO:teuthology.orchestra.run.smithi115.stdout: "scrub_backtrace_repaired": 0, 2024-03-19T02:06:08.879 INFO:teuthology.orchestra.run.smithi115.stdout: "scrub_inotable_repaired": 0, 2024-03-19T02:06:08.879 INFO:teuthology.orchestra.run.smithi115.stdout: "scrub_dir_inodes": 0, 2024-03-19T02:06:08.879 INFO:teuthology.orchestra.run.smithi115.stdout: "scrub_dir_base_inodes": 0, 2024-03-19T02:06:08.879 INFO:teuthology.orchestra.run.smithi115.stdout: "scrub_dirfrag_rstats": 0, 2024-03-19T02:06:08.879 INFO:teuthology.orchestra.run.smithi115.stdout: "scrub_file_inodes": 0, 2024-03-19T02:06:08.879 INFO:teuthology.orchestra.run.smithi115.stdout: "handle_inode_file_caps": 0, 2024-03-19T02:06:08.879 INFO:teuthology.orchestra.run.smithi115.stdout: "ceph_cap_op_revoke": 0, 2024-03-19T02:06:08.880 INFO:teuthology.orchestra.run.smithi115.stdout: "ceph_cap_op_grant": 0, 2024-03-19T02:06:08.880 INFO:teuthology.orchestra.run.smithi115.stdout: "ceph_cap_op_trunc": 0, 2024-03-19T02:06:08.880 INFO:teuthology.orchestra.run.smithi115.stdout: "ceph_cap_op_flushsnap_ack": 0, 2024-03-19T02:06:08.880 INFO:teuthology.orchestra.run.smithi115.stdout: "ceph_cap_op_flush_ack": 0, 2024-03-19T02:06:08.880 INFO:teuthology.orchestra.run.smithi115.stdout: "handle_client_caps": 0, 2024-03-19T02:06:08.880 INFO:teuthology.orchestra.run.smithi115.stdout: "handle_client_caps_dirty": 0, 2024-03-19T02:06:08.880 INFO:teuthology.orchestra.run.smithi115.stdout: "handle_client_cap_release": 0, 2024-03-19T02:06:08.880 INFO:teuthology.orchestra.run.smithi115.stdout: "process_request_cap_release": 0 2024-03-19T02:06:08.880 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.880 INFO:teuthology.orchestra.run.smithi115.stdout: "mds_cache": { 2024-03-19T02:06:08.880 INFO:teuthology.orchestra.run.smithi115.stdout: "dir_update": 0, 2024-03-19T02:06:08.880 INFO:teuthology.orchestra.run.smithi115.stdout: "dir_update_receipt": 0, 2024-03-19T02:06:08.880 INFO:teuthology.orchestra.run.smithi115.stdout: "dir_try_discover": 0, 2024-03-19T02:06:08.880 INFO:teuthology.orchestra.run.smithi115.stdout: "dir_send_discover": 0, 2024-03-19T02:06:08.880 INFO:teuthology.orchestra.run.smithi115.stdout: "dir_handle_discover": 0, 2024-03-19T02:06:08.881 INFO:teuthology.orchestra.run.smithi115.stdout: "num_strays": 0, 2024-03-19T02:06:08.881 INFO:teuthology.orchestra.run.smithi115.stdout: "num_strays_delayed": 0, 2024-03-19T02:06:08.881 INFO:teuthology.orchestra.run.smithi115.stdout: "num_strays_enqueuing": 0, 2024-03-19T02:06:08.881 INFO:teuthology.orchestra.run.smithi115.stdout: "strays_created": 0, 2024-03-19T02:06:08.881 INFO:teuthology.orchestra.run.smithi115.stdout: "strays_enqueued": 0, 2024-03-19T02:06:08.881 INFO:teuthology.orchestra.run.smithi115.stdout: "strays_reintegrated": 0, 2024-03-19T02:06:08.881 INFO:teuthology.orchestra.run.smithi115.stdout: "strays_migrated": 0, 2024-03-19T02:06:08.881 INFO:teuthology.orchestra.run.smithi115.stdout: "num_recovering_processing": 0, 2024-03-19T02:06:08.881 INFO:teuthology.orchestra.run.smithi115.stdout: "num_recovering_enqueued": 0, 2024-03-19T02:06:08.881 INFO:teuthology.orchestra.run.smithi115.stdout: "num_recovering_prioritized": 0, 2024-03-19T02:06:08.881 INFO:teuthology.orchestra.run.smithi115.stdout: "recovery_started": 0, 2024-03-19T02:06:08.881 INFO:teuthology.orchestra.run.smithi115.stdout: "recovery_completed": 0, 2024-03-19T02:06:08.881 INFO:teuthology.orchestra.run.smithi115.stdout: "ireq_quiesce_path": 0, 2024-03-19T02:06:08.881 INFO:teuthology.orchestra.run.smithi115.stdout: "ireq_quiesce_inode": 0, 2024-03-19T02:06:08.881 INFO:teuthology.orchestra.run.smithi115.stdout: "ireq_enqueue_scrub": 0, 2024-03-19T02:06:08.882 INFO:teuthology.orchestra.run.smithi115.stdout: "ireq_exportdir": 0, 2024-03-19T02:06:08.882 INFO:teuthology.orchestra.run.smithi115.stdout: "ireq_flush": 0, 2024-03-19T02:06:08.882 INFO:teuthology.orchestra.run.smithi115.stdout: "ireq_fragmentdir": 0, 2024-03-19T02:06:08.882 INFO:teuthology.orchestra.run.smithi115.stdout: "ireq_fragstats": 0, 2024-03-19T02:06:08.882 INFO:teuthology.orchestra.run.smithi115.stdout: "ireq_inodestats": 0 2024-03-19T02:06:08.882 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.882 INFO:teuthology.orchestra.run.smithi115.stdout: "mds_log": { 2024-03-19T02:06:08.882 INFO:teuthology.orchestra.run.smithi115.stdout: "evlrg": 0, 2024-03-19T02:06:08.882 INFO:teuthology.orchestra.run.smithi115.stdout: "evadd": 0, 2024-03-19T02:06:08.882 INFO:teuthology.orchestra.run.smithi115.stdout: "evex": 0, 2024-03-19T02:06:08.882 INFO:teuthology.orchestra.run.smithi115.stdout: "evtrm": 0, 2024-03-19T02:06:08.882 INFO:teuthology.orchestra.run.smithi115.stdout: "ev": 293317, 2024-03-19T02:06:08.882 INFO:teuthology.orchestra.run.smithi115.stdout: "evexg": 0, 2024-03-19T02:06:08.882 INFO:teuthology.orchestra.run.smithi115.stdout: "evexd": 0, 2024-03-19T02:06:08.883 INFO:teuthology.orchestra.run.smithi115.stdout: "segadd": 0, 2024-03-19T02:06:08.883 INFO:teuthology.orchestra.run.smithi115.stdout: "segex": 0, 2024-03-19T02:06:08.883 INFO:teuthology.orchestra.run.smithi115.stdout: "segtrm": 0, 2024-03-19T02:06:08.883 INFO:teuthology.orchestra.run.smithi115.stdout: "seg": 6, 2024-03-19T02:06:08.883 INFO:teuthology.orchestra.run.smithi115.stdout: "segmjr": 58, 2024-03-19T02:06:08.883 INFO:teuthology.orchestra.run.smithi115.stdout: "segexg": 0, 2024-03-19T02:06:08.883 INFO:teuthology.orchestra.run.smithi115.stdout: "segexd": 0, 2024-03-19T02:06:08.886 INFO:teuthology.orchestra.run.smithi115.stdout: "expos": 658779933, 2024-03-19T02:06:08.886 INFO:teuthology.orchestra.run.smithi115.stdout: "wrpos": 666974406, 2024-03-19T02:06:08.886 INFO:teuthology.orchestra.run.smithi115.stdout: "rdpos": 666972321, 2024-03-19T02:06:08.886 INFO:teuthology.orchestra.run.smithi115.stdout: "jlat": { 2024-03-19T02:06:08.886 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:08.886 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:08.886 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:08.886 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.886 INFO:teuthology.orchestra.run.smithi115.stdout: "replayed": 293317 2024-03-19T02:06:08.887 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.887 INFO:teuthology.orchestra.run.smithi115.stdout: "mds_mem": { 2024-03-19T02:06:08.887 INFO:teuthology.orchestra.run.smithi115.stdout: "ino": 83, 2024-03-19T02:06:08.887 INFO:teuthology.orchestra.run.smithi115.stdout: "ino+": 59682, 2024-03-19T02:06:08.887 INFO:teuthology.orchestra.run.smithi115.stdout: "ino-": 59599, 2024-03-19T02:06:08.887 INFO:teuthology.orchestra.run.smithi115.stdout: "dir": 91, 2024-03-19T02:06:08.887 INFO:teuthology.orchestra.run.smithi115.stdout: "dir+": 392, 2024-03-19T02:06:08.887 INFO:teuthology.orchestra.run.smithi115.stdout: "dir-": 301, 2024-03-19T02:06:08.887 INFO:teuthology.orchestra.run.smithi115.stdout: "dn": 59974, 2024-03-19T02:06:08.887 INFO:teuthology.orchestra.run.smithi115.stdout: "dn+": 60828, 2024-03-19T02:06:08.887 INFO:teuthology.orchestra.run.smithi115.stdout: "dn-": 854, 2024-03-19T02:06:08.887 INFO:teuthology.orchestra.run.smithi115.stdout: "cap": 0, 2024-03-19T02:06:08.887 INFO:teuthology.orchestra.run.smithi115.stdout: "cap+": 0, 2024-03-19T02:06:08.887 INFO:teuthology.orchestra.run.smithi115.stdout: "cap-": 0, 2024-03-19T02:06:08.887 INFO:teuthology.orchestra.run.smithi115.stdout: "rss": 103032, 2024-03-19T02:06:08.887 INFO:teuthology.orchestra.run.smithi115.stdout: "heap": 215396 2024-03-19T02:06:08.888 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.888 INFO:teuthology.orchestra.run.smithi115.stdout: "mds_server": { 2024-03-19T02:06:08.888 INFO:teuthology.orchestra.run.smithi115.stdout: "dispatch_client_request": 0, 2024-03-19T02:06:08.888 INFO:teuthology.orchestra.run.smithi115.stdout: "dispatch_server_request": 0, 2024-03-19T02:06:08.888 INFO:teuthology.orchestra.run.smithi115.stdout: "handle_client_request": 0, 2024-03-19T02:06:08.888 INFO:teuthology.orchestra.run.smithi115.stdout: "handle_client_session": 0, 2024-03-19T02:06:08.888 INFO:teuthology.orchestra.run.smithi115.stdout: "handle_peer_request": 0, 2024-03-19T02:06:08.888 INFO:teuthology.orchestra.run.smithi115.stdout: "req_create_latency": { 2024-03-19T02:06:08.888 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:08.888 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:08.888 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:08.888 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.888 INFO:teuthology.orchestra.run.smithi115.stdout: "req_getattr_latency": { 2024-03-19T02:06:08.888 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:08.888 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:08.889 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:08.889 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.889 INFO:teuthology.orchestra.run.smithi115.stdout: "req_getfilelock_latency": { 2024-03-19T02:06:08.889 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:08.889 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:08.889 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:08.889 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.889 INFO:teuthology.orchestra.run.smithi115.stdout: "req_link_latency": { 2024-03-19T02:06:08.889 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:08.889 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:08.889 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:08.889 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.889 INFO:teuthology.orchestra.run.smithi115.stdout: "req_lookup_latency": { 2024-03-19T02:06:08.889 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:08.889 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:08.890 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:08.890 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.890 INFO:teuthology.orchestra.run.smithi115.stdout: "req_lookuphash_latency": { 2024-03-19T02:06:08.890 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:08.890 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:08.890 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:08.890 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.890 INFO:teuthology.orchestra.run.smithi115.stdout: "req_lookupino_latency": { 2024-03-19T02:06:08.890 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:08.890 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:08.890 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:08.890 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.890 INFO:teuthology.orchestra.run.smithi115.stdout: "req_lookupname_latency": { 2024-03-19T02:06:08.890 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:08.890 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:08.891 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:08.891 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.891 INFO:teuthology.orchestra.run.smithi115.stdout: "req_lookupparent_latency": { 2024-03-19T02:06:08.891 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:08.891 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:08.891 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:08.891 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.891 INFO:teuthology.orchestra.run.smithi115.stdout: "req_lookupsnap_latency": { 2024-03-19T02:06:08.891 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:08.891 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:08.891 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:08.891 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.891 INFO:teuthology.orchestra.run.smithi115.stdout: "req_lssnap_latency": { 2024-03-19T02:06:08.891 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:08.891 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:08.892 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:08.892 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.892 INFO:teuthology.orchestra.run.smithi115.stdout: "req_mkdir_latency": { 2024-03-19T02:06:08.893 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:08.893 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:08.893 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:08.893 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.893 INFO:teuthology.orchestra.run.smithi115.stdout: "req_mknod_latency": { 2024-03-19T02:06:08.893 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:08.893 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:08.893 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:08.893 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.893 INFO:teuthology.orchestra.run.smithi115.stdout: "req_mksnap_latency": { 2024-03-19T02:06:08.893 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:08.893 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:08.893 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:08.893 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.894 INFO:teuthology.orchestra.run.smithi115.stdout: "req_open_latency": { 2024-03-19T02:06:08.894 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:08.894 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:08.894 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:08.894 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.894 INFO:teuthology.orchestra.run.smithi115.stdout: "req_readdir_latency": { 2024-03-19T02:06:08.894 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:08.894 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:08.894 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:08.894 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.894 INFO:teuthology.orchestra.run.smithi115.stdout: "req_rename_latency": { 2024-03-19T02:06:08.894 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:08.894 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:08.894 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:08.895 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.895 INFO:teuthology.orchestra.run.smithi115.stdout: "req_renamesnap_latency": { 2024-03-19T02:06:08.895 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:08.895 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:08.895 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:08.895 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.895 INFO:teuthology.orchestra.run.smithi115.stdout: "req_snapdiff_latency": { 2024-03-19T02:06:08.895 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:08.895 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:08.895 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:08.895 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.895 INFO:teuthology.orchestra.run.smithi115.stdout: "req_rmdir_latency": { 2024-03-19T02:06:08.895 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:08.895 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:08.895 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:08.896 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.896 INFO:teuthology.orchestra.run.smithi115.stdout: "req_rmsnap_latency": { 2024-03-19T02:06:08.896 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:08.896 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:08.896 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:08.896 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.896 INFO:teuthology.orchestra.run.smithi115.stdout: "req_rmxattr_latency": { 2024-03-19T02:06:08.896 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:08.896 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:08.896 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:08.896 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.896 INFO:teuthology.orchestra.run.smithi115.stdout: "req_setattr_latency": { 2024-03-19T02:06:08.896 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:08.896 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:08.896 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:08.896 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.897 INFO:teuthology.orchestra.run.smithi115.stdout: "req_setdirlayout_latency": { 2024-03-19T02:06:08.897 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:08.897 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:08.897 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:08.897 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.897 INFO:teuthology.orchestra.run.smithi115.stdout: "req_setfilelock_latency": { 2024-03-19T02:06:08.897 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:08.897 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:08.897 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:08.897 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.897 INFO:teuthology.orchestra.run.smithi115.stdout: "req_setlayout_latency": { 2024-03-19T02:06:08.897 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:08.897 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:08.897 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:08.898 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.898 INFO:teuthology.orchestra.run.smithi115.stdout: "req_setxattr_latency": { 2024-03-19T02:06:08.898 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:08.898 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:08.898 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:08.898 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.898 INFO:teuthology.orchestra.run.smithi115.stdout: "req_symlink_latency": { 2024-03-19T02:06:08.898 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:08.898 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:08.898 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:08.898 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.898 INFO:teuthology.orchestra.run.smithi115.stdout: "req_unlink_latency": { 2024-03-19T02:06:08.898 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:08.898 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:08.898 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:08.899 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.899 INFO:teuthology.orchestra.run.smithi115.stdout: "cap_revoke_eviction": 0, 2024-03-19T02:06:08.899 INFO:teuthology.orchestra.run.smithi115.stdout: "cap_acquisition_throttle": 0, 2024-03-19T02:06:08.899 INFO:teuthology.orchestra.run.smithi115.stdout: "req_getvxattr_latency": { 2024-03-19T02:06:08.899 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:08.899 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:08.899 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:08.899 INFO:teuthology.orchestra.run.smithi115.stdout: } 2024-03-19T02:06:08.899 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.899 INFO:teuthology.orchestra.run.smithi115.stdout: "mds_sessions": { 2024-03-19T02:06:08.899 INFO:teuthology.orchestra.run.smithi115.stdout: "session_count": 1, 2024-03-19T02:06:08.899 INFO:teuthology.orchestra.run.smithi115.stdout: "session_add": 1, 2024-03-19T02:06:08.899 INFO:teuthology.orchestra.run.smithi115.stdout: "session_remove": 0, 2024-03-19T02:06:08.899 INFO:teuthology.orchestra.run.smithi115.stdout: "sessions_open": 0, 2024-03-19T02:06:08.900 INFO:teuthology.orchestra.run.smithi115.stdout: "sessions_stale": 0, 2024-03-19T02:06:08.900 INFO:teuthology.orchestra.run.smithi115.stdout: "total_load": 0, 2024-03-19T02:06:08.900 INFO:teuthology.orchestra.run.smithi115.stdout: "average_load": 0, 2024-03-19T02:06:08.900 INFO:teuthology.orchestra.run.smithi115.stdout: "avg_session_uptime": 6297, 2024-03-19T02:06:08.900 INFO:teuthology.orchestra.run.smithi115.stdout: "mdthresh_evicted": 0 2024-03-19T02:06:08.900 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.900 INFO:teuthology.orchestra.run.smithi115.stdout: "mempool": { 2024-03-19T02:06:08.900 INFO:teuthology.orchestra.run.smithi115.stdout: "bloom_filter_bytes": 0, 2024-03-19T02:06:08.900 INFO:teuthology.orchestra.run.smithi115.stdout: "bloom_filter_items": 0, 2024-03-19T02:06:08.900 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_alloc_bytes": 0, 2024-03-19T02:06:08.900 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_alloc_items": 0, 2024-03-19T02:06:08.900 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_cache_data_bytes": 0, 2024-03-19T02:06:08.900 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_cache_data_items": 0, 2024-03-19T02:06:08.900 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_cache_onode_bytes": 0, 2024-03-19T02:06:08.901 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_cache_onode_items": 0, 2024-03-19T02:06:08.901 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_cache_meta_bytes": 0, 2024-03-19T02:06:08.901 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_cache_meta_items": 0, 2024-03-19T02:06:08.901 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_cache_other_bytes": 0, 2024-03-19T02:06:08.901 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_cache_other_items": 0, 2024-03-19T02:06:08.901 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_cache_buffer_bytes": 0, 2024-03-19T02:06:08.901 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_cache_buffer_items": 0, 2024-03-19T02:06:08.901 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_extent_bytes": 0, 2024-03-19T02:06:08.901 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_extent_items": 0, 2024-03-19T02:06:08.901 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_blob_bytes": 0, 2024-03-19T02:06:08.901 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_blob_items": 0, 2024-03-19T02:06:08.901 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_shared_blob_bytes": 0, 2024-03-19T02:06:08.901 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_shared_blob_items": 0, 2024-03-19T02:06:08.901 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_inline_bl_bytes": 0, 2024-03-19T02:06:08.901 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_inline_bl_items": 0, 2024-03-19T02:06:08.902 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_fsck_bytes": 0, 2024-03-19T02:06:08.902 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_fsck_items": 0, 2024-03-19T02:06:08.902 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_txc_bytes": 0, 2024-03-19T02:06:08.902 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_txc_items": 0, 2024-03-19T02:06:08.902 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_writing_deferred_bytes": 0, 2024-03-19T02:06:08.902 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_writing_deferred_items": 0, 2024-03-19T02:06:08.902 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_writing_bytes": 0, 2024-03-19T02:06:08.902 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_writing_items": 0, 2024-03-19T02:06:08.902 INFO:teuthology.orchestra.run.smithi115.stdout: "bluefs_bytes": 0, 2024-03-19T02:06:08.902 INFO:teuthology.orchestra.run.smithi115.stdout: "bluefs_items": 0, 2024-03-19T02:06:08.902 INFO:teuthology.orchestra.run.smithi115.stdout: "bluefs_file_reader_bytes": 0, 2024-03-19T02:06:08.902 INFO:teuthology.orchestra.run.smithi115.stdout: "bluefs_file_reader_items": 0, 2024-03-19T02:06:08.902 INFO:teuthology.orchestra.run.smithi115.stdout: "bluefs_file_writer_bytes": 0, 2024-03-19T02:06:08.902 INFO:teuthology.orchestra.run.smithi115.stdout: "bluefs_file_writer_items": 0, 2024-03-19T02:06:08.903 INFO:teuthology.orchestra.run.smithi115.stdout: "buffer_anon_bytes": 241459, 2024-03-19T02:06:08.903 INFO:teuthology.orchestra.run.smithi115.stdout: "buffer_anon_items": 152, 2024-03-19T02:06:08.903 INFO:teuthology.orchestra.run.smithi115.stdout: "buffer_meta_bytes": 0, 2024-03-19T02:06:08.903 INFO:teuthology.orchestra.run.smithi115.stdout: "buffer_meta_items": 0, 2024-03-19T02:06:08.903 INFO:teuthology.orchestra.run.smithi115.stdout: "osd_bytes": 0, 2024-03-19T02:06:08.903 INFO:teuthology.orchestra.run.smithi115.stdout: "osd_items": 0, 2024-03-19T02:06:08.903 INFO:teuthology.orchestra.run.smithi115.stdout: "osd_mapbl_bytes": 0, 2024-03-19T02:06:08.903 INFO:teuthology.orchestra.run.smithi115.stdout: "osd_mapbl_items": 0, 2024-03-19T02:06:08.903 INFO:teuthology.orchestra.run.smithi115.stdout: "osd_pglog_bytes": 0, 2024-03-19T02:06:08.903 INFO:teuthology.orchestra.run.smithi115.stdout: "osd_pglog_items": 0, 2024-03-19T02:06:08.903 INFO:teuthology.orchestra.run.smithi115.stdout: "osdmap_bytes": 7816, 2024-03-19T02:06:08.903 INFO:teuthology.orchestra.run.smithi115.stdout: "osdmap_items": 164, 2024-03-19T02:06:08.903 INFO:teuthology.orchestra.run.smithi115.stdout: "osdmap_mapping_bytes": 0, 2024-03-19T02:06:08.903 INFO:teuthology.orchestra.run.smithi115.stdout: "osdmap_mapping_items": 0, 2024-03-19T02:06:08.903 INFO:teuthology.orchestra.run.smithi115.stdout: "pgmap_bytes": 0, 2024-03-19T02:06:08.904 INFO:teuthology.orchestra.run.smithi115.stdout: "pgmap_items": 0, 2024-03-19T02:06:08.904 INFO:teuthology.orchestra.run.smithi115.stdout: "mds_co_bytes": 42569056, 2024-03-19T02:06:08.904 INFO:teuthology.orchestra.run.smithi115.stdout: "mds_co_items": 250014, 2024-03-19T02:06:08.904 INFO:teuthology.orchestra.run.smithi115.stdout: "unittest_1_bytes": 0, 2024-03-19T02:06:08.904 INFO:teuthology.orchestra.run.smithi115.stdout: "unittest_1_items": 0, 2024-03-19T02:06:08.904 INFO:teuthology.orchestra.run.smithi115.stdout: "unittest_2_bytes": 0, 2024-03-19T02:06:08.904 INFO:teuthology.orchestra.run.smithi115.stdout: "unittest_2_items": 0 2024-03-19T02:06:08.904 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.904 INFO:teuthology.orchestra.run.smithi115.stdout: "objecter": { 2024-03-19T02:06:08.904 INFO:teuthology.orchestra.run.smithi115.stdout: "op_active": 0, 2024-03-19T02:06:08.904 INFO:teuthology.orchestra.run.smithi115.stdout: "op_laggy": 0, 2024-03-19T02:06:08.904 INFO:teuthology.orchestra.run.smithi115.stdout: "op_send": 3577, 2024-03-19T02:06:08.904 INFO:teuthology.orchestra.run.smithi115.stdout: "op_send_bytes": 26, 2024-03-19T02:06:08.904 INFO:teuthology.orchestra.run.smithi115.stdout: "op_resend": 4, 2024-03-19T02:06:08.905 INFO:teuthology.orchestra.run.smithi115.stdout: "op_reply": 3577, 2024-03-19T02:06:08.905 INFO:teuthology.orchestra.run.smithi115.stdout: "op_latency": { 2024-03-19T02:06:08.905 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 3577, 2024-03-19T02:06:08.905 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 9.031824247, 2024-03-19T02:06:08.905 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.002524971 2024-03-19T02:06:08.905 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.905 INFO:teuthology.orchestra.run.smithi115.stdout: "op_inflight": 0, 2024-03-19T02:06:08.905 INFO:teuthology.orchestra.run.smithi115.stdout: "oplen_avg": { 2024-03-19T02:06:08.905 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 3577, 2024-03-19T02:06:08.905 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 3578 2024-03-19T02:06:08.905 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.905 INFO:teuthology.orchestra.run.smithi115.stdout: "op": 3577, 2024-03-19T02:06:08.905 INFO:teuthology.orchestra.run.smithi115.stdout: "op_r": 3577, 2024-03-19T02:06:08.905 INFO:teuthology.orchestra.run.smithi115.stdout: "op_w": 0, 2024-03-19T02:06:08.905 INFO:teuthology.orchestra.run.smithi115.stdout: "op_rmw": 0, 2024-03-19T02:06:08.905 INFO:teuthology.orchestra.run.smithi115.stdout: "op_pg": 0, 2024-03-19T02:06:08.906 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_stat": 1910, 2024-03-19T02:06:08.906 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_create": 0, 2024-03-19T02:06:08.906 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_read": 1664, 2024-03-19T02:06:08.906 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_write": 0, 2024-03-19T02:06:08.906 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_writefull": 0, 2024-03-19T02:06:08.906 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_writesame": 0, 2024-03-19T02:06:08.906 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_append": 0, 2024-03-19T02:06:08.906 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_zero": 0, 2024-03-19T02:06:08.906 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_truncate": 0, 2024-03-19T02:06:08.906 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_delete": 0, 2024-03-19T02:06:08.906 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_mapext": 0, 2024-03-19T02:06:08.906 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_sparse_read": 0, 2024-03-19T02:06:08.906 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_clonerange": 0, 2024-03-19T02:06:08.906 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_getxattr": 2, 2024-03-19T02:06:08.907 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_setxattr": 0, 2024-03-19T02:06:08.907 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_cmpxattr": 0, 2024-03-19T02:06:08.907 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_rmxattr": 0, 2024-03-19T02:06:08.907 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_resetxattrs": 0, 2024-03-19T02:06:08.907 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_call": 0, 2024-03-19T02:06:08.907 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_watch": 0, 2024-03-19T02:06:08.907 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_notify": 0, 2024-03-19T02:06:08.907 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_src_cmpxattr": 0, 2024-03-19T02:06:08.907 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_pgls": 0, 2024-03-19T02:06:08.907 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_pgls_filter": 0, 2024-03-19T02:06:08.907 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_other": 0, 2024-03-19T02:06:08.907 INFO:teuthology.orchestra.run.smithi115.stdout: "linger_active": 0, 2024-03-19T02:06:08.907 INFO:teuthology.orchestra.run.smithi115.stdout: "linger_send": 0, 2024-03-19T02:06:08.907 INFO:teuthology.orchestra.run.smithi115.stdout: "linger_resend": 0, 2024-03-19T02:06:08.907 INFO:teuthology.orchestra.run.smithi115.stdout: "linger_ping": 0, 2024-03-19T02:06:08.908 INFO:teuthology.orchestra.run.smithi115.stdout: "poolop_active": 0, 2024-03-19T02:06:08.908 INFO:teuthology.orchestra.run.smithi115.stdout: "poolop_send": 0, 2024-03-19T02:06:08.908 INFO:teuthology.orchestra.run.smithi115.stdout: "poolop_resend": 0, 2024-03-19T02:06:08.908 INFO:teuthology.orchestra.run.smithi115.stdout: "poolstat_active": 0, 2024-03-19T02:06:08.908 INFO:teuthology.orchestra.run.smithi115.stdout: "poolstat_send": 0, 2024-03-19T02:06:08.908 INFO:teuthology.orchestra.run.smithi115.stdout: "poolstat_resend": 0, 2024-03-19T02:06:08.908 INFO:teuthology.orchestra.run.smithi115.stdout: "statfs_active": 0, 2024-03-19T02:06:08.908 INFO:teuthology.orchestra.run.smithi115.stdout: "statfs_send": 0, 2024-03-19T02:06:08.908 INFO:teuthology.orchestra.run.smithi115.stdout: "statfs_resend": 0, 2024-03-19T02:06:08.908 INFO:teuthology.orchestra.run.smithi115.stdout: "command_active": 0, 2024-03-19T02:06:08.908 INFO:teuthology.orchestra.run.smithi115.stdout: "command_send": 0, 2024-03-19T02:06:08.908 INFO:teuthology.orchestra.run.smithi115.stdout: "command_resend": 0, 2024-03-19T02:06:08.908 INFO:teuthology.orchestra.run.smithi115.stdout: "map_epoch": 0, 2024-03-19T02:06:08.908 INFO:teuthology.orchestra.run.smithi115.stdout: "map_full": 0, 2024-03-19T02:06:08.908 INFO:teuthology.orchestra.run.smithi115.stdout: "map_inc": 0, 2024-03-19T02:06:08.909 INFO:teuthology.orchestra.run.smithi115.stdout: "osd_sessions": 11, 2024-03-19T02:06:08.909 INFO:teuthology.orchestra.run.smithi115.stdout: "osd_session_open": 11, 2024-03-19T02:06:08.909 INFO:teuthology.orchestra.run.smithi115.stdout: "osd_session_close": 0, 2024-03-19T02:06:08.909 INFO:teuthology.orchestra.run.smithi115.stdout: "osd_laggy": 0, 2024-03-19T02:06:08.909 INFO:teuthology.orchestra.run.smithi115.stdout: "omap_wr": 0, 2024-03-19T02:06:08.909 INFO:teuthology.orchestra.run.smithi115.stdout: "omap_rd": 2, 2024-03-19T02:06:08.909 INFO:teuthology.orchestra.run.smithi115.stdout: "omap_del": 0 2024-03-19T02:06:08.909 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.909 INFO:teuthology.orchestra.run.smithi115.stdout: "oft": { 2024-03-19T02:06:08.909 INFO:teuthology.orchestra.run.smithi115.stdout: "omap_total_objs": 0, 2024-03-19T02:06:08.909 INFO:teuthology.orchestra.run.smithi115.stdout: "omap_total_kv_pairs": 0, 2024-03-19T02:06:08.909 INFO:teuthology.orchestra.run.smithi115.stdout: "omap_total_updates": 0, 2024-03-19T02:06:08.909 INFO:teuthology.orchestra.run.smithi115.stdout: "omap_total_removes": 0 2024-03-19T02:06:08.909 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.909 INFO:teuthology.orchestra.run.smithi115.stdout: "osd-slow-ops": { 2024-03-19T02:06:08.910 INFO:teuthology.orchestra.run.smithi115.stdout: "slow_ops_count": 0 2024-03-19T02:06:08.910 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.910 INFO:teuthology.orchestra.run.smithi115.stdout: "purge_queue": { 2024-03-19T02:06:08.910 INFO:teuthology.orchestra.run.smithi115.stdout: "pq_executing_ops": 0, 2024-03-19T02:06:08.910 INFO:teuthology.orchestra.run.smithi115.stdout: "pq_executing_ops_high_water": 0, 2024-03-19T02:06:08.910 INFO:teuthology.orchestra.run.smithi115.stdout: "pq_executing": 0, 2024-03-19T02:06:08.910 INFO:teuthology.orchestra.run.smithi115.stdout: "pq_executing_high_water": 0, 2024-03-19T02:06:08.910 INFO:teuthology.orchestra.run.smithi115.stdout: "pq_executed_ops": 0, 2024-03-19T02:06:08.910 INFO:teuthology.orchestra.run.smithi115.stdout: "pq_executed": 0, 2024-03-19T02:06:08.910 INFO:teuthology.orchestra.run.smithi115.stdout: "pq_item_in_journal": 0 2024-03-19T02:06:08.910 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.910 INFO:teuthology.orchestra.run.smithi115.stdout: "throttle-msgr_dispatch_throttler-mds": { 2024-03-19T02:06:08.910 INFO:teuthology.orchestra.run.smithi115.stdout: "val": 0, 2024-03-19T02:06:08.910 INFO:teuthology.orchestra.run.smithi115.stdout: "max": 104857600, 2024-03-19T02:06:08.910 INFO:teuthology.orchestra.run.smithi115.stdout: "get_started": 0, 2024-03-19T02:06:08.911 INFO:teuthology.orchestra.run.smithi115.stdout: "get": 3822, 2024-03-19T02:06:08.911 INFO:teuthology.orchestra.run.smithi115.stdout: "get_sum": 600738282, 2024-03-19T02:06:08.911 INFO:teuthology.orchestra.run.smithi115.stdout: "get_or_fail_fail": 0, 2024-03-19T02:06:08.911 INFO:teuthology.orchestra.run.smithi115.stdout: "get_or_fail_success": 3822, 2024-03-19T02:06:08.911 INFO:teuthology.orchestra.run.smithi115.stdout: "take": 0, 2024-03-19T02:06:08.911 INFO:teuthology.orchestra.run.smithi115.stdout: "take_sum": 0, 2024-03-19T02:06:08.911 INFO:teuthology.orchestra.run.smithi115.stdout: "put": 3822, 2024-03-19T02:06:08.911 INFO:teuthology.orchestra.run.smithi115.stdout: "put_sum": 600738282, 2024-03-19T02:06:08.911 INFO:teuthology.orchestra.run.smithi115.stdout: "wait": { 2024-03-19T02:06:08.911 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:08.911 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:08.911 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:08.911 INFO:teuthology.orchestra.run.smithi115.stdout: } 2024-03-19T02:06:08.911 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.912 INFO:teuthology.orchestra.run.smithi115.stdout: "throttle-objecter_bytes": { 2024-03-19T02:06:08.912 INFO:teuthology.orchestra.run.smithi115.stdout: "val": 0, 2024-03-19T02:06:08.912 INFO:teuthology.orchestra.run.smithi115.stdout: "max": 104857600, 2024-03-19T02:06:08.912 INFO:teuthology.orchestra.run.smithi115.stdout: "get_started": 0, 2024-03-19T02:06:08.912 INFO:teuthology.orchestra.run.smithi115.stdout: "get": 0, 2024-03-19T02:06:08.912 INFO:teuthology.orchestra.run.smithi115.stdout: "get_sum": 0, 2024-03-19T02:06:08.912 INFO:teuthology.orchestra.run.smithi115.stdout: "get_or_fail_fail": 0, 2024-03-19T02:06:08.912 INFO:teuthology.orchestra.run.smithi115.stdout: "get_or_fail_success": 0, 2024-03-19T02:06:08.912 INFO:teuthology.orchestra.run.smithi115.stdout: "take": 3577, 2024-03-19T02:06:08.912 INFO:teuthology.orchestra.run.smithi115.stdout: "take_sum": 599818554, 2024-03-19T02:06:08.912 INFO:teuthology.orchestra.run.smithi115.stdout: "put": 3577, 2024-03-19T02:06:08.912 INFO:teuthology.orchestra.run.smithi115.stdout: "put_sum": 599818554, 2024-03-19T02:06:08.912 INFO:teuthology.orchestra.run.smithi115.stdout: "wait": { 2024-03-19T02:06:08.913 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:08.913 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:08.913 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:08.913 INFO:teuthology.orchestra.run.smithi115.stdout: } 2024-03-19T02:06:08.913 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.913 INFO:teuthology.orchestra.run.smithi115.stdout: "throttle-objecter_ops": { 2024-03-19T02:06:08.913 INFO:teuthology.orchestra.run.smithi115.stdout: "val": 0, 2024-03-19T02:06:08.913 INFO:teuthology.orchestra.run.smithi115.stdout: "max": 1024, 2024-03-19T02:06:08.913 INFO:teuthology.orchestra.run.smithi115.stdout: "get_started": 0, 2024-03-19T02:06:08.913 INFO:teuthology.orchestra.run.smithi115.stdout: "get": 0, 2024-03-19T02:06:08.913 INFO:teuthology.orchestra.run.smithi115.stdout: "get_sum": 0, 2024-03-19T02:06:08.913 INFO:teuthology.orchestra.run.smithi115.stdout: "get_or_fail_fail": 0, 2024-03-19T02:06:08.913 INFO:teuthology.orchestra.run.smithi115.stdout: "get_or_fail_success": 0, 2024-03-19T02:06:08.913 INFO:teuthology.orchestra.run.smithi115.stdout: "take": 3577, 2024-03-19T02:06:08.913 INFO:teuthology.orchestra.run.smithi115.stdout: "take_sum": 3577, 2024-03-19T02:06:08.914 INFO:teuthology.orchestra.run.smithi115.stdout: "put": 3577, 2024-03-19T02:06:08.914 INFO:teuthology.orchestra.run.smithi115.stdout: "put_sum": 3577, 2024-03-19T02:06:08.914 INFO:teuthology.orchestra.run.smithi115.stdout: "wait": { 2024-03-19T02:06:08.914 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:08.914 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:08.914 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:08.914 INFO:teuthology.orchestra.run.smithi115.stdout: } 2024-03-19T02:06:08.914 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.914 INFO:teuthology.orchestra.run.smithi115.stdout: "throttle-write_buf_throttle": { 2024-03-19T02:06:08.914 INFO:teuthology.orchestra.run.smithi115.stdout: "val": 0, 2024-03-19T02:06:08.914 INFO:teuthology.orchestra.run.smithi115.stdout: "max": 3758096384, 2024-03-19T02:06:08.914 INFO:teuthology.orchestra.run.smithi115.stdout: "get_started": 0, 2024-03-19T02:06:08.914 INFO:teuthology.orchestra.run.smithi115.stdout: "get": 0, 2024-03-19T02:06:08.914 INFO:teuthology.orchestra.run.smithi115.stdout: "get_sum": 0, 2024-03-19T02:06:08.914 INFO:teuthology.orchestra.run.smithi115.stdout: "get_or_fail_fail": 0, 2024-03-19T02:06:08.915 INFO:teuthology.orchestra.run.smithi115.stdout: "get_or_fail_success": 0, 2024-03-19T02:06:08.915 INFO:teuthology.orchestra.run.smithi115.stdout: "take": 0, 2024-03-19T02:06:08.915 INFO:teuthology.orchestra.run.smithi115.stdout: "take_sum": 0, 2024-03-19T02:06:08.915 INFO:teuthology.orchestra.run.smithi115.stdout: "put": 0, 2024-03-19T02:06:08.915 INFO:teuthology.orchestra.run.smithi115.stdout: "put_sum": 0, 2024-03-19T02:06:08.915 INFO:teuthology.orchestra.run.smithi115.stdout: "wait": { 2024-03-19T02:06:08.915 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:08.915 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:08.915 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:08.915 INFO:teuthology.orchestra.run.smithi115.stdout: } 2024-03-19T02:06:08.915 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:08.915 INFO:teuthology.orchestra.run.smithi115.stdout: "throttle-write_buf_throttle-0x5613cee3c640": { 2024-03-19T02:06:08.915 INFO:teuthology.orchestra.run.smithi115.stdout: "val": 0, 2024-03-19T02:06:08.915 INFO:teuthology.orchestra.run.smithi115.stdout: "max": 3758096384, 2024-03-19T02:06:08.915 INFO:teuthology.orchestra.run.smithi115.stdout: "get_started": 0, 2024-03-19T02:06:08.915 INFO:teuthology.orchestra.run.smithi115.stdout: "get": 0, 2024-03-19T02:06:08.916 INFO:teuthology.orchestra.run.smithi115.stdout: "get_sum": 0, 2024-03-19T02:06:08.916 INFO:teuthology.orchestra.run.smithi115.stdout: "get_or_fail_fail": 0, 2024-03-19T02:06:08.916 INFO:teuthology.orchestra.run.smithi115.stdout: "get_or_fail_success": 0, 2024-03-19T02:06:08.916 INFO:teuthology.orchestra.run.smithi115.stdout: "take": 0, 2024-03-19T02:06:08.916 INFO:teuthology.orchestra.run.smithi115.stdout: "take_sum": 0, 2024-03-19T02:06:08.916 INFO:teuthology.orchestra.run.smithi115.stdout: "put": 0, 2024-03-19T02:06:08.916 INFO:teuthology.orchestra.run.smithi115.stdout: "put_sum": 0, 2024-03-19T02:06:08.916 INFO:teuthology.orchestra.run.smithi115.stdout: "wait": { 2024-03-19T02:06:08.916 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:08.916 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:08.916 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:08.916 INFO:teuthology.orchestra.run.smithi115.stdout: } 2024-03-19T02:06:08.916 INFO:teuthology.orchestra.run.smithi115.stdout: } 2024-03-19T02:06:08.916 INFO:teuthology.orchestra.run.smithi115.stdout:} 2024-03-19T02:06:09.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:06:08 smithi115 ceph-mon[67332]: pgmap v1102: 97 pgs: 97 active+clean; 440 MiB data, 9.0 GiB used, 1.0 TiB / 1.0 TiB avail; 175 KiB/s rd, 3.1 MiB/s wr, 125 op/s 2024-03-19T02:06:09.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:06:08 smithi099 ceph-mon[62951]: pgmap v1102: 97 pgs: 97 active+clean; 440 MiB data, 9.0 GiB used, 1.0 TiB / 1.0 TiB avail; 175 KiB/s rd, 3.1 MiB/s wr, 125 op/s 2024-03-19T02:06:09.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:06:08 smithi112 ceph-mon[66819]: pgmap v1102: 97 pgs: 97 active+clean; 440 MiB data, 9.0 GiB used, 1.0 TiB / 1.0 TiB avail; 175 KiB/s rd, 3.1 MiB/s wr, 125 op/s 2024-03-19T02:06:09.697 INFO:tasks.check_counter:Daemon mds.f mds.exported=0 2024-03-19T02:06:09.697 INFO:tasks.check_counter:Daemon mds.f mds.imported=0 2024-03-19T02:06:09.698 INFO:tasks.check_counter:Daemon mds.f mds_cache.dir_update=0 2024-03-19T02:06:09.698 INFO:tasks.check_counter:Daemon mds.f mds_cache.dir_update_receipt=0 2024-03-19T02:06:09.698 DEBUG:tasks.check_counter:Getting stats from i 2024-03-19T02:06:09.698 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph daemon mds.i perf dump 2024-03-19T02:06:09.950 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.c/config 2024-03-19T02:06:11.284 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:06:10 smithi115 ceph-mon[67332]: pgmap v1103: 97 pgs: 97 active+clean; 436 MiB data, 9.0 GiB used, 1.0 TiB / 1.0 TiB avail; 115 KiB/s rd, 2.6 MiB/s wr, 103 op/s 2024-03-19T02:06:11.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:06:10 smithi099 ceph-mon[62951]: pgmap v1103: 97 pgs: 97 active+clean; 436 MiB data, 9.0 GiB used, 1.0 TiB / 1.0 TiB avail; 115 KiB/s rd, 2.6 MiB/s wr, 103 op/s 2024-03-19T02:06:11.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:06:10 smithi112 ceph-mon[66819]: pgmap v1103: 97 pgs: 97 active+clean; 436 MiB data, 9.0 GiB used, 1.0 TiB / 1.0 TiB avail; 115 KiB/s rd, 2.6 MiB/s wr, 103 op/s 2024-03-19T02:06:11.535 INFO:teuthology.orchestra.run.smithi115.stdout:{ 2024-03-19T02:06:11.536 INFO:teuthology.orchestra.run.smithi115.stdout: "AsyncMessenger::Worker-0": { 2024-03-19T02:06:11.536 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_recv_messages": 730429, 2024-03-19T02:06:11.536 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_send_messages": 1069516, 2024-03-19T02:06:11.536 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_recv_bytes": 218377790, 2024-03-19T02:06:11.536 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_send_bytes": 1257007831, 2024-03-19T02:06:11.536 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_created_connections": 226, 2024-03-19T02:06:11.536 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_active_connections": 9, 2024-03-19T02:06:11.536 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_running_total_time": 51.274403665, 2024-03-19T02:06:11.536 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_running_send_time": 22.144225215, 2024-03-19T02:06:11.536 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_running_recv_time": 31.016633219, 2024-03-19T02:06:11.536 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_running_fast_dispatch_time": 4.496909014, 2024-03-19T02:06:11.536 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_send_messages_queue_lat": { 2024-03-19T02:06:11.536 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 1069511, 2024-03-19T02:06:11.536 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 124.383337496, 2024-03-19T02:06:11.537 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000116299 2024-03-19T02:06:11.537 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.537 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_handle_ack_lat": { 2024-03-19T02:06:11.537 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 1508008, 2024-03-19T02:06:11.537 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.283816891, 2024-03-19T02:06:11.537 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000188 2024-03-19T02:06:11.537 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.537 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_recv_encrypted_bytes": 84016, 2024-03-19T02:06:11.537 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_send_encrypted_bytes": 31216 2024-03-19T02:06:11.537 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.537 INFO:teuthology.orchestra.run.smithi115.stdout: "AsyncMessenger::Worker-1": { 2024-03-19T02:06:11.537 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_recv_messages": 456484, 2024-03-19T02:06:11.537 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_send_messages": 458352, 2024-03-19T02:06:11.537 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_recv_bytes": 117459827, 2024-03-19T02:06:11.537 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_send_bytes": 328318544, 2024-03-19T02:06:11.538 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_created_connections": 18, 2024-03-19T02:06:11.538 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_active_connections": 5, 2024-03-19T02:06:11.538 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_running_total_time": 38.162924359, 2024-03-19T02:06:11.538 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_running_send_time": 18.187766480, 2024-03-19T02:06:11.538 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_running_recv_time": 19.645749624, 2024-03-19T02:06:11.538 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_running_fast_dispatch_time": 3.784950575, 2024-03-19T02:06:11.538 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_send_messages_queue_lat": { 2024-03-19T02:06:11.538 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 458353, 2024-03-19T02:06:11.538 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 96.236505855, 2024-03-19T02:06:11.538 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000209961 2024-03-19T02:06:11.538 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.538 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_handle_ack_lat": { 2024-03-19T02:06:11.538 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 774066, 2024-03-19T02:06:11.538 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.144379829, 2024-03-19T02:06:11.542 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000186 2024-03-19T02:06:11.542 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.542 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_recv_encrypted_bytes": 160, 2024-03-19T02:06:11.543 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_send_encrypted_bytes": 679328 2024-03-19T02:06:11.543 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.543 INFO:teuthology.orchestra.run.smithi115.stdout: "AsyncMessenger::Worker-2": { 2024-03-19T02:06:11.543 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_recv_messages": 277746, 2024-03-19T02:06:11.543 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_send_messages": 319094, 2024-03-19T02:06:11.543 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_recv_bytes": 66169695, 2024-03-19T02:06:11.543 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_send_bytes": 220996839, 2024-03-19T02:06:11.543 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_created_connections": 7, 2024-03-19T02:06:11.543 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_active_connections": 6, 2024-03-19T02:06:11.544 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_running_total_time": 21.770323189, 2024-03-19T02:06:11.544 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_running_send_time": 8.613346178, 2024-03-19T02:06:11.544 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_running_recv_time": 11.580953388, 2024-03-19T02:06:11.544 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_running_fast_dispatch_time": 3.026386683, 2024-03-19T02:06:11.544 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_send_messages_queue_lat": { 2024-03-19T02:06:11.544 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 319094, 2024-03-19T02:06:11.544 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 67.930479998, 2024-03-19T02:06:11.544 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000212885 2024-03-19T02:06:11.544 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.544 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_handle_ack_lat": { 2024-03-19T02:06:11.544 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 490902, 2024-03-19T02:06:11.544 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.106732034, 2024-03-19T02:06:11.544 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000217 2024-03-19T02:06:11.544 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.545 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_recv_encrypted_bytes": 967600, 2024-03-19T02:06:11.545 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_send_encrypted_bytes": 279344 2024-03-19T02:06:11.545 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.545 INFO:teuthology.orchestra.run.smithi115.stdout: "cct": { 2024-03-19T02:06:11.545 INFO:teuthology.orchestra.run.smithi115.stdout: "total_workers": 1, 2024-03-19T02:06:11.545 INFO:teuthology.orchestra.run.smithi115.stdout: "unhealthy_workers": 0 2024-03-19T02:06:11.545 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.545 INFO:teuthology.orchestra.run.smithi115.stdout: "finisher-MDSRank": { 2024-03-19T02:06:11.545 INFO:teuthology.orchestra.run.smithi115.stdout: "queue_len": 0, 2024-03-19T02:06:11.545 INFO:teuthology.orchestra.run.smithi115.stdout: "complete_latency": { 2024-03-19T02:06:11.545 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 117619, 2024-03-19T02:06:11.545 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 364.339452639, 2024-03-19T02:06:11.545 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.003097624 2024-03-19T02:06:11.545 INFO:teuthology.orchestra.run.smithi115.stdout: } 2024-03-19T02:06:11.545 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.546 INFO:teuthology.orchestra.run.smithi115.stdout: "finisher-PurgeQueue": { 2024-03-19T02:06:11.546 INFO:teuthology.orchestra.run.smithi115.stdout: "queue_len": 0, 2024-03-19T02:06:11.546 INFO:teuthology.orchestra.run.smithi115.stdout: "complete_latency": { 2024-03-19T02:06:11.546 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 105340, 2024-03-19T02:06:11.546 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 47.761536466, 2024-03-19T02:06:11.546 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000453403 2024-03-19T02:06:11.546 INFO:teuthology.orchestra.run.smithi115.stdout: } 2024-03-19T02:06:11.546 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.546 INFO:teuthology.orchestra.run.smithi115.stdout: "mds": { 2024-03-19T02:06:11.546 INFO:teuthology.orchestra.run.smithi115.stdout: "request": 447680, 2024-03-19T02:06:11.546 INFO:teuthology.orchestra.run.smithi115.stdout: "reply": 395431, 2024-03-19T02:06:11.546 INFO:teuthology.orchestra.run.smithi115.stdout: "reply_latency": { 2024-03-19T02:06:11.546 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 395431, 2024-03-19T02:06:11.546 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 4978.014357275, 2024-03-19T02:06:11.547 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.012588831 2024-03-19T02:06:11.547 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.547 INFO:teuthology.orchestra.run.smithi115.stdout: "slow_reply": 0, 2024-03-19T02:06:11.547 INFO:teuthology.orchestra.run.smithi115.stdout: "forward": 52249, 2024-03-19T02:06:11.547 INFO:teuthology.orchestra.run.smithi115.stdout: "dir_fetch_complete": 0, 2024-03-19T02:06:11.547 INFO:teuthology.orchestra.run.smithi115.stdout: "dir_fetch_keys": 0, 2024-03-19T02:06:11.548 INFO:teuthology.orchestra.run.smithi115.stdout: "dir_commit": 8241, 2024-03-19T02:06:11.548 INFO:teuthology.orchestra.run.smithi115.stdout: "dir_split": 29, 2024-03-19T02:06:11.548 INFO:teuthology.orchestra.run.smithi115.stdout: "dir_merge": 43, 2024-03-19T02:06:11.548 INFO:teuthology.orchestra.run.smithi115.stdout: "inodes": 2640, 2024-03-19T02:06:11.548 INFO:teuthology.orchestra.run.smithi115.stdout: "inodes_top": 1138, 2024-03-19T02:06:11.548 INFO:teuthology.orchestra.run.smithi115.stdout: "inodes_bottom": 1492, 2024-03-19T02:06:11.548 INFO:teuthology.orchestra.run.smithi115.stdout: "inodes_pin_tail": 10, 2024-03-19T02:06:11.548 INFO:teuthology.orchestra.run.smithi115.stdout: "inodes_pinned": 1013, 2024-03-19T02:06:11.548 INFO:teuthology.orchestra.run.smithi115.stdout: "inodes_expired": 244, 2024-03-19T02:06:11.548 INFO:teuthology.orchestra.run.smithi115.stdout: "inodes_with_caps": 1, 2024-03-19T02:06:11.548 INFO:teuthology.orchestra.run.smithi115.stdout: "caps": 1, 2024-03-19T02:06:11.548 INFO:teuthology.orchestra.run.smithi115.stdout: "subtrees": 4, 2024-03-19T02:06:11.548 INFO:teuthology.orchestra.run.smithi115.stdout: "traverse": 721511, 2024-03-19T02:06:11.548 INFO:teuthology.orchestra.run.smithi115.stdout: "traverse_hit": 361021, 2024-03-19T02:06:11.549 INFO:teuthology.orchestra.run.smithi115.stdout: "traverse_forward": 52248, 2024-03-19T02:06:11.549 INFO:teuthology.orchestra.run.smithi115.stdout: "traverse_discover": 43, 2024-03-19T02:06:11.549 INFO:teuthology.orchestra.run.smithi115.stdout: "traverse_dir_fetch": 0, 2024-03-19T02:06:11.549 INFO:teuthology.orchestra.run.smithi115.stdout: "traverse_remote_ino": 0, 2024-03-19T02:06:11.549 INFO:teuthology.orchestra.run.smithi115.stdout: "traverse_lock": 6056, 2024-03-19T02:06:11.549 INFO:teuthology.orchestra.run.smithi115.stdout: "load_cent": 0, 2024-03-19T02:06:11.549 INFO:teuthology.orchestra.run.smithi115.stdout: "q": 0, 2024-03-19T02:06:11.549 INFO:teuthology.orchestra.run.smithi115.stdout: "exported": 14, 2024-03-19T02:06:11.549 INFO:teuthology.orchestra.run.smithi115.stdout: "exported_inodes": 186, 2024-03-19T02:06:11.549 INFO:teuthology.orchestra.run.smithi115.stdout: "imported": 4, 2024-03-19T02:06:11.549 INFO:teuthology.orchestra.run.smithi115.stdout: "imported_inodes": 62, 2024-03-19T02:06:11.549 INFO:teuthology.orchestra.run.smithi115.stdout: "openino_dir_fetch": 0, 2024-03-19T02:06:11.549 INFO:teuthology.orchestra.run.smithi115.stdout: "openino_backtrace_fetch": 0, 2024-03-19T02:06:11.549 INFO:teuthology.orchestra.run.smithi115.stdout: "openino_peer_discover": 0, 2024-03-19T02:06:11.550 INFO:teuthology.orchestra.run.smithi115.stdout: "root_rfiles": 4, 2024-03-19T02:06:11.550 INFO:teuthology.orchestra.run.smithi115.stdout: "root_rbytes": 234, 2024-03-19T02:06:11.550 INFO:teuthology.orchestra.run.smithi115.stdout: "root_rsnaps": 0, 2024-03-19T02:06:11.550 INFO:teuthology.orchestra.run.smithi115.stdout: "scrub_backtrace_fetch": 0, 2024-03-19T02:06:11.550 INFO:teuthology.orchestra.run.smithi115.stdout: "scrub_set_tag": 0, 2024-03-19T02:06:11.550 INFO:teuthology.orchestra.run.smithi115.stdout: "scrub_backtrace_repaired": 0, 2024-03-19T02:06:11.550 INFO:teuthology.orchestra.run.smithi115.stdout: "scrub_inotable_repaired": 0, 2024-03-19T02:06:11.550 INFO:teuthology.orchestra.run.smithi115.stdout: "scrub_dir_inodes": 0, 2024-03-19T02:06:11.550 INFO:teuthology.orchestra.run.smithi115.stdout: "scrub_dir_base_inodes": 0, 2024-03-19T02:06:11.550 INFO:teuthology.orchestra.run.smithi115.stdout: "scrub_dirfrag_rstats": 0, 2024-03-19T02:06:11.550 INFO:teuthology.orchestra.run.smithi115.stdout: "scrub_file_inodes": 0, 2024-03-19T02:06:11.550 INFO:teuthology.orchestra.run.smithi115.stdout: "handle_inode_file_caps": 60, 2024-03-19T02:06:11.550 INFO:teuthology.orchestra.run.smithi115.stdout: "ceph_cap_op_revoke": 35977, 2024-03-19T02:06:11.550 INFO:teuthology.orchestra.run.smithi115.stdout: "ceph_cap_op_grant": 276514, 2024-03-19T02:06:11.551 INFO:teuthology.orchestra.run.smithi115.stdout: "ceph_cap_op_trunc": 0, 2024-03-19T02:06:11.551 INFO:teuthology.orchestra.run.smithi115.stdout: "ceph_cap_op_flushsnap_ack": 0, 2024-03-19T02:06:11.551 INFO:teuthology.orchestra.run.smithi115.stdout: "ceph_cap_op_flush_ack": 0, 2024-03-19T02:06:11.551 INFO:teuthology.orchestra.run.smithi115.stdout: "handle_client_caps": 141666, 2024-03-19T02:06:11.551 INFO:teuthology.orchestra.run.smithi115.stdout: "handle_client_caps_dirty": 61256, 2024-03-19T02:06:11.551 INFO:teuthology.orchestra.run.smithi115.stdout: "handle_client_cap_release": 26697, 2024-03-19T02:06:11.551 INFO:teuthology.orchestra.run.smithi115.stdout: "process_request_cap_release": 285554 2024-03-19T02:06:11.551 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.551 INFO:teuthology.orchestra.run.smithi115.stdout: "mds_cache": { 2024-03-19T02:06:11.551 INFO:teuthology.orchestra.run.smithi115.stdout: "dir_update": 693, 2024-03-19T02:06:11.552 INFO:teuthology.orchestra.run.smithi115.stdout: "dir_update_receipt": 625, 2024-03-19T02:06:11.552 INFO:teuthology.orchestra.run.smithi115.stdout: "dir_try_discover": 322, 2024-03-19T02:06:11.552 INFO:teuthology.orchestra.run.smithi115.stdout: "dir_send_discover": 641, 2024-03-19T02:06:11.552 INFO:teuthology.orchestra.run.smithi115.stdout: "dir_handle_discover": 1043, 2024-03-19T02:06:11.552 INFO:teuthology.orchestra.run.smithi115.stdout: "num_strays": 31, 2024-03-19T02:06:11.552 INFO:teuthology.orchestra.run.smithi115.stdout: "num_strays_delayed": 0, 2024-03-19T02:06:11.552 INFO:teuthology.orchestra.run.smithi115.stdout: "num_strays_enqueuing": 0, 2024-03-19T02:06:11.552 INFO:teuthology.orchestra.run.smithi115.stdout: "strays_created": 66008, 2024-03-19T02:06:11.552 INFO:teuthology.orchestra.run.smithi115.stdout: "strays_enqueued": 65977, 2024-03-19T02:06:11.552 INFO:teuthology.orchestra.run.smithi115.stdout: "strays_reintegrated": 0, 2024-03-19T02:06:11.552 INFO:teuthology.orchestra.run.smithi115.stdout: "strays_migrated": 0, 2024-03-19T02:06:11.552 INFO:teuthology.orchestra.run.smithi115.stdout: "num_recovering_processing": 0, 2024-03-19T02:06:11.552 INFO:teuthology.orchestra.run.smithi115.stdout: "num_recovering_enqueued": 0, 2024-03-19T02:06:11.552 INFO:teuthology.orchestra.run.smithi115.stdout: "num_recovering_prioritized": 0, 2024-03-19T02:06:11.552 INFO:teuthology.orchestra.run.smithi115.stdout: "recovery_started": 0, 2024-03-19T02:06:11.553 INFO:teuthology.orchestra.run.smithi115.stdout: "recovery_completed": 0, 2024-03-19T02:06:11.553 INFO:teuthology.orchestra.run.smithi115.stdout: "ireq_quiesce_path": 0, 2024-03-19T02:06:11.553 INFO:teuthology.orchestra.run.smithi115.stdout: "ireq_quiesce_inode": 0, 2024-03-19T02:06:11.553 INFO:teuthology.orchestra.run.smithi115.stdout: "ireq_enqueue_scrub": 0, 2024-03-19T02:06:11.553 INFO:teuthology.orchestra.run.smithi115.stdout: "ireq_exportdir": 15, 2024-03-19T02:06:11.553 INFO:teuthology.orchestra.run.smithi115.stdout: "ireq_flush": 0, 2024-03-19T02:06:11.553 INFO:teuthology.orchestra.run.smithi115.stdout: "ireq_fragmentdir": 72, 2024-03-19T02:06:11.553 INFO:teuthology.orchestra.run.smithi115.stdout: "ireq_fragstats": 0, 2024-03-19T02:06:11.553 INFO:teuthology.orchestra.run.smithi115.stdout: "ireq_inodestats": 0 2024-03-19T02:06:11.553 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.553 INFO:teuthology.orchestra.run.smithi115.stdout: "mds_log": { 2024-03-19T02:06:11.553 INFO:teuthology.orchestra.run.smithi115.stdout: "evlrg": 0, 2024-03-19T02:06:11.553 INFO:teuthology.orchestra.run.smithi115.stdout: "evadd": 340860, 2024-03-19T02:06:11.553 INFO:teuthology.orchestra.run.smithi115.stdout: "evex": 336084, 2024-03-19T02:06:11.553 INFO:teuthology.orchestra.run.smithi115.stdout: "evtrm": 336084, 2024-03-19T02:06:11.553 INFO:teuthology.orchestra.run.smithi115.stdout: "ev": 4776, 2024-03-19T02:06:11.554 INFO:teuthology.orchestra.run.smithi115.stdout: "evexg": 0, 2024-03-19T02:06:11.554 INFO:teuthology.orchestra.run.smithi115.stdout: "evexd": 5221, 2024-03-19T02:06:11.554 INFO:teuthology.orchestra.run.smithi115.stdout: "segadd": 454, 2024-03-19T02:06:11.554 INFO:teuthology.orchestra.run.smithi115.stdout: "segex": 448, 2024-03-19T02:06:11.554 INFO:teuthology.orchestra.run.smithi115.stdout: "segtrm": 448, 2024-03-19T02:06:11.554 INFO:teuthology.orchestra.run.smithi115.stdout: "seg": 6, 2024-03-19T02:06:11.554 INFO:teuthology.orchestra.run.smithi115.stdout: "segmjr": 1, 2024-03-19T02:06:11.554 INFO:teuthology.orchestra.run.smithi115.stdout: "segexg": 0, 2024-03-19T02:06:11.554 INFO:teuthology.orchestra.run.smithi115.stdout: "segexd": 8, 2024-03-19T02:06:11.554 INFO:teuthology.orchestra.run.smithi115.stdout: "expos": 658779933, 2024-03-19T02:06:11.554 INFO:teuthology.orchestra.run.smithi115.stdout: "wrpos": 666995371, 2024-03-19T02:06:11.554 INFO:teuthology.orchestra.run.smithi115.stdout: "rdpos": 0, 2024-03-19T02:06:11.554 INFO:teuthology.orchestra.run.smithi115.stdout: "jlat": { 2024-03-19T02:06:11.554 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 74888, 2024-03-19T02:06:11.555 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 1143.354694285, 2024-03-19T02:06:11.555 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.015267528 2024-03-19T02:06:11.555 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.555 INFO:teuthology.orchestra.run.smithi115.stdout: "replayed": 0 2024-03-19T02:06:11.555 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.555 INFO:teuthology.orchestra.run.smithi115.stdout: "mds_mem": { 2024-03-19T02:06:11.555 INFO:teuthology.orchestra.run.smithi115.stdout: "ino": 89, 2024-03-19T02:06:11.555 INFO:teuthology.orchestra.run.smithi115.stdout: "ino+": 66194, 2024-03-19T02:06:11.555 INFO:teuthology.orchestra.run.smithi115.stdout: "ino-": 66105, 2024-03-19T02:06:11.555 INFO:teuthology.orchestra.run.smithi115.stdout: "dir": 97, 2024-03-19T02:06:11.555 INFO:teuthology.orchestra.run.smithi115.stdout: "dir+": 720, 2024-03-19T02:06:11.555 INFO:teuthology.orchestra.run.smithi115.stdout: "dir-": 623, 2024-03-19T02:06:11.555 INFO:teuthology.orchestra.run.smithi115.stdout: "dn": 2630, 2024-03-19T02:06:11.555 INFO:teuthology.orchestra.run.smithi115.stdout: "dn+": 152279, 2024-03-19T02:06:11.555 INFO:teuthology.orchestra.run.smithi115.stdout: "dn-": 149649, 2024-03-19T02:06:11.556 INFO:teuthology.orchestra.run.smithi115.stdout: "cap": 1, 2024-03-19T02:06:11.556 INFO:teuthology.orchestra.run.smithi115.stdout: "cap+": 71440, 2024-03-19T02:06:11.556 INFO:teuthology.orchestra.run.smithi115.stdout: "cap-": 71439, 2024-03-19T02:06:11.556 INFO:teuthology.orchestra.run.smithi115.stdout: "rss": 102216, 2024-03-19T02:06:11.556 INFO:teuthology.orchestra.run.smithi115.stdout: "heap": 231804 2024-03-19T02:06:11.556 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.556 INFO:teuthology.orchestra.run.smithi115.stdout: "mds_server": { 2024-03-19T02:06:11.556 INFO:teuthology.orchestra.run.smithi115.stdout: "dispatch_client_request": 612653, 2024-03-19T02:06:11.556 INFO:teuthology.orchestra.run.smithi115.stdout: "dispatch_server_request": 105, 2024-03-19T02:06:11.556 INFO:teuthology.orchestra.run.smithi115.stdout: "handle_client_request": 447680, 2024-03-19T02:06:11.556 INFO:teuthology.orchestra.run.smithi115.stdout: "handle_client_session": 10086, 2024-03-19T02:06:11.556 INFO:teuthology.orchestra.run.smithi115.stdout: "handle_peer_request": 23146, 2024-03-19T02:06:11.556 INFO:teuthology.orchestra.run.smithi115.stdout: "req_create_latency": { 2024-03-19T02:06:11.556 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 66000, 2024-03-19T02:06:11.556 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 893.027629399, 2024-03-19T02:06:11.557 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.013530721 2024-03-19T02:06:11.557 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.557 INFO:teuthology.orchestra.run.smithi115.stdout: "req_getattr_latency": { 2024-03-19T02:06:11.557 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 14863, 2024-03-19T02:06:11.557 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 348.333048095, 2024-03-19T02:06:11.557 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.023436254 2024-03-19T02:06:11.557 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.557 INFO:teuthology.orchestra.run.smithi115.stdout: "req_getfilelock_latency": { 2024-03-19T02:06:11.557 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:11.557 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:11.557 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:11.557 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.557 INFO:teuthology.orchestra.run.smithi115.stdout: "req_link_latency": { 2024-03-19T02:06:11.557 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:11.557 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:11.558 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:11.558 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.558 INFO:teuthology.orchestra.run.smithi115.stdout: "req_lookup_latency": { 2024-03-19T02:06:11.558 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 130078, 2024-03-19T02:06:11.558 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 1283.394015488, 2024-03-19T02:06:11.558 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.009866341 2024-03-19T02:06:11.558 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.558 INFO:teuthology.orchestra.run.smithi115.stdout: "req_lookuphash_latency": { 2024-03-19T02:06:11.558 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:11.558 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:11.558 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:11.558 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.558 INFO:teuthology.orchestra.run.smithi115.stdout: "req_lookupino_latency": { 2024-03-19T02:06:11.558 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:11.559 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:11.559 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:11.559 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.559 INFO:teuthology.orchestra.run.smithi115.stdout: "req_lookupname_latency": { 2024-03-19T02:06:11.559 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:11.559 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:11.559 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:11.559 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.559 INFO:teuthology.orchestra.run.smithi115.stdout: "req_lookupparent_latency": { 2024-03-19T02:06:11.559 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:11.559 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:11.559 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:11.559 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.559 INFO:teuthology.orchestra.run.smithi115.stdout: "req_lookupsnap_latency": { 2024-03-19T02:06:11.560 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:11.560 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:11.560 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:11.560 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.560 INFO:teuthology.orchestra.run.smithi115.stdout: "req_lssnap_latency": { 2024-03-19T02:06:11.560 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:11.561 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:11.561 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:11.561 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.561 INFO:teuthology.orchestra.run.smithi115.stdout: "req_mkdir_latency": { 2024-03-19T02:06:11.561 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 109, 2024-03-19T02:06:11.561 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.927139105, 2024-03-19T02:06:11.561 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.008505863 2024-03-19T02:06:11.561 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.561 INFO:teuthology.orchestra.run.smithi115.stdout: "req_mknod_latency": { 2024-03-19T02:06:11.561 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:11.561 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:11.561 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:11.561 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.561 INFO:teuthology.orchestra.run.smithi115.stdout: "req_mksnap_latency": { 2024-03-19T02:06:11.562 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:11.562 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:11.562 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:11.562 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.562 INFO:teuthology.orchestra.run.smithi115.stdout: "req_open_latency": { 2024-03-19T02:06:11.562 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 10, 2024-03-19T02:06:11.562 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.013362700, 2024-03-19T02:06:11.562 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.001336270 2024-03-19T02:06:11.562 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.562 INFO:teuthology.orchestra.run.smithi115.stdout: "req_readdir_latency": { 2024-03-19T02:06:11.562 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 103701, 2024-03-19T02:06:11.562 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 1463.667783112, 2024-03-19T02:06:11.562 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.014114307 2024-03-19T02:06:11.562 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.563 INFO:teuthology.orchestra.run.smithi115.stdout: "req_rename_latency": { 2024-03-19T02:06:11.563 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 12529, 2024-03-19T02:06:11.563 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 178.827473907, 2024-03-19T02:06:11.563 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.014273084 2024-03-19T02:06:11.563 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.563 INFO:teuthology.orchestra.run.smithi115.stdout: "req_renamesnap_latency": { 2024-03-19T02:06:11.563 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:11.563 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:11.563 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:11.563 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.563 INFO:teuthology.orchestra.run.smithi115.stdout: "req_snapdiff_latency": { 2024-03-19T02:06:11.563 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:11.563 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:11.563 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:11.564 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.564 INFO:teuthology.orchestra.run.smithi115.stdout: "req_rmdir_latency": { 2024-03-19T02:06:11.564 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 95, 2024-03-19T02:06:11.564 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.935221617, 2024-03-19T02:06:11.564 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.009844438 2024-03-19T02:06:11.564 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.564 INFO:teuthology.orchestra.run.smithi115.stdout: "req_rmsnap_latency": { 2024-03-19T02:06:11.564 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:11.564 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:11.564 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:11.564 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.564 INFO:teuthology.orchestra.run.smithi115.stdout: "req_rmxattr_latency": { 2024-03-19T02:06:11.564 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:11.564 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:11.565 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:11.565 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.565 INFO:teuthology.orchestra.run.smithi115.stdout: "req_setattr_latency": { 2024-03-19T02:06:11.565 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 350, 2024-03-19T02:06:11.565 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 11.791009717, 2024-03-19T02:06:11.565 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.033688599 2024-03-19T02:06:11.565 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.565 INFO:teuthology.orchestra.run.smithi115.stdout: "req_setdirlayout_latency": { 2024-03-19T02:06:11.565 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:11.565 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:11.565 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:11.565 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.565 INFO:teuthology.orchestra.run.smithi115.stdout: "req_setfilelock_latency": { 2024-03-19T02:06:11.566 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 1764, 2024-03-19T02:06:11.566 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 19.390379013, 2024-03-19T02:06:11.566 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.010992278 2024-03-19T02:06:11.566 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.566 INFO:teuthology.orchestra.run.smithi115.stdout: "req_setlayout_latency": { 2024-03-19T02:06:11.566 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:11.566 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:11.566 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:11.566 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.566 INFO:teuthology.orchestra.run.smithi115.stdout: "req_setxattr_latency": { 2024-03-19T02:06:11.566 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 8, 2024-03-19T02:06:11.566 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.017853221, 2024-03-19T02:06:11.566 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.002231652 2024-03-19T02:06:11.566 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.567 INFO:teuthology.orchestra.run.smithi115.stdout: "req_symlink_latency": { 2024-03-19T02:06:11.567 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:11.567 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:11.567 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:11.567 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.567 INFO:teuthology.orchestra.run.smithi115.stdout: "req_unlink_latency": { 2024-03-19T02:06:11.567 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 65924, 2024-03-19T02:06:11.567 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 777.689441901, 2024-03-19T02:06:11.567 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.011796757 2024-03-19T02:06:11.567 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.567 INFO:teuthology.orchestra.run.smithi115.stdout: "cap_revoke_eviction": 0, 2024-03-19T02:06:11.567 INFO:teuthology.orchestra.run.smithi115.stdout: "cap_acquisition_throttle": 0, 2024-03-19T02:06:11.567 INFO:teuthology.orchestra.run.smithi115.stdout: "req_getvxattr_latency": { 2024-03-19T02:06:11.567 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:11.568 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:11.568 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:11.568 INFO:teuthology.orchestra.run.smithi115.stdout: } 2024-03-19T02:06:11.568 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.568 INFO:teuthology.orchestra.run.smithi115.stdout: "mds_sessions": { 2024-03-19T02:06:11.568 INFO:teuthology.orchestra.run.smithi115.stdout: "session_count": 1, 2024-03-19T02:06:11.568 INFO:teuthology.orchestra.run.smithi115.stdout: "session_add": 3, 2024-03-19T02:06:11.568 INFO:teuthology.orchestra.run.smithi115.stdout: "session_remove": 2, 2024-03-19T02:06:11.568 INFO:teuthology.orchestra.run.smithi115.stdout: "sessions_open": 1, 2024-03-19T02:06:11.568 INFO:teuthology.orchestra.run.smithi115.stdout: "sessions_stale": 0, 2024-03-19T02:06:11.568 INFO:teuthology.orchestra.run.smithi115.stdout: "total_load": 39845, 2024-03-19T02:06:11.568 INFO:teuthology.orchestra.run.smithi115.stdout: "average_load": 39845, 2024-03-19T02:06:11.568 INFO:teuthology.orchestra.run.smithi115.stdout: "avg_session_uptime": 1519, 2024-03-19T02:06:11.568 INFO:teuthology.orchestra.run.smithi115.stdout: "mdthresh_evicted": 0 2024-03-19T02:06:11.568 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.568 INFO:teuthology.orchestra.run.smithi115.stdout: "mempool": { 2024-03-19T02:06:11.569 INFO:teuthology.orchestra.run.smithi115.stdout: "bloom_filter_bytes": 360, 2024-03-19T02:06:11.569 INFO:teuthology.orchestra.run.smithi115.stdout: "bloom_filter_items": 360, 2024-03-19T02:06:11.569 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_alloc_bytes": 0, 2024-03-19T02:06:11.569 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_alloc_items": 0, 2024-03-19T02:06:11.569 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_cache_data_bytes": 0, 2024-03-19T02:06:11.569 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_cache_data_items": 0, 2024-03-19T02:06:11.569 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_cache_onode_bytes": 0, 2024-03-19T02:06:11.569 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_cache_onode_items": 0, 2024-03-19T02:06:11.569 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_cache_meta_bytes": 0, 2024-03-19T02:06:11.569 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_cache_meta_items": 0, 2024-03-19T02:06:11.569 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_cache_other_bytes": 0, 2024-03-19T02:06:11.569 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_cache_other_items": 0, 2024-03-19T02:06:11.569 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_cache_buffer_bytes": 0, 2024-03-19T02:06:11.569 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_cache_buffer_items": 0, 2024-03-19T02:06:11.569 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_extent_bytes": 0, 2024-03-19T02:06:11.570 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_extent_items": 0, 2024-03-19T02:06:11.570 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_blob_bytes": 0, 2024-03-19T02:06:11.570 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_blob_items": 0, 2024-03-19T02:06:11.570 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_shared_blob_bytes": 0, 2024-03-19T02:06:11.570 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_shared_blob_items": 0, 2024-03-19T02:06:11.570 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_inline_bl_bytes": 0, 2024-03-19T02:06:11.570 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_inline_bl_items": 0, 2024-03-19T02:06:11.570 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_fsck_bytes": 0, 2024-03-19T02:06:11.570 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_fsck_items": 0, 2024-03-19T02:06:11.570 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_txc_bytes": 0, 2024-03-19T02:06:11.570 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_txc_items": 0, 2024-03-19T02:06:11.570 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_writing_deferred_bytes": 0, 2024-03-19T02:06:11.570 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_writing_deferred_items": 0, 2024-03-19T02:06:11.570 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_writing_bytes": 0, 2024-03-19T02:06:11.571 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_writing_items": 0, 2024-03-19T02:06:11.571 INFO:teuthology.orchestra.run.smithi115.stdout: "bluefs_bytes": 0, 2024-03-19T02:06:11.571 INFO:teuthology.orchestra.run.smithi115.stdout: "bluefs_items": 0, 2024-03-19T02:06:11.571 INFO:teuthology.orchestra.run.smithi115.stdout: "bluefs_file_reader_bytes": 0, 2024-03-19T02:06:11.571 INFO:teuthology.orchestra.run.smithi115.stdout: "bluefs_file_reader_items": 0, 2024-03-19T02:06:11.571 INFO:teuthology.orchestra.run.smithi115.stdout: "bluefs_file_writer_bytes": 0, 2024-03-19T02:06:11.571 INFO:teuthology.orchestra.run.smithi115.stdout: "bluefs_file_writer_items": 0, 2024-03-19T02:06:11.571 INFO:teuthology.orchestra.run.smithi115.stdout: "buffer_anon_bytes": 797332, 2024-03-19T02:06:11.571 INFO:teuthology.orchestra.run.smithi115.stdout: "buffer_anon_items": 314, 2024-03-19T02:06:11.571 INFO:teuthology.orchestra.run.smithi115.stdout: "buffer_meta_bytes": 0, 2024-03-19T02:06:11.571 INFO:teuthology.orchestra.run.smithi115.stdout: "buffer_meta_items": 0, 2024-03-19T02:06:11.571 INFO:teuthology.orchestra.run.smithi115.stdout: "osd_bytes": 0, 2024-03-19T02:06:11.571 INFO:teuthology.orchestra.run.smithi115.stdout: "osd_items": 0, 2024-03-19T02:06:11.571 INFO:teuthology.orchestra.run.smithi115.stdout: "osd_mapbl_bytes": 0, 2024-03-19T02:06:11.571 INFO:teuthology.orchestra.run.smithi115.stdout: "osd_mapbl_items": 0, 2024-03-19T02:06:11.571 INFO:teuthology.orchestra.run.smithi115.stdout: "osd_pglog_bytes": 0, 2024-03-19T02:06:11.571 INFO:teuthology.orchestra.run.smithi115.stdout: "osd_pglog_items": 0, 2024-03-19T02:06:11.572 INFO:teuthology.orchestra.run.smithi115.stdout: "osdmap_bytes": 7816, 2024-03-19T02:06:11.572 INFO:teuthology.orchestra.run.smithi115.stdout: "osdmap_items": 164, 2024-03-19T02:06:11.572 INFO:teuthology.orchestra.run.smithi115.stdout: "osdmap_mapping_bytes": 0, 2024-03-19T02:06:11.572 INFO:teuthology.orchestra.run.smithi115.stdout: "osdmap_mapping_items": 0, 2024-03-19T02:06:11.572 INFO:teuthology.orchestra.run.smithi115.stdout: "pgmap_bytes": 0, 2024-03-19T02:06:11.572 INFO:teuthology.orchestra.run.smithi115.stdout: "pgmap_items": 0, 2024-03-19T02:06:11.572 INFO:teuthology.orchestra.run.smithi115.stdout: "mds_co_bytes": 2445691, 2024-03-19T02:06:11.572 INFO:teuthology.orchestra.run.smithi115.stdout: "mds_co_items": 42071, 2024-03-19T02:06:11.572 INFO:teuthology.orchestra.run.smithi115.stdout: "unittest_1_bytes": 0, 2024-03-19T02:06:11.572 INFO:teuthology.orchestra.run.smithi115.stdout: "unittest_1_items": 0, 2024-03-19T02:06:11.572 INFO:teuthology.orchestra.run.smithi115.stdout: "unittest_2_bytes": 0, 2024-03-19T02:06:11.572 INFO:teuthology.orchestra.run.smithi115.stdout: "unittest_2_items": 0 2024-03-19T02:06:11.572 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.572 INFO:teuthology.orchestra.run.smithi115.stdout: "objecter": { 2024-03-19T02:06:11.572 INFO:teuthology.orchestra.run.smithi115.stdout: "op_active": 0, 2024-03-19T02:06:11.573 INFO:teuthology.orchestra.run.smithi115.stdout: "op_laggy": 0, 2024-03-19T02:06:11.573 INFO:teuthology.orchestra.run.smithi115.stdout: "op_send": 197296, 2024-03-19T02:06:11.573 INFO:teuthology.orchestra.run.smithi115.stdout: "op_send_bytes": 728360426, 2024-03-19T02:06:11.573 INFO:teuthology.orchestra.run.smithi115.stdout: "op_resend": 31, 2024-03-19T02:06:11.573 INFO:teuthology.orchestra.run.smithi115.stdout: "op_reply": 197296, 2024-03-19T02:06:11.573 INFO:teuthology.orchestra.run.smithi115.stdout: "op_latency": { 2024-03-19T02:06:11.573 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 197296, 2024-03-19T02:06:11.573 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 8803.591616289, 2024-03-19T02:06:11.573 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.044621237 2024-03-19T02:06:11.573 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.573 INFO:teuthology.orchestra.run.smithi115.stdout: "op_inflight": 0, 2024-03-19T02:06:11.573 INFO:teuthology.orchestra.run.smithi115.stdout: "oplen_avg": { 2024-03-19T02:06:11.573 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 197296, 2024-03-19T02:06:11.574 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 301215 2024-03-19T02:06:11.574 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.574 INFO:teuthology.orchestra.run.smithi115.stdout: "op": 197296, 2024-03-19T02:06:11.574 INFO:teuthology.orchestra.run.smithi115.stdout: "op_r": 1844, 2024-03-19T02:06:11.574 INFO:teuthology.orchestra.run.smithi115.stdout: "op_w": 195452, 2024-03-19T02:06:11.574 INFO:teuthology.orchestra.run.smithi115.stdout: "op_rmw": 0, 2024-03-19T02:06:11.574 INFO:teuthology.orchestra.run.smithi115.stdout: "op_pg": 0, 2024-03-19T02:06:11.574 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_stat": 7865, 2024-03-19T02:06:11.574 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_create": 40496, 2024-03-19T02:06:11.574 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_read": 1844, 2024-03-19T02:06:11.574 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_write": 76732, 2024-03-19T02:06:11.574 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_writefull": 1307, 2024-03-19T02:06:11.574 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_writesame": 0, 2024-03-19T02:06:11.574 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_append": 0, 2024-03-19T02:06:11.575 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_zero": 0, 2024-03-19T02:06:11.575 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_truncate": 27, 2024-03-19T02:06:11.575 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_delete": 67009, 2024-03-19T02:06:11.575 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_mapext": 0, 2024-03-19T02:06:11.575 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_sparse_read": 0, 2024-03-19T02:06:11.575 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_clonerange": 0, 2024-03-19T02:06:11.575 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_getxattr": 0, 2024-03-19T02:06:11.575 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_setxattr": 80992, 2024-03-19T02:06:11.575 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_cmpxattr": 0, 2024-03-19T02:06:11.575 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_rmxattr": 0, 2024-03-19T02:06:11.575 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_resetxattrs": 0, 2024-03-19T02:06:11.575 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_call": 0, 2024-03-19T02:06:11.575 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_watch": 0, 2024-03-19T02:06:11.575 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_notify": 0, 2024-03-19T02:06:11.575 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_src_cmpxattr": 0, 2024-03-19T02:06:11.576 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_pgls": 0, 2024-03-19T02:06:11.576 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_pgls_filter": 0, 2024-03-19T02:06:11.576 INFO:teuthology.orchestra.run.smithi115.stdout: "osdop_other": 0, 2024-03-19T02:06:11.576 INFO:teuthology.orchestra.run.smithi115.stdout: "linger_active": 0, 2024-03-19T02:06:11.576 INFO:teuthology.orchestra.run.smithi115.stdout: "linger_send": 0, 2024-03-19T02:06:11.576 INFO:teuthology.orchestra.run.smithi115.stdout: "linger_resend": 0, 2024-03-19T02:06:11.576 INFO:teuthology.orchestra.run.smithi115.stdout: "linger_ping": 0, 2024-03-19T02:06:11.576 INFO:teuthology.orchestra.run.smithi115.stdout: "poolop_active": 0, 2024-03-19T02:06:11.576 INFO:teuthology.orchestra.run.smithi115.stdout: "poolop_send": 0, 2024-03-19T02:06:11.576 INFO:teuthology.orchestra.run.smithi115.stdout: "poolop_resend": 0, 2024-03-19T02:06:11.576 INFO:teuthology.orchestra.run.smithi115.stdout: "poolstat_active": 0, 2024-03-19T02:06:11.576 INFO:teuthology.orchestra.run.smithi115.stdout: "poolstat_send": 0, 2024-03-19T02:06:11.576 INFO:teuthology.orchestra.run.smithi115.stdout: "poolstat_resend": 0, 2024-03-19T02:06:11.576 INFO:teuthology.orchestra.run.smithi115.stdout: "statfs_active": 0, 2024-03-19T02:06:11.577 INFO:teuthology.orchestra.run.smithi115.stdout: "statfs_send": 0, 2024-03-19T02:06:11.577 INFO:teuthology.orchestra.run.smithi115.stdout: "statfs_resend": 0, 2024-03-19T02:06:11.577 INFO:teuthology.orchestra.run.smithi115.stdout: "command_active": 0, 2024-03-19T02:06:11.577 INFO:teuthology.orchestra.run.smithi115.stdout: "command_send": 0, 2024-03-19T02:06:11.577 INFO:teuthology.orchestra.run.smithi115.stdout: "command_resend": 0, 2024-03-19T02:06:11.577 INFO:teuthology.orchestra.run.smithi115.stdout: "map_epoch": 92, 2024-03-19T02:06:11.577 INFO:teuthology.orchestra.run.smithi115.stdout: "map_full": 0, 2024-03-19T02:06:11.577 INFO:teuthology.orchestra.run.smithi115.stdout: "map_inc": 6, 2024-03-19T02:06:11.577 INFO:teuthology.orchestra.run.smithi115.stdout: "osd_sessions": 12, 2024-03-19T02:06:11.577 INFO:teuthology.orchestra.run.smithi115.stdout: "osd_session_open": 12, 2024-03-19T02:06:11.577 INFO:teuthology.orchestra.run.smithi115.stdout: "osd_session_close": 0, 2024-03-19T02:06:11.577 INFO:teuthology.orchestra.run.smithi115.stdout: "osd_laggy": 0, 2024-03-19T02:06:11.577 INFO:teuthology.orchestra.run.smithi115.stdout: "omap_wr": 18375, 2024-03-19T02:06:11.577 INFO:teuthology.orchestra.run.smithi115.stdout: "omap_rd": 0, 2024-03-19T02:06:11.577 INFO:teuthology.orchestra.run.smithi115.stdout: "omap_del": 6568 2024-03-19T02:06:11.578 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.578 INFO:teuthology.orchestra.run.smithi115.stdout: "oft": { 2024-03-19T02:06:11.578 INFO:teuthology.orchestra.run.smithi115.stdout: "omap_total_objs": 1, 2024-03-19T02:06:11.578 INFO:teuthology.orchestra.run.smithi115.stdout: "omap_total_kv_pairs": 0, 2024-03-19T02:06:11.578 INFO:teuthology.orchestra.run.smithi115.stdout: "omap_total_updates": 0, 2024-03-19T02:06:11.578 INFO:teuthology.orchestra.run.smithi115.stdout: "omap_total_removes": 0 2024-03-19T02:06:11.578 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.578 INFO:teuthology.orchestra.run.smithi115.stdout: "osd-slow-ops": { 2024-03-19T02:06:11.578 INFO:teuthology.orchestra.run.smithi115.stdout: "slow_ops_count": 0 2024-03-19T02:06:11.578 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.578 INFO:teuthology.orchestra.run.smithi115.stdout: "purge_queue": { 2024-03-19T02:06:11.578 INFO:teuthology.orchestra.run.smithi115.stdout: "pq_executing_ops": 0, 2024-03-19T02:06:11.578 INFO:teuthology.orchestra.run.smithi115.stdout: "pq_executing_ops_high_water": 10, 2024-03-19T02:06:11.578 INFO:teuthology.orchestra.run.smithi115.stdout: "pq_executing": 0, 2024-03-19T02:06:11.578 INFO:teuthology.orchestra.run.smithi115.stdout: "pq_executing_high_water": 10, 2024-03-19T02:06:11.579 INFO:teuthology.orchestra.run.smithi115.stdout: "pq_executed_ops": 65977, 2024-03-19T02:06:11.579 INFO:teuthology.orchestra.run.smithi115.stdout: "pq_executed": 65977, 2024-03-19T02:06:11.579 INFO:teuthology.orchestra.run.smithi115.stdout: "pq_item_in_journal": 0 2024-03-19T02:06:11.579 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.579 INFO:teuthology.orchestra.run.smithi115.stdout: "throttle-msgr_dispatch_throttler-mds": { 2024-03-19T02:06:11.579 INFO:teuthology.orchestra.run.smithi115.stdout: "val": 0, 2024-03-19T02:06:11.579 INFO:teuthology.orchestra.run.smithi115.stdout: "max": 104857600, 2024-03-19T02:06:11.579 INFO:teuthology.orchestra.run.smithi115.stdout: "get_started": 0, 2024-03-19T02:06:11.579 INFO:teuthology.orchestra.run.smithi115.stdout: "get": 1464655, 2024-03-19T02:06:11.579 INFO:teuthology.orchestra.run.smithi115.stdout: "get_sum": 270149982, 2024-03-19T02:06:11.579 INFO:teuthology.orchestra.run.smithi115.stdout: "get_or_fail_fail": 0, 2024-03-19T02:06:11.579 INFO:teuthology.orchestra.run.smithi115.stdout: "get_or_fail_success": 1464655, 2024-03-19T02:06:11.579 INFO:teuthology.orchestra.run.smithi115.stdout: "take": 0, 2024-03-19T02:06:11.579 INFO:teuthology.orchestra.run.smithi115.stdout: "take_sum": 0, 2024-03-19T02:06:11.580 INFO:teuthology.orchestra.run.smithi115.stdout: "put": 1464655, 2024-03-19T02:06:11.580 INFO:teuthology.orchestra.run.smithi115.stdout: "put_sum": 270149982, 2024-03-19T02:06:11.580 INFO:teuthology.orchestra.run.smithi115.stdout: "wait": { 2024-03-19T02:06:11.580 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:11.580 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:11.580 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:11.580 INFO:teuthology.orchestra.run.smithi115.stdout: } 2024-03-19T02:06:11.580 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.580 INFO:teuthology.orchestra.run.smithi115.stdout: "throttle-objecter_bytes": { 2024-03-19T02:06:11.580 INFO:teuthology.orchestra.run.smithi115.stdout: "val": 0, 2024-03-19T02:06:11.580 INFO:teuthology.orchestra.run.smithi115.stdout: "max": 104857600, 2024-03-19T02:06:11.580 INFO:teuthology.orchestra.run.smithi115.stdout: "get_started": 0, 2024-03-19T02:06:11.580 INFO:teuthology.orchestra.run.smithi115.stdout: "get": 0, 2024-03-19T02:06:11.580 INFO:teuthology.orchestra.run.smithi115.stdout: "get_sum": 0, 2024-03-19T02:06:11.580 INFO:teuthology.orchestra.run.smithi115.stdout: "get_or_fail_fail": 0, 2024-03-19T02:06:11.581 INFO:teuthology.orchestra.run.smithi115.stdout: "get_or_fail_success": 0, 2024-03-19T02:06:11.581 INFO:teuthology.orchestra.run.smithi115.stdout: "take": 197296, 2024-03-19T02:06:11.581 INFO:teuthology.orchestra.run.smithi115.stdout: "take_sum": 735024103, 2024-03-19T02:06:11.581 INFO:teuthology.orchestra.run.smithi115.stdout: "put": 197296, 2024-03-19T02:06:11.581 INFO:teuthology.orchestra.run.smithi115.stdout: "put_sum": 735024103, 2024-03-19T02:06:11.581 INFO:teuthology.orchestra.run.smithi115.stdout: "wait": { 2024-03-19T02:06:11.581 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:11.581 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:11.581 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:11.581 INFO:teuthology.orchestra.run.smithi115.stdout: } 2024-03-19T02:06:11.581 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.581 INFO:teuthology.orchestra.run.smithi115.stdout: "throttle-objecter_ops": { 2024-03-19T02:06:11.581 INFO:teuthology.orchestra.run.smithi115.stdout: "val": 0, 2024-03-19T02:06:11.581 INFO:teuthology.orchestra.run.smithi115.stdout: "max": 1024, 2024-03-19T02:06:11.581 INFO:teuthology.orchestra.run.smithi115.stdout: "get_started": 0, 2024-03-19T02:06:11.582 INFO:teuthology.orchestra.run.smithi115.stdout: "get": 0, 2024-03-19T02:06:11.582 INFO:teuthology.orchestra.run.smithi115.stdout: "get_sum": 0, 2024-03-19T02:06:11.582 INFO:teuthology.orchestra.run.smithi115.stdout: "get_or_fail_fail": 0, 2024-03-19T02:06:11.582 INFO:teuthology.orchestra.run.smithi115.stdout: "get_or_fail_success": 0, 2024-03-19T02:06:11.582 INFO:teuthology.orchestra.run.smithi115.stdout: "take": 197296, 2024-03-19T02:06:11.582 INFO:teuthology.orchestra.run.smithi115.stdout: "take_sum": 197296, 2024-03-19T02:06:11.582 INFO:teuthology.orchestra.run.smithi115.stdout: "put": 197296, 2024-03-19T02:06:11.582 INFO:teuthology.orchestra.run.smithi115.stdout: "put_sum": 197296, 2024-03-19T02:06:11.582 INFO:teuthology.orchestra.run.smithi115.stdout: "wait": { 2024-03-19T02:06:11.582 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:11.582 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:11.582 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:11.582 INFO:teuthology.orchestra.run.smithi115.stdout: } 2024-03-19T02:06:11.582 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.583 INFO:teuthology.orchestra.run.smithi115.stdout: "throttle-write_buf_throttle": { 2024-03-19T02:06:11.583 INFO:teuthology.orchestra.run.smithi115.stdout: "val": 0, 2024-03-19T02:06:11.583 INFO:teuthology.orchestra.run.smithi115.stdout: "max": 3758096384, 2024-03-19T02:06:11.583 INFO:teuthology.orchestra.run.smithi115.stdout: "get_started": 0, 2024-03-19T02:06:11.583 INFO:teuthology.orchestra.run.smithi115.stdout: "get": 65977, 2024-03-19T02:06:11.583 INFO:teuthology.orchestra.run.smithi115.stdout: "get_sum": 6663677, 2024-03-19T02:06:11.583 INFO:teuthology.orchestra.run.smithi115.stdout: "get_or_fail_fail": 0, 2024-03-19T02:06:11.583 INFO:teuthology.orchestra.run.smithi115.stdout: "get_or_fail_success": 65977, 2024-03-19T02:06:11.583 INFO:teuthology.orchestra.run.smithi115.stdout: "take": 0, 2024-03-19T02:06:11.583 INFO:teuthology.orchestra.run.smithi115.stdout: "take_sum": 0, 2024-03-19T02:06:11.583 INFO:teuthology.orchestra.run.smithi115.stdout: "put": 1844, 2024-03-19T02:06:11.583 INFO:teuthology.orchestra.run.smithi115.stdout: "put_sum": 6663677, 2024-03-19T02:06:11.583 INFO:teuthology.orchestra.run.smithi115.stdout: "wait": { 2024-03-19T02:06:11.583 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:11.583 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:11.584 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:11.584 INFO:teuthology.orchestra.run.smithi115.stdout: } 2024-03-19T02:06:11.584 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:11.584 INFO:teuthology.orchestra.run.smithi115.stdout: "throttle-write_buf_throttle-0x55f85e322500": { 2024-03-19T02:06:11.584 INFO:teuthology.orchestra.run.smithi115.stdout: "val": 20965, 2024-03-19T02:06:11.584 INFO:teuthology.orchestra.run.smithi115.stdout: "max": 3758096384, 2024-03-19T02:06:11.584 INFO:teuthology.orchestra.run.smithi115.stdout: "get_started": 0, 2024-03-19T02:06:11.584 INFO:teuthology.orchestra.run.smithi115.stdout: "get": 340860, 2024-03-19T02:06:11.584 INFO:teuthology.orchestra.run.smithi115.stdout: "get_sum": 662801067, 2024-03-19T02:06:11.584 INFO:teuthology.orchestra.run.smithi115.stdout: "get_or_fail_fail": 0, 2024-03-19T02:06:11.584 INFO:teuthology.orchestra.run.smithi115.stdout: "get_or_fail_success": 340860, 2024-03-19T02:06:11.584 INFO:teuthology.orchestra.run.smithi115.stdout: "take": 0, 2024-03-19T02:06:11.584 INFO:teuthology.orchestra.run.smithi115.stdout: "take_sum": 0, 2024-03-19T02:06:11.584 INFO:teuthology.orchestra.run.smithi115.stdout: "put": 74888, 2024-03-19T02:06:11.585 INFO:teuthology.orchestra.run.smithi115.stdout: "put_sum": 662780102, 2024-03-19T02:06:11.585 INFO:teuthology.orchestra.run.smithi115.stdout: "wait": { 2024-03-19T02:06:11.585 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:11.585 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:11.585 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:11.585 INFO:teuthology.orchestra.run.smithi115.stdout: } 2024-03-19T02:06:11.585 INFO:teuthology.orchestra.run.smithi115.stdout: } 2024-03-19T02:06:11.585 INFO:teuthology.orchestra.run.smithi115.stdout:} 2024-03-19T02:06:12.287 INFO:tasks.check_counter:Daemon mds.i mds.exported=14 2024-03-19T02:06:12.287 INFO:tasks.check_counter:Daemon mds.i mds.imported=4 2024-03-19T02:06:12.287 INFO:tasks.check_counter:Daemon mds.i mds_cache.dir_update=693 2024-03-19T02:06:12.287 INFO:tasks.check_counter:Daemon mds.i mds_cache.dir_update_receipt=625 2024-03-19T02:06:12.287 DEBUG:tasks.check_counter:Getting stats from l 2024-03-19T02:06:12.288 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810 shell --fsid 438fb69c-e590-11ee-95c9-87774f69a715 -- ceph daemon mds.l perf dump 2024-03-19T02:06:12.551 INFO:teuthology.orchestra.run.smithi115.stderr:Inferring config /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/mon.c/config 2024-03-19T02:06:13.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:06:12 smithi115 ceph-mon[67332]: pgmap v1104: 97 pgs: 97 active+clean; 433 MiB data, 9.0 GiB used, 1.0 TiB / 1.0 TiB avail; 128 KiB/s rd, 2.0 MiB/s wr, 91 op/s 2024-03-19T02:06:13.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:06:12 smithi099 ceph-mon[62951]: pgmap v1104: 97 pgs: 97 active+clean; 433 MiB data, 9.0 GiB used, 1.0 TiB / 1.0 TiB avail; 128 KiB/s rd, 2.0 MiB/s wr, 91 op/s 2024-03-19T02:06:13.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:06:12 smithi112 ceph-mon[66819]: pgmap v1104: 97 pgs: 97 active+clean; 433 MiB data, 9.0 GiB used, 1.0 TiB / 1.0 TiB avail; 128 KiB/s rd, 2.0 MiB/s wr, 91 op/s 2024-03-19T02:06:14.185 INFO:teuthology.orchestra.run.smithi115.stdout:{ 2024-03-19T02:06:14.185 INFO:teuthology.orchestra.run.smithi115.stdout: "AsyncMessenger::Worker-0": { 2024-03-19T02:06:14.185 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_recv_messages": 4, 2024-03-19T02:06:14.185 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_send_messages": 4, 2024-03-19T02:06:14.185 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_recv_bytes": 3072, 2024-03-19T02:06:14.185 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_send_bytes": 448, 2024-03-19T02:06:14.186 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_created_connections": 4, 2024-03-19T02:06:14.186 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_active_connections": 0, 2024-03-19T02:06:14.186 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_running_total_time": 0.001447586, 2024-03-19T02:06:14.186 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_running_send_time": 0.000031281, 2024-03-19T02:06:14.186 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_running_recv_time": 0.000657270, 2024-03-19T02:06:14.186 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_running_fast_dispatch_time": 0.000000000, 2024-03-19T02:06:14.186 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_send_messages_queue_lat": { 2024-03-19T02:06:14.186 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 3, 2024-03-19T02:06:14.186 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000725898, 2024-03-19T02:06:14.186 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000241966 2024-03-19T02:06:14.186 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:14.186 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_handle_ack_lat": { 2024-03-19T02:06:14.186 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:14.187 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:14.187 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:14.187 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:14.187 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_recv_encrypted_bytes": 3072, 2024-03-19T02:06:14.187 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_send_encrypted_bytes": 448 2024-03-19T02:06:14.187 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:14.187 INFO:teuthology.orchestra.run.smithi115.stdout: "AsyncMessenger::Worker-1": { 2024-03-19T02:06:14.187 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_recv_messages": 2, 2024-03-19T02:06:14.187 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_send_messages": 339, 2024-03-19T02:06:14.187 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_recv_bytes": 320, 2024-03-19T02:06:14.187 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_send_bytes": 215600, 2024-03-19T02:06:14.187 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_created_connections": 4, 2024-03-19T02:06:14.187 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_active_connections": 2, 2024-03-19T02:06:14.187 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_running_total_time": 0.013196908, 2024-03-19T02:06:14.187 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_running_send_time": 0.010211484, 2024-03-19T02:06:14.188 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_running_recv_time": 0.000989163, 2024-03-19T02:06:14.188 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_running_fast_dispatch_time": 0.000000000, 2024-03-19T02:06:14.188 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_send_messages_queue_lat": { 2024-03-19T02:06:14.188 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 337, 2024-03-19T02:06:14.188 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.011132549, 2024-03-19T02:06:14.188 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000033034 2024-03-19T02:06:14.188 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:14.188 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_handle_ack_lat": { 2024-03-19T02:06:14.188 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:14.188 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:14.188 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:14.188 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:14.188 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_recv_encrypted_bytes": 320, 2024-03-19T02:06:14.188 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_send_encrypted_bytes": 215600 2024-03-19T02:06:14.189 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:14.189 INFO:teuthology.orchestra.run.smithi115.stdout: "AsyncMessenger::Worker-2": { 2024-03-19T02:06:14.189 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_recv_messages": 449, 2024-03-19T02:06:14.189 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_send_messages": 444, 2024-03-19T02:06:14.189 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_recv_bytes": 817104, 2024-03-19T02:06:14.189 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_send_bytes": 274832, 2024-03-19T02:06:14.189 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_created_connections": 2, 2024-03-19T02:06:14.189 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_active_connections": 1, 2024-03-19T02:06:14.189 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_running_total_time": 0.053394192, 2024-03-19T02:06:14.189 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_running_send_time": 0.019312764, 2024-03-19T02:06:14.189 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_running_recv_time": 0.025755094, 2024-03-19T02:06:14.189 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_running_fast_dispatch_time": 0.008982039, 2024-03-19T02:06:14.189 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_send_messages_queue_lat": { 2024-03-19T02:06:14.189 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 443, 2024-03-19T02:06:14.189 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.014167319, 2024-03-19T02:06:14.189 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000031980 2024-03-19T02:06:14.190 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:14.190 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_handle_ack_lat": { 2024-03-19T02:06:14.190 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:14.190 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:14.190 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:14.190 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:14.190 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_recv_encrypted_bytes": 817104, 2024-03-19T02:06:14.190 INFO:teuthology.orchestra.run.smithi115.stdout: "msgr_send_encrypted_bytes": 274832 2024-03-19T02:06:14.190 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:14.190 INFO:teuthology.orchestra.run.smithi115.stdout: "cct": { 2024-03-19T02:06:14.190 INFO:teuthology.orchestra.run.smithi115.stdout: "total_workers": 0, 2024-03-19T02:06:14.190 INFO:teuthology.orchestra.run.smithi115.stdout: "unhealthy_workers": 0 2024-03-19T02:06:14.190 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:14.190 INFO:teuthology.orchestra.run.smithi115.stdout: "mempool": { 2024-03-19T02:06:14.190 INFO:teuthology.orchestra.run.smithi115.stdout: "bloom_filter_bytes": 0, 2024-03-19T02:06:14.190 INFO:teuthology.orchestra.run.smithi115.stdout: "bloom_filter_items": 0, 2024-03-19T02:06:14.191 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_alloc_bytes": 0, 2024-03-19T02:06:14.191 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_alloc_items": 0, 2024-03-19T02:06:14.191 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_cache_data_bytes": 0, 2024-03-19T02:06:14.191 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_cache_data_items": 0, 2024-03-19T02:06:14.191 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_cache_onode_bytes": 0, 2024-03-19T02:06:14.191 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_cache_onode_items": 0, 2024-03-19T02:06:14.191 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_cache_meta_bytes": 0, 2024-03-19T02:06:14.191 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_cache_meta_items": 0, 2024-03-19T02:06:14.191 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_cache_other_bytes": 0, 2024-03-19T02:06:14.191 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_cache_other_items": 0, 2024-03-19T02:06:14.191 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_cache_buffer_bytes": 0, 2024-03-19T02:06:14.191 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_cache_buffer_items": 0, 2024-03-19T02:06:14.191 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_extent_bytes": 0, 2024-03-19T02:06:14.191 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_extent_items": 0, 2024-03-19T02:06:14.191 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_blob_bytes": 0, 2024-03-19T02:06:14.192 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_blob_items": 0, 2024-03-19T02:06:14.192 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_shared_blob_bytes": 0, 2024-03-19T02:06:14.192 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_shared_blob_items": 0, 2024-03-19T02:06:14.192 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_inline_bl_bytes": 0, 2024-03-19T02:06:14.192 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_inline_bl_items": 0, 2024-03-19T02:06:14.192 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_fsck_bytes": 0, 2024-03-19T02:06:14.192 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_fsck_items": 0, 2024-03-19T02:06:14.192 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_txc_bytes": 0, 2024-03-19T02:06:14.192 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_txc_items": 0, 2024-03-19T02:06:14.192 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_writing_deferred_bytes": 0, 2024-03-19T02:06:14.192 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_writing_deferred_items": 0, 2024-03-19T02:06:14.192 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_writing_bytes": 0, 2024-03-19T02:06:14.192 INFO:teuthology.orchestra.run.smithi115.stdout: "bluestore_writing_items": 0, 2024-03-19T02:06:14.192 INFO:teuthology.orchestra.run.smithi115.stdout: "bluefs_bytes": 0, 2024-03-19T02:06:14.192 INFO:teuthology.orchestra.run.smithi115.stdout: "bluefs_items": 0, 2024-03-19T02:06:14.192 INFO:teuthology.orchestra.run.smithi115.stdout: "bluefs_file_reader_bytes": 0, 2024-03-19T02:06:14.193 INFO:teuthology.orchestra.run.smithi115.stdout: "bluefs_file_reader_items": 0, 2024-03-19T02:06:14.193 INFO:teuthology.orchestra.run.smithi115.stdout: "bluefs_file_writer_bytes": 0, 2024-03-19T02:06:14.193 INFO:teuthology.orchestra.run.smithi115.stdout: "bluefs_file_writer_items": 0, 2024-03-19T02:06:14.193 INFO:teuthology.orchestra.run.smithi115.stdout: "buffer_anon_bytes": 148784, 2024-03-19T02:06:14.193 INFO:teuthology.orchestra.run.smithi115.stdout: "buffer_anon_items": 31, 2024-03-19T02:06:14.193 INFO:teuthology.orchestra.run.smithi115.stdout: "buffer_meta_bytes": 0, 2024-03-19T02:06:14.193 INFO:teuthology.orchestra.run.smithi115.stdout: "buffer_meta_items": 0, 2024-03-19T02:06:14.193 INFO:teuthology.orchestra.run.smithi115.stdout: "osd_bytes": 0, 2024-03-19T02:06:14.193 INFO:teuthology.orchestra.run.smithi115.stdout: "osd_items": 0, 2024-03-19T02:06:14.193 INFO:teuthology.orchestra.run.smithi115.stdout: "osd_mapbl_bytes": 0, 2024-03-19T02:06:14.193 INFO:teuthology.orchestra.run.smithi115.stdout: "osd_mapbl_items": 0, 2024-03-19T02:06:14.193 INFO:teuthology.orchestra.run.smithi115.stdout: "osd_pglog_bytes": 0, 2024-03-19T02:06:14.193 INFO:teuthology.orchestra.run.smithi115.stdout: "osd_pglog_items": 0, 2024-03-19T02:06:14.193 INFO:teuthology.orchestra.run.smithi115.stdout: "osdmap_bytes": 0, 2024-03-19T02:06:14.193 INFO:teuthology.orchestra.run.smithi115.stdout: "osdmap_items": 0, 2024-03-19T02:06:14.193 INFO:teuthology.orchestra.run.smithi115.stdout: "osdmap_mapping_bytes": 0, 2024-03-19T02:06:14.194 INFO:teuthology.orchestra.run.smithi115.stdout: "osdmap_mapping_items": 0, 2024-03-19T02:06:14.194 INFO:teuthology.orchestra.run.smithi115.stdout: "pgmap_bytes": 0, 2024-03-19T02:06:14.194 INFO:teuthology.orchestra.run.smithi115.stdout: "pgmap_items": 0, 2024-03-19T02:06:14.194 INFO:teuthology.orchestra.run.smithi115.stdout: "mds_co_bytes": 928, 2024-03-19T02:06:14.194 INFO:teuthology.orchestra.run.smithi115.stdout: "mds_co_items": 2, 2024-03-19T02:06:14.194 INFO:teuthology.orchestra.run.smithi115.stdout: "unittest_1_bytes": 0, 2024-03-19T02:06:14.194 INFO:teuthology.orchestra.run.smithi115.stdout: "unittest_1_items": 0, 2024-03-19T02:06:14.194 INFO:teuthology.orchestra.run.smithi115.stdout: "unittest_2_bytes": 0, 2024-03-19T02:06:14.194 INFO:teuthology.orchestra.run.smithi115.stdout: "unittest_2_items": 0 2024-03-19T02:06:14.194 INFO:teuthology.orchestra.run.smithi115.stdout: }, 2024-03-19T02:06:14.194 INFO:teuthology.orchestra.run.smithi115.stdout: "throttle-msgr_dispatch_throttler-mds": { 2024-03-19T02:06:14.194 INFO:teuthology.orchestra.run.smithi115.stdout: "val": 0, 2024-03-19T02:06:14.194 INFO:teuthology.orchestra.run.smithi115.stdout: "max": 104857600, 2024-03-19T02:06:14.194 INFO:teuthology.orchestra.run.smithi115.stdout: "get_started": 0, 2024-03-19T02:06:14.194 INFO:teuthology.orchestra.run.smithi115.stdout: "get": 451, 2024-03-19T02:06:14.194 INFO:teuthology.orchestra.run.smithi115.stdout: "get_sum": 753630, 2024-03-19T02:06:14.195 INFO:teuthology.orchestra.run.smithi115.stdout: "get_or_fail_fail": 0, 2024-03-19T02:06:14.195 INFO:teuthology.orchestra.run.smithi115.stdout: "get_or_fail_success": 451, 2024-03-19T02:06:14.195 INFO:teuthology.orchestra.run.smithi115.stdout: "take": 0, 2024-03-19T02:06:14.195 INFO:teuthology.orchestra.run.smithi115.stdout: "take_sum": 0, 2024-03-19T02:06:14.195 INFO:teuthology.orchestra.run.smithi115.stdout: "put": 451, 2024-03-19T02:06:14.195 INFO:teuthology.orchestra.run.smithi115.stdout: "put_sum": 753630, 2024-03-19T02:06:14.195 INFO:teuthology.orchestra.run.smithi115.stdout: "wait": { 2024-03-19T02:06:14.195 INFO:teuthology.orchestra.run.smithi115.stdout: "avgcount": 0, 2024-03-19T02:06:14.195 INFO:teuthology.orchestra.run.smithi115.stdout: "sum": 0.000000000, 2024-03-19T02:06:14.195 INFO:teuthology.orchestra.run.smithi115.stdout: "avgtime": 0.000000000 2024-03-19T02:06:14.195 INFO:teuthology.orchestra.run.smithi115.stdout: } 2024-03-19T02:06:14.195 INFO:teuthology.orchestra.run.smithi115.stdout: } 2024-03-19T02:06:14.195 INFO:teuthology.orchestra.run.smithi115.stdout:} 2024-03-19T02:06:14.811 WARNING:tasks.check_counter:Counter 'mds.exported' not found on daemon mds.l 2024-03-19T02:06:14.811 WARNING:tasks.check_counter:Counter 'mds.imported' not found on daemon mds.l 2024-03-19T02:06:14.811 WARNING:tasks.check_counter:Counter 'mds_cache.dir_update' not found on daemon mds.l 2024-03-19T02:06:14.811 WARNING:tasks.check_counter:Counter 'mds_cache.dir_update_receipt' not found on daemon mds.l 2024-03-19T02:06:14.812 DEBUG:teuthology.run_tasks:Unwinding manager kclient 2024-03-19T02:06:14.822 INFO:tasks.kclient:Unmounting kernel clients... 2024-03-19T02:06:14.823 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2024-03-19T02:06:14.823 DEBUG:teuthology.orchestra.run.smithi099:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-19T02:06:14.841 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2024-03-19T02:06:14.841 DEBUG:teuthology.orchestra.run.smithi099:> dd if=/proc/self/mounts of=/dev/stdout 2024-03-19T02:06:14.898 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-19T02:06:15.048 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.045+0000 7f15016ac640 1 -- 172.21.15.99:0/2868856285 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f14fc102380 msgr2=0x7f14fc104770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:15.048 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.045+0000 7f15016ac640 1 --2- 172.21.15.99:0/2868856285 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f14fc102380 0x7f14fc104770 secure :-1 s=READY pgs=813 cs=0 l=1 rev1=1 crypto rx=0x7f14ec009ca0 tx=0x7f14ec02c0f0 comp rx=0 tx=0).stop 2024-03-19T02:06:15.048 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.045+0000 7f15016ac640 1 -- 172.21.15.99:0/2868856285 shutdown_connections 2024-03-19T02:06:15.048 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.045+0000 7f15016ac640 1 --2- 172.21.15.99:0/2868856285 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f14fc104cb0 0x7f14fc1070a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:15.048 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.045+0000 7f15016ac640 1 --2- 172.21.15.99:0/2868856285 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f14fc1075e0 0x7f14fc1099d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:15.048 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.045+0000 7f15016ac640 1 --2- 172.21.15.99:0/2868856285 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f14fc102380 0x7f14fc104770 unknown :-1 s=CLOSED pgs=813 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:15.048 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.045+0000 7f15016ac640 1 -- 172.21.15.99:0/2868856285 >> 172.21.15.99:0/2868856285 conn(0x7f14fc0fbcb0 msgr2=0x7f14fc0fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:06:15.048 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.045+0000 7f15016ac640 1 -- 172.21.15.99:0/2868856285 shutdown_connections 2024-03-19T02:06:15.049 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.045+0000 7f15016ac640 1 -- 172.21.15.99:0/2868856285 wait complete. 2024-03-19T02:06:15.049 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.045+0000 7f15016ac640 1 Processor -- start 2024-03-19T02:06:15.049 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.046+0000 7f15016ac640 1 -- start start 2024-03-19T02:06:15.049 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.046+0000 7f15016ac640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f14fc102380 0x7f14fc197ba0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:06:15.049 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.046+0000 7f15016ac640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f14fc104cb0 0x7f14fc1980e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:06:15.049 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.046+0000 7f15016ac640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f14fc1075e0 0x7f14fc19f150 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:06:15.049 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.046+0000 7f15016ac640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f14fc10d4f0 con 0x7f14fc102380 2024-03-19T02:06:15.049 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.046+0000 7f15016ac640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f14fc10d370 con 0x7f14fc104cb0 2024-03-19T02:06:15.049 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.046+0000 7f15016ac640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f14fc10d670 con 0x7f14fc1075e0 2024-03-19T02:06:15.049 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.046+0000 7f14faffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f14fc102380 0x7f14fc197ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:06:15.049 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.046+0000 7f14faffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f14fc102380 0x7f14fc197ba0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58254/0 (socket says 172.21.15.99:58254) 2024-03-19T02:06:15.049 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.046+0000 7f14faffd640 1 -- 172.21.15.99:0/1055288936 learned_addr learned my addr 172.21.15.99:0/1055288936 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:06:15.049 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.046+0000 7f14fa7fc640 1 --2- 172.21.15.99:0/1055288936 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f14fc104cb0 0x7f14fc1980e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:06:15.049 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.046+0000 7f14fb7fe640 1 --2- 172.21.15.99:0/1055288936 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f14fc1075e0 0x7f14fc19f150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:06:15.049 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.046+0000 7f14faffd640 1 -- 172.21.15.99:0/1055288936 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f14fc1075e0 msgr2=0x7f14fc19f150 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:15.050 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.046+0000 7f14faffd640 1 --2- 172.21.15.99:0/1055288936 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f14fc1075e0 0x7f14fc19f150 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:15.050 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.046+0000 7f14faffd640 1 -- 172.21.15.99:0/1055288936 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f14fc104cb0 msgr2=0x7f14fc1980e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:15.050 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.046+0000 7f14faffd640 1 --2- 172.21.15.99:0/1055288936 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f14fc104cb0 0x7f14fc1980e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:15.050 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.046+0000 7f14faffd640 1 -- 172.21.15.99:0/1055288936 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f14fc19f7c0 con 0x7f14fc102380 2024-03-19T02:06:15.050 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.047+0000 7f14faffd640 1 --2- 172.21.15.99:0/1055288936 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f14fc102380 0x7f14fc197ba0 secure :-1 s=READY pgs=814 cs=0 l=1 rev1=1 crypto rx=0x7f14ec009dc0 tx=0x7f14ec032f20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:06:15.050 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.047+0000 7f14e3fff640 1 -- 172.21.15.99:0/1055288936 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f14ec041020 con 0x7f14fc102380 2024-03-19T02:06:15.050 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.047+0000 7f14e3fff640 1 -- 172.21.15.99:0/1055288936 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f14ec034cd0 con 0x7f14fc102380 2024-03-19T02:06:15.050 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.047+0000 7f15016ac640 1 -- 172.21.15.99:0/1055288936 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f14fc19fa50 con 0x7f14fc102380 2024-03-19T02:06:15.050 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.047+0000 7f14e3fff640 1 -- 172.21.15.99:0/1055288936 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f14ec039040 con 0x7f14fc102380 2024-03-19T02:06:15.050 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.047+0000 7f15016ac640 1 -- 172.21.15.99:0/1055288936 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f14fc19ff40 con 0x7f14fc102380 2024-03-19T02:06:15.051 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.048+0000 7f15016ac640 1 -- 172.21.15.99:0/1055288936 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f14fc0fd7a0 con 0x7f14fc102380 2024-03-19T02:06:15.051 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.049+0000 7f14e3fff640 1 -- 172.21.15.99:0/1055288936 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f14ec044020 con 0x7f14fc102380 2024-03-19T02:06:15.051 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.049+0000 7f14e3fff640 1 --2- 172.21.15.99:0/1055288936 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f14e40a2800 0x7f14e40a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:06:15.051 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.049+0000 7f14fa7fc640 1 --2- 172.21.15.99:0/1055288936 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f14e40a2800 0x7f14e40a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:06:15.051 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.049+0000 7f14e3fff640 1 -- 172.21.15.99:0/1055288936 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f14ec0f0c90 con 0x7f14fc102380 2024-03-19T02:06:15.051 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.050+0000 7f14fa7fc640 1 --2- 172.21.15.99:0/1055288936 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f14e40a2800 0x7f14e40a4cc0 secure :-1 s=READY pgs=403 cs=0 l=1 rev1=1 crypto rx=0x7f14f00095f0 tx=0x7f14f0005f90 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:06:15.053 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.053+0000 7f14e3fff640 1 -- 172.21.15.99:0/1055288936 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f14ec04b020 con 0x7f14fc102380 2024-03-19T02:06:15.056 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:06:14 smithi115 ceph-mon[67332]: pgmap v1105: 97 pgs: 97 active+clean; 432 MiB data, 9.0 GiB used, 1.0 TiB / 1.0 TiB avail; 121 KiB/s rd, 1.4 MiB/s wr, 71 op/s 2024-03-19T02:06:15.235 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.234+0000 7f15016ac640 1 -- 172.21.15.99:0/1055288936 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f14fc062420 con 0x7f14fc102380 2024-03-19T02:06:15.236 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.235+0000 7f14e3fff640 1 -- 172.21.15.99:0/1055288936 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v92) ==== 74+0+22804 (secure 0 0 0) 0x7f14ec0bdcf0 con 0x7f14fc102380 2024-03-19T02:06:15.236 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T02:06:15.237 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":92,"fsid":"438fb69c-e590-11ee-95c9-87774f69a715","created":"2024-03-19T01:31:10.165073+0000","modified":"2024-03-19T01:52:30.348737+0000","last_up_change":"2024-03-19T01:38:09.082688+0000","last_in_change":"2024-03-19T01:37:49.115793+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-19T01:34:31.001309+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-19T01:38:54.018623+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"86","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-19T01:38:55.319036+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"87","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"0c407a0e-a40e-43d7-ad9a-bb8f3cd4a9ba","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6803","nonce":2935260112}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6805","nonce":2935260112}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6808","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6809","nonce":2935260112}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6807","nonce":2935260112}]},"public_addr":"172.21.15.99:6803/2935260112","cluster_addr":"172.21.15.99:6805/2935260112","heartbeat_back_addr":"172.21.15.99:6809/2935260112","heartbeat_front_addr":"172.21.15.99:6807/2935260112","state":["exists","up"]},{"osd":1,"uuid":"a48f9173-4a06-49f5-bca2-2c9047b8f666","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6800","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6801","nonce":1468854289}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6802","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6803","nonce":1468854289}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6806","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6807","nonce":1468854289}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6804","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6805","nonce":1468854289}]},"public_addr":"172.21.15.112:6801/1468854289","cluster_addr":"172.21.15.112:6803/1468854289","heartbeat_back_addr":"172.21.15.112:6807/1468854289","heartbeat_front_addr":"172.21.15.112:6805/1468854289","state":["exists","up"]},{"osd":2,"uuid":"b7715fdd-b6c9-4348-bd16-5eb61b427eef","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6800","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6801","nonce":3766690472}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6802","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6803","nonce":3766690472}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6806","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6807","nonce":3766690472}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6804","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6805","nonce":3766690472}]},"public_addr":"172.21.15.115:6801/3766690472","cluster_addr":"172.21.15.115:6803/3766690472","heartbeat_back_addr":"172.21.15.115:6807/3766690472","heartbeat_front_addr":"172.21.15.115:6805/3766690472","state":["exists","up"]},{"osd":3,"uuid":"6697ed1b-96bf-42de-8440-d17e9785b359","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6810","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6811","nonce":3756732895}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6812","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6813","nonce":3756732895}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6816","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6817","nonce":3756732895}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6814","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6815","nonce":3756732895}]},"public_addr":"172.21.15.99:6811/3756732895","cluster_addr":"172.21.15.99:6813/3756732895","heartbeat_back_addr":"172.21.15.99:6817/3756732895","heartbeat_front_addr":"172.21.15.99:6815/3756732895","state":["exists","up"]},{"osd":4,"uuid":"d958d6b9-67f6-43d0-b704-aa55dc9df83a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":34,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6808","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6809","nonce":3717986391}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6810","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6811","nonce":3717986391}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6814","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6815","nonce":3717986391}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6812","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6813","nonce":3717986391}]},"public_addr":"172.21.15.112:6809/3717986391","cluster_addr":"172.21.15.112:6811/3717986391","heartbeat_back_addr":"172.21.15.112:6815/3717986391","heartbeat_front_addr":"172.21.15.112:6813/3717986391","state":["exists","up"]},{"osd":5,"uuid":"a73eb49a-7925-49ba-bc4f-515e17f8b8ae","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6808","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6809","nonce":1971806408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6810","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6811","nonce":1971806408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6814","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6815","nonce":1971806408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6812","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6813","nonce":1971806408}]},"public_addr":"172.21.15.115:6809/1971806408","cluster_addr":"172.21.15.115:6811/1971806408","heartbeat_back_addr":"172.21.15.115:6815/1971806408","heartbeat_front_addr":"172.21.15.115:6813/1971806408","state":["exists","up"]},{"osd":6,"uuid":"caa718cd-e08b-48de-832a-29546d71e508","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6818","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6819","nonce":1130832772}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6820","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6821","nonce":1130832772}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6824","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6825","nonce":1130832772}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6822","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6823","nonce":1130832772}]},"public_addr":"172.21.15.99:6819/1130832772","cluster_addr":"172.21.15.99:6821/1130832772","heartbeat_back_addr":"172.21.15.99:6825/1130832772","heartbeat_front_addr":"172.21.15.99:6823/1130832772","state":["exists","up"]},{"osd":7,"uuid":"dbee19a1-2315-40d3-88e7-876fcde28903","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":53,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6816","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6817","nonce":3610356576}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6818","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6819","nonce":3610356576}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6822","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6823","nonce":3610356576}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6820","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6821","nonce":3610356576}]},"public_addr":"172.21.15.112:6817/3610356576","cluster_addr":"172.21.15.112:6819/3610356576","heartbeat_back_addr":"172.21.15.112:6823/3610356576","heartbeat_front_addr":"172.21.15.112:6821/3610356576","state":["exists","up"]},{"osd":8,"uuid":"14a866dc-5ce3-42fc-8bcf-9088b68726cb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":60,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6816","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6817","nonce":4255803111}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6818","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6819","nonce":4255803111}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6822","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6823","nonce":4255803111}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6820","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6821","nonce":4255803111}]},"public_addr":"172.21.15.115:6817/4255803111","cluster_addr":"172.21.15.115:6819/4255803111","heartbeat_back_addr":"172.21.15.115:6823/4255803111","heartbeat_front_addr":"172.21.15.115:6821/4255803111","state":["exists","up"]},{"osd":9,"uuid":"1db93652-6035-4906-91ae-d64d5a909502","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":68,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6826","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6827","nonce":1410921523}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6828","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6829","nonce":1410921523}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6832","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6833","nonce":1410921523}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6830","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6831","nonce":1410921523}]},"public_addr":"172.21.15.99:6827/1410921523","cluster_addr":"172.21.15.99:6829/1410921523","heartbeat_back_addr":"172.21.15.99:6833/1410921523","heartbeat_front_addr":"172.21.15.99:6831/1410921523","state":["exists","up"]},{"osd":10,"uuid":"6c50a8a7-3345-4f4a-af0e-5606c7aca453","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":74,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6824","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6825","nonce":891805922}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6826","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6827","nonce":891805922}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6830","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6831","nonce":891805922}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6828","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6829","nonce":891805922}]},"public_addr":"172.21.15.112:6825/891805922","cluster_addr":"172.21.15.112:6827/891805922","heartbeat_back_addr":"172.21.15.112:6831/891805922","heartbeat_front_addr":"172.21.15.112:6829/891805922","state":["exists","up"]},{"osd":11,"uuid":"fd63b7fd-0db5-46d5-82fa-9707bbd0b495","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":82,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6824","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6825","nonce":938436244}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6826","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6827","nonce":938436244}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6830","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6831","nonce":938436244}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6828","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6829","nonce":938436244}]},"public_addr":"172.21.15.115:6825/938436244","cluster_addr":"172.21.15.115:6827/938436244","heartbeat_back_addr":"172.21.15.115:6831/938436244","heartbeat_front_addr":"172.21.15.115:6829/938436244","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:33:39.036822+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:34:03.339566+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:34:27.149273+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:34:51.315561+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:35:15.396031+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:35:38.957168+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:36:03.213914+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:36:27.763699+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:36:51.688670+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:37:17.280165+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:37:42.554633+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:38:07.438039+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.0","mappings":[{"from":8,"to":1}]},{"pgid":"2.8","mappings":[{"from":2,"to":11}]},{"pgid":"2.b","mappings":[{"from":8,"to":11}]},{"pgid":"2.e","mappings":[{"from":2,"to":4}]},{"pgid":"3.0","mappings":[{"from":5,"to":9}]},{"pgid":"3.2","mappings":[{"from":3,"to":9}]},{"pgid":"3.14","mappings":[{"from":1,"to":9}]},{"pgid":"3.17","mappings":[{"from":4,"to":8}]},{"pgid":"3.31","mappings":[{"from":4,"to":8}]},{"pgid":"3.37","mappings":[{"from":5,"to":7}]},{"pgid":"3.3a","mappings":[{"from":4,"to":6}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.99:6801/1328245934":"2024-03-20T01:32:06.661002+0000","172.21.15.99:0/1618965837":"2024-03-20T01:32:06.661002+0000","172.21.15.99:0/2491429851":"2024-03-20T01:32:06.661002+0000","172.21.15.99:6800/1328245934":"2024-03-20T01:32:06.661002+0000","172.21.15.99:0/2112231766":"2024-03-20T01:31:40.161751+0000","172.21.15.99:0/3341836545":"2024-03-20T01:31:40.161751+0000","172.21.15.99:6800/4223048293":"2024-03-20T01:31:40.161751+0000","172.21.15.99:6801/4223048293":"2024-03-20T01:31:40.161751+0000","172.21.15.99:0/2377621863":"2024-03-20T01:31:40.161751+0000","172.21.15.99:0/1904400935":"2024-03-20T01:32:06.661002+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-19T02:06:15.238 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.237+0000 7f15016ac640 1 -- 172.21.15.99:0/1055288936 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f14e40a2800 msgr2=0x7f14e40a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:15.238 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.237+0000 7f15016ac640 1 --2- 172.21.15.99:0/1055288936 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f14e40a2800 0x7f14e40a4cc0 secure :-1 s=READY pgs=403 cs=0 l=1 rev1=1 crypto rx=0x7f14f00095f0 tx=0x7f14f0005f90 comp rx=0 tx=0).stop 2024-03-19T02:06:15.239 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.237+0000 7f15016ac640 1 -- 172.21.15.99:0/1055288936 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f14fc102380 msgr2=0x7f14fc197ba0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:15.239 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.237+0000 7f15016ac640 1 --2- 172.21.15.99:0/1055288936 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f14fc102380 0x7f14fc197ba0 secure :-1 s=READY pgs=814 cs=0 l=1 rev1=1 crypto rx=0x7f14ec009dc0 tx=0x7f14ec032f20 comp rx=0 tx=0).stop 2024-03-19T02:06:15.239 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.237+0000 7f15016ac640 1 -- 172.21.15.99:0/1055288936 shutdown_connections 2024-03-19T02:06:15.239 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.237+0000 7f15016ac640 1 --2- 172.21.15.99:0/1055288936 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f14e40a2800 0x7f14e40a4cc0 unknown :-1 s=CLOSED pgs=403 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:15.239 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.237+0000 7f15016ac640 1 --2- 172.21.15.99:0/1055288936 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f14fc104cb0 0x7f14fc1980e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:15.239 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.237+0000 7f15016ac640 1 --2- 172.21.15.99:0/1055288936 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f14fc1075e0 0x7f14fc19f150 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:15.239 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.237+0000 7f15016ac640 1 --2- 172.21.15.99:0/1055288936 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f14fc102380 0x7f14fc197ba0 unknown :-1 s=CLOSED pgs=814 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:15.239 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.237+0000 7f15016ac640 1 -- 172.21.15.99:0/1055288936 >> 172.21.15.99:0/1055288936 conn(0x7f14fc0fbcb0 msgr2=0x7f14fc102c40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:06:15.239 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.238+0000 7f15016ac640 1 -- 172.21.15.99:0/1055288936 shutdown_connections 2024-03-19T02:06:15.239 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.238+0000 7f15016ac640 1 -- 172.21.15.99:0/1055288936 wait complete. 2024-03-19T02:06:15.248 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:06:14 smithi099 ceph-mon[62951]: pgmap v1105: 97 pgs: 97 active+clean; 432 MiB data, 9.0 GiB used, 1.0 TiB / 1.0 TiB avail; 121 KiB/s rd, 1.4 MiB/s wr, 71 op/s 2024-03-19T02:06:15.250 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-19T01:34:31.001309+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 12, 'score_stable': 12, 'optimal_score': 0.25, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-03-19T01:38:54.018623+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '86', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.8799999952316284, 'score_stable': 1.8799999952316284, 'optimal_score': 1, 'raw_score_acting': 1.8799999952316284, 'raw_score_stable': 1.8799999952316284, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-03-19T01:38:55.319036+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '87', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-19T02:06:15.250 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-03-19T02:06:15.358 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.355+0000 7f7051859640 1 -- 172.21.15.99:0/3503990862 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f704c101920 msgr2=0x7f704c101da0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:15.358 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.355+0000 7f7051859640 1 --2- 172.21.15.99:0/3503990862 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f704c101920 0x7f704c101da0 secure :-1 s=READY pgs=815 cs=0 l=1 rev1=1 crypto rx=0x7f7034009900 tx=0x7f703402bc30 comp rx=0 tx=0).stop 2024-03-19T02:06:15.358 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.355+0000 7f7051859640 1 -- 172.21.15.99:0/3503990862 shutdown_connections 2024-03-19T02:06:15.358 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.355+0000 7f7051859640 1 --2- 172.21.15.99:0/3503990862 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f704c1022e0 0x7f704c108b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:15.358 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.355+0000 7f7051859640 1 --2- 172.21.15.99:0/3503990862 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f704c101920 0x7f704c101da0 unknown :-1 s=CLOSED pgs=815 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:15.358 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.355+0000 7f7051859640 1 --2- 172.21.15.99:0/3503990862 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f704c100730 0x7f704c100b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:15.358 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.355+0000 7f7051859640 1 -- 172.21.15.99:0/3503990862 >> 172.21.15.99:0/3503990862 conn(0x7f704c0fbcc0 msgr2=0x7f704c0fe0e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:06:15.358 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.355+0000 7f7051859640 1 -- 172.21.15.99:0/3503990862 shutdown_connections 2024-03-19T02:06:15.358 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.355+0000 7f7051859640 1 -- 172.21.15.99:0/3503990862 wait complete. 2024-03-19T02:06:15.359 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.355+0000 7f7051859640 1 Processor -- start 2024-03-19T02:06:15.359 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.355+0000 7f7051859640 1 -- start start 2024-03-19T02:06:15.359 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.356+0000 7f7051859640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f704c100730 0x7f704c06a900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:06:15.359 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.356+0000 7f7051859640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f704c101920 0x7f704c19c8b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:06:15.359 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.356+0000 7f7051859640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f704c1022e0 0x7f704c19eed0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:06:15.359 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.356+0000 7f7051859640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f704c10c3b0 con 0x7f704c100730 2024-03-19T02:06:15.359 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.356+0000 7f7051859640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f704c10c230 con 0x7f704c1022e0 2024-03-19T02:06:15.359 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.356+0000 7f7051859640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f704c10c530 con 0x7f704c101920 2024-03-19T02:06:15.359 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.356+0000 7f704affd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f704c100730 0x7f704c06a900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:06:15.359 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.356+0000 7f704affd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f704c100730 0x7f704c06a900 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58276/0 (socket says 172.21.15.99:58276) 2024-03-19T02:06:15.359 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.356+0000 7f704affd640 1 -- 172.21.15.99:0/4162475428 learned_addr learned my addr 172.21.15.99:0/4162475428 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:06:15.359 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.356+0000 7f704b7fe640 1 --2- 172.21.15.99:0/4162475428 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f704c1022e0 0x7f704c19eed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:06:15.359 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.356+0000 7f7043fff640 1 --2- 172.21.15.99:0/4162475428 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f704c101920 0x7f704c19c8b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:06:15.359 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.356+0000 7f704affd640 1 -- 172.21.15.99:0/4162475428 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f704c101920 msgr2=0x7f704c19c8b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:15.360 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.356+0000 7f704affd640 1 --2- 172.21.15.99:0/4162475428 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f704c101920 0x7f704c19c8b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:15.360 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.356+0000 7f704affd640 1 -- 172.21.15.99:0/4162475428 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f704c1022e0 msgr2=0x7f704c19eed0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:15.360 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.356+0000 7f704affd640 1 --2- 172.21.15.99:0/4162475428 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f704c1022e0 0x7f704c19eed0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:15.360 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.356+0000 7f704affd640 1 -- 172.21.15.99:0/4162475428 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f704c19f7e0 con 0x7f704c100730 2024-03-19T02:06:15.360 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.356+0000 7f704affd640 1 --2- 172.21.15.99:0/4162475428 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f704c100730 0x7f704c06a900 secure :-1 s=READY pgs=816 cs=0 l=1 rev1=1 crypto rx=0x7f703c00e9c0 tx=0x7f703c00ee80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:06:15.360 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.356+0000 7f7050857640 1 -- 172.21.15.99:0/4162475428 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f703c00cd80 con 0x7f704c100730 2024-03-19T02:06:15.360 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.356+0000 7f7050857640 1 -- 172.21.15.99:0/4162475428 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f703c0035f0 con 0x7f704c100730 2024-03-19T02:06:15.360 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.356+0000 7f7051859640 1 -- 172.21.15.99:0/4162475428 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f704c19fa70 con 0x7f704c100730 2024-03-19T02:06:15.360 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.356+0000 7f7050857640 1 -- 172.21.15.99:0/4162475428 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f703c010690 con 0x7f704c100730 2024-03-19T02:06:15.360 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.356+0000 7f7051859640 1 -- 172.21.15.99:0/4162475428 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f704c19ff40 con 0x7f704c100730 2024-03-19T02:06:15.360 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.357+0000 7f7051859640 1 -- 172.21.15.99:0/4162475428 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f704c107ad0 con 0x7f704c100730 2024-03-19T02:06:15.361 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.358+0000 7f7050857640 1 -- 172.21.15.99:0/4162475428 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f703c0037a0 con 0x7f704c100730 2024-03-19T02:06:15.361 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.359+0000 7f7050857640 1 --2- 172.21.15.99:0/4162475428 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f70380a2800 0x7f70380a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:06:15.361 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.359+0000 7f7043fff640 1 --2- 172.21.15.99:0/4162475428 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f70380a2800 0x7f70380a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:06:15.361 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.359+0000 7f7050857640 1 -- 172.21.15.99:0/4162475428 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f703c0cee00 con 0x7f704c100730 2024-03-19T02:06:15.361 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.359+0000 7f7043fff640 1 --2- 172.21.15.99:0/4162475428 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f70380a2800 0x7f70380a4cc0 secure :-1 s=READY pgs=404 cs=0 l=1 rev1=1 crypto rx=0x7f703402c7b0 tx=0x7f7034005fd0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:06:15.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:06:14 smithi112 ceph-mon[66819]: pgmap v1105: 97 pgs: 97 active+clean; 432 MiB data, 9.0 GiB used, 1.0 TiB / 1.0 TiB avail; 121 KiB/s rd, 1.4 MiB/s wr, 71 op/s 2024-03-19T02:06:15.364 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.363+0000 7f7050857640 1 -- 172.21.15.99:0/4162475428 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f703c09be60 con 0x7f704c100730 2024-03-19T02:06:15.539 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.538+0000 7f7051859640 1 -- 172.21.15.99:0/4162475428 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) -- 0x7f704c0624c0 con 0x7f704c100730 2024-03-19T02:06:15.539 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.539+0000 7f7050857640 1 -- 172.21.15.99:0/4162475428 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v92) ==== 93+0+10 (secure 0 0 0) 0x7f703c09b760 con 0x7f704c100730 2024-03-19T02:06:15.539 INFO:teuthology.orchestra.run.smithi099.stdout:pg_num: 1 2024-03-19T02:06:15.542 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.541+0000 7f7051859640 1 -- 172.21.15.99:0/4162475428 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f70380a2800 msgr2=0x7f70380a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:15.542 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.541+0000 7f7051859640 1 --2- 172.21.15.99:0/4162475428 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f70380a2800 0x7f70380a4cc0 secure :-1 s=READY pgs=404 cs=0 l=1 rev1=1 crypto rx=0x7f703402c7b0 tx=0x7f7034005fd0 comp rx=0 tx=0).stop 2024-03-19T02:06:15.542 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.541+0000 7f7051859640 1 -- 172.21.15.99:0/4162475428 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f704c100730 msgr2=0x7f704c06a900 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:15.542 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.541+0000 7f7051859640 1 --2- 172.21.15.99:0/4162475428 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f704c100730 0x7f704c06a900 secure :-1 s=READY pgs=816 cs=0 l=1 rev1=1 crypto rx=0x7f703c00e9c0 tx=0x7f703c00ee80 comp rx=0 tx=0).stop 2024-03-19T02:06:15.542 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.541+0000 7f7051859640 1 -- 172.21.15.99:0/4162475428 shutdown_connections 2024-03-19T02:06:15.542 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.541+0000 7f7051859640 1 --2- 172.21.15.99:0/4162475428 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f70380a2800 0x7f70380a4cc0 unknown :-1 s=CLOSED pgs=404 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:15.542 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.541+0000 7f7051859640 1 --2- 172.21.15.99:0/4162475428 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f704c1022e0 0x7f704c19eed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:15.542 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.541+0000 7f7051859640 1 --2- 172.21.15.99:0/4162475428 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f704c101920 0x7f704c19c8b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:15.542 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.541+0000 7f7051859640 1 --2- 172.21.15.99:0/4162475428 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f704c100730 0x7f704c06a900 unknown :-1 s=CLOSED pgs=816 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:15.542 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.541+0000 7f7051859640 1 -- 172.21.15.99:0/4162475428 >> 172.21.15.99:0/4162475428 conn(0x7f704c0fbcc0 msgr2=0x7f704c104ca0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:06:15.542 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.541+0000 7f7051859640 1 -- 172.21.15.99:0/4162475428 shutdown_connections 2024-03-19T02:06:15.542 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.541+0000 7f7051859640 1 -- 172.21.15.99:0/4162475428 wait complete. 2024-03-19T02:06:15.553 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-03-19T02:06:15.660 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.657+0000 7f2a84c4b640 1 -- 172.21.15.99:0/838123568 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2a80101910 msgr2=0x7f2a80101d90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:15.660 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.657+0000 7f2a84c4b640 1 --2- 172.21.15.99:0/838123568 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2a80101910 0x7f2a80101d90 secure :-1 s=READY pgs=817 cs=0 l=1 rev1=1 crypto rx=0x7f2a68009900 tx=0x7f2a6802bc30 comp rx=0 tx=0).stop 2024-03-19T02:06:15.660 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.657+0000 7f2a84c4b640 1 -- 172.21.15.99:0/838123568 shutdown_connections 2024-03-19T02:06:15.660 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.657+0000 7f2a84c4b640 1 --2- 172.21.15.99:0/838123568 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f2a801022d0 0x7f2a80108b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:15.660 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.657+0000 7f2a84c4b640 1 --2- 172.21.15.99:0/838123568 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2a80101910 0x7f2a80101d90 unknown :-1 s=CLOSED pgs=817 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:15.660 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.657+0000 7f2a84c4b640 1 --2- 172.21.15.99:0/838123568 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f2a80100720 0x7f2a80100b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:15.660 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.657+0000 7f2a84c4b640 1 -- 172.21.15.99:0/838123568 >> 172.21.15.99:0/838123568 conn(0x7f2a800fbcb0 msgr2=0x7f2a800fe0d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:06:15.660 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.657+0000 7f2a84c4b640 1 -- 172.21.15.99:0/838123568 shutdown_connections 2024-03-19T02:06:15.661 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.657+0000 7f2a84c4b640 1 -- 172.21.15.99:0/838123568 wait complete. 2024-03-19T02:06:15.661 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.657+0000 7f2a84c4b640 1 Processor -- start 2024-03-19T02:06:15.661 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.657+0000 7f2a84c4b640 1 -- start start 2024-03-19T02:06:15.661 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.658+0000 7f2a84c4b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2a80100720 0x7f2a80196a60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:06:15.661 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.658+0000 7f2a84c4b640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f2a80101910 0x7f2a80196fa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:06:15.661 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.658+0000 7f2a84c4b640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f2a801022d0 0x7f2a8019e010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:06:15.661 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.658+0000 7f2a84c4b640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f2a8010c380 con 0x7f2a80100720 2024-03-19T02:06:15.661 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.658+0000 7f2a84c4b640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f2a8010c200 con 0x7f2a80101910 2024-03-19T02:06:15.661 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.658+0000 7f2a84c4b640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f2a8010c500 con 0x7f2a801022d0 2024-03-19T02:06:15.661 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.658+0000 7f2a7e59c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2a80100720 0x7f2a80196a60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:06:15.661 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.658+0000 7f2a7e59c640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2a80100720 0x7f2a80196a60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58294/0 (socket says 172.21.15.99:58294) 2024-03-19T02:06:15.661 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.658+0000 7f2a7e59c640 1 -- 172.21.15.99:0/2275079258 learned_addr learned my addr 172.21.15.99:0/2275079258 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:06:15.661 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.658+0000 7f2a77fff640 1 --2- 172.21.15.99:0/2275079258 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f2a80101910 0x7f2a80196fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:06:15.661 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.658+0000 7f2a7ed9d640 1 --2- 172.21.15.99:0/2275079258 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f2a801022d0 0x7f2a8019e010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:06:15.662 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.658+0000 7f2a7e59c640 1 -- 172.21.15.99:0/2275079258 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f2a801022d0 msgr2=0x7f2a8019e010 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:15.662 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.658+0000 7f2a7e59c640 1 --2- 172.21.15.99:0/2275079258 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f2a801022d0 0x7f2a8019e010 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:15.662 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.658+0000 7f2a7e59c640 1 -- 172.21.15.99:0/2275079258 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f2a80101910 msgr2=0x7f2a80196fa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:15.662 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.658+0000 7f2a7e59c640 1 --2- 172.21.15.99:0/2275079258 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f2a80101910 0x7f2a80196fa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:15.662 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.658+0000 7f2a7e59c640 1 -- 172.21.15.99:0/2275079258 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f2a8019e680 con 0x7f2a80100720 2024-03-19T02:06:15.662 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.658+0000 7f2a7e59c640 1 --2- 172.21.15.99:0/2275079258 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2a80100720 0x7f2a80196a60 secure :-1 s=READY pgs=818 cs=0 l=1 rev1=1 crypto rx=0x7f2a7000e9c0 tx=0x7f2a7000edc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:06:15.662 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.658+0000 7f2a777fe640 1 -- 172.21.15.99:0/2275079258 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2a7000cd80 con 0x7f2a80100720 2024-03-19T02:06:15.662 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.658+0000 7f2a777fe640 1 -- 172.21.15.99:0/2275079258 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f2a70010910 con 0x7f2a80100720 2024-03-19T02:06:15.662 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.658+0000 7f2a777fe640 1 -- 172.21.15.99:0/2275079258 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2a70018950 con 0x7f2a80100720 2024-03-19T02:06:15.662 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.658+0000 7f2a84c4b640 1 -- 172.21.15.99:0/2275079258 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f2a8019e970 con 0x7f2a80100720 2024-03-19T02:06:15.662 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.658+0000 7f2a84c4b640 1 -- 172.21.15.99:0/2275079258 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f2a8019ee40 con 0x7f2a80100720 2024-03-19T02:06:15.662 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.659+0000 7f2a84c4b640 1 -- 172.21.15.99:0/2275079258 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f2a80107ac0 con 0x7f2a80100720 2024-03-19T02:06:15.663 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.660+0000 7f2a777fe640 1 -- 172.21.15.99:0/2275079258 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f2a7001f020 con 0x7f2a80100720 2024-03-19T02:06:15.663 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.661+0000 7f2a777fe640 1 --2- 172.21.15.99:0/2275079258 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f2a6c0a2800 0x7f2a6c0a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:06:15.663 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.661+0000 7f2a77fff640 1 --2- 172.21.15.99:0/2275079258 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f2a6c0a2800 0x7f2a6c0a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:06:15.664 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.661+0000 7f2a777fe640 1 -- 172.21.15.99:0/2275079258 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f2a700ceb60 con 0x7f2a80100720 2024-03-19T02:06:15.664 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.661+0000 7f2a77fff640 1 --2- 172.21.15.99:0/2275079258 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f2a6c0a2800 0x7f2a6c0a4cc0 secure :-1 s=READY pgs=405 cs=0 l=1 rev1=1 crypto rx=0x7f2a6802c7b0 tx=0x7f2a68005fd0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:06:15.666 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.666+0000 7f2a777fe640 1 -- 172.21.15.99:0/2275079258 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f2a7009bbc0 con 0x7f2a80100720 2024-03-19T02:06:15.844 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.843+0000 7f2a84c4b640 1 -- 172.21.15.99:0/2275079258 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) -- 0x7f2a800624c0 con 0x7f2a80100720 2024-03-19T02:06:15.844 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.843+0000 7f2a777fe640 1 -- 172.21.15.99:0/2275079258 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v92) ==== 104+0+11 (secure 0 0 0) 0x7f2a7009b4c0 con 0x7f2a80100720 2024-03-19T02:06:15.845 INFO:teuthology.orchestra.run.smithi099.stdout:pg_num: 32 2024-03-19T02:06:15.846 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.845+0000 7f2a84c4b640 1 -- 172.21.15.99:0/2275079258 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f2a6c0a2800 msgr2=0x7f2a6c0a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:15.846 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.845+0000 7f2a84c4b640 1 --2- 172.21.15.99:0/2275079258 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f2a6c0a2800 0x7f2a6c0a4cc0 secure :-1 s=READY pgs=405 cs=0 l=1 rev1=1 crypto rx=0x7f2a6802c7b0 tx=0x7f2a68005fd0 comp rx=0 tx=0).stop 2024-03-19T02:06:15.846 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.845+0000 7f2a84c4b640 1 -- 172.21.15.99:0/2275079258 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2a80100720 msgr2=0x7f2a80196a60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:15.846 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.845+0000 7f2a84c4b640 1 --2- 172.21.15.99:0/2275079258 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2a80100720 0x7f2a80196a60 secure :-1 s=READY pgs=818 cs=0 l=1 rev1=1 crypto rx=0x7f2a7000e9c0 tx=0x7f2a7000edc0 comp rx=0 tx=0).stop 2024-03-19T02:06:15.846 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.845+0000 7f2a84c4b640 1 -- 172.21.15.99:0/2275079258 shutdown_connections 2024-03-19T02:06:15.846 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.845+0000 7f2a84c4b640 1 --2- 172.21.15.99:0/2275079258 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f2a6c0a2800 0x7f2a6c0a4cc0 unknown :-1 s=CLOSED pgs=405 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:15.846 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.845+0000 7f2a84c4b640 1 --2- 172.21.15.99:0/2275079258 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f2a80101910 0x7f2a80196fa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:15.847 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.845+0000 7f2a84c4b640 1 --2- 172.21.15.99:0/2275079258 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f2a801022d0 0x7f2a8019e010 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:15.847 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.845+0000 7f2a84c4b640 1 --2- 172.21.15.99:0/2275079258 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2a80100720 0x7f2a80196a60 unknown :-1 s=CLOSED pgs=818 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:15.847 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.845+0000 7f2a84c4b640 1 -- 172.21.15.99:0/2275079258 >> 172.21.15.99:0/2275079258 conn(0x7f2a800fbcb0 msgr2=0x7f2a80104c90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:06:15.847 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.845+0000 7f2a84c4b640 1 -- 172.21.15.99:0/2275079258 shutdown_connections 2024-03-19T02:06:15.847 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.845+0000 7f2a84c4b640 1 -- 172.21.15.99:0/2275079258 wait complete. 2024-03-19T02:06:15.857 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-03-19T02:06:15.967 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.964+0000 7fb83eaf7640 1 -- 172.21.15.99:0/3392408129 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb8381033e0 msgr2=0x7fb838109c70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:15.967 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.964+0000 7fb83eaf7640 1 --2- 172.21.15.99:0/3392408129 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb8381033e0 0x7fb838109c70 secure :-1 s=READY pgs=819 cs=0 l=1 rev1=1 crypto rx=0x7fb830009ca0 tx=0x7fb83002c0f0 comp rx=0 tx=0).stop 2024-03-19T02:06:15.967 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.964+0000 7fb83eaf7640 1 -- 172.21.15.99:0/3392408129 shutdown_connections 2024-03-19T02:06:15.967 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.964+0000 7fb83eaf7640 1 --2- 172.21.15.99:0/3392408129 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb8381033e0 0x7fb838109c70 unknown :-1 s=CLOSED pgs=819 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:15.968 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.964+0000 7fb83eaf7640 1 --2- 172.21.15.99:0/3392408129 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb838102a20 0x7fb838102ea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:15.968 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.964+0000 7fb83eaf7640 1 --2- 172.21.15.99:0/3392408129 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb838101830 0x7fb838101c50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:15.968 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.964+0000 7fb83eaf7640 1 -- 172.21.15.99:0/3392408129 >> 172.21.15.99:0/3392408129 conn(0x7fb8380fcdc0 msgr2=0x7fb8380ff1e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:06:15.968 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.964+0000 7fb83eaf7640 1 -- 172.21.15.99:0/3392408129 shutdown_connections 2024-03-19T02:06:15.968 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.964+0000 7fb83eaf7640 1 -- 172.21.15.99:0/3392408129 wait complete. 2024-03-19T02:06:15.968 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.965+0000 7fb83eaf7640 1 Processor -- start 2024-03-19T02:06:15.968 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.965+0000 7fb83eaf7640 1 -- start start 2024-03-19T02:06:15.968 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.965+0000 7fb83eaf7640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb838101830 0x7fb838197af0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:06:15.968 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.965+0000 7fb83eaf7640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb838102a20 0x7fb838198030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:06:15.968 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.965+0000 7fb83eaf7640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb8381033e0 0x7fb83819f0a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:06:15.968 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.965+0000 7fb83eaf7640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fb83810d4b0 con 0x7fb838101830 2024-03-19T02:06:15.968 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.965+0000 7fb83eaf7640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fb83810d330 con 0x7fb838102a20 2024-03-19T02:06:15.968 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.965+0000 7fb83eaf7640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fb83810d630 con 0x7fb8381033e0 2024-03-19T02:06:15.969 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.965+0000 7fb83c893640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb838101830 0x7fb838197af0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:06:15.969 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.965+0000 7fb83c893640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb838101830 0x7fb838197af0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58320/0 (socket says 172.21.15.99:58320) 2024-03-19T02:06:15.969 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.965+0000 7fb83c893640 1 -- 172.21.15.99:0/2138925953 learned_addr learned my addr 172.21.15.99:0/2138925953 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:06:15.969 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.965+0000 7fb837fff640 1 --2- 172.21.15.99:0/2138925953 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb838102a20 0x7fb838198030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:06:15.969 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.965+0000 7fb83d094640 1 --2- 172.21.15.99:0/2138925953 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb8381033e0 0x7fb83819f0a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:06:15.969 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.965+0000 7fb83c893640 1 -- 172.21.15.99:0/2138925953 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb8381033e0 msgr2=0x7fb83819f0a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:15.969 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.965+0000 7fb83c893640 1 --2- 172.21.15.99:0/2138925953 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb8381033e0 0x7fb83819f0a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:15.969 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.965+0000 7fb83c893640 1 -- 172.21.15.99:0/2138925953 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb838102a20 msgr2=0x7fb838198030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:15.969 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.965+0000 7fb83c893640 1 --2- 172.21.15.99:0/2138925953 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb838102a20 0x7fb838198030 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:15.969 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.965+0000 7fb83c893640 1 -- 172.21.15.99:0/2138925953 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb83819f710 con 0x7fb838101830 2024-03-19T02:06:15.969 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.965+0000 7fb83c893640 1 --2- 172.21.15.99:0/2138925953 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb838101830 0x7fb838197af0 secure :-1 s=READY pgs=820 cs=0 l=1 rev1=1 crypto rx=0x7fb82c00e9c0 tx=0x7fb82c00ee80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:06:15.969 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.966+0000 7fb835ffb640 1 -- 172.21.15.99:0/2138925953 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb82c00cde0 con 0x7fb838101830 2024-03-19T02:06:15.969 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.966+0000 7fb835ffb640 1 -- 172.21.15.99:0/2138925953 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fb82c010910 con 0x7fb838101830 2024-03-19T02:06:15.970 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.966+0000 7fb83eaf7640 1 -- 172.21.15.99:0/2138925953 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb83819fa00 con 0x7fb838101830 2024-03-19T02:06:15.970 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.966+0000 7fb835ffb640 1 -- 172.21.15.99:0/2138925953 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb82c018a00 con 0x7fb838101830 2024-03-19T02:06:15.970 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.966+0000 7fb83eaf7640 1 -- 172.21.15.99:0/2138925953 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fb83819ffa0 con 0x7fb838101830 2024-03-19T02:06:15.970 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.967+0000 7fb83eaf7640 1 -- 172.21.15.99:0/2138925953 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fb838108b50 con 0x7fb838101830 2024-03-19T02:06:15.971 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.968+0000 7fb835ffb640 1 -- 172.21.15.99:0/2138925953 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7fb82c01f020 con 0x7fb838101830 2024-03-19T02:06:15.971 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.968+0000 7fb835ffb640 1 --2- 172.21.15.99:0/2138925953 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb8280a2880 0x7fb8280a4d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:06:15.971 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.969+0000 7fb837fff640 1 --2- 172.21.15.99:0/2138925953 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb8280a2880 0x7fb8280a4d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:06:15.971 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.969+0000 7fb835ffb640 1 -- 172.21.15.99:0/2138925953 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fb82c0cf1f0 con 0x7fb838101830 2024-03-19T02:06:15.971 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.969+0000 7fb837fff640 1 --2- 172.21.15.99:0/2138925953 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb8280a2880 0x7fb8280a4d40 secure :-1 s=READY pgs=406 cs=0 l=1 rev1=1 crypto rx=0x7fb838199000 tx=0x7fb824005f90 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:06:15.972 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:15.972+0000 7fb835ffb640 1 -- 172.21.15.99:0/2138925953 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7fb82c09c250 con 0x7fb838101830 2024-03-19T02:06:16.148 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.147+0000 7fb83eaf7640 1 -- 172.21.15.99:0/2138925953 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) -- 0x7fb83802d3a0 con 0x7fb838101830 2024-03-19T02:06:16.148 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.148+0000 7fb835ffb640 1 -- 172.21.15.99:0/2138925953 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v92) ==== 100+0+11 (secure 0 0 0) 0x7fb82c09bb50 con 0x7fb838101830 2024-03-19T02:06:16.149 INFO:teuthology.orchestra.run.smithi099.stdout:pg_num: 64 2024-03-19T02:06:16.151 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.151+0000 7fb83eaf7640 1 -- 172.21.15.99:0/2138925953 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb8280a2880 msgr2=0x7fb8280a4d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:16.151 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.151+0000 7fb83eaf7640 1 --2- 172.21.15.99:0/2138925953 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb8280a2880 0x7fb8280a4d40 secure :-1 s=READY pgs=406 cs=0 l=1 rev1=1 crypto rx=0x7fb838199000 tx=0x7fb824005f90 comp rx=0 tx=0).stop 2024-03-19T02:06:16.151 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.151+0000 7fb83eaf7640 1 -- 172.21.15.99:0/2138925953 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb838101830 msgr2=0x7fb838197af0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:16.151 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.151+0000 7fb83eaf7640 1 --2- 172.21.15.99:0/2138925953 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb838101830 0x7fb838197af0 secure :-1 s=READY pgs=820 cs=0 l=1 rev1=1 crypto rx=0x7fb82c00e9c0 tx=0x7fb82c00ee80 comp rx=0 tx=0).stop 2024-03-19T02:06:16.151 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.151+0000 7fb83eaf7640 1 -- 172.21.15.99:0/2138925953 shutdown_connections 2024-03-19T02:06:16.151 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.151+0000 7fb83eaf7640 1 --2- 172.21.15.99:0/2138925953 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb8280a2880 0x7fb8280a4d40 unknown :-1 s=CLOSED pgs=406 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:16.151 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.151+0000 7fb83eaf7640 1 --2- 172.21.15.99:0/2138925953 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb838102a20 0x7fb838198030 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:16.152 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.151+0000 7fb83eaf7640 1 --2- 172.21.15.99:0/2138925953 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb8381033e0 0x7fb83819f0a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:16.152 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.151+0000 7fb83eaf7640 1 --2- 172.21.15.99:0/2138925953 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb838101830 0x7fb838197af0 unknown :-1 s=CLOSED pgs=820 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:16.152 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.151+0000 7fb83eaf7640 1 -- 172.21.15.99:0/2138925953 >> 172.21.15.99:0/2138925953 conn(0x7fb8380fcdc0 msgr2=0x7fb838105d20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:06:16.152 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.151+0000 7fb83eaf7640 1 -- 172.21.15.99:0/2138925953 shutdown_connections 2024-03-19T02:06:16.152 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.151+0000 7fb83eaf7640 1 -- 172.21.15.99:0/2138925953 wait complete. 2024-03-19T02:06:16.161 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:06:15 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/1055288936' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-19T02:06:16.161 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:06:15 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/4162475428' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-03-19T02:06:16.161 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:06:15 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/2275079258' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-03-19T02:06:16.162 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist ls 2024-03-19T02:06:16.267 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.264+0000 7f152c663640 1 -- 172.21.15.99:0/1726887646 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1524100a00 msgr2=0x7f1524100e80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:16.267 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.264+0000 7f152c663640 1 --2- 172.21.15.99:0/1726887646 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1524100a00 0x7f1524100e80 secure :-1 s=READY pgs=821 cs=0 l=1 rev1=1 crypto rx=0x7f1514009900 tx=0x7f151402bc30 comp rx=0 tx=0).stop 2024-03-19T02:06:16.268 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.264+0000 7f152c663640 1 -- 172.21.15.99:0/1726887646 shutdown_connections 2024-03-19T02:06:16.268 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.264+0000 7f152c663640 1 --2- 172.21.15.99:0/1726887646 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f15241013c0 0x7f1524108c60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:16.268 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.264+0000 7f152c663640 1 --2- 172.21.15.99:0/1726887646 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1524100a00 0x7f1524100e80 unknown :-1 s=CLOSED pgs=821 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:16.268 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.264+0000 7f152c663640 1 --2- 172.21.15.99:0/1726887646 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f1524101cd0 0x7f15241020f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:16.268 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.264+0000 7f152c663640 1 -- 172.21.15.99:0/1726887646 >> 172.21.15.99:0/1726887646 conn(0x7f15240fb450 msgr2=0x7f15240fd870 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:06:16.268 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.265+0000 7f152c663640 1 -- 172.21.15.99:0/1726887646 shutdown_connections 2024-03-19T02:06:16.268 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.265+0000 7f152c663640 1 -- 172.21.15.99:0/1726887646 wait complete. 2024-03-19T02:06:16.268 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.265+0000 7f152c663640 1 Processor -- start 2024-03-19T02:06:16.268 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.265+0000 7f152c663640 1 -- start start 2024-03-19T02:06:16.268 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.265+0000 7f152c663640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1524100a00 0x7f1524197bb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:06:16.268 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.265+0000 7f152c663640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f15241013c0 0x7f15241980f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:06:16.268 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.265+0000 7f152c663640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1524101cd0 0x7f152419f160 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:06:16.268 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.265+0000 7f152c663640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f152410d4f0 con 0x7f1524100a00 2024-03-19T02:06:16.269 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.265+0000 7f152c663640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f152410d370 con 0x7f1524101cd0 2024-03-19T02:06:16.269 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.265+0000 7f152c663640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f152410d670 con 0x7f15241013c0 2024-03-19T02:06:16.269 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.265+0000 7f152a3ff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1524100a00 0x7f1524197bb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:06:16.269 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.265+0000 7f152a3ff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1524100a00 0x7f1524197bb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58348/0 (socket says 172.21.15.99:58348) 2024-03-19T02:06:16.269 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.265+0000 7f152a3ff640 1 -- 172.21.15.99:0/4045078630 learned_addr learned my addr 172.21.15.99:0/4045078630 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:06:16.269 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.265+0000 7f1529bfe640 1 --2- 172.21.15.99:0/4045078630 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f15241013c0 0x7f15241980f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:06:16.269 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.265+0000 7f152ac00640 1 --2- 172.21.15.99:0/4045078630 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1524101cd0 0x7f152419f160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:06:16.269 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.266+0000 7f152a3ff640 1 -- 172.21.15.99:0/4045078630 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f15241013c0 msgr2=0x7f15241980f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:16.269 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.266+0000 7f152a3ff640 1 --2- 172.21.15.99:0/4045078630 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f15241013c0 0x7f15241980f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:16.269 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.266+0000 7f152a3ff640 1 -- 172.21.15.99:0/4045078630 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1524101cd0 msgr2=0x7f152419f160 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:16.269 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.266+0000 7f152a3ff640 1 --2- 172.21.15.99:0/4045078630 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1524101cd0 0x7f152419f160 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:16.269 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.266+0000 7f152a3ff640 1 -- 172.21.15.99:0/4045078630 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f152419f7d0 con 0x7f1524100a00 2024-03-19T02:06:16.269 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.266+0000 7f152a3ff640 1 --2- 172.21.15.99:0/4045078630 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1524100a00 0x7f1524197bb0 secure :-1 s=READY pgs=822 cs=0 l=1 rev1=1 crypto rx=0x7f152000e9c0 tx=0x7f152000edc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:06:16.269 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.266+0000 7f151b7fe640 1 -- 172.21.15.99:0/4045078630 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f152000ccc0 con 0x7f1524100a00 2024-03-19T02:06:16.270 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.266+0000 7f151b7fe640 1 -- 172.21.15.99:0/4045078630 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f15200035f0 con 0x7f1524100a00 2024-03-19T02:06:16.270 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.266+0000 7f152c663640 1 -- 172.21.15.99:0/4045078630 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f152419fac0 con 0x7f1524100a00 2024-03-19T02:06:16.270 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.266+0000 7f151b7fe640 1 -- 172.21.15.99:0/4045078630 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f152000ce60 con 0x7f1524100a00 2024-03-19T02:06:16.270 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.266+0000 7f152c663640 1 -- 172.21.15.99:0/4045078630 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f152419ff90 con 0x7f1524100a00 2024-03-19T02:06:16.270 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.267+0000 7f152c663640 1 -- 172.21.15.99:0/4045078630 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f1524107b60 con 0x7f1524100a00 2024-03-19T02:06:16.271 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.268+0000 7f151b7fe640 1 -- 172.21.15.99:0/4045078630 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f15200037a0 con 0x7f1524100a00 2024-03-19T02:06:16.271 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.269+0000 7f151b7fe640 1 --2- 172.21.15.99:0/4045078630 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f15100a2800 0x7f15100a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:06:16.271 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.269+0000 7f1529bfe640 1 --2- 172.21.15.99:0/4045078630 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f15100a2800 0x7f15100a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:06:16.271 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.269+0000 7f151b7fe640 1 -- 172.21.15.99:0/4045078630 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f15200ceeb0 con 0x7f1524100a00 2024-03-19T02:06:16.271 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.269+0000 7f1529bfe640 1 --2- 172.21.15.99:0/4045078630 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f15100a2800 0x7f15100a4cc0 secure :-1 s=READY pgs=407 cs=0 l=1 rev1=1 crypto rx=0x7f151402c7b0 tx=0x7f1514000bc0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:06:16.272 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.273+0000 7f151b7fe640 1 -- 172.21.15.99:0/4045078630 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f152009bf10 con 0x7f1524100a00 2024-03-19T02:06:16.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:06:15 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/1055288936' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-19T02:06:16.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:06:15 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/4162475428' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-03-19T02:06:16.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:06:15 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/2275079258' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-03-19T02:06:16.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:06:15 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/1055288936' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-19T02:06:16.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:06:15 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/4162475428' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-03-19T02:06:16.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:06:15 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/2275079258' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-03-19T02:06:16.459 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.458+0000 7f152c663640 1 -- 172.21.15.99:0/4045078630 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "osd blocklist ls"} v 0) -- 0x7f152402d460 con 0x7f1524100a00 2024-03-19T02:06:16.459 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.459+0000 7f151b7fe640 1 -- 172.21.15.99:0/4045078630 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist ls"}]=0 listed 10 entries v92) ==== 81+0+592 (secure 0 0 0) 0x7f152009b810 con 0x7f1524100a00 2024-03-19T02:06:16.460 INFO:teuthology.orchestra.run.smithi099.stdout:172.21.15.99:6801/1328245934 2024-03-20T01:32:06.661002+0000 2024-03-19T02:06:16.460 INFO:teuthology.orchestra.run.smithi099.stdout:172.21.15.99:0/1618965837 2024-03-20T01:32:06.661002+0000 2024-03-19T02:06:16.460 INFO:teuthology.orchestra.run.smithi099.stdout:172.21.15.99:0/2491429851 2024-03-20T01:32:06.661002+0000 2024-03-19T02:06:16.460 INFO:teuthology.orchestra.run.smithi099.stdout:172.21.15.99:6800/1328245934 2024-03-20T01:32:06.661002+0000 2024-03-19T02:06:16.460 INFO:teuthology.orchestra.run.smithi099.stdout:172.21.15.99:0/2112231766 2024-03-20T01:31:40.161751+0000 2024-03-19T02:06:16.460 INFO:teuthology.orchestra.run.smithi099.stdout:172.21.15.99:0/3341836545 2024-03-20T01:31:40.161751+0000 2024-03-19T02:06:16.460 INFO:teuthology.orchestra.run.smithi099.stdout:172.21.15.99:6800/4223048293 2024-03-20T01:31:40.161751+0000 2024-03-19T02:06:16.460 INFO:teuthology.orchestra.run.smithi099.stdout:172.21.15.99:6801/4223048293 2024-03-20T01:31:40.161751+0000 2024-03-19T02:06:16.460 INFO:teuthology.orchestra.run.smithi099.stdout:172.21.15.99:0/2377621863 2024-03-20T01:31:40.161751+0000 2024-03-19T02:06:16.460 INFO:teuthology.orchestra.run.smithi099.stdout:172.21.15.99:0/1904400935 2024-03-20T01:32:06.661002+0000 2024-03-19T02:06:16.461 INFO:teuthology.orchestra.run.smithi099.stderr:listed 10 entries 2024-03-19T02:06:16.462 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.461+0000 7f152c663640 1 -- 172.21.15.99:0/4045078630 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f15100a2800 msgr2=0x7f15100a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:16.462 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.461+0000 7f152c663640 1 --2- 172.21.15.99:0/4045078630 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f15100a2800 0x7f15100a4cc0 secure :-1 s=READY pgs=407 cs=0 l=1 rev1=1 crypto rx=0x7f151402c7b0 tx=0x7f1514000bc0 comp rx=0 tx=0).stop 2024-03-19T02:06:16.462 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.461+0000 7f152c663640 1 -- 172.21.15.99:0/4045078630 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1524100a00 msgr2=0x7f1524197bb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:16.462 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.461+0000 7f152c663640 1 --2- 172.21.15.99:0/4045078630 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1524100a00 0x7f1524197bb0 secure :-1 s=READY pgs=822 cs=0 l=1 rev1=1 crypto rx=0x7f152000e9c0 tx=0x7f152000edc0 comp rx=0 tx=0).stop 2024-03-19T02:06:16.462 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.461+0000 7f152c663640 1 -- 172.21.15.99:0/4045078630 shutdown_connections 2024-03-19T02:06:16.462 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.461+0000 7f152c663640 1 --2- 172.21.15.99:0/4045078630 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f15100a2800 0x7f15100a4cc0 unknown :-1 s=CLOSED pgs=407 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:16.462 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.461+0000 7f152c663640 1 --2- 172.21.15.99:0/4045078630 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f1524101cd0 0x7f152419f160 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:16.462 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.461+0000 7f152c663640 1 --2- 172.21.15.99:0/4045078630 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f15241013c0 0x7f15241980f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:16.462 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.461+0000 7f152c663640 1 --2- 172.21.15.99:0/4045078630 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f1524100a00 0x7f1524197bb0 unknown :-1 s=CLOSED pgs=822 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:16.462 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.461+0000 7f152c663640 1 -- 172.21.15.99:0/4045078630 >> 172.21.15.99:0/4045078630 conn(0x7f15240fb450 msgr2=0x7f1524104d30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:06:16.462 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.461+0000 7f152c663640 1 -- 172.21.15.99:0/4045078630 shutdown_connections 2024-03-19T02:06:16.462 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:16.461+0000 7f152c663640 1 -- 172.21.15.99:0/4045078630 wait complete. 2024-03-19T02:06:16.473 DEBUG:tasks.cephfs.kernel_mount:Unmounting client client.0... 2024-03-19T02:06:16.473 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-19T02:06:16.473 DEBUG:teuthology.orchestra.run.smithi099:> sudo umount /home/ubuntu/cephtest/mnt.0 2024-03-19T02:06:16.676 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi099.front.sepia.ceph.com 2024-03-19T02:06:16.676 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-19T02:06:16.676 DEBUG:teuthology.orchestra.run.smithi099:> (cd /home/ubuntu/cephtest && exec rmdir -- /home/ubuntu/cephtest/mnt.0) 2024-03-19T02:06:16.718 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-19T02:06:16.719 DEBUG:teuthology.orchestra.run.smithi099:> ip netns list 2024-03-19T02:06:16.794 INFO:teuthology.orchestra.run.smithi099.stdout:ceph-ns--home-ubuntu-cephtest-mnt.0 (id: 0) 2024-03-19T02:06:16.795 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-19T02:06:16.795 DEBUG:teuthology.orchestra.run.smithi099:> sudo ip netns delete ceph-ns--home-ubuntu-cephtest-mnt.0 2024-03-19T02:06:16.903 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-03-19T02:06:16.903 DEBUG:teuthology.orchestra.run.smithi099:> sudo ip link delete ceph-brx 2024-03-19T02:06:17.005 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-03-19T02:06:17.016 INFO:tasks.cephadm:Teardown begin 2024-03-19T02:06:17.016 INFO:tasks.cephfs.filesystem:Destroying file system cephfs and related pools 2024-03-19T02:06:17.017 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-19T02:06:17.155 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.152+0000 7fb4ead22640 1 -- 172.21.15.99:0/1891724310 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb4e4104c10 msgr2=0x7fb4e4107000 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:17.155 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.152+0000 7fb4ead22640 1 --2- 172.21.15.99:0/1891724310 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb4e4104c10 0x7fb4e4107000 secure :-1 s=READY pgs=823 cs=0 l=1 rev1=1 crypto rx=0x7fb4d0009900 tx=0x7fb4d002bc30 comp rx=0 tx=0).stop 2024-03-19T02:06:17.155 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.152+0000 7fb4ead22640 1 -- 172.21.15.99:0/1891724310 shutdown_connections 2024-03-19T02:06:17.155 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.152+0000 7fb4ead22640 1 --2- 172.21.15.99:0/1891724310 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb4e4107540 0x7fb4e4109970 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:17.155 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.152+0000 7fb4ead22640 1 --2- 172.21.15.99:0/1891724310 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb4e4104c10 0x7fb4e4107000 unknown :-1 s=CLOSED pgs=823 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:17.155 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.152+0000 7fb4ead22640 1 --2- 172.21.15.99:0/1891724310 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb4e41022e0 0x7fb4e41046d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:17.155 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.152+0000 7fb4ead22640 1 -- 172.21.15.99:0/1891724310 >> 172.21.15.99:0/1891724310 conn(0x7fb4e40fbcb0 msgr2=0x7fb4e40fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:06:17.155 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.152+0000 7fb4ead22640 1 -- 172.21.15.99:0/1891724310 shutdown_connections 2024-03-19T02:06:17.156 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.152+0000 7fb4ead22640 1 -- 172.21.15.99:0/1891724310 wait complete. 2024-03-19T02:06:17.156 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.153+0000 7fb4ead22640 1 Processor -- start 2024-03-19T02:06:17.156 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.153+0000 7fb4ead22640 1 -- start start 2024-03-19T02:06:17.156 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.153+0000 7fb4ead22640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb4e41022e0 0x7fb4e4199da0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:06:17.156 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.153+0000 7fb4ead22640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb4e4104c10 0x7fb4e419a2e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:06:17.156 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.153+0000 7fb4ead22640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb4e4107540 0x7fb4e41a1310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:06:17.156 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.153+0000 7fb4ead22640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7fb4e410d4d0 con 0x7fb4e4104c10 2024-03-19T02:06:17.156 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.153+0000 7fb4ead22640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7fb4e410d350 con 0x7fb4e41022e0 2024-03-19T02:06:17.156 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.153+0000 7fb4ead22640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7fb4e410d650 con 0x7fb4e4107540 2024-03-19T02:06:17.156 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.153+0000 7fb4e8abe640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb4e41022e0 0x7fb4e4199da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:06:17.156 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.153+0000 7fb4e3fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb4e4104c10 0x7fb4e419a2e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:06:17.156 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.153+0000 7fb4e8abe640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb4e41022e0 0x7fb4e4199da0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.112:3300/0 says I am v2:172.21.15.99:60702/0 (socket says 172.21.15.99:60702) 2024-03-19T02:06:17.156 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.153+0000 7fb4e3fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb4e4104c10 0x7fb4e419a2e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58374/0 (socket says 172.21.15.99:58374) 2024-03-19T02:06:17.156 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.153+0000 7fb4e8abe640 1 -- 172.21.15.99:0/13146027 learned_addr learned my addr 172.21.15.99:0/13146027 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:06:17.156 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.153+0000 7fb4e92bf640 1 --2- 172.21.15.99:0/13146027 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb4e4107540 0x7fb4e41a1310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:06:17.157 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.153+0000 7fb4e3fff640 1 -- 172.21.15.99:0/13146027 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb4e4107540 msgr2=0x7fb4e41a1310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:17.157 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.153+0000 7fb4e3fff640 1 --2- 172.21.15.99:0/13146027 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb4e4107540 0x7fb4e41a1310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:17.157 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.153+0000 7fb4e3fff640 1 -- 172.21.15.99:0/13146027 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb4e41022e0 msgr2=0x7fb4e4199da0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:17.157 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.153+0000 7fb4e3fff640 1 --2- 172.21.15.99:0/13146027 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb4e41022e0 0x7fb4e4199da0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:17.157 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.153+0000 7fb4e3fff640 1 -- 172.21.15.99:0/13146027 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb4e41a19b0 con 0x7fb4e4104c10 2024-03-19T02:06:17.157 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.154+0000 7fb4e3fff640 1 --2- 172.21.15.99:0/13146027 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb4e4104c10 0x7fb4e419a2e0 secure :-1 s=READY pgs=824 cs=0 l=1 rev1=1 crypto rx=0x7fb4d002c170 tx=0x7fb4d002c650 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:06:17.157 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.154+0000 7fb4e1ffb640 1 -- 172.21.15.99:0/13146027 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb4d0041070 con 0x7fb4e4104c10 2024-03-19T02:06:17.157 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.154+0000 7fb4ead22640 1 -- 172.21.15.99:0/13146027 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb4e41a1c40 con 0x7fb4e4104c10 2024-03-19T02:06:17.157 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.154+0000 7fb4e1ffb640 1 -- 172.21.15.99:0/13146027 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7fb4d0004450 con 0x7fb4e4104c10 2024-03-19T02:06:17.157 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.154+0000 7fb4e1ffb640 1 -- 172.21.15.99:0/13146027 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb4d0003e50 con 0x7fb4e4104c10 2024-03-19T02:06:17.157 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.154+0000 7fb4ead22640 1 -- 172.21.15.99:0/13146027 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fb4e41a2100 con 0x7fb4e4104c10 2024-03-19T02:06:17.158 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.156+0000 7fb4ead22640 1 -- 172.21.15.99:0/13146027 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fb4e4109070 con 0x7fb4e4104c10 2024-03-19T02:06:17.158 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.156+0000 7fb4e1ffb640 1 -- 172.21.15.99:0/13146027 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7fb4d0044070 con 0x7fb4e4104c10 2024-03-19T02:06:17.158 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.157+0000 7fb4e1ffb640 1 --2- 172.21.15.99:0/13146027 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb4d40a29a0 0x7fb4d40a4e60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:06:17.158 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.157+0000 7fb4e1ffb640 1 -- 172.21.15.99:0/13146027 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7fb4d00f0780 con 0x7fb4e4104c10 2024-03-19T02:06:17.158 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.157+0000 7fb4e8abe640 1 --2- 172.21.15.99:0/13146027 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb4d40a29a0 0x7fb4d40a4e60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:06:17.159 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.157+0000 7fb4e8abe640 1 --2- 172.21.15.99:0/13146027 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb4d40a29a0 0x7fb4d40a4e60 secure :-1 s=READY pgs=408 cs=0 l=1 rev1=1 crypto rx=0x7fb4e419b270 tx=0x7fb4d8005f90 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:06:17.161 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.160+0000 7fb4e1ffb640 1 -- 172.21.15.99:0/13146027 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7fb4d00bd190 con 0x7fb4e4104c10 2024-03-19T02:06:17.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:06:16 smithi115 ceph-mon[67332]: pgmap v1106: 97 pgs: 97 active+clean; 433 MiB data, 9.0 GiB used, 1.0 TiB / 1.0 TiB avail; 95 KiB/s rd, 527 KiB/s wr, 47 op/s 2024-03-19T02:06:17.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:06:16 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/2138925953' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-03-19T02:06:17.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:06:16 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/4045078630' entity='client.admin' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-03-19T02:06:17.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:06:16 smithi099 ceph-mon[62951]: pgmap v1106: 97 pgs: 97 active+clean; 433 MiB data, 9.0 GiB used, 1.0 TiB / 1.0 TiB avail; 95 KiB/s rd, 527 KiB/s wr, 47 op/s 2024-03-19T02:06:17.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:06:16 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/2138925953' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-03-19T02:06:17.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:06:16 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/4045078630' entity='client.admin' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-03-19T02:06:17.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:06:16 smithi112 ceph-mon[66819]: pgmap v1106: 97 pgs: 97 active+clean; 433 MiB data, 9.0 GiB used, 1.0 TiB / 1.0 TiB avail; 95 KiB/s rd, 527 KiB/s wr, 47 op/s 2024-03-19T02:06:17.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:06:16 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/2138925953' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-03-19T02:06:17.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:06:16 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/4045078630' entity='client.admin' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-03-19T02:06:17.402 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.402+0000 7fb4ead22640 1 -- 172.21.15.99:0/13146027 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7fb4e402d1f0 con 0x7fb4e4104c10 2024-03-19T02:06:17.402 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.403+0000 7fb4e1ffb640 1 -- 172.21.15.99:0/13146027 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 47 v47) ==== 94+0+11888 (secure 0 0 0) 0x7fb4d00c1230 con 0x7fb4e4104c10 2024-03-19T02:06:17.403 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T02:06:17.403 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":47,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14562,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6835/1631815621","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6834","nonce":1631815621},{"type":"v1","addr":"172.21.15.99:6835","nonce":1631815621}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":4},{"gid":14646,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6839/2236145144","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6838","nonce":2236145144},{"type":"v1","addr":"172.21.15.99:6839","nonce":2236145144}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11},{"gid":14658,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6841/1660990159","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6840","nonce":1660990159},{"type":"v1","addr":"172.21.15.99:6841","nonce":1660990159}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":12},{"gid":24383,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.112:6833/2969810977","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6832","nonce":2969810977},{"type":"v1","addr":"172.21.15.112:6833","nonce":2969810977}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3},{"gid":24404,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6837/2038634879","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6836","nonce":2038634879},{"type":"v1","addr":"172.21.15.99:6837","nonce":2038634879}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":24415,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.115:6833/2219320481","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6832","nonce":2219320481},{"type":"v1","addr":"172.21.15.115:6833","nonce":2219320481}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":47,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-03-19T01:38:56.124529+0000","modified":"2024-03-19T02:05:40.088167+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24487,"mds_1":24479,"mds_2":24472},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24416":{"gid":24416,"name":"e","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.112:6835/3690587528","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6834","nonce":3690587528},{"type":"v1","addr":"172.21.15.112:6835","nonce":3690587528}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24440":{"gid":24440,"name":"h","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.112:6837/2339823159","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6836","nonce":2339823159},{"type":"v1","addr":"172.21.15.112:6837","nonce":2339823159}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24472":{"gid":24472,"name":"c","rank":2,"incarnation":19,"state":"up:active","state_seq":7,"addr":"172.21.15.115:6837/1368988191","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6836","nonce":1368988191},{"type":"v1","addr":"172.21.15.115:6837","nonce":1368988191}]},"join_fscid":-1,"export_targets":[0,1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24479":{"gid":24479,"name":"b","rank":1,"incarnation":17,"state":"up:active","state_seq":4,"addr":"172.21.15.112:6839/986249680","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6838","nonce":986249680},{"type":"v1","addr":"172.21.15.112:6839","nonce":986249680}]},"join_fscid":-1,"export_targets":[0],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24487":{"gid":24487,"name":"i","rank":0,"incarnation":14,"state":"up:active","state_seq":5,"addr":"172.21.15.115:6839/2743761968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6838","nonce":2743761968},{"type":"v1","addr":"172.21.15.115:6839","nonce":2743761968}]},"join_fscid":-1,"export_targets":[1,2],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_26042":{"gid":26042,"name":"f","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.115:6835/1572636732","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6834","nonce":1572636732},{"type":"v1","addr":"172.21.15.115:6835","nonce":1572636732}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24487,"qdb_cluster":[24487,24472,24479]},"id":1}]} 2024-03-19T02:06:17.403 INFO:teuthology.orchestra.run.smithi099.stderr:dumped fsmap epoch 47 2024-03-19T02:06:17.404 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.405+0000 7fb4ead22640 1 -- 172.21.15.99:0/13146027 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb4d40a29a0 msgr2=0x7fb4d40a4e60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:17.405 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.405+0000 7fb4ead22640 1 --2- 172.21.15.99:0/13146027 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb4d40a29a0 0x7fb4d40a4e60 secure :-1 s=READY pgs=408 cs=0 l=1 rev1=1 crypto rx=0x7fb4e419b270 tx=0x7fb4d8005f90 comp rx=0 tx=0).stop 2024-03-19T02:06:17.405 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.405+0000 7fb4ead22640 1 -- 172.21.15.99:0/13146027 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb4e4104c10 msgr2=0x7fb4e419a2e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:17.405 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.405+0000 7fb4ead22640 1 --2- 172.21.15.99:0/13146027 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb4e4104c10 0x7fb4e419a2e0 secure :-1 s=READY pgs=824 cs=0 l=1 rev1=1 crypto rx=0x7fb4d002c170 tx=0x7fb4d002c650 comp rx=0 tx=0).stop 2024-03-19T02:06:17.405 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.405+0000 7fb4ead22640 1 -- 172.21.15.99:0/13146027 shutdown_connections 2024-03-19T02:06:17.405 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.405+0000 7fb4ead22640 1 --2- 172.21.15.99:0/13146027 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7fb4e4107540 0x7fb4e41a1310 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:17.405 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.405+0000 7fb4ead22640 1 --2- 172.21.15.99:0/13146027 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7fb4e4104c10 0x7fb4e419a2e0 unknown :-1 s=CLOSED pgs=824 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:17.405 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.405+0000 7fb4ead22640 1 --2- 172.21.15.99:0/13146027 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7fb4d40a29a0 0x7fb4d40a4e60 unknown :-1 s=CLOSED pgs=408 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:17.405 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.405+0000 7fb4ead22640 1 --2- 172.21.15.99:0/13146027 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7fb4e41022e0 0x7fb4e4199da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:17.405 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.405+0000 7fb4ead22640 1 -- 172.21.15.99:0/13146027 >> 172.21.15.99:0/13146027 conn(0x7fb4e40fbcb0 msgr2=0x7fb4e4101a40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:06:17.405 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.405+0000 7fb4ead22640 1 -- 172.21.15.99:0/13146027 shutdown_connections 2024-03-19T02:06:17.405 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.405+0000 7fb4ead22640 1 -- 172.21.15.99:0/13146027 wait complete. 2024-03-19T02:06:17.416 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-03-19T02:06:17.520 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.518+0000 7ffa24a27640 1 -- 172.21.15.99:0/4021356201 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ffa20102340 msgr2=0x7ffa20104730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:17.520 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.518+0000 7ffa24a27640 1 --2- 172.21.15.99:0/4021356201 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ffa20102340 0x7ffa20104730 secure :-1 s=READY pgs=825 cs=0 l=1 rev1=1 crypto rx=0x7ffa10009900 tx=0x7ffa1002bc30 comp rx=0 tx=0).stop 2024-03-19T02:06:17.520 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.518+0000 7ffa24a27640 1 -- 172.21.15.99:0/4021356201 shutdown_connections 2024-03-19T02:06:17.520 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.518+0000 7ffa24a27640 1 --2- 172.21.15.99:0/4021356201 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ffa20104c70 0x7ffa20107060 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:17.520 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.518+0000 7ffa24a27640 1 --2- 172.21.15.99:0/4021356201 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ffa201075a0 0x7ffa20109990 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:17.521 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.518+0000 7ffa24a27640 1 --2- 172.21.15.99:0/4021356201 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ffa20102340 0x7ffa20104730 unknown :-1 s=CLOSED pgs=825 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:17.521 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.518+0000 7ffa24a27640 1 -- 172.21.15.99:0/4021356201 >> 172.21.15.99:0/4021356201 conn(0x7ffa200fbcd0 msgr2=0x7ffa200fe110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:06:17.521 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.518+0000 7ffa24a27640 1 -- 172.21.15.99:0/4021356201 shutdown_connections 2024-03-19T02:06:17.521 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.518+0000 7ffa24a27640 1 -- 172.21.15.99:0/4021356201 wait complete. 2024-03-19T02:06:17.521 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.518+0000 7ffa24a27640 1 Processor -- start 2024-03-19T02:06:17.521 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.518+0000 7ffa24a27640 1 -- start start 2024-03-19T02:06:17.521 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.519+0000 7ffa24a27640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ffa20102340 0x7ffa20197b80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:06:17.521 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.519+0000 7ffa24a27640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ffa20104c70 0x7ffa201980c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:06:17.521 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.519+0000 7ffa24a27640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ffa201075a0 0x7ffa2019f130 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:06:17.521 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.519+0000 7ffa24a27640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7ffa2010d500 con 0x7ffa20104c70 2024-03-19T02:06:17.521 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.519+0000 7ffa24a27640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7ffa2010d380 con 0x7ffa20102340 2024-03-19T02:06:17.522 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.519+0000 7ffa24a27640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7ffa2010d680 con 0x7ffa201075a0 2024-03-19T02:06:17.522 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.519+0000 7ffa17fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ffa20104c70 0x7ffa201980c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:06:17.522 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.519+0000 7ffa1e59c640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ffa20102340 0x7ffa20197b80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:06:17.522 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.519+0000 7ffa17fff640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ffa20104c70 0x7ffa201980c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58392/0 (socket says 172.21.15.99:58392) 2024-03-19T02:06:17.522 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.519+0000 7ffa17fff640 1 -- 172.21.15.99:0/3545031292 learned_addr learned my addr 172.21.15.99:0/3545031292 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:06:17.522 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.519+0000 7ffa1ed9d640 1 --2- 172.21.15.99:0/3545031292 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ffa201075a0 0x7ffa2019f130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:06:17.522 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.519+0000 7ffa17fff640 1 -- 172.21.15.99:0/3545031292 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ffa201075a0 msgr2=0x7ffa2019f130 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:17.522 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.519+0000 7ffa17fff640 1 --2- 172.21.15.99:0/3545031292 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ffa201075a0 0x7ffa2019f130 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:17.522 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.519+0000 7ffa17fff640 1 -- 172.21.15.99:0/3545031292 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ffa20102340 msgr2=0x7ffa20197b80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:17.522 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.519+0000 7ffa17fff640 1 --2- 172.21.15.99:0/3545031292 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ffa20102340 0x7ffa20197b80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:17.522 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.519+0000 7ffa17fff640 1 -- 172.21.15.99:0/3545031292 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ffa2019f7a0 con 0x7ffa20104c70 2024-03-19T02:06:17.522 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.519+0000 7ffa1e59c640 1 --2- 172.21.15.99:0/3545031292 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ffa20102340 0x7ffa20197b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:06:17.523 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.519+0000 7ffa17fff640 1 --2- 172.21.15.99:0/3545031292 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ffa20104c70 0x7ffa201980c0 secure :-1 s=READY pgs=826 cs=0 l=1 rev1=1 crypto rx=0x7ffa0800e9a0 tx=0x7ffa0800ee60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:06:17.523 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.519+0000 7ffa177fe640 1 -- 172.21.15.99:0/3545031292 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffa0800cd80 con 0x7ffa20104c70 2024-03-19T02:06:17.523 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.519+0000 7ffa177fe640 1 -- 172.21.15.99:0/3545031292 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7ffa080035f0 con 0x7ffa20104c70 2024-03-19T02:06:17.523 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.519+0000 7ffa177fe640 1 -- 172.21.15.99:0/3545031292 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffa08010670 con 0x7ffa20104c70 2024-03-19T02:06:17.523 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.519+0000 7ffa24a27640 1 -- 172.21.15.99:0/3545031292 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ffa2019fa90 con 0x7ffa20104c70 2024-03-19T02:06:17.523 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.519+0000 7ffa24a27640 1 -- 172.21.15.99:0/3545031292 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7ffa2019ff90 con 0x7ffa20104c70 2024-03-19T02:06:17.523 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.520+0000 7ffa24a27640 1 -- 172.21.15.99:0/3545031292 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7ffa200fd7c0 con 0x7ffa20104c70 2024-03-19T02:06:17.524 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.521+0000 7ffa177fe640 1 -- 172.21.15.99:0/3545031292 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7ffa080037a0 con 0x7ffa20104c70 2024-03-19T02:06:17.524 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.522+0000 7ffa177fe640 1 --2- 172.21.15.99:0/3545031292 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ffa0c0a2800 0x7ffa0c0a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:06:17.524 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.522+0000 7ffa1e59c640 1 --2- 172.21.15.99:0/3545031292 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ffa0c0a2800 0x7ffa0c0a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:06:17.524 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.522+0000 7ffa177fe640 1 -- 172.21.15.99:0/3545031292 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7ffa080ced10 con 0x7ffa20104c70 2024-03-19T02:06:17.524 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.522+0000 7ffa1e59c640 1 --2- 172.21.15.99:0/3545031292 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ffa0c0a2800 0x7ffa0c0a4cc0 secure :-1 s=READY pgs=409 cs=0 l=1 rev1=1 crypto rx=0x7ffa1002c7b0 tx=0x7ffa10005fd0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:06:17.526 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.526+0000 7ffa177fe640 1 -- 172.21.15.99:0/3545031292 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7ffa0809bd70 con 0x7ffa20104c70 2024-03-19T02:06:17.740 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.740+0000 7ffa24a27640 1 -- 172.21.15.99:0/3545031292 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7ffa20062420 con 0x7ffa20104c70 2024-03-19T02:06:17.740 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.740+0000 7ffa177fe640 1 -- 172.21.15.99:0/3545031292 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 47 v47) ==== 94+0+11888 (secure 0 0 0) 0x7ffa0809b670 con 0x7ffa20104c70 2024-03-19T02:06:17.741 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T02:06:17.741 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":47,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14562,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6835/1631815621","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6834","nonce":1631815621},{"type":"v1","addr":"172.21.15.99:6835","nonce":1631815621}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":4},{"gid":14646,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6839/2236145144","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6838","nonce":2236145144},{"type":"v1","addr":"172.21.15.99:6839","nonce":2236145144}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11},{"gid":14658,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6841/1660990159","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6840","nonce":1660990159},{"type":"v1","addr":"172.21.15.99:6841","nonce":1660990159}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":12},{"gid":24383,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.112:6833/2969810977","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6832","nonce":2969810977},{"type":"v1","addr":"172.21.15.112:6833","nonce":2969810977}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3},{"gid":24404,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.99:6837/2038634879","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6836","nonce":2038634879},{"type":"v1","addr":"172.21.15.99:6837","nonce":2038634879}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":24415,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.115:6833/2219320481","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6832","nonce":2219320481},{"type":"v1","addr":"172.21.15.115:6833","nonce":2219320481}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":47,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-03-19T01:38:56.124529+0000","modified":"2024-03-19T02:05:40.088167+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24487,"mds_1":24479,"mds_2":24472},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24416":{"gid":24416,"name":"e","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.112:6835/3690587528","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6834","nonce":3690587528},{"type":"v1","addr":"172.21.15.112:6835","nonce":3690587528}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24440":{"gid":24440,"name":"h","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.112:6837/2339823159","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6836","nonce":2339823159},{"type":"v1","addr":"172.21.15.112:6837","nonce":2339823159}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24472":{"gid":24472,"name":"c","rank":2,"incarnation":19,"state":"up:active","state_seq":7,"addr":"172.21.15.115:6837/1368988191","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6836","nonce":1368988191},{"type":"v1","addr":"172.21.15.115:6837","nonce":1368988191}]},"join_fscid":-1,"export_targets":[0,1],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24479":{"gid":24479,"name":"b","rank":1,"incarnation":17,"state":"up:active","state_seq":4,"addr":"172.21.15.112:6839/986249680","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6838","nonce":986249680},{"type":"v1","addr":"172.21.15.112:6839","nonce":986249680}]},"join_fscid":-1,"export_targets":[0],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24487":{"gid":24487,"name":"i","rank":0,"incarnation":14,"state":"up:active","state_seq":5,"addr":"172.21.15.115:6839/2743761968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6838","nonce":2743761968},{"type":"v1","addr":"172.21.15.115:6839","nonce":2743761968}]},"join_fscid":-1,"export_targets":[1,2],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_26042":{"gid":26042,"name":"f","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.115:6835/1572636732","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6834","nonce":1572636732},{"type":"v1","addr":"172.21.15.115:6835","nonce":1572636732}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24487,"qdb_cluster":[24487,24472,24479]},"id":1}]} 2024-03-19T02:06:17.741 INFO:teuthology.orchestra.run.smithi099.stderr:dumped fsmap epoch 47 2024-03-19T02:06:17.742 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.742+0000 7ffa24a27640 1 -- 172.21.15.99:0/3545031292 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ffa0c0a2800 msgr2=0x7ffa0c0a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:17.742 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.742+0000 7ffa24a27640 1 --2- 172.21.15.99:0/3545031292 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ffa0c0a2800 0x7ffa0c0a4cc0 secure :-1 s=READY pgs=409 cs=0 l=1 rev1=1 crypto rx=0x7ffa1002c7b0 tx=0x7ffa10005fd0 comp rx=0 tx=0).stop 2024-03-19T02:06:17.743 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.742+0000 7ffa24a27640 1 -- 172.21.15.99:0/3545031292 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ffa20104c70 msgr2=0x7ffa201980c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:17.743 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.742+0000 7ffa24a27640 1 --2- 172.21.15.99:0/3545031292 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ffa20104c70 0x7ffa201980c0 secure :-1 s=READY pgs=826 cs=0 l=1 rev1=1 crypto rx=0x7ffa0800e9a0 tx=0x7ffa0800ee60 comp rx=0 tx=0).stop 2024-03-19T02:06:17.743 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.742+0000 7ffa24a27640 1 -- 172.21.15.99:0/3545031292 shutdown_connections 2024-03-19T02:06:17.743 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.742+0000 7ffa24a27640 1 --2- 172.21.15.99:0/3545031292 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7ffa201075a0 0x7ffa2019f130 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:17.743 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.742+0000 7ffa24a27640 1 --2- 172.21.15.99:0/3545031292 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7ffa20104c70 0x7ffa201980c0 unknown :-1 s=CLOSED pgs=826 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:17.743 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.742+0000 7ffa24a27640 1 --2- 172.21.15.99:0/3545031292 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7ffa0c0a2800 0x7ffa0c0a4cc0 unknown :-1 s=CLOSED pgs=409 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:17.743 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.742+0000 7ffa24a27640 1 --2- 172.21.15.99:0/3545031292 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7ffa20102340 0x7ffa20197b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:17.743 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.742+0000 7ffa24a27640 1 -- 172.21.15.99:0/3545031292 >> 172.21.15.99:0/3545031292 conn(0x7ffa200fbcd0 msgr2=0x7ffa20102d80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:06:17.743 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.742+0000 7ffa24a27640 1 -- 172.21.15.99:0/3545031292 shutdown_connections 2024-03-19T02:06:17.743 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.742+0000 7ffa24a27640 1 -- 172.21.15.99:0/3545031292 wait complete. 2024-03-19T02:06:17.754 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-03-19T02:06:17.865 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.857+0000 7f54fd552640 1 -- 172.21.15.99:0/2055595382 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f54f81022f0 msgr2=0x7f54f8108b80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:17.865 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.857+0000 7f54fd552640 1 --2- 172.21.15.99:0/2055595382 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f54f81022f0 0x7f54f8108b80 secure :-1 s=READY pgs=827 cs=0 l=1 rev1=1 crypto rx=0x7f54f0009ca0 tx=0x7f54f002c0f0 comp rx=0 tx=0).stop 2024-03-19T02:06:17.866 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.857+0000 7f54fd552640 1 -- 172.21.15.99:0/2055595382 shutdown_connections 2024-03-19T02:06:17.866 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.857+0000 7f54fd552640 1 --2- 172.21.15.99:0/2055595382 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f54f8101930 0x7f54f8101db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:17.866 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.857+0000 7f54fd552640 1 --2- 172.21.15.99:0/2055595382 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f54f81022f0 0x7f54f8108b80 unknown :-1 s=CLOSED pgs=827 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:17.866 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.857+0000 7f54fd552640 1 --2- 172.21.15.99:0/2055595382 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f54f8100740 0x7f54f8100b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:17.866 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.857+0000 7f54fd552640 1 -- 172.21.15.99:0/2055595382 >> 172.21.15.99:0/2055595382 conn(0x7f54f80fbcb0 msgr2=0x7f54f80fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:06:17.866 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.857+0000 7f54fd552640 1 -- 172.21.15.99:0/2055595382 shutdown_connections 2024-03-19T02:06:17.867 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.857+0000 7f54fd552640 1 -- 172.21.15.99:0/2055595382 wait complete. 2024-03-19T02:06:17.867 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.858+0000 7f54fd552640 1 Processor -- start 2024-03-19T02:06:17.867 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.858+0000 7f54fd552640 1 -- start start 2024-03-19T02:06:17.867 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.858+0000 7f54fd552640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f54f8100740 0x7f54f8196a30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:06:17.867 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.858+0000 7f54fd552640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f54f8101930 0x7f54f8196f70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:06:17.867 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.858+0000 7f54fd552640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f54f81022f0 0x7f54f819dfe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:06:17.867 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.858+0000 7f54fd552640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f54f810c3c0 con 0x7f54f8100740 2024-03-19T02:06:17.868 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.858+0000 7f54fd552640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f54f810c240 con 0x7f54f8101930 2024-03-19T02:06:17.868 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.858+0000 7f54fd552640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f54f810c540 con 0x7f54f81022f0 2024-03-19T02:06:17.868 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.858+0000 7f54f6ffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f54f8100740 0x7f54f8196a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:06:17.868 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.858+0000 7f54f6ffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f54f8100740 0x7f54f8196a30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58418/0 (socket says 172.21.15.99:58418) 2024-03-19T02:06:17.868 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.858+0000 7f54f6ffd640 1 -- 172.21.15.99:0/3669511095 learned_addr learned my addr 172.21.15.99:0/3669511095 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:06:17.868 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.858+0000 7f54f67fc640 1 --2- 172.21.15.99:0/3669511095 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f54f8101930 0x7f54f8196f70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:06:17.868 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.858+0000 7f54f77fe640 1 --2- 172.21.15.99:0/3669511095 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f54f81022f0 0x7f54f819dfe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:06:17.868 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.859+0000 7f54f6ffd640 1 -- 172.21.15.99:0/3669511095 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f54f81022f0 msgr2=0x7f54f819dfe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:17.868 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.859+0000 7f54f6ffd640 1 --2- 172.21.15.99:0/3669511095 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f54f81022f0 0x7f54f819dfe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:17.868 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.859+0000 7f54f6ffd640 1 -- 172.21.15.99:0/3669511095 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f54f8101930 msgr2=0x7f54f8196f70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:17.868 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.859+0000 7f54f6ffd640 1 --2- 172.21.15.99:0/3669511095 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f54f8101930 0x7f54f8196f70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:17.868 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.859+0000 7f54f6ffd640 1 -- 172.21.15.99:0/3669511095 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f54f819e650 con 0x7f54f8100740 2024-03-19T02:06:17.868 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.859+0000 7f54f6ffd640 1 --2- 172.21.15.99:0/3669511095 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f54f8100740 0x7f54f8196a30 secure :-1 s=READY pgs=828 cs=0 l=1 rev1=1 crypto rx=0x7f54ec00e9c0 tx=0x7f54ec00ee80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:06:17.868 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.859+0000 7f54dffff640 1 -- 172.21.15.99:0/3669511095 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f54ec00cd80 con 0x7f54f8100740 2024-03-19T02:06:17.869 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.859+0000 7f54dffff640 1 -- 172.21.15.99:0/3669511095 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f54ec0035f0 con 0x7f54f8100740 2024-03-19T02:06:17.869 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.859+0000 7f54fd552640 1 -- 172.21.15.99:0/3669511095 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f54f819e940 con 0x7f54f8100740 2024-03-19T02:06:17.869 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.859+0000 7f54dffff640 1 -- 172.21.15.99:0/3669511095 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f54ec010690 con 0x7f54f8100740 2024-03-19T02:06:17.869 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.859+0000 7f54fd552640 1 -- 172.21.15.99:0/3669511095 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f54f819ee40 con 0x7f54f8100740 2024-03-19T02:06:17.869 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.860+0000 7f54fd552640 1 -- 172.21.15.99:0/3669511095 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f54f8068610 con 0x7f54f8100740 2024-03-19T02:06:17.869 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.861+0000 7f54dffff640 1 -- 172.21.15.99:0/3669511095 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f54ec0037a0 con 0x7f54f8100740 2024-03-19T02:06:17.869 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.862+0000 7f54dffff640 1 --2- 172.21.15.99:0/3669511095 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f54e00a2800 0x7f54e00a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:06:17.869 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.862+0000 7f54dffff640 1 -- 172.21.15.99:0/3669511095 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f54ec0ceed0 con 0x7f54f8100740 2024-03-19T02:06:17.869 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.862+0000 7f54f67fc640 1 --2- 172.21.15.99:0/3669511095 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f54e00a2800 0x7f54e00a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:06:17.869 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.862+0000 7f54f67fc640 1 --2- 172.21.15.99:0/3669511095 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f54e00a2800 0x7f54e00a4cc0 secure :-1 s=READY pgs=410 cs=0 l=1 rev1=1 crypto rx=0x7f54e80095f0 tx=0x7f54e8005f90 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:06:17.877 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:17.865+0000 7f54dffff640 1 -- 172.21.15.99:0/3669511095 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f54ec09bf30 con 0x7f54f8100740 2024-03-19T02:06:18.044 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.044+0000 7f54fd552640 1 -- 172.21.15.99:0/3669511095 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f54f802d1f0 con 0x7f54f8100740 2024-03-19T02:06:18.046 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.046+0000 7f54dffff640 1 -- 172.21.15.99:0/3669511095 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v92) ==== 74+0+22804 (secure 0 0 0) 0x7f54f802d1f0 con 0x7f54f8100740 2024-03-19T02:06:18.046 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-19T02:06:18.046 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":92,"fsid":"438fb69c-e590-11ee-95c9-87774f69a715","created":"2024-03-19T01:31:10.165073+0000","modified":"2024-03-19T01:52:30.348737+0000","last_up_change":"2024-03-19T01:38:09.082688+0000","last_in_change":"2024-03-19T01:37:49.115793+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-19T01:34:31.001309+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-03-19T01:38:54.018623+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"86","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-03-19T01:38:55.319036+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"87","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"0c407a0e-a40e-43d7-ad9a-bb8f3cd4a9ba","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6803","nonce":2935260112}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6805","nonce":2935260112}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6808","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6809","nonce":2935260112}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":2935260112},{"type":"v1","addr":"172.21.15.99:6807","nonce":2935260112}]},"public_addr":"172.21.15.99:6803/2935260112","cluster_addr":"172.21.15.99:6805/2935260112","heartbeat_back_addr":"172.21.15.99:6809/2935260112","heartbeat_front_addr":"172.21.15.99:6807/2935260112","state":["exists","up"]},{"osd":1,"uuid":"a48f9173-4a06-49f5-bca2-2c9047b8f666","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6800","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6801","nonce":1468854289}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6802","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6803","nonce":1468854289}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6806","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6807","nonce":1468854289}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6804","nonce":1468854289},{"type":"v1","addr":"172.21.15.112:6805","nonce":1468854289}]},"public_addr":"172.21.15.112:6801/1468854289","cluster_addr":"172.21.15.112:6803/1468854289","heartbeat_back_addr":"172.21.15.112:6807/1468854289","heartbeat_front_addr":"172.21.15.112:6805/1468854289","state":["exists","up"]},{"osd":2,"uuid":"b7715fdd-b6c9-4348-bd16-5eb61b427eef","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6800","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6801","nonce":3766690472}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6802","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6803","nonce":3766690472}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6806","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6807","nonce":3766690472}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6804","nonce":3766690472},{"type":"v1","addr":"172.21.15.115:6805","nonce":3766690472}]},"public_addr":"172.21.15.115:6801/3766690472","cluster_addr":"172.21.15.115:6803/3766690472","heartbeat_back_addr":"172.21.15.115:6807/3766690472","heartbeat_front_addr":"172.21.15.115:6805/3766690472","state":["exists","up"]},{"osd":3,"uuid":"6697ed1b-96bf-42de-8440-d17e9785b359","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6810","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6811","nonce":3756732895}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6812","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6813","nonce":3756732895}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6816","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6817","nonce":3756732895}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6814","nonce":3756732895},{"type":"v1","addr":"172.21.15.99:6815","nonce":3756732895}]},"public_addr":"172.21.15.99:6811/3756732895","cluster_addr":"172.21.15.99:6813/3756732895","heartbeat_back_addr":"172.21.15.99:6817/3756732895","heartbeat_front_addr":"172.21.15.99:6815/3756732895","state":["exists","up"]},{"osd":4,"uuid":"d958d6b9-67f6-43d0-b704-aa55dc9df83a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":34,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6808","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6809","nonce":3717986391}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6810","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6811","nonce":3717986391}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6814","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6815","nonce":3717986391}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6812","nonce":3717986391},{"type":"v1","addr":"172.21.15.112:6813","nonce":3717986391}]},"public_addr":"172.21.15.112:6809/3717986391","cluster_addr":"172.21.15.112:6811/3717986391","heartbeat_back_addr":"172.21.15.112:6815/3717986391","heartbeat_front_addr":"172.21.15.112:6813/3717986391","state":["exists","up"]},{"osd":5,"uuid":"a73eb49a-7925-49ba-bc4f-515e17f8b8ae","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6808","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6809","nonce":1971806408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6810","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6811","nonce":1971806408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6814","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6815","nonce":1971806408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6812","nonce":1971806408},{"type":"v1","addr":"172.21.15.115:6813","nonce":1971806408}]},"public_addr":"172.21.15.115:6809/1971806408","cluster_addr":"172.21.15.115:6811/1971806408","heartbeat_back_addr":"172.21.15.115:6815/1971806408","heartbeat_front_addr":"172.21.15.115:6813/1971806408","state":["exists","up"]},{"osd":6,"uuid":"caa718cd-e08b-48de-832a-29546d71e508","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6818","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6819","nonce":1130832772}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6820","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6821","nonce":1130832772}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6824","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6825","nonce":1130832772}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6822","nonce":1130832772},{"type":"v1","addr":"172.21.15.99:6823","nonce":1130832772}]},"public_addr":"172.21.15.99:6819/1130832772","cluster_addr":"172.21.15.99:6821/1130832772","heartbeat_back_addr":"172.21.15.99:6825/1130832772","heartbeat_front_addr":"172.21.15.99:6823/1130832772","state":["exists","up"]},{"osd":7,"uuid":"dbee19a1-2315-40d3-88e7-876fcde28903","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":53,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6816","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6817","nonce":3610356576}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6818","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6819","nonce":3610356576}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6822","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6823","nonce":3610356576}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6820","nonce":3610356576},{"type":"v1","addr":"172.21.15.112:6821","nonce":3610356576}]},"public_addr":"172.21.15.112:6817/3610356576","cluster_addr":"172.21.15.112:6819/3610356576","heartbeat_back_addr":"172.21.15.112:6823/3610356576","heartbeat_front_addr":"172.21.15.112:6821/3610356576","state":["exists","up"]},{"osd":8,"uuid":"14a866dc-5ce3-42fc-8bcf-9088b68726cb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":60,"up_thru":85,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6816","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6817","nonce":4255803111}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6818","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6819","nonce":4255803111}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6822","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6823","nonce":4255803111}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6820","nonce":4255803111},{"type":"v1","addr":"172.21.15.115:6821","nonce":4255803111}]},"public_addr":"172.21.15.115:6817/4255803111","cluster_addr":"172.21.15.115:6819/4255803111","heartbeat_back_addr":"172.21.15.115:6823/4255803111","heartbeat_front_addr":"172.21.15.115:6821/4255803111","state":["exists","up"]},{"osd":9,"uuid":"1db93652-6035-4906-91ae-d64d5a909502","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":68,"up_thru":90,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6826","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6827","nonce":1410921523}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6828","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6829","nonce":1410921523}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6832","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6833","nonce":1410921523}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6830","nonce":1410921523},{"type":"v1","addr":"172.21.15.99:6831","nonce":1410921523}]},"public_addr":"172.21.15.99:6827/1410921523","cluster_addr":"172.21.15.99:6829/1410921523","heartbeat_back_addr":"172.21.15.99:6833/1410921523","heartbeat_front_addr":"172.21.15.99:6831/1410921523","state":["exists","up"]},{"osd":10,"uuid":"6c50a8a7-3345-4f4a-af0e-5606c7aca453","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":74,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6824","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6825","nonce":891805922}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6826","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6827","nonce":891805922}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6830","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6831","nonce":891805922}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6828","nonce":891805922},{"type":"v1","addr":"172.21.15.112:6829","nonce":891805922}]},"public_addr":"172.21.15.112:6825/891805922","cluster_addr":"172.21.15.112:6827/891805922","heartbeat_back_addr":"172.21.15.112:6831/891805922","heartbeat_front_addr":"172.21.15.112:6829/891805922","state":["exists","up"]},{"osd":11,"uuid":"fd63b7fd-0db5-46d5-82fa-9707bbd0b495","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":82,"up_thru":88,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6824","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6825","nonce":938436244}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6826","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6827","nonce":938436244}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6830","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6831","nonce":938436244}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.115:6828","nonce":938436244},{"type":"v1","addr":"172.21.15.115:6829","nonce":938436244}]},"public_addr":"172.21.15.115:6825/938436244","cluster_addr":"172.21.15.115:6827/938436244","heartbeat_back_addr":"172.21.15.115:6831/938436244","heartbeat_front_addr":"172.21.15.115:6829/938436244","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:33:39.036822+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:34:03.339566+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:34:27.149273+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:34:51.315561+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:35:15.396031+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:35:38.957168+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:36:03.213914+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:36:27.763699+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:36:51.688670+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:37:17.280165+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:37:42.554633+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-19T01:38:07.438039+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.0","mappings":[{"from":8,"to":1}]},{"pgid":"2.8","mappings":[{"from":2,"to":11}]},{"pgid":"2.b","mappings":[{"from":8,"to":11}]},{"pgid":"2.e","mappings":[{"from":2,"to":4}]},{"pgid":"3.0","mappings":[{"from":5,"to":9}]},{"pgid":"3.2","mappings":[{"from":3,"to":9}]},{"pgid":"3.14","mappings":[{"from":1,"to":9}]},{"pgid":"3.17","mappings":[{"from":4,"to":8}]},{"pgid":"3.31","mappings":[{"from":4,"to":8}]},{"pgid":"3.37","mappings":[{"from":5,"to":7}]},{"pgid":"3.3a","mappings":[{"from":4,"to":6}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.99:6801/1328245934":"2024-03-20T01:32:06.661002+0000","172.21.15.99:0/1618965837":"2024-03-20T01:32:06.661002+0000","172.21.15.99:0/2491429851":"2024-03-20T01:32:06.661002+0000","172.21.15.99:6800/1328245934":"2024-03-20T01:32:06.661002+0000","172.21.15.99:0/2112231766":"2024-03-20T01:31:40.161751+0000","172.21.15.99:0/3341836545":"2024-03-20T01:31:40.161751+0000","172.21.15.99:6800/4223048293":"2024-03-20T01:31:40.161751+0000","172.21.15.99:6801/4223048293":"2024-03-20T01:31:40.161751+0000","172.21.15.99:0/2377621863":"2024-03-20T01:31:40.161751+0000","172.21.15.99:0/1904400935":"2024-03-20T01:32:06.661002+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-19T02:06:18.048 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.048+0000 7f54fd552640 1 -- 172.21.15.99:0/3669511095 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f54e00a2800 msgr2=0x7f54e00a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:18.048 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.048+0000 7f54fd552640 1 --2- 172.21.15.99:0/3669511095 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f54e00a2800 0x7f54e00a4cc0 secure :-1 s=READY pgs=410 cs=0 l=1 rev1=1 crypto rx=0x7f54e80095f0 tx=0x7f54e8005f90 comp rx=0 tx=0).stop 2024-03-19T02:06:18.048 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.048+0000 7f54fd552640 1 -- 172.21.15.99:0/3669511095 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f54f8100740 msgr2=0x7f54f8196a30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:18.048 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.048+0000 7f54fd552640 1 --2- 172.21.15.99:0/3669511095 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f54f8100740 0x7f54f8196a30 secure :-1 s=READY pgs=828 cs=0 l=1 rev1=1 crypto rx=0x7f54ec00e9c0 tx=0x7f54ec00ee80 comp rx=0 tx=0).stop 2024-03-19T02:06:18.048 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.048+0000 7f54fd552640 1 -- 172.21.15.99:0/3669511095 shutdown_connections 2024-03-19T02:06:18.049 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.048+0000 7f54fd552640 1 --2- 172.21.15.99:0/3669511095 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f54e00a2800 0x7f54e00a4cc0 unknown :-1 s=CLOSED pgs=410 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:18.049 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.048+0000 7f54fd552640 1 --2- 172.21.15.99:0/3669511095 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f54f8101930 0x7f54f8196f70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:18.049 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.048+0000 7f54fd552640 1 --2- 172.21.15.99:0/3669511095 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f54f81022f0 0x7f54f819dfe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:18.049 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.048+0000 7f54fd552640 1 --2- 172.21.15.99:0/3669511095 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f54f8100740 0x7f54f8196a30 unknown :-1 s=CLOSED pgs=828 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:18.049 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.048+0000 7f54fd552640 1 -- 172.21.15.99:0/3669511095 >> 172.21.15.99:0/3669511095 conn(0x7f54f80fbcb0 msgr2=0x7f54f8104c30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:06:18.049 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.048+0000 7f54fd552640 1 -- 172.21.15.99:0/3669511095 shutdown_connections 2024-03-19T02:06:18.049 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.048+0000 7f54fd552640 1 -- 172.21.15.99:0/3669511095 wait complete. 2024-03-19T02:06:18.058 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:06:17 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/13146027' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T02:06:18.058 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:06:17 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/3545031292' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T02:06:18.061 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs fail cephfs 2024-03-19T02:06:18.167 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.164+0000 7f7fe7d62640 1 -- 172.21.15.99:0/412736219 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7fe01022f0 msgr2=0x7f7fe0108b80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:18.167 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.164+0000 7f7fe7d62640 1 --2- 172.21.15.99:0/412736219 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7fe01022f0 0x7f7fe0108b80 secure :-1 s=READY pgs=829 cs=0 l=1 rev1=1 crypto rx=0x7f7fd8009ca0 tx=0x7f7fd802c0f0 comp rx=0 tx=0).stop 2024-03-19T02:06:18.167 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.164+0000 7f7fe7d62640 1 -- 172.21.15.99:0/412736219 shutdown_connections 2024-03-19T02:06:18.167 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.164+0000 7f7fe7d62640 1 --2- 172.21.15.99:0/412736219 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7fe01022f0 0x7f7fe0108b80 unknown :-1 s=CLOSED pgs=829 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:18.167 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.164+0000 7f7fe7d62640 1 --2- 172.21.15.99:0/412736219 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f7fe0101930 0x7f7fe0101db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:18.167 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.164+0000 7f7fe7d62640 1 --2- 172.21.15.99:0/412736219 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f7fe0100740 0x7f7fe0100b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:18.167 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.164+0000 7f7fe7d62640 1 -- 172.21.15.99:0/412736219 >> 172.21.15.99:0/412736219 conn(0x7f7fe00fbcd0 msgr2=0x7f7fe00fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:06:18.167 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.165+0000 7f7fe7d62640 1 -- 172.21.15.99:0/412736219 shutdown_connections 2024-03-19T02:06:18.167 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.165+0000 7f7fe7d62640 1 -- 172.21.15.99:0/412736219 wait complete. 2024-03-19T02:06:18.167 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.165+0000 7f7fe7d62640 1 Processor -- start 2024-03-19T02:06:18.168 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.165+0000 7f7fe7d62640 1 -- start start 2024-03-19T02:06:18.168 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.165+0000 7f7fe7d62640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f7fe0100740 0x7f7fe0197ae0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:06:18.168 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.165+0000 7f7fe7d62640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7fe0101930 0x7f7fe0198020 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:06:18.168 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.165+0000 7f7fe7d62640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f7fe01022f0 0x7f7fe019f090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:06:18.168 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.165+0000 7f7fe7d62640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f7fe010d4a0 con 0x7f7fe0101930 2024-03-19T02:06:18.168 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.165+0000 7f7fe7d62640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f7fe010d320 con 0x7f7fe01022f0 2024-03-19T02:06:18.168 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.165+0000 7f7fe7d62640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f7fe010d620 con 0x7f7fe0100740 2024-03-19T02:06:18.168 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.165+0000 7f7fe52fd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7fe0101930 0x7f7fe0198020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:06:18.168 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.165+0000 7f7fe5afe640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f7fe0100740 0x7f7fe0197ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:06:18.168 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.165+0000 7f7fe5afe640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f7fe0100740 0x7f7fe0197ae0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.115:3300/0 says I am v2:172.21.15.99:32934/0 (socket says 172.21.15.99:32934) 2024-03-19T02:06:18.168 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.165+0000 7f7fe5afe640 1 -- 172.21.15.99:0/441142250 learned_addr learned my addr 172.21.15.99:0/441142250 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:06:18.168 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.165+0000 7f7fe62ff640 1 --2- 172.21.15.99:0/441142250 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f7fe01022f0 0x7f7fe019f090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:06:18.168 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.166+0000 7f7fe52fd640 1 -- 172.21.15.99:0/441142250 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f7fe0100740 msgr2=0x7f7fe0197ae0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:18.168 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.166+0000 7f7fe52fd640 1 --2- 172.21.15.99:0/441142250 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f7fe0100740 0x7f7fe0197ae0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:18.169 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.166+0000 7f7fe52fd640 1 -- 172.21.15.99:0/441142250 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f7fe01022f0 msgr2=0x7f7fe019f090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:18.169 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.166+0000 7f7fe52fd640 1 --2- 172.21.15.99:0/441142250 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f7fe01022f0 0x7f7fe019f090 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:18.169 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.166+0000 7f7fe52fd640 1 -- 172.21.15.99:0/441142250 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7fe019f700 con 0x7f7fe0101930 2024-03-19T02:06:18.169 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.166+0000 7f7fe52fd640 1 --2- 172.21.15.99:0/441142250 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7fe0101930 0x7f7fe0198020 secure :-1 s=READY pgs=830 cs=0 l=1 rev1=1 crypto rx=0x7f7fd000e9f0 tx=0x7f7fd000eeb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:06:18.169 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.166+0000 7f7fd6ffd640 1 -- 172.21.15.99:0/441142250 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7fd000cd80 con 0x7f7fe0101930 2024-03-19T02:06:18.169 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.166+0000 7f7fd6ffd640 1 -- 172.21.15.99:0/441142250 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f7fd00035f0 con 0x7f7fe0101930 2024-03-19T02:06:18.169 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.166+0000 7f7fe7d62640 1 -- 172.21.15.99:0/441142250 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f7fe019f9f0 con 0x7f7fe0101930 2024-03-19T02:06:18.169 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.166+0000 7f7fd6ffd640 1 -- 172.21.15.99:0/441142250 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7fd0010690 con 0x7f7fe0101930 2024-03-19T02:06:18.169 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.166+0000 7f7fe7d62640 1 -- 172.21.15.99:0/441142250 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f7fe019ff90 con 0x7f7fe0101930 2024-03-19T02:06:18.169 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.167+0000 7f7fe7d62640 1 -- 172.21.15.99:0/441142250 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f7fe0107b30 con 0x7f7fe0101930 2024-03-19T02:06:18.170 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.168+0000 7f7fd6ffd640 1 -- 172.21.15.99:0/441142250 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f7fd0003b30 con 0x7f7fe0101930 2024-03-19T02:06:18.170 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.169+0000 7f7fd6ffd640 1 --2- 172.21.15.99:0/441142250 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f7fcc0a2880 0x7f7fcc0a4d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:06:18.170 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.169+0000 7f7fe5afe640 1 --2- 172.21.15.99:0/441142250 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f7fcc0a2880 0x7f7fcc0a4d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:06:18.170 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.169+0000 7f7fd6ffd640 1 -- 172.21.15.99:0/441142250 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8298+0+0 (secure 0 0 0) 0x7f7fd00cece0 con 0x7f7fe0101930 2024-03-19T02:06:18.170 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.169+0000 7f7fe5afe640 1 --2- 172.21.15.99:0/441142250 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f7fcc0a2880 0x7f7fcc0a4d40 secure :-1 s=READY pgs=411 cs=0 l=1 rev1=1 crypto rx=0x7f7fdc0095f0 tx=0x7f7fdc005f90 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:06:18.173 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.173+0000 7f7fd6ffd640 1 -- 172.21.15.99:0/441142250 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f7fd009bd40 con 0x7f7fe0101930 2024-03-19T02:06:18.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:06:17 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/13146027' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T02:06:18.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:06:17 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/3545031292' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T02:06:18.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:06:17 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/13146027' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T02:06:18.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:06:17 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/3545031292' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-03-19T02:06:18.387 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.387+0000 7f7fe7d62640 1 -- 172.21.15.99:0/441142250 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "fs fail", "fs_name": "cephfs"} v 0) -- 0x7f7fe0062420 con 0x7f7fe0101930 2024-03-19T02:06:18.389 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.390+0000 7f7fd6ffd640 1 -- 172.21.15.99:0/441142250 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "fs fail", "fs_name": "cephfs"}]=0 cephfs marked not joinable; MDS cannot join the cluster. All MDS ranks marked failed. v48) ==== 161+0+0 (secure 0 0 0) 0x7f7fd0097030 con 0x7f7fe0101930 2024-03-19T02:06:18.389 INFO:teuthology.orchestra.run.smithi099.stderr:cephfs marked not joinable; MDS cannot join the cluster. All MDS ranks marked failed. 2024-03-19T02:06:18.392 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.392+0000 7f7fe7d62640 1 -- 172.21.15.99:0/441142250 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f7fcc0a2880 msgr2=0x7f7fcc0a4d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:18.392 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.392+0000 7f7fe7d62640 1 --2- 172.21.15.99:0/441142250 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f7fcc0a2880 0x7f7fcc0a4d40 secure :-1 s=READY pgs=411 cs=0 l=1 rev1=1 crypto rx=0x7f7fdc0095f0 tx=0x7f7fdc005f90 comp rx=0 tx=0).stop 2024-03-19T02:06:18.392 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.392+0000 7f7fe7d62640 1 -- 172.21.15.99:0/441142250 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7fe0101930 msgr2=0x7f7fe0198020 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:18.392 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.392+0000 7f7fe7d62640 1 --2- 172.21.15.99:0/441142250 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7fe0101930 0x7f7fe0198020 secure :-1 s=READY pgs=830 cs=0 l=1 rev1=1 crypto rx=0x7f7fd000e9f0 tx=0x7f7fd000eeb0 comp rx=0 tx=0).stop 2024-03-19T02:06:18.392 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.392+0000 7f7fe7d62640 1 -- 172.21.15.99:0/441142250 shutdown_connections 2024-03-19T02:06:18.392 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.392+0000 7f7fe7d62640 1 --2- 172.21.15.99:0/441142250 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f7fcc0a2880 0x7f7fcc0a4d40 unknown :-1 s=CLOSED pgs=411 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:18.392 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.392+0000 7f7fe7d62640 1 --2- 172.21.15.99:0/441142250 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f7fe01022f0 0x7f7fe019f090 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:18.392 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.392+0000 7f7fe7d62640 1 --2- 172.21.15.99:0/441142250 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f7fe0101930 0x7f7fe0198020 unknown :-1 s=CLOSED pgs=830 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:18.392 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.392+0000 7f7fe7d62640 1 --2- 172.21.15.99:0/441142250 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f7fe0100740 0x7f7fe0197ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:18.392 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.392+0000 7f7fe7d62640 1 -- 172.21.15.99:0/441142250 >> 172.21.15.99:0/441142250 conn(0x7f7fe00fbcd0 msgr2=0x7f7fe0104d00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:06:18.393 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.392+0000 7f7fe7d62640 1 -- 172.21.15.99:0/441142250 shutdown_connections 2024-03-19T02:06:18.393 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.392+0000 7f7fe7d62640 1 -- 172.21.15.99:0/441142250 wait complete. 2024-03-19T02:06:18.404 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs rm cephfs --yes-i-really-mean-it 2024-03-19T02:06:18.513 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.511+0000 7f9fe1380640 1 -- 172.21.15.99:0/1939825172 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9fdc101850 msgr2=0x7f9fdc101c70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:18.513 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.511+0000 7f9fe1380640 1 --2- 172.21.15.99:0/1939825172 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9fdc101850 0x7f9fdc101c70 secure :-1 s=READY pgs=834 cs=0 l=1 rev1=1 crypto rx=0x7f9fcc009ca0 tx=0x7f9fcc02c0f0 comp rx=0 tx=0).stop 2024-03-19T02:06:18.513 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.511+0000 7f9fe1380640 1 -- 172.21.15.99:0/1939825172 shutdown_connections 2024-03-19T02:06:18.513 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.511+0000 7f9fe1380640 1 --2- 172.21.15.99:0/1939825172 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9fdc103400 0x7f9fdc109c90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:18.513 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.511+0000 7f9fe1380640 1 --2- 172.21.15.99:0/1939825172 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9fdc102a40 0x7f9fdc102ec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:18.513 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.511+0000 7f9fe1380640 1 --2- 172.21.15.99:0/1939825172 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9fdc101850 0x7f9fdc101c70 unknown :-1 s=CLOSED pgs=834 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:18.513 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.511+0000 7f9fe1380640 1 -- 172.21.15.99:0/1939825172 >> 172.21.15.99:0/1939825172 conn(0x7f9fdc0fcdc0 msgr2=0x7f9fdc0ff200 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:06:18.514 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.511+0000 7f9fe1380640 1 -- 172.21.15.99:0/1939825172 shutdown_connections 2024-03-19T02:06:18.514 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.511+0000 7f9fe1380640 1 -- 172.21.15.99:0/1939825172 wait complete. 2024-03-19T02:06:18.514 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.512+0000 7f9fe1380640 1 Processor -- start 2024-03-19T02:06:18.514 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.512+0000 7f9fe1380640 1 -- start start 2024-03-19T02:06:18.514 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.512+0000 7f9fe1380640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9fdc101850 0x7f9fdc197b10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:06:18.514 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.512+0000 7f9fe1380640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9fdc102a40 0x7f9fdc198050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:06:18.514 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.512+0000 7f9fe1380640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9fdc103400 0x7f9fdc19f0c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:06:18.514 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.512+0000 7f9fe1380640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f9fdc10d4a0 con 0x7f9fdc102a40 2024-03-19T02:06:18.514 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.512+0000 7f9fe1380640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f9fdc10d320 con 0x7f9fdc101850 2024-03-19T02:06:18.514 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.512+0000 7f9fe1380640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f9fdc10d620 con 0x7f9fdc103400 2024-03-19T02:06:18.514 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.512+0000 7f9fda7fc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9fdc102a40 0x7f9fdc198050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:06:18.514 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.512+0000 7f9fda7fc640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9fdc102a40 0x7f9fdc198050 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58464/0 (socket says 172.21.15.99:58464) 2024-03-19T02:06:18.515 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.512+0000 7f9fda7fc640 1 -- 172.21.15.99:0/1197144428 learned_addr learned my addr 172.21.15.99:0/1197144428 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:06:18.515 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.512+0000 7f9fdb7fe640 1 --2- 172.21.15.99:0/1197144428 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9fdc103400 0x7f9fdc19f0c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:06:18.515 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.512+0000 7f9fdaffd640 1 --2- 172.21.15.99:0/1197144428 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9fdc101850 0x7f9fdc197b10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:06:18.515 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.513+0000 7f9fda7fc640 1 -- 172.21.15.99:0/1197144428 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9fdc103400 msgr2=0x7f9fdc19f0c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:18.515 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.513+0000 7f9fda7fc640 1 --2- 172.21.15.99:0/1197144428 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9fdc103400 0x7f9fdc19f0c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:18.515 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.513+0000 7f9fda7fc640 1 -- 172.21.15.99:0/1197144428 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9fdc101850 msgr2=0x7f9fdc197b10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:18.515 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.513+0000 7f9fda7fc640 1 --2- 172.21.15.99:0/1197144428 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9fdc101850 0x7f9fdc197b10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:18.515 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.513+0000 7f9fda7fc640 1 -- 172.21.15.99:0/1197144428 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f9fdc19f730 con 0x7f9fdc102a40 2024-03-19T02:06:18.515 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.513+0000 7f9fdb7fe640 1 --2- 172.21.15.99:0/1197144428 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9fdc103400 0x7f9fdc19f0c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-03-19T02:06:18.515 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.513+0000 7f9fda7fc640 1 --2- 172.21.15.99:0/1197144428 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9fdc102a40 0x7f9fdc198050 secure :-1 s=READY pgs=835 cs=0 l=1 rev1=1 crypto rx=0x7f9fd000c890 tx=0x7f9fd000cd50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:06:18.515 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.513+0000 7f9fc3fff640 1 -- 172.21.15.99:0/1197144428 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9fd0004220 con 0x7f9fdc102a40 2024-03-19T02:06:18.515 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.513+0000 7f9fe1380640 1 -- 172.21.15.99:0/1197144428 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f9fdc19fa20 con 0x7f9fdc102a40 2024-03-19T02:06:18.515 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.513+0000 7f9fc3fff640 1 -- 172.21.15.99:0/1197144428 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f9fd00043c0 con 0x7f9fdc102a40 2024-03-19T02:06:18.515 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.513+0000 7f9fc3fff640 1 -- 172.21.15.99:0/1197144428 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9fd0004a00 con 0x7f9fdc102a40 2024-03-19T02:06:18.516 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.513+0000 7f9fe1380640 1 -- 172.21.15.99:0/1197144428 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f9fdc19ff70 con 0x7f9fdc102a40 2024-03-19T02:06:18.516 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.514+0000 7f9fe1380640 1 -- 172.21.15.99:0/1197144428 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f9fdc108bc0 con 0x7f9fdc102a40 2024-03-19T02:06:18.516 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.515+0000 7f9fc3fff640 1 -- 172.21.15.99:0/1197144428 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f9fd00079b0 con 0x7f9fdc102a40 2024-03-19T02:06:18.517 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.515+0000 7f9fc3fff640 1 --2- 172.21.15.99:0/1197144428 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f9fc40a2800 0x7f9fc40a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:06:18.517 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.516+0000 7f9fdaffd640 1 --2- 172.21.15.99:0/1197144428 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f9fc40a2800 0x7f9fc40a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:06:18.517 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.516+0000 7f9fc3fff640 1 -- 172.21.15.99:0/1197144428 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(93..93 src has 1..93) ==== 8556+0+0 (secure 0 0 0) 0x7f9fd00cec70 con 0x7f9fdc102a40 2024-03-19T02:06:18.517 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.516+0000 7f9fdaffd640 1 --2- 172.21.15.99:0/1197144428 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f9fc40a2800 0x7f9fc40a4cc0 secure :-1 s=READY pgs=412 cs=0 l=1 rev1=1 crypto rx=0x7f9fcc02c700 tx=0x7f9fcc032590 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:06:18.519 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.519+0000 7f9fc3fff640 1 -- 172.21.15.99:0/1197144428 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f9fd0014030 con 0x7f9fdc102a40 2024-03-19T02:06:18.732 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:18.732+0000 7f9fe1380640 1 -- 172.21.15.99:0/1197144428 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true} v 0) -- 0x7f9fdc02d440 con 0x7f9fdc102a40 2024-03-19T02:06:18.806 INFO:journalctl@ceph.mds.c.smithi115.stdout:Mar 19 02:06:18 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-c[97153]: ignoring --setuser ceph since I am not root 2024-03-19T02:06:18.807 INFO:journalctl@ceph.mds.c.smithi115.stdout:Mar 19 02:06:18 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-c[97153]: ignoring --setgroup ceph since I am not root 2024-03-19T02:06:18.807 INFO:journalctl@ceph.mds.c.smithi115.stdout:Mar 19 02:06:18 smithi115 ceph-mds[97157]: -- 172.21.15.115:0/1435071195 <== mon.2 v2:172.21.15.112:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55a10220da40 con 0x55a1021e0800 2024-03-19T02:06:18.807 INFO:journalctl@ceph.mds.c.smithi115.stdout:Mar 19 02:06:18 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-c[97153]: starting mds.c at 2024-03-19T02:06:18.807 INFO:journalctl@ceph.mds.f.smithi115.stdout:Mar 19 02:06:18 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-f[96845]: ignoring --setuser ceph since I am not root 2024-03-19T02:06:18.807 INFO:journalctl@ceph.mds.f.smithi115.stdout:Mar 19 02:06:18 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-f[96845]: ignoring --setgroup ceph since I am not root 2024-03-19T02:06:18.807 INFO:journalctl@ceph.mds.f.smithi115.stdout:Mar 19 02:06:18 smithi115 ceph-mds[96849]: -- 172.21.15.115:0/3287977010 <== mon.0 v2:172.21.15.99:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55d8f1d95a40 con 0x55d8f1d69000 2024-03-19T02:06:18.807 INFO:journalctl@ceph.mds.f.smithi115.stdout:Mar 19 02:06:18 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-f[96845]: starting mds.f at 2024-03-19T02:06:18.807 INFO:journalctl@ceph.mds.i.smithi115.stdout:Mar 19 02:06:18 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-i[97459]: ignoring --setuser ceph since I am not root 2024-03-19T02:06:18.808 INFO:journalctl@ceph.mds.i.smithi115.stdout:Mar 19 02:06:18 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-i[97459]: ignoring --setgroup ceph since I am not root 2024-03-19T02:06:18.808 INFO:journalctl@ceph.mds.i.smithi115.stdout:Mar 19 02:06:18 smithi115 ceph-mds[97463]: -- 172.21.15.115:0/2219122861 <== mon.2 v2:172.21.15.112:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x562a682eba40 con 0x562a682be800 2024-03-19T02:06:18.808 INFO:journalctl@ceph.mds.i.smithi115.stdout:Mar 19 02:06:18 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-i[97459]: starting mds.i at 2024-03-19T02:06:18.840 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:06:18 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mon-a[62947]: 2024-03-19T02:06:18.388+0000 7f4e6c0bc700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-19T02:06:18.862 INFO:journalctl@ceph.mds.b.smithi112.stdout:Mar 19 02:06:18 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-b[97403]: ignoring --setuser ceph since I am not root 2024-03-19T02:06:18.862 INFO:journalctl@ceph.mds.b.smithi112.stdout:Mar 19 02:06:18 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-b[97403]: ignoring --setgroup ceph since I am not root 2024-03-19T02:06:18.862 INFO:journalctl@ceph.mds.b.smithi112.stdout:Mar 19 02:06:18 smithi112 ceph-mds[97407]: -- 172.21.15.112:0/218821401 <== mon.2 v2:172.21.15.112:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55b3590e3a40 con 0x55b3590b6800 2024-03-19T02:06:18.863 INFO:journalctl@ceph.mds.b.smithi112.stdout:Mar 19 02:06:18 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-b[97403]: starting mds.b at 2024-03-19T02:06:18.863 INFO:journalctl@ceph.mds.e.smithi112.stdout:Mar 19 02:06:18 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-e[96773]: ignoring --setuser ceph since I am not root 2024-03-19T02:06:18.863 INFO:journalctl@ceph.mds.e.smithi112.stdout:Mar 19 02:06:18 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-e[96773]: ignoring --setgroup ceph since I am not root 2024-03-19T02:06:18.863 INFO:journalctl@ceph.mds.e.smithi112.stdout:Mar 19 02:06:18 smithi112 ceph-mds[96777]: -- 172.21.15.112:0/1277878694 <== mon.2 v2:172.21.15.112:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x555c46765a40 con 0x555c46738800 2024-03-19T02:06:18.863 INFO:journalctl@ceph.mds.e.smithi112.stdout:Mar 19 02:06:18 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-e[96773]: starting mds.e at 2024-03-19T02:06:18.863 INFO:journalctl@ceph.mds.h.smithi112.stdout:Mar 19 02:06:18 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-h[97080]: ignoring --setuser ceph since I am not root 2024-03-19T02:06:18.863 INFO:journalctl@ceph.mds.h.smithi112.stdout:Mar 19 02:06:18 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-h[97080]: ignoring --setgroup ceph since I am not root 2024-03-19T02:06:18.863 INFO:journalctl@ceph.mds.h.smithi112.stdout:Mar 19 02:06:18 smithi112 ceph-mds[97084]: -- 172.21.15.112:0/2835817830 <== mon.2 v2:172.21.15.112:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55f57d0e1a40 con 0x55f57d0b4c00 2024-03-19T02:06:18.864 INFO:journalctl@ceph.mds.h.smithi112.stdout:Mar 19 02:06:18 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-h[97080]: starting mds.h at 2024-03-19T02:06:19.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:06:18 smithi115 ceph-mon[67332]: pgmap v1107: 97 pgs: 97 active+clean; 433 MiB data, 9.0 GiB used, 1.0 TiB / 1.0 TiB avail; 85 KiB/s rd, 281 KiB/s wr, 40 op/s 2024-03-19T02:06:19.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:06:18 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/3669511095' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-19T02:06:19.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:06:18 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/441142250' entity='client.admin' cmd=[{"prefix": "fs fail", "fs_name": "cephfs"}]: dispatch 2024-03-19T02:06:19.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:06:18 smithi115 ceph-mon[67332]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-03-19T02:06:19.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:06:18 smithi115 ceph-mon[67332]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-19T02:06:19.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:06:18 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/441142250' entity='client.admin' cmd='[{"prefix": "fs fail", "fs_name": "cephfs"}]': finished 2024-03-19T02:06:19.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:06:18 smithi115 ceph-mon[67332]: osdmap e93: 12 total, 12 up, 12 in 2024-03-19T02:06:19.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:06:18 smithi115 ceph-mon[67332]: fsmap cephfs:0/3 6 up:standby, 3 failed 2024-03-19T02:06:19.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:06:18 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/1197144428' entity='client.admin' cmd=[{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]: dispatch 2024-03-19T02:06:19.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:06:18 smithi099 ceph-mon[62951]: pgmap v1107: 97 pgs: 97 active+clean; 433 MiB data, 9.0 GiB used, 1.0 TiB / 1.0 TiB avail; 85 KiB/s rd, 281 KiB/s wr, 40 op/s 2024-03-19T02:06:19.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:06:18 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/3669511095' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-19T02:06:19.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:06:18 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/441142250' entity='client.admin' cmd=[{"prefix": "fs fail", "fs_name": "cephfs"}]: dispatch 2024-03-19T02:06:19.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:06:18 smithi099 ceph-mon[62951]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-03-19T02:06:19.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:06:18 smithi099 ceph-mon[62951]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-19T02:06:19.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:06:18 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/441142250' entity='client.admin' cmd='[{"prefix": "fs fail", "fs_name": "cephfs"}]': finished 2024-03-19T02:06:19.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:06:18 smithi099 ceph-mon[62951]: osdmap e93: 12 total, 12 up, 12 in 2024-03-19T02:06:19.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:06:18 smithi099 ceph-mon[62951]: fsmap cephfs:0/3 6 up:standby, 3 failed 2024-03-19T02:06:19.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:06:18 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/1197144428' entity='client.admin' cmd=[{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]: dispatch 2024-03-19T02:06:19.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:06:18 smithi112 ceph-mon[66819]: pgmap v1107: 97 pgs: 97 active+clean; 433 MiB data, 9.0 GiB used, 1.0 TiB / 1.0 TiB avail; 85 KiB/s rd, 281 KiB/s wr, 40 op/s 2024-03-19T02:06:19.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:06:18 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/3669511095' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-19T02:06:19.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:06:18 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/441142250' entity='client.admin' cmd=[{"prefix": "fs fail", "fs_name": "cephfs"}]: dispatch 2024-03-19T02:06:19.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:06:18 smithi112 ceph-mon[66819]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-03-19T02:06:19.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:06:18 smithi112 ceph-mon[66819]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-19T02:06:19.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:06:18 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/441142250' entity='client.admin' cmd='[{"prefix": "fs fail", "fs_name": "cephfs"}]': finished 2024-03-19T02:06:19.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:06:18 smithi112 ceph-mon[66819]: osdmap e93: 12 total, 12 up, 12 in 2024-03-19T02:06:19.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:06:18 smithi112 ceph-mon[66819]: fsmap cephfs:0/3 6 up:standby, 3 failed 2024-03-19T02:06:19.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:06:18 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/1197144428' entity='client.admin' cmd=[{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]: dispatch 2024-03-19T02:06:19.391 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:19.391+0000 7f9fc3fff640 1 -- 172.21.15.99:0/1197144428 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]=0 v49) ==== 104+0+0 (secure 0 0 0) 0x7f9fd009bbd0 con 0x7f9fdc102a40 2024-03-19T02:06:19.395 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:19.395+0000 7f9fe1380640 1 -- 172.21.15.99:0/1197144428 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f9fc40a2800 msgr2=0x7f9fc40a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:19.395 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:19.395+0000 7f9fe1380640 1 --2- 172.21.15.99:0/1197144428 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f9fc40a2800 0x7f9fc40a4cc0 secure :-1 s=READY pgs=412 cs=0 l=1 rev1=1 crypto rx=0x7f9fcc02c700 tx=0x7f9fcc032590 comp rx=0 tx=0).stop 2024-03-19T02:06:19.395 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:19.395+0000 7f9fe1380640 1 -- 172.21.15.99:0/1197144428 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9fdc102a40 msgr2=0x7f9fdc198050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:19.395 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:19.395+0000 7f9fe1380640 1 --2- 172.21.15.99:0/1197144428 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9fdc102a40 0x7f9fdc198050 secure :-1 s=READY pgs=835 cs=0 l=1 rev1=1 crypto rx=0x7f9fd000c890 tx=0x7f9fd000cd50 comp rx=0 tx=0).stop 2024-03-19T02:06:19.396 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:19.395+0000 7f9fe1380640 1 -- 172.21.15.99:0/1197144428 shutdown_connections 2024-03-19T02:06:19.396 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:19.395+0000 7f9fe1380640 1 --2- 172.21.15.99:0/1197144428 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f9fdc103400 0x7f9fdc19f0c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:19.396 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:19.395+0000 7f9fe1380640 1 --2- 172.21.15.99:0/1197144428 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f9fdc102a40 0x7f9fdc198050 unknown :-1 s=CLOSED pgs=835 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:19.396 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:19.395+0000 7f9fe1380640 1 --2- 172.21.15.99:0/1197144428 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f9fc40a2800 0x7f9fc40a4cc0 unknown :-1 s=CLOSED pgs=412 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:19.396 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:19.395+0000 7f9fe1380640 1 --2- 172.21.15.99:0/1197144428 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f9fdc101850 0x7f9fdc197b10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:19.396 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:19.395+0000 7f9fe1380640 1 -- 172.21.15.99:0/1197144428 >> 172.21.15.99:0/1197144428 conn(0x7f9fdc0fcdc0 msgr2=0x7f9fdc105d90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:06:19.396 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:19.396+0000 7f9fe1380640 1 -- 172.21.15.99:0/1197144428 shutdown_connections 2024-03-19T02:06:19.396 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:19.396+0000 7f9fe1380640 1 -- 172.21.15.99:0/1197144428 wait complete. 2024-03-19T02:06:19.410 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool rm cephfs_metadata cephfs_metadata --yes-i-really-really-mean-it 2024-03-19T02:06:19.524 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:19.522+0000 7f2d053df640 1 -- 172.21.15.99:0/3837225903 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2d00101930 msgr2=0x7f2d00101db0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:19.524 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:19.522+0000 7f2d053df640 1 --2- 172.21.15.99:0/3837225903 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2d00101930 0x7f2d00101db0 secure :-1 s=READY pgs=836 cs=0 l=1 rev1=1 crypto rx=0x7f2ce8009900 tx=0x7f2ce802bc30 comp rx=0 tx=0).stop 2024-03-19T02:06:19.524 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:19.522+0000 7f2d053df640 1 -- 172.21.15.99:0/3837225903 shutdown_connections 2024-03-19T02:06:19.524 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:19.522+0000 7f2d053df640 1 --2- 172.21.15.99:0/3837225903 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f2d001022f0 0x7f2d00108b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:19.524 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:19.522+0000 7f2d053df640 1 --2- 172.21.15.99:0/3837225903 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2d00101930 0x7f2d00101db0 unknown :-1 s=CLOSED pgs=836 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:19.524 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:19.522+0000 7f2d053df640 1 --2- 172.21.15.99:0/3837225903 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f2d00100740 0x7f2d00100b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:19.524 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:19.522+0000 7f2d053df640 1 -- 172.21.15.99:0/3837225903 >> 172.21.15.99:0/3837225903 conn(0x7f2d000fbcb0 msgr2=0x7f2d000fe0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:06:19.524 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:19.522+0000 7f2d053df640 1 -- 172.21.15.99:0/3837225903 shutdown_connections 2024-03-19T02:06:19.524 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:19.522+0000 7f2d053df640 1 -- 172.21.15.99:0/3837225903 wait complete. 2024-03-19T02:06:19.524 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:19.522+0000 7f2d053df640 1 Processor -- start 2024-03-19T02:06:19.525 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:19.522+0000 7f2d053df640 1 -- start start 2024-03-19T02:06:19.525 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:19.522+0000 7f2d053df640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2d00100740 0x7f2d00197c20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:06:19.525 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:19.523+0000 7f2d053df640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f2d00101930 0x7f2d00198160 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:06:19.525 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:19.523+0000 7f2d053df640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f2d001022f0 0x7f2d0019f180 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:06:19.525 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:19.523+0000 7f2d053df640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f2d0010c3c0 con 0x7f2d00100740 2024-03-19T02:06:19.525 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:19.523+0000 7f2d053df640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f2d0010c240 con 0x7f2d00101930 2024-03-19T02:06:19.525 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:19.523+0000 7f2d053df640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f2d0010c540 con 0x7f2d001022f0 2024-03-19T02:06:19.525 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:19.523+0000 7f2cfeffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2d00100740 0x7f2d00197c20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:06:19.525 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:19.523+0000 7f2cfeffd640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2d00100740 0x7f2d00197c20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58484/0 (socket says 172.21.15.99:58484) 2024-03-19T02:06:19.525 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:19.523+0000 7f2cfeffd640 1 -- 172.21.15.99:0/1608510425 learned_addr learned my addr 172.21.15.99:0/1608510425 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:06:19.525 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:19.523+0000 7f2cff7fe640 1 --2- 172.21.15.99:0/1608510425 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f2d001022f0 0x7f2d0019f180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:06:19.525 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:19.523+0000 7f2cf67fc640 1 --2- 172.21.15.99:0/1608510425 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f2d00101930 0x7f2d00198160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:06:19.525 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:19.523+0000 7f2cfeffd640 1 -- 172.21.15.99:0/1608510425 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f2d001022f0 msgr2=0x7f2d0019f180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:19.525 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:19.523+0000 7f2cfeffd640 1 --2- 172.21.15.99:0/1608510425 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f2d001022f0 0x7f2d0019f180 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:19.525 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:19.523+0000 7f2cfeffd640 1 -- 172.21.15.99:0/1608510425 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f2d00101930 msgr2=0x7f2d00198160 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:19.526 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:19.523+0000 7f2cfeffd640 1 --2- 172.21.15.99:0/1608510425 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f2d00101930 0x7f2d00198160 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:19.526 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:19.523+0000 7f2cfeffd640 1 -- 172.21.15.99:0/1608510425 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f2d0019f7f0 con 0x7f2d00100740 2024-03-19T02:06:19.526 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:19.523+0000 7f2cf67fc640 1 --2- 172.21.15.99:0/1608510425 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f2d00101930 0x7f2d00198160 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-03-19T02:06:19.526 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:19.523+0000 7f2cfeffd640 1 --2- 172.21.15.99:0/1608510425 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2d00100740 0x7f2d00197c20 secure :-1 s=READY pgs=837 cs=0 l=1 rev1=1 crypto rx=0x7f2cf000d950 tx=0x7f2cf000de10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:06:19.526 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:19.523+0000 7f2cfcff9640 1 -- 172.21.15.99:0/1608510425 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2cf0017070 con 0x7f2d00100740 2024-03-19T02:06:19.526 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:19.523+0000 7f2cfcff9640 1 -- 172.21.15.99:0/1608510425 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f2cf0009c30 con 0x7f2d00100740 2024-03-19T02:06:19.526 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:19.523+0000 7f2d053df640 1 -- 172.21.15.99:0/1608510425 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f2d0019fae0 con 0x7f2d00100740 2024-03-19T02:06:19.526 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:19.523+0000 7f2cfcff9640 1 -- 172.21.15.99:0/1608510425 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2cf000f600 con 0x7f2d00100740 2024-03-19T02:06:19.526 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:19.523+0000 7f2d053df640 1 -- 172.21.15.99:0/1608510425 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f2d001a0030 con 0x7f2d00100740 2024-03-19T02:06:19.526 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:19.524+0000 7f2d053df640 1 -- 172.21.15.99:0/1608510425 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f2d001a0200 con 0x7f2d00100740 2024-03-19T02:06:19.527 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:19.525+0000 7f2cfcff9640 1 -- 172.21.15.99:0/1608510425 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f2cf001a070 con 0x7f2d00100740 2024-03-19T02:06:19.527 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:19.526+0000 7f2cfcff9640 1 --2- 172.21.15.99:0/1608510425 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f2cec0a2880 0x7f2cec0a4d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:06:19.527 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:19.526+0000 7f2cf67fc640 1 --2- 172.21.15.99:0/1608510425 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f2cec0a2880 0x7f2cec0a4d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:06:19.527 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:19.526+0000 7f2cfcff9640 1 -- 172.21.15.99:0/1608510425 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(93..93 src has 1..93) ==== 8556+0+0 (secure 0 0 0) 0x7f2cf00ce640 con 0x7f2d00100740 2024-03-19T02:06:19.527 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:19.527+0000 7f2cf67fc640 1 --2- 172.21.15.99:0/1608510425 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f2cec0a2880 0x7f2cec0a4d40 secure :-1 s=READY pgs=419 cs=0 l=1 rev1=1 crypto rx=0x7f2ce8036040 tx=0x7f2ce8000bc0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:06:19.529 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:19.530+0000 7f2cfcff9640 1 -- 172.21.15.99:0/1608510425 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f2cf0009de0 con 0x7f2d00100740 2024-03-19T02:06:19.706 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:19.707+0000 7f2d053df640 1 -- 172.21.15.99:0/1608510425 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true} v 0) -- 0x7f2d001a03d0 con 0x7f2d00100740 2024-03-19T02:06:20.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:06:19 smithi115 ceph-mon[67332]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-03-19T02:06:20.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:06:19 smithi115 ceph-mon[67332]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-03-19T02:06:20.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:06:19 smithi115 ceph-mon[67332]: Cluster is now healthy 2024-03-19T02:06:20.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:06:19 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/1197144428' entity='client.admin' cmd='[{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]': finished 2024-03-19T02:06:20.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:06:19 smithi115 ceph-mon[67332]: mds.? [v2:172.21.15.112:6836/3891050149,v1:172.21.15.112:6838/3891050149] up:boot 2024-03-19T02:06:20.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:06:19 smithi115 ceph-mon[67332]: mds.? [v2:172.21.15.112:6834/1562431297,v1:172.21.15.112:6835/1562431297] up:boot 2024-03-19T02:06:20.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:06:19 smithi115 ceph-mon[67332]: mds.? [v2:172.21.15.112:6837/3017913889,v1:172.21.15.112:6839/3017913889] up:boot 2024-03-19T02:06:20.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:06:19 smithi115 ceph-mon[67332]: mds.? [v2:172.21.15.115:6835/2026468910,v1:172.21.15.115:6838/2026468910] up:boot 2024-03-19T02:06:20.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:06:19 smithi115 ceph-mon[67332]: mds.? [v2:172.21.15.115:6836/1713250482,v1:172.21.15.115:6839/1713250482] up:boot 2024-03-19T02:06:20.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:06:19 smithi115 ceph-mon[67332]: mds.? [v2:172.21.15.115:6834/771221114,v1:172.21.15.115:6837/771221114] up:boot 2024-03-19T02:06:20.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:06:19 smithi115 ceph-mon[67332]: fsmap 12 up:standby 2024-03-19T02:06:20.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:06:19 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "e"}]: dispatch 2024-03-19T02:06:20.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:06:19 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "f"}]: dispatch 2024-03-19T02:06:20.307 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:06:19 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "h"}]: dispatch 2024-03-19T02:06:20.308 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:06:19 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "c"}]: dispatch 2024-03-19T02:06:20.308 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:06:19 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-03-19T02:06:20.308 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:06:19 smithi115 ceph-mon[67332]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "i"}]: dispatch 2024-03-19T02:06:20.308 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:06:19 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/1608510425' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]: dispatch 2024-03-19T02:06:20.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:06:19 smithi099 ceph-mon[62951]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-03-19T02:06:20.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:06:19 smithi099 ceph-mon[62951]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-03-19T02:06:20.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:06:19 smithi099 ceph-mon[62951]: Cluster is now healthy 2024-03-19T02:06:20.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:06:19 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/1197144428' entity='client.admin' cmd='[{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]': finished 2024-03-19T02:06:20.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:06:19 smithi099 ceph-mon[62951]: mds.? [v2:172.21.15.112:6836/3891050149,v1:172.21.15.112:6838/3891050149] up:boot 2024-03-19T02:06:20.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:06:19 smithi099 ceph-mon[62951]: mds.? [v2:172.21.15.112:6834/1562431297,v1:172.21.15.112:6835/1562431297] up:boot 2024-03-19T02:06:20.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:06:19 smithi099 ceph-mon[62951]: mds.? [v2:172.21.15.112:6837/3017913889,v1:172.21.15.112:6839/3017913889] up:boot 2024-03-19T02:06:20.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:06:19 smithi099 ceph-mon[62951]: mds.? [v2:172.21.15.115:6835/2026468910,v1:172.21.15.115:6838/2026468910] up:boot 2024-03-19T02:06:20.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:06:19 smithi099 ceph-mon[62951]: mds.? [v2:172.21.15.115:6836/1713250482,v1:172.21.15.115:6839/1713250482] up:boot 2024-03-19T02:06:20.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:06:19 smithi099 ceph-mon[62951]: mds.? [v2:172.21.15.115:6834/771221114,v1:172.21.15.115:6837/771221114] up:boot 2024-03-19T02:06:20.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:06:19 smithi099 ceph-mon[62951]: fsmap 12 up:standby 2024-03-19T02:06:20.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:06:19 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "e"}]: dispatch 2024-03-19T02:06:20.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:06:19 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "f"}]: dispatch 2024-03-19T02:06:20.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:06:19 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "h"}]: dispatch 2024-03-19T02:06:20.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:06:19 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "c"}]: dispatch 2024-03-19T02:06:20.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:06:19 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-03-19T02:06:20.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:06:19 smithi099 ceph-mon[62951]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "i"}]: dispatch 2024-03-19T02:06:20.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:06:19 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/1608510425' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]: dispatch 2024-03-19T02:06:20.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:06:19 smithi112 ceph-mon[66819]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-03-19T02:06:20.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:06:19 smithi112 ceph-mon[66819]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-03-19T02:06:20.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:06:19 smithi112 ceph-mon[66819]: Cluster is now healthy 2024-03-19T02:06:20.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:06:19 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/1197144428' entity='client.admin' cmd='[{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]': finished 2024-03-19T02:06:20.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:06:19 smithi112 ceph-mon[66819]: mds.? [v2:172.21.15.112:6836/3891050149,v1:172.21.15.112:6838/3891050149] up:boot 2024-03-19T02:06:20.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:06:19 smithi112 ceph-mon[66819]: mds.? [v2:172.21.15.112:6834/1562431297,v1:172.21.15.112:6835/1562431297] up:boot 2024-03-19T02:06:20.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:06:19 smithi112 ceph-mon[66819]: mds.? [v2:172.21.15.112:6837/3017913889,v1:172.21.15.112:6839/3017913889] up:boot 2024-03-19T02:06:20.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:06:19 smithi112 ceph-mon[66819]: mds.? [v2:172.21.15.115:6835/2026468910,v1:172.21.15.115:6838/2026468910] up:boot 2024-03-19T02:06:20.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:06:19 smithi112 ceph-mon[66819]: mds.? [v2:172.21.15.115:6836/1713250482,v1:172.21.15.115:6839/1713250482] up:boot 2024-03-19T02:06:20.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:06:19 smithi112 ceph-mon[66819]: mds.? [v2:172.21.15.115:6834/771221114,v1:172.21.15.115:6837/771221114] up:boot 2024-03-19T02:06:20.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:06:19 smithi112 ceph-mon[66819]: fsmap 12 up:standby 2024-03-19T02:06:20.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:06:19 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "e"}]: dispatch 2024-03-19T02:06:20.363 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:06:19 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "f"}]: dispatch 2024-03-19T02:06:20.363 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:06:19 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "h"}]: dispatch 2024-03-19T02:06:20.363 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:06:19 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "c"}]: dispatch 2024-03-19T02:06:20.363 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:06:19 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-03-19T02:06:20.363 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:06:19 smithi112 ceph-mon[66819]: from='mgr.14150 172.21.15.99:0/2500444532' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "i"}]: dispatch 2024-03-19T02:06:20.363 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:06:19 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/1608510425' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]: dispatch 2024-03-19T02:06:20.393 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:20.393+0000 7f2cfcff9640 1 -- 172.21.15.99:0/1608510425 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]=0 pool 'cephfs_metadata' removed v94) ==== 181+0+0 (secure 0 0 0) 0x7f2cf009b5c0 con 0x7f2d00100740 2024-03-19T02:06:20.393 INFO:teuthology.orchestra.run.smithi099.stderr:pool 'cephfs_metadata' removed 2024-03-19T02:06:20.396 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:20.397+0000 7f2d053df640 1 -- 172.21.15.99:0/1608510425 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f2cec0a2880 msgr2=0x7f2cec0a4d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:20.397 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:20.397+0000 7f2d053df640 1 --2- 172.21.15.99:0/1608510425 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f2cec0a2880 0x7f2cec0a4d40 secure :-1 s=READY pgs=419 cs=0 l=1 rev1=1 crypto rx=0x7f2ce8036040 tx=0x7f2ce8000bc0 comp rx=0 tx=0).stop 2024-03-19T02:06:20.397 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:20.397+0000 7f2d053df640 1 -- 172.21.15.99:0/1608510425 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2d00100740 msgr2=0x7f2d00197c20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:20.397 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:20.397+0000 7f2d053df640 1 --2- 172.21.15.99:0/1608510425 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2d00100740 0x7f2d00197c20 secure :-1 s=READY pgs=837 cs=0 l=1 rev1=1 crypto rx=0x7f2cf000d950 tx=0x7f2cf000de10 comp rx=0 tx=0).stop 2024-03-19T02:06:20.397 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:20.397+0000 7f2d053df640 1 -- 172.21.15.99:0/1608510425 shutdown_connections 2024-03-19T02:06:20.397 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:20.397+0000 7f2d053df640 1 --2- 172.21.15.99:0/1608510425 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f2cec0a2880 0x7f2cec0a4d40 unknown :-1 s=CLOSED pgs=419 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:20.397 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:20.397+0000 7f2d053df640 1 --2- 172.21.15.99:0/1608510425 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f2d00101930 0x7f2d00198160 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:20.397 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:20.397+0000 7f2d053df640 1 --2- 172.21.15.99:0/1608510425 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f2d001022f0 0x7f2d0019f180 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:20.397 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:20.397+0000 7f2d053df640 1 --2- 172.21.15.99:0/1608510425 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2d00100740 0x7f2d00197c20 unknown :-1 s=CLOSED pgs=837 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:20.397 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:20.397+0000 7f2d053df640 1 -- 172.21.15.99:0/1608510425 >> 172.21.15.99:0/1608510425 conn(0x7f2d000fbcb0 msgr2=0x7f2d00069ac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:06:20.398 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:20.397+0000 7f2d053df640 1 -- 172.21.15.99:0/1608510425 shutdown_connections 2024-03-19T02:06:20.398 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:20.397+0000 7f2d053df640 1 -- 172.21.15.99:0/1608510425 wait complete. 2024-03-19T02:06:20.416 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool rm cephfs_data cephfs_data --yes-i-really-really-mean-it 2024-03-19T02:06:20.534 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:20.532+0000 7f256be0b640 1 -- 172.21.15.99:0/3368081634 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2564102300 msgr2=0x7f25641046f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:20.534 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:20.532+0000 7f256be0b640 1 --2- 172.21.15.99:0/3368081634 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2564102300 0x7f25641046f0 secure :-1 s=READY pgs=838 cs=0 l=1 rev1=1 crypto rx=0x7f2560009900 tx=0x7f256002bc30 comp rx=0 tx=0).stop 2024-03-19T02:06:20.534 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:20.532+0000 7f256be0b640 1 -- 172.21.15.99:0/3368081634 shutdown_connections 2024-03-19T02:06:20.534 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:20.532+0000 7f256be0b640 1 --2- 172.21.15.99:0/3368081634 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f2564104c30 0x7f2564107020 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:20.534 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:20.532+0000 7f256be0b640 1 --2- 172.21.15.99:0/3368081634 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f2564107560 0x7f2564109950 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:20.534 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:20.532+0000 7f256be0b640 1 --2- 172.21.15.99:0/3368081634 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2564102300 0x7f25641046f0 unknown :-1 s=CLOSED pgs=838 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:20.534 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:20.532+0000 7f256be0b640 1 -- 172.21.15.99:0/3368081634 >> 172.21.15.99:0/3368081634 conn(0x7f25640fbcb0 msgr2=0x7f25640fe0d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:06:20.534 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:20.532+0000 7f256be0b640 1 -- 172.21.15.99:0/3368081634 shutdown_connections 2024-03-19T02:06:20.534 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:20.532+0000 7f256be0b640 1 -- 172.21.15.99:0/3368081634 wait complete. 2024-03-19T02:06:20.534 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:20.533+0000 7f256be0b640 1 Processor -- start 2024-03-19T02:06:20.535 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:20.533+0000 7f256be0b640 1 -- start start 2024-03-19T02:06:20.535 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:20.533+0000 7f256be0b640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f2564102300 0x7f2564197ba0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:06:20.535 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:20.533+0000 7f256be0b640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f2564104c30 0x7f25641980e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:06:20.535 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:20.533+0000 7f256be0b640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2564107560 0x7f256419f150 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:06:20.535 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:20.533+0000 7f256be0b640 1 -- --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_getmap magic: 0 -- 0x7f256410d490 con 0x7f2564107560 2024-03-19T02:06:20.535 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:20.533+0000 7f256be0b640 1 -- --> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] -- mon_getmap magic: 0 -- 0x7f256410d310 con 0x7f2564104c30 2024-03-19T02:06:20.535 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:20.533+0000 7f256be0b640 1 -- --> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] -- mon_getmap magic: 0 -- 0x7f256410d610 con 0x7f2564102300 2024-03-19T02:06:20.535 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:20.533+0000 7f256a3a8640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2564107560 0x7f256419f150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:06:20.535 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:20.533+0000 7f2569ba7640 1 --2- >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f2564102300 0x7f2564197ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:06:20.535 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:20.533+0000 7f256a3a8640 1 --2- >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2564107560 0x7f256419f150 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.99:3300/0 says I am v2:172.21.15.99:58510/0 (socket says 172.21.15.99:58510) 2024-03-19T02:06:20.535 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:20.533+0000 7f25693a6640 1 --2- >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f2564104c30 0x7f25641980e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:06:20.535 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:20.533+0000 7f256a3a8640 1 -- 172.21.15.99:0/2148705669 learned_addr learned my addr 172.21.15.99:0/2148705669 (peer_addr_for_me v2:172.21.15.99:0/0) 2024-03-19T02:06:20.535 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:20.533+0000 7f256a3a8640 1 -- 172.21.15.99:0/2148705669 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f2564102300 msgr2=0x7f2564197ba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:20.536 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:20.533+0000 7f256a3a8640 1 --2- 172.21.15.99:0/2148705669 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f2564102300 0x7f2564197ba0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:20.536 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:20.533+0000 7f256a3a8640 1 -- 172.21.15.99:0/2148705669 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f2564104c30 msgr2=0x7f25641980e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:20.536 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:20.533+0000 7f256a3a8640 1 --2- 172.21.15.99:0/2148705669 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f2564104c30 0x7f25641980e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:20.536 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:20.533+0000 7f256a3a8640 1 -- 172.21.15.99:0/2148705669 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f256419f7c0 con 0x7f2564107560 2024-03-19T02:06:20.536 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:20.533+0000 7f25693a6640 1 --2- 172.21.15.99:0/2148705669 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f2564104c30 0x7f25641980e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-03-19T02:06:20.536 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:20.533+0000 7f256a3a8640 1 --2- 172.21.15.99:0/2148705669 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2564107560 0x7f256419f150 secure :-1 s=READY pgs=839 cs=0 l=1 rev1=1 crypto rx=0x7f255c00eaa0 tx=0x7f255c00ef60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:06:20.536 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:20.534+0000 7f255affd640 1 -- 172.21.15.99:0/2148705669 <== mon.0 v2:172.21.15.99:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f255c00cdb0 con 0x7f2564107560 2024-03-19T02:06:20.536 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:20.534+0000 7f256be0b640 1 -- 172.21.15.99:0/2148705669 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f256419fab0 con 0x7f2564107560 2024-03-19T02:06:20.536 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:20.534+0000 7f255affd640 1 -- 172.21.15.99:0/2148705669 <== mon.0 v2:172.21.15.99:3300/0 2 ==== config(27 keys) ==== 1140+0+0 (secure 0 0 0) 0x7f255c005020 con 0x7f2564107560 2024-03-19T02:06:20.536 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:20.534+0000 7f255affd640 1 -- 172.21.15.99:0/2148705669 <== mon.0 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f255c0116b0 con 0x7f2564107560 2024-03-19T02:06:20.536 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:20.534+0000 7f256be0b640 1 -- 172.21.15.99:0/2148705669 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f256419ffb0 con 0x7f2564107560 2024-03-19T02:06:20.537 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:20.535+0000 7f256be0b640 1 -- 172.21.15.99:0/2148705669 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f25640fd7a0 con 0x7f2564107560 2024-03-19T02:06:20.537 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:20.536+0000 7f255affd640 1 -- 172.21.15.99:0/2148705669 <== mon.0 v2:172.21.15.99:3300/0 4 ==== mgrmap(e 20) ==== 137290+0+0 (secure 0 0 0) 0x7f255c005560 con 0x7f2564107560 2024-03-19T02:06:20.537 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:20.536+0000 7f255affd640 1 --2- 172.21.15.99:0/2148705669 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f25500a2800 0x7f25500a4cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-03-19T02:06:20.537 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:20.537+0000 7f2569ba7640 1 --2- 172.21.15.99:0/2148705669 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f25500a2800 0x7f25500a4cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-03-19T02:06:20.537 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:20.537+0000 7f255affd640 1 -- 172.21.15.99:0/2148705669 <== mon.0 v2:172.21.15.99:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 8002+0+0 (secure 0 0 0) 0x7f255c0ceae0 con 0x7f2564107560 2024-03-19T02:06:20.538 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:20.537+0000 7f2569ba7640 1 --2- 172.21.15.99:0/2148705669 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f25500a2800 0x7f25500a4cc0 secure :-1 s=READY pgs=422 cs=0 l=1 rev1=1 crypto rx=0x7f256002c7b0 tx=0x7f2560005fd0 comp rx=0 tx=0).ready entity=mgr.14150 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-03-19T02:06:20.540 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:20.541+0000 7f255affd640 1 -- 172.21.15.99:0/2148705669 <== mon.0 v2:172.21.15.99:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+186149 (secure 0 0 0) 0x7f255c016030 con 0x7f2564107560 2024-03-19T02:06:20.728 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:20.728+0000 7f256be0b640 1 -- 172.21.15.99:0/2148705669 --> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] -- mon_command({"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true} v 0) -- 0x7f256402d1f0 con 0x7f2564107560 2024-03-19T02:06:21.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:06:20 smithi115 ceph-mon[67332]: pgmap v1109: 97 pgs: 97 active+clean; 433 MiB data, 9.0 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 57 KiB/s wr, 18 op/s 2024-03-19T02:06:21.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:06:20 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/1608510425' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]': finished 2024-03-19T02:06:21.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:06:20 smithi115 ceph-mon[67332]: osdmap e94: 12 total, 12 up, 12 in 2024-03-19T02:06:21.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:06:20 smithi115 ceph-mon[67332]: from='client.? 172.21.15.99:0/2148705669' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]: dispatch 2024-03-19T02:06:21.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:06:20 smithi099 ceph-mon[62951]: pgmap v1109: 97 pgs: 97 active+clean; 433 MiB data, 9.0 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 57 KiB/s wr, 18 op/s 2024-03-19T02:06:21.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:06:20 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/1608510425' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]': finished 2024-03-19T02:06:21.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:06:20 smithi099 ceph-mon[62951]: osdmap e94: 12 total, 12 up, 12 in 2024-03-19T02:06:21.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:06:20 smithi099 ceph-mon[62951]: from='client.? 172.21.15.99:0/2148705669' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]: dispatch 2024-03-19T02:06:21.361 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:06:20 smithi112 ceph-mon[66819]: pgmap v1109: 97 pgs: 97 active+clean; 433 MiB data, 9.0 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 57 KiB/s wr, 18 op/s 2024-03-19T02:06:21.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:06:20 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/1608510425' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]': finished 2024-03-19T02:06:21.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:06:20 smithi112 ceph-mon[66819]: osdmap e94: 12 total, 12 up, 12 in 2024-03-19T02:06:21.362 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:06:20 smithi112 ceph-mon[66819]: from='client.? 172.21.15.99:0/2148705669' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]: dispatch 2024-03-19T02:06:21.394 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:21.395+0000 7f255affd640 1 -- 172.21.15.99:0/2148705669 <== mon.0 v2:172.21.15.99:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]=0 pool 'cephfs_data' removed v95) ==== 169+0+0 (secure 0 0 0) 0x7f255c09b660 con 0x7f2564107560 2024-03-19T02:06:21.394 INFO:teuthology.orchestra.run.smithi099.stderr:pool 'cephfs_data' removed 2024-03-19T02:06:21.397 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:21.397+0000 7f256be0b640 1 -- 172.21.15.99:0/2148705669 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f25500a2800 msgr2=0x7f25500a4cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:21.397 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:21.397+0000 7f256be0b640 1 --2- 172.21.15.99:0/2148705669 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f25500a2800 0x7f25500a4cc0 secure :-1 s=READY pgs=422 cs=0 l=1 rev1=1 crypto rx=0x7f256002c7b0 tx=0x7f2560005fd0 comp rx=0 tx=0).stop 2024-03-19T02:06:21.397 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:21.397+0000 7f256be0b640 1 -- 172.21.15.99:0/2148705669 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2564107560 msgr2=0x7f256419f150 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-03-19T02:06:21.397 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:21.397+0000 7f256be0b640 1 --2- 172.21.15.99:0/2148705669 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2564107560 0x7f256419f150 secure :-1 s=READY pgs=839 cs=0 l=1 rev1=1 crypto rx=0x7f255c00eaa0 tx=0x7f255c00ef60 comp rx=0 tx=0).stop 2024-03-19T02:06:21.397 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:21.397+0000 7f256be0b640 1 -- 172.21.15.99:0/2148705669 shutdown_connections 2024-03-19T02:06:21.398 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:21.397+0000 7f256be0b640 1 --2- 172.21.15.99:0/2148705669 >> [v2:172.21.15.99:6800/4272410295,v1:172.21.15.99:6801/4272410295] conn(0x7f25500a2800 0x7f25500a4cc0 unknown :-1 s=CLOSED pgs=422 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:21.398 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:21.397+0000 7f256be0b640 1 --2- 172.21.15.99:0/2148705669 >> [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] conn(0x7f2564104c30 0x7f25641980e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:21.398 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:21.397+0000 7f256be0b640 1 --2- 172.21.15.99:0/2148705669 >> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] conn(0x7f2564107560 0x7f256419f150 unknown :-1 s=CLOSED pgs=839 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:21.398 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:21.397+0000 7f256be0b640 1 --2- 172.21.15.99:0/2148705669 >> [v2:172.21.15.115:3300/0,v1:172.21.15.115:6789/0] conn(0x7f2564102300 0x7f2564197ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-03-19T02:06:21.398 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:21.397+0000 7f256be0b640 1 -- 172.21.15.99:0/2148705669 >> 172.21.15.99:0/2148705669 conn(0x7f25640fbcb0 msgr2=0x7f2564102de0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-03-19T02:06:21.398 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:21.397+0000 7f256be0b640 1 -- 172.21.15.99:0/2148705669 shutdown_connections 2024-03-19T02:06:21.398 INFO:teuthology.orchestra.run.smithi099.stderr:2024-03-19T02:06:21.397+0000 7f256be0b640 1 -- 172.21.15.99:0/2148705669 wait complete. 2024-03-19T02:06:21.409 DEBUG:teuthology.orchestra.run.smithi099:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-03-19T02:06:21.435 DEBUG:teuthology.orchestra.run.smithi112:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-03-19T02:06:21.464 DEBUG:teuthology.orchestra.run.smithi115:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-03-19T02:06:21.614 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-03-19T02:06:21.614 DEBUG:teuthology.orchestra.run.smithi099:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-03-19T02:06:21.628 DEBUG:teuthology.orchestra.run.smithi112:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-03-19T02:06:21.642 DEBUG:teuthology.orchestra.run.smithi115:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-03-19T02:06:21.656 INFO:tasks.cephadm:Stopping all daemons... 2024-03-19T02:06:21.656 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-03-19T02:06:21.656 DEBUG:teuthology.orchestra.run.smithi099:> sudo systemctl stop ceph-438fb69c-e590-11ee-95c9-87774f69a715@mon.a 2024-03-19T02:06:21.989 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:06:21 smithi099 systemd[1]: Stopping Ceph mon.a for 438fb69c-e590-11ee-95c9-87774f69a715... 2024-03-19T02:06:22.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:06:21 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mon-a[62947]: 2024-03-19T02:06:21.989+0000 7f4e718c7700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-03-19T02:06:22.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:06:21 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mon-a[62947]: 2024-03-19T02:06:21.989+0000 7f4e718c7700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-03-19T02:06:22.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:06:22 smithi099 podman[126590]: 2024-03-19 02:06:22.10207322 +0000 UTC m=+0.243842987 container died e320e9f5ecea50796d28569f5e924ed9515a79ba91911c669944c318a5d0037c (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mon-a, org.label-schema.build-date=20240307, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.license=GPLv2) 2024-03-19T02:06:22.774 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:06:22 smithi099 podman[126590]: 2024-03-19 02:06:22.490486781 +0000 UTC m=+0.632256546 container cleanup e320e9f5ecea50796d28569f5e924ed9515a79ba91911c669944c318a5d0037c (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mon-a, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.build-date=20240307, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True) 2024-03-19T02:06:22.774 INFO:journalctl@ceph.mon.a.smithi099.stdout:Mar 19 02:06:22 smithi099 bash[126590]: ceph-438fb69c-e590-11ee-95c9-87774f69a715-mon-a 2024-03-19T02:06:22.902 DEBUG:teuthology.orchestra.run.smithi099:> sudo pkill -f 'journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@mon.a.service' 2024-03-19T02:06:22.951 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-19T02:06:22.951 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-03-19T02:06:22.951 INFO:tasks.cephadm.mon.c:Stopping mon.b... 2024-03-19T02:06:22.951 DEBUG:teuthology.orchestra.run.smithi112:> sudo systemctl stop ceph-438fb69c-e590-11ee-95c9-87774f69a715@mon.b 2024-03-19T02:06:23.328 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:06:22 smithi112 systemd[1]: Stopping Ceph mon.b for 438fb69c-e590-11ee-95c9-87774f69a715... 2024-03-19T02:06:23.611 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:06:23 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mon-b[66815]: 2024-03-19T02:06:23.326+0000 7f7f00036700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-03-19T02:06:23.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:06:23 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mon-b[66815]: 2024-03-19T02:06:23.326+0000 7f7f00036700 -1 mon.b@2(peon) e3 *** Got Signal Terminated *** 2024-03-19T02:06:23.612 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:06:23 smithi112 podman[106211]: 2024-03-19 02:06:23.480264249 +0000 UTC m=+0.309460734 container died d293a73943f87f7340eabc303ef0ff99db8bc08a92f0b8105844ae628f2ac93e (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mon-b, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.build-date=20240307, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-19T02:06:24.111 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:06:23 smithi112 podman[106211]: 2024-03-19 02:06:23.836511309 +0000 UTC m=+0.665707801 container cleanup d293a73943f87f7340eabc303ef0ff99db8bc08a92f0b8105844ae628f2ac93e (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mon-b, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240307, org.label-schema.schema-version=1.0, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.license=GPLv2, RELEASE=wip-batrick-testing-20240318.181317-b6ae249) 2024-03-19T02:06:24.112 INFO:journalctl@ceph.mon.b.smithi112.stdout:Mar 19 02:06:23 smithi112 bash[106211]: ceph-438fb69c-e590-11ee-95c9-87774f69a715-mon-b 2024-03-19T02:06:24.278 DEBUG:teuthology.orchestra.run.smithi112:> sudo pkill -f 'journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@mon.b.service' 2024-03-19T02:06:24.324 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-19T02:06:24.324 INFO:tasks.cephadm.mon.c:Stopped mon.b 2024-03-19T02:06:24.324 INFO:tasks.cephadm.mon.c:Stopping mon.c... 2024-03-19T02:06:24.324 DEBUG:teuthology.orchestra.run.smithi115:> sudo systemctl stop ceph-438fb69c-e590-11ee-95c9-87774f69a715@mon.c 2024-03-19T02:06:24.778 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:06:24 smithi115 systemd[1]: Stopping Ceph mon.c for 438fb69c-e590-11ee-95c9-87774f69a715... 2024-03-19T02:06:25.053 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:06:24 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mon-c[67306]: 2024-03-19T02:06:24.777+0000 7f1ece53d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-03-19T02:06:25.053 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:06:24 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mon-c[67306]: 2024-03-19T02:06:24.777+0000 7f1ece53d700 -1 mon.c@1(peon) e3 *** Got Signal Terminated *** 2024-03-19T02:06:25.054 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:06:24 smithi115 podman[105661]: 2024-03-19 02:06:24.798618819 +0000 UTC m=+0.144270377 container died 30f5aa3b44b18e0930bfcf3673fa632f7856352342cbabc3e17d7dddfc17505c (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mon-c, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240307, CEPH_POINT_RELEASE=, ceph=True) 2024-03-19T02:06:25.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:06:25 smithi115 podman[105661]: 2024-03-19 02:06:25.15359597 +0000 UTC m=+0.499247522 container cleanup 30f5aa3b44b18e0930bfcf3673fa632f7856352342cbabc3e17d7dddfc17505c (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mon-c, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, ceph=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240307, org.label-schema.license=GPLv2) 2024-03-19T02:06:25.306 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:06:25 smithi115 bash[105661]: ceph-438fb69c-e590-11ee-95c9-87774f69a715-mon-c 2024-03-19T02:06:25.642 INFO:journalctl@ceph.mon.c.smithi115.stdout:Mar 19 02:06:25 smithi115 podman[105673]: 2024-03-19 02:06:25.380198438 +0000 UTC m=+0.577555116 container remove 30f5aa3b44b18e0930bfcf3673fa632f7856352342cbabc3e17d7dddfc17505c (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mon-c, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240307, GIT_CLEAN=True) 2024-03-19T02:06:25.647 DEBUG:teuthology.orchestra.run.smithi115:> sudo pkill -f 'journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@mon.c.service' 2024-03-19T02:06:25.711 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-19T02:06:25.711 INFO:tasks.cephadm.mon.c:Stopped mon.c 2024-03-19T02:06:25.711 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-03-19T02:06:25.712 DEBUG:teuthology.orchestra.run.smithi099:> sudo systemctl stop ceph-438fb69c-e590-11ee-95c9-87774f69a715@mgr.x 2024-03-19T02:06:26.044 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 02:06:25 smithi099 systemd[1]: Stopping Ceph mgr.x for 438fb69c-e590-11ee-95c9-87774f69a715... 2024-03-19T02:06:26.340 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 02:06:26 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x[63186]: 2024-03-19T02:06:26.044+0000 7f6855f7d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.x -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-19T02:06:26.341 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 02:06:26 smithi099 podman[126711]: 2024-03-19 02:06:26.086071849 +0000 UTC m=+0.182146641 container died 565767728d8fdc5c7e21cfd8562041ee2d26964daad91e707c80f641c8e8a644 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240307, ceph=True) 2024-03-19T02:06:26.629 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 02:06:26 smithi099 podman[126711]: 2024-03-19 02:06:26.428918009 +0000 UTC m=+0.524992795 container cleanup 565767728d8fdc5c7e21cfd8562041ee2d26964daad91e707c80f641c8e8a644 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:b6ae24918b03b1b8a19c9263a50655bba80bb810, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240307, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-03-19T02:06:26.629 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 19 02:06:26 smithi099 bash[126711]: ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-x 2024-03-19T02:06:26.845 DEBUG:teuthology.orchestra.run.smithi099:> sudo pkill -f 'journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@mgr.x.service' 2024-03-19T02:06:26.882 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-19T02:06:26.882 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-03-19T02:06:26.882 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-03-19T02:06:26.882 DEBUG:teuthology.orchestra.run.smithi112:> sudo systemctl stop ceph-438fb69c-e590-11ee-95c9-87774f69a715@mgr.y 2024-03-19T02:06:27.187 INFO:journalctl@ceph.mgr.y.smithi112.stdout:Mar 19 02:06:26 smithi112 systemd[1]: Stopping Ceph mgr.y for 438fb69c-e590-11ee-95c9-87774f69a715... 2024-03-19T02:06:27.484 INFO:journalctl@ceph.mgr.y.smithi112.stdout:Mar 19 02:06:27 smithi112 podman[106335]: 2024-03-19 02:06:27.187501744 +0000 UTC m=+0.124301079 container died c0916e7a53e9babca1da4ae0a3ad92ecad1e923624d89a3c7847949878e39490 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-y, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240307, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-03-19T02:06:27.819 INFO:journalctl@ceph.mgr.y.smithi112.stdout:Mar 19 02:06:27 smithi112 podman[106335]: 2024-03-19 02:06:27.625446135 +0000 UTC m=+0.562245467 container cleanup c0916e7a53e9babca1da4ae0a3ad92ecad1e923624d89a3c7847949878e39490 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-y, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.build-date=20240307, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30) 2024-03-19T02:06:27.819 INFO:journalctl@ceph.mgr.y.smithi112.stdout:Mar 19 02:06:27 smithi112 bash[106335]: ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-y 2024-03-19T02:06:28.035 DEBUG:teuthology.orchestra.run.smithi112:> sudo pkill -f 'journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@mgr.y.service' 2024-03-19T02:06:28.072 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-19T02:06:28.072 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-03-19T02:06:28.072 INFO:tasks.cephadm.mgr.z:Stopping mgr.z... 2024-03-19T02:06:28.072 DEBUG:teuthology.orchestra.run.smithi115:> sudo systemctl stop ceph-438fb69c-e590-11ee-95c9-87774f69a715@mgr.z 2024-03-19T02:06:28.462 INFO:journalctl@ceph.mgr.z.smithi115.stdout:Mar 19 02:06:28 smithi115 systemd[1]: Stopping Ceph mgr.z for 438fb69c-e590-11ee-95c9-87774f69a715... 2024-03-19T02:06:28.735 INFO:journalctl@ceph.mgr.z.smithi115.stdout:Mar 19 02:06:28 smithi115 podman[105783]: 2024-03-19 02:06:28.462665466 +0000 UTC m=+0.198233206 container died b2c0b74c91363968eeca6c8bb8d7e012f574e5797037b5a338196015ca290688 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-z, maintainer=Guillaume Abrioux , RELEASE=wip-batrick-testing-20240318.181317-b6ae249, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240307, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-19T02:06:29.056 INFO:journalctl@ceph.mgr.z.smithi115.stdout:Mar 19 02:06:28 smithi115 podman[105783]: 2024-03-19 02:06:28.841602619 +0000 UTC m=+0.577170362 container cleanup b2c0b74c91363968eeca6c8bb8d7e012f574e5797037b5a338196015ca290688 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-z, org.label-schema.build-date=20240307, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, ceph=True, maintainer=Guillaume Abrioux ) 2024-03-19T02:06:29.056 INFO:journalctl@ceph.mgr.z.smithi115.stdout:Mar 19 02:06:28 smithi115 bash[105783]: ceph-438fb69c-e590-11ee-95c9-87774f69a715-mgr-z 2024-03-19T02:06:29.322 DEBUG:teuthology.orchestra.run.smithi115:> sudo pkill -f 'journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@mgr.z.service' 2024-03-19T02:06:29.358 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-19T02:06:29.359 INFO:tasks.cephadm.mgr.z:Stopped mgr.z 2024-03-19T02:06:29.359 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-03-19T02:06:29.359 DEBUG:teuthology.orchestra.run.smithi099:> sudo systemctl stop ceph-438fb69c-e590-11ee-95c9-87774f69a715@osd.0 2024-03-19T02:06:29.656 INFO:journalctl@ceph.osd.0.smithi099.stdout:Mar 19 02:06:29 smithi099 systemd[1]: Stopping Ceph osd.0 for 438fb69c-e590-11ee-95c9-87774f69a715... 2024-03-19T02:06:30.090 INFO:journalctl@ceph.osd.0.smithi099.stdout:Mar 19 02:06:29 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-0[77885]: 2024-03-19T02:06:29.656+0000 7f97121fe700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-19T02:06:30.090 INFO:journalctl@ceph.osd.0.smithi099.stdout:Mar 19 02:06:29 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-0[77885]: 2024-03-19T02:06:29.656+0000 7f97121fe700 -1 osd.0 95 *** Got signal Terminated *** 2024-03-19T02:06:30.090 INFO:journalctl@ceph.osd.0.smithi099.stdout:Mar 19 02:06:29 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-0[77885]: 2024-03-19T02:06:29.656+0000 7f97121fe700 -1 osd.0 95 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-19T02:06:35.491 INFO:journalctl@ceph.osd.0.smithi099.stdout:Mar 19 02:06:35 smithi099 podman[126831]: 2024-03-19 02:06:35.197488437 +0000 UTC m=+5.648926563 container died 81c9f16c5f7cfb0c048c16b76b49f268ec6e99a9e42fcceb4c07c1774b1d5fa3 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-0, ceph=True, org.label-schema.build-date=20240307, maintainer=Guillaume Abrioux , RELEASE=wip-batrick-testing-20240318.181317-b6ae249, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0) 2024-03-19T02:06:35.832 INFO:journalctl@ceph.osd.0.smithi099.stdout:Mar 19 02:06:35 smithi099 podman[126831]: 2024-03-19 02:06:35.574194631 +0000 UTC m=+6.025632766 container cleanup 81c9f16c5f7cfb0c048c16b76b49f268ec6e99a9e42fcceb4c07c1774b1d5fa3 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.build-date=20240307, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5) 2024-03-19T02:06:35.832 INFO:journalctl@ceph.osd.0.smithi099.stdout:Mar 19 02:06:35 smithi099 bash[126831]: ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-0 2024-03-19T02:06:36.090 INFO:journalctl@ceph.osd.0.smithi099.stdout:Mar 19 02:06:35 smithi099 podman[126847]: 2024-03-19 02:06:35.833852071 +0000 UTC m=+0.634527545 container remove 81c9f16c5f7cfb0c048c16b76b49f268ec6e99a9e42fcceb4c07c1774b1d5fa3 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-0, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240307, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=wip-batrick-testing-20240318.181317-b6ae249) 2024-03-19T02:06:36.462 INFO:journalctl@ceph.osd.0.smithi099.stdout:Mar 19 02:06:36 smithi099 podman[126941]: 2024-03-19 02:06:36.083519781 +0000 UTC m=+0.017947801 image pull 2312c8e42d0069a2dbb2364bddb00264db4b800e7f18a7e7e4eb8f3ecab2ff1c quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639 2024-03-19T02:06:36.462 INFO:journalctl@ceph.osd.0.smithi099.stdout:Mar 19 02:06:36 smithi099 podman[126941]: 2024-03-19 02:06:36.218574529 +0000 UTC m=+0.153002538 container create cefb879936d92dc5480c5b8ce5da368590c62ca325a8df51be0b025c12d96a1e (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.build-date=20240307, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-03-19T02:06:36.462 INFO:journalctl@ceph.osd.0.smithi099.stdout:Mar 19 02:06:36 smithi099 podman[126941]: 2024-03-19 02:06:36.462963798 +0000 UTC m=+0.397391838 container init cefb879936d92dc5480c5b8ce5da368590c62ca325a8df51be0b025c12d96a1e (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-0-deactivate, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.build-date=20240307, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-03-19T02:06:36.750 INFO:journalctl@ceph.osd.0.smithi099.stdout:Mar 19 02:06:36 smithi099 podman[126941]: 2024-03-19 02:06:36.466401253 +0000 UTC m=+0.400829271 container start cefb879936d92dc5480c5b8ce5da368590c62ca325a8df51be0b025c12d96a1e (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-0-deactivate, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.build-date=20240307, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-03-19T02:06:36.751 INFO:journalctl@ceph.osd.0.smithi099.stdout:Mar 19 02:06:36 smithi099 podman[126941]: 2024-03-19 02:06:36.562658465 +0000 UTC m=+0.497086488 container attach cefb879936d92dc5480c5b8ce5da368590c62ca325a8df51be0b025c12d96a1e (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-0-deactivate, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.build-date=20240307, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-03-19T02:06:37.023 INFO:journalctl@ceph.osd.0.smithi099.stdout:Mar 19 02:06:36 smithi099 podman[126941]: 2024-03-19 02:06:36.751844108 +0000 UTC m=+0.686272132 container died cefb879936d92dc5480c5b8ce5da368590c62ca325a8df51be0b025c12d96a1e (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-0-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240307, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30) 2024-03-19T02:06:37.287 INFO:journalctl@ceph.osd.0.smithi099.stdout:Mar 19 02:06:37 smithi099 podman[126968]: 2024-03-19 02:06:37.266579266 +0000 UTC m=+0.509197324 container remove cefb879936d92dc5480c5b8ce5da368590c62ca325a8df51be0b025c12d96a1e (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-0-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240307, io.buildah.version=1.33.5, RELEASE=wip-batrick-testing-20240318.181317-b6ae249) 2024-03-19T02:06:37.292 DEBUG:teuthology.orchestra.run.smithi099:> sudo pkill -f 'journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@osd.0.service' 2024-03-19T02:06:37.334 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-19T02:06:37.334 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-03-19T02:06:37.335 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-03-19T02:06:37.335 DEBUG:teuthology.orchestra.run.smithi112:> sudo systemctl stop ceph-438fb69c-e590-11ee-95c9-87774f69a715@osd.1 2024-03-19T02:06:37.612 INFO:journalctl@ceph.osd.1.smithi112.stdout:Mar 19 02:06:37 smithi112 systemd[1]: Stopping Ceph osd.1 for 438fb69c-e590-11ee-95c9-87774f69a715... 2024-03-19T02:06:38.111 INFO:journalctl@ceph.osd.1.smithi112.stdout:Mar 19 02:06:37 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-1[71392]: 2024-03-19T02:06:37.648+0000 7fe305532700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-19T02:06:38.112 INFO:journalctl@ceph.osd.1.smithi112.stdout:Mar 19 02:06:37 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-1[71392]: 2024-03-19T02:06:37.648+0000 7fe305532700 -1 osd.1 95 *** Got signal Terminated *** 2024-03-19T02:06:38.112 INFO:journalctl@ceph.osd.1.smithi112.stdout:Mar 19 02:06:37 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-1[71392]: 2024-03-19T02:06:37.648+0000 7fe305532700 -1 osd.1 95 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-19T02:06:43.482 INFO:journalctl@ceph.osd.1.smithi112.stdout:Mar 19 02:06:43 smithi112 podman[106467]: 2024-03-19 02:06:43.198850319 +0000 UTC m=+5.672692042 container died 787465b965698bd88a9bdf13998d18e6dd277bcb8a99dc22dd3b24135f36bfdc (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-1, GIT_BRANCH=HEAD, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240307) 2024-03-19T02:06:43.861 INFO:journalctl@ceph.osd.1.smithi112.stdout:Mar 19 02:06:43 smithi112 podman[106467]: 2024-03-19 02:06:43.581889837 +0000 UTC m=+6.055731566 container cleanup 787465b965698bd88a9bdf13998d18e6dd277bcb8a99dc22dd3b24135f36bfdc (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-1, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240307, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-03-19T02:06:43.862 INFO:journalctl@ceph.osd.1.smithi112.stdout:Mar 19 02:06:43 smithi112 bash[106467]: ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-1 2024-03-19T02:06:44.191 INFO:journalctl@ceph.osd.1.smithi112.stdout:Mar 19 02:06:43 smithi112 podman[106485]: 2024-03-19 02:06:43.892668092 +0000 UTC m=+0.690027003 container remove 787465b965698bd88a9bdf13998d18e6dd277bcb8a99dc22dd3b24135f36bfdc (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-1, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240307, GIT_CLEAN=True, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, CEPH_POINT_RELEASE=) 2024-03-19T02:06:44.478 INFO:journalctl@ceph.osd.1.smithi112.stdout:Mar 19 02:06:44 smithi112 podman[106578]: 2024-03-19 02:06:44.094623886 +0000 UTC m=+0.017751381 image pull 2312c8e42d0069a2dbb2364bddb00264db4b800e7f18a7e7e4eb8f3ecab2ff1c quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639 2024-03-19T02:06:44.478 INFO:journalctl@ceph.osd.1.smithi112.stdout:Mar 19 02:06:44 smithi112 podman[106578]: 2024-03-19 02:06:44.209485306 +0000 UTC m=+0.132612808 container create e22000af7c40d9cb5014d350acbd7d78c4cd3f5060934d7bd9cff7d4c82e1c16 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-1-deactivate, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240307, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True) 2024-03-19T02:06:44.478 INFO:journalctl@ceph.osd.1.smithi112.stdout:Mar 19 02:06:44 smithi112 podman[106578]: 2024-03-19 02:06:44.403775215 +0000 UTC m=+0.326902698 container init e22000af7c40d9cb5014d350acbd7d78c4cd3f5060934d7bd9cff7d4c82e1c16 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240307, maintainer=Guillaume Abrioux ) 2024-03-19T02:06:44.478 INFO:journalctl@ceph.osd.1.smithi112.stdout:Mar 19 02:06:44 smithi112 podman[106578]: 2024-03-19 02:06:44.406980589 +0000 UTC m=+0.330108074 container start e22000af7c40d9cb5014d350acbd7d78c4cd3f5060934d7bd9cff7d4c82e1c16 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-1-deactivate, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, GIT_CLEAN=True, org.label-schema.build-date=20240307, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30) 2024-03-19T02:06:44.478 INFO:journalctl@ceph.osd.1.smithi112.stdout:Mar 19 02:06:44 smithi112 podman[106578]: 2024-03-19 02:06:44.478505327 +0000 UTC m=+0.401632826 container attach e22000af7c40d9cb5014d350acbd7d78c4cd3f5060934d7bd9cff7d4c82e1c16 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-1-deactivate, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240307, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-03-19T02:06:45.064 INFO:journalctl@ceph.osd.1.smithi112.stdout:Mar 19 02:06:44 smithi112 podman[106578]: 2024-03-19 02:06:44.774383594 +0000 UTC m=+0.697511093 container died e22000af7c40d9cb5014d350acbd7d78c4cd3f5060934d7bd9cff7d4c82e1c16 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-1-deactivate, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240307, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30) 2024-03-19T02:06:45.372 DEBUG:teuthology.orchestra.run.smithi112:> sudo pkill -f 'journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@osd.1.service' 2024-03-19T02:06:45.416 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-19T02:06:45.416 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-03-19T02:06:45.416 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-03-19T02:06:45.417 DEBUG:teuthology.orchestra.run.smithi115:> sudo systemctl stop ceph-438fb69c-e590-11ee-95c9-87774f69a715@osd.2 2024-03-19T02:06:45.711 INFO:journalctl@ceph.osd.2.smithi115.stdout:Mar 19 02:06:45 smithi115 systemd[1]: Stopping Ceph osd.2 for 438fb69c-e590-11ee-95c9-87774f69a715... 2024-03-19T02:06:46.056 INFO:journalctl@ceph.osd.2.smithi115.stdout:Mar 19 02:06:45 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-2[72330]: 2024-03-19T02:06:45.711+0000 7fe99a304700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-19T02:06:46.056 INFO:journalctl@ceph.osd.2.smithi115.stdout:Mar 19 02:06:45 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-2[72330]: 2024-03-19T02:06:45.711+0000 7fe99a304700 -1 osd.2 95 *** Got signal Terminated *** 2024-03-19T02:06:46.056 INFO:journalctl@ceph.osd.2.smithi115.stdout:Mar 19 02:06:45 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-2[72330]: 2024-03-19T02:06:45.711+0000 7fe99a304700 -1 osd.2 95 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-19T02:06:51.556 INFO:journalctl@ceph.osd.2.smithi115.stdout:Mar 19 02:06:51 smithi115 podman[105919]: 2024-03-19 02:06:51.274996012 +0000 UTC m=+5.664875753 container died eda57b9f9f7886cf313dab9280eff4a97b33cd27724883110a080670400c78f8 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240307, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, GIT_CLEAN=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30) 2024-03-19T02:06:52.056 INFO:journalctl@ceph.osd.2.smithi115.stdout:Mar 19 02:06:51 smithi115 podman[105919]: 2024-03-19 02:06:51.602187168 +0000 UTC m=+5.992066921 container cleanup eda57b9f9f7886cf313dab9280eff4a97b33cd27724883110a080670400c78f8 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.build-date=20240307, org.label-schema.schema-version=1.0, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True) 2024-03-19T02:06:52.056 INFO:journalctl@ceph.osd.2.smithi115.stdout:Mar 19 02:06:51 smithi115 bash[105919]: ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-2 2024-03-19T02:06:52.056 INFO:journalctl@ceph.osd.2.smithi115.stdout:Mar 19 02:06:51 smithi115 podman[105939]: 2024-03-19 02:06:51.803238808 +0000 UTC m=+0.532381438 container remove eda57b9f9f7886cf313dab9280eff4a97b33cd27724883110a080670400c78f8 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-2, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, maintainer=Guillaume Abrioux , RELEASE=wip-batrick-testing-20240318.181317-b6ae249, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.build-date=20240307) 2024-03-19T02:06:52.472 INFO:journalctl@ceph.osd.2.smithi115.stdout:Mar 19 02:06:52 smithi115 podman[106029]: 2024-03-19 02:06:52.121835313 +0000 UTC m=+0.017913984 image pull 2312c8e42d0069a2dbb2364bddb00264db4b800e7f18a7e7e4eb8f3ecab2ff1c quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639 2024-03-19T02:06:52.472 INFO:journalctl@ceph.osd.2.smithi115.stdout:Mar 19 02:06:52 smithi115 podman[106029]: 2024-03-19 02:06:52.25397949 +0000 UTC m=+0.150058146 container create 9330c576c13258325c4f02aad32940958bed88b6b820e617623d9448389d6206 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-2-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, GIT_CLEAN=True, org.label-schema.build-date=20240307, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5) 2024-03-19T02:06:52.806 INFO:journalctl@ceph.osd.2.smithi115.stdout:Mar 19 02:06:52 smithi115 podman[106029]: 2024-03-19 02:06:52.47304047 +0000 UTC m=+0.369119134 container init 9330c576c13258325c4f02aad32940958bed88b6b820e617623d9448389d6206 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-2-deactivate, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.build-date=20240307, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-19T02:06:52.806 INFO:journalctl@ceph.osd.2.smithi115.stdout:Mar 19 02:06:52 smithi115 podman[106029]: 2024-03-19 02:06:52.476231861 +0000 UTC m=+0.372310513 container start 9330c576c13258325c4f02aad32940958bed88b6b820e617623d9448389d6206 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-2-deactivate, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240307, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, io.buildah.version=1.33.5) 2024-03-19T02:06:52.806 INFO:journalctl@ceph.osd.2.smithi115.stdout:Mar 19 02:06:52 smithi115 podman[106029]: 2024-03-19 02:06:52.581352261 +0000 UTC m=+0.477431005 container attach 9330c576c13258325c4f02aad32940958bed88b6b820e617623d9448389d6206 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-2-deactivate, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240307, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-03-19T02:06:53.306 INFO:journalctl@ceph.osd.2.smithi115.stdout:Mar 19 02:06:53 smithi115 podman[106029]: 2024-03-19 02:06:53.051258367 +0000 UTC m=+0.947337020 container died 9330c576c13258325c4f02aad32940958bed88b6b820e617623d9448389d6206 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-2-deactivate, GIT_CLEAN=True, org.label-schema.build-date=20240307, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, ceph=True) 2024-03-19T02:06:53.578 INFO:journalctl@ceph.osd.2.smithi115.stdout:Mar 19 02:06:53 smithi115 podman[106055]: 2024-03-19 02:06:53.568116381 +0000 UTC m=+0.511688298 container remove 9330c576c13258325c4f02aad32940958bed88b6b820e617623d9448389d6206 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-2-deactivate, org.label-schema.build-date=20240307, org.label-schema.vendor=CentOS, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-03-19T02:06:53.583 DEBUG:teuthology.orchestra.run.smithi115:> sudo pkill -f 'journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@osd.2.service' 2024-03-19T02:06:53.625 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-19T02:06:53.626 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-03-19T02:06:53.626 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-03-19T02:06:53.626 DEBUG:teuthology.orchestra.run.smithi099:> sudo systemctl stop ceph-438fb69c-e590-11ee-95c9-87774f69a715@osd.3 2024-03-19T02:06:53.917 INFO:journalctl@ceph.osd.3.smithi099.stdout:Mar 19 02:06:53 smithi099 systemd[1]: Stopping Ceph osd.3 for 438fb69c-e590-11ee-95c9-87774f69a715... 2024-03-19T02:06:54.245 INFO:journalctl@ceph.osd.3.smithi099.stdout:Mar 19 02:06:53 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-3[84808]: 2024-03-19T02:06:53.917+0000 7f5094693700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-19T02:06:54.246 INFO:journalctl@ceph.osd.3.smithi099.stdout:Mar 19 02:06:53 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-3[84808]: 2024-03-19T02:06:53.917+0000 7f5094693700 -1 osd.3 95 *** Got signal Terminated *** 2024-03-19T02:06:54.246 INFO:journalctl@ceph.osd.3.smithi099.stdout:Mar 19 02:06:53 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-3[84808]: 2024-03-19T02:06:53.917+0000 7f5094693700 -1 osd.3 95 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-19T02:06:56.340 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:06:55 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:06:55.934+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170696+0000 front 2024-03-19T02:06:34.170736+0000 (oldest deadline 2024-03-19T02:06:55.870953+0000) 2024-03-19T02:06:57.340 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:06:56 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:06:56.930+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170696+0000 front 2024-03-19T02:06:34.170736+0000 (oldest deadline 2024-03-19T02:06:55.870953+0000) 2024-03-19T02:06:57.806 INFO:journalctl@ceph.osd.5.smithi115.stdout:Mar 19 02:06:57 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-5[78850]: 2024-03-19T02:06:57.549+0000 7fba3c793700 -1 osd.5 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:33.323399+0000 front 2024-03-19T02:06:33.323427+0000 (oldest deadline 2024-03-19T02:06:57.423498+0000) 2024-03-19T02:06:58.270 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:06:57 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:06:57.973+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170696+0000 front 2024-03-19T02:06:34.170736+0000 (oldest deadline 2024-03-19T02:06:55.870953+0000) 2024-03-19T02:06:58.590 INFO:journalctl@ceph.osd.3.smithi099.stdout:Mar 19 02:06:58 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-3[84808]: 2024-03-19T02:06:58.412+0000 7f508cc7f700 -1 osd.3 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:32.952483+0000 front 2024-03-19T02:06:32.952503+0000 (oldest deadline 2024-03-19T02:06:57.652607+0000) 2024-03-19T02:06:58.591 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:06:58 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:06:58.271+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:31.579664+0000 front 2024-03-19T02:06:31.579687+0000 (oldest deadline 2024-03-19T02:06:57.479977+0000) 2024-03-19T02:06:58.806 INFO:journalctl@ceph.osd.5.smithi115.stdout:Mar 19 02:06:58 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-5[78850]: 2024-03-19T02:06:58.505+0000 7fba3c793700 -1 osd.5 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:33.323399+0000 front 2024-03-19T02:06:33.323427+0000 (oldest deadline 2024-03-19T02:06:57.423498+0000) 2024-03-19T02:06:58.807 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:06:58 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:06:58.471+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:06:58.862 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:06:58 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:06:58.574+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170251+0000 front 2024-03-19T02:06:34.170405+0000 (oldest deadline 2024-03-19T02:06:58.270386+0000) 2024-03-19T02:06:59.230 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:06:58 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:06:58.927+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170696+0000 front 2024-03-19T02:06:34.170736+0000 (oldest deadline 2024-03-19T02:06:55.870953+0000) 2024-03-19T02:06:59.546 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:06:59 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:06:59.229+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:31.579664+0000 front 2024-03-19T02:06:31.579687+0000 (oldest deadline 2024-03-19T02:06:57.479977+0000) 2024-03-19T02:06:59.806 INFO:journalctl@ceph.osd.5.smithi115.stdout:Mar 19 02:06:59 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-5[78850]: 2024-03-19T02:06:59.473+0000 7fba3c793700 -1 osd.5 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:33.323399+0000 front 2024-03-19T02:06:33.323427+0000 (oldest deadline 2024-03-19T02:06:57.423498+0000) 2024-03-19T02:06:59.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:06:59 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:06:59.473+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:06:59.833 INFO:journalctl@ceph.osd.3.smithi099.stdout:Mar 19 02:06:59 smithi099 podman[127049]: 2024-03-19 02:06:59.547731274 +0000 UTC m=+5.730066754 container died 3ca45436f39e94f5e0ef0cc0271ecee221ecad635b170e3c6c5cd25dd8feece9 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-3, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=wip-batrick-testing-20240318.181317-b6ae249, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240307, CEPH_POINT_RELEASE=) 2024-03-19T02:06:59.861 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:06:59 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:06:59.586+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170251+0000 front 2024-03-19T02:06:34.170405+0000 (oldest deadline 2024-03-19T02:06:58.270386+0000) 2024-03-19T02:07:00.090 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:06:59 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:06:59.920+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170696+0000 front 2024-03-19T02:06:34.170736+0000 (oldest deadline 2024-03-19T02:06:55.870953+0000) 2024-03-19T02:07:00.091 INFO:journalctl@ceph.osd.3.smithi099.stdout:Mar 19 02:06:59 smithi099 podman[127049]: 2024-03-19 02:06:59.942410372 +0000 UTC m=+6.124745851 container cleanup 3ca45436f39e94f5e0ef0cc0271ecee221ecad635b170e3c6c5cd25dd8feece9 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-3, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.build-date=20240307, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.schema-version=1.0) 2024-03-19T02:07:00.091 INFO:journalctl@ceph.osd.3.smithi099.stdout:Mar 19 02:06:59 smithi099 bash[127049]: ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-3 2024-03-19T02:07:00.476 INFO:journalctl@ceph.osd.3.smithi099.stdout:Mar 19 02:07:00 smithi099 podman[127077]: 2024-03-19 02:07:00.177003418 +0000 UTC m=+0.625274327 container remove 3ca45436f39e94f5e0ef0cc0271ecee221ecad635b170e3c6c5cd25dd8feece9 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-3, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=wip-batrick-testing-20240318.181317-b6ae249, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240307, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.33.5) 2024-03-19T02:07:00.477 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:00 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:00.185+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:31.579664+0000 front 2024-03-19T02:06:31.579687+0000 (oldest deadline 2024-03-19T02:06:57.479977+0000) 2024-03-19T02:07:00.755 INFO:journalctl@ceph.osd.3.smithi099.stdout:Mar 19 02:07:00 smithi099 podman[127169]: 2024-03-19 02:07:00.477953645 +0000 UTC m=+0.090911703 container create 3c8183d7770565f5311f95adf321a452bb7576e57ed5bd9ef2e00d5f421817d9 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-3-deactivate, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.build-date=20240307, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-batrick-testing-20240318.181317-b6ae249) 2024-03-19T02:07:00.755 INFO:journalctl@ceph.osd.3.smithi099.stdout:Mar 19 02:07:00 smithi099 podman[127169]: 2024-03-19 02:07:00.404925298 +0000 UTC m=+0.017883379 image pull 2312c8e42d0069a2dbb2364bddb00264db4b800e7f18a7e7e4eb8f3ecab2ff1c quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639 2024-03-19T02:07:00.755 INFO:journalctl@ceph.osd.3.smithi099.stdout:Mar 19 02:07:00 smithi099 podman[127169]: 2024-03-19 02:07:00.689207607 +0000 UTC m=+0.302165665 container init 3c8183d7770565f5311f95adf321a452bb7576e57ed5bd9ef2e00d5f421817d9 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-3-deactivate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.build-date=20240307, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30) 2024-03-19T02:07:00.755 INFO:journalctl@ceph.osd.3.smithi099.stdout:Mar 19 02:07:00 smithi099 podman[127169]: 2024-03-19 02:07:00.692226379 +0000 UTC m=+0.305184442 container start 3c8183d7770565f5311f95adf321a452bb7576e57ed5bd9ef2e00d5f421817d9 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-3-deactivate, maintainer=Guillaume Abrioux , GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.build-date=20240307, org.label-schema.vendor=CentOS) 2024-03-19T02:07:00.755 INFO:journalctl@ceph.osd.3.smithi099.stdout:Mar 19 02:07:00 smithi099 podman[127169]: 2024-03-19 02:07:00.755746509 +0000 UTC m=+0.368704585 container attach 3c8183d7770565f5311f95adf321a452bb7576e57ed5bd9ef2e00d5f421817d9 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-3-deactivate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240307, org.label-schema.license=GPLv2) 2024-03-19T02:07:00.806 INFO:journalctl@ceph.osd.5.smithi115.stdout:Mar 19 02:07:00 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-5[78850]: 2024-03-19T02:07:00.447+0000 7fba3c793700 -1 osd.5 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:33.323399+0000 front 2024-03-19T02:06:33.323427+0000 (oldest deadline 2024-03-19T02:06:57.423498+0000) 2024-03-19T02:07:00.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:00 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:00.496+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:00.861 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:00 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:00.590+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170251+0000 front 2024-03-19T02:06:34.170405+0000 (oldest deadline 2024-03-19T02:06:58.270386+0000) 2024-03-19T02:07:01.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:00 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:00.955+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170696+0000 front 2024-03-19T02:06:34.170736+0000 (oldest deadline 2024-03-19T02:06:55.870953+0000) 2024-03-19T02:07:01.091 INFO:journalctl@ceph.osd.3.smithi099.stdout:Mar 19 02:07:00 smithi099 podman[127169]: 2024-03-19 02:07:00.979817706 +0000 UTC m=+0.592775759 container died 3c8183d7770565f5311f95adf321a452bb7576e57ed5bd9ef2e00d5f421817d9 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-3-deactivate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_CLEAN=True, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.build-date=20240307, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-19T02:07:01.491 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:01 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:01.186+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:31.579664+0000 front 2024-03-19T02:06:31.579687+0000 (oldest deadline 2024-03-19T02:06:57.479977+0000) 2024-03-19T02:07:01.516 DEBUG:teuthology.orchestra.run.smithi099:> sudo pkill -f 'journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@osd.3.service' 2024-03-19T02:07:01.559 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-19T02:07:01.559 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-03-19T02:07:01.559 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-03-19T02:07:01.559 DEBUG:teuthology.orchestra.run.smithi112:> sudo systemctl stop ceph-438fb69c-e590-11ee-95c9-87774f69a715@osd.4 2024-03-19T02:07:01.806 INFO:journalctl@ceph.osd.5.smithi115.stdout:Mar 19 02:07:01 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-5[78850]: 2024-03-19T02:07:01.464+0000 7fba3c793700 -1 osd.5 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:33.323399+0000 front 2024-03-19T02:06:33.323427+0000 (oldest deadline 2024-03-19T02:06:57.423498+0000) 2024-03-19T02:07:01.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:01 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:01.538+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:01.861 INFO:journalctl@ceph.osd.4.smithi112.stdout:Mar 19 02:07:01 smithi112 systemd[1]: Stopping Ceph osd.4 for 438fb69c-e590-11ee-95c9-87774f69a715... 2024-03-19T02:07:01.862 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:01 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:01.583+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170251+0000 front 2024-03-19T02:06:34.170405+0000 (oldest deadline 2024-03-19T02:06:58.270386+0000) 2024-03-19T02:07:02.341 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:02 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:02.164+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:31.579664+0000 front 2024-03-19T02:06:31.579687+0000 (oldest deadline 2024-03-19T02:06:57.479977+0000) 2024-03-19T02:07:02.341 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:01 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:01.969+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170696+0000 front 2024-03-19T02:06:34.170736+0000 (oldest deadline 2024-03-19T02:06:55.870953+0000) 2024-03-19T02:07:02.362 INFO:journalctl@ceph.osd.4.smithi112.stdout:Mar 19 02:07:02 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-4[78359]: 2024-03-19T02:07:02.044+0000 7f39ad75d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-19T02:07:02.362 INFO:journalctl@ceph.osd.4.smithi112.stdout:Mar 19 02:07:02 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-4[78359]: 2024-03-19T02:07:02.044+0000 7f39ad75d700 -1 osd.4 95 *** Got signal Terminated *** 2024-03-19T02:07:02.362 INFO:journalctl@ceph.osd.4.smithi112.stdout:Mar 19 02:07:02 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-4[78359]: 2024-03-19T02:07:02.044+0000 7f39ad75d700 -1 osd.4 95 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-19T02:07:02.806 INFO:journalctl@ceph.osd.5.smithi115.stdout:Mar 19 02:07:02 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-5[78850]: 2024-03-19T02:07:02.511+0000 7fba3c793700 -1 osd.5 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:33.323399+0000 front 2024-03-19T02:06:33.323427+0000 (oldest deadline 2024-03-19T02:06:57.423498+0000) 2024-03-19T02:07:02.807 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:02 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:02.576+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:02.861 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:02 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:02.559+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170251+0000 front 2024-03-19T02:06:34.170405+0000 (oldest deadline 2024-03-19T02:06:58.270386+0000) 2024-03-19T02:07:03.340 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:03 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:03.199+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:31.579664+0000 front 2024-03-19T02:06:31.579687+0000 (oldest deadline 2024-03-19T02:06:57.479977+0000) 2024-03-19T02:07:03.341 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:02 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:02.950+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170696+0000 front 2024-03-19T02:06:34.170736+0000 (oldest deadline 2024-03-19T02:06:55.870953+0000) 2024-03-19T02:07:03.806 INFO:journalctl@ceph.osd.5.smithi115.stdout:Mar 19 02:07:03 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-5[78850]: 2024-03-19T02:07:03.536+0000 7fba3c793700 -1 osd.5 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:33.323399+0000 front 2024-03-19T02:06:33.323427+0000 (oldest deadline 2024-03-19T02:06:57.423498+0000) 2024-03-19T02:07:03.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:03 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:03.570+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:03.861 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:03 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:03.603+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170251+0000 front 2024-03-19T02:06:34.170405+0000 (oldest deadline 2024-03-19T02:06:58.270386+0000) 2024-03-19T02:07:03.862 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:03 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:03.746+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.951647+0000 front 2024-03-19T02:06:38.951558+0000 (oldest deadline 2024-03-19T02:07:03.651603+0000) 2024-03-19T02:07:04.194 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:03 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:03.943+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170696+0000 front 2024-03-19T02:06:34.170736+0000 (oldest deadline 2024-03-19T02:06:55.870953+0000) 2024-03-19T02:07:04.339 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:04 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:04.065+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:41.493849+0000 front 2024-03-19T02:06:41.493734+0000 (oldest deadline 2024-03-19T02:07:03.793726+0000) 2024-03-19T02:07:04.576 INFO:journalctl@ceph.osd.4.smithi112.stdout:Mar 19 02:07:04 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-4[78359]: 2024-03-19T02:07:04.305+0000 7f39a9d51700 -1 osd.4 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:42.199034+0000 front 2024-03-19T02:06:42.199000+0000 (oldest deadline 2024-03-19T02:07:03.899192+0000) 2024-03-19T02:07:04.591 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:04 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:04.194+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:31.579664+0000 front 2024-03-19T02:06:31.579687+0000 (oldest deadline 2024-03-19T02:06:57.479977+0000) 2024-03-19T02:07:04.591 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:04 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:04.194+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.980985+0000 front 2024-03-19T02:06:40.981003+0000 (oldest deadline 2024-03-19T02:07:03.281058+0000) 2024-03-19T02:07:04.806 INFO:journalctl@ceph.osd.5.smithi115.stdout:Mar 19 02:07:04 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-5[78850]: 2024-03-19T02:07:04.527+0000 7fba3c793700 -1 osd.5 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:33.323399+0000 front 2024-03-19T02:06:33.323427+0000 (oldest deadline 2024-03-19T02:06:57.423498+0000) 2024-03-19T02:07:04.807 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:04 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:04.529+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:04.862 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:04 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:04.733+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.951647+0000 front 2024-03-19T02:06:38.951558+0000 (oldest deadline 2024-03-19T02:07:03.651603+0000) 2024-03-19T02:07:04.862 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:04 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:04.575+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170251+0000 front 2024-03-19T02:06:34.170405+0000 (oldest deadline 2024-03-19T02:06:58.270386+0000) 2024-03-19T02:07:04.862 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:04 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:04.575+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.270741+0000 front 2024-03-19T02:06:38.270719+0000 (oldest deadline 2024-03-19T02:07:04.170884+0000) 2024-03-19T02:07:05.241 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:04 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:04.935+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170696+0000 front 2024-03-19T02:06:34.170736+0000 (oldest deadline 2024-03-19T02:06:55.870953+0000) 2024-03-19T02:07:05.494 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:05 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:05.067+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:41.493849+0000 front 2024-03-19T02:06:41.493734+0000 (oldest deadline 2024-03-19T02:07:03.793726+0000) 2024-03-19T02:07:05.590 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:05 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:05.241+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:31.579664+0000 front 2024-03-19T02:06:31.579687+0000 (oldest deadline 2024-03-19T02:06:57.479977+0000) 2024-03-19T02:07:05.590 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:05 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:05.241+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.980985+0000 front 2024-03-19T02:06:40.981003+0000 (oldest deadline 2024-03-19T02:07:03.281058+0000) 2024-03-19T02:07:05.603 INFO:journalctl@ceph.osd.4.smithi112.stdout:Mar 19 02:07:05 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-4[78359]: 2024-03-19T02:07:05.265+0000 7f39a9d51700 -1 osd.4 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:42.199034+0000 front 2024-03-19T02:06:42.199000+0000 (oldest deadline 2024-03-19T02:07:03.899192+0000) 2024-03-19T02:07:05.806 INFO:journalctl@ceph.osd.5.smithi115.stdout:Mar 19 02:07:05 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-5[78850]: 2024-03-19T02:07:05.494+0000 7fba3c793700 -1 osd.5 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:33.323399+0000 front 2024-03-19T02:06:33.323427+0000 (oldest deadline 2024-03-19T02:06:57.423498+0000) 2024-03-19T02:07:05.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:05 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:05.523+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:05.861 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:05 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:05.762+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.951647+0000 front 2024-03-19T02:06:38.951558+0000 (oldest deadline 2024-03-19T02:07:03.651603+0000) 2024-03-19T02:07:05.862 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:05 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:05.603+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170251+0000 front 2024-03-19T02:06:34.170405+0000 (oldest deadline 2024-03-19T02:06:58.270386+0000) 2024-03-19T02:07:05.862 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:05 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:05.603+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.270741+0000 front 2024-03-19T02:06:38.270719+0000 (oldest deadline 2024-03-19T02:07:04.170884+0000) 2024-03-19T02:07:06.273 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:05 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:05.890+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170696+0000 front 2024-03-19T02:06:34.170736+0000 (oldest deadline 2024-03-19T02:06:55.870953+0000) 2024-03-19T02:07:06.482 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:06 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:06.109+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:41.493849+0000 front 2024-03-19T02:06:41.493734+0000 (oldest deadline 2024-03-19T02:07:03.793726+0000) 2024-03-19T02:07:06.590 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:06 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:06.273+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:31.579664+0000 front 2024-03-19T02:06:31.579687+0000 (oldest deadline 2024-03-19T02:06:57.479977+0000) 2024-03-19T02:07:06.590 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:06 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:06.273+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.980985+0000 front 2024-03-19T02:06:40.981003+0000 (oldest deadline 2024-03-19T02:07:03.281058+0000) 2024-03-19T02:07:06.611 INFO:journalctl@ceph.osd.4.smithi112.stdout:Mar 19 02:07:06 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-4[78359]: 2024-03-19T02:07:06.288+0000 7f39a9d51700 -1 osd.4 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:42.199034+0000 front 2024-03-19T02:06:42.199000+0000 (oldest deadline 2024-03-19T02:07:03.899192+0000) 2024-03-19T02:07:06.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:06 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:06.482+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:06.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:06 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:06.482+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:07:06.806 INFO:journalctl@ceph.osd.5.smithi115.stdout:Mar 19 02:07:06 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-5[78850]: 2024-03-19T02:07:06.500+0000 7fba3c793700 -1 osd.5 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:33.323399+0000 front 2024-03-19T02:06:33.323427+0000 (oldest deadline 2024-03-19T02:06:57.423498+0000) 2024-03-19T02:07:06.807 INFO:journalctl@ceph.osd.5.smithi115.stdout:Mar 19 02:07:06 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-5[78850]: 2024-03-19T02:07:06.500+0000 7fba3c793700 -1 osd.5 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.225204+0000 front 2024-03-19T02:06:40.225256+0000 (oldest deadline 2024-03-19T02:07:05.525146+0000) 2024-03-19T02:07:07.112 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:06 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:06.739+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.951647+0000 front 2024-03-19T02:06:38.951558+0000 (oldest deadline 2024-03-19T02:07:03.651603+0000) 2024-03-19T02:07:07.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:06 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:06.647+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170251+0000 front 2024-03-19T02:06:34.170405+0000 (oldest deadline 2024-03-19T02:06:58.270386+0000) 2024-03-19T02:07:07.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:06 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:06.647+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.270741+0000 front 2024-03-19T02:06:38.270719+0000 (oldest deadline 2024-03-19T02:07:04.170884+0000) 2024-03-19T02:07:07.277 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:06 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:06.855+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170696+0000 front 2024-03-19T02:06:34.170736+0000 (oldest deadline 2024-03-19T02:06:55.870953+0000) 2024-03-19T02:07:07.278 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:06 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:06.855+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.572097+0000 front 2024-03-19T02:06:40.571972+0000 (oldest deadline 2024-03-19T02:07:06.372675+0000) 2024-03-19T02:07:07.493 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:07 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:07.095+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:41.493849+0000 front 2024-03-19T02:06:41.493734+0000 (oldest deadline 2024-03-19T02:07:03.793726+0000) 2024-03-19T02:07:07.590 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:07 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:07.277+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:31.579664+0000 front 2024-03-19T02:06:31.579687+0000 (oldest deadline 2024-03-19T02:06:57.479977+0000) 2024-03-19T02:07:07.590 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:07 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:07.277+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.980985+0000 front 2024-03-19T02:06:40.981003+0000 (oldest deadline 2024-03-19T02:07:03.281058+0000) 2024-03-19T02:07:07.745 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:07 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:07.711+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.951647+0000 front 2024-03-19T02:06:38.951558+0000 (oldest deadline 2024-03-19T02:07:03.651603+0000) 2024-03-19T02:07:07.745 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:07 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:07.668+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170251+0000 front 2024-03-19T02:06:34.170405+0000 (oldest deadline 2024-03-19T02:06:58.270386+0000) 2024-03-19T02:07:07.745 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:07 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:07.668+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.270741+0000 front 2024-03-19T02:06:38.270719+0000 (oldest deadline 2024-03-19T02:07:04.170884+0000) 2024-03-19T02:07:07.746 INFO:journalctl@ceph.osd.4.smithi112.stdout:Mar 19 02:07:07 smithi112 podman[106687]: 2024-03-19 02:07:07.490699567 +0000 UTC m=+5.742233126 container died f51be7fcdd3dc0db17601ee48f1818f4010082d5e1cb6ea0361c3e4deb75733b (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-4, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, GIT_CLEAN=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240307, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True) 2024-03-19T02:07:07.806 INFO:journalctl@ceph.osd.5.smithi115.stdout:Mar 19 02:07:07 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-5[78850]: 2024-03-19T02:07:07.529+0000 7fba3c793700 -1 osd.5 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:33.323399+0000 front 2024-03-19T02:06:33.323427+0000 (oldest deadline 2024-03-19T02:06:57.423498+0000) 2024-03-19T02:07:07.806 INFO:journalctl@ceph.osd.5.smithi115.stdout:Mar 19 02:07:07 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-5[78850]: 2024-03-19T02:07:07.529+0000 7fba3c793700 -1 osd.5 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.225204+0000 front 2024-03-19T02:06:40.225256+0000 (oldest deadline 2024-03-19T02:07:05.525146+0000) 2024-03-19T02:07:07.807 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:07 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:07.492+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:07.807 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:07 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:07.492+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:07:08.090 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:07 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:07.837+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170696+0000 front 2024-03-19T02:06:34.170736+0000 (oldest deadline 2024-03-19T02:06:55.870953+0000) 2024-03-19T02:07:08.090 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:07 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:07.837+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.572097+0000 front 2024-03-19T02:06:40.571972+0000 (oldest deadline 2024-03-19T02:07:06.372675+0000) 2024-03-19T02:07:08.111 INFO:journalctl@ceph.osd.4.smithi112.stdout:Mar 19 02:07:07 smithi112 podman[106687]: 2024-03-19 02:07:07.844577746 +0000 UTC m=+6.096111288 container cleanup f51be7fcdd3dc0db17601ee48f1818f4010082d5e1cb6ea0361c3e4deb75733b (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-4, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240307, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, io.buildah.version=1.33.5) 2024-03-19T02:07:08.112 INFO:journalctl@ceph.osd.4.smithi112.stdout:Mar 19 02:07:07 smithi112 bash[106687]: ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-4 2024-03-19T02:07:08.461 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:08 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:08.078+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:41.493849+0000 front 2024-03-19T02:06:41.493734+0000 (oldest deadline 2024-03-19T02:07:03.793726+0000) 2024-03-19T02:07:08.490 INFO:journalctl@ceph.osd.4.smithi112.stdout:Mar 19 02:07:08 smithi112 podman[106704]: 2024-03-19 02:07:08.138534602 +0000 UTC m=+0.645268051 container remove f51be7fcdd3dc0db17601ee48f1818f4010082d5e1cb6ea0361c3e4deb75733b (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-4, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240307, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-03-19T02:07:08.590 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:08 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:08.301+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:31.579664+0000 front 2024-03-19T02:06:31.579687+0000 (oldest deadline 2024-03-19T02:06:57.479977+0000) 2024-03-19T02:07:08.590 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:08 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:08.301+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.980985+0000 front 2024-03-19T02:06:40.981003+0000 (oldest deadline 2024-03-19T02:07:03.281058+0000) 2024-03-19T02:07:08.757 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:08 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:08.678+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.951647+0000 front 2024-03-19T02:06:38.951558+0000 (oldest deadline 2024-03-19T02:07:03.651603+0000) 2024-03-19T02:07:08.757 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:08 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:08.645+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170251+0000 front 2024-03-19T02:06:34.170405+0000 (oldest deadline 2024-03-19T02:06:58.270386+0000) 2024-03-19T02:07:08.757 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:08 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:08.645+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.270741+0000 front 2024-03-19T02:06:38.270719+0000 (oldest deadline 2024-03-19T02:07:04.170884+0000) 2024-03-19T02:07:08.758 INFO:journalctl@ceph.osd.4.smithi112.stdout:Mar 19 02:07:08 smithi112 podman[106797]: 2024-03-19 02:07:08.393518427 +0000 UTC m=+0.017446983 image pull 2312c8e42d0069a2dbb2364bddb00264db4b800e7f18a7e7e4eb8f3ecab2ff1c quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639 2024-03-19T02:07:08.758 INFO:journalctl@ceph.osd.4.smithi112.stdout:Mar 19 02:07:08 smithi112 podman[106797]: 2024-03-19 02:07:08.496742699 +0000 UTC m=+0.120671237 container create ae5fd18c2fd708c146fc5090f24bc14b29e5975db16e86a5af89aeffbdcd3240 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-4-deactivate, CEPH_POINT_RELEASE=, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.build-date=20240307, org.label-schema.vendor=CentOS, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, ceph=True) 2024-03-19T02:07:08.758 INFO:journalctl@ceph.osd.4.smithi112.stdout:Mar 19 02:07:08 smithi112 podman[106797]: 2024-03-19 02:07:08.68263468 +0000 UTC m=+0.306563242 container init ae5fd18c2fd708c146fc5090f24bc14b29e5975db16e86a5af89aeffbdcd3240 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-4-deactivate, org.label-schema.build-date=20240307, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-03-19T02:07:08.758 INFO:journalctl@ceph.osd.4.smithi112.stdout:Mar 19 02:07:08 smithi112 podman[106797]: 2024-03-19 02:07:08.685775415 +0000 UTC m=+0.309703963 container start ae5fd18c2fd708c146fc5090f24bc14b29e5975db16e86a5af89aeffbdcd3240 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-4-deactivate, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.build-date=20240307, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, ceph=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=) 2024-03-19T02:07:08.758 INFO:journalctl@ceph.osd.4.smithi112.stdout:Mar 19 02:07:08 smithi112 podman[106797]: 2024-03-19 02:07:08.757576048 +0000 UTC m=+0.381504611 container attach ae5fd18c2fd708c146fc5090f24bc14b29e5975db16e86a5af89aeffbdcd3240 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-4-deactivate, io.buildah.version=1.33.5, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, maintainer=Guillaume Abrioux , RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.build-date=20240307, org.label-schema.vendor=CentOS) 2024-03-19T02:07:08.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:08 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:08.461+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:08.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:08 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:08.461+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:07:08.806 INFO:journalctl@ceph.osd.5.smithi115.stdout:Mar 19 02:07:08 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-5[78850]: 2024-03-19T02:07:08.527+0000 7fba3c793700 -1 osd.5 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:33.323399+0000 front 2024-03-19T02:06:33.323427+0000 (oldest deadline 2024-03-19T02:06:57.423498+0000) 2024-03-19T02:07:08.807 INFO:journalctl@ceph.osd.5.smithi115.stdout:Mar 19 02:07:08 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-5[78850]: 2024-03-19T02:07:08.527+0000 7fba3c793700 -1 osd.5 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.225204+0000 front 2024-03-19T02:06:40.225256+0000 (oldest deadline 2024-03-19T02:07:05.525146+0000) 2024-03-19T02:07:09.090 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:08 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:08.825+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170696+0000 front 2024-03-19T02:06:34.170736+0000 (oldest deadline 2024-03-19T02:06:55.870953+0000) 2024-03-19T02:07:09.090 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:08 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:08.825+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.572097+0000 front 2024-03-19T02:06:40.571972+0000 (oldest deadline 2024-03-19T02:07:06.372675+0000) 2024-03-19T02:07:09.112 INFO:journalctl@ceph.osd.4.smithi112.stdout:Mar 19 02:07:08 smithi112 podman[106797]: 2024-03-19 02:07:08.97885478 +0000 UTC m=+0.602783334 container died ae5fd18c2fd708c146fc5090f24bc14b29e5975db16e86a5af89aeffbdcd3240 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-4-deactivate, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.build-date=20240307, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30) 2024-03-19T02:07:09.507 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:09 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:09.116+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:41.493849+0000 front 2024-03-19T02:06:41.493734+0000 (oldest deadline 2024-03-19T02:07:03.793726+0000) 2024-03-19T02:07:09.572 DEBUG:teuthology.orchestra.run.smithi112:> sudo pkill -f 'journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@osd.4.service' 2024-03-19T02:07:09.590 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:09 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:09.254+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:31.579664+0000 front 2024-03-19T02:06:31.579687+0000 (oldest deadline 2024-03-19T02:06:57.479977+0000) 2024-03-19T02:07:09.590 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:09 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:09.254+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.980985+0000 front 2024-03-19T02:06:40.981003+0000 (oldest deadline 2024-03-19T02:07:03.281058+0000) 2024-03-19T02:07:09.613 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-19T02:07:09.613 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-03-19T02:07:09.614 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-03-19T02:07:09.614 DEBUG:teuthology.orchestra.run.smithi115:> sudo systemctl stop ceph-438fb69c-e590-11ee-95c9-87774f69a715@osd.5 2024-03-19T02:07:09.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:09 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:09.507+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:09.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:09 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:09.507+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:07:09.806 INFO:journalctl@ceph.osd.5.smithi115.stdout:Mar 19 02:07:09 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-5[78850]: 2024-03-19T02:07:09.536+0000 7fba3c793700 -1 osd.5 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:33.323399+0000 front 2024-03-19T02:06:33.323427+0000 (oldest deadline 2024-03-19T02:06:57.423498+0000) 2024-03-19T02:07:09.807 INFO:journalctl@ceph.osd.5.smithi115.stdout:Mar 19 02:07:09 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-5[78850]: 2024-03-19T02:07:09.536+0000 7fba3c793700 -1 osd.5 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.225204+0000 front 2024-03-19T02:06:40.225256+0000 (oldest deadline 2024-03-19T02:07:05.525146+0000) 2024-03-19T02:07:09.807 INFO:journalctl@ceph.osd.5.smithi115.stdout:Mar 19 02:07:09 smithi115 systemd[1]: Stopping Ceph osd.5 for 438fb69c-e590-11ee-95c9-87774f69a715... 2024-03-19T02:07:09.861 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:09 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:09.649+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.951647+0000 front 2024-03-19T02:06:38.951558+0000 (oldest deadline 2024-03-19T02:07:03.651603+0000) 2024-03-19T02:07:09.862 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:09 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:09.623+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170251+0000 front 2024-03-19T02:06:34.170405+0000 (oldest deadline 2024-03-19T02:06:58.270386+0000) 2024-03-19T02:07:09.862 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:09 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:09.623+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.270741+0000 front 2024-03-19T02:06:38.270719+0000 (oldest deadline 2024-03-19T02:07:04.170884+0000) 2024-03-19T02:07:10.090 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:09 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:09.790+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170696+0000 front 2024-03-19T02:06:34.170736+0000 (oldest deadline 2024-03-19T02:06:55.870953+0000) 2024-03-19T02:07:10.090 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:09 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:09.790+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.572097+0000 front 2024-03-19T02:06:40.571972+0000 (oldest deadline 2024-03-19T02:07:06.372675+0000) 2024-03-19T02:07:10.306 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:10 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:10.115+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:41.493849+0000 front 2024-03-19T02:06:41.493734+0000 (oldest deadline 2024-03-19T02:07:03.793726+0000) 2024-03-19T02:07:10.306 INFO:journalctl@ceph.osd.5.smithi115.stdout:Mar 19 02:07:09 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-5[78850]: 2024-03-19T02:07:09.926+0000 7fba441a7700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-19T02:07:10.306 INFO:journalctl@ceph.osd.5.smithi115.stdout:Mar 19 02:07:09 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-5[78850]: 2024-03-19T02:07:09.926+0000 7fba441a7700 -1 osd.5 95 *** Got signal Terminated *** 2024-03-19T02:07:10.307 INFO:journalctl@ceph.osd.5.smithi115.stdout:Mar 19 02:07:09 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-5[78850]: 2024-03-19T02:07:09.926+0000 7fba441a7700 -1 osd.5 95 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-19T02:07:10.590 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:10 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:10.261+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:31.579664+0000 front 2024-03-19T02:06:31.579687+0000 (oldest deadline 2024-03-19T02:06:57.479977+0000) 2024-03-19T02:07:10.591 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:10 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:10.261+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.980985+0000 front 2024-03-19T02:06:40.981003+0000 (oldest deadline 2024-03-19T02:07:03.281058+0000) 2024-03-19T02:07:10.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:10 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:10.539+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:10.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:10 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:10.539+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:07:10.807 INFO:journalctl@ceph.osd.5.smithi115.stdout:Mar 19 02:07:10 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-5[78850]: 2024-03-19T02:07:10.569+0000 7fba3c793700 -1 osd.5 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:33.323399+0000 front 2024-03-19T02:06:33.323427+0000 (oldest deadline 2024-03-19T02:06:57.423498+0000) 2024-03-19T02:07:10.807 INFO:journalctl@ceph.osd.5.smithi115.stdout:Mar 19 02:07:10 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-5[78850]: 2024-03-19T02:07:10.569+0000 7fba3c793700 -1 osd.5 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.225204+0000 front 2024-03-19T02:06:40.225256+0000 (oldest deadline 2024-03-19T02:07:05.525146+0000) 2024-03-19T02:07:11.090 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:10 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:10.807+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170696+0000 front 2024-03-19T02:06:34.170736+0000 (oldest deadline 2024-03-19T02:06:55.870953+0000) 2024-03-19T02:07:11.090 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:10 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:10.807+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.572097+0000 front 2024-03-19T02:06:40.571972+0000 (oldest deadline 2024-03-19T02:07:06.372675+0000) 2024-03-19T02:07:11.111 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:10 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:10.655+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.951647+0000 front 2024-03-19T02:06:38.951558+0000 (oldest deadline 2024-03-19T02:07:03.651603+0000) 2024-03-19T02:07:11.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:10 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:10.659+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170251+0000 front 2024-03-19T02:06:34.170405+0000 (oldest deadline 2024-03-19T02:06:58.270386+0000) 2024-03-19T02:07:11.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:10 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:10.659+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.270741+0000 front 2024-03-19T02:06:38.270719+0000 (oldest deadline 2024-03-19T02:07:04.170884+0000) 2024-03-19T02:07:11.527 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:11 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:11.132+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:41.493849+0000 front 2024-03-19T02:06:41.493734+0000 (oldest deadline 2024-03-19T02:07:03.793726+0000) 2024-03-19T02:07:11.565 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:11 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:11.273+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:31.579664+0000 front 2024-03-19T02:06:31.579687+0000 (oldest deadline 2024-03-19T02:06:57.479977+0000) 2024-03-19T02:07:11.565 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:11 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:11.273+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.980985+0000 front 2024-03-19T02:06:40.981003+0000 (oldest deadline 2024-03-19T02:07:03.281058+0000) 2024-03-19T02:07:11.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:11 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:11.526+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:11.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:11 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:11.526+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:07:11.807 INFO:journalctl@ceph.osd.5.smithi115.stdout:Mar 19 02:07:11 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-5[78850]: 2024-03-19T02:07:11.546+0000 7fba3c793700 -1 osd.5 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:33.323399+0000 front 2024-03-19T02:06:33.323427+0000 (oldest deadline 2024-03-19T02:06:57.423498+0000) 2024-03-19T02:07:11.807 INFO:journalctl@ceph.osd.5.smithi115.stdout:Mar 19 02:07:11 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-5[78850]: 2024-03-19T02:07:11.546+0000 7fba3c793700 -1 osd.5 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.225204+0000 front 2024-03-19T02:06:40.225256+0000 (oldest deadline 2024-03-19T02:07:05.525146+0000) 2024-03-19T02:07:12.111 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:11 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:11.692+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.951647+0000 front 2024-03-19T02:06:38.951558+0000 (oldest deadline 2024-03-19T02:07:03.651603+0000) 2024-03-19T02:07:12.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:11 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:11.666+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170251+0000 front 2024-03-19T02:06:34.170405+0000 (oldest deadline 2024-03-19T02:06:58.270386+0000) 2024-03-19T02:07:12.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:11 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:11.666+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.270741+0000 front 2024-03-19T02:06:38.270719+0000 (oldest deadline 2024-03-19T02:07:04.170884+0000) 2024-03-19T02:07:12.269 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:11 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:11.843+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170696+0000 front 2024-03-19T02:06:34.170736+0000 (oldest deadline 2024-03-19T02:06:55.870953+0000) 2024-03-19T02:07:12.269 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:11 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:11.843+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.572097+0000 front 2024-03-19T02:06:40.571972+0000 (oldest deadline 2024-03-19T02:07:06.372675+0000) 2024-03-19T02:07:12.510 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:12 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:12.115+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:41.493849+0000 front 2024-03-19T02:06:41.493734+0000 (oldest deadline 2024-03-19T02:07:03.793726+0000) 2024-03-19T02:07:12.590 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:12 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:12.269+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:31.579664+0000 front 2024-03-19T02:06:31.579687+0000 (oldest deadline 2024-03-19T02:06:57.479977+0000) 2024-03-19T02:07:12.591 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:12 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:12.269+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.980985+0000 front 2024-03-19T02:06:40.981003+0000 (oldest deadline 2024-03-19T02:07:03.281058+0000) 2024-03-19T02:07:12.591 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:12 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:12.269+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.182086+0000 front 2024-03-19T02:06:46.182039+0000 (oldest deadline 2024-03-19T02:07:12.082249+0000) 2024-03-19T02:07:12.806 INFO:journalctl@ceph.osd.5.smithi115.stdout:Mar 19 02:07:12 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-5[78850]: 2024-03-19T02:07:12.592+0000 7fba3c793700 -1 osd.5 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:33.323399+0000 front 2024-03-19T02:06:33.323427+0000 (oldest deadline 2024-03-19T02:06:57.423498+0000) 2024-03-19T02:07:12.806 INFO:journalctl@ceph.osd.5.smithi115.stdout:Mar 19 02:07:12 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-5[78850]: 2024-03-19T02:07:12.592+0000 7fba3c793700 -1 osd.5 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.225204+0000 front 2024-03-19T02:06:40.225256+0000 (oldest deadline 2024-03-19T02:07:05.525146+0000) 2024-03-19T02:07:12.807 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:12 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:12.510+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:12.807 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:12 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:12.510+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:07:12.807 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:12 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:12.510+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.116306+0000 front 2024-03-19T02:06:46.116324+0000 (oldest deadline 2024-03-19T02:07:12.016484+0000) 2024-03-19T02:07:13.090 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:12 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:12.803+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170696+0000 front 2024-03-19T02:06:34.170736+0000 (oldest deadline 2024-03-19T02:06:55.870953+0000) 2024-03-19T02:07:13.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:12 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:12.803+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.572097+0000 front 2024-03-19T02:06:40.571972+0000 (oldest deadline 2024-03-19T02:07:06.372675+0000) 2024-03-19T02:07:13.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:12 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:12.803+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.675305+0000 front 2024-03-19T02:06:49.675286+0000 (oldest deadline 2024-03-19T02:07:11.975395+0000) 2024-03-19T02:07:13.112 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:12 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:12.735+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.951647+0000 front 2024-03-19T02:06:38.951558+0000 (oldest deadline 2024-03-19T02:07:03.651603+0000) 2024-03-19T02:07:13.112 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:12 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:12.735+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.453491+0000 front 2024-03-19T02:06:49.453554+0000 (oldest deadline 2024-03-19T02:07:11.753578+0000) 2024-03-19T02:07:13.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:12 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:12.712+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170251+0000 front 2024-03-19T02:06:34.170405+0000 (oldest deadline 2024-03-19T02:06:58.270386+0000) 2024-03-19T02:07:13.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:12 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:12.713+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.270741+0000 front 2024-03-19T02:06:38.270719+0000 (oldest deadline 2024-03-19T02:07:04.170884+0000) 2024-03-19T02:07:13.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:12 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:12.713+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:47.072167+0000 front 2024-03-19T02:06:47.072106+0000 (oldest deadline 2024-03-19T02:07:12.372271+0000) 2024-03-19T02:07:13.518 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:13 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:13.133+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:41.493849+0000 front 2024-03-19T02:06:41.493734+0000 (oldest deadline 2024-03-19T02:07:03.793726+0000) 2024-03-19T02:07:13.590 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:13 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:13.277+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:31.579664+0000 front 2024-03-19T02:06:31.579687+0000 (oldest deadline 2024-03-19T02:06:57.479977+0000) 2024-03-19T02:07:13.590 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:13 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:13.277+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.980985+0000 front 2024-03-19T02:06:40.981003+0000 (oldest deadline 2024-03-19T02:07:03.281058+0000) 2024-03-19T02:07:13.590 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:13 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:13.277+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.182086+0000 front 2024-03-19T02:06:46.182039+0000 (oldest deadline 2024-03-19T02:07:12.082249+0000) 2024-03-19T02:07:13.806 INFO:journalctl@ceph.osd.5.smithi115.stdout:Mar 19 02:07:13 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-5[78850]: 2024-03-19T02:07:13.619+0000 7fba3c793700 -1 osd.5 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:33.323399+0000 front 2024-03-19T02:06:33.323427+0000 (oldest deadline 2024-03-19T02:06:57.423498+0000) 2024-03-19T02:07:13.806 INFO:journalctl@ceph.osd.5.smithi115.stdout:Mar 19 02:07:13 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-5[78850]: 2024-03-19T02:07:13.619+0000 7fba3c793700 -1 osd.5 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.225204+0000 front 2024-03-19T02:06:40.225256+0000 (oldest deadline 2024-03-19T02:07:05.525146+0000) 2024-03-19T02:07:13.807 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:13 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:13.518+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:13.807 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:13 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:13.518+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:07:13.807 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:13 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:13.518+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.116306+0000 front 2024-03-19T02:06:46.116324+0000 (oldest deadline 2024-03-19T02:07:12.016484+0000) 2024-03-19T02:07:14.090 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:13 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:13.793+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170696+0000 front 2024-03-19T02:06:34.170736+0000 (oldest deadline 2024-03-19T02:06:55.870953+0000) 2024-03-19T02:07:14.090 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:13 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:13.793+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.572097+0000 front 2024-03-19T02:06:40.571972+0000 (oldest deadline 2024-03-19T02:07:06.372675+0000) 2024-03-19T02:07:14.090 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:13 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:13.793+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.675305+0000 front 2024-03-19T02:06:49.675286+0000 (oldest deadline 2024-03-19T02:07:11.975395+0000) 2024-03-19T02:07:14.112 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:13 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:13.750+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.951647+0000 front 2024-03-19T02:06:38.951558+0000 (oldest deadline 2024-03-19T02:07:03.651603+0000) 2024-03-19T02:07:14.112 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:13 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:13.750+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.453491+0000 front 2024-03-19T02:06:49.453554+0000 (oldest deadline 2024-03-19T02:07:11.753578+0000) 2024-03-19T02:07:14.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:13 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:13.689+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170251+0000 front 2024-03-19T02:06:34.170405+0000 (oldest deadline 2024-03-19T02:06:58.270386+0000) 2024-03-19T02:07:14.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:13 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:13.689+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.270741+0000 front 2024-03-19T02:06:38.270719+0000 (oldest deadline 2024-03-19T02:07:04.170884+0000) 2024-03-19T02:07:14.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:13 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:13.689+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:47.072167+0000 front 2024-03-19T02:06:47.072106+0000 (oldest deadline 2024-03-19T02:07:12.372271+0000) 2024-03-19T02:07:14.547 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:14 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:14.121+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:41.493849+0000 front 2024-03-19T02:06:41.493734+0000 (oldest deadline 2024-03-19T02:07:03.793726+0000) 2024-03-19T02:07:14.547 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:14 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:14.121+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.996743+0000 front 2024-03-19T02:06:49.996766+0000 (oldest deadline 2024-03-19T02:07:14.096879+0000) 2024-03-19T02:07:14.590 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:14 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:14.322+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:31.579664+0000 front 2024-03-19T02:06:31.579687+0000 (oldest deadline 2024-03-19T02:06:57.479977+0000) 2024-03-19T02:07:14.590 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:14 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:14.322+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.980985+0000 front 2024-03-19T02:06:40.981003+0000 (oldest deadline 2024-03-19T02:07:03.281058+0000) 2024-03-19T02:07:14.590 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:14 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:14.322+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.182086+0000 front 2024-03-19T02:06:46.182039+0000 (oldest deadline 2024-03-19T02:07:12.082249+0000) 2024-03-19T02:07:14.806 INFO:journalctl@ceph.osd.5.smithi115.stdout:Mar 19 02:07:14 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-5[78850]: 2024-03-19T02:07:14.597+0000 7fba3c793700 -1 osd.5 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:33.323399+0000 front 2024-03-19T02:06:33.323427+0000 (oldest deadline 2024-03-19T02:06:57.423498+0000) 2024-03-19T02:07:14.806 INFO:journalctl@ceph.osd.5.smithi115.stdout:Mar 19 02:07:14 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-5[78850]: 2024-03-19T02:07:14.597+0000 7fba3c793700 -1 osd.5 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.225204+0000 front 2024-03-19T02:06:40.225256+0000 (oldest deadline 2024-03-19T02:07:05.525146+0000) 2024-03-19T02:07:14.807 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:14 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:14.547+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:14.807 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:14 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:14.547+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:07:14.807 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:14 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:14.547+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.116306+0000 front 2024-03-19T02:06:46.116324+0000 (oldest deadline 2024-03-19T02:07:12.016484+0000) 2024-03-19T02:07:15.090 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:14 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:14.818+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170696+0000 front 2024-03-19T02:06:34.170736+0000 (oldest deadline 2024-03-19T02:06:55.870953+0000) 2024-03-19T02:07:15.090 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:14 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:14.818+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.572097+0000 front 2024-03-19T02:06:40.571972+0000 (oldest deadline 2024-03-19T02:07:06.372675+0000) 2024-03-19T02:07:15.090 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:14 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:14.818+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.675305+0000 front 2024-03-19T02:06:49.675286+0000 (oldest deadline 2024-03-19T02:07:11.975395+0000) 2024-03-19T02:07:15.112 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:14 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:14.704+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.951647+0000 front 2024-03-19T02:06:38.951558+0000 (oldest deadline 2024-03-19T02:07:03.651603+0000) 2024-03-19T02:07:15.112 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:14 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:14.704+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.453491+0000 front 2024-03-19T02:06:49.453554+0000 (oldest deadline 2024-03-19T02:07:11.753578+0000) 2024-03-19T02:07:15.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:14 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:14.734+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170251+0000 front 2024-03-19T02:06:34.170405+0000 (oldest deadline 2024-03-19T02:06:58.270386+0000) 2024-03-19T02:07:15.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:14 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:14.734+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.270741+0000 front 2024-03-19T02:06:38.270719+0000 (oldest deadline 2024-03-19T02:07:04.170884+0000) 2024-03-19T02:07:15.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:14 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:14.734+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:47.072167+0000 front 2024-03-19T02:06:47.072106+0000 (oldest deadline 2024-03-19T02:07:12.372271+0000) 2024-03-19T02:07:15.557 INFO:journalctl@ceph.osd.5.smithi115.stdout:Mar 19 02:07:15 smithi115 podman[106146]: 2024-03-19 02:07:15.340105349 +0000 UTC m=+5.532549808 container died d9abad9bc8798e02075b6d046cddfc20d279ae123504a11d4f248111556be839 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-5, org.label-schema.build-date=20240307, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-19T02:07:15.557 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:15 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:15.107+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:41.493849+0000 front 2024-03-19T02:06:41.493734+0000 (oldest deadline 2024-03-19T02:07:03.793726+0000) 2024-03-19T02:07:15.557 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:15 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:15.107+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.996743+0000 front 2024-03-19T02:06:49.996766+0000 (oldest deadline 2024-03-19T02:07:14.096879+0000) 2024-03-19T02:07:15.590 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:15 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:15.295+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:31.579664+0000 front 2024-03-19T02:06:31.579687+0000 (oldest deadline 2024-03-19T02:06:57.479977+0000) 2024-03-19T02:07:15.590 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:15 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:15.295+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.980985+0000 front 2024-03-19T02:06:40.981003+0000 (oldest deadline 2024-03-19T02:07:03.281058+0000) 2024-03-19T02:07:15.590 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:15 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:15.295+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.182086+0000 front 2024-03-19T02:06:46.182039+0000 (oldest deadline 2024-03-19T02:07:12.082249+0000) 2024-03-19T02:07:16.011 INFO:journalctl@ceph.osd.5.smithi115.stdout:Mar 19 02:07:15 smithi115 podman[106146]: 2024-03-19 02:07:15.726584207 +0000 UTC m=+5.919028677 container cleanup d9abad9bc8798e02075b6d046cddfc20d279ae123504a11d4f248111556be839 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-5, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240307, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-batrick-testing-20240318.181317-b6ae249) 2024-03-19T02:07:16.011 INFO:journalctl@ceph.osd.5.smithi115.stdout:Mar 19 02:07:15 smithi115 bash[106146]: ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-5 2024-03-19T02:07:16.011 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:15 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:15.595+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:16.012 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:15 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:15.595+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:07:16.012 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:15 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:15.595+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.116306+0000 front 2024-03-19T02:06:46.116324+0000 (oldest deadline 2024-03-19T02:07:12.016484+0000) 2024-03-19T02:07:16.111 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:15 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:15.749+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.951647+0000 front 2024-03-19T02:06:38.951558+0000 (oldest deadline 2024-03-19T02:07:03.651603+0000) 2024-03-19T02:07:16.112 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:15 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:15.749+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.453491+0000 front 2024-03-19T02:06:49.453554+0000 (oldest deadline 2024-03-19T02:07:11.753578+0000) 2024-03-19T02:07:16.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:15 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:15.729+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170251+0000 front 2024-03-19T02:06:34.170405+0000 (oldest deadline 2024-03-19T02:06:58.270386+0000) 2024-03-19T02:07:16.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:15 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:15.729+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.270741+0000 front 2024-03-19T02:06:38.270719+0000 (oldest deadline 2024-03-19T02:07:04.170884+0000) 2024-03-19T02:07:16.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:15 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:15.729+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:47.072167+0000 front 2024-03-19T02:06:47.072106+0000 (oldest deadline 2024-03-19T02:07:12.372271+0000) 2024-03-19T02:07:16.246 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:15 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:15.853+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170696+0000 front 2024-03-19T02:06:34.170736+0000 (oldest deadline 2024-03-19T02:06:55.870953+0000) 2024-03-19T02:07:16.246 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:15 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:15.853+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.572097+0000 front 2024-03-19T02:06:40.571972+0000 (oldest deadline 2024-03-19T02:07:06.372675+0000) 2024-03-19T02:07:16.246 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:15 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:15.853+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.675305+0000 front 2024-03-19T02:06:49.675286+0000 (oldest deadline 2024-03-19T02:07:11.975395+0000) 2024-03-19T02:07:16.306 INFO:journalctl@ceph.osd.5.smithi115.stdout:Mar 19 02:07:16 smithi115 podman[106162]: 2024-03-19 02:07:16.01198065 +0000 UTC m=+0.667116837 container remove d9abad9bc8798e02075b6d046cddfc20d279ae123504a11d4f248111556be839 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-5, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, ceph=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240307, org.label-schema.schema-version=1.0) 2024-03-19T02:07:16.306 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:16 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:16.093+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:41.493849+0000 front 2024-03-19T02:06:41.493734+0000 (oldest deadline 2024-03-19T02:07:03.793726+0000) 2024-03-19T02:07:16.306 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:16 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:16.093+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.996743+0000 front 2024-03-19T02:06:49.996766+0000 (oldest deadline 2024-03-19T02:07:14.096879+0000) 2024-03-19T02:07:16.590 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:16 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:16.246+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:31.579664+0000 front 2024-03-19T02:06:31.579687+0000 (oldest deadline 2024-03-19T02:06:57.479977+0000) 2024-03-19T02:07:16.590 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:16 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:16.246+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.980985+0000 front 2024-03-19T02:06:40.981003+0000 (oldest deadline 2024-03-19T02:07:03.281058+0000) 2024-03-19T02:07:16.591 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:16 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:16.246+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.182086+0000 front 2024-03-19T02:06:46.182039+0000 (oldest deadline 2024-03-19T02:07:12.082249+0000) 2024-03-19T02:07:16.609 INFO:journalctl@ceph.osd.5.smithi115.stdout:Mar 19 02:07:16 smithi115 podman[106253]: 2024-03-19 02:07:16.238177292 +0000 UTC m=+0.017395738 image pull 2312c8e42d0069a2dbb2364bddb00264db4b800e7f18a7e7e4eb8f3ecab2ff1c quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639 2024-03-19T02:07:16.609 INFO:journalctl@ceph.osd.5.smithi115.stdout:Mar 19 02:07:16 smithi115 podman[106253]: 2024-03-19 02:07:16.345588902 +0000 UTC m=+0.124807349 container create 58b455a6c866e5a81528e46fce264013c4ac3df579ea79c241ede51685eb7e3c (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-5-deactivate, maintainer=Guillaume Abrioux , GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, ceph=True, org.label-schema.build-date=20240307, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-19T02:07:16.609 INFO:journalctl@ceph.osd.5.smithi115.stdout:Mar 19 02:07:16 smithi115 podman[106253]: 2024-03-19 02:07:16.581553078 +0000 UTC m=+0.360771525 container init 58b455a6c866e5a81528e46fce264013c4ac3df579ea79c241ede51685eb7e3c (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-5-deactivate, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=wip-batrick-testing-20240318.181317-b6ae249, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240307, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-03-19T02:07:16.609 INFO:journalctl@ceph.osd.5.smithi115.stdout:Mar 19 02:07:16 smithi115 podman[106253]: 2024-03-19 02:07:16.584938505 +0000 UTC m=+0.364156936 container start 58b455a6c866e5a81528e46fce264013c4ac3df579ea79c241ede51685eb7e3c (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-5-deactivate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.build-date=20240307, org.label-schema.schema-version=1.0) 2024-03-19T02:07:16.876 INFO:journalctl@ceph.osd.5.smithi115.stdout:Mar 19 02:07:16 smithi115 podman[106253]: 2024-03-19 02:07:16.656476638 +0000 UTC m=+0.435695096 container attach 58b455a6c866e5a81528e46fce264013c4ac3df579ea79c241ede51685eb7e3c (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-5-deactivate, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.vendor=CentOS, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_BRANCH=HEAD, org.label-schema.build-date=20240307) 2024-03-19T02:07:16.876 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:16 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:16.608+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:16.876 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:16 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:16.608+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:07:16.876 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:16 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:16.608+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.116306+0000 front 2024-03-19T02:06:46.116324+0000 (oldest deadline 2024-03-19T02:07:12.016484+0000) 2024-03-19T02:07:17.112 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:16 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:16.789+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.951647+0000 front 2024-03-19T02:06:38.951558+0000 (oldest deadline 2024-03-19T02:07:03.651603+0000) 2024-03-19T02:07:17.112 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:16 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:16.789+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.453491+0000 front 2024-03-19T02:06:49.453554+0000 (oldest deadline 2024-03-19T02:07:11.753578+0000) 2024-03-19T02:07:17.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:16 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:16.757+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170251+0000 front 2024-03-19T02:06:34.170405+0000 (oldest deadline 2024-03-19T02:06:58.270386+0000) 2024-03-19T02:07:17.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:16 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:16.757+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.270741+0000 front 2024-03-19T02:06:38.270719+0000 (oldest deadline 2024-03-19T02:07:04.170884+0000) 2024-03-19T02:07:17.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:16 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:16.757+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:47.072167+0000 front 2024-03-19T02:06:47.072106+0000 (oldest deadline 2024-03-19T02:07:12.372271+0000) 2024-03-19T02:07:17.244 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:16 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:16.859+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170696+0000 front 2024-03-19T02:06:34.170736+0000 (oldest deadline 2024-03-19T02:06:55.870953+0000) 2024-03-19T02:07:17.244 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:16 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:16.859+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.572097+0000 front 2024-03-19T02:06:40.571972+0000 (oldest deadline 2024-03-19T02:07:06.372675+0000) 2024-03-19T02:07:17.245 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:16 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:16.859+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.675305+0000 front 2024-03-19T02:06:49.675286+0000 (oldest deadline 2024-03-19T02:07:11.975395+0000) 2024-03-19T02:07:17.306 INFO:journalctl@ceph.osd.5.smithi115.stdout:Mar 19 02:07:16 smithi115 podman[106253]: 2024-03-19 02:07:16.876952591 +0000 UTC m=+0.656171035 container died 58b455a6c866e5a81528e46fce264013c4ac3df579ea79c241ede51685eb7e3c (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-5-deactivate, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, CEPH_POINT_RELEASE=, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_CLEAN=True, org.label-schema.build-date=20240307, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-03-19T02:07:17.306 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:17 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:17.098+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:41.493849+0000 front 2024-03-19T02:06:41.493734+0000 (oldest deadline 2024-03-19T02:07:03.793726+0000) 2024-03-19T02:07:17.306 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:17 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:17.098+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.996743+0000 front 2024-03-19T02:06:49.996766+0000 (oldest deadline 2024-03-19T02:07:14.096879+0000) 2024-03-19T02:07:17.379 DEBUG:teuthology.orchestra.run.smithi115:> sudo pkill -f 'journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@osd.5.service' 2024-03-19T02:07:17.420 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-19T02:07:17.420 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-03-19T02:07:17.420 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-03-19T02:07:17.420 DEBUG:teuthology.orchestra.run.smithi099:> sudo systemctl stop ceph-438fb69c-e590-11ee-95c9-87774f69a715@osd.6 2024-03-19T02:07:17.590 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:17 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:17.245+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:31.579664+0000 front 2024-03-19T02:06:31.579687+0000 (oldest deadline 2024-03-19T02:06:57.479977+0000) 2024-03-19T02:07:17.590 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:17 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:17.245+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.980985+0000 front 2024-03-19T02:06:40.981003+0000 (oldest deadline 2024-03-19T02:07:03.281058+0000) 2024-03-19T02:07:17.591 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:17 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:17.245+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.182086+0000 front 2024-03-19T02:06:46.182039+0000 (oldest deadline 2024-03-19T02:07:12.082249+0000) 2024-03-19T02:07:17.591 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:17 smithi099 systemd[1]: Stopping Ceph osd.6 for 438fb69c-e590-11ee-95c9-87774f69a715... 2024-03-19T02:07:17.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:17 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:17.590+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:17.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:17 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:17.590+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:07:17.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:17 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:17.590+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.116306+0000 front 2024-03-19T02:06:46.116324+0000 (oldest deadline 2024-03-19T02:07:12.016484+0000) 2024-03-19T02:07:18.090 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:17 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:17.735+0000 7f6b8d881700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-19T02:07:18.091 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:17 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:17.735+0000 7f6b8d881700 -1 osd.6 95 *** Got signal Terminated *** 2024-03-19T02:07:18.091 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:17 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:17.735+0000 7f6b8d881700 -1 osd.6 95 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-19T02:07:18.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:17 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:17.839+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170696+0000 front 2024-03-19T02:06:34.170736+0000 (oldest deadline 2024-03-19T02:06:55.870953+0000) 2024-03-19T02:07:18.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:17 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:17.839+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.572097+0000 front 2024-03-19T02:06:40.571972+0000 (oldest deadline 2024-03-19T02:07:06.372675+0000) 2024-03-19T02:07:18.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:17 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:17.839+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.675305+0000 front 2024-03-19T02:06:49.675286+0000 (oldest deadline 2024-03-19T02:07:11.975395+0000) 2024-03-19T02:07:18.111 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:17 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:17.772+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.951647+0000 front 2024-03-19T02:06:38.951558+0000 (oldest deadline 2024-03-19T02:07:03.651603+0000) 2024-03-19T02:07:18.112 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:17 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:17.772+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.453491+0000 front 2024-03-19T02:06:49.453554+0000 (oldest deadline 2024-03-19T02:07:11.753578+0000) 2024-03-19T02:07:18.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:17 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:17.734+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170251+0000 front 2024-03-19T02:06:34.170405+0000 (oldest deadline 2024-03-19T02:06:58.270386+0000) 2024-03-19T02:07:18.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:17 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:17.734+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.270741+0000 front 2024-03-19T02:06:38.270719+0000 (oldest deadline 2024-03-19T02:07:04.170884+0000) 2024-03-19T02:07:18.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:17 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:17.734+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:47.072167+0000 front 2024-03-19T02:06:47.072106+0000 (oldest deadline 2024-03-19T02:07:12.372271+0000) 2024-03-19T02:07:18.556 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:18 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:18.120+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:41.493849+0000 front 2024-03-19T02:06:41.493734+0000 (oldest deadline 2024-03-19T02:07:03.793726+0000) 2024-03-19T02:07:18.556 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:18 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:18.120+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.996743+0000 front 2024-03-19T02:06:49.996766+0000 (oldest deadline 2024-03-19T02:07:14.096879+0000) 2024-03-19T02:07:18.590 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:18 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:18.201+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:31.579664+0000 front 2024-03-19T02:06:31.579687+0000 (oldest deadline 2024-03-19T02:06:57.479977+0000) 2024-03-19T02:07:18.591 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:18 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:18.201+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.980985+0000 front 2024-03-19T02:06:40.981003+0000 (oldest deadline 2024-03-19T02:07:03.281058+0000) 2024-03-19T02:07:18.591 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:18 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:18.201+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.182086+0000 front 2024-03-19T02:06:46.182039+0000 (oldest deadline 2024-03-19T02:07:12.082249+0000) 2024-03-19T02:07:19.056 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:18 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:18.587+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:19.057 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:18 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:18.587+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:07:19.057 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:18 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:18.587+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.116306+0000 front 2024-03-19T02:06:46.116324+0000 (oldest deadline 2024-03-19T02:07:12.016484+0000) 2024-03-19T02:07:19.090 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:18 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:18.817+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170696+0000 front 2024-03-19T02:06:34.170736+0000 (oldest deadline 2024-03-19T02:06:55.870953+0000) 2024-03-19T02:07:19.090 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:18 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:18.817+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.572097+0000 front 2024-03-19T02:06:40.571972+0000 (oldest deadline 2024-03-19T02:07:06.372675+0000) 2024-03-19T02:07:19.090 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:18 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:18.817+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.675305+0000 front 2024-03-19T02:06:49.675286+0000 (oldest deadline 2024-03-19T02:07:11.975395+0000) 2024-03-19T02:07:19.111 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:18 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:18.811+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.951647+0000 front 2024-03-19T02:06:38.951558+0000 (oldest deadline 2024-03-19T02:07:03.651603+0000) 2024-03-19T02:07:19.112 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:18 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:18.811+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.453491+0000 front 2024-03-19T02:06:49.453554+0000 (oldest deadline 2024-03-19T02:07:11.753578+0000) 2024-03-19T02:07:19.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:18 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:18.785+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170251+0000 front 2024-03-19T02:06:34.170405+0000 (oldest deadline 2024-03-19T02:06:58.270386+0000) 2024-03-19T02:07:19.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:18 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:18.785+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.270741+0000 front 2024-03-19T02:06:38.270719+0000 (oldest deadline 2024-03-19T02:07:04.170884+0000) 2024-03-19T02:07:19.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:18 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:18.785+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:47.072167+0000 front 2024-03-19T02:06:47.072106+0000 (oldest deadline 2024-03-19T02:07:12.372271+0000) 2024-03-19T02:07:19.556 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:19 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:19.088+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:41.493849+0000 front 2024-03-19T02:06:41.493734+0000 (oldest deadline 2024-03-19T02:07:03.793726+0000) 2024-03-19T02:07:19.556 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:19 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:19.088+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.996743+0000 front 2024-03-19T02:06:49.996766+0000 (oldest deadline 2024-03-19T02:07:14.096879+0000) 2024-03-19T02:07:19.590 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:19 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:19.168+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:31.579664+0000 front 2024-03-19T02:06:31.579687+0000 (oldest deadline 2024-03-19T02:06:57.479977+0000) 2024-03-19T02:07:19.590 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:19 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:19.168+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.980985+0000 front 2024-03-19T02:06:40.981003+0000 (oldest deadline 2024-03-19T02:07:03.281058+0000) 2024-03-19T02:07:19.591 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:19 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:19.168+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.182086+0000 front 2024-03-19T02:06:46.182039+0000 (oldest deadline 2024-03-19T02:07:12.082249+0000) 2024-03-19T02:07:20.060 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:19 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:19.604+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:20.060 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:19 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:19.604+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:07:20.060 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:19 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:19.604+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.116306+0000 front 2024-03-19T02:06:46.116324+0000 (oldest deadline 2024-03-19T02:07:12.016484+0000) 2024-03-19T02:07:20.112 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:19 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:19.778+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.951647+0000 front 2024-03-19T02:06:38.951558+0000 (oldest deadline 2024-03-19T02:07:03.651603+0000) 2024-03-19T02:07:20.112 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:19 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:19.778+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.453491+0000 front 2024-03-19T02:06:49.453554+0000 (oldest deadline 2024-03-19T02:07:11.753578+0000) 2024-03-19T02:07:20.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:19 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:19.801+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170251+0000 front 2024-03-19T02:06:34.170405+0000 (oldest deadline 2024-03-19T02:06:58.270386+0000) 2024-03-19T02:07:20.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:19 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:19.801+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.270741+0000 front 2024-03-19T02:06:38.270719+0000 (oldest deadline 2024-03-19T02:07:04.170884+0000) 2024-03-19T02:07:20.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:19 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:19.801+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:47.072167+0000 front 2024-03-19T02:06:47.072106+0000 (oldest deadline 2024-03-19T02:07:12.372271+0000) 2024-03-19T02:07:20.201 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:19 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:19.852+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170696+0000 front 2024-03-19T02:06:34.170736+0000 (oldest deadline 2024-03-19T02:06:55.870953+0000) 2024-03-19T02:07:20.201 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:19 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:19.852+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.572097+0000 front 2024-03-19T02:06:40.571972+0000 (oldest deadline 2024-03-19T02:07:06.372675+0000) 2024-03-19T02:07:20.201 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:19 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:19.852+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.675305+0000 front 2024-03-19T02:06:49.675286+0000 (oldest deadline 2024-03-19T02:07:11.975395+0000) 2024-03-19T02:07:20.556 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:20 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:20.122+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:41.493849+0000 front 2024-03-19T02:06:41.493734+0000 (oldest deadline 2024-03-19T02:07:03.793726+0000) 2024-03-19T02:07:20.556 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:20 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:20.122+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.996743+0000 front 2024-03-19T02:06:49.996766+0000 (oldest deadline 2024-03-19T02:07:14.096879+0000) 2024-03-19T02:07:20.556 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:20 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:20.122+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:54.097324+0000 front 2024-03-19T02:06:54.097227+0000 (oldest deadline 2024-03-19T02:07:19.997301+0000) 2024-03-19T02:07:20.590 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:20 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:20.201+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:31.579664+0000 front 2024-03-19T02:06:31.579687+0000 (oldest deadline 2024-03-19T02:06:57.479977+0000) 2024-03-19T02:07:20.590 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:20 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:20.201+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.980985+0000 front 2024-03-19T02:06:40.981003+0000 (oldest deadline 2024-03-19T02:07:03.281058+0000) 2024-03-19T02:07:20.590 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:20 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:20.201+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.182086+0000 front 2024-03-19T02:06:46.182039+0000 (oldest deadline 2024-03-19T02:07:12.082249+0000) 2024-03-19T02:07:21.056 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:20 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:20.621+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:21.056 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:20 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:20.621+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:07:21.056 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:20 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:20.621+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.116306+0000 front 2024-03-19T02:06:46.116324+0000 (oldest deadline 2024-03-19T02:07:12.016484+0000) 2024-03-19T02:07:21.090 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:20 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:20.815+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170696+0000 front 2024-03-19T02:06:34.170736+0000 (oldest deadline 2024-03-19T02:06:55.870953+0000) 2024-03-19T02:07:21.090 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:20 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:20.815+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.572097+0000 front 2024-03-19T02:06:40.571972+0000 (oldest deadline 2024-03-19T02:07:06.372675+0000) 2024-03-19T02:07:21.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:20 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:20.815+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.675305+0000 front 2024-03-19T02:06:49.675286+0000 (oldest deadline 2024-03-19T02:07:11.975395+0000) 2024-03-19T02:07:21.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:20 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:20.815+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.276447+0000 front 2024-03-19T02:06:57.276673+0000 (oldest deadline 2024-03-19T02:07:20.176691+0000) 2024-03-19T02:07:21.111 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:20 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:20.788+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.951647+0000 front 2024-03-19T02:06:38.951558+0000 (oldest deadline 2024-03-19T02:07:03.651603+0000) 2024-03-19T02:07:21.111 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:20 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:20.788+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.453491+0000 front 2024-03-19T02:06:49.453554+0000 (oldest deadline 2024-03-19T02:07:11.753578+0000) 2024-03-19T02:07:21.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:20 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:20.849+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170251+0000 front 2024-03-19T02:06:34.170405+0000 (oldest deadline 2024-03-19T02:06:58.270386+0000) 2024-03-19T02:07:21.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:20 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:20.849+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.270741+0000 front 2024-03-19T02:06:38.270719+0000 (oldest deadline 2024-03-19T02:07:04.170884+0000) 2024-03-19T02:07:21.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:20 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:20.849+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:47.072167+0000 front 2024-03-19T02:06:47.072106+0000 (oldest deadline 2024-03-19T02:07:12.372271+0000) 2024-03-19T02:07:21.556 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:21 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:21.122+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:41.493849+0000 front 2024-03-19T02:06:41.493734+0000 (oldest deadline 2024-03-19T02:07:03.793726+0000) 2024-03-19T02:07:21.556 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:21 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:21.122+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.996743+0000 front 2024-03-19T02:06:49.996766+0000 (oldest deadline 2024-03-19T02:07:14.096879+0000) 2024-03-19T02:07:21.556 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:21 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:21.122+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:54.097324+0000 front 2024-03-19T02:06:54.097227+0000 (oldest deadline 2024-03-19T02:07:19.997301+0000) 2024-03-19T02:07:21.590 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:21 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:21.180+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:31.579664+0000 front 2024-03-19T02:06:31.579687+0000 (oldest deadline 2024-03-19T02:06:57.479977+0000) 2024-03-19T02:07:21.590 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:21 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:21.180+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.980985+0000 front 2024-03-19T02:06:40.981003+0000 (oldest deadline 2024-03-19T02:07:03.281058+0000) 2024-03-19T02:07:21.590 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:21 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:21.180+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.182086+0000 front 2024-03-19T02:06:46.182039+0000 (oldest deadline 2024-03-19T02:07:12.082249+0000) 2024-03-19T02:07:21.591 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:21 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:21.180+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:58.484036+0000 front 2024-03-19T02:06:58.484064+0000 (oldest deadline 2024-03-19T02:07:20.784064+0000) 2024-03-19T02:07:22.056 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:21 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:21.598+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:22.056 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:21 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:21.598+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:07:22.056 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:21 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:21.598+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.116306+0000 front 2024-03-19T02:06:46.116324+0000 (oldest deadline 2024-03-19T02:07:12.016484+0000) 2024-03-19T02:07:22.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:21 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:21.819+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170696+0000 front 2024-03-19T02:06:34.170736+0000 (oldest deadline 2024-03-19T02:06:55.870953+0000) 2024-03-19T02:07:22.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:21 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:21.819+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.572097+0000 front 2024-03-19T02:06:40.571972+0000 (oldest deadline 2024-03-19T02:07:06.372675+0000) 2024-03-19T02:07:22.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:21 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:21.819+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.675305+0000 front 2024-03-19T02:06:49.675286+0000 (oldest deadline 2024-03-19T02:07:11.975395+0000) 2024-03-19T02:07:22.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:21 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:21.819+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.276447+0000 front 2024-03-19T02:06:57.276673+0000 (oldest deadline 2024-03-19T02:07:20.176691+0000) 2024-03-19T02:07:22.112 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:21 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:21.812+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.951647+0000 front 2024-03-19T02:06:38.951558+0000 (oldest deadline 2024-03-19T02:07:03.651603+0000) 2024-03-19T02:07:22.112 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:21 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:21.812+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.453491+0000 front 2024-03-19T02:06:49.453554+0000 (oldest deadline 2024-03-19T02:07:11.753578+0000) 2024-03-19T02:07:22.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:21 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:21.818+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170251+0000 front 2024-03-19T02:06:34.170405+0000 (oldest deadline 2024-03-19T02:06:58.270386+0000) 2024-03-19T02:07:22.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:21 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:21.818+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.270741+0000 front 2024-03-19T02:06:38.270719+0000 (oldest deadline 2024-03-19T02:07:04.170884+0000) 2024-03-19T02:07:22.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:21 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:21.818+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:47.072167+0000 front 2024-03-19T02:06:47.072106+0000 (oldest deadline 2024-03-19T02:07:12.372271+0000) 2024-03-19T02:07:22.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:21 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:21.818+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.574305+0000 front 2024-03-19T02:06:57.574422+0000 (oldest deadline 2024-03-19T02:07:20.974748+0000) 2024-03-19T02:07:22.555 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:22 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:22.079+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:41.493849+0000 front 2024-03-19T02:06:41.493734+0000 (oldest deadline 2024-03-19T02:07:03.793726+0000) 2024-03-19T02:07:22.555 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:22 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:22.079+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.996743+0000 front 2024-03-19T02:06:49.996766+0000 (oldest deadline 2024-03-19T02:07:14.096879+0000) 2024-03-19T02:07:22.556 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:22 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:22.079+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:54.097324+0000 front 2024-03-19T02:06:54.097227+0000 (oldest deadline 2024-03-19T02:07:19.997301+0000) 2024-03-19T02:07:22.590 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:22 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:22.205+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:31.579664+0000 front 2024-03-19T02:06:31.579687+0000 (oldest deadline 2024-03-19T02:06:57.479977+0000) 2024-03-19T02:07:22.590 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:22 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:22.205+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.980985+0000 front 2024-03-19T02:06:40.981003+0000 (oldest deadline 2024-03-19T02:07:03.281058+0000) 2024-03-19T02:07:22.591 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:22 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:22.205+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.182086+0000 front 2024-03-19T02:06:46.182039+0000 (oldest deadline 2024-03-19T02:07:12.082249+0000) 2024-03-19T02:07:22.591 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:22 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6[91342]: 2024-03-19T02:07:22.205+0000 7f6b85e6d700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:58.484036+0000 front 2024-03-19T02:06:58.484064+0000 (oldest deadline 2024-03-19T02:07:20.784064+0000) 2024-03-19T02:07:22.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:22 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:22.554+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:22.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:22 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:22.554+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:07:22.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:22 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:22.554+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.116306+0000 front 2024-03-19T02:06:46.116324+0000 (oldest deadline 2024-03-19T02:07:12.016484+0000) 2024-03-19T02:07:23.090 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:22 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:22.783+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170696+0000 front 2024-03-19T02:06:34.170736+0000 (oldest deadline 2024-03-19T02:06:55.870953+0000) 2024-03-19T02:07:23.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:22 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:22.783+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.572097+0000 front 2024-03-19T02:06:40.571972+0000 (oldest deadline 2024-03-19T02:07:06.372675+0000) 2024-03-19T02:07:23.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:22 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:22.783+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.675305+0000 front 2024-03-19T02:06:49.675286+0000 (oldest deadline 2024-03-19T02:07:11.975395+0000) 2024-03-19T02:07:23.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:22 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:22.783+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.276447+0000 front 2024-03-19T02:06:57.276673+0000 (oldest deadline 2024-03-19T02:07:20.176691+0000) 2024-03-19T02:07:23.112 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:22 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:22.761+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.951647+0000 front 2024-03-19T02:06:38.951558+0000 (oldest deadline 2024-03-19T02:07:03.651603+0000) 2024-03-19T02:07:23.112 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:22 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:22.761+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.453491+0000 front 2024-03-19T02:06:49.453554+0000 (oldest deadline 2024-03-19T02:07:11.753578+0000) 2024-03-19T02:07:23.112 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:22 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:22.761+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:53.454959+0000 front 2024-03-19T02:06:53.454882+0000 (oldest deadline 2024-03-19T02:07:22.255438+0000) 2024-03-19T02:07:23.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:22 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:22.848+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170251+0000 front 2024-03-19T02:06:34.170405+0000 (oldest deadline 2024-03-19T02:06:58.270386+0000) 2024-03-19T02:07:23.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:22 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:22.848+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.270741+0000 front 2024-03-19T02:06:38.270719+0000 (oldest deadline 2024-03-19T02:07:04.170884+0000) 2024-03-19T02:07:23.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:22 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:22.848+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:47.072167+0000 front 2024-03-19T02:06:47.072106+0000 (oldest deadline 2024-03-19T02:07:12.372271+0000) 2024-03-19T02:07:23.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:22 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:22.848+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.574305+0000 front 2024-03-19T02:06:57.574422+0000 (oldest deadline 2024-03-19T02:07:20.974748+0000) 2024-03-19T02:07:23.306 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:23 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:23.048+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:41.493849+0000 front 2024-03-19T02:06:41.493734+0000 (oldest deadline 2024-03-19T02:07:03.793726+0000) 2024-03-19T02:07:23.306 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:23 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:23.048+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.996743+0000 front 2024-03-19T02:06:49.996766+0000 (oldest deadline 2024-03-19T02:07:14.096879+0000) 2024-03-19T02:07:23.307 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:23 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:23.048+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:54.097324+0000 front 2024-03-19T02:06:54.097227+0000 (oldest deadline 2024-03-19T02:07:19.997301+0000) 2024-03-19T02:07:23.373 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:23 smithi099 podman[127274]: 2024-03-19 02:07:23.232398345 +0000 UTC m=+5.613791546 container died 5bc2c043b8ab9abf1b5d40c649af4fef44f6098db6705209f77feec144238e94 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240307) 2024-03-19T02:07:23.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:23 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:23.524+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:23.807 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:23 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:23.524+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:07:23.807 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:23 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:23.524+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.116306+0000 front 2024-03-19T02:06:46.116324+0000 (oldest deadline 2024-03-19T02:07:12.016484+0000) 2024-03-19T02:07:24.091 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:23 smithi099 podman[127274]: 2024-03-19 02:07:23.627104514 +0000 UTC m=+6.008497712 container cleanup 5bc2c043b8ab9abf1b5d40c649af4fef44f6098db6705209f77feec144238e94 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6, GIT_CLEAN=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240307) 2024-03-19T02:07:24.091 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:23 smithi099 bash[127274]: ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6 2024-03-19T02:07:24.091 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:23 smithi099 podman[127292]: 2024-03-19 02:07:23.845095387 +0000 UTC m=+0.610705249 container remove 5bc2c043b8ab9abf1b5d40c649af4fef44f6098db6705209f77feec144238e94 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.build-date=20240307, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.license=GPLv2, RELEASE=wip-batrick-testing-20240318.181317-b6ae249) 2024-03-19T02:07:24.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:23 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:23.763+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170696+0000 front 2024-03-19T02:06:34.170736+0000 (oldest deadline 2024-03-19T02:06:55.870953+0000) 2024-03-19T02:07:24.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:23 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:23.763+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.572097+0000 front 2024-03-19T02:06:40.571972+0000 (oldest deadline 2024-03-19T02:07:06.372675+0000) 2024-03-19T02:07:24.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:23 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:23.763+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.675305+0000 front 2024-03-19T02:06:49.675286+0000 (oldest deadline 2024-03-19T02:07:11.975395+0000) 2024-03-19T02:07:24.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:23 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:23.763+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.276447+0000 front 2024-03-19T02:06:57.276673+0000 (oldest deadline 2024-03-19T02:07:20.176691+0000) 2024-03-19T02:07:24.112 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:23 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:23.764+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.951647+0000 front 2024-03-19T02:06:38.951558+0000 (oldest deadline 2024-03-19T02:07:03.651603+0000) 2024-03-19T02:07:24.112 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:23 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:23.764+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.453491+0000 front 2024-03-19T02:06:49.453554+0000 (oldest deadline 2024-03-19T02:07:11.753578+0000) 2024-03-19T02:07:24.112 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:23 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:23.764+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:53.454959+0000 front 2024-03-19T02:06:53.454882+0000 (oldest deadline 2024-03-19T02:07:22.255438+0000) 2024-03-19T02:07:24.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:23 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:23.889+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170251+0000 front 2024-03-19T02:06:34.170405+0000 (oldest deadline 2024-03-19T02:06:58.270386+0000) 2024-03-19T02:07:24.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:23 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:23.889+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.270741+0000 front 2024-03-19T02:06:38.270719+0000 (oldest deadline 2024-03-19T02:07:04.170884+0000) 2024-03-19T02:07:24.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:23 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:23.889+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:47.072167+0000 front 2024-03-19T02:06:47.072106+0000 (oldest deadline 2024-03-19T02:07:12.372271+0000) 2024-03-19T02:07:24.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:23 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:23.889+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.574305+0000 front 2024-03-19T02:06:57.574422+0000 (oldest deadline 2024-03-19T02:07:20.974748+0000) 2024-03-19T02:07:24.522 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:24 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:24.091+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:41.493849+0000 front 2024-03-19T02:06:41.493734+0000 (oldest deadline 2024-03-19T02:07:03.793726+0000) 2024-03-19T02:07:24.522 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:24 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:24.091+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.996743+0000 front 2024-03-19T02:06:49.996766+0000 (oldest deadline 2024-03-19T02:07:14.096879+0000) 2024-03-19T02:07:24.522 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:24 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:24.091+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:54.097324+0000 front 2024-03-19T02:06:54.097227+0000 (oldest deadline 2024-03-19T02:07:19.997301+0000) 2024-03-19T02:07:24.522 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:24 smithi099 podman[127394]: 2024-03-19 02:07:24.104654366 +0000 UTC m=+0.017855852 image pull 2312c8e42d0069a2dbb2364bddb00264db4b800e7f18a7e7e4eb8f3ecab2ff1c quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639 2024-03-19T02:07:24.522 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:24 smithi099 podman[127394]: 2024-03-19 02:07:24.204212159 +0000 UTC m=+0.117413617 container create faae30b0e5f88fca8b61097982d630a5899e7c90dfdd7da53d940e5851ba2852 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6-deactivate, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240307, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-03-19T02:07:24.522 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:24 smithi099 podman[127394]: 2024-03-19 02:07:24.44020875 +0000 UTC m=+0.353410225 container init faae30b0e5f88fca8b61097982d630a5899e7c90dfdd7da53d940e5851ba2852 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6-deactivate, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240307, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5) 2024-03-19T02:07:24.522 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:24 smithi099 podman[127394]: 2024-03-19 02:07:24.443558688 +0000 UTC m=+0.356760154 container start faae30b0e5f88fca8b61097982d630a5899e7c90dfdd7da53d940e5851ba2852 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6-deactivate, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.build-date=20240307, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, ceph=True) 2024-03-19T02:07:24.523 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:24 smithi099 podman[127394]: 2024-03-19 02:07:24.523294593 +0000 UTC m=+0.436496070 container attach faae30b0e5f88fca8b61097982d630a5899e7c90dfdd7da53d940e5851ba2852 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6-deactivate, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.build-date=20240307, io.buildah.version=1.33.5, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True) 2024-03-19T02:07:24.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:24 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:24.522+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:24.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:24 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:24.522+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:07:24.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:24 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:24.522+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.116306+0000 front 2024-03-19T02:06:46.116324+0000 (oldest deadline 2024-03-19T02:07:12.016484+0000) 2024-03-19T02:07:24.840 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 19 02:07:24 smithi099 podman[127394]: 2024-03-19 02:07:24.737340395 +0000 UTC m=+0.650541859 container died faae30b0e5f88fca8b61097982d630a5899e7c90dfdd7da53d940e5851ba2852 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-6-deactivate, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240307) 2024-03-19T02:07:24.841 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:24 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:24.734+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170696+0000 front 2024-03-19T02:06:34.170736+0000 (oldest deadline 2024-03-19T02:06:55.870953+0000) 2024-03-19T02:07:24.841 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:24 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:24.734+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.572097+0000 front 2024-03-19T02:06:40.571972+0000 (oldest deadline 2024-03-19T02:07:06.372675+0000) 2024-03-19T02:07:24.841 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:24 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:24.734+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.675305+0000 front 2024-03-19T02:06:49.675286+0000 (oldest deadline 2024-03-19T02:07:11.975395+0000) 2024-03-19T02:07:24.841 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:24 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:24.734+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.276447+0000 front 2024-03-19T02:06:57.276673+0000 (oldest deadline 2024-03-19T02:07:20.176691+0000) 2024-03-19T02:07:25.112 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:24 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:24.718+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.951647+0000 front 2024-03-19T02:06:38.951558+0000 (oldest deadline 2024-03-19T02:07:03.651603+0000) 2024-03-19T02:07:25.112 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:24 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:24.718+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.453491+0000 front 2024-03-19T02:06:49.453554+0000 (oldest deadline 2024-03-19T02:07:11.753578+0000) 2024-03-19T02:07:25.112 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:24 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:24.718+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:53.454959+0000 front 2024-03-19T02:06:53.454882+0000 (oldest deadline 2024-03-19T02:07:22.255438+0000) 2024-03-19T02:07:25.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:24 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:24.869+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170251+0000 front 2024-03-19T02:06:34.170405+0000 (oldest deadline 2024-03-19T02:06:58.270386+0000) 2024-03-19T02:07:25.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:24 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:24.869+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.270741+0000 front 2024-03-19T02:06:38.270719+0000 (oldest deadline 2024-03-19T02:07:04.170884+0000) 2024-03-19T02:07:25.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:24 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:24.869+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:47.072167+0000 front 2024-03-19T02:06:47.072106+0000 (oldest deadline 2024-03-19T02:07:12.372271+0000) 2024-03-19T02:07:25.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:24 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:24.869+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.574305+0000 front 2024-03-19T02:06:57.574422+0000 (oldest deadline 2024-03-19T02:07:20.974748+0000) 2024-03-19T02:07:25.379 DEBUG:teuthology.orchestra.run.smithi099:> sudo pkill -f 'journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@osd.6.service' 2024-03-19T02:07:25.421 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-19T02:07:25.421 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-03-19T02:07:25.421 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-03-19T02:07:25.421 DEBUG:teuthology.orchestra.run.smithi112:> sudo systemctl stop ceph-438fb69c-e590-11ee-95c9-87774f69a715@osd.7 2024-03-19T02:07:25.505 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:25 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:25.077+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:41.493849+0000 front 2024-03-19T02:06:41.493734+0000 (oldest deadline 2024-03-19T02:07:03.793726+0000) 2024-03-19T02:07:25.505 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:25 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:25.077+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.996743+0000 front 2024-03-19T02:06:49.996766+0000 (oldest deadline 2024-03-19T02:07:14.096879+0000) 2024-03-19T02:07:25.505 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:25 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:25.077+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:54.097324+0000 front 2024-03-19T02:06:54.097227+0000 (oldest deadline 2024-03-19T02:07:19.997301+0000) 2024-03-19T02:07:25.722 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:25 smithi112 systemd[1]: Stopping Ceph osd.7 for 438fb69c-e590-11ee-95c9-87774f69a715... 2024-03-19T02:07:25.722 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:25 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:25.698+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.951647+0000 front 2024-03-19T02:06:38.951558+0000 (oldest deadline 2024-03-19T02:07:03.651603+0000) 2024-03-19T02:07:25.722 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:25 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:25.698+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.453491+0000 front 2024-03-19T02:06:49.453554+0000 (oldest deadline 2024-03-19T02:07:11.753578+0000) 2024-03-19T02:07:25.722 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:25 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:25.698+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:53.454959+0000 front 2024-03-19T02:06:53.454882+0000 (oldest deadline 2024-03-19T02:07:22.255438+0000) 2024-03-19T02:07:25.722 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:25 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:25.721+0000 7f02a1a9f700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-19T02:07:25.722 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:25 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:25.721+0000 7f02a1a9f700 -1 osd.7 95 *** Got signal Terminated *** 2024-03-19T02:07:25.723 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:25 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:25.721+0000 7f02a1a9f700 -1 osd.7 95 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-19T02:07:25.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:25 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:25.505+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:25.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:25 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:25.505+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:07:25.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:25 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:25.505+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.116306+0000 front 2024-03-19T02:06:46.116324+0000 (oldest deadline 2024-03-19T02:07:12.016484+0000) 2024-03-19T02:07:26.090 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:25 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:25.713+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170696+0000 front 2024-03-19T02:06:34.170736+0000 (oldest deadline 2024-03-19T02:06:55.870953+0000) 2024-03-19T02:07:26.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:25 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:25.713+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.572097+0000 front 2024-03-19T02:06:40.571972+0000 (oldest deadline 2024-03-19T02:07:06.372675+0000) 2024-03-19T02:07:26.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:25 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:25.713+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.675305+0000 front 2024-03-19T02:06:49.675286+0000 (oldest deadline 2024-03-19T02:07:11.975395+0000) 2024-03-19T02:07:26.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:25 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:25.713+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.276447+0000 front 2024-03-19T02:06:57.276673+0000 (oldest deadline 2024-03-19T02:07:20.176691+0000) 2024-03-19T02:07:26.111 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:25 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:25.904+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170251+0000 front 2024-03-19T02:06:34.170405+0000 (oldest deadline 2024-03-19T02:06:58.270386+0000) 2024-03-19T02:07:26.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:25 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:25.904+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.270741+0000 front 2024-03-19T02:06:38.270719+0000 (oldest deadline 2024-03-19T02:07:04.170884+0000) 2024-03-19T02:07:26.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:25 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:25.904+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:47.072167+0000 front 2024-03-19T02:06:47.072106+0000 (oldest deadline 2024-03-19T02:07:12.372271+0000) 2024-03-19T02:07:26.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:25 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:25.904+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.574305+0000 front 2024-03-19T02:06:57.574422+0000 (oldest deadline 2024-03-19T02:07:20.974748+0000) 2024-03-19T02:07:26.484 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:26 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:26.124+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:41.493849+0000 front 2024-03-19T02:06:41.493734+0000 (oldest deadline 2024-03-19T02:07:03.793726+0000) 2024-03-19T02:07:26.484 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:26 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:26.124+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.996743+0000 front 2024-03-19T02:06:49.996766+0000 (oldest deadline 2024-03-19T02:07:14.096879+0000) 2024-03-19T02:07:26.484 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:26 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:26.124+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:54.097324+0000 front 2024-03-19T02:06:54.097227+0000 (oldest deadline 2024-03-19T02:07:19.997301+0000) 2024-03-19T02:07:26.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:26 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:26.484+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:26.811 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:26 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:26.484+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:07:26.811 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:26 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:26.484+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.116306+0000 front 2024-03-19T02:06:46.116324+0000 (oldest deadline 2024-03-19T02:07:12.016484+0000) 2024-03-19T02:07:27.090 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:26 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:26.723+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170696+0000 front 2024-03-19T02:06:34.170736+0000 (oldest deadline 2024-03-19T02:06:55.870953+0000) 2024-03-19T02:07:27.090 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:26 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:26.723+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.572097+0000 front 2024-03-19T02:06:40.571972+0000 (oldest deadline 2024-03-19T02:07:06.372675+0000) 2024-03-19T02:07:27.090 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:26 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:26.723+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.675305+0000 front 2024-03-19T02:06:49.675286+0000 (oldest deadline 2024-03-19T02:07:11.975395+0000) 2024-03-19T02:07:27.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:26 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:26.723+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.276447+0000 front 2024-03-19T02:06:57.276673+0000 (oldest deadline 2024-03-19T02:07:20.176691+0000) 2024-03-19T02:07:27.111 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:26 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:26.724+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.951647+0000 front 2024-03-19T02:06:38.951558+0000 (oldest deadline 2024-03-19T02:07:03.651603+0000) 2024-03-19T02:07:27.112 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:26 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:26.724+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.453491+0000 front 2024-03-19T02:06:49.453554+0000 (oldest deadline 2024-03-19T02:07:11.753578+0000) 2024-03-19T02:07:27.112 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:26 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:26.724+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:53.454959+0000 front 2024-03-19T02:06:53.454882+0000 (oldest deadline 2024-03-19T02:07:22.255438+0000) 2024-03-19T02:07:27.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:26 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:26.886+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170251+0000 front 2024-03-19T02:06:34.170405+0000 (oldest deadline 2024-03-19T02:06:58.270386+0000) 2024-03-19T02:07:27.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:26 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:26.886+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.270741+0000 front 2024-03-19T02:06:38.270719+0000 (oldest deadline 2024-03-19T02:07:04.170884+0000) 2024-03-19T02:07:27.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:26 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:26.886+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:47.072167+0000 front 2024-03-19T02:06:47.072106+0000 (oldest deadline 2024-03-19T02:07:12.372271+0000) 2024-03-19T02:07:27.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:26 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:26.886+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.574305+0000 front 2024-03-19T02:06:57.574422+0000 (oldest deadline 2024-03-19T02:07:20.974748+0000) 2024-03-19T02:07:27.455 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:27 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:27.078+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:41.493849+0000 front 2024-03-19T02:06:41.493734+0000 (oldest deadline 2024-03-19T02:07:03.793726+0000) 2024-03-19T02:07:27.456 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:27 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:27.078+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.996743+0000 front 2024-03-19T02:06:49.996766+0000 (oldest deadline 2024-03-19T02:07:14.096879+0000) 2024-03-19T02:07:27.456 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:27 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:27.078+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:54.097324+0000 front 2024-03-19T02:06:54.097227+0000 (oldest deadline 2024-03-19T02:07:19.997301+0000) 2024-03-19T02:07:27.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:27 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:27.456+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:27.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:27 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:27.456+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:07:27.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:27 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:27.456+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.116306+0000 front 2024-03-19T02:06:46.116324+0000 (oldest deadline 2024-03-19T02:07:12.016484+0000) 2024-03-19T02:07:28.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:27 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:27.738+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170696+0000 front 2024-03-19T02:06:34.170736+0000 (oldest deadline 2024-03-19T02:06:55.870953+0000) 2024-03-19T02:07:28.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:27 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:27.738+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.572097+0000 front 2024-03-19T02:06:40.571972+0000 (oldest deadline 2024-03-19T02:07:06.372675+0000) 2024-03-19T02:07:28.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:27 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:27.738+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.675305+0000 front 2024-03-19T02:06:49.675286+0000 (oldest deadline 2024-03-19T02:07:11.975395+0000) 2024-03-19T02:07:28.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:27 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:27.738+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.276447+0000 front 2024-03-19T02:06:57.276673+0000 (oldest deadline 2024-03-19T02:07:20.176691+0000) 2024-03-19T02:07:28.112 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:27 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:27.691+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.951647+0000 front 2024-03-19T02:06:38.951558+0000 (oldest deadline 2024-03-19T02:07:03.651603+0000) 2024-03-19T02:07:28.112 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:27 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:27.691+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.453491+0000 front 2024-03-19T02:06:49.453554+0000 (oldest deadline 2024-03-19T02:07:11.753578+0000) 2024-03-19T02:07:28.112 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:27 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:27.691+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:53.454959+0000 front 2024-03-19T02:06:53.454882+0000 (oldest deadline 2024-03-19T02:07:22.255438+0000) 2024-03-19T02:07:28.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:27 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:27.893+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170251+0000 front 2024-03-19T02:06:34.170405+0000 (oldest deadline 2024-03-19T02:06:58.270386+0000) 2024-03-19T02:07:28.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:27 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:27.893+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.270741+0000 front 2024-03-19T02:06:38.270719+0000 (oldest deadline 2024-03-19T02:07:04.170884+0000) 2024-03-19T02:07:28.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:27 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:27.893+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:47.072167+0000 front 2024-03-19T02:06:47.072106+0000 (oldest deadline 2024-03-19T02:07:12.372271+0000) 2024-03-19T02:07:28.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:27 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:27.893+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.574305+0000 front 2024-03-19T02:06:57.574422+0000 (oldest deadline 2024-03-19T02:07:20.974748+0000) 2024-03-19T02:07:28.306 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:28 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:28.049+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:41.493849+0000 front 2024-03-19T02:06:41.493734+0000 (oldest deadline 2024-03-19T02:07:03.793726+0000) 2024-03-19T02:07:28.306 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:28 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:28.049+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.996743+0000 front 2024-03-19T02:06:49.996766+0000 (oldest deadline 2024-03-19T02:07:14.096879+0000) 2024-03-19T02:07:28.306 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:28 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:28.049+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:54.097324+0000 front 2024-03-19T02:06:54.097227+0000 (oldest deadline 2024-03-19T02:07:19.997301+0000) 2024-03-19T02:07:28.306 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:28 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:28.049+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:05.799241+0000 front 2024-03-19T02:07:05.799447+0000 (oldest deadline 2024-03-19T02:07:27.499223+0000) 2024-03-19T02:07:28.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:28 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:28.467+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:28.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:28 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:28.467+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:07:28.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:28 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:28.467+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.116306+0000 front 2024-03-19T02:06:46.116324+0000 (oldest deadline 2024-03-19T02:07:12.016484+0000) 2024-03-19T02:07:28.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:28 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:28.467+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.819160+0000 front 2024-03-19T02:07:04.819180+0000 (oldest deadline 2024-03-19T02:07:27.719305+0000) 2024-03-19T02:07:29.090 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:28 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:28.785+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170696+0000 front 2024-03-19T02:06:34.170736+0000 (oldest deadline 2024-03-19T02:06:55.870953+0000) 2024-03-19T02:07:29.090 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:28 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:28.785+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.572097+0000 front 2024-03-19T02:06:40.571972+0000 (oldest deadline 2024-03-19T02:07:06.372675+0000) 2024-03-19T02:07:29.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:28 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:28.785+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.675305+0000 front 2024-03-19T02:06:49.675286+0000 (oldest deadline 2024-03-19T02:07:11.975395+0000) 2024-03-19T02:07:29.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:28 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:28.785+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.276447+0000 front 2024-03-19T02:06:57.276673+0000 (oldest deadline 2024-03-19T02:07:20.176691+0000) 2024-03-19T02:07:29.111 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:28 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:28.887+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170251+0000 front 2024-03-19T02:06:34.170405+0000 (oldest deadline 2024-03-19T02:06:58.270386+0000) 2024-03-19T02:07:29.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:28 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:28.887+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.270741+0000 front 2024-03-19T02:06:38.270719+0000 (oldest deadline 2024-03-19T02:07:04.170884+0000) 2024-03-19T02:07:29.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:28 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:28.887+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:47.072167+0000 front 2024-03-19T02:06:47.072106+0000 (oldest deadline 2024-03-19T02:07:12.372271+0000) 2024-03-19T02:07:29.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:28 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:28.887+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.574305+0000 front 2024-03-19T02:06:57.574422+0000 (oldest deadline 2024-03-19T02:07:20.974748+0000) 2024-03-19T02:07:29.112 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:28 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:28.713+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.951647+0000 front 2024-03-19T02:06:38.951558+0000 (oldest deadline 2024-03-19T02:07:03.651603+0000) 2024-03-19T02:07:29.112 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:28 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:28.713+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.453491+0000 front 2024-03-19T02:06:49.453554+0000 (oldest deadline 2024-03-19T02:07:11.753578+0000) 2024-03-19T02:07:29.112 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:28 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:28.713+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:53.454959+0000 front 2024-03-19T02:06:53.454882+0000 (oldest deadline 2024-03-19T02:07:22.255438+0000) 2024-03-19T02:07:29.112 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:28 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:28.713+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:06.956349+0000 front 2024-03-19T02:07:06.956303+0000 (oldest deadline 2024-03-19T02:07:28.656302+0000) 2024-03-19T02:07:29.420 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:29 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:29.075+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:41.493849+0000 front 2024-03-19T02:06:41.493734+0000 (oldest deadline 2024-03-19T02:07:03.793726+0000) 2024-03-19T02:07:29.421 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:29 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:29.075+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.996743+0000 front 2024-03-19T02:06:49.996766+0000 (oldest deadline 2024-03-19T02:07:14.096879+0000) 2024-03-19T02:07:29.421 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:29 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:29.075+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:54.097324+0000 front 2024-03-19T02:06:54.097227+0000 (oldest deadline 2024-03-19T02:07:19.997301+0000) 2024-03-19T02:07:29.421 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:29 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:29.075+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:05.799241+0000 front 2024-03-19T02:07:05.799447+0000 (oldest deadline 2024-03-19T02:07:27.499223+0000) 2024-03-19T02:07:29.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:29 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:29.420+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:29.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:29 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:29.420+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:07:29.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:29 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:29.420+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.116306+0000 front 2024-03-19T02:06:46.116324+0000 (oldest deadline 2024-03-19T02:07:12.016484+0000) 2024-03-19T02:07:29.807 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:29 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:29.420+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.819160+0000 front 2024-03-19T02:07:04.819180+0000 (oldest deadline 2024-03-19T02:07:27.719305+0000) 2024-03-19T02:07:30.090 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:29 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:29.782+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170696+0000 front 2024-03-19T02:06:34.170736+0000 (oldest deadline 2024-03-19T02:06:55.870953+0000) 2024-03-19T02:07:30.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:29 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:29.782+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.572097+0000 front 2024-03-19T02:06:40.571972+0000 (oldest deadline 2024-03-19T02:07:06.372675+0000) 2024-03-19T02:07:30.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:29 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:29.782+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.675305+0000 front 2024-03-19T02:06:49.675286+0000 (oldest deadline 2024-03-19T02:07:11.975395+0000) 2024-03-19T02:07:30.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:29 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:29.782+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.276447+0000 front 2024-03-19T02:06:57.276673+0000 (oldest deadline 2024-03-19T02:07:20.176691+0000) 2024-03-19T02:07:30.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:29 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:29.782+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.778329+0000 front 2024-03-19T02:07:04.778433+0000 (oldest deadline 2024-03-19T02:07:29.478432+0000) 2024-03-19T02:07:30.112 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:29 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:29.688+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.951647+0000 front 2024-03-19T02:06:38.951558+0000 (oldest deadline 2024-03-19T02:07:03.651603+0000) 2024-03-19T02:07:30.112 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:29 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:29.688+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.453491+0000 front 2024-03-19T02:06:49.453554+0000 (oldest deadline 2024-03-19T02:07:11.753578+0000) 2024-03-19T02:07:30.112 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:29 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:29.688+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:53.454959+0000 front 2024-03-19T02:06:53.454882+0000 (oldest deadline 2024-03-19T02:07:22.255438+0000) 2024-03-19T02:07:30.112 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:29 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:29.688+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:06.956349+0000 front 2024-03-19T02:07:06.956303+0000 (oldest deadline 2024-03-19T02:07:28.656302+0000) 2024-03-19T02:07:30.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:29 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:29.846+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170251+0000 front 2024-03-19T02:06:34.170405+0000 (oldest deadline 2024-03-19T02:06:58.270386+0000) 2024-03-19T02:07:30.113 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:29 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:29.846+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.270741+0000 front 2024-03-19T02:06:38.270719+0000 (oldest deadline 2024-03-19T02:07:04.170884+0000) 2024-03-19T02:07:30.113 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:29 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:29.846+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:47.072167+0000 front 2024-03-19T02:06:47.072106+0000 (oldest deadline 2024-03-19T02:07:12.372271+0000) 2024-03-19T02:07:30.113 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:29 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:29.846+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.574305+0000 front 2024-03-19T02:06:57.574422+0000 (oldest deadline 2024-03-19T02:07:20.974748+0000) 2024-03-19T02:07:30.113 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:29 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:29.846+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.475657+0000 front 2024-03-19T02:07:04.475681+0000 (oldest deadline 2024-03-19T02:07:29.175780+0000) 2024-03-19T02:07:30.306 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:30 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:30.055+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:41.493849+0000 front 2024-03-19T02:06:41.493734+0000 (oldest deadline 2024-03-19T02:07:03.793726+0000) 2024-03-19T02:07:30.307 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:30 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:30.055+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.996743+0000 front 2024-03-19T02:06:49.996766+0000 (oldest deadline 2024-03-19T02:07:14.096879+0000) 2024-03-19T02:07:30.307 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:30 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:30.055+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:54.097324+0000 front 2024-03-19T02:06:54.097227+0000 (oldest deadline 2024-03-19T02:07:19.997301+0000) 2024-03-19T02:07:30.307 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:30 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:30.055+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:05.799241+0000 front 2024-03-19T02:07:05.799447+0000 (oldest deadline 2024-03-19T02:07:27.499223+0000) 2024-03-19T02:07:30.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:30 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:30.458+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:30.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:30 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:30.458+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:07:30.807 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:30 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:30.458+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.116306+0000 front 2024-03-19T02:06:46.116324+0000 (oldest deadline 2024-03-19T02:07:12.016484+0000) 2024-03-19T02:07:30.807 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:30 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:30.458+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.819160+0000 front 2024-03-19T02:07:04.819180+0000 (oldest deadline 2024-03-19T02:07:27.719305+0000) 2024-03-19T02:07:31.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:30 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:30.798+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170696+0000 front 2024-03-19T02:06:34.170736+0000 (oldest deadline 2024-03-19T02:06:55.870953+0000) 2024-03-19T02:07:31.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:30 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:30.798+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.572097+0000 front 2024-03-19T02:06:40.571972+0000 (oldest deadline 2024-03-19T02:07:06.372675+0000) 2024-03-19T02:07:31.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:30 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:30.798+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.675305+0000 front 2024-03-19T02:06:49.675286+0000 (oldest deadline 2024-03-19T02:07:11.975395+0000) 2024-03-19T02:07:31.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:30 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:30.798+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.276447+0000 front 2024-03-19T02:06:57.276673+0000 (oldest deadline 2024-03-19T02:07:20.176691+0000) 2024-03-19T02:07:31.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:30 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:30.798+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.778329+0000 front 2024-03-19T02:07:04.778433+0000 (oldest deadline 2024-03-19T02:07:29.478432+0000) 2024-03-19T02:07:31.112 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:30 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:30.666+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.951647+0000 front 2024-03-19T02:06:38.951558+0000 (oldest deadline 2024-03-19T02:07:03.651603+0000) 2024-03-19T02:07:31.112 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:30 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:30.666+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.453491+0000 front 2024-03-19T02:06:49.453554+0000 (oldest deadline 2024-03-19T02:07:11.753578+0000) 2024-03-19T02:07:31.112 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:30 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:30.666+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:53.454959+0000 front 2024-03-19T02:06:53.454882+0000 (oldest deadline 2024-03-19T02:07:22.255438+0000) 2024-03-19T02:07:31.112 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:30 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7[84975]: 2024-03-19T02:07:30.666+0000 7f029a08b700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:06.956349+0000 front 2024-03-19T02:07:06.956303+0000 (oldest deadline 2024-03-19T02:07:28.656302+0000) 2024-03-19T02:07:31.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:30 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:30.875+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170251+0000 front 2024-03-19T02:06:34.170405+0000 (oldest deadline 2024-03-19T02:06:58.270386+0000) 2024-03-19T02:07:31.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:30 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:30.875+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.270741+0000 front 2024-03-19T02:06:38.270719+0000 (oldest deadline 2024-03-19T02:07:04.170884+0000) 2024-03-19T02:07:31.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:30 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:30.875+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:47.072167+0000 front 2024-03-19T02:06:47.072106+0000 (oldest deadline 2024-03-19T02:07:12.372271+0000) 2024-03-19T02:07:31.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:30 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:30.875+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.574305+0000 front 2024-03-19T02:06:57.574422+0000 (oldest deadline 2024-03-19T02:07:20.974748+0000) 2024-03-19T02:07:31.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:30 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:30.875+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.475657+0000 front 2024-03-19T02:07:04.475681+0000 (oldest deadline 2024-03-19T02:07:29.175780+0000) 2024-03-19T02:07:31.306 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:31 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:31.055+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:41.493849+0000 front 2024-03-19T02:06:41.493734+0000 (oldest deadline 2024-03-19T02:07:03.793726+0000) 2024-03-19T02:07:31.306 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:31 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:31.055+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.996743+0000 front 2024-03-19T02:06:49.996766+0000 (oldest deadline 2024-03-19T02:07:14.096879+0000) 2024-03-19T02:07:31.306 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:31 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:31.055+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:54.097324+0000 front 2024-03-19T02:06:54.097227+0000 (oldest deadline 2024-03-19T02:07:19.997301+0000) 2024-03-19T02:07:31.307 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:31 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:31.055+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:05.799241+0000 front 2024-03-19T02:07:05.799447+0000 (oldest deadline 2024-03-19T02:07:27.499223+0000) 2024-03-19T02:07:31.598 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:31 smithi112 podman[106902]: 2024-03-19 02:07:31.334978911 +0000 UTC m=+5.722541645 container died e99f85ee12abf716a79154128951e220f6f4efb9585232bacd9b4155ff9ec980 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.build-date=20240307, maintainer=Guillaume Abrioux , RELEASE=wip-batrick-testing-20240318.181317-b6ae249) 2024-03-19T02:07:31.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:31 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:31.469+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:31.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:31 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:31.469+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:07:31.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:31 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:31.469+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.116306+0000 front 2024-03-19T02:06:46.116324+0000 (oldest deadline 2024-03-19T02:07:12.016484+0000) 2024-03-19T02:07:31.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:31 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:31.469+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.819160+0000 front 2024-03-19T02:07:04.819180+0000 (oldest deadline 2024-03-19T02:07:27.719305+0000) 2024-03-19T02:07:31.861 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:31 smithi112 podman[106902]: 2024-03-19 02:07:31.689348989 +0000 UTC m=+6.076911721 container cleanup e99f85ee12abf716a79154128951e220f6f4efb9585232bacd9b4155ff9ec980 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.build-date=20240307, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-19T02:07:31.861 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:31 smithi112 bash[106902]: ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7 2024-03-19T02:07:32.090 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:31 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:31.816+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170696+0000 front 2024-03-19T02:06:34.170736+0000 (oldest deadline 2024-03-19T02:06:55.870953+0000) 2024-03-19T02:07:32.090 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:31 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:31.816+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.572097+0000 front 2024-03-19T02:06:40.571972+0000 (oldest deadline 2024-03-19T02:07:06.372675+0000) 2024-03-19T02:07:32.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:31 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:31.816+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.675305+0000 front 2024-03-19T02:06:49.675286+0000 (oldest deadline 2024-03-19T02:07:11.975395+0000) 2024-03-19T02:07:32.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:31 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:31.816+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.276447+0000 front 2024-03-19T02:06:57.276673+0000 (oldest deadline 2024-03-19T02:07:20.176691+0000) 2024-03-19T02:07:32.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:31 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:31.816+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.778329+0000 front 2024-03-19T02:07:04.778433+0000 (oldest deadline 2024-03-19T02:07:29.478432+0000) 2024-03-19T02:07:32.262 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:31 smithi112 podman[106917]: 2024-03-19 02:07:31.965687522 +0000 UTC m=+0.629210354 container remove e99f85ee12abf716a79154128951e220f6f4efb9585232bacd9b4155ff9ec980 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.build-date=20240307, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, RELEASE=wip-batrick-testing-20240318.181317-b6ae249) 2024-03-19T02:07:32.262 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:31 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:31.890+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170251+0000 front 2024-03-19T02:06:34.170405+0000 (oldest deadline 2024-03-19T02:06:58.270386+0000) 2024-03-19T02:07:32.263 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:31 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:31.890+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.270741+0000 front 2024-03-19T02:06:38.270719+0000 (oldest deadline 2024-03-19T02:07:04.170884+0000) 2024-03-19T02:07:32.263 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:31 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:31.890+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:47.072167+0000 front 2024-03-19T02:06:47.072106+0000 (oldest deadline 2024-03-19T02:07:12.372271+0000) 2024-03-19T02:07:32.263 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:31 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:31.890+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.574305+0000 front 2024-03-19T02:06:57.574422+0000 (oldest deadline 2024-03-19T02:07:20.974748+0000) 2024-03-19T02:07:32.263 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:31 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:31.890+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.475657+0000 front 2024-03-19T02:07:04.475681+0000 (oldest deadline 2024-03-19T02:07:29.175780+0000) 2024-03-19T02:07:32.306 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:32 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:32.006+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:41.493849+0000 front 2024-03-19T02:06:41.493734+0000 (oldest deadline 2024-03-19T02:07:03.793726+0000) 2024-03-19T02:07:32.306 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:32 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:32.006+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.996743+0000 front 2024-03-19T02:06:49.996766+0000 (oldest deadline 2024-03-19T02:07:14.096879+0000) 2024-03-19T02:07:32.306 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:32 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:32.006+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:54.097324+0000 front 2024-03-19T02:06:54.097227+0000 (oldest deadline 2024-03-19T02:07:19.997301+0000) 2024-03-19T02:07:32.306 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:32 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:32.006+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:05.799241+0000 front 2024-03-19T02:07:05.799447+0000 (oldest deadline 2024-03-19T02:07:27.499223+0000) 2024-03-19T02:07:32.551 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:32 smithi112 podman[107009]: 2024-03-19 02:07:32.165379863 +0000 UTC m=+0.017051003 image pull 2312c8e42d0069a2dbb2364bddb00264db4b800e7f18a7e7e4eb8f3ecab2ff1c quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639 2024-03-19T02:07:32.551 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:32 smithi112 podman[107009]: 2024-03-19 02:07:32.266368734 +0000 UTC m=+0.118039852 container create d109e7002ef1e8e0cdc3109ef78140ae78df848cf83932ae4f82c1b5519a84ba (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7-deactivate, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240307, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-19T02:07:32.552 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:32 smithi112 podman[107009]: 2024-03-19 02:07:32.493832592 +0000 UTC m=+0.345503735 container init d109e7002ef1e8e0cdc3109ef78140ae78df848cf83932ae4f82c1b5519a84ba (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7-deactivate, org.label-schema.schema-version=1.0, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240307, maintainer=Guillaume Abrioux , GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30) 2024-03-19T02:07:32.552 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:32 smithi112 podman[107009]: 2024-03-19 02:07:32.497104214 +0000 UTC m=+0.348775339 container start d109e7002ef1e8e0cdc3109ef78140ae78df848cf83932ae4f82c1b5519a84ba (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7-deactivate, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240307, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30) 2024-03-19T02:07:32.552 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:32 smithi112 podman[107009]: 2024-03-19 02:07:32.551991259 +0000 UTC m=+0.403662411 container attach d109e7002ef1e8e0cdc3109ef78140ae78df848cf83932ae4f82c1b5519a84ba (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7-deactivate, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240307, CEPH_POINT_RELEASE=) 2024-03-19T02:07:32.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:32 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:32.431+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:32.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:32 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:32.431+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:07:32.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:32 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:32.431+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.116306+0000 front 2024-03-19T02:06:46.116324+0000 (oldest deadline 2024-03-19T02:07:12.016484+0000) 2024-03-19T02:07:32.807 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:32 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:32.431+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.819160+0000 front 2024-03-19T02:07:04.819180+0000 (oldest deadline 2024-03-19T02:07:27.719305+0000) 2024-03-19T02:07:32.862 INFO:journalctl@ceph.osd.7.smithi112.stdout:Mar 19 02:07:32 smithi112 podman[107009]: 2024-03-19 02:07:32.794768785 +0000 UTC m=+0.646439915 container died d109e7002ef1e8e0cdc3109ef78140ae78df848cf83932ae4f82c1b5519a84ba (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-7-deactivate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240307, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD) 2024-03-19T02:07:33.090 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:32 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:32.819+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170696+0000 front 2024-03-19T02:06:34.170736+0000 (oldest deadline 2024-03-19T02:06:55.870953+0000) 2024-03-19T02:07:33.090 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:32 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:32.819+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.572097+0000 front 2024-03-19T02:06:40.571972+0000 (oldest deadline 2024-03-19T02:07:06.372675+0000) 2024-03-19T02:07:33.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:32 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:32.819+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.675305+0000 front 2024-03-19T02:06:49.675286+0000 (oldest deadline 2024-03-19T02:07:11.975395+0000) 2024-03-19T02:07:33.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:32 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:32.819+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.276447+0000 front 2024-03-19T02:06:57.276673+0000 (oldest deadline 2024-03-19T02:07:20.176691+0000) 2024-03-19T02:07:33.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:32 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:32.819+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.778329+0000 front 2024-03-19T02:07:04.778433+0000 (oldest deadline 2024-03-19T02:07:29.478432+0000) 2024-03-19T02:07:33.306 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:32 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:32.965+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:41.493849+0000 front 2024-03-19T02:06:41.493734+0000 (oldest deadline 2024-03-19T02:07:03.793726+0000) 2024-03-19T02:07:33.306 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:32 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:32.965+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.996743+0000 front 2024-03-19T02:06:49.996766+0000 (oldest deadline 2024-03-19T02:07:14.096879+0000) 2024-03-19T02:07:33.306 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:32 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:32.965+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:54.097324+0000 front 2024-03-19T02:06:54.097227+0000 (oldest deadline 2024-03-19T02:07:19.997301+0000) 2024-03-19T02:07:33.307 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:32 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:32.965+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:05.799241+0000 front 2024-03-19T02:07:05.799447+0000 (oldest deadline 2024-03-19T02:07:27.499223+0000) 2024-03-19T02:07:33.330 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:32 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:32.890+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170251+0000 front 2024-03-19T02:06:34.170405+0000 (oldest deadline 2024-03-19T02:06:58.270386+0000) 2024-03-19T02:07:33.330 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:32 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:32.890+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.270741+0000 front 2024-03-19T02:06:38.270719+0000 (oldest deadline 2024-03-19T02:07:04.170884+0000) 2024-03-19T02:07:33.330 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:32 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:32.890+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:47.072167+0000 front 2024-03-19T02:06:47.072106+0000 (oldest deadline 2024-03-19T02:07:12.372271+0000) 2024-03-19T02:07:33.330 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:32 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:32.890+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.574305+0000 front 2024-03-19T02:06:57.574422+0000 (oldest deadline 2024-03-19T02:07:20.974748+0000) 2024-03-19T02:07:33.330 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:32 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:32.890+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.475657+0000 front 2024-03-19T02:07:04.475681+0000 (oldest deadline 2024-03-19T02:07:29.175780+0000) 2024-03-19T02:07:33.353 DEBUG:teuthology.orchestra.run.smithi112:> sudo pkill -f 'journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@osd.7.service' 2024-03-19T02:07:33.394 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-19T02:07:33.394 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-03-19T02:07:33.394 INFO:tasks.cephadm.osd.8:Stopping osd.8... 2024-03-19T02:07:33.394 DEBUG:teuthology.orchestra.run.smithi115:> sudo systemctl stop ceph-438fb69c-e590-11ee-95c9-87774f69a715@osd.8 2024-03-19T02:07:33.694 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:33 smithi115 systemd[1]: Stopping Ceph osd.8 for 438fb69c-e590-11ee-95c9-87774f69a715... 2024-03-19T02:07:33.694 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:33 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:33.694+0000 7f3e981fa700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.8 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-19T02:07:33.694 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:33 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:33.694+0000 7f3e981fa700 -1 osd.8 95 *** Got signal Terminated *** 2024-03-19T02:07:33.694 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:33 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:33.694+0000 7f3e981fa700 -1 osd.8 95 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-19T02:07:33.694 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:33 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:33.410+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:33.694 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:33 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:33.410+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:07:33.694 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:33 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:33.410+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.116306+0000 front 2024-03-19T02:06:46.116324+0000 (oldest deadline 2024-03-19T02:07:12.016484+0000) 2024-03-19T02:07:33.694 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:33 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:33.410+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.819160+0000 front 2024-03-19T02:07:04.819180+0000 (oldest deadline 2024-03-19T02:07:27.719305+0000) 2024-03-19T02:07:34.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:33 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:33.772+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170696+0000 front 2024-03-19T02:06:34.170736+0000 (oldest deadline 2024-03-19T02:06:55.870953+0000) 2024-03-19T02:07:34.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:33 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:33.772+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.572097+0000 front 2024-03-19T02:06:40.571972+0000 (oldest deadline 2024-03-19T02:07:06.372675+0000) 2024-03-19T02:07:34.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:33 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:33.772+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.675305+0000 front 2024-03-19T02:06:49.675286+0000 (oldest deadline 2024-03-19T02:07:11.975395+0000) 2024-03-19T02:07:34.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:33 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:33.772+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.276447+0000 front 2024-03-19T02:06:57.276673+0000 (oldest deadline 2024-03-19T02:07:20.176691+0000) 2024-03-19T02:07:34.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:33 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:33.772+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.778329+0000 front 2024-03-19T02:07:04.778433+0000 (oldest deadline 2024-03-19T02:07:29.478432+0000) 2024-03-19T02:07:34.306 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:34 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:34.004+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:41.493849+0000 front 2024-03-19T02:06:41.493734+0000 (oldest deadline 2024-03-19T02:07:03.793726+0000) 2024-03-19T02:07:34.306 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:34 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:34.004+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.996743+0000 front 2024-03-19T02:06:49.996766+0000 (oldest deadline 2024-03-19T02:07:14.096879+0000) 2024-03-19T02:07:34.306 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:34 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:34.004+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:54.097324+0000 front 2024-03-19T02:06:54.097227+0000 (oldest deadline 2024-03-19T02:07:19.997301+0000) 2024-03-19T02:07:34.307 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:34 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:34.004+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:05.799241+0000 front 2024-03-19T02:07:05.799447+0000 (oldest deadline 2024-03-19T02:07:27.499223+0000) 2024-03-19T02:07:34.362 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:33 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:33.892+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170251+0000 front 2024-03-19T02:06:34.170405+0000 (oldest deadline 2024-03-19T02:06:58.270386+0000) 2024-03-19T02:07:34.362 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:33 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:33.892+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.270741+0000 front 2024-03-19T02:06:38.270719+0000 (oldest deadline 2024-03-19T02:07:04.170884+0000) 2024-03-19T02:07:34.362 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:33 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:33.892+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:47.072167+0000 front 2024-03-19T02:06:47.072106+0000 (oldest deadline 2024-03-19T02:07:12.372271+0000) 2024-03-19T02:07:34.362 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:33 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:33.892+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.574305+0000 front 2024-03-19T02:06:57.574422+0000 (oldest deadline 2024-03-19T02:07:20.974748+0000) 2024-03-19T02:07:34.362 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:33 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:33.892+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.475657+0000 front 2024-03-19T02:07:04.475681+0000 (oldest deadline 2024-03-19T02:07:29.175780+0000) 2024-03-19T02:07:34.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:34 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:34.368+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:34.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:34 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:34.368+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:07:34.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:34 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:34.368+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.116306+0000 front 2024-03-19T02:06:46.116324+0000 (oldest deadline 2024-03-19T02:07:12.016484+0000) 2024-03-19T02:07:34.807 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:34 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:34.368+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.819160+0000 front 2024-03-19T02:07:04.819180+0000 (oldest deadline 2024-03-19T02:07:27.719305+0000) 2024-03-19T02:07:34.851 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:34 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:34.851+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170251+0000 front 2024-03-19T02:06:34.170405+0000 (oldest deadline 2024-03-19T02:06:58.270386+0000) 2024-03-19T02:07:34.851 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:34 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:34.851+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.270741+0000 front 2024-03-19T02:06:38.270719+0000 (oldest deadline 2024-03-19T02:07:04.170884+0000) 2024-03-19T02:07:35.090 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:34 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:34.774+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170696+0000 front 2024-03-19T02:06:34.170736+0000 (oldest deadline 2024-03-19T02:06:55.870953+0000) 2024-03-19T02:07:35.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:34 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:34.774+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.572097+0000 front 2024-03-19T02:06:40.571972+0000 (oldest deadline 2024-03-19T02:07:06.372675+0000) 2024-03-19T02:07:35.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:34 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:34.774+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.675305+0000 front 2024-03-19T02:06:49.675286+0000 (oldest deadline 2024-03-19T02:07:11.975395+0000) 2024-03-19T02:07:35.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:34 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:34.774+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.276447+0000 front 2024-03-19T02:06:57.276673+0000 (oldest deadline 2024-03-19T02:07:20.176691+0000) 2024-03-19T02:07:35.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:34 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:34.774+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.778329+0000 front 2024-03-19T02:07:04.778433+0000 (oldest deadline 2024-03-19T02:07:29.478432+0000) 2024-03-19T02:07:35.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:34 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:34.851+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:47.072167+0000 front 2024-03-19T02:06:47.072106+0000 (oldest deadline 2024-03-19T02:07:12.372271+0000) 2024-03-19T02:07:35.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:34 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:34.851+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.574305+0000 front 2024-03-19T02:06:57.574422+0000 (oldest deadline 2024-03-19T02:07:20.974748+0000) 2024-03-19T02:07:35.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:34 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:34.851+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.475657+0000 front 2024-03-19T02:07:04.475681+0000 (oldest deadline 2024-03-19T02:07:29.175780+0000) 2024-03-19T02:07:35.306 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:35 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:35.043+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:41.493849+0000 front 2024-03-19T02:06:41.493734+0000 (oldest deadline 2024-03-19T02:07:03.793726+0000) 2024-03-19T02:07:35.306 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:35 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:35.043+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.996743+0000 front 2024-03-19T02:06:49.996766+0000 (oldest deadline 2024-03-19T02:07:14.096879+0000) 2024-03-19T02:07:35.306 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:35 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:35.043+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:54.097324+0000 front 2024-03-19T02:06:54.097227+0000 (oldest deadline 2024-03-19T02:07:19.997301+0000) 2024-03-19T02:07:35.306 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:35 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:35.043+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:05.799241+0000 front 2024-03-19T02:07:05.799447+0000 (oldest deadline 2024-03-19T02:07:27.499223+0000) 2024-03-19T02:07:35.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:35 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:35.319+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:35.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:35 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:35.319+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:07:35.807 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:35 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:35.319+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.116306+0000 front 2024-03-19T02:06:46.116324+0000 (oldest deadline 2024-03-19T02:07:12.016484+0000) 2024-03-19T02:07:35.807 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:35 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:35.319+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.819160+0000 front 2024-03-19T02:07:04.819180+0000 (oldest deadline 2024-03-19T02:07:27.719305+0000) 2024-03-19T02:07:36.090 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:35 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:35.765+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170696+0000 front 2024-03-19T02:06:34.170736+0000 (oldest deadline 2024-03-19T02:06:55.870953+0000) 2024-03-19T02:07:36.090 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:35 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:35.765+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.572097+0000 front 2024-03-19T02:06:40.571972+0000 (oldest deadline 2024-03-19T02:07:06.372675+0000) 2024-03-19T02:07:36.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:35 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:35.765+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.675305+0000 front 2024-03-19T02:06:49.675286+0000 (oldest deadline 2024-03-19T02:07:11.975395+0000) 2024-03-19T02:07:36.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:35 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:35.765+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.276447+0000 front 2024-03-19T02:06:57.276673+0000 (oldest deadline 2024-03-19T02:07:20.176691+0000) 2024-03-19T02:07:36.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:35 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:35.765+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.778329+0000 front 2024-03-19T02:07:04.778433+0000 (oldest deadline 2024-03-19T02:07:29.478432+0000) 2024-03-19T02:07:36.111 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:35 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:35.814+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170251+0000 front 2024-03-19T02:06:34.170405+0000 (oldest deadline 2024-03-19T02:06:58.270386+0000) 2024-03-19T02:07:36.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:35 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:35.814+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.270741+0000 front 2024-03-19T02:06:38.270719+0000 (oldest deadline 2024-03-19T02:07:04.170884+0000) 2024-03-19T02:07:36.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:35 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:35.814+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:47.072167+0000 front 2024-03-19T02:06:47.072106+0000 (oldest deadline 2024-03-19T02:07:12.372271+0000) 2024-03-19T02:07:36.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:35 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:35.814+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.574305+0000 front 2024-03-19T02:06:57.574422+0000 (oldest deadline 2024-03-19T02:07:20.974748+0000) 2024-03-19T02:07:36.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:35 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:35.814+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.475657+0000 front 2024-03-19T02:07:04.475681+0000 (oldest deadline 2024-03-19T02:07:29.175780+0000) 2024-03-19T02:07:36.556 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:36 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:36.062+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:41.493849+0000 front 2024-03-19T02:06:41.493734+0000 (oldest deadline 2024-03-19T02:07:03.793726+0000) 2024-03-19T02:07:36.557 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:36 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:36.062+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.996743+0000 front 2024-03-19T02:06:49.996766+0000 (oldest deadline 2024-03-19T02:07:14.096879+0000) 2024-03-19T02:07:36.557 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:36 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:36.062+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:54.097324+0000 front 2024-03-19T02:06:54.097227+0000 (oldest deadline 2024-03-19T02:07:19.997301+0000) 2024-03-19T02:07:36.557 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:36 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:36.062+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:05.799241+0000 front 2024-03-19T02:07:05.799447+0000 (oldest deadline 2024-03-19T02:07:27.499223+0000) 2024-03-19T02:07:36.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:36 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:36.292+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:36.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:36 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:36.292+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:07:36.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:36 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:36.292+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.116306+0000 front 2024-03-19T02:06:46.116324+0000 (oldest deadline 2024-03-19T02:07:12.016484+0000) 2024-03-19T02:07:36.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:36 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:36.292+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.819160+0000 front 2024-03-19T02:07:04.819180+0000 (oldest deadline 2024-03-19T02:07:27.719305+0000) 2024-03-19T02:07:36.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:36 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:36.292+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6812 osd.5 since back 2024-03-19T02:07:13.521002+0000 front 2024-03-19T02:07:13.520954+0000 (oldest deadline 2024-03-19T02:07:35.821183+0000) 2024-03-19T02:07:37.090 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:36 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:36.735+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170696+0000 front 2024-03-19T02:06:34.170736+0000 (oldest deadline 2024-03-19T02:06:55.870953+0000) 2024-03-19T02:07:37.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:36 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:36.735+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.572097+0000 front 2024-03-19T02:06:40.571972+0000 (oldest deadline 2024-03-19T02:07:06.372675+0000) 2024-03-19T02:07:37.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:36 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:36.735+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.675305+0000 front 2024-03-19T02:06:49.675286+0000 (oldest deadline 2024-03-19T02:07:11.975395+0000) 2024-03-19T02:07:37.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:36 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:36.735+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.276447+0000 front 2024-03-19T02:06:57.276673+0000 (oldest deadline 2024-03-19T02:07:20.176691+0000) 2024-03-19T02:07:37.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:36 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:36.735+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.778329+0000 front 2024-03-19T02:07:04.778433+0000 (oldest deadline 2024-03-19T02:07:29.478432+0000) 2024-03-19T02:07:37.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:36 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:36.735+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.115:6812 osd.5 since back 2024-03-19T02:07:11.179460+0000 front 2024-03-19T02:07:11.179551+0000 (oldest deadline 2024-03-19T02:07:35.879442+0000) 2024-03-19T02:07:37.111 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:36 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:36.779+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170251+0000 front 2024-03-19T02:06:34.170405+0000 (oldest deadline 2024-03-19T02:06:58.270386+0000) 2024-03-19T02:07:37.111 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:36 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:36.779+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.270741+0000 front 2024-03-19T02:06:38.270719+0000 (oldest deadline 2024-03-19T02:07:04.170884+0000) 2024-03-19T02:07:37.111 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:36 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:36.779+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:47.072167+0000 front 2024-03-19T02:06:47.072106+0000 (oldest deadline 2024-03-19T02:07:12.372271+0000) 2024-03-19T02:07:37.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:36 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:36.779+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.574305+0000 front 2024-03-19T02:06:57.574422+0000 (oldest deadline 2024-03-19T02:07:20.974748+0000) 2024-03-19T02:07:37.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:36 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:36.779+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.475657+0000 front 2024-03-19T02:07:04.475681+0000 (oldest deadline 2024-03-19T02:07:29.175780+0000) 2024-03-19T02:07:37.311 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:37 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:37.060+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:41.493849+0000 front 2024-03-19T02:06:41.493734+0000 (oldest deadline 2024-03-19T02:07:03.793726+0000) 2024-03-19T02:07:37.311 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:37 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:37.060+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.996743+0000 front 2024-03-19T02:06:49.996766+0000 (oldest deadline 2024-03-19T02:07:14.096879+0000) 2024-03-19T02:07:37.312 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:37 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:37.060+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:54.097324+0000 front 2024-03-19T02:06:54.097227+0000 (oldest deadline 2024-03-19T02:07:19.997301+0000) 2024-03-19T02:07:37.312 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:37 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:37.060+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:05.799241+0000 front 2024-03-19T02:07:05.799447+0000 (oldest deadline 2024-03-19T02:07:27.499223+0000) 2024-03-19T02:07:37.570 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:37 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:37.311+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:37.570 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:37 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:37.311+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:07:37.570 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:37 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:37.311+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.116306+0000 front 2024-03-19T02:06:46.116324+0000 (oldest deadline 2024-03-19T02:07:12.016484+0000) 2024-03-19T02:07:37.571 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:37 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:37.311+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.819160+0000 front 2024-03-19T02:07:04.819180+0000 (oldest deadline 2024-03-19T02:07:27.719305+0000) 2024-03-19T02:07:37.571 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:37 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:37.311+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6812 osd.5 since back 2024-03-19T02:07:13.521002+0000 front 2024-03-19T02:07:13.520954+0000 (oldest deadline 2024-03-19T02:07:35.821183+0000) 2024-03-19T02:07:38.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:37 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:37.767+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170696+0000 front 2024-03-19T02:06:34.170736+0000 (oldest deadline 2024-03-19T02:06:55.870953+0000) 2024-03-19T02:07:38.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:37 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:37.767+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.572097+0000 front 2024-03-19T02:06:40.571972+0000 (oldest deadline 2024-03-19T02:07:06.372675+0000) 2024-03-19T02:07:38.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:37 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:37.767+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.675305+0000 front 2024-03-19T02:06:49.675286+0000 (oldest deadline 2024-03-19T02:07:11.975395+0000) 2024-03-19T02:07:38.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:37 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:37.767+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.276447+0000 front 2024-03-19T02:06:57.276673+0000 (oldest deadline 2024-03-19T02:07:20.176691+0000) 2024-03-19T02:07:38.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:37 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:37.767+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.778329+0000 front 2024-03-19T02:07:04.778433+0000 (oldest deadline 2024-03-19T02:07:29.478432+0000) 2024-03-19T02:07:38.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:37 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:37.767+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.115:6812 osd.5 since back 2024-03-19T02:07:11.179460+0000 front 2024-03-19T02:07:11.179551+0000 (oldest deadline 2024-03-19T02:07:35.879442+0000) 2024-03-19T02:07:38.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:37 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:37.801+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170251+0000 front 2024-03-19T02:06:34.170405+0000 (oldest deadline 2024-03-19T02:06:58.270386+0000) 2024-03-19T02:07:38.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:37 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:37.801+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.270741+0000 front 2024-03-19T02:06:38.270719+0000 (oldest deadline 2024-03-19T02:07:04.170884+0000) 2024-03-19T02:07:38.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:37 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:37.801+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:47.072167+0000 front 2024-03-19T02:06:47.072106+0000 (oldest deadline 2024-03-19T02:07:12.372271+0000) 2024-03-19T02:07:38.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:37 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:37.801+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.574305+0000 front 2024-03-19T02:06:57.574422+0000 (oldest deadline 2024-03-19T02:07:20.974748+0000) 2024-03-19T02:07:38.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:37 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:37.801+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.475657+0000 front 2024-03-19T02:07:04.475681+0000 (oldest deadline 2024-03-19T02:07:29.175780+0000) 2024-03-19T02:07:38.350 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:38 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:38.093+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:41.493849+0000 front 2024-03-19T02:06:41.493734+0000 (oldest deadline 2024-03-19T02:07:03.793726+0000) 2024-03-19T02:07:38.351 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:38 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:38.093+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.996743+0000 front 2024-03-19T02:06:49.996766+0000 (oldest deadline 2024-03-19T02:07:14.096879+0000) 2024-03-19T02:07:38.351 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:38 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:38.093+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:54.097324+0000 front 2024-03-19T02:06:54.097227+0000 (oldest deadline 2024-03-19T02:07:19.997301+0000) 2024-03-19T02:07:38.351 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:38 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8[85432]: 2024-03-19T02:07:38.093+0000 7f3e947ee700 -1 osd.8 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:05.799241+0000 front 2024-03-19T02:07:05.799447+0000 (oldest deadline 2024-03-19T02:07:27.499223+0000) 2024-03-19T02:07:38.764 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:38 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:38.350+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:38.764 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:38 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:38.350+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:07:38.764 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:38 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:38.350+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.116306+0000 front 2024-03-19T02:06:46.116324+0000 (oldest deadline 2024-03-19T02:07:12.016484+0000) 2024-03-19T02:07:38.765 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:38 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:38.350+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.819160+0000 front 2024-03-19T02:07:04.819180+0000 (oldest deadline 2024-03-19T02:07:27.719305+0000) 2024-03-19T02:07:38.765 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:38 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:38.350+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6812 osd.5 since back 2024-03-19T02:07:13.521002+0000 front 2024-03-19T02:07:13.520954+0000 (oldest deadline 2024-03-19T02:07:35.821183+0000) 2024-03-19T02:07:39.090 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:38 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:38.816+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170696+0000 front 2024-03-19T02:06:34.170736+0000 (oldest deadline 2024-03-19T02:06:55.870953+0000) 2024-03-19T02:07:39.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:38 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:38.816+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.572097+0000 front 2024-03-19T02:06:40.571972+0000 (oldest deadline 2024-03-19T02:07:06.372675+0000) 2024-03-19T02:07:39.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:38 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:38.816+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.675305+0000 front 2024-03-19T02:06:49.675286+0000 (oldest deadline 2024-03-19T02:07:11.975395+0000) 2024-03-19T02:07:39.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:38 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:38.816+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.276447+0000 front 2024-03-19T02:06:57.276673+0000 (oldest deadline 2024-03-19T02:07:20.176691+0000) 2024-03-19T02:07:39.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:38 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:38.816+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.778329+0000 front 2024-03-19T02:07:04.778433+0000 (oldest deadline 2024-03-19T02:07:29.478432+0000) 2024-03-19T02:07:39.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:38 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:38.816+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.115:6812 osd.5 since back 2024-03-19T02:07:11.179460+0000 front 2024-03-19T02:07:11.179551+0000 (oldest deadline 2024-03-19T02:07:35.879442+0000) 2024-03-19T02:07:39.111 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:38 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:38.811+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170251+0000 front 2024-03-19T02:06:34.170405+0000 (oldest deadline 2024-03-19T02:06:58.270386+0000) 2024-03-19T02:07:39.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:38 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:38.811+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.270741+0000 front 2024-03-19T02:06:38.270719+0000 (oldest deadline 2024-03-19T02:07:04.170884+0000) 2024-03-19T02:07:39.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:38 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:38.811+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:47.072167+0000 front 2024-03-19T02:06:47.072106+0000 (oldest deadline 2024-03-19T02:07:12.372271+0000) 2024-03-19T02:07:39.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:38 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:38.811+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.574305+0000 front 2024-03-19T02:06:57.574422+0000 (oldest deadline 2024-03-19T02:07:20.974748+0000) 2024-03-19T02:07:39.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:38 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:38.811+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.475657+0000 front 2024-03-19T02:07:04.475681+0000 (oldest deadline 2024-03-19T02:07:29.175780+0000) 2024-03-19T02:07:39.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:38 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:38.811+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6812 osd.5 since back 2024-03-19T02:07:13.876885+0000 front 2024-03-19T02:07:13.876797+0000 (oldest deadline 2024-03-19T02:07:38.576783+0000) 2024-03-19T02:07:39.477 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:39 smithi115 podman[106357]: 2024-03-19 02:07:39.208362559 +0000 UTC m=+5.623911177 container died edb84f05142c485aeee049e2515586a9e5176f828f9f0ad94950ed497fa89981 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240307, org.label-schema.schema-version=1.0, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True) 2024-03-19T02:07:39.477 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:39 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:39.392+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:39.477 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:39 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:39.392+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:07:39.477 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:39 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:39.392+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.116306+0000 front 2024-03-19T02:06:46.116324+0000 (oldest deadline 2024-03-19T02:07:12.016484+0000) 2024-03-19T02:07:39.477 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:39 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:39.392+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.819160+0000 front 2024-03-19T02:07:04.819180+0000 (oldest deadline 2024-03-19T02:07:27.719305+0000) 2024-03-19T02:07:39.478 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:39 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:39.392+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6812 osd.5 since back 2024-03-19T02:07:13.521002+0000 front 2024-03-19T02:07:13.520954+0000 (oldest deadline 2024-03-19T02:07:35.821183+0000) 2024-03-19T02:07:39.806 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:39 smithi115 podman[106357]: 2024-03-19 02:07:39.584792866 +0000 UTC m=+6.000341483 container cleanup edb84f05142c485aeee049e2515586a9e5176f828f9f0ad94950ed497fa89981 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240307, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.vendor=CentOS) 2024-03-19T02:07:39.806 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:39 smithi115 bash[106357]: ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8 2024-03-19T02:07:40.090 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:39 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:39.778+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170696+0000 front 2024-03-19T02:06:34.170736+0000 (oldest deadline 2024-03-19T02:06:55.870953+0000) 2024-03-19T02:07:40.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:39 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:39.778+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.572097+0000 front 2024-03-19T02:06:40.571972+0000 (oldest deadline 2024-03-19T02:07:06.372675+0000) 2024-03-19T02:07:40.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:39 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:39.778+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.675305+0000 front 2024-03-19T02:06:49.675286+0000 (oldest deadline 2024-03-19T02:07:11.975395+0000) 2024-03-19T02:07:40.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:39 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:39.778+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.276447+0000 front 2024-03-19T02:06:57.276673+0000 (oldest deadline 2024-03-19T02:07:20.176691+0000) 2024-03-19T02:07:40.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:39 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:39.778+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.778329+0000 front 2024-03-19T02:07:04.778433+0000 (oldest deadline 2024-03-19T02:07:29.478432+0000) 2024-03-19T02:07:40.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:39 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:39.778+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.115:6812 osd.5 since back 2024-03-19T02:07:11.179460+0000 front 2024-03-19T02:07:11.179551+0000 (oldest deadline 2024-03-19T02:07:35.879442+0000) 2024-03-19T02:07:40.111 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:39 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:39.808+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170251+0000 front 2024-03-19T02:06:34.170405+0000 (oldest deadline 2024-03-19T02:06:58.270386+0000) 2024-03-19T02:07:40.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:39 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:39.808+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.270741+0000 front 2024-03-19T02:06:38.270719+0000 (oldest deadline 2024-03-19T02:07:04.170884+0000) 2024-03-19T02:07:40.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:39 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:39.808+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:47.072167+0000 front 2024-03-19T02:06:47.072106+0000 (oldest deadline 2024-03-19T02:07:12.372271+0000) 2024-03-19T02:07:40.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:39 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:39.808+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.574305+0000 front 2024-03-19T02:06:57.574422+0000 (oldest deadline 2024-03-19T02:07:20.974748+0000) 2024-03-19T02:07:40.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:39 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:39.808+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.475657+0000 front 2024-03-19T02:07:04.475681+0000 (oldest deadline 2024-03-19T02:07:29.175780+0000) 2024-03-19T02:07:40.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:39 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:39.808+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6812 osd.5 since back 2024-03-19T02:07:13.876885+0000 front 2024-03-19T02:07:13.876797+0000 (oldest deadline 2024-03-19T02:07:38.576783+0000) 2024-03-19T02:07:40.190 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:39 smithi115 podman[106386]: 2024-03-19 02:07:39.844196525 +0000 UTC m=+0.639481243 container remove edb84f05142c485aeee049e2515586a9e5176f828f9f0ad94950ed497fa89981 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240307, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-19T02:07:40.488 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:40 smithi115 podman[106474]: 2024-03-19 02:07:40.094606039 +0000 UTC m=+0.018022119 image pull 2312c8e42d0069a2dbb2364bddb00264db4b800e7f18a7e7e4eb8f3ecab2ff1c quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639 2024-03-19T02:07:40.488 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:40 smithi115 podman[106474]: 2024-03-19 02:07:40.194857739 +0000 UTC m=+0.118273859 container create f39fe6ad3dbf4e8b5e17feff257bdf438bac1a7b650ec4abb66095a5ad739f00 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8-deactivate, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240307, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=wip-batrick-testing-20240318.181317-b6ae249) 2024-03-19T02:07:40.488 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:40 smithi115 podman[106474]: 2024-03-19 02:07:40.413891917 +0000 UTC m=+0.337307994 container init f39fe6ad3dbf4e8b5e17feff257bdf438bac1a7b650ec4abb66095a5ad739f00 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8-deactivate, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240307, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , RELEASE=wip-batrick-testing-20240318.181317-b6ae249) 2024-03-19T02:07:40.488 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:40 smithi115 podman[106474]: 2024-03-19 02:07:40.416917977 +0000 UTC m=+0.340334056 container start f39fe6ad3dbf4e8b5e17feff257bdf438bac1a7b650ec4abb66095a5ad739f00 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8-deactivate, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240307, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, GIT_CLEAN=True) 2024-03-19T02:07:40.488 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:40 smithi115 podman[106474]: 2024-03-19 02:07:40.488588714 +0000 UTC m=+0.412004801 container attach f39fe6ad3dbf4e8b5e17feff257bdf438bac1a7b650ec4abb66095a5ad739f00 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8-deactivate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.build-date=20240307, org.label-schema.schema-version=1.0) 2024-03-19T02:07:40.488 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:40 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:40.358+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:40.489 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:40 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:40.358+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:07:40.489 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:40 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:40.358+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.116306+0000 front 2024-03-19T02:06:46.116324+0000 (oldest deadline 2024-03-19T02:07:12.016484+0000) 2024-03-19T02:07:40.489 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:40 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:40.358+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.819160+0000 front 2024-03-19T02:07:04.819180+0000 (oldest deadline 2024-03-19T02:07:27.719305+0000) 2024-03-19T02:07:40.489 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:40 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:40.358+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6812 osd.5 since back 2024-03-19T02:07:13.521002+0000 front 2024-03-19T02:07:13.520954+0000 (oldest deadline 2024-03-19T02:07:35.821183+0000) 2024-03-19T02:07:40.806 INFO:journalctl@ceph.osd.8.smithi115.stdout:Mar 19 02:07:40 smithi115 podman[106474]: 2024-03-19 02:07:40.700728528 +0000 UTC m=+0.624144611 container died f39fe6ad3dbf4e8b5e17feff257bdf438bac1a7b650ec4abb66095a5ad739f00 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-8-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240307, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2) 2024-03-19T02:07:41.090 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:40 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:40.736+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170696+0000 front 2024-03-19T02:06:34.170736+0000 (oldest deadline 2024-03-19T02:06:55.870953+0000) 2024-03-19T02:07:41.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:40 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:40.736+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.572097+0000 front 2024-03-19T02:06:40.571972+0000 (oldest deadline 2024-03-19T02:07:06.372675+0000) 2024-03-19T02:07:41.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:40 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:40.736+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.675305+0000 front 2024-03-19T02:06:49.675286+0000 (oldest deadline 2024-03-19T02:07:11.975395+0000) 2024-03-19T02:07:41.092 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:40 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:40.736+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.276447+0000 front 2024-03-19T02:06:57.276673+0000 (oldest deadline 2024-03-19T02:07:20.176691+0000) 2024-03-19T02:07:41.092 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:40 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:40.736+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.778329+0000 front 2024-03-19T02:07:04.778433+0000 (oldest deadline 2024-03-19T02:07:29.478432+0000) 2024-03-19T02:07:41.092 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:40 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:40.736+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.115:6812 osd.5 since back 2024-03-19T02:07:11.179460+0000 front 2024-03-19T02:07:11.179551+0000 (oldest deadline 2024-03-19T02:07:35.879442+0000) 2024-03-19T02:07:41.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:40 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:40.767+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170251+0000 front 2024-03-19T02:06:34.170405+0000 (oldest deadline 2024-03-19T02:06:58.270386+0000) 2024-03-19T02:07:41.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:40 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:40.767+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.270741+0000 front 2024-03-19T02:06:38.270719+0000 (oldest deadline 2024-03-19T02:07:04.170884+0000) 2024-03-19T02:07:41.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:40 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:40.767+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:47.072167+0000 front 2024-03-19T02:06:47.072106+0000 (oldest deadline 2024-03-19T02:07:12.372271+0000) 2024-03-19T02:07:41.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:40 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:40.767+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.574305+0000 front 2024-03-19T02:06:57.574422+0000 (oldest deadline 2024-03-19T02:07:20.974748+0000) 2024-03-19T02:07:41.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:40 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:40.767+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.475657+0000 front 2024-03-19T02:07:04.475681+0000 (oldest deadline 2024-03-19T02:07:29.175780+0000) 2024-03-19T02:07:41.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:40 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:40.767+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6812 osd.5 since back 2024-03-19T02:07:13.876885+0000 front 2024-03-19T02:07:13.876797+0000 (oldest deadline 2024-03-19T02:07:38.576783+0000) 2024-03-19T02:07:41.341 DEBUG:teuthology.orchestra.run.smithi115:> sudo pkill -f 'journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@osd.8.service' 2024-03-19T02:07:41.384 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-19T02:07:41.385 INFO:tasks.cephadm.osd.8:Stopped osd.8 2024-03-19T02:07:41.385 INFO:tasks.cephadm.osd.9:Stopping osd.9... 2024-03-19T02:07:41.385 DEBUG:teuthology.orchestra.run.smithi099:> sudo systemctl stop ceph-438fb69c-e590-11ee-95c9-87774f69a715@osd.9 2024-03-19T02:07:41.677 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:41 smithi099 systemd[1]: Stopping Ceph osd.9 for 438fb69c-e590-11ee-95c9-87774f69a715... 2024-03-19T02:07:41.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:41 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:41.327+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:41.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:41 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:41.327+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:07:41.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:41 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:41.327+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.116306+0000 front 2024-03-19T02:06:46.116324+0000 (oldest deadline 2024-03-19T02:07:12.016484+0000) 2024-03-19T02:07:41.806 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:41 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:41.327+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.819160+0000 front 2024-03-19T02:07:04.819180+0000 (oldest deadline 2024-03-19T02:07:27.719305+0000) 2024-03-19T02:07:41.807 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:41 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:41.327+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6812 osd.5 since back 2024-03-19T02:07:13.521002+0000 front 2024-03-19T02:07:13.520954+0000 (oldest deadline 2024-03-19T02:07:35.821183+0000) 2024-03-19T02:07:42.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:41 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:41.677+0000 7f2fc3dbf700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.9 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-19T02:07:42.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:41 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:41.677+0000 7f2fc3dbf700 -1 osd.9 95 *** Got signal Terminated *** 2024-03-19T02:07:42.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:41 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:41.677+0000 7f2fc3dbf700 -1 osd.9 95 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-19T02:07:42.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:41 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:41.783+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170696+0000 front 2024-03-19T02:06:34.170736+0000 (oldest deadline 2024-03-19T02:06:55.870953+0000) 2024-03-19T02:07:42.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:41 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:41.783+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.572097+0000 front 2024-03-19T02:06:40.571972+0000 (oldest deadline 2024-03-19T02:07:06.372675+0000) 2024-03-19T02:07:42.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:41 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:41.783+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.675305+0000 front 2024-03-19T02:06:49.675286+0000 (oldest deadline 2024-03-19T02:07:11.975395+0000) 2024-03-19T02:07:42.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:41 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:41.783+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.276447+0000 front 2024-03-19T02:06:57.276673+0000 (oldest deadline 2024-03-19T02:07:20.176691+0000) 2024-03-19T02:07:42.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:41 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:41.783+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.778329+0000 front 2024-03-19T02:07:04.778433+0000 (oldest deadline 2024-03-19T02:07:29.478432+0000) 2024-03-19T02:07:42.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:41 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:41.783+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.115:6812 osd.5 since back 2024-03-19T02:07:11.179460+0000 front 2024-03-19T02:07:11.179551+0000 (oldest deadline 2024-03-19T02:07:35.879442+0000) 2024-03-19T02:07:42.111 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:41 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:41.769+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170251+0000 front 2024-03-19T02:06:34.170405+0000 (oldest deadline 2024-03-19T02:06:58.270386+0000) 2024-03-19T02:07:42.111 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:41 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:41.769+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.270741+0000 front 2024-03-19T02:06:38.270719+0000 (oldest deadline 2024-03-19T02:07:04.170884+0000) 2024-03-19T02:07:42.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:41 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:41.769+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:47.072167+0000 front 2024-03-19T02:06:47.072106+0000 (oldest deadline 2024-03-19T02:07:12.372271+0000) 2024-03-19T02:07:42.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:41 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:41.769+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.574305+0000 front 2024-03-19T02:06:57.574422+0000 (oldest deadline 2024-03-19T02:07:20.974748+0000) 2024-03-19T02:07:42.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:41 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:41.769+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.475657+0000 front 2024-03-19T02:07:04.475681+0000 (oldest deadline 2024-03-19T02:07:29.175780+0000) 2024-03-19T02:07:42.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:41 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:41.769+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6812 osd.5 since back 2024-03-19T02:07:13.876885+0000 front 2024-03-19T02:07:13.876797+0000 (oldest deadline 2024-03-19T02:07:38.576783+0000) 2024-03-19T02:07:42.556 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:42 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:42.294+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:42.556 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:42 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:42.294+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:07:42.556 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:42 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:42.294+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.116306+0000 front 2024-03-19T02:06:46.116324+0000 (oldest deadline 2024-03-19T02:07:12.016484+0000) 2024-03-19T02:07:42.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:42 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:42.294+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.819160+0000 front 2024-03-19T02:07:04.819180+0000 (oldest deadline 2024-03-19T02:07:27.719305+0000) 2024-03-19T02:07:42.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:42 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:42.294+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6812 osd.5 since back 2024-03-19T02:07:13.521002+0000 front 2024-03-19T02:07:13.520954+0000 (oldest deadline 2024-03-19T02:07:35.821183+0000) 2024-03-19T02:07:43.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:42 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:42.805+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170696+0000 front 2024-03-19T02:06:34.170736+0000 (oldest deadline 2024-03-19T02:06:55.870953+0000) 2024-03-19T02:07:43.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:42 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:42.805+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.572097+0000 front 2024-03-19T02:06:40.571972+0000 (oldest deadline 2024-03-19T02:07:06.372675+0000) 2024-03-19T02:07:43.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:42 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:42.805+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.675305+0000 front 2024-03-19T02:06:49.675286+0000 (oldest deadline 2024-03-19T02:07:11.975395+0000) 2024-03-19T02:07:43.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:42 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:42.805+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.276447+0000 front 2024-03-19T02:06:57.276673+0000 (oldest deadline 2024-03-19T02:07:20.176691+0000) 2024-03-19T02:07:43.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:42 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:42.805+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.778329+0000 front 2024-03-19T02:07:04.778433+0000 (oldest deadline 2024-03-19T02:07:29.478432+0000) 2024-03-19T02:07:43.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:42 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:42.805+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.115:6812 osd.5 since back 2024-03-19T02:07:11.179460+0000 front 2024-03-19T02:07:11.179551+0000 (oldest deadline 2024-03-19T02:07:35.879442+0000) 2024-03-19T02:07:43.111 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:42 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:42.729+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170251+0000 front 2024-03-19T02:06:34.170405+0000 (oldest deadline 2024-03-19T02:06:58.270386+0000) 2024-03-19T02:07:43.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:42 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:42.729+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.270741+0000 front 2024-03-19T02:06:38.270719+0000 (oldest deadline 2024-03-19T02:07:04.170884+0000) 2024-03-19T02:07:43.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:42 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:42.729+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:47.072167+0000 front 2024-03-19T02:06:47.072106+0000 (oldest deadline 2024-03-19T02:07:12.372271+0000) 2024-03-19T02:07:43.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:42 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:42.729+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.574305+0000 front 2024-03-19T02:06:57.574422+0000 (oldest deadline 2024-03-19T02:07:20.974748+0000) 2024-03-19T02:07:43.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:42 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:42.729+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.475657+0000 front 2024-03-19T02:07:04.475681+0000 (oldest deadline 2024-03-19T02:07:29.175780+0000) 2024-03-19T02:07:43.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:42 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:42.729+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6812 osd.5 since back 2024-03-19T02:07:13.876885+0000 front 2024-03-19T02:07:13.876797+0000 (oldest deadline 2024-03-19T02:07:38.576783+0000) 2024-03-19T02:07:43.556 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:43 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:43.273+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:43.556 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:43 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:43.273+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:07:43.556 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:43 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:43.273+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.116306+0000 front 2024-03-19T02:06:46.116324+0000 (oldest deadline 2024-03-19T02:07:12.016484+0000) 2024-03-19T02:07:43.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:43 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:43.273+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.819160+0000 front 2024-03-19T02:07:04.819180+0000 (oldest deadline 2024-03-19T02:07:27.719305+0000) 2024-03-19T02:07:43.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:43 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:43.273+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6812 osd.5 since back 2024-03-19T02:07:13.521002+0000 front 2024-03-19T02:07:13.520954+0000 (oldest deadline 2024-03-19T02:07:35.821183+0000) 2024-03-19T02:07:44.090 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:43 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:43.828+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170696+0000 front 2024-03-19T02:06:34.170736+0000 (oldest deadline 2024-03-19T02:06:55.870953+0000) 2024-03-19T02:07:44.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:43 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:43.828+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.572097+0000 front 2024-03-19T02:06:40.571972+0000 (oldest deadline 2024-03-19T02:07:06.372675+0000) 2024-03-19T02:07:44.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:43 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:43.828+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.675305+0000 front 2024-03-19T02:06:49.675286+0000 (oldest deadline 2024-03-19T02:07:11.975395+0000) 2024-03-19T02:07:44.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:43 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:43.828+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.276447+0000 front 2024-03-19T02:06:57.276673+0000 (oldest deadline 2024-03-19T02:07:20.176691+0000) 2024-03-19T02:07:44.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:43 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:43.828+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.778329+0000 front 2024-03-19T02:07:04.778433+0000 (oldest deadline 2024-03-19T02:07:29.478432+0000) 2024-03-19T02:07:44.091 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:43 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:43.828+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.115:6812 osd.5 since back 2024-03-19T02:07:11.179460+0000 front 2024-03-19T02:07:11.179551+0000 (oldest deadline 2024-03-19T02:07:35.879442+0000) 2024-03-19T02:07:44.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:43 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:43.768+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170251+0000 front 2024-03-19T02:06:34.170405+0000 (oldest deadline 2024-03-19T02:06:58.270386+0000) 2024-03-19T02:07:44.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:43 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:43.768+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.270741+0000 front 2024-03-19T02:06:38.270719+0000 (oldest deadline 2024-03-19T02:07:04.170884+0000) 2024-03-19T02:07:44.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:43 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:43.768+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:47.072167+0000 front 2024-03-19T02:06:47.072106+0000 (oldest deadline 2024-03-19T02:07:12.372271+0000) 2024-03-19T02:07:44.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:43 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:43.768+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.574305+0000 front 2024-03-19T02:06:57.574422+0000 (oldest deadline 2024-03-19T02:07:20.974748+0000) 2024-03-19T02:07:44.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:43 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:43.768+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.475657+0000 front 2024-03-19T02:07:04.475681+0000 (oldest deadline 2024-03-19T02:07:29.175780+0000) 2024-03-19T02:07:44.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:43 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:43.768+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6812 osd.5 since back 2024-03-19T02:07:13.876885+0000 front 2024-03-19T02:07:13.876797+0000 (oldest deadline 2024-03-19T02:07:38.576783+0000) 2024-03-19T02:07:44.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:43 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:43.768+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6822 osd.6 since back 2024-03-19T02:07:18.577293+0000 front 2024-03-19T02:07:18.577378+0000 (oldest deadline 2024-03-19T02:07:43.277238+0000) 2024-03-19T02:07:44.556 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:44 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:44.224+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:44.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:44 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:44.224+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:07:44.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:44 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:44.224+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.116306+0000 front 2024-03-19T02:06:46.116324+0000 (oldest deadline 2024-03-19T02:07:12.016484+0000) 2024-03-19T02:07:44.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:44 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:44.224+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.819160+0000 front 2024-03-19T02:07:04.819180+0000 (oldest deadline 2024-03-19T02:07:27.719305+0000) 2024-03-19T02:07:44.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:44 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:44.224+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6812 osd.5 since back 2024-03-19T02:07:13.521002+0000 front 2024-03-19T02:07:13.520954+0000 (oldest deadline 2024-03-19T02:07:35.821183+0000) 2024-03-19T02:07:44.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:44 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:44.224+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6822 osd.6 since back 2024-03-19T02:07:18.122266+0000 front 2024-03-19T02:07:18.122291+0000 (oldest deadline 2024-03-19T02:07:43.422218+0000) 2024-03-19T02:07:45.111 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:44 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:44.810+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170251+0000 front 2024-03-19T02:06:34.170405+0000 (oldest deadline 2024-03-19T02:06:58.270386+0000) 2024-03-19T02:07:45.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:44 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:44.810+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.270741+0000 front 2024-03-19T02:06:38.270719+0000 (oldest deadline 2024-03-19T02:07:04.170884+0000) 2024-03-19T02:07:45.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:44 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:44.810+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:47.072167+0000 front 2024-03-19T02:06:47.072106+0000 (oldest deadline 2024-03-19T02:07:12.372271+0000) 2024-03-19T02:07:45.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:44 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:44.810+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.574305+0000 front 2024-03-19T02:06:57.574422+0000 (oldest deadline 2024-03-19T02:07:20.974748+0000) 2024-03-19T02:07:45.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:44 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:44.810+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.475657+0000 front 2024-03-19T02:07:04.475681+0000 (oldest deadline 2024-03-19T02:07:29.175780+0000) 2024-03-19T02:07:45.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:44 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:44.810+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6812 osd.5 since back 2024-03-19T02:07:13.876885+0000 front 2024-03-19T02:07:13.876797+0000 (oldest deadline 2024-03-19T02:07:38.576783+0000) 2024-03-19T02:07:45.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:44 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:44.810+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6822 osd.6 since back 2024-03-19T02:07:18.577293+0000 front 2024-03-19T02:07:18.577378+0000 (oldest deadline 2024-03-19T02:07:43.277238+0000) 2024-03-19T02:07:45.340 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:44 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:44.859+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170696+0000 front 2024-03-19T02:06:34.170736+0000 (oldest deadline 2024-03-19T02:06:55.870953+0000) 2024-03-19T02:07:45.341 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:44 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:44.859+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.572097+0000 front 2024-03-19T02:06:40.571972+0000 (oldest deadline 2024-03-19T02:07:06.372675+0000) 2024-03-19T02:07:45.341 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:44 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:44.859+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.675305+0000 front 2024-03-19T02:06:49.675286+0000 (oldest deadline 2024-03-19T02:07:11.975395+0000) 2024-03-19T02:07:45.341 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:44 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:44.859+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.276447+0000 front 2024-03-19T02:06:57.276673+0000 (oldest deadline 2024-03-19T02:07:20.176691+0000) 2024-03-19T02:07:45.341 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:44 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:44.859+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.778329+0000 front 2024-03-19T02:07:04.778433+0000 (oldest deadline 2024-03-19T02:07:29.478432+0000) 2024-03-19T02:07:45.341 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:44 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:44.859+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.115:6812 osd.5 since back 2024-03-19T02:07:11.179460+0000 front 2024-03-19T02:07:11.179551+0000 (oldest deadline 2024-03-19T02:07:35.879442+0000) 2024-03-19T02:07:45.341 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:44 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:44.859+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6822 osd.6 since back 2024-03-19T02:07:18.680869+0000 front 2024-03-19T02:07:18.680913+0000 (oldest deadline 2024-03-19T02:07:44.580978+0000) 2024-03-19T02:07:45.556 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:45 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:45.219+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:45.556 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:45 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:45.219+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:07:45.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:45 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:45.219+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.116306+0000 front 2024-03-19T02:06:46.116324+0000 (oldest deadline 2024-03-19T02:07:12.016484+0000) 2024-03-19T02:07:45.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:45 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:45.219+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.819160+0000 front 2024-03-19T02:07:04.819180+0000 (oldest deadline 2024-03-19T02:07:27.719305+0000) 2024-03-19T02:07:45.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:45 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:45.219+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6812 osd.5 since back 2024-03-19T02:07:13.521002+0000 front 2024-03-19T02:07:13.520954+0000 (oldest deadline 2024-03-19T02:07:35.821183+0000) 2024-03-19T02:07:45.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:45 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:45.219+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6822 osd.6 since back 2024-03-19T02:07:18.122266+0000 front 2024-03-19T02:07:18.122291+0000 (oldest deadline 2024-03-19T02:07:43.422218+0000) 2024-03-19T02:07:46.063 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:45 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:45.778+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170251+0000 front 2024-03-19T02:06:34.170405+0000 (oldest deadline 2024-03-19T02:06:58.270386+0000) 2024-03-19T02:07:46.384 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:45 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:45.778+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.270741+0000 front 2024-03-19T02:06:38.270719+0000 (oldest deadline 2024-03-19T02:07:04.170884+0000) 2024-03-19T02:07:46.385 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:45 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:45.778+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:47.072167+0000 front 2024-03-19T02:06:47.072106+0000 (oldest deadline 2024-03-19T02:07:12.372271+0000) 2024-03-19T02:07:46.385 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:45 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:45.778+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.574305+0000 front 2024-03-19T02:06:57.574422+0000 (oldest deadline 2024-03-19T02:07:20.974748+0000) 2024-03-19T02:07:46.385 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:45 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:45.778+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.475657+0000 front 2024-03-19T02:07:04.475681+0000 (oldest deadline 2024-03-19T02:07:29.175780+0000) 2024-03-19T02:07:46.385 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:45 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:45.778+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6812 osd.5 since back 2024-03-19T02:07:13.876885+0000 front 2024-03-19T02:07:13.876797+0000 (oldest deadline 2024-03-19T02:07:38.576783+0000) 2024-03-19T02:07:46.385 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:45 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:45.778+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6822 osd.6 since back 2024-03-19T02:07:18.577293+0000 front 2024-03-19T02:07:18.577378+0000 (oldest deadline 2024-03-19T02:07:43.277238+0000) 2024-03-19T02:07:46.386 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:45 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:45.876+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170696+0000 front 2024-03-19T02:06:34.170736+0000 (oldest deadline 2024-03-19T02:06:55.870953+0000) 2024-03-19T02:07:46.386 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:45 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:45.876+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.572097+0000 front 2024-03-19T02:06:40.571972+0000 (oldest deadline 2024-03-19T02:07:06.372675+0000) 2024-03-19T02:07:46.386 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:45 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:45.876+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:49.675305+0000 front 2024-03-19T02:06:49.675286+0000 (oldest deadline 2024-03-19T02:07:11.975395+0000) 2024-03-19T02:07:46.386 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:45 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:45.876+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.276447+0000 front 2024-03-19T02:06:57.276673+0000 (oldest deadline 2024-03-19T02:07:20.176691+0000) 2024-03-19T02:07:46.386 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:45 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:45.876+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.778329+0000 front 2024-03-19T02:07:04.778433+0000 (oldest deadline 2024-03-19T02:07:29.478432+0000) 2024-03-19T02:07:46.386 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:45 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:45.876+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.115:6812 osd.5 since back 2024-03-19T02:07:11.179460+0000 front 2024-03-19T02:07:11.179551+0000 (oldest deadline 2024-03-19T02:07:35.879442+0000) 2024-03-19T02:07:46.386 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:45 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9[97951]: 2024-03-19T02:07:45.876+0000 7f2fbc3ab700 -1 osd.9 95 heartbeat_check: no reply from 172.21.15.99:6822 osd.6 since back 2024-03-19T02:07:18.680869+0000 front 2024-03-19T02:07:18.680913+0000 (oldest deadline 2024-03-19T02:07:44.580978+0000) 2024-03-19T02:07:46.556 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:46 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:46.177+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:46.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:46 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:46.177+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:07:46.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:46 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:46.177+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.116306+0000 front 2024-03-19T02:06:46.116324+0000 (oldest deadline 2024-03-19T02:07:12.016484+0000) 2024-03-19T02:07:46.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:46 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:46.177+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.819160+0000 front 2024-03-19T02:07:04.819180+0000 (oldest deadline 2024-03-19T02:07:27.719305+0000) 2024-03-19T02:07:46.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:46 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:46.177+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6812 osd.5 since back 2024-03-19T02:07:13.521002+0000 front 2024-03-19T02:07:13.520954+0000 (oldest deadline 2024-03-19T02:07:35.821183+0000) 2024-03-19T02:07:46.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:46 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:46.177+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6822 osd.6 since back 2024-03-19T02:07:18.122266+0000 front 2024-03-19T02:07:18.122291+0000 (oldest deadline 2024-03-19T02:07:43.422218+0000) 2024-03-19T02:07:47.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:46 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:46.743+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170251+0000 front 2024-03-19T02:06:34.170405+0000 (oldest deadline 2024-03-19T02:06:58.270386+0000) 2024-03-19T02:07:47.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:46 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:46.743+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.270741+0000 front 2024-03-19T02:06:38.270719+0000 (oldest deadline 2024-03-19T02:07:04.170884+0000) 2024-03-19T02:07:47.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:46 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:46.743+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:47.072167+0000 front 2024-03-19T02:06:47.072106+0000 (oldest deadline 2024-03-19T02:07:12.372271+0000) 2024-03-19T02:07:47.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:46 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:46.743+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.574305+0000 front 2024-03-19T02:06:57.574422+0000 (oldest deadline 2024-03-19T02:07:20.974748+0000) 2024-03-19T02:07:47.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:46 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:46.743+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.475657+0000 front 2024-03-19T02:07:04.475681+0000 (oldest deadline 2024-03-19T02:07:29.175780+0000) 2024-03-19T02:07:47.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:46 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:46.743+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6812 osd.5 since back 2024-03-19T02:07:13.876885+0000 front 2024-03-19T02:07:13.876797+0000 (oldest deadline 2024-03-19T02:07:38.576783+0000) 2024-03-19T02:07:47.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:46 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:46.743+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6822 osd.6 since back 2024-03-19T02:07:18.577293+0000 front 2024-03-19T02:07:18.577378+0000 (oldest deadline 2024-03-19T02:07:43.277238+0000) 2024-03-19T02:07:47.340 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:47 smithi099 podman[127501]: 2024-03-19 02:07:47.086424141 +0000 UTC m=+5.509332375 container died 36df48bbbcfdd6d2e284f186c2cc76d451ee0ea73d061a0ba09edf21f069c493 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9, org.label-schema.build-date=20240307, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-19T02:07:47.556 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:47 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:47.207+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:47.556 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:47 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:47.207+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:07:47.556 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:47 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:47.207+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.116306+0000 front 2024-03-19T02:06:46.116324+0000 (oldest deadline 2024-03-19T02:07:12.016484+0000) 2024-03-19T02:07:47.556 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:47 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:47.207+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.819160+0000 front 2024-03-19T02:07:04.819180+0000 (oldest deadline 2024-03-19T02:07:27.719305+0000) 2024-03-19T02:07:47.556 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:47 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:47.207+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6812 osd.5 since back 2024-03-19T02:07:13.521002+0000 front 2024-03-19T02:07:13.520954+0000 (oldest deadline 2024-03-19T02:07:35.821183+0000) 2024-03-19T02:07:47.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:47 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:47.207+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6822 osd.6 since back 2024-03-19T02:07:18.122266+0000 front 2024-03-19T02:07:18.122291+0000 (oldest deadline 2024-03-19T02:07:43.422218+0000) 2024-03-19T02:07:47.840 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:47 smithi099 podman[127501]: 2024-03-19 02:07:47.377209671 +0000 UTC m=+5.800117911 container cleanup 36df48bbbcfdd6d2e284f186c2cc76d451ee0ea73d061a0ba09edf21f069c493 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240307, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-03-19T02:07:47.840 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:47 smithi099 bash[127501]: ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9 2024-03-19T02:07:47.840 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:47 smithi099 podman[127518]: 2024-03-19 02:07:47.612205512 +0000 UTC m=+0.523389569 container remove 36df48bbbcfdd6d2e284f186c2cc76d451ee0ea73d061a0ba09edf21f069c493 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240307, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, ceph=True) 2024-03-19T02:07:48.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:47 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:47.725+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170251+0000 front 2024-03-19T02:06:34.170405+0000 (oldest deadline 2024-03-19T02:06:58.270386+0000) 2024-03-19T02:07:48.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:47 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:47.725+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.270741+0000 front 2024-03-19T02:06:38.270719+0000 (oldest deadline 2024-03-19T02:07:04.170884+0000) 2024-03-19T02:07:48.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:47 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:47.725+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:47.072167+0000 front 2024-03-19T02:06:47.072106+0000 (oldest deadline 2024-03-19T02:07:12.372271+0000) 2024-03-19T02:07:48.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:47 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:47.725+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.574305+0000 front 2024-03-19T02:06:57.574422+0000 (oldest deadline 2024-03-19T02:07:20.974748+0000) 2024-03-19T02:07:48.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:47 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:47.725+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.475657+0000 front 2024-03-19T02:07:04.475681+0000 (oldest deadline 2024-03-19T02:07:29.175780+0000) 2024-03-19T02:07:48.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:47 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:47.725+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6812 osd.5 since back 2024-03-19T02:07:13.876885+0000 front 2024-03-19T02:07:13.876797+0000 (oldest deadline 2024-03-19T02:07:38.576783+0000) 2024-03-19T02:07:48.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:47 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:47.725+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6822 osd.6 since back 2024-03-19T02:07:18.577293+0000 front 2024-03-19T02:07:18.577378+0000 (oldest deadline 2024-03-19T02:07:43.277238+0000) 2024-03-19T02:07:48.223 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:47 smithi099 podman[127608]: 2024-03-19 02:07:47.832847922 +0000 UTC m=+0.018692407 image pull 2312c8e42d0069a2dbb2364bddb00264db4b800e7f18a7e7e4eb8f3ecab2ff1c quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639 2024-03-19T02:07:48.223 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:47 smithi099 podman[127608]: 2024-03-19 02:07:47.955101635 +0000 UTC m=+0.140946116 container create bd71d7fc4ce88074ff1d16ad0ce42af479f77ba85460c2c603b68d7d1f8c47e4 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9-deactivate, GIT_CLEAN=True, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240307, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True) 2024-03-19T02:07:48.223 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:48 smithi099 podman[127608]: 2024-03-19 02:07:48.165620221 +0000 UTC m=+0.351464706 container init bd71d7fc4ce88074ff1d16ad0ce42af479f77ba85460c2c603b68d7d1f8c47e4 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9-deactivate, org.label-schema.build-date=20240307, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-19T02:07:48.223 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:48 smithi099 podman[127608]: 2024-03-19 02:07:48.168813954 +0000 UTC m=+0.354658432 container start bd71d7fc4ce88074ff1d16ad0ce42af479f77ba85460c2c603b68d7d1f8c47e4 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9-deactivate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.build-date=20240307, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5) 2024-03-19T02:07:48.223 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:48 smithi099 podman[127608]: 2024-03-19 02:07:48.22393029 +0000 UTC m=+0.409774787 container attach bd71d7fc4ce88074ff1d16ad0ce42af479f77ba85460c2c603b68d7d1f8c47e4 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9-deactivate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240307, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2) 2024-03-19T02:07:48.556 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:48 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:48.177+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:48.556 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:48 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:48.177+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:07:48.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:48 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:48.177+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.116306+0000 front 2024-03-19T02:06:46.116324+0000 (oldest deadline 2024-03-19T02:07:12.016484+0000) 2024-03-19T02:07:48.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:48 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:48.177+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.819160+0000 front 2024-03-19T02:07:04.819180+0000 (oldest deadline 2024-03-19T02:07:27.719305+0000) 2024-03-19T02:07:48.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:48 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:48.177+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6812 osd.5 since back 2024-03-19T02:07:13.521002+0000 front 2024-03-19T02:07:13.520954+0000 (oldest deadline 2024-03-19T02:07:35.821183+0000) 2024-03-19T02:07:48.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:48 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:48.177+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6822 osd.6 since back 2024-03-19T02:07:18.122266+0000 front 2024-03-19T02:07:18.122291+0000 (oldest deadline 2024-03-19T02:07:43.422218+0000) 2024-03-19T02:07:48.590 INFO:journalctl@ceph.osd.9.smithi099.stdout:Mar 19 02:07:48 smithi099 podman[127608]: 2024-03-19 02:07:48.452025958 +0000 UTC m=+0.637870449 container died bd71d7fc4ce88074ff1d16ad0ce42af479f77ba85460c2c603b68d7d1f8c47e4 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-9-deactivate, ceph=True, org.label-schema.build-date=20240307, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-03-19T02:07:49.034 DEBUG:teuthology.orchestra.run.smithi099:> sudo pkill -f 'journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@osd.9.service' 2024-03-19T02:07:49.075 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-19T02:07:49.076 INFO:tasks.cephadm.osd.9:Stopped osd.9 2024-03-19T02:07:49.076 INFO:tasks.cephadm.osd.10:Stopping osd.10... 2024-03-19T02:07:49.076 DEBUG:teuthology.orchestra.run.smithi112:> sudo systemctl stop ceph-438fb69c-e590-11ee-95c9-87774f69a715@osd.10 2024-03-19T02:07:49.105 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:48 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:48.680+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170251+0000 front 2024-03-19T02:06:34.170405+0000 (oldest deadline 2024-03-19T02:06:58.270386+0000) 2024-03-19T02:07:49.105 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:48 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:48.680+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.270741+0000 front 2024-03-19T02:06:38.270719+0000 (oldest deadline 2024-03-19T02:07:04.170884+0000) 2024-03-19T02:07:49.105 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:48 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:48.680+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:47.072167+0000 front 2024-03-19T02:06:47.072106+0000 (oldest deadline 2024-03-19T02:07:12.372271+0000) 2024-03-19T02:07:49.106 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:48 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:48.680+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.574305+0000 front 2024-03-19T02:06:57.574422+0000 (oldest deadline 2024-03-19T02:07:20.974748+0000) 2024-03-19T02:07:49.106 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:48 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:48.680+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.475657+0000 front 2024-03-19T02:07:04.475681+0000 (oldest deadline 2024-03-19T02:07:29.175780+0000) 2024-03-19T02:07:49.106 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:48 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:48.680+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6812 osd.5 since back 2024-03-19T02:07:13.876885+0000 front 2024-03-19T02:07:13.876797+0000 (oldest deadline 2024-03-19T02:07:38.576783+0000) 2024-03-19T02:07:49.106 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:48 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:48.680+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6822 osd.6 since back 2024-03-19T02:07:18.577293+0000 front 2024-03-19T02:07:18.577378+0000 (oldest deadline 2024-03-19T02:07:43.277238+0000) 2024-03-19T02:07:49.361 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:49 smithi112 systemd[1]: Stopping Ceph osd.10 for 438fb69c-e590-11ee-95c9-87774f69a715... 2024-03-19T02:07:49.556 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:49 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:49.196+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:49.556 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:49 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:49.196+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:07:49.556 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:49 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:49.196+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.116306+0000 front 2024-03-19T02:06:46.116324+0000 (oldest deadline 2024-03-19T02:07:12.016484+0000) 2024-03-19T02:07:49.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:49 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:49.196+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.819160+0000 front 2024-03-19T02:07:04.819180+0000 (oldest deadline 2024-03-19T02:07:27.719305+0000) 2024-03-19T02:07:49.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:49 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:49.196+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6812 osd.5 since back 2024-03-19T02:07:13.521002+0000 front 2024-03-19T02:07:13.520954+0000 (oldest deadline 2024-03-19T02:07:35.821183+0000) 2024-03-19T02:07:49.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:49 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:49.196+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6822 osd.6 since back 2024-03-19T02:07:18.122266+0000 front 2024-03-19T02:07:18.122291+0000 (oldest deadline 2024-03-19T02:07:43.422218+0000) 2024-03-19T02:07:49.683 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:49 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:49.375+0000 7f1e0e5b7700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.10 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-19T02:07:49.683 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:49 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:49.375+0000 7f1e0e5b7700 -1 osd.10 95 *** Got signal Terminated *** 2024-03-19T02:07:49.684 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:49 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:49.375+0000 7f1e0e5b7700 -1 osd.10 95 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-19T02:07:50.111 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:49 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:49.684+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170251+0000 front 2024-03-19T02:06:34.170405+0000 (oldest deadline 2024-03-19T02:06:58.270386+0000) 2024-03-19T02:07:50.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:49 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:49.684+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.270741+0000 front 2024-03-19T02:06:38.270719+0000 (oldest deadline 2024-03-19T02:07:04.170884+0000) 2024-03-19T02:07:50.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:49 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:49.684+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:47.072167+0000 front 2024-03-19T02:06:47.072106+0000 (oldest deadline 2024-03-19T02:07:12.372271+0000) 2024-03-19T02:07:50.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:49 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:49.684+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.574305+0000 front 2024-03-19T02:06:57.574422+0000 (oldest deadline 2024-03-19T02:07:20.974748+0000) 2024-03-19T02:07:50.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:49 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:49.684+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.475657+0000 front 2024-03-19T02:07:04.475681+0000 (oldest deadline 2024-03-19T02:07:29.175780+0000) 2024-03-19T02:07:50.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:49 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:49.684+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6812 osd.5 since back 2024-03-19T02:07:13.876885+0000 front 2024-03-19T02:07:13.876797+0000 (oldest deadline 2024-03-19T02:07:38.576783+0000) 2024-03-19T02:07:50.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:49 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:49.684+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6822 osd.6 since back 2024-03-19T02:07:18.577293+0000 front 2024-03-19T02:07:18.577378+0000 (oldest deadline 2024-03-19T02:07:43.277238+0000) 2024-03-19T02:07:50.556 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:50 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:50.159+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:50.556 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:50 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:50.159+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:07:50.556 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:50 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:50.159+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.116306+0000 front 2024-03-19T02:06:46.116324+0000 (oldest deadline 2024-03-19T02:07:12.016484+0000) 2024-03-19T02:07:50.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:50 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:50.159+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.819160+0000 front 2024-03-19T02:07:04.819180+0000 (oldest deadline 2024-03-19T02:07:27.719305+0000) 2024-03-19T02:07:50.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:50 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:50.159+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6812 osd.5 since back 2024-03-19T02:07:13.521002+0000 front 2024-03-19T02:07:13.520954+0000 (oldest deadline 2024-03-19T02:07:35.821183+0000) 2024-03-19T02:07:50.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:50 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:50.159+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6822 osd.6 since back 2024-03-19T02:07:18.122266+0000 front 2024-03-19T02:07:18.122291+0000 (oldest deadline 2024-03-19T02:07:43.422218+0000) 2024-03-19T02:07:51.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:50 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:50.669+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170251+0000 front 2024-03-19T02:06:34.170405+0000 (oldest deadline 2024-03-19T02:06:58.270386+0000) 2024-03-19T02:07:51.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:50 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:50.669+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.270741+0000 front 2024-03-19T02:06:38.270719+0000 (oldest deadline 2024-03-19T02:07:04.170884+0000) 2024-03-19T02:07:51.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:50 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:50.669+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:47.072167+0000 front 2024-03-19T02:06:47.072106+0000 (oldest deadline 2024-03-19T02:07:12.372271+0000) 2024-03-19T02:07:51.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:50 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:50.669+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.574305+0000 front 2024-03-19T02:06:57.574422+0000 (oldest deadline 2024-03-19T02:07:20.974748+0000) 2024-03-19T02:07:51.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:50 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:50.669+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.475657+0000 front 2024-03-19T02:07:04.475681+0000 (oldest deadline 2024-03-19T02:07:29.175780+0000) 2024-03-19T02:07:51.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:50 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:50.669+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6812 osd.5 since back 2024-03-19T02:07:13.876885+0000 front 2024-03-19T02:07:13.876797+0000 (oldest deadline 2024-03-19T02:07:38.576783+0000) 2024-03-19T02:07:51.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:50 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:50.669+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6822 osd.6 since back 2024-03-19T02:07:18.577293+0000 front 2024-03-19T02:07:18.577378+0000 (oldest deadline 2024-03-19T02:07:43.277238+0000) 2024-03-19T02:07:51.556 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:51 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:51.192+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:51.556 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:51 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:51.192+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:07:51.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:51 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:51.192+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.116306+0000 front 2024-03-19T02:06:46.116324+0000 (oldest deadline 2024-03-19T02:07:12.016484+0000) 2024-03-19T02:07:51.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:51 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:51.192+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.819160+0000 front 2024-03-19T02:07:04.819180+0000 (oldest deadline 2024-03-19T02:07:27.719305+0000) 2024-03-19T02:07:51.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:51 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:51.192+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6812 osd.5 since back 2024-03-19T02:07:13.521002+0000 front 2024-03-19T02:07:13.520954+0000 (oldest deadline 2024-03-19T02:07:35.821183+0000) 2024-03-19T02:07:51.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:51 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:51.192+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6822 osd.6 since back 2024-03-19T02:07:18.122266+0000 front 2024-03-19T02:07:18.122291+0000 (oldest deadline 2024-03-19T02:07:43.422218+0000) 2024-03-19T02:07:52.111 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:51 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:51.633+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170251+0000 front 2024-03-19T02:06:34.170405+0000 (oldest deadline 2024-03-19T02:06:58.270386+0000) 2024-03-19T02:07:52.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:51 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:51.633+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.270741+0000 front 2024-03-19T02:06:38.270719+0000 (oldest deadline 2024-03-19T02:07:04.170884+0000) 2024-03-19T02:07:52.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:51 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:51.633+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:47.072167+0000 front 2024-03-19T02:06:47.072106+0000 (oldest deadline 2024-03-19T02:07:12.372271+0000) 2024-03-19T02:07:52.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:51 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:51.633+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.574305+0000 front 2024-03-19T02:06:57.574422+0000 (oldest deadline 2024-03-19T02:07:20.974748+0000) 2024-03-19T02:07:52.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:51 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:51.633+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.475657+0000 front 2024-03-19T02:07:04.475681+0000 (oldest deadline 2024-03-19T02:07:29.175780+0000) 2024-03-19T02:07:52.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:51 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:51.633+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6812 osd.5 since back 2024-03-19T02:07:13.876885+0000 front 2024-03-19T02:07:13.876797+0000 (oldest deadline 2024-03-19T02:07:38.576783+0000) 2024-03-19T02:07:52.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:51 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:51.633+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6822 osd.6 since back 2024-03-19T02:07:18.577293+0000 front 2024-03-19T02:07:18.577378+0000 (oldest deadline 2024-03-19T02:07:43.277238+0000) 2024-03-19T02:07:52.556 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:52 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:52.214+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:52.556 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:52 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:52.214+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:07:52.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:52 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:52.214+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.116306+0000 front 2024-03-19T02:06:46.116324+0000 (oldest deadline 2024-03-19T02:07:12.016484+0000) 2024-03-19T02:07:52.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:52 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:52.214+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.819160+0000 front 2024-03-19T02:07:04.819180+0000 (oldest deadline 2024-03-19T02:07:27.719305+0000) 2024-03-19T02:07:52.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:52 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:52.214+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6812 osd.5 since back 2024-03-19T02:07:13.521002+0000 front 2024-03-19T02:07:13.520954+0000 (oldest deadline 2024-03-19T02:07:35.821183+0000) 2024-03-19T02:07:52.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:52 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:52.214+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6822 osd.6 since back 2024-03-19T02:07:18.122266+0000 front 2024-03-19T02:07:18.122291+0000 (oldest deadline 2024-03-19T02:07:43.422218+0000) 2024-03-19T02:07:53.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:52 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:52.668+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170251+0000 front 2024-03-19T02:06:34.170405+0000 (oldest deadline 2024-03-19T02:06:58.270386+0000) 2024-03-19T02:07:53.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:52 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:52.668+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.270741+0000 front 2024-03-19T02:06:38.270719+0000 (oldest deadline 2024-03-19T02:07:04.170884+0000) 2024-03-19T02:07:53.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:52 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:52.668+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:47.072167+0000 front 2024-03-19T02:06:47.072106+0000 (oldest deadline 2024-03-19T02:07:12.372271+0000) 2024-03-19T02:07:53.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:52 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:52.668+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.574305+0000 front 2024-03-19T02:06:57.574422+0000 (oldest deadline 2024-03-19T02:07:20.974748+0000) 2024-03-19T02:07:53.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:52 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:52.668+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.475657+0000 front 2024-03-19T02:07:04.475681+0000 (oldest deadline 2024-03-19T02:07:29.175780+0000) 2024-03-19T02:07:53.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:52 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:52.668+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6812 osd.5 since back 2024-03-19T02:07:13.876885+0000 front 2024-03-19T02:07:13.876797+0000 (oldest deadline 2024-03-19T02:07:38.576783+0000) 2024-03-19T02:07:53.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:52 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:52.668+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6822 osd.6 since back 2024-03-19T02:07:18.577293+0000 front 2024-03-19T02:07:18.577378+0000 (oldest deadline 2024-03-19T02:07:43.277238+0000) 2024-03-19T02:07:53.556 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:53 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:53.178+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:53.556 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:53 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:53.178+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:07:53.556 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:53 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:53.178+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.116306+0000 front 2024-03-19T02:06:46.116324+0000 (oldest deadline 2024-03-19T02:07:12.016484+0000) 2024-03-19T02:07:53.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:53 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:53.178+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.819160+0000 front 2024-03-19T02:07:04.819180+0000 (oldest deadline 2024-03-19T02:07:27.719305+0000) 2024-03-19T02:07:53.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:53 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:53.178+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6812 osd.5 since back 2024-03-19T02:07:13.521002+0000 front 2024-03-19T02:07:13.520954+0000 (oldest deadline 2024-03-19T02:07:35.821183+0000) 2024-03-19T02:07:53.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:53 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:53.178+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6822 osd.6 since back 2024-03-19T02:07:18.122266+0000 front 2024-03-19T02:07:18.122291+0000 (oldest deadline 2024-03-19T02:07:43.422218+0000) 2024-03-19T02:07:53.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:53 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:53.178+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6820 osd.7 since back 2024-03-19T02:07:29.824127+0000 front 2024-03-19T02:07:29.824052+0000 (oldest deadline 2024-03-19T02:07:52.624568+0000) 2024-03-19T02:07:54.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:53 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:53.655+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:34.170251+0000 front 2024-03-19T02:06:34.170405+0000 (oldest deadline 2024-03-19T02:06:58.270386+0000) 2024-03-19T02:07:54.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:53 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:53.655+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:38.270741+0000 front 2024-03-19T02:06:38.270719+0000 (oldest deadline 2024-03-19T02:07:04.170884+0000) 2024-03-19T02:07:54.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:53 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:53.655+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:47.072167+0000 front 2024-03-19T02:06:47.072106+0000 (oldest deadline 2024-03-19T02:07:12.372271+0000) 2024-03-19T02:07:54.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:53 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:53.655+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6814 osd.3 since back 2024-03-19T02:06:57.574305+0000 front 2024-03-19T02:06:57.574422+0000 (oldest deadline 2024-03-19T02:07:20.974748+0000) 2024-03-19T02:07:54.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:53 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:53.655+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.475657+0000 front 2024-03-19T02:07:04.475681+0000 (oldest deadline 2024-03-19T02:07:29.175780+0000) 2024-03-19T02:07:54.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:53 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:53.655+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.115:6812 osd.5 since back 2024-03-19T02:07:13.876885+0000 front 2024-03-19T02:07:13.876797+0000 (oldest deadline 2024-03-19T02:07:38.576783+0000) 2024-03-19T02:07:54.113 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:53 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10[91747]: 2024-03-19T02:07:53.655+0000 7f1e06ba3700 -1 osd.10 95 heartbeat_check: no reply from 172.21.15.99:6822 osd.6 since back 2024-03-19T02:07:18.577293+0000 front 2024-03-19T02:07:18.577378+0000 (oldest deadline 2024-03-19T02:07:43.277238+0000) 2024-03-19T02:07:54.556 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:54 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:54.165+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:54.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:54 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:54.165+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:07:54.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:54 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:54.165+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.116306+0000 front 2024-03-19T02:06:46.116324+0000 (oldest deadline 2024-03-19T02:07:12.016484+0000) 2024-03-19T02:07:54.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:54 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:54.165+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.819160+0000 front 2024-03-19T02:07:04.819180+0000 (oldest deadline 2024-03-19T02:07:27.719305+0000) 2024-03-19T02:07:54.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:54 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:54.165+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6812 osd.5 since back 2024-03-19T02:07:13.521002+0000 front 2024-03-19T02:07:13.520954+0000 (oldest deadline 2024-03-19T02:07:35.821183+0000) 2024-03-19T02:07:54.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:54 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:54.165+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6822 osd.6 since back 2024-03-19T02:07:18.122266+0000 front 2024-03-19T02:07:18.122291+0000 (oldest deadline 2024-03-19T02:07:43.422218+0000) 2024-03-19T02:07:54.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:54 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:54.165+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6820 osd.7 since back 2024-03-19T02:07:29.824127+0000 front 2024-03-19T02:07:29.824052+0000 (oldest deadline 2024-03-19T02:07:52.624568+0000) 2024-03-19T02:07:55.112 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:54 smithi112 podman[107126]: 2024-03-19 02:07:54.842535067 +0000 UTC m=+5.568852393 container died 141d2a5e8f280f6bf69d77b8ea7e618b8d29bddff292c9b617c2d8029ac6d592 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240307, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True) 2024-03-19T02:07:55.483 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:55 smithi112 podman[107126]: 2024-03-19 02:07:55.249175551 +0000 UTC m=+5.975492896 container cleanup 141d2a5e8f280f6bf69d77b8ea7e618b8d29bddff292c9b617c2d8029ac6d592 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240307, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True) 2024-03-19T02:07:55.483 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:55 smithi112 bash[107126]: ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10 2024-03-19T02:07:55.556 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:55 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:55.196+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:55.556 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:55 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:55.196+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:07:55.556 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:55 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:55.196+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.116306+0000 front 2024-03-19T02:06:46.116324+0000 (oldest deadline 2024-03-19T02:07:12.016484+0000) 2024-03-19T02:07:55.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:55 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:55.196+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.819160+0000 front 2024-03-19T02:07:04.819180+0000 (oldest deadline 2024-03-19T02:07:27.719305+0000) 2024-03-19T02:07:55.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:55 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:55.196+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6812 osd.5 since back 2024-03-19T02:07:13.521002+0000 front 2024-03-19T02:07:13.520954+0000 (oldest deadline 2024-03-19T02:07:35.821183+0000) 2024-03-19T02:07:55.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:55 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:55.196+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6822 osd.6 since back 2024-03-19T02:07:18.122266+0000 front 2024-03-19T02:07:18.122291+0000 (oldest deadline 2024-03-19T02:07:43.422218+0000) 2024-03-19T02:07:55.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:55 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:55.196+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6820 osd.7 since back 2024-03-19T02:07:29.824127+0000 front 2024-03-19T02:07:29.824052+0000 (oldest deadline 2024-03-19T02:07:52.624568+0000) 2024-03-19T02:07:55.774 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:55 smithi112 podman[107143]: 2024-03-19 02:07:55.483645446 +0000 UTC m=+0.631920905 container remove 141d2a5e8f280f6bf69d77b8ea7e618b8d29bddff292c9b617c2d8029ac6d592 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.build-date=20240307, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, RELEASE=wip-batrick-testing-20240318.181317-b6ae249) 2024-03-19T02:07:56.045 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:55 smithi112 podman[107236]: 2024-03-19 02:07:55.677689317 +0000 UTC m=+0.018614306 image pull 2312c8e42d0069a2dbb2364bddb00264db4b800e7f18a7e7e4eb8f3ecab2ff1c quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639 2024-03-19T02:07:56.045 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:55 smithi112 podman[107236]: 2024-03-19 02:07:55.784244686 +0000 UTC m=+0.125169651 container create 6a97c39e833228d1bae5135ebb2837702e6faca2263d6df7836c0869873ec5e8 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10-deactivate, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240307, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-batrick-testing-20240318.181317-b6ae249) 2024-03-19T02:07:56.045 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:56 smithi112 podman[107236]: 2024-03-19 02:07:56.045532087 +0000 UTC m=+0.386457081 container init 6a97c39e833228d1bae5135ebb2837702e6faca2263d6df7836c0869873ec5e8 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10-deactivate, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, ceph=True, org.label-schema.build-date=20240307, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-19T02:07:56.331 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:56 smithi112 podman[107236]: 2024-03-19 02:07:56.048650498 +0000 UTC m=+0.389575468 container start 6a97c39e833228d1bae5135ebb2837702e6faca2263d6df7836c0869873ec5e8 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10-deactivate, org.label-schema.build-date=20240307, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5) 2024-03-19T02:07:56.331 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:56 smithi112 podman[107236]: 2024-03-19 02:07:56.103716296 +0000 UTC m=+0.444641292 container attach 6a97c39e833228d1bae5135ebb2837702e6faca2263d6df7836c0869873ec5e8 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240307, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True) 2024-03-19T02:07:56.556 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:56 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:56.168+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:56.556 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:56 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:56.168+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:07:56.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:56 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:56.168+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.116306+0000 front 2024-03-19T02:06:46.116324+0000 (oldest deadline 2024-03-19T02:07:12.016484+0000) 2024-03-19T02:07:56.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:56 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:56.168+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.819160+0000 front 2024-03-19T02:07:04.819180+0000 (oldest deadline 2024-03-19T02:07:27.719305+0000) 2024-03-19T02:07:56.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:56 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:56.168+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6812 osd.5 since back 2024-03-19T02:07:13.521002+0000 front 2024-03-19T02:07:13.520954+0000 (oldest deadline 2024-03-19T02:07:35.821183+0000) 2024-03-19T02:07:56.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:56 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:56.168+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6822 osd.6 since back 2024-03-19T02:07:18.122266+0000 front 2024-03-19T02:07:18.122291+0000 (oldest deadline 2024-03-19T02:07:43.422218+0000) 2024-03-19T02:07:56.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:56 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:56.168+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6820 osd.7 since back 2024-03-19T02:07:29.824127+0000 front 2024-03-19T02:07:29.824052+0000 (oldest deadline 2024-03-19T02:07:52.624568+0000) 2024-03-19T02:07:56.612 INFO:journalctl@ceph.osd.10.smithi112.stdout:Mar 19 02:07:56 smithi112 podman[107236]: 2024-03-19 02:07:56.332188694 +0000 UTC m=+0.673113670 container died 6a97c39e833228d1bae5135ebb2837702e6faca2263d6df7836c0869873ec5e8 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-10-deactivate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240307, io.buildah.version=1.33.5, ceph=True, org.label-schema.license=GPLv2, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30) 2024-03-19T02:07:56.881 DEBUG:teuthology.orchestra.run.smithi112:> sudo pkill -f 'journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@osd.10.service' 2024-03-19T02:07:56.964 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-19T02:07:56.964 INFO:tasks.cephadm.osd.10:Stopped osd.10 2024-03-19T02:07:56.965 INFO:tasks.cephadm.osd.11:Stopping osd.11... 2024-03-19T02:07:56.965 DEBUG:teuthology.orchestra.run.smithi115:> sudo systemctl stop ceph-438fb69c-e590-11ee-95c9-87774f69a715@osd.11 2024-03-19T02:07:57.251 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:57 smithi115 systemd[1]: Stopping Ceph osd.11 for 438fb69c-e590-11ee-95c9-87774f69a715... 2024-03-19T02:07:57.251 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:57 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:57.207+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:57.251 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:57 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:57.207+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:07:57.251 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:57 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:57.207+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.116306+0000 front 2024-03-19T02:06:46.116324+0000 (oldest deadline 2024-03-19T02:07:12.016484+0000) 2024-03-19T02:07:57.251 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:57 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:57.207+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.819160+0000 front 2024-03-19T02:07:04.819180+0000 (oldest deadline 2024-03-19T02:07:27.719305+0000) 2024-03-19T02:07:57.252 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:57 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:57.207+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6812 osd.5 since back 2024-03-19T02:07:13.521002+0000 front 2024-03-19T02:07:13.520954+0000 (oldest deadline 2024-03-19T02:07:35.821183+0000) 2024-03-19T02:07:57.252 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:57 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:57.207+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6822 osd.6 since back 2024-03-19T02:07:18.122266+0000 front 2024-03-19T02:07:18.122291+0000 (oldest deadline 2024-03-19T02:07:43.422218+0000) 2024-03-19T02:07:57.252 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:57 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:57.207+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6820 osd.7 since back 2024-03-19T02:07:29.824127+0000 front 2024-03-19T02:07:29.824052+0000 (oldest deadline 2024-03-19T02:07:52.624568+0000) 2024-03-19T02:07:57.252 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:57 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:57.250+0000 7fd8c5bc4700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.11 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-19T02:07:57.252 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:57 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:57.250+0000 7fd8c5bc4700 -1 osd.11 95 *** Got signal Terminated *** 2024-03-19T02:07:57.252 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:57 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:57.250+0000 7fd8c5bc4700 -1 osd.11 95 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-19T02:07:58.556 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:58 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:58.163+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:58.556 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:58 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:58.163+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:07:58.556 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:58 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:58.163+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.116306+0000 front 2024-03-19T02:06:46.116324+0000 (oldest deadline 2024-03-19T02:07:12.016484+0000) 2024-03-19T02:07:58.556 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:58 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:58.163+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.819160+0000 front 2024-03-19T02:07:04.819180+0000 (oldest deadline 2024-03-19T02:07:27.719305+0000) 2024-03-19T02:07:58.556 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:58 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:58.163+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6812 osd.5 since back 2024-03-19T02:07:13.521002+0000 front 2024-03-19T02:07:13.520954+0000 (oldest deadline 2024-03-19T02:07:35.821183+0000) 2024-03-19T02:07:58.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:58 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:58.163+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6822 osd.6 since back 2024-03-19T02:07:18.122266+0000 front 2024-03-19T02:07:18.122291+0000 (oldest deadline 2024-03-19T02:07:43.422218+0000) 2024-03-19T02:07:58.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:58 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:58.163+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6820 osd.7 since back 2024-03-19T02:07:29.824127+0000 front 2024-03-19T02:07:29.824052+0000 (oldest deadline 2024-03-19T02:07:52.624568+0000) 2024-03-19T02:07:59.556 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:59 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:59.119+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:07:59.556 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:59 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:59.119+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:07:59.556 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:59 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:59.119+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.116306+0000 front 2024-03-19T02:06:46.116324+0000 (oldest deadline 2024-03-19T02:07:12.016484+0000) 2024-03-19T02:07:59.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:59 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:59.119+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.819160+0000 front 2024-03-19T02:07:04.819180+0000 (oldest deadline 2024-03-19T02:07:27.719305+0000) 2024-03-19T02:07:59.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:59 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:59.119+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6812 osd.5 since back 2024-03-19T02:07:13.521002+0000 front 2024-03-19T02:07:13.520954+0000 (oldest deadline 2024-03-19T02:07:35.821183+0000) 2024-03-19T02:07:59.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:59 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:59.119+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6822 osd.6 since back 2024-03-19T02:07:18.122266+0000 front 2024-03-19T02:07:18.122291+0000 (oldest deadline 2024-03-19T02:07:43.422218+0000) 2024-03-19T02:07:59.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:07:59 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:07:59.119+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6820 osd.7 since back 2024-03-19T02:07:29.824127+0000 front 2024-03-19T02:07:29.824052+0000 (oldest deadline 2024-03-19T02:07:52.624568+0000) 2024-03-19T02:08:00.556 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:08:00 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:08:00.119+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:08:00.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:08:00 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:08:00.119+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:08:00.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:08:00 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:08:00.119+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.116306+0000 front 2024-03-19T02:06:46.116324+0000 (oldest deadline 2024-03-19T02:07:12.016484+0000) 2024-03-19T02:08:00.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:08:00 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:08:00.119+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.819160+0000 front 2024-03-19T02:07:04.819180+0000 (oldest deadline 2024-03-19T02:07:27.719305+0000) 2024-03-19T02:08:00.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:08:00 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:08:00.119+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6812 osd.5 since back 2024-03-19T02:07:13.521002+0000 front 2024-03-19T02:07:13.520954+0000 (oldest deadline 2024-03-19T02:07:35.821183+0000) 2024-03-19T02:08:00.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:08:00 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:08:00.119+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6822 osd.6 since back 2024-03-19T02:07:18.122266+0000 front 2024-03-19T02:07:18.122291+0000 (oldest deadline 2024-03-19T02:07:43.422218+0000) 2024-03-19T02:08:00.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:08:00 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:08:00.119+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6820 osd.7 since back 2024-03-19T02:07:29.824127+0000 front 2024-03-19T02:07:29.824052+0000 (oldest deadline 2024-03-19T02:07:52.624568+0000) 2024-03-19T02:08:01.556 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:08:01 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:08:01.083+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:08:01.556 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:08:01 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:08:01.083+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:08:01.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:08:01 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:08:01.083+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.116306+0000 front 2024-03-19T02:06:46.116324+0000 (oldest deadline 2024-03-19T02:07:12.016484+0000) 2024-03-19T02:08:01.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:08:01 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:08:01.083+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.819160+0000 front 2024-03-19T02:07:04.819180+0000 (oldest deadline 2024-03-19T02:07:27.719305+0000) 2024-03-19T02:08:01.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:08:01 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:08:01.083+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6812 osd.5 since back 2024-03-19T02:07:13.521002+0000 front 2024-03-19T02:07:13.520954+0000 (oldest deadline 2024-03-19T02:07:35.821183+0000) 2024-03-19T02:08:01.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:08:01 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:08:01.083+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6822 osd.6 since back 2024-03-19T02:07:18.122266+0000 front 2024-03-19T02:07:18.122291+0000 (oldest deadline 2024-03-19T02:07:43.422218+0000) 2024-03-19T02:08:01.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:08:01 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:08:01.083+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6820 osd.7 since back 2024-03-19T02:07:29.824127+0000 front 2024-03-19T02:07:29.824052+0000 (oldest deadline 2024-03-19T02:07:52.624568+0000) 2024-03-19T02:08:01.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:08:01 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:08:01.083+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6820 osd.8 since back 2024-03-19T02:07:36.725592+0000 front 2024-03-19T02:07:36.725610+0000 (oldest deadline 2024-03-19T02:08:00.225715+0000) 2024-03-19T02:08:02.556 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:08:02 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:08:02.065+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-03-19T02:06:30.313925+0000 front 2024-03-19T02:06:30.313826+0000 (oldest deadline 2024-03-19T02:06:57.914663+0000) 2024-03-19T02:08:02.556 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:08:02 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:08:02.065+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6804 osd.1 since back 2024-03-19T02:06:40.815881+0000 front 2024-03-19T02:06:40.815831+0000 (oldest deadline 2024-03-19T02:07:06.115979+0000) 2024-03-19T02:08:02.556 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:08:02 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:08:02.065+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6804 osd.2 since back 2024-03-19T02:06:46.116306+0000 front 2024-03-19T02:06:46.116324+0000 (oldest deadline 2024-03-19T02:07:12.016484+0000) 2024-03-19T02:08:02.556 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:08:02 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:08:02.065+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6812 osd.4 since back 2024-03-19T02:07:04.819160+0000 front 2024-03-19T02:07:04.819180+0000 (oldest deadline 2024-03-19T02:07:27.719305+0000) 2024-03-19T02:08:02.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:08:02 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:08:02.065+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6812 osd.5 since back 2024-03-19T02:07:13.521002+0000 front 2024-03-19T02:07:13.520954+0000 (oldest deadline 2024-03-19T02:07:35.821183+0000) 2024-03-19T02:08:02.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:08:02 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:08:02.065+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.99:6822 osd.6 since back 2024-03-19T02:07:18.122266+0000 front 2024-03-19T02:07:18.122291+0000 (oldest deadline 2024-03-19T02:07:43.422218+0000) 2024-03-19T02:08:02.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:08:02 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:08:02.065+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.112:6820 osd.7 since back 2024-03-19T02:07:29.824127+0000 front 2024-03-19T02:07:29.824052+0000 (oldest deadline 2024-03-19T02:07:52.624568+0000) 2024-03-19T02:08:02.557 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:08:02 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11[92094]: 2024-03-19T02:08:02.065+0000 7fd8c21b8700 -1 osd.11 95 heartbeat_check: no reply from 172.21.15.115:6820 osd.8 since back 2024-03-19T02:07:36.725592+0000 front 2024-03-19T02:07:36.725610+0000 (oldest deadline 2024-03-19T02:08:00.225715+0000) 2024-03-19T02:08:02.974 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:08:02 smithi115 podman[106579]: 2024-03-19 02:08:02.689827325 +0000 UTC m=+5.538171570 container died bdb2160f4732de90f22b8375399bfa25dac236c0e29e6a6dc36c480410716f0f (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240307, GIT_BRANCH=HEAD, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30) 2024-03-19T02:08:02.974 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:08:02 smithi115 podman[106579]: 2024-03-19 02:08:02.975007532 +0000 UTC m=+5.823351749 container cleanup bdb2160f4732de90f22b8375399bfa25dac236c0e29e6a6dc36c480410716f0f (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.build-date=20240307, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-03-19T02:08:03.234 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:08:02 smithi115 bash[106579]: ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11 2024-03-19T02:08:03.556 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:08:03 smithi115 podman[106596]: 2024-03-19 02:08:03.234832361 +0000 UTC m=+0.542203995 container remove bdb2160f4732de90f22b8375399bfa25dac236c0e29e6a6dc36c480410716f0f (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11, GIT_BRANCH=HEAD, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.build-date=20240307, CEPH_POINT_RELEASE=, ceph=True, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-03-19T02:08:03.903 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:08:03 smithi115 podman[106685]: 2024-03-19 02:08:03.48634773 +0000 UTC m=+0.017811072 image pull 2312c8e42d0069a2dbb2364bddb00264db4b800e7f18a7e7e4eb8f3ecab2ff1c quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639 2024-03-19T02:08:03.904 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:08:03 smithi115 podman[106685]: 2024-03-19 02:08:03.602011094 +0000 UTC m=+0.133474429 container create e9c7bc378da338cbc7db6b279b1a06cbcf089dfdb3c43c6cac4e91e8cfd8f240 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11-deactivate, org.label-schema.build-date=20240307, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_CLEAN=True, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-03-19T02:08:03.904 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:08:03 smithi115 podman[106685]: 2024-03-19 02:08:03.829156089 +0000 UTC m=+0.360619446 container init e9c7bc378da338cbc7db6b279b1a06cbcf089dfdb3c43c6cac4e91e8cfd8f240 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240307, CEPH_POINT_RELEASE=) 2024-03-19T02:08:03.904 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:08:03 smithi115 podman[106685]: 2024-03-19 02:08:03.832223424 +0000 UTC m=+0.363686773 container start e9c7bc378da338cbc7db6b279b1a06cbcf089dfdb3c43c6cac4e91e8cfd8f240 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11-deactivate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.build-date=20240307, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.33.5) 2024-03-19T02:08:03.904 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:08:03 smithi115 podman[106685]: 2024-03-19 02:08:03.90404024 +0000 UTC m=+0.435503599 container attach e9c7bc378da338cbc7db6b279b1a06cbcf089dfdb3c43c6cac4e91e8cfd8f240 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11-deactivate, org.label-schema.build-date=20240307, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=wip-batrick-testing-20240318.181317-b6ae249, CEPH_POINT_RELEASE=) 2024-03-19T02:08:04.306 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:08:04 smithi115 podman[106685]: 2024-03-19 02:08:04.119024778 +0000 UTC m=+0.650488129 container died e9c7bc378da338cbc7db6b279b1a06cbcf089dfdb3c43c6cac4e91e8cfd8f240 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11-deactivate, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240307) 2024-03-19T02:08:04.650 INFO:journalctl@ceph.osd.11.smithi115.stdout:Mar 19 02:08:04 smithi115 podman[106711]: 2024-03-19 02:08:04.648679383 +0000 UTC m=+0.524561598 container remove e9c7bc378da338cbc7db6b279b1a06cbcf089dfdb3c43c6cac4e91e8cfd8f240 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-osd-11-deactivate, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240307, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-03-19T02:08:04.662 DEBUG:teuthology.orchestra.run.smithi115:> sudo pkill -f 'journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@osd.11.service' 2024-03-19T02:08:04.703 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-19T02:08:04.703 INFO:tasks.cephadm.osd.11:Stopped osd.11 2024-03-19T02:08:04.703 INFO:tasks.cephadm.mds.a:Stopping mds.a... 2024-03-19T02:08:04.703 DEBUG:teuthology.orchestra.run.smithi099:> sudo systemctl stop ceph-438fb69c-e590-11ee-95c9-87774f69a715@mds.a 2024-03-19T02:08:05.090 INFO:journalctl@ceph.mds.a.smithi099.stdout:Mar 19 02:08:04 smithi099 systemd[1]: Stopping Ceph mds.a for 438fb69c-e590-11ee-95c9-87774f69a715... 2024-03-19T02:08:05.590 INFO:journalctl@ceph.mds.a.smithi099.stdout:Mar 19 02:08:05 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-a[104734]: 2024-03-19T02:08:05.103+0000 7f12cbb99700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-19T02:08:05.590 INFO:journalctl@ceph.mds.a.smithi099.stdout:Mar 19 02:08:05 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-a[104734]: 2024-03-19T02:08:05.103+0000 7f12cbb99700 -1 mds.a *** got signal Terminated *** 2024-03-19T02:08:07.678 INFO:journalctl@ceph.mds.a.smithi099.stdout:Mar 19 02:08:07 smithi099 podman[127728]: 2024-03-19 02:08:07.346669799 +0000 UTC m=+2.448076079 container stop 4b3805d58da22fca839a8e7fa803a6033ce20cac387521647211070b8f5d9c2d (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-a, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.build-date=20240307, org.label-schema.schema-version=1.0, ceph=True, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.vendor=CentOS) 2024-03-19T02:08:07.678 INFO:journalctl@ceph.mds.a.smithi099.stdout:Mar 19 02:08:07 smithi099 podman[127728]: 2024-03-19 02:08:07.372323286 +0000 UTC m=+2.473729580 container died 4b3805d58da22fca839a8e7fa803a6033ce20cac387521647211070b8f5d9c2d (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-a, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.build-date=20240307, org.label-schema.vendor=CentOS, RELEASE=wip-batrick-testing-20240318.181317-b6ae249) 2024-03-19T02:08:08.090 INFO:journalctl@ceph.mds.a.smithi099.stdout:Mar 19 02:08:07 smithi099 podman[127728]: 2024-03-19 02:08:07.76168841 +0000 UTC m=+2.863094688 container cleanup 4b3805d58da22fca839a8e7fa803a6033ce20cac387521647211070b8f5d9c2d (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-a, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240307, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, io.buildah.version=1.33.5) 2024-03-19T02:08:08.090 INFO:journalctl@ceph.mds.a.smithi099.stdout:Mar 19 02:08:07 smithi099 bash[127728]: ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-a 2024-03-19T02:08:08.090 INFO:journalctl@ceph.mds.a.smithi099.stdout:Mar 19 02:08:07 smithi099 podman[127743]: 2024-03-19 02:08:07.912941677 +0000 UTC m=+0.559738851 container remove 4b3805d58da22fca839a8e7fa803a6033ce20cac387521647211070b8f5d9c2d (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-a, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.build-date=20240307, org.label-schema.license=GPLv2, RELEASE=wip-batrick-testing-20240318.181317-b6ae249) 2024-03-19T02:08:08.126 DEBUG:teuthology.orchestra.run.smithi099:> sudo pkill -f 'journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@mds.a.service' 2024-03-19T02:08:08.164 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-19T02:08:08.164 INFO:tasks.cephadm.mds.a:Stopped mds.a 2024-03-19T02:08:08.164 INFO:tasks.cephadm.mds.d:Stopping mds.d... 2024-03-19T02:08:08.165 DEBUG:teuthology.orchestra.run.smithi099:> sudo systemctl stop ceph-438fb69c-e590-11ee-95c9-87774f69a715@mds.d 2024-03-19T02:08:08.481 INFO:journalctl@ceph.mds.d.smithi099.stdout:Mar 19 02:08:08 smithi099 systemd[1]: Stopping Ceph mds.d for 438fb69c-e590-11ee-95c9-87774f69a715... 2024-03-19T02:08:08.840 INFO:journalctl@ceph.mds.d.smithi099.stdout:Mar 19 02:08:08 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-d[103781]: 2024-03-19T02:08:08.481+0000 7f4794865700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.d -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-19T02:08:08.840 INFO:journalctl@ceph.mds.d.smithi099.stdout:Mar 19 02:08:08 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-d[103781]: 2024-03-19T02:08:08.481+0000 7f4794865700 -1 mds.d *** got signal Terminated *** 2024-03-19T02:08:12.840 INFO:journalctl@ceph.mds.d.smithi099.stdout:Mar 19 02:08:12 smithi099 podman[127852]: 2024-03-19 02:08:12.493177072 +0000 UTC m=+4.113876259 container died 3051f64272da14623a4fdedb0aea3b46e2dc28e0e793865d93e6baf863e49fcf (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-d, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240307, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-03-19T02:08:13.340 INFO:journalctl@ceph.mds.d.smithi099.stdout:Mar 19 02:08:13 smithi099 podman[127852]: 2024-03-19 02:08:13.098726993 +0000 UTC m=+4.719426189 container cleanup 3051f64272da14623a4fdedb0aea3b46e2dc28e0e793865d93e6baf863e49fcf (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-d, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=wip-batrick-testing-20240318.181317-b6ae249, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.build-date=20240307, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-03-19T02:08:13.341 INFO:journalctl@ceph.mds.d.smithi099.stdout:Mar 19 02:08:13 smithi099 bash[127852]: ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-d 2024-03-19T02:08:13.341 INFO:journalctl@ceph.mds.d.smithi099.stdout:Mar 19 02:08:13 smithi099 podman[127866]: 2024-03-19 02:08:13.266617955 +0000 UTC m=+0.770951196 container remove 3051f64272da14623a4fdedb0aea3b46e2dc28e0e793865d93e6baf863e49fcf (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-d, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240307, org.label-schema.license=GPLv2, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS) 2024-03-19T02:08:13.483 DEBUG:teuthology.orchestra.run.smithi099:> sudo pkill -f 'journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@mds.d.service' 2024-03-19T02:08:13.518 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-19T02:08:13.518 INFO:tasks.cephadm.mds.d:Stopped mds.d 2024-03-19T02:08:13.518 INFO:tasks.cephadm.mds.g:Stopping mds.g... 2024-03-19T02:08:13.518 DEBUG:teuthology.orchestra.run.smithi099:> sudo systemctl stop ceph-438fb69c-e590-11ee-95c9-87774f69a715@mds.g 2024-03-19T02:08:13.841 INFO:journalctl@ceph.mds.g.smithi099.stdout:Mar 19 02:08:13 smithi099 systemd[1]: Stopping Ceph mds.g for 438fb69c-e590-11ee-95c9-87774f69a715... 2024-03-19T02:08:14.340 INFO:journalctl@ceph.mds.g.smithi099.stdout:Mar 19 02:08:13 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-g[104386]: 2024-03-19T02:08:13.843+0000 7fb935c1d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.g -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-19T02:08:14.341 INFO:journalctl@ceph.mds.g.smithi099.stdout:Mar 19 02:08:13 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-g[104386]: 2024-03-19T02:08:13.843+0000 7fb935c1d700 -1 mds.g *** got signal Terminated *** 2024-03-19T02:08:16.978 INFO:journalctl@ceph.mds.g.smithi099.stdout:Mar 19 02:08:16 smithi099 podman[127972]: 2024-03-19 02:08:16.672066396 +0000 UTC m=+2.934442819 container died f397a20a10888b49909a5be7e4d609542647d10ce1cfeb60a3357664293dc19e (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-g, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240307, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-03-19T02:08:17.329 INFO:journalctl@ceph.mds.g.smithi099.stdout:Mar 19 02:08:17 smithi099 podman[127972]: 2024-03-19 02:08:17.060403131 +0000 UTC m=+3.322779559 container cleanup f397a20a10888b49909a5be7e4d609542647d10ce1cfeb60a3357664293dc19e (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-g, GIT_BRANCH=HEAD, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240307, io.buildah.version=1.33.5) 2024-03-19T02:08:17.330 INFO:journalctl@ceph.mds.g.smithi099.stdout:Mar 19 02:08:17 smithi099 bash[127972]: ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-g 2024-03-19T02:08:17.330 INFO:journalctl@ceph.mds.g.smithi099.stdout:Mar 19 02:08:17 smithi099 podman[127986]: 2024-03-19 02:08:17.228269335 +0000 UTC m=+0.545820424 container remove f397a20a10888b49909a5be7e4d609542647d10ce1cfeb60a3357664293dc19e (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-g, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240307, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-19T02:08:17.344 DEBUG:teuthology.orchestra.run.smithi099:> sudo pkill -f 'journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@mds.g.service' 2024-03-19T02:08:17.380 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-19T02:08:17.381 INFO:tasks.cephadm.mds.g:Stopped mds.g 2024-03-19T02:08:17.381 INFO:tasks.cephadm.mds.j:Stopping mds.j... 2024-03-19T02:08:17.381 DEBUG:teuthology.orchestra.run.smithi099:> sudo systemctl stop ceph-438fb69c-e590-11ee-95c9-87774f69a715@mds.j 2024-03-19T02:08:17.696 INFO:journalctl@ceph.mds.j.smithi099.stdout:Mar 19 02:08:17 smithi099 systemd[1]: Stopping Ceph mds.j for 438fb69c-e590-11ee-95c9-87774f69a715... 2024-03-19T02:08:18.090 INFO:journalctl@ceph.mds.j.smithi099.stdout:Mar 19 02:08:17 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-j[103434]: 2024-03-19T02:08:17.697+0000 7f1c188f9700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.j -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-19T02:08:18.090 INFO:journalctl@ceph.mds.j.smithi099.stdout:Mar 19 02:08:17 smithi099 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-j[103434]: 2024-03-19T02:08:17.697+0000 7f1c188f9700 -1 mds.j *** got signal Terminated *** 2024-03-19T02:08:22.063 INFO:journalctl@ceph.mds.j.smithi099.stdout:Mar 19 02:08:21 smithi099 podman[128093]: 2024-03-19 02:08:21.713696313 +0000 UTC m=+4.122375969 container died ca9adb08a0032a36ab7bf0f5ba80051f7f04a9b2d23aba5bb3fa647b91cb0345 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-j, org.label-schema.build-date=20240307, io.buildah.version=1.33.5, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-03-19T02:08:22.340 INFO:journalctl@ceph.mds.j.smithi099.stdout:Mar 19 02:08:22 smithi099 podman[128093]: 2024-03-19 02:08:22.155631348 +0000 UTC m=+4.564311005 container cleanup ca9adb08a0032a36ab7bf0f5ba80051f7f04a9b2d23aba5bb3fa647b91cb0345 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-j, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, io.buildah.version=1.33.5, ceph=True, org.label-schema.build-date=20240307, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-03-19T02:08:22.340 INFO:journalctl@ceph.mds.j.smithi099.stdout:Mar 19 02:08:22 smithi099 bash[128093]: ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-j 2024-03-19T02:08:22.569 DEBUG:teuthology.orchestra.run.smithi099:> sudo pkill -f 'journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@mds.j.service' 2024-03-19T02:08:22.603 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-19T02:08:22.603 INFO:tasks.cephadm.mds.j:Stopped mds.j 2024-03-19T02:08:22.604 INFO:tasks.cephadm.mds.b:Stopping mds.b... 2024-03-19T02:08:22.604 DEBUG:teuthology.orchestra.run.smithi112:> sudo systemctl stop ceph-438fb69c-e590-11ee-95c9-87774f69a715@mds.b 2024-03-19T02:08:22.900 INFO:journalctl@ceph.mds.b.smithi112.stdout:Mar 19 02:08:22 smithi112 systemd[1]: Stopping Ceph mds.b for 438fb69c-e590-11ee-95c9-87774f69a715... 2024-03-19T02:08:23.154 INFO:journalctl@ceph.mds.b.smithi112.stdout:Mar 19 02:08:22 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-b[97403]: 2024-03-19T02:08:22.898+0000 7fda4e37b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-19T02:08:23.154 INFO:journalctl@ceph.mds.b.smithi112.stdout:Mar 19 02:08:22 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-b[97403]: 2024-03-19T02:08:22.898+0000 7fda4e37b700 -1 mds.b *** got signal Terminated *** 2024-03-19T02:08:26.734 INFO:journalctl@ceph.mds.b.smithi112.stdout:Mar 19 02:08:26 smithi112 podman[107344]: 2024-03-19 02:08:26.466589909 +0000 UTC m=+3.670782440 container died de00165d1e05d16e711ff811983f3ce64e232cb4953d8b1ef7eda6e813b1ae81 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-b, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.schema-version=1.0, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.build-date=20240307, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-19T02:08:27.083 INFO:journalctl@ceph.mds.b.smithi112.stdout:Mar 19 02:08:26 smithi112 podman[107344]: 2024-03-19 02:08:26.832723949 +0000 UTC m=+4.036916480 container cleanup de00165d1e05d16e711ff811983f3ce64e232cb4953d8b1ef7eda6e813b1ae81 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-b, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240307, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, CEPH_POINT_RELEASE=) 2024-03-19T02:08:27.083 INFO:journalctl@ceph.mds.b.smithi112.stdout:Mar 19 02:08:26 smithi112 bash[107344]: ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-b 2024-03-19T02:08:27.213 DEBUG:teuthology.orchestra.run.smithi112:> sudo pkill -f 'journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@mds.b.service' 2024-03-19T02:08:27.254 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-19T02:08:27.254 INFO:tasks.cephadm.mds.b:Stopped mds.b 2024-03-19T02:08:27.255 INFO:tasks.cephadm.mds.e:Stopping mds.e... 2024-03-19T02:08:27.255 DEBUG:teuthology.orchestra.run.smithi112:> sudo systemctl stop ceph-438fb69c-e590-11ee-95c9-87774f69a715@mds.e 2024-03-19T02:08:27.611 INFO:journalctl@ceph.mds.e.smithi112.stdout:Mar 19 02:08:27 smithi112 systemd[1]: Stopping Ceph mds.e for 438fb69c-e590-11ee-95c9-87774f69a715... 2024-03-19T02:08:27.612 INFO:journalctl@ceph.mds.e.smithi112.stdout:Mar 19 02:08:27 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-e[96773]: 2024-03-19T02:08:27.577+0000 7f6054622700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.e -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-19T02:08:27.612 INFO:journalctl@ceph.mds.e.smithi112.stdout:Mar 19 02:08:27 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-e[96773]: 2024-03-19T02:08:27.577+0000 7f6054622700 -1 mds.e *** got signal Terminated *** 2024-03-19T02:08:30.714 INFO:journalctl@ceph.mds.e.smithi112.stdout:Mar 19 02:08:30 smithi112 podman[107476]: 2024-03-19 02:08:30.456985721 +0000 UTC m=+2.983443805 container died 3378ef045bc81c8fad3b97487361d96b709ccb0f59097b080e6356d3ad8ca902 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-e, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240307, CEPH_POINT_RELEASE=) 2024-03-19T02:08:30.995 INFO:journalctl@ceph.mds.e.smithi112.stdout:Mar 19 02:08:30 smithi112 podman[107476]: 2024-03-19 02:08:30.811137197 +0000 UTC m=+3.337595282 container cleanup 3378ef045bc81c8fad3b97487361d96b709ccb0f59097b080e6356d3ad8ca902 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-e, org.label-schema.build-date=20240307, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5) 2024-03-19T02:08:30.995 INFO:journalctl@ceph.mds.e.smithi112.stdout:Mar 19 02:08:30 smithi112 bash[107476]: ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-e 2024-03-19T02:08:31.232 DEBUG:teuthology.orchestra.run.smithi112:> sudo pkill -f 'journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@mds.e.service' 2024-03-19T02:08:31.255 INFO:journalctl@ceph.mds.e.smithi112.stdout:Mar 19 02:08:30 smithi112 podman[107490]: 2024-03-19 02:08:30.995571837 +0000 UTC m=+0.538346931 container remove 3378ef045bc81c8fad3b97487361d96b709ccb0f59097b080e6356d3ad8ca902 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-e, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240307, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, ceph=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30) 2024-03-19T02:08:31.255 INFO:journalctl@ceph.mds.e.smithi112.stdout:Mar 19 02:08:31 smithi112 systemd[1]: ceph-438fb69c-e590-11ee-95c9-87774f69a715@mds.e.service: Deactivated successfully. 2024-03-19T02:08:31.255 INFO:journalctl@ceph.mds.e.smithi112.stdout:Mar 19 02:08:31 smithi112 systemd[1]: Stopped Ceph mds.e for 438fb69c-e590-11ee-95c9-87774f69a715. 2024-03-19T02:08:31.256 INFO:journalctl@ceph.mds.e.smithi112.stdout:Mar 19 02:08:31 smithi112 systemd[1]: ceph-438fb69c-e590-11ee-95c9-87774f69a715@mds.e.service: Consumed 56.076s CPU time. 2024-03-19T02:08:31.267 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-19T02:08:31.267 INFO:tasks.cephadm.mds.e:Stopped mds.e 2024-03-19T02:08:31.267 INFO:tasks.cephadm.mds.h:Stopping mds.h... 2024-03-19T02:08:31.267 DEBUG:teuthology.orchestra.run.smithi112:> sudo systemctl stop ceph-438fb69c-e590-11ee-95c9-87774f69a715@mds.h 2024-03-19T02:08:31.589 INFO:journalctl@ceph.mds.h.smithi112.stdout:Mar 19 02:08:31 smithi112 systemd[1]: Stopping Ceph mds.h for 438fb69c-e590-11ee-95c9-87774f69a715... 2024-03-19T02:08:31.861 INFO:journalctl@ceph.mds.h.smithi112.stdout:Mar 19 02:08:31 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-h[97080]: 2024-03-19T02:08:31.590+0000 7f252fc0f700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.h -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-19T02:08:31.862 INFO:journalctl@ceph.mds.h.smithi112.stdout:Mar 19 02:08:31 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-h[97080]: 2024-03-19T02:08:31.590+0000 7f252fc0f700 -1 mds.h *** got signal Terminated *** 2024-03-19T02:08:34.715 INFO:journalctl@ceph.mds.h.smithi112.stdout:Mar 19 02:08:34 smithi112 podman[107598]: 2024-03-19 02:08:34.458909746 +0000 UTC m=+2.966628196 container died 1b8f536d04b2ecd12906ef8b2dfaf3f65b8543ba65829a593a2e204f706f91d0 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-h, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.build-date=20240307, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30) 2024-03-19T02:08:34.965 INFO:journalctl@ceph.mds.h.smithi112.stdout:Mar 19 02:08:34 smithi112 podman[107598]: 2024-03-19 02:08:34.781819531 +0000 UTC m=+3.289537980 container cleanup 1b8f536d04b2ecd12906ef8b2dfaf3f65b8543ba65829a593a2e204f706f91d0 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-h, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.33.5, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240307, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30) 2024-03-19T02:08:34.966 INFO:journalctl@ceph.mds.h.smithi112.stdout:Mar 19 02:08:34 smithi112 bash[107598]: ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-h 2024-03-19T02:08:35.178 DEBUG:teuthology.orchestra.run.smithi112:> sudo pkill -f 'journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@mds.h.service' 2024-03-19T02:08:35.214 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-19T02:08:35.214 INFO:tasks.cephadm.mds.h:Stopped mds.h 2024-03-19T02:08:35.217 INFO:tasks.cephadm.mds.k:Stopping mds.k... 2024-03-19T02:08:35.218 DEBUG:teuthology.orchestra.run.smithi112:> sudo systemctl stop ceph-438fb69c-e590-11ee-95c9-87774f69a715@mds.k 2024-03-19T02:08:35.535 INFO:journalctl@ceph.mds.k.smithi112.stdout:Mar 19 02:08:35 smithi112 systemd[1]: Stopping Ceph mds.k for 438fb69c-e590-11ee-95c9-87774f69a715... 2024-03-19T02:08:35.862 INFO:journalctl@ceph.mds.k.smithi112.stdout:Mar 19 02:08:35 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-k[96466]: 2024-03-19T02:08:35.535+0000 7f122643e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.k -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-19T02:08:35.862 INFO:journalctl@ceph.mds.k.smithi112.stdout:Mar 19 02:08:35 smithi112 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-k[96466]: 2024-03-19T02:08:35.535+0000 7f122643e700 -1 mds.k *** got signal Terminated *** 2024-03-19T02:08:39.710 INFO:journalctl@ceph.mds.k.smithi112.stdout:Mar 19 02:08:39 smithi112 podman[107718]: 2024-03-19 02:08:39.438736963 +0000 UTC m=+4.010004848 container died 20eddeea6ac326bfd03b1e87140250fd370b351ab039147839bfadf141893f8e (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-k, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240307, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30) 2024-03-19T02:08:39.961 INFO:journalctl@ceph.mds.k.smithi112.stdout:Mar 19 02:08:39 smithi112 podman[107718]: 2024-03-19 02:08:39.785341147 +0000 UTC m=+4.356609062 container cleanup 20eddeea6ac326bfd03b1e87140250fd370b351ab039147839bfadf141893f8e (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-k, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240307, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-03-19T02:08:39.961 INFO:journalctl@ceph.mds.k.smithi112.stdout:Mar 19 02:08:39 smithi112 bash[107718]: ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-k 2024-03-19T02:08:40.204 DEBUG:teuthology.orchestra.run.smithi112:> sudo pkill -f 'journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@mds.k.service' 2024-03-19T02:08:40.227 INFO:journalctl@ceph.mds.k.smithi112.stdout:Mar 19 02:08:39 smithi112 podman[107732]: 2024-03-19 02:08:39.961286861 +0000 UTC m=+0.515875391 container remove 20eddeea6ac326bfd03b1e87140250fd370b351ab039147839bfadf141893f8e (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-k, org.label-schema.build-date=20240307, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-03-19T02:08:40.227 INFO:journalctl@ceph.mds.k.smithi112.stdout:Mar 19 02:08:40 smithi112 systemd[1]: ceph-438fb69c-e590-11ee-95c9-87774f69a715@mds.k.service: Deactivated successfully. 2024-03-19T02:08:40.227 INFO:journalctl@ceph.mds.k.smithi112.stdout:Mar 19 02:08:40 smithi112 systemd[1]: Stopped Ceph mds.k for 438fb69c-e590-11ee-95c9-87774f69a715. 2024-03-19T02:08:40.227 INFO:journalctl@ceph.mds.k.smithi112.stdout:Mar 19 02:08:40 smithi112 systemd[1]: ceph-438fb69c-e590-11ee-95c9-87774f69a715@mds.k.service: Consumed 1.109s CPU time. 2024-03-19T02:08:40.238 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-19T02:08:40.239 INFO:tasks.cephadm.mds.k:Stopped mds.k 2024-03-19T02:08:40.239 INFO:tasks.cephadm.mds.c:Stopping mds.c... 2024-03-19T02:08:40.239 DEBUG:teuthology.orchestra.run.smithi115:> sudo systemctl stop ceph-438fb69c-e590-11ee-95c9-87774f69a715@mds.c 2024-03-19T02:08:40.522 INFO:journalctl@ceph.mds.c.smithi115.stdout:Mar 19 02:08:40 smithi115 systemd[1]: Stopping Ceph mds.c for 438fb69c-e590-11ee-95c9-87774f69a715... 2024-03-19T02:08:40.806 INFO:journalctl@ceph.mds.c.smithi115.stdout:Mar 19 02:08:40 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-c[97153]: 2024-03-19T02:08:40.521+0000 7f8aa91e9700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-19T02:08:40.806 INFO:journalctl@ceph.mds.c.smithi115.stdout:Mar 19 02:08:40 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-c[97153]: 2024-03-19T02:08:40.521+0000 7f8aa91e9700 -1 mds.c *** got signal Terminated *** 2024-03-19T02:08:42.806 INFO:journalctl@ceph.mds.c.smithi115.stdout:Mar 19 02:08:42 smithi115 podman[106791]: 2024-03-19 02:08:42.50161036 +0000 UTC m=+2.073149073 container died ff3f5e67fed9cbb582e3ab4f836771004c3f1a5c0c76d93d802b3b3610630173 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-c, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240307, io.buildah.version=1.33.5, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-03-19T02:08:43.266 INFO:journalctl@ceph.mds.c.smithi115.stdout:Mar 19 02:08:42 smithi115 podman[106791]: 2024-03-19 02:08:42.914023454 +0000 UTC m=+2.485562175 container cleanup ff3f5e67fed9cbb582e3ab4f836771004c3f1a5c0c76d93d802b3b3610630173 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-c, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, GIT_BRANCH=HEAD, org.label-schema.build-date=20240307, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5) 2024-03-19T02:08:43.266 INFO:journalctl@ceph.mds.c.smithi115.stdout:Mar 19 02:08:42 smithi115 bash[106791]: ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-c 2024-03-19T02:08:43.266 INFO:journalctl@ceph.mds.c.smithi115.stdout:Mar 19 02:08:43 smithi115 podman[106814]: 2024-03-19 02:08:43.081794676 +0000 UTC m=+0.579170595 container remove ff3f5e67fed9cbb582e3ab4f836771004c3f1a5c0c76d93d802b3b3610630173 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-c, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, io.buildah.version=1.33.5, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240307, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD) 2024-03-19T02:08:43.278 DEBUG:teuthology.orchestra.run.smithi115:> sudo pkill -f 'journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@mds.c.service' 2024-03-19T02:08:43.317 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-19T02:08:43.317 INFO:tasks.cephadm.mds.c:Stopped mds.c 2024-03-19T02:08:43.317 INFO:tasks.cephadm.mds.f:Stopping mds.f... 2024-03-19T02:08:43.317 DEBUG:teuthology.orchestra.run.smithi115:> sudo systemctl stop ceph-438fb69c-e590-11ee-95c9-87774f69a715@mds.f 2024-03-19T02:08:43.556 INFO:journalctl@ceph.mds.f.smithi115.stdout:Mar 19 02:08:43 smithi115 systemd[1]: Stopping Ceph mds.f for 438fb69c-e590-11ee-95c9-87774f69a715... 2024-03-19T02:08:44.056 INFO:journalctl@ceph.mds.f.smithi115.stdout:Mar 19 02:08:43 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-f[96845]: 2024-03-19T02:08:43.684+0000 7f70228d3700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.f -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-19T02:08:44.056 INFO:journalctl@ceph.mds.f.smithi115.stdout:Mar 19 02:08:43 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-f[96845]: 2024-03-19T02:08:43.684+0000 7f70228d3700 -1 mds.f *** got signal Terminated *** 2024-03-19T02:08:46.769 INFO:journalctl@ceph.mds.f.smithi115.stdout:Mar 19 02:08:46 smithi115 podman[106925]: 2024-03-19 02:08:46.504604555 +0000 UTC m=+2.972557860 container died 8b886dd7bfadaf5c445cd81d1af10c7064a66a24cfe43d7ddc0e456b87a80206 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-f, org.label-schema.build-date=20240307, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-03-19T02:08:47.056 INFO:journalctl@ceph.mds.f.smithi115.stdout:Mar 19 02:08:46 smithi115 podman[106925]: 2024-03-19 02:08:46.850814275 +0000 UTC m=+3.318767571 container cleanup 8b886dd7bfadaf5c445cd81d1af10c7064a66a24cfe43d7ddc0e456b87a80206 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-f, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240307, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.33.5, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, ceph=True) 2024-03-19T02:08:47.056 INFO:journalctl@ceph.mds.f.smithi115.stdout:Mar 19 02:08:46 smithi115 bash[106925]: ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-f 2024-03-19T02:08:47.273 DEBUG:teuthology.orchestra.run.smithi115:> sudo pkill -f 'journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@mds.f.service' 2024-03-19T02:08:47.310 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-19T02:08:47.310 INFO:tasks.cephadm.mds.f:Stopped mds.f 2024-03-19T02:08:47.310 INFO:tasks.cephadm.mds.i:Stopping mds.i... 2024-03-19T02:08:47.310 DEBUG:teuthology.orchestra.run.smithi115:> sudo systemctl stop ceph-438fb69c-e590-11ee-95c9-87774f69a715@mds.i 2024-03-19T02:08:47.556 INFO:journalctl@ceph.mds.i.smithi115.stdout:Mar 19 02:08:47 smithi115 systemd[1]: Stopping Ceph mds.i for 438fb69c-e590-11ee-95c9-87774f69a715... 2024-03-19T02:08:48.056 INFO:journalctl@ceph.mds.i.smithi115.stdout:Mar 19 02:08:47 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-i[97459]: 2024-03-19T02:08:47.665+0000 7fcff62d6700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.i -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-19T02:08:48.348 INFO:journalctl@ceph.mds.i.smithi115.stdout:Mar 19 02:08:47 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-i[97459]: 2024-03-19T02:08:47.665+0000 7fcff62d6700 -1 mds.i *** got signal Terminated *** 2024-03-19T02:08:50.806 INFO:journalctl@ceph.mds.i.smithi115.stdout:Mar 19 02:08:50 smithi115 podman[107048]: 2024-03-19 02:08:50.504462972 +0000 UTC m=+2.982800642 container died 03bb723802445487840469d87aff49344f197edfb94d992c33eff8a8ca3c3c65 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-i, org.label-schema.vendor=CentOS, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240307, io.buildah.version=1.33.5, ceph=True, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-03-19T02:08:51.138 INFO:journalctl@ceph.mds.i.smithi115.stdout:Mar 19 02:08:50 smithi115 podman[107048]: 2024-03-19 02:08:50.937816661 +0000 UTC m=+3.416154325 container cleanup 03bb723802445487840469d87aff49344f197edfb94d992c33eff8a8ca3c3c65 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-i, GIT_BRANCH=HEAD, org.label-schema.build-date=20240307, io.buildah.version=1.33.5, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, maintainer=Guillaume Abrioux ) 2024-03-19T02:08:51.138 INFO:journalctl@ceph.mds.i.smithi115.stdout:Mar 19 02:08:50 smithi115 bash[107048]: ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-i 2024-03-19T02:08:51.356 DEBUG:teuthology.orchestra.run.smithi115:> sudo pkill -f 'journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@mds.i.service' 2024-03-19T02:08:51.390 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-19T02:08:51.390 INFO:tasks.cephadm.mds.i:Stopped mds.i 2024-03-19T02:08:51.390 INFO:tasks.cephadm.mds.l:Stopping mds.l... 2024-03-19T02:08:51.390 DEBUG:teuthology.orchestra.run.smithi115:> sudo systemctl stop ceph-438fb69c-e590-11ee-95c9-87774f69a715@mds.l 2024-03-19T02:08:51.707 INFO:journalctl@ceph.mds.l.smithi115.stdout:Mar 19 02:08:51 smithi115 systemd[1]: Stopping Ceph mds.l for 438fb69c-e590-11ee-95c9-87774f69a715... 2024-03-19T02:08:52.056 INFO:journalctl@ceph.mds.l.smithi115.stdout:Mar 19 02:08:51 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-l[96536]: 2024-03-19T02:08:51.706+0000 7f408f9ac700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.l -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-19T02:08:52.056 INFO:journalctl@ceph.mds.l.smithi115.stdout:Mar 19 02:08:51 smithi115 ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-l[96536]: 2024-03-19T02:08:51.706+0000 7f408f9ac700 -1 mds.l *** got signal Terminated *** 2024-03-19T02:08:53.056 INFO:journalctl@ceph.mds.l.smithi115.stdout:Mar 19 02:08:52 smithi115 podman[107169]: 2024-03-19 02:08:52.813341534 +0000 UTC m=+1.210215558 container died 5547b1cc5bf1dc8362bce22f02cc601bb7ab6a1f75399567269f53ed1c811e88 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-l, RELEASE=wip-batrick-testing-20240318.181317-b6ae249, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240307, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-03-19T02:08:53.458 INFO:journalctl@ceph.mds.l.smithi115.stdout:Mar 19 02:08:53 smithi115 podman[107169]: 2024-03-19 02:08:53.174676985 +0000 UTC m=+1.571551003 container cleanup 5547b1cc5bf1dc8362bce22f02cc601bb7ab6a1f75399567269f53ed1c811e88 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c851f2a8983bc273fa71dd1a3e84be4b397f9c2099e180242b453d87e7c2b639, name=ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-l, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=wip-batrick-testing-20240318.181317-b6ae249, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.build-date=20240307, GIT_CLEAN=True) 2024-03-19T02:08:53.458 INFO:journalctl@ceph.mds.l.smithi115.stdout:Mar 19 02:08:53 smithi115 bash[107169]: ceph-438fb69c-e590-11ee-95c9-87774f69a715-mds-l 2024-03-19T02:08:53.582 DEBUG:teuthology.orchestra.run.smithi115:> sudo pkill -f 'journalctl -f -n 0 -u ceph-438fb69c-e590-11ee-95c9-87774f69a715@mds.l.service' 2024-03-19T02:08:53.618 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-19T02:08:53.618 INFO:tasks.cephadm.mds.l:Stopped mds.l 2024-03-19T02:08:53.618 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 438fb69c-e590-11ee-95c9-87774f69a715 --force --keep-logs 2024-03-19T02:08:53.809 INFO:teuthology.orchestra.run.smithi099.stdout:Deleting cluster with fsid: 438fb69c-e590-11ee-95c9-87774f69a715 2024-03-19T02:08:56.410 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 438fb69c-e590-11ee-95c9-87774f69a715 --force --keep-logs 2024-03-19T02:08:56.600 INFO:teuthology.orchestra.run.smithi112.stdout:Deleting cluster with fsid: 438fb69c-e590-11ee-95c9-87774f69a715 2024-03-19T02:08:59.084 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 438fb69c-e590-11ee-95c9-87774f69a715 --force --keep-logs 2024-03-19T02:08:59.273 INFO:teuthology.orchestra.run.smithi115.stdout:Deleting cluster with fsid: 438fb69c-e590-11ee-95c9-87774f69a715 2024-03-19T02:09:01.783 DEBUG:teuthology.orchestra.run.smithi099:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-03-19T02:09:01.818 DEBUG:teuthology.orchestra.run.smithi112:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-03-19T02:09:01.855 DEBUG:teuthology.orchestra.run.smithi115:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-03-19T02:09:01.884 INFO:tasks.cephadm:Archiving crash dumps... 2024-03-19T02:09:01.886 DEBUG:teuthology.misc:Transferring archived files from smithi099:/var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/crash to /home/teuthworker/archive/pdonnell-2024-03-18_21:05:49-fs-wip-batrick-testing-20240318.181317-distro-default-smithi/7609258/remote/smithi099/crash 2024-03-19T02:09:01.887 DEBUG:teuthology.orchestra.run.smithi099:> sudo tar c -f - -C /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/crash -- . 2024-03-19T02:09:01.937 INFO:teuthology.orchestra.run.smithi099.stderr:tar: /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/crash: Cannot open: No such file or directory 2024-03-19T02:09:01.938 INFO:teuthology.orchestra.run.smithi099.stderr:tar: Error is not recoverable: exiting now 2024-03-19T02:09:01.939 DEBUG:teuthology.misc:Transferring archived files from smithi112:/var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/crash to /home/teuthworker/archive/pdonnell-2024-03-18_21:05:49-fs-wip-batrick-testing-20240318.181317-distro-default-smithi/7609258/remote/smithi112/crash 2024-03-19T02:09:01.940 DEBUG:teuthology.orchestra.run.smithi112:> sudo tar c -f - -C /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/crash -- . 2024-03-19T02:09:01.985 INFO:teuthology.orchestra.run.smithi112.stderr:tar: /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/crash: Cannot open: No such file or directory 2024-03-19T02:09:01.985 INFO:teuthology.orchestra.run.smithi112.stderr:tar: Error is not recoverable: exiting now 2024-03-19T02:09:01.987 DEBUG:teuthology.misc:Transferring archived files from smithi115:/var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/crash to /home/teuthworker/archive/pdonnell-2024-03-18_21:05:49-fs-wip-batrick-testing-20240318.181317-distro-default-smithi/7609258/remote/smithi115/crash 2024-03-19T02:09:01.988 DEBUG:teuthology.orchestra.run.smithi115:> sudo tar c -f - -C /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/crash -- . 2024-03-19T02:09:02.021 INFO:teuthology.orchestra.run.smithi115.stderr:tar: /var/lib/ceph/438fb69c-e590-11ee-95c9-87774f69a715/crash: Cannot open: No such file or directory 2024-03-19T02:09:02.021 INFO:teuthology.orchestra.run.smithi115.stderr:tar: Error is not recoverable: exiting now 2024-03-19T02:09:02.022 INFO:tasks.cephadm:Checking cluster log for badness... 2024-03-19T02:09:02.023 DEBUG:teuthology.orchestra.run.smithi099:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v FS_DEGRADED | egrep -v FS_INLINE_DATA_DEPRECATED | egrep -v FS_WITH_FAILED_MDS | egrep -v MDS_ALL_DOWN | egrep -v MDS_DAMAGE | egrep -v MDS_DEGRADED | egrep -v MDS_FAILED | egrep -v MDS_INSUFFICIENT_STANDBY | egrep -v MDS_UP_LESS_THAN_MAX | egrep -v POOL_APP_NOT_ENABLED | egrep -v 'overall HEALTH_' | egrep -v 'Replacing daemon' | egrep -v 'deprecated feature inline_data' | egrep -v PG_AVAILABILITY | egrep -v PG_DEGRADED | egrep -v 'overall HEALTH_' | egrep -v '\(OSD_DOWN\)' | egrep -v '\(OSD_' | egrep -v 'but it is still running' | egrep -v 'is not responding' | head -n 1 2024-03-19T02:09:02.058 INFO:teuthology.orchestra.run.smithi099.stdout:2024-03-19T01:33:04.595808+0000 mon.a (mon.0) 209 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-03-19T02:09:02.058 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-03-19T02:09:02.059 DEBUG:teuthology.orchestra.run.smithi099:> sudo egrep '\[SEC\]' /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v FS_DEGRADED | egrep -v FS_INLINE_DATA_DEPRECATED | egrep -v FS_WITH_FAILED_MDS | egrep -v MDS_ALL_DOWN | egrep -v MDS_DAMAGE | egrep -v MDS_DEGRADED | egrep -v MDS_FAILED | egrep -v MDS_INSUFFICIENT_STANDBY | egrep -v MDS_UP_LESS_THAN_MAX | egrep -v POOL_APP_NOT_ENABLED | egrep -v 'overall HEALTH_' | egrep -v 'Replacing daemon' | egrep -v 'deprecated feature inline_data' | egrep -v PG_AVAILABILITY | egrep -v PG_DEGRADED | egrep -v 'overall HEALTH_' | egrep -v '\(OSD_DOWN\)' | egrep -v '\(OSD_' | egrep -v 'but it is still running' | egrep -v 'is not responding' | head -n 1 2024-03-19T02:09:02.133 DEBUG:teuthology.orchestra.run.smithi099:> sudo egrep '\[ERR\]' /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v FS_DEGRADED | egrep -v FS_INLINE_DATA_DEPRECATED | egrep -v FS_WITH_FAILED_MDS | egrep -v MDS_ALL_DOWN | egrep -v MDS_DAMAGE | egrep -v MDS_DEGRADED | egrep -v MDS_FAILED | egrep -v MDS_INSUFFICIENT_STANDBY | egrep -v MDS_UP_LESS_THAN_MAX | egrep -v POOL_APP_NOT_ENABLED | egrep -v 'overall HEALTH_' | egrep -v 'Replacing daemon' | egrep -v 'deprecated feature inline_data' | egrep -v PG_AVAILABILITY | egrep -v PG_DEGRADED | egrep -v 'overall HEALTH_' | egrep -v '\(OSD_DOWN\)' | egrep -v '\(OSD_' | egrep -v 'but it is still running' | egrep -v 'is not responding' | head -n 1 2024-03-19T02:09:02.207 DEBUG:teuthology.orchestra.run.smithi099:> sudo egrep '\[WRN\]' /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v FS_DEGRADED | egrep -v FS_INLINE_DATA_DEPRECATED | egrep -v FS_WITH_FAILED_MDS | egrep -v MDS_ALL_DOWN | egrep -v MDS_DAMAGE | egrep -v MDS_DEGRADED | egrep -v MDS_FAILED | egrep -v MDS_INSUFFICIENT_STANDBY | egrep -v MDS_UP_LESS_THAN_MAX | egrep -v POOL_APP_NOT_ENABLED | egrep -v 'overall HEALTH_' | egrep -v 'Replacing daemon' | egrep -v 'deprecated feature inline_data' | egrep -v PG_AVAILABILITY | egrep -v PG_DEGRADED | egrep -v 'overall HEALTH_' | egrep -v '\(OSD_DOWN\)' | egrep -v '\(OSD_' | egrep -v 'but it is still running' | egrep -v 'is not responding' | head -n 1 2024-03-19T02:09:02.280 INFO:teuthology.orchestra.run.smithi099.stdout:2024-03-19T01:33:04.595808+0000 mon.a (mon.0) 209 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-03-19T02:09:02.280 INFO:tasks.cephadm:Compressing logs... 2024-03-19T02:09:02.280 DEBUG:teuthology.orchestra.run.smithi099:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-03-19T02:09:02.323 DEBUG:teuthology.orchestra.run.smithi112:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-03-19T02:09:02.326 DEBUG:teuthology.orchestra.run.smithi115:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-03-19T02:09:02.349 INFO:teuthology.orchestra.run.smithi099.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-03-19T02:09:02.349 INFO:teuthology.orchestra.run.smithi099.stderr:: No such file or directory 2024-03-19T02:09:02.349 INFO:teuthology.orchestra.run.smithi099.stderr:gzip -5 --verbose -- /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-osd.6.log 2024-03-19T02:09:02.349 INFO:teuthology.orchestra.run.smithi099.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-mon.a.log 2024-03-19T02:09:02.349 INFO:teuthology.orchestra.run.smithi099.stderr:/var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph.audit.log 2024-03-19T02:09:02.349 INFO:teuthology.orchestra.run.smithi099.stderr:/var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-mon.a.log: gzip -5 --verbose -- /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-mds.g.log 2024-03-19T02:09:02.349 INFO:teuthology.orchestra.run.smithi099.stderr:/var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-mds.j.log 2024-03-19T02:09:02.351 INFO:teuthology.orchestra.run.smithi099.stderr:/var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-mds.g.log: gzip -5 --verbose -- /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-osd.3.log 2024-03-19T02:09:02.351 INFO:teuthology.orchestra.run.smithi099.stderr:/var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-mds.j.log: gzip -5 --verbose -- /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-mgr.x.log 2024-03-19T02:09:02.353 INFO:teuthology.orchestra.run.smithi099.stderr:/var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-osd.3.log: gzip -5 --verbose -- /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-osd.0.log 2024-03-19T02:09:02.353 INFO:teuthology.orchestra.run.smithi099.stderr:/var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-mgr.x.log: 90.5% -- replaced with /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph.audit.log.gz 2024-03-19T02:09:02.353 INFO:teuthology.orchestra.run.smithi099.stderr:gzip -5 --verbose -- /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph.cephadm.log 2024-03-19T02:09:02.359 INFO:teuthology.orchestra.run.smithi099.stderr:/var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-osd.0.log: 93.3% -- replaced with /var/log/ceph/cephadm.log.gz 2024-03-19T02:09:02.359 INFO:teuthology.orchestra.run.smithi099.stderr:gzip -5 --verbose -- /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph.log 2024-03-19T02:09:02.359 INFO:teuthology.orchestra.run.smithi099.stderr:/var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph.cephadm.log: 94.7% -- replaced with /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-mds.g.log.gz 2024-03-19T02:09:02.359 INFO:teuthology.orchestra.run.smithi099.stderr: 83.4% -- replaced with /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph.cephadm.log.gz 2024-03-19T02:09:02.359 INFO:teuthology.orchestra.run.smithi099.stderr:gzip -5 --verbose -- /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-volume.log 2024-03-19T02:09:02.360 INFO:teuthology.orchestra.run.smithi099.stderr: 94.7% -- replaced with /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-mds.j.log.gz 2024-03-19T02:09:02.360 INFO:teuthology.orchestra.run.smithi099.stderr:/var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph.log: gzip -5 --verbose -- /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-osd.9.log 2024-03-19T02:09:02.360 INFO:teuthology.orchestra.run.smithi099.stderr:/var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-mds.d.log 2024-03-19T02:09:02.362 INFO:teuthology.orchestra.run.smithi099.stderr:/var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-osd.9.log: 86.9% -- replaced with /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph.log.gz 2024-03-19T02:09:02.362 INFO:teuthology.orchestra.run.smithi099.stderr:gzip -5 --verbose -- /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-mds.a.log 2024-03-19T02:09:02.373 INFO:teuthology.orchestra.run.smithi115.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-03-19T02:09:02.373 INFO:teuthology.orchestra.run.smithi115.stderr:: No such file or directory 2024-03-19T02:09:02.373 INFO:teuthology.orchestra.run.smithi115.stderr:gzip -5 --verbose -- /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-mds.l.log 2024-03-19T02:09:02.373 INFO:teuthology.orchestra.run.smithi115.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph.audit.log 2024-03-19T02:09:02.373 INFO:teuthology.orchestra.run.smithi115.stderr:gzip/var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-mds.l.log: -5 --verbose -- /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-mds.i.log 2024-03-19T02:09:02.373 INFO:teuthology.orchestra.run.smithi115.stderr:/var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-osd.11.log 2024-03-19T02:09:02.373 INFO:teuthology.orchestra.run.smithi115.stderr:/var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-mds.i.log: gzip -5 --verbose -- /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-mgr.z.log 2024-03-19T02:09:02.373 INFO:teuthology.orchestra.run.smithi115.stderr:/var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-osd.11.log: gzip -5 --verbose -- /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-osd.8.log 2024-03-19T02:09:02.374 INFO:teuthology.orchestra.run.smithi115.stderr:/var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-mgr.z.log: gzip -5 --verbose -- /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-osd.5.log 2024-03-19T02:09:02.374 INFO:teuthology.orchestra.run.smithi115.stderr:/var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-osd.8.log: gzip -5 --verbose -- /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-mds.f.log 2024-03-19T02:09:02.374 INFO:teuthology.orchestra.run.smithi115.stderr:/var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph.cephadm.log 2024-03-19T02:09:02.374 INFO:teuthology.orchestra.run.smithi115.stderr:/var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-mds.f.log: gzip -5 --verbose -- /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph.log 2024-03-19T02:09:02.374 INFO:teuthology.orchestra.run.smithi115.stderr:/var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-osd.2.log 2024-03-19T02:09:02.374 INFO:teuthology.orchestra.run.smithi115.stderr:/var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph.log: gzip -5 --verbose -- /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-mds.c.log 2024-03-19T02:09:02.374 INFO:teuthology.orchestra.run.smithi115.stderr:/var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-volume.log 2024-03-19T02:09:02.374 INFO:teuthology.orchestra.run.smithi115.stderr:/var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-mds.c.log: gzip -5 --verbose -- /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-mon.c.log 2024-03-19T02:09:02.376 INFO:teuthology.orchestra.run.smithi099.stderr:/var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-mds.d.log: 94.5% -- replaced with /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-mds.d.log.gz 2024-03-19T02:09:02.376 INFO:teuthology.orchestra.run.smithi099.stderr:/var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-mds.a.log: 91.0% -- replaced with /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-volume.log.gz 2024-03-19T02:09:02.379 INFO:teuthology.orchestra.run.smithi112.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2024-03-19T02:09:02.380 INFO:teuthology.orchestra.run.smithi112.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-03-19T02:09:02.380 INFO:teuthology.orchestra.run.smithi112.stderr:gzip -5 --verbose -- /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-osd.10.log 2024-03-19T02:09:02.380 INFO:teuthology.orchestra.run.smithi112.stderr:gzip/var/log/ceph/cephadm.log: -5 --verbose -- /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph.audit.log 2024-03-19T02:09:02.380 INFO:teuthology.orchestra.run.smithi112.stderr:/var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-osd.10.log: gzip -5 --verbose -- /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-mds.b.log 2024-03-19T02:09:02.380 INFO:teuthology.orchestra.run.smithi112.stderr:/var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-osd.4.log 2024-03-19T02:09:02.380 INFO:teuthology.orchestra.run.smithi112.stderr:/var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-mds.b.log: gzip -5 --verbose -- /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-osd.1.log 2024-03-19T02:09:02.380 INFO:teuthology.orchestra.run.smithi112.stderr:/var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-mds.k.log 2024-03-19T02:09:02.380 INFO:teuthology.orchestra.run.smithi112.stderr:/var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-mds.h.log 2024-03-19T02:09:02.380 INFO:teuthology.orchestra.run.smithi112.stderr:/var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-mds.k.log: gzip -5 --verbose -- /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-osd.7.log 2024-03-19T02:09:02.380 INFO:teuthology.orchestra.run.smithi112.stderr:/var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-mds.h.log: gzip -5 --verbose -- /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-mds.e.log 2024-03-19T02:09:02.380 INFO:teuthology.orchestra.run.smithi112.stderr:/var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-mgr.y.log 2024-03-19T02:09:02.380 INFO:teuthology.orchestra.run.smithi112.stderr:/var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-mds.e.log: gzip -5 --verbose -- /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph.cephadm.log 2024-03-19T02:09:02.380 INFO:teuthology.orchestra.run.smithi112.stderr:/var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-mgr.y.log: gzip -5 --verbose -- /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph.log 2024-03-19T02:09:02.381 INFO:teuthology.orchestra.run.smithi112.stderr:/var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-volume.log 2024-03-19T02:09:02.381 INFO:teuthology.orchestra.run.smithi112.stderr:/var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph.log: gzip -5 --verbose -- /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-mon.b.log 2024-03-19T02:09:02.381 INFO:teuthology.orchestra.run.smithi112.stderr:/var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-volume.log: /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-mon.b.log: 94.7% -- replaced with /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-mds.k.log.gz 2024-03-19T02:09:02.381 INFO:teuthology.orchestra.run.smithi112.stderr: 93.3% -- replaced with /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-mgr.y.log.gz 2024-03-19T02:09:02.382 INFO:teuthology.orchestra.run.smithi099.stderr: 94.7% -- replaced with /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-mds.a.log.gz 2024-03-19T02:09:02.382 INFO:teuthology.orchestra.run.smithi112.stderr: 92.1% -- replaced with /var/log/ceph/cephadm.log.gz 2024-03-19T02:09:02.382 INFO:teuthology.orchestra.run.smithi115.stderr:/var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-volume.log: /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-mon.c.log: 94.7% -- replaced with /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-mds.l.log.gz 2024-03-19T02:09:02.382 INFO:teuthology.orchestra.run.smithi115.stderr: 93.4% -- replaced with /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-mgr.z.log.gz 2024-03-19T02:09:02.382 INFO:teuthology.orchestra.run.smithi112.stderr: 83.4% -- replaced with /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph.cephadm.log.gz 2024-03-19T02:09:02.434 INFO:teuthology.orchestra.run.smithi112.stderr: 87.0% -- replaced with /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph.log.gz 2024-03-19T02:09:02.453 INFO:teuthology.orchestra.run.smithi112.stderr: 90.7% -- replaced with /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph.audit.log.gz 2024-03-19T02:09:02.493 INFO:teuthology.orchestra.run.smithi115.stderr: 83.4% -- replaced with /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph.cephadm.log.gz 2024-03-19T02:09:02.529 INFO:teuthology.orchestra.run.smithi112.stderr: 91.0% -- replaced with /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-volume.log.gz 2024-03-19T02:09:02.579 INFO:teuthology.orchestra.run.smithi115.stderr: 90.9% -- replaced with /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-volume.log.gz 2024-03-19T02:09:02.624 INFO:teuthology.orchestra.run.smithi099.stderr: 90.1% -- replaced with /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-mgr.x.log.gz 2024-03-19T02:09:03.567 INFO:teuthology.orchestra.run.smithi099.stderr: 90.3% -- replaced with /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-mon.a.log.gz 2024-03-19T02:09:03.635 INFO:teuthology.orchestra.run.smithi115.stderr: 90.7% 92.3% -- replaced with /var/log/ceph/cephadm.log.gz -- replaced with /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph.audit.log.gz 2024-03-19T02:09:03.636 INFO:teuthology.orchestra.run.smithi115.stderr: 2024-03-19T02:09:04.287 INFO:teuthology.orchestra.run.smithi115.stderr: 87.0% -- replaced with /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph.log.gz 2024-03-19T02:09:04.405 INFO:teuthology.orchestra.run.smithi112.stderr: 91.9% -- replaced with /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-mon.b.log.gz 2024-03-19T02:09:19.441 INFO:teuthology.orchestra.run.smithi099.stderr: 94.1% -- replaced with /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-osd.9.log.gz 2024-03-19T02:09:19.604 INFO:teuthology.orchestra.run.smithi099.stderr: 93.8% -- replaced with /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-osd.0.log.gz 2024-03-19T02:09:21.569 INFO:teuthology.orchestra.run.smithi099.stderr: 94.0% -- replaced with /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-osd.6.log.gz 2024-03-19T02:09:24.082 INFO:teuthology.orchestra.run.smithi099.stderr: 94.0% -- replaced with /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-osd.3.log.gz 2024-03-19T02:09:24.084 INFO:teuthology.orchestra.run.smithi099.stderr: 2024-03-19T02:09:24.084 INFO:teuthology.orchestra.run.smithi099.stderr:real 0m21.749s 2024-03-19T02:09:24.084 INFO:teuthology.orchestra.run.smithi099.stderr:user 1m13.965s 2024-03-19T02:09:24.084 INFO:teuthology.orchestra.run.smithi099.stderr:sys 0m2.724s 2024-03-19T02:09:49.826 INFO:teuthology.orchestra.run.smithi112.stderr: 93.9% -- replaced with /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-osd.1.log.gz 2024-03-19T02:09:51.661 INFO:teuthology.orchestra.run.smithi112.stderr: 94.1% -- replaced with /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-osd.4.log.gz 2024-03-19T02:09:56.820 INFO:teuthology.orchestra.run.smithi112.stderr: 95.8% -- replaced with /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-mds.e.log.gz 2024-03-19T02:10:00.546 INFO:teuthology.orchestra.run.smithi112.stderr: 95.5% -- replaced with /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-mds.h.log.gz 2024-03-19T02:10:01.183 INFO:teuthology.orchestra.run.smithi112.stderr: 94.0% -- replaced with /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-osd.10.log.gz 2024-03-19T02:10:09.361 INFO:teuthology.orchestra.run.smithi112.stderr: 94.0% -- replaced with /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-osd.7.log.gz 2024-03-19T02:10:24.095 INFO:teuthology.orchestra.run.smithi115.stderr: 93.6% -- replaced with /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-mon.c.log.gz 2024-03-19T02:12:04.753 INFO:teuthology.orchestra.run.smithi115.stderr: 93.9% -- replaced with /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-osd.11.log.gz 2024-03-19T02:12:18.354 INFO:teuthology.orchestra.run.smithi115.stderr: 94.0% -- replaced with /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-osd.8.log.gz 2024-03-19T02:12:27.788 INFO:teuthology.orchestra.run.smithi112.stderr: 93.4% -- replaced with /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-mds.b.log.gz 2024-03-19T02:12:27.790 INFO:teuthology.orchestra.run.smithi112.stderr: 2024-03-19T02:12:27.790 INFO:teuthology.orchestra.run.smithi112.stderr:real 3m25.450s 2024-03-19T02:12:27.790 INFO:teuthology.orchestra.run.smithi112.stderr:user 3m55.622s 2024-03-19T02:12:27.790 INFO:teuthology.orchestra.run.smithi112.stderr:sys 0m8.799s 2024-03-19T02:13:21.152 INFO:teuthology.orchestra.run.smithi115.stderr: 94.1% -- replaced with /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-mds.f.log.gz 2024-03-19T02:13:38.628 INFO:teuthology.orchestra.run.smithi115.stderr: 94.1% -- replaced with /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-osd.2.log.gz 2024-03-19T02:14:21.049 INFO:teuthology.orchestra.run.smithi115.stderr: 94.0% -- replaced with /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-osd.5.log.gz 2024-03-19T02:49:08.766 INFO:teuthology.orchestra.run.smithi115.stderr: 93.6% -- replaced with /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-mds.c.log.gz 2024-03-19T02:49:34.690 INFO:teuthology.orchestra.run.smithi115.stderr: 93.0% -- replaced with /var/log/ceph/438fb69c-e590-11ee-95c9-87774f69a715/ceph-mds.i.log.gz 2024-03-19T02:49:34.701 INFO:teuthology.orchestra.run.smithi115.stderr: 2024-03-19T02:49:34.702 INFO:teuthology.orchestra.run.smithi115.stderr:real 40m32.361s 2024-03-19T02:49:34.702 INFO:teuthology.orchestra.run.smithi115.stderr:user 12m13.264s 2024-03-19T02:49:34.702 INFO:teuthology.orchestra.run.smithi115.stderr:sys 0m35.490s 2024-03-19T02:49:34.702 INFO:tasks.cephadm:Archiving logs... 2024-03-19T02:49:34.704 DEBUG:teuthology.misc:Transferring archived files from smithi099:/var/log/ceph to /home/teuthworker/archive/pdonnell-2024-03-18_21:05:49-fs-wip-batrick-testing-20240318.181317-distro-default-smithi/7609258/remote/smithi099/log 2024-03-19T02:49:34.706 DEBUG:teuthology.orchestra.run.smithi099:> sudo tar c -f - -C /var/log/ceph -- . 2024-03-19T02:49:45.420 DEBUG:teuthology.misc:Transferring archived files from smithi112:/var/log/ceph to /home/teuthworker/archive/pdonnell-2024-03-18_21:05:49-fs-wip-batrick-testing-20240318.181317-distro-default-smithi/7609258/remote/smithi112/log 2024-03-19T02:49:45.421 DEBUG:teuthology.orchestra.run.smithi112:> sudo tar c -f - -C /var/log/ceph -- . 2024-03-19T02:50:21.796 DEBUG:teuthology.misc:Transferring archived files from smithi115:/var/log/ceph to /home/teuthworker/archive/pdonnell-2024-03-18_21:05:49-fs-wip-batrick-testing-20240318.181317-distro-default-smithi/7609258/remote/smithi115/log 2024-03-19T02:50:21.798 DEBUG:teuthology.orchestra.run.smithi115:> sudo tar c -f - -C /var/log/ceph -- . 2024-03-19T02:52:10.011 INFO:tasks.cephadm:Removing cluster... 2024-03-19T02:52:10.012 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 438fb69c-e590-11ee-95c9-87774f69a715 --force 2024-03-19T02:52:10.207 INFO:teuthology.orchestra.run.smithi099.stdout:Deleting cluster with fsid: 438fb69c-e590-11ee-95c9-87774f69a715 2024-03-19T02:52:10.561 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 438fb69c-e590-11ee-95c9-87774f69a715 --force 2024-03-19T02:52:10.753 INFO:teuthology.orchestra.run.smithi112.stdout:Deleting cluster with fsid: 438fb69c-e590-11ee-95c9-87774f69a715 2024-03-19T02:52:11.324 DEBUG:teuthology.orchestra.run.smithi115:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 438fb69c-e590-11ee-95c9-87774f69a715 --force 2024-03-19T02:52:11.717 INFO:teuthology.orchestra.run.smithi115.stdout:Deleting cluster with fsid: 438fb69c-e590-11ee-95c9-87774f69a715 2024-03-19T02:52:12.565 INFO:tasks.cephadm:Removing cephadm ... 2024-03-19T02:52:12.566 DEBUG:teuthology.orchestra.run.smithi099:> rm -rf /home/ubuntu/cephtest/cephadm 2024-03-19T02:52:12.585 DEBUG:teuthology.orchestra.run.smithi112:> rm -rf /home/ubuntu/cephtest/cephadm 2024-03-19T02:52:12.682 DEBUG:teuthology.orchestra.run.smithi115:> rm -rf /home/ubuntu/cephtest/cephadm 2024-03-19T02:52:12.697 INFO:tasks.cephadm:Teardown complete 2024-03-19T02:52:12.697 DEBUG:teuthology.run_tasks:Unwinding manager install 2024-03-19T02:52:12.709 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2024-03-19T02:52:12.709 DEBUG:teuthology.orchestra.run.smithi099:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-03-19T02:52:12.712 DEBUG:teuthology.orchestra.run.smithi112:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-03-19T02:52:12.714 DEBUG:teuthology.orchestra.run.smithi115:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-03-19T02:52:12.848 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-03-19T02:52:12.859 INFO:teuthology.task.clock:Checking final clock skew... 2024-03-19T02:52:12.859 DEBUG:teuthology.orchestra.run.smithi099:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-03-19T02:52:12.862 DEBUG:teuthology.orchestra.run.smithi112:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-03-19T02:52:12.864 DEBUG:teuthology.orchestra.run.smithi115:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-03-19T02:52:12.877 INFO:teuthology.orchestra.run.smithi099.stderr:bash: line 1: ntpq: command not found 2024-03-19T02:52:12.878 INFO:teuthology.orchestra.run.smithi112.stderr:bash: line 1: ntpq: command not found 2024-03-19T02:52:12.878 INFO:teuthology.orchestra.run.smithi115.stderr:bash: line 1: ntpq: command not found 2024-03-19T02:52:12.881 INFO:teuthology.orchestra.run.smithi099.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-03-19T02:52:12.881 INFO:teuthology.orchestra.run.smithi099.stdout:=============================================================================== 2024-03-19T02:52:12.881 INFO:teuthology.orchestra.run.smithi099.stdout:^+ hv01.front.sepia.ceph.com 2 8 377 164 +2379us[+2404us] +/- 124ms 2024-03-19T02:52:12.881 INFO:teuthology.orchestra.run.smithi099.stdout:^* hv02.front.sepia.ceph.com 3 7 377 33 -1205us[-1177us] +/- 68ms 2024-03-19T02:52:12.882 INFO:teuthology.orchestra.run.smithi099.stdout:^+ hv03.front.sepia.ceph.com 3 8 377 164 -104us[ -78us] +/- 89ms 2024-03-19T02:52:12.882 INFO:teuthology.orchestra.run.smithi099.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2024-03-19T02:52:12.933 INFO:teuthology.orchestra.run.smithi112.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-03-19T02:52:12.933 INFO:teuthology.orchestra.run.smithi112.stdout:=============================================================================== 2024-03-19T02:52:12.933 INFO:teuthology.orchestra.run.smithi112.stdout:^+ hv01.front.sepia.ceph.com 2 7 377 103 +2485us[+2465us] +/- 125ms 2024-03-19T02:52:12.933 INFO:teuthology.orchestra.run.smithi112.stdout:^* hv02.front.sepia.ceph.com 3 7 377 32 -1123us[-1142us] +/- 68ms 2024-03-19T02:52:12.933 INFO:teuthology.orchestra.run.smithi112.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 33 -79us[ -98us] +/- 60ms 2024-03-19T02:52:12.933 INFO:teuthology.orchestra.run.smithi112.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2024-03-19T02:52:12.992 INFO:teuthology.orchestra.run.smithi115.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-03-19T02:52:12.992 INFO:teuthology.orchestra.run.smithi115.stdout:=============================================================================== 2024-03-19T02:52:12.992 INFO:teuthology.orchestra.run.smithi115.stdout:^+ hv01.front.sepia.ceph.com 2 7 377 40 +2422us[+2491us] +/- 94ms 2024-03-19T02:52:12.992 INFO:teuthology.orchestra.run.smithi115.stdout:^* hv02.front.sepia.ceph.com 3 7 377 32 -1235us[-1166us] +/- 68ms 2024-03-19T02:52:12.992 INFO:teuthology.orchestra.run.smithi115.stdout:^+ hv03.front.sepia.ceph.com 3 8 377 231 -91us[ -108us] +/- 88ms 2024-03-19T02:52:12.992 INFO:teuthology.orchestra.run.smithi115.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2024-03-19T02:52:12.993 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-03-19T02:52:13.005 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-03-19T02:52:13.005 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-03-19T02:52:13.028 DEBUG:teuthology.orchestra.run.smithi099:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-03-19T02:52:13.113 DEBUG:teuthology.orchestra.run.smithi112:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-03-19T02:52:13.222 DEBUG:teuthology.orchestra.run.smithi115:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-03-19T02:52:13.360 DEBUG:teuthology.orchestra.run.smithi099:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-03-19T02:52:13.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-19T02:52:13.393 DEBUG:teuthology.orchestra.run.smithi112:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-03-19T02:52:13.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-19T02:52:13.427 DEBUG:teuthology.orchestra.run.smithi115:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-03-19T02:52:13.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-19T02:52:13.457 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-03-19T02:52:13.469 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-03-19T02:52:13.511 INFO:teuthology.task.internal:Duration was 6855.110952 seconds 2024-03-19T02:52:13.512 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-03-19T02:52:13.563 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-03-19T02:52:13.564 DEBUG:teuthology.orchestra.run.smithi099:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-03-19T02:52:13.566 DEBUG:teuthology.orchestra.run.smithi112:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-03-19T02:52:13.569 DEBUG:teuthology.orchestra.run.smithi115:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-03-19T02:52:13.610 INFO:teuthology.orchestra.run.smithi099.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-03-19T02:52:13.627 INFO:teuthology.orchestra.run.smithi115.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-03-19T02:52:13.648 INFO:teuthology.orchestra.run.smithi112.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-03-19T02:52:14.129 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-03-19T02:52:14.129 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi099.front.sepia.ceph.com 2024-03-19T02:52:14.129 DEBUG:teuthology.orchestra.run.smithi099:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-03-19T02:52:14.154 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi112.front.sepia.ceph.com 2024-03-19T02:52:14.154 DEBUG:teuthology.orchestra.run.smithi112:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-03-19T02:52:14.184 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi115.front.sepia.ceph.com 2024-03-19T02:52:14.184 DEBUG:teuthology.orchestra.run.smithi115:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-03-19T02:52:14.205 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-03-19T02:52:14.205 DEBUG:teuthology.orchestra.run.smithi099:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-03-19T02:52:14.208 DEBUG:teuthology.orchestra.run.smithi112:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-03-19T02:52:14.227 DEBUG:teuthology.orchestra.run.smithi115:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-03-19T02:52:14.281 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-03-19T02:52:14.282 DEBUG:teuthology.orchestra.run.smithi099:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-03-19T02:52:14.285 DEBUG:teuthology.orchestra.run.smithi112:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-03-19T02:52:14.301 DEBUG:teuthology.orchestra.run.smithi115:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-03-19T02:52:14.656 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-03-19T02:52:14.683 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-03-19T02:52:14.684 DEBUG:teuthology.orchestra.run.smithi099:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-03-19T02:52:14.712 DEBUG:teuthology.orchestra.run.smithi112:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-03-19T02:52:14.771 DEBUG:teuthology.orchestra.run.smithi115:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-03-19T02:52:14.849 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-03-19T02:52:15.047 DEBUG:teuthology.orchestra.run.smithi099:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-03-19T02:52:15.050 DEBUG:teuthology.orchestra.run.smithi112:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-03-19T02:52:15.053 DEBUG:teuthology.orchestra.run.smithi115:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-03-19T02:52:15.074 INFO:teuthology.orchestra.run.smithi099.stdout:kernel.core_pattern = core 2024-03-19T02:52:15.078 INFO:teuthology.orchestra.run.smithi112.stdout:kernel.core_pattern = core 2024-03-19T02:52:15.090 INFO:teuthology.orchestra.run.smithi115.stdout:kernel.core_pattern = core 2024-03-19T02:52:15.115 DEBUG:teuthology.orchestra.run.smithi099:> test -e /home/ubuntu/cephtest/archive/coredump 2024-03-19T02:52:15.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-19T02:52:15.145 DEBUG:teuthology.orchestra.run.smithi112:> test -e /home/ubuntu/cephtest/archive/coredump 2024-03-19T02:52:15.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-19T02:52:15.161 DEBUG:teuthology.orchestra.run.smithi115:> test -e /home/ubuntu/cephtest/archive/coredump 2024-03-19T02:52:15.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-19T02:52:15.177 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-03-19T02:52:15.334 INFO:teuthology.task.internal:Transferring archived files... 2024-03-19T02:52:15.343 DEBUG:teuthology.misc:Transferring archived files from smithi099:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/pdonnell-2024-03-18_21:05:49-fs-wip-batrick-testing-20240318.181317-distro-default-smithi/7609258/remote/smithi099 2024-03-19T02:52:15.343 DEBUG:teuthology.orchestra.run.smithi099:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-03-19T02:52:15.392 DEBUG:teuthology.misc:Transferring archived files from smithi112:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/pdonnell-2024-03-18_21:05:49-fs-wip-batrick-testing-20240318.181317-distro-default-smithi/7609258/remote/smithi112 2024-03-19T02:52:15.392 DEBUG:teuthology.orchestra.run.smithi112:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-03-19T02:52:15.435 DEBUG:teuthology.misc:Transferring archived files from smithi115:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/pdonnell-2024-03-18_21:05:49-fs-wip-batrick-testing-20240318.181317-distro-default-smithi/7609258/remote/smithi115 2024-03-19T02:52:15.435 DEBUG:teuthology.orchestra.run.smithi115:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-03-19T02:52:15.474 INFO:teuthology.task.internal:Removing archive directory... 2024-03-19T02:52:15.474 DEBUG:teuthology.orchestra.run.smithi099:> rm -rf -- /home/ubuntu/cephtest/archive 2024-03-19T02:52:15.477 DEBUG:teuthology.orchestra.run.smithi112:> rm -rf -- /home/ubuntu/cephtest/archive 2024-03-19T02:52:15.479 DEBUG:teuthology.orchestra.run.smithi115:> rm -rf -- /home/ubuntu/cephtest/archive 2024-03-19T02:52:15.522 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-03-19T02:52:15.536 INFO:teuthology.task.internal:Not uploading archives. 2024-03-19T02:52:15.536 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-03-19T02:52:15.571 INFO:teuthology.task.internal:Tidying up after the test... 2024-03-19T02:52:15.571 DEBUG:teuthology.orchestra.run.smithi099:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-03-19T02:52:15.574 DEBUG:teuthology.orchestra.run.smithi112:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-03-19T02:52:15.576 DEBUG:teuthology.orchestra.run.smithi115:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-03-19T02:52:15.586 INFO:teuthology.orchestra.run.smithi099.stdout: 265823 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Mar 19 02:52 /home/ubuntu/cephtest 2024-03-19T02:52:15.587 INFO:teuthology.orchestra.run.smithi112.stdout: 262162 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Mar 19 02:52 /home/ubuntu/cephtest 2024-03-19T02:52:15.590 INFO:teuthology.orchestra.run.smithi115.stdout: 262162 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Mar 19 02:52 /home/ubuntu/cephtest 2024-03-19T02:52:15.592 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-03-19T02:52:15.620 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-03-19T02:52:15.721 INFO:teuthology.nuke:Checking targets against current locks 2024-03-19T02:52:15.746 DEBUG:teuthology.nuke:shortname: smithi099 2024-03-19T02:52:15.747 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-03-19T02:52:15.766 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi099.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/pdonnell-2024-03-18_21:05:49-fs-wip-batrick-testing-20240318.181317-distro-default-smithi/7609258', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-03-19 00:16:53.420364', 'locked_by': 'scheduled_pdonnell@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBMWOkmh1fiqamXWIQTPEThz+6CiVZmPEHUZMQAJUAAEvdGqeZxrzOqraKfT4dEVey7xyQartsdDv292AAKfiP7Q='} 2024-03-19T02:52:15.772 DEBUG:teuthology.nuke:shortname: smithi112 2024-03-19T02:52:15.772 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-03-19T02:52:15.795 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi112.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/pdonnell-2024-03-18_21:05:49-fs-wip-batrick-testing-20240318.181317-distro-default-smithi/7609258', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-03-19 00:16:53.422319', 'locked_by': 'scheduled_pdonnell@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCd7Wmd7bQcoJeCcw/Zzufv4EK8DxlS46DEW+9X2hp+OA9SCNLUuEhu7gkAob5SpREz6z6IXduH3m1ZL1u9Al3Y='} 2024-03-19T02:52:15.796 DEBUG:teuthology.nuke:shortname: smithi115 2024-03-19T02:52:15.797 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-03-19T02:52:15.818 INFO:teuthology.orchestra.console.smithi099:Power off 2024-03-19T02:52:15.818 DEBUG:teuthology.orchestra.console.smithi099:pexpect command: ipmitool -H smithi099.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-03-19T02:52:15.831 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi115.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/pdonnell-2024-03-18_21:05:49-fs-wip-batrick-testing-20240318.181317-distro-default-smithi/7609258', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-03-19 00:16:53.423979', 'locked_by': 'scheduled_pdonnell@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBO4emzpff0NrFFf+cZK9dZxkf4KE96HuWp15+KTIk5TJFRiV8fCyISa3xjNfpZKAlAofA9JjR5CO+jYYrdWAZOU='} 2024-03-19T02:52:15.842 DEBUG:teuthology.orchestra.console.smithi099:power off output: Chassis Power Control: Down/Off 2024-03-19T02:52:15.842 DEBUG:teuthology.orchestra.console.smithi099:pexpect command: ipmitool -H smithi099.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-19T02:52:15.859 INFO:teuthology.orchestra.console.smithi112:Power off 2024-03-19T02:52:15.860 DEBUG:teuthology.orchestra.console.smithi112:pexpect command: ipmitool -H smithi112.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-03-19T02:52:15.872 DEBUG:teuthology.orchestra.console.smithi099:check power output: Chassis Power is on 2024-03-19T02:52:15.881 DEBUG:teuthology.orchestra.console.smithi112:power off output: Chassis Power Control: Down/Off 2024-03-19T02:52:15.881 DEBUG:teuthology.orchestra.console.smithi112:pexpect command: ipmitool -H smithi112.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-19T02:52:15.900 INFO:teuthology.orchestra.console.smithi115:Power off 2024-03-19T02:52:15.900 DEBUG:teuthology.orchestra.console.smithi115:pexpect command: ipmitool -H smithi115.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-03-19T02:52:15.909 DEBUG:teuthology.orchestra.console.smithi112:check power output: Chassis Power is on 2024-03-19T02:52:15.921 DEBUG:teuthology.orchestra.console.smithi115:power off output: Chassis Power Control: Down/Off 2024-03-19T02:52:15.921 DEBUG:teuthology.orchestra.console.smithi115:pexpect command: ipmitool -H smithi115.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-19T02:52:15.943 DEBUG:teuthology.orchestra.console.smithi115:check power output: Chassis Power is on 2024-03-19T02:52:19.873 DEBUG:teuthology.orchestra.console.smithi099:pexpect command: ipmitool -H smithi099.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-19T02:52:19.910 DEBUG:teuthology.orchestra.console.smithi112:pexpect command: ipmitool -H smithi112.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-19T02:52:19.944 DEBUG:teuthology.orchestra.console.smithi115:pexpect command: ipmitool -H smithi115.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-19T02:52:19.986 DEBUG:teuthology.orchestra.console.smithi099:check power output: Chassis Power is on 2024-03-19T02:52:20.021 DEBUG:teuthology.orchestra.console.smithi112:check power output: Chassis Power is on 2024-03-19T02:52:20.053 DEBUG:teuthology.orchestra.console.smithi115:check power output: Chassis Power is on 2024-03-19T02:52:23.991 DEBUG:teuthology.orchestra.console.smithi099:pexpect command: ipmitool -H smithi099.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-19T02:52:24.022 DEBUG:teuthology.orchestra.console.smithi112:pexpect command: ipmitool -H smithi112.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-19T02:52:24.054 DEBUG:teuthology.orchestra.console.smithi115:pexpect command: ipmitool -H smithi115.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-19T02:52:24.101 DEBUG:teuthology.orchestra.console.smithi099:check power output: Chassis Power is on 2024-03-19T02:52:24.131 DEBUG:teuthology.orchestra.console.smithi112:check power output: Chassis Power is off 2024-03-19T02:52:24.164 DEBUG:teuthology.orchestra.console.smithi115:check power output: Chassis Power is off 2024-03-19T02:52:24.232 INFO:teuthology.orchestra.console.smithi112:Power off completed 2024-03-19T02:52:24.265 INFO:teuthology.orchestra.console.smithi115:Power off completed 2024-03-19T02:52:28.102 DEBUG:teuthology.orchestra.console.smithi099:pexpect command: ipmitool -H smithi099.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-19T02:52:28.212 DEBUG:teuthology.orchestra.console.smithi099:check power output: Chassis Power is off 2024-03-19T02:52:28.313 INFO:teuthology.orchestra.console.smithi099:Power off completed 2024-03-19T02:52:28.417 INFO:teuthology.run:Summary data: description: fs/workload/{0-centos_9.stream begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v1} mount overrides/{distro/stock/{centos_9.stream k-stock} ms-die-on-skipped}} ms_mode/crc wsync/no} objectstore-ec/bluestore-bitmap omap_limit/10 overrides/{cephsqlite-timeout frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/multi/{balancer/random export-check n/3 replication/always} standby-replay tasks/{0-subvolume/{with-no-extra-options} 1-check-counter 2-scrub/no 3-snaps/no 4-flush/yes 5-workunit/suites/dbench}} duration: 6855.110951662064 failure_reason: '"2024-03-19T01:33:04.595808+0000 mon.a (mon.0) 209 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,c (MON_DOWN)" in cluster log' flavor: default owner: scheduled_pdonnell@teuthology success: false 2024-03-19T02:52:28.418 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-03-19T02:52:28.518 INFO:teuthology.run:FAIL